From 755e02adc162dc6779386b8097744acb976bbc3a Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Tue, 6 Oct 2026 11:16:16 -0400 Subject: [PATCH 01/24] ci: run pull request and push validation for feature/pq-transport feature/pq-transport collects the #184 hybrid post-quantum transport work before it goes to 1.x.x. Add it to the branch lists of Full Validation, Core Checks and the Required Merge Gate, so pull requests into it and merges to it get the same validation as 1.x.x. Remove the entry when the branch merges into 1.x.x. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 3 +++ .github/workflows/core-checks.yml | 3 +++ .github/workflows/required-merge-gate.yml | 3 +++ 3 files changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c01dbb2097..b4db0ffa7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,7 @@ on: - 1.0.0 - 1.x.x - develop + - feature/pq-transport push: branches: - main @@ -27,6 +28,7 @@ on: - 1.0.0 - 1.x.x - develop + - feature/pq-transport merge_group: branches: - main @@ -35,6 +37,7 @@ on: - 1.0.0 - 1.x.x - develop + - feature/pq-transport workflow_dispatch: # Manual trigger for full validation on any branch permissions: diff --git a/.github/workflows/core-checks.yml b/.github/workflows/core-checks.yml index 30111a8636..2e699513cf 100644 --- a/.github/workflows/core-checks.yml +++ b/.github/workflows/core-checks.yml @@ -13,6 +13,7 @@ on: - 1.0.0 - 1.x.x - develop + - feature/pq-transport push: branches: - main @@ -21,6 +22,7 @@ on: - 1.0.0 - 1.x.x - develop + - feature/pq-transport merge_group: branches: - main @@ -29,6 +31,7 @@ on: - 1.0.0 - 1.x.x - develop + - feature/pq-transport permissions: contents: read diff --git a/.github/workflows/required-merge-gate.yml b/.github/workflows/required-merge-gate.yml index 2fc2fa372b..a8be0dff67 100644 --- a/.github/workflows/required-merge-gate.yml +++ b/.github/workflows/required-merge-gate.yml @@ -19,6 +19,7 @@ on: - 1.0.0 - 1.x.x - develop + - feature/pq-transport push: branches: - main @@ -26,6 +27,7 @@ on: - 1.0.0 - 1.x.x - develop + - feature/pq-transport merge_group: branches: - main @@ -33,6 +35,7 @@ on: - 1.0.0 - 1.x.x - develop + - feature/pq-transport permissions: contents: read From 5cbf4e10eecb55a6cfd4e7e5aace116e87cf8754 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Tue, 6 Oct 2026 11:20:23 -0400 Subject: [PATCH 02/24] ci: build pull request RPC docs from the event commit (#228) The pull request build of the development RPC docs checked out refs/pull//merge and then required it to be the event's merge commit. That ref moves whenever the base branch or the pull request's stack changes, so a run that overlapped such a change checked out a different merge commit and failed "Validate version source checkout". Check out the event's merge commit by hash instead, as the release entries already do, and drop the now unused --development-checkout-ref plumbing. Co-authored-by: Claude Opus 5.5 --- .github/workflows/rpc-docs-pages.yml | 9 ++++----- doc/rpc/versioned_site.py | 5 +---- test/rpc_docs/test_versioned_site.py | 7 +++---- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/.github/workflows/rpc-docs-pages.yml b/.github/workflows/rpc-docs-pages.yml index c4b77f3583..df54dbe68c 100644 --- a/.github/workflows/rpc-docs-pages.yml +++ b/.github/workflows/rpc-docs-pages.yml @@ -85,20 +85,20 @@ jobs: DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} DISPATCH_REF: ${{ inputs.ref || '' }} EVENT_NAME: ${{ github.event_name }} - PULL_REQUEST_NUMBER: ${{ github.event_name == 'pull_request' && github.event.pull_request.number || '' }} PULL_REQUEST_SHA: ${{ github.event_name == 'pull_request' && github.sha || '' }} run: | set -euo pipefail development_ref="${DEFAULT_BRANCH}" - development_checkout_ref="" + # A pull request builds its event merge commit, checked out by hash: + # refs/pull//merge moves whenever the base branch or the pull + # request's stack changes, so it can name a different commit by the + # time the build job checks it out. if [[ "${EVENT_NAME}" == "pull_request" ]]; then development_ref="${PULL_REQUEST_SHA}" - development_checkout_ref="refs/pull/${PULL_REQUEST_NUMBER}/merge" elif [[ "${EVENT_NAME}" == "workflow_dispatch" && -n "${DISPATCH_REF}" ]]; then development_ref="${DISPATCH_REF}" fi echo "ref=${development_ref}" >> "${GITHUB_OUTPUT}" - echo "checkout_ref=${development_checkout_ref}" >> "${GITHUB_OUTPUT}" - name: Determine publication mode id: mode @@ -128,7 +128,6 @@ jobs: python3 doc/rpc/versioned_site.py discover \ --repository "${REPOSITORY}" \ --development-ref "${DEVELOPMENT_REF}" \ - --development-checkout-ref "${{ steps.development.outputs.checkout_ref }}" \ --publisher-sha "${publisher_sha}" \ --output "${RUNNER_TEMP}/rpc-docs-build-plan.json" \ --github-output "${GITHUB_OUTPUT}" diff --git a/doc/rpc/versioned_site.py b/doc/rpc/versioned_site.py index e1047c989b..288d466713 100644 --- a/doc/rpc/versioned_site.py +++ b/doc/rpc/versioned_site.py @@ -329,7 +329,6 @@ def build_plan( development_ref: str, publisher_sha: str, repository: str, - development_checkout_ref: str = "", release_resolver: Callable[[dict[str, Any]], str] | None = None, development_resolver: Callable[[str], str] = resolve_commit, ) -> dict[str, Any]: @@ -342,7 +341,7 @@ def build_plan( "path": "main", "source_ref": development_ref, "source_sha": development_sha, - "source_checkout_ref": development_checkout_ref or development_sha, + "source_checkout_ref": development_sha, } return { "schema_version": SCHEMA_VERSION, @@ -588,7 +587,6 @@ def parse_args() -> argparse.Namespace: discover = subparsers.add_parser("discover") discover.add_argument("--repository", required=True) discover.add_argument("--development-ref", required=True) - discover.add_argument("--development-checkout-ref", default="") discover.add_argument("--publisher-sha", required=True) discover.add_argument("--output", required=True) discover.add_argument("--github-output") @@ -612,7 +610,6 @@ def main() -> int: plan = build_plan( releases, development_ref=args.development_ref, - development_checkout_ref=args.development_checkout_ref, publisher_sha=args.publisher_sha, repository=args.repository, ) diff --git a/test/rpc_docs/test_versioned_site.py b/test/rpc_docs/test_versioned_site.py index 9b1c7cd868..662ece6d00 100644 --- a/test/rpc_docs/test_versioned_site.py +++ b/test/rpc_docs/test_versioned_site.py @@ -80,7 +80,6 @@ def test_build_plan_adds_development_after_releases(self) -> None: plan = versioned_site.build_plan( [release("v1.0.0")], development_ref="topic/ref", - development_checkout_ref="refs/pull/12/merge", publisher_sha="2" * 40, repository="Qbit-Org/qbit", release_resolver=lambda _release: "1" * 40, @@ -90,9 +89,9 @@ def test_build_plan_adds_development_after_releases(self) -> None: self.assertEqual(plan["latest"], "v1.0.0") self.assertEqual([entry["id"] for entry in plan["builds"]], ["v1.0.0", "main"]) self.assertEqual(plan["development"]["source_ref"], "topic/ref") - self.assertEqual( - plan["development"]["source_checkout_ref"], "refs/pull/12/merge" - ) + # The build checks out the resolved commit itself, never a ref that can move. + self.assertEqual(plan["development"]["source_sha"], "3" * 40) + self.assertEqual(plan["development"]["source_checkout_ref"], "3" * 40) self.assertIn("33333333", plan["development"]["label"]) def test_duplicate_release_tag_is_rejected(self) -> None: From 152117ac34efc4cc7385a939f267500f4fd3120e Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Tue, 6 Oct 2026 11:24:36 -0400 Subject: [PATCH 03/24] i2p: request post-quantum leasesets (6,4) and retry once with 4,0 on explicit rejection (#223) * net: support bounded interruptible SAM session creation Proxy::Connect() waits up to -timeout for a connection and does not watch any interrupt, so a caller that has to finish several dependent steps by one deadline, and stop promptly on shutdown, cannot use it. Add a Proxy::Connect(deadline, interrupt) overload for that. It waits for the connection in slices of at most MAX_CONNECT_POLL_INTERVAL (100 ms), rechecking the interrupt and the deadline between them, and creates no socket if either is already over. It does not use nConnectTimeout; callers cap the deadline as they need. Proxy::Connect() and ConnectDirectly() behave as before. The new netbase tests use a socket whose connect stays in progress to check that the overload connects, gives up at the deadline rather than at -timeout, notices an interrupt within one slice, never waits longer than a slice, and that the existing overload still waits once for -timeout. TCP and Unix socket proxies are both covered. Part of #185. Co-Authored-By: Claude Opus 5.5 * i2p: prefer hybrid leasesets and retry explicit router rejection once Backport Bitcoin Core #35696: ask the SAM router for the post-quantum hybrid leaseset encryption type, i2cp.leaseSetEncType=6,4 (MLKEM768-X25519, then ECIES-X25519), instead of 4,0, for persistent and transient sessions. Java I2P 2.10.0 or newer and i2pd 2.58.0 or newer built with OpenSSL 3.5 or newer use it; other i2pd versions ignore it. Java I2P older than 2.10.0 fails the session instead. qbit-only, as upstream has no retry: on an explicit rejection of 6,4, a well-formed SESSION STATUS whose RESULT is present, not OK and not one of DUPLICATED_ID, DUPLICATED_DEST, INVALID_ID or INVALID_KEY, close that control socket and retry once on a fresh connection with a fresh HELLO and session id, keeping the persistent destination, with 4,0. A SAM reply with RESULT=TIMEOUT is such a rejection and is retried. No reply (a timeout waiting for one, EOF, a socket error), a malformed reply, a missing RESULT or a lost success reply is never retried. The original rejection is logged at debug level with RESULT and a sanitized MESSAGE capped at 200 characters. m_session_id and m_control_sock are set only on success. Only a rejection whose MESSAGE names the encryption type is evidence that the router lacks 6,4. Java I2P 2.9.0 replies "... Failed to build tunnels - ...: Unsupported crypto type: 6". The MESSAGE, parsed like the logged one but before it is sanitized or capped, names the type if it contains "crypto type", "encryption type", "enc type" or "enctype", ASCII case-insensitively. A process-lifetime, mutex-protected record per router (its TCP endpoint or Unix socket path), shared by all sessions, counts the sessions in a row in which the router rejected 6,4 naming the encryption type and accepted 4,0. After two, later sessions through it start with 4,0 and never retry, until qbit restarts. A session created with 6,4 starts the count anew, so one transient rejection does not turn 6,4 off, and a failed retry counts for nothing. The record keeps at most 64 routers and forgets the least recently updated one. At the first such session through a router, this is logged once at warning level, with RESULT shown only if it is one of the SAM protocol's RESULT values, otherwise "other": I2P: the router rejected leaseSetEncType=6,4 (RESULT=I2P_ERROR) and accepted 4,0, so this session has no post-quantum leaseset. Java I2P older than 2.10.0 does this; upgrade the router to get it. An info line says when later sessions through the router start using 4,0 until qbit restarts. A rejection that does not name the encryption type, such as a passing failure to build tunnels, RESULT=TIMEOUT or CANT_REACH_PEER, is retried the same way but is no evidence either way: it neither counts nor starts the count anew, and does not warn. One info line says that the session uses 4,0 because of that rejection, with RESULT shown as in the warning and the sanitized MESSAGE, and that the next session asks for 6,4 again. Session creation now has one deadline, SAM_CREATE_TIMEOUT (180 s), over connecting, HELLO, the optional DEST GENERATE and both SESSION CREATE attempts, through new Hello() and SendRequestAndGetReply() overloads that take it. Each connect uses Proxy::Connect(deadline, interrupt), capped by -timeout as before, so shutdown interrupts it promptly. Other SAM requests behave as before. Session constructors take an optional creation timeout for tests. An unexpected reply to SESSION CREATE is now reported with the sanitized request, "SESSION CREATE ...", instead of the raw one, which carried the private key. The new i2p tests use a recording mock SAM router to cover creation with 6,4, the retry, the two-sessions-in-a-row rule and its reset, rejections that name the encryption type (with the reply of Java I2P 2.9.0) and ones that do not, which are retried but neutral, the phrase matching, the bounded record, sessions through a remembered router including concurrent ones, rejections that are not retried, a failing retry, transport failures and lost replies, interruption during reconnect, HELLO and CREATE, an exhausted deadline across steps, during DEST GENERATE and while connecting, the -timeout cap on connecting, and that the persistent private key is never logged. Deadline checks use the timeouts the mock router saw rather than wall-clock margins. Tests and the i2p fuzz target forget the router record between runs. The existing tests are unchanged. Part of #185. Co-Authored-By: Claude Opus 5.5 * doc: document I2P hybrid router requirements and compatibility retry State in i2p.md which routers use the post-quantum leaseset type that qbit now asks for, that Java I2P older than 2.10.0 rejects it naming the encryption type, and how qbit retries once with 4,0 and warns at the first such session through a router. After two such sessions in a row, qbit uses 4,0 with that router until qbit restarts; a session that gets the post-quantum type starts the count anew. Other rejections, a SAM RESULT=TIMEOUT reply among them, are retried but neither count nor warn. Tell users to restart qbit after upgrading the router. Note that the retry is qbit-only, that no reply from the router is not retried, and the single 3-minute limit on creating a session. Add a release-note fragment, to be renamed to the pull request number. Part of #185. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- doc/release-notes-223.md | 23 + doc/user/network/i2p.md | 45 ++ src/i2p.cpp | 398 ++++++++++-- src/i2p.h | 86 ++- src/netbase.cpp | 87 ++- src/netbase.h | 19 + src/test/fuzz/i2p.cpp | 2 + src/test/i2p_tests.cpp | 1231 +++++++++++++++++++++++++++++++++++- src/test/netbase_tests.cpp | 172 ++++- 9 files changed, 2002 insertions(+), 61 deletions(-) create mode 100644 doc/release-notes-223.md diff --git a/doc/release-notes-223.md b/doc/release-notes-223.md new file mode 100644 index 0000000000..cc590acbf6 --- /dev/null +++ b/doc/release-notes-223.md @@ -0,0 +1,23 @@ +I2P post-quantum encryption +--------------------------- + +qbit now asks the I2P router for post-quantum hybrid session encryption, +MLKEM768-X25519 with ECIES-X25519 for peers without it +(`i2cp.leaseSetEncType=6,4`), as Bitcoin Core does. Java I2P 2.10.0 or newer, +and i2pd 2.58.0 or newer built with OpenSSL 3.5 or newer, use it; other i2pd +versions ignore it. + +Java I2P older than 2.10.0 rejects the request, naming the encryption type in +its reply. qbit then retries once with the previous setting, `4,0`, and warns +the first time this happens with a router. After two sessions in a row in +which the router rejects the post-quantum type this way and accepts `4,0`, +qbit uses `4,0` with that router until qbit restarts; a session that gets the +post-quantum type starts the count anew, so one passing router error does not +turn it off. Other rejections, such as a failure to build tunnels or a SAM +`RESULT=TIMEOUT` reply, are retried the same way but do not count and do not +warn. Upgrade the router to get the post-quantum type, then restart qbit. No +reply from the router (a timeout while waiting for it, or a closed or broken +connection) is not retried. + +Creating an I2P session now has a single 3-minute limit that covers every +step, and stops promptly when qbit shuts down. (#223) diff --git a/doc/user/network/i2p.md b/doc/user/network/i2p.md index c6ed76563b..32c9a3ae66 100644 --- a/doc/user/network/i2p.md +++ b/doc/user/network/i2p.md @@ -121,6 +121,51 @@ You can use the `getnodeaddresses` RPC to fetch a number of I2P peers known to y qbit uses the [SAM v3.1](https://geti2p.net/en/docs/api/samv3) protocol to connect to the I2P network. Any I2P router that supports it can be used. +### Post-quantum encryption of I2P sessions + +qbit asks the I2P router to encrypt its sessions with the post-quantum hybrid +type MLKEM768-X25519, and with ECIES-X25519 for peers that lack it +(`i2cp.leaseSetEncType=6,4`). Whether you get the post-quantum type depends on +your router: + +- Java I2P 2.10.0 or newer, and i2pd 2.58.0 or newer built with OpenSSL 3.5 or + newer, use it. +- Other i2pd versions ignore it and use ECIES-X25519 as before. +- Java I2P older than 2.10.0 rejects it and refuses to create the session. + +When the router rejects the session, qbit retries once without the +post-quantum type (`i2cp.leaseSetEncType=4,0`, what earlier versions of qbit +used). Java I2P older than 2.10.0 names the encryption type in its rejection, +for example `Unsupported crypto type: 6`. The first time such a rejection is +followed by a working retry with a router, qbit logs a warning such as: + +``` +I2P: the router rejected leaseSetEncType=6,4 (RESULT=I2P_ERROR) and accepted 4,0, so this session has no post-quantum leaseset. Java I2P older than 2.10.0 does this; upgrade the router to get it. +``` + +One such rejection may be a passing router error, so qbit asks for the +post-quantum type again in the next session. Only when the router rejects it +this way and accepts `4,0` in two sessions in a row does qbit stop asking: +later sessions through that router use `4,0` until qbit restarts. A session in +which the router accepts the post-quantum type starts the count anew. + +Only a rejection that names the encryption type counts and warns. Other +rejections, such as a failure to build tunnels or a SAM `RESULT=TIMEOUT` reply, +are retried the same way but do not count: they neither add to the count nor +start it anew. qbit logs that the session uses `4,0` and that the next session +asks for the post-quantum type again. + +Upgrade the router to get the post-quantum type, then restart qbit so that it +asks for it again. The retry is specific to qbit; Bitcoin Core has none. qbit +retries only when the router answers with a rejection, and not when the +rejection is about the session id or the destination. A SAM `RESULT=TIMEOUT` +reply is a rejection and is retried. No reply at all is not: a timeout while +waiting for the router, or a closed or broken connection to it, fails as +before. + +Creating an I2P session, with the retry, must complete within 3 minutes, and +stops promptly when qbit shuts down. Use `-debug=i2p` to see each attempt. + ## Ports in I2P and qbit One particularity of SAM v3.1 is that it does not support ports, diff --git a/src/i2p.cpp b/src/i2p.cpp index 8b48615afe..d822fdeaaa 100644 --- a/src/i2p.cpp +++ b/src/i2p.cpp @@ -21,11 +21,17 @@ #include #include +#include +#include #include +#include +#include #include #include #include #include +#include +#include using util::Split; @@ -117,19 +123,199 @@ static CNetAddr DestB64ToAddr(const std::string& dest) namespace sam { +namespace { +/** + * i2cp.leaseSetEncType for new sessions: the post-quantum hybrid MLKEM768-X25519 (6), and + * ECIES-X25519 (4) for peers without it. + */ +constexpr std::string_view LEASESET_ENC_TYPES_HYBRID{"6,4"}; + +/** + * i2cp.leaseSetEncType for routers that reject the hybrid type: ECIES-X25519 (4) and ElGamal (0). + */ +constexpr std::string_view LEASESET_ENC_TYPES_LEGACY{"4,0"}; + +/** + * Longest part of a router-supplied RESULT= or MESSAGE= value that is logged. + */ +constexpr size_t MAX_LOGGED_REPLY_VALUE{200}; + +/** + * qbit-only: what is known about a SAM router's support for i2cp.leaseSetEncType=6,4. + */ +struct RouterHybridState { + /** + * Sessions in a row in which the router rejected 6,4 naming the encryption type and accepted + * 4,0. Sessions in which the rejection did not name it are not counted and do not break the row. + */ + unsigned fallbacks{0}; + /** Whether the warning about the router was logged. */ + bool warned{false}; + /** Position of the latest update among all updates, to forget the oldest router. */ + uint64_t updated{0}; +}; + +/** + * qbit-only: `RouterHybridState` of SAM routers, by `Proxy::ToString()` (TCP endpoint or Unix + * socket path). Kept for the process lifetime and shared by all sessions, so that later sessions + * through a router (e.g. transient ones) start with 4,0 once it has rejected 6,4 naming the + * encryption type in `SAM_HYBRID_FALLBACKS_TO_REMEMBER` sessions in a row. Only such rejections + * add a router. At most `SAM_MAX_REMEMBERED_ROUTERS` routers are kept, so a forgotten router may + * be warned about again. + */ +GlobalMutex g_router_hybrid_mutex; +std::map g_router_hybrid GUARDED_BY(g_router_hybrid_mutex); +uint64_t g_router_hybrid_updates GUARDED_BY(g_router_hybrid_mutex){0}; + +/** + * Whether sessions through `router` start with 4,0 without asking for 6,4. + */ +bool RouterLacksHybrid(const std::string& router) +{ + LOCK(g_router_hybrid_mutex); + const auto it{g_router_hybrid.find(router)}; + return it != g_router_hybrid.end() && it->second.fallbacks >= SAM_HYBRID_FALLBACKS_TO_REMEMBER; +} + +/** + * A session through `router` was created with 6,4: its rejections so far were not in a row. + */ +void NoteHybridAccepted(const std::string& router) +{ + LOCK(g_router_hybrid_mutex); + const auto it{g_router_hybrid.find(router)}; + if (it != g_router_hybrid.end()) { + it->second.fallbacks = 0; + it->second.updated = ++g_router_hybrid_updates; + } +} + +struct HybridFallbackNoted { + /** This is the first such session through the router: warn about it. */ + bool warn; + /** From now on sessions through the router start with 4,0. */ + bool remembered; +}; + +/** + * A session through `router` was created with 4,0 after the router rejected 6,4 naming the + * encryption type (see `NamesEncType()`), evidence that it lacks 6,4. A rejection that does not + * name it may be a passing router error and must not be noted here. + */ +HybridFallbackNoted NoteHybridFallback(const std::string& router) +{ + LOCK(g_router_hybrid_mutex); + auto it{g_router_hybrid.find(router)}; + if (it == g_router_hybrid.end()) { + if (g_router_hybrid.size() >= SAM_MAX_REMEMBERED_ROUTERS) { + g_router_hybrid.erase(std::ranges::min_element(g_router_hybrid, {}, [](const auto& entry) { return entry.second.updated; })); + } + it = g_router_hybrid.emplace(router, RouterHybridState{}).first; + } + RouterHybridState& state{it->second}; + state.updated = ++g_router_hybrid_updates; + const bool was_remembered{state.fallbacks >= SAM_HYBRID_FALLBACKS_TO_REMEMBER}; + state.fallbacks = std::min(state.fallbacks + 1, SAM_HYBRID_FALLBACKS_TO_REMEMBER); + return {.warn = !std::exchange(state.warned, true), + .remembered = !was_remembered && state.fallbacks >= SAM_HYBRID_FALLBACKS_TO_REMEMBER}; +} + +/** + * Whether a non-OK RESULT of "SESSION STATUS" can be about the leaseset encryption type. + * These cannot: the session id or the destination is the problem. + */ +bool MayBeEncTypeRejection(std::string_view result) +{ + return result != "DUPLICATED_ID" && result != "DUPLICATED_DEST" && result != "INVALID_ID" && + result != "INVALID_KEY"; +} + +/** + * `result` if it is one of the RESULT values of the SAM v3 protocol, otherwise "other". + */ +std::string_view KnownSamResult(std::string_view result) +{ + static constexpr std::array known{ + "OK", "CANT_REACH_PEER", "DUPLICATED_ID", "DUPLICATED_DEST", "I2P_ERROR", + "INVALID_ID", "INVALID_KEY", "KEY_NOT_FOUND", "PEER_NOT_FOUND", "TIMEOUT", + "NOVERSION", "LEASESET_NOT_FOUND", "ALREADY_ACCEPTING"}; + return std::ranges::find(known, result) != known.end() ? result : "other"; +} + +/** + * Router-supplied text, made safe and short enough for the log. + */ +std::string LoggableReplyValue(std::string_view value) +{ + return SanitizeString(value.substr(0, MAX_LOGGED_REPLY_VALUE)); +} + +/** + * The MESSAGE= value of a SAM reply, which may be quoted and contain spaces. It is router text, not + * safe for the log as is. + */ +std::string_view ReplyMessage(std::string_view reply) +{ + static constexpr std::string_view key{" MESSAGE="}; + const auto pos{reply.find(key)}; + if (pos == std::string_view::npos) { + return ""; + } + std::string_view value{reply.substr(pos + key.size())}; + if (value.starts_with('"')) { + value.remove_prefix(1); + return value.substr(0, value.find('"')); + } + return value.substr(0, value.find(' ')); +} + +/** + * The MESSAGE= value of a SAM reply, which may be quoted and contain spaces, ready for the log. + */ +std::string LoggableReplyMessage(std::string_view reply) +{ + return LoggableReplyValue(ReplyMessage(reply)); +} + +/** + * Whether the MESSAGE= value of a rejection names the encryption type, as Java I2P older than + * 2.10.0 does when asked for 6,4: "... Unsupported crypto type: 6". Only that is evidence that + * the router lacks 6,4; another rejection may be a passing router error. Matched before the + * value is sanitized or capped for the log, and case-insensitively for ASCII. + */ +bool NamesEncType(std::string_view message) +{ + static constexpr std::array phrases{"crypto type", "encryption type", "enc type", "enctype"}; + const std::string lower{ToLower(message)}; + return std::ranges::any_of(phrases, [&](std::string_view phrase) { return lower.find(phrase) != std::string::npos; }); +} +} // namespace + +void ResetRouterHybridStateForTest() +{ + LOCK(g_router_hybrid_mutex); + g_router_hybrid.clear(); + g_router_hybrid_updates = 0; +} + Session::Session(const fs::path& private_key_file, const Proxy& control_host, - CThreadInterrupt* interrupt) + CThreadInterrupt* interrupt, + std::chrono::steady_clock::duration create_timeout) : m_private_key_file{private_key_file}, m_control_host{control_host}, m_interrupt{interrupt}, + m_create_timeout{create_timeout}, m_transient{false} { } -Session::Session(const Proxy& control_host, CThreadInterrupt* interrupt) +Session::Session(const Proxy& control_host, + CThreadInterrupt* interrupt, + std::chrono::steady_clock::duration create_timeout) : m_control_host{control_host}, m_interrupt{interrupt}, + m_create_timeout{create_timeout}, m_transient{true} { } @@ -294,20 +480,36 @@ Session::Reply Session::SendRequestAndGetReply(const Sock& sock, const std::string& request, bool check_result_ok) const { - sock.SendComplete(request + "\n", MAX_WAIT_FOR_IO, *m_interrupt); + return SendRequestAndGetReply(sock, request, check_result_ok, SAMDeadline::max()); +} +Session::Reply Session::SendRequestAndGetReply(const Sock& sock, + const std::string& request, + bool check_result_ok, + SAMDeadline deadline) const +{ Reply reply; // Don't log the full "SESSION CREATE ..." because it contains our private key. reply.request = request.starts_with("SESSION CREATE") ? "SESSION CREATE ..." : request; + const auto time_left = [&]() -> std::chrono::milliseconds { + const auto now{std::chrono::steady_clock::now()}; + if (now >= deadline) { + throw std::runtime_error(strprintf("Timed out during \"%s\"", reply.request)); + } + return std::chrono::ceil(deadline - now); + }; + + sock.SendComplete(request + "\n", std::min(MAX_WAIT_FOR_IO, time_left()), *m_interrupt); + // It could take a few minutes for the I2P router to reply as it is querying the I2P network // (when doing name lookup, for example). Notice: `RecvUntilTerminator()` is checking // `m_interrupt` more often, so we would not be stuck here for long if `m_interrupt` is // signaled. static constexpr auto recv_timeout = 3min; - reply.full = sock.RecvUntilTerminator('\n', recv_timeout, *m_interrupt, MAX_MSG_SIZE); + reply.full = sock.RecvUntilTerminator('\n', std::min(recv_timeout, time_left()), *m_interrupt, MAX_MSG_SIZE); for (const auto& kv : Split(reply.full, ' ')) { const auto pos{std::ranges::find(kv, '=')}; @@ -320,7 +522,7 @@ Session::Reply Session::SendRequestAndGetReply(const Sock& sock, if (check_result_ok && reply.Get("RESULT") != "OK") { throw std::runtime_error( - strprintf("Unexpected reply to \"%s\": \"%s\"", request, reply.full)); + strprintf("Unexpected reply to \"%s\": \"%s\"", reply.request, reply.full)); } return reply; @@ -339,6 +541,29 @@ std::unique_ptr Session::Hello() const return sock; } +std::unique_ptr Session::Hello(SAMDeadline deadline) const +{ + // Like Proxy::Connect() without a deadline, give up connecting after -timeout. + const SAMDeadline connect_deadline{ + std::min(deadline, std::chrono::steady_clock::now() + std::chrono::milliseconds{nConnectTimeout})}; + + auto sock = m_control_host.Connect(connect_deadline, *m_interrupt); + + if (!sock) { + if (*m_interrupt) { + throw std::runtime_error(strprintf("Interrupted while connecting to %s", m_control_host.ToString())); + } + if (std::chrono::steady_clock::now() >= deadline) { + throw std::runtime_error(strprintf("Timed out connecting to %s", m_control_host.ToString())); + } + throw std::runtime_error(strprintf("Cannot connect to %s", m_control_host.ToString())); + } + + SendRequestAndGetReply(*sock, "HELLO VERSION MIN=3.1 MAX=3.1", /*check_result_ok=*/true, deadline); + + return sock; +} + void Session::CheckControlSock() { LOCK(m_mutex); @@ -350,20 +575,20 @@ void Session::CheckControlSock() } } -void Session::DestGenerate(const Sock& sock) +void Session::DestGenerate(const Sock& sock, SAMDeadline deadline) { // https://geti2p.net/spec/common-structures#key-certificates // "7" or "EdDSA_SHA512_Ed25519" - "Recent Router Identities and Destinations". // Use "7" because i2pd <2.24.0 does not recognize the textual form. // If SIGNATURE_TYPE is not specified, then the default one is DSA_SHA1. - const Reply& reply = SendRequestAndGetReply(sock, "DEST GENERATE SIGNATURE_TYPE=7", false); + const Reply& reply = SendRequestAndGetReply(sock, "DEST GENERATE SIGNATURE_TYPE=7", false, deadline); m_private_key = DecodeI2PBase64(reply.Get("PRIV")); } -void Session::GenerateAndSavePrivateKey(const Sock& sock) +void Session::GenerateAndSavePrivateKey(const Sock& sock, SAMDeadline deadline) { - DestGenerate(sock); + DestGenerate(sock, deadline); // umask is set to 0077 in common/system.cpp, which is ok. if (!WriteBinaryFile(m_private_key_file, @@ -413,44 +638,141 @@ void Session::CreateIfNotCreatedAlready() } const auto session_type = m_transient ? "transient" : "persistent"; - const auto session_id = GetRandHash().GetHex().substr(0, 10); // full is overkill, too verbose in the logs + const SAMDeadline deadline{std::chrono::steady_clock::now() + m_create_timeout}; + const std::string router{m_control_host.ToString()}; + + bool hybrid{!RouterLacksHybrid(router)}; + if (!hybrid) { + LogPrintLevel(BCLog::I2P, BCLog::Level::Debug, "%s rejected leaseSetEncType=%s in earlier sessions, using %s\n", + router, LEASESET_ENC_TYPES_HYBRID, LEASESET_ENC_TYPES_LEGACY); + } + // The rejection of 6,4, if this is the retry with 4,0. + struct HybridRejection { + std::string result; + /** MESSAGE=, ready for the log. */ + std::string message; + /** Whether MESSAGE= names the encryption type, see `NamesEncType()`. */ + bool names_enc_type; + }; + std::optional hybrid_rejection; + + // Our persistent destination (private key), read or generated once and kept for a retry. + std::optional private_key_b64; + + for (unsigned attempt{1};; ++attempt) { + const auto session_id = GetRandHash().GetHex().substr(0, 10); // full is overkill, too verbose in the logs + const std::string_view enc_types{hybrid ? LEASESET_ENC_TYPES_HYBRID : LEASESET_ENC_TYPES_LEGACY}; + + LogPrintLevel(BCLog::I2P, BCLog::Level::Debug, "Creating %s SAM session %s with %s\n", session_type, session_id, router); + + auto sock = Hello(deadline); + + std::string request; + if (m_transient) { + // The destination (private key) is generated upon session creation and returned + // in the reply in DESTINATION=. + request = strprintf("SESSION CREATE STYLE=STREAM ID=%s DESTINATION=TRANSIENT SIGNATURE_TYPE=7 " + "i2cp.leaseSetEncType=%s inbound.quantity=1 outbound.quantity=1", + session_id, + enc_types); + } else { + if (!private_key_b64) { + // Read our persistent destination (private key) from disk or generate + // one and save it to disk. Then use it when creating the session. + const auto& [read_ok, data] = ReadBinaryFile(m_private_key_file); + if (read_ok) { + m_private_key.assign(data.begin(), data.end()); + } else { + GenerateAndSavePrivateKey(*sock, deadline); + } + private_key_b64 = SwapBase64(EncodeBase64(m_private_key)); + } + request = strprintf("SESSION CREATE STYLE=STREAM ID=%s DESTINATION=%s " + "i2cp.leaseSetEncType=%s inbound.quantity=3 outbound.quantity=3", + session_id, + *private_key_b64, + enc_types); + } - LogPrintLevel(BCLog::I2P, BCLog::Level::Debug, "Creating %s SAM session %s with %s\n", session_type, session_id, m_control_host.ToString()); + const Reply reply = SendRequestAndGetReply(*sock, request, /*check_result_ok=*/false, deadline); - auto sock = Hello(); + // A reply without RESULT= is not a rejection, this throws. + const std::string result{reply.Get("RESULT")}; - if (m_transient) { - // The destination (private key) is generated upon session creation and returned - // in the reply in DESTINATION=. - const Reply& reply = SendRequestAndGetReply( - *sock, - strprintf("SESSION CREATE STYLE=STREAM ID=%s DESTINATION=TRANSIENT SIGNATURE_TYPE=7 " - "i2cp.leaseSetEncType=4,0 inbound.quantity=1 outbound.quantity=1", - session_id)); + if (result == "OK") { + if (m_transient) { + m_private_key = DecodeI2PBase64(reply.Get("DESTINATION")); + } + const CService my_addr{DestBinToAddr(MyDestination()), I2P_SAM31_PORT}; - m_private_key = DecodeI2PBase64(reply.Get("DESTINATION")); - } else { - // Read our persistent destination (private key) from disk or generate - // one and save it to disk. Then use it when creating the session. - const auto& [read_ok, data] = ReadBinaryFile(m_private_key_file); - if (read_ok) { - m_private_key.assign(data.begin(), data.end()); - } else { - GenerateAndSavePrivateKey(*sock); + m_my_addr = my_addr; + m_session_id = session_id; + m_control_sock = std::move(sock); + break; } - const std::string& private_key_b64 = SwapBase64(EncodeBase64(m_private_key)); + // qbit-only, upstream Bitcoin Core does not retry: Java I2P older than 2.10.0 fails the + // session if asked for the hybrid type, so retry once without it. Any explicit rejection, + // a well-formed "SESSION STATUS" whose RESULT may be about the encryption type (including + // RESULT=TIMEOUT), is retried; whether it counts against the router is decided below. No + // reply (a timeout waiting for one, EOF, a socket error) or a malformed reply never is. + const bool retry{hybrid && attempt < SAM_CREATE_MAX_ATTEMPTS && + reply.full.starts_with("SESSION STATUS ") && !result.empty() && + MayBeEncTypeRejection(result)}; + if (!retry) { + throw std::runtime_error(strprintf("Unexpected reply to \"%s\": \"%s\"", reply.request, reply.full)); + } - SendRequestAndGetReply(*sock, - strprintf("SESSION CREATE STYLE=STREAM ID=%s DESTINATION=%s " - "i2cp.leaseSetEncType=4,0 inbound.quantity=3 outbound.quantity=3", - session_id, - private_key_b64)); + hybrid_rejection = HybridRejection{.result = result, + .message = LoggableReplyMessage(reply.full), + .names_enc_type = NamesEncType(ReplyMessage(reply.full))}; + + LogPrintLevel(BCLog::I2P, BCLog::Level::Debug, + "%s rejected SAM session %s with leaseSetEncType=%s: RESULT=%s MESSAGE=\"%s\", retrying with %s\n", + router, + session_id, + LEASESET_ENC_TYPES_HYBRID, + LoggableReplyValue(result), + hybrid_rejection->message, + LEASESET_ENC_TYPES_LEGACY); + + hybrid = false; + // `sock` is closed here, the retry uses a fresh connection, HELLO and session id. } - m_my_addr = CService(DestBinToAddr(MyDestination()), I2P_SAM31_PORT); - m_session_id = session_id; - m_control_sock = std::move(sock); + if (hybrid_rejection && hybrid_rejection->names_enc_type) { + // Only such rejections in a row turn 6,4 off for the router: one may be a transient error. + const auto noted{NoteHybridFallback(router)}; + if (noted.warn) { + LogWarning("I2P: the router rejected leaseSetEncType=6,4 (RESULT=%s) and accepted 4,0, so this session has no post-quantum leaseset. Java I2P older than 2.10.0 does this; upgrade the router to get it.\n", + KnownSamResult(hybrid_rejection->result)); + } + if (noted.remembered) { + LogPrintLevel(BCLog::I2P, BCLog::Level::Info, + "%s rejected leaseSetEncType=%s and accepted %s in %u sessions in a row, later sessions through it use %s until qbit restarts\n", + router, + LEASESET_ENC_TYPES_HYBRID, + LEASESET_ENC_TYPES_LEGACY, + SAM_HYBRID_FALLBACKS_TO_REMEMBER, + LEASESET_ENC_TYPES_LEGACY); + } + } else if (hybrid_rejection) { + // A rejection that does not name the encryption type may be a passing router error + // (e.g. failing to build tunnels): it is no evidence either way, so the count of + // rejections in a row neither grows nor starts anew, and there is no warning. + LogPrintLevel(BCLog::I2P, BCLog::Level::Info, + "%s SAM session %s uses leaseSetEncType=%s because %s rejected %s with RESULT=%s MESSAGE=\"%s\", which does not name the encryption type; the next session asks for %s again\n", + Capitalize(session_type), + m_session_id, + LEASESET_ENC_TYPES_LEGACY, + router, + LEASESET_ENC_TYPES_HYBRID, + KnownSamResult(hybrid_rejection->result), + hybrid_rejection->message, + LEASESET_ENC_TYPES_HYBRID); + } else if (hybrid) { + NoteHybridAccepted(router); + } LogPrintLevel(BCLog::I2P, BCLog::Level::Info, "%s SAM session %s created, my address=%s\n", Capitalize(session_type), diff --git a/src/i2p.h b/src/i2p.h index d7b2f84d6e..62f2b4f9d3 100644 --- a/src/i2p.h +++ b/src/i2p.h @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -50,6 +51,43 @@ namespace sam { */ static constexpr size_t MAX_MSG_SIZE{65536}; +/** + * Point in time by which a multi-step exchange with the SAM proxy must complete. + */ +using SAMDeadline = std::chrono::steady_clock::time_point; + +/** + * Bound on creating a SAM session. It covers connecting to the SAM proxy, "HELLO", the + * optional "DEST GENERATE" and every "SESSION CREATE" attempt together. + */ +static constexpr auto SAM_CREATE_TIMEOUT = std::chrono::seconds{180}; + +/** + * Most "SESSION CREATE" attempts when creating one session: the first one with + * i2cp.leaseSetEncType=6,4 and, if the router explicitly rejects it, one with 4,0. + */ +static constexpr unsigned SAM_CREATE_MAX_ATTEMPTS = 2; + +/** + * Sessions in a row in which a router rejects i2cp.leaseSetEncType=6,4 naming the encryption type + * and accepts 4,0, after which later sessions through it start with 4,0 until qbit restarts. A + * session created with 6,4 starts the count anew, so one transient rejection does not turn 6,4 + * off. A session whose rejection does not name the encryption type neither counts nor starts the + * count anew. + */ +static constexpr unsigned SAM_HYBRID_FALLBACKS_TO_REMEMBER = 2; + +/** + * Most SAM routers whose rejections of i2cp.leaseSetEncType=6,4 are remembered. Beyond that, + * the least recently updated router is forgotten. + */ +static constexpr size_t SAM_MAX_REMEMBERED_ROUTERS = 64; + +/** + * Forget the rejections of i2cp.leaseSetEncType=6,4 remembered for all routers. Only for tests. + */ +void ResetRouterHybridStateForTest(); + /** * I2P SAM session. */ @@ -66,10 +104,12 @@ class Session * possible and executing methods throw an exception. Notice: only a pointer to the * `CThreadInterrupt` object is saved, so it must not be destroyed earlier than this * `Session` object. + * @param[in] create_timeout Bound on creating the SAM session, see `SAM_CREATE_TIMEOUT`. */ Session(const fs::path& private_key_file, const Proxy& control_host, - CThreadInterrupt* interrupt); + CThreadInterrupt* interrupt, + std::chrono::steady_clock::duration create_timeout = SAM_CREATE_TIMEOUT); /** * Construct a transient session which will generate its own I2P private key @@ -81,8 +121,11 @@ class Session * possible and executing methods throw an exception. Notice: only a pointer to the * `CThreadInterrupt` object is saved, so it must not be destroyed earlier than this * `Session` object. + * @param[in] create_timeout Bound on creating the SAM session, see `SAM_CREATE_TIMEOUT`. */ - Session(const Proxy& control_host, CThreadInterrupt* interrupt); + Session(const Proxy& control_host, + CThreadInterrupt* interrupt, + std::chrono::steady_clock::duration create_timeout = SAM_CREATE_TIMEOUT); /** * Destroy the session, closing the internally used sockets. The sockets that have been @@ -168,6 +211,21 @@ class Session const std::string& request, bool check_result_ok = true) const; + /** + * Send request and get a reply from the SAM proxy, giving up at `deadline`. + * Like the overload above, except that sending and receiving also end at `deadline`. + * @param[in] sock A socket that is connected to the SAM proxy. + * @param[in] request Raw request to send, a newline terminator is appended to it. + * @param[in] check_result_ok If true then after receiving the reply a check is made + * whether it contains "RESULT=OK" and an exception is thrown if it does not. + * @param[in] deadline Give up at this time. + * @throws std::runtime_error if an error occurs, `deadline` passes or `m_interrupt` is signaled + */ + Reply SendRequestAndGetReply(const Sock& sock, + const std::string& request, + bool check_result_ok, + SAMDeadline deadline) const; + /** * Open a new connection to the SAM proxy. * @return a connected socket @@ -175,6 +233,15 @@ class Session */ std::unique_ptr Hello() const EXCLUSIVE_LOCKS_REQUIRED(m_mutex); + /** + * Open a new connection to the SAM proxy, giving up at `deadline` or when `m_interrupt` is + * signaled. Connecting is also limited by `nConnectTimeout`, whichever ends first. + * @param[in] deadline Give up at this time. + * @return a connected socket + * @throws std::runtime_error if an error occurs + */ + std::unique_ptr Hello(SAMDeadline deadline) const EXCLUSIVE_LOCKS_REQUIRED(m_mutex); + /** * Check the control socket for errors and possibly disconnect. */ @@ -183,17 +250,19 @@ class Session /** * Generate a new destination with the SAM proxy and set `m_private_key` to it. * @param[in] sock Socket to use for talking to the SAM proxy. + * @param[in] deadline Give up at this time. * @throws std::runtime_error if an error occurs */ - void DestGenerate(const Sock& sock) EXCLUSIVE_LOCKS_REQUIRED(m_mutex); + void DestGenerate(const Sock& sock, SAMDeadline deadline) EXCLUSIVE_LOCKS_REQUIRED(m_mutex); /** * Generate a new destination with the SAM proxy, set `m_private_key` to it and save * it on disk to `m_private_key_file`. * @param[in] sock Socket to use for talking to the SAM proxy. + * @param[in] deadline Give up at this time. * @throws std::runtime_error if an error occurs */ - void GenerateAndSavePrivateKey(const Sock& sock) EXCLUSIVE_LOCKS_REQUIRED(m_mutex); + void GenerateAndSavePrivateKey(const Sock& sock, SAMDeadline deadline) EXCLUSIVE_LOCKS_REQUIRED(m_mutex); /** * Derive own destination from `m_private_key`. @@ -204,7 +273,9 @@ class Session /** * Create the session if not already created. Reads the private key file and connects to the - * SAM proxy. + * SAM proxy. Asks for i2cp.leaseSetEncType=6,4 unless this router is known to reject it, and + * retries once with 4,0 if the router explicitly rejects it. All of it must complete within + * `m_create_timeout`. `m_session_id` and `m_control_sock` are set only on success. * @throws std::runtime_error if an error occurs */ void CreateIfNotCreatedAlready() EXCLUSIVE_LOCKS_REQUIRED(m_mutex); @@ -237,6 +308,11 @@ class Session */ CThreadInterrupt* const m_interrupt; + /** + * Bound on creating the SAM session, see `SAM_CREATE_TIMEOUT`. + */ + const std::chrono::steady_clock::duration m_create_timeout; + /** * Mutex protecting the members that can be concurrently accessed. */ diff --git a/src/netbase.cpp b/src/netbase.cpp index 7c93b27c3e..d5f21b4f4d 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -591,7 +591,23 @@ static void LogConnectFailure(bool manual_connection, util::ConstevalFormatStrin } } -static bool ConnectToSocket(const Sock& sock, struct sockaddr* sockaddr, socklen_t len, const std::string& dest_str, bool manual_connection) +namespace { +/** Limits of a connection attempt made by `Proxy::Connect(deadline, interrupt)`. */ +struct ConnectBound { + /** Give up at this time. */ + std::chrono::steady_clock::time_point deadline; + /** Give up as soon as this is signaled. */ + CThreadInterrupt& interrupt; +}; +} // namespace + +/** + * Connect to `sockaddr` using `sock`. + * @param[in] bound If null, wait at most `nConnectTimeout` for the connection to be established. + * Otherwise wait until `bound->deadline` in slices of at most `MAX_CONNECT_POLL_INTERVAL` and + * give up as soon as `bound->interrupt` is signaled. + */ +static bool ConnectToSocket(const Sock& sock, struct sockaddr* sockaddr, socklen_t len, const std::string& dest_str, bool manual_connection, const ConnectBound* bound) { // Connect to `sockaddr` using `sock`. if (sock.Connect(sockaddr, len) == SOCKET_ERROR) { @@ -603,13 +619,34 @@ static bool ConnectToSocket(const Sock& sock, struct sockaddr* sockaddr, socklen // asynchronously. Thus, use async I/O api (select/poll) // synchronously to check for successful connection with a timeout. const Sock::Event requested = Sock::RECV | Sock::SEND; - Sock::Event occurred; - if (!sock.Wait(std::chrono::milliseconds{nConnectTimeout}, requested, &occurred)) { - LogPrintf("wait for connect to %s failed: %s\n", - dest_str, - NetworkErrorString(WSAGetLastError())); - return false; - } else if (occurred == 0) { + Sock::Event occurred{0}; + if (bound == nullptr) { + if (!sock.Wait(std::chrono::milliseconds{nConnectTimeout}, requested, &occurred)) { + LogPrintf("wait for connect to %s failed: %s\n", + dest_str, + NetworkErrorString(WSAGetLastError())); + return false; + } + } else { + // Wait in short slices, so that an interrupt is noticed promptly. + while (occurred == 0) { + if (bound->interrupt) { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "connection attempt to %s interrupted\n", dest_str); + return false; + } + const auto now{std::chrono::steady_clock::now()}; + if (now >= bound->deadline) break; + const auto slice{std::min(std::chrono::ceil(bound->deadline - now), + MAX_CONNECT_POLL_INTERVAL)}; + if (!sock.Wait(slice, requested, &occurred)) { + LogPrintf("wait for connect to %s failed: %s\n", + dest_str, + NetworkErrorString(WSAGetLastError())); + return false; + } + } + } + if (occurred == 0) { LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "connection attempt to %s timed out\n", dest_str); return false; } @@ -646,7 +683,7 @@ static bool ConnectToSocket(const Sock& sock, struct sockaddr* sockaddr, socklen return true; } -std::unique_ptr ConnectDirectly(const CService& dest, bool manual_connection) +static std::unique_ptr ConnectDirectly(const CService& dest, bool manual_connection, const ConnectBound* bound) { auto sock = CreateSock(dest.GetSAFamily(), SOCK_STREAM, IPPROTO_TCP); if (!sock) { @@ -662,27 +699,32 @@ std::unique_ptr ConnectDirectly(const CService& dest, bool manual_connecti return {}; } - if (!ConnectToSocket(*sock, (struct sockaddr*)&sockaddr, len, dest.ToStringAddrPort(), manual_connection)) { + if (!ConnectToSocket(*sock, (struct sockaddr*)&sockaddr, len, dest.ToStringAddrPort(), manual_connection, bound)) { return {}; } return sock; } -std::unique_ptr Proxy::Connect() const +std::unique_ptr ConnectDirectly(const CService& dest, bool manual_connection) +{ + return ConnectDirectly(dest, manual_connection, /*bound=*/nullptr); +} + +static std::unique_ptr ConnectToProxy(const Proxy& proxy, const ConnectBound* bound) { - if (!IsValid()) return {}; + if (!proxy.IsValid()) return {}; - if (!m_is_unix_socket) return ConnectDirectly(proxy, /*manual_connection=*/true); + if (!proxy.m_is_unix_socket) return ConnectDirectly(proxy.proxy, /*manual_connection=*/true, bound); #ifdef HAVE_SOCKADDR_UN auto sock = CreateSock(AF_UNIX, SOCK_STREAM, 0); if (!sock) { - LogPrintLevel(BCLog::NET, BCLog::Level::Error, "Cannot create a socket for connecting to %s\n", m_unix_socket_path); + LogPrintLevel(BCLog::NET, BCLog::Level::Error, "Cannot create a socket for connecting to %s\n", proxy.m_unix_socket_path); return {}; } - const std::string path{m_unix_socket_path.substr(ADDR_PREFIX_UNIX.length())}; + const std::string path{proxy.m_unix_socket_path.substr(ADDR_PREFIX_UNIX.length())}; struct sockaddr_un addrun; memset(&addrun, 0, sizeof(addrun)); @@ -691,7 +733,7 @@ std::unique_ptr Proxy::Connect() const memcpy(addrun.sun_path, path.c_str(), std::min(sizeof(addrun.sun_path) - 1, path.length())); socklen_t len = sizeof(addrun); - if(!ConnectToSocket(*sock, (struct sockaddr*)&addrun, len, path, /*manual_connection=*/true)) { + if(!ConnectToSocket(*sock, (struct sockaddr*)&addrun, len, path, /*manual_connection=*/true, bound)) { return {}; } @@ -701,6 +743,19 @@ std::unique_ptr Proxy::Connect() const #endif } +std::unique_ptr Proxy::Connect() const +{ + return ConnectToProxy(*this, /*bound=*/nullptr); +} + +std::unique_ptr Proxy::Connect(std::chrono::steady_clock::time_point deadline, + CThreadInterrupt& interrupt) const +{ + if (interrupt || std::chrono::steady_clock::now() >= deadline) return {}; + const ConnectBound bound{.deadline = deadline, .interrupt = interrupt}; + return ConnectToProxy(*this, &bound); +} + bool SetProxy(enum Network net, const Proxy &addrProxy) { assert(net >= 0 && net < NET_MAX); if (!addrProxy.IsValid()) diff --git a/src/netbase.h b/src/netbase.h index 031e981566..18f6522790 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -27,6 +28,12 @@ static const int DEFAULT_CONNECT_TIMEOUT = 5000; //! -dns default static const int DEFAULT_NAME_LOOKUP = true; +/** + * Longest single wait of `Proxy::Connect(deadline, interrupt)` for the connection to be + * established, before it checks the interrupt and the deadline again. + */ +static constexpr std::chrono::milliseconds MAX_CONNECT_POLL_INTERVAL{100}; + /** Prefix for unix domain socket addresses (which are local filesystem paths) */ const std::string ADDR_PREFIX_UNIX = "unix:"; @@ -86,6 +93,18 @@ class Proxy } std::unique_ptr Connect() const; + + /** + * Connect to the proxy, giving up at `deadline` or as soon as `interrupt` is signaled. + * Unlike `Connect()`, this does not use `nConnectTimeout`. It waits for the connection in + * slices of at most `MAX_CONNECT_POLL_INTERVAL`, so it returns promptly after an interrupt. + * No socket is created if `interrupt` is already signaled or `deadline` has already passed. + * @param[in] deadline Give up at this time. + * @param[in] interrupt Give up as soon as this is signaled. + * @returns the connected socket if the operation succeeded, empty unique_ptr otherwise + */ + std::unique_ptr Connect(std::chrono::steady_clock::time_point deadline, + CThreadInterrupt& interrupt) const; }; /** Credentials for proxy authentication */ diff --git a/src/test/fuzz/i2p.cpp b/src/test/fuzz/i2p.cpp index 29a11123d9..38c2805d94 100644 --- a/src/test/fuzz/i2p.cpp +++ b/src/test/fuzz/i2p.cpp @@ -22,6 +22,8 @@ void initialize_i2p() FUZZ_TARGET(i2p, .init = initialize_i2p) { SeedRandomStateForTest(SeedRand::ZEROS); + // Forget routers' rejections of the post-quantum leaseset type from earlier inputs. + i2p::sam::ResetRouterHybridStateForTest(); FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; SetMockTime(ConsumeTime(fuzzed_data_provider)); diff --git a/src/test/i2p_tests.cpp b/src/test/i2p_tests.cpp index bdb3408b66..3fb8bf94ec 100644 --- a/src/test/i2p_tests.cpp +++ b/src/test/i2p_tests.cpp @@ -2,23 +2,51 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include // IWYU pragma: keep + #include #include #include #include #include +#include #include #include #include +#include #include +#include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include +#include +#include #include +#include +#include +#include +#include + +#ifdef HAVE_SOCKADDR_UN +#include +#endif + +using namespace std::chrono_literals; -/// Save the log level and the value of CreateSock and restore them when the test ends. +/// Save the log level and the value of CreateSock and restore them when the test ends. Forget the +/// routers' rejections of the post-quantum leaseset type before and after. class EnvTestingSetup : public BasicTestingSetup { public: @@ -29,10 +57,12 @@ class EnvTestingSetup : public BasicTestingSetup m_create_sock_orig{CreateSock} { LogInstance().SetLogLevel(BCLog::Level::Trace); + i2p::sam::ResetRouterHybridStateForTest(); } ~EnvTestingSetup() { + i2p::sam::ResetRouterHybridStateForTest(); CreateSock = m_create_sock_orig; LogInstance().SetLogLevel(m_prev_log_level); } @@ -42,6 +72,471 @@ class EnvTestingSetup : public BasicTestingSetup const decltype(CreateSock) m_create_sock_orig; }; +namespace { +/** A valid I2P private key (destination and its keys), in I2P-style Base64. */ +const std::string TEST_PRIV{"WnGOLXRBqHQhdVjFlWqRxJwz9hxx~2~wGc2Vplta1KhacY4tdEGodCF1WMWVapHEnDP2HHH~b~AZzZWmW1rUqFpxji10Qah0IXVYxZVqkcScM~Yccf9v8BnNlaZbWtSoWnGOLXRBqHQhdVjFlWqRxJwz9hxx~2~wGc2Vplta1KhacY4tdEGodCF1WMWVapHEnDP2HHH~b~AZzZWmW1rUqFpxji10Qah0IXVYxZVqkcScM~Yccf9v8BnNlaZbWtSoWnGOLXRBqHQhdVjFlWqRxJwz9hxx~2~wGc2Vplta1KhacY4tdEGodCF1WMWVapHEnDP2HHH~b~AZzZWmW1rUqFpxji10Qah0IXVYxZVqkcScM~Yccf9v8BnNlaZbWtSoWnGOLXRBqHQhdVjFlWqRxJwz9hxx~2~wGc2Vplta1KhacY4tdEGodCF1WMWVapHEnDP2HHH~b~AZzZWmW1rUqLE4SD-yjT48UNI7qiTUfIPiDitCoiTTz2cr4QGfw89rBQAEAAcAAOvuCIKTyv5f~1QgGq7XQl-IqBULTB5WzB3gw5yGPtd1p0AeoADrq1ccZggLPQ4ZLUsGK-HVw373rcTfvxrcuwenqVjiN4tbbYLWtP7xXGWj6fM6HyORhU63GphrjEePpMUHDHXd3o7pWGM-ieVVQSK~1MzF9P93pQWI3Do52EeNAayz4HbpPjNhVBzG1hUEFwznfPmUZBPuaOR4-uBm1NEWEuONlNOCctE4-U0Ukh94z-Qb55U5vXjR5G4apmBblr68t6Wm1TKlzpgFHzSqLryh3stWqrOKY1H0z9eZ2z1EkHFOpD5LyF6nf51e-lV7HLMl44TYzoEHK8RRVodtLcW9lacVdBpv~tOzlZERIiDziZODPETENZMz5oy9DQ7UUw=="}; + +/** The start of the warning about a router that rejects the post-quantum leaseset type. */ +const std::string HYBRID_WARNING{"I2P: the router rejected leaseSetEncType=6,4"}; + +/** The full warning, for a rejection with `result`. */ +std::string HybridWarning(const std::string& result) +{ + return strprintf("I2P: the router rejected leaseSetEncType=6,4 (RESULT=%s) and accepted 4,0, so this session has " + "no post-quantum leaseset. Java I2P older than 2.10.0 does this; upgrade the router to get it.", + result); +} + +/** The reply of Java I2P 2.9.0, which lacks the post-quantum leaseset type, to a request for it. */ +const std::string JAVA_I2P_2_9_0_REJECTION{"SESSION STATUS RESULT=I2P_ERROR MESSAGE=\"Error creating I2PSocketManager: " + "[SAM TCP Client(CLOSED)]: Failed to build tunnels - Disconnected from router " + "while waiting for tunnels: Unsupported crypto type: 6\""}; + +/** Its MESSAGE=, as logged. */ +const std::string JAVA_I2P_2_9_0_MESSAGE_LOGGED{"Error creating I2PSocketManager: SAM TCP Client(CLOSED): Failed to build tunnels - " + "Disconnected from router while waiting for tunnels: Unsupported crypto type: 6"}; + +/** A rejection like a passing router error: it does not name the encryption type. */ +const std::string TUNNEL_BUILD_FAILURE{"SESSION STATUS RESULT=I2P_ERROR MESSAGE=\"Failed to build tunnels\""}; + +/** Part of the info line about a session that uses 4,0 after a rejection that does not name the encryption type. */ +const std::string UNNAMED_REJECTION{"which does not name the encryption type"}; + +/** The full info line, for transient session `session_id` through `router`. */ +std::string UnnamedRejectionInfo(const Proxy& router, const std::string& session_id, const std::string& result, const std::string& message) +{ + return strprintf("Transient SAM session %s uses leaseSetEncType=4,0 because %s rejected 6,4 with RESULT=%s MESSAGE=\"%s\", " + "which does not name the encryption type; the next session asks for 6,4 again", + session_id, router.ToString(), result, message); +} + +/** Errors when creating a session runs out of time, depending on the step it was at. */ +const std::vector TIMEOUT_ERRORS{"Receive timeout", "Timed out during", "Timed out connecting to"}; + +void SetSockError(int err) +{ +#ifdef WIN32 + WSASetLastError(err); +#else + errno = err; +#endif +} + +#ifdef WIN32 +constexpr int SOCK_ERROR_IN_PROGRESS{WSAEWOULDBLOCK}; +constexpr int SOCK_ERROR_RESET{WSAECONNRESET}; +#else +constexpr int SOCK_ERROR_IN_PROGRESS{EINPROGRESS}; +constexpr int SOCK_ERROR_RESET{ECONNRESET}; +#endif + +/** + * Recording mock of SAM routers. Every socket from `NewSock()` is one connection. The router + * it connects to, named like `Proxy::ToString()`, answers each request as its script says. + * Every request is recorded. + */ +class MockSamRouters +{ +public: + enum class Action { + REPLY, //!< Reply with `line` after `delay`. + STALL, //!< Never reply. + CLOSE, //!< Close the connection. + FAIL_SEND, //!< Fail sending the request, as if the connection was reset. + }; + + struct Response { + Action action{Action::REPLY}; + std::string line{}; + std::chrono::milliseconds delay{0}; + }; + + /** Answers `request` received on the router's connection number `conn` (0-based). */ + using Script = std::function; + + struct Router { + Script script; + /** Connections (by number) whose connect never completes. */ + std::set pending_connects{}; + /** Called with the 1-based number of every wait for a pending connect. */ + std::function on_connect_wait{}; + }; + + void AddRouter(const std::string& name, Router router) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) + { + LOCK(m_mutex); + m_routers.insert_or_assign(name, RouterState{.router = std::move(router)}); + } + + std::unique_ptr NewSock() EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); + + /** The requests a router received, per connection. */ + std::vector> Requests(const std::string& name) const EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) + { + LOCK(m_mutex); + return m_routers.at(name).requests; + } + + /** + * The timeout of every reply a router's connections waited for, per connection. On a + * connection, the n-th one is for the n-th request. + */ + std::vector> RecvTimeouts(const std::string& name) const EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) + { + LOCK(m_mutex); + auto recv_timeouts{m_routers.at(name).recv_timeouts}; + recv_timeouts.resize(m_routers.at(name).requests.size()); + return recv_timeouts; + } + + /** The timeout of every wait for a pending connect to a router. */ + std::vector ConnectWaits(const std::string& name) const EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) + { + LOCK(m_mutex); + return m_routers.at(name).connect_waits; + } + + size_t OpenSocks() const EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) + { + LOCK(m_mutex); + return m_socks.size(); + } + + /** The most sockets open at the same time since the last `ResetMaxOpenSocks()`. */ + size_t MaxOpenSocks() const EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) + { + LOCK(m_mutex); + return m_max_open_socks; + } + + void ResetMaxOpenSocks() EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) + { + LOCK(m_mutex); + m_max_open_socks = m_socks.size(); + } + +private: + class MockSamSock; + + struct RouterState { + Router router; + std::vector> requests{}; + std::vector> recv_timeouts{}; + std::vector connect_waits{}; + }; + + struct SockState { + std::string router{}; + size_t conn{0}; + bool connect_pending{false}; + std::string partial_request{}; + std::string inbox{}; + std::chrono::steady_clock::time_point inbox_ready{}; + bool closed{false}; + }; + + static bool Readable(const SockState& sock) + { + return sock.inbox.empty() ? sock.closed : std::chrono::steady_clock::now() >= sock.inbox_ready; + } + + void Close(uint64_t id) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) + { + LOCK(m_mutex); + m_socks.erase(id); + } + + int Connect(uint64_t id, const sockaddr* addr, socklen_t len) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) + { + std::string name; + if (addr->sa_family == AF_INET || addr->sa_family == AF_INET6) { + CService service; + if (service.SetSockAddr(addr, len)) name = service.ToStringAddrPort(); +#ifdef HAVE_SOCKADDR_UN + } else if (addr->sa_family == AF_UNIX) { + name = ADDR_PREFIX_UNIX + std::string{reinterpret_cast(addr)->sun_path}; +#endif + } + LOCK(m_mutex); + const auto router{m_routers.find(name)}; + if (router == m_routers.end()) { + SetSockError(SOCK_ERROR_RESET); + return SOCKET_ERROR; + } + SockState& sock{m_socks.at(id)}; + sock.router = name; + sock.conn = router->second.requests.size(); + router->second.requests.emplace_back(); + if (router->second.router.pending_connects.contains(sock.conn)) { + sock.connect_pending = true; + SetSockError(SOCK_ERROR_IN_PROGRESS); + return SOCKET_ERROR; + } + return 0; + } + + ssize_t Send(uint64_t id, const void* data, size_t len) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) + { + LOCK(m_mutex); + SockState& sock{m_socks.at(id)}; + RouterState& router{m_routers.at(sock.router)}; + sock.partial_request.append(static_cast(data), len); + for (auto end{sock.partial_request.find('\n')}; end != std::string::npos; end = sock.partial_request.find('\n')) { + const std::string request{sock.partial_request.substr(0, end)}; + sock.partial_request.erase(0, end + 1); + router.requests.at(sock.conn).push_back(request); + const Response response{router.router.script(sock.conn, request)}; + switch (response.action) { + case Action::REPLY: + sock.inbox += response.line + "\n"; + sock.inbox_ready = std::chrono::steady_clock::now() + response.delay; + break; + case Action::STALL: + break; + case Action::CLOSE: + sock.closed = true; + break; + case Action::FAIL_SEND: + SetSockError(SOCK_ERROR_RESET); + return -1; + } + } + return len; + } + + void RecordRecvTimeout(uint64_t id, std::chrono::milliseconds timeout) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) + { + LOCK(m_mutex); + const SockState& sock{m_socks.at(id)}; + RouterState& router{m_routers.at(sock.router)}; + router.recv_timeouts.resize(router.requests.size()); + router.recv_timeouts.at(sock.conn).push_back(timeout); + } + + ssize_t Recv(uint64_t id, void* buf, size_t len, int flags) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) + { + LOCK(m_mutex); + SockState& sock{m_socks.at(id)}; + if (!Readable(sock)) { + SetSockError(WSAEWOULDBLOCK); + return -1; + } + const size_t n{std::min(len, sock.inbox.size())}; + std::copy_n(sock.inbox.begin(), n, static_cast(buf)); + if ((flags & MSG_PEEK) == 0) sock.inbox.erase(0, n); + return n; + } + + bool Wait(uint64_t id, std::chrono::milliseconds timeout, Sock::Event requested, Sock::Event* occurred) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) + { + std::function on_connect_wait; + size_t wait_number{0}; + std::chrono::steady_clock::duration sleep{std::min(timeout, 5ms)}; + { + LOCK(m_mutex); + const SockState& sock{m_socks.at(id)}; + if (sock.connect_pending) { + RouterState& router{m_routers.at(sock.router)}; + router.connect_waits.push_back(timeout); + on_connect_wait = router.router.on_connect_wait; + wait_number = router.connect_waits.size(); + } else if ((requested & Sock::RECV) && Readable(sock)) { + if (occurred) *occurred = Sock::RECV; + return true; + } else if (requested & Sock::SEND) { + if (occurred) *occurred = Sock::SEND; + return true; + } else if (!sock.inbox.empty()) { + sleep = std::min(sleep, sock.inbox_ready - std::chrono::steady_clock::now()); + } + } + if (on_connect_wait) on_connect_wait(wait_number); + if (sleep > 0s) std::this_thread::sleep_for(sleep); + LOCK(m_mutex); + const SockState& sock{m_socks.at(id)}; + if (occurred) *occurred = !sock.connect_pending && (requested & Sock::RECV) && Readable(sock) ? Sock::RECV : 0; + return true; + } + + mutable Mutex m_mutex; + std::map m_routers GUARDED_BY(m_mutex); + std::map m_socks GUARDED_BY(m_mutex); + uint64_t m_next_sock_id GUARDED_BY(m_mutex){0}; + size_t m_max_open_socks GUARDED_BY(m_mutex){0}; +}; + +class MockSamRouters::MockSamSock : public ZeroSock +{ +public: + MockSamSock(MockSamRouters& routers, uint64_t id) : m_routers{routers}, m_id{id} {} + ~MockSamSock() override { m_routers.Close(m_id); } + + int Connect(const sockaddr* addr, socklen_t len) const override { return m_routers.Connect(m_id, addr, len); } + ssize_t Send(const void* data, size_t len, int) const override { return m_routers.Send(m_id, data, len); } + ssize_t Recv(void* buf, size_t len, int flags) const override { return m_routers.Recv(m_id, buf, len, flags); } + bool Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred = nullptr) const override + { + return m_routers.Wait(m_id, timeout, requested, occurred); + } + bool IsConnected(std::string&) const override { return true; } + + std::string RecvUntilTerminator(uint8_t terminator, + std::chrono::milliseconds timeout, + CThreadInterrupt& interrupt, + size_t max_data) const override + { + m_routers.RecordRecvTimeout(m_id, timeout); + return Sock::RecvUntilTerminator(terminator, timeout, interrupt, max_data); + } + + MockSamSock& operator=(Sock&&) override + { + assert(false && "Move of Sock into MockSamSock not allowed."); + return *this; + } + +private: + MockSamRouters& m_routers; + const uint64_t m_id; +}; + +std::unique_ptr MockSamRouters::NewSock() +{ + LOCK(m_mutex); + const uint64_t id{m_next_sock_id++}; + m_socks.emplace(id, SockState{}); + m_max_open_socks = std::max(m_max_open_socks, m_socks.size()); + return std::make_unique(*this, id); +} + +using Response = MockSamRouters::Response; +using Action = MockSamRouters::Action; + +/** The value of `key=` in a SAM request or reply, or "" if it has none. */ +std::string Field(const std::string& line, const std::string& key) +{ + for (const auto& token : util::SplitString(line, ' ')) { + if (token.starts_with(key + "=")) return token.substr(key.size() + 1); + } + return ""; +} + +/** The "SESSION CREATE" requests in `requests`, each with the number of its connection. */ +std::vector> Creates(const std::vector>& requests, size_t first_conn = 0) +{ + std::vector> creates; + for (size_t conn{first_conn}; conn < requests.size(); ++conn) { + for (const auto& request : requests[conn]) { + if (request.starts_with("SESSION CREATE")) creates.emplace_back(conn, request); + } + } + return creates; +} + +size_t CountRequests(const std::vector>& requests, const std::string& prefix) +{ + size_t count{0}; + for (const auto& conn : requests) { + count += std::ranges::count_if(conn, [&](const auto& request) { return request.starts_with(prefix); }); + } + return count; +} + +Response CreateOk() { return {.line = "SESSION STATUS RESULT=OK DESTINATION=" + TEST_PRIV}; } + +Response RejectHybrid(std::chrono::milliseconds delay = 0ms) +{ + return {.line = JAVA_I2P_2_9_0_REJECTION, .delay = delay}; +} + +/** + * A router script. "SESSION CREATE" is answered by `create`, given the requested + * i2cp.leaseSetEncType; other requests like a working router does, after `delay`. + */ +MockSamRouters::Script RouterScript(std::function create, + std::chrono::milliseconds delay = 0ms) +{ + return [create, delay](size_t, const std::string& request) -> Response { + if (request.starts_with("HELLO")) return {.line = "HELLO REPLY RESULT=OK VERSION=3.1", .delay = delay}; + if (request.starts_with("DEST GENERATE")) return {.line = "DEST REPLY PRIV=" + TEST_PRIV, .delay = delay}; + if (request.starts_with("SESSION CREATE")) return create(Field(request, "i2cp.leaseSetEncType")); + if (request.starts_with("STREAM ACCEPT")) return {.line = "STREAM STATUS RESULT=OK", .delay = delay}; + return {.action = Action::STALL}; + }; +} + +/** A router with the post-quantum leaseset type. */ +MockSamRouters::Script Hybrid() +{ + return RouterScript([](const std::string&) { return CreateOk(); }); +} + +/** + * A router without the post-quantum leaseset type, like Java I2P older than 2.10.0. It takes + * `delay` to answer anything but an acceptable "SESSION CREATE". + */ +MockSamRouters::Script NoHybrid(std::chrono::milliseconds delay = 0ms) +{ + return RouterScript([delay](const std::string& enc_types) { return enc_types == "6,4" ? RejectHybrid(delay) : CreateOk(); }, delay); +} + +Proxy TestRouter(uint16_t port) { return Proxy{LookupNumeric("127.0.0.1", port)}; } + +/** Captures the log lines while alive. */ +class LogCapture +{ +public: + LogCapture() + { + m_callback = LogInstance().PushBackCallback([this](const std::string& line) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) { + LOCK(m_mutex); + m_lines.push_back(line); + }); + } + ~LogCapture() { LogInstance().DeleteCallback(m_callback); } + + /** The number of lines that contain `needle`. */ + size_t Count(std::string_view needle) const EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) + { + LOCK(m_mutex); + return std::ranges::count_if(m_lines, [&](const auto& line) { return line.find(needle) != std::string::npos; }); + } + + /** The latest line that contains `needle`, or "" if there is none. */ + std::string Last(std::string_view needle) const EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) + { + LOCK(m_mutex); + const auto it{std::ranges::find_if(m_lines | std::views::reverse, [&](const auto& line) { return line.find(needle) != std::string::npos; })}; + return it == (m_lines | std::views::reverse).end() ? "" : *it; + } + +private: + mutable Mutex m_mutex; + std::vector m_lines GUARDED_BY(m_mutex); + std::list>::iterator m_callback; +}; + +/** Whether the latest "Couldn't listen" error in `log` contains one of `errors`. */ +bool ListenFailedWith(const LogCapture& log, const std::vector& errors) +{ + const std::string line{log.Last("Couldn't listen")}; + return std::ranges::any_of(errors, [&](const auto& error) { return line.find(error) != std::string::npos; }); +} + +/** The "SESSION CREATE" requests in `creates` that ask for `enc_types`. */ +size_t CountEncTypes(const std::vector>& creates, std::string_view enc_types) +{ + return std::ranges::count_if(creates, [&](const auto& c) { return Field(c.second, "i2cp.leaseSetEncType") == enc_types; }); +} + +/** Whether the session's next "SESSION CREATE" after connection `first_conn` asked for `enc_types`. */ +bool FirstCreateAsks(const MockSamRouters& routers, const Proxy& router, size_t first_conn, std::string_view enc_types) +{ + const auto creates{Creates(routers.Requests(router.ToString()), first_conn)}; + return !creates.empty() && Field(creates[0].second, "i2cp.leaseSetEncType") == enc_types; +} +} // namespace + BOOST_FIXTURE_TEST_SUITE(i2p_tests, EnvTestingSetup) BOOST_AUTO_TEST_CASE(unlimited_recv) @@ -171,4 +666,738 @@ BOOST_AUTO_TEST_CASE(damaged_private_key) } } +BOOST_AUTO_TEST_CASE(hybrid_create_success) +{ + MockSamRouters routers; + CreateSock = [&routers](int, int, int) { return routers.NewSock(); }; + LogCapture log; + + for (const bool transient : {false, true}) { + const Proxy router{TestRouter(transient ? 17002 : 17001)}; + routers.AddRouter(router.ToString(), {.script = Hybrid()}); + CThreadInterrupt interrupt; + auto session{transient ? std::make_unique(router, &interrupt) : + std::make_unique(m_args.GetDataDirNet() / "i2p_key_success", router, &interrupt)}; + + i2p::Connection conn; + BOOST_REQUIRE(session->Listen(conn)); + + // One connection created the session, the next one listens with it. + const auto requests{routers.Requests(router.ToString())}; + BOOST_REQUIRE_EQUAL(requests.size(), 2U); + const auto creates{Creates(requests)}; + BOOST_REQUIRE_EQUAL(creates.size(), 1U); + BOOST_CHECK_EQUAL(creates[0].first, 0U); + BOOST_CHECK_EQUAL(Field(creates[0].second, "i2cp.leaseSetEncType"), "6,4"); + BOOST_CHECK_EQUAL(Field(creates[0].second, "DESTINATION"), transient ? "TRANSIENT" : TEST_PRIV); + BOOST_CHECK_EQUAL(requests[1].back(), strprintf("STREAM ACCEPT ID=%s SILENT=false", Field(creates[0].second, "ID"))); + } + BOOST_CHECK_EQUAL(log.Count(HYBRID_WARNING), 0U); +} + +BOOST_AUTO_TEST_CASE(hybrid_rejection_retry) +{ + MockSamRouters routers; + CreateSock = [&routers](int, int, int) { return routers.NewSock(); }; + + for (const bool transient : {false, true}) { + const Proxy router{TestRouter(transient ? 17012 : 17011)}; + routers.AddRouter(router.ToString(), {.script = NoHybrid()}); + CThreadInterrupt interrupt; + auto session{transient ? std::make_unique(router, &interrupt) : + std::make_unique(m_args.GetDataDirNet() / "i2p_key_retry", router, &interrupt)}; + + LogCapture log; + routers.ResetMaxOpenSocks(); + i2p::Connection conn; + BOOST_REQUIRE(session->Listen(conn)); + BOOST_CHECK_EQUAL(log.Count("with leaseSetEncType=6,4: RESULT=I2P_ERROR MESSAGE=\"" + JAVA_I2P_2_9_0_MESSAGE_LOGGED + "\", retrying with 4,0"), 1U); + BOOST_CHECK_EQUAL(log.Count(HYBRID_WARNING), 1U); + BOOST_CHECK_EQUAL(log.Count(HybridWarning("I2P_ERROR")), 1U); + BOOST_CHECK_EQUAL(log.Count(UNNAMED_REJECTION), 0U); + + // Rejected, retried, then listening. + const auto requests{routers.Requests(router.ToString())}; + BOOST_REQUIRE_EQUAL(requests.size(), 3U); + const auto creates{Creates(requests)}; + BOOST_REQUIRE_EQUAL(creates.size(), 2U); + BOOST_CHECK_EQUAL(Field(creates[0].second, "i2cp.leaseSetEncType"), "6,4"); + BOOST_CHECK_EQUAL(Field(creates[1].second, "i2cp.leaseSetEncType"), "4,0"); + // The retry uses a fresh connection, HELLO and session id. + BOOST_CHECK_EQUAL(creates[0].first, 0U); + BOOST_CHECK_EQUAL(creates[1].first, 1U); + BOOST_CHECK(requests[1].front().starts_with("HELLO")); + BOOST_CHECK(Field(creates[0].second, "ID") != Field(creates[1].second, "ID")); + // The rejected connection is closed before the retry: at most the control socket and + // the listening one were open at the same time. + BOOST_CHECK_EQUAL(routers.MaxOpenSocks(), 2U); + // The persistent destination is generated once and kept for the retry. + BOOST_CHECK_EQUAL(CountRequests(requests, "DEST GENERATE"), transient ? 0U : 1U); + BOOST_CHECK_EQUAL(Field(creates[1].second, "DESTINATION"), transient ? "TRANSIENT" : TEST_PRIV); + BOOST_CHECK_EQUAL(Field(creates[0].second, "DESTINATION"), Field(creates[1].second, "DESTINATION")); + // The session uses the id of the retried one. + BOOST_CHECK_EQUAL(requests[2].back(), strprintf("STREAM ACCEPT ID=%s SILENT=false", Field(creates[1].second, "ID"))); + } + + // The warning names the RESULT only if it is one of the SAM protocol's. Whatever the RESULT, + // the MESSAGE names the encryption type. + uint16_t port{17013}; + for (const auto& [result, shown] : std::vector>{ + {"CANT_REACH_PEER", "CANT_REACH_PEER"}, + {"TIMEOUT", "TIMEOUT"}, + {"NOT_A_SAM_RESULT", "other"}, + {"I2P_ERROR\x07", "other"}}) { + const Proxy router{TestRouter(port++)}; + routers.AddRouter(router.ToString(), {.script = RouterScript([&](const std::string& enc_types) { + return enc_types == "6,4" ? Response{.line = "SESSION STATUS RESULT=" + result + " MESSAGE=\"Unsupported crypto type: 6\""} : CreateOk(); + })}); + LogCapture log; + CThreadInterrupt interrupt; + i2p::sam::Session session{router, &interrupt}; + i2p::Connection conn; + BOOST_REQUIRE(session.Listen(conn)); + BOOST_CHECK_EQUAL(log.Count(HybridWarning(shown)), 1U); + } +} + +BOOST_AUTO_TEST_CASE(hybrid_rejection_cached) +{ + MockSamRouters routers; + CreateSock = [&routers](int, int, int) { return routers.NewSock(); }; + CThreadInterrupt interrupt; + LogCapture log; + + const Proxy router{TestRouter(17021)}; + bool accept_legacy{true}; + routers.AddRouter(router.ToString(), {.script = RouterScript([&](const std::string& enc_types) { + return enc_types == "6,4" || !accept_legacy ? RejectHybrid() : CreateOk(); + })}); + // Two sessions in a row in which the router rejects 6,4 like Java I2P 2.9.0, naming the + // encryption type, and accepts 4,0. + for (unsigned i{0}; i < i2p::sam::SAM_HYBRID_FALLBACKS_TO_REMEMBER; ++i) { + const size_t first_conn{routers.Requests(router.ToString()).size()}; + i2p::sam::Session session{router, &interrupt}; + i2p::Connection conn; + BOOST_REQUIRE(session.Listen(conn)); + BOOST_CHECK(FirstCreateAsks(routers, router, first_conn, "6,4")); + } + BOOST_CHECK_EQUAL(log.Count(HYBRID_WARNING), 1U); + BOOST_CHECK_EQUAL(log.Count("until qbit restarts"), 1U); + + // Later sessions through the same router start with 4,0 and never retry, even when the router + // rejects 4,0 as well. + for (const bool legacy_ok : {true, false}) { + accept_legacy = legacy_ok; + const size_t first_conn{routers.Requests(router.ToString()).size()}; + i2p::sam::Session session{router, &interrupt}; + i2p::Connection conn; + BOOST_CHECK_EQUAL(session.Listen(conn), legacy_ok); + const auto creates{Creates(routers.Requests(router.ToString()), first_conn)}; + BOOST_REQUIRE_EQUAL(creates.size(), 1U); + BOOST_CHECK_EQUAL(Field(creates[0].second, "i2cp.leaseSetEncType"), "4,0"); + } + BOOST_CHECK_EQUAL(log.Count(HYBRID_WARNING), 1U); + + // A different router is unaffected. + std::vector other_routers{TestRouter(17022)}; +#ifdef HAVE_SOCKADDR_UN + other_routers.emplace_back("unix:/tmp/qbit-i2p-tests-sam.sock"); +#endif + for (const auto& other : other_routers) { + routers.AddRouter(other.ToString(), {.script = Hybrid()}); + i2p::sam::Session session{other, &interrupt}; + i2p::Connection conn; + BOOST_REQUIRE(session.Listen(conn)); + const auto creates{Creates(routers.Requests(other.ToString()))}; + BOOST_REQUIRE_EQUAL(creates.size(), 1U); + BOOST_CHECK_EQUAL(Field(creates[0].second, "i2cp.leaseSetEncType"), "6,4"); + } + + // Sessions created at the same time through one router share one record and warn once. In + // any order, at least two of them see the router reject 6,4 before it is remembered. + const Proxy busy_router{TestRouter(17023)}; + routers.AddRouter(busy_router.ToString(), {.script = NoHybrid(/*delay=*/50ms)}); + { + LogCapture busy_log; + constexpr int num_sessions{4}; + std::atomic listening{0}; + std::vector threads; + for (int i{0}; i < num_sessions; ++i) { + threads.emplace_back([&] { + i2p::sam::Session session{busy_router, &interrupt}; + i2p::Connection conn; + if (session.Listen(conn)) ++listening; + }); + } + for (auto& thread : threads) thread.join(); + BOOST_CHECK_EQUAL(listening, num_sessions); + BOOST_CHECK_EQUAL(busy_log.Count(HYBRID_WARNING), 1U); + BOOST_CHECK_EQUAL(busy_log.Count("until qbit restarts"), 1U); + BOOST_CHECK_EQUAL(CountEncTypes(Creates(routers.Requests(busy_router.ToString())), "4,0"), size_t{num_sessions}); + + const size_t first_conn{routers.Requests(busy_router.ToString()).size()}; + i2p::sam::Session session{busy_router, &interrupt}; + i2p::Connection conn; + BOOST_REQUIRE(session.Listen(conn)); + BOOST_CHECK(FirstCreateAsks(routers, busy_router, first_conn, "4,0")); + } +} + +BOOST_AUTO_TEST_CASE(hybrid_two_strikes) +{ + MockSamRouters routers; + CreateSock = [&routers](int, int, int) { return routers.NewSock(); }; + CThreadInterrupt interrupt; + LogCapture log; + + const Proxy router{TestRouter(17081)}; + // The router's answer to 6,4: a rejection, or acceptance if none. + std::optional rejection; + routers.AddRouter(router.ToString(), {.script = RouterScript([&](const std::string& enc_types) { + return enc_types == "6,4" && rejection ? Response{.line = *rejection} : CreateOk(); + })}); + // Creates a session and returns the leaseset types its first "SESSION CREATE" asked for. + const auto session_starts_with = [&](std::optional reply_to_hybrid) { + rejection = std::move(reply_to_hybrid); + const size_t first_conn{routers.Requests(router.ToString()).size()}; + i2p::sam::Session session{router, &interrupt}; + i2p::Connection conn; + BOOST_REQUIRE(session.Listen(conn)); + const auto creates{Creates(routers.Requests(router.ToString()), first_conn)}; + BOOST_REQUIRE(!creates.empty()); + return Field(creates[0].second, "i2cp.leaseSetEncType"); + }; + + // A rejection, then a session that gets 6,4: one rejection does not turn 6,4 off. + BOOST_CHECK_EQUAL(session_starts_with(JAVA_I2P_2_9_0_REJECTION), "6,4"); + BOOST_CHECK_EQUAL(log.Count(HybridWarning("I2P_ERROR")), 1U); + BOOST_CHECK_EQUAL(session_starts_with(std::nullopt), "6,4"); + + // That success started the count anew, so this rejection is only the first in a row. + BOOST_CHECK_EQUAL(session_starts_with(JAVA_I2P_2_9_0_REJECTION), "6,4"); + BOOST_CHECK_EQUAL(log.Count("until qbit restarts"), 0U); + // A rejection that does not name the encryption type neither counts nor starts the count anew. + BOOST_CHECK_EQUAL(session_starts_with(TUNNEL_BUILD_FAILURE), "6,4"); + BOOST_CHECK_EQUAL(log.Count("until qbit restarts"), 0U); + BOOST_CHECK_EQUAL(log.Count(UNNAMED_REJECTION), 1U); + // So this is the second in a row: from now on, sessions start with 4,0. + BOOST_CHECK_EQUAL(session_starts_with(JAVA_I2P_2_9_0_REJECTION), "6,4"); + BOOST_CHECK_EQUAL(log.Count("until qbit restarts"), 1U); + BOOST_CHECK_EQUAL(session_starts_with(JAVA_I2P_2_9_0_REJECTION), "4,0"); + BOOST_CHECK_EQUAL(session_starts_with(std::nullopt), "4,0"); + + // Warned once per router, at the first rejection. + BOOST_CHECK_EQUAL(log.Count(HYBRID_WARNING), 1U); +} + +BOOST_AUTO_TEST_CASE(hybrid_unnamed_rejection) +{ + MockSamRouters routers; + CreateSock = [&routers](int, int, int) { return routers.NewSock(); }; + CThreadInterrupt interrupt; + + // Rejections of 6,4 that do not name the encryption type, and their RESULT= and MESSAGE= as + // the info line shows them. + struct Unnamed { + std::string reply; + std::string result; + std::string message; + }; + uint16_t port{17101}; + for (const auto& c : std::vector{ + {TUNNEL_BUILD_FAILURE, "I2P_ERROR", "Failed to build tunnels"}, + {"SESSION STATUS RESULT=TIMEOUT", "TIMEOUT", ""}, + {"SESSION STATUS RESULT=CANT_REACH_PEER", "CANT_REACH_PEER", ""}, + {"SESSION STATUS RESULT=I2P_ERROR", "I2P_ERROR", ""}, + // Router text is sanitized and capped. + {"SESSION STATUS RESULT=NOT_A_SAM_RESULT MESSAGE=\"Tunnel\x07 build failed " + std::string(300, 'x') + "\"", + "other", "Tunnel build failed " + std::string(179, 'x')}, + }) { + BOOST_TEST_MESSAGE(c.reply); + const Proxy router{TestRouter(port++)}; + routers.AddRouter(router.ToString(), {.script = RouterScript([&](const std::string& enc_types) { + return enc_types == "6,4" ? Response{.line = c.reply} : CreateOk(); + })}); + LogCapture log; + + // Each session is retried with 4,0 and created. After three in a row, the next session + // still asks for 6,4. + for (size_t i{1}; i <= 4; ++i) { + const size_t first_conn{routers.Requests(router.ToString()).size()}; + i2p::sam::Session session{router, &interrupt}; + i2p::Connection conn; + BOOST_REQUIRE(session.Listen(conn)); + const auto creates{Creates(routers.Requests(router.ToString()), first_conn)}; + BOOST_REQUIRE_EQUAL(creates.size(), 2U); + BOOST_CHECK_EQUAL(Field(creates[0].second, "i2cp.leaseSetEncType"), "6,4"); + BOOST_CHECK_EQUAL(Field(creates[1].second, "i2cp.leaseSetEncType"), "4,0"); + // One info line per session. + BOOST_CHECK_EQUAL(log.Count(UNNAMED_REJECTION), i); + BOOST_CHECK_EQUAL(log.Count(UnnamedRejectionInfo(router, Field(creates[1].second, "ID"), c.result, c.message)), 1U); + } + // Not counted, and no warning. + BOOST_CHECK_EQUAL(log.Count("until qbit restarts"), 0U); + BOOST_CHECK_EQUAL(log.Count(HYBRID_WARNING), 0U); + } +} + +BOOST_AUTO_TEST_CASE(hybrid_rejection_names_enc_type) +{ + MockSamRouters routers; + CreateSock = [&routers](int, int, int) { return routers.NewSock(); }; + CThreadInterrupt interrupt; + + uint16_t port{17121}; + for (const auto& [reply, named] : std::vector>{ + {JAVA_I2P_2_9_0_REJECTION, true}, + // Each phrase, in any case. + {"SESSION STATUS RESULT=I2P_ERROR MESSAGE=\"Unsupported CRYPTO TYPE 6\"", true}, + {"SESSION STATUS RESULT=I2P_ERROR MESSAGE=\"Encryption Type 6 is not supported\"", true}, + {"SESSION STATUS RESULT=I2P_ERROR MESSAGE=\"bad enc TYPE: 6\"", true}, + {"SESSION STATUS RESULT=I2P_ERROR MESSAGE=\"unknown EncType 6\"", true}, + {"SESSION STATUS RESULT=I2P_ERROR MESSAGE=unsupported_encType", true}, + // Matched before MESSAGE= is capped for the log. + {"SESSION STATUS RESULT=I2P_ERROR MESSAGE=\"" + std::string(300, 'x') + " crypto type 6\"", true}, + // Only "type" or "crypto". + {"SESSION STATUS RESULT=I2P_ERROR MESSAGE=\"Unsupported type: 6\"", false}, + {"SESSION STATUS RESULT=I2P_ERROR MESSAGE=\"crypto failure\"", false}, + {"SESSION STATUS RESULT=I2P_ERROR MESSAGE=\"crypto: unsupported type 6\"", false}, + // Not in MESSAGE=: an unquoted one ends at the first space. + {"SESSION STATUS RESULT=I2P_ERROR MESSAGE=Unsupported crypto type: 6", false}, + {"SESSION STATUS RESULT=I2P_ERROR MESSAGE=\"Failed\" enctype=6", false}, + }) { + BOOST_TEST_MESSAGE(reply); + const Proxy router{TestRouter(port++)}; + routers.AddRouter(router.ToString(), {.script = RouterScript([&](const std::string& enc_types) { + return enc_types == "6,4" ? Response{.line = reply} : CreateOk(); + })}); + LogCapture log; + i2p::sam::Session session{router, &interrupt}; + i2p::Connection conn; + BOOST_REQUIRE(session.Listen(conn)); + BOOST_CHECK_EQUAL(CountEncTypes(Creates(routers.Requests(router.ToString())), "4,0"), 1U); + // Only a rejection that names the encryption type counts and warns. + BOOST_CHECK_EQUAL(log.Count(HYBRID_WARNING), named ? 1U : 0U); + BOOST_CHECK_EQUAL(log.Count(UNNAMED_REJECTION), named ? 0U : 1U); + } +} + +BOOST_AUTO_TEST_CASE(hybrid_remembered_routers_bounded) +{ + MockSamRouters routers; + CreateSock = [&routers](int, int, int) { return routers.NewSock(); }; + CThreadInterrupt interrupt; + + const auto fallback_session = [&](const Proxy& router) { + const size_t first_conn{routers.Requests(router.ToString()).size()}; + i2p::sam::Session session{router, &interrupt}; + i2p::Connection conn; + BOOST_REQUIRE(session.Listen(conn)); + const auto creates{Creates(routers.Requests(router.ToString()), first_conn)}; + BOOST_REQUIRE(!creates.empty()); + return Field(creates[0].second, "i2cp.leaseSetEncType"); + }; + + // The oldest router does not sort first, so forgetting the first one would not pass. + const Proxy oldest{TestRouter(17299)}; + routers.AddRouter(oldest.ToString(), {.script = NoHybrid()}); + for (unsigned i{0}; i < i2p::sam::SAM_HYBRID_FALLBACKS_TO_REMEMBER; ++i) { + BOOST_CHECK_EQUAL(fallback_session(oldest), "6,4"); + } + BOOST_CHECK_EQUAL(fallback_session(oldest), "4,0"); + + // As many newer routers as can be remembered: the oldest one is forgotten. + std::vector newer; + for (size_t i{0}; i < i2p::sam::SAM_MAX_REMEMBERED_ROUTERS; ++i) { + newer.push_back(TestRouter(17200 + i)); + routers.AddRouter(newer.back().ToString(), {.script = NoHybrid()}); + BOOST_CHECK_EQUAL(fallback_session(newer.back()), "6,4"); + } + // The newer ones are remembered: a second rejection in a row turns 6,4 off. + BOOST_CHECK_EQUAL(fallback_session(newer.front()), "6,4"); + BOOST_CHECK_EQUAL(fallback_session(newer.front()), "4,0"); + BOOST_CHECK_EQUAL(fallback_session(oldest), "6,4"); +} + +BOOST_AUTO_TEST_CASE(hybrid_unrelated_rejection) +{ + MockSamRouters routers; + CreateSock = [&routers](int, int, int) { return routers.NewSock(); }; + CThreadInterrupt interrupt; + LogCapture log; + + uint16_t port{17031}; + for (const std::string result : {"DUPLICATED_ID", "DUPLICATED_DEST", "INVALID_ID", "INVALID_KEY"}) { + const Proxy router{TestRouter(port++)}; + bool accept{false}; + routers.AddRouter(router.ToString(), {.script = RouterScript([&](const std::string&) { + return accept ? CreateOk() : Response{.line = "SESSION STATUS RESULT=" + result}; + })}); + { + i2p::sam::Session session{router, &interrupt}; + i2p::Connection conn; + BOOST_CHECK(!session.Listen(conn)); + BOOST_CHECK(ListenFailedWith(log, {"Unexpected reply to \"SESSION CREATE ...\": \"SESSION STATUS RESULT=" + result + "\""})); + } + // No retry. + const auto requests{routers.Requests(router.ToString())}; + BOOST_CHECK_EQUAL(requests.size(), 1U); + BOOST_CHECK_EQUAL(Creates(requests).size(), 1U); + + // Nothing remembered: the next session through this router asks for 6,4 again. + accept = true; + i2p::sam::Session session{router, &interrupt}; + i2p::Connection conn; + BOOST_REQUIRE(session.Listen(conn)); + BOOST_CHECK(FirstCreateAsks(routers, router, requests.size(), "6,4")); + } + BOOST_CHECK_EQUAL(log.Count(HYBRID_WARNING), 0U); +} + +BOOST_AUTO_TEST_CASE(hybrid_retry_also_fails) +{ + MockSamRouters routers; + CreateSock = [&routers](int, int, int) { return routers.NewSock(); }; + CThreadInterrupt interrupt; + LogCapture log; + + const Proxy router{TestRouter(17041)}; + bool accept{false}; + // MESSAGE is router text: logged sanitized and capped. It names the encryption type, so only + // the failed retry keeps the rejections from counting. + const std::string named_message{"Bad\x07 crypto type " + std::string(300, 'x')}; + std::string message; + routers.AddRouter(router.ToString(), {.script = RouterScript([&](const std::string&) { + return accept ? CreateOk() : Response{.line = "SESSION STATUS RESULT=I2P_ERROR MESSAGE=\"" + message + "\""}; + })}); + // Two failed retries in a row, after rejections that name the encryption type, are not + // remembered. A failed retry after one that does not name it logs no info line either. + for (const std::string& m : {named_message, named_message, std::string{"Failed to build tunnels"}}) { + message = m; + const size_t first_conn{routers.Requests(router.ToString()).size()}; + i2p::sam::Session session{router, &interrupt}; + i2p::Connection conn; + BOOST_CHECK(!session.Listen(conn)); + BOOST_CHECK(ListenFailedWith(log, {"Unexpected reply to \"SESSION CREATE ...\""})); + const auto creates{Creates(routers.Requests(router.ToString()), first_conn)}; + BOOST_REQUIRE_EQUAL(creates.size(), 2U); + BOOST_CHECK_EQUAL(Field(creates[0].second, "i2cp.leaseSetEncType"), "6,4"); + BOOST_CHECK_EQUAL(Field(creates[1].second, "i2cp.leaseSetEncType"), "4,0"); + BOOST_CHECK_EQUAL(routers.OpenSocks(), 0U); + } + BOOST_CHECK_EQUAL(log.Count("RESULT=I2P_ERROR MESSAGE=\"Bad crypto type " + std::string(183, 'x') + "\", retrying with 4,0"), 2U); + BOOST_CHECK_EQUAL(log.Count("RESULT=I2P_ERROR MESSAGE=\"Failed to build tunnels\", retrying with 4,0"), 1U); + BOOST_CHECK_EQUAL(log.Count(UNNAMED_REJECTION), 0U); + + // Nothing remembered: the next session through this router asks for 6,4 again. + accept = true; + const size_t first_conn{routers.Requests(router.ToString()).size()}; + i2p::sam::Session session{router, &interrupt}; + i2p::Connection conn; + BOOST_REQUIRE(session.Listen(conn)); + BOOST_CHECK(FirstCreateAsks(routers, router, first_conn, "6,4")); + BOOST_CHECK_EQUAL(log.Count(HYBRID_WARNING), 0U); +} + +BOOST_AUTO_TEST_CASE(hybrid_no_retry_on_transport_failure) +{ + MockSamRouters routers; + CreateSock = [&routers](int, int, int) { return routers.NewSock(); }; + CThreadInterrupt interrupt; + LogCapture log; + + struct Failure { + std::string name; + Response response; + std::vector errors; + std::chrono::milliseconds create_timeout{i2p::sam::SAM_CREATE_TIMEOUT}; + }; + const std::vector failures{ + {"send failure", {.action = Action::FAIL_SEND}, {"send():"}}, + {"EOF", {.action = Action::CLOSE}, {"Connection unexpectedly closed by peer"}}, + {"malformed reply, empty RESULT", {.line = "SESSION STATUS RESULT="}, {"Unexpected reply"}}, + {"malformed reply, not SESSION STATUS", {.line = "STREAM STATUS RESULT=I2P_ERROR"}, {"Unexpected reply"}}, + {"missing RESULT", {.line = "SESSION STATUS MESSAGE=\"no result\""}, {"Missing RESULT="}}, + {"RESULT without a value", {.line = "SESSION STATUS RESULT"}, {"Missing RESULT="}}, + // The router got the request, but its reply is lost. + {"lost success reply", {.action = Action::STALL}, TIMEOUT_ERRORS, 1s}, + }; + + uint16_t port{17051}; + for (const auto& failure : failures) { + BOOST_TEST_MESSAGE(failure.name); + const Proxy router{TestRouter(port++)}; + bool accept{false}; + routers.AddRouter(router.ToString(), {.script = RouterScript([&](const std::string&) { + return accept ? CreateOk() : failure.response; + })}); + { + i2p::sam::Session session{router, &interrupt, failure.create_timeout}; + i2p::Connection conn; + BOOST_CHECK(!session.Listen(conn)); + BOOST_CHECK(ListenFailedWith(log, failure.errors)); + } + // No retry: no second connection. (Only an exhausted deadline can keep the request + // from being sent.) + const auto requests{routers.Requests(router.ToString())}; + BOOST_CHECK_EQUAL(requests.size(), 1U); + BOOST_CHECK_LE(Creates(requests).size(), 1U); + BOOST_CHECK_EQUAL(routers.OpenSocks(), 0U); + + // Nothing remembered: the next session through this router asks for 6,4 again. + accept = true; + i2p::sam::Session session{router, &interrupt}; + i2p::Connection conn; + BOOST_REQUIRE(session.Listen(conn)); + BOOST_CHECK(FirstCreateAsks(routers, router, requests.size(), "6,4")); + } + BOOST_CHECK_EQUAL(log.Count(HYBRID_WARNING), 0U); +} + +// Shutdown must interrupt connecting to the SAM proxy within 100 ms. +static_assert(MAX_CONNECT_POLL_INTERVAL <= 100ms); + +BOOST_AUTO_TEST_CASE(hybrid_retry_deadline_and_interrupt) +{ + MockSamRouters routers; + CreateSock = [&routers](int, int, int) { return routers.NewSock(); }; + LogCapture log; + + // Signal the interrupt during the retry: while it connects, or on a request. + struct Interruption { + std::string name; + /** "connect", or the start of the request. */ + std::string on; + std::string error; + /** The requests of the retry that the router gets. */ + size_t retry_requests; + }; + uint16_t port{17061}; + for (const auto& c : std::vector{ + {"interrupt during reconnect", "connect", "Interrupted while connecting to", 0}, + {"interrupt during HELLO", "HELLO", "Receive interrupted", 1}, + {"interrupt during CREATE", "SESSION CREATE", "Receive interrupted", 2}, + }) { + BOOST_TEST_MESSAGE(c.name); + const Proxy router{TestRouter(port++)}; + CThreadInterrupt interrupt; + bool accept{false}; + routers.AddRouter(router.ToString(), + {.script = [&](size_t conn, const std::string& request) -> Response { + if (accept) return Hybrid()(conn, request); + if (conn == 1 && request.starts_with(c.on)) { + interrupt(); + return {.action = Action::STALL}; + } + return NoHybrid()(conn, request); + }, + .pending_connects = c.on == "connect" ? std::set{1} : std::set{}, + .on_connect_wait = [&](size_t wait) { + if (wait == 2) interrupt(); + }}); + + // The deadline is far: only the interrupt can end this. + i2p::sam::Session session{router, &interrupt}; + i2p::Connection conn; + const auto start{std::chrono::steady_clock::now()}; + BOOST_CHECK(!session.Listen(conn)); + BOOST_CHECK(ListenFailedWith(log, {c.error})); + BOOST_CHECK(std::chrono::steady_clock::now() - start < i2p::sam::SAM_CREATE_TIMEOUT / 3); + + const auto requests{routers.Requests(router.ToString())}; + BOOST_REQUIRE_EQUAL(requests.size(), 2U); + BOOST_CHECK_EQUAL(requests[1].size(), c.retry_requests); + BOOST_CHECK(std::ranges::all_of(routers.ConnectWaits(router.ToString()), [](auto w) { return w <= MAX_CONNECT_POLL_INTERVAL; })); + // Sockets released. + BOOST_CHECK_EQUAL(routers.OpenSocks(), 0U); + + // No half-published session: the next use creates the session anew. + interrupt.reset(); + accept = true; + BOOST_REQUIRE(session.Listen(conn)); + const auto next_creates{Creates(routers.Requests(router.ToString()), requests.size())}; + BOOST_REQUIRE_EQUAL(next_creates.size(), 1U); + BOOST_CHECK_EQUAL(routers.Requests(router.ToString()).back().back(), + strprintf("STREAM ACCEPT ID=%s SILENT=false", Field(next_creates[0].second, "ID"))); + } + + { + BOOST_TEST_MESSAGE("deadline exhausted across steps"); + // Each answer of the first attempt is quicker than the deadline, the retry's HELLO is + // answered only long after it. Only one deadline over all the steps ends the retry in time. + constexpr auto create_timeout{1000ms}; + constexpr auto delay{400ms}; + const Proxy router{TestRouter(port++)}; + CThreadInterrupt interrupt; + bool accept{false}; + routers.AddRouter(router.ToString(), + {.script = [&](size_t conn, const std::string& request) -> Response { + if (accept) return Hybrid()(conn, request); + if (conn == 0) return NoHybrid(delay)(conn, request); + return {.line = "HELLO REPLY RESULT=OK VERSION=3.1", .delay = 20s}; + }}); + + i2p::sam::Session session{router, &interrupt, create_timeout}; + i2p::Connection conn; + const auto start{std::chrono::steady_clock::now()}; + BOOST_CHECK(!session.Listen(conn)); + const auto elapsed{std::chrono::steady_clock::now() - start}; + BOOST_CHECK(ListenFailedWith(log, TIMEOUT_ERRORS)); + // The first attempt's two answers took that long, and the retry's HELLO was not waited for. + BOOST_CHECK(elapsed >= 2 * delay); + BOOST_CHECK(elapsed < 15s); + + const auto requests{routers.Requests(router.ToString())}; + BOOST_CHECK_EQUAL(CountEncTypes(Creates(requests), "4,0"), 0U); + // Every wait for an answer ends by the deadline: the retry's waits got what was left + // after the first attempt's two answers. Deadlines are rounded up to milliseconds. + const auto recv_timeouts{routers.RecvTimeouts(router.ToString())}; + for (size_t c{0}; c < recv_timeouts.size(); ++c) { + for (const auto timeout : recv_timeouts[c]) { + BOOST_CHECK(timeout <= (c == 0 ? create_timeout : create_timeout - 2 * delay) + 1ms); + } + } + BOOST_CHECK_EQUAL(routers.OpenSocks(), 0U); + + // No half-published session: the next use creates the session anew. + accept = true; + BOOST_REQUIRE(session.Listen(conn)); + BOOST_CHECK_EQUAL(Creates(routers.Requests(router.ToString()), requests.size()).size(), 1U); + } + + { + BOOST_TEST_MESSAGE("deadline exhausted while connecting"); + constexpr auto create_timeout{300ms}; + const Proxy router{TestRouter(port++)}; + CThreadInterrupt interrupt; + routers.AddRouter(router.ToString(), {.script = Hybrid(), .pending_connects = {0}}); + + i2p::sam::Session session{router, &interrupt, create_timeout}; + i2p::Connection conn; + const auto start{std::chrono::steady_clock::now()}; + BOOST_CHECK(!session.Listen(conn)); + const auto elapsed{std::chrono::steady_clock::now() - start}; + BOOST_CHECK(ListenFailedWith(log, {"Timed out connecting to"})); + BOOST_CHECK(elapsed >= create_timeout); + BOOST_CHECK(elapsed < 15s); + BOOST_CHECK(std::ranges::all_of(routers.ConnectWaits(router.ToString()), [](auto w) { return w <= MAX_CONNECT_POLL_INTERVAL; })); + BOOST_CHECK_EQUAL(routers.OpenSocks(), 0U); + + // No half-published session: the next use creates the session anew. + BOOST_REQUIRE(session.Listen(conn)); + BOOST_CHECK_EQUAL(Creates(routers.Requests(router.ToString())).size(), 1U); + } + BOOST_CHECK_EQUAL(log.Count(HYBRID_WARNING), 0U); +} + +BOOST_AUTO_TEST_CASE(hybrid_deadline_covers_dest_generate) +{ + MockSamRouters routers; + CreateSock = [&routers](int, int, int) { return routers.NewSock(); }; + CThreadInterrupt interrupt; + LogCapture log; + + // A persistent session without a key generates one first. The router answers that only + // long after the deadline. + constexpr auto create_timeout{500ms}; + const Proxy router{TestRouter(17091)}; + bool accept{false}; + routers.AddRouter(router.ToString(), {.script = [&](size_t conn, const std::string& request) -> Response { + if (!accept && request.starts_with("DEST GENERATE")) { + return {.line = "DEST REPLY PRIV=" + TEST_PRIV, .delay = 20s}; + } + return Hybrid()(conn, request); + }}); + const fs::path key_file{m_args.GetDataDirNet() / "i2p_key_dest_generate"}; + i2p::sam::Session session{key_file, router, &interrupt, create_timeout}; + i2p::Connection conn; + const auto start{std::chrono::steady_clock::now()}; + BOOST_CHECK(!session.Listen(conn)); + BOOST_CHECK(std::chrono::steady_clock::now() - start < 15s); + BOOST_CHECK(ListenFailedWith(log, TIMEOUT_ERRORS)); + + const auto requests{routers.Requests(router.ToString())}; + BOOST_REQUIRE_EQUAL(requests.size(), 1U); + BOOST_REQUIRE_EQUAL(requests[0].size(), 2U); + BOOST_CHECK(requests[0][1].starts_with("DEST GENERATE")); + BOOST_CHECK_EQUAL(Creates(requests).size(), 0U); + // The wait for the new key ends by the deadline, not after the usual 3 minutes. + const auto recv_timeouts{routers.RecvTimeouts(router.ToString())}; + BOOST_REQUIRE_EQUAL(recv_timeouts[0].size(), 2U); + BOOST_CHECK(recv_timeouts[0][1] <= create_timeout + 1ms); + BOOST_CHECK(!fs::exists(key_file)); + BOOST_CHECK_EQUAL(routers.OpenSocks(), 0U); + + accept = true; + BOOST_REQUIRE(session.Listen(conn)); + BOOST_CHECK(fs::exists(key_file)); +} + +BOOST_AUTO_TEST_CASE(hybrid_connect_capped_by_timeout) +{ + MockSamRouters routers; + CreateSock = [&routers](int, int, int) { return routers.NewSock(); }; + CThreadInterrupt interrupt; + LogCapture log; + + // Connecting to the router never completes. -timeout ends it long before the creation + // deadline. (Should it not, the interrupt ends it eventually, with another error.) + const Proxy router{TestRouter(17092)}; + routers.AddRouter(router.ToString(), {.script = Hybrid(), + .pending_connects = {0}, + .on_connect_wait = [&](size_t wait) { + if (wait == 3000) interrupt(); + }}); + const int connect_timeout_orig{nConnectTimeout}; + nConnectTimeout = 200; + i2p::sam::Session session{router, &interrupt}; + i2p::Connection conn; + BOOST_CHECK(!session.Listen(conn)); + nConnectTimeout = connect_timeout_orig; + BOOST_CHECK(ListenFailedWith(log, {"Cannot connect to"})); + BOOST_CHECK(std::ranges::all_of(routers.ConnectWaits(router.ToString()), [](auto w) { return w <= MAX_CONNECT_POLL_INTERVAL; })); + BOOST_CHECK_EQUAL(routers.OpenSocks(), 0U); +} + +BOOST_AUTO_TEST_CASE(hybrid_private_key_never_logged) +{ + MockSamRouters routers; + CreateSock = [&routers](int, int, int) { return routers.NewSock(); }; + CThreadInterrupt interrupt; + LogCapture log; + + // How the router answers "SESSION CREATE". + std::function create; + const Proxy router{TestRouter(17093)}; + routers.AddRouter(router.ToString(), {.script = RouterScript([&](const std::string& enc_types) { return create(enc_types); })}); + + const fs::path key_file{m_args.GetDataDirNet() / "i2p_key_never_logged"}; + for (const auto& [name, answer] : std::vector>>{ + {"retried", [](const std::string& enc_types) { return enc_types == "6,4" ? RejectHybrid() : CreateOk(); }}, + {"retried, not named", [](const std::string& enc_types) { return enc_types == "6,4" ? Response{.line = TUNNEL_BUILD_FAILURE} : CreateOk(); }}, + {"retry rejected", [](const std::string&) { return RejectHybrid(); }}, + {"missing RESULT", [](const std::string&) { return Response{.line = "SESSION STATUS MESSAGE=\"no result\""}; }}, + {"unrelated rejection", [](const std::string&) { return Response{.line = "SESSION STATUS RESULT=DUPLICATED_DEST"}; }}, + }) { + BOOST_TEST_MESSAGE(name); + create = answer; + i2p::sam::Session session{key_file, router, &interrupt}; + i2p::Connection conn; + (void)session.Listen(conn); + } + + // Every "SESSION CREATE" carried the key, and the errors were logged without it. + const auto creates{Creates(routers.Requests(router.ToString()))}; + BOOST_REQUIRE_EQUAL(creates.size(), 8U); + BOOST_CHECK(std::ranges::all_of(creates, [](const auto& c) { return Field(c.second, "DESTINATION") == TEST_PRIV; })); + BOOST_CHECK_GE(log.Count("SESSION CREATE ..."), 3U); + // In I2P's Base64 alphabet and the standard one. + std::string standard_b64{TEST_PRIV}; + std::ranges::replace(standard_b64, '-', '+'); + std::ranges::replace(standard_b64, '~', '/'); + BOOST_CHECK_EQUAL(log.Count(TEST_PRIV), 0U); + BOOST_CHECK_EQUAL(log.Count(standard_b64), 0U); + BOOST_CHECK_EQUAL(log.Count(TEST_PRIV.substr(400, 64)), 0U); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index a088bf9df7..9e36a47764 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -2,6 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include // IWYU pragma: keep + #include #include #include @@ -9,12 +11,22 @@ #include #include #include +#include #include #include +#include #include -#include +#include +#include +#include +#include +#include +#include #include +#include +#include +#include #include @@ -610,4 +622,162 @@ BOOST_AUTO_TEST_CASE(isbadport) BOOST_CHECK_EQUAL(std::ranges::count_if(ports, IsBadPort), 85); } +namespace { +/** What happened to the sockets of `PendingConnectSock`. */ +struct PendingConnectLog { + /** Sockets created. */ + size_t socks{0}; + /** The timeout of every `Wait()` for the connection to be established. */ + std::vector waits; + /** If set, the connection is established on this `Wait()` (1-based). */ + size_t connect_on_wait{0}; + /** Called on every `Wait()` with its 1-based number, before the wait. */ + std::function on_wait; +}; + +/** A socket whose connect stays in progress, see `PendingConnectLog`. */ +class PendingConnectSock : public ZeroSock +{ +public: + explicit PendingConnectSock(PendingConnectLog& log) : m_log{log} { ++m_log.socks; } + + int Connect(const sockaddr*, socklen_t) const override + { +#ifdef WIN32 + WSASetLastError(WSAEWOULDBLOCK); +#else + errno = EINPROGRESS; +#endif + return SOCKET_ERROR; + } + + bool Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred) const override + { + m_log.waits.push_back(timeout); + if (m_log.on_wait) m_log.on_wait(m_log.waits.size()); + if (m_log.waits.size() == m_log.connect_on_wait) { + *occurred = requested; + return true; + } + std::this_thread::sleep_for(timeout); + *occurred = 0; + return true; + } + + PendingConnectSock& operator=(Sock&&) override + { + assert(false && "Move of Sock into PendingConnectSock not allowed."); + return *this; + } + +private: + PendingConnectLog& m_log; +}; + +/** Make `CreateSock` return `PendingConnectSock`s while alive. */ +class PendingConnectSetup +{ +public: + PendingConnectSetup() + : m_create_sock_orig{CreateSock} + { + CreateSock = [this](int, int, int) { return std::make_unique(log); }; + } + ~PendingConnectSetup() { CreateSock = m_create_sock_orig; } + + PendingConnectLog log; + +private: + const decltype(CreateSock) m_create_sock_orig; +}; + +std::vector TestProxies() +{ + std::vector proxies{Proxy{CService{in6_addr(IN6ADDR_LOOPBACK_INIT), 7656}}}; +#ifdef HAVE_SOCKADDR_UN + proxies.emplace_back("unix:/tmp/qbit-netbase-tests-sam.sock"); +#endif + return proxies; +} + +bool WithinPollInterval(const std::vector& waits) +{ + return std::ranges::all_of(waits, [](auto w) { return w > 0ms && w <= MAX_CONNECT_POLL_INTERVAL; }); +} +} // namespace + +BOOST_AUTO_TEST_CASE(proxy_connect_without_deadline_unchanged) +{ + const int connect_timeout_orig{nConnectTimeout}; + nConnectTimeout = 20; + for (const auto& proxy : TestProxies()) { + PendingConnectSetup setup; + BOOST_CHECK(!proxy.Connect()); + // One wait of -timeout, as before the deadline overload was added. + BOOST_REQUIRE_EQUAL(setup.log.waits.size(), 1U); + BOOST_CHECK(setup.log.waits[0] == 20ms); + } + nConnectTimeout = connect_timeout_orig; +} + +BOOST_AUTO_TEST_CASE(proxy_connect_deadline_connects) +{ + for (const auto& proxy : TestProxies()) { + PendingConnectSetup setup; + setup.log.connect_on_wait = 3; + CThreadInterrupt interrupt; + BOOST_CHECK(proxy.Connect(std::chrono::steady_clock::now() + 1h, interrupt)); + BOOST_CHECK_EQUAL(setup.log.waits.size(), 3U); + BOOST_CHECK(WithinPollInterval(setup.log.waits)); + } +} + +BOOST_AUTO_TEST_CASE(proxy_connect_deadline_expires) +{ + const int connect_timeout_orig{nConnectTimeout}; + // The deadline, not -timeout, ends the attempt. + nConnectTimeout = 50; + for (const auto& proxy : TestProxies()) { + PendingConnectSetup setup; + CThreadInterrupt interrupt; + const auto start{std::chrono::steady_clock::now()}; + const auto deadline{start + 350ms}; + BOOST_CHECK(!proxy.Connect(deadline, interrupt)); + const auto end{std::chrono::steady_clock::now()}; + BOOST_CHECK(end >= deadline); + BOOST_CHECK(end - start < 60s); + BOOST_CHECK(!setup.log.waits.empty()); + BOOST_CHECK(WithinPollInterval(setup.log.waits)); + } + nConnectTimeout = connect_timeout_orig; +} + +BOOST_AUTO_TEST_CASE(proxy_connect_deadline_interrupted) +{ + for (const auto& proxy : TestProxies()) { + PendingConnectSetup setup; + CThreadInterrupt interrupt; + setup.log.on_wait = [&](size_t wait) { + if (wait == 3) interrupt(); + }; + BOOST_CHECK(!proxy.Connect(std::chrono::steady_clock::now() + 30s, interrupt)); + // Noticed right after the wait in progress when it was signaled, not at the deadline. + BOOST_CHECK_EQUAL(setup.log.waits.size(), 3U); + BOOST_CHECK(WithinPollInterval(setup.log.waits)); + } +} + +BOOST_AUTO_TEST_CASE(proxy_connect_deadline_already_over) +{ + for (const auto& proxy : TestProxies()) { + PendingConnectSetup setup; + CThreadInterrupt interrupt; + BOOST_CHECK(!proxy.Connect(std::chrono::steady_clock::now(), interrupt)); + interrupt(); + BOOST_CHECK(!proxy.Connect(std::chrono::steady_clock::now() + 1h, interrupt)); + // No socket is created at all. + BOOST_CHECK_EQUAL(setup.log.socks, 0U); + } +} + BOOST_AUTO_TEST_SUITE_END() From 104016bd915263a6d03a2db56d2d753534c5593d Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Tue, 6 Oct 2026 14:07:23 -0400 Subject: [PATCH 04/24] bench: bound the Cadence benchmark chain without -asymptote (#229) StarvationDistance() cast nanobench's complexityN() to size_t. It is -1 unless -asymptote sets it, so a plain bench_qbit run (including ctest's bench_sanity_check) built a header chain of about 2^64 blocks and grew until the host ran out of memory: 27 GiB before it was killed. Use complexityN() only when -asymptote sets it to at least 1, and default to 10,000 legacy blocks since the last AuxPoW block otherwise. Both Cadence benchmarks now run in about 17 MB. Co-authored-by: Claude Opus 5.5 --- src/bench/cadence.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/bench/cadence.cpp b/src/bench/cadence.cpp index a1be0a710b..73a4669210 100644 --- a/src/bench/cadence.cpp +++ b/src/bench/cadence.cpp @@ -12,7 +12,6 @@ #include #include -#include #include #include #include @@ -73,9 +72,15 @@ class CadenceBenchChain CBlockHeader m_next_auxpow; }; +//! Legacy blocks since the last AuxPoW block when -asymptote does not set one. +constexpr size_t DEFAULT_STARVATION_DISTANCE{10'000}; + size_t StarvationDistance(const benchmark::Bench& bench) { - return std::max(1, static_cast(bench.complexityN())); + // complexityN() is -1 unless -asymptote sets it: never cast that to size_t, + // which would build a chain until memory runs out. + const double n{bench.complexityN()}; + return n >= 1 ? static_cast(n) : DEFAULT_STARVATION_DISTANCE; } void CadenceLaneLookup(benchmark::Bench& bench) From 514cd748a7e96fadc5a2e5cd1b5a47611265a45a Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Wed, 21 Oct 2026 13:00:22 -0400 Subject: [PATCH 05/24] multi_index: fix compilation failure with boost >= 1.91 (#237) This effectively reverts a3cb309e7c31853f272bffaa65fb6ab0a7cc4083 from PR #30194. That PR reduced the multi_index type signatures as recommended upstream, but this is no longer supported as of boost 1.91 because it is no longer necessary. 1.91 drops support for the pre-c++11 work-arounds that bloated the type signatures to begin with. The upstream `BOOST_MULTI_INDEX_ENABLE_MPL_SUPPORT` define is meant to provide compatibility with removed features, but it does not work for this case. Using `indexed_by` directly when defining the `multi_index` (as opposed to inheriting from it) works with all versions, and avoids the use of the back-compat define. This is a slight regression when building against boost < 1.91 because the bloated type signatures are reintroduced in that case, but it's not significant enough to go to the trouble of introducing version detection and ifdefs. Github-Pull: #35175 Rebased-From: 0bc9d354dfd8074d1c36a891a69b6585a8775c65 (cherry picked from commit 9632450ff65c1fee29c2564a6ad64dfaad4d663c) Backport of bitcoin/bitcoin#35175. Co-authored-by: Cory Fields --- src/node/miner.h | 11 ++++------- src/node/txorphanage.cpp | 13 +++++++------ src/txmempool.h | 10 ++++------ src/txrequest.cpp | 12 +++++------- 4 files changed, 20 insertions(+), 26 deletions(-) diff --git a/src/node/miner.h b/src/node/miner.h index 96c8c3bc2c..331e4877c3 100644 --- a/src/node/miner.h +++ b/src/node/miner.h @@ -109,7 +109,9 @@ struct CompareTxIterByAncestorCount { }; -struct CTxMemPoolModifiedEntry_Indices final : boost::multi_index::indexed_by< +using indexed_modified_transaction_set = boost::multi_index_container< + CTxMemPoolModifiedEntry, + boost::multi_index::indexed_by< boost::multi_index::ordered_unique< modifiedentry_iter, CompareCTxMemPoolIter @@ -122,12 +124,7 @@ struct CTxMemPoolModifiedEntry_Indices final : boost::multi_index::indexed_by< CompareTxMemPoolEntryByAncestorFee > > -{}; - -typedef boost::multi_index_container< - CTxMemPoolModifiedEntry, - CTxMemPoolModifiedEntry_Indices -> indexed_modified_transaction_set; +>; typedef indexed_modified_transaction_set::nth_index<0>::type::iterator modtxiter; typedef indexed_modified_transaction_set::index::type::iterator modtxscoreiter; diff --git a/src/node/txorphanage.cpp b/src/node/txorphanage.cpp index 12683b2018..a92dbc9b32 100644 --- a/src/node/txorphanage.cpp +++ b/src/node/txorphanage.cpp @@ -91,12 +91,13 @@ class TxOrphanageImpl final : public TxOrphanage { } }; - struct OrphanIndices final : boost::multi_index::indexed_by< - boost::multi_index::ordered_unique, WtxidExtractor>, - boost::multi_index::ordered_unique, ByPeerViewExtractor> - >{}; - - using AnnouncementMap = boost::multi_index::multi_index_container; + using AnnouncementMap = boost::multi_index::multi_index_container< + Announcement, + boost::multi_index::indexed_by< + boost::multi_index::ordered_unique, WtxidExtractor>, + boost::multi_index::ordered_unique, ByPeerViewExtractor> + > + >; template using Iter = typename AnnouncementMap::index::type::iterator; AnnouncementMap m_orphans; diff --git a/src/txmempool.h b/src/txmempool.h index 611b159d3e..2ee9282118 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -306,7 +306,9 @@ class CTxMemPool static const int ROLLING_FEE_HALFLIFE = 60 * 60 * 12; // public only for testing - struct CTxMemPoolEntry_Indices final : boost::multi_index::indexed_by< + using indexed_transaction_set = boost::multi_index_container< + CTxMemPoolEntry, + boost::multi_index::indexed_by< // sorted by txid boost::multi_index::hashed_unique, // sorted by wtxid @@ -334,11 +336,7 @@ class CTxMemPool CompareTxMemPoolEntryByAncestorFee > > - {}; - typedef boost::multi_index_container< - CTxMemPoolEntry, - CTxMemPoolEntry_Indices - > indexed_transaction_set; + >; /** * This mutex needs to be locked when accessing `mapTx` or other members diff --git a/src/txrequest.cpp b/src/txrequest.cpp index 4d7240bee0..53136a8e71 100644 --- a/src/txrequest.cpp +++ b/src/txrequest.cpp @@ -208,17 +208,15 @@ struct ByTimeViewExtractor } }; -struct Announcement_Indices final : boost::multi_index::indexed_by< - boost::multi_index::ordered_unique, ByPeerViewExtractor>, - boost::multi_index::ordered_non_unique, ByTxHashViewExtractor>, - boost::multi_index::ordered_non_unique, ByTimeViewExtractor> -> -{}; /** Data type for the main data structure (Announcement objects with ByPeer/ByTxHash/ByTime indexes). */ using Index = boost::multi_index_container< Announcement, - Announcement_Indices + boost::multi_index::indexed_by< + boost::multi_index::ordered_unique, ByPeerViewExtractor>, + boost::multi_index::ordered_non_unique, ByTxHashViewExtractor>, + boost::multi_index::ordered_non_unique, ByTimeViewExtractor> + > >; /** Helper type to simplify syntax of iterator types. */ From 173cd22d5ac1e29f431c3f1149cfae1a31c7b039 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Wed, 21 Oct 2026 17:02:12 -0400 Subject: [PATCH 06/24] ci: wait for a newer run instead of failing on an earlier one's result (#231) * ci: wait for a newer run instead of failing on an earlier one's result The Required Merge Gate reads the newest check run of each required name on the commit. When CI runs again on the same commit (a re-run, a reopened pull request, a label), the gate starts within seconds, before the new workflows have created their own gate checks. It then found the previous run's result and, if that run had been cancelled or failed, failed at once, although the new run was still on its way. PR #229's gate failed this way three seconds after a reopen. Treat a check that concluded other than success before this gate run started as pending: it belongs to an earlier run, and a newer one may not have reported yet. The gate keeps polling and fails, with the reason named, only if no newer result arrives before its deadline. A failure after the gate run started still fails at once, and an earlier success still counts, as a label starts no new Core Checks. Co-Authored-By: Claude Opus 5.5 * ci: date the merge gate from its workflow run, not its step The gate step starts only after the jobs it needs, so a check from the current run can fail before the step's own start time and be taken for an earlier run's result, making the gate wait out its deadline instead of failing at once. Read this run attempt's run_started_at from the API instead, and only when checks are polled, so profiles that poll nothing still make no API calls. If it can't be read, fall back to the step's start with a warning. Co-Authored-By: Claude Opus 5.5 * ci: never take a cancelled check as the merge gate's verdict A newer push, reopen or label cancels the running Full Validation (cancel-in-progress), and that cancellation can complete after this gate run started, so the run-start cutoff alone took it for a current failure. A cancelled check means superseded or never run, so treat it as pending and wait for the run that replaces it; the gate still fails at its deadline if none arrives. Co-Authored-By: Claude Opus 5.5 * ci: let the merge gate read its run attempt's start time Getting a workflow run attempt needs Actions: read, and the workflow grants only contents and checks, so the start-time lookup always fell back to the step's own start. Grant actions: read, and pin it in the poller contract test. Co-Authored-By: Claude Opus 5.5 * ci: start Core Checks on every event that starts the merge gate The gate now waits for a newer Core Checks Gate instead of trusting an earlier failure, but Core Checks ran only on opened, synchronize and reopened, so a label, unlabel or ready-for-review event left the gate waiting out its deadline for a run that could not start. Give Core Checks the same pull_request activity types as the gate and Full Validation, and pin that in the poller contract test. Co-Authored-By: Claude Opus 5.5 * ci: tell the merge gate's checks apart by the event that started them The gate waited on a check that failed before its own run started, taking it for an earlier run's result. A time cutoff cannot tell the two apart: Core Checks, Full Validation and the gate are separate workflows, so this event's check can fail before the gate's run starts, an earlier event's can fail or be cancelled after it, and re-running the gate alone moves its start but starts nothing else. Date each failed check by its event instead. Every workflow run an event starts is created at that event, and a re-run keeps the run, its check suite and its created_at. The gate maps a failed check's check suite to its workflow run (GET /actions/runs?check_suite_id=) and compares that run's created_at with its own run's (GET /actions/runs/{run_id}), allowing 60 seconds between one event's runs. - An earlier event's failure is waited on until a newer run reports. - A failure from this event or a later one fails at once, including after a gate-only re-run. - A success counts whichever event produced it, and a cancellation is never a verdict; neither needs a lookup. - If either run can't be read after the usual retries, the gate fails closed under its own error title. Co-Authored-By: Claude Opus 5.5 * ci: let the newest event's check decide, not the newest-started one The gate judged only the newest check run of each required name by started_at. A manual re-run of an earlier event's workflow starts later than this event's check, so it hid this event's result: a failed old re-run was taken as an earlier event's failure and waited on, poll after poll, until the deadline, even with this event's success already on the commit. Read every check run of the name instead. When they come from more than one workflow run and not all of them succeeded, map each check suite to its workflow run (cached per suite) and let the newest workflow run's check decide, by created_at, then run_number for runs created in the same second, then started_at for its attempts. - An earlier event's re-run never outranks this or a later event's result, whether it failed, passed, was cancelled or is still running. - Attempts within one workflow run still rank by started_at alone, and a check whose runs all succeeded passes, both without a lookup. - If the runs to rank can't be read, or a check run names no check suite, the gate fails closed under its "could not correlate" title. Co-Authored-By: Claude Opus 5.5 * ci: tell close events apart by a newer run of the check's workflow The gate took any failure whose workflow run was created within 60 s of its own as this event's. Two events that close together (a label, then an unlabel or a reopen) made the second gate fail at once on the first event's result. That is the usual case, not an edge one: the newer event cancels the earlier event's Core Checks and Full Validation, and their if: always() gate jobs then conclude failure (7 of 8 cancelled Core Checks runs sampled here did), after the newer event has started. GitHub's REST API names no event on a workflow run, check suite or check run, so there is no marker to compare. It does give each workflow's runs a run_number that grows with every new run and is kept by re-runs. Before failing at once, the gate now lists the check's workflow's runs on the commit (GET /actions/workflows/{id}/runs ?head_sha=): if one has a higher run_number, an event since has started the run that replaces this one, and the failure is waited on. That tells events apart at any spacing, even within the same second. Tighten the allowance from 60 s to 10 s. Across 325 pull request events on this repository (June to October 2026), one event's runs were created within 4 s of each other, and the gate's up to 3 s after another's, so exact equality would misdate a current failure; the closest two events on one commit were 28 s apart. - A current failure with no newer run of its workflow still fails at once, including after a gate-only re-run. - A current failure whose workflow already has a newer run, from a later event, is waited on, and that run's result decides. - A newer run, once listed, is cached; a listing that can't be read, or lacks the check's own run, fails closed. - Left open: two events within 10 s where the earlier one's check fails and the later one's run is not yet listed when the gate judges it. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .github/workflows/core-checks.yml | 9 + .github/workflows/required-merge-gate.yml | 506 ++++++++-- ci/checks/test_merge_gate_poller.py | 1049 ++++++++++++++++++++- 3 files changed, 1450 insertions(+), 114 deletions(-) diff --git a/.github/workflows/core-checks.yml b/.github/workflows/core-checks.yml index 2e699513cf..fc69e7a364 100644 --- a/.github/workflows/core-checks.yml +++ b/.github/workflows/core-checks.yml @@ -6,6 +6,15 @@ name: Core Checks on: pull_request: + # The same activity types as the Required Merge Gate, which waits for + # its own event's Core Checks Gate instead of an earlier event's failure. + types: + - opened + - synchronize + - reopened + - ready_for_review + - labeled + - unlabeled branches: - main - 1.0.0 diff --git a/.github/workflows/required-merge-gate.yml b/.github/workflows/required-merge-gate.yml index a8be0dff67..cea6f908b3 100644 --- a/.github/workflows/required-merge-gate.yml +++ b/.github/workflows/required-merge-gate.yml @@ -40,6 +40,9 @@ on: permissions: contents: read checks: read + # The gate reads workflow runs, its own and those behind the checks it + # waits for, to tell which event produced a failed check. + actions: read concurrency: group: required-merge-gate-${{ github.event_name != 'pull_request' && github.run_id || github.ref }} @@ -714,6 +717,7 @@ jobs: TOUCHED_OPERATOR_KEYS: ${{ needs.classify-changes.outputs.touched_operator_keys }} REPOSITORY: ${{ github.repository }} SHA: ${{ github.event.pull_request.head.sha || github.sha }} + RUN_ID: ${{ github.run_id }} run: | set -euo pipefail @@ -763,11 +767,17 @@ jobs: # # * a page without a check_runs array is malformed and must never be # mistaken for "this commit has no check runs"; - # * a re-run check appears more than once, so the newest run by - # started_at (created_at while a run has not started) wins; + # * every check run with the name is reported, one per line, oldest + # first by started_at (created_at while a run has not started): a + # re-run check appears more than once, and so does a check that + # more than one event ran (judge_check below picks the one that + # decides); # * "absent" means the query succeeded and reported no run with that # name, which is a different fact from a query that could not be - # read at all. + # read at all; + # * each check run's check suite travels with it: it names the + # workflow run, and so the event, that produced the check (see + # date_check_suite below). # $name is a jq variable bound by --arg below, not a shell expansion. # shellcheck disable=SC2016 check_run_filter=' @@ -776,33 +786,44 @@ jobs: | add | map(select(.name == $name)) | sort_by(.started_at // .created_at // "") - | last - | if . == null then "absent" else ["found", (.status // ""), (.conclusion // "")] | @tsv end + | if length == 0 then "absent" + else .[] | ["found", (.status // ""), (.conclusion // ""), (((.check_suite // {}).id // "") | tostring), (.started_at // .created_at // "")] | @tsv + end ' + # "foundstatusconclusioncheck suite idstarted_at" + check_run_line=$'found\t[a-z_]+\t[^\t\n]*\t[^\t\n]*\t[^\t\n]*' + check_run_pattern="^(absent|${check_run_line}("$'\n'"${check_run_line})*)\$" summarize_stderr() { tr '\n' ' ' < "$1" | cut -c1-300 } - # Read the newest check run named $1 on ${SHA}. + # GET the API path $4 with `gh api --paginate`, pass the slurped + # pages through the jq program $6 (arguments from $7 on go to jq), + # and trust the result only if it matches the bash regex $5. $1 + # names the check for the log and $2 what is read. # - # On success sets QUERY_OBSERVATION to "absent" or to - # "foundstatusconclusion" and returns 0. Returns 1 when the - # query could not be read within GATE_QUERY_MAX_ATTEMPTS attempts or - # before $2, the wall-clock deadline shared with the caller: an - # unreadable query is never success and never "not found yet". + # On success sets API_RESULT and returns 0. Returns 1 when the path + # could not be read within GATE_QUERY_MAX_ATTEMPTS attempts or + # before $3, the wall-clock deadline shared with the caller: an + # unreadable response is never success and never "not found yet". # # The result travels in a variable rather than on stdout because this # function also logs there, and errexit is not in force for a function # called from a condition, so every failure is handled explicitly. - query_check_run() { + read_api() { local check_name="$1" - local deadline="$2" + local what="$2" + local deadline="$3" + local path="$4" + local pattern="$5" + local filter="$6" + shift 6 local attempt=1 local backoff="${GATE_QUERY_BACKOFF_SECONDS}" - local body fields observation reason status stderr_path now rc + local body reason result stderr_path now rc - QUERY_OBSERVATION="" + API_RESULT="" while (( attempt <= GATE_QUERY_MAX_ATTEMPTS )); do stderr_path="$(mktemp)" @@ -811,39 +832,27 @@ jobs: timeout "${GATE_QUERY_TIMEOUT_SECONDS}" gh api \ -H "Accept: application/vnd.github+json" \ --paginate \ - "/repos/${REPOSITORY}/commits/${SHA}/check-runs?per_page=100" \ + "${path}" \ 2>"${stderr_path}" )" || rc=$? if (( rc != 0 )); then - reason="check-runs query failed (exit ${rc}): $(summarize_stderr "${stderr_path}")" + reason="${what} query failed (exit ${rc}): $(summarize_stderr "${stderr_path}")" else rc=0 - observation="$( + result="$( printf '%s' "${body}" \ - | jq -e -r -s --arg name "${check_name}" "${check_run_filter}" 2>"${stderr_path}" + | jq -e -r -s "$@" "${filter}" 2>"${stderr_path}" )" || rc=$? if (( rc != 0 )); then - reason="check-runs response could not be read (jq exit ${rc}): $(summarize_stderr "${stderr_path}")" + reason="${what} response could not be read (jq exit ${rc}): $(summarize_stderr "${stderr_path}")" + elif [[ "${result}" =~ ${pattern} ]]; then + rm -f "${stderr_path}" + API_RESULT="${result}" + return 0 else - case "${observation}" in - absent) - rm -f "${stderr_path}" - QUERY_OBSERVATION="${observation}" - return 0 - ;; - found$'\t'*) - fields="${observation#found$'\t'}" - status="${fields%%$'\t'*}" - if [[ "${status}" =~ ^[a-z_]+$ ]]; then - rm -f "${stderr_path}" - QUERY_OBSERVATION="${observation}" - return 0 - fi - ;; - esac - reason="check-runs response was malformed: ${observation}" + reason="${what} response was malformed: ${result}" fi fi @@ -857,7 +866,7 @@ jobs: now="$(date +%s)" if (( now + backoff > deadline )); then - echo "${check_name}: wait deadline reached while the check-runs query was unreadable" + echo "${check_name}: wait deadline reached while the ${what} query was unreadable" break fi @@ -871,16 +880,391 @@ jobs: return 1 } + # Read every check run named $1 on ${SHA}. + # + # On success sets QUERY_OBSERVATION to "absent" or to one line per + # check run (see check_run_pattern) and returns 0. Returns 1 when the + # query could not be read (see read_api; $2 is the deadline). + query_check_run() { + QUERY_OBSERVATION="" + read_api "$1" "check-runs" "$2" \ + "/repos/${REPOSITORY}/commits/${SHA}/check-runs?per_page=100" \ + "${check_run_pattern}" "${check_run_filter}" --arg name "$1" || return 1 + QUERY_OBSERVATION="${API_RESULT}" + } + + # Which event a check belongs to. + # + # When CI runs again on a commit (a reopen, a label, a re-run), this + # gate starts within seconds, but Core Checks and Full Validation + # create their gate checks only at their last job, minutes later. + # Until then the newest check of that name is an earlier event's, + # and its failure says nothing about the run under way. A cutoff + # in time cannot tell that apart from a failure of this event: + # Core Checks, Full Validation and this gate are separate workflows, + # scheduled independently, so a check this event started can fail + # before this gate's run even starts; an earlier event's check can be + # cancelled, or fail, after it started; and re-running this gate + # alone starts it again but starts nothing else. + # + # What the checks of one event share is the event, but GitHub's REST + # API names it nowhere: a workflow run, check suite or check run + # carries no delivery id or activity type. What it does give: + # + # * every workflow run a push, pull request event or merge group + # starts is created at that event, within seconds of the others + # (see EVENT_SKEW_SECONDS for how close), and a re-run keeps the + # workflow run: its id, run_number, check suite and created_at + # stay, and only run_attempt and run_started_at move (checked on + # this repository's PR #229); + # * a workflow's run_number grows with each new run and is kept by a + # re-run (GitHub's contexts reference), so of two runs of one + # workflow on this commit the higher run_number is the later + # event's, even when both were created in the same second. + # + # So: + # + # * a check run's check suite names the workflow run that produced + # it (GET /actions/runs?check_suite_id=), whose created_at dates + # the event that started it; once read, a check suite's workflow + # run is cached, as it cannot change; + # * this gate's own workflow run's created_at (GET + # /actions/runs/{run_id}, never the attempt's) dates the event it + # judges, so a gate-only re-run still judges its original event; + # * a check whose workflow run was created more than + # EVENT_SKEW_SECONDS before this gate's belongs to an earlier event; + # * any other belongs to this event or a later one, unless a newer + # run of its workflow is on this commit (GET + # /actions/workflows/{id}/runs?head_sha=): then an event since, + # however soon after, has started the run that replaces it. + # + # That newer run is the usual case, not an edge one. A newer event + # cancels an earlier event's Core Checks and Full Validation runs + # (their concurrency groups cancel in progress), whose gate jobs + # still run (if: always()) and conclude failure: of 8 such cancelled + # Core Checks runs sampled here, 7 did. So the earlier event's + # failure lands after the newer event, and the newer run already + # exists, having caused the cancellation. + # + # What is left: two events whose runs are created within + # EVENT_SKEW_SECONDS of each other, the earlier one's check failing, + # and the later one's run of that workflow not yet listed when this + # gate judges it. That failure is taken as current and fails at once. + # And a current failure whose workflow has a newer run, from an event + # after this gate's, is waited on: that run's result decides. + # + # Only a check that concluded neither success nor cancelled is + # dated against this gate: a success on this commit passes whichever + # event produced it, and a cancellation is never a verdict. Which of + # a check's runs is judged at all is judge_check's to say, below. + # + # Workflow runs started by one event are created this close together. + # Across 325 pull request events on this repository (June to October + # 2026), one event's runs were created within 4 s of each other, and + # this gate's run up to 3 s after another's; the closest two events + # on one commit were 28 s apart. A documented allowance, not a + # tunable: a smaller value risks taking a current failure as earlier + # (it is waited on until the deadline), a larger one widens the + # window left above, where only the newer run tells events apart. + EVENT_SKEW_SECONDS=10 + + # GET /actions/runs/{run_id}: this gate's own workflow run. + # $run_id is bound by --argjson below. + # shellcheck disable=SC2016 + gate_run_filter=' + if type != "array" or length != 1 then error("expected one workflow run") else .[0] end + | if .id != $run_id then error("not workflow run \($run_id)") else . end + | .created_at + | [(fromdateiso8601 | tostring), .] + | @tsv + ' + # "" + created_at_pattern=$'^[0-9]+\t[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$' + # GET /actions/runs?check_suite_id={id}: the one workflow run that + # owns a check suite. A run of another suite is never taken for it, + # even if the filter were ignored. $suite is bound by --argjson. + # shellcheck disable=SC2016 + suite_run_filter=' + if type != "array" or length == 0 then error("empty workflow-runs response") else . end + | map(if (.workflow_runs | type) == "array" then .workflow_runs[] else error("malformed workflow-runs page") end) + | map(select(.check_suite_id == $suite)) + | if length != 1 then error("\(length) workflow runs own check suite \($suite)") else .[0] end + | [(.created_at | fromdateiso8601 | tostring), (.run_number | tostring), (.workflow_id | tostring), .created_at] + | @tsv + ' + # "" + suite_run_pattern=$'^[0-9]+\t[1-9][0-9]*\t[1-9][0-9]*\t[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$' + # GET /actions/workflows/{id}/runs?head_sha=: a workflow's runs on + # this commit, and of them the one with the highest run_number. Runs + # of another workflow or commit are never counted, even if the + # filters were ignored, and a listing that lacks the failed check's + # own run is not trusted. $workflow and $suite are bound by + # --argjson, $sha by --arg. + # shellcheck disable=SC2016 + newest_run_filter=' + if type != "array" or length == 0 then error("empty workflow-runs response") else . end + | map(if (.workflow_runs | type) == "array" then .workflow_runs[] else error("malformed workflow-runs page") end) + | map(select(.workflow_id == $workflow and .head_sha == $sha)) + | if any(.[]; .check_suite_id == $suite) then . else error("check suite \($suite) is not among the runs of workflow \($workflow) on \($sha)") end + | if all(.[]; (.run_number | type) == "number") then . else error("a run of workflow \($workflow) has no run_number") end + | max_by(.run_number) + | [(.run_number | tostring), .created_at] + | @tsv + ' + # "" + newest_run_pattern=$'^[1-9][0-9]*\t[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$' + + GATE_RUN_CREATED="" + declare -A SUITE_RUN=() + declare -A SUITE_NEWER_RUN=() + + # Read this gate's own workflow run once, into GATE_RUN_CREATED + # (""). Returns 1 if it could not be read by + # $2, the shared deadline ($1 names the check for the log). + date_gate_run() { + if [[ -n "${GATE_RUN_CREATED}" ]]; then + return 0 + fi + if [[ ! "${RUN_ID}" =~ ^[1-9][0-9]*$ ]]; then + echo "$1: this gate's workflow run id is not a number: '${RUN_ID}'" + return 1 + fi + read_api "$1" "workflow-run" "$2" \ + "/repos/${REPOSITORY}/actions/runs/${RUN_ID}" \ + "${created_at_pattern}" "${gate_run_filter}" --argjson run_id "${RUN_ID}" || return 1 + GATE_RUN_CREATED="${API_RESULT}" + echo "This gate's workflow run was created at ${GATE_RUN_CREATED#*$'\t'}, by the event it judges" + } + + # Read the workflow run that owns check suite $2 once, into + # SUITE_RUN[$2] (see suite_run_pattern). Returns 1 if $2 is not a + # check suite id, or its workflow run could not be read by $3, the + # shared deadline ($1 names the check for the log). + date_check_suite() { + if [[ ! "$2" =~ ^[1-9][0-9]*$ ]]; then + echo "$1: the check run names no check suite: '$2'" + return 1 + fi + if [[ -n "${SUITE_RUN[$2]:-}" ]]; then + return 0 + fi + read_api "$1" "workflow-run" "$3" \ + "/repos/${REPOSITORY}/actions/runs?check_suite_id=$2&exclude_pull_requests=true&per_page=100" \ + "${suite_run_pattern}" "${suite_run_filter}" --argjson suite "$2" || return 1 + SUITE_RUN[$2]="${API_RESULT}" + } + + # Date the event of check $1, from check suite $2 (see above). + # + # Sets CHECK_EVENT to "earlier", "superseded" (a newer run of its + # workflow is on this commit, named in CHECK_NEWER_RUN as + # "") or "current", and CHECK_RUN_NUMBER + # and CHECK_RUN_CREATED_AT to its own workflow run's, and returns 0. + # Returns 1 when the check run names no check suite, or a workflow + # run or listing it needs could not be read by $3, the shared + # deadline: an event that cannot be dated is never success, never + # earlier and never current. + date_check_event() { + local check_name="$1" + local check_suite_id="$2" + local deadline="$3" + local check_created gate_created workflow_id newest_number + + CHECK_EVENT="" + CHECK_RUN_NUMBER="" + CHECK_RUN_CREATED_AT="" + CHECK_NEWER_RUN="" + + date_gate_run "${check_name}" "${deadline}" || return 1 + date_check_suite "${check_name}" "${check_suite_id}" "${deadline}" || return 1 + + check_created="${SUITE_RUN[${check_suite_id}]}" + CHECK_RUN_CREATED_AT="${check_created##*$'\t'}" + CHECK_RUN_NUMBER="${check_created#*$'\t'}" + workflow_id="${CHECK_RUN_NUMBER#*$'\t'}" + CHECK_RUN_NUMBER="${CHECK_RUN_NUMBER%%$'\t'*}" + workflow_id="${workflow_id%%$'\t'*}" + check_created="${check_created%%$'\t'*}" + gate_created="${GATE_RUN_CREATED%%$'\t'*}" + if (( check_created < gate_created - EVENT_SKEW_SECONDS )); then + CHECK_EVENT="earlier" + return 0 + fi + + # A newer run, once listed, stays on the commit, so it is cached; + # its absence is read when the verdict hangs on it, and a failure + # with no newer run is a verdict that ends the wait. + if [[ -z "${SUITE_NEWER_RUN[${check_suite_id}]:-}" ]]; then + read_api "${check_name}" "workflow-runs" "${deadline}" \ + "/repos/${REPOSITORY}/actions/workflows/${workflow_id}/runs?head_sha=${SHA}&exclude_pull_requests=true&per_page=100" \ + "${newest_run_pattern}" "${newest_run_filter}" \ + --argjson workflow "${workflow_id}" --argjson suite "${check_suite_id}" --arg sha "${SHA}" || return 1 + newest_number="${API_RESULT%%$'\t'*}" + if (( newest_number > CHECK_RUN_NUMBER )); then + SUITE_NEWER_RUN[${check_suite_id}]="${API_RESULT}" + fi + fi + if [[ -n "${SUITE_NEWER_RUN[${check_suite_id}]:-}" ]]; then + CHECK_NEWER_RUN="${SUITE_NEWER_RUN[${check_suite_id}]}" + CHECK_EVENT="superseded" + else + CHECK_EVENT="current" + fi + } + + # Judge check $1 from its check runs in QUERY_OBSERVATION, by $2, the + # shared deadline. + # + # A check has a run per event that ran CI on this commit, each in + # that event's workflow run, and a re-run adds another to the same + # workflow run. The check run of the newest workflow run decides: by + # its created_at, which dates its event, then its run_number, which + # orders one workflow's runs created in the same second; and within + # that workflow run the newest attempt, by started_at. So a manual + # re-run of an earlier event's workflow, however recent, never + # outranks this event's (or a later event's) result, and an earlier + # event's result is judged only while no later one has reported. + # Workflow runs are read only to rank check runs from more than one + # of them, and only when not all of those succeeded: a check whose + # runs all succeeded passes whichever decides. + # + # Sets CHECK_VERDICT to "success", "failed" or "pending" and returns 0; + # a failed check's conclusion is in CHECK_CONCLUSION, and a pending + # one is named in CHECK_PENDING the way the timeout reports it. + # Returns 1, with the reason in CHECK_UNCORRELATED, when the workflow + # runs the verdict depends on could not be read. + judge_check() { + local check_name="$1" + local deadline="$2" + local lines=() statuses=() conclusions=() suites=() stamps=() + local line fields index decide ranked run run_number + local status conclusion all_success=1 + local -A distinct=() + + CHECK_VERDICT="" + CHECK_CONCLUSION="" + CHECK_PENDING="${check_name}" + CHECK_UNCORRELATED="" + + if [[ "${QUERY_OBSERVATION}" == "absent" ]]; then + echo "${check_name}: query succeeded, no such check run on ${SHA} yet" + CHECK_VERDICT="pending" + return 0 + fi + + mapfile -t lines <<< "${QUERY_OBSERVATION}" + for line in "${lines[@]}"; do + fields="${line#found$'\t'}" + statuses+=("${fields%%$'\t'*}") + fields="${fields#*$'\t'}" + conclusions+=("${fields%%$'\t'*}") + fields="${fields#*$'\t'}" + suites+=("${fields%%$'\t'*}") + stamps+=("${fields#*$'\t'}") + if [[ "${statuses[-1]}" != "completed" || "${conclusions[-1]}" != "success" ]]; then + all_success=0 + fi + done + + # The newest by started_at, which is right within one workflow run. + decide=$(( ${#lines[@]} - 1 )) + if (( ${#lines[@]} > 1 && ! all_success )); then + for index in "${!lines[@]}"; do + if [[ ! "${suites[index]}" =~ ^[1-9][0-9]*$ ]]; then + echo "${check_name}: a check run names no check suite: '${suites[index]}'" + CHECK_UNCORRELATED="${check_name} on ${SHA} has ${#lines[@]} check runs, but not every one names the workflow run that tells which event is newest" + return 1 + fi + distinct[${suites[index]}]=1 + done + if (( ${#distinct[@]} > 1 )); then + ranked="" + for index in "${!lines[@]}"; do + if ! date_check_suite "${check_name}" "${suites[index]}" "${deadline}"; then + CHECK_UNCORRELATED="${check_name} on ${SHA} has check runs from ${#distinct[@]} workflow runs, but the workflow runs that tell which event is newest could not be read" + return 1 + fi + run="${SUITE_RUN[${suites[index]}]}" + run_number="${run#*$'\t'}" + run_number="${run_number%%$'\t'*}" + ranked+="${run%%$'\t'*}"$'\t'"${run_number}"$'\t'"${stamps[index]}"$'\t'"${index}"$'\n' + done + decide="$(printf '%s' "${ranked}" | LC_ALL=C sort -t $'\t' -k1,1n -k2,2n -k3,3 -k4,4n | tail -n 1)" + decide="${decide##*$'\t'}" + if [[ ! "${decide}" =~ ^[0-9]+$ ]] || (( decide >= ${#lines[@]} )); then + CHECK_UNCORRELATED="${check_name} on ${SHA} has check runs from ${#distinct[@]} workflow runs, which could not be ranked" + return 1 + fi + run="${SUITE_RUN[${suites[decide]}]}" + echo "${check_name}: ${#lines[@]} check runs from ${#distinct[@]} workflow runs; the newest workflow run's, created at ${run##*$'\t'}, decides" + fi + fi + + status="${statuses[decide]}" + conclusion="${conclusions[decide]}" + echo "${check_name}: status=${status} conclusion=${conclusion:-pending}" + + if [[ "${status}" != "completed" ]]; then + CHECK_VERDICT="pending" + return 0 + fi + if [[ "${conclusion}" == "success" ]]; then + CHECK_VERDICT="success" + return 0 + fi + if [[ "${conclusion}" == "cancelled" ]]; then + # A cancelled check is never a verdict: it was superseded (a + # newer push, reopen or label cancels the running workflow, + # possibly after this gate started) or it never ran. Wait for + # the run that replaces it; the deadline still fails the gate. + echo "${check_name}: cancelled; waiting for the run that replaces it" + CHECK_PENDING="${check_name} (cancelled)" + CHECK_VERDICT="pending" + return 0 + fi + + if ! date_check_event "${check_name}" "${suites[decide]}" "${deadline}"; then + CHECK_UNCORRELATED="${check_name} on ${SHA} concluded ${conclusion:-unknown}, but the workflow runs that tell whether it belongs to this event could not be read" + return 1 + fi + case "${CHECK_EVENT}" in + earlier) + echo "${check_name}: concluded ${conclusion:-unknown} in a workflow run created at ${CHECK_RUN_CREATED_AT}, by an earlier event than this gate's (${GATE_RUN_CREATED#*$'\t'}); waiting for a newer run" + CHECK_PENDING="${check_name} (last ${conclusion:-unknown} from an earlier event)" + CHECK_VERDICT="pending" + ;; + superseded) + echo "${check_name}: concluded ${conclusion:-unknown} in run #${CHECK_RUN_NUMBER} of its workflow, created at ${CHECK_RUN_CREATED_AT}, but run #${CHECK_NEWER_RUN%%$'\t'*}, created at ${CHECK_NEWER_RUN#*$'\t'}, has since started on ${SHA}; waiting for its result" + CHECK_PENDING="${check_name} (last ${conclusion:-unknown} superseded by a newer run)" + CHECK_VERDICT="pending" + ;; + current) + CHECK_CONCLUSION="${conclusion}" + CHECK_VERDICT="failed" + ;; + *) + CHECK_UNCORRELATED="${check_name} on ${SHA} concluded ${conclusion:-unknown} in an event that could not be told ('${CHECK_EVENT}')" + return 1 + ;; + esac + } + # Wait until every named check on ${SHA} has concluded successfully. # - # Returns 1 under three distinct error titles: a check concluded - # something other than success, the check runs could not be read, or - # the shared deadline passed with checks still pending. Only a - # completed check with a success conclusion is success. + # Returns 1 under four distinct error titles: a check concluded + # something other than success, the check runs could not be read, a + # check's runs could not be traced to their events, or the shared + # deadline passed with checks still pending. Only a completed check + # with a success conclusion is success. A check that concluded + # otherwise is a verdict only if it belongs to the event this gate + # judges, or a later one, and no newer run of its workflow is on this + # commit; an earlier event's, or a superseded one, is waited on, as + # its newer run may not have reported yet, and a cancelled one is + # always waited on. wait_for_checks() { local check_names=("$@") local pending=() - local check_name conclusion fields now observation status deadline + local check_name now deadline now="$(date +%s)" deadline=$(( now + GATE_WAIT_TIMEOUT_SECONDS )) @@ -893,28 +1277,26 @@ jobs: echo "::error title=Required Merge Gate could not read check runs::Check runs for ${check_name} on ${SHA} could not be read after ${GATE_QUERY_MAX_ATTEMPTS} attempts; failing closed with no verdict for: ${check_names[*]}" return 1 fi - observation="${QUERY_OBSERVATION}" - - if [[ "${observation}" == "absent" ]]; then - echo "${check_name}: query succeeded, no such check run on ${SHA} yet" - pending+=("${check_name}") - continue + if ! judge_check "${check_name}" "${deadline}"; then + echo "::error title=Required Merge Gate could not correlate check runs::${CHECK_UNCORRELATED}; failing closed with no verdict for: ${check_names[*]}" + return 1 fi - fields="${observation#found$'\t'}" - status="${fields%%$'\t'*}" - conclusion="${fields#*$'\t'}" - echo "${check_name}: status=${status} conclusion=${conclusion:-pending}" - - if [[ "${status}" == "completed" ]]; then - if [[ "${conclusion}" != "success" ]]; then - echo "::error title=Required Merge Gate failed::${check_name} concluded ${conclusion:-unknown}." + case "${CHECK_VERDICT}" in + success) + ;; + pending) + pending+=("${CHECK_PENDING}") + ;; + failed) + echo "::error title=Required Merge Gate failed::${check_name} concluded ${CHECK_CONCLUSION:-unknown}." return 1 - fi - continue - fi - - pending+=("${check_name}") + ;; + *) + echo "::error title=Required Merge Gate failed::${check_name} has no verdict ('${CHECK_VERDICT}')." + return 1 + ;; + esac done if (( ${#pending[@]} == 0 )); then diff --git a/ci/checks/test_merge_gate_poller.py b/ci/checks/test_merge_gate_poller.py index 4ea097df16..1d1e6cb2b8 100755 --- a/ci/checks/test_merge_gate_poller.py +++ b/ci/checks/test_merge_gate_poller.py @@ -17,9 +17,17 @@ test, so the production defaults stay the values CI actually uses while the suite still runs in seconds. +The stub also answers the workflow-run lookups the gate makes to rank a +check's runs by the event that produced them and to tell which event produced +a failed one (its own run, and the run that owns each such check suite), so +those are exercised by the same real script. + Not provable here: whether the hosted ``Required Merge Gate`` check is attached to a branch ruleset, and whether the real API agrees with the shapes -modelled below. +modelled below. The workflow-run shapes and their re-run semantics (a re-run +keeps the run's ``check_suite_id`` and ``created_at`` and moves only +``run_attempt`` and ``run_started_at``) were read from GitHub's REST API +description and checked against this repository's PR #229 on 2026-10-06. """ from __future__ import annotations @@ -31,6 +39,7 @@ import tempfile import time import unittest +from datetime import datetime, timedelta, timezone from pathlib import Path CHECKS_DIR = Path(__file__).resolve().parent @@ -85,16 +94,119 @@ ERROR_CHECK_FAILED = "::error title=Required Merge Gate failed::" ERROR_UNREADABLE = "::error title=Required Merge Gate could not read check runs::" +ERROR_UNCORRELATED = "::error title=Required Merge Gate could not correlate check runs::" ERROR_TIMED_OUT = "::error title=Required Merge Gate timed out::" WARNING_RETRYING = "::warning title=Required Merge Gate retrying::" +# The documented allowance between the workflow runs one event creates: 325 +# pull request events on this repository (June to October 2026) spread their +# runs over at most 4 s, and the closest two events on one commit were 28 s +# apart. +EVENT_SKEW = re.compile(r"^\s*EVENT_SKEW_SECONDS=(?P\d+)\s*$", re.MULTILINE) +DOCUMENTED_EVENT_SKEW_SECONDS = 10 + +# -- the events a commit has seen ------------------------------------------- +# +# Every workflow run an event starts is created at that event, and a re-run +# keeps the run's created_at. The gate's own run (RUN_ID) was created by +# THIS_EVENT unless a test says otherwise. Check suites, one per workflow run, +# are numbered by event: 1xx earlier, 2xx this one, 3xx a later one. Each +# workflow's run_number grows by one per run. +GATE_RUN_ID = 12345 +EARLIER_EVENT = "2026-10-05T20:17:51Z" +THIS_EVENT = "2026-10-06T14:12:00Z" +LATER_EVENT = "2026-10-06T14:40:00Z" +CORE_WORKFLOW, FULL_WORKFLOW, RPC_DOCS_WORKFLOW = 7001, 7002, 7003 +CORE_EARLIER, FULL_EARLIER = 101, 102 +CORE_THIS, FULL_THIS, RPC_DOCS_THIS = 201, 202, 203 +CORE_LATER = 301 +# check suite: (workflow id, run_number, created_at) of the run that owns it. +# These are the workflow runs on the commit unless a test says otherwise; the +# later event's run is on it only in the tests that ask for it. +SUITE_RUNS = { + CORE_EARLIER: (CORE_WORKFLOW, 452, EARLIER_EVENT), + FULL_EARLIER: (FULL_WORKFLOW, 445, EARLIER_EVENT), + CORE_THIS: (CORE_WORKFLOW, 457, THIS_EVENT), + FULL_THIS: (FULL_WORKFLOW, 450, THIS_EVENT), + RPC_DOCS_THIS: (RPC_DOCS_WORKFLOW, 30, THIS_EVENT), +} +LATER_RUNS = {CORE_LATER: (CORE_WORKFLOW, 460, LATER_EVENT)} + +SHA = "0" * 40 +GATE_RUN_PATH = re.escape(f"/actions/runs/{GATE_RUN_ID}") + "$" + + +def suite_lookup(suite: int) -> str: + """The route for the lookup of the workflow run that owns ``suite``.""" + return re.escape(f"/actions/runs?check_suite_id={suite}&exclude_pull_requests=true&per_page=100") + "$" + + +def commit_runs_lookup(workflow: int) -> str: + """The route for the listing of ``workflow``'s runs on the commit.""" + return re.escape(f"/actions/workflows/{workflow}/runs?head_sha={SHA}&exclude_pull_requests=true&per_page=100") + "$" -def check_run(name: str, status: str, conclusion: str | None = None, **stamps: str) -> dict: + +def check_run( + name: str, status: str, conclusion: str | None = None, *, suite: int | None = None, **stamps: str +) -> dict: run = {"name": name, "status": status, "conclusion": conclusion} + if suite is not None: + run["check_suite"] = {"id": suite} run.update(stamps) return run +def workflow_run( + run_id: int, suite: int, created_at: str | None, *, workflow: int = CORE_WORKFLOW, number: int | None = 1, **fields +) -> dict: + run = { + "id": run_id, + "workflow_id": workflow, + "run_number": number, + "check_suite_id": suite, + "head_sha": SHA, + "created_at": created_at, + "run_started_at": created_at, + "run_attempt": 1, + } + run.update(fields) + return run + + +def workflow_runs(*runs: dict) -> str: + return json.dumps({"total_count": len(runs), "workflow_runs": list(runs)}) + + +def suite_run(suite: int, owner: tuple[int, int, str]) -> dict: + workflow, number, created_at = owner + return workflow_run(50_000 + suite, suite, created_at, workflow=workflow, number=number) + + +def event_routes( + gate_run: dict | None = None, suites: dict[int, tuple[int, int, str]] | None = None +) -> dict[str, tuple]: + """Answers for the gate's own run, for the run behind each check suite, and + for each workflow's runs on the commit: exactly the runs in ``suites``.""" + routes: dict[str, tuple] = { + GATE_RUN_PATH: ok(json.dumps(gate_run or workflow_run(GATE_RUN_ID, 999, THIS_EVENT))), + } + by_workflow: dict[int, list[dict]] = {} + for suite, owner in (SUITE_RUNS if suites is None else suites).items(): + routes[suite_lookup(suite)] = ok(workflow_runs(suite_run(suite, owner))) + by_workflow.setdefault(owner[0], []).append(suite_run(suite, owner)) + for workflow, runs in by_workflow.items(): + routes[commit_runs_lookup(workflow)] = ok(workflow_runs(*runs)) + return routes + + +def iso(moment: datetime) -> str: + return moment.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def parse_iso(stamp: str) -> datetime: + return datetime.strptime(stamp, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) + + def page(*runs: dict) -> dict: return {"total_count": len(runs), "check_runs": list(runs)} @@ -109,65 +221,99 @@ def ok(payload: str) -> tuple[int, str]: class FakeGh: - """A ``gh`` on PATH that scripts one response per call. - - ``responses`` is consumed in order; once exhausted, ``default`` answers - every further call. Each entry is ``(exit_code, stdout)``, optionally with - a third element written to stderr the way ``gh`` reports an HTTP error. - ``delay`` makes every call take that many seconds, so a test can show the - wait deadline counts time spent in the API and not only in ``sleep``. + """A ``gh`` on PATH that scripts its answers. + + Check-runs queries are answered in order from ``responses``; once those are + exhausted, ``default`` answers every further one. ``routes`` answers any + other call whose arguments match its regular expression, with one reply or + a list of replies whose last one repeats. A call nothing answers fails as + an unscripted call, which ``run_gate`` refuses. Each reply is + ``(exit_code, stdout)``, optionally with a third element written to stderr + the way ``gh`` reports an HTTP error. ``delay`` makes every call take that + many seconds, so a test can show the wait deadline counts time spent in the + API and not only in ``sleep``. """ + STUB = """#!/usr/bin/env python3 +import json, re, sys, time +from pathlib import Path + +state = Path(__file__).resolve().parent +config = json.loads((state / "config.json").read_text(encoding="utf8")) +args = " ".join(sys.argv[1:]) +with open(state / "calls", "a", encoding="utf8") as log: + log.write(args + "\\n") +counts_path = state / "counts.json" +counts = json.loads(counts_path.read_text(encoding="utf8")) if counts_path.exists() else {} +key, replies, fallback = None, [], None +for index, (pattern, route_replies) in enumerate(config["routes"]): + if re.search(pattern, args): + key, replies, fallback = f"route-{index}", route_replies, route_replies[-1] + break +if key is None and "/check-runs" in args: + key, replies, fallback = "check-runs", config["responses"], config["default"] +served = counts.get(key, 0) if key else 0 +reply = replies[served] if served < len(replies) else fallback +if reply is None: + sys.stderr.write(f"fake gh: unscripted call: {args}\\n") + sys.exit(97) +counts[key] = served + 1 +counts_path.write_text(json.dumps(counts), encoding="utf8") +time.sleep(config["delay"]) +sys.stdout.write(reply[1]) +sys.stderr.write(reply[2] if len(reply) > 2 and reply[2] else "") +sys.exit(reply[0]) +""" + def __init__( self, directory: Path, *, responses: list[tuple] | None = None, default: tuple | None = None, + routes: dict[str, tuple | list[tuple]] | None = None, delay: int = 0, ) -> None: directory.mkdir(parents=True, exist_ok=True) self.directory = directory self.calls_path = directory / "calls" - replies = directory / "replies" - replies.mkdir() - for name, reply in [(str(i), r) for i, r in enumerate(responses or [], start=1)]: - self._write_reply(replies, name, reply) - if default is not None: - self._write_reply(replies, "default", default) + config = { + "responses": list(responses or []), + "default": default, + "routes": [ + [pattern, list(reply) if isinstance(reply, list) else [reply]] + for pattern, reply in (routes or {}).items() + ], + "delay": delay, + } + (directory / "config.json").write_text(json.dumps(config), encoding="utf8") script = directory / "gh" - script.write_text( - "#!/usr/bin/env bash\n" - f'printf "%s\\n" "$*" >> {self.calls_path!s}\n' - f'call="$(wc -l < {self.calls_path!s})"\n' - f'reply="{replies!s}/${{call}}"\n' - f'[[ -f "${{reply}}" ]] || reply="{replies!s}/default"\n' - 'if [[ ! -f "${reply}" ]]; then\n' - ' echo "fake gh: unscripted call: $*" >&2\n' - " exit 97\n" - "fi\n" - f"{f'sleep {delay}' if delay else ''}\n" - 'code="$(head -n 1 "${reply}")"\n' - '[[ -f "${reply}.err" ]] && cat "${reply}.err" >&2\n' - 'tail -n +2 "${reply}"\n' - 'exit "${code}"\n', - encoding="utf8", - ) + script.write_text(self.STUB, encoding="utf8") script.chmod(0o755) - @staticmethod - def _write_reply(replies: Path, name: str, reply: tuple) -> None: - code, payload = reply[0], reply[1] - (replies / name).write_text(f"{code}\n{payload}", encoding="utf8") - if len(reply) > 2 and reply[2]: - (replies / f"{name}.err").write_text(reply[2], encoding="utf8") - @property def calls(self) -> list[str]: if not self.calls_path.exists(): return [] return self.calls_path.read_text(encoding="utf8").splitlines() + def calls_matching(self, pattern: str) -> list[str]: + return [call for call in self.calls if re.search(pattern, call)] + + @property + def check_run_calls(self) -> list[str]: + return [call for call in self.calls if "/check-runs" in call] + + @property + def workflow_run_calls(self) -> list[str]: + return [call for call in self.calls if "/actions/runs" in call] + + +def load_yaml(path: Path) -> dict: + import yaml + from test_classifier_ci_contract import WorkflowLoader # noqa: E402 keeps "on" a string key + return yaml.load(path.read_text(encoding="utf8"), Loader=WorkflowLoader) + class MergeGatePollerTest(unittest.TestCase): maxDiff = None @@ -192,6 +338,9 @@ def setUp(self) -> None: match.group("name"): int(match.group("default")) for match in TUNABLE.finditer(self.script) } + skews = EVENT_SKEW.findall(self.script) + self.assertEqual(len(skews), 1, "expected the gate to declare EVENT_SKEW_SECONDS once") + self.event_skew = int(skews[0]) # -- helpers ----------------------------------------------------------- @@ -215,7 +364,8 @@ def gate_env(self, fake: FakeGh, profile: str, **overrides: str) -> dict[str, st "SOURCE_VALIDATION_REQUIRED": "true", "GH_TOKEN": "fake-token", "REPOSITORY": "example/repo", - "SHA": "0" * 40, + "SHA": SHA, + "RUN_ID": str(GATE_RUN_ID), } declared = set(self.step["env"]) self.assertFalse( @@ -302,27 +452,28 @@ def test_completed_success_on_every_required_check_passes(self) -> None: self.assertIn("Full Validation Gate: status=completed conclusion=success", completed.stdout) def test_failed_conclusion_fails_as_a_failed_check_not_as_pending(self) -> None: - for conclusion in ("failure", "cancelled", "timed_out", "action_required", "neutral", "skipped"): + for conclusion in ("failure", "timed_out", "action_required", "neutral", "skipped", "stale"): with self.subTest(conclusion=conclusion): fake = FakeGh( self.tmp(f"concluded-{conclusion}"), default=ok( body( page( - check_run("Core Checks Gate", "completed", conclusion, started_at="1"), - check_run("Full Validation Gate", "completed", "success", started_at="1"), + check_run("Core Checks Gate", "completed", conclusion, suite=CORE_THIS, started_at="1"), + check_run("Full Validation Gate", "completed", "success", suite=FULL_THIS, started_at="1"), ) ) ), + routes=event_routes(), ) completed = self.run_gate(fake) self.assertGateFailed( completed, f"{ERROR_CHECK_FAILED}Core Checks Gate concluded {conclusion}.", - forbidden=(ERROR_TIMED_OUT, ERROR_UNREADABLE), + forbidden=(ERROR_TIMED_OUT, ERROR_UNREADABLE, ERROR_UNCORRELATED), ) # A verdict is reached on the first poll, not after a wait. - self.assertEqual(len(fake.calls), 1, fake.calls) + self.assertEqual(len(fake.check_run_calls), 1, fake.calls) def test_a_check_still_running_is_pending_and_the_timeout_names_it(self) -> None: fake = FakeGh( @@ -452,8 +603,14 @@ def test_non_numeric_tunable_fails_closed_before_polling(self) -> None: # -- selection --------------------------------------------------------- def test_the_newest_check_run_wins_when_a_check_was_rerun(self) -> None: - older_failed = check_run("rpc-docs", "completed", "failure", started_at="2026-09-17T00:00:00Z") - newer_green = check_run("rpc-docs", "completed", "success", started_at="2026-09-18T00:00:00Z") + # A re-run adds a check run to the same workflow run, and so to the + # same check suite: its attempts rank by started_at alone. + older_failed = check_run( + "rpc-docs", "completed", "failure", suite=RPC_DOCS_THIS, started_at="2026-09-17T00:00:00Z" + ) + newer_green = check_run( + "rpc-docs", "completed", "success", suite=RPC_DOCS_THIS, started_at="2026-09-18T00:00:00Z" + ) for label, runs in { "newest last": [older_failed, newer_green], @@ -464,24 +621,812 @@ def test_the_newest_check_run_wins_when_a_check_was_rerun(self) -> None: completed = self.run_gate(fake, profile="rpc-docs") self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) self.assertIn("rpc-docs: status=completed conclusion=success", completed.stdout) + self.assertEqual(fake.workflow_run_calls, []) # ... and the reverse: a green first attempt must not mask a red re-run. - older_green = check_run("rpc-docs", "completed", "success", started_at="2026-09-17T00:00:00Z") - newer_failed = check_run("rpc-docs", "completed", "failure", started_at="2026-09-18T00:00:00Z") - fake = FakeGh(self.tmp("rerun-red"), default=ok(body(page(older_green, newer_failed)))) + older_green = check_run( + "rpc-docs", "completed", "success", suite=RPC_DOCS_THIS, started_at="2026-09-17T00:00:00Z" + ) + newer_failed = check_run( + "rpc-docs", "completed", "failure", suite=RPC_DOCS_THIS, started_at="2026-09-18T00:00:00Z" + ) + fake = FakeGh( + self.tmp("rerun-red"), default=ok(body(page(older_green, newer_failed))), routes=event_routes() + ) self.assertGateFailed( self.run_gate(fake, profile="rpc-docs"), f"{ERROR_CHECK_FAILED}rpc-docs concluded failure.", ) + # -- which event a failed check belongs to ------------------------------- + # + # A reopen, a label or a re-run starts this gate within seconds, while the + # workflows it waits for create their gate checks only at their last job. + # Until then the newest check of a name can be an earlier event's, whose + # failure says nothing about the run under way. + + def failed_core(self, suite: int, **stamps: str) -> dict: + stamps.setdefault("started_at", "2026-10-06T14:20:00Z") + stamps.setdefault("completed_at", "2026-10-06T14:26:19Z") + return check_run("Core Checks Gate", "completed", "failure", suite=suite, **stamps) + + def green_validation(self) -> dict: + return check_run( + "Full Validation Gate", "completed", "success", suite=FULL_THIS, + started_at="2026-10-06T14:30:00Z", completed_at="2026-10-06T14:31:00Z", + ) + + def test_a_failure_from_this_event_fails_at_once(self) -> None: + # Core Checks, Full Validation and this gate are separate workflows the + # same event starts, so a free runner can let Core Checks fail before + # this gate's step, or even its run, has started. + for conclusion in ("failure", "timed_out"): + with self.subTest(conclusion=conclusion): + fake = FakeGh( + self.tmp(f"this-event-{conclusion}"), + default=ok(body(page( + check_run( + "Core Checks Gate", "completed", conclusion, suite=CORE_THIS, + started_at="2026-10-06T14:12:01Z", completed_at="2026-10-06T14:12:02Z", + ), + check_run("Full Validation Gate", "in_progress", None, suite=FULL_THIS, started_at="2026-10-06T14:12:01Z"), + ))), + routes=event_routes(), + ) + completed = self.run_gate(fake) + self.assertGateFailed( + completed, + f"{ERROR_CHECK_FAILED}Core Checks Gate concluded {conclusion}.", + forbidden=(ERROR_TIMED_OUT, ERROR_UNREADABLE, ERROR_UNCORRELATED), + ) + self.assertEqual(len(fake.check_run_calls), 1, fake.calls) + self.assertIn(f"This gate's workflow run was created at {THIS_EVENT}", completed.stdout) + # The workflow run, not one of its attempts: an attempt has its own created_at. + self.assertEqual(len(fake.calls_matching(GATE_RUN_PATH)), 1, fake.calls) + self.assertEqual(len(fake.calls_matching(suite_lookup(CORE_THIS))), 1, fake.calls) + + def test_an_earlier_events_failure_is_waited_on_and_the_timeout_names_it(self) -> None: + # It ended after this gate's event, as a failure that outlives a newer + # push or label can: when a check ended says nothing about its event. + for conclusion in ("failure", "timed_out", "action_required"): + with self.subTest(conclusion=conclusion): + fake = FakeGh( + self.tmp(f"earlier-{conclusion}"), + default=ok(body(page( + check_run( + "Core Checks Gate", "completed", conclusion, suite=CORE_EARLIER, + started_at=EARLIER_EVENT, completed_at=LATER_EVENT, + ), + self.green_validation(), + ))), + routes=event_routes(), + ) + completed = self.run_gate(fake, GATE_WAIT_TIMEOUT_SECONDS="3") + self.assertGateFailed( + completed, + f"{ERROR_TIMED_OUT}Required checks did not complete within 3s and are still pending: " + f"Core Checks Gate (last {conclusion} from an earlier event)", + forbidden=(ERROR_CHECK_FAILED, ERROR_UNREADABLE, ERROR_UNCORRELATED), + ) + self.assertIn( + f"Core Checks Gate: concluded {conclusion} in a workflow run created at {EARLIER_EVENT}, " + f"by an earlier event than this gate's ({THIS_EVENT}); waiting for a newer run", + completed.stdout, + ) + self.assertGreater(len(fake.check_run_calls), 2, "an earlier event's failure must be polled again") + # Dated once: the workflow run behind a check suite cannot change. + self.assertEqual(len(fake.calls_matching(suite_lookup(CORE_EARLIER))), 1, fake.calls) + self.assertEqual(len(fake.calls_matching(GATE_RUN_PATH)), 1, fake.calls) + + def test_the_newer_run_replaces_an_earlier_events_failure(self) -> None: + stale = self.failed_core(CORE_EARLIER, started_at=EARLIER_EVENT, completed_at=EARLIER_EVENT) + fresh_running = check_run("Core Checks Gate", "in_progress", None, suite=CORE_THIS, started_at="2026-10-06T14:30:00Z") + fresh_green = check_run( + "Core Checks Gate", "completed", "success", suite=CORE_THIS, + started_at="2026-10-06T14:30:00Z", completed_at="2026-10-06T14:44:00Z", + ) + validation = self.green_validation() + fake = FakeGh( + self.tmp("earlier-then-green"), + responses=[ + ok(body(page(stale, validation))), + ok(body(page(stale, validation))), + ok(body(page(stale, fresh_running, validation))), + ok(body(page(stale, fresh_running, validation))), + ], + default=ok(body(page(stale, fresh_green, validation))), + routes=event_routes(), + ) + completed = self.run_gate(fake) + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + self.assertIn("waiting for a newer run", completed.stdout) + self.assertIn("Core Checks Gate: status=in_progress conclusion=pending", completed.stdout) + self.assertIn("Core Checks Gate: status=completed conclusion=success", completed.stdout) + + def test_a_rerun_of_only_this_gate_fails_at_once_on_this_events_failure(self) -> None: + # Re-running this gate alone starts no Core Checks or Full Validation, + # so no newer run is coming. The re-run keeps its workflow run and that + # run's created_at, and moves only run_attempt and run_started_at, so a + # failure that ended hours before the re-run is still this event's. + rerun = workflow_run(GATE_RUN_ID, 999, THIS_EVENT, run_attempt=4, run_started_at="2026-10-06T18:02:04Z") + fake = FakeGh( + self.tmp("gate-only-rerun"), + default=ok(body(page(self.failed_core(CORE_THIS), self.green_validation()))), + routes=event_routes(gate_run=rerun), + ) + completed = self.run_gate(fake) + self.assertGateFailed( + completed, + f"{ERROR_CHECK_FAILED}Core Checks Gate concluded failure.", + forbidden=(ERROR_TIMED_OUT, ERROR_UNCORRELATED), + ) + self.assertEqual(len(fake.check_run_calls), 1, fake.calls) + + def test_a_rerun_of_an_earlier_events_workflow_is_still_that_events(self) -> None: + # Re-running an earlier event's Core Checks keeps its workflow run's + # created_at, so its failure, however recent, is still waited on: the + # check this event started will be newer. + rerun = workflow_run( + 50_101, CORE_EARLIER, EARLIER_EVENT, number=452, run_attempt=2, run_started_at="2026-10-06T15:31:19Z" + ) + routes = event_routes() + routes[suite_lookup(CORE_EARLIER)] = ok(workflow_runs(rerun)) + fake = FakeGh( + self.tmp("earlier-workflow-rerun"), + default=ok(body(page( + self.failed_core(CORE_EARLIER, started_at="2026-10-06T15:31:30Z", completed_at="2026-10-06T15:45:16Z"), + self.green_validation(), + ))), + routes=routes, + ) + completed = self.run_gate(fake, GATE_WAIT_TIMEOUT_SECONDS="2") + self.assertGateFailed( + completed, + f"{ERROR_TIMED_OUT}Required checks did not complete within 2s and are still pending: " + "Core Checks Gate (last failure from an earlier event)", + forbidden=(ERROR_CHECK_FAILED, ERROR_UNCORRELATED), + ) + + def test_a_label_event_fails_at_once_on_its_own_core_checks_failure(self) -> None: + # A label on a commit whose Core Checks had passed starts this gate, + # Core Checks and Full Validation again. This event's Core Checks + # failed before this gate first polled: it is the newest Core Checks + # Gate and the verdict, while Full Validation is still running. + earlier_green = check_run( + "Core Checks Gate", "completed", "success", suite=CORE_EARLIER, + started_at=EARLIER_EVENT, completed_at=EARLIER_EVENT, + ) + this_failed = self.failed_core(CORE_THIS, started_at="2026-10-06T14:12:40Z", completed_at="2026-10-06T14:13:10Z") + validation_running = check_run("Full Validation Gate", "in_progress", None, suite=FULL_THIS, started_at="2026-10-06T14:12:05Z") + fake = FakeGh( + self.tmp("label-this-event"), + default=ok(body(page(earlier_green, this_failed, validation_running))), + routes=event_routes(), + ) + completed = self.run_gate(fake) + self.assertGateFailed( + completed, + f"{ERROR_CHECK_FAILED}Core Checks Gate concluded failure.", + forbidden=(ERROR_TIMED_OUT, ERROR_UNCORRELATED), + ) + self.assertEqual(len(fake.check_run_calls), 1, fake.calls) + + def test_a_later_events_failure_is_current(self) -> None: + # Nothing this gate could wait for is newer than a later event's run. + fake = FakeGh( + self.tmp("later-event"), + default=ok(body(page(self.failed_core(CORE_LATER, started_at=LATER_EVENT), self.green_validation()))), + routes=event_routes(suites={**SUITE_RUNS, **LATER_RUNS}), + ) + completed = self.run_gate(fake) + self.assertGateFailed( + completed, + f"{ERROR_CHECK_FAILED}Core Checks Gate concluded failure.", + forbidden=(ERROR_TIMED_OUT, ERROR_UNCORRELATED), + ) + self.assertEqual(len(fake.check_run_calls), 1, fake.calls) + + # -- which of a check's runs decides ------------------------------------- + # + # A check has a run per event that ran CI on the commit, and a re-run adds + # another to the same workflow run. The newest workflow run's decides, at + # its latest attempt, so a manual re-run of an earlier event's workflow, + # which starts later, never outranks this event's result. + + def earlier_rerun(self, status: str, conclusion: str | None) -> dict: + if status == "queued": + return check_run("Core Checks Gate", status, conclusion, suite=CORE_EARLIER, created_at="2026-10-06T15:31:30Z") + stamps = {"started_at": "2026-10-06T15:31:30Z"} + if status == "completed": + stamps["completed_at"] = "2026-10-06T15:45:16Z" + return check_run("Core Checks Gate", status, conclusion, suite=CORE_EARLIER, **stamps) + + def this_events_core(self, status: str, conclusion: str | None) -> dict: + stamps = {"started_at": "2026-10-06T14:30:00Z"} + if status == "completed": + stamps["completed_at"] = "2026-10-06T14:44:00Z" + return check_run("Core Checks Gate", status, conclusion, suite=CORE_THIS, **stamps) + + def test_an_earlier_events_rerun_does_not_hide_this_events_success(self) -> None: + for status, conclusion in ( + ("completed", "failure"), + ("completed", "timed_out"), + ("completed", "cancelled"), + ("in_progress", None), + ("queued", None), + ): + with self.subTest(rerun=conclusion or status): + fake = FakeGh( + self.tmp(f"earlier-rerun-{conclusion or status}"), + default=ok(body(page( + self.this_events_core("completed", "success"), + self.earlier_rerun(status, conclusion), + self.green_validation(), + ))), + routes=event_routes(), + ) + completed = self.run_gate(fake) + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + self.assertIn( + "Core Checks Gate: 2 check runs from 2 workflow runs; the newest workflow run's, " + f"created at {THIS_EVENT}, decides", + completed.stdout, + ) + self.assertIn("Core Checks Gate: status=completed conclusion=success", completed.stdout) + # A verdict on the first poll, from one lookup per check suite. + self.assertEqual(len(fake.check_run_calls), 2, fake.calls) + self.assertEqual(len(fake.calls_matching(suite_lookup(CORE_EARLIER))), 1, fake.calls) + self.assertEqual(len(fake.calls_matching(suite_lookup(CORE_THIS))), 1, fake.calls) + self.assertEqual(fake.calls_matching(GATE_RUN_PATH), [], "a success is never dated") + + def test_an_earlier_events_rerun_does_not_hide_this_events_failure(self) -> None: + fake = FakeGh( + self.tmp("earlier-rerun-green"), + default=ok(body(page( + self.this_events_core("completed", "failure"), + self.earlier_rerun("completed", "success"), + self.green_validation(), + ))), + routes=event_routes(), + ) + completed = self.run_gate(fake) + self.assertGateFailed( + completed, + f"{ERROR_CHECK_FAILED}Core Checks Gate concluded failure.", + forbidden=(ERROR_TIMED_OUT, ERROR_UNCORRELATED), + ) + self.assertEqual(len(fake.check_run_calls), 1, fake.calls) + + def test_an_earlier_events_rerun_failure_waits_for_this_events_result(self) -> None: + # The re-run failed before this event's check existed, and stays the + # newest by started_at once it does: only this event's result decides. + rerun_failed = self.earlier_rerun("completed", "failure") + validation = self.green_validation() + fake = FakeGh( + self.tmp("earlier-rerun-then-green"), + responses=[ + ok(body(page(rerun_failed, validation))), + ok(body(page(rerun_failed, validation))), + ok(body(page(rerun_failed, self.this_events_core("in_progress", None), validation))), + ok(body(page(rerun_failed, self.this_events_core("in_progress", None), validation))), + ], + default=ok(body(page(rerun_failed, self.this_events_core("completed", "success"), validation))), + routes=event_routes(), + ) + completed = self.run_gate(fake) + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + self.assertIn("by an earlier event than this gate's", completed.stdout) + self.assertIn("Core Checks Gate: status=in_progress conclusion=pending", completed.stdout) + self.assertGreater(len(fake.check_run_calls), 4, fake.calls) + # Cached per check suite across polls: a suite's workflow run cannot change. + self.assertEqual(len(fake.calls_matching(suite_lookup(CORE_EARLIER))), 1, fake.calls) + self.assertEqual(len(fake.calls_matching(suite_lookup(CORE_THIS))), 1, fake.calls) + self.assertEqual(len(fake.calls_matching(GATE_RUN_PATH)), 1, fake.calls) + + def test_check_runs_that_all_succeeded_are_never_ranked(self) -> None: + fake = FakeGh( + self.tmp("all-runs-green"), + default=ok(body(page( + self.this_events_core("completed", "success"), + self.earlier_rerun("completed", "success"), + self.green_validation(), + ))), + ) + completed = self.run_gate(fake) + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + self.assertEqual(fake.workflow_run_calls, []) + + def test_runs_created_in_the_same_second_rank_by_run_number(self) -> None: + # Two labels applied at once start two runs of a workflow in the same + # second; the later one has the higher run_number, whichever check + # started last. + first, second = 401, 402 + suites = { + first: (CORE_WORKFLOW, 456, THIS_EVENT), + second: (CORE_WORKFLOW, 457, THIS_EVENT), + FULL_THIS: SUITE_RUNS[FULL_THIS], + } + for label, (newer, older) in { + "the newer run failed": ("failure", "success"), + "the newer run passed": ("success", "failure"), + }.items(): + with self.subTest(case=label): + fake = FakeGh( + self.tmp(f"same-second-{newer}"), + default=ok(body(page( + check_run("Core Checks Gate", "completed", newer, suite=second, started_at="2026-10-06T14:20:00Z"), + check_run("Core Checks Gate", "completed", older, suite=first, started_at="2026-10-06T15:31:30Z"), + self.green_validation(), + ))), + routes=event_routes(suites=suites), + ) + completed = self.run_gate(fake) + if newer == "failure": + self.assertGateFailed( + completed, + f"{ERROR_CHECK_FAILED}Core Checks Gate concluded failure.", + forbidden=(ERROR_TIMED_OUT, ERROR_UNCORRELATED), + ) + else: + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + self.assertEqual(len(fake.check_run_calls), 1 if newer == "failure" else 2, fake.calls) + + def test_check_runs_that_cannot_be_ranked_fail_closed(self) -> None: + # Which of several workflow runs is newest must be read, never guessed. + server_error = (1, "", "gh: Server Error (HTTP 502)\n") + earlier_failed = self.failed_core(CORE_EARLIER, started_at=EARLIER_EVENT, completed_at=EARLIER_EVENT) + this_running = self.this_events_core("in_progress", None) + unranked = ( + f"{ERROR_UNCORRELATED}Core Checks Gate on {'0' * 40} has check runs from 2 workflow runs, but the " + "workflow runs that tell which event is newest could not be read; failing closed with no verdict " + "for: Core Checks Gate Full Validation Gate" + ) + no_number = workflow_run(50_201, CORE_THIS, THIS_EVENT, number=None) + cases = { + "a check suite's run cannot be read": ( + {suite_lookup(CORE_THIS): server_error}, this_running, unranked, suite_lookup(CORE_THIS), + ), + "a check suite's run has no run_number": ( + {suite_lookup(CORE_THIS): ok(workflow_runs(no_number))}, this_running, unranked, suite_lookup(CORE_THIS), + ), + "a check run names no check suite": ( + {}, + check_run("Core Checks Gate", "in_progress", None, started_at="2026-10-06T14:30:00Z"), + f"{ERROR_UNCORRELATED}Core Checks Gate on {'0' * 40} has 2 check runs, but not every one names " + "the workflow run that tells which event is newest; failing closed with no verdict for: " + "Core Checks Gate Full Validation Gate", + None, + ), + } + for index, (label, (overrides, other, expected, gave_up)) in enumerate(cases.items()): + with self.subTest(case=label): + routes = event_routes() + routes.update(overrides) + fake = FakeGh( + self.tmp(f"unranked-{index}"), + default=ok(body(page(earlier_failed, other, self.green_validation()))), + routes=routes, + ) + completed = self.run_gate(fake, GATE_QUERY_MAX_ATTEMPTS="2") + self.assertGateFailed( + completed, expected, forbidden=(ERROR_CHECK_FAILED, ERROR_TIMED_OUT, ERROR_UNREADABLE) + ) + self.assertEqual(len(fake.check_run_calls), 1, fake.calls) + if gave_up is not None: + self.assertEqual(len(fake.calls_matching(gave_up)), 2, fake.calls) + + def test_the_event_skew_allowance_is_a_documented_constant(self) -> None: + self.assertEqual(self.event_skew, DOCUMENTED_EVENT_SKEW_SECONDS) + self.assertNotIn("EVENT_SKEW_SECONDS:-", self.script, "the allowance must not be overridable") + + def test_the_event_skew_boundary(self) -> None: + # Workflow runs one event starts are created within the allowance of + # each other; a run created further before this gate's is an earlier + # event's. The documented value, not the script's, sets the boundary. + gate = parse_iso(THIS_EVENT) + skew = timedelta(seconds=DOCUMENTED_EVENT_SKEW_SECONDS) + one = timedelta(seconds=1) + cases = { + "the same second": (gate, "current"), + "the allowance before": (gate - skew, "current"), + "one second more": (gate - skew - one, "earlier"), + "after this gate's run": (gate + skew + one, "current"), + } + for label, (created, expected) in cases.items(): + with self.subTest(case=label, created=iso(created)): + fake = FakeGh( + self.tmp(f"skew-{label}"), + default=ok(body(page(self.failed_core(777), self.green_validation()))), + routes=event_routes(suites={777: (CORE_WORKFLOW, 1, iso(created))}), + ) + completed = self.run_gate(fake, GATE_WAIT_TIMEOUT_SECONDS="2") + if expected == "current": + self.assertGateFailed( + completed, + f"{ERROR_CHECK_FAILED}Core Checks Gate concluded failure.", + forbidden=(ERROR_TIMED_OUT, ERROR_UNCORRELATED), + ) + self.assertEqual(len(fake.check_run_calls), 1, fake.calls) + else: + self.assertGateFailed( + completed, + f"{ERROR_TIMED_OUT}Required checks did not complete within 2s and are still pending: " + "Core Checks Gate (last failure from an earlier event)", + forbidden=(ERROR_CHECK_FAILED, ERROR_UNCORRELATED), + ) + + # -- two events on one commit, close together ---------------------------- + # + # No marker names a workflow run's event, so events are told apart by when + # their runs were created and, closer together than the allowance, by a + # newer run of the same workflow on the commit. A newer event cancels the + # earlier event's run, whose always() gate job then concludes failure, + # after the newer run exists. + + def earlier_events_failure(self, suite: int) -> dict: + # Concluded after this event, as a cancelled run's gate job does. + return check_run( + "Core Checks Gate", "completed", "failure", suite=suite, + started_at="2026-10-06T14:12:40Z", completed_at="2026-10-06T14:12:45Z", + ) + + def test_an_earlier_event_seconds_before_waits_for_this_events_run(self) -> None: + # 0 s is two labels applied at once. Inside the allowance only the + # newer run of the workflow tells the earlier event's failure apart. + gate = parse_iso(THIS_EVENT) + first = 401 + for seconds in (0, 2, DOCUMENTED_EVENT_SKEW_SECONDS): + with self.subTest(seconds_apart=seconds): + created = iso(gate - timedelta(seconds=seconds)) + suites = { + first: (CORE_WORKFLOW, 456, created), + CORE_THIS: SUITE_RUNS[CORE_THIS], + FULL_THIS: SUITE_RUNS[FULL_THIS], + } + stale = self.earlier_events_failure(first) + validation = self.green_validation() + running = self.this_events_core("in_progress", None) + fake = FakeGh( + self.tmp(f"apart-{seconds}"), + responses=[ok(body(page(stale, validation)))] * 4 + [ok(body(page(stale, running, validation)))] * 2, + default=ok(body(page(stale, self.this_events_core("completed", "success"), validation))), + routes=event_routes(suites=suites), + ) + completed = self.run_gate(fake) + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + self.assertIn( + f"Core Checks Gate: concluded failure in run #456 of its workflow, created at {created}, " + f"but run #457, created at {THIS_EVENT}, has since started on {SHA}; waiting for its result", + completed.stdout, + ) + self.assertIn("Core Checks Gate: status=in_progress conclusion=pending", completed.stdout) + # Listed once: a newer run, once seen, stays on the commit. + self.assertEqual(len(fake.calls_matching(commit_runs_lookup(CORE_WORKFLOW))), 1, fake.calls) + + def test_a_superseded_failure_is_named_by_the_timeout(self) -> None: + gate = parse_iso(THIS_EVENT) + suites = { + 401: (CORE_WORKFLOW, 456, iso(gate - timedelta(seconds=2))), + CORE_THIS: SUITE_RUNS[CORE_THIS], + FULL_THIS: SUITE_RUNS[FULL_THIS], + } + fake = FakeGh( + self.tmp("superseded-timeout"), + default=ok(body(page(self.earlier_events_failure(401), self.green_validation()))), + routes=event_routes(suites=suites), + ) + completed = self.run_gate(fake, GATE_WAIT_TIMEOUT_SECONDS="3") + self.assertGateFailed( + completed, + f"{ERROR_TIMED_OUT}Required checks did not complete within 3s and are still pending: " + "Core Checks Gate (last failure superseded by a newer run)", + forbidden=(ERROR_CHECK_FAILED, ERROR_UNCORRELATED, ERROR_UNREADABLE), + ) + + def test_an_earlier_event_30_seconds_before_is_told_apart_by_time(self) -> None: + # Outside the allowance the failure is an earlier event's whether or + # not this event's run of the workflow is listed yet, and no listing + # is read for it. + gate = parse_iso(THIS_EVENT) + first = 401 + for label, listed in {"this event's run listed": True, "not listed yet": False}.items(): + with self.subTest(case=label): + suites = { + first: (CORE_WORKFLOW, 456, iso(gate - timedelta(seconds=30))), + FULL_THIS: SUITE_RUNS[FULL_THIS], + } + if listed: + suites[CORE_THIS] = SUITE_RUNS[CORE_THIS] + fake = FakeGh( + self.tmp(f"apart-30-{listed}"), + default=ok(body(page(self.earlier_events_failure(first), self.green_validation()))), + routes=event_routes(suites=suites), + ) + completed = self.run_gate(fake, GATE_WAIT_TIMEOUT_SECONDS="2") + self.assertGateFailed( + completed, + f"{ERROR_TIMED_OUT}Required checks did not complete within 2s and are still pending: " + "Core Checks Gate (last failure from an earlier event)", + forbidden=(ERROR_CHECK_FAILED, ERROR_UNCORRELATED), + ) + self.assertEqual(fake.calls_matching(commit_runs_lookup(CORE_WORKFLOW)), [], fake.calls) + + def test_a_current_failure_with_no_newer_run_fails_at_once(self) -> None: + # One event's runs are created seconds apart, this gate's up to 3 s + # after another's: such a failure is this event's, and with no newer + # run of its workflow on the commit it fails at once, also when only + # this gate is re-run, hours later. + gate = parse_iso(THIS_EVENT) + rerun = workflow_run(GATE_RUN_ID, 999, THIS_EVENT, run_attempt=3, run_started_at="2026-10-06T19:40:00Z") + for label, gate_run in {"first attempt": None, "gate-only re-run": rerun}.items(): + for seconds in (3, DOCUMENTED_EVENT_SKEW_SECONDS): + with self.subTest(gate=label, seconds_before=seconds): + suites = { + CORE_EARLIER: SUITE_RUNS[CORE_EARLIER], + 401: (CORE_WORKFLOW, 457, iso(gate - timedelta(seconds=seconds))), + FULL_THIS: SUITE_RUNS[FULL_THIS], + } + fake = FakeGh( + self.tmp(f"current-{label}-{seconds}"), + default=ok(body(page(self.failed_core(401), self.green_validation()))), + routes=event_routes(gate_run=gate_run, suites=suites), + ) + completed = self.run_gate(fake) + self.assertGateFailed( + completed, + f"{ERROR_CHECK_FAILED}Core Checks Gate concluded failure.", + forbidden=(ERROR_TIMED_OUT, ERROR_UNCORRELATED), + ) + self.assertEqual(len(fake.check_run_calls), 1, fake.calls) + self.assertEqual(len(fake.calls_matching(commit_runs_lookup(CORE_WORKFLOW))), 1, fake.calls) + + def test_runs_of_another_workflow_or_commit_never_supersede(self) -> None: + # Even were the listing's filters ignored. + routes = event_routes() + routes[commit_runs_lookup(CORE_WORKFLOW)] = ok(workflow_runs( + suite_run(CORE_THIS, SUITE_RUNS[CORE_THIS]), + workflow_run(60_001, 901, LATER_EVENT, workflow=FULL_WORKFLOW, number=999), + dict(workflow_run(60_002, 902, LATER_EVENT, number=999), head_sha="1" * 40), + )) + fake = FakeGh( + self.tmp("other-runs"), + default=ok(body(page(self.failed_core(CORE_THIS), self.green_validation()))), + routes=routes, + ) + completed = self.run_gate(fake) + self.assertGateFailed( + completed, + f"{ERROR_CHECK_FAILED}Core Checks Gate concluded failure.", + forbidden=(ERROR_TIMED_OUT, ERROR_UNCORRELATED), + ) + + def test_a_later_events_run_decides_over_this_events_failure(self) -> None: + # An event after this gate's has already started a newer run of the + # workflow: this event's failure is waited on, and that run decides. + suites = {**SUITE_RUNS, **LATER_RUNS} + validation = self.green_validation() + this_failed = self.failed_core(CORE_THIS) + for conclusion in ("success", "failure"): + with self.subTest(later=conclusion): + later = check_run( + "Core Checks Gate", "completed", conclusion, suite=CORE_LATER, + started_at="2026-10-06T14:50:00Z", completed_at="2026-10-06T15:04:00Z", + ) + fake = FakeGh( + self.tmp(f"later-decides-{conclusion}"), + responses=[ok(body(page(this_failed, validation)))] * 2, + default=ok(body(page(this_failed, later, validation))), + routes=event_routes(suites=suites), + ) + completed = self.run_gate(fake) + self.assertIn("but run #460, created at " + LATER_EVENT, completed.stdout) + if conclusion == "success": + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + else: + self.assertGateFailed( + completed, + f"{ERROR_CHECK_FAILED}Core Checks Gate concluded failure.", + forbidden=(ERROR_TIMED_OUT, ERROR_UNCORRELATED), + ) + + def test_a_success_from_an_earlier_event_still_counts(self) -> None: + # A success on this commit is a verdict whichever event produced it, + # so it is never dated: no workflow run is read for it. + fake = FakeGh( + self.tmp("earlier-success"), + default=ok(body(page( + check_run( + "Core Checks Gate", "completed", "success", suite=CORE_EARLIER, + started_at=EARLIER_EVENT, completed_at=EARLIER_EVENT, + ), + self.green_validation(), + ))), + ) + completed = self.run_gate(fake) + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + self.assertEqual(fake.workflow_run_calls, []) + + def test_a_cancelled_check_is_never_a_verdict(self) -> None: + # A newer push, reopen or label cancels the running Full Validation, + # and that cancellation can complete after this gate run started. + # Whichever event's it is, it is waited on and never dated. + core = check_run( + "Core Checks Gate", "completed", "success", suite=CORE_THIS, + started_at="2026-10-06T14:12:05Z", completed_at="2026-10-06T14:26:00Z", + ) + for label, suite in {"earlier": FULL_EARLIER, "this": FULL_THIS}.items(): + with self.subTest(event=label): + cancelled = check_run( + "Full Validation Gate", "completed", "cancelled", suite=suite, + started_at=EARLIER_EVENT, completed_at=LATER_EVENT, + ) + fake = FakeGh(self.tmp(f"cancelled-{label}"), default=ok(body(page(core, cancelled)))) + completed = self.run_gate(fake, GATE_WAIT_TIMEOUT_SECONDS="3") + self.assertGateFailed( + completed, + f"{ERROR_TIMED_OUT}Required checks did not complete within 3s and are still pending: " + "Full Validation Gate (cancelled)", + forbidden=(ERROR_CHECK_FAILED, ERROR_UNREADABLE, ERROR_UNCORRELATED), + ) + self.assertEqual(fake.workflow_run_calls, []) + + cancelled = check_run( + "Full Validation Gate", "completed", "cancelled", suite=FULL_THIS, + started_at="2026-10-06T14:12:05Z", completed_at="2026-10-06T14:13:00Z", + ) + newer_green = check_run( + "Full Validation Gate", "completed", "success", suite=FULL_THIS, + started_at="2026-10-06T14:20:00Z", completed_at="2026-10-06T15:00:00Z", + ) + fake = FakeGh( + self.tmp("cancelled-then-green"), + responses=[ok(body(page(core, cancelled))), ok(body(page(core, cancelled)))], + default=ok(body(page(core, cancelled, newer_green))), + ) + completed = self.run_gate(fake) + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + + def test_a_failure_that_cannot_be_dated_fails_closed_with_a_distinct_error(self) -> None: + # Not knowing a failure's event is never success, never a wait for a + # run that may not come, and never reported as the check's own verdict. + gate_run = ok(json.dumps(workflow_run(GATE_RUN_ID, 999, THIS_EVENT))) + core_run = workflow_run(60_201, CORE_THIS, THIS_EVENT) + server_error = (1, "", "gh: Server Error (HTTP 502)\n") + forbidden = (1, "", "gh: Resource not accessible by integration (HTTP 403)\n") + failed = self.failed_core(CORE_THIS) + suite_route = suite_lookup(CORE_THIS) + own_run = suite_run(CORE_THIS, SUITE_RUNS[CORE_THIS]) + dated = {GATE_RUN_PATH: gate_run, suite_route: ok(workflow_runs(own_run))} + listing = commit_runs_lookup(CORE_WORKFLOW) + # case: (routes, check run, env overrides, the route read until it gave up) + cases = { + "the check's run cannot be read": ({GATE_RUN_PATH: gate_run, suite_route: server_error}, failed, {}, suite_route), + "the token cannot read workflow runs": ({GATE_RUN_PATH: gate_run, suite_route: forbidden}, failed, {}, suite_route), + "no run owns the check suite": ({GATE_RUN_PATH: gate_run, suite_route: ok(workflow_runs())}, failed, {}, suite_route), + "only another suite's run": ( + {GATE_RUN_PATH: gate_run, suite_route: ok(workflow_runs(workflow_run(1, 555, THIS_EVENT)))}, + failed, {}, suite_route, + ), + "two runs claim the check suite": ( + {GATE_RUN_PATH: gate_run, suite_route: ok(workflow_runs(core_run, workflow_run(2, CORE_THIS, EARLIER_EVENT)))}, + failed, {}, suite_route, + ), + "the check's run has no created_at": ( + {GATE_RUN_PATH: gate_run, suite_route: ok(workflow_runs(workflow_run(1, CORE_THIS, None)))}, + failed, {}, suite_route, + ), + "the check's run has no run_number": ( + {GATE_RUN_PATH: gate_run, suite_route: ok(workflow_runs(workflow_run(1, CORE_THIS, THIS_EVENT, number=None)))}, + failed, {}, suite_route, + ), + "the check's run names no workflow": ( + {GATE_RUN_PATH: gate_run, suite_route: ok(workflow_runs(workflow_run(1, CORE_THIS, THIS_EVENT, workflow=None)))}, + failed, {}, suite_route, + ), + "the check's run has fractional seconds": ( + {GATE_RUN_PATH: gate_run, suite_route: ok(workflow_runs(workflow_run(1, CORE_THIS, "2026-10-06T14:12:00.5Z")))}, + failed, {}, suite_route, + ), + "the response is not json": ({GATE_RUN_PATH: gate_run, suite_route: ok("502")}, failed, {}, suite_route), + "this gate's run cannot be read": ({GATE_RUN_PATH: server_error}, failed, {}, GATE_RUN_PATH), + "this gate's run is another run": ( + {GATE_RUN_PATH: ok(json.dumps(workflow_run(GATE_RUN_ID + 1, 999, THIS_EVENT)))}, failed, {}, GATE_RUN_PATH, + ), + "this gate's run has no created_at": ( + {GATE_RUN_PATH: ok(json.dumps(workflow_run(GATE_RUN_ID, 999, "")))}, failed, {}, GATE_RUN_PATH, + ), + "this gate's run id is missing": ({}, failed, {"RUN_ID": ""}, None), + # Whether a newer run of the check's workflow is on the commit. + "its workflow's runs cannot be listed": ({**dated, listing: server_error}, failed, {}, listing), + "the token cannot list its workflow's runs": ({**dated, listing: forbidden}, failed, {}, listing), + "the listing is not json": ({**dated, listing: ok("502")}, failed, {}, listing), + "the listing is empty": ({**dated, listing: ok(workflow_runs())}, failed, {}, listing), + "the listing lacks the check's own run": ( + {**dated, listing: ok(workflow_runs(suite_run(CORE_EARLIER, SUITE_RUNS[CORE_EARLIER])))}, + failed, {}, listing, + ), + "the listing has the run under another workflow": ( + {**dated, listing: ok(workflow_runs(dict(own_run, workflow_id=FULL_WORKFLOW)))}, failed, {}, listing, + ), + "the listing has the run under another commit": ( + {**dated, listing: ok(workflow_runs(dict(own_run, head_sha="1" * 40)))}, failed, {}, listing, + ), + "a listed run has no run_number": ( + {**dated, listing: ok(workflow_runs(own_run, workflow_run(1, 999, LATER_EVENT, number=None)))}, + failed, {}, listing, + ), + "the check run names no check suite": ( + {GATE_RUN_PATH: gate_run}, + check_run("Core Checks Gate", "completed", "failure", started_at="2026-10-06T14:20:00Z"), + {}, None, + ), + } + for index, (label, (routes, core, env, gave_up)) in enumerate(cases.items()): + with self.subTest(case=label): + fake = FakeGh( + self.tmp(f"undatable-{index}"), + default=ok(body(page(core, self.green_validation()))), + routes=routes, + ) + completed = self.run_gate(fake, GATE_QUERY_MAX_ATTEMPTS="2", **env) + self.assertGateFailed( + completed, + f"{ERROR_UNCORRELATED}Core Checks Gate on {'0' * 40} concluded failure, but the workflow " + "runs that tell whether it belongs to this event could not be read; failing closed with " + "no verdict for: Core Checks Gate Full Validation Gate", + forbidden=(ERROR_CHECK_FAILED, ERROR_TIMED_OUT, ERROR_UNREADABLE), + ) + self.assertEqual(len(fake.check_run_calls), 1, fake.calls) + if gave_up is not None: + # Bounded by the same retries as the check-runs query. + self.assertEqual(len(fake.calls_matching(gave_up)), 2, fake.calls) + self.assertIn(f"{WARNING_RETRYING}Core Checks Gate: workflow-run", completed.stdout) + + def test_a_transient_lookup_failure_is_retried(self) -> None: + routes = event_routes() + routes[suite_lookup(CORE_THIS)] = [ + (1, "", "gh: Server Error (HTTP 502)\n"), + ok(workflow_runs(suite_run(CORE_THIS, SUITE_RUNS[CORE_THIS]))), + ] + fake = FakeGh( + self.tmp("lookup-transient"), + default=ok(body(page(self.failed_core(CORE_THIS), self.green_validation()))), + routes=routes, + ) + completed = self.run_gate(fake) + self.assertGateFailed( + completed, + f"{ERROR_CHECK_FAILED}Core Checks Gate concluded failure.", + forbidden=(ERROR_UNCORRELATED, ERROR_TIMED_OUT), + ) + self.assertIn(f"{WARNING_RETRYING}Core Checks Gate: workflow-run query failed (exit 1)", completed.stdout) + self.assertEqual(len(fake.calls_matching(suite_lookup(CORE_THIS))), 2, fake.calls) + + def test_every_gate_event_can_start_the_checks_it_waits_for(self) -> None: + # The gate waits for this event's run instead of trusting an earlier + # event's failure, so each event that starts the gate must also start + # the gated workflows. + gate_types = set(self.workflow["on"]["pull_request"]["types"]) + for name in ("core-checks.yml", "ci.yml"): + other = load_yaml(REPO_ROOT / ".github" / "workflows" / name) + types = set(other["on"]["pull_request"].get("types", ["opened", "synchronize", "reopened"])) + self.assertEqual(gate_types - types, set(), f"{name} misses events that start the Required Merge Gate") + + def test_the_workflow_token_can_read_workflow_runs(self) -> None: + # Getting or listing workflow runs needs Actions: read; without it every + # failure would fail closed as undatable. + permissions = self.workflow["permissions"] + self.assertEqual(permissions.get("actions"), "read", permissions) + self.assertEqual(permissions.get("checks"), "read", permissions) + def test_a_queued_rerun_falls_back_to_created_at_and_is_pending(self) -> None: # A re-run that has not started yet has no started_at at all; ordering # by created_at keeps it, so an old green run cannot pass the gate. old_green = check_run( - "rpc-docs", "completed", "success", + "rpc-docs", "completed", "success", suite=RPC_DOCS_THIS, started_at="2026-09-17T00:00:00Z", created_at="2026-09-17T00:00:00Z", ) - queued_rerun = check_run("rpc-docs", "queued", None, created_at="2026-09-18T00:00:00Z") + queued_rerun = check_run("rpc-docs", "queued", None, suite=RPC_DOCS_THIS, created_at="2026-09-18T00:00:00Z") fake = FakeGh(self.tmp("queued-rerun"), default=ok(body(page(old_green, queued_rerun)))) completed = self.run_gate(fake, profile="rpc-docs", GATE_WAIT_TIMEOUT_SECONDS="3") self.assertIn("rpc-docs: status=queued conclusion=pending", completed.stdout) From ad1d6944dd55d76ca98c75ba9ae6015cde2bd33a Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Wed, 21 Oct 2026 17:03:00 -0400 Subject: [PATCH 07/24] test, ci: foundations for hybrid v2 transport (Python ML-KEM, qbit releases, v1.0.0 interop, PQ CI, canary tools) (#224) * test: add an independent ML-KEM-1024 Python reference Add test_framework/crypto/mlkem.py, a pure-Python FIPS 203 ML-KEM-1024 written from the standard and independent of the C library: KeyGen from a 64-byte d || z seed, Encaps from a 32-byte m, Decaps with implicit rejection, and the section 7.2 (length and modulus) and 7.3 (length and hash) input checks. Hashing uses hashlib's SHA3 and SHAKE only. TestMLKEM1024, registered in feature_framework_unit_tests.py, checks: - NIST ACVP-Server ML-KEM-1024 keyGen and encapsulation vectors (exact ek, dk, ct and shared secret); - ACVP decapsulation vectors, including modified ciphertexts whose expected secret is the implicit-rejection value J(z || ct); - ACVP encapsulationKeyCheck vectors: valid keys pass the section 7.2 check and encapsulate, the others are rejected; - C2SP CCTV modulus vectors (q and 4095 at the first and last coefficient of every polynomial, plus a sample of the full run), each rejected, and accepted once the bad coefficient is set to 3328; - round trips, implicit rejection on corrupted ciphertexts, and input size and hash checks. mlkem1024_test_vectors.txt holds the subset with its sources and pinned commits in a header comment. The new tests take about 0.07 s; the whole feature_framework_unit_tests.py run about 4 s. Part of #184 (part 2, #186, work item B2). Co-Authored-By: Claude Opus 5.5 * test: identify qbit releases independently of Core versions Let functional tests download and run qbit release binaries the way they run Bitcoin Core releases, starting with v1.0.0. get_previous_releases.py resolves each tag through resolve_download(). qbit tags come from explicit descriptors: the GitHub release URL, the archive per host (the "-unsigned" tarballs on macOS, whose -unsigned.zip holds only the GUI app, and the "-unsigned" zip on win64), the archive root and each archive's SHA256 copied from the release's SHA256SUMS. Unsigned arm64 macOS binaries are self-signed as old Core ones are. A host without a qbit archive fails with a message instead of a guessed download. Bitcoin Core URLs, archives, hashes and messages are unchanged, and qbit tags join the default download list, so the previous-releases and Windows jobs fetch v1.0.0. The test framework maps client versions through explicit ReleaseInfo metadata (test_node.py): 10000 is qbit v1.0.0, based on Core v30.2 (core_compat_version 300200), with qbitd/qbit-cli binary names. A version from 140000 (Core v0.14) up is the Core release it names, as before; any lower version not in QBIT_RELEASES is an unknown qbit release and raises, instead of being read as a Core version. version_is_at_least() compares core_compat_version, so v1.0.0 nodes get -v2transport and the logging options, and qbit_version_is_at_least() gates qbit-only options: -p2mronly=0 now reaches qbit releases that have it, and supports_v2_pq marks which nodes may get -v2pqtransport. tool_qbit_release_resolution.py (qbit_release_resolution) checks v1.0.0 on every published host triple including win64 against a verbatim copy of SHA256SUMS, the Core mappings, the rejection of unknown qbit versions, and the options a v1.0.0, Core and current node get. It needs no downloads, so it runs in every functional job, Linux and Windows. Part of #184 (part 2, #186, work item B1). Co-Authored-By: Claude Opus 5.5 * test: baseline v1.0.0 v2 transport interoperability Add p2p_v2_previous_release.py, which pins how the qbit v1.0.0 release binary treats non-empty BIP324 version-packet contents, where the hybrid post-quantum handshake (#184) will carry its ML-KEM-1024 key and ciphertext. It drives a BIP324 peer on a plain socket, so it controls every byte and pause, against v1.0.0 in both roles: - test_version_contents: with 0, 1,572 and 65,000 bytes of contents, and with record-shaped contents (CompactSize(len) || header || payload): one 0xF0 record exactly as the hybrid offer and accept carry it (fd 21 06 f0 || 1,568 bytes), a 0xF0 record followed by an unknown 0xF1 record, and a single 65,000-byte record, each sent after three decoy packets, both immediately and after a 10 s pause under the default 60 s -peertimeout, v1.0.0 completes the handshake, exchanges version/verack and ping/pong, sends empty contents itself, and reports the plain BIP324 session id. As initiator it sends its version packet and VERSION message before receiving ours. - test_limits: v1.0.0 keeps a connection whose packet length is exactly MAX_CONTENTS_LEN (4,000,013), drops one at 4,000,014 bytes at once ("packet too large"), and drops a handshake stalled after the keys at -peertimeout ("V2 handshake timeout"), within a bounded time. The test skips without previous releases and runs in the previous-releases job, which downloads v1.0.0 since the previous commit. It takes about 24 s. Part of #184 (part 2, #186, work item B3). Co-Authored-By: Claude Opus 5.5 * ci: add reusable maintained-branch validation entry points Add .github/workflows/ci-pq.yml, one reusable workflow for the post-quantum transport jobs, and make ci-nightly-heavy.yml callable and job-selectable, so main's callers (C4), on a schedule and on demand, can run either against a recorded commit. GitHub dispatches only workflows whose file is on the default branch, so on-demand PQ runs go through C4's caller on main (workflow_dispatch with source_ref and jobs, calling ci-pq.yml@1.x.x), available after this part merges; ci-pq.yml keeps its own workflow_dispatch trigger for when it is on main. ci-nightly-heavy.yml is already on main, so it can also be dispatched on a branch directly. ci-pq.yml runs on workflow_call and workflow_dispatch with two inputs: - source_ref: resolved once by the resolve-source job to a full commit (empty: the triggering commit; a branch or tag, looked up on the remote with annotated tags peeled; or a full hash). A missing, ambiguous or malformed ref fails the run. Every job checks out that commit, verifies HEAD against it, and the summaries record it. - jobs: a comma-separated allowlist; empty runs all jobs, and an unknown, repeated or empty name fails the run. The resolve-source job emits the matrix, since a job-level if cannot see matrix values. Its first job, aarch64-unit, runs on GitHub's ubuntu-24.04-arm runner with the public-runner environment (docker.io/library images, CI_ENFORCE_INTERNAL_REGISTRY=0, public DNS, no tailnet search domains), builds only test_bitcoin and runs bip324_tests and net_tests (ci/test/00_setup_env_native_aarch64_pq.sh). Its summary shows the source commit, labels this the pre-library phase, and lists both suites with their results. ci/test/03_test_script.sh: - CI_BUILD_TARGET (default all) selects the target of the build and of its verbose retry; GOAL's install/deploy targets apply to full builds; - CTEST_REGEX selects tests by name and is rejected together with CTEST_INCLUDE_RANGE; - ctest runs with --no-tests=error, lists the selected tests first and writes JUnit results; when CTEST_EXPECTED_SUITES is set, ci/checks/ctest_evidence.py fails the job unless every expected suite was selected, executed and passed (ctest itself passes a skipped suite, for example one compiled out). ci-nightly-heavy.yml gains workflow_call inputs mirroring workflow_dispatch (choices become strings, which run-scanners.py validates), an optional LIBBITCOINPQC_READ_TOKEN secret, and a jobs input over arm32-unit-1, arm32-unit-2, previous-releases, fuzz and scanner. The matrix entries are unchanged and now come from resolve-source. A called run gets its caller's github context, so github.event_name, github.ref and github.sha are main's and cannot choose what a called run validates. Only workflow_call takes a source_ref input (default 1.x.x, must not be empty), and a non-empty source_ref marks a run as called: - a called run resolves source_ref once, with the same function as ci-pq.yml (which the contract test keeps identical), never reads github.ref or github.sha, and records source_mode maintained when source_ref names 1.x.x and requested otherwise; it honors run_test_matrix, run_scanners, the scanner settings and jobs whatever its caller's event; - a direct scheduled run still resolves 1.x.x and follows the CI_NIGHTLY_HEAVY_RUN_* variables and scheduled defaults, and a direct dispatch keeps its selected ref and event commit; - a first step fails the run when source_ref disagrees with GitHub's job.workflow_ref, for example a caller that passes an empty source_ref, and skips that check, saying so in its summary, where job.workflow_ref is unavailable; - a called run's concurrency group names its caller's event and source_ref, so callers for different sources, and an on-demand call beside the scheduled one, never cancel each other; - the summaries record the source_ref input and source_mode. ci/checks/test_ci_pq_contract.py executes the workflows' inline steps and the 03_test_script.sh blocks against git fixtures and stub cmake/ctest: job selection, rejection of unknown jobs, an immutable source commit, suites that really ran, and called and direct nightly runs with their concurrency groups. The shared expression evaluator follows GitHub's loose equality and supports format() and toJSON(). test_scheduled_validation_contract.py reads the nightly matrix from its new catalog. Both now run in required CI: the Required Merge Gate's classify-changes job (.github/workflows/required-merge-gate.yml) gains one step that runs them, beside the classifier contract. Part of #184 (part 2, #186, work item C1). Co-Authored-By: Claude Opus 5.5 * tools: add PQ canary sampling and evidence evaluation Scripted evidence for the mainnet canary of the hybrid post-quantum v2 transport (#184, R5), ready before the 14-day R0 baseline starts on v1.0.0 nodes. Everything lives in contrib/pq-canary/. pq-canary-sample.sh runs every 5 minutes from cron or a systemd timer on each canary node and the control node. Through qbit-cli it appends one row to samples.csv: time, host label, uptime, version; enabled, instance_id, backends, since, every per-direction handshake counter, load_shedding.active and each ring's last_sequence and dropped from getpqtransportinfo; connections_in and connections_pq; and, on the pool node (--pinned), the pinned archive node's getpeerinfo connection_type, transport_pq and transport_pq_status. sample_ok is 1 only when uptime, getnetworkinfo and, on the pool node, getpeerinfo answered. A failed call or a missing field is NA, never 0, and a real 0 stays 0, so a v1.0.0 node writes NA for the new fields with sample_ok=1. The raw recent_failures and fallback_set go to failures.jsonl, which keeps its addresses between the team's canary nodes. --pinned and --host are validated. pq_canary.py holds the format shared with the evaluator. pq-canary-eval.py prints each figure with its sample count, coverage and pass, fail or unknown: the pinned link manual and hybrid in at least 99% of samples excluding restarts (with the disconnectnode remedy when an automatic connection holds the address); no malformed_record, first_packet_failed or fallback entry between known-good endpoints, on every monitored canary node (one node's failures file cannot pass it); closed_after_switch entries between them, listed for triage; and the archive node's connections_in within +-20% of the control's, relative to their baseline ratio, per UTC day, excluding 48 hours after a restart. A jump in boot time is a restart gap. A ring that wrapped stays judged while the entries read continue from the last sequence processed; only a wrap that evicted unread entries, a changed instance_id or an inconsistent ring makes that ring's interval unknown. A figure, node or day below --min-coverage (default 90% of the expected 5-minute samples) is unknown, not judged, and every figure reports its coverage. Missing evidence is unknown, never clean. The report names known-good peers by label and never prints a peer address, so it can be posted. Unknown, malformed or missing arguments are rejected with a message and exit status 2. Tests: contrib/pq-canary/test/test_pq_canary.py (stdlib only, fixtures in the #184 getpqtransportinfo schema) covers, among others, a monitored outbound failure surviving 300 unrelated inbound failures, unrelated noise, a restart gap, an RPC outage, a wrapped ring with and without lost entries, per-node evidence, coverage thresholds, address redaction and argument validation. The Required Merge Gate's PQ contract step (.github/workflows/required-merge-gate.yml) now runs it as well, and test/functional/tool_pq_canary.py runs the real sampler against the node under test and, when previous releases are present, a v1.0.0 node. The README covers install, the cron lines and how to read the output. Part of #184 (part 2, #186, work item C5). Co-Authored-By: Claude Opus 5.5 * contrib: treat missing PQ canary fallback-set evidence as unknown The evaluator read a missing or malformed fallback_set as an empty list, so a sample without fallback evidence could pass Figure 2. Record it as an unknown interval for that host instead, as for malformed rings, and iterate fallback_set only when it is a list. Co-Authored-By: Claude Opus 5.5 * contrib: treat PQ canary records without an instance_id as unknown A missing or malformed instance_id was compared like a real process identity, so a restart between samples whose new ring did not go backwards went unnoticed and lost failures could pass Figure 2. Mark both rings unknown from the previous sample through the next record with a valid 64-hex-digit instance_id. Co-Authored-By: Claude Opus 5.5 * contrib: treat PQ canary entries with malformed endpoints as unknown A ring entry whose endpoint was missing or malformed got no known-good label and was skipped like a valid unrelated peer, so a fallback, malformed_record or first_packet_failed entry without endpoint telemetry could still let the failure figure pass. Check that an endpoint has the getpqtransportinfo Endpoint shape: a kind and network that agree, an address of that network without a port, and a port from 0 to 65535. An entry that counts towards figure 2 or 3 and has neither a known-good nor a well-formed endpoint now makes that figure unknown. A malformed endpoint that still names a known-good peer remains that peer's failure. Co-Authored-By: Claude Opus 5.5 * contrib: reject PQ canary samples that compare a node with itself Giving the same samples.csv for --archive and --control, or two files with the same host label, made figure 4 compare a node with itself: a perfect ratio that could pass at full coverage. The same applied to --pool and --archive, which also set the nodes monitored for figure 2. Require --pool, --archive and --control to be three different nodes. A file given for two roles, two roles sharing a host label or an instance_id, and a file whose rows carry no host label or more than one are bad input: the evaluator exits with status 2 and names the roles. The node test gave one node's samples as both pool and archive; give it the pool samples alone and check the rejection instead. The unit tests' simulated nodes now get distinct instance_ids, as real nodes do. Co-Authored-By: Claude Opus 5.5 * test: leave qbit releases without a host archive out of the default list On powerpc64le, a host set_host() recognizes, running get_previous_releases.py without tags now always included v1.0.0, which has no powerpc64le archive, so the default download stopped with "No qbit v1.0.0 archive". Without tags, download every Core tag as before but only the qbit releases that have an archive for the host, and say which were skipped. A tag given explicitly still fails when it has no archive for the host. Co-Authored-By: Claude Opus 5.5 * contrib: treat malformed PQ canary failure evidence as unknown Audit every failures.jsonl field the evaluator reads, so that a missing, malformed or contradictory value makes its stretch unknown rather than clean, and never crashes the evaluator into a "fail" exit status: - recent_failures that is not an object, and ring counters or sequences that are booleans, negative or zero, are a malformed ring (a non-object recent_failures raised AttributeError); - an entry outcome outside the getpqtransportinfo vocabulary may be a monitored or a triage one, so both figures are unknown (it was skipped as clean; an object outcome raised TypeError); - an entry time that is missing, malformed, before its process started or more than CLOCK_SLACK after the sample that read it is unknown (it was read as 1970 and skipped, or raised TypeError); - a fallback-set entry that is not an object or lacks a well-formed endpoint is unknown, once per distinct entry (it was skipped); - since that is missing, malformed, after the sample or changed within one instance_id makes the record's process identity unusable, like a missing instance_id (it fell back to the sample time); - a failures.jsonl line needs a time in Unix seconds up to year 9999 and a host label, else it is bad input (a boolean time was accepted, and a huge one crashed the report); - a known-good failure whose reason, cause, connection type, peer id, streak or times are malformed is still a failure, and those fields print as "malformed": a malformed field could put an address in the report, and malformed times or unhashable values raised. Labels are now matched in full, so a host label with a trailing newline is rejected. The unit tests' simulated nodes now report only failures recorded by the time of each sample, and boot before their first sample. Co-Authored-By: Claude Opus 5.5 * contrib: treat malformed PQ canary sample values as unknown Audit every samples.csv field the evaluator reads, so that a missing, malformed or contradictory value is unknown rather than a pass, a miss or a crash: - time, uptime and connections_in must be plain non-negative decimal integers. int() also took "-30", " 30", "3_0" and non-ASCII digits, so a negative uptime looked like a fresh restart and excluded samples from figures 1 and 4; a time such as "soon" raised ValueError. A row without a usable time is now a missing sample. --canary-start and --end must lie between 1970 and 9999, and so must a sample time, which the report prints. - A row with more or fewer fields than the header, such as a line cut short by a crash and continued by the next sample, has values in the wrong columns, so it is not a sample. - A connected pinned peer needs a connection type, transport_pq 0 or 1 and a transport_pq_status from the getpeerinfo vocabulary, with transport_pq 1 exactly when the status is hybrid. A malformed or contradictory value is unknown: it counted as a miss, and transport_pq=1 with status "pending" counted as hybrid. - Figure 4 pairs only samples with sample_ok=1. - An uptime whose boot time contradicts the one already seen for the same instance_id is unknown: it was read as a restart and excluded samples. Co-Authored-By: Claude Opus 5.5 * contrib: write only well-typed values to PQ canary samples The sampler wrote any integer, string or boolean it found: true became a connections_in or counter of 1, a negative uptime or since was kept, and an instance_id of any shape was recorded. Write each column only when the RPC value has its contract type (counts are non-negative integers, flags booleans, backends and statuses strings, instance_id 64 lowercase hex digits); anything else is NA, which the evaluator reads as unknown. Reject --host NA, which would read back as a missing host label. Co-Authored-By: Claude Opus 5.5 * contrib: keep one stray PQ canary value from making whole stretches unknown Two of the new unknown rules reached further than the evidence: - An uptime was checked against the first boot time seen for its instance_id. A restart between the sampler's uptime and getpqtransportinfo calls pairs the old uptime with the new instance_id once, which made every later uptime of that process unknown. Check each uptime against the median boot time of its instance_id instead, so one racy or malformed value is the only unknown. - An entry with an implausible time made figure 2 or 3 unknown even when the sample that first read it was taken long before the window. Such an entry was recorded before that sample, so it cannot be in the window. Also rename the samples() helper to sample_rows(), which evaluate_failures shadowed with its sample count. Co-Authored-By: Claude Opus 5.5 * test: check the release default list without the real host tool_qbit_release_resolution's default-tags case set HOST to make get_previous_releases.py act as a powerpc64le host. set_host() ignores HOST on Windows and answers win64, so on the Windows CI job the case downloaded the v1.0.0 win64 archive and failed. Stand in for set_host() so the case checks main()'s use of the host on any platform. Co-Authored-By: Claude Opus 5.5 * test: skip the v1.0.0 transport baseline unless v1.0.0 is downloaded p2p_v2_previous_release.py only checked that previous releases were enabled. A releases directory that holds other releases, such as Bitcoin Core's, but not qbit v1.0.0 enables them, so the test went on and setup_nodes() aborted on the missing qbitd and qbit-cli. Skip unless both v1.0.0 binaries are present, as tool_pq_canary.py leaves out its v1.0.0 sample. Co-Authored-By: Claude Opus 5.5 * contrib: count each expected PQ canary sample once Figure 1 counted every samples.csv row in the window, so rows repeated for one sampling slot (overlapping files, a manual run beside cron) added to its total: 95 of 100 expected samples is unknown, but the same rows recorded six times read as 99.1% hybrid and passed. Expected samples are now --interval apart from --canary-start, a sample stands for the nearest one, and every figure counts an expected sample once. Several samples for one count as the least favorable of them, so an extra sample can never improve the evidence: for the pinned link a miss, then unknown, then excluded, then hybrid; for inbound connections unknown, then excluded, then a measurement, where measurements that disagree are unknown (figure 4 kept whichever row came last). Figure 2 counted a record nearest to an expected sample just outside the window as covering one inside it; it now uses the same expected samples. A row repeated exactly is one sample, so repeated rows cannot outvote the median boot time that decides whether an uptime is a restart: one racy uptime copied a hundred times made two misses look like a restart and excluded them. Co-Authored-By: Claude Opus 5.5 * test: find every host qbit publishes v1.0.0 archives for get_previous_releases.py registers the arm-linux-gnueabihf and powerpc64-linux-gnu v1.0.0 archives, but set_host() had no pattern for either host, so on 32-bit ARM or big-endian POWER it exited "Not sure which binary to download" before resolve_download() saw them. Map both hosts. Each pattern for a qbit host now accepts the depends triplet given in HOST, which the script's help documents, as well as config.guess output. This also fixes the upstream-derived aarch64, riscv64 and arm64 macOS patterns, which only matched config.guess output: HOST=aarch64-linux-gnu, riscv64-linux-gnu or arm64-apple-darwin exited the same way. The ARM patterns accept only little-endian ARMv7 or ARMv8 hosts with hard float, because the v1.0.0 ARM binaries are ARMv7 VFPv3 armhf builds; an ARMv6 host still gets "Not sure which binary". powerpc64le (Core archives only) and x86_64-w64-mingw32 (win64 comes from platform.system(), not HOST) are left as they were. tool_qbit_release_resolution now runs set_host() with a mocked platform, HOST and config.guess output for every published host, and checks that each answer resolves to that host's v1.0.0 archive. Co-Authored-By: Claude Opus 5.5 * test: skip the v1.0.0 canary sample unless both executables exist tool_pq_canary enabled its v1.0.0 node whenever releases/v1.0.0/bin existed. get_previous_releases.py creates the release directory before extracting into it and keeps it after a failed extraction, so a stale or partial download could leave bin/ without qbitd or qbit-cli, and add_nodes() then aborted the whole test. Check both resolved executables, as p2p_v2_previous_release does, and otherwise skip only the optional v1.0.0 sample. Co-Authored-By: Claude Opus 5.5 * contrib: pass a PQ canary day exactly 20% off its baseline ratio Figure 4 passes a day whose archive/control connections_in ratio is within 20% of the baseline ratio, inclusive. The evaluator computed the relative ratio in floats with two divisions, so a day exactly on the bound could round past it: 24/10 against a baseline of 3 came out as 0.7999999999999999, and 108/10 against 9 as 1.2000000000000002, and both failed the figure and the release gate. Compute the ratios as exact fractions, and read --baseline-ratio exactly too, since the float nearest 1.2 is below it. float() still screens the argument first, so nan, infinity and exponents outside its range are rejected as before. Co-Authored-By: Claude Opus 5.5 * contrib: match a known-good IP address however it is written The failure figures look endpoints up in the known-good list by string. The node reports an IPv6 address compressed, so a list entry spelled out, such as [2001:0db8:0:0:0:0:0:5]:8333, never matched 2001:db8::5: its first_packet_failed entries were counted as another endpoint's, and Figure 2 passed at full coverage despite a failure between known-good peers. Give every IP address one spelling on both sides, the list and the node's endpoints, address-only entries included: compressed, lower case, and an IPv4-mapped IPv6 address as IPv4, which is how the node reports it. Two spellings of one entry are now "listed twice". Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .github/workflows/ci-nightly-heavy.yml | 362 ++++- .github/workflows/ci-pq.yml | 335 ++++ .github/workflows/required-merge-gate.yml | 9 + ci/checks/ctest_evidence.py | 137 ++ ci/checks/test_ci_pq_contract.py | 1006 ++++++++++++ .../test_scheduled_validation_contract.py | 105 +- ci/test/00_setup_env_native_aarch64_pq.sh | 29 + ci/test/03_test_script.sh | 43 +- contrib/pq-canary/README.md | 240 +++ contrib/pq-canary/pq-canary-eval.py | 1012 +++++++++++++ contrib/pq-canary/pq-canary-sample.sh | 85 ++ contrib/pq-canary/pq_canary.py | 224 +++ .../test/fixtures/getnetworkinfo-v1.0.0.json | 10 + .../test/fixtures/getnetworkinfo.json | 11 + .../pq-canary/test/fixtures/getpeerinfo.json | 18 + .../test/fixtures/getpqtransportinfo.json | 89 ++ .../pq-canary/test/fixtures/known-good.txt | 7 + contrib/pq-canary/test/test_pq_canary.py | 1348 +++++++++++++++++ .../feature_framework_unit_tests.py | 1 + test/functional/p2p_v2_previous_release.py | 388 +++++ .../functional/test_framework/crypto/mlkem.py | 461 ++++++ .../crypto/mlkem1024_test_vectors.txt | 64 + .../test_framework/test_framework.py | 63 +- test/functional/test_framework/test_node.py | 77 +- test/functional/test_runner.py | 3 + test/functional/tool_pq_canary.py | 157 ++ .../tool_qbit_release_resolution.py | 300 ++++ test/get_previous_releases.py | 145 +- 28 files changed, 6610 insertions(+), 119 deletions(-) create mode 100644 .github/workflows/ci-pq.yml create mode 100755 ci/checks/ctest_evidence.py create mode 100755 ci/checks/test_ci_pq_contract.py create mode 100755 ci/test/00_setup_env_native_aarch64_pq.sh create mode 100644 contrib/pq-canary/README.md create mode 100755 contrib/pq-canary/pq-canary-eval.py create mode 100755 contrib/pq-canary/pq-canary-sample.sh create mode 100755 contrib/pq-canary/pq_canary.py create mode 100644 contrib/pq-canary/test/fixtures/getnetworkinfo-v1.0.0.json create mode 100644 contrib/pq-canary/test/fixtures/getnetworkinfo.json create mode 100644 contrib/pq-canary/test/fixtures/getpeerinfo.json create mode 100644 contrib/pq-canary/test/fixtures/getpqtransportinfo.json create mode 100644 contrib/pq-canary/test/fixtures/known-good.txt create mode 100755 contrib/pq-canary/test/test_pq_canary.py create mode 100755 test/functional/p2p_v2_previous_release.py create mode 100644 test/functional/test_framework/crypto/mlkem.py create mode 100644 test/functional/test_framework/crypto/mlkem1024_test_vectors.txt create mode 100755 test/functional/tool_pq_canary.py create mode 100755 test/functional/tool_qbit_release_resolution.py diff --git a/.github/workflows/ci-nightly-heavy.yml b/.github/workflows/ci-nightly-heavy.yml index 6966149d7b..7755b84ea9 100644 --- a/.github/workflows/ci-nightly-heavy.yml +++ b/.github/workflows/ci-nightly-heavy.yml @@ -1,8 +1,69 @@ name: Nightly CI +# Heavy validation, entered three ways: +# - schedule: validates the maintained branch 1.x.x, with the repository +# variables CI_NIGHTLY_HEAVY_RUN_TEST_MATRIX and CI_NIGHTLY_HEAVY_RUN_SCANNERS +# and the scheduled defaults below. GitHub schedules only the default +# branch's copy of a workflow. +# - workflow_dispatch: validates the selected ref at its event commit, with +# the dispatch inputs. +# - workflow_call: main's callers (C4, part 2 of #184) run this copy on their +# schedule or on demand. A called run gets its caller's github context +# (GitHub's "Reusing workflows" reference), so github.event_name, github.ref +# and github.sha describe main's caller and cannot choose the source or the +# settings. A called run is recognized by its source_ref input instead: only +# workflow_call declares it, and its default is not empty. It validates +# source_ref and honors every input, whatever its caller's event. A caller +# must not pass an empty source_ref; the first step fails a run whose +# source_ref disagrees with GitHub's job.workflow_ref. +# The source is resolved once to a commit; every job checks out and verifies +# that commit, and the summaries record the source_ref input and source_mode +# (maintained, manual, or requested for a called run that names another source). + on: schedule: - cron: '30 3 * * *' + # The inputs mirror workflow_dispatch, plus source_ref. + workflow_call: + inputs: + source_ref: + description: 'Branch, tag or full commit hash to validate; defaults to the maintained branch 1.x.x; must not be empty' + required: false + type: string + default: '1.x.x' + run_test_matrix: + description: 'Run the existing heavy test matrix' + required: false + type: boolean + default: true + run_scanners: + description: 'Run scanners' + required: false + type: boolean + default: true + scanner_set: + description: 'Scanner set' + required: false + type: string + default: minimum + scanner_fail_policy: + description: 'Scanner failure policy' + required: false + type: string + default: infra-and-secrets + skip_build_scanners: + description: 'Skip build-backed scanners such as scan-build' + required: false + type: boolean + default: false + jobs: + description: 'Comma-separated jobs to run (arm32-unit-1, arm32-unit-2, previous-releases, fuzz, scanner); empty runs all' + required: false + type: string + default: '' + secrets: + LIBBITCOINPQC_READ_TOKEN: + required: false workflow_dispatch: inputs: run_test_matrix: @@ -42,9 +103,22 @@ on: required: false type: boolean default: false + jobs: + description: 'Comma-separated jobs to run (arm32-unit-1, arm32-unit-2, previous-releases, fuzz, scanner); empty runs all' + required: false + type: string + default: '' +# Concurrency groups are repository-wide, and every call from main shares +# main's github.ref. A called run's group therefore names its caller's event and +# its source_ref, so callers for different sources, and an on-demand call beside +# the scheduled one, never cancel each other; direct runs keep their groups. +# GitHub's "Reusing workflows" reference warns that a caller and a called +# workflow sharing a group with cancel-in-progress cancel the run already in +# progress, and that github.workflow is the caller's name in a called run: a +# caller must not use a nightly-heavy-call-* group. concurrency: - group: nightly-heavy-${{ github.event_name == 'schedule' && 'nightly-1.x.x' || github.ref }} + group: nightly-heavy-${{ inputs.source_ref != '' && format('call-{0}-{1}', github.event_name, inputs.source_ref) || github.event_name == 'schedule' && 'nightly-1.x.x' || github.ref }} cancel-in-progress: true env: @@ -62,14 +136,81 @@ jobs: name: Resolve source runs-on: ${{ github.repository == 'Qbit-Org/qbit' && fromJSON('["self-hosted", "linux", "x64", "qbit-trusted-ci"]') || fromJSON('["self-hosted", "linux", "x64"]') }} timeout-minutes: 10 + # Exactly one resolver step runs: resolve for a direct run, requested for a + # called one. outputs: - source_mode: ${{ steps.resolve.outputs.source_mode }} - requested_ref: ${{ steps.resolve.outputs.requested_ref }} - requested_ref_name: ${{ steps.resolve.outputs.requested_ref_name }} - resolved_sha: ${{ steps.resolve.outputs.resolved_sha }} + source_mode: ${{ steps.resolve.outputs.source_mode || steps.requested.outputs.source_mode }} + requested_ref: ${{ steps.resolve.outputs.requested_ref || steps.requested.outputs.requested_ref }} + requested_ref_name: ${{ steps.resolve.outputs.requested_ref_name || steps.requested.outputs.requested_ref_name }} + resolved_sha: ${{ steps.resolve.outputs.resolved_sha || steps.requested.outputs.resolved_sha }} + matrix: ${{ steps.select.outputs.matrix }} + matrix_count: ${{ steps.select.outputs.matrix_count }} + selected: ${{ steps.select.outputs.selected }} + run_scanner: ${{ steps.select.outputs.run_scanner }} steps: + # Fail closed when the source_ref rule and GitHub's record of how the run + # started disagree, for example a caller that passes an empty source_ref, + # which GitHub's loose equality cannot tell from no source_ref. The + # Contexts reference says of job.workflow_ref: "For jobs defined directly + # in a workflow file, this is the same as github.workflow_ref. For jobs + # defined in a [reusable workflow], this refers to the reusable workflow + # file." The full refs are compared, including @ref, so that main's own + # ci-nightly-heavy.yml calling this copy is still a called run. The value + # is read from toJSON(job) because actionlint does not know the property; + # where GitHub does not provide it (GitHub Enterprise Server), the check + # is skipped and the summary says so. + - name: Check how the run started + id: trigger + shell: python3 {0} + env: + RUN_KIND: ${{ inputs.source_ref != '' && 'called' || 'direct' }} + SOURCE_REF: ${{ inputs.source_ref }} + JOB_CONTEXT: ${{ toJSON(job) }} + RUN_WORKFLOW_REF: ${{ github.workflow_ref }} + EVENT_NAME: ${{ github.event_name }} + run: | + import json + import os + import sys + + + def fail(message): + print(f"::error::{message}", file=sys.stderr) + sys.exit(1) + + + kind = os.environ["RUN_KIND"] + run_ref = os.environ.get("RUN_WORKFLOW_REF", "") + try: + job = json.loads(os.environ.get("JOB_CONTEXT") or "null") or {} + except ValueError: + fail("the job context is not JSON") + job_ref = job.get("workflow_ref") or "" + if not job_ref: + check = "skipped: job.workflow_ref is not available" + else: + started = "direct" if job_ref == run_ref else "called" + if started == "called" and kind != "called": + fail(f"this run was called (job.workflow_ref {job_ref} is not github.workflow_ref {run_ref}) " + "with an empty source_ref; the caller must pass the branch, tag or commit to validate") + if started == "direct" and kind != "direct": + fail(f"source_ref '{os.environ.get('SOURCE_REF', '')}' is set, but this run was not called " + f"(job.workflow_ref is github.workflow_ref {run_ref})") + check = f"passed: {started}" + with open(os.environ.get("GITHUB_STEP_SUMMARY", os.devnull), "a", encoding="utf8") as summary: + summary.write("## Run\n\n| field | value |\n| --- | --- |\n" + f"| run | {kind} ({'source_ref given' if kind == 'called' else 'no source_ref'}) |\n" + f"| event | {os.environ.get('EVENT_NAME', '')} |\n" + f"| job.workflow_ref | {job_ref or '(not available)'} |\n" + f"| github.workflow_ref | {run_ref} |\n" + f"| cross-check | {check} |\n\n") + + # A direct run: a schedule validates the maintained branch, a dispatch + # its selected ref. This script is identical in the IBD and RPC perf + # workflows. - name: Resolve source id: resolve + if: ${{ inputs.source_ref == '' }} shell: bash env: SOURCE_MODE: ${{ github.event_name == 'schedule' && 'maintained' || 'manual' }} @@ -140,8 +281,169 @@ jobs: echo "| workflow_sha | $WORKFLOW_SHA |" } >> "${GITHUB_STEP_SUMMARY:-/dev/null}" + # A called run: its source_ref names the source, resolved exactly as + # ci-pq.yml resolves its own. github.ref and github.sha are the caller's, + # so this step never reads them. + - name: Resolve requested source + id: requested + if: ${{ inputs.source_ref != '' }} + shell: bash + env: + SOURCE_REF: ${{ inputs.source_ref }} + MAINTAINED_BRANCH: 1.x.x + CALLER_EVENT: ${{ github.event_name }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + SOURCE_REPOSITORY_URL: ${{ github.server_url }}/${{ github.repository }}.git + GITHUB_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + # Resolve the validation source once per run, before any checkout, so + # every job builds and reports the same immutable commit. source_mode + # is maintained when source_ref names the maintained branch, and + # requested otherwise. + fail() { echo "::error::$*" >&2; exit 1; } + is_full_sha() { printf '%s\n' "$1" | LC_ALL=C grep -Eq '^[0-9a-f]{40}$'; } + + # Sets requested_ref and resolved_sha from a non-empty source_ref, or + # fails the run: a full commit hash is used as is; a branch or tag + # name is looked up on the remote, and an annotated tag is peeled to + # its commit. ci-pq.yml and ci-nightly-heavy.yml (for a called run) + # share this function; ci/checks/test_ci_pq_contract.py keeps the two + # copies identical. + resolve_source_ref() { + local source_ref="$1" candidate status listing matches peeled + local -a candidates patterns auth + if is_full_sha "$source_ref"; then + requested_ref="$source_ref" + resolved_sha="$source_ref" + return 0 + fi + case "$source_ref" in + refs/heads/?*|refs/tags/?*) candidates=("$source_ref") ;; + refs/*) fail "source_ref '$source_ref' must be a branch, a tag or a full commit hash" ;; + *) candidates=("refs/heads/$source_ref" "refs/tags/$source_ref") ;; + esac + patterns=() + for candidate in "${candidates[@]}"; do + git check-ref-format "$candidate" || fail "source_ref '$source_ref' is not a valid ref name" + patterns+=("$candidate" "$candidate^{}") + done + auth=() + if [ -n "${GITHUB_TOKEN:-}" ]; then + auth=(-c "http.extraheader=AUTHORIZATION: basic $(printf 'x-access-token:%s' "$GITHUB_TOKEN" | base64 | tr -d '\n')") + fi + status=0 + listing="$(git ${auth[@]+"${auth[@]}"} ls-remote --exit-code -- "$SOURCE_REPOSITORY_URL" "${patterns[@]}")" || status=$? + case "$status" in + 0) ;; + 2) fail "no branch or tag '$source_ref' in '$SOURCE_REPOSITORY_URL'" ;; + *) fail "could not list refs from '$SOURCE_REPOSITORY_URL' (git exit code $status)" ;; + esac + matches="$(printf '%s\n' "$listing" | awk -v a="${candidates[0]}" -v b="${candidates[1]:-}" '$2 == a || $2 == b { print $2 }')" + [ "$(printf '%s\n' "$matches" | grep -c .)" -eq 1 ] \ + || fail "source_ref '$source_ref' must name exactly one branch or tag, found: ${matches:-none}" + requested_ref="$matches" + peeled="$(printf '%s\n' "$listing" | awk -v ref="$requested_ref^{}" '$2 == ref { print $1 }')" + resolved_sha="${peeled:-$(printf '%s\n' "$listing" | awk -v ref="$requested_ref" '$2 == ref { print $1 }')}" + } + + source_ref="$(printf '%s' "$SOURCE_REF" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + [ -n "$source_ref" ] || fail "source_ref is empty; a called run must name the branch, tag or commit to validate" + [ -n "$MAINTAINED_BRANCH" ] || fail "MAINTAINED_BRANCH is empty" + resolve_source_ref "$source_ref" + if [ "$requested_ref" = "refs/heads/${MAINTAINED_BRANCH}" ]; then + source_mode=maintained + else + source_mode=requested + fi + is_full_sha "$resolved_sha" || fail "resolved source '$resolved_sha' is not a full commit hash" + is_full_sha "$WORKFLOW_SHA" || fail "workflow revision '$WORKFLOW_SHA' is not a full commit hash" + requested_ref_name="${requested_ref#refs/heads/}" + requested_ref_name="${requested_ref_name#refs/tags/}" + + { + echo "source_mode=$source_mode" + echo "requested_ref=$requested_ref" + echo "requested_ref_name=$requested_ref_name" + echo "resolved_sha=$resolved_sha" + } >> "$GITHUB_OUTPUT" + { + echo "## Validation source" + echo + echo "| field | value |" + echo "| --- | --- |" + echo "| source_ref input | $source_ref |" + echo "| source_mode | $source_mode |" + echo "| requested_ref | $requested_ref |" + echo "| resolved_sha | $resolved_sha |" + echo "| caller_event | $CALLER_EVENT |" + echo "| workflow_sha | $WORKFLOW_SHA |" + } >> "${GITHUB_STEP_SUMMARY:-/dev/null}" + + - name: Select jobs + id: select + shell: python3 {0} + env: + JOBS_INPUT: ${{ inputs.jobs }} + # The nightly-matrix entries; "job" is the name the jobs input uses. + JOB_CATALOG: | + [ + {"job": "arm32-unit-1", "name": "32 bit ARM, unit tests shard 1/2, no functional tests", "timeout-minutes": 180, "file-env": "./ci/test/00_setup_env_arm.sh", "ctest_include_range": "1,139", "container-cpus": "8", "container-memory": "48g"}, + {"job": "arm32-unit-2", "name": "32 bit ARM, unit tests shard 2/2, no functional tests", "timeout-minutes": 180, "file-env": "./ci/test/00_setup_env_arm.sh", "ctest_include_range": "140,99999", "container-cpus": "8", "container-memory": "48g"}, + {"job": "previous-releases", "name": "previous releases, depends DEBUG", "timeout-minutes": 180, "file-env": "./ci/test/00_setup_env_native_previous_releases.sh", "container-cpus": "8", "container-memory": "48g"}, + {"job": "fuzz", "name": "fuzzer,address,undefined,integer, no depends", "timeout-minutes": 240, "file-env": "./ci/test/00_setup_env_native_fuzz.sh", "makejobs": "-j12", "container-cpus": "12", "container-memory": "64g", "fuzz-mutate-min-time": "120"} + ] + # Selectable jobs that are not matrix entries; each gets a run_ output. + NON_MATRIX_JOBS: scanner + run: | + # Turn the comma-separated jobs input into the matrix: empty selects + # every job; an empty, unknown or repeated name fails the run. + import json + import os + import sys + + + def fail(message): + print(f"::error::{message}", file=sys.stderr) + sys.exit(1) + + + catalog = json.loads(os.environ["JOB_CATALOG"]) + non_matrix = os.environ.get("NON_MATRIX_JOBS", "").split() + known = [entry["job"] for entry in catalog] + non_matrix + if len(set(known)) != len(known): + fail(f"job catalog repeats a name: {known}") + + raw = os.environ.get("JOBS_INPUT", "") + if raw.strip(): + requested = [name.strip() for name in raw.split(",")] + if "" in requested: + fail(f"jobs input '{raw}' contains an empty name") + repeated = sorted({name for name in requested if requested.count(name) > 1}) + if repeated: + fail(f"jobs input names a job more than once: {', '.join(repeated)}") + unknown = [name for name in requested if name not in known] + if unknown: + fail(f"unknown job(s) {', '.join(unknown)}; known jobs: {', '.join(known)}") + else: + requested = known + + include = [entry for entry in catalog if entry["job"] in requested] + selected = [name for name in known if name in requested] + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf8") as out: + out.write(f"matrix={json.dumps({'include': include}, separators=(',', ':'))}\n") + out.write(f"matrix_count={len(include)}\n") + out.write(f"selected={json.dumps(selected, separators=(',', ':'))}\n") + for name in non_matrix: + out.write(f"run_{name.replace('-', '_')}={'true' if name in requested else 'false'}\n") + with open(os.environ.get("GITHUB_STEP_SUMMARY", os.devnull), "a", encoding="utf8") as summary: + summary.write(f"\nSelected jobs: {', '.join(selected) or 'none'} (jobs input: {raw.strip() or 'all'})\n") + nightly-matrix: - if: ${{ (github.event_name == 'schedule' && vars.CI_NIGHTLY_HEAVY_RUN_TEST_MATRIX != 'false') || (github.event_name == 'workflow_dispatch' && inputs.run_test_matrix) }} + # A direct scheduled run follows the repository variable; a called run or a + # direct dispatch follows its inputs. A called run reports its caller's + # event, so github.event_name alone cannot tell it from a direct one. + if: ${{ ((inputs.source_ref == '' && github.event_name == 'schedule' && vars.CI_NIGHTLY_HEAVY_RUN_TEST_MATRIX != 'false') || ((inputs.source_ref != '' || github.event_name != 'schedule') && inputs.run_test_matrix)) && needs.resolve-source.outputs.matrix_count != '0' }} name: ${{ matrix.name }} needs: resolve-source runs-on: ${{ github.repository == 'Qbit-Org/qbit' && fromJSON('["self-hosted", "linux", "x64", "qbit-trusted-ci"]') || fromJSON('["self-hosted", "linux", "x64"]') }} @@ -165,32 +467,8 @@ jobs: QBIT_FUZZ_MUTATE_MIN_TIME: ${{ matrix.fuzz-mutate-min-time || '' }} strategy: fail-fast: false - matrix: - include: - - name: '32 bit ARM, unit tests shard 1/2, no functional tests' - timeout-minutes: 180 - file-env: './ci/test/00_setup_env_arm.sh' - ctest_include_range: '1,139' - container-cpus: '8' - container-memory: '48g' - - name: '32 bit ARM, unit tests shard 2/2, no functional tests' - timeout-minutes: 180 - file-env: './ci/test/00_setup_env_arm.sh' - ctest_include_range: '140,99999' - container-cpus: '8' - container-memory: '48g' - - name: 'previous releases, depends DEBUG' - timeout-minutes: 180 - file-env: './ci/test/00_setup_env_native_previous_releases.sh' - container-cpus: '8' - container-memory: '48g' - - name: 'fuzzer,address,undefined,integer, no depends' - timeout-minutes: 240 - file-env: './ci/test/00_setup_env_native_fuzz.sh' - makejobs: '-j12' - container-cpus: '12' - container-memory: '64g' - fuzz-mutate-min-time: '120' + # The entries are JOB_CATALOG in resolve-source, filtered by the jobs input. + matrix: ${{ fromJSON(needs.resolve-source.outputs.matrix) }} steps: - name: Require resolved source @@ -215,6 +493,7 @@ jobs: - name: Verify checked-out source shell: bash env: + SOURCE_REF_INPUT: ${{ inputs.source_ref }} SOURCE_MODE: ${{ needs.resolve-source.outputs.source_mode }} REQUESTED_REF: ${{ needs.resolve-source.outputs.requested_ref }} RESOLVED_SHA: ${{ needs.resolve-source.outputs.resolved_sha }} @@ -230,6 +509,7 @@ jobs: { echo "### Checked-out source: ${GITHUB_JOB}${MATRIX_NAME:+ ($MATRIX_NAME)}" echo + echo "- source_ref input: ${SOURCE_REF_INPUT:-(none)}" echo "- source_mode: ${SOURCE_MODE}" echo "- requested_ref: ${REQUESTED_REF}" echo "- resolved_sha: ${RESOLVED_SHA}" @@ -256,7 +536,8 @@ jobs: uses: ./.github/actions/save-caches scanner-readiness: - if: ${{ (github.event_name == 'schedule' && vars.CI_NIGHTLY_HEAVY_RUN_SCANNERS != 'false') || (github.event_name == 'workflow_dispatch' && inputs.run_scanners) }} + # The same rule as nightly-matrix, with the scanner's variable and input. + if: ${{ ((inputs.source_ref == '' && github.event_name == 'schedule' && vars.CI_NIGHTLY_HEAVY_RUN_SCANNERS != 'false') || ((inputs.source_ref != '' || github.event_name != 'schedule') && inputs.run_scanners)) && needs.resolve-source.outputs.run_scanner == 'true' }} name: Scanner needs: resolve-source runs-on: ${{ github.repository == 'Qbit-Org/qbit' && fromJSON('["self-hosted", "linux", "x64", "qbit-trusted-ci"]') || fromJSON('["self-hosted", "linux", "x64"]') }} @@ -264,7 +545,13 @@ jobs: env: SCANNER_SET: ${{ inputs.scanner_set || 'minimum' }} SCANNER_FAIL_POLICY: ${{ inputs.scanner_fail_policy || 'infra-and-secrets' }} - SKIP_BUILD_SCANNERS: ${{ github.event_name == 'schedule' && 'true' || inputs.skip_build_scanners && 'true' || 'false' }} + # A direct scheduled run skips build-backed scanners; a called run or a + # direct dispatch follows skip_build_scanners. + SKIP_BUILD_SCANNERS: ${{ inputs.source_ref == '' && github.event_name == 'schedule' && 'true' || inputs.skip_build_scanners && 'true' || 'false' }} + # The run's purpose, which the caller's event does give: nightly for a + # scheduled run, direct or called by main's scheduled caller, and a manual + # dry run for anything on demand. + SCANNER_MODE: ${{ github.event_name == 'schedule' && 'nightly' || 'manual_dry_run' }} steps: - name: Require resolved source shell: bash @@ -288,6 +575,7 @@ jobs: - name: Verify checked-out source shell: bash env: + SOURCE_REF_INPUT: ${{ inputs.source_ref }} SOURCE_MODE: ${{ needs.resolve-source.outputs.source_mode }} REQUESTED_REF: ${{ needs.resolve-source.outputs.requested_ref }} RESOLVED_SHA: ${{ needs.resolve-source.outputs.resolved_sha }} @@ -302,6 +590,7 @@ jobs: { echo "### Checked-out source: ${GITHUB_JOB}${MATRIX_NAME:+ ($MATRIX_NAME)}" echo + echo "- source_ref input: ${SOURCE_REF_INPUT:-(none)}" echo "- source_mode: ${SOURCE_MODE}" echo "- requested_ref: ${REQUESTED_REF}" echo "- resolved_sha: ${RESOLVED_SHA}" @@ -321,10 +610,7 @@ jobs: LIBBITCOINPQC_READ_TOKEN: ${{ secrets.LIBBITCOINPQC_READ_TOKEN }} run: | set -euo pipefail - mode="nightly" - if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - mode="manual_dry_run" - fi + mode="$SCANNER_MODE" git fetch --no-tags origin +main:refs/remotes/origin/main source_commit="$(git rev-parse HEAD)" history_diff_base_ref="origin/main" diff --git a/.github/workflows/ci-pq.yml b/.github/workflows/ci-pq.yml new file mode 100644 index 0000000000..c252804291 --- /dev/null +++ b/.github/workflows/ci-pq.yml @@ -0,0 +1,335 @@ +# Copyright (c) 2026-present The qbit core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit. + +# Post-quantum transport validation (#184). One reusable workflow, run through +# callers on main, because GitHub schedules and dispatches only workflows whose +# file is on the default branch: +# - main's nightly caller runs it on a schedule with source_ref: 1.x.x; +# - on-demand runs go through main's PQ caller (C4, part 2 of #184), which takes +# workflow_dispatch inputs source_ref and jobs and calls ci-pq.yml@1.x.x, for +# example gh workflow run ci-maintained-pq.yml -f source_ref= -f jobs=aarch64-unit. +# That caller lands on main after this part merges. +# The workflow_dispatch trigger below stays for when this file is on main itself; +# until then GitHub rejects gh workflow run ci-pq.yml. +# The source is resolved once to a commit, every job checks out and verifies +# that commit, and the summaries record it. + +name: PQ CI +run-name: "PQ CI: ${{ inputs.source_ref || github.ref_name }} (${{ inputs.jobs || 'all jobs' }})" + +on: + workflow_call: + inputs: + source_ref: + description: 'Branch, tag or full commit hash to validate; empty validates the triggering commit' + required: false + type: string + default: '' + jobs: + description: 'Comma-separated jobs to run (aarch64-unit); empty runs all' + required: false + type: string + default: '' + workflow_dispatch: + inputs: + source_ref: + description: 'Branch, tag or full commit hash to validate; empty validates the selected ref' + required: false + type: string + default: '' + jobs: + description: 'Comma-separated jobs to run (aarch64-unit); empty runs all' + required: false + type: string + default: '' + +permissions: + contents: read + +jobs: + resolve-source: + name: Resolve source and jobs + runs-on: ubuntu-24.04 + timeout-minutes: 10 + outputs: + source_mode: ${{ steps.resolve.outputs.source_mode }} + requested_ref: ${{ steps.resolve.outputs.requested_ref }} + resolved_sha: ${{ steps.resolve.outputs.resolved_sha }} + matrix: ${{ steps.select.outputs.matrix }} + matrix_count: ${{ steps.select.outputs.matrix_count }} + selected: ${{ steps.select.outputs.selected }} + steps: + - name: Resolve source + id: resolve + shell: bash + env: + SOURCE_REF: ${{ inputs.source_ref }} + EVENT_REF: ${{ github.ref }} + EVENT_SHA: ${{ github.sha }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + SOURCE_REPOSITORY_URL: ${{ github.server_url }}/${{ github.repository }}.git + GITHUB_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + # Resolve the validation source once per run, before any checkout, so + # every job builds and reports the same immutable commit. An empty + # source_ref keeps the triggering ref and commit; otherwise + # resolve_source_ref resolves it. + fail() { echo "::error::$*" >&2; exit 1; } + is_full_sha() { printf '%s\n' "$1" | LC_ALL=C grep -Eq '^[0-9a-f]{40}$'; } + + # Sets requested_ref and resolved_sha from a non-empty source_ref, or + # fails the run: a full commit hash is used as is; a branch or tag + # name is looked up on the remote, and an annotated tag is peeled to + # its commit. ci-pq.yml and ci-nightly-heavy.yml (for a called run) + # share this function; ci/checks/test_ci_pq_contract.py keeps the two + # copies identical. + resolve_source_ref() { + local source_ref="$1" candidate status listing matches peeled + local -a candidates patterns auth + if is_full_sha "$source_ref"; then + requested_ref="$source_ref" + resolved_sha="$source_ref" + return 0 + fi + case "$source_ref" in + refs/heads/?*|refs/tags/?*) candidates=("$source_ref") ;; + refs/*) fail "source_ref '$source_ref' must be a branch, a tag or a full commit hash" ;; + *) candidates=("refs/heads/$source_ref" "refs/tags/$source_ref") ;; + esac + patterns=() + for candidate in "${candidates[@]}"; do + git check-ref-format "$candidate" || fail "source_ref '$source_ref' is not a valid ref name" + patterns+=("$candidate" "$candidate^{}") + done + auth=() + if [ -n "${GITHUB_TOKEN:-}" ]; then + auth=(-c "http.extraheader=AUTHORIZATION: basic $(printf 'x-access-token:%s' "$GITHUB_TOKEN" | base64 | tr -d '\n')") + fi + status=0 + listing="$(git ${auth[@]+"${auth[@]}"} ls-remote --exit-code -- "$SOURCE_REPOSITORY_URL" "${patterns[@]}")" || status=$? + case "$status" in + 0) ;; + 2) fail "no branch or tag '$source_ref' in '$SOURCE_REPOSITORY_URL'" ;; + *) fail "could not list refs from '$SOURCE_REPOSITORY_URL' (git exit code $status)" ;; + esac + matches="$(printf '%s\n' "$listing" | awk -v a="${candidates[0]}" -v b="${candidates[1]:-}" '$2 == a || $2 == b { print $2 }')" + [ "$(printf '%s\n' "$matches" | grep -c .)" -eq 1 ] \ + || fail "source_ref '$source_ref' must name exactly one branch or tag, found: ${matches:-none}" + requested_ref="$matches" + peeled="$(printf '%s\n' "$listing" | awk -v ref="$requested_ref^{}" '$2 == ref { print $1 }')" + resolved_sha="${peeled:-$(printf '%s\n' "$listing" | awk -v ref="$requested_ref" '$2 == ref { print $1 }')}" + } + + source_ref="$(printf '%s' "$SOURCE_REF" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + if [ -z "$source_ref" ]; then + source_mode=event + requested_ref="$EVENT_REF" + resolved_sha="$EVENT_SHA" + else + resolve_source_ref "$source_ref" + if is_full_sha "$source_ref"; then + source_mode=commit + else + source_mode=ref + fi + fi + + is_full_sha "$resolved_sha" || fail "resolved source '$resolved_sha' is not a full commit hash" + [ -n "$requested_ref" ] || fail "requested ref is empty" + + { + echo "source_mode=$source_mode" + echo "requested_ref=$requested_ref" + echo "resolved_sha=$resolved_sha" + } >> "$GITHUB_OUTPUT" + { + echo "## Validation source" + echo + echo "| field | value |" + echo "| --- | --- |" + echo "| source_ref input | ${source_ref:-(empty)} |" + echo "| source_mode | $source_mode |" + echo "| requested_ref | $requested_ref |" + echo "| resolved_sha | $resolved_sha |" + echo "| event_sha | $EVENT_SHA |" + echo "| workflow_sha | $WORKFLOW_SHA |" + } >> "${GITHUB_STEP_SUMMARY:-/dev/null}" + + - name: Select jobs + id: select + shell: python3 {0} + env: + JOBS_INPUT: ${{ inputs.jobs }} + # Jobs run as matrix entries; "job" is the name the jobs input uses. + JOB_CATALOG: | + [ + { + "job": "aarch64-unit", + "name": "AArch64 unit tests (pre-library phase)", + "runs-on": "ubuntu-24.04-arm", + "timeout-minutes": 120, + "file-env": "./ci/test/00_setup_env_native_aarch64_pq.sh", + "phase": "Pre-library phase: bip324_tests and net_tests on AArch64. Part 3 (#190) adds mlkem_tests with the ML-KEM library." + } + ] + # Selectable jobs that are not matrix entries; each gets a run_ output. + NON_MATRIX_JOBS: '' + run: | + # Turn the comma-separated jobs input into the matrix: empty selects + # every job; an empty, unknown or repeated name fails the run. + import json + import os + import sys + + + def fail(message): + print(f"::error::{message}", file=sys.stderr) + sys.exit(1) + + + catalog = json.loads(os.environ["JOB_CATALOG"]) + non_matrix = os.environ.get("NON_MATRIX_JOBS", "").split() + known = [entry["job"] for entry in catalog] + non_matrix + if len(set(known)) != len(known): + fail(f"job catalog repeats a name: {known}") + + raw = os.environ.get("JOBS_INPUT", "") + if raw.strip(): + requested = [name.strip() for name in raw.split(",")] + if "" in requested: + fail(f"jobs input '{raw}' contains an empty name") + repeated = sorted({name for name in requested if requested.count(name) > 1}) + if repeated: + fail(f"jobs input names a job more than once: {', '.join(repeated)}") + unknown = [name for name in requested if name not in known] + if unknown: + fail(f"unknown job(s) {', '.join(unknown)}; known jobs: {', '.join(known)}") + else: + requested = known + + include = [entry for entry in catalog if entry["job"] in requested] + selected = [name for name in known if name in requested] + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf8") as out: + out.write(f"matrix={json.dumps({'include': include}, separators=(',', ':'))}\n") + out.write(f"matrix_count={len(include)}\n") + out.write(f"selected={json.dumps(selected, separators=(',', ':'))}\n") + for name in non_matrix: + out.write(f"run_{name.replace('-', '_')}={'true' if name in requested else 'false'}\n") + with open(os.environ.get("GITHUB_STEP_SUMMARY", os.devnull), "a", encoding="utf8") as summary: + summary.write(f"\nSelected jobs: {', '.join(selected) or 'none'} (jobs input: {raw.strip() or 'all'})\n") + + unit: + name: ${{ matrix.name }} + needs: resolve-source + if: ${{ needs.resolve-source.outputs.matrix_count != '0' }} + runs-on: ${{ matrix.runs-on }} + timeout-minutes: ${{ matrix.timeout-minutes }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.resolve-source.outputs.matrix) }} + env: + CI_FAILFAST_TEST_LEAVE_DANGLING: 1 + DANGER_CI_ON_HOST_FOLDERS: 1 + FILE_ENV: ${{ matrix.file-env }} + TEST_RUNNER_FAILFAST: 'false' + MAKEJOBS: '-j4' + TEST_RUNNER_JOBS: '4' + CTEST_JOBS: '4' + # GitHub-hosted runners use the public registry and resolvers, not the + # internal registry and tailnet DNS that the self-hosted jobs use. + CI_PROFILE: public + CI_IMAGE_REGISTRY_PREFIX: docker.io/library + CI_ENFORCE_INTERNAL_REGISTRY: '0' + CI_BUILDKIT_DNS_NAMESERVERS: '1.1.1.1,8.8.8.8' + CI_BUILDKIT_DNS_SEARCH_DOMAINS: '' + steps: + - name: Require resolved source + shell: bash + env: + RESOLVED_SHA: ${{ needs.resolve-source.outputs.resolved_sha }} + REQUESTED_REF: ${{ needs.resolve-source.outputs.requested_ref }} + run: | + set -euo pipefail + # Refuse to check out anything but the per-run resolved commit; an + # empty ref would silently fall back to the triggering commit. + printf '%s\n' "$RESOLVED_SHA" | LC_ALL=C grep -Eq '^[0-9a-f]{40}$' \ + || { echo "::error::resolved source '$RESOLVED_SHA' is not a full commit hash" >&2; exit 1; } + [ -n "$REQUESTED_REF" ] || { echo "::error::requested ref is empty" >&2; exit 1; } + + - name: Checkout + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + ref: ${{ needs.resolve-source.outputs.resolved_sha }} + fetch-depth: 0 + + - name: Verify checked-out source + shell: bash + env: + SOURCE_MODE: ${{ needs.resolve-source.outputs.source_mode }} + REQUESTED_REF: ${{ needs.resolve-source.outputs.requested_ref }} + RESOLVED_SHA: ${{ needs.resolve-source.outputs.resolved_sha }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + MATRIX_NAME: ${{ matrix.name }} + run: | + set -euo pipefail + actual_head="$(git rev-parse --verify 'HEAD^{commit}')" + if [ "$actual_head" != "$RESOLVED_SHA" ]; then + echo "::error::checked-out HEAD $actual_head does not match resolved source $RESOLVED_SHA" >&2 + exit 1 + fi + { + echo "### Checked-out source: ${GITHUB_JOB}${MATRIX_NAME:+ ($MATRIX_NAME)}" + echo + echo "- source_mode: ${SOURCE_MODE}" + echo "- requested_ref: ${REQUESTED_REF}" + echo "- resolved_sha: ${RESOLVED_SHA}" + echo "- actual_head: ${actual_head}" + echo "- workflow_sha: ${WORKFLOW_SHA}" + } >> "${GITHUB_STEP_SUMMARY:-/dev/null}" + + - name: Configure environment + uses: ./.github/actions/configure-environment + + - name: Restore caches + id: restore-cache + uses: ./.github/actions/restore-caches + + - name: Configure Docker + uses: ./.github/actions/configure-docker + with: + cache-provider: gha + + - name: CI script + run: ./ci/test_run_all.sh + + - name: Report unit-test evidence + if: ${{ always() }} + shell: bash + env: + RESOLVED_SHA: ${{ needs.resolve-source.outputs.resolved_sha }} + REQUESTED_REF: ${{ needs.resolve-source.outputs.requested_ref }} + MATRIX_NAME: ${{ matrix.name }} + PHASE: ${{ matrix.phase }} + run: | + set -euo pipefail + # ci/test/03_test_script.sh writes this after ctest_evidence.py has + # checked that every expected suite was selected, executed and passed. + evidence="${BASE_BUILD_DIR}/ctest-evidence/summary.md" + { + echo "## ${MATRIX_NAME}" + echo + echo "- source: ${RESOLVED_SHA} (${REQUESTED_REF})" + echo "- ${PHASE}" + echo + if [ -f "$evidence" ]; then + cat "$evidence" + else + echo "No unit-test evidence was produced." + fi + } >> "${GITHUB_STEP_SUMMARY:-/dev/null}" + [ -f "$evidence" ] || { echo "::error::no unit-test evidence at $evidence" >&2; exit 1; } + + - name: Save caches + uses: ./.github/actions/save-caches diff --git a/.github/workflows/required-merge-gate.yml b/.github/workflows/required-merge-gate.yml index cea6f908b3..e9e4008c2a 100644 --- a/.github/workflows/required-merge-gate.yml +++ b/.github/workflows/required-merge-gate.yml @@ -141,6 +141,15 @@ jobs: - name: Test merge gate polling contract run: /usr/bin/python3 ci/checks/test_merge_gate_poller.py -v + # Job selection, source resolution and unit-test evidence of the PQ and + # nightly workflows, which run outside pull requests, the nightly + # workflow's scheduled source, and the PQ canary tools. + - name: Test PQ CI workflow contract + run: | + /usr/bin/python3 ci/checks/test_ci_pq_contract.py -v + /usr/bin/python3 ci/checks/test_scheduled_validation_contract.py -v + /usr/bin/python3 contrib/pq-canary/test/test_pq_canary.py -v + - name: Classify validation profile id: classify run: | diff --git a/ci/checks/ctest_evidence.py b/ci/checks/ctest_evidence.py new file mode 100755 index 0000000000..c2f501c0e4 --- /dev/null +++ b/ci/checks/ctest_evidence.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026-present The qbit core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. +"""Prove that a CI job's expected unit-test suites really ran and passed. + +ctest exits 0 when a suite is skipped: a Boost suite that was compiled out, or +filtered away, prints "no test cases matching filter", which the suite's +SKIP_REGULAR_EXPRESSION turns into a skip. A selection regex can also drift and +quietly select fewer suites. ci/test/03_test_script.sh therefore runs this +check after ctest whenever CTEST_EXPECTED_SUITES is set: + +* the tests ctest selected (``ctest --show-only=json-v1`` with the same + arguments as the run) must be exactly the expected suites; +* every expected suite must appear once in ctest's JUnit file with status + ``run`` and no failure; skipped (``notrun``), disabled, failed and missing + suites fail the check. + +It writes a Markdown table of the results, which the workflow adds to the job +summary, and exits non-zero on any violation. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +PASSED = "passed" + + +def parse_expected(text: str) -> list[str]: + names = text.split() + if not names: + raise ValueError("no expected suites given") + duplicates = sorted({name for name in names if names.count(name) > 1}) + if duplicates: + raise ValueError(f"expected suites listed more than once: {', '.join(duplicates)}") + return names + + +def selected_tests(listing: str) -> list[str]: + """Test names from ``ctest --show-only=json-v1`` output.""" + return [test["name"] for test in json.loads(listing).get("tests", [])] + + +def junit_outcomes(junit: str) -> dict[str, tuple[str, str]]: + """Map each test case in a ctest JUnit file to (outcome, time in seconds).""" + root = ET.fromstring(junit) + outcomes: dict[str, tuple[str, str]] = {} + for case in root.iter("testcase"): + name = case.get("name", "") + status = case.get("status", "") + if case.find("failure") is not None or case.find("error") is not None or status == "fail": + outcome = "failed" + elif case.find("skipped") is not None or status == "notrun": + outcome = "skipped" + elif status == "disabled": + outcome = "disabled" + elif status == "run": + outcome = PASSED + else: + outcome = f"unknown status {status!r}" + if name in outcomes: + outcome = "reported more than once" + outcomes[name] = (outcome, case.get("time", "")) + return outcomes + + +def check(expected: list[str], selected: list[str], outcomes: dict[str, tuple[str, str]]) -> list[str]: + errors = [] + missing = [name for name in expected if name not in selected] + unexpected = [name for name in selected if name not in expected] + if missing: + errors.append(f"ctest did not select expected suite(s): {', '.join(missing)}") + if unexpected: + errors.append(f"ctest selected suite(s) that are not expected: {', '.join(unexpected)}") + for name in expected: + outcome = outcomes.get(name, ("missing from the JUnit results", ""))[0] + if outcome != PASSED: + errors.append(f"{name}: {outcome}") + return errors + + +def summary(expected: list[str], outcomes: dict[str, tuple[str, str]], errors: list[str]) -> str: + lines = ["| suite | result | time (s) |", "| --- | --- | --- |"] + for name in expected: + outcome, seconds = outcomes.get(name, ("missing", "")) + lines.append(f"| {name} | {outcome} | {seconds} |") + lines.append("") + if errors: + lines.append("Unit-test evidence check failed:") + lines.extend(f"- {error}" for error in errors) + else: + lines.append(f"All {len(expected)} expected suites were selected, executed and passed; none was skipped.") + return "\n".join(lines) + "\n" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--expected", required=True, help="space-separated suite names (CTEST_EXPECTED_SUITES)") + parser.add_argument("--listing", required=True, type=Path, help="ctest --show-only=json-v1 output") + parser.add_argument("--junit", required=True, type=Path, help="ctest --output-junit file") + parser.add_argument("--summary", type=Path, help="write a Markdown summary here") + args = parser.parse_args(argv) + + try: + expected = parse_expected(args.expected) + except ValueError as e: + print(f"Error: CTEST_EXPECTED_SUITES: {e}", file=sys.stderr) + return 1 + errors: list[str] = [] + try: + selected = selected_tests(args.listing.read_text(encoding="utf8")) + except (OSError, ValueError, KeyError, TypeError) as e: + selected = [] + errors.append(f"unreadable ctest test listing {args.listing}: {e}") + try: + outcomes = junit_outcomes(args.junit.read_text(encoding="utf8")) + except (OSError, ET.ParseError) as e: + outcomes = {} + errors.append(f"unreadable JUnit results {args.junit}: {e}") + errors.extend(check(expected, selected, outcomes)) + + text = summary(expected, outcomes, errors) + if args.summary is not None: + args.summary.write_text(text, encoding="utf8") + print(text, end="") + for error in errors: + print(f"Error: {error}", file=sys.stderr) + return 1 if errors else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/checks/test_ci_pq_contract.py b/ci/checks/test_ci_pq_contract.py new file mode 100755 index 0000000000..c54bbe596c --- /dev/null +++ b/ci/checks/test_ci_pq_contract.py @@ -0,0 +1,1006 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026-present The qbit core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. +"""Executable contract for the PQ CI entry points (#184 part 2, work item C1). + +``ci-pq.yml`` and ``ci-nightly-heavy.yml`` are reusable and job-selectable, and +the PQ unit job proves which suites ran. These tests read the real workflow +files and scripts and execute their inline steps and shell blocks: + +* job selection: the ``jobs`` input of a call or dispatch becomes the matrix, + and only the selected jobs pass their ``if``. On-demand PQ runs reach + ``ci-pq.yml`` through main's caller (C4), which forwards ``source_ref`` and + ``jobs``; GitHub dispatches only workflows whose file is on the default + branch, so ``gh workflow run ci-pq.yml --ref `` is not the path; +* an unknown, repeated or empty job name fails the run; +* the source is resolved once to a full commit (branch, peeled tag, commit or + the triggering commit), every job checks out and verifies exactly that + commit, and a scheduled call from main still validates the maintained branch; +* a called nightly run gets its caller's github context, so it is recognized by + its ``source_ref`` input: it validates that source (resolved by the same + function as ``ci-pq.yml``) and honors its inputs whatever the caller's event, + while direct runs keep their source, the repository variables of a schedule + and their concurrency groups; callers for different sources never share one; +* suites really ran: ``CI_BUILD_TARGET`` and ``CTEST_REGEX`` reach the build and + ctest commands of ``ci/test/03_test_script.sh``, and ``ctest_evidence.py`` + fails a skipped, failed, missing or unselected expected suite. + +ctest and cmake are replaced by recording stubs; GitHub itself is not called. +""" + +from __future__ import annotations + +import contextlib +import copy +import io +import json +import os +import re +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from typing import Any + +import ctest_evidence +import test_scheduled_validation_contract as svc + +REPO_ROOT = svc.REPO_ROOT +PQ = "pq" +NIGHTLY = "nightly" +WORKFLOWS = { + PQ: svc.WORKFLOW_DIR / "ci-pq.yml", + NIGHTLY: svc.WORKFLOW_DIR / "ci-nightly-heavy.yml", +} +GATE_WORKFLOW = svc.WORKFLOW_DIR / "required-merge-gate.yml" +TEST_SCRIPT = REPO_ROOT / "ci" / "test" / "03_test_script.sh" +RESOLVER_JOB = svc.RESOLVER_JOB +TRIGGER_STEP = "Check how the run started" +REQUESTED_STEP = "Resolve requested source" +SHARED_RESOLVER = "resolve_source_ref" +SELECT_STEP = "Select jobs" +REPORT_STEP = "Report unit-test evidence" +PQ_UNIT_JOB = "unit" +AARCH64_JOB = "aarch64-unit" +AARCH64_SUITES = ["bip324_tests", "net_tests"] +NIGHTLY_MATRIX_JOBS = ["arm32-unit-1", "arm32-unit-2", "previous-releases", "fuzz"] +NIGHTLY_ALL_JOBS: dict[str, list[str]] = {"nightly-matrix": NIGHTLY_MATRIX_JOBS, "scanner-readiness": []} +NIGHTLY_VARS_OFF = {"CI_NIGHTLY_HEAVY_RUN_TEST_MATRIX": "false", "CI_NIGHTLY_HEAVY_RUN_SCANNERS": "false"} +SCANNER_SETTINGS = ("SCANNER_SET", "SCANNER_FAIL_POLICY", "SKIP_BUILD_SCANNERS", "SCANNER_MODE") +CALLER_EVENTS = ("schedule", "workflow_dispatch") +CALLER_WORKFLOW = "ci-maintained-heavy.yml" # main's heavy caller (C4) +MATRIX_EXPRESSION = "${{ fromJSON(needs." + RESOLVER_JOB + ".outputs.matrix) }}" +ANNOTATED_TAG = "v9.9.9-annotated" +AMBIGUOUS_NAME = "dup" +SUITE_INVENTORY = ["addrman_tests", "bip324_tests", "bip324_tests_extra", "net_tests", "netbase_tests", "util_tests"] + + +# --------------------------------------------------------------------------- +# Workflow helpers +# --------------------------------------------------------------------------- + + +def run_python_step(fixture: svc.Fixture, step: dict[str, Any], env: dict[str, str], cwd: Path) -> svc.StepResult: + """Execute a ``shell: python3 {0}`` step the way the runner does.""" + assert step.get("shell") == "python3 {0}", step.get("shell") + with tempfile.TemporaryDirectory(prefix="step-", dir=fixture.root) as tmp: + script = Path(tmp) / "step.py" + script.write_text(step["run"], encoding="utf8") + output_path = Path(tmp) / "github-output" + summary_path = Path(tmp) / "step-summary" + output_path.touch() + summary_path.touch() + full_env = fixture.base_env() + full_env.update(env) + full_env["GITHUB_OUTPUT"] = str(output_path) + full_env["GITHUB_STEP_SUMMARY"] = str(summary_path) + completed = subprocess.run([sys.executable, str(script)], cwd=cwd, env=full_env, text=True, capture_output=True) + outputs = svc.parse_key_values(output_path.read_text(encoding="utf8")) + summary = summary_path.read_text(encoding="utf8") + return svc.StepResult(completed, outputs, summary) + + +def resolver_outputs(workflow: dict[str, Any], steps: dict[str, dict[str, str]]) -> dict[str, str]: + """Evaluate the resolver job's ``outputs`` from the outputs of the steps that ran, by step id.""" + context = {"steps": {step_id: {"outputs": outputs} for step_id, outputs in steps.items()}} + job = workflow["jobs"][RESOLVER_JOB] + return {name: svc._to_text(svc.render(value, context)) for name, value in job["outputs"].items()} + + +def call_inputs(workflow: dict[str, Any]) -> dict[str, Any]: + """A called run's inputs: GitHub gives every input the caller does not pass its default.""" + declared = workflow["on"]["workflow_call"].get("inputs") or {} + return {name: spec.get("default") for name, spec in declared.items()} + + +def workflow_ref(github: dict[str, Any], file: str, ref: str) -> str: + """A workflow_ref as GitHub writes it: /.github/workflows/@.""" + return f"{github['repository']}/.github/workflows/{file}@{ref}" + + +def shell_function(script: str, name: str) -> str: + """The definition of a top-level shell function in a step script.""" + lines = script.splitlines() + first = next(index for index, line in enumerate(lines) if line.startswith(f"{name}() {{")) + last = next(index for index in range(first, len(lines)) if lines[index] == "}") + return "\n".join(lines[first:last + 1]) + + +def catalog(workflow: dict[str, Any]) -> list[dict[str, Any]]: + step = svc.find_step(workflow["jobs"][RESOLVER_JOB], SELECT_STEP) + return json.loads(step["env"]["JOB_CATALOG"]) + + +def running_jobs(workflow: dict[str, Any], context: dict[str, Any]) -> dict[str, list[str]]: + """Jobs whose ``if`` passes, with the matrix entries each one would run.""" + outputs = context["needs"][RESOLVER_JOB]["outputs"] + running: dict[str, list[str]] = {} + for job_id, job in workflow["jobs"].items(): + if job_id == RESOLVER_JOB: + continue + condition = job.get("if") + if condition is not None and not svc._truthy(svc.render(condition, context)): + continue + matrix = (job.get("strategy") or {}).get("matrix") + if matrix is None: + running[job_id] = [] + continue + assert matrix == MATRIX_EXPRESSION, f"{job_id} matrix must come from the resolver: {matrix!r}" + running[job_id] = [entry["job"] for entry in json.loads(outputs["matrix"])["include"]] + return running + + +def source_env(env_file: str, names: list[str], extra_env: dict[str, str] | None = None) -> dict[str, str]: + """Source a ci/test env file in bash and return the effective values of names.""" + script = f"set -e\nsource {env_file}\n" + "".join(f'printf "%s=%s\\n" {name} "${{{name}:-}}"\n' for name in names) + env = {"PATH": os.environ["PATH"], "HOME": os.environ.get("HOME", "/tmp")} + env.update(extra_env or {}) + completed = subprocess.run(["bash", "-c", script], cwd=REPO_ROOT, env=env, text=True, capture_output=True, check=True) + return svc.parse_key_values(completed.stdout) + + +def shell_block(text: str, start: str, end_line: str) -> str: + """The lines of text from the line starting with start through the next line equal to end_line.""" + lines = text.splitlines() + first = next(index for index, line in enumerate(lines) if line.startswith(start)) + last = next(index for index in range(first + 1, len(lines)) if lines[index] == end_line) + return "\n".join(lines[first:last + 1]) + "\n" + + +STUB_CTEST = r'''#!/usr/bin/env python3 +import json, os, re, sys +args = sys.argv[1:] +with open(os.environ["STUB_LOG"], "a", encoding="utf8") as log: + log.write(json.dumps(["ctest"] + args) + "\n") +selected = os.environ["STUB_INVENTORY"].split() +junit = None +for flag, value in zip(args, args[1:]): + if flag == "-I": + start, end = (int(part) for part in value.split(",")) + selected = selected[start - 1:end] + elif flag == "-R": + selected = [name for name in selected if re.search(value, name)] + elif flag == "--output-junit": + junit = value +if "--show-only=json-v1" in args: + print(json.dumps({"kind": "ctestInfo", "version": {"major": 1, "minor": 0}, "tests": [{"name": n} for n in selected]})) + sys.exit(0) +if "--show-only" in args: + print("\n".join(selected)) + sys.exit(0) +if not selected and "--no-tests=error" in args: + print("No tests were found!!!") + sys.exit(8) +outcomes = json.loads(os.environ.get("STUB_OUTCOMES") or "{}") +cases = [] +for name in selected: + status = outcomes.get(name, "run") + body = {"notrun": '', "fail": ''}.get(status, "") + cases.append(f'{body}') +if junit: + with open(junit, "w", encoding="utf8") as out: + out.write(f'\n' + "".join(cases) + "\n") +sys.exit(8 if "fail" in outcomes.values() else 0) +''' + +STUB_CMAKE = r'''#!/usr/bin/env python3 +import json, os, sys +with open(os.environ["STUB_LOG"], "a", encoding="utf8") as log: + log.write(json.dumps(["cmake"] + sys.argv[1:]) + "\n") +counter = os.environ["STUB_LOG"] + ".cmake-calls" +calls = int(open(counter).read()) if os.path.exists(counter) else 0 +open(counter, "w").write(str(calls + 1)) +sys.exit(1 if os.environ.get("STUB_FAIL_FIRST") == "1" and calls == 0 else 0) +''' + + +class ScriptRun: + def __init__(self, completed: subprocess.CompletedProcess[str], calls: list[list[str]], files: dict[str, str]) -> None: + self.completed = completed + self.calls = calls + self.files = files # ctest-evidence/ -> contents + + @property + def returncode(self) -> int: + return self.completed.returncode + + +def run_test_script_block(block: str, env: dict[str, str]) -> ScriptRun: + """Run a block of 03_test_script.sh under its own shell options with stub cmake and ctest.""" + with tempfile.TemporaryDirectory(prefix="ci-test-script-") as tmp: + root = Path(tmp) + stubs = root / "stubs" + stubs.mkdir() + for name, body in (("ctest", STUB_CTEST), ("cmake", STUB_CMAKE)): + (stubs / name).write_text(body, encoding="utf8") + (stubs / name).chmod(0o755) + build = root / "build" + build.mkdir() + log = root / "calls.jsonl" + log.touch() + full_env = { + "PATH": f"{stubs}:{os.environ['PATH']}", + "HOME": str(root), + "LC_ALL": "C", + "BASE_ROOT_DIR": str(REPO_ROOT), + "BASE_BUILD_DIR": str(build), + "DEPENDS_DIR": str(root / "depends"), + "HOST": "x86_64-pc-linux-gnu", + "MAKEJOBS": "-j2", + "CTEST_JOBS": "2", + "TEST_RUNNER_TIMEOUT_FACTOR": "1", + "RUN_UNIT_TESTS": "true", + "DIR_UNIT_TEST_DATA": "", + "GOAL": "install", + "STUB_LOG": str(log), + "STUB_INVENTORY": " ".join(SUITE_INVENTORY), + } + full_env.update(env) + script = root / "block.sh" + script.write_text("set -ex\n" + block, encoding="utf8") + completed = subprocess.run(["bash", str(script)], env=full_env, text=True, capture_output=True) + calls = [json.loads(line) for line in log.read_text(encoding="utf8").splitlines()] + evidence = build / "ctest-evidence" + files = {path.name: path.read_text(encoding="utf8") for path in evidence.glob("*")} if evidence.is_dir() else {} + return ScriptRun(completed, calls, files) + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class PQWorkflowContractTest(unittest.TestCase): + fixture: svc.Fixture + tag_object: str + workflows: dict[str, dict[str, Any]] + workflow_text: dict[str, str] + + @classmethod + def setUpClass(cls) -> None: + cls.fixture = svc.Fixture() + fixture = cls.fixture + # An annotated tag and a name that is both a branch and a tag. + fixture.git("tag", "-a", ANNOTATED_TAG, "-m", "annotated", fixture.maintained_sha, cwd=fixture.work) + fixture.git("tag", AMBIGUOUS_NAME, fixture.manual_sha, cwd=fixture.work) + fixture.git("push", "-q", "origin", f"refs/tags/{ANNOTATED_TAG}", f"refs/tags/{AMBIGUOUS_NAME}", cwd=fixture.work) + fixture.git("push", "-q", "origin", f"{fixture.default_sha}:refs/heads/{AMBIGUOUS_NAME}", cwd=fixture.work) + cls.tag_object = fixture.git("rev-parse", ANNOTATED_TAG, cwd=fixture.work).strip() + cls.workflows = {key: svc.load_workflow(path) for key, path in WORKFLOWS.items()} + cls.workflow_text = {key: path.read_text(encoding="utf8") for key, path in WORKFLOWS.items()} + + @classmethod + def tearDownClass(cls) -> None: + cls.fixture.close() + + def tearDown(self) -> None: + for key, path in WORKFLOWS.items(): + self.assertEqual(path.read_text(encoding="utf8"), self.workflow_text[key], f"{path} was modified") + + # -- contexts and step runs ------------------------------------------------ + + def direct(self, key: str, context: dict[str, Any]) -> dict[str, Any]: + """A direct run: the job is defined in the workflow file that runs, so job.workflow_ref is github.workflow_ref.""" + github = dict(context["github"]) + github["workflow_ref"] = workflow_ref(github, WORKFLOWS[key].name, github["ref"]) + return dict(context, github=github, job={"status": "success", "workflow_ref": github["workflow_ref"]}) + + def dispatch(self, key: str, **inputs: Any) -> dict[str, Any]: + declared = svc.dispatch_inputs(self.workflows[key]) + unknown = set(inputs) - set(declared) + self.assertFalse(unknown, f"{key} has no workflow_dispatch input(s) {unknown}") + declared.update(inputs) + return self.direct(key, svc.dispatch_context(self.fixture, f"refs/heads/{self.fixture.manual_branch}", self.fixture.manual_sha, declared)) + + def scheduled(self, key: str, **context: Any) -> dict[str, Any]: + """A direct scheduled run on main.""" + return self.direct(key, dict(svc.schedule_context(self.fixture), **context)) + + def call(self, key: str, caller_event: str, *, variables: dict[str, str] | None = None, remote: Path | None = None, + caller_workflow: str = CALLER_WORKFLOW, **inputs: Any) -> dict[str, Any]: + """A run called by main's caller: the github context is the caller's event and + workflow on main, job.workflow_ref is this file at 1.x.x, and every input the + caller does not pass has its default.""" + declared = call_inputs(self.workflows[key]) + unknown = set(inputs) - set(declared) + self.assertFalse(unknown, f"{key} has no workflow_call input(s) {unknown}") + declared.update(inputs) + github = self.fixture.github_context(caller_event, "refs/heads/main", self.fixture.default_sha, remote) + github["workflow_ref"] = workflow_ref(github, caller_workflow, "refs/heads/main") + job = {"status": "success", "workflow_ref": workflow_ref(github, WORKFLOWS[key].name, svc.MAINTAINED_REF)} + return {"github": github, "inputs": declared, "needs": {}, "matrix": {}, "vars": dict(variables or {}), "steps": {}, "job": job} + + def run_resolver_step(self, key: str, name: str, context: dict[str, Any]) -> svc.StepResult: + """Run one bash step of the resolver job, whatever its if.""" + workflow = self.workflows[key] + job = workflow["jobs"][RESOLVER_JOB] + step = svc.find_step(job, name) + env = svc.step_env(workflow, job, step, context) + return svc.run_step(self.fixture, step, env, self.fixture.root, svc.github_env_for(context, RESOLVER_JOB, workflow["name"])) + + def run_resolver(self, key: str, context: dict[str, Any]) -> svc.StepResult: + return self.run_resolver_step(key, svc.RESOLVER_STEP, context) + + def run_resolver_job(self, key: str, context: dict[str, Any]) -> tuple[list[str], dict[str, svc.StepResult]]: + """Run the resolver job's steps the way the runner does: a step whose if is + false is skipped, and a failed step ends the job. Returns the names of the + steps that ran and their results by step id.""" + workflow = self.workflows[key] + job = workflow["jobs"][RESOLVER_JOB] + ran: list[str] = [] + results: dict[str, svc.StepResult] = {} + step_context = dict(context, steps={}) + for step in job["steps"]: + condition = step.get("if") + if condition is not None and not svc._truthy(svc.render(condition, step_context)): + continue + env = svc.step_env(workflow, job, step, step_context) + if step.get("shell") == "python3 {0}": + result = run_python_step(self.fixture, step, env, self.fixture.root) + else: + github_env = svc.github_env_for(context, RESOLVER_JOB, workflow["name"]) + result = svc.run_step(self.fixture, step, env, self.fixture.root, github_env) + ran.append(step["name"]) + results[step["id"]] = result + if result.returncode != 0: + break + step_context["steps"] = dict(step_context["steps"], **{step["id"]: {"outputs": result.outputs}}) + return ran, results + + def run_select(self, key: str, context: dict[str, Any]) -> svc.StepResult: + workflow = self.workflows[key] + job = workflow["jobs"][RESOLVER_JOB] + step = svc.find_step(job, SELECT_STEP) + return run_python_step(self.fixture, step, svc.step_env(workflow, job, step, context), self.fixture.root) + + def resolve(self, key: str, context: dict[str, Any]) -> dict[str, Any]: + """Run the resolver job; return the context with its outputs as ``needs``.""" + _ran, results = self.run_resolver_job(key, context) + for result in results.values(): + self.assertEqual(result.returncode, 0, result.stderr) + steps = {step_id: result.outputs for step_id, result in results.items()} + return svc.with_needs(context, resolver_outputs(self.workflows[key], steps)) + + def nightly_run(self, context: dict[str, Any]) -> dict[str, Any]: + """What a nightly run does with a context: the resolver step that ran, the + source, the jobs that run, the scanner settings and the concurrency group.""" + workflow = self.workflows[NIGHTLY] + ran, results = self.run_resolver_job(NIGHTLY, context) + for result in results.values(): + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(ran[-1], SELECT_STEP) + context = svc.with_needs(context, resolver_outputs(workflow, {step_id: result.outputs for step_id, result in results.items()})) + outputs = context["needs"][RESOLVER_JOB]["outputs"] + scanner_env = svc.job_env(workflow, workflow["jobs"]["scanner-readiness"], context) + self.assertEqual(ran[0], TRIGGER_STEP) + return { + "resolvers": ran[1:-1], + "source": (outputs["source_mode"], outputs["requested_ref"], outputs["resolved_sha"]), + "jobs": running_jobs(workflow, context), + "scanner": {name: scanner_env[name] for name in SCANNER_SETTINGS}, + "group": svc.render(workflow["concurrency"]["group"], context), + "summary": "".join(result.summary for result in results.values()), + "context": context, + } + + # -- entry points ------------------------------------------------------------ + + def test_entry_points(self) -> None: + pq = self.workflows[PQ] + for trigger in ("workflow_call", "workflow_dispatch"): + with self.subTest(workflow=PQ, trigger=trigger): + inputs = pq["on"][trigger]["inputs"] + self.assertEqual(set(inputs), {"source_ref", "jobs"}) + for spec in inputs.values(): + self.assertEqual((spec["type"], spec["default"], spec["required"]), ("string", "", False)) + + nightly = self.workflows[NIGHTLY] + self.assertIn("schedule", nightly["on"]) + call = nightly["on"]["workflow_call"]["inputs"] + dispatch = nightly["on"]["workflow_dispatch"]["inputs"] + # Only a call names its source, so a direct dispatch validates the ref it was dispatched on. + self.assertNotIn("source_ref", dispatch) + source = call["source_ref"] + self.assertEqual((source["type"], source["default"], source["required"]), ("string", svc.MAINTAINED_BRANCH, False)) + self.assertIn("must not be empty", source["description"]) + self.assertEqual([name for name in call if name != "source_ref"], list(dispatch), + "the other workflow_call inputs must mirror workflow_dispatch") + self.assertIn("jobs", call) + for name, spec in dispatch.items(): + with self.subTest(workflow=NIGHTLY, input=name): + mirrored = call[name] + expected_type = "string" if spec["type"] == "choice" else spec["type"] + self.assertEqual(mirrored["type"], expected_type) + self.assertEqual(mirrored["default"], spec["default"]) + self.assertEqual(mirrored["required"], spec["required"]) + if spec["type"] == "choice": + self.assertIn(spec["default"], spec["options"]) + # The scanner's secret reaches a called run. + self.assertEqual(set(nightly["on"]["workflow_call"]["secrets"]), {"LIBBITCOINPQC_READ_TOKEN"}) + self.assertIn("secrets.LIBBITCOINPQC_READ_TOKEN", self.workflow_text[NIGHTLY]) + + def test_selection_scripts_are_identical(self) -> None: + scripts = {key: svc.find_step(workflow["jobs"][RESOLVER_JOB], SELECT_STEP)["run"] for key, workflow in self.workflows.items()} + self.assertEqual(scripts[PQ], scripts[NIGHTLY]) + for key, workflow in self.workflows.items(): + resolver = workflow["jobs"][RESOLVER_JOB] + self.assertFalse(any("uses" in step for step in resolver["steps"]), f"{key}: selection must run inline") + names = [entry["job"] for entry in catalog(workflow)] + self.assertEqual(len(names), len(set(names)), key) + + # -- named-dispatch selection -------------------------------------------------- + + def test_named_dispatch_runs_only_selected_jobs(self) -> None: + cases: list[tuple[str, dict[str, Any], dict[str, list[str]]]] = [ + (PQ, {"jobs": AARCH64_JOB}, {PQ_UNIT_JOB: [AARCH64_JOB]}), + (PQ, {"jobs": ""}, {PQ_UNIT_JOB: [AARCH64_JOB]}), + (NIGHTLY, {"run_test_matrix": True, "run_scanners": False, "jobs": "previous-releases"}, + {"nightly-matrix": ["previous-releases"]}), + (NIGHTLY, {"jobs": ""}, {"nightly-matrix": NIGHTLY_MATRIX_JOBS, "scanner-readiness": []}), + (NIGHTLY, {"jobs": " fuzz , previous-releases "}, {"nightly-matrix": ["previous-releases", "fuzz"]}), + (NIGHTLY, {"jobs": "scanner"}, {"scanner-readiness": []}), + (NIGHTLY, {"run_test_matrix": False, "jobs": "previous-releases"}, {}), + ] + for key, inputs, expected in cases: + with self.subTest(workflow=key, inputs=inputs): + context = self.resolve(key, self.dispatch(key, **inputs)) + self.assertEqual(running_jobs(self.workflows[key], context), expected) + + # The nightly run keeps its entries' settings, such as the shard ranges and the fuzz mutation time. + entries = {entry["job"]: entry for entry in catalog(self.workflows[NIGHTLY])} + self.assertEqual(entries["arm32-unit-2"]["ctest_include_range"], "140,99999") + self.assertEqual(entries["fuzz"]["fuzz-mutate-min-time"], "120") + + def test_scheduled_call_from_main_runs_everything_on_the_maintained_branch(self) -> None: + context = self.resolve(NIGHTLY, self.call(NIGHTLY, "schedule")) + outputs = context["needs"][RESOLVER_JOB]["outputs"] + self.assertEqual(outputs["resolved_sha"], self.fixture.maintained_sha) + self.assertEqual(outputs["requested_ref"], svc.MAINTAINED_REF) + self.assertEqual(running_jobs(self.workflows[NIGHTLY], context), + {"nightly-matrix": NIGHTLY_MATRIX_JOBS, "scanner-readiness": []}) + refs = svc.check_source_wiring(self.workflows[NIGHTLY], context) + self.assertEqual(set(refs.values()), {self.fixture.maintained_sha}) + + # The PQ workflow is told which branch to validate. + context = self.resolve(PQ, self.call(PQ, "schedule", source_ref=svc.MAINTAINED_BRANCH)) + outputs = context["needs"][RESOLVER_JOB]["outputs"] + self.assertEqual(outputs["resolved_sha"], self.fixture.maintained_sha) + self.assertNotEqual(outputs["resolved_sha"], context["github"]["sha"]) + self.assertEqual(running_jobs(self.workflows[PQ], context), {PQ_UNIT_JOB: [AARCH64_JOB]}) + + # -- called and direct nightly runs --------------------------------------------- + + def test_called_nightly_run_validates_source_ref_whatever_the_caller(self) -> None: + fixture = self.fixture + workflow = self.workflows[NIGHTLY] + defaults = {"SCANNER_SET": "minimum", "SCANNER_FAIL_POLICY": "infra-and-secrets", "SKIP_BUILD_SCANNERS": "false"} + for event in CALLER_EVENTS: + mode = "nightly" if event == "schedule" else "manual_dry_run" + with self.subTest(caller=event): + # The repository variables that switch a direct scheduled run off do not apply to a called one. + run = self.nightly_run(self.call(NIGHTLY, event, variables=NIGHTLY_VARS_OFF)) + github = run["context"]["github"] + self.assertEqual((github["ref"], github["sha"]), ("refs/heads/main", fixture.default_sha)) + self.assertEqual(run["resolvers"], [REQUESTED_STEP]) + self.assertEqual(run["source"], ("maintained", svc.MAINTAINED_REF, fixture.maintained_sha)) + self.assertEqual(run["jobs"], NIGHTLY_ALL_JOBS) + self.assertEqual(run["scanner"], dict(defaults, SCANNER_MODE=mode)) + for row in ("| source_ref input | 1.x.x |", "| source_mode | maintained |", f"| caller_event | {event} |", + f"| resolved_sha | {fixture.maintained_sha} |"): + self.assertIn(row, run["summary"]) + refs = svc.check_source_wiring(workflow, run["context"]) + self.assertEqual(set(refs.values()), {fixture.maintained_sha}) + fixture.checkout_work(fixture.maintained_sha) + for job_id in sorted(refs): + job_context = svc.matrix_context(workflow, job_id, run["context"]) + verify = svc.run_job_step(fixture, workflow, job_id, svc.VERIFY_STEP, job_context, fixture.work) + self.assertEqual(verify.returncode, 0, verify.stderr) + self.assertIn("- source_ref input: 1.x.x\n- source_mode: maintained\n", verify.summary) + + # The default source is looked up on the remote, never taken from the caller. + ran, results = self.run_resolver_job(NIGHTLY, self.call(NIGHTLY, event, remote=fixture.remote_without_maintained)) + self.assertEqual(ran, [TRIGGER_STEP, REQUESTED_STEP]) + self.assertNotEqual(results["requested"].returncode, 0) + self.assertIn(f"::error::no branch or tag '{svc.MAINTAINED_BRANCH}'", results["requested"].stderr) + + # Every input is honored. + honored: list[tuple[dict[str, Any], dict[str, list[str]], dict[str, str]]] = [ + ({"run_test_matrix": False}, {"scanner-readiness": []}, {}), + ({"run_scanners": False}, {"nightly-matrix": NIGHTLY_MATRIX_JOBS}, {}), + ({"run_test_matrix": False, "run_scanners": False}, {}, {}), + ({"jobs": "fuzz"}, {"nightly-matrix": ["fuzz"]}, {}), + ({"scanner_set": "secrets-only", "scanner_fail_policy": "never", "skip_build_scanners": True}, NIGHTLY_ALL_JOBS, + {"SCANNER_SET": "secrets-only", "SCANNER_FAIL_POLICY": "never", "SKIP_BUILD_SCANNERS": "true"}), + ] + for inputs, jobs, scanner in honored: + with self.subTest(caller=event, inputs=inputs): + run = self.nightly_run(self.call(NIGHTLY, event, **inputs)) + self.assertEqual(run["source"][2], fixture.maintained_sha) + self.assertEqual(run["jobs"], jobs) + self.assertEqual(run["scanner"], dict(defaults, SCANNER_MODE=mode, **scanner)) + + def test_called_nightly_run_resolves_each_kind_of_source(self) -> None: + fixture = self.fixture + manual_ref = f"refs/heads/{fixture.manual_branch}" + cases = [ + (fixture.manual_branch, manual_ref, fixture.manual_sha, "requested"), + (manual_ref, manual_ref, fixture.manual_sha, "requested"), + (fixture.manual_tag, f"refs/tags/{fixture.manual_tag}", fixture.manual_sha, "requested"), + (ANNOTATED_TAG, f"refs/tags/{ANNOTATED_TAG}", fixture.maintained_sha, "requested"), + (fixture.manual_sha, fixture.manual_sha, fixture.manual_sha, "requested"), + # A hash never names the maintained branch, even at its tip. + (fixture.maintained_sha, fixture.maintained_sha, fixture.maintained_sha, "requested"), + (svc.MAINTAINED_REF, svc.MAINTAINED_REF, fixture.maintained_sha, "maintained"), + (f" {svc.MAINTAINED_BRANCH} ", svc.MAINTAINED_REF, fixture.maintained_sha, "maintained"), + ] + # A dispatched caller: github.ref and github.sha are main's. + for source_ref, requested_ref, resolved_sha, mode in cases: + with self.subTest(source_ref=source_ref): + run = self.nightly_run(self.call(NIGHTLY, "workflow_dispatch", source_ref=source_ref)) + self.assertEqual(run["resolvers"], [REQUESTED_STEP]) + self.assertEqual(run["source"], (mode, requested_ref, resolved_sha)) + self.assertIn(f"| source_ref input | {source_ref.strip()} |", run["summary"]) + self.assertIn(f"| source_mode | {mode} |", run["summary"]) + self.assertEqual(set(svc.check_source_wiring(self.workflows[NIGHTLY], run["context"]).values()), {resolved_sha}) + + rejected = { + "no-such-branch": "::error::no branch or tag 'no-such-branch'", + fixture.default_sha[:12]: "::error::no branch or tag", + AMBIGUOUS_NAME: f"::error::source_ref '{AMBIGUOUS_NAME}' must name exactly one branch or tag", + "refs/pull/1/head": "::error::source_ref 'refs/pull/1/head' must be a branch, a tag or a full commit hash", + "bad..name": "::error::source_ref 'bad..name' is not a valid ref name", + " ": "::error::source_ref is empty", + } + for source_ref, message in rejected.items(): + with self.subTest(rejected=source_ref): + ran, results = self.run_resolver_job(NIGHTLY, self.call(NIGHTLY, "workflow_dispatch", source_ref=source_ref)) + self.assertEqual(ran, [TRIGGER_STEP, REQUESTED_STEP], "a failed resolution ends the job") + self.assertNotEqual(results["requested"].returncode, 0) + self.assertIn(message, results["requested"].stderr) + self.assertNotIn("resolved_sha", results["requested"].outputs) + + def test_nightly_run_cross_checks_how_it_started(self) -> None: + fixture = self.fixture + direct = self.dispatch(NIGHTLY) + cases: list[tuple[str, dict[str, Any], list[str], str]] = [] + for event in CALLER_EVENTS: + cases += [ + (f"called by a {event} caller with a source_ref", self.call(NIGHTLY, event), + [TRIGGER_STEP, REQUESTED_STEP, SELECT_STEP], "| cross-check | passed: called |"), + # Only @ref tells main's own ci-nightly-heavy.yml calling this copy from a direct run. + (f"called by main's own copy on {event}", self.call(NIGHTLY, event, caller_workflow=WORKFLOWS[NIGHTLY].name), + [TRIGGER_STEP, REQUESTED_STEP, SELECT_STEP], "| cross-check | passed: called |"), + (f"called by a {event} caller with an empty source_ref", self.call(NIGHTLY, event, source_ref=""), + [TRIGGER_STEP], "with an empty source_ref; the caller must pass the branch, tag or commit to validate"), + ] + cases += [ + ("direct without a source_ref", direct, [TRIGGER_STEP, svc.RESOLVER_STEP, SELECT_STEP], "| cross-check | passed: direct |"), + ("direct schedule without a source_ref", self.scheduled(NIGHTLY), [TRIGGER_STEP, svc.RESOLVER_STEP, SELECT_STEP], + "| cross-check | passed: direct |"), + ("direct with a source_ref", dict(direct, inputs=dict(direct["inputs"], source_ref=svc.MAINTAINED_BRANCH)), + [TRIGGER_STEP], f"source_ref '{svc.MAINTAINED_BRANCH}' is set, but this run was not called"), + ] + for label, context, steps, expected in cases: + with self.subTest(label): + ran, results = self.run_resolver_job(NIGHTLY, context) + self.assertEqual(ran, steps) + check = results["trigger"] + if steps == [TRIGGER_STEP]: + self.assertNotEqual(check.returncode, 0) + self.assertIn("::error::", check.stderr) + self.assertIn(expected, check.stderr) + self.assertNotIn(fixture.default_sha, json.dumps({key: result.outputs for key, result in results.items()})) + else: + self.assertEqual(check.returncode, 0, check.stderr) + self.assertIn(expected, check.summary) + self.assertIn(f"| job.workflow_ref | {context['job']['workflow_ref']} |", check.summary) + + # Without job.workflow_ref the check is skipped and says so, and the + # source_ref rule alone decides: then a caller's empty source_ref is a + # direct run on main's ref, which is the gap the check closes. + for label, context, source in ( + ("called with a source_ref", self.call(NIGHTLY, "workflow_dispatch"), + ("maintained", svc.MAINTAINED_REF, fixture.maintained_sha)), + ("direct", self.dispatch(NIGHTLY), ("manual", f"refs/heads/{fixture.manual_branch}", fixture.manual_sha)), + ("called with an empty source_ref", self.call(NIGHTLY, "workflow_dispatch", source_ref=""), + ("manual", "refs/heads/main", fixture.default_sha)), + ): + for job in ({"status": "success"}, None): + with self.subTest(f"{label}, job.workflow_ref unavailable", job=job): + run = self.nightly_run(dict(context, job=job)) + self.assertEqual(run["source"], source) + self.assertIn("| job.workflow_ref | (not available) |", run["summary"]) + self.assertIn("| cross-check | skipped: job.workflow_ref is not available |", run["summary"]) + + def test_called_nightly_and_pq_resolve_source_ref_alike(self) -> None: + nightly_step = svc.find_step(self.workflows[NIGHTLY]["jobs"][RESOLVER_JOB], REQUESTED_STEP) + pq_step = svc.find_step(self.workflows[PQ]["jobs"][RESOLVER_JOB], svc.RESOLVER_STEP) + self.assertEqual(shell_function(nightly_step["run"], SHARED_RESOLVER), shell_function(pq_step["run"], SHARED_RESOLVER)) + # The caller's ref and commit never reach the called run's resolver. + for name, value in nightly_step["env"].items(): + self.assertNotRegex(str(value), r"github\.(ref|sha)\b", name) + self.assertNotRegex(nightly_step["run"], r"GITHUB_(REF|SHA)\b") + sources = [svc.MAINTAINED_BRANCH, f"refs/tags/{self.fixture.manual_tag}", ANNOTATED_TAG, self.fixture.manual_sha, + "no-such-branch", AMBIGUOUS_NAME, "refs/pull/1/head", "bad..name", self.fixture.manual_sha[:12]] + for source_ref in sources: + with self.subTest(source_ref=source_ref): + pq = self.run_resolver(PQ, self.dispatch(PQ, source_ref=source_ref)) + nightly = self.run_resolver_step(NIGHTLY, REQUESTED_STEP, self.call(NIGHTLY, "workflow_dispatch", source_ref=source_ref)) + self.assertEqual(pq.returncode == 0, nightly.returncode == 0, pq.stderr + nightly.stderr) + if pq.returncode == 0: + for output in ("requested_ref", "resolved_sha"): + self.assertEqual(pq.outputs[output], nightly.outputs[output]) + else: + self.assertEqual(pq.stderr.strip().splitlines()[-1], nightly.stderr.strip().splitlines()[-1]) + + def test_direct_nightly_runs_keep_their_source_and_settings(self) -> None: + fixture = self.fixture + # A direct scheduled run: the maintained branch, the repository variables and the scheduled defaults. + scheduled = {"SCANNER_SET": "minimum", "SCANNER_FAIL_POLICY": "infra-and-secrets", "SKIP_BUILD_SCANNERS": "true", + "SCANNER_MODE": "nightly"} + scheduled_cases: tuple[tuple[dict[str, str], dict[str, list[str]]], ...] = ( + ({}, NIGHTLY_ALL_JOBS), + ({"CI_NIGHTLY_HEAVY_RUN_TEST_MATRIX": "true", "CI_NIGHTLY_HEAVY_RUN_SCANNERS": "true"}, NIGHTLY_ALL_JOBS), + ({"CI_NIGHTLY_HEAVY_RUN_TEST_MATRIX": "false"}, {"scanner-readiness": []}), + ({"CI_NIGHTLY_HEAVY_RUN_SCANNERS": "false"}, {"nightly-matrix": NIGHTLY_MATRIX_JOBS}), + (NIGHTLY_VARS_OFF, {}), + ) + for variables, jobs in scheduled_cases: + with self.subTest(trigger="schedule", variables=variables): + run = self.nightly_run(self.scheduled(NIGHTLY, vars=variables)) + self.assertEqual(run["resolvers"], [svc.RESOLVER_STEP]) + self.assertEqual(run["source"], ("maintained", svc.MAINTAINED_REF, fixture.maintained_sha)) + self.assertEqual(run["jobs"], jobs) + self.assertEqual(run["scanner"], scheduled) + self.assertEqual(run["group"], "nightly-heavy-nightly-1.x.x") + + # A direct dispatch: manual mode on the selected ref and its commit, following its inputs, not the variables. + manual = ("manual", f"refs/heads/{fixture.manual_branch}", fixture.manual_sha) + dispatched = {"SCANNER_SET": "minimum", "SCANNER_FAIL_POLICY": "infra-and-secrets", "SKIP_BUILD_SCANNERS": "false", + "SCANNER_MODE": "manual_dry_run"} + dispatched_cases: tuple[tuple[dict[str, Any], dict[str, list[str]], dict[str, str]], ...] = ( + ({}, NIGHTLY_ALL_JOBS, {}), + ({"run_test_matrix": False, "skip_build_scanners": True}, {"scanner-readiness": []}, {"SKIP_BUILD_SCANNERS": "true"}), + ({"run_scanners": False, "jobs": "previous-releases"}, {"nightly-matrix": ["previous-releases"]}, {}), + ) + for inputs, jobs, scanner in dispatched_cases: + with self.subTest(trigger="workflow_dispatch", inputs=inputs): + run = self.nightly_run(dict(self.dispatch(NIGHTLY, **inputs), vars=NIGHTLY_VARS_OFF)) + self.assertEqual(run["resolvers"], [svc.RESOLVER_STEP]) + self.assertEqual(run["source"], manual) + self.assertEqual(run["jobs"], jobs) + self.assertEqual(run["scanner"], dict(dispatched, **scanner)) + self.assertEqual(run["group"], f"nightly-heavy-refs/heads/{fixture.manual_branch}") + + # The verify step records that no source_ref was given. + run = self.nightly_run(self.scheduled(NIGHTLY)) + fixture.checkout_work(fixture.maintained_sha) + context = svc.matrix_context(self.workflows[NIGHTLY], "nightly-matrix", run["context"]) + verify = svc.run_job_step(fixture, self.workflows[NIGHTLY], "nightly-matrix", svc.VERIFY_STEP, context, fixture.work) + self.assertEqual(verify.returncode, 0, verify.stderr) + self.assertIn("- source_ref input: (none)\n- source_mode: maintained\n", verify.summary) + + def test_nightly_concurrency_keeps_sources_and_callers_apart(self) -> None: + fixture = self.fixture + workflow = self.workflows[NIGHTLY] + + def group(context: dict[str, Any]) -> str: + return svc._to_text(svc.render(workflow["concurrency"]["group"], context)) + + groups = { + "direct schedule": group(self.scheduled(NIGHTLY)), + "direct dispatch": group(self.dispatch(NIGHTLY)), + "scheduled caller": group(self.call(NIGHTLY, "schedule")), + "dispatched caller, maintained branch": group(self.call(NIGHTLY, "workflow_dispatch")), + "dispatched caller, branch": group(self.call(NIGHTLY, "workflow_dispatch", source_ref=fixture.manual_branch)), + "dispatched caller, tag": group(self.call(NIGHTLY, "workflow_dispatch", source_ref=fixture.manual_tag)), + "dispatched caller, commit": group(self.call(NIGHTLY, "workflow_dispatch", source_ref=fixture.manual_sha)), + } + self.assertEqual(len(set(groups.values())), len(groups), groups) + self.assertEqual(groups["scheduled caller"], "nightly-heavy-call-schedule-1.x.x") + self.assertEqual(groups["dispatched caller, branch"], f"nightly-heavy-call-workflow_dispatch-{fixture.manual_branch}") + # A second call for the same source still replaces the one in progress. + self.assertEqual(group(self.call(NIGHTLY, "workflow_dispatch", source_ref=fixture.manual_branch, jobs="fuzz")), + groups["dispatched caller, branch"]) + # The group is evaluated before any job runs, from the contexts GitHub allows there. + self.assertNotRegex(workflow["concurrency"]["group"], r"\b(needs|steps|env|job)\.") + + def test_rejects_unknown_repeated_and_empty_job_names(self) -> None: + bad = { + PQ: ["aarch64", "AARCH64-UNIT", f"{AARCH64_JOB},{AARCH64_JOB}", f"{AARCH64_JOB},", ",", "s390x-unit"], + NIGHTLY: ["previous_releases", "previous-releases,previous-releases", "previous-releases,,fuzz", "unit"], + } + for key, values in bad.items(): + for value in values: + with self.subTest(workflow=key, jobs=value): + result = self.run_select(key, self.dispatch(key, jobs=value)) + self.assertNotEqual(result.returncode, 0) + self.assertIn("::error::", result.stderr) + self.assertNotIn("matrix", result.outputs) + + # -- source resolution --------------------------------------------------------- + + def test_source_ref_resolves_once_to_a_commit(self) -> None: + fixture = self.fixture + cases = [ + ("", f"refs/heads/{fixture.manual_branch}", fixture.manual_sha, "event"), + (svc.MAINTAINED_BRANCH, svc.MAINTAINED_REF, fixture.maintained_sha, "ref"), + (f" {svc.MAINTAINED_BRANCH} ", svc.MAINTAINED_REF, fixture.maintained_sha, "ref"), + (svc.MAINTAINED_REF, svc.MAINTAINED_REF, fixture.maintained_sha, "ref"), + (ANNOTATED_TAG, f"refs/tags/{ANNOTATED_TAG}", fixture.maintained_sha, "ref"), + (fixture.manual_tag, f"refs/tags/{fixture.manual_tag}", fixture.manual_sha, "ref"), + (fixture.default_sha, fixture.default_sha, fixture.default_sha, "commit"), + ] + for source_ref, requested_ref, resolved_sha, mode in cases: + with self.subTest(source_ref=source_ref): + result = self.run_resolver(PQ, self.dispatch(PQ, source_ref=source_ref)) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.outputs["requested_ref"], requested_ref) + self.assertEqual(result.outputs["resolved_sha"], resolved_sha) + self.assertEqual(result.outputs["source_mode"], mode) + self.assertIn(f"| resolved_sha | {resolved_sha} |", result.summary) + # The annotated tag is peeled: its tag object is not what gets checked out. + self.assertNotEqual(self.tag_object, fixture.maintained_sha) + + rejected = { + "no-such-branch": "no branch or tag 'no-such-branch'", + fixture.default_sha[:12]: "no branch or tag", + AMBIGUOUS_NAME: "must name exactly one branch or tag", + "refs/pull/1/head": "must be a branch, a tag or a full commit hash", + "bad..name": "is not a valid ref name", + } + for source_ref, message in rejected.items(): + with self.subTest(rejected=source_ref): + result = self.run_resolver(PQ, self.dispatch(PQ, source_ref=source_ref)) + self.assertNotEqual(result.returncode, 0) + self.assertIn(f"::error::source_ref '{source_ref}' " if "source_ref" in message else "::error::", result.stderr) + self.assertIn(message, result.stderr) + self.assertNotIn("resolved_sha", result.outputs) + + def test_every_pq_job_checks_out_the_resolved_commit(self) -> None: + fixture = self.fixture + workflow = self.workflows[PQ] + context = self.resolve(PQ, self.dispatch(PQ, source_ref=svc.MAINTAINED_BRANCH, jobs=AARCH64_JOB)) + refs = svc.check_source_wiring(workflow, context) + self.assertEqual(refs, {PQ_UNIT_JOB: fixture.maintained_sha}) + + mutated = copy.deepcopy(workflow) + checkout = svc.find_step(mutated["jobs"][PQ_UNIT_JOB], svc.CHECKOUT_STEP) + checkout["with"]["ref"] = "${{ github.sha }}" + with self.assertRaises(AssertionError): + svc.check_source_wiring(mutated, context) + + job_context = dict(context, matrix=catalog(workflow)[0]) + fixture.checkout_work(fixture.maintained_sha) + guard = svc.run_job_step(fixture, workflow, PQ_UNIT_JOB, svc.GUARD_STEP, job_context, fixture.root) + self.assertEqual(guard.returncode, 0, guard.stderr) + verify = svc.run_job_step(fixture, workflow, PQ_UNIT_JOB, svc.VERIFY_STEP, job_context, fixture.work) + self.assertEqual(verify.returncode, 0, verify.stderr) + self.assertIn(f"- actual_head: {fixture.maintained_sha}", verify.summary) + + # A checkout that moved (for example a branch push between jobs) is refused. + fixture.checkout_work(fixture.manual_sha) + verify = svc.run_job_step(fixture, workflow, PQ_UNIT_JOB, svc.VERIFY_STEP, job_context, fixture.work) + self.assertNotEqual(verify.returncode, 0) + self.assertIn("does not match resolved source", verify.stderr) + + # -- the aarch64 job and its suites ---------------------------------------------- + + def test_aarch64_job_runs_on_a_public_arm_runner(self) -> None: + workflow = self.workflows[PQ] + (entry,) = [entry for entry in catalog(workflow) if entry["job"] == AARCH64_JOB] + self.assertEqual(entry["runs-on"], "ubuntu-24.04-arm") + self.assertIn("pre-library", entry["name"]) + env_file = entry["file-env"] + self.assertTrue(os.access(REPO_ROOT / env_file, os.X_OK), env_file) + + context = dict(self.resolve(PQ, self.dispatch(PQ, jobs=AARCH64_JOB)), matrix=entry) + env = svc.job_env(workflow, workflow["jobs"][PQ_UNIT_JOB], context) + self.assertEqual(env["FILE_ENV"], env_file) + self.assertEqual(env["CI_IMAGE_REGISTRY_PREFIX"], "docker.io/library") + self.assertEqual(env["CI_ENFORCE_INTERNAL_REGISTRY"], "0") + self.assertEqual(env["CI_PROFILE"], "public") + self.assertNotIn("100.100.100.100", env["CI_BUILDKIT_DNS_NAMESERVERS"]) + self.assertNotIn("ts.net", env["CI_BUILDKIT_DNS_SEARCH_DOMAINS"]) + self.assertEqual(env["DANGER_CI_ON_HOST_FOLDERS"], "1", "the evidence must reach the host through BASE_BUILD_DIR") + + names = ["CI_BUILD_TARGET", "CTEST_REGEX", "CTEST_EXPECTED_SUITES", "CTEST_INCLUDE_RANGE", "CI_IMAGE_NAME_TAG", "RUN_FUNCTIONAL_TESTS"] + effective = source_env(env_file, names, {key: env[key] for key in ("CI_IMAGE_REGISTRY_PREFIX", "CI_ENFORCE_INTERNAL_REGISTRY")}) + self.assertEqual(effective["CI_BUILD_TARGET"], "test_bitcoin") + self.assertEqual(effective["CTEST_EXPECTED_SUITES"].split(), AARCH64_SUITES) + self.assertEqual(effective["CTEST_INCLUDE_RANGE"], "") + self.assertEqual(effective["CI_IMAGE_NAME_TAG"], "docker.io/library/ubuntu:24.04") + self.assertEqual(effective["RUN_FUNCTIONAL_TESTS"], "false") + regex = re.compile(effective["CTEST_REGEX"]) + self.assertEqual([name for name in SUITE_INVENTORY if regex.search(name)], AARCH64_SUITES) + + # 02_run_container.py passes only variables exported by some ci/test/00_setup_env*.sh into the container. + exported = set() + for path in (REPO_ROOT / "ci" / "test").glob("00_setup_env*.sh"): + exported.update(re.findall(r"^\s*export ([A-Z_][A-Z0-9_]*)=", path.read_text(encoding="utf8"), re.M)) + for name in ("CI_BUILD_TARGET", "CTEST_REGEX", "CTEST_EXPECTED_SUITES", "CTEST_INCLUDE_RANGE"): + self.assertIn(name, exported) + + def test_build_target_reaches_both_build_commands(self) -> None: + block = shell_block(TEST_SCRIPT.read_text(encoding="utf8"), "CI_BUILD_TARGET=${CI_BUILD_TARGET:-all}", ")") + for target, expected in (("test_bitcoin", ["test_bitcoin"]), ("", ["all", "install"])): + with self.subTest(CI_BUILD_TARGET=target): + run = run_test_script_block(block, {"CI_BUILD_TARGET": target, "STUB_FAIL_FIRST": "1"}) + self.assertNotEqual(run.returncode, 0, "the verbose retry still fails the build") + builds = [call for call in run.calls if call[:2] == ["cmake", "--build"]] + self.assertEqual(len(builds), 2, run.completed.stderr) + for call in builds: + targets = call[call.index("--target") + 1:] + self.assertEqual([t for t in targets if t != "--verbose"], expected) + self.assertIn("--verbose", builds[1]) + + def test_ctest_selection_reaches_listing_run_and_evidence(self) -> None: + block = shell_block(TEST_SCRIPT.read_text(encoding="utf8"), 'if [ "$RUN_UNIT_TESTS" = "true" ]; then', "fi") + regex = "^(bip324_tests|net_tests)$" + expected = " ".join(AARCH64_SUITES) + + run = run_test_script_block(block, {"CTEST_REGEX": regex, "CTEST_EXPECTED_SUITES": expected}) + self.assertEqual(run.returncode, 0, run.completed.stderr) + ctest_calls = [call[1:] for call in run.calls if call[0] == "ctest"] + self.assertEqual(len(ctest_calls), 3) + listing, human, ran = ctest_calls + self.assertIn("--show-only=json-v1", listing) + self.assertIn("--show-only", human) + for call in ctest_calls: + self.assertEqual(call[call.index("-R") + 1], regex) + self.assertNotIn("-I", call) + self.assertIn("--no-tests=error", ran) + self.assertIn("--output-junit", ran) + summary = run.files["summary.md"] + for suite in AARCH64_SUITES: + self.assertIn(f"| {suite} | passed |", summary) + + failures = { + "skipped suite": ({"CTEST_REGEX": regex, "CTEST_EXPECTED_SUITES": expected, + "STUB_OUTCOMES": json.dumps({"bip324_tests": "notrun"})}, "bip324_tests: skipped"), + "regex selects too little": ({"CTEST_REGEX": "^(net_tests)$", "CTEST_EXPECTED_SUITES": expected}, + "did not select expected suite(s): bip324_tests"), + "regex selects too much": ({"CTEST_REGEX": "^bip324_tests|^net_tests$", "CTEST_EXPECTED_SUITES": expected}, + "not expected: bip324_tests_extra"), + "suite missing from the build": ({"CTEST_REGEX": "^(bip324_tests|net_tests|mlkem_tests)$", + "CTEST_EXPECTED_SUITES": expected + " mlkem_tests"}, "mlkem_tests"), + "range and regex together": ({"CTEST_REGEX": regex, "CTEST_INCLUDE_RANGE": "1,2"}, "set only one"), + "nothing selected": ({"CTEST_REGEX": "^nothing$"}, "No tests were found"), + } + for label, (env, message) in failures.items(): + with self.subTest(label): + run = run_test_script_block(block, env) + self.assertNotEqual(run.returncode, 0) + self.assertIn(message, run.completed.stderr + run.completed.stdout) + run = run_test_script_block(block, {"CTEST_REGEX": regex, "CTEST_INCLUDE_RANGE": "1,2"}) + self.assertEqual(run.calls, [], "conflicting selections are rejected before ctest runs") + + # A job without expected suites, such as a range shard, keeps its old behavior. + run = run_test_script_block(block, {"CTEST_INCLUDE_RANGE": "1,3", "STUB_OUTCOMES": json.dumps({"addrman_tests": "notrun"})}) + self.assertEqual(run.returncode, 0, run.completed.stderr) + ran = [call[1:] for call in run.calls if call[0] == "ctest"][-1] + self.assertEqual(ran[ran.index("-I") + 1], "1,3") + self.assertNotIn("summary.md", run.files) + + def test_report_step_publishes_source_and_suites(self) -> None: + fixture = self.fixture + workflow = self.workflows[PQ] + context = self.resolve(PQ, self.dispatch(PQ, jobs=AARCH64_JOB)) + entry = catalog(workflow)[0] + job_context = dict(context, matrix=entry) + step = svc.find_step(workflow["jobs"][PQ_UNIT_JOB], REPORT_STEP) + self.assertEqual(step.get("if"), "${{ always() }}") + # The step reads the file 03_test_script.sh writes. + self.assertIn('CTEST_EVIDENCE_DIR="${BASE_BUILD_DIR}/ctest-evidence"', TEST_SCRIPT.read_text(encoding="utf8")) + self.assertIn('--summary "${CTEST_EVIDENCE_DIR}/summary.md"', TEST_SCRIPT.read_text(encoding="utf8")) + with tempfile.TemporaryDirectory(dir=fixture.root) as tmp: + build = Path(tmp) + env = svc.step_env(workflow, workflow["jobs"][PQ_UNIT_JOB], step, job_context) + env["BASE_BUILD_DIR"] = str(build) + github_env = svc.github_env_for(job_context, PQ_UNIT_JOB, workflow["name"]) + missing = svc.run_step(fixture, step, env, fixture.root, github_env) + self.assertNotEqual(missing.returncode, 0) + self.assertIn("No unit-test evidence was produced.", missing.summary) + + (build / "ctest-evidence").mkdir() + (build / "ctest-evidence" / "summary.md").write_text("| bip324_tests | passed | 0.5 |\n", encoding="utf8") + report = svc.run_step(fixture, step, env, fixture.root, github_env) + self.assertEqual(report.returncode, 0, report.stderr) + self.assertIn(f"- source: {fixture.manual_sha} (refs/heads/{fixture.manual_branch})", report.summary) + self.assertIn("Pre-library phase", report.summary) + self.assertIn("| bip324_tests | passed | 0.5 |", report.summary) + + def test_required_gate_runs_the_contracts(self) -> None: + gate = svc.load_workflow(GATE_WORKFLOW) + commands = [step.get("run", "") for step in gate["jobs"]["classify-changes"]["steps"]] + for contract in ("ci/checks/test_ci_pq_contract.py", "ci/checks/test_scheduled_validation_contract.py"): + self.assertTrue(any(contract in command for command in commands), contract) + + +class CtestEvidenceTest(unittest.TestCase): + """ctest_evidence.py against JUnit in the format ctest --output-junit writes.""" + + @staticmethod + def junit(*cases: tuple[str, str, str]) -> str: + body = "".join(f'{inner}' + for name, status, inner in cases) + return f'\n{body}\n' + + def test_outcomes(self) -> None: + outcomes = ctest_evidence.junit_outcomes(self.junit( + ("passed_suite", "run", ""), + ("skipped_suite", "notrun", ''), + ("failed_suite", "fail", ''), + ("disabled_suite", "disabled", "Disabled"), + ("odd_suite", "weird", ""), + )) + self.assertEqual({name: outcome for name, (outcome, _) in outcomes.items()}, { + "passed_suite": "passed", + "skipped_suite": "skipped", + "failed_suite": "failed", + "disabled_suite": "disabled", + "odd_suite": "unknown status 'weird'", + }) + twice = ctest_evidence.junit_outcomes(self.junit(("a", "run", ""), ("a", "run", ""))) + self.assertEqual(twice["a"][0], "reported more than once") + + def test_check(self) -> None: + passed = {"a": ("passed", "1"), "b": ("passed", "2")} + self.assertEqual(ctest_evidence.check(["a", "b"], ["a", "b"], passed), []) + self.assertEqual(ctest_evidence.check(["a", "b"], ["a"], {"a": ("passed", "1")}), + ["ctest did not select expected suite(s): b", "b: missing from the JUnit results"]) + self.assertEqual(ctest_evidence.check(["a"], ["a", "c"], passed), ["ctest selected suite(s) that are not expected: c"]) + self.assertEqual(ctest_evidence.check(["a"], ["a"], {"a": ("skipped", "0")}), ["a: skipped"]) + with self.assertRaises(ValueError): + ctest_evidence.parse_expected("") + with self.assertRaises(ValueError): + ctest_evidence.parse_expected("a b a") + + def test_main_reports_unreadable_inputs(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + listing = root / "selected.json" + junit = root / "junit.xml" + summary = root / "summary.md" + listing.write_text(json.dumps({"tests": [{"name": "a"}]}), encoding="utf8") + junit.write_text(" int: + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + return ctest_evidence.main(args) + + self.assertEqual(main(), 1) + self.assertIn("unreadable JUnit results", summary.read_text(encoding="utf8")) + junit.write_text(self.junit(("a", "run", "")), encoding="utf8") + self.assertEqual(main(), 0) + self.assertIn("| a | passed | 0.1 |", summary.read_text(encoding="utf8")) + listing.unlink() + self.assertEqual(main(), 1) + self.assertIn("unreadable ctest test listing", summary.read_text(encoding="utf8")) + + +if __name__ == "__main__": + unittest.main() diff --git a/ci/checks/test_scheduled_validation_contract.py b/ci/checks/test_scheduled_validation_contract.py index d94ba136f1..e458a17673 100755 --- a/ci/checks/test_scheduled_validation_contract.py +++ b/ci/checks/test_scheduled_validation_contract.py @@ -29,7 +29,7 @@ import tempfile import unittest from pathlib import Path, PurePosixPath -from typing import Any +from typing import Any, Callable import yaml @@ -136,10 +136,56 @@ def _to_text(value: Any) -> str: return str(value) +_JSON_NUMBER = re.compile(r"-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?") + + +def _to_number(value: Any) -> float: + """GitHub's conversion of an operand when the types of a comparison differ.""" + if value is None: + return 0.0 + if isinstance(value, bool): + return 1.0 if value else 0.0 + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, str): + if value == "": + return 0.0 + return float(value) if _JSON_NUMBER.fullmatch(value) else float("nan") + return float("nan") + + def _equal(left: Any, right: Any) -> bool: - if isinstance(left, str) or isinstance(right, str) or left is None or right is None: - return _to_text(left).casefold() == _to_text(right).casefold() - return left == right + """GitHub's loose equality: strings compare case-insensitively, and operands + of different types compare as numbers, so null, '' and false are all equal.""" + if isinstance(left, str) and isinstance(right, str): + return left.casefold() == right.casefold() + if type(left) is type(right): + return bool(left == right) + return _to_number(left) == _to_number(right) + + +def _format(template: Any, *values: Any) -> str: + """format(): {N} is replaced by the Nth value, and {{ and }} are literal braces.""" + if not values: + raise ExpressionError("format() needs at least one replacement value") + + def replace(match: re.Match[str]) -> str: + if match.group(1) is None: + return match.group(0)[0] + index = int(match.group(1)) + if index >= len(values): + raise ExpressionError(f"format() has no value {{{index}}}") + return _to_text(values[index]) + + return re.sub(r"\{\{|\}\}|\{(\d+)\}", replace, _to_text(template)) + + +def _to_json(value: Any) -> str: + """toJSON(): pretty-printed JSON, as GitHub prints it.""" + return json.dumps(value, indent=2) + + +_FUNCTIONS: dict[str, Callable[..., Any]] = {"format": _format, "tojson": _to_json} class _Parser: @@ -211,7 +257,7 @@ def _parse_primary(self) -> Any: return float(text) if "." in text else int(text) if kind == "path": if self._peek() == ("op", "("): - raise ExpressionError(f"function calls are not supported: {text}") + return self._call(text) if text == "true": return True if text == "false": @@ -221,6 +267,20 @@ def _parse_primary(self) -> Any: return self._lookup(text) raise ExpressionError(f"unexpected token {text!r}") + def _call(self, name: str) -> Any: + function = _FUNCTIONS.get(name.casefold()) + if function is None: + raise ExpressionError(f"function calls are not supported: {name}") + self._take("(") + arguments = [] + if self._peek() != ("op", ")"): + arguments.append(self._parse_or()) + while self._peek() == ("op", ","): + self._take() + arguments.append(self._parse_or()) + self._take(")") + return function(*arguments) + def _lookup(self, path: str) -> Any: value: Any = self.context for part in path.split("."): @@ -610,9 +670,19 @@ def run_job_step( return run_step(fixture, step, env, cwd, github_env_for(context, job_id, workflow["name"])) -def matrix_context(workflow: dict[str, Any], job_id: str, context: dict[str, Any]) -> dict[str, Any]: +def matrix_include(workflow: dict[str, Any], job_id: str) -> list[dict[str, Any]]: + """A job's matrix entries: inline, or the resolver's job catalog that its matrix is selected from.""" strategy = workflow["jobs"][job_id].get("strategy") or {} - include = (strategy.get("matrix") or {}).get("include") or [] + matrix = strategy.get("matrix") or {} + if isinstance(matrix, str): + assert matrix == "${{ fromJSON(needs." + RESOLVER_JOB + ".outputs.matrix) }}", matrix + catalog = find_step(workflow["jobs"][RESOLVER_JOB], "Select jobs")["env"]["JOB_CATALOG"] + return json.loads(catalog) + return matrix.get("include") or [] + + +def matrix_context(workflow: dict[str, Any], job_id: str, context: dict[str, Any]) -> dict[str, Any]: + include = matrix_include(workflow, job_id) if not include: return context return dict(context, matrix=include[-1]) @@ -659,7 +729,7 @@ def test_workflow_triggers_are_preserved(self) -> None: self.assertIn("workflow_dispatch", workflow["on"]) self.assertNotIn(RETIRED_BRANCH, self.workflow_text[key]) self.assertEqual( - workflow["jobs"][RESOLVER_JOB]["steps"][0]["env"]["MAINTAINED_BRANCH"], MAINTAINED_BRANCH + find_step(workflow["jobs"][RESOLVER_JOB], RESOLVER_STEP)["env"]["MAINTAINED_BRANCH"], MAINTAINED_BRANCH ) def test_scheduled_jobs_use_approved_source(self) -> None: @@ -956,7 +1026,7 @@ def test_nightly_mutation_env_only_on_native_fuzz_matrix(self) -> None: workflow = self.workflows["nightly"] context, _outputs = self.resolve_schedule("nightly") job = workflow["jobs"]["nightly-matrix"] - include = job["strategy"]["matrix"]["include"] + include = matrix_include(workflow, "nightly-matrix") native_fuzz = [entry for entry in include if entry["file-env"].endswith("/00_setup_env_native_fuzz.sh")] self.assertEqual(len(native_fuzz), 1) for entry in include: @@ -981,6 +1051,23 @@ def test_expression_evaluator_matches_actions_semantics(self) -> None: self.assertEqual(json.dumps(render("${{ inputs.missing }}", context)), "null") with self.assertRaises(ExpressionError): evaluate("fromJSON('[]')", context) + # Loose equality: operands of different types compare as numbers, so an + # absent input, an empty string and false are indistinguishable. + context["inputs"] = {"empty": "", "no": False, "yes": True, "one": "1", "word": "false"} + for expression in ("inputs.missing == ''", "inputs.empty == null", "inputs.no == ''", "inputs.no == null", + "inputs.missing == 0", "inputs.one == 1", "inputs.yes == 1", "'ABC' == 'abc'"): + with self.subTest(expression=expression): + self.assertIs(evaluate(expression, context), True) + for expression in ("inputs.word == inputs.no", "'true' == inputs.yes", "inputs.missing != ''", "'abc' == 0", + "'1.x.x' == ''"): + with self.subTest(expression=expression): + self.assertIs(evaluate(expression, context), False) + self.assertEqual(evaluate("format('call-{0}-{1}', github.event_name, '1.x.x')", context), "call-schedule-1.x.x") + self.assertEqual(evaluate("FORMAT('{{{0}}} {1}', 'a', inputs.missing)", context), "{a} ") + self.assertEqual(json.loads(evaluate("toJSON(github)", context)), context["github"]) + self.assertEqual(evaluate("toJSON(inputs.missing)", context), "null") + with self.assertRaises(ExpressionError): + evaluate("format('{1}', 'a')", context) if __name__ == "__main__": diff --git a/ci/test/00_setup_env_native_aarch64_pq.sh b/ci/test/00_setup_env_native_aarch64_pq.sh new file mode 100755 index 0000000000..ddb42f03c7 --- /dev/null +++ b/ci/test/00_setup_env_native_aarch64_pq.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2026-present The qbit core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +# shellcheck source=ci/test/00_setup_env_base_image.sh +source "$( dirname "${BASH_SOURCE[0]}" )/00_setup_env_base_image.sh" + +# The aarch64-unit job of .github/workflows/ci-pq.yml: build only the unit-test +# binary natively on an AArch64 runner and run the post-quantum transport suites. +export CONTAINER_NAME=ci_native_aarch64_pq +ci_set_base_image_name_tag "ubuntu:24.04" +export PACKAGES="libboost-dev libevent-dev" +export NO_DEPENDS=1 +export RUN_UNIT_TESTS=true +export RUN_FUNCTIONAL_TESTS=false +export CI_BUILD_TARGET=test_bitcoin +# The suites this job must run; ctest_evidence.py fails the job unless each was +# selected, executed and passed. CTEST_REGEX is derived from the same list. +export CTEST_EXPECTED_SUITES="bip324_tests net_tests" +export CTEST_REGEX="^(${CTEST_EXPECTED_SUITES// /|})\$" +# -Wno-psabi silences GCC's AArch64 "parameter passing ... changed" notes. +export BITCOIN_CONFIG="\ + -DBUILD_GUI=OFF -DENABLE_WALLET=OFF -DENABLE_IPC=OFF -DWITH_ZMQ=OFF -DREDUCE_EXPORTS=ON \ + -DCMAKE_CXX_FLAGS='-Wno-psabi' \ +" diff --git a/ci/test/03_test_script.sh b/ci/test/03_test_script.sh index 7ee547774d..20dad10606 100755 --- a/ci/test/03_test_script.sh +++ b/ci/test/03_test_script.sh @@ -152,11 +152,22 @@ bash -c "cmake -S $BASE_ROOT_DIR -B ${BASE_BUILD_DIR} $BITCOIN_CONFIG_ALL $BITCO false ) +# CI_BUILD_TARGET narrows the build to the named target(s), e.g. test_bitcoin +# for a job that only runs unit tests. GOAL's install and deploy targets need +# the whole build, so they apply only when everything is built. +CI_BUILD_TARGET=${CI_BUILD_TARGET:-all} +if [ "${CI_BUILD_TARGET}" = "all" ]; then + BUILD_TARGETS="all ${GOAL}" +else + BUILD_TARGETS="${CI_BUILD_TARGET}" +fi +echo "Building target(s): ${BUILD_TARGETS}" + # shellcheck disable=SC2086 -cmake --build "${BASE_BUILD_DIR}" "$MAKEJOBS" --target all $GOAL || ( +cmake --build "${BASE_BUILD_DIR}" "$MAKEJOBS" --target ${BUILD_TARGETS} || ( echo "Build failure. Verbose build follows." # shellcheck disable=SC2086 - cmake --build "${BASE_BUILD_DIR}" -j1 --target all $GOAL --verbose + cmake --build "${BASE_BUILD_DIR}" -j1 --target ${BUILD_TARGETS} --verbose false ) @@ -218,19 +229,47 @@ if [ "$RUN_CHECK_DEPS" = "true" ]; then fi if [ "$RUN_UNIT_TESTS" = "true" ]; then + # Select tests by ordinal range (CTEST_INCLUDE_RANGE, for shards) or by name + # (CTEST_REGEX), never both. + if [ -n "${CTEST_INCLUDE_RANGE}" ] && [ -n "${CTEST_REGEX}" ]; then + echo "Error: CTEST_INCLUDE_RANGE and CTEST_REGEX are both set; set only one." >&2 + exit 1 + fi CTEST_ARGS=() if [ -n "${CTEST_INCLUDE_RANGE}" ]; then CTEST_ARGS+=(-I "${CTEST_INCLUDE_RANGE}") fi + if [ -n "${CTEST_REGEX}" ]; then + CTEST_ARGS+=(-R "${CTEST_REGEX}") + fi + + # List the selected tests before running them, with the same arguments. + CTEST_EVIDENCE_DIR="${BASE_BUILD_DIR}/ctest-evidence" + rm -rf "${CTEST_EVIDENCE_DIR}" + mkdir -p "${CTEST_EVIDENCE_DIR}" + ctest --test-dir "${BASE_BUILD_DIR}" --show-only=json-v1 "${CTEST_ARGS[@]}" > "${CTEST_EVIDENCE_DIR}/selected.json" + ctest --test-dir "${BASE_BUILD_DIR}" --show-only "${CTEST_ARGS[@]}" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" \ LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" \ CTEST_OUTPUT_ON_FAILURE=ON \ ctest --test-dir "${BASE_BUILD_DIR}" \ --stop-on-failure \ + --no-tests=error \ + --output-junit "${CTEST_EVIDENCE_DIR}/junit.xml" \ -j "${CTEST_JOBS}" \ --timeout $(( TEST_RUNNER_TIMEOUT_FACTOR * 60 )) \ "${CTEST_ARGS[@]}" + + # ctest passes a skipped test, so a job that names its suites checks that + # each one was selected, executed and passed. + if [ -n "${CTEST_EXPECTED_SUITES}" ]; then + python3 "${BASE_ROOT_DIR}/ci/checks/ctest_evidence.py" \ + --expected "${CTEST_EXPECTED_SUITES}" \ + --listing "${CTEST_EVIDENCE_DIR}/selected.json" \ + --junit "${CTEST_EVIDENCE_DIR}/junit.xml" \ + --summary "${CTEST_EVIDENCE_DIR}/summary.md" + fi fi # Build qbit-photon relay daemon for integration tests. diff --git a/contrib/pq-canary/README.md b/contrib/pq-canary/README.md new file mode 100644 index 0000000000..f168513b61 --- /dev/null +++ b/contrib/pq-canary/README.md @@ -0,0 +1,240 @@ +# PQ canary sampler and evaluator + +Scripted evidence for the mainnet canary of the hybrid post-quantum v2 +transport (#184, release gate R5, with the R0 baseline before it). Two team +mainnet nodes run the release candidate: an archive node and a pool node pinned +to it with `-addnode`. A third team archive node stays on v1.0.0 as the +control. Every 5 minutes each of them records one sample, and +`pq-canary-eval.py` turns the samples into the canary figures. + +The tools work against nodes without the new RPCs (v1.0.0, and the R0 baseline): +whatever a node cannot report is recorded as `NA`. + +## Files + +| File | Purpose | +|---|---| +| `pq-canary-sample.sh` | Records one sample; run it from cron or a systemd timer. | +| `pq_canary.py` | The sample format, shared by the sampler and the evaluator. | +| `pq-canary-eval.py` | Prints the canary figures with their sample counts. | +| `test/test_pq_canary.py` | Tests, with RPC fixtures in the `getpqtransportinfo` schema of #184. | + +## Install + +Requirements: bash, Python 3.10 or later, and `qbit-cli` able to reach the node. + +1. Copy this directory to the node, for example to `/opt/qbit/pq-canary`, or use + a checkout of the repository. +2. Pick an output directory owned by the user that runs the sampler, for example + `/var/lib/qbit-canary`. +3. Add a cron line (`crontab -e`) for that user. Use a short `--host` label, + other than `NA`; the evaluator tells nodes apart by it. + +Canary archive node: + +``` +*/5 * * * * /opt/qbit/pq-canary/pq-canary-sample.sh --host archive --out /var/lib/qbit-canary -- /usr/local/bin/qbit-cli -datadir=/var/lib/qbit -rpcclienttimeout=30 +``` + +Pool node, pinned to the archive node with `-addnode=`; +`--pinned` takes the same `address:port`, as `getpeerinfo` shows it: + +``` +*/5 * * * * /opt/qbit/pq-canary/pq-canary-sample.sh --host pool --pinned 203.0.113.5:8333 --out /var/lib/qbit-canary -- /usr/local/bin/qbit-cli -datadir=/var/lib/qbit -rpcclienttimeout=30 +``` + +Control node (v1.0.0): the archive line with `--host control`. + +With a systemd timer instead of cron, run the same command from a oneshot +service and set `OnCalendar=*:0/5` on the timer. + +Each RPC call is also bounded by `timeout 60` when `timeout(1)` is installed, so +a hung node cannot pile up sampler runs. The sampler exits 0 for a complete +sample, 2 when `sample_ok=0`, and 1 on a usage error. + +## What a sample contains + +Each run appends one row to `samples.csv` (the header is written once) and, +when `getpqtransportinfo` answers, one line to `failures.jsonl`. + +`samples.csv` columns: + +- `time` (Unix seconds), `utc`, `host`, `uptime`, `version` (from `getnetworkinfo`); +- `sample_ok`: 1 when `uptime`, `getnetworkinfo` and, on the pool node, + `getpeerinfo` answered, else 0; +- from `getpqtransportinfo`: `pq_enabled`, `instance_id`, `arith_backend`, + `keccak_backend`, `since`, the inbound counters `in_switched` ... + `in_shed`, the outbound counters `out_switched` ... `out_fallback`, + `load_shedding_active`, and each ring's `*_ring_last_sequence` and + `*_ring_dropped`; +- `connections_in` and `connections_pq` from `getnetworkinfo`; +- on the pool node, from the `getpeerinfo` entry for the pinned address: + `pinned_present` (1 or 0), `pinned_connection_type`, `pinned_transport_pq` + and `pinned_transport_pq_status`. + +Booleans are written as 1 or 0. A failed call, or a field the node does not +report, is `NA`, never 0; a real 0 stays 0. So is a value of the wrong type: a +count that is not a non-negative integer (`true` is never written as 1), a flag +that is not a boolean, or an `instance_id` that is not 64 lowercase hex digits. A v1.0.0 node therefore writes `NA` +for every `getpqtransportinfo` column and for `connections_pq`, with +`sample_ok=1`. The CSV holds no peer addresses. + +`failures.jsonl` holds each sample's raw `recent_failures` and `fallback_set`, +with the sample time, host label, `instance_id` and `since`. It contains peer +addresses: keep it on the node, and run the evaluator there. + +## Known-good endpoints + +The failure figures only count entries between known-good endpoints: the canary +nodes and their pinned peers. Keep the list in a file on the evaluating node, +one entry per line: the endpoint, then an optional label (letters, digits, `.`, +`_`, `-`). `#` starts a comment. + +``` +# The archive node, as the pool node dials it. +203.0.113.5:8333 archive +# The pool node, as the archive node sees it: inbound entries carry the +# peer's source port, so list the address alone to match any port. +198.51.100.7 pool +# IPv6 entries go in brackets. Without a label this one is known-good-3. +[2001:db8::5]:8333 +``` + +The report names a known-good peer by its label and never prints an address, +so it can be posted. Unlabeled entries are `known-good-1`, `known-good-2`, ... +in file order. Errors about the list name the line, not the address. + +An IP address matches however it is written: `[2001:0db8:0:0:0:0:0:5]:8333` is +the same entry as `[2001:db8::5]:8333`, and `[::ffff:203.0.113.5]:8333` the same +as `203.0.113.5:8333`, as the node reports them. + +## Running the evaluator + +Figure 2 needs the failures of every monitored canary node, by default the +hosts of `--pool` and `--archive`: one node's file cannot pass it. Copy the +pool node's `failures.jsonl` to the archive node, or the other way round, over +a private channel between the two team nodes (for example `scp`), and run the +evaluator there. The CSV files hold no addresses. For example, on the archive +node: + +``` +pq-canary-eval.py --canary-start 2026-11-02T00:00:00Z \ + --pool pool/samples.csv --archive /var/lib/qbit-canary/samples.csv --control control/samples.csv \ + --failures pool/failures.jsonl --failures /var/lib/qbit-canary/failures.jsonl \ + --known-good /etc/qbit-canary/known-good.txt +``` + +Samples before `--canary-start` are the baseline (R0); the canary window runs +from there to `--end`, or to the latest sample. Other options: + +- `--baseline-ratio R` sets the archive/control connection ratio instead of + measuring it from the baseline. +- `--min-coverage S` (default 0.9) is the share of the expected 5-minute samples + that a figure, node or day needs before it is judged. +- `--monitored HOST,...` names the nodes whose failures must be present, instead + of the hosts of `--pool` and `--archive`. +- `--interval` (default 300 s) and `--restart-grace` (default 600 s). + +Unknown, misspelled or abbreviated options, malformed times or numbers, missing +files, `--failures` without `--known-good` (or the reverse) and an `--end` +before `--canary-start` are rejected with exit status 2 and a message. So are +samples that would compare a node with itself: `--pool`, `--archive` and +`--control` must each be one node, so a file given for two roles, two roles +with the same host label or `instance_id`, or a file whose rows carry more than +one host label (or none) is rejected. + +Output: one line per figure, `
: samples= coverage=
`, +where the result is `pass`, `fail` or `unknown`, followed by indented details, +then the verdict over figures 1, 2 and 4: + +1. **Pinned link**: the pool node's pinned peer is connected as `manual` with + `transport_pq` true in at least 99% of samples, excluding restarts. +2. **Failure entries**: no `malformed_record` or `first_packet_failed` entry, + no `fallback` entry and no fallback-set entry between known-good endpoints, + on every monitored node. Its details give each node's coverage. +3. **closed_after_switch**: such entries between known-good endpoints, listed + for triage; not part of the verdict. +4. **Inbound connections**: per UTC day, the archive node's `connections_in` + divided by the control's stays within ±20% of the baseline ratio, excluding + the 48 hours after a restart of either node. Each day shows its coverage. A + baseline in which either node's `connections_in` sums to 0 gives no usable + ratio, so the figure is unknown unless `--baseline-ratio` is given. + +The exit status is 0 when the verdict is pass, 1 on fail, 3 on unknown and 2 for +bad arguments or input. + +### How evidence is judged + +- **Restarts.** A jump in a node's boot time (`time - uptime`), or a new + `instance_id`, is a restart. For the pinned link, the samples from the last + sample before the restart to 10 minutes after it (`--restart-grace`) are a + gap: neither pass nor fail. The restarts are listed at the top of the report. + An uptime whose boot time differs from the one most samples with the same + `instance_id` agree on contradicts it: that uptime is unknown, not a restart. +- **Coverage.** Coverage is the share of the expected 5-minute samples that are + present and usable. Below `--min-coverage`, the pinned link, a monitored + node's failure evidence and an inbound-connections day are unknown, not + judged. +- **One sample per expected sample.** The expected samples are `--interval` + apart from `--canary-start`, and a sample stands for the expected one nearest + to it. However many samples stand for one expected sample (a repeated row, + overlapping files, a manual run beside cron), every figure counts it once, as + the least favorable of them: for the pinned link a miss, then unknown, then + excluded, then hybrid; for inbound connections unknown, then excluded, then a + measurement, where two measurements that disagree are unknown. A row + repeated exactly is one sample. Give `--canary-start` on the sampler's + 5-minute schedule, such as a whole hour, so that each sample is near the + expected sample it stands for. +- **Unknown, never clean.** A sample with `sample_ok=0`, an `NA` where the + figure needs a value, or a missing sample (for example while cron was not + running) is unknown. So is a malformed value: a `time`, `uptime` or + `connections_in` that is not a plain non-negative integer (a row without a + usable `time` counts as missing), a row with more or fewer fields than the + header (such as a line cut short by a crash), and a connected pinned peer + whose connection type, `transport_pq` or `transport_pq_status` is missing or + malformed, or whose `transport_pq` is not 1 exactly when the status is + `hybrid`. The pinned link passes only if it would pass with every + unknown sample counted as a failure, and fails only if it would fail with + every unknown sample counted as a success; otherwise it is unknown. +- **Rings.** Failure entries are read from each ring (`inbound`, `outbound`) by + its per-ring sequence. A ring holds the latest 256 entries, so it wraps: that + is fine as long as the entries read reach back to the sequence after the last + one already processed, and the ring stays judged. Entries are lost, and that + stretch is unknown for that ring, when a wrap between two samples evicted + entries never read (the oldest entry read is past that point), across a + changed `instance_id`, or when a ring is malformed or goes backwards. A flood + of inbound failures therefore cannot hide an outbound one: the rings are + separate. Unrelated endpoints never fail the canary, but only a well-formed + endpoint is unrelated: a `malformed_record`, `first_packet_failed`, + `fallback` or `closed_after_switch` entry whose endpoint is missing or + malformed makes its figure unknown. +- **Malformed evidence is unknown.** Only values of the `getpqtransportinfo` + types count as evidence. A record whose `instance_id` or `since` is missing + or malformed, whose `since` is after the sample, or whose `since` changed + within one `instance_id`; a ring that is not an object with non-negative + integer counters; a failure entry whose `outcome` is outside the vocabulary, + or whose `time` is missing, malformed, before its process started or more + than 5 minutes after the sample that read it; and a fallback-set entry + without a well-formed endpoint: each makes that stretch unknown. A + `failures.jsonl` line without a time in Unix seconds and a valid host label + is bad input (exit status 2). A known-good failure whose other fields are + malformed is still a failure, and those fields print as `malformed`, so a + malformed field can never put an address in the report. + +### When the pinned link is not manual + +If the pool node already has an automatic connection to the archive node's +address, `GetAddedNodeInfo` skips the address and the `-addnode` thread does +not open its manual connection, so the figure fails and the report says so. The +remedy: run `qbit-cli disconnectnode ` once on the pool +node; the `-addnode` thread reconnects it as manual within about 60 seconds. + +## Tests + +``` +python3 contrib/pq-canary/test/test_pq_canary.py +``` + +The Required Merge Gate runs them, and `test/functional/tool_pq_canary.py` runs +the sampler against a regtest node, and against v1.0.0 when previous releases +are downloaded. diff --git a/contrib/pq-canary/pq-canary-eval.py b/contrib/pq-canary/pq-canary-eval.py new file mode 100755 index 0000000000..1521acd9b1 --- /dev/null +++ b/contrib/pq-canary/pq-canary-eval.py @@ -0,0 +1,1012 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026-present The qbit core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. +"""Turn PQ canary samples into the R5 canary figures (#184). + +Each figure is printed with its sample count, its coverage and pass, fail or +unknown: + +1. the pool node's pinned link to the archive node is manual and hybrid in at + least 99% of samples, excluding restarts; +2. no malformed_record, first_packet_failed or fallback entry between + known-good endpoints, on every monitored canary node; +3. closed_after_switch entries between known-good endpoints, listed for triage + (not part of the canary verdict); +4. the canary archive node's connections_in stays within +-20% of the control + node's, relative to their baseline ratio, per UTC day, excluding the 48 hours + after a restart of either node. + +Evidence rules: +- A jump in a node's boot time (time - uptime) is a restart. The samples from + the last sample before it to the end of the grace period after it are a gap, + not a failure. An uptime whose boot time contradicts the one most samples of + the same instance_id agree on is unknown, not a restart. +- Coverage is the share of the expected 5-minute samples that are present and + usable. A figure, node or day below --min-coverage (default 90%) is unknown, + not judged. +- The expected samples are --interval apart from --canary-start (so give one on + the sampler's schedule, such as a whole hour), and a sample stands for the + expected one nearest to it. Every figure counts an expected sample once, + however many samples stand for it: several count as the least favorable of + them, so a repeated row, overlapping files or an extra sampler run never add + evidence. A row repeated exactly is one sample. +- Failure rings are read by their per-ring sequence. A ring that wrapped is + still complete when the entries read continue from the last sequence already + processed; entries are lost only when the oldest entry read is past that point + (a wrap between samples that hid unread entries), and across a changed + instance_id. A lost stretch makes that interval unknown for that ring. +- Missing evidence is unknown, never clean: the pinned link passes only if it + would pass with every unknown sample counted against it. A malformed value is + missing: a time, uptime or connections_in that is not a plain non-negative + integer, a samples.csv row with the wrong number of fields, and a connected + pinned peer whose fields are malformed or disagree. +- Only well-formed evidence is clean. A malformed ring, a record whose + instance_id or since is missing, malformed or contradictory, a failure entry + whose outcome, time or endpoint is missing, malformed or implausible, and a + fallback-set entry without a well-formed endpoint make that stretch unknown. + A failure entry is implausible when it is dated before its process started or + more than CLOCK_SLACK after the sample that read it. +- The report names known-good peers by their label, never by address; peer + addresses stay in failures.jsonl and the known-good list on the node. +- --pool, --archive and --control are three different nodes: samples that share + a file, a host label or an instance_id, or one file with several host labels, + are rejected as bad input rather than compared with themselves. + +Exit status: 0 when the verdict (figures 1, 2 and 4) is pass, 1 when any of +them fails, 3 when none fails but one is unknown, 2 for bad arguments or input. +""" + +from __future__ import annotations + +import argparse +import ipaddress +import json +import re +import sys +from collections import defaultdict +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from fractions import Fraction +from pathlib import Path +from typing import Any, Iterable, TypeGuard + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import pq_canary # noqa: E402 + +PASS, FAIL, UNKNOWN = "pass", "fail", "unknown" +PINNED_THRESHOLD = 0.99 +CONNECTIONS_TOLERANCE = Fraction(1, 5) +CONNECTIONS_RESTART_EXCLUSION = 48 * 3600 +DEFAULT_MIN_COVERAGE = 0.90 +RESTART_TOLERANCE = 120 # seconds of boot-time jitter that is not a restart +MONITORED_OUTCOMES = {"malformed_record", "first_packet_failed", "fallback"} +TRIAGE_OUTCOMES = {"closed_after_switch"} +OUTCOMES = MONITORED_OUTCOMES | TRIAGE_OUTCOMES | {"legacy_peer", "abandoned", "internal_error"} +# Seconds a node's own times may lead the sample time: the sampler takes the +# time, then makes up to three RPC calls of at most 60 s each. +CLOCK_SLACK = 300 +MAX_TIME = 253402300799 # 9999-12-31T23:59:59Z, the latest time the report can print +RINGS = ("inbound", "outbound") +TRANSPORT_STATUSES = {"pending", "hybrid", "legacy_peer", "fallback", "off", "v1"} # getpeerinfo transport_pq_status +ENDPOINT_KINDS = {"address", "name_proxy"} +NETWORKS = {"ipv4", "ipv6", "onion", "i2p", "cjdns", "name_proxy"} +LABEL = re.compile(r"[A-Za-z0-9._-]+") # use with fullmatch +TOKEN = re.compile(r"[a-z_-]{1,40}") # a vocabulary word, such as a reason or a connection type +REMEDY = ("an automatic connection held the pinned address in {count} sample(s), so the -addnode thread " + "(GetAddedNodeInfo) skipped it. Remedy: run `qbit-cli disconnectnode ` once; " + "the -addnode thread reconnects it as manual within about 60 s.") + + +def parse_time(text: str) -> int: + """Epoch seconds from Unix seconds or an ISO 8601 time such as 2026-10-20T00:00:00Z (UTC unless it says otherwise).""" + text = text.strip() + if re.fullmatch(r"[0-9]+", text): + value = int(text) + else: + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + except ValueError: + raise ValueError(f"expected Unix seconds or an ISO 8601 time such as 2026-11-02T00:00:00Z, got {text!r}") from None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + value = int(parsed.timestamp()) + if not is_time(value): + raise ValueError(f"expected a time from 1970 to 9999, got {text!r}") + return value + + +def utc(epoch: int | float) -> str: + return datetime.fromtimestamp(epoch, timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def is_count(value: Any) -> TypeGuard[int]: + """Whether a JSON value is a non-negative integer (not a boolean).""" + return isinstance(value, int) and not isinstance(value, bool) and value >= 0 + + +def is_time(value: Any) -> TypeGuard[int]: + """Whether a JSON value is Unix seconds that the report can print.""" + return is_count(value) and value <= MAX_TIME + + +def when(value: Any) -> str: + return utc(value) if is_time(value) else "malformed" + + +def token(value: Any) -> str: + """A vocabulary word for the report, or "malformed": a malformed value may hold an address.""" + return value if isinstance(value, str) and TOKEN.fullmatch(value) else "malformed" + + +def number(value: Any) -> str: + return str(value) if is_count(value) else "malformed" + + +def as_count(value: str | None) -> int | None: + """A CSV value that is a plain non-negative decimal integer, else None.""" + return int(value) if value is not None and re.fullmatch(r"[0-9]{1,18}", value) else None + + +def percent(share: float | None) -> str: + return "n/a" if share is None else f"{100 * share:.1f}%" + + +def expected_samples(begin: int, end: int, interval: int, phase: int | None = None) -> int: + """Samples expected in [begin, end] at phase + k * interval (phase defaults to begin).""" + phase = begin if phase is None else phase + first = phase + -(-(begin - phase) // interval) * interval + return 0 if end < first else (end - first) // interval + 1 + + +@dataclass +class Figure: + name: str + result: str + samples: int + coverage: float | None + summary: str = "" + details: list[str] = field(default_factory=list) + + +@dataclass +class Window: + start: int + end: int + + def contains(self, time: int) -> bool: + return self.start <= time <= self.end + + def overlaps(self, begin: int, finish: int) -> bool: + return begin <= self.end and finish >= self.start + + +def slot_of(time: int, window: Window, interval: int) -> int | None: + """The expected sample window.start + k * interval that a sample at time stands for, as k, or None. + + A sample stands for the expected sample nearest to it; k < 0 is before the + window (the connections baseline). A sample on the other side of the + window's start or end from that expected sample stands for none. + """ + slot = (time - window.start + interval // 2) // interval + if (slot < 0) != (time < window.start) or time > window.end or slot > (window.end - window.start) // interval: + return None + return slot + + +# A sample's state, from the most to the least favorable: several samples that stand for one +# expected sample count as the least favorable of them. Figure 1 uses all four; figure 4 the +# first three, where the first is a connections_in measurement rather than a hybrid link. +HYBRID = MEASURED = 0 +EXCLUDED, UNUSABLE, NOT_HYBRID = 1, 2, 3 + + +# --------------------------------------------------------------------------- +# Restarts +# --------------------------------------------------------------------------- + + +@dataclass +class Restart: + last_seen: int # time of the last sample of the previous run + boot: int # boot time of the new run + + +Row = dict[str, str | None] # a samples.csv row, with NA as None + + +def sample_rows(rows: Iterable[Row]) -> list[tuple[int, Row]]: + """The rows that are samples with their times, in time order, with the values the figures read checked. + + A row without a time in Unix seconds is not a sample: it counts as missing. + A row repeated exactly, as in overlapping or concatenated files, is one + sample, so repeating rows cannot outvote the others' boot time. + An uptime that is not a non-negative integer, or whose boot time (time - + uptime) contradicts the boot time of its instance_id, becomes None: unknown, + never a restart. A process's boot time is the median over its samples, so + neither a malformed uptime nor a restart between the sampler's uptime and + getpqtransportinfo calls (the old uptime with the new instance_id) moves it. + """ + timed = [] + seen: set[frozenset[tuple[str, str | None]]] = set() + for row in rows: + time, key = as_count(row["time"]), frozenset(row.items()) + if is_time(time) and key not in seen: + seen.add(key) + timed.append((time, row)) + timed.sort(key=lambda item: item[0]) + boots: dict[str, list[int]] = defaultdict(list) + for time, row in timed: + uptime, instance = as_count(row["uptime"]), row["instance_id"] + if uptime is not None and valid_instance_id(instance): + boots[instance].append(time - uptime) + boot_of = {instance: sorted(values)[len(values) // 2] for instance, values in boots.items()} + checked = [] + for time, row in timed: + row = dict(row) + uptime, instance = as_count(row["uptime"]), row["instance_id"] + if uptime is not None and valid_instance_id(instance) and abs(time - uptime - boot_of[instance]) > RESTART_TOLERANCE: + uptime = None + row["time"], row["uptime"] = str(time), None if uptime is None else str(uptime) + checked.append((time, row)) + return checked + + +def restarts(rows: list[Row]) -> list[Restart]: + """Restarts seen as a jump in boot time (time - uptime) or a new instance_id.""" + found = [] + previous: tuple[int, int, str | None] | None = None + for time, row in sample_rows(rows): + uptime = as_count(row["uptime"]) + if uptime is None: + continue + boot, instance = time - uptime, row["instance_id"] + if previous is not None: + last_time, last_boot, last_instance = previous + new_instance = instance is not None and last_instance is not None and instance != last_instance + if boot > last_boot + RESTART_TOLERANCE or new_instance: + found.append(Restart(last_seen=last_time, boot=boot)) + previous = (time, boot, instance) + return found + + +def merged_gaps(node_restarts: Iterable[Restart], grace: int) -> list[tuple[int, int]]: + """Restart gaps as merged (start, end] intervals.""" + merged: list[tuple[int, int]] = [] + for low, high in sorted((restart.last_seen, restart.boot + grace) for restart in node_restarts): + if merged and low <= merged[-1][1]: + merged[-1] = (merged[-1][0], max(merged[-1][1], high)) + else: + merged.append((low, high)) + return merged + + +def missing_samples(after: int, before: int, start: int, interval: int, gaps: list[tuple[int, int]]) -> tuple[int, int]: + """(unknown, excluded) counts of the expected samples start + k * interval with after < k < before.""" + count = before - after - 1 + if count <= 0: + return 0, 0 + excluded = 0 + for low, high in gaps: + # Expected times inside (low, high]. + first = max(after + 1, (low - start) // interval + 1) + last = min(before - 1, (high - start) // interval) + excluded += max(0, last - first + 1) + return count - excluded, excluded + + +# --------------------------------------------------------------------------- +# Figure 1: the pinned link +# --------------------------------------------------------------------------- + + +def pinned_peer_ok(row: Row) -> bool: + """Whether a connected pinned peer's fields are well formed and agree: transport_pq is 1 + exactly when transport_pq_status is hybrid.""" + connection_type, transport_pq, status = (row[name] for name in + ("pinned_connection_type", "pinned_transport_pq", "pinned_transport_pq_status")) + return (connection_type is not None and TOKEN.fullmatch(connection_type) is not None and transport_pq in ("0", "1") + and status in TRANSPORT_STATUSES and (transport_pq == "1") == (status == "hybrid")) + + +def evaluate_pinned(pool: list[Row] | None, archive: list[Row] | None, + window: Window, interval: int, grace: int, min_coverage: float) -> Figure: + name = "pinned link manual and hybrid in >= 99% of samples, excluding restarts" + if not pool: + return Figure(name, UNKNOWN, 0, None, "no pool node samples") + gaps = merged_gaps(restarts(pool) + (restarts(archive) if archive else []), grace) + # Each expected sample once: several samples for it count as the least favorable. + states: dict[int, int] = {} + automatic: set[int] = set() + for time, row in sample_rows(pool): + slot = slot_of(time, window, interval) + if slot is None or slot < 0: + continue + uptime = as_count(row["uptime"]) + if any(low < time <= high for low, high in gaps) or (uptime is not None and uptime < grace): + state = EXCLUDED + elif row["sample_ok"] != "1" or row["pinned_present"] not in ("0", "1"): + state = UNUSABLE + elif row["pinned_present"] == "0": + state = NOT_HYBRID + elif not pinned_peer_ok(row): + state = UNUSABLE + elif row["pinned_connection_type"] == "manual" and row["pinned_transport_pq"] == "1": + state = HYBRID + else: + state = NOT_HYBRID + if row["pinned_connection_type"] != "manual": + automatic.add(slot) + states[slot] = max(states.get(slot, state), state) + good, excluded, unknown, bad = (sum(state == value for state in states.values()) + for value in (HYBRID, EXCLUDED, UNUSABLE, NOT_HYBRID)) + # Expected samples without a sample are unknown, or excluded inside a restart gap. + previous = -1 + for slot in sorted(states) + [(window.end - window.start) // interval + 1]: + missing, skipped = missing_samples(previous, slot, window.start, interval, gaps) + unknown += missing + excluded += skipped + previous = slot + total = good + bad + unknown + if total == 0: + return Figure(name, UNKNOWN, 0, None, f"no samples outside restarts (excluded={excluded})") + coverage = (good + bad) / total + if coverage < min_coverage: + result = UNKNOWN + elif good / total >= PINNED_THRESHOLD: + result = PASS + elif (good + unknown) / total < PINNED_THRESHOLD: + result = FAIL + else: + result = UNKNOWN + figure = Figure(name, result, total, coverage, + f"hybrid={good} not_hybrid={bad} unknown={unknown} excluded={excluded} ({100 * good / total:.2f}% hybrid)") + if automatic: + figure.details.append(REMEDY.format(count=len(automatic))) + return figure + + +# --------------------------------------------------------------------------- +# Figures 2 and 3: failure entries between known-good endpoints +# --------------------------------------------------------------------------- + + +def canonical_address(address: str) -> str: + """An IP address in one spelling, compressed, and an IPv4-mapped IPv6 address + as IPv4, as the node reports it; any other address lower-cased.""" + address = address.strip().lower() + try: + ip = ipaddress.ip_address(address) + except ValueError: + return address + if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None: + ip = ip.ipv4_mapped + return str(ip) + + +def endpoint_key(address: str, port: Any) -> str: + address = canonical_address(address) + return f"[{address}]:{port}" if ":" in address else f"{address}:{port}" + + +def valid_endpoint(endpoint: Any) -> bool: + """Whether endpoint has the getpqtransportinfo Endpoint shape: a kind and network that + agree, an address of that network without a port, and a port from 0 to 65535.""" + if not isinstance(endpoint, dict): + return False + kind, network, address, port = (endpoint.get(key) for key in ("kind", "network", "address", "port")) + if not (isinstance(kind, str) and isinstance(network, str) and isinstance(address, str)): + return False + if not is_count(port) or port > 65535: + return False + if kind not in ENDPOINT_KINDS or network not in NETWORKS or (kind == "name_proxy") != (network == "name_proxy"): + return False + if network in ("ipv4", "ipv6", "cjdns"): + try: + return ipaddress.ip_address(address).version == (4 if network == "ipv4" else 6) + except ValueError: + return False + return LABEL.fullmatch(address) is not None + + +class KnownGood: + """Known-good endpoints, each with a label that the report prints instead of the address. + + An entry is address:port, or an address alone to match any port: inbound + failures carry the peer's source port, which changes per connection. + """ + + def __init__(self, endpoints: dict[str, str], addresses: dict[str, str]) -> None: + self.endpoints = endpoints # address:port -> label + self.addresses = addresses # address -> label + + def __bool__(self) -> bool: + return bool(self.endpoints or self.addresses) + + def label(self, endpoint: Any) -> str | None: + """The label of a known-good endpoint, else None.""" + if not isinstance(endpoint, dict) or not isinstance(endpoint.get("address"), str): + return None + key = endpoint_key(endpoint["address"], endpoint.get("port")) + return self.endpoints.get(key) or self.addresses.get(canonical_address(endpoint["address"])) + + +def read_known_good(path: Path) -> KnownGood: + """One endpoint per line, then an optional label: address:port, [ipv6]:port, an address or [ipv6]. + + # starts a comment. Unlabeled entries are known-good-1, known-good-2, ... in file + order. An IP address matches however it is written. Errors name the line, never + its address. + """ + endpoints: dict[str, str] = {} + addresses: dict[str, str] = {} + count = 0 + for number, raw in enumerate(path.read_text(encoding="utf8").splitlines(), 1): + fields = raw.split("#", 1)[0].split() + if not fields: + continue + if len(fields) > 2: + raise ValueError(f"{path}:{number}: expected an endpoint and at most one label") + line = fields[0].lower() + if line.startswith("["): + address, close, rest = line[1:].partition("]") + port = rest[1:] if rest.startswith(":") else None + if not close or (rest and port is None): + raise ValueError(f"{path}:{number}: expected [ipv6] or [ipv6]:port") + elif line.count(":") == 1: + address, _, port = line.partition(":") + elif ":" in line: + raise ValueError(f"{path}:{number}: write an IPv6 endpoint as [address] or [address]:port") + else: + address, port = line, None + if not address or (port is not None and not (port.isdigit() and 0 < int(port) < 65536)): + raise ValueError(f"{path}:{number}: expected address or address:port with a port from 1 to 65535") + count += 1 + label = fields[1] if len(fields) == 2 else f"known-good-{count}" + if not LABEL.fullmatch(label): + raise ValueError(f"{path}:{number}: a label is letters, digits, '.', '_' or '-'") + target = addresses if port is None else endpoints + key = canonical_address(address) if port is None else endpoint_key(address, int(port)) + if key in target: + raise ValueError(f"{path}:{number}: endpoint listed twice") + target[key] = label + return KnownGood(endpoints, addresses) + + +def valid_instance_id(value: Any) -> TypeGuard[str]: + """Whether value is the 64-hex-digit instance_id that getpqtransportinfo reports.""" + return isinstance(value, str) and re.fullmatch(r"[0-9a-f]{64}", value) is not None + + +def ring_of(record: dict[str, Any], direction: str) -> dict[str, Any] | None: + """A structurally valid ring: non-negative integer counters and entries with consecutive + sequences from 1 up, ending at last_sequence.""" + rings = record.get("recent_failures") + ring = rings.get(direction) if isinstance(rings, dict) else None + if not isinstance(ring, dict) or not isinstance(ring.get("entries"), list): + return None + last = ring.get("last_sequence") + if not is_count(last) or not is_count(ring.get("dropped")): + return None + sequences: list[Any] = [entry.get("sequence") if isinstance(entry, dict) else None for entry in ring["entries"]] + if any(not is_count(sequence) or sequence == 0 for sequence in sequences): + return None + if any(after != before + 1 for before, after in zip(sequences, sequences[1:])): + return None + if sequences and sequences[-1] != last: + return None + return ring + + +def identity_problem(record: dict[str, Any], started: dict[str, int]) -> str | None: + """Why a record's process identity, its instance_id and since, cannot be used, or None. + + started holds the since of each instance_id seen so far. + """ + instance, since = record.get("instance_id"), record.get("since") + if not valid_instance_id(instance): + return "instance_id missing or malformed" + if not is_time(since) or since > record["time"] + CLOCK_SLACK: + return "since missing, malformed or after the sample" + if started.setdefault(instance, since) != since: + return "since changed within one instance_id" + return None + + +@dataclass +class RingState: + instance: Any + last_sequence: int + dropped: int + time: int + + +def unknown_intervals(records: list[dict[str, Any]], end: int, interval: int) -> list[tuple[str, str, int, int, str]]: + """(host, ring, begin, end, reason) for every stretch whose failure entries may be incomplete.""" + found = [] + by_host: dict[str, list[dict[str, Any]]] = defaultdict(list) + for record in records: + by_host[str(record.get("host"))].append(record) + for host, host_records in by_host.items(): + host_records.sort(key=lambda record: record["time"]) + state: dict[str, RingState] = {} + started: dict[str, int] = {} # instance_id -> since + previous_time: int | None = None + # Time and reason of the latest record without a usable process identity, until a usable one follows. + identity_lost: tuple[int, str] | None = None + for record in host_records: + time = record["time"] + if not isinstance(record.get("fallback_set"), list): + # Missing fallback evidence is unknown, never an empty set. + found.append((host, "fallback_set", time if previous_time is None else previous_time, time, + "fallback_set missing or malformed")) + problem = identity_problem(record, started) + if problem is not None: + # Without the process identity a restart can't be told apart from + # a continuing run, so neither ring is judged across this record. + for direction in RINGS: + found.append((host, direction, time if previous_time is None else previous_time, time, problem)) + state.clear() + identity_lost = (time, problem) + previous_time = time + continue + if identity_lost is not None: + for direction in RINGS: + found.append((host, direction, identity_lost[0], time, identity_lost[1])) + identity_lost = None + instance, since = record["instance_id"], record["since"] + for direction in RINGS: + ring = ring_of(record, direction) + before = state.get(direction) + if ring is None: + found.append((host, direction, before.time if before else time, time, "malformed ring")) + continue + entries = ring["entries"] + oldest_sequence = entries[0]["sequence"] if entries else ring["last_sequence"] + 1 + oldest_time = entries[0]["time"] if entries and is_time(entries[0].get("time")) else time + if before is None or before.instance != instance: + if before is not None: + found.append((host, direction, before.time, max(before.time, since), + "instance_id changed; the previous run's later entries are lost")) + if ring["last_sequence"] > 0 and oldest_sequence > 1: + found.append((host, direction, since, max(since, oldest_time), + f"sequences 1..{oldest_sequence - 1} were dropped before the first sample")) + elif ring["last_sequence"] < before.last_sequence or ring["dropped"] < before.dropped: + found.append((host, direction, before.time, time, "last_sequence or dropped went backwards")) + elif ring["last_sequence"] == before.last_sequence: + if ring["dropped"] != before.dropped: + found.append((host, direction, before.time, time, "dropped changed without new entries")) + elif oldest_sequence > before.last_sequence + 1: + # The ring wrapped past entries never read: they lie between the + # previous sample and the oldest entry still held. + found.append((host, direction, before.time, max(before.time, oldest_time), + f"sequences {before.last_sequence + 1}..{oldest_sequence - 1} were lost when the ring wrapped")) + state[direction] = RingState(instance, ring["last_sequence"], ring["dropped"], time) + previous_time = time + if host_records and host_records[-1]["time"] < end - 2 * interval: + for direction in RINGS: + found.append((host, direction, host_records[-1]["time"], end, "no samples since")) + return found + + +def fingerprint(value: Any) -> str: + """A hashable stand-in for a JSON value.""" + return json.dumps(value, sort_keys=True) + + +def describe(host: str, direction: str, entry: dict[str, Any], label: str) -> str: + return (f"{utc(entry['time'])} {host} {direction} {entry['outcome']}/{token(entry.get('reason'))} " + f"conn_type={token(entry.get('connection_type'))} peer_id={number(entry.get('peer_id'))} known_good={label}") + + +def evaluate_failures(records: list[dict[str, Any]] | None, known_good: KnownGood | None, monitored: list[str], + window: Window, interval: int, min_coverage: float) -> tuple[Figure, Figure]: + monitored_name = "no malformed_record, first_packet_failed or fallback entry between known-good endpoints" + triage_name = "closed_after_switch entries between known-good endpoints (triage, not in the verdict)" + + def both_unknown(reason: str) -> tuple[Figure, Figure]: + return Figure(monitored_name, UNKNOWN, 0, None, reason), Figure(triage_name, UNKNOWN, 0, None, reason) + + if records is None: + return both_unknown("no failures files") + if not known_good: + return both_unknown("no known-good endpoint list") + if not monitored: + return both_unknown("no monitored canary nodes named (--monitored, or the hosts of --pool and --archive)") + + # Every monitored node must have its own evidence: one node's file cannot pass the figure. + expected = expected_samples(window.start, window.end, interval) + slots: dict[str, set[int]] = defaultdict(set) + for record in records: + slot = slot_of(record["time"], window, interval) + if slot is not None and slot >= 0: + slots[str(record.get("host"))].add(slot) + coverage = {host: min(1.0, len(slots[host]) / expected) if expected else 0.0 for host in monitored} + low = [host for host in monitored if coverage[host] < min_coverage] + + seen: set[tuple[Any, ...]] = set() + found_monitored: list[str] = [] + found_triage: list[str] = [] + # Entries that may be between known-good endpoints but cannot be told apart, per figure. + unknown_monitored: list[str] = [] + unknown_triage: list[str] = [] + for record in records: + host = str(record.get("host")) + instance, since = record.get("instance_id"), record.get("since") + for direction in RINGS: + ring = ring_of(record, direction) + for entry in (ring or {}).get("entries", []): + key: tuple[Any, ...] = (host, instance if valid_instance_id(instance) else None, direction, entry["sequence"]) + if key in seen: + continue + seen.add(key) + outcome, time = entry.get("outcome"), entry.get("time") + outcome = outcome if isinstance(outcome, str) else None + # The figures the entry may count towards, as (found, unknown); an + # entry with a malformed outcome may belong to either. + if outcome in MONITORED_OUTCOMES: + targets = [(found_monitored, unknown_monitored)] + elif outcome in TRIAGE_OUTCOMES: + targets = [(found_triage, unknown_triage)] + elif outcome in OUTCOMES: + continue + else: + targets = [(found_monitored, unknown_monitored), (found_triage, unknown_triage)] + # An entry is recorded after its process started and before the sample that read it. + earliest = since - CLOCK_SLACK if is_time(since) else 0 + if not is_time(time) or time > record["time"] + CLOCK_SLACK or time < earliest: + if not window.overlaps(earliest, record["time"] + CLOCK_SLACK): + continue # whenever it was recorded, it was outside the window + problem = f"unknown {host} {direction}: entry {entry['sequence']} has a missing or implausible time" + elif not window.contains(time): + continue + elif outcome not in OUTCOMES: + problem = f"unknown {host} {direction} {utc(time)}: entry {entry['sequence']} has a missing or malformed outcome" + else: + label = known_good.label(entry.get("endpoint")) + if label is not None: + targets[0][0].append(describe(host, direction, entry, label)) + continue + if valid_endpoint(entry.get("endpoint")): + continue + # Not a valid unrelated peer either: the entry may be between known-good endpoints. + problem = (f"unknown {host} {direction} {utc(time)}: {outcome} entry {entry['sequence']} " + "has a missing or malformed endpoint") + for _, unknown in targets: + unknown.append(problem) + if not window.contains(record["time"]): + continue + fallback_set = record.get("fallback_set") + for item in fallback_set if isinstance(fallback_set, list) else []: + endpoint = item.get("endpoint") if isinstance(item, dict) else None + label = known_good.label(endpoint) + if label is None: + key = (host, "fallback_set", fingerprint(item)) + if not valid_endpoint(endpoint) and key not in seen: + seen.add(key) + unknown_monitored.append(f"unknown {host} fallback_set {utc(record['time'])}: " + "an entry without a well-formed endpoint") + continue + key = (host, "fallback_set", label, fingerprint(item.get("entered"))) + if key not in seen: + seen.add(key) + found_monitored.append(f"{when(item.get('entered'))} {host} fallback_set cause={token(item.get('cause'))}/" + f"{token(item.get('reason'))} streak={number(item.get('streak'))} " + f"expires={when(item.get('expires'))} known_good={label}") + + gaps = [gap for gap in unknown_intervals(records, window.end, interval) if window.overlaps(gap[2], gap[3])] + notes = [f"unknown {host} {ring} {utc(max(begin, window.start))}..{utc(min(finish, window.end))}: {reason}" + for host, ring, begin, finish, reason in gaps] + notes += [f"unknown {host}: coverage {percent(coverage[host])} is below {percent(min_coverage)}" for host in low] + samples = sum(len(slots[host]) for host in monitored) + per_host = ",".join(f"{host}:{percent(coverage[host])}" for host in monitored) + + def figure(name: str, found: list[str], unknown: list[str]) -> Figure: + result = FAIL if found else (UNKNOWN if gaps or low or unknown else PASS) + return Figure(name, result, samples, min(coverage.values()), + f"entries={len(found)} unknown_intervals={len(gaps) + len(unknown)} nodes={per_host}", + found + unknown + notes) + + return figure(monitored_name, found_monitored, unknown_monitored), figure(triage_name, found_triage, unknown_triage) + + +# --------------------------------------------------------------------------- +# Figure 4: inbound connections against the control +# --------------------------------------------------------------------------- + + +def evaluate_connections(archive: list[Row] | None, control: list[Row] | None, + canary_start: int, window: Window, interval: int, baseline_ratio: Fraction | None, + min_coverage: float) -> Figure: + """Daily archive/control connections_in ratios, relative to the baseline ratio. + + Samples are paired by the expected sample they stand for. A day is judged + only when its valid pairs cover at least min_coverage of the samples + expected outside restart exclusions; otherwise the day is unknown. + """ + name = "archive connections_in within +-20% of control, relative to the baseline ratio, excluding 48 h after restarts" + if not archive or not control: + return Figure(name, UNKNOWN, 0, None, "needs both archive and control samples") + + def by_slot(rows: list[Row]) -> dict[int, tuple[int, int | None]]: + """(state, connections_in) per expected sample: EXCLUDED within 48 h of a restart. Several + samples for one count as the least favorable, and measurements that disagree as unusable.""" + slots: dict[int, tuple[int, int | None]] = {} + for time, row in sample_rows(rows): + slot = slot_of(time, window, interval) + if slot is None: + continue + uptime, count = as_count(row["uptime"]), as_count(row["connections_in"]) + if uptime is not None and uptime < CONNECTIONS_RESTART_EXCLUSION: + sample: tuple[int, int | None] = (EXCLUDED, None) + elif row["sample_ok"] == "1" and uptime is not None and count is not None: + sample = (MEASURED, count) + else: + sample = (UNUSABLE, None) + held = slots.setdefault(slot, sample) + if held != sample: + slots[slot] = max(held, sample) if held[0] != sample[0] else (UNUSABLE, None) + return slots + + archive_slots, control_slots = by_slot(archive), by_slot(control) + baseline = [0, 0, 0] # archive sum, control sum, pairs + days: dict[str, dict[str, int]] = defaultdict(lambda: {"archive": 0, "control": 0, "pairs": 0, "excluded": 0}) + for slot in sorted(set(archive_slots) | set(control_slots)): + pair = [node_slots.get(slot, (UNUSABLE, None)) for node_slots in (archive_slots, control_slots)] + counts = [count for _, count in pair] + restarted = any(state == EXCLUDED for state, _ in pair) + valid = all(state == MEASURED for state, _ in pair) + time = window.start + slot * interval + if time < canary_start: + if valid: + baseline[0] += counts[0] # type: ignore[operator] + baseline[1] += counts[1] # type: ignore[operator] + baseline[2] += 1 + continue + if slot < 0: + continue + day = days[utc(time)[:10]] + if restarted: + day["excluded"] += 1 + elif valid: + day["archive"] += counts[0] # type: ignore[operator] + day["control"] += counts[1] # type: ignore[operator] + day["pairs"] += 1 + + if baseline_ratio is None: + # Every day is compared against this ratio, so a zero on either side is + # no evidence: the ratio is undefined, or every day divides by zero. + if baseline[2] == 0: + return Figure(name, UNKNOWN, 0, None, "no baseline: no valid paired samples before the canary start") + for node, total in (("archive", baseline[0]), ("control", baseline[1])): + if total == 0: + return Figure(name, UNKNOWN, 0, None, + f"no baseline: {node} connections_in summed to 0 over {baseline[2]} valid pre-canary pairs") + baseline_ratio = Fraction(baseline[0], baseline[1]) + baseline_text = f"baseline_ratio={float(baseline_ratio):.3f} baseline_pairs={baseline[2]}" + else: + baseline_text = f"baseline_ratio={float(baseline_ratio):.3f} (given)" + details = [] + results = [] + pairs = judged_expected = 0 + day_start = datetime.fromtimestamp(window.start, timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0) + while int(day_start.timestamp()) <= window.end: + label = day_start.date().isoformat() + begin = max(window.start, int(day_start.timestamp())) + finish = min(window.end, int((day_start + timedelta(days=1)).timestamp()) - 1) + day_start += timedelta(days=1) + day = days.get(label) or {"archive": 0, "control": 0, "pairs": 0, "excluded": 0} + expected = expected_samples(begin, finish, interval, phase=window.start) - day["excluded"] + if expected <= 0: + details.append(f"{label} excluded: excluded={day['excluded']}") + continue + pairs += day["pairs"] + judged_expected += expected + coverage = min(1.0, day["pairs"] / expected) + counts_text = f"pairs={day['pairs']} expected={expected} excluded={day['excluded']} coverage={percent(coverage)}" + if coverage < min_coverage or day["control"] == 0: + details.append(f"{label} unknown: {counts_text}") + results.append(UNKNOWN) + continue + # Exact, so a day exactly 20% off passes: in floats, 24/10 against a + # baseline of 3 is 0.7999999999999999. + relative = Fraction(day["archive"], day["control"]) / baseline_ratio + state = PASS if abs(relative - 1) <= CONNECTIONS_TOLERANCE else FAIL + results.append(state) + details.append(f"{label} {state}: relative={float(relative):.3f} {counts_text}") + if FAIL in results: + result = FAIL + elif UNKNOWN in results or PASS not in results: + result = UNKNOWN + else: + result = PASS + overall = min(1.0, pairs / judged_expected) if judged_expected else None + return Figure(name, result, pairs, overall, f"{baseline_text} days={len(results)}", details) + + +# --------------------------------------------------------------------------- +# Command line +# --------------------------------------------------------------------------- + + +def verdict(figures: list[Figure]) -> str: + results = [figure.result for figure in figures] + if FAIL in results: + return FAIL + return UNKNOWN if UNKNOWN in results else PASS + + +def time_arg(text: str) -> int: + try: + return parse_time(text) + except ValueError as e: + raise argparse.ArgumentTypeError(str(e)) from None + + +def positive_int(text: str) -> int: + if not text.isdigit() or int(text) == 0: + raise argparse.ArgumentTypeError(f"expected a positive whole number of seconds, got {text!r}") + return int(text) + + +def non_negative_int(text: str) -> int: + if not text.isdigit(): + raise argparse.ArgumentTypeError(f"expected a whole number of seconds, got {text!r}") + return int(text) + + +def share_arg(text: str) -> float: + try: + value = float(text) + except ValueError: + value = -1.0 + if not 0 < value <= 1: + raise argparse.ArgumentTypeError(f"expected a share greater than 0 and at most 1, such as 0.9, got {text!r}") + return value + + +def ratio_arg(text: str) -> Fraction: + """The ratio as written, exactly: 1.2 is 6/5, not the float nearest it.""" + try: + # float() rejects nan and infinity, and keeps the exponent within its range. + value = Fraction(text) if 0 < float(text) < float("inf") else Fraction(0) + except ValueError: + value = Fraction(0) + if value <= 0: + raise argparse.ArgumentTypeError(f"expected a positive ratio, got {text!r}") + return value + + +def file_arg(text: str) -> Path: + path = Path(text) + if not path.is_file(): + raise argparse.ArgumentTypeError(f"no such file: {text}") + return path + + +def role_error(roles: list[tuple[str, Path, list[Row]]]) -> str | None: + """Why the samples given for --pool, --archive and --control are not each one distinct node, or None. + + A role's samples carry one host label; no two roles share a file, a host + label or an instance_id, so no node is compared with itself. + """ + hosts: dict[str, str] = {} + instances: dict[str, str] = {} + for index, (role, path, rows) in enumerate(roles): + for other, other_path, _ in roles[:index]: + if path.samefile(other_path): + return f"--{other} and --{role} are the same file; each role needs its own node's samples" + labels = {row["host"] for row in rows} + if any(label is None or not LABEL.fullmatch(label) for label in labels): + return f"--{role} has a row without a valid host label" + if len(labels) > 1: + return f"--{role} mixes the host labels {', '.join(sorted(map(str, labels)))}; give one node's samples per role" + for label in map(str, labels): + if label in hosts: + return f"--{hosts[label]} and --{role} both have the host label {label}; each role needs its own node" + hosts[label] = role + for instance in {row["instance_id"] for row in rows if row["instance_id"] is not None}: + if instances.setdefault(instance, role) != role: + return f"--{instances[instance]} and --{role} report the same instance_id; each role needs its own node" + return None + + +def hosts_arg(text: str) -> list[str]: + hosts = [host.strip() for host in text.split(",")] + if not all(LABEL.fullmatch(host) for host in hosts) or len(set(hosts)) != len(hosts): + raise argparse.ArgumentTypeError(f"expected distinct comma-separated host labels, got {text!r}") + return hosts + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0], allow_abbrev=False) + parser.add_argument("--canary-start", required=True, type=time_arg, + help="start of the canary window (Unix seconds or ISO 8601); earlier samples are the baseline") + parser.add_argument("--end", type=time_arg, help="end of the window (default: the latest sample)") + parser.add_argument("--pool", type=file_arg, help="the pool node's samples.csv") + parser.add_argument("--archive", type=file_arg, help="the canary archive node's samples.csv") + parser.add_argument("--control", type=file_arg, help="the control archive node's samples.csv") + parser.add_argument("--failures", type=file_arg, action="append", default=[], + help="a canary node's failures.jsonl (repeatable; every monitored node needs one)") + parser.add_argument("--known-good", type=file_arg, help="known-good endpoints with labels, required with --failures") + parser.add_argument("--monitored", type=hosts_arg, + help="host labels whose failures must all be present (default: the hosts of --pool and --archive)") + parser.add_argument("--baseline-ratio", type=ratio_arg, + help="archive/control connections_in ratio to use instead of the samples before --canary-start") + parser.add_argument("--min-coverage", type=share_arg, default=DEFAULT_MIN_COVERAGE, + help="share of expected samples a figure, node or day needs to be judged (default: 0.9)") + parser.add_argument("--interval", type=positive_int, default=300, help="sampling interval in seconds (default: 300)") + parser.add_argument("--restart-grace", type=non_negative_int, default=600, + help="seconds after a restart excluded from the pinned-link figure (default: 600)") + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + if not (args.pool or args.archive or args.control or args.failures): + parser.error("give at least one of --pool, --archive, --control or --failures") + if args.failures and not args.known_good: + parser.error("--failures needs --known-good") + if args.known_good and not args.failures: + parser.error("--known-good is only used with --failures") + if args.monitored and not args.failures: + parser.error("--monitored is only used with --failures") + if args.end is not None and args.end < args.canary_start: + parser.error("--end is before --canary-start") + + try: + pool = pq_canary.read_rows(args.pool) if args.pool else None + archive = pq_canary.read_rows(args.archive) if args.archive else None + control = pq_canary.read_rows(args.control) if args.control else None + records = [record for path in args.failures for record in pq_canary.read_failures(path)] if args.failures else None + known_good = read_known_good(args.known_good) if args.known_good else None + except (OSError, ValueError) as e: + print(f"Error: {e}", file=sys.stderr) + return 2 + error = role_error([(role, path, rows) for role, path, rows in + (("pool", args.pool, pool), ("archive", args.archive, archive), ("control", args.control, control)) + if path is not None and rows is not None]) + if error is not None: + print(f"Error: {error}", file=sys.stderr) + return 2 + for record in records or []: + if not isinstance(record, dict) or not is_time(record.get("time")) or not isinstance(record.get("host"), str) \ + or not LABEL.fullmatch(record["host"]): + print("Error: a failures.jsonl line lacks a time in Unix seconds or a host label", file=sys.stderr) + return 2 + + monitored = args.monitored + if monitored is None: + monitored = sorted({row["host"] for rows in (pool, archive) if rows for row in rows if row["host"]}) + + start = args.canary_start + latest = [time for rows in (pool, archive, control) if rows for time, _ in sample_rows(rows)] + latest += [record["time"] for record in records or []] + end = args.end if args.end is not None else max(latest, default=start) + window = Window(start, end) + + pinned = evaluate_pinned(pool, archive, window, args.interval, args.restart_grace, args.min_coverage) + failures, triage = evaluate_failures(records, known_good, monitored, window, args.interval, args.min_coverage) + connections = evaluate_connections(archive, control, start, window, args.interval, args.baseline_ratio, args.min_coverage) + + print(f"PQ canary evaluation: {utc(start)} to {utc(end)}, minimum coverage {percent(args.min_coverage)}") + for label, rows in (("pool", pool), ("archive", archive), ("control", control)): + if rows is None: + print(f"restarts: {label} no samples") + continue + node_restarts = [restart for restart in restarts(rows) if window.overlaps(restart.last_seen, restart.boot)] + print(f"restarts: {label} {len(node_restarts)}" + "".join(f" {utc(restart.boot)}" for restart in node_restarts)) + print() + for figure in (pinned, failures, triage, connections): + print(f"{figure.result:<8} {figure.name}: samples={figure.samples} coverage={percent(figure.coverage)} {figure.summary}".rstrip()) + for detail in figure.details: + print(f" {detail}") + final = verdict([pinned, failures, connections]) + print() + print(f"canary verdict (figures 1, 2 and 4): {final}") + return {PASS: 0, FAIL: 1, UNKNOWN: 3}[final] + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/contrib/pq-canary/pq-canary-sample.sh b/contrib/pq-canary/pq-canary-sample.sh new file mode 100755 index 0000000000..59bbfe2a99 --- /dev/null +++ b/contrib/pq-canary/pq-canary-sample.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Copyright (c) 2026-present The qbit core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. +# +# Record one post-quantum transport sample for the mainnet canary (#184, R5). +# Run every 5 minutes from cron or a systemd timer on each canary node and on +# the control node; see README.md. +# +# pq-canary-sample.sh --host LABEL --out DIR [--pinned ADDR:PORT] -- QBIT_CLI [ARGS...] +# +# Appends one row to DIR/samples.csv and, when getpqtransportinfo answers, one +# line to DIR/failures.jsonl. A failed call or a missing field is written as NA, +# never 0. --pinned names the archive node the pool node is pinned to; it makes +# getpeerinfo part of the sample. The addresses in failures.jsonl stay on the node. +# Exit status: 0 for a complete sample, 2 when sample_ok=0, 1 on a usage error. + +export LC_ALL=C +set -o nounset -o pipefail + +usage() { + echo "Usage: $0 --host LABEL --out DIR [--pinned ADDR:PORT] -- QBIT_CLI [ARGS...]" >&2 + exit 1 +} + +host="" +out="" +pinned="" +while [ $# -gt 0 ]; do + case "$1" in + --host) [ $# -ge 2 ] || usage; host="$2"; shift 2 ;; + --out) [ $# -ge 2 ] || usage; out="$2"; shift 2 ;; + --pinned) [ $# -ge 2 ] || usage; pinned="$2"; shift 2 ;; + --) shift; break ;; + *) usage ;; + esac +done +[ -n "$host" ] && [ -n "$out" ] && [ $# -gt 0 ] || usage +case "$host" in + *[!A-Za-z0-9._-]*) echo "Error: --host must be letters, digits, '.', '_' or '-'" >&2; exit 1 ;; + NA) echo "Error: --host NA would read back as a missing value; pick another label" >&2; exit 1 ;; +esac +# The pinned peer as getpeerinfo shows it: host:port or [ipv6]:port (I2P peers show port 0). +if [ -n "$pinned" ]; then + if ! [[ "$pinned" =~ ^(\[[0-9A-Fa-f:.]+\]|[A-Za-z0-9.-]+):([0-9]{1,5})$ ]] || [ "${BASH_REMATCH[2]}" -gt 65535 ]; then + echo "Error: --pinned must be ADDRESS:PORT or [IPV6]:PORT, as getpeerinfo shows the pinned peer" >&2 + exit 1 + fi +fi + +mkdir -p "$out" || exit 1 +rpc_dir="$(mktemp -d)" || exit 1 +trap 'rm -rf "$rpc_dir"' EXIT + +# One timestamp for the whole sample. +now="$(date -u +%s)" +utc="$(date -u -d "@$now" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -r "$now" +%Y-%m-%dT%H:%M:%SZ)" + +cli=("$@") +# A hung node must not stall cron: bound each call when timeout(1) exists. +limit=() +if command -v timeout > /dev/null; then + limit=(timeout 60) +fi + +# Run one RPC; a failure leaves a marker so the formatter writes NA, not 0. +rpc() { + local method="$1" + if ! ${limit[@]+"${limit[@]}"} "${cli[@]}" "$method" > "$rpc_dir/$method" 2> "$rpc_dir/$method.err"; then + : > "$rpc_dir/$method.failed" + fi +} + +rpc uptime +rpc getnetworkinfo +rpc getpqtransportinfo +pinned_args=() +if [ -n "$pinned" ]; then + rpc getpeerinfo + pinned_args=(--pinned "$pinned") +fi + +python3 "$(dirname "$0")/pq_canary.py" sample \ + --rpc-dir "$rpc_dir" --out "$out" --host "$host" --time "$now" --utc "$utc" \ + ${pinned_args[@]+"${pinned_args[@]}"} diff --git a/contrib/pq-canary/pq_canary.py b/contrib/pq-canary/pq_canary.py new file mode 100755 index 0000000000..d1042bf711 --- /dev/null +++ b/contrib/pq-canary/pq_canary.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026-present The qbit core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. +"""Sample format shared by pq-canary-sample.sh and pq-canary-eval.py. + +One sample is one CSV row (COLUMNS) plus, when getpqtransportinfo answered, one +JSON line with the raw recent_failures and fallback_set. The rules that keep the +evidence honest: + +* a field that is missing, or whose RPC failed, is written as NA, never 0; + v1.0.0 has neither getnetworkinfo.connections_pq nor getpqtransportinfo; +* so is a value of the wrong type: a count that is not a non-negative integer, + a flag that is not a boolean, an instance_id that is not 64 lowercase hex + digits; true is never written as a count of 1; +* a real 0 stays 0; +* sample_ok is 1 only when uptime, getnetworkinfo and, on the pool node, + getpeerinfo all answered. + +The RPC fields follow the getpqtransportinfo contract in #184. + +Run as ``pq_canary.py sample ...`` by pq-canary-sample.sh, which gathers the RPC +results into files; see that script for the arguments. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import re +import sys +from pathlib import Path +from typing import Any, Iterator + +NA = "NA" + +DIRECTION_COUNTS = ["switched", "legacy_peer", "malformed_record", "first_packet_failed", "abandoned", "internal_error"] +INBOUND_COUNTS = DIRECTION_COUNTS + ["shed"] +OUTBOUND_COUNTS = DIRECTION_COUNTS + ["closed_after_switch", "fallback"] + +COLUMNS = ( + ["time", "utc", "host", "uptime", "version", "sample_ok", + "pq_enabled", "instance_id", "arith_backend", "keccak_backend", "since"] + + [f"in_{name}" for name in INBOUND_COUNTS] + + [f"out_{name}" for name in OUTBOUND_COUNTS] + + ["load_shedding_active", + "in_ring_last_sequence", "in_ring_dropped", "out_ring_last_sequence", "out_ring_dropped", + "connections_in", "connections_pq", + "pinned_present", "pinned_connection_type", "pinned_transport_pq", "pinned_transport_pq_status"] +) + + +def count(value: Any) -> str: + """CSV text for a JSON non-negative integer, else NA.""" + return str(value) if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else NA + + +def flag(value: Any) -> str: + """CSV text for a JSON boolean, 1 or 0, else NA.""" + return ("1" if value else "0") if isinstance(value, bool) else NA + + +def text(value: Any) -> str: + """CSV text for a JSON string, else NA.""" + return value if isinstance(value, str) else NA + + +def instance_id(value: Any) -> str: + """The 64-lowercase-hex-digit instance_id, else NA.""" + return value if isinstance(value, str) and re.fullmatch(r"[0-9a-f]{64}", value) else NA + + +def lookup(document: Any, *path: str) -> Any: + """document[path[0]][path[1]]..., or None if any step is missing or not an object.""" + for key in path: + if not isinstance(document, dict) or key not in document: + return None + document = document[key] + return document + + +def build_row(*, time: int, utc: str, host: str, uptime: Any, netinfo: Any, pqinfo: Any, + peers: Any, pinned: str | None) -> dict[str, str]: + """One CSV row. uptime, netinfo, pqinfo and peers are the parsed RPC results, or None + when the call failed; peers and pinned are None on nodes that do not pin a peer.""" + row = {name: NA for name in COLUMNS} + row.update(time=str(time), utc=utc, host=host) + row["uptime"] = count(uptime) + row["version"] = count(lookup(netinfo, "version")) + row["connections_in"] = count(lookup(netinfo, "connections_in")) + row["connections_pq"] = count(lookup(netinfo, "connections_pq")) + + row["pq_enabled"] = flag(lookup(pqinfo, "enabled")) + row["instance_id"] = instance_id(lookup(pqinfo, "instance_id")) + row["arith_backend"] = text(lookup(pqinfo, "arith_backend")) + row["keccak_backend"] = text(lookup(pqinfo, "keccak_backend")) + row["since"] = count(lookup(pqinfo, "since")) + for name in INBOUND_COUNTS: + row[f"in_{name}"] = count(lookup(pqinfo, "handshakes", "inbound", name)) + for name in OUTBOUND_COUNTS: + row[f"out_{name}"] = count(lookup(pqinfo, "handshakes", "outbound", name)) + row["load_shedding_active"] = flag(lookup(pqinfo, "load_shedding", "active")) + for prefix, direction in (("in", "inbound"), ("out", "outbound")): + row[f"{prefix}_ring_last_sequence"] = count(lookup(pqinfo, "recent_failures", direction, "last_sequence")) + row[f"{prefix}_ring_dropped"] = count(lookup(pqinfo, "recent_failures", direction, "dropped")) + + pool_node = pinned is not None + peers_ok = isinstance(peers, list) + if pool_node and peers_ok: + matches = [peer for peer in peers if isinstance(peer, dict) and peer.get("addr") == pinned] + # Prefer the manual connection if the address is connected more than once. + matches.sort(key=lambda peer: peer.get("connection_type") != "manual") + row["pinned_present"] = "1" if matches else "0" + if matches: + peer = matches[0] + row["pinned_connection_type"] = text(peer.get("connection_type")) + row["pinned_transport_pq"] = flag(peer.get("transport_pq")) + row["pinned_transport_pq_status"] = text(peer.get("transport_pq_status")) + + answered = row["uptime"] != NA and isinstance(netinfo, dict) and (peers_ok or not pool_node) + row["sample_ok"] = "1" if answered else "0" + return row + + +def build_failures_record(*, time: int, host: str, pqinfo: Any) -> dict[str, Any] | None: + """The JSON line kept for the evaluator, or None when getpqtransportinfo did not answer.""" + if not isinstance(pqinfo, dict): + return None + return { + "time": time, + "host": host, + "instance_id": pqinfo.get("instance_id"), + "since": pqinfo.get("since"), + "recent_failures": pqinfo.get("recent_failures"), + "fallback_set": pqinfo.get("fallback_set"), + } + + +def append_row(path: Path, row: dict[str, str]) -> None: + """Append a row, writing the header first when the file is new or empty.""" + new = not path.exists() or path.stat().st_size == 0 + if not new: + with path.open(encoding="utf8", newline="") as existing: + header = next(csv.reader(existing), None) + if header != COLUMNS: + raise ValueError(f"{path} has a different header; move it aside and start a new file") + with path.open("a", encoding="utf8", newline="") as out: + writer = csv.DictWriter(out, fieldnames=COLUMNS, lineterminator="\n") + if new: + writer.writeheader() + writer.writerow(row) + + +def read_rows(path: Path) -> list[dict[str, str | None]]: + """Rows of a samples file, with NA turned into None. + + A row with more or fewer fields than the header, such as one cut short by a + crash mid-write and continued by the next sample, is not a sample: its values + may sit in the wrong columns, so it is left out and counts as missing. + """ + with path.open(encoding="utf8", newline="") as source: + reader = csv.reader(source) + header = next(reader, None) + if header != COLUMNS: + raise ValueError(f"{path}: unexpected header {header}") + return [{key: (None if value == NA else value) for key, value in zip(COLUMNS, fields)} + for fields in reader if len(fields) == len(COLUMNS)] + + +def read_failures(path: Path) -> Iterator[dict[str, Any]]: + with path.open(encoding="utf8") as source: + for number, line in enumerate(source, 1): + if line.strip(): + try: + yield json.loads(line) + except json.JSONDecodeError as e: + raise ValueError(f"{path}:{number}: {e}") from e + + +def load_rpc_result(path: Path) -> Any: + """A parsed RPC result, or None when the call failed (marker file) or printed nothing parsable.""" + if not path.exists() or path.with_name(path.name + ".failed").exists(): + return None + text = path.read_text(encoding="utf8").strip() + try: + return json.loads(text) + except json.JSONDecodeError: + return None + + +def sample_main(args: argparse.Namespace) -> int: + rpc = Path(args.rpc_dir) + uptime = load_rpc_result(rpc / "uptime") + netinfo = load_rpc_result(rpc / "getnetworkinfo") + pqinfo = load_rpc_result(rpc / "getpqtransportinfo") + peers = load_rpc_result(rpc / "getpeerinfo") if args.pinned else None + row = build_row(time=args.time, utc=args.utc, host=args.host, uptime=uptime, netinfo=netinfo, + pqinfo=pqinfo, peers=peers, pinned=args.pinned) + out = Path(args.out) + append_row(out / "samples.csv", row) + record = build_failures_record(time=args.time, host=args.host, pqinfo=pqinfo) + if record is not None: + with (out / "failures.jsonl").open("a", encoding="utf8") as failures: + failures.write(json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n") + return 0 if row["sample_ok"] == "1" else 2 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="PQ canary sample formatting (used by pq-canary-sample.sh).") + commands = parser.add_subparsers(dest="command", required=True) + sample = commands.add_parser("sample", help="append one sample from RPC results gathered in --rpc-dir") + sample.add_argument("--rpc-dir", required=True) + sample.add_argument("--out", required=True) + sample.add_argument("--host", required=True) + sample.add_argument("--time", required=True, type=int) + sample.add_argument("--utc", required=True) + sample.add_argument("--pinned") + args = parser.parse_args(argv) + return sample_main(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/contrib/pq-canary/test/fixtures/getnetworkinfo-v1.0.0.json b/contrib/pq-canary/test/fixtures/getnetworkinfo-v1.0.0.json new file mode 100644 index 0000000000..39b250a518 --- /dev/null +++ b/contrib/pq-canary/test/fixtures/getnetworkinfo-v1.0.0.json @@ -0,0 +1,10 @@ +{ + "version": 10000, + "subversion": "/qbit:1.0.0/", + "protocolversion": 70016, + "localrelay": true, + "networkactive": true, + "connections": 38, + "connections_in": 28, + "connections_out": 10 +} diff --git a/contrib/pq-canary/test/fixtures/getnetworkinfo.json b/contrib/pq-canary/test/fixtures/getnetworkinfo.json new file mode 100644 index 0000000000..2d5f110fda --- /dev/null +++ b/contrib/pq-canary/test/fixtures/getnetworkinfo.json @@ -0,0 +1,11 @@ +{ + "version": 10100, + "subversion": "/qbit:1.1.0/", + "protocolversion": 70016, + "localrelay": true, + "networkactive": true, + "connections": 41, + "connections_in": 30, + "connections_out": 11, + "connections_pq": 12 +} diff --git a/contrib/pq-canary/test/fixtures/getpeerinfo.json b/contrib/pq-canary/test/fixtures/getpeerinfo.json new file mode 100644 index 0000000000..a5222c2312 --- /dev/null +++ b/contrib/pq-canary/test/fixtures/getpeerinfo.json @@ -0,0 +1,18 @@ +[ + { + "id": 3, + "addr": "203.0.113.5:8333", + "connection_type": "manual", + "transport_protocol_type": "v2", + "transport_pq": true, + "transport_pq_status": "hybrid" + }, + { + "id": 4, + "addr": "198.51.100.20:8333", + "connection_type": "outbound-full-relay", + "transport_protocol_type": "v2", + "transport_pq": false, + "transport_pq_status": "legacy_peer" + } +] diff --git a/contrib/pq-canary/test/fixtures/getpqtransportinfo.json b/contrib/pq-canary/test/fixtures/getpqtransportinfo.json new file mode 100644 index 0000000000..e0f8fba06d --- /dev/null +++ b/contrib/pq-canary/test/fixtures/getpqtransportinfo.json @@ -0,0 +1,89 @@ +{ + "enabled": true, + "status": "enabled", + "instance_id": "5e1f0c3a9b7d4e2f8a6c1b3d5f7e9a0c2e4f6a8b0d1c3e5f7a9b0c2d4e6f8a1b", + "arith_backend": "x86_64-avx2", + "keccak_backend": "x86_64-avx2", + "since": 1760000000, + "handshakes": { + "inbound": { + "switched": 120, + "legacy_peer": 30, + "malformed_record": 1, + "first_packet_failed": 0, + "abandoned": 2, + "internal_error": 0, + "shed": 0 + }, + "outbound": { + "switched": 40, + "legacy_peer": 8, + "malformed_record": 0, + "first_packet_failed": 1, + "abandoned": 0, + "internal_error": 0, + "closed_after_switch": 1, + "fallback": 0 + } + }, + "load_shedding": { + "active": false, + "threshold_per_s": 1000, + "since": 0 + }, + "fallback_set": [], + "failure_streaks": [ + { + "endpoint": {"kind": "address", "network": "ipv4", "address": "203.0.113.9", "port": 8333}, + "cause": "first_packet_failed", + "reason": "tag", + "streak": 1, + "last_failure": 1760003000, + "next_window_seconds": 3600 + } + ], + "recent_failures": { + "inbound": { + "last_sequence": 1, + "dropped": 0, + "entries": [ + { + "sequence": 1, + "time": 1760001000, + "endpoint": {"kind": "address", "network": "ipv4", "address": "198.51.100.77", "port": 51234}, + "direction": "inbound", + "connection_type": "inbound", + "peer_id": 17, + "outcome": "malformed_record", + "reason": "ek_length" + } + ] + }, + "outbound": { + "last_sequence": 2, + "dropped": 0, + "entries": [ + { + "sequence": 1, + "time": 1760003000, + "endpoint": {"kind": "address", "network": "ipv4", "address": "203.0.113.9", "port": 8333}, + "direction": "outbound", + "connection_type": "outbound-full-relay", + "peer_id": 21, + "outcome": "first_packet_failed", + "reason": "tag" + }, + { + "sequence": 2, + "time": 1760004000, + "endpoint": {"kind": "address", "network": "ipv6", "address": "2001:db8::5", "port": 8333}, + "direction": "outbound", + "connection_type": "block-relay-only", + "peer_id": 25, + "outcome": "closed_after_switch", + "reason": "eof" + } + ] + } + } +} diff --git a/contrib/pq-canary/test/fixtures/known-good.txt b/contrib/pq-canary/test/fixtures/known-good.txt new file mode 100644 index 0000000000..723eabf617 --- /dev/null +++ b/contrib/pq-canary/test/fixtures/known-good.txt @@ -0,0 +1,7 @@ +# endpoint [label]: the report prints the label, never the address. +# The archive node, as the pool node dials it. +203.0.113.5:8333 archive +# The pool node, as the archive node sees its inbound connection (any source port). +198.51.100.7 pool +# An IPv6 canary peer, unlabeled: it is reported as known-good-3. +[2001:db8::5]:8333 diff --git a/contrib/pq-canary/test/test_pq_canary.py b/contrib/pq-canary/test/test_pq_canary.py new file mode 100755 index 0000000000..e6c2b05d52 --- /dev/null +++ b/contrib/pq-canary/test/test_pq_canary.py @@ -0,0 +1,1348 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026-present The qbit core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. +"""Tests for the PQ canary sampler and evaluator (contrib/pq-canary). + +The RPC fixtures in fixtures/ follow the getpqtransportinfo contract in #184. +Standard library only, so the Required Merge Gate can run it as is. +""" + +from __future__ import annotations + +import hashlib +import importlib.util +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from fractions import Fraction +from pathlib import Path +from typing import Any + +HERE = Path(__file__).resolve().parent +TOOL_DIR = HERE.parent +FIXTURES = HERE / "fixtures" +SAMPLER = TOOL_DIR / "pq-canary-sample.sh" +EVALUATOR = TOOL_DIR / "pq-canary-eval.py" + +sys.path.insert(0, str(TOOL_DIR)) +import pq_canary # noqa: E402 + +_spec = importlib.util.spec_from_file_location("pq_canary_eval", EVALUATOR) +assert _spec is not None and _spec.loader is not None +pq_eval = importlib.util.module_from_spec(_spec) +sys.modules[_spec.name] = pq_eval # dataclasses look their module up while it loads +_spec.loader.exec_module(pq_eval) + +NA = pq_canary.NA +PINNED = "203.0.113.5:8333" +POOL_ADDRESS = "198.51.100.7" +T0 = 1_760_000_000 # canary start +INTERVAL = 300 +DAY = 86_400 +LONG_UPTIME = 30 * DAY + + +def fixture(name: str) -> Any: + return json.loads((FIXTURES / name).read_text(encoding="utf8")) + + +def parsed(row: dict[str, str]) -> dict[str, str | None]: + """A row as read_rows returns it: NA becomes None.""" + return {key: (None if value == NA else value) for key, value in row.items()} + + +class Node: + """A simulated node producing samples the way pq-canary-sample.sh records them.""" + + def __init__(self, host: str, *, boot: int, pinned: str | None = None, v1_0_0: bool = False) -> None: + self.host = host + self.boot = boot + self.pinned = pinned + self.v1_0_0 = v1_0_0 + self.instance = 0 + + def restart(self, boot: int) -> None: + self.boot = boot + self.instance += 1 + + def pqinfo(self, time: int) -> dict[str, Any] | None: + """getpqtransportinfo at time: the rings hold the fixture's failures recorded by then.""" + if self.v1_0_0: + return None + info = fixture("getpqtransportinfo.json") + # Random per process start on a real node; distinct per node and run here. + info["instance_id"] = hashlib.sha256(f"{self.host}/{self.instance}".encode()).hexdigest() + info["since"] = self.boot + for failures in info["recent_failures"].values(): + failures["entries"] = [item for item in failures["entries"] if item["time"] <= time] + failures["last_sequence"] = failures["entries"][-1]["sequence"] if failures["entries"] else 0 + return info + + def sample(self, time: int, *, down: bool = False, connections_in: int | None = 30, + pinned_peer: dict[str, Any] | None | str = "manual-hybrid") -> dict[str, str | None]: + netinfo = None if down else fixture("getnetworkinfo-v1.0.0.json" if self.v1_0_0 else "getnetworkinfo.json") + if netinfo is not None: + if connections_in is None: + del netinfo["connections_in"] + else: + netinfo["connections_in"] = connections_in + peers = None + if self.pinned and not down: + peers = [peer for peer in fixture("getpeerinfo.json") if peer["addr"] != PINNED] + if pinned_peer == "manual-hybrid": + peers.append({"addr": self.pinned, "connection_type": "manual", "transport_pq": True, "transport_pq_status": "hybrid"}) + elif isinstance(pinned_peer, dict): + peers.append(dict(pinned_peer, addr=self.pinned)) + row = pq_canary.build_row( + time=time, utc=pq_eval.utc(time), host=self.host, + uptime=None if down else time - self.boot, + netinfo=netinfo, pqinfo=None if down else self.pqinfo(time), peers=peers, pinned=self.pinned) + return parsed(row) + + +def window(count: int) -> Any: + return pq_eval.Window(T0, T0 + (count - 1) * INTERVAL) + + +def entry(sequence: int, time: int, address: str, port: int, outcome: str, *, direction: str = "outbound", + reason: str = "tag", connection_type: str = "manual") -> dict[str, Any]: + network = "ipv6" if ":" in address else "ipv4" + return {"sequence": sequence, "time": time, "direction": direction, "connection_type": connection_type, + "peer_id": sequence, "outcome": outcome, "reason": reason, + "endpoint": {"kind": "address", "network": network, "address": address, "port": port}} + + +def ring(last_sequence: Any, dropped: Any, entries: list[Any]) -> dict[str, Any]: + return {"last_sequence": last_sequence, "dropped": dropped, "entries": entries} + + +def record(time: int, *, inbound: Any, outbound: Any, host: str = "archive", + instance: Any = "a" * 64, since: Any = T0 - DAY, fallback_set: list[Any] | None = None) -> dict[str, Any]: + return {"time": time, "host": host, "instance_id": instance, "since": since, + "recent_failures": {"inbound": inbound, "outbound": outbound}, "fallback_set": fallback_set or []} + + +def known_good() -> Any: + return pq_eval.read_known_good(FIXTURES / "known-good.txt") + + +# --------------------------------------------------------------------------- +# Sampling +# --------------------------------------------------------------------------- + + +class SampleRowTest(unittest.TestCase): + def test_full_schema_fills_every_column(self) -> None: + row = pq_canary.build_row(time=T0, utc="2025-10-09T08:53:20Z", host="pool", uptime=1234, + netinfo=fixture("getnetworkinfo.json"), pqinfo=fixture("getpqtransportinfo.json"), + peers=fixture("getpeerinfo.json"), pinned=PINNED) + self.assertEqual(list(row), pq_canary.COLUMNS) + self.assertNotIn(NA, row.values()) + self.assertEqual(row["sample_ok"], "1") + self.assertEqual((row["pq_enabled"], row["load_shedding_active"]), ("1", "0")) + self.assertEqual((row["in_switched"], row["out_closed_after_switch"], row["out_fallback"]), ("120", "1", "0")) + self.assertEqual((row["in_ring_last_sequence"], row["out_ring_last_sequence"], row["out_ring_dropped"]), ("1", "2", "0")) + self.assertEqual((row["connections_in"], row["connections_pq"]), ("30", "12")) + self.assertEqual((row["pinned_present"], row["pinned_connection_type"], row["pinned_transport_pq"], + row["pinned_transport_pq_status"]), ("1", "manual", "1", "hybrid")) + # A real zero is a measurement, not a missing value. + self.assertEqual(row["in_internal_error"], "0") + self.assertNotIn(PINNED, ",".join(row.values()), "addresses stay out of the CSV") + + def test_v1_0_0_writes_na_and_a_complete_sample(self) -> None: + row = Node("control", boot=T0 - DAY, v1_0_0=True).sample(T0) + self.assertEqual(row["sample_ok"], "1") + self.assertEqual(row["version"], "10000") + self.assertEqual(row["connections_in"], "30") + new_fields = [name for name in pq_canary.COLUMNS if name not in + ("time", "utc", "host", "uptime", "version", "sample_ok", "connections_in")] + self.assertEqual({name: row[name] for name in new_fields}, {name: None for name in new_fields}) + + def test_failed_or_missing_values_are_na_never_zero(self) -> None: + down = Node("pool", boot=T0 - DAY, pinned=PINNED).sample(T0, down=True) + self.assertEqual(down["sample_ok"], "0") + self.assertIsNone(down["uptime"]) + self.assertIsNone(down["connections_in"]) + self.assertIsNone(down["pinned_present"]) + + info = fixture("getpqtransportinfo.json") + del info["handshakes"]["inbound"]["shed"] + info["recent_failures"]["outbound"] = "not a ring" + row = pq_canary.build_row(time=T0, utc="x", host="archive", uptime=10, netinfo={"version": 10100}, + pqinfo=info, peers=None, pinned=None) + self.assertEqual(row["in_shed"], NA) + self.assertEqual(row["out_ring_last_sequence"], NA) + self.assertEqual(row["connections_in"], NA) + self.assertEqual(row["in_switched"], "120") + self.assertEqual(row["sample_ok"], "1", "getpeerinfo is only part of the pool node's sample") + + # The pool node needs getpeerinfo; a pinned peer that is not connected is a 0, not NA. + pool = Node("pool", boot=T0 - DAY, pinned=PINNED) + absent = pool.sample(T0, pinned_peer=None) + self.assertEqual((absent["sample_ok"], absent["pinned_present"], absent["pinned_connection_type"]), ("1", "0", None)) + row = pq_canary.build_row(time=T0, utc="x", host="pool", uptime=10, netinfo={}, pqinfo=None, peers=None, pinned=PINNED) + self.assertEqual((row["sample_ok"], row["pinned_present"]), ("0", NA)) + + def test_values_of_the_wrong_type_are_na(self) -> None: + info = fixture("getpqtransportinfo.json") + info.update(enabled="yes", instance_id=info["instance_id"].upper(), arith_backend=3, since=-5) + info["handshakes"]["inbound"]["switched"] = True + info["handshakes"]["outbound"]["fallback"] = -1 + info["handshakes"]["outbound"]["switched"] = "40" + info["load_shedding"]["active"] = 1 + info["recent_failures"]["inbound"]["last_sequence"] = 1.0 + netinfo = dict(fixture("getnetworkinfo.json"), version="10100", connections_in=True, connections_pq=-2) + peers = [{"addr": PINNED, "connection_type": 5, "transport_pq": "true", "transport_pq_status": ["hybrid"]}] + wrong = ["uptime", "version", "connections_in", "connections_pq", "pq_enabled", "instance_id", "arith_backend", "since", + "in_switched", "out_fallback", "out_switched", "load_shedding_active", "in_ring_last_sequence", + "pinned_connection_type", "pinned_transport_pq", "pinned_transport_pq_status"] + for uptime in (-1, True, 12.5, "4300"): + with self.subTest(uptime=uptime): + row = pq_canary.build_row(time=T0, utc="x", host="pool", uptime=uptime, netinfo=netinfo, pqinfo=info, + peers=peers, pinned=PINNED) + self.assertEqual({name: row[name] for name in wrong}, dict.fromkeys(wrong, NA)) + # The values of the right type are kept. + self.assertEqual((row["keccak_backend"], row["in_legacy_peer"], row["out_ring_dropped"], row["pinned_present"]), + ("x86_64-avx2", "30", "0", "1")) + self.assertEqual(row["sample_ok"], "0", "a sample without a valid uptime is incomplete") + + def test_files(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "samples.csv" + row = pq_canary.build_row(time=T0, utc="x", host="pool", uptime=1, netinfo={}, pqinfo=None, peers=None, pinned=None) + pq_canary.append_row(path, row) + pq_canary.append_row(path, row) + lines = path.read_text(encoding="utf8").splitlines() + self.assertEqual(lines[0], ",".join(pq_canary.COLUMNS)) + self.assertEqual(len(lines), 3) + self.assertEqual(len(pq_canary.read_rows(path)), 2) + path.write_text("time,host\n1,pool\n", encoding="utf8") + with self.assertRaises(ValueError): + pq_canary.append_row(path, row) + + def test_rows_with_the_wrong_number_of_fields_are_not_samples(self) -> None: + pool = Node("pool", boot=T0 - DAY, pinned=PINNED) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "samples.csv" + for index in range(3): + pq_canary.append_row(path, {key: NA if value is None else value for key, value in pool.sample(T0 + index * INTERVAL).items()}) + header, first, second, third = path.read_text(encoding="utf8").splitlines() + # A crash cut the second row short and the next sample continued its line; + # a later crash left a short last line. + path.write_text("\n".join([header, first, second[:len(second) // 2] + third, first[:40]]) + "\n", encoding="utf8") + rows = pq_canary.read_rows(path) + self.assertEqual([row["time"] for row in rows], [str(T0)]) + + +@unittest.skipIf(shutil.which("bash") is None, "needs bash") +class SamplerScriptTest(unittest.TestCase): + """pq-canary-sample.sh end to end, with a fake qbit-cli serving the fixtures.""" + + FAKE_CLI = ( + "import json, os, sys\n" + "method = sys.argv[-1]\n" + "path = os.path.join(os.environ['FAKE_RPC'], method)\n" + "if not os.path.exists(path):\n" + " print('error code: -32601\\nerror message:\\nMethod not found', file=sys.stderr)\n" + " sys.exit(89)\n" + "print(open(path, encoding='utf8').read())\n" + "sys.exit(int(os.environ.get('FAKE_EXIT_' + method, '0')))\n" + ) + + def run_sampler(self, root: Path, served: dict[str, Any], *args: str, exits: dict[str, int] | None = None) -> subprocess.CompletedProcess[str]: + rpc = root / "rpc" + shutil.rmtree(rpc, ignore_errors=True) + rpc.mkdir() + for method, result in served.items(): + (rpc / method).write_text(json.dumps(result), encoding="utf8") + cli = root / "fake_cli.py" + cli.write_text(self.FAKE_CLI, encoding="utf8") + env = dict(os.environ, FAKE_RPC=str(rpc), **{f"FAKE_EXIT_{method}": str(code) for method, code in (exits or {}).items()}) + return subprocess.run(["bash", str(SAMPLER), *args, "--", sys.executable, str(cli), "-datadir=/nonexistent"], + env=env, text=True, capture_output=True) + + def test_samples(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + out = root / "out" + v1_0_0 = {"uptime": 4000, "getnetworkinfo": fixture("getnetworkinfo-v1.0.0.json")} + result = self.run_sampler(root, v1_0_0, "--host", "control", "--out", str(out)) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse((out / "failures.jsonl").exists()) + + full = {"uptime": 4300, "getnetworkinfo": fixture("getnetworkinfo.json"), + "getpqtransportinfo": fixture("getpqtransportinfo.json"), "getpeerinfo": fixture("getpeerinfo.json")} + result = self.run_sampler(root, full, "--host", "control", "--out", str(out)) + self.assertEqual(result.returncode, 0, result.stderr) + + result = self.run_sampler(root, {}, "--host", "control", "--out", str(out)) + self.assertEqual(result.returncode, 2, "an unreachable node is an incomplete sample") + + # A call that fails is NA even if it printed something that parses. + result = self.run_sampler(root, full, "--host", "control", "--out", str(out), exits={"uptime": 1}) + self.assertEqual(result.returncode, 2) + + rows = pq_canary.read_rows(out / "samples.csv") + self.assertEqual([row["sample_ok"] for row in rows], ["1", "1", "0", "0"]) + self.assertEqual([row["connections_pq"] for row in rows], [None, "12", None, "12"]) + self.assertEqual([row["instance_id"] is None for row in rows], [True, False, True, False]) + self.assertEqual([row["uptime"] for row in rows], ["4000", "4300", None, None]) + records = list(pq_canary.read_failures(out / "failures.jsonl")) + self.assertEqual(len(records), 2) + self.assertEqual(records[0]["recent_failures"], fixture("getpqtransportinfo.json")["recent_failures"]) + self.assertEqual(records[0]["host"], "control") + + result = self.run_sampler(root, full, "--host", "pool", "--out", str(root / "pool"), "--pinned", PINNED) + self.assertEqual(result.returncode, 0, result.stderr) + for pinned in ("[2001:db8::5]:8333", "example.b32.i2p:0"): + result = self.run_sampler(root, full, "--host", "pool", "--out", str(root / "other"), "--pinned", pinned) + self.assertEqual(result.returncode, 0, result.stderr) + (row,) = pq_canary.read_rows(root / "pool" / "samples.csv") + self.assertEqual((row["pinned_present"], row["pinned_connection_type"], row["pinned_transport_pq"]), ("1", "manual", "1")) + + def test_usage_errors(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + bad_pinned = [("--host", "pool", "--out", tmp, "--pinned", value) + for value in ("pool-node", "203.0.113.5", "[2001:db8::5", "203.0.113.5:99999", "a b:8333", ":8333")] + for args in ((), ("--host", "pool"), ("--host", "bad label", "--out", tmp), ("--host", "NA", "--out", tmp), + ("--host", "pool", "--out", tmp, "--bogus"), + *bad_pinned): + result = subprocess.run(["bash", str(SAMPLER), *args, "--", "true"], text=True, capture_output=True) + self.assertEqual(result.returncode, 1, args) + self.assertFalse((root / "samples.csv").exists()) + + +# --------------------------------------------------------------------------- +# Figure 1: the pinned link +# --------------------------------------------------------------------------- + + +class PinnedLinkTest(unittest.TestCase): + def evaluate(self, pool_rows: list[dict[str, str | None]], count: int, archive_rows: Any = None, + min_coverage: float = 0.9) -> Any: + return pq_eval.evaluate_pinned(pool_rows, archive_rows, window(count), INTERVAL, 600, min_coverage) + + def test_pass(self) -> None: + pool = Node("pool", boot=T0 - DAY, pinned=PINNED) + figure = self.evaluate([pool.sample(T0 + i * INTERVAL) for i in range(200)], 200) + self.assertEqual((figure.result, figure.samples), ("pass", 200)) + + def test_restart_is_a_gap(self) -> None: + pool = Node("pool", boot=T0 - DAY, pinned=PINNED) + rows = [] + for i in range(100): + time = T0 + i * INTERVAL + if i in (40, 41): + rows.append(pool.sample(time, down=True)) # the node is restarting + continue + if i == 42: + pool.restart(time - 30) + rows.append(pool.sample(time)) + figure = self.evaluate(rows, 100) + self.assertEqual(figure.result, "pass", figure.summary) + self.assertIn("excluded=4", figure.summary) # two down, then uptime 30 s and 330 s < 600 s grace + self.assertEqual(figure.samples, 96) + self.assertEqual(len(pq_eval.restarts(rows)), 1) + + def test_archive_restart_is_a_gap_for_the_pool(self) -> None: + pool = Node("pool", boot=T0 - DAY, pinned=PINNED) + archive = Node("archive", boot=T0 - DAY) + pool_rows, archive_rows = [], [] + for i in range(50): + time = T0 + i * INTERVAL + if i == 25: + archive.restart(time - 60) + pool_rows.append(pool.sample(time, pinned_peer=None if i == 25 else "manual-hybrid")) + archive_rows.append(archive.sample(time)) + # Without the archive node's samples the dropped link is a real miss: 49 of 50 is below 99%. + self.assertEqual(self.evaluate(pool_rows, 50).result, "fail") + self.assertEqual(self.evaluate(pool_rows, 50, archive_rows).result, "pass") + + def test_rpc_outage_is_unknown(self) -> None: + pool = Node("pool", boot=T0 - DAY, pinned=PINNED) + rows = [pool.sample(T0 + i * INTERVAL, down=i in (10, 11)) for i in range(100)] + figure = self.evaluate(rows, 100) + self.assertEqual(figure.result, "unknown", figure.summary) + self.assertIn("unknown=2", figure.summary) + # Missing rows, such as a stopped cron, are unknown too. + figure = self.evaluate([row for index, row in enumerate(rows) if index not in (10, 11)], 100) + self.assertEqual(figure.result, "unknown", figure.summary) + self.assertIn("unknown=2", figure.summary) + + def test_missing_rows_are_counted_not_listed(self) -> None: + pool = Node("pool", boot=T0 - DAY, pinned=PINNED) + rows = [pool.sample(T0 + i * INTERVAL) for i in range(100)] + # A window starting long before the sampler did is mostly unknown, and cheap to evaluate. + early = pq_eval.evaluate_pinned(rows, None, pq_eval.Window(0, T0 + 99 * INTERVAL), INTERVAL, 600, 0.9) + self.assertEqual(early.result, "unknown") + self.assertIn(f"unknown={int(T0 / INTERVAL + 0.5)}", early.summary) # samples at 0, 300, ... before T0 + # Rows missing because the whole machine was down for a restart are part of the gap. + pool.restart(T0 + 50 * INTERVAL - 30) + rows = rows[:40] + [pool.sample(T0 + i * INTERVAL) for i in range(50, 100)] + figure = self.evaluate(rows, 100) + self.assertEqual(figure.result, "pass", figure.summary) + self.assertIn("unknown=0 excluded=12", figure.summary) + + def test_automatic_connection_fails_with_remedy(self) -> None: + pool = Node("pool", boot=T0 - DAY, pinned=PINNED) + automatic = {"connection_type": "outbound-full-relay", "transport_pq": True, "transport_pq_status": "hybrid"} + rows = [pool.sample(T0 + i * INTERVAL, pinned_peer=automatic if i < 5 else "manual-hybrid") for i in range(100)] + figure = self.evaluate(rows, 100) + self.assertEqual(figure.result, "fail") + self.assertIn("disconnectnode", " ".join(figure.details)) + + def test_no_samples_is_unknown(self) -> None: + self.assertEqual(self.evaluate([], 10).result, "unknown") + # Two rows for one expected sample (for example a manual run beside cron) are one sample. + pool = Node("pool", boot=T0 - DAY, pinned=PINNED) + self.assertEqual(self.evaluate([pool.sample(T0), pool.sample(T0)], 1).samples, 1) + self.assertEqual(pq_eval.evaluate_pinned(None, None, window(10), INTERVAL, 600, 0.9).result, "unknown") + + def test_malformed_numbers_are_missing(self) -> None: + pool = Node("pool", boot=T0 - DAY, pinned=PINNED) + rows = [pool.sample(T0 + i * INTERVAL, pinned_peer=None if i in (40, 41) else "manual-hybrid") for i in range(100)] + # The link was down in two samples: 98 of 100 fails. + self.assertEqual(self.evaluate(rows, 100).result, "fail") + for label, uptime in (("negative", "-30"), ("decimal", "30.0"), ("spaced", " 30"), ("non-ASCII digits", "\u0663\u0660"), + ("underscored", "3_0")): + with self.subTest(uptime=label): + # Not an uptime, so not a restart: the two samples are judged, not excluded. + figure = self.evaluate([dict(row, uptime=uptime) if index in (40, 41) else row for index, row in enumerate(rows)], 100) + self.assertEqual(figure.result, "fail", figure.summary) + self.assertIn("unknown=0 excluded=0", figure.summary) + good = [pool.sample(T0 + i * INTERVAL) for i in range(100)] + for label, time in (("text", "soon"), ("decimal", f"{T0 + 40 * INTERVAL}.0"), ("negative", "-1"), + ("beyond year 9999", str(10**14))): + with self.subTest(time=label): + # A row without a time is a missing sample: unknown. + figure = self.evaluate([dict(row, time=time) if index == 40 else row for index, row in enumerate(good)], 100) + self.assertIn("hybrid=99 not_hybrid=0 unknown=1", figure.summary) + + def test_malformed_pinned_peer_is_unknown(self) -> None: + pool = Node("pool", boot=T0 - DAY, pinned=PINNED) + rows = [pool.sample(T0 + i * INTERVAL) for i in range(100)] + malformed = { + "pinned_present not 0 or 1": {"pinned_present": "yes"}, + "no connection type": {"pinned_connection_type": None}, + "malformed connection type": {"pinned_connection_type": "Manual connection"}, + "transport_pq not 0 or 1": {"pinned_transport_pq": "true"}, + "no transport_pq": {"pinned_transport_pq": None}, + "no status": {"pinned_transport_pq_status": None}, + "unknown status": {"pinned_transport_pq_status": "quantum"}, + "unknown status, not hybrid": {"pinned_transport_pq": "0", "pinned_transport_pq_status": "quantum"}, + "no status, not hybrid": {"pinned_transport_pq": "0", "pinned_transport_pq_status": None}, + "hybrid without transport_pq": {"pinned_transport_pq": "0"}, + "transport_pq without hybrid": {"pinned_transport_pq_status": "pending"}, + } + for label, change in malformed.items(): + with self.subTest(label): + # Neither hybrid nor a miss: two unknown samples in 100 cannot be judged against 99%. + figure = self.evaluate([dict(row, **change) if index in (10, 11) else row for index, row in enumerate(rows)], 100) + self.assertEqual(figure.result, "unknown", figure.summary) + self.assertIn("hybrid=98 not_hybrid=0 unknown=2", figure.summary) + # Well-formed values that are not a manual hybrid link are misses. + misses = { + "not connected": {"pinned_present": "0", "pinned_connection_type": None, "pinned_transport_pq": None, + "pinned_transport_pq_status": None}, + "not hybrid": {"pinned_transport_pq": "0", "pinned_transport_pq_status": "fallback"}, + "automatic": {"pinned_connection_type": "outbound-full-relay"}, + } + for label, change in misses.items(): + with self.subTest(label): + figure = self.evaluate([dict(row, **change) if index in (10, 11) else row for index, row in enumerate(rows)], 100) + self.assertEqual(figure.result, "fail", figure.summary) + self.assertIn("hybrid=98 not_hybrid=2 unknown=0", figure.summary) + + def test_uptime_that_contradicts_the_instance_is_unknown(self) -> None: + pool = Node("pool", boot=T0 - DAY, pinned=PINNED) + rows = [pool.sample(T0 + i * INTERVAL, pinned_peer=None if i in (40, 41) else "manual-hybrid") for i in range(100)] + # The node reports a fresh boot in the two samples where the link was down, + # but its instance_id did not change: that is no restart, so they are judged. + rows[40]["uptime"], rows[41]["uptime"] = "30", "330" + figure = self.evaluate(rows, 100) + self.assertEqual(figure.result, "fail", figure.summary) + self.assertIn("excluded=0", figure.summary) + self.assertEqual(pq_eval.restarts(rows), []) + self.assertEqual([row["uptime"] for _, row in pq_eval.sample_rows(rows)[39:43]], [rows[39]["uptime"], None, None, rows[42]["uptime"]]) + + # A restart between the sampler's uptime and getpqtransportinfo calls pairs the + # old uptime with the new instance_id once: only that uptime is unknown. + pool = Node("pool", boot=T0 - DAY, pinned=PINNED) + rows = [] + for i in range(100): + time = T0 + i * INTERVAL + if i == 50: + old_uptime = str(time - pool.boot) + pool.restart(time - 30) + rows.append(pool.sample(time)) + rows[50]["uptime"] = old_uptime + self.assertEqual([index for index, (_, row) in enumerate(pq_eval.sample_rows(rows)) if row["uptime"] is None], [50]) + self.assertEqual([restart.last_seen for restart in pq_eval.restarts(rows)], [T0 + 49 * INTERVAL]) + figure = self.evaluate(rows, 100) + self.assertEqual(figure.result, "pass", figure.summary) + self.assertIn("excluded=2", figure.summary) # 30 s and 330 s after the restart + + def test_low_coverage_is_unknown_not_judged(self) -> None: + pool = Node("pool", boot=T0 - DAY, pinned=PINNED) + automatic = {"connection_type": "outbound-full-relay", "transport_pq": True, "transport_pq_status": "hybrid"} + # 15 of 100 samples missing and 5 bad: with the gaps it cannot be judged. + rows = [pool.sample(T0 + i * INTERVAL, pinned_peer=automatic if i < 5 else "manual-hybrid") for i in range(100) if i % 7 or i > 98] + self.assertEqual(len(rows), 85) + figure = self.evaluate(rows, 100) + self.assertEqual(figure.result, "unknown", figure.summary) + self.assertAlmostEqual(figure.coverage, 0.85) + self.assertEqual(self.evaluate(rows, 100, min_coverage=0.8).result, "fail") + + def test_each_expected_sample_counts_once(self) -> None: + pool = Node("pool", boot=T0 - DAY, pinned=PINNED) + # 95 of 100 expected samples cannot show 99%, however often each one is recorded. + rows = [pool.sample(T0 + i * INTERVAL) for i in range(100) if i % 20] + self.assertEqual(len(rows), 95) + cases = { + "once": rows, + "each row six times": [row for row in rows for _ in range(6)], + "files that overlap": rows + rows[10:] + rows[40:], + "six sampler runs per expected sample": + [pool.sample(T0 + i * INTERVAL + offset) for i in range(100) if i % 20 for offset in range(-100, 101, 40)], + } + for label, case in cases.items(): + with self.subTest(label): + figure = self.evaluate(case, 100) + self.assertEqual((figure.result, figure.samples), ("unknown", 100), figure.summary) + self.assertIn("hybrid=95 not_hybrid=0 unknown=5 excluded=0", figure.summary) + self.assertAlmostEqual(figure.coverage, 0.95) + + def test_several_samples_for_one_expected_sample_count_as_the_least_favorable(self) -> None: + pool = Node("pool", boot=T0 - DAY, pinned=PINNED) + rows = [pool.sample(T0 + i * INTERVAL) for i in range(100)] + + def with_extra(extra: Any, before: bool) -> list[Any]: + """rows with an extra sample a minute before or after expected samples 10 and 11.""" + offset = -60 if before else 60 + return rows + [extra(T0 + i * INTERVAL + offset) for i in (10, 11)] + cases = { + "not connected": (lambda time: pool.sample(time, pinned_peer=None), "fail", "hybrid=98 not_hybrid=2 unknown=0"), + "unanswered": (lambda time: pool.sample(time, down=True), "unknown", "hybrid=98 not_hybrid=0 unknown=2"), + "hybrid": (pool.sample, "pass", "hybrid=100 not_hybrid=0 unknown=0"), + } + for label, (extra, result, summary) in cases.items(): + for before in (True, False): + with self.subTest(label, before=before): + figure = self.evaluate(with_extra(extra, before), 100) + self.assertEqual((figure.result, figure.samples), (result, 100), figure.summary) + self.assertIn(summary, figure.summary) + # Not connected in one sample and unanswered in the other is a miss. + unanswered = [dict(row, sample_ok="0") if index in (10, 11) else row for index, row in enumerate(rows)] + figure = self.evaluate(unanswered + [pool.sample(T0 + i * INTERVAL + 60, pinned_peer=None) for i in (10, 11)], 100) + self.assertIn("hybrid=98 not_hybrid=2 unknown=0", figure.summary) + + # In a restart gap (the archive node restarts after expected sample 49), a sample + # excludes its expected sample from the figure, unless another one is a miss. + archive = Node("archive", boot=T0 - DAY) + archive_rows = [archive.sample(T0 + i * INTERVAL) for i in range(50)] + archive.restart(T0 + 50 * INTERVAL - 60) + archive_rows += [archive.sample(T0 + i * INTERVAL) for i in range(50, 100)] + figure = self.evaluate(rows, 100, archive_rows) + self.assertIn("hybrid=98 not_hybrid=0 unknown=0 excluded=2", figure.summary) + in_gap = pool.sample(T0 + 49 * INTERVAL + 60) + figure = self.evaluate(rows + [in_gap], 100, archive_rows) + self.assertIn("hybrid=97 not_hybrid=0 unknown=0 excluded=3", figure.summary) + missed = [dict(row, pinned_present="0") if index == 49 else row for index, row in enumerate(rows)] + figure = self.evaluate(missed + [in_gap], 100, archive_rows) + self.assertIn("hybrid=97 not_hybrid=1 unknown=0 excluded=2", figure.summary) + + def test_repeated_rows_do_not_outvote_the_boot_time(self) -> None: + pool = Node("pool", boot=T0 - DAY, pinned=PINNED) + rows = [pool.sample(T0 + i * INTERVAL, pinned_peer=None if i in (40, 41) else "manual-hybrid") for i in range(100)] + # A fresh boot in the two misses, without a new instance_id, is no restart: they are judged. + rows[40]["uptime"], rows[41]["uptime"] = "30", "330" + for copies in (1, 100): + with self.subTest(copies=copies): + figure = self.evaluate(rows + [dict(rows[index]) for index in (40, 41) for _ in range(copies - 1)], 100) + self.assertEqual(figure.result, "fail", figure.summary) + self.assertIn("hybrid=98 not_hybrid=2 unknown=0 excluded=0", figure.summary) + + +# --------------------------------------------------------------------------- +# Figures 2 and 3: failure entries +# --------------------------------------------------------------------------- + + +class FailureEntriesTest(unittest.TestCase): + ADDRESSES = ("203.0.113.5", "198.51.100.7", "2001:db8::5", "192.0.2.") + + def evaluate(self, records: list[dict[str, Any]], *, good: Any = None, monitored: list[str] | None = None, + min_coverage: float = 0.9, end: int | None = None) -> tuple[Any, Any]: + win = pq_eval.Window(T0, max(record["time"] for record in records) if end is None else end) + hosts = sorted({record["host"] for record in records}) if monitored is None else monitored + return pq_eval.evaluate_failures(records, known_good() if good is None else good, hosts, win, INTERVAL, min_coverage) + + def assert_no_addresses(self, *figures: Any) -> None: + text = " ".join(" ".join([figure.summary] + figure.details) for figure in figures) + for address in self.ADDRESSES: + self.assertNotIn(address, text) + + def test_monitored_outbound_failure_survives_an_inbound_flood(self) -> None: + first = record(T0, inbound=ring(0, 0, []), outbound=ring(0, 0, [])) + flood = [entry(seq, T0 + 100, "192.0.2.200", 40000 + seq, "malformed_record", direction="inbound", + reason="ek_length", connection_type="inbound") for seq in range(45, 301)] + monitored = entry(1, T0 + 50, "203.0.113.5", 8333, "first_packet_failed") + second = record(T0 + INTERVAL, inbound=ring(300, 44, flood), outbound=ring(1, 0, [monitored])) + figure, triage = self.evaluate([first, second]) + self.assertEqual(figure.result, "fail") + self.assertIn("entries=1", figure.summary) + self.assertTrue(any("first_packet_failed/tag" in line and "known_good=archive" in line for line in figure.details)) + # 300 inbound failures between two samples overflow the ring: those lost entries are unknown. + self.assertTrue(any("unknown archive inbound" in line and "sequences 1..44 were lost" in line for line in figure.details)) + self.assertEqual(triage.result, "unknown") + self.assert_no_addresses(figure, triage) + + def test_wrapped_ring_without_loss_stays_judged(self) -> None: + def unrelated(first: int, last: int) -> list[dict[str, Any]]: + return [entry(seq, T0 + seq, "192.0.2.9", 9000 + seq, "malformed_record", direction="inbound") for seq in range(first, last + 1)] + # The ring holds 256 entries and has wrapped twice, but every sample reaches back past the previous one. + records = [record(T0, inbound=ring(256, 0, unrelated(1, 256)), outbound=ring(0, 0, [])), + record(T0 + INTERVAL, inbound=ring(300, 44, unrelated(45, 300)), outbound=ring(0, 0, [])), + record(T0 + 2 * INTERVAL, inbound=ring(556, 300, unrelated(301, 556)), outbound=ring(0, 0, []))] + figure, triage = self.evaluate(records) + self.assertEqual((figure.result, triage.result), ("pass", "pass"), figure.details) + self.assertIn("unknown_intervals=0", figure.summary) + # It is judged, so a known-good failure inside the wrapped stretch fails it. + records[2]["recent_failures"]["inbound"]["entries"][-1] = entry( + 556, T0 + 556, "198.51.100.7", 50000, "first_packet_failed", direction="inbound") + figure, _ = self.evaluate(records) + self.assertEqual(figure.result, "fail") + self.assertIn("known_good=pool", figure.details[0]) + + def test_wrap_that_loses_entries_is_unknown(self) -> None: + unrelated = [entry(seq, T0 + seq, "192.0.2.9", 9000 + seq, "malformed_record", direction="inbound") for seq in range(1, 301)] + first = record(T0, inbound=ring(10, 0, unrelated[:10]), outbound=ring(0, 0, [])) + wrapped = record(T0 + INTERVAL, inbound=ring(300, 44, unrelated[44:]), outbound=ring(0, 0, [])) + figure, _ = self.evaluate([first, wrapped]) + self.assertEqual(figure.result, "unknown") + self.assertIn("sequences 11..44 were lost when the ring wrapped", " ".join(figure.details)) + # Read again with nothing new, the ring is complete. + self.assertEqual(self.evaluate([first, record(T0 + INTERVAL, inbound=ring(10, 0, unrelated[:10]), + outbound=ring(0, 0, []))])[0].result, "pass") + + def test_inconsistent_rings_are_unknown(self) -> None: + entries = [entry(seq, T0 + seq, "192.0.2.9", 9000 + seq, "malformed_record", direction="inbound") for seq in range(1, 11)] + first = record(T0, inbound=ring(10, 0, entries), outbound=ring(0, 0, [])) + cases = { + "dropped changed without new entries": ring(10, 3, entries[3:]), + "went backwards": ring(5, 0, entries[:5]), + "malformed ring": ring(10, 0, entries[:4] + entries[5:]), + } + for reason, inbound in cases.items(): + with self.subTest(reason): + figure, _ = self.evaluate([first, record(T0 + INTERVAL, inbound=inbound, outbound=ring(0, 0, []))]) + self.assertEqual(figure.result, "unknown") + self.assertIn(reason, " ".join(figure.details)) + + def test_unrelated_noise_does_not_fail(self) -> None: + noise = [entry(seq, T0 + seq, "192.0.2.9", 8333, outcome) for seq, outcome in + enumerate(["malformed_record", "first_packet_failed", "fallback", "closed_after_switch"], 1)] + records = [record(T0, inbound=ring(0, 0, []), outbound=ring(0, 0, [])), + record(T0 + INTERVAL, inbound=ring(0, 0, []), outbound=ring(4, 0, noise)), + record(T0 + 2 * INTERVAL, inbound=ring(0, 0, []), outbound=ring(4, 0, noise))] + figure, triage = self.evaluate(records) + self.assertEqual((figure.result, figure.samples), ("pass", 3)) + self.assertEqual(triage.result, "pass") + self.assertAlmostEqual(figure.coverage, 1.0) + + def test_malformed_endpoint_is_unknown(self) -> None: + start = record(T0, inbound=ring(0, 0, []), outbound=ring(0, 0, [])) + + def evaluate(outcome: str, endpoint: Any) -> tuple[Any, Any]: + failure = entry(1, T0 + 5, "192.0.2.9", 8333, outcome) + if endpoint is ...: + del failure["endpoint"] + else: + failure["endpoint"] = endpoint + return self.evaluate([start, record(T0 + INTERVAL, inbound=ring(0, 0, []), outbound=ring(1, 0, [failure]))]) + + good = {"kind": "address", "network": "ipv4", "address": "192.0.2.9", "port": 8333} + malformed = { + "absent": ..., "null": None, "not an object": "192.0.2.9:8333", + "no address": {key: value for key, value in good.items() if key != "address"}, + "address not a string": dict(good, address=3221225993), "empty address": dict(good, address=""), + "address with a port": dict(good, address="192.0.2.9:8333"), "ipv6 address as ipv4": dict(good, address="2001:db8::9"), + "no port": {key: value for key, value in good.items() if key != "port"}, "port as text": dict(good, port="8333"), + "port true": dict(good, port=True), "port out of range": dict(good, port=65536), "negative port": dict(good, port=-1), + "unknown kind": dict(good, kind="service"), "kind a list": dict(good, kind=["address"]), + "unknown network": dict(good, network="tor"), + "kind and network disagree": dict(good, kind="name_proxy"), + } + for outcome in ("malformed_record", "first_packet_failed", "fallback", "closed_after_switch"): + monitored = outcome != "closed_after_switch" + for label, endpoint in malformed.items(): + with self.subTest(outcome=outcome, endpoint=label): + figure, triage = evaluate(outcome, endpoint) + judged, other = (figure, triage) if monitored else (triage, figure) + self.assertEqual((judged.result, other.result), ("unknown", "pass"), judged.details) + self.assertIn(f"{outcome} entry 1 has a missing or malformed endpoint", " ".join(judged.details)) + self.assert_no_addresses(figure, triage) + # Valid endpoints of unrelated peers, on every network, are judged clean. + valid = [good, dict(good, network="ipv6", address="2001:db8::9"), dict(good, network="cjdns", address="fc00::9"), + dict(good, network="onion", address="example.onion"), dict(good, network="i2p", address="example.b32.i2p", port=0), + {"kind": "name_proxy", "network": "name_proxy", "address": "seed.example.org", "port": 8333}] + for endpoint in valid: + with self.subTest(valid=endpoint["network"]): + self.assertEqual(tuple(figure.result for figure in evaluate("fallback", endpoint)), ("pass", "pass")) + # Outcomes outside both figures do not need an endpoint. + self.assertEqual(evaluate("legacy_peer", None)[0].result, "pass") + # A malformed endpoint that still names a known-good peer is that peer's failure. + figure, _ = evaluate("fallback", {"address": "203.0.113.5", "port": 8333}) + self.assertEqual(figure.result, "fail") + self.assertIn("known_good=archive", figure.details[0]) + + def test_malformed_rings_are_unknown(self) -> None: + start = record(T0, inbound=ring(0, 0, []), outbound=ring(0, 0, [])) + failure = entry(1, T0 + 5, "192.0.2.9", 8333, "malformed_record", direction="inbound") + cases: dict[str, Any] = { + "recent_failures absent": ..., "recent_failures null": None, "recent_failures a list": [failure], + "recent_failures text": "inbound", + "last_sequence true": ring(True, 0, [failure]), "dropped true": ring(1, True, [failure]), + "negative dropped": ring(1, -1, [failure]), "sequence true": ring(1, 0, [dict(failure, sequence=True)]), + "sequence 0": ring(0, 0, [dict(failure, sequence=0)]), "entry not an object": ring(1, 0, ["entry"]), + } + for label, value in cases.items(): + with self.subTest(label): + later = record(T0 + INTERVAL, inbound=value, outbound=ring(0, 0, [])) + if label.startswith("recent_failures"): + if value is ...: + del later["recent_failures"] + else: + later["recent_failures"] = value + figure, triage = self.evaluate([start, later]) + self.assertEqual((figure.result, triage.result), ("unknown", "unknown")) + self.assertIn("malformed ring", " ".join(figure.details)) + + def test_malformed_outcome_is_unknown(self) -> None: + start = record(T0, inbound=ring(0, 0, []), outbound=ring(0, 0, [])) + + def evaluate(outcome: Any) -> tuple[Any, Any]: + failure = entry(1, T0 + 5, "192.0.2.9", 8333, "fallback") + if outcome is ...: + del failure["outcome"] + else: + failure["outcome"] = outcome + return self.evaluate([start, record(T0 + INTERVAL, inbound=ring(0, 0, []), outbound=ring(1, 0, [failure]))]) + + # A malformed outcome may be a monitored or a triage one: both figures are unknown. + for label, outcome in (("absent", ...), ("null", None), ("unknown word", "timeout"), ("capitalized", "Fallback"), + ("number", 3), ("object", {"outcome": "fallback"})): + with self.subTest(label): + figures = evaluate(outcome) + self.assertEqual(tuple(figure.result for figure in figures), ("unknown", "unknown")) + for figure in figures: + self.assertIn("entry 1 has a missing or malformed outcome", " ".join(figure.details)) + # The other outcomes in the vocabulary count towards neither figure. + for outcome in ("legacy_peer", "abandoned", "internal_error"): + with self.subTest(outcome): + self.assertEqual(tuple(figure.result for figure in evaluate(outcome)), ("pass", "pass")) + + def test_implausible_entry_time_is_unknown(self) -> None: + start = record(T0, inbound=ring(0, 0, []), outbound=ring(0, 0, [])) + + def evaluate(time: Any) -> Any: + # A known-good failure: with its time it fails the figure, or is outside the window. + failure = entry(1, T0 + 5, "203.0.113.5", 8333, "first_packet_failed") + if time is ...: + del failure["time"] + else: + failure["time"] = time + later = record(T0 + INTERVAL, inbound=ring(0, 0, []), outbound=ring(1, 0, [failure])) + return self.evaluate([start, later], end=T0 + 2 * INTERVAL)[0] + + implausible = {"absent": ..., "null": None, "text": str(T0 + 5), "true": True, "negative": -1, + "beyond year 9999": 10**12, "milliseconds": (T0 + 5) * 1000, + "after the sample that read it": T0 + INTERVAL + 301, "before its process started": T0 - DAY - 301} + for label, time in implausible.items(): + with self.subTest(label): + figure = evaluate(time) + self.assertEqual(figure.result, "unknown", figure.details) + self.assertIn("entry 1 has a missing or implausible time", " ".join(figure.details)) + self.assert_no_addresses(figure) + # Within the clock slack it is judged by its time: in the window, or before it. + self.assertEqual(evaluate(T0 + INTERVAL + 300).result, "fail") + # First read long before the window, it was recorded before the window, whatever its time says. + held = ring(1, 0, [dict(entry(1, 0, "203.0.113.5", 8333, "fallback"), time="x")]) + early, later = (record(time, inbound=ring(0, 0, []), outbound=held) for time in (T0 - DAY + 600, T0)) + self.assertEqual(self.evaluate([early, later])[0].result, "pass") + figure = evaluate(T0 - DAY - 300) + self.assertEqual(figure.result, "unknown") # only the coverage: 2 of 3 samples + self.assertIn("entries=0 unknown_intervals=0", figure.summary) + + def test_malformed_fallback_set_entry_is_unknown(self) -> None: + start = record(T0, inbound=ring(0, 0, []), outbound=ring(0, 0, [])) + unrelated = {"endpoint": {"kind": "address", "network": "ipv4", "address": "192.0.2.9", "port": 8333}, + "cause": "first_packet_failed", "reason": "tag", "streak": 3, "entered": T0 + 5, "expires": T0 + 3605, + "window_seconds": 3600} + malformed = {"not an object": "192.0.2.9:8333", "null": None, "a list": [unrelated], + "no endpoint": {key: value for key, value in unrelated.items() if key != "endpoint"}, + "malformed endpoint": dict(unrelated, endpoint={"address": "192.0.2.9"})} + for label, item in malformed.items(): + with self.subTest(label): + # Held over three samples, it is one unknown entry. + records = [start] + [record(T0 + i * INTERVAL, inbound=ring(0, 0, []), outbound=ring(0, 0, []), fallback_set=[item]) + for i in (1, 2, 3)] + figure, triage = self.evaluate(records) + self.assertEqual((figure.result, triage.result), ("unknown", "pass")) + self.assertEqual(sum("fallback_set" in detail and "without a well-formed endpoint" in detail + for detail in figure.details), 1, figure.details) + self.assert_no_addresses(figure, triage) + figure, _ = self.evaluate([start, record(T0 + INTERVAL, inbound=ring(0, 0, []), outbound=ring(0, 0, []), fallback_set=[unrelated])]) + self.assertEqual(figure.result, "pass") + + def test_malformed_since_is_unknown(self) -> None: + start = record(T0, inbound=ring(0, 0, []), outbound=ring(0, 0, [])) + cases = {"absent": ..., "null": None, "text": str(T0 - DAY), "true": True, "negative": -1, "beyond year 9999": 10**12, + "after the sample": T0 + INTERVAL + 301} + reasons = dict.fromkeys(cases, "since missing, malformed or after the sample") + cases["changed within one instance_id"] = T0 - DAY + 1 + reasons["changed within one instance_id"] = "since changed within one instance_id" + for label, since in cases.items(): + with self.subTest(label): + later = record(T0 + INTERVAL, inbound=ring(0, 0, []), outbound=ring(0, 0, []), since=since) + if since is ...: + del later["since"] + after = record(T0 + 2 * INTERVAL, inbound=ring(0, 0, []), outbound=ring(0, 0, [])) + figure, _ = self.evaluate([start, later, after]) + self.assertEqual(figure.result, "unknown") + self.assertIn(reasons[label], " ".join(figure.details)) + # A process that started while the sampler ran is not after the sample. + self.assertEqual(self.evaluate([record(T0, inbound=ring(0, 0, []), outbound=ring(0, 0, []), since=T0 + 300)])[0].result, "pass") + + def test_report_prints_only_well_formed_values(self) -> None: + start = record(T0, inbound=ring(0, 0, []), outbound=ring(0, 0, [])) + # Known-good failures whose other fields are malformed: still failures, and an + # address in a malformed field never reaches the report. + failure = dict(entry(1, T0 + 5, "203.0.113.5", 8333, "first_packet_failed"), + reason="203.0.113.9", connection_type="2001:db8::7", peer_id="198.51.100.20") + held = {"endpoint": {"kind": "address", "network": "ipv6", "address": "2001:db8::5", "port": 8333}, "cause": ["tag"], + "reason": "198.51.100.20", "streak": True, "entered": {"time": T0}, "expires": 10**30} + later = record(T0 + INTERVAL, inbound=ring(0, 0, []), outbound=ring(1, 0, [failure]), fallback_set=[held, held]) + figure, triage = self.evaluate([start, later]) + self.assertEqual(figure.result, "fail") + self.assertIn("entries=2", figure.summary) + self.assertIn("first_packet_failed/malformed conn_type=malformed peer_id=malformed known_good=archive", figure.details[0]) + self.assertEqual(figure.details[1], "malformed archive fallback_set cause=malformed/malformed streak=malformed " + "expires=malformed known_good=known-good-3") + text = " ".join(figure.details + triage.details) + for address in ("203.0.113.9", "2001:db8::7", "198.51.100.20"): + self.assertNotIn(address, text) + self.assert_no_addresses(figure, triage) + + def test_restart_is_an_unknown_interval(self) -> None: + first = record(T0, inbound=ring(0, 0, []), outbound=ring(0, 0, [])) + restarted = record(T0 + 2 * INTERVAL, inbound=ring(0, 0, []), outbound=ring(0, 0, []), + instance="b" * 64, since=T0 + INTERVAL) + figure, _ = self.evaluate([first, restarted], min_coverage=0.5) + self.assertEqual(figure.result, "unknown") + self.assertIn("instance_id changed", " ".join(figure.details)) + # A fresh process whose rings are complete since startup is clean. + fresh = record(T0, inbound=ring(0, 0, []), outbound=ring(1, 0, [entry(1, T0 - 5, "192.0.2.1", 1, "fallback")]), + since=T0 - 10) + self.assertEqual(self.evaluate([fresh])[0].result, "pass") + incomplete = record(T0, inbound=ring(0, 0, []), outbound=ring(300, 44, []), since=T0 - 10) + figure, _ = self.evaluate([incomplete]) + self.assertEqual(figure.result, "unknown") + self.assertIn("sequences 1..300 were dropped before the first sample", " ".join(figure.details)) + + def test_known_good_failures(self) -> None: + start = record(T0, inbound=ring(0, 0, []), outbound=ring(0, 0, [])) + cases: dict[str, tuple[dict[str, Any], dict[str, Any], list[Any], str]] = { + "inbound from the pool node, any source port": + (ring(1, 0, [entry(1, T0 + 5, POOL_ADDRESS, 50123, "malformed_record", direction="inbound")]), ring(0, 0, []), [], "pool"), + "fallback entry": + (ring(0, 0, []), ring(1, 0, [entry(1, T0 + 5, "203.0.113.5", 8333, "fallback", reason="first_packet_failed")]), [], "archive"), + "fallback_set": + (ring(0, 0, []), ring(0, 0, []), [{"endpoint": {"kind": "address", "network": "ipv6", "address": "2001:db8::5", "port": 8333}, + "cause": "closed_after_switch", "reason": "eof", "streak": 3, + "entered": T0 + 5, "expires": T0 + 3605, "window_seconds": 3600}], "known-good-3"), + } + for label, (inbound, outbound, fallback_set, name) in cases.items(): + with self.subTest(label): + figure, triage = self.evaluate([start, record(T0 + INTERVAL, inbound=inbound, outbound=outbound, fallback_set=fallback_set)]) + self.assertEqual(figure.result, "fail") + self.assertIn("entries=1", figure.summary) + self.assertIn(f"known_good={name}", figure.details[0]) + self.assert_no_addresses(figure, triage) + + def test_known_good_address_in_any_spelling(self) -> None: + # The node reports an IPv6 address compressed; the list may spell it out, and either + # side may write an IPv4 address as IPv4-mapped IPv6. + start = record(T0, inbound=ring(0, 0, []), outbound=ring(0, 0, [])) + cases = (("[2001:0DB8:0:0:0:0:0:5]:8333 peer", "2001:db8::5"), ("[2001:db8:0:0::5] peer", "2001:db8::5"), + ("[2001:db8::5]:8333 peer", "2001:0db8:0:0:0:0:0:5"), ("[2001:db8::5] peer", "2001:0db8:0:0:0:0:0:5"), + ("203.0.113.5:8333 peer", "::ffff:203.0.113.5")) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "known-good.txt" + for line, address in cases: + with self.subTest(line=line, address=address): + path.write_text(line + "\n", encoding="utf8") + failed = record(T0 + INTERVAL, inbound=ring(0, 0, []), + outbound=ring(1, 0, [entry(1, T0 + 5, address, 8333, "first_packet_failed")])) + figure, triage = self.evaluate([start, failed], good=pq_eval.read_known_good(path)) + self.assertEqual(figure.result, "fail", figure.details) + self.assertIn("known_good=peer", figure.details[0]) + self.assert_no_addresses(figure, triage) + + def test_missing_instance_id_is_unknown(self) -> None: + start = record(T0, inbound=ring(0, 0, []), outbound=ring(0, 0, [])) + for label, value in (("null", None), ("short", "abc"), ("number", 7), ("list", ["a" * 64])): + with self.subTest(label): + broken = record(T0 + INTERVAL, inbound=ring(0, 0, []), outbound=ring(1, 0, [entry(1, T0 + 5, "192.0.2.9", 1, "fallback")]), + instance=value) + after = record(T0 + 2 * INTERVAL, inbound=ring(0, 0, []), outbound=ring(0, 0, [])) + figure, _ = self.evaluate([start, broken, after]) + self.assertEqual(figure.result, "unknown") + self.assertTrue(any("instance_id missing or malformed" in detail for detail in figure.details)) + + def test_missing_fallback_set_is_unknown(self) -> None: + start = record(T0, inbound=ring(0, 0, []), outbound=ring(0, 0, [])) + for label, value in (("null", None), ("not a list", {"endpoint": "x"}), ("absent", ...)): + with self.subTest(label): + later = record(T0 + INTERVAL, inbound=ring(0, 0, []), outbound=ring(0, 0, [])) + if value is ...: + del later["fallback_set"] + else: + later["fallback_set"] = value + figure, _ = self.evaluate([start, later]) + self.assertEqual(figure.result, "unknown") + self.assertTrue(any("fallback_set missing or malformed" in detail for detail in figure.details)) + + def test_closed_after_switch_is_listed_for_triage(self) -> None: + closed = entry(1, T0 + 5, "2001:db8::5", 8333, "closed_after_switch", reason="eof", connection_type="block-relay-only") + records = [record(T0, inbound=ring(0, 0, []), outbound=ring(0, 0, [])), + record(T0 + INTERVAL, inbound=ring(0, 0, []), outbound=ring(1, 0, [closed]))] + figure, triage = self.evaluate(records) + self.assertEqual(figure.result, "pass") + self.assertEqual(triage.result, "fail") + self.assertIn("known_good=known-good-3", triage.details[0]) + self.assert_no_addresses(figure, triage) + + def test_every_monitored_node_needs_evidence(self) -> None: + archive = [record(T0 + i * INTERVAL, inbound=ring(0, 0, []), outbound=ring(0, 0, [])) for i in range(3)] + figure, _ = self.evaluate(archive, monitored=["archive", "pool"]) + self.assertEqual(figure.result, "unknown") + self.assertIn("unknown pool: coverage 0.0% is below 90.0%", figure.details) + self.assertIn("nodes=archive:100.0%,pool:0.0%", figure.summary) + pool = [dict(item, host="pool") for item in archive] + self.assertEqual(self.evaluate(archive + pool, monitored=["archive", "pool"])[0].result, "pass") + + def test_low_coverage_is_unknown(self) -> None: + # Records for half of the expected samples; the rings say nothing was lost. + records = [record(T0 + i * INTERVAL, inbound=ring(0, 0, []), outbound=ring(0, 0, [])) for i in range(0, 10, 2)] + figure, _ = self.evaluate(records, end=T0 + 9 * INTERVAL) + self.assertEqual(figure.result, "unknown") + self.assertAlmostEqual(figure.coverage, 0.5) + self.assertEqual(self.evaluate(records, end=T0 + 9 * INTERVAL, min_coverage=0.5)[0].result, "pass") + # Repeated records, files given twice and extra sampler runs add no coverage. + for label, case in (("repeated", records * 3), + ("extra runs", records + [dict(item, time=item["time"] + offset) for item in records for offset in (-100, 100)])): + with self.subTest(label): + figure, _ = self.evaluate(case, end=T0 + 9 * INTERVAL) + self.assertEqual((figure.result, figure.samples), ("unknown", 5)) + self.assertAlmostEqual(figure.coverage, 0.5) + + def test_coverage_counts_only_expected_samples_in_the_window(self) -> None: + # Records for expected samples 0 to 8, and one 160 s after sample 9: nearest to + # sample 10, which is after the window, so sample 9 is missing. + def empty(time: int) -> dict[str, Any]: + return record(time, inbound=ring(0, 0, []), outbound=ring(0, 0, [])) + records = [empty(T0 + i * INTERVAL) for i in range(9)] + [empty(T0 + 9 * INTERVAL + 160)] + figure, _ = self.evaluate(records, min_coverage=0.95) + self.assertEqual((figure.result, figure.samples), ("unknown", 9)) + self.assertAlmostEqual(figure.coverage, 0.9) + # Likewise a record 100 s before the window starts is nearest to sample 0, but not in the window. + records = [empty(T0 - 100)] + [empty(T0 + i * INTERVAL) for i in range(1, 10)] + figure, _ = self.evaluate(records, min_coverage=0.95) + self.assertEqual((figure.result, figure.samples), ("unknown", 9)) + + def test_repeated_records_count_each_entry_once(self) -> None: + start = record(T0, inbound=ring(0, 0, []), outbound=ring(0, 0, [])) + failed = record(T0 + INTERVAL, inbound=ring(0, 0, []), outbound=ring(1, 0, [entry(1, T0 + 5, "203.0.113.5", 8333, "first_packet_failed")])) + once = self.evaluate([start, failed]) + twice = self.evaluate([start, failed, start, failed]) + self.assertEqual([(figure.result, figure.samples, figure.summary, figure.details) for figure in twice], + [(figure.result, figure.samples, figure.summary, figure.details) for figure in once]) + self.assertIn("entries=1", twice[0].summary) + + def test_missing_evidence_is_unknown(self) -> None: + records = [record(T0, inbound=ring(0, 0, []), outbound=ring(0, 0, []))] + win = pq_eval.Window(T0, T0) + self.assertEqual(pq_eval.evaluate_failures(None, known_good(), ["archive"], win, INTERVAL, 0.9)[0].result, "unknown") + empty = pq_eval.KnownGood({}, {}) + self.assertEqual(pq_eval.evaluate_failures(records, empty, ["archive"], win, INTERVAL, 0.9)[0].result, "unknown") + self.assertEqual(pq_eval.evaluate_failures(records, known_good(), [], win, INTERVAL, 0.9)[0].result, "unknown") + self.assertEqual(pq_eval.evaluate_failures([], known_good(), ["archive"], win, INTERVAL, 0.9)[0].result, "unknown") + # A node that stopped reporting leaves the rest of the window unknown. + figure, _ = self.evaluate(records, end=T0 + 10 * INTERVAL, min_coverage=0.05) + self.assertEqual(figure.result, "unknown") + self.assertIn("no samples since", " ".join(figure.details)) + broken = record(T0, inbound=ring(0, 0, []), outbound={"entries": []}) + self.assertEqual(self.evaluate([broken])[0].result, "unknown") + + def test_known_good_file(self) -> None: + good = known_good() + self.assertEqual(good.endpoints, {"203.0.113.5:8333": "archive", "[2001:db8::5]:8333": "known-good-3"}) + self.assertEqual(good.addresses, {POOL_ADDRESS: "pool"}) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "known-good.txt" + bad = ["2001:db8::1:8333", "198.51.100.1:port", "[2001:db8::1", ":8333", "198.51.100.1:0", "198.51.100.1:70000", + "198.51.100.1 two labels", "198.51.100.1 bad/label", "198.51.100.1\n198.51.100.1", + "[2001:db8::1]\n[2001:0db8:0:0:0:0:0:1]", "198.51.100.1:8333\n[::ffff:198.51.100.1]:8333"] + for line in bad: + path.write_text(line.replace("\\n", "\n") + "\n", encoding="utf8") + with self.subTest(line), self.assertRaises(ValueError) as raised: + pq_eval.read_known_good(path) + self.assertNotIn("198.51.100.1", str(raised.exception).replace(str(path), "")) + self.assertNotIn("2001:db8", str(raised.exception)) + path.write_text("[2001:db8::1]\nExample.org:8333 seed # comment\n", encoding="utf8") + good = pq_eval.read_known_good(path) + self.assertEqual((good.endpoints, good.addresses), ({"example.org:8333": "seed"}, {"2001:db8::1": "known-good-1"})) + # Each IP address has one spelling: compressed, lower case, and IPv4 for an IPv4-mapped address. + path.write_text("[2001:0DB8:0:0:0:0:0:1]\n[2001:db8:0::0:5]:8333 peer\n[::ffff:203.0.113.5]:8333 archive\n", encoding="utf8") + good = pq_eval.read_known_good(path) + self.assertEqual(good.endpoints, {"[2001:db8::5]:8333": "peer", "203.0.113.5:8333": "archive"}) + self.assertEqual(good.addresses, {"2001:db8::1": "known-good-1"}) + + +# --------------------------------------------------------------------------- +# Figure 4: connections_in against the control +# --------------------------------------------------------------------------- + + +class ConnectionsTest(unittest.TestCase): + BASELINE_DAYS = 3 + CANARY_DAYS = 4 + + def series(self, archive_in: Any, control_in: Any, *, archive_restart: int | None = None, + control_down: range | None = None) -> tuple[list[Any], list[Any], Any]: + archive = Node("archive", boot=T0 - 60 * DAY) + control = Node("control", boot=T0 - 60 * DAY, v1_0_0=True) + archive_rows, control_rows = [], [] + per_day = DAY // INTERVAL + for index in range(-self.BASELINE_DAYS * per_day, self.CANARY_DAYS * per_day): + time = T0 + index * INTERVAL + if archive_restart is not None and index == archive_restart: + archive.restart(time - 60) + archive_rows.append(archive.sample(time, connections_in=archive_in(index))) + down = control_down is not None and index in control_down + control_rows.append(control.sample(time, connections_in=control_in(index), down=down)) + end = T0 + (self.CANARY_DAYS * per_day - 1) * INTERVAL + return archive_rows, control_rows, pq_eval.Window(T0, end) + + def evaluate(self, archive_rows: Any, control_rows: Any, win: Any, baseline_ratio: Fraction | None = None, + min_coverage: float = 0.9) -> Any: + return pq_eval.evaluate_connections(archive_rows, control_rows, T0, win, INTERVAL, baseline_ratio, min_coverage) + + def test_within_tolerance(self) -> None: + # The archive runs at 1.2 times the control in the baseline; 1.3 times is within 20% of that. + figure = self.evaluate(*self.series(lambda i: 36 if i < 0 else 39, lambda i: 30)) + self.assertEqual(figure.result, "pass", figure.details) + self.assertIn("baseline_ratio=1.200", figure.summary) + self.assertEqual(figure.samples, self.CANARY_DAYS * DAY // INTERVAL) + + def test_outside_tolerance(self) -> None: + figure = self.evaluate(*self.series(lambda i: 36 if i < 288 else 25, lambda i: 30)) + self.assertEqual(figure.result, "fail") + self.assertTrue(any(" fail: relative=0.694" in line for line in figure.details)) + + def test_tolerance_is_inclusive(self) -> None: + # Exactly 20% off the baseline ratio passes. In floats, 24/10 against a + # baseline of 3 is 0.7999999999999999, and 108/10 against 9 is 1.2000000000000002. + for baseline_in, canary_in, result in ((30, 24, "pass"), (30, 23, "fail"), (90, 108, "pass"), (90, 109, "fail")): + with self.subTest(baseline_in=baseline_in, canary_in=canary_in): + figure = self.evaluate(*self.series(lambda i, b=baseline_in, c=canary_in: b if i < 0 else c, lambda i: 10)) + self.assertEqual(figure.result, result, figure.details) + # A given ratio is read exactly too: the float nearest 1.2 is below it, + # which would put 36/25 against it just outside 20%. + self.assertEqual(pq_eval.ratio_arg("1.2"), Fraction(6, 5)) + archive_rows, control_rows, win = self.series(lambda i: 36, lambda i: 25) + self.assertEqual(self.evaluate(archive_rows, control_rows, win, baseline_ratio=pq_eval.ratio_arg("1.2")).result, "pass") + + def test_48_hours_after_a_restart_are_excluded(self) -> None: + # Inbound connections recover slowly after the restart at the canary start. + archive_rows, control_rows, win = self.series( + lambda i: 36 if i < 0 or i >= 2 * 288 else 5, lambda i: 30, archive_restart=0) + figure = self.evaluate(archive_rows, control_rows, win) + self.assertEqual(figure.result, "pass", figure.details) + self.assertEqual(sum(" excluded:" in line for line in figure.details), 2) + + def test_missing_measurements_are_unknown(self) -> None: + archive_rows, control_rows, win = self.series(lambda i: 36, lambda i: 30, control_down=range(288, 2 * 288)) + figure = self.evaluate(archive_rows, control_rows, win) + self.assertEqual(figure.result, "unknown", figure.details) + archive_rows, control_rows, win = self.series(lambda i: 36, lambda i: None if 0 <= i < 288 else 30) + self.assertEqual(self.evaluate(archive_rows, control_rows, win).result, "unknown") + # Without a baseline there is nothing to compare against, unless one is given. + archive_rows, control_rows, win = self.series(lambda i: 36, lambda i: 30) + canary_only = ([row for row in archive_rows if int(row["time"]) >= T0], [row for row in control_rows if int(row["time"]) >= T0]) + self.assertEqual(self.evaluate(*canary_only, win).result, "unknown") + self.assertEqual(self.evaluate(*canary_only, win, baseline_ratio=Fraction(6, 5)).result, "pass") + self.assertEqual(pq_eval.evaluate_connections(archive_rows, None, T0, win, INTERVAL, None, 0.9).result, "unknown") + + def test_zero_baseline_is_unknown(self) -> None: + # Valid pre-canary pairs whose connections_in sum to 0 on one side give no usable ratio. + pairs = self.BASELINE_DAYS * DAY // INTERVAL + cases = { + "archive": (lambda i: 0 if i < 0 else 36, lambda i: 30), + "control": (lambda i: 36, lambda i: 0 if i < 0 else 30), + } + for node, (archive_in, control_in) in cases.items(): + with self.subTest(node=node): + archive_rows, control_rows, win = self.series(archive_in, control_in) + figure = self.evaluate(archive_rows, control_rows, win) + self.assertEqual(figure.result, "unknown") + self.assertEqual(figure.summary, f"no baseline: {node} connections_in summed to 0 over {pairs} valid pre-canary pairs") + self.assertEqual((figure.samples, figure.coverage), (0, None)) + # A given ratio still judges the canary days. + self.assertEqual(self.evaluate(archive_rows, control_rows, win, baseline_ratio=Fraction(6, 5)).result, "pass") + + def test_malformed_numbers_are_missing(self) -> None: + # Day 0 holds the baseline ratio; the archive then loses a third of its connections. + archive_rows, control_rows, win = self.series(lambda i: 36 if i < 288 else 24, lambda i: 30) + self.assertEqual(self.evaluate(archive_rows, control_rows, win).result, "fail") + later = T0 + DAY + for label, uptime in (("negative", "-1"), ("non-ASCII digits", "\u0661")): + with self.subTest(uptime=label): + # Not an uptime, so not a restart: the later days are unknown, not excluded. + broken = [dict(row, uptime=uptime) if int(row["time"]) >= later else row for row in archive_rows] + figure = self.evaluate(broken, control_rows, win) + self.assertEqual(figure.result, "unknown", figure.details) + self.assertFalse(any(" excluded:" in line for line in figure.details), figure.details) + for label, count in (("negative", "-30"), ("decimal", "30.5")): + with self.subTest(connections_in=label): + broken = [dict(row, connections_in=count) if int(row["time"]) >= later else row for row in control_rows] + self.assertEqual(self.evaluate(archive_rows, broken, win).result, "unknown") + + def test_unanswered_samples_are_not_pairs(self) -> None: + archive_rows, control_rows, win = self.series(lambda i: 36, lambda i: 30) + # sample_ok=0 says a call failed: its values are not a measurement, even when present. + control_rows = [dict(row, sample_ok="0") if T0 + DAY <= int(row["time"]) < T0 + 2 * DAY else row for row in control_rows] + figure = self.evaluate(archive_rows, control_rows, win) + self.assertEqual(figure.result, "unknown", figure.details) + self.assertEqual(figure.samples, (self.CANARY_DAYS - 1) * DAY // INTERVAL) + + def test_days_need_coverage(self) -> None: + # 30 of a full day's 288 samples missing on the control: 89.6% coverage. + archive_rows, control_rows, win = self.series(lambda i: 36, lambda i: 30) + gap = {T0 + index * INTERVAL for index in range(400, 430)} + control_rows = [row for row in control_rows if int(row["time"]) not in gap] + figure = self.evaluate(archive_rows, control_rows, win) + self.assertEqual(figure.result, "unknown") + self.assertTrue(any(line.startswith("2025-10-10 unknown:") and "coverage=89.6%" in line for line in figure.details), figure.details) + self.assertEqual(self.evaluate(archive_rows, control_rows, win, min_coverage=0.85).result, "pass") + # Recording the other samples twice does not fill the gap. + twice = control_rows + [dict(row, time=str(int(row["time"]) + 60)) for row in control_rows] + figure = self.evaluate(archive_rows * 2, twice, win) + self.assertEqual(figure.result, "unknown") + self.assertTrue(any(line.startswith("2025-10-10 unknown:") and "coverage=89.6%" in line for line in figure.details), figure.details) + + def test_several_samples_for_one_expected_sample_count_as_the_least_favorable(self) -> None: + archive_rows, control_rows, win = self.series(lambda i: 36, lambda i: 30) + full = self.CANARY_DAYS * DAY // INTERVAL + self.assertEqual(self.evaluate(archive_rows, control_rows, win).samples, full) + + def shifted(rows: list[Any], indexes: range, offset: int, **changes: str) -> list[Any]: + """Copies of the canary samples at indexes, offset seconds later, with changes.""" + return [dict(rows[self.BASELINE_DAYS * 288 + index], time=str(T0 + index * INTERVAL + offset), **changes) for index in indexes] + cases = { + # An unanswered sample a minute before some expected samples and after others. + "unanswered": shifted(control_rows, range(300, 310), -60, sample_ok="0") + shifted(control_rows, range(400, 410), 60, sample_ok="0"), + # A second measurement that disagrees with the first. + "disagreeing": shifted(control_rows, range(300, 310), -60, connections_in="31") + shifted(control_rows, range(400, 410), 60, connections_in="29"), + } + for label, extra in cases.items(): + with self.subTest(label): + figure = self.evaluate(archive_rows, control_rows + extra, win) + self.assertEqual(figure.samples, full - 20, figure.details) + # A measurement that agrees is the same measurement. + figure = self.evaluate(archive_rows, control_rows + shifted(control_rows, range(300, 310), -60), win) + self.assertEqual((figure.result, figure.samples), ("pass", full)) + + # A sample of the node 30 s after a restart excludes its expected sample, + # unless another sample for it is unusable. + restarted = Node("archive", boot=T0 + 300 * INTERVAL + 30) + restarted.instance = 1 + after_restart = restarted.sample(T0 + 300 * INTERVAL + 60) + figure = self.evaluate(archive_rows + [after_restart], control_rows, win) + self.assertEqual(figure.samples, full - 1) + self.assertTrue(any(" excluded=1 " in line for line in figure.details), figure.details) + unanswered = [dict(row, sample_ok="0") if int(row["time"]) == T0 + 300 * INTERVAL else row for row in archive_rows] + figure = self.evaluate(unanswered + [after_restart], control_rows, win) + self.assertEqual(figure.samples, full - 1) + self.assertFalse(any(" excluded=1 " in line for line in figure.details), figure.details) + + +# --------------------------------------------------------------------------- +# The command line +# --------------------------------------------------------------------------- + + +class EvaluatorCommandTest(unittest.TestCase): + # Every address in the fixtures; none may reach the report. + ADDRESSES = ("203.0.113.5", "203.0.113.9", "198.51.100.7", "198.51.100.20", "198.51.100.77", "2001:db8") + + @staticmethod + def run_evaluator(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run([sys.executable, str(EVALUATOR), *args], text=True, capture_output=True) + + def write_samples(self, root: Path) -> None: + nodes = {"pool": Node("pool", boot=T0 - 3 * DAY, pinned=PINNED), "archive": Node("archive", boot=T0 - 3 * DAY), + "control": Node("control", boot=T0 - 10 * DAY, v1_0_0=True)} + for index in range(-2 * 288, 3 * 288): + time = T0 + index * INTERVAL + for name, node in nodes.items(): + row = node.sample(time) + pq_canary.append_row(root / f"{name}.csv", {key: NA if value is None else value for key, value in row.items()}) + failures = pq_canary.build_failures_record(time=time, host=name, pqinfo=node.pqinfo(time)) + if failures is not None and name != "control": + with (root / f"{name}.jsonl").open("a", encoding="utf8") as out: + out.write(json.dumps(failures) + "\n") + + def test_prints_every_figure_without_addresses(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self.write_samples(root) + inputs = ["--canary-start", pq_eval.utc(T0), "--pool", str(root / "pool.csv"), "--archive", str(root / "archive.csv"), + "--control", str(root / "control.csv"), "--known-good", str(FIXTURES / "known-good.txt"), "--baseline-ratio", "1.0"] + result = self.run_evaluator(*inputs, "--failures", str(root / "pool.jsonl"), "--failures", str(root / "archive.jsonl")) + # The pool and archive nodes are monitored by default: one node's file is not enough. + one_node = self.run_evaluator(*inputs, "--failures", str(root / "archive.jsonl")) + lines = result.stdout.splitlines() + figures = [line for line in lines if line.split(" ", 1)[0] in ("pass", "fail", "unknown")] + self.assertEqual(len(figures), 4, result.stdout + result.stderr) + for line in figures: + self.assertIn("samples=", line) + self.assertIn("coverage=", line) + self.assertTrue(figures[0].startswith("pass ") and "pinned link" in figures[0], figures[0]) + self.assertTrue(figures[1].startswith("pass ") and "nodes=archive:100.0%,pool:100.0%" in figures[1], figures[1]) + # The fixture's closed_after_switch entry comes from a known-good IPv6 peer, reported by its label. + self.assertTrue(figures[2].startswith("fail ") and "closed_after_switch" in figures[2], figures[2]) + self.assertIn("known_good=known-good-3", result.stdout) + for address in self.ADDRESSES: + self.assertNotIn(address, result.stdout + result.stderr) + self.assertIn("canary verdict (figures 1, 2 and 4):", lines[-1]) + verdict = lines[-1].rsplit(" ", 1)[1] + self.assertEqual(result.returncode, {"pass": 0, "fail": 1, "unknown": 3}[verdict]) + + failures_line = next(line for line in one_node.stdout.splitlines() if "first_packet_failed or fallback" in line) + self.assertTrue(failures_line.startswith("unknown "), failures_line) + self.assertIn("unknown pool: coverage 0.0%", one_node.stdout) + + def test_sample_time_beyond_year_9999_is_not_a_sample(self) -> None: + pool = Node("pool", boot=T0 - DAY, pinned=PINNED) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "pool.csv" + for index in range(3): + row = pool.sample(T0 + index * INTERVAL) + pq_canary.append_row(path, {key: NA if value is None else value for key, value in dict(row, time=str(10**14) if index == 2 else row["time"]).items()}) + result = self.run_evaluator("--canary-start", str(T0), "--pool", str(path)) + self.assertNotIn("Traceback", result.stderr) + self.assertIn(f"PQ canary evaluation: {pq_eval.utc(T0)} to {pq_eval.utc(T0 + INTERVAL)}", result.stdout) + + def test_zero_baseline_is_reported_not_raised(self) -> None: + archive, control = Node("archive", boot=T0 - 60 * DAY), Node("control", boot=T0 - 60 * DAY, v1_0_0=True) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for index in range(-288, 288): + time = T0 + index * INTERVAL + for name, row in (("archive", archive.sample(time, connections_in=0 if index < 0 else 36)), ("control", control.sample(time))): + pq_canary.append_row(root / f"{name}.csv", {key: NA if value is None else value for key, value in row.items()}) + result = self.run_evaluator("--canary-start", str(T0), "--archive", str(root / "archive.csv"), "--control", str(root / "control.csv")) + self.assertEqual(result.returncode, 3, result.stdout + result.stderr) + self.assertNotIn("Traceback", result.stderr) + line = next(line for line in result.stdout.splitlines() if "connections_in within" in line) + self.assertTrue(line.startswith("unknown "), line) + self.assertIn("no baseline: archive connections_in summed to 0 over 288 valid pre-canary pairs", line) + + def test_argument_validation(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self.write_samples(root) + pool, failures, good = str(root / "pool.csv"), str(root / "archive.jsonl"), str(FIXTURES / "known-good.txt") + bad_csv = root / "bad.csv" + bad_csv.write_text("not,a,sample\n", encoding="utf8") + bad_good = root / "bad-known-good.txt" + bad_good.write_text("203.0.113.5:99999\n", encoding="utf8") + archive, control = str(root / "archive.csv"), str(root / "control.csv") + + def samples(name: str, *sources: str, **changes: str) -> str: + """A samples file with the rows of sources, with changes applied to each row.""" + for source in sources: + for row in pq_canary.read_rows(Path(source)): + pq_canary.append_row(root / name, {key: NA if value is None else value for key, value in dict(row, **changes).items()}) + return str(root / name) + start = ["--canary-start", str(T0)] + cases = { + "the following arguments are required: --canary-start": ["--pool", pool], + "expected Unix seconds or an ISO 8601 time": ["--canary-start", "yesterday", "--pool", pool], + "expected a time from 1970 to 9999, got '99999999999999'": start + ["--pool", pool, "--end", "99999999999999"], + "give at least one of --pool": start, + "unrecognized arguments: --bogus": start + ["--pool", pool, "--bogus"], + "unrecognized arguments: --pool-csv": start + ["--pool-csv", pool], + "unrecognized arguments: --canary-st": start + ["--canary-st", str(T0), "--pool", pool], + "no such file": start + ["--pool", str(root / "missing.csv")], + "expected a positive whole number": start + ["--pool", pool, "--interval", "0"], + "expected a whole number of seconds": start + ["--pool", pool, "--restart-grace", "-1"], + "expected a share greater than 0": start + ["--pool", pool, "--min-coverage", "1.5"], + "expected a positive ratio": start + ["--pool", pool, "--baseline-ratio", "inf"], + "expected a positive ratio, got '0'": start + ["--pool", pool, "--baseline-ratio", "0"], + "--failures needs --known-good": start + ["--failures", failures], + "--known-good is only used with --failures": start + ["--pool", pool, "--known-good", good], + "--monitored is only used with --failures": start + ["--pool", pool, "--monitored", "pool"], + "expected distinct comma-separated host labels": start + ["--failures", failures, "--known-good", good, "--monitored", "pool,pool"], + "--end is before --canary-start": start + ["--pool", pool, "--end", str(T0 - 1)], + "unexpected header": start + ["--pool", str(bad_csv)], + # Each role is a different node: never compare a node with itself. + "--archive and --control are the same file": start + ["--archive", archive, "--control", archive], + "--archive and --control both have the host label archive": + start + ["--archive", archive, "--control", samples("relabeled.csv", control, host="archive")], + "--archive mixes the host labels archive, control": start + ["--archive", samples("mixed.csv", archive, control)], + "--pool has a row without a valid host label": start + ["--pool", samples("no-host.csv", pool, host=NA)], + "--archive and --control report the same instance_id": + start + ["--archive", archive, "--control", samples("same-node.csv", archive, host="control")], + "--pool and --archive both have the host label pool": + start + ["--pool", pool, "--archive", samples("pool-as-archive.csv", archive, host="pool")], + "expected address or address:port with a port from 1 to 65535": start + ["--failures", failures, "--known-good", str(bad_good)], + } + for message, args in cases.items(): + with self.subTest(message): + result = self.run_evaluator(*args) + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertIn(message, result.stderr) + self.assertNotIn("Traceback", result.stderr) + self.assertNotIn("203.0.113.5", result.stderr) + # A failures.jsonl line that cannot be placed in time or attributed to a node. + for label, change in (("time true", {"time": True}), ("time text", {"time": str(T0)}), ("time negative", {"time": -1}), + ("time beyond year 9999", {"time": 10**12}), ("host null", {"host": None}), + ("host not a label", {"host": "archive node"}), ("host with a newline", {"host": "archive\n"})): + with self.subTest(label): + bad_failures = root / "bad.jsonl" + line = dict(record(T0, inbound=ring(0, 0, []), outbound=ring(0, 0, [])), **change) + bad_failures.write_text(json.dumps(line) + "\n", encoding="utf8") + result = self.run_evaluator(*start, "--failures", str(bad_failures), "--known-good", good) + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertIn("a failures.jsonl line lacks a time in Unix seconds or a host label", result.stderr) + self.assertNotIn("Traceback", result.stderr) + + def test_time_parsing(self) -> None: + self.assertEqual(pq_eval.parse_time("1760000000"), T0) + self.assertEqual(pq_eval.parse_time(pq_eval.utc(T0)), T0) + self.assertEqual(pq_eval.parse_time("2025-10-09T08:53:20+00:00"), T0) + with self.assertRaises(ValueError): + pq_eval.parse_time("2025-13-01") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/functional/feature_framework_unit_tests.py b/test/functional/feature_framework_unit_tests.py index 90dc316453..1343b93220 100755 --- a/test/functional/feature_framework_unit_tests.py +++ b/test/functional/feature_framework_unit_tests.py @@ -23,6 +23,7 @@ "crypto.ellswift", "key", "messages", + "crypto.mlkem", "network_sim", "crypto.muhash", "crypto.poly1305", diff --git a/test/functional/p2p_v2_previous_release.py b/test/functional/p2p_v2_previous_release.py new file mode 100755 index 0000000000..bf4b133edd --- /dev/null +++ b/test/functional/p2p_v2_previous_release.py @@ -0,0 +1,388 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 The qbit developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Baseline qbit v1.0.0's v2 transport toward non-empty version-packet contents. + +The hybrid post-quantum handshake (#184) puts an ML-KEM-1024 key or ciphertext +into the BIP324 version packet, whose contents v1.0.0 ignores. This test pins +that behavior on the v1.0.0 release binary, in both roles, before any node +change lands: + +- test_version_contents: v1.0.0 completes the handshake and exchanges + application messages when the peer's version packet carries 0, 1,572 or + 65,000 bytes of contents, or record-shaped contents (CompactSize(len) || + header || payload): one 0xF0 record exactly as the hybrid offer and accept + carry it (fd 21 06 f0 || 1,568 bytes), a 0xF0 record followed by an unknown + record, and a single 65,000-byte record. Each is sent after decoy packets, + and again after a 10 s pause, and v1.0.0's session id stays the plain BIP324 + one. As initiator it sends its version packet and its VERSION message + without waiting for ours. +- test_limits: v1.0.0 disconnects at once when a packet's length exceeds + MAX_CONTENTS_LEN (4,000,013 bytes), not at that length, and disconnects a + stalled handshake at -peertimeout. + +The peer is a BIP324 implementation on a plain socket, so the test decides +every byte it sends and when. +""" +from concurrent.futures import ThreadPoolExecutor +from io import BytesIO +import os +import shutil +import socket +import time + +from test_framework.messages import ( + msg_ping, + msg_verack, + msg_version, + ser_compact_size, +) +from test_framework.p2p import ( + MESSAGEMAP, + P2P_SERVICES, + P2P_SUBVERSION, + P2P_VERSION, + P2P_VERSION_RELAY, +) +from test_framework.test_framework import BitcoinTestFramework, SkipTest +from test_framework.test_node import release_info +from test_framework.util import ( + MAX_NODES, + assert_equal, + assert_greater_than_or_equal, + p2p_port, +) +from test_framework.v2_p2p import ( + LENGTH_FIELD_LEN, + MAX_GARBAGE_LEN, + MSGTYPE_TO_SHORTID, + SHORTID, + EncryptedP2PState, +) + +V1_0_0 = 10000 +# V2Transport's limit: the long message type encoding (1 + 12 bytes) plus MAX_PROTOCOL_MESSAGE_LENGTH. +MAX_CONTENTS_LEN = 1 + 12 + 4_000_000 +PQ_MLKEM1024 = 0xF0 +MLKEM1024_PAYLOAD_LEN = 1568 # an encapsulation key or a ciphertext + + +def record(header, payload): + """One version-contents record: CompactSize(len) || header || payload, len counting the header.""" + return ser_compact_size(1 + len(payload)) + bytes([header]) + payload + + +MLKEM1024_RECORD = record(PQ_MLKEM1024, bytes((i * 7 + 3) & 0xff for i in range(MLKEM1024_PAYLOAD_LEN))) +assert MLKEM1024_RECORD[:4] == bytes.fromhex("fd2106f0") and len(MLKEM1024_RECORD) == 1572 +# Version-packet contents to send, by name: raw sizes and record-shaped contents. +CONTENTS = { + "empty": b"", + "1,572 zero bytes": bytes(len(MLKEM1024_RECORD)), + "65,000 zero bytes": bytes(65_000), + "one 0xF0 ML-KEM-1024 record": MLKEM1024_RECORD, + "a 0xF0 record and an unknown 0xF1 record": MLKEM1024_RECORD + record(0xF1, bytes(100)), + "one 65,000-byte record": record(0xF1, bytes(65_000 - 4)), +} +assert len(CONTENTS["one 65,000-byte record"]) == 65_000 +DECOY_SIZES = [0, 37, 1000] +VERSION_DELAY = 10 +DEFAULT_PEER_TIMEOUT = 60 +STALL_PEER_TIMEOUT = 3 + + +class RawV2Peer: + """A BIP324 peer on a blocking socket, driven step by step by the test.""" + + def __init__(self, sock, *, initiating, timeout): + self.sock = sock + self.sock.settimeout(timeout) + self.state = EncryptedP2PState(initiating=initiating, net="regtest") + self.recvbuf = b"" + self.send_aad = b"" # our garbage until our first packet is sent + self.recv_aad = None # their garbage until their first packet is authenticated + # The port v1.0.0 sees for this connection: our source port when we connect + # to it, our listening port when it connects to us. + self.local_port = sock.getsockname()[1] + + @classmethod + def connect(cls, port, *, timeout): + return cls(socket.create_connection(("127.0.0.1", port), timeout=timeout), initiating=True, timeout=timeout) + + def send(self, data): + self.sock.sendall(data) + + def _recv_more(self): + data = self.sock.recv(65536) + if not data: + raise EOFError("peer closed the connection") + self.recvbuf += data + + def _recv_exact(self, n): + while len(self.recvbuf) < n: + self._recv_more() + data, self.recvbuf = self.recvbuf[:n], self.recvbuf[n:] + return data + + def exchange_keys(self, garbage_len=100): + """Send our key and garbage and read theirs; derive the BIP324 keys.""" + st = self.state + if st.initiating: + self.send(st.generate_keypair_and_garbage(garbage_len)) + ellswift_theirs = self._recv_exact(64) + else: + ellswift_theirs = self._recv_exact(64) + self.send(st.generate_keypair_and_garbage(garbage_len)) + st.initialize_v2_transport(st.v2_ecdh(st.privkey_ours, ellswift_theirs, st.ellswift_ours, st.initiating)) + self.send_aad = st.sent_garbage + + def send_terminator_and_decoys(self, decoy_sizes): + data = self.state.peer['send_garbage_terminator'] + for size in decoy_sizes: + data += self.state.v2_enc_packet(bytes(size), aad=self.send_aad, ignore=True) + self.send_aad = b"" + self.send(data) + + def send_version_packet(self, contents): + self.send(self.state.v2_enc_packet(contents, aad=self.send_aad)) + self.send_aad = b"" + + def send_length_only(self, length): + """Send just the encrypted length of a packet with the given contents length.""" + self.send(self.state.peer['send_L'].crypt(length.to_bytes(LENGTH_FIELD_LEN, 'little'))) + + def _receive_packet(self): + """Return the next non-decoy packet's contents, skipping their garbage first.""" + if self.recv_aad is None: + terminator = self.state.peer['recv_garbage_terminator'] + while (pos := self.recvbuf.find(terminator)) == -1: + assert len(self.recvbuf) < MAX_GARBAGE_LEN + len(terminator), "no garbage terminator" + self._recv_more() + self.recv_aad = self.recvbuf[:pos] + self.recvbuf = self.recvbuf[pos + len(terminator):] + while True: + length, contents = self.state.v2_receive_packet(self.recvbuf, aad=self.recv_aad) + assert length != -1, "packet authentication failed" + if length == 0: + self._recv_more() + continue + self.recvbuf = self.recvbuf[length:] + self.recv_aad = b"" + if contents is not None: + return contents + + def receive_version_packet(self): + return self._receive_packet() + + def send_message(self, message): + msgtype = message.msgtype + if msgtype in MSGTYPE_TO_SHORTID: + data = MSGTYPE_TO_SHORTID[msgtype].to_bytes(1, 'big') + else: + data = b"\x00" + msgtype.ljust(12, b"\x00") + self.send(self.state.v2_enc_packet(data + message.serialize())) + + def receive_message(self): + contents = self._receive_packet() + if contents[0] == 0: + msgtype, payload = contents[1:13].rstrip(b"\x00"), contents[13:] + else: + msgtype, payload = SHORTID.get(contents[0], b"unknown"), contents[1:] + if msgtype not in MESSAGEMAP: + return msgtype, None + message = MESSAGEMAP[msgtype]() + message.deserialize(BytesIO(payload)) + return msgtype, message + + def wait_for_message(self, wanted): + while True: + msgtype, message = self.receive_message() + if msgtype == wanted: + return message + + def assert_stays_open(self, seconds): + """Read for the given time and fail if the other side closes the connection.""" + timeout = self.sock.gettimeout() + deadline = time.time() + seconds + self.sock.settimeout(seconds) + try: + while time.time() < deadline: + self._recv_more() + except socket.timeout: + pass + finally: + self.sock.settimeout(timeout) + + def wait_for_close(self): + """Wait until the other side closes the connection; return the time it took.""" + start = time.time() + try: + while True: + self._recv_more() + except (EOFError, ConnectionError): + return time.time() - start + + def close(self): + self.sock.close() + + +def our_version_message(): + version = msg_version() + version.nVersion = P2P_VERSION + version.strSubVer = P2P_SUBVERSION + version.relay = P2P_VERSION_RELAY + version.nServices = P2P_SERVICES + version.addrFrom.ip = "0.0.0.0" + version.addrFrom.port = 0 + return version + + +class P2PV2PreviousReleaseTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + self.extra_args = [["-v2transport=1", f"-peertimeout={DEFAULT_PEER_TIMEOUT}"]] + + def skip_test_if_missing_module(self): + self.skip_if_no_previous_releases() + # The releases directory can hold other releases, such as Bitcoin Core's, without v1.0.0. + release = release_info(V1_0_0) + binaries = self.get_binaries(os.path.join(self.options.previous_releases_path, release.tag, "bin"), release.binary_prefix) + if any(shutil.which(argv[0]) is None for argv in (binaries.node_argv(), binaries.rpc_argv())): + raise SkipTest(f"the qbit {release.tag} release is not downloaded") + + def setup_nodes(self): + self.add_nodes(self.num_nodes, self.extra_args, versions=[V1_0_0]) + self.start_nodes() + + def run_test(self): + self.node = self.nodes[0] + assert_equal(self.node.getnetworkinfo()["version"], V1_0_0) + assert "P2P_V2" in self.node.getnetworkinfo()["localservicesnames"] + self.timeout = 30 * self.options.timeout_factor + + self.test_version_contents() + self.test_limits() + + def open_peers(self, roles): + """Open one raw v2 connection per role: "responder" means v1.0.0 responds (we + connect to it), "initiator" means v1.0.0 initiates (it connects to us).""" + peers = [] + listeners = 0 + for role in roles: + if role == "responder": + peers.append(RawV2Peer.connect(p2p_port(0), timeout=self.timeout)) + continue + # Ports of unused node indexes, distinct within one call and reused by later calls. + listeners += 1 + assert listeners < MAX_NODES + port = p2p_port(MAX_NODES - listeners) + with socket.create_server(("127.0.0.1", port)) as listener: + listener.settimeout(self.timeout) + self.node.addconnection(f"127.0.0.1:{port}", "outbound-full-relay", True) + sock, _ = listener.accept() + peers.append(RawV2Peer(sock, initiating=False, timeout=self.timeout)) + return peers + + def peer_info(self, peer, role): + """v1.0.0's getpeerinfo entry for a raw peer, or None once it dropped the peer.""" + inbound = role == "responder" + infos = [p for p in self.node.getpeerinfo() if p["addr"] == f"127.0.0.1:{peer.local_port}" and p["inbound"] == inbound] + return infos[0] if infos else None + + def run_handshake(self, peer, role, contents, delay): + """Handshake with decoys and the given contents, then exchange application messages. + + Runs on a worker thread, so it uses only the socket, never RPC.""" + peer.exchange_keys() + peer.send_terminator_and_decoys(DECOY_SIZES) + early = [] + if role == "initiator": + # v1.0.0 as initiator does not wait for our version packet: its version + # packet and its VERSION message are already on the way. + early.append(peer.receive_version_packet()) + peer.wait_for_message(b"version") + if delay: + time.sleep(delay) + peer.send_version_packet(contents) + if role == "responder": + early.append(peer.receive_version_packet()) + peer.send_message(our_version_message()) + peer.wait_for_message(b"version") + else: + peer.send_message(our_version_message()) + peer.send_message(msg_verack()) + peer.wait_for_message(b"verack") + ping = msg_ping(nonce=0x5a5a0000 + len(contents)) + peer.send_message(ping) + while peer.wait_for_message(b"pong").nonce != ping.nonce: + pass + return early[0] + + def check_cases(self, delay): + cases = [(role, name) for role in ("responder", "initiator") for name in CONTENTS] + peers = self.open_peers([role for role, _ in cases]) + with ThreadPoolExecutor(len(cases)) as pool: + futures = [pool.submit(self.run_handshake, peer, role, CONTENTS[name], delay) for peer, (role, name) in zip(peers, cases)] + their_contents = [f.result() for f in futures] + for peer, (role, name), contents in zip(peers, cases, their_contents): + self.log.debug(f"v1.0.0 as {role}, contents: {name}, {delay} s delay") + # v1.0.0 always sends empty contents. + assert_equal(contents, b"") + info = self.peer_info(peer, role) + assert info is not None, f"v1.0.0 as {role} dropped a peer whose contents were {name}" + assert_equal(info["transport_protocol_type"], "v2") + # The contents do not touch key derivation: the session id is plain BIP324's. + assert_equal(info["session_id"], peer.state.peer['session_id'].hex()) + peer.close() + self.wait_until(lambda: not self.node.getpeerinfo()) + + def test_version_contents(self): + self.log.info("v1.0.0 accepts raw and record-shaped version contents after decoys, in both roles") + self.check_cases(delay=0) + self.log.info(f"... and after a {VERSION_DELAY} s pause before the version packet") + self.check_cases(delay=VERSION_DELAY) + + def test_limits(self): + for role in ("responder", "initiator"): + self.log.info(f"v1.0.0 as {role} waits for a packet of exactly MAX_CONTENTS_LEN bytes") + peer, = self.open_peers([role]) + peer.exchange_keys() + peer.send_terminator_and_decoys([]) + peer.send_length_only(MAX_CONTENTS_LEN) + peer.assert_stays_open(2) + assert self.peer_info(peer, role) is not None + peer.close() + self.wait_until(lambda: not self.node.getpeerinfo()) + + self.log.info(f"v1.0.0 as {role} disconnects at once above MAX_CONTENTS_LEN") + with self.node.assert_debug_log([f"V2 transport error: packet too large ({MAX_CONTENTS_LEN + 1} bytes)"]): + peer, = self.open_peers([role]) + peer.exchange_keys() + peer.send_terminator_and_decoys([]) + peer.send_length_only(MAX_CONTENTS_LEN + 1) + assert peer.wait_for_close() < self.timeout + peer.close() + self.wait_until(lambda: not self.node.getpeerinfo()) + + self.log.info(f"v1.0.0 disconnects a stalled handshake at -peertimeout ({STALL_PEER_TIMEOUT} s), in both roles") + self.restart_node(0, extra_args=["-v2transport=1", f"-peertimeout={STALL_PEER_TIMEOUT}"]) + for role in ("responder", "initiator"): + # Until a version packet arrives the transport counts as detecting, hence "V2 handshake timeout". + with self.node.assert_debug_log(["V2 handshake timeout"]): + start = time.time() + peer, = self.open_peers([role]) + # Keys only: no garbage terminator, no version packet. + peer.exchange_keys() + peer.wait_for_close() + elapsed = time.time() - start + self.log.debug(f"v1.0.0 as {role} disconnected the stalled handshake after {elapsed:.1f} s") + # The check runs once m_connected + peertimeout < now, in whole seconds. + assert_greater_than_or_equal(elapsed, STALL_PEER_TIMEOUT - 1) + assert elapsed < STALL_PEER_TIMEOUT + 2 + 5 * self.options.timeout_factor, elapsed + peer.close() + self.wait_until(lambda: not self.node.getpeerinfo()) + + +if __name__ == '__main__': + P2PV2PreviousReleaseTest(__file__).main() diff --git a/test/functional/test_framework/crypto/mlkem.py b/test/functional/test_framework/crypto/mlkem.py new file mode 100644 index 0000000000..f06a2a16ff --- /dev/null +++ b/test/functional/test_framework/crypto/mlkem.py @@ -0,0 +1,461 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 The qbit developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +"""Test-only implementation of ML-KEM-1024 (FIPS 203). + +It is written from the FIPS 203 text, independently of the C library the node +uses, so the node's implementation can be checked against a second one. It uses +only Python's built-in hashlib for SHA3 and SHAKE, and imports nothing from the +C++ build. + +Interface (all byte strings): + mlkem1024_keygen(seed) -> (ek, dk) seed = d || z, 64 bytes + mlkem1024_encaps(ek, m) -> (ss, ct) m is 32 bytes + mlkem1024_decaps(dk, ct) -> ss implicit rejection on a bad ct + mlkem1024_check_ek(ek) -> bool FIPS 203 section 7.2 input check + +The functions are the deterministic ML-KEM.KeyGen_internal and +ML-KEM.Encaps_internal plus the FIPS 203 section 7.2 and 7.3 input checks; +callers supply the randomness (for example os.urandom). Inputs that fail a +check raise ValueError. + +WARNING: This code is slow and trivially vulnerable to side channel attacks. Do not use for +anything but tests. +""" + +import hashlib +import os +import unittest + +# ML-KEM-1024 parameters (FIPS 203 section 8, table 2). +N = 256 +Q = 3329 +K = 4 +ETA1 = 2 +ETA2 = 2 +DU = 11 +DV = 5 + +EK_SIZE = 384 * K + 32 # 1568 +DK_SIZE = 768 * K + 96 # 3168 +CT_SIZE = 32 * (DU * K + DV) # 1568 +SS_SIZE = 32 +SEED_SIZE = 64 # d || z +MSG_SIZE = 32 + + +def _bitrev7(i): + return int(f"{i:07b}"[::-1], 2) + + +# zeta^BitRev7(i) and zeta^(2*BitRev7(i)+1) with zeta = 17 (FIPS 203 appendix A). +ZETAS = [pow(17, _bitrev7(i), Q) for i in range(128)] +GAMMAS = [pow(17, 2 * _bitrev7(i) + 1, Q) for i in range(128)] + + +def _h(s): + return hashlib.sha3_256(s).digest() + + +def _j(s): + return hashlib.shake_256(s).digest(32) + + +def _g(c): + out = hashlib.sha3_512(c).digest() + return out[:32], out[32:] + + +def _prf(eta, s, b): + return hashlib.shake_256(s + bytes([b])).digest(64 * eta) + + +def byte_encode(f, d): + """ByteEncode_d (FIPS 203 algorithm 5): pack 256 d-bit integers, least significant bit first.""" + value = 0 + for i, a in enumerate(f): + value |= a << (i * d) + return value.to_bytes(32 * d, "little") + + +def byte_decode_raw(b, d): + """ByteDecode_d without the final reduction mod q (used by the modulus check).""" + assert len(b) == 32 * d + value = int.from_bytes(b, "little") + mask = (1 << d) - 1 + return [(value >> (i * d)) & mask for i in range(N)] + + +def byte_decode(b, d): + """ByteDecode_d (FIPS 203 algorithm 6).""" + f = byte_decode_raw(b, d) + if d == 12: + f = [a % Q for a in f] + return f + + +def compress(f, d): + """Compress_d: round(2^d / q * x) mod 2^d, rounding halves up.""" + return [(((x << (d + 1)) + Q) // (2 * Q)) & ((1 << d) - 1) for x in f] + + +def decompress(f, d): + """Decompress_d: round(q / 2^d * y), rounding halves up.""" + return [(Q * y + (1 << (d - 1))) >> d for y in f] + + +def ntt(f): + """NTT (FIPS 203 algorithm 9).""" + f = list(f) + i = 1 + length = 128 + while length >= 2: + for start in range(0, N, 2 * length): + zeta = ZETAS[i] + i += 1 + for j in range(start, start + length): + t = zeta * f[j + length] % Q + f[j + length] = (f[j] - t) % Q + f[j] = (f[j] + t) % Q + length //= 2 + return f + + +def ntt_inv(f): + """NTT^-1 (FIPS 203 algorithm 10).""" + f = list(f) + i = 127 + length = 2 + while length <= 128: + for start in range(0, N, 2 * length): + zeta = ZETAS[i] + i -= 1 + for j in range(start, start + length): + t = f[j] + f[j] = (t + f[j + length]) % Q + f[j + length] = zeta * (f[j + length] - t) % Q + length *= 2 + return [x * 3303 % Q for x in f] + + +def multiply_ntts(f, g): + """MultiplyNTTs (FIPS 203 algorithms 11 and 12).""" + h = [0] * N + for i in range(128): + a0, a1 = f[2 * i], f[2 * i + 1] + b0, b1 = g[2 * i], g[2 * i + 1] + h[2 * i] = (a0 * b0 + a1 * b1 * GAMMAS[i]) % Q + h[2 * i + 1] = (a0 * b1 + a1 * b0) % Q + return h + + +def _poly_add(f, g): + return [(a + b) % Q for a, b in zip(f, g)] + + +def _poly_sub(f, g): + return [(a - b) % Q for a, b in zip(f, g)] + + +def sample_ntt(seed34): + """SampleNTT (FIPS 203 algorithm 7): rejection-sample a polynomial from SHAKE128(seed34).""" + assert len(seed34) == 34 + # hashlib has no incremental squeeze, but SHAKE output is prefix-stable, so + # ask for a longer stream if the first one runs out. + stream_len = 168 * 4 + while True: + stream = hashlib.shake_128(seed34).digest(stream_len) + a = [] + pos = 0 + while len(a) < N and pos + 3 <= stream_len: + c0, c1, c2 = stream[pos], stream[pos + 1], stream[pos + 2] + pos += 3 + d1 = c0 + 256 * (c1 % 16) + d2 = c1 // 16 + 16 * c2 + if d1 < Q: + a.append(d1) + if d2 < Q and len(a) < N: + a.append(d2) + if len(a) == N: + return a + stream_len *= 2 + + +def sample_poly_cbd(b, eta): + """SamplePolyCBD_eta (FIPS 203 algorithm 8).""" + assert len(b) == 64 * eta + bits = int.from_bytes(b, "little") + f = [] + for i in range(N): + x = sum((bits >> (2 * i * eta + j)) & 1 for j in range(eta)) + y = sum((bits >> (2 * i * eta + eta + j)) & 1 for j in range(eta)) + f.append((x - y) % Q) + return f + + +def _sample_matrix(rho): + """A_hat[i][j] = SampleNTT(rho || j || i).""" + return [[sample_ntt(rho + bytes([j, i])) for j in range(K)] for i in range(K)] + + +def kpke_keygen(d): + """K-PKE.KeyGen (FIPS 203 algorithm 13).""" + rho, sigma = _g(d + bytes([K])) + a_hat = _sample_matrix(rho) + s = [sample_poly_cbd(_prf(ETA1, sigma, i), ETA1) for i in range(K)] + e = [sample_poly_cbd(_prf(ETA1, sigma, K + i), ETA1) for i in range(K)] + s_hat = [ntt(p) for p in s] + e_hat = [ntt(p) for p in e] + t_hat = [] + for i in range(K): + acc = e_hat[i] + for j in range(K): + acc = _poly_add(acc, multiply_ntts(a_hat[i][j], s_hat[j])) + t_hat.append(acc) + ek_pke = b"".join(byte_encode(p, 12) for p in t_hat) + rho + dk_pke = b"".join(byte_encode(p, 12) for p in s_hat) + return ek_pke, dk_pke + + +def kpke_encrypt(ek_pke, m, r): + """K-PKE.Encrypt (FIPS 203 algorithm 14).""" + t_hat = [byte_decode(ek_pke[384 * i:384 * (i + 1)], 12) for i in range(K)] + rho = ek_pke[384 * K:384 * K + 32] + a_hat = _sample_matrix(rho) + y = [sample_poly_cbd(_prf(ETA1, r, i), ETA1) for i in range(K)] + e1 = [sample_poly_cbd(_prf(ETA2, r, K + i), ETA2) for i in range(K)] + e2 = sample_poly_cbd(_prf(ETA2, r, 2 * K), ETA2) + y_hat = [ntt(p) for p in y] + u = [] + for i in range(K): + acc = [0] * N + for j in range(K): + acc = _poly_add(acc, multiply_ntts(a_hat[j][i], y_hat[j])) + u.append(_poly_add(ntt_inv(acc), e1[i])) + mu = decompress(byte_decode(m, 1), 1) + acc = [0] * N + for j in range(K): + acc = _poly_add(acc, multiply_ntts(t_hat[j], y_hat[j])) + v = _poly_add(_poly_add(ntt_inv(acc), e2), mu) + c1 = b"".join(byte_encode(compress(p, DU), DU) for p in u) + c2 = byte_encode(compress(v, DV), DV) + return c1 + c2 + + +def kpke_decrypt(dk_pke, c): + """K-PKE.Decrypt (FIPS 203 algorithm 15).""" + c1 = c[:32 * DU * K] + c2 = c[32 * DU * K:] + u = [decompress(byte_decode(c1[32 * DU * i:32 * DU * (i + 1)], DU), DU) for i in range(K)] + v = decompress(byte_decode(c2, DV), DV) + s_hat = [byte_decode(dk_pke[384 * i:384 * (i + 1)], 12) for i in range(K)] + acc = [0] * N + for i in range(K): + acc = _poly_add(acc, multiply_ntts(s_hat[i], ntt(u[i]))) + w = _poly_sub(v, ntt_inv(acc)) + return byte_encode(compress(w, 1), 1) + + +def _check_bytes(name, value, size): + if not isinstance(value, (bytes, bytearray)) or len(value) != size: + raise ValueError(f"{name} must be {size} bytes") + return bytes(value) + + +def mlkem1024_check_ek(ek): + """FIPS 203 section 7.2 encapsulation-key check: the type (length) check and the modulus check. + + The modulus check requires every 12-bit coefficient of the encoded t_hat to be below q, which + is the same as ByteEncode12(ByteDecode12(ek[0:1536])) == ek[0:1536]. + """ + if not isinstance(ek, (bytes, bytearray)) or len(ek) != EK_SIZE: + return False + return all(a < Q for i in range(K) for a in byte_decode_raw(bytes(ek[384 * i:384 * (i + 1)]), 12)) + + +def mlkem1024_keygen(seed): + """ML-KEM.KeyGen_internal (FIPS 203 algorithm 16) from seed = d || z. Returns (ek, dk).""" + seed = _check_bytes("seed", seed, SEED_SIZE) + d, z = seed[:32], seed[32:] + ek_pke, dk_pke = kpke_keygen(d) + ek = ek_pke + dk = dk_pke + ek + _h(ek) + z + return ek, dk + + +def mlkem1024_encaps(ek, m): + """ML-KEM.Encaps_internal (FIPS 203 algorithm 17) after the section 7.2 checks. Returns (ss, ct).""" + ek = _check_bytes("ek", ek, EK_SIZE) + m = _check_bytes("m", m, MSG_SIZE) + if not mlkem1024_check_ek(ek): + raise ValueError("ek fails the modulus check") + ss, r = _g(m + _h(ek)) + ct = kpke_encrypt(ek, m, r) + return ss, ct + + +def mlkem1024_decaps(dk, ct): + """ML-KEM.Decaps_internal (FIPS 203 algorithm 18) after the section 7.3 checks. Returns ss. + + A ciphertext of the right length that does not re-encrypt to itself yields the implicit + rejection secret J(z || ct), as the standard requires. + """ + dk = _check_bytes("dk", dk, DK_SIZE) + ct = _check_bytes("ct", ct, CT_SIZE) + dk_pke = dk[:384 * K] + ek_pke = dk[384 * K:768 * K + 32] + h = dk[768 * K + 32:768 * K + 64] + z = dk[768 * K + 64:] + if _h(ek_pke) != h: + raise ValueError("dk fails the hash check") + m_prime = kpke_decrypt(dk_pke, ct) + ss_prime, r_prime = _g(m_prime + h) + ss_reject = _j(z + ct) + ct_prime = kpke_encrypt(ek_pke, m_prime, r_prime) + if ct != ct_prime: + return ss_reject + return ss_prime + + +def _read_test_vectors(): + """Parse mlkem1024_test_vectors.txt into {kind: [fields]} (see that file's header).""" + vectors_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "mlkem1024_test_vectors.txt") + vectors = {} + with open(vectors_file, encoding="utf8") as f: + for line in f: + if line.startswith("#") or not line.strip(): + continue + kind, *fields = line.split() + vectors.setdefault(kind, []).append(dict(field.split("=", 1) for field in fields)) + return vectors + + +class TestMLKEM1024(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.vectors = _read_test_vectors() + + def test_ntt_roundtrip(self): + f = [(i * 1009 + 7) % Q for i in range(N)] + self.assertEqual(ntt_inv(ntt(f)), f) + + def test_acvp_keygen(self): + """ACVP keyGen: exact ek and dk from d || z.""" + self.assertTrue(self.vectors["keyGen"]) + for v in self.vectors["keyGen"]: + ek, dk = mlkem1024_keygen(bytes.fromhex(v["d"] + v["z"])) + self.assertEqual((len(ek), len(dk)), (EK_SIZE, DK_SIZE)) + self.assertEqual(ek.hex(), v["ek"], f"tcId {v['tcId']}") + self.assertEqual(dk.hex(), v["dk"], f"tcId {v['tcId']}") + + def test_acvp_encaps(self): + """ACVP encapsulation: exact ct and shared secret from ek and m.""" + self.assertTrue(self.vectors["encapsulation"]) + for v in self.vectors["encapsulation"]: + ek = bytes.fromhex(v["ek"]) + self.assertTrue(mlkem1024_check_ek(ek)) + ss, ct = mlkem1024_encaps(ek, bytes.fromhex(v["m"])) + self.assertEqual((len(ss), len(ct)), (SS_SIZE, CT_SIZE)) + self.assertEqual(ct.hex(), v["c"], f"tcId {v['tcId']}") + self.assertEqual(ss.hex(), v["k"], f"tcId {v['tcId']}") + + def test_acvp_decaps(self): + """ACVP decapsulation, including modified ciphertexts that need the implicit-rejection secret.""" + reasons = set() + for v in self.vectors["decapsulation"]: + dk, ct = bytes.fromhex(v["dk"]), bytes.fromhex(v["c"]) + ss = mlkem1024_decaps(dk, ct) + self.assertEqual(ss.hex(), v["k"], f"tcId {v['tcId']}") + if v["reason"] == "modified_ciphertext": + # The rejection secret is J(z || ct), with z the last 32 bytes of dk. + self.assertEqual(ss, hashlib.shake_256(dk[-32:] + ct).digest(32)) + reasons.add(v["reason"]) + self.assertEqual(reasons, {"modified_ciphertext", "valid_decapsulation"}) + + def test_acvp_encapsulation_key_check(self): + """ACVP encapsulationKeyCheck: valid keys pass the section 7.2 check and encapsulate; the others are rejected.""" + vectors = self.vectors["encapsulationKeyCheck"] + self.assertEqual({v["passed"] for v in vectors}, {"0", "1"}) + for v in vectors: + ek = bytes.fromhex(v["ek"]) + passed = v["passed"] == "1" + self.assertEqual(mlkem1024_check_ek(ek), passed, f"tcId {v['tcId']}") + if passed: + ss, ct = mlkem1024_encaps(ek, bytes(MSG_SIZE)) + self.assertEqual((len(ss), len(ct)), (SS_SIZE, CT_SIZE)) + else: + with self.assertRaises(ValueError): + mlkem1024_encaps(ek, bytes(MSG_SIZE)) + + def test_cctv_modulus(self): + """C2SP CCTV keys with one coefficient in [q, 4095] fail the modulus check; q - 1 there passes.""" + base = None + for v in self.vectors["modulus"]: + if "ek" in v: + ek = bytes.fromhex(v["ek"]) + base = ek + else: + ek = bytearray(base) + for patch in v["patch"].split(","): + offset, data = patch.split(":") + ek[int(offset):int(offset) + len(data) // 2] = bytes.fromhex(data) + ek = bytes(ek) + self.assertEqual(hashlib.sha256(ek).hexdigest(), v["sha256"], f"line {v['line']}") + self.assertEqual(len(ek), EK_SIZE) + self.assertFalse(mlkem1024_check_ek(ek), f"line {v['line']}") + with self.assertRaises(ValueError): + mlkem1024_encaps(ek, bytes(MSG_SIZE)) + # Each key has exactly one coefficient >= q. Setting it to q - 1 = 3328, the largest + # valid value, must make the key pass. + polys = [byte_decode_raw(ek[384 * i:384 * (i + 1)], 12) for i in range(K)] + bad = [(i, j) for i in range(K) for j in range(N) if polys[i][j] >= Q] + self.assertEqual(len(bad), 1, f"line {v['line']}") + i, j = bad[0] + polys[i][j] = Q - 1 + fixed = b"".join(byte_encode(p, 12) for p in polys) + ek[384 * K:] + self.assertTrue(mlkem1024_check_ek(fixed), f"line {v['line']}") + self.assertEqual(len(self.vectors["modulus"]), 32) + # A key at the boundary is usable, not only accepted by the check. + ss, ct = mlkem1024_encaps(fixed, bytes(MSG_SIZE)) + self.assertEqual((len(ss), len(ct)), (SS_SIZE, CT_SIZE)) + + def test_roundtrip_and_implicit_rejection(self): + ek, dk = mlkem1024_keygen(bytes(range(SEED_SIZE))) + ss, ct = mlkem1024_encaps(ek, bytes(range(100, 100 + MSG_SIZE))) + self.assertEqual(mlkem1024_decaps(dk, ct), ss) + for pos in (0, CT_SIZE - 1): + bad_ct = bytearray(ct) + bad_ct[pos] ^= 0x01 + bad_ct = bytes(bad_ct) + ss_bad = mlkem1024_decaps(dk, bad_ct) + self.assertNotEqual(ss_bad, ss) + self.assertEqual(ss_bad, hashlib.shake_256(dk[-32:] + bad_ct).digest(32)) + + def test_input_checks(self): + ek, dk = mlkem1024_keygen(bytes(SEED_SIZE)) + _, ct = mlkem1024_encaps(ek, bytes(MSG_SIZE)) + for size in (0, SEED_SIZE - 1, SEED_SIZE + 1): + with self.assertRaises(ValueError): + mlkem1024_keygen(bytes(size)) + for bad_ek in (b"", ek[:-1], ek + b"\x00"): + self.assertFalse(mlkem1024_check_ek(bad_ek)) + with self.assertRaises(ValueError): + mlkem1024_encaps(bad_ek, bytes(MSG_SIZE)) + for bad_m in (b"", bytes(MSG_SIZE - 1), bytes(MSG_SIZE + 1)): + with self.assertRaises(ValueError): + mlkem1024_encaps(ek, bad_m) + for bad_ct in (b"", ct[:-1], ct + b"\x00"): + with self.assertRaises(ValueError): + mlkem1024_decaps(dk, bad_ct) + for bad_dk in (b"", dk[:-1], dk + b"\x00"): + with self.assertRaises(ValueError): + mlkem1024_decaps(bad_dk, ct) + # FIPS 203 section 7.3 hash check: H(ek) stored in dk must match the embedded ek. + bad_dk = bytearray(dk) + bad_dk[768 * K + 32] ^= 0x01 + with self.assertRaises(ValueError): + mlkem1024_decaps(bytes(bad_dk), ct) + with self.assertRaises(ValueError): + mlkem1024_encaps(list(ek), bytes(MSG_SIZE)) diff --git a/test/functional/test_framework/crypto/mlkem1024_test_vectors.txt b/test/functional/test_framework/crypto/mlkem1024_test_vectors.txt new file mode 100644 index 0000000000..1dcfbac707 --- /dev/null +++ b/test/functional/test_framework/crypto/mlkem1024_test_vectors.txt @@ -0,0 +1,64 @@ +# ML-KEM-1024 test vectors for test_framework/crypto/mlkem.py. A small subset of the sources below; +# each record is one line: a kind, then space-separated name=value fields, hex in lowercase. +# +# keyGen, encapsulation, decapsulation: NIST ACVP-Server (https://github.com/usnistgov/ACVP-Server), +# commit 975de31eb83d87039ec88934fdc47d8c312b892d, gen-val/json-files/ML-KEM-keyGen-FIPS203/internalProjection.json +# (tgId 3, ML-KEM-1024, tcId 51, 52) and +# gen-val/json-files/ML-KEM-encapDecap-FIPS203/internalProjection.json (tgId 3 encapsulation, +# tcId 51, 52; tgId 6 decapsulation, tcId 96, 97, 98; tgId 12 +# encapsulationKeyCheck, tcId 156, 159, 160, 161, 163). Field names follow ACVP: c is the ciphertext +# and k the shared secret; a decapsulation with reason=modified_ciphertext expects the +# implicit-rejection secret; an encapsulationKeyCheck with passed=0 is an ek that must fail the +# FIPS 203 section 7.2 check. +# +# modulus: C2SP CCTV (https://github.com/C2SP/CCTV/tree/main/ML-KEM), commit +# 50a8ecf2a220f4c8bdc4f085789b8e85c26829e7, ML-KEM/modulus/ML-KEM-1024.txt.gz: encapsulation keys that each +# carry one coefficient in [q, 4095] and must fail the FIPS 203 modulus check. line is the line +# number in that file: lines 1-16 (q and 4095 at the first and last coefficient of each +# polynomial) and every 64th line from line 18. The keys share all other coefficients, so line 1 is +# stored in full and the others as byte patches against it (patch=offset:hex,...); sha256 is the +# SHA-256 of the line's key bytes in the source file, which the tests check after patching. +keyGen tcId=51 d=f3a706faf090c03db506863ab0b20bd8a1627956318e88c67eb875e8e7266009 z=35d2bc43dd1cc879f765bf2a0c5e297889dde910e57e2bb0eae417b90ab7a275 ek=8d0923ca8a2da2b4146ec25321122b8a5aa8afe0c03415273008a46ee83031e98aaaa125abc75d3b30322560c197e75dd0e48a348099f7b2144d7b8a8660a4a97bcf19c0583bd9bb2123033cd7bb5a14b08b817831a673a28170f5f6443c0551913a327cba18c3a053c4040250403b70ab9588832403aa0fc37665e04980fe1602e7d2715d9cbc00515df432a4f5b32b3bc92ae3f31700166d498123e94576509b712b18491b1435ee7ab7aeb1ad30d72348c3cc083abe24a8b12097bf32f792476288eecc3bf630adcdac6aca7950d9839501a448500742bae37f109203a809b2b960a307e25347a32c3eab79288173a878789b296e9e8c1c28c5bb3ac472601c9765f7b77225a810c7b85370bef4a5b079d2015ada54236b8f33840675f9b2eb427a1b5974cd5c61b24010886c5a7bda5bbed974af7217f3338ad719cb308a8bcb1b6d6ed2a1643736c29095e8a8452a3a36c7bb5ae58cbfdc61529466a90f454ed6895b0861083dd1371999b2f559a3a487cf59a074fb49215ea6a6be656f9af17b121a2447cb7985590e9738842b899ba57ac311810ae2d9794f37483dd6bccb64af6d56588ae94665961c025c3aa2861974c236bca4bd8ff5509f7ab774593e7c5549e57c2f18d15c0515094ad9a0dfaa0601e524f8231156b627bb25a0dae04dacd0a66c041cef400583fc13bae640291a39a5c5ca8bca1ad5c683cdd8290891a76940817dd8c9f52678780548e37a05806600801426dbb950c3b2ba34e24cc77864dd91b39f1408c716a69df63342854e50a245fb50977b9410ded2c93f86b1f9d5a78b87bf81e51ca620a7e8566b19ab700964a40e3266415228d432156e5cbdf52364a90483a55c39b3fb16fc7465a3f8ac801b70b9fb28b583444ba5c1a73722d417a9d6d9b7deb08bc6b330ff27cf61ab8831e27758c64af3b12150cb7b33abc29858106d63686d8762459abf9413850ae53ed6313f76f83d0fb8ab34374e7df693e4a1b3e5a8ad0ce820afe1cf401acde650a8101b0946022d52178e19613c42b88b07cc04eaa81dfb28ac9dc076236b67219a30f8f945dd57bd2f335c52d59372308d38993467db53da3382b74867b616481bd0091a2232c1116dc88a589db9107224a681008c67c589186a6929549beef92253db02b0c8aa9f9c875a670266c72bcbdb4f5625043703c1a0457395832e4c335180462ed2220c59e7361903c107d85457f6cd82eb820d0855d97675c2e0151cdb73c2885ddb7849d74541580124e890116a65bc068093b57914e20c937c60a3eb25576f1a976a9583839b672144cd4a45c3477a45c29b4e0bc2bdbd206585c9b7a7741c8b6b5793a92797a15ae7a5b73a74b2971463634ba52aa792af05530730b6d0a89a346156b733677932bd36593a7496130cc458dcc5ca987c21960604ec8a8c5396056680cbf3f1aac4f401aa5029fb2150434bb4706c31a2d54e4297939fa7c9c6f85700613ceb65c7f03ac56eb86e2d27ccc6dcb7b9394dcdb942ff222d86958a996c0cb6a8a44f97a70441c95fa71250116eec20863c0b5a643458788ab001f8869d909922f51ee547a1e889255b3a0599c65842e5ab8d73872f053bc62392ea53896d328102d460bf1609583c22c3b43780ec6dad0319eb4a5a65b4756c3cb40eaa935183bf8bd46abe76ba46e199103a5313c3235f49c915e097bca804de680781d8365731beac6789a9203fb8787c4c070e00a13a6722a66a28236db179825653d33ccf898b72c6b8450d97afd3276bb13340519cbeda708d12a858f54c49f4547195b7788a9150b2649e36aa394121926d568a488b16d3557b2a32af57d11fc3373f80a28c0723273d362502e7c428ab44d3cbabf9ea585fd1bd0c9846556a1e196b78cf951592984a0a8487a78c2317d7aca4118e1049750a0788f0d66ad9e48e34731130aba0b427360a856d96d80b3f028fdd3aba9035c10106ba1c0934bed36c6d7c7434249654ea89fc22137f4ab903653b75fb25b6f01635e6cc7d39cf1508690562826b49b6ffc59e0dd35022e541f8ba0d304aa5b4e20606907c424395666c54abc2b8fb009847c86317685000c231215c8c15945860f6a85ddb98a8c3a527f2749d3c027e694e8f0b0f0fa454913aadb635aadd452f7128bf7752569669a8b93290eb92e78f6adff23e89f57f3890753b51f12f3f3a8a654e677847 dk=6cbbb491d7b3a3439f53c9027bd727a0e1c78004b4cdb66760e50f1bf97fff0139f82862f1f3178a077b6af77735cb802e91316bc207b9f1485ff7a868891e003bb9c6558be6052c0826af1cb5836b94c7d5d714eefc7cfd983e9884b71348499dca05def33b49d1c14bbbb95ab195bd72219b18bd0c704275cc94ab06653bbb9e0eec74181903a0081aae068b81ebae8925335b8134b1480a60f758282c2f011b909641266774513a19090bc672eb3a0708cb577f134151c155c1e560d9b335a35a4d75f223502752cd250c9a7778efc76a52709813a24679b987a6172f1714641ea6902d364f96357e92b44275ebcce7f6207bb0a215e98b3aa92daa46a39ad9132f3874c6239391e716ba600e42bc7084425051f865756a89dab46a76104e183a12031b99be86257b583c2e88104fec3bfcaa50f238beec20abf31219685885f2c3ce2f80c09b874e73a8712d35b8ee819d84b1c59a37947c0b9e25f066ee030ed28172fd5914b562408ba9b9de9bab2b95a4f30c22ece458e133371a693d9458058265a75ad6ca5707617b38a1b6f396fb199cc1c77100741de37500f1bbb10b9204eeb7c3f5b91a1cc91b6d4072af410957358acb0625488b9f6a9915758b7cc9b4afe7d5ad6a95231a7a711b3a8996e23afb3695b45b6d919c771d7685d90a7550a9b5b1e7538a365451a5a815009e49326c6541caf0ba685056bf82ea4ab504161b555568583802800554a9480602437b0b9677fac638a2508154a658e0623c62666e078d21a5a545a818d66c9547b9749458b735a4ad51812950e8a625a1b2a753abbba16415006300ad32d34c8b0b3b2af4d1cfbde08b4f06c70de4ba28fb3763745cce133dd2e488f89a51e7a586a86ac928867b0541438af5645de3211aa94aca4a1095896ebc363c374a85565c1eba030b5ea21b9b8a140e131ed07580d5b8590446988da65939a823a84661b26ca12461afe373519f43c13987a96a9069cd6b9177b205b25b5197dbcde66c6457892c1df96fe50ab7ef3b3dab0a87436b3451476c44aa22aef918ae5a1a8ae46920263032e3bc5debcfee43b8aa290cedf06966939ee6cc7c6af83cb9d90cc889ab8b5a3d4c224e73916ae057bf5455528cacbed31b554043c6b0ea196854461d2644461470efc55080e326fed49d62b5a4bb06b14f505eb4486c9a1b033b927836da656c2475e5b8b823986dca99b298512d94b77fe0588dbb96ba53a67369b82c2beab2ccc0103e549317a99dd97b598fa0a506184eb55c96b17302533c7ec3334e3cec2bbb22be0f89442840884fab4a766a41bc8061d4abcac5190c729309ac6151bd434b818bc1e7d0cd435c9ca6dc7182228ff93867134a5cc65c6167c4a8bb830e7feb863c15bf87b259eb0518ff5248792a625f37c589589799746d36197dc63aabd134a00ef1b49c79a2df67b28e6b6e989cc9d0577ea5aa1b2fe64b322c7a8b241d503523d875ccdc479d28e3c01bd8383dcb2b684281e6949171f518ea97009237b5cd42ac8fc8a571716eb784b336529db67728499988b73b80ef5c6d33f3b42ada7a0a36a7c6a8a0dfb804b3d7bb05803957b913827a16f52acdd7650ee2971a85dc21f6a94a5b31988e4438a9c1733ab8c48c32653fe800bf81ba66658cd2001acf3a83e3a0426ab6c3b2600a53621277a43d723ab96758b120f120248c15edf4294fc38f96ec490bd52094655841d9355038a379d8b666875171f39218e21d37023f5108b6d15389f07c1241e15071c43cd7d160eb5341b719c2055424ef6cbf66815f84943c9481cd1de685a74c5b9db37a0713952fd4a67bec8ca7496f6c9c5fb84a8eafe68d54654c6bac9c3ab712c2bc32c0cab072c18de3cabbd9224dfaa305dfdb3f85019d7b988577b40f27514822db42d87489502c53cce6a095b486d9143d6d160b1c7313bf288913657698117eb145c0c9c55d66c15bf1a2b268f6a5fa259139551abfe3468f1cc49c01b4232543def0bf9ae503b998536310751a8bc302b4b4440b835233797c967e7f9308155b27ba63c65e08bdd1946828196b30c042cda0b56e4008215ac51f373604f5930bb66297c32be41a162ca21471cc86657b5cb5307d89091da9a5137f1c1433ab8ac1d59de421c7c17a883eab99ba4714f841998d0923ca8a2da2b4146ec25321122b8a5aa8afe0c03415273008a46ee83031e98aaaa125abc75d3b30322560c197e75dd0e48a348099f7b2144d7b8a8660a4a97bcf19c0583bd9bb2123033cd7bb5a14b08b817831a673a28170f5f6443c0551913a327cba18c3a053c4040250403b70ab9588832403aa0fc37665e04980fe1602e7d2715d9cbc00515df432a4f5b32b3bc92ae3f31700166d498123e94576509b712b18491b1435ee7ab7aeb1ad30d72348c3cc083abe24a8b12097bf32f792476288eecc3bf630adcdac6aca7950d9839501a448500742bae37f109203a809b2b960a307e25347a32c3eab79288173a878789b296e9e8c1c28c5bb3ac472601c9765f7b77225a810c7b85370bef4a5b079d2015ada54236b8f33840675f9b2eb427a1b5974cd5c61b24010886c5a7bda5bbed974af7217f3338ad719cb308a8bcb1b6d6ed2a1643736c29095e8a8452a3a36c7bb5ae58cbfdc61529466a90f454ed6895b0861083dd1371999b2f559a3a487cf59a074fb49215ea6a6be656f9af17b121a2447cb7985590e9738842b899ba57ac311810ae2d9794f37483dd6bccb64af6d56588ae94665961c025c3aa2861974c236bca4bd8ff5509f7ab774593e7c5549e57c2f18d15c0515094ad9a0dfaa0601e524f8231156b627bb25a0dae04dacd0a66c041cef400583fc13bae640291a39a5c5ca8bca1ad5c683cdd8290891a76940817dd8c9f52678780548e37a05806600801426dbb950c3b2ba34e24cc77864dd91b39f1408c716a69df63342854e50a245fb50977b9410ded2c93f86b1f9d5a78b87bf81e51ca620a7e8566b19ab700964a40e3266415228d432156e5cbdf52364a90483a55c39b3fb16fc7465a3f8ac801b70b9fb28b583444ba5c1a73722d417a9d6d9b7deb08bc6b330ff27cf61ab8831e27758c64af3b12150cb7b33abc29858106d63686d8762459abf9413850ae53ed6313f76f83d0fb8ab34374e7df693e4a1b3e5a8ad0ce820afe1cf401acde650a8101b0946022d52178e19613c42b88b07cc04eaa81dfb28ac9dc076236b67219a30f8f945dd57bd2f335c52d59372308d38993467db53da3382b74867b616481bd0091a2232c1116dc88a589db9107224a681008c67c589186a6929549beef92253db02b0c8aa9f9c875a670266c72bcbdb4f5625043703c1a0457395832e4c335180462ed2220c59e7361903c107d85457f6cd82eb820d0855d97675c2e0151cdb73c2885ddb7849d74541580124e890116a65bc068093b57914e20c937c60a3eb25576f1a976a9583839b672144cd4a45c3477a45c29b4e0bc2bdbd206585c9b7a7741c8b6b5793a92797a15ae7a5b73a74b2971463634ba52aa792af05530730b6d0a89a346156b733677932bd36593a7496130cc458dcc5ca987c21960604ec8a8c5396056680cbf3f1aac4f401aa5029fb2150434bb4706c31a2d54e4297939fa7c9c6f85700613ceb65c7f03ac56eb86e2d27ccc6dcb7b9394dcdb942ff222d86958a996c0cb6a8a44f97a70441c95fa71250116eec20863c0b5a643458788ab001f8869d909922f51ee547a1e889255b3a0599c65842e5ab8d73872f053bc62392ea53896d328102d460bf1609583c22c3b43780ec6dad0319eb4a5a65b4756c3cb40eaa935183bf8bd46abe76ba46e199103a5313c3235f49c915e097bca804de680781d8365731beac6789a9203fb8787c4c070e00a13a6722a66a28236db179825653d33ccf898b72c6b8450d97afd3276bb13340519cbeda708d12a858f54c49f4547195b7788a9150b2649e36aa394121926d568a488b16d3557b2a32af57d11fc3373f80a28c0723273d362502e7c428ab44d3cbabf9ea585fd1bd0c9846556a1e196b78cf951592984a0a8487a78c2317d7aca4118e1049750a0788f0d66ad9e48e34731130aba0b427360a856d96d80b3f028fdd3aba9035c10106ba1c0934bed36c6d7c7434249654ea89fc22137f4ab903653b75fb25b6f01635e6cc7d39cf1508690562826b49b6ffc59e0dd35022e541f8ba0d304aa5b4e20606907c424395666c54abc2b8fb009847c86317685000c231215c8c15945860f6a85ddb98a8c3a527f2749d3c027e694e8f0b0f0fa454913aadb635aadd452f7128bf7752569669a8b93290eb92e78f6adff23e89f57f3890753b51f12f3f3a8a654e6778479370fe5b05ddc92c939f62cbde4c0fea36f45cd20c5748cf3ac891a4c260449635d2bc43dd1cc879f765bf2a0c5e297889dde910e57e2bb0eae417b90ab7a275 +keyGen tcId=52 d=52f2df12173088140ca73333d1db4341949be452049028ef0dd904bc7e0ddef7 z=cc96de9e1f0506bf490a154a6db46bd2bd1413c3d1dc163190ff171ef0d11e22 ek=88c29c2a91ce80b13eebe182aa709a1694a456b139deb1977d531ad2722ac7d2b356aa2684d767c60012b1a2345a69ac52c7a814858e8c741a80c4b5e942ceee0045b957748e2847ea835e1fe17c70456dbea13fcee0384bf431225c8a6b499624f086bf71518c8c269640b436e7783057372825768670b2478c96c4a1a2adcb1f83a057263362476197c5d878d76b025f83ac802ccc02296a99296cf6b23c5617aa95387c2b0071c23ab7bf614371bc7001e29d7109057dc03053fb38c0084c48890612369c649c0ba3c2603e377c84d15b6db6bb6e9513ada9a47210861aa055d14683f0556729192caea8ae9a538ff810596d64bc0964b380cbc2f5d706a0a8135c1b87c3480c6d54cfc28a53c339666e24213d33a62ec872d803103987a36256840dd98da8bc17ed42c34f5c008f6c678c748f3f7404d17b46f29012f079a46a0abf77c04538254ed4b6bdea929724aa60014674c427c55b1b0192cb284ed02194f9615417b08a9580dca78944f43094e48aebbc8bf0126310d9aa79530457e04dba9b0d02b045ab28391f923c25caa69ad038b4586350613d9ae22896f2979d56bbafdaa294041aa42c21b9ac59e41cbedbc33ad6996c3451b7df5b9ae2d003f5092b91847290336a62e804b5f167d7cc4e31f40f6c640c10eb99d7f5519c80147b77843cd083b3b5c03718b1a7ea2dbe25adbd56762e9150b46b6b4e82b769c384cab39b3610070c6b6e5981c3b182105981a50567514ca15eb44620c231740feb6bcbf4980f7b1824a5602a982a68bc8961bbc441b111c93b2374ec88fef30d4931ab0ce3b5bea680b1099cfa79b3fff2b56bb3031a8c07519c7df4f75a8c887c6a1b7249a700717b2798c02f0ea04e720752d8323376e1635331b07435cdd5ba6d7228281612bf060c694bab6a00a8700977518bbac8195853fdd34d44e29fea7286dcfabdf47c9b2f2347ee7b3133cc275e601aa1874bd6974894c99d9659c1910030ad033138a57f8d9b312208873f733db9c73007f45c49a007bfab03aaac3f107c8c0a032cbe24602a2bcae4d2658d21899ea89ec5e832c0565701e5baf0da89e1b55ecab7802ec830b8268f6d9ca3883c97aacc20fc4861eb59707afa18a7b24534956301f238dffb68bd20a8264acb6be4918c85282f63cbaca87a3530515041c5a00cadac1a0b365b8a6cf626b30a49338461e94b44e20c250a23c7473952a20404fc17055e124792621d417cc7e20381bb24a701d428b1643bc0054b5ff46ba9129c0126889156bc6418a4c3b8c6d41b3d8646a915592305c30886b3995400a2a80771a494795d4b4b9ea6c21e0b21e0933f9f7c5fcb3b4c20aa303f655cb003526f8b04b3c85260c00c8e316db495417da22bd3a975fe784434c04dffa545f91555f01033696bc4b87abc0665b7a5c972d9a08fc8911147da3cef156b5ce85391161f32d09f6f8c0f8a845e268aa044e150944247e2b4354cdc72d3ea01b5970a6061b4f83563390bb33cf1cf9f910858dccdd6c65e463607df5289a3684440546816d4c2d59b214a15347ea9580310786a182ec6c2abb73062fa281ac6806a6a042c0e95312027ad66ea6e245a9f2681a24cf505d5db43f8a8c39bdc12e07c9f805634c9e5725e18a4229a20a681a0ca5144a8611dc372bfddf102e5254ac1955450b94ce601ad68b5156996ac8c8a1bf1962a2758b4cbaaab3a341e9c7c19005ab82bd236cb6294aef22f1a1bbb32dc6ebd1c15feb409e6c495ef1013e979981db92bff97bbb3b34dc385176c440b29637f68540bd10143ac33cde4d2331aab1da5d9c7a41344a073b41fe536d5890239a96a319852593671f6a22e9480775f265c66d68521a73e9ec83dc6ac2963d3b70c3453861766048b18a135374ed53521a458e1fb024100116f434d1a261ca1c579f4311549243ad629525ebc465f8779c7e1b50344009c487d84a7b2683298ec39c27af80fb3f28c61d388959a99721857cfd30f98f43a19716cd9e698ec946e69fac477cbcb8e060eb9c96d3ee64b894c748133a9bdb5758ce45ddf791a68d87d1ef70d9af626c3a909451a132c0702f0731d5d5959a732684c1759d8d31092669e70ecab9f804cef4ca416563af1f27a219b3fd6bc482901201ada9836b824af9bb870db2c9ccdac525933e58a16c00b373dd81a3b89c39e9893960d7f6cc246a53dffb9fe dk=09766134e63c40a74d70107a121b44651ca43f837316f63f01290286304da07b45fbcc418092b9fec00ea7c88a4f20a0adf9188e23707c88cffdd6cd86f35a9e9a3d4f842c84a5c8662048395099037ace5fba75495a6370b9118d11324788bd4a841c90d219aa6bbc5c829ef2c291af3ac99cf08672d95225dcc4af7aaa428b7ee7201eef8966a10b88067988f62c27fb52470c036535d762df868bdad167a383c74246b218ca68ef9b5106979e30ba4b278a731ec370750c2b37135d39b32f811ac667e717be098b3f278f93815b2270c01723c322f1a3f4d8b1026a39808a9de51b8453507464e283694bb937778161e604e127ca85f10cea05cd28458503f100caa530874ba500a3102caa3802291dbcf72d457637c557c8f70012eb78b9df1948f6055974c81af1fb6dee601fe40c3d185251441437582b57ab29627bca1804c3b53ea76325b925982c8cebf50474a5bd00e43274437229d94494b7536bf473435a1f3c552c45064f4df31cae2338394baffc8354d1005cdf0cc7d5a8782906135739079ddc727b55a67515c609223fa3c34b2ec24d033397b0d8c6777547e5c635f5e546df52a207d2ce1ac3a9f85179a8898050078e93093b24122a7f553cec39bf42b35a07f128288a23c91c9232b835a31288461640cfab5eaf9a9baa63ad40044160369c899737ffc8590c83a20caa997fda5be4367dd00b91131c0f4ef61b0d207eec750a8f4abd5caca994b779183051c6f52dca8696edc65807758ababb04203234113052c7f894a478b564ea69d14a1970d2bbb437908f2288d0084ee7d538a3fc628e311d5d558cc2113dd1762dc3fcb336a899b583ceb457a8d55935f6926e1b5a2968352d6b1a35c986915c248e8c193ea3816300c67173930560e8455c560f3817c58848cc9fb3888037b449aa9268876ae4f665b5838329387431408068125f7da65027917afd42bd01389fe1f64bef5b0ec688396ce58d622975d69818888029f6c0005082b9aa14b0ebf9188ce93c3b127237597fefcc4dcf15b7f70518d8a9b34f1826c5b01ca9f48d1122ad41139c2e625f37dbb431957cd4db6818292436846745a933014200efb80fa9b4a4d38824cdd05c1422243a440126855da2948024e136a23732b98a3319809948b9ae25abc9b6a5a96aec8a50167df81c475ee586dc137dd5abb7d91a0211258a82982de2114a808a4da29b79483a26f3cb73e5985bd68010f233ad7c0464c39cb0ef02938f036e1b099313318e81089ce78c1d008496b69764c08c7ba7dc00328846268b0709104aebb8c757f22a8a45274cf75e80b1b377a971940852dbe1579b59b7b7901e06abb7972b60c9357ccc0024c14341eee51502d79d51d87a6f42897f5955a2b44c73766662f940d47b7dcee02e22b21e138014d735404a13523ec23adef21a3184939aeb0bda5264798146e1702f42c58185c3b063e1b884455c2b508de26c5d334c0020c7430d774aaa85b2f9007fa41495a50bad7315c4cad429bf2b5b733690a8e422506c36fbc877708bb9f51cb899a48d3885b73933972f97a31df12c37e2ccff5608fe500f0b44585cfa0dffd6932a2096cfe13518fa06b2e93c6f31cfc6e5c1973c2ffeab73c0d5aa0f7a2c1b922a62065a423187b30b1dd261104933a036a89bf1357961008707d7c4819c6746e137531797e2b554e4c06f57a1811f9b1c0b459b2cbbc4c90293821ac6627b20a1bb8f93752222ba795e0448735840de552ae26c2b8bbcafeff61e0b7898ef606df88156e9540ccddb6f4ed180e8e95d2404cd7fc15dacdc4873a2276dcc1892a36b4eb8c973d6858a478fa669052a04ca890c4feed830553670e5f07fc43c59a0584802e541c26ac4c3c5774f79cf2251274c54bdbfa5b726ca710530c87ee49918a41bc50a3ce5a67c05689646516f2e77a832622ec11c9a69f70b4d014ff35182fd8b6cf2e181710c2efda85358712962aa2f04c58360756469ac4bfe861ea019923b68a4b63975064596f39b83fa7083620c20f578b7c96cccf4fb5d485726deccbdfbbc6eeee2185fb46379575b5a875b28653a6945a7409c0402ea40a386a4f0f47d5bb1c972e96216384f9c48489cbb9defa92b2b983e11b796f0f176c6d5264290ac88db13aeb93c022bc8c2266e88c29c2a91ce80b13eebe182aa709a1694a456b139deb1977d531ad2722ac7d2b356aa2684d767c60012b1a2345a69ac52c7a814858e8c741a80c4b5e942ceee0045b957748e2847ea835e1fe17c70456dbea13fcee0384bf431225c8a6b499624f086bf71518c8c269640b436e7783057372825768670b2478c96c4a1a2adcb1f83a057263362476197c5d878d76b025f83ac802ccc02296a99296cf6b23c5617aa95387c2b0071c23ab7bf614371bc7001e29d7109057dc03053fb38c0084c48890612369c649c0ba3c2603e377c84d15b6db6bb6e9513ada9a47210861aa055d14683f0556729192caea8ae9a538ff810596d64bc0964b380cbc2f5d706a0a8135c1b87c3480c6d54cfc28a53c339666e24213d33a62ec872d803103987a36256840dd98da8bc17ed42c34f5c008f6c678c748f3f7404d17b46f29012f079a46a0abf77c04538254ed4b6bdea929724aa60014674c427c55b1b0192cb284ed02194f9615417b08a9580dca78944f43094e48aebbc8bf0126310d9aa79530457e04dba9b0d02b045ab28391f923c25caa69ad038b4586350613d9ae22896f2979d56bbafdaa294041aa42c21b9ac59e41cbedbc33ad6996c3451b7df5b9ae2d003f5092b91847290336a62e804b5f167d7cc4e31f40f6c640c10eb99d7f5519c80147b77843cd083b3b5c03718b1a7ea2dbe25adbd56762e9150b46b6b4e82b769c384cab39b3610070c6b6e5981c3b182105981a50567514ca15eb44620c231740feb6bcbf4980f7b1824a5602a982a68bc8961bbc441b111c93b2374ec88fef30d4931ab0ce3b5bea680b1099cfa79b3fff2b56bb3031a8c07519c7df4f75a8c887c6a1b7249a700717b2798c02f0ea04e720752d8323376e1635331b07435cdd5ba6d7228281612bf060c694bab6a00a8700977518bbac8195853fdd34d44e29fea7286dcfabdf47c9b2f2347ee7b3133cc275e601aa1874bd6974894c99d9659c1910030ad033138a57f8d9b312208873f733db9c73007f45c49a007bfab03aaac3f107c8c0a032cbe24602a2bcae4d2658d21899ea89ec5e832c0565701e5baf0da89e1b55ecab7802ec830b8268f6d9ca3883c97aacc20fc4861eb59707afa18a7b24534956301f238dffb68bd20a8264acb6be4918c85282f63cbaca87a3530515041c5a00cadac1a0b365b8a6cf626b30a49338461e94b44e20c250a23c7473952a20404fc17055e124792621d417cc7e20381bb24a701d428b1643bc0054b5ff46ba9129c0126889156bc6418a4c3b8c6d41b3d8646a915592305c30886b3995400a2a80771a494795d4b4b9ea6c21e0b21e0933f9f7c5fcb3b4c20aa303f655cb003526f8b04b3c85260c00c8e316db495417da22bd3a975fe784434c04dffa545f91555f01033696bc4b87abc0665b7a5c972d9a08fc8911147da3cef156b5ce85391161f32d09f6f8c0f8a845e268aa044e150944247e2b4354cdc72d3ea01b5970a6061b4f83563390bb33cf1cf9f910858dccdd6c65e463607df5289a3684440546816d4c2d59b214a15347ea9580310786a182ec6c2abb73062fa281ac6806a6a042c0e95312027ad66ea6e245a9f2681a24cf505d5db43f8a8c39bdc12e07c9f805634c9e5725e18a4229a20a681a0ca5144a8611dc372bfddf102e5254ac1955450b94ce601ad68b5156996ac8c8a1bf1962a2758b4cbaaab3a341e9c7c19005ab82bd236cb6294aef22f1a1bbb32dc6ebd1c15feb409e6c495ef1013e979981db92bff97bbb3b34dc385176c440b29637f68540bd10143ac33cde4d2331aab1da5d9c7a41344a073b41fe536d5890239a96a319852593671f6a22e9480775f265c66d68521a73e9ec83dc6ac2963d3b70c3453861766048b18a135374ed53521a458e1fb024100116f434d1a261ca1c579f4311549243ad629525ebc465f8779c7e1b50344009c487d84a7b2683298ec39c27af80fb3f28c61d388959a99721857cfd30f98f43a19716cd9e698ec946e69fac477cbcb8e060eb9c96d3ee64b894c748133a9bdb5758ce45ddf791a68d87d1ef70d9af626c3a909451a132c0702f0731d5d5959a732684c1759d8d31092669e70ecab9f804cef4ca416563af1f27a219b3fd6bc482901201ada9836b824af9bb870db2c9ccdac525933e58a16c00b373dd81a3b89c39e9893960d7f6cc246a53dffb9fe7708c928659dfe465ae813012df3ff6eecb1e081391b0a94172aff81aa3296c4cc96de9e1f0506bf490a154a6db46bd2bd1413c3d1dc163190ff171ef0d11e22 +encapsulation tcId=51 ek=2191abb6d6beee29c5780758a970349879b61a028deea5404731292346c81eeb1d17766afbcaa68c867d91132f34a494e28caa767241b50902f4825771865fc8d633736248963a253dd52c1a07c7177cb6df74c43c3c74d7133da9dc915fe14b5bc30b86153e7b86be04189f4ce8ceb6e5cb69808d35640ac50335b1633162e28260300646153be7b26a84c1c565956f8577c95cfa80b68978cd7a79e49c7e808b21cdb460ed8388c4859670333b69980b2f9a0f78d7a5b9d8554893a0b5c52447b15318828989816f43406abfa4950607a5a57cba62f5c4340ac3d294191687c7d2b60ef2d83f6bec15b1618129b78577447bd1e36a87142b9c552c8cd2304f35b81b3bc5196c3840c0346d513ec8862ba14a172c051402f7398707a6133c8e88e5b57f9b9eec18aed1404ea1c47be5169e5d558e262a231a75925d25c499650c8ac031be580cfb185a0e0362ce2194c53713a630cb59600c1942908ff5142b02364b74b3f5474301ab2da4188dca640961142f28584ebe810683c046b59cc2cc07025da63a5e8c5e99e24fbae183329886279c2d58d0b8cac1c7b3033f05e765c75414748687ff03c00b36709d41c5968528f178cd03b813b69054d3e20fb6a04a713b1b169ab5f0369810fc5cd967a866b70a2d87514605794930be9b7b5d49d08f5cb08d84fc5beecb994dd2a5fc10aeeb4cce1ed251fd4815a9d06eea6a77f18c56acabb1f7c478dd52218b7bbf9770a51c3ac96402548a1bb22dd0c29f422330057179ec2d91e1aee27bbcb1eb4671324265e28676090896e332535ba2e9169c54113f19a45229c2448078ae08f7213f347244527ef726855ca054ab7a1ed1a40a285a60d04c00ccfa8fd318cf182705ddb888df652669b3224cf54ed6d00db5121331b40c7e35b1945925e5ac1503a999899317a2370020d81873330327db7cc1d430e32c6570fccffeea90a871a4b04ca9a8065ab20a6459f1806221a5482802f14a68893b111607c4e085a5a6410c027c1bf5fa33adeccea576211f272cc942acacc879de398477780a26e7545ed693edd66ad24772b0c0c4b033a73ed94e1d02c384d1a288a78f5d8838b5ac0b37855dc59b987ec9b0f5c6868b0319f3372024fc15b247144f925d3edc453fb8551ec1ca734a37a3972b8cc9b84a217a4d0bc0d08c815359aba59a07eed998133587e062cf735a8521a18ac2059e3f142f12ccc5ae024ca33b8e4a285be5a3ced460c2c3675b956294ef2c61c96a88441665d56b2029688efe4540dc35abf5957bf9477f58277851719547510f22d01a13d0047c5b88372a654c732aff395dbf825829a06e6ae6923296218ab9ad33d0b07a0b6a3f3b3dd8428a1f82b72a921853d94c5d400494494f0d168148b5392f386b16c6056d6b8d767b7f4d918a9384c855c8b4de8450eb6ccc2df18528cba0a3552d5704683aa3aaf52751fd5a17801a76a58a75038533c1576e8668b5d290ceb20cb1ff91317d9b418311c11e617d10b4b0e9055aef5cc6ebf8b91f0244ed9854fde463e55112a238038750cd016735613b9c461996b90a3b4d4b0d8b8825ec087fb138ab868a04cb3c9249b5b118f81ed06b99e6b521f206a32a4013e876c6c80b36af39ac197a21b78b5f2713aa45c74335f32676c2ac2902710ec42af5984687a672e0466831c7b436300b30351795b22b3a939347a7b744831bafe78c3bf09a8224a958568f75d71f822941dfd64d8969ada5b73c847272b5914491bbce5f0c09ef501babd95190f93201ab413e833f7924b696d5c2b62272245932a6d9c35e1670aa3bc57cd41b8d61a52aaa6d80bb774f922ecb41cd58469c4dcac698f6730890237602064b148650c17e6edcb3713382fe755e46f3abe0c6c45184ae5c5c861e35a0e2bab2b4e8a4775584351170152512c7e99e96f285b916605b53b91d6c501f51a93129483046cb2dfa16d4a540ed2b6ab18bbdfba2b201baa20297ab1fe7133757c07dc041971aa4219157d77628f31aaf520650a2250adad1524e58268d21cc62e2469c8425e86359dbcaa066acb466e06947a1c144e53ee0dc930432c806c493b2bb25757b1e2c775a27a896ec25b58e8a78d4dac91815a1f3d90635ea25ae13cefa77877dc0c2f0678ade307243b438c54aa69dda539b8a41c1b3c98c77fceb2c0bc2025a272d927943ea338cdf32a0dec8b187df5c m=2f1e2ca7bd72af847cac38cdefc4d345909d7517543edf32e2fc491ba05eb5c3 c=d892e0948544d020ece56f00c6495b9bd1c469ac0f134002c864d10cf61c6c7887890276a7546310ea077741d83428f22b60e2a40ed8bbf5d9227893bb0b7417df4380323426ebc9744ef35a1bb6dab1181ff0b677e8c9b6574360994f96eb87c3524e15e468283169d90d8a994bed0da9cd778ca239ca6c225390221fd408a3eae541a032d714f3d078dd1ec722ca83fffc92a416f46fc1e8710c3f8e9cd452c016f483985a5c1d951fe2b03c4ac2c9a0ba71f6ffa4b29ede76df69cad594d10618b94e783fddaf11cc801ed5036feb4a70d071079944b7d9f1ffbb98f123dc34a52a40de57e079a1f8b180da9e6ecaa47a111f3c054dd9563d7e74b95ac65ab453afe0bf0dcec5578fe6fbc2d9eb91932799ea64b2dfa95b2c9c6e931ffb0a5a499ddc42c6d6b563d5a6c1f34dbcb36f57e6c10c69feadc70a3d01f5f3715f6bea0d3e2e7ac2b04e01ca80a2e187bb7d9efa5ee257b9c3caea3947d785ca56a76d0b6692a69418935dfc36fd9b26758f5e1043d5c2a6d5cc1556aed592bbcd652f4024bf324910f8caa2415171f3ec8b1cc494f2b519cbd2b13d317672de60f0a3d400e6079f85c3cdad0da5a17f2efaccec03d3c2e85b063ac3cb29f2fc4850705b8d472e35dc12d06a7d021db302ca883dc7d4a57b7a9e1d023960d4ea5e39c9d5bf328b8e4ac0afc3a6c990f598f373059ad28edbeb90412472c637877ea82e7105caf467a15ce961695f2ff04c750e277e78c1ce9e490a5c45168936bcf79261b95fb20afccdce790e0aedde204da4f6734700fc1ce05239f4ead0a496fd8f733049ef9b6ecc54d5c635b56d6fcac7aba1ba32d707d888b820ce3e794c2562798210d726c6c5db3d22a214f3f2a1f2477fcb77377af41d0f3f0ac635e7bf0674cebb672ea93b9504efa8c5aafb7458e41a1d85c28841877ff2f444005cd1c1e4227573e8151184786489c7cc39549f8a29b9c4d68be38f24081fe239c5d0e14ad5e361720836a72a99ce88e47bf3e6a1a9eb57cdc3724cb4789fb88c135ca9ccd94ccc9bd73ee8a796bfea36e894ed27b0791e8ad23fcda5ae7c167d5774aa00468d97bd0fd2b9bd78d01e1e39a66994a369212d95d7886be00f2b7a1f187c85b873f33cb7d3e4c23993f7bc9ff7abbc6fd9d5c54f06e5f31b79da992bc0f2aa79a36bd00483ca0add2dcb2be36c27dc913668f030128e82281e5e66e87c75dce3fc60431ce6a19c6c45e19a2d84430e945034b7da6c506ef009493af22f701088e01c20acb21972ff28d284d2a57a69d108161938f028011ba985e2b925532b52eed878646579183eb0ef3fb57be1e05c1bd2b68d0e4a9ce104e0ecdfb30268e6c007812fd88498d97795447e7f88ce8a3bcd161b1e34dcf77413b778f88a3ad66f247f7664d0f6af31ec1dadc00fec09d6ac63fc8a802c3752484b5b43555381043fdd30ef5d7dcf4f5b68eb89f08a0fe5ecb65d6d9418ca5283ca398d3129cd3b44edf5e3568c953ce2b66a28b473cab0c43918a05c5a4358d1392604495d06b395187caa6b36050b436b218485466cb6643eda7ac66aaeea758715a22a8849066879cc966da7e0f7e843a1e234920aecf2a4fed2f78c69a6c103a5a535d77976b1e40bb0d75d6e370212017734f2b0be9f3f87c5583634ea6a998d8fe9ce5e5905f2a6abb9d635425d06731c1227eb634a3603d081cb1a7c2b0c1685942efd6f65992dd39aa67cd954fa0310a0f5866ac6121e27e349d5c2adf37a672c1dfb021a855000f0c0c29489926b997930b20df641120ef8605cfd9482eea9344918aec689f580a94508f318e63eadc3eb7486b9fcd7a92ba16f5e02cf78ef73f528fae3a43d451c58261a82b3735e09d4f4679ce112803505006cbae6eaf641a6f66b0c51ee90095735d87ad88c3d19cd7888248fcc3872939605f205976ef6c6aa52f8316e9def841594373fcd177f0e24d8975653a29f738fa8f0d457f2d72b7c00a4b7a66fb080f705a8bd599314354e65842af598b8a2a40f6cb3208762ac6cd467d06a2e987a7af72e355bea297da87bc9250ebe8d8f85cf292200a21d93475ba46be78c17c1605c10b97659859f08f980114955e68361e180c98015aa46a776070c4f2b328a903a70ad226742f0022279179fd2530390f190e3008f7202c83a6a15866df848c8a150d12287451dec8ee7f04c1c3121e4bba687d7 k=5087e3b0c90bf601dd6501e071270eff8683621e9f5d67a7a668e50c4f460a75 +encapsulation tcId=52 ek=7ed2911897c09e483affa62da7db18bcfa54d700745d055844778c0804b94bfabc3212b6032c6ba86ccac5b844a0c1c1130922423a628686af96eb08af8bb51482a531cb648b4c535047179742388fa4cbc04118486aa340e50d01f38c4e305833d04d90bacef4338f59b91af2d17ecc583516813b8476bc5e4ac5cf4a1e0c621da498b87fdb6126ecbb79d3cdc047c4eb8b07bb246f2e11bc76d033ca22a92dc041cd646154c0528ea207dad8066cc3068a026dba587acd78215c6bc794c3a61689b6151c7e412148a55a7ad955b85b284c58c56baebca1aa7186b09162a6035b6ea55122c67b823996d2c548c620843d6a939bb2791148cbd18145bae3aad82c8a7330716148567ab4aa2c93c739c64b3e63164fcb1a8cd05d9b6c3efcd3b350f57fc98223c097af02b77411f9ae3b4285e375923bc63e61b587c396460fc668e9369e5d1341685bc49a090c3b5c52597a7e84c1c9ca2924682808ef45b4e4459a66ec368c80be3ce2ca5c2007fad566e7e07c78d009d3ca006f2769476871c57b758967cf39b3166a75625914b90146a1d5a1a3a22164b1d61431e98de06185436506f349044962a6c5d034c1ba9faae2b58e25aa611bcf39383ce0c43de10b8eca58347a9a970e56651fe4b6e163ccb9415d7a79b4760193a5cc5c9ad4251ea3aae9231246d82c7b4abe1d5932cbf9c439b50c3ac741f9a50fae32c34b662f7c12937a27ba76d357cf42aecba7c2075ac4d3050476b43dfab4380d1854e48c91eae54b022380156c240f6018f35a5a3933745a887b76ba9625c868c5938e6c618bc2485d20483237065c53f535c5508a8fe82c282a298f2915263059c0b8899ee2b29cb1a45d9a28dcc60179b15f58fa0cdb79338cca8d8943b2d32b0f4b6408cfb9190ee0697f81ad4285b3e95519083c315cac73b2920f4ed0b672a95f7fc3873dca5337e931f79719687078d876a88b6417da2447827987a04217fc509fcfa99fc2966dddc05298784bcee39390b3b48e59555844826a72709fe210350669496c66ff846b60e66eeb957ae834b85c3065340b4cf25574dbcaaf5b732f376256d7267412010b2cec49653c7222366ca274339090a402e2200e045ae6228848219a08da8c33403c5779b4de89475e8466a956c6f378a4a0f79f8f22c17a516ba5ca5c3d21c80866639987a270619b5eabb34b818fc035773a9bba1ad1179349489a49744a46aa826a7045e1aba6833043d611679938c29c06ee896665024178db47cf242798e30278c664c90a39335a33196259a4831f5d875085f64488b054d348ad996a94b37610ae978efa7926f844983430a797381c0649384e299832f0953c10341e89783b70618a60ab07680d8a677b5237743b37ac4972ca72c5137e8aa4939913db91b2556aa4a17c09fad7064289bef6879baacbbed5849c93616d3a2ba5e1c0a8f037a7aca7b89541233586788c9b29f07197b56aa26eb59c727a7502f98f65641129d310a9b036e783552548588a2c5ac789114fc04416602044432df8469951453d58704279d47700a257944cc1ce1a2a0df0bf6de6630edab65d4995a023c4ad351b3607578b51b8ddcb43c2549952727ec97066c24230deaa4bfa55ce0d7394b8f0b84c969f8e443e71c3c38f0b71535b1e68e7a3b08b5bde9c9d78e655014388af8cb54ac5b9ca41ab5e2a4bfbb58d0628b4155a102574bb4ff8c172b34be9ba2408e0c8df7339a009452d7023f1f912c7c84d5d1a415f1894897b7514eabb771ccae56c677c506495595012b25ac3409153952699b9a691c09bba597f128b7b7634004d5bad0199c213dc892a7a9b7f51a0012c81c000bfb50b2cf7b85a94b2903365be3b8cb4f92a6392b75866c0908f52cf402a4c8746a2abfc91161c70b2c21bddca2907e29c35fc39ec610b84d28b58d178de04388764afaec4c7e596c953f5a8396cba69eb99bfc1cf7f0c1f4dc20367f1ba365192f7c35d96d089e0a5934f467eab3483c8c0b009627150706be172243ce0cd14684564e586a1e985702cae6f90b8a6e2771d6059f8c9041f8aaa7f5c06cda09428dc9904b6aa2b857ee87a3f07ac037af379dc6ab7adcb59f0e82e73d474e0c66036b71061eb8f111869eff94aca27ab15d02e84f146b9b6b0729a314c96c71a72c583fc446859b01dc664931a3aeda27e7197bfd78421c3a5e0bc629e393f m=ae818f586ec2adadc7afed63c03344b9dc0c834133d202b43e2372eeed31817b c=cb327be1baedeac4cdd83977486eae157634e3a7623260197b96e4956172febaceb6a4c22781095ec6db8a91204fd3d2ff660cc52026aac9ed73cef79b0d39191c601a81923dc309cf5bc526fb76d8200b98438ada64bfcd0b2105be5663d14b3bcf0818c53bcd171fa7927e1cfe6b7fc0afadd15d040949652b779ce05b49c937e6b7e43fad1283ab3583e8d7a4a3ea0287eafbfcbf10d07c8dcd6aacf44904027299f24d08c0868caafbedaba6a45f5e00b1f4ffa31d8010b45d5529d3a4d1dd82ec22cc03bb75cf21fa4957ef81fcefc2d25c53ac5c2559c57aade716ab26ed3ef8d250c2cacd3f4945194e979c5d5df64b097694e3052313ce86c7d36ce74fdd9bc4d3cc1fa968e5984ae93e3a4ebb1cb3309dab6df63fcb978140624c2240ab80dedd5eaea395fd0fa6a5892ea1a162923fb4372cc1bea7b60295eae03d200e8b2537b127ece6ee36ee1d80da9191029ee666cf29a35c32f9b34d126748c2aadd183bc4bf05d627e1f4b61aedbf39f0038471a839a141af16fa371d60b97fe47d450917ec19f5062baf4c02d2e158adca6bf8302f5f18a867f9ce2f010ba079cf39a76aaece8cd1a7129288b2bfb8562da666084d7733bdf4c419639beb1e8c980f86c7c884213a1325cb7be693543bc056960ae29543d3aa3fc57c994c185642044fa3105bc8ffa80bb020cad148eb6154a1fd9160c32bf56373084023d470ed7c003fa0a94caa5c95cf4cddaca3710926bc7a4f1da5cd8d66041b1a557547dadab39712d359228177cdfb5fa779f3446fe8931ffc78bb738a6ebb0f6560f14d6b8bc1e898f35d55d085335d92e8559e7fcb467b04a195efc179085f5a8b400dc04cec807e935e6f9f6d38c833dd845d1e4fef8564bf576d75cf23d1ff74ed1e03dfb96c3b4454794d60fc88c76ad190b21f4940d0be343434de890cdce2deb58808390b709df2b84133824a9e5c769713b2d7befef842aa23a92ef9f6fdb6e0871e514aa55a80132c7a8b5ecd39441ec76d862aea1e93ef98b9ae15cf2458dc7918926102c4c993c14b898ec430f3343647767163fb979943d1185ba420840b876d2f95c709d43c7d29f002deddad4ef03b55825e19e54b1c9fc56b79e8bdca7591f712dff92cb9145151b3320d433a45d8b4a47651398e33ac9da9de758b6d80373c7deadf937c293df202489f99b0248f3a1ef831e6177c907f19f38758bafcdfd7e56b5232a87e4e86735f055a164335c85dbcdb55d4b99055d00d708eb6a9067ed215c20629d532911480d2ab8600c8b474fc6ac6c3753ca7cede2978680032c9fb9aff2dadf76a81ad7a7fd0639c9170ec2a78f37db2425fee690f40ce6d02997d538631f65f01c7f5d7a614edfda4117a6242d1575c65afe5d1808a7f3fb5edeb429bb53a0aa2584f9958f519b5fc0a4a727c8c5a92583596766dcbfdba467351c86e3d2231ba22f656c8449ce54d726b069c790bd5c19b6147139f1e2e4b48182e609844d9d858748f7c416a6a1ba35c70a98936c2d9bb1008c390f4b09527eca0fa719dc42016e0a8622cebad83f636e780ef79c2216e7da50e5db69bf1660a059ac6cb68ae3969a7210f9e7f8dd172080fa98b60abce8831c6ced8b809c85818c263597d3a1a3f012269bb3bbf0f40cae60bb9f8725a3dac68cbe49402c90867030f9fa9414a2bb4f5daeeb292a00f68184150d9e7a6a38ac2094ae61bf06db01b356c95c52bf6c0ca84dc0d9ee48802fa9ef45c07de48d4f8e8f715497874886511c9300b950cfb9c110ef9402c70ef54c2132a932e7d5fbbc93522577592dd2f76e2e638aef9c823a9731bdf2ec047c8b8813f086c7233f2bd8c1bc0b2a2153a9e0b406e19aa57609abd0df4f95806cf11eb75e440b9f4d4b2bcebc880336c97f7104140222a9f6be97ce3dad5a8e5980f4c8c81069f850925cdd11799bcc5511769f7ba6434b0e3508163bbc8d9ee8829cf0ab53a7980aa0aab5127c4a6f3f28960ccdc492be5e79a50e44906cb4c507df3c69f363d5aa9ceb18f8df13667a97519a68f31d8d93b1cd188db7ccf5bf2b2a3389da5cd8e755e1e1beb2649868a13ddca3050a18bed253568b2fe7d16072d94add2d18ba7acded627ecc9fee2a9b90098e3d4323cc807362d90d3b10ae1992d41a4ca5afa032338bdcbf8ed165d6d7581e6a6c1bef67c35796265bf2eb37920c2d564e5b9 k=c904a4b2ac6d2d8690799af342cfc78ca0d61e2b6bdd1d5738f32e87e07905c3 +decapsulation tcId=96 reason=modified_ciphertext dk=36bb5d4b256dcfcb14028b1efd5a74e5606617950da40b2408cb2c0462a222d4373a7a290fcb2997daacae414dd8c3aa7b38773c79c949059cf15b12fa8c522c538fdde905e798a3e7d5a6f27c93330b8243a12983e5c83880971827b01c2a03b4fabe4b15071ad124a5a00c69d98afd6bc63f305693b50040595995e99ad375bbd58989cdd2aa92e95532ea88336c5dd9f6c4a1b254fbcc4921b96de593c3134a2219b7a192e87309bbc6ef95499d594b30a4661f809d03297b6bcb2ea894b724083115b0c6e6c2af2d239ab69574ea95a746cb23a0dc429ef42a79234cad8b9e85597beec196e85870ca1415cac40213b64ca34a8247595c89fbc19ad73b715ab3990b7e9691c9e5c767d0780e4e3026dc172494bb4ed87311d20896571608b065224b2c08d3b754837c0b8ad8cdac7775a8158a0fa93872c972f4048de1e9c9a8e33d59397e8de188251a6286ba2ce6f258d1f8a99e78c464f576fce80a999c6475ec03bc597014da6084f2c97eb88795f28cdb27437fb11f67977be1f865722b85c3371d4534cddd396bd6b12298f4287a7caa87542afadcc70a222fc1830182831bd1319d980ba03937b2cee6c2417a766e841609d8417e9c9d8dfcbc956217a5ec02deba6ba6717a0c71009aa663dd4cafd5fb5142ea14fb52c72fab33b0070fb38c9c9670323dec00d9eb9f02f64bf4e65d86c2c216799093040f6ffb80fdf6bdcc649f1f39bcd2c613221b528803a75e16bb5b8b067eba7765738af0647da1026dea6717cbe4116bb28dff91c224243c1a72350165c30184ad901a9425538d255b72fd834c8e0090cb0289c2f79ef7997cc2aa5928d453198bc392f6524a88b5ae67b96a1c1782267f044a0aa0d8a011332cb8d34e1c6cc149a4a6a3b99621a308818322fd336edfb867e4495ec9d92b4bc70f7b170a0d28b3ec365e5da02c7e50373b3059248a439e0a7a0b3a656863785ca749e7419a9b1595ee28b7df838f1f2baf02a065b1242c17da7974c5332bd7724c77644e88549fb572ba8964979979b5a7901cf05823b4b985e4b3d531c21620689f036646d528dbccb4645b72ec920fd0f5426614bc469664a48a9ee0b5c7d9a52164e3163c299441e11dd246731180c9bd70219171712c7bb371b213fc976236043f9367569561cc00f033d473809b3c49fa3725f111907015c6214c63347c4b8672980dd856e30b55bdb7b3d22554242393ab54314715cebb43ab224a911b090df1cc5e6950504ed063bab0a0d72a5c0fa8a1824962de9cb685745659ecadbdc3b335627ddb5959fed1b66108af4f5548f151acbb3336a94c59a0a6b28d94b9359ac7c766cd7b9015c3b99ef45280470c37e985cce280acabe67afc54151239125845b94fa26656c4cad63b0494416464f4c7fe5936bc284e9ae388fa844ef71227d1aa0e872bc6fde92f894c26e885087523cfb56c7440222150a62bb27c067173025ba2953999bb3746094d546a6ae03baf8630fb3bc6f57239e7a35c20691fbe1611e6c46efc25b77029abaa947d81c286c4816e24f35d883b4d315a529a593421bcc5765b34ef6c4d416c3cd95443f9ba2fa248659ee77aa201987f6a5ebda78d307559a874963b1927198b3524518acd6618f2d25afa4b11c4b396c9111a19398d55079ef62054a812c37fe81db4565f67038b9eda7a4b63b640252701844917e8cc7b3b1ede433d0356a2c0ea7e2df3c237498231a65e8e87874bd78487ebc1ce800c7d1b56476c220a8bcfba568ba20b773c993a98302c084b2ba25297049841355a16ff098c420a2220837ba6c89b44356b810576760316254646b0a5a41754a18af17d998541400cac0dc830ad9616f4f17840a9ce2d69b3bbe8cf9d60b01b00ac7ec8b1e08b3d886a804b4a4f2fe30cee898e42109f3a16c91fca5d1c328b90f00da0996f2365be54927d93f83f8f319d8ffc375a5126356a94c6b6c3dec98cc9f2cf0089b3a6b878c73bbef04a4442b1b0fbe273a994bfb6e88ef627b6fae415c37691c9d30e10b45a83e02ac3f16a5b1249e8b15c0304ab6f3482477c44a6fa8b5db797fcba271fc1544ca333a6144e72b02e929cab8fb6011eb66399b8af76cb188aac3ee43b992482c4585934a5075428fb9966aba67b414114e379a05c7f81c351f6e18eeb8c3042e4a8dad0ced9a28cc492a2573590b15a914cd100ba119939610c6cd77d73fcb5bbbc50609152b3946e2d5969bda6586a7b26bd4284f41269dec3c124fc3c73a0a369818d9a491931607afbc85b54410444f9a426c9491099cb4ad40edcab36f6924764403ade37288feac93c2544f16acb41f0355e2491fd948fb8774a62c0ab9fd3bdf48948c94206e386bdf8d463fae547b580b1d9531d48e11d1df6539a7911b5577f7d8b55b17547306ccdf4224950776a2c90adce2673983c3ff286260fc6b08138cdc93ca3297873aa9443c7165021d767d3c132b1a66b413b8bd3849e8f6547665601c776a558209ffcfc84ef1415dcd6091f12c790f1cfa2c5cc9c170bb7d0072ae997ff443d853b021b1999e1686ac6580ea6a0a946a2b417c17eef967a51cbcab67992a16a8c12623ca327ac7fac5e4b616844e28f43ba45ad9252c695c4ee6707d60c10085917a3d5ce9d46cc9ea52a7759827fd64571b7b4630554946796b308ab65680fb23091515683d1cb815221b294fa403d79124a9989eb7402f9990712e91d78c89ae45b33647a8fc11c5770ab738a3573e2c32e485a07dd0a009df27b4456834776ade5c454e5c70d00cc455115169e8c5d9ac35844e5126e784596b191cacc1c8ff6c2c0e371a81726930808caf3680ca194d0156bb1662be1928d3b3a432f102ee9c8ba950c3cd05b394320c568474d78dc7e75862a155cc73cf2a093c87a96653fc1b1288fb130caa41500541e34725e612c475740079bea8ef227bf1269b9f0f86e2ca929c3d611ce52552859c7cfca9bb0f125e2ca1bc1cc67eff49e27ea6b087700b8723c9dc0175a424d1578458d0bce007c5858361279b0c76e62a03e1977b880552396320976040d9155dbe7a4ae2aae1a312b010cc430b310cda08ebbe690e0a102d24bc508b2a31f61171efc3c80e0038c20469436776c65656f189a972544529ccce3d917eeec4912f779e3435a890000042468f93a3d1f104780d26ffaab53aa33bea1a3103c3a51a86c6a7e178bb65c8145f6a41efb6dca856fe4847ac574476dd768271031e2e8a70220caaca92ec136120de09ffa31c034c0a243eb48762286b1b2218ed2b34c551572b6bf741a9d0d8a31bc1678067a3fd13995cb559325b29c39a78c7798c3806682d3b0c810f5740d438a1861c8b5f11343b9b21a11694bb1049dcc0c872b2575911de0e8997bd19f2dd7b7d2ac501098cc54444abcf1c266721122a6acac20b17dc5ca3f9b5089136e6edb6061382e96d08b27d869003b8794ba83a50b1e2e72910699bf02513b35a1630f0bc2e8a666087787dd3b0b10529af8499f6314039b827786b71ca5dc706dd0610dd220eaf024d577ce14f80649b86ef5f839ecf23f5c595223a294dd9bc0cb7bb59f039082e68b57fbb325911c2d8cac1c694cdce01d9e890561b43fea7bb5e2e00c6d3a650d9838801810bdf5013a8b95e9d76b329c1f2a0a40c4964fc52823e55b2af6412136314937f98150ca4a2679444773b66cc679519815746b74728a0c9b8346187106aa2185669023a1983a38580505d04e4847b2aa2a4127125013db519f141b5fe640b06935c2b673819a665d9039848803128204c4b223533c9a3f12c9553c8a4161490fcc5b51ca2b6a81385169c38c78ada8ba7c42877138a50fbfc5863e4967de01593597cfc7110a0b9c82506c9fdc7bc4d4d6b245751ccee2119be688ea0985c5647294a3b9abd07296473b70578fbdb0c4c5f16ebe034375818a63e408141b57129093fad761c54a99133111707656811a2351319c04b99259946fe197b7e89283659623ba26185f887680943c1cb88341420c6c32ce2b5b6304fa99bd35a2d7cab3e39060056bb4d756a78050835f301ac355ba7a412227027dfd2c681440af9c81664a9c4dc5955f4bbc2479888a7536a9199199ac6710b495949ca82bb9b1a2b06823bb1a79c6162cfb6a8b8a8893152928a169b9801765deb2ab890ba2d32c5c4409912c86c12c08972cb3a042bba1b76675987279067c288fab89ddd93a75368888a40c2c9a20b60a0fcee8495dabb859b797c7b763cdb9ceb405213b7bb45f134cb6cb106cd7252ea069feb250138018fc4a5071473ca81c6a495a46aef27210d556fabc5d12281ca6d03bec1e62e762b589499cc3deb96e2a1119663db91a6a308e2360a02413ccf01948541c54d5589829053e8513a916f58e7318073cc06a308a53fcba7d197e3a2ded7516d56c84ba0b51b7ffa0e5d77342681eb856686b73d7942c37d8e2c8 c=4f18ed0d9ff40079ec3a6d67f903adee4d5caf5b4194d969ca222537eecd10abff3ce7cda292b2f632f65a1c73e8ce44b6d7890e123987a8c08ff23f7345313294fb6b3e8e382eafb352958637a6a01e0839dd1218f1bca9167894383be7f651ab0c0e74ca9b19dfa157db8656e5c4367ded62f7b4432f43f5ba02f394271d79f1ac3dc4b1caed50723730b337ec50303182ed297aa3908cc28eea64e80819cb1509fe54302e3e03ebc39bdf0c00a755650af3610af7834ffffef5e6a2f7b66d68634e5b1b76f2482652b785dc8051494bce4f41867baa07ee88c9680e19139eb9bc5128fd5a348e01ee2a7178b3b29bd0c6d6096a7dc3059da039e2aeb64d5dde1154bfdfcfaee6cb6ccdda5039c64ec3ed3a2cd58fc4f7a4228995cb240b1ba31cbea59bf6483e100c3816badae7414f0e744451ff1fa0912d5c0a2472b1b7c0abdfb175424db48ab6f7be4db871788af54960d244035d0e3ffa2fdbcf143c22001d4d7d3ce3ba848562c9ceb024d5dfb8f07689e72b0f2ad31d05890bf96f46002e204eda434511daa17385b81e0c513cc36f9d53a6adb36c1b23c2e37cac31e5997e1ac3102d4fc2c3a5b3ba3c9772541fcfff57ae162f5f55fe21dc47a66da2ab98233de8ee3058635fe5f3315e7bfba87e8dd2280930240945e972cc29cca19060439d317c146dff758fb183d66542ed00f2841c203095e9020d0f10f416c0b7e641f4b02a9f71f599e42678c85ceace3ef586ec1b4c4c62d21f2cc75cfa7f58a525cb6a55868eaa31043596e2985d52f32938b304678a89c72d3b91aeb6943a4b34541da42e1060f6dc03ab7d6aa90b2ec7a4d8b2e1fce969a8614765f8212a08bceafff50a48c2a522e1a41fc2c2cc89827830d34ad556dbcab5e69121c22dd035437fc6ad0dd95c0e22916d5284e833cb84286ddfc47dcbe9758b980d78e4dc294a3ce5a1175f7ecbe2d103801089ca744f2bbcee7212f64b0ccff10d4e9643027b942134ff2b50db9084d66857c4df62b2c6919d9de84d51170efc7942b3871fd536d26310ff39ae099e0fa562fb13b0418ffdfd398657259a822a458e3688b42d492117f16e291f913566cfd6ccb097c0c45c2e5416b7d3707e93fec60037937b3930d1bb6a6cca5418c1d26083b3286d00eb0aed6c2b7e77028fc70896789ab29f424d91d393f1f63ed8e0d02d3a19f16b80481cad1809bf92db925d11651e7dc1b0499e1853865b708d79446166a1c25ede6ffcd277689b298da836bf52d4a885c6e8f4320ad031ffbd676378093c0f854236733b002e55636d44a5db9ecd7358ca24a3ef7638ef03b4b5d8e79def61221fde892987860dfced702d4ca1536348fccd99fa58c1031d934cde4593538474b2ebc8248bd4c4792b2b63b16a152970007574969f34cacb80dc978240c9755903e9f7951aca608b682fa60a90a8282b3e54862b60cec8eeb737fbc31b6511052856083cad928d2e41f226c89a3de0fc36797714471216ea0ff8a3dc1eee64e24afc46a16a818b4fb212410975034f5486a4e84078f59dad65e5d1ebd37021eede2b459041d4250468459d535ca974683cd4584b62b45756ef0d2c0e0f2af7b946597dcb86129279078c4fea6563ef1e378bef179d65b1c8b0432660af5afb461478b9a9cba980925e8035cafe83c3dbad397f2bdfaaa08ba05c623115cdea9a62129d271f237d33cf1266b38f122cd7a365956b9a08d611646cbda81ad64cf096344f9b905386e7a231526d4c5de966263b7c8299fc7ac8dae2cc188ac205f60a14aa108c394260a4630fccd65aa9f56fb26425b02c6319c305b042eb9e10dba41221299bd6dd4e02ce5e4085db6439ef118bb45b6d4c086af359a3c4b79e9b8bcb93cab682eb050c745380b65e8775acdd27f97e934a059b3d663e35bedc977fbcbbecb3d10c2d8ae2cd8b1940968ff7ce054ea7eebd0e81e65a372ccd133e098867ede82de7c58d99e908b416584645710e2b013a849fc15c209931918cbfb0fcc35d87ca63315719cf8de8025682f123b6a4288ec654438c6d75522a7e0b80f59bbb44ab8dfcb5372597590ab7644dd1ed74162f5ff389b417981fe46b27396508b28965bc564f8b45d68ddeed1cbb0b3a50ead14087f3b9e5d8abd007bc38258d127ec34413dfc4f41ee0b549e106f77c1696dbf04f31a2dca4691e71d8bdecf4f2721d98e1c8 k=5d1350c0937b6222662f3d0a6194b8f1b60d4aeaf94fb8426c91078b6c12041d +decapsulation tcId=97 reason=valid_decapsulation dk=97250e3dbccec512adc4cac174bc8f95f32a2a180ad65c0a4005a86b79bf85b47cecac4f0fc8cdb5d929f422c866ac88ebb91906f26b6a077125cb9d509b24b16aa7c1c39971693f63a83f91aa8303f107d8a54a793bc1e178932314c0ee71a7d1f1a3c53623d66621178067e6085f37b7c2401a50bc50acec5814d2f620cb80398918a266293a10032269fac992205a276c141604639f62c99c557a46338e2570ce54232d7171bdd4cccc6b0accb26742ca100eb9b2a52bc72e7587677c9b0b9c43c16bc56093141d627b76984c6fde04984604243d286431c8a57c312bab8cbe468a593613814f6739048cb3aacb9c32a9971938552fdc0bf056b1df2b0901482534a255f44a79c17b00c485aaeae57391564fe127c5693695987b67b11a6a4043876d5a77eb4163ffd0a2a538a703da5e1c1696b1134eddf43b6ec158895a612db5a4d993cc09e43957c76f1e24b573478c1b4242ff389d0195b1ffe8a6579828462cb7849c6904326108027f16d8bd5bd872de80a4d2e77f3cc3001aa69e4d471981e731e6033a3b66092f215146508bfe17372ab424b3974a5f9b0b4c76c9c600823a919502a1414203ad248baaf2519e6c03647fc9c82a3abc26796f46416fcab55d4ff73b64393564c7682bc0703417ca96c3bfca97950ce29a459c71d9d84a61f02f05127f2f9a549a7650802029752aa126e07f50c2afad0159b6cab32f773b8f9ca6a49a0fafda807c62667e4b9ea5075ccca22a9f791b83a9920b62692baa3a9b4ac920f5b46eb99b6f869a06769b04c20c4a4c75d35a131ef0cc0ea9b973a7685295cc4a8983ac1a2e4acc8bddb86a11ea9596e8a9ee338a6a8570e88b28b35bb0d400ba83c41faaba7c2636217870b41fc182a6d1a109d54983047a29d372f5eac59129017203b1be7566f9b518ab8ba22ea5b8f824bbde8717f6f559e379981fd68a1fdcb57e2c75ba0c13fa2b598977c9f4343a945a3b130c306a8a8980bc75665504403413d4805a4aa77dd9a43549971bc8d3204786990f7c0470f73cd716be9e351fd056b28887cd253445d81115bfda50026779ae7775cb160a4b7379aa5a34bef4488b15a25d177715748a46db5c0ca94203c652b2d20737b1abcd530c61d26f69137c344c5f030228c1646e2e567414063800b268425cb0e008b98c36700d186bae0763cc030029ccc68cc8af4c1009f5c4613a3b1066c12e90c9c48e19b31197758ac2a69f2aab2bc942562b5d78c0831b3b7f44211973ec33978a3cafec3318b680bec68ba5544331eb38427baac266b67b663840671e150c9992a5b522517e96a97db38b241da868acd23676327211f460560196f4bc881c2212f1583005c5530c784ead51a3e9c745579b056e4155cfda14c6628a9e7b0202563e59bc084fd1baa662b8a38aaca8e4cf99969d0c243a8895320d80a3948512d9aac9e0d08d7fdccadb91a1436276ebd8ac88cc893606a181cac857435f262ab36b07bec3a78882b81e533a79e5e54c8fa49d68f033aee991eb8293e8022d4344097b06c0fd5224a019552510709458a620b98e8a6656d8f75241b1bb68e822e96a3c02339bff99c4553493c38abbed5620e2217cf388a97729223d00622aca95b5cbc1e2c075f1db5493767130d2afe4588636851a8676409d1b55b6577650a7ba26db6c885b04a9169568bc5019824de600c1e947635f30b5128392a061c739590750a3bb9e975f8159673f199ca8824f8bd35bf1e062ceda8ac9cccf3e0a7413f42c2f65965230247ef46c80a08c50584c887715e4bc56ea478ede317c5eeb0fc4573b62fb29dc795a9069c523b3ce6d16bdfe6b440ec6922818b973833f7484a959e7b3b981cec0a136d4581429c75c6211c29a29ca36566b0bc6b2023b96da991a3ab520988914077a8b67b70947bb4c2c3974c458a71272afe03c459ceb04f889487a8365a6648c759486504156c6334782cc71ebf06534dc3a59738651426dfb527d46d469322777d752a96ce0586d701324f13a781490b475baffaa763ce00b6d9abb08c61fa97a73dde3800b365e61b5ac1efbc26586623fbc1ed4502e8274c3c11626f0e01dfc5bcfd4e768cb7c69bdd493fda01c0ad476804289b3b02185f75ea26a7925949c66f595d90b033faa4401590fec389c966b4f9e86a65394569173c309ab4bf7e92d2eeb21e8d12733da1d5173255f0ac080abb7c143a0013577dba999b37022f1d7538613b03b40596e8ccbe92c52cb8961fc8a9431d88a94658bfe906b2ed295c4539bfd191931b89ddbd6063cd3372d6a7e17c74cfd55c9230cc3c7c46811436c92b66f05d88287878d74bc7f644554ca5b913e5391d2f37425f697b604ad9f0cc0e9aa66945386481a1ef177809a597cac153638643314a9c1aec06f175918bf8b04ac9005566b4f8be608a825b3d393436f09d0ca33a5a787b88d7242f9c750c998a15d99cf27c6b0a5d00a8e47a39e09ae6dd8b24fb6b01e35934f493cdb273ec8a6adf66533c2e777ad639e7167691baca4c3b4947b465377c35bb10043acb420d481add110a38f0a900a23b387fb78d8aa4e40a32a2e02b8ddc10de6377194e51eb01b8c9a2a958e85bcd6dc7c21093b296895ee921f7853183ee1bde3a9a9dfe0cc95dc9536897e69982816f5ceab300b2ae5c85384b2516c92ec369a4b6b7db51cb45261afd2c86a4c3842389b80907532ce76ad20d9a2e850c07983c1753315a1121f10b1c66cc47f7f30701d12cb4e486d24c66754c3b01ec86eb6b056ba77636e4757ccfa11642212754426e6374e95685cb9728d6f357e3eba3ecf241d118515d9ba4096e55da801256ec13d182034dd2000da02305fc282a4b40594ea429bd606261045f3b87d7cf06040c71d14141b4b4150fcd5ac3fec11eeb9a0edb3cff1e80b1cd915b7464101958b8bc3788d3440c7f47f06140e5aa3be8ba291fe2bbbb91671c9fa077619c6a79968d2e8768bcc052cdb1657e987d1c693be262e75ba2c088bc8adf9b7b2525b0b168ff26a5ac77c81adac2a2eb2bb53d97b2d78680e830a36152678661816372973e257d70655f1e9c2dff95449b4c7511886d7d1430677a12cf1843b771b0fd76fe71081b7e165af37cce7d0ccfa71124d6a15e100a66e1a8d9b150c91babec681c2fbd96a33884032a8bd462305de0327cbf63a090c2fe4b568c2b54b4aa5b3189c609fdc08ef2803c0dc59254ccb4a956136f19c63768faed91b343213876acc0e141afc30a36865bed609cac905632872680e59adbbe76332e13f257097bb4cc8bb4bb278db331fd8440f0a64c576bb347173ffab65cc351ad2dabfa822542a477ed1455aec01467a5453838bb71df38c261a72887210c4ebbc6d762867c8479cac682c4652e248a6677132be56b1f9c2b74e008a878c44bee92de8435197583513b409db87b29dfa32920ca144b437738c70f0c68a58202979d66fe8c50bc53c2817605207cb4fc4784feeb3275d8aa4a74687fa776c55374e67eaac488595b5a1445dbc28541cab5d324db2d967c6d371ac144a9ceba82588040bb0c8d4329b81a97eaa42bb649a960a424babd31df9379726807ee1245532e0be252563acc336cb743f698839ee665f4e294123767031e30224c979786805f369b0c7227732ecb55e90cc6a42b08f652672bb414472ab094b32887803e6db57da64b0b9094dfe29154bd0b853e0cd052c2e534902e448561293ca1cf60a24d6122a7b154af59763a48661f40d11d2c3bb60ab306021f5659e3fd196ed8b59c517b892c457a8f8bbb806ab31b8b7c3f8377b79a214002ce01cae9975875d2a9cc660ae8f386103b7a276dc883d145ebfe9363d0290e5b63fb6a04cd28b83d8116102c73dd40b21b2e00958924ed0fa33f0546288e6a453914cc7a182876b435026442d050fa19c3bdd451598003f5892a156e718802aa2a1f4882a9146fa78b77bab9cbb522c3597295da57940396cea70b62982b41c07ae5ef33f591a6d1478c19925025d2437dd60b99433097aa463d136c49aeca8162c37f9290c5cda4f0eda410c580a203c25bb0c360a99771cd852b73c3fa6fc52b2909d95529d9eb69647a42892390b1cc3bcc6f9c86aa109f8732be976a85c98401e46c00f5aac50286dd648b11a3b4523bb126086963c748592a7261ed9bb65e62887c22290f2c9d3e86f289141023a47a1b09b202a0448b444c6321424c5cd8afbacd04bc17886541ac90c4fd600513034775077b4037b09aa11ff687ee26ac5715b3bf3430fe98c028617cd0040441985aad41863f819c95a54940f00498f0439f5a04b518506ee96571d9508e5caa574c6687ae834f8781caeb17fed8c8721ab3fec93ebd097a0dc10a493436080a89ed15190d6825c6f3f4db7b8283e1a2ddfdbfd4c8a3779bc55965a9ac9ae909a994ecefa6279adfc260f3686d78ac8c96e95f5c61a739d20db8c c=7a0a63634cd704edaf7fdcb2ab069be8cad96be23dfa96c02bd4eed0ba60e43ec8b79a668b6ec2259d60b748ce5c2bc428a045ff2e6687127be9d14811213f148d39ed6512509e5160543d9b737fd5c074d5273eeff6e4e5b09508106c18e604fde7c927d25dd4cd2f76b7370f58bb6e0798698db855bd687019ff5c9d2ab6ac398239335d3d4ba465af994c9146410cab5376d913d13078c0b101151394b7f5847c90cca38708c3d4fa257e6144e0100eed9899601802ef337d7b5b1a260273ea5b35a5f61fb48d13fcbe43869e84f4f7cf1d1b6aae61dbe95d7ff6da587caa3d0b4ef865b49bb3149a43a9f89cb76387bf363b0c7fdacaf4cdaa851815e490287cca86f1f424660294630470a90468d5b39839e1c275a07af30318ffc50edc34a24fa531196735e78beaad162057e6b5af92170d1bec86a415243213fcd7050a2cd34e1b2842c762e13c39666a950964178603627dfcc2da0a583cec0d2a51eaff9e1f5ce4fd58cd239dfd91d3ce683ad5e8d3005fde4e349fb2c6de5292561531e868ac91218e2c123747bfe23298481fec2f127ac884576c6944c79b2aca24f1de67dfd178100632fb3aae44d920dc68e913e82d7d817cd44af11946856eea887f321209ea08f394cdb72108211f70b1836a7fd6b6422e987b7f8cdc34e3b2f56243875e441fe050a5e2e963ba804e02add36f524f30b6d0fbca4c8e833a52be516ad1bf9d7089d80cdf48a7ab09f4175a1906059e7fa6ddfb704243d2e79afad7f46c02ccbe5a64dc7f8be6ee79edd35deacb476ada83113a078a65c96889643ea2bf0f2b297df76d4ec9e9bbe9fd93dd839fa0827fb2133ba17928ffe64dce9d74afe9cff2b7979aee75b6ffa85173e97d10a3ec16462bd0ad16550aaa45d716584fa747992d4f3a8e2fb0869a3435a26f21b773c88f4376a8f68757c7678ffff04c22d566de7b3146532b9321c1a6408601833abf2e5046fc4a192e6a0bba38399e3b7368aa0b41b9886f397162ad1dc321b92c735140a2808250e08df557b515d3235f35b540fba3b76747d0377b517d24ff1c5148f62d54f71e54eebda2c0af7b2ac9b420dbce59647b1f9cc00d16e5d08f6fdecc1f4222c75c19c4023812ebec86e11dedd337f15e18131bb5d856ac19c5f7e1fbe411336818812b472d55ada4a9d202322eb6270e4a2ba7b1bce1c1352a84518eb456b6f570e428bbac4b5653ac529327cbf312ee23df1e72d94143f035a9e4d3c6ec380b0795a418187e5c90fe3872e0d1f82aac2f206ace6ea74590bfc42cb05957623d7c02202354149ef5e6323a0d968d715cabad07b6fe22084f74c8dca6aa3e0a40df4c8d85811589d44718b53760359fe2c39f95c98ebedc70f9832c3a70815f99238a3a9a29c479f31fec54f91d742914a79c9f9a29889dc2cf159264c5d26bb9bc2b9680a1d2835090855d680f9d48e5cfb82ed8200ffb1fd8626cd7c6bcdeada75025f55f184f32c2b40eaa11626c873501dc8a3fd1cff14be88726e718f1736e93d9c58a67b62e5c952a3926feeb0fe8080f198e2f00d77ab5788e795ce445994becdd5d4ef7a282eaf323c81a39276aeb9d4f7f7b11d22904a6b512145f176c2bca6371178c18ec9b12d233d1b10553fc5262358ff7aed33b0155a79d7d0d637d1bc538d6d5817c83edd614170864941b3527a1f615d18085258cfe837bdf73520d94cd7aba32a48a0a80c36680d7c3c8d83b80950c1dd9aea2406b755f6cda567b513563514f9e71c1582e9b95fa6e05e3059c2ee2e4317e5d67181eb17a2e65812c6824d2ebfe63e664c06bbd40d162b9f28371d61d8ae363232eadafe8b571abe21063fd1e9cd7915d9573d42331710e17eab4c268ba95532c5a8a7c7cb2ec035847134f8d8260e9a760bd755c32aaa7bccfbf5a20da39acd382566d2bc30176e80b8acbd129de56083784dfe7cfa8e36aee1b911cbca6101f0786a6791bb6025b0d9c92bd85a426cac2d38c838062b3208c49ac06f9dbf4d436965fcbcfd3222b093da213fe641a623ed3dba61a8faec417c14d40cfcc2636a12901633ceb3dc0290ec05c5e7ad3ad828d7f379b1347e11ab25e9f4f2fe8e4ccdd15b3b3abf961955138849c348d9e9c6a4a5cbc55fe22dc87790bae23e350fa8f4d27a91454ea4a10589cc95d09066fe7a5e037510c2a6cc1dd0592037619c6cb6294d56c5a k=8d77e1681dfbee67ff380ab5c5f66a0c5a6d30e34cca71523b4ed7dfd51a1a56 +decapsulation tcId=98 reason=modified_ciphertext dk=9c004d41eb2eac59ad6ca481475914c5f07378b05edf3b7fca5ca844185449c61678953e72cc0bc5559b062b0f63e29fdb7209a2546de4b21539e5948a38a42eb0cbfe8325c219288ab7adaab772ab46c7fb071b93f2356a22967867901ee7605c689fe8cb13c693b08a3aa302c2b707e4a93243c2b933083a04603ca05a9f996dc205a2cf4a20762babf970bd32b8481f183d5a8c7170b234aee2b50bbb88b52ab3696388ae0aafda484a5178b015cba4c2ac29539823c384b45c65aac95b2496c02132640e21fc04396c696afa9f77619de82448b1530a42b5c455072e7462c8f7f35758740c023c8040341824d965adacb445d52c3fc91607770ab2d8508b96c407b7a104021f6461447a730a38551526f09bfd462d4733918fb588d5cbcebfabb458111565f809977229696b8fe173b3204556031acebb514f49a563711b0a3c637594cbb42ee46ffde34ee92430f4c41a2ec16f68180d44d21a67525bd81947f8350e5159c9d54c0f295642d0812fc11611cfe3797d7b2b34291b82372e79b6a5b120737297376aa30f6c91caa7ba4a8fb4173e46aeb7a24eb315443dfacb67d067a5d93af9c844a32a9048f94f7b2968e35956072ca5c57707a41235c68b8a971c790427cb53b12e32dc5556103f0fc7919ce0a79ef6991a92349d5663e5973c2dcabe876b084958b0a63c38e3778537e86b85237e5367c176e3bc2f2029f7b524b499b84c28b1ede9b9345ba5b070ac61586d63d79436805ea2ea11b353839736ab5dc384b997025f80c4ef65559f0370ad1b5d385754e806321b906c307a3684bbbc919b01899b5cbea35d5db99ecb38a7943c135e49564a3b3c29b18e42f018b50b9c6b7782566209ed93166c121fba88585a165ecc7aa69219c1de9c4e019677621b3870997efa96909a331ab2aa63fc704b86d745f8228e5ef7c6d8e242aa285a06321bb1fc55f36b47bfa22cd8f849689ac3691024537808522816d7f71914b6115e0b71f9f41ce04565926782bfd7c75524af6b8b63debca86dd97efbb7c0d73946a0f24316046f23fc4939776d9d9156cea01071283e24fc02838448c54aa5c03a0ad77b563805022e9131ce77201d746567132b76707fd310ca512517081c2591e03133faa06e97bb58e625591ba5b5f901f068728008a8188a16e3034cf08c21a379acaa610b63e5afeefa9acc21191ee618a6777b979980b893b0534c44fbcc770e837b34d60030c47a2a790a09d42c72b4be53198eb38ba88839368095c375552fbed00841e08f2db7b991393342d22f3ad6a9f0fc763ef0b5706acc402b6526cb9db1ea65af79a43d184b1588481475bd0175404de0a266144867924f748a3d0c776062856627d2a21f39ce766cbbf64358fcc86ada67769583273aeac70f7277af501299628eae607ea4f38473643b3b983943d28ce3411495a88413b531d9d200b2928ab9c6324a5621f2c3352c0549d0a5c146c763530611a292263547cd7b2ba8c54510b9121056a3470d2b5e09cc7ca1377d3fcc18ce74b2c6e5c66b1b3d73ab1d7910679ff821a8525882cbbae2fa65aa20beaa78c555461e2d331be364100a946e5ef90c8ed74e917c79c0e81b3f556aae5c46ffe15f4c85b6e436bfae3441aad9c0215c6bb0d2970e3417565ba104249404e5817871309bb14587a807c402adc1f45a6b927cb04ca885951abc6353974214d4804ee7b281e3857be6bcbd97e47202f5bd28a71e3d179e0475c262b489d63c37c5630ce0a210b4e5cf7df19b81b1206da4ab9d367a793194e23122a37823e09b35f718bc9026a26c970f9fe6c0969a1e75f6aca3255291facd3397935e349a50778e690aa734e7778b6c4ef0e4915a0106a2d8080c676e00b28ba090101561b204c1791e00a064595f6c7a5799324b4f700ac41b1b1a923234638c8597c5d3d2abbcc6ca52595248036f9d277706785ed2951fb789a97ca061059245ea249c297ca7e88456cd8a0a0fbb27bdd6a71c0049a6c86f44792c7b470e6a73482ab38a94f247c12a10e546ab184ab0a8e97ae23c1c758a86cfe4bda5db2d1d1c8c5a1739a0a4503d82246886cf3cdb3369d2a51616c43d2850459395726bb975a46fbd8c73fdf726008013b17c98d47332ecf0725ab7722974c62021a5fd479e82f442d4dc6c768b171c83025f5b0d676c6a9882b6bed90a6e861247ea0a96fa4183275ab5d484695a1c3994696f38a920225cb2041d29d096dd3645efa8350337542ae09fc6c5cb49fc3e74bac9daf5bd853406ef8cadb206b810e31564b6ada0c3547e00a90a04793a58a824b4a489e1cf932c1fb12268364b296acb615cf81229ca33569caf28929d8fe52527967267606e96e49ce4a11b13e12258d4846b0a2345f053a53c4d4ff14e8970bc3c1b614f074667a624b858c54147b674d7b90330a09d67a31e5961c3b5c162082481b9354695778cf09a1f0a4c33182f969039820033fd071e8e33c87bd2bee1050a88b86dec251a976c3d4d3c6e5613a7987b44d37a8cfeb5bf8ce5cb6434284633922fbc8f9a169eb8f0bf20e637ecd28f50c940f2424186722ed6bc9e57547d3da241f5e517683795b8757275476d511614ea405d29a1cd70ec0fc7ea90dd80c99d4567fcc58b91ab2544417188a6cb55c8ccfb212b57ec1b023668c5a043518b7e17082eaf7a1f60125e938cb7d600a746f10e4a486dfc96a13e50b3b0d166038666c6c02d9df624754aa6c4824809a6bae60736f368488ad8949523ca70427b57362049a3958dfbcf27a0a6a1046355c1a4fb2503ad7098a472a22b3ab5e53a91b5c4912431a3d9a18e0ec5075fb720cfc0048830127f78a96ff9837b8716b5d88675d7696793b62f5524827b7a4da1034f65a8d3d509d95a51cc51bbf49181cb696977b2710c0b9ec9d945e3647d930b207f7ccbd36b6bda2586f312073a020cc27185ca26411e4750db354737143cf5540c3190bc225b49aae549e296c39eb6cc6c28079456174be7acfb8a3116d25ae851a116898ed38552439c28f13b83fa78bea8bb001f8785d0859f9c21697fdb3a7bb54859384e7315c5fd0b5bad34917f2c3bf1827578354b97984a8a53360ea58cd96753d01502bff21f1abb31c9a7b69094761ec957ae08965c4936e9176114684294a631c0e12879592770262b9bd56604451ef6b5548145b316f290a1d2a68d1743b760cb4c373321d977de48032d20cbf61bc6788c43fb1701e4f65e2821b884e44fcad016a8865fe0cabf9e682773734b7ab50ccb3697d74441fec87417965f5b6520591b6b1d550776d1cefc37a1b5d12cc9e462a23332c79ccba8c69989f60897696e738656964131c68417c1e89b0f8071c6511d1ea47b6db37d314a63fc847cd625bdb6e8458c5a0b8b070ae7d1a2e659637d9073ac1b6e99a2c1ae520fe6c6a527b6a64f329672374505d773c0ec11343c24c3e71e422072fbe7121644009bccc89e92239d7c761d092755800713e821929523bf6a1f52c307857a22ba9b54146b15ea159403db8ce3c2ad57e665b5190c23e9c3f7c4b90a566292f9c3f1b98afe683418615727a876db6078a2b40524e325fd881625c76ad5bc6874a47ba562beb7d11b46115cb3e10a8cf61d3ac73fbeb8945f4a3128107238268701dbb574c217e618aaba40aa56678219b66376e429975b4ba6f18ceea451dff657a6182ef1db42eb918071f7c0bf1c4f16450df1823aba8649b772c273d310af25be7f250b6adb5898d74d2c244bcc05744d074c579825156738ec2b8020dace305117f173b6fef86243193f8ac81d782b36f4b6565f22a3525248264c4bd8162e724b6140b1846f9ba0dd8464b2acbce11447e6581891c98aa39b5b5c4a299f473e57fb6aa5994e1cf71edea78f28c77c1c9b8c00a15f4ce7bcc56bcd3e6075329012a16377a2b2cd29d7cf97c7b34311486589a043c58291da6afad42262a192f70c24e09abf4d9272907b76ba832711176711271d7e9caa30bcc5b0228b90a12a5ce182f1f2046830a009e0b6794844a301b6ac1646af68c2f6bc16b094286d212c3353a26c7a7c6935a8ccc43156888fedda0971768f7e558c099ab5aac79900660500883e8fb078e91120648aaab0083ea0cb026df22f82547e99ba0abfb1741d075dce1a84a567c4acb99cf475bf4dfca20e0650ee278683b6a2c5115024c7af7ec16c7c65074b8535a13143cd590dd4960b8048498ec1be5a48bbac9a5d48002de4000bdbd8898729c40362a9b04070afda881adbbfe695a01d73b8d722b6c73b0d58a9401b803b1e882c5bfa1cdf4395d7b28ec69614b1c80fc74949aabf5523b52ed2f7da22260efd872dcdbdd51c1980ed2ce1524038644e72971011454ae50a899899e840aa421746d93acc94fb8bda6619ddad22f88d9336a144491206b47b3ee659dfabf1e99a66d6df3543df0414fd2df7bc5cc8ea245d0fdb c=63b513e1172cec1034c6a626c8018657a5ee6bddf4c781186eeb42d676cb998e00589ea7da35756dba56ca139fc8fd1e19ed21d41da8a486f4fab877d38a8fe4ef97bc51f338b086557004d7a53fa4f281de3fb3fb2383177dc3af358c6377fd4146e72b5e590c9c1ef2d57b5b75bd84e44d6a8a3cc9e1107c3f642a227e1c35c7954e2499393a8f48cd616890d36bd6a274473c14ca6ce68d84594389fa0d9ec8936f6b1a42af4601bc909d52b13bc726f5be1084c7e63d5fa65e16b7d1a36a1c38400ebd8b73c76af1ef838395acb2aaac1761d114432f7fa5243b5ec88bf8126f58a9750f7cedeb69d732fa12eac65568d64443cfe4cf60b392c2d1d23a0576165a1af54b8f21cd145d897084dc863b52e0afd618ce8666d5f551498179200858607bf1a7fe0b4c0bbbf62e65ce86cfae3b2e7f2fc8582491e6c8af96ceeb56cac2320d2e16da251f6fdbbe40f86b4dcdb9fbbf80288fac98686a7e0270ab991877d0094e6762d76eee5817bef9718f0976fcd7fa9f56f07db47a0f0b1b01687f1e7364f99613c9845e697281f29843f2f40f1c1cf0a8bbd810a8ebc35a9e3b7598eca61751da922078def662a8b80405ef019cad33dc124eeb339a48be48fbe703100281e9b4af4060df20f45b7e48086059d7a8ff44863ec4bd12f7e6ff22258f7c00e7da2364f42061e06c4a3c3e19b5a33fdc0e32c663cf8ab2448c3898b0e052b3cc51635eea77183c7257a6fa2a19e463bd4e29946b79d41ef8e8ce154d7e0cecdaf985a815dc194d1dc436629f0542dfd5a276928ab1a225959be8ba5e18873c188d1b07a55296f831077505d4774d26dda37086a67239f134d129291db08cfec921b4f4649cda8f45bdbe49e4456b5c6360b932b628f3f749d0a4059ff98f5acfb5723770614d916249065534f4ffaad2f31dde7d3ff50e538eded40d68fb0f748fc341ca5b13904d2dcb9fae332861fd81a1bfe7dbeb4187cd10167d3ef990bf533abd2d9aedd8e6e0b5143fc89ffa640ffa1de761011c8fa8e30b87299053a9ae2bbcc7540b82a78de99cbae258caee29b0af4f75ff8d7bdbc7bb67d789b374b71e467eab0a9ba4add8d3f90f6a2c12af642f3a5fe2e50404d9f0a897a4e04cf1d1434eeb4e3aaca845000a27092c3c97eb2813684c466130c673fca1a3ebe17d66d8491d986eebd4152c9b2c21fb70b4de6a2ef576baf76c5256e630350d7eb43f164b274108d9dd72b9d3624fb9609fcdbfbe85e253ebf7725156665f9d05d4c0bfe2415fa31b282b71db83e6b5cda39f3632b44695aa2f894f7403f367dbf26b117031baa296655148cba5588ee35e903047e2960c5f70f65d4c5110cca521a8aef81a743f8130da7786c584bac7b50c14e6de49765df8d2f5b1b4611f95a4b3031e1e7aae759f3458560dfa9738346bd11400a9e460a74dbb524c27d70c67390193e4b5e515972ffcdc487ac26d347250b9523bf8e4fc4b3f88774b4c1673a597c7e214679be9e3c2b640c273d163f8654ae1304cbf1a1fad2eabaa1febce15f0ff808530dee24eb5257256328bbea9ad65db1513eec31300c3a5ea37a0726deb74468f158bcf7474759235e74db6874f1b1df9ef4d29e091cfea32b9b81d29b51cd6124358ca2bc47a0113f5dc66bd0a2318702a2133c7a43b8106cd2d8f08971ff205a0f2ff4193038a6ac8902d9a7c980102cd87a6603ac9c3ef3005e886483ea3cf7b649977064d4cc0d33fb8400825fd715d69a20bc76f24e137a63bcc05ba801760f85b7afc3901d7a11e13f1e5e14a82c0aa5b01dc79eb44844687748c057347ee8b90e3d5b2acb7b5d81c9f084c8068ec744e3172a043e4c9da639552ba91cfdf52603d3876892250868e74465ccf735673d71b49e95456a670c0786ad65fd45aed9b51a334ddb2dedb54a02b2f6c43ea82b2c93375a5f1e165a9adeb803c91bfb06b85230663ef16de7fb608f0d85cd8cec9e7c24a863f18c4e6abdfd2382fdbfd91812a4b8afed6b688e76a36b08f38d2f56b4e40d5b8e95afa86c6ec8ce017863892d07385b2240dc7f10953f68c1440de5531e089efa01390382bdd8962b8ccba235d4ba9a5b46b62ac1d7a3cfb87b6904087db8ee01e1c455138f5e7e30773813012f5238ad2a35fcd160d685ddfc1cdbe5e6b92f398c5265188cc7916d48810d95290a28a7246791853551659b559d32d k=ae84fee1f4075e4ceef6961d122e621c1380387ff0804b8b40230bd03a44aee4 +encapsulationKeyCheck tcId=156 passed=1 reason=valid_encapsulation_key ek=521ca0f404b0dc436d02ec4d4c783a9fe60a027716754cc83a478ef9b292c71b1b4b78038b582944aa857fec42eca10f1d42cf97c6790f4b26152c5328279cc5a0ac9c369dfeb82a79e498398b54357c5b6b7aa3935a5bf456865c93669e0721acc1c1233bc75adb9a2845befad66045d93a9a1a22024688da726b38bb84d8c246a7e67f909a5329715212ea2f8d3a4631645887e0a32db14957a39dbdb550395352944b7d86368bd4b41f11bbabda9360b5f7160d8a123439b6e327ce6a55bdf08acdfa1b71c382abcfe9209b8c960cd95c7ee8b1a9a98cafa4242ff25d957a82efaa926e99372203381f801e0fa1898855a1a228c533f295e342122574936ce4925db87eeec11ed7306b0340a44b37c1c819acdc474adc004fc6cb2c578a2b089cc4894bc53bc4830f3b54d73aa2c10ca7e46c322af962e3aa6e45287538b920edfa71356b0bad9a0d1e0855701ca4c0e23f66ab3c7a4615a5076848d564a9a154a00958cdc9152d61a9915484e65154c4d33bf29285a2632ae16c81040b2c3618718f807fa0821da33a7b165379ec3757ecc21fb9a08ddad77a125476e6266d42e20efbd99488192dd735a196a5b2d310cd10e3cf711b441c642887cbcf5133b056eb8a1e5089540ba2e4404f25b71fd787bacc2bca73e0b9de76cee2664aecf5499a687c7c167f08836a15b832e69b66a1509725720543a123311159063137f25711250016ae983684112a0e5b0ac03ac28468606acc5cde2450af086e386149c76015ca6abc228948d8029b224b63a0a485518753e9271b74db35eb080b839a174326a57e332c24f3506ee748d1c56835799eece13c16e53d3a3966a17548526b0d9132c22788687c8bcf3288784b31539d615344abce04258b6f218e48a78a4732b91f75455024acd237a3b2f143e447533786889693ac3b96492d58765eca406317085dc659bf893b7c11ce4655b2452a389cd912bb98a63dfa37dfbb71bd58a5cd6a6094114101940cab2b90ef215a0a356098ab06ef068002e165bc21abaa68b58891445d434139746cb54402972b12dc75b53c471ed6951498da464cc4bfa76b2c5e8165d6d1cc76c3506261052e411b1db34bdb6cbbc4a8cb0efb980691af6508ca8ae97518ea0e689747d2ca89fdd44f017c49fdcc6dfb10465c8412b175697f052940909be126c5d35c0e642b6d251b7d526c11407962dedc06c8dc138de5233c5c976265a8da7b3683c4467497592e8bcc741449aaac41050bad867c36872c2119a4c7db34067a943f0c951168252ea0fb5fc6d769f940bffde7c46dab183e161e02046cc708ce817046ea8063f469148d2336432b36efc266077aa88c85a26606aaf7eb5ab6f3b64ed95fac5c3c6da380d0dc93a30c437a14816c25499b29132d492c3792c1880865b3670857b1758a97b14a29b569844775444c915707e2870b79ca8695759719a34d73062f51560fff19684f1048ff9730e2669443ea8635f303214a629b71a04e6042b06584e6386f3b396304535bb8356240353ec8ea8158c01565154baea1509de657625c2731a01bf741c5242ba7b909bdd6d47175681b92898ac8872f013852546a982081895729a369f40a70073b7a59bfee3460982307bdb163d3a05664a91961b12eb4248c190b8d980392ba0accd9705b4d3ba54535426f10099fa8198537adb66656b2d2843be15f42b93fb4e521280031b27798afab68ea084ec0767fb5c76f3a129d15e985aa18c899a9532df21d3366024d9c214b338da5891bcaf287c4940e5b3837f66a4aefd3c4a8406532fb8fb1cb8e45691bc1e960c976501fd532ab164d20a3368b0780c71a0e68c76a59f6cfc7f6663798b102c0a9ffa90c18e33121d127f6467e49471681b186a752b63b593c02f85f18384bb8f6087e59ae3ef879178534de3361ac4a03448486bafc394d369f450c234208377833a5b9768e13eb4c855c7893020a8091a8a9709a13693cf44cb842a747274273989c28fe3cc0753573bffc565ba3be02d7348fe98cc9bc616afac99308805ed0a1480888168790f4fb5417f27bb7f96ec963611d2cb468a98c508140cf5267c90a21e07bc471064efaa4476d0b95c9a16ebdc34965b3b219dc759af65392ca3cd486a2b9744aaf5bb33cb727e5bba176cc4fc28398cd1ef5eee6429066edf3fba01694c6aff0f246418712a4be53e1c13a75 +encapsulationKeyCheck tcId=159 passed=0 reason=noisy_linear_system_values_too_large ek=020d58896b0bc35c2d70da8992c424899a8ba9414a1a8319a70659fcc6a3f2b794e84b753b0c8822abce063106f4a8706358a0a422af8ff287eac05a3d8aa84339996763a8f7a9833d2980f19754d18c37e8c7b46425bf8cc1a75cbb7e1df5ac70e2056ad37f79f4ab83f87e5e32aa2e36cca999aa9a4c82e81cc94b4ab471785661349f0fd832297a3cab38cefdb45fb020495ad1bc1108186bf4b31d8354422b21ae719809762132e076e5a5b692f40c704acf1d3904d690af79f217ea38a8545281f3309cd6ec565a5921fdec938853a63eba0a1e8a68187cb5a3d639d35439ce5a80985bbb9c7a9762ec14d3eb5501380d7c298f9fa177eb5a510b04d0ca7a19c763abe4494a72a5bfc985bc403739dd8a9a13681dd39bcc93e965e4a09e44b97690c36d0d58b4f02126e4a0a07e6ac10e58c24b554ed89aaeb15c49ff53c16433835dd72fb4f69458d1bc99330891f1be9ab172242412ff819f82377914b269b7120137375be39b9d32112326ea5cfb5a071b479a73e0532b643b53e58acf1a987473a3ab017a66da9ac5946097679bbadab1a0827783e07f7bc198903215a6777fa3b77dc264ce37636aaa48a5f4208c03636e71c7746083bb7d403035390dd78118fdfc2f7df282287c96afe8774a20caa6db69104038eda590c648c69511811e9c28d52c3540721428336ee3d7732a287e51109ba8589227864f5c27c5dd07556b33cb4842340e5c5b027464a74734f088a591ebbc3caa0899359e75c395a6a57670b04bb38619bbb82a27c85f063cb286160ca46c372604b794d4202ea16f8a757e9dd8a54194c3384850d58213c7533cada33dd6704f082b4edfa67fafa2cab0c89897fc4c8f2255b839cd653c4f03a728f0d25cf2b99afdd0419c515dbdbcca0fc6947b0a0c936ab1f2f290dce5474183ad4dc85218ea13afc5b671926842a15d7c367d04ea97890b371297491d21720de148f9780866e1414ec6844eda97ae951afb0ab07fe61b1b856209db6dd522792c03332645a63c460ec595ac5c49b1de1980dde8b4bd5505ffa7cc4749c42f0694bf3abe4cc6acf8a7235766513319915b58522487c7f9eb9c2754863a40036e28332265c2a4fc420d95af87063c244233d5c27ca5b0261669386b624b5afc992cab773ec948b8957079b3b3fd27c16cf969005b27b4c438b7269fe317024814ae56862d18362fa47a0035339a485528540bcd27a288489c80ab890bc3992be39249e507ccc92a250ee631297ab19e6c39ac446ea5d54c675368bd5bc4002463d8cb9a294c3e427a2940ba5f00b97fb2b1714cb6880f5b6a13f868fb936e23fc4b7eb563e2f1c5fb421a449715a1f8ca0d3c6af726caa5d93c255bbcb7e60843a54eba470b32d19e3088c17ae5bec5408ed9758639c2ae50fcc6c5471b638210f8bb1509691b312082dda85c91b31517d78d4cf96cd94335494a66b8ec3f9b2478dc264a1b7462deea1c44cc0d357c00604abcc2437404771640d01fa1a9a73e900eb0b7cc8cb3a5ef621ea0118796ac7907612f339654cb519b50a04b6c38a1aa7464e34a8e9b9029a4c1687149b0959070ccf4744a03951b4ab9f6b88d85487e2929338cc6cac39651ef7c9b802657938b7c368c2b6b0539ec0c152ec95ab702662c21685748a968c68c150a96a9050bf3c597ade0699930c9780287d8811a68a74aeec8c335d4666c64aeb257c508a45e32d7a6cf86448b81486296cfee29c3ca7995fb9b55a8d62c6bd3be1c267627f4754dfcc1a7818051bb0448219794b87089b8969c72b01ffc168703cd11992383552e14197ab1c99aaf848231579eddc305d4942b27957700ed5ad44797e0b5953ae06eb25cb0cdc94c6b67a423cc6700d1604e362fb35105a8e6a00ed6587a759b98e37bd358c9ffcc4465f9c5d945a93954550a2b45a6ca90a3593e07163d7a8aada9848924b135c4911224b52fb5c2823f499f98d0238811b86b55a0d444a2b14b885a0b23f510b5b80280830b5dc705122595578fe076ff859b5fe0878e7b17932a3b99ac58d9a86812ab2006073b8d80bb940931ccfba269d319d68cc6b1fb5f6ebc288ad208379364efa73384a8a441c1759bca9d90016024eb2316d50e8042cb74f81320d089f6d08008007b7031a02bf773bb1c620a73b3a1579ffdbcaef6652400008b53f560f13d5dcc0b716895116434f2ee28 +encapsulationKeyCheck tcId=160 passed=0 reason=noisy_linear_system_values_too_large ek=02bdb4ea986ea8c4815085c9df937348e86bf9cba6bb910f2bb760014b2fa93c86280699af5323d0d2c1de689f11139bdc58ccc49771d5b1b2f17c5fdd1662eb0c663cc4935de26e64dcbd855bbdd42ca9f37c9263575c6008aea8803831124156f71f22743dd257bad514c49df4a4c2828e0070130ec1c2026213ad9694cdda308cd82f0c437557251807392508067d3e3a044ab0665b4a23afa209128a010c8c91d685ba77c65f119046d14a196c98882d135b14303f7852b1e4daa8fa873dd2d889186941266b744eecaf47eb84e47a69c86c16c71279ddb8b3dbc7a52b7b2d8d06aab4f65992e79d82368c6c8a7edff47f05e00d2cb35dcb2653fbf51b7a695721c68c602b6b08bbb52ad713480151dd221dbbb129f6d30521ecbecaa98e847470517968fe34cb22ec2245cb0ea0f0b1a4909406eb864990ad7948884e1562f3f4aa823a00553832dba5b5e15027184133b446baea9216cdc3c633481f6afb0263965f2b66c01eb180564c7187572073c31a00979f591bb6516c0cee373d117882434a8024e316d1d192ef8ab1dc1013ad07a8e78c226a3884c2a1b8ca372499e784fa10cf9f98a1c22a2f47eb4121b7157ac0c2da66350fc961367b92f57cae774443f705115c21793efb365bcc3db7d19f18119add93620014a69395c31b73a3709685b113b576711c6756132687840f2541bdf6a70a0704e4b4c2dcc33cdc9134586a50af2010fb640a3d03411a0b8d96e394d27733e7b58c521184bc8ac74592ba596220a1d00ad47240e4593c438b94ef29356d65415eea24c11644e5c726991a5718a9b21a91cecd83aa172c9495aa1b059845c81550dacb195ae6bb25f32f2cb7928dc1b83fe32700c78c542641573b241c92ace3766522a84669808dc6f94f98107c3cd15c1830afe473008594bfd1c779e0017ab9cbbad49344fcfc6c38b9a5f11956316cc34836768ca99d8c48bdcae189ab93a556493530cc4e8775ace118b4095c353c9702ab023928e16981e61f325921605659825034757564a494b8955759dca7b10593afdaf396da7b24fa9570e516bdc2ebbc36d875917224af4059f6863b511bcc5b186968db28f122415189b103f4b7ae003e87998244b71326cbb19c34017e892d2676a018268a29a77e29a2697c0655e2455aa544844939974e961f204994dde56573158c6120a2e61b56d7663926702a1dc77be18957d789a512632852f60fc6220789ca9fb2868e49f95903c62e0ca13979f582dab615047b06392bb0dd9b6555d934e9e18bc1987cca01a0917c82de6029b593994a195bb501a43ea689fd8b833bda0f9db2890c69846f625a84d2c97f47a21e345774dcc8c58ab7ae34bdd838595cf182dfd837ebfc67f4b96f2fb2c2ce046fe62b5fc4291c26b63f258606294101bc5588d13482fc7a88645111b9fc195a8c15fce93afd7b1b173c987c282b879a4651939f83d2af413bbe3f6027299ca09a8838ec79700da19e92fcc717d69abb775bed5c19bc9798d6c32ae6279af473356f5b7994676291e00a2333766c5342fbb2c7b261c3a10b147c8abf768b4781495c434199a4da7bc18b2bffa1268ab745ce972d5265a9735cbb84d67b7b236c3213058680a69c99006cd0850035414b23a40b6670542a98335664ef513700da6a48ea8290370f5974b868bb0aa7b3a145a77b83a01def743404385749b177b5259474f28377ab811d2075f0e49021f184d1b866f3a97baaea7e54a292c353ab8c974d29b02ece23835b163a120cb990d81310085a6a18bc6d69596baa0c1142b297c18b7b09a1d486a48a812eb5c2875b71ad2d4cbdf8a6901c3c64a8472730b963e7da999269a849b376cc6947e3218326b96ce175bdb7e79cb1a20fd8439f276c7acdc47c0e266ac8f270cb349cafb05d2c02bfcf5321dc3a3320506100e27576d58f0a0202eb2ac800e0801cfc81dc0ac680b3bf25ca9c3b5a40b852a547999637619063828e1c6c0a6fc73b61b011fba9101944ce179a504bf6a5434948628a0f0a35279480a287e9146690bdde5a41aaf5148351a3a7c537007100c4764c18bb0c8480c7b34414e833257f1434733896caa22135db24c7d2b3add53b5a110de4a5a883b591e029462537657c73a24fb92d6da735e6035372758299963ce1c91fc62ec34416c0ce6482495c96bd8298d5ad3d88f9738c005a1198abace6 +encapsulationKeyCheck tcId=161 passed=1 reason=valid_encapsulation_key ek=7935a67b16aed3340520932a0bf507660b944387a68da88b60921a780cc2e964b0eae3ae8742638471c6daa27148387c8ed516b7eb474d5c224fc777a6030aa98bc479138b70f2763b8c129cec752856a9a4ec92382426d365467c3754adc2409adb8bb658cda86098728095f5aa624e049ecce7774f153c08ac6f3da3044fca2f7c6936b24715e4000e49784d6d766a5eab9f4ba4c15c224b58ec595a7c1de9c61b798402e0e15a2db27b597344acc667929acf91297358e81692296c1364594f2b9bbde0bb6d07c347462f4a0480a7a410a3a63e91c692c383ce4ab0b1def98718f7118a5c3d39e8b9bba8aa3c62383143907e1bb04c7596731b4ace0c62268c450ea744fb884632075fa8d8baba02c5532a3fe6aa6891284d5db0ccaa950c564c9d16d813581ab8e76b8fdb0aa2d1b40e7f40560d3448bd3276557602c9da2de090b83db8802db18d962b4e8cc59797315e5ee26d9608b872c6191cd454cca2283275a7f8c00f70292802db7c3b9bbf320cc8f3e6ac26c2cd34b58b5a8b28fc3490d6630b7f12156d17627d616133540da97cad13e67f8e295aa0956a9743adcb799a5d50451613c8c91a7fad201d64e1283da18a18197320235f0f450a52400f153a3339eb0ef14c5b7af833f46c705c020a16749ad331286168bb0b865a8474330683b7db892788f5a421f6ade4547eb093c18fb454ae68821b439a79e52a3f3722244401b0bcb8262231dbf6ba16c807f48c118a088e55aac1afd2734a0994274b68a0527bc9d9587e4000a76c7f3a499c40990f5e4179330042b22b9b6dc4ba2e019512a76866669da284053bb727e68b6853348821a8760f630aa2d250abd55b8535733c5298ff0acd6a957e8c72705a6c6bb868a7b27367a7984811cb2657f994d355a2a42421d6288d38eb7313828ee922104951c234806f382a6ee863b8a0e23b85a5b3da7112bfbca56867a6ff6aa12d961f17112d9f6454a2678a1627114efa26dba199b71222aaa60453d02659ca1a1cf4532d118628ecc5fa2a48c27347a80a67ee271421b05c24ca50c3283357cc30182c02b2e7705f710dda5c06bcf57a1c049e9d902600f210c425a091343d05d1b019933612f858e3cb818ad1c6b96c50f4d1539c9098f6250ea170720d58134bf18186d1b2dc5354b583375274786ee887db4539fdb327db8bc619589a9579036541240278c59ac13bc75a3d006993ba473e14a93f5deb4900e205229293beba5c20400d1504c5b578c2306405fbe84d43059ac9863124a05bcdca93e7b522a3f800b4f218bdc98728108e5c2215b5428b504158732705b139258f9983a6cc560d81432cdab30602c6a1ab43c8122646898f342a6f3b464f9b163a140350b27384821350bfe532439b98c4eb4df852ce15e8c6a2422a5d5c2ab4723731794b079a163354442a5339c7c22d0707204560207141b71e7242fb25ce95c988a4c223e8c3b077f226cec7c32c0504bea81a2c187b0e08196176387165b57c7b9f22f1b6dc0b02f929547af19852390dffc86f076526e22aaaf1748bd4995cd56b5356d8c0c91857801301d8bccc45c7b6b39cb8fbb31d9e6028b18c1e79940cea800d1d6aca16e83cc46b203bc9bc9ea3754ba9cd48c4b53758351c243ba714beb5e3ca71fa7554294a809c45a4a16584702f9448525a2866bcc0a30878b024ba739259043c1871b3f24887a9a726843883c015ea318165db06fc1730ca7a77feab989b1a29e8d34626b700238b4f5af6bfc801b8e6643dc8d39ff6d747d07597774581ad1b2ac6053d763b1575720db0586cf0a906f8ca30387a079b86c609612aaf93139991c17224a0a52b086323a89c0cb81a441039b43502db87938ac217abbca6b37596524d025bcc36434ec11910ebf7510a77487fb5a21470974efca0b16aa8b71cc5de73bf5f00a607784df0d14f4b02c7a836cc78c06ae76588daf5756bb84dd6b2aaadf66904e5b60af3b8499bac91770b1c91263055658142891c3823367717579abb0f42908a3993d69bc65591827eb11c10c71dc15bcd87d7a134b25e12409f301a6725a0513bba6b51f5651f41c4912a4af4ab4690692535049de1a00edc504990fb609b34a3ae028bff0582b348635ea1368db89c376c0bce04011191b25a9069246c1212c117afa0e974fd9550da9b35203dad85b146effc811722cc1a591efa2b8d1f +encapsulationKeyCheck tcId=163 passed=0 reason=noisy_linear_system_values_too_large ek=02ed6489606b2ca56577b3578e6259c14acbdc2980df103dd8d40cc0f5854dd035dae89317ab508168706dba784cb45938a22243e1a40b539f4b94332e11c7eb25403703a967a6008d1770babb5afb92281fc28ffb537fca32b77c23c3c3a3aad2b792abc739b86bcea8f3589876b371ca0f700514c6a967271ba7fa2cc30766b622f4926c44a9c8b3b04bb571b833cc8018309cf257a1d77b71a8597e475fd9b5252637806f20b2ac58a280439a9c450eec4282e8615ae4f970f896375fd208d47a83bdd989e731b680d91b0e4687c2eab9bc568dc926c4f6f053f463bd01a83974b83ef9801daf6b814a94c96da949acb3165955a255ab8913f24c36d1536dcc014d1375721732b39cca8f0ca0168ca0b0f2184fe65e1c4c4c97720a03a84d02048ebc1228b45222195449572bc622b50a33a16bac6baf3bd9112c07c816f088bbb4baf9c29ba2cc03b1bc50be3122cedcc7590b6ad5e69977d239bce629802c775665c94634a07de6ace363562cf3385b0ca688740842e5532fbb668f78a4203609c0499bab04a1a4e37cd62362ee6480e53c1441c72bbf586779f907b4b625840bc3b0fc23e5eb5247450ddf0852a619a830134ffcb2846c825f94b38674ac524b090ddc3656ce348d925b5eb1c7a1e5272f19bbaa424263b4b80c8f51017fb20767644433ab45f0e29403e8cf4b943d8f566c0b807e7be33cd646129da97147d1911e0224cf8c75b412b5232602d73b386a80861301463b0b8053c66567c6626e47cec5769be6ec9b9e914225253fffeb3bc16622f5e1c667da9decb49f03f4802ad401a808c964b862b7fc63474299d30c8701d02caa626535330a5b290eba787594310b55ecbc9891caf1110f19136d1098ce51c95e7589a0f078bddb528698e08d721a7b99a8be89a5b196ea3773fba4bc281d56197ae781024050b6b41cc809436235eb1045e1ba1874196464c6af3935d1e93487d507cc538236d545ede6180f2009bcaa2b3b640c18aabd2ad65e97e57d2d501b1ef529c12057b02c6430d0c7e2e585e36a63918792ec661432ba416a54021e8700fdf27f87d04130c9c2673bbd9f5b8d91dc1e60d65b25215a27624ed7c044832cbfd6216138986c198545b6d6060523cb33da07e2fa220f231fa731a35e039fce591706a37937878b377368282c1e9a486fad1b1f78537988ba831bda26c0657e68b68594ea697834cf2352ab8d127641b0731df77abc35c0945171ac1816185211791031048b7587974a406a1cd97a813552314bc502d3a4bdb31b74680a85d5b6429bd68be2635c9542460294221bf365f3848efcbc7d3c3b33cf6b95a786c6124970c110049202b551d098d2537e204a3f98a02a4d649dca6524eff78cb4782dc738984aacc0759cb949b4b32aa2281b590c85757fbaa90e00e3789afb6983a61b474637355c2d47e5c90f835ad37b9f004aa86576166d504e5ac1a5edf6beb7217497cb8edf24a06ccc94c031aed62c1361e0222ff515ff1131a30421a8dc1f3f7341bb975c4d39202fe38433144942539e9db0650eec9dbbc90c7e4052046bbef9b45780bc0e4f105fcf927483c5b760a9aa78e15a5f88c19101063e771b71324541d3af49138860cb51d0dc9445fb653590b749b89a8181adaf101610d0806c7a12b7d3439deab948f5a38b56661b653299f4955307aab5011d94f81162d45353101674e156ed2cb50e4bc303fa53098852bde5347b7931bdab1ff3b086e0aa1871f375958991715c627b8457958c4dce389535f70f87aa9bfcd7c25c9191ba1a4b34a54d7b40cb49642f5f9a4a27bcb9ae8058c4b5c64098b08e5c3560565489e74bc12252a5419798b489881b7a5f57c9ec2014f3c98b920522e74128e7517b5f229a9a991ed193461ed17d3b83123785bd45279e2e6948cec9c0d2c63d73d6a7b043bab8bc22349cc1086443d2e9892711292f1b45e7292ed2021060e365cc86b42d11bb77b68c740c2294a32e5368a84840b7b560a314b854148907ec735930e74f67937767921f2a37c2da7bbdf14b8769564dba302e316999e6567edcb85f5eda0921dc4ff0d76befb03c6bc11879c6212ec8c30ab1ab4707782998c64b00158309c2d850382de25f53d528bfa2083ba01430aa5fc0172f2eda55e37a6f01217baa3a518c931417ca43b17052aeee6ea5d1e9e6314bb60d948be094e2690857cb +modulus line=1 sha256=f8d3b87f5d1d12ccf2ca1e4ef240348df6eb9c3540ef9057ed7f64c9efe3b5c6 ek=01cd9c6874b42683550244a579ca3bdb4bac1658603bc0c8afd035d0a5415e2c4310d647b6a3984244c3e7456af0e180d453240477461f49cdc6111e47cabf82fbaa36880c869262103a4a20cac30c304ce6990999039e7ef648d08253bc195c980071e61c959f02b409bcb737098af3c6acc98b579c99521d791295153b672a2d17d112d5e87d4878274e01568f9b81ccd50fa8ea34bbf15fc892c82ad46a21a60eef635aa89ca8c9b8cf944a5773ca01488b0ad35aa9ab5babe4c0cd63a90bc929bc3f9a0290851b79626229c54603b883120c0318c7113077aa4d507a366506c6936b56d1883f019918132f87592b5b6343e842323ab3713681b0de78ae607365de49267fa61d5b90ae69d378ab25626791a71dfb93ea06002cd8c32ef28d7e38587924363c1055554b4d7c632b144a958a9139a9ca61523a7e4cb38385f2547aa812c3dbb97ea49e7cd83815a486a6373e179b29a65a005df32b4f72bcb2a5c5371c2bcf8441bea92fe8c2183c6ca97b643480d8cd62d84fe17599f568af4204bd9a1bba594cb5668b3fc44a79397a052faa0569436c6515c5098a9bc25593cea3b694d82ac65516268b58a0f0a5cafb073cf943099b558f6c519a0b96fc4093dbf2362c237872160d7ca7b627606ad1c8169320285b1a3ddeb90d7806183b2bcce6228cb5a166904a6181c789041612af83b2c03326aaf8c468c6a2fa2ac2d8182da2941d7f34265ed54a0ac6b8f819c997a66c26d48c16086a9d77b25d80705da2bf198305ba3115a50ac655253044c4261e1b1d72876aae59b1e63787bf2ab0773c95d0707321ab91b5216d937506b4e3b3e49c18c1f1bbd0e4c1b10a5c2f13cf496769cc463f65e6cd26953692722218005bdcb25f165c9abbcc18b06b8050cb58550c319cf843c560583d0b499750231234c65b243055c68d16834d2a6baf14f96186a65d5de05eaa0b5a5af076d574a1da7334ee45489e337e147b297e963936fc084f3b7a8a324406422d3f677a4a5437965b5115730e949353ac04858d0a56af7ccf340093ee05309736799e8792018433bf55901574c341b2a9df1ccab2955624e58f0413074b1810f957bd202a728855ba1c123a0d3bc88855c59c8a8bafc6329db779b815c5d08297c8e0b328c08fa526aba4aab6c21b8c653c5b0b135b99ec24c11b9edccc4cc3d231efb7a045b61af546c24458ac876c151626cf3a0502f2463d850c9a038a200211abc7c085d29812e0395ebff3c236a34f44c64153e49a2ad002f1e16976b1c18766706fd3b7dde69d29834bee186f4bf4b7abe207175389d1b1b5c9c674e6facdd93158f17628e1ac1197b30902b3277366af0702a0576212cfd9707b47424971bfbb75011cba31b2a58ca062a479586aad4ca5890a8fd805151d5294df479803f4b80d055b19a1b7c483b280c12064213b2ab278eecca661ab635fbcc436fa8412861171f8cc49bc56cf1ba8050c07fc8c670886ae29d05762c26540d1c63888b7d6c9047cd2cbe8f9aa378114722b1d2ca87ce2599dce524ea6c4b83cd32133c36db8162e3c34681fda8ea2c7c9e80c87d34a73e4812aad40af218c75c2551cf86607d43c9f3706a9c30c737a89418ca703d2b1a015728ccb313e9d04117aeb8284f8012fab777640be1ef2592f533468851f8ca70087a83b68bb67cd9947334975e6355a9e50bc32418801458b3aab0d7b68542b581ee5e65990897830d37a72a6b849bb61e8c51cee56832793642f053a29f06bc1a616b912943deb1090a9cb671a904b503f45fa052f140cf0c557ee843c2d8818472c102a91cd7afc5afd9687a61765676508de1050a0fcb7bba82d0740063ac001eb336e81fb5a60c4a7143aaafaf93a62218e760b9dded8c44a6b4de9397d9c677badd9942f346f5b8022ac8c2a54c53856224bdae0146136636acc0b4815bd908b6bc525c0f06732034b8142c16490d419fc85873eeb56f2c002f1126bb3075f5bb16c99642e80f57b4df0479a564e50c78833485936a63c38d1cb8932c9050a0a7ad74a54f2a28788bae082c946da9e8d757afe1880b5d5747717698103538098b177386d36d7bfb96ca1a6b68d6ababc5a4531ea33cb053279b81c84c6d45bb1d14aa12258fc407de6a9a5dd66a9e16b6bd6b9b3bdb45a4a06d763455b4da200013e681c43ea7c943eb186edf29664c5dd42fb0f240f61 +modulus line=2 sha256=82a8c9f4b3c06702bf041aea8bd28d7683172e8792dd68a72e0334236dd02af3 patch=0:36c9,382:18d0 +modulus line=3 sha256=07df38591c9d4e46b0ef277415126870212b1cdd334ea32314589c3e33e9ff4f patch=0:ffcf +modulus line=4 sha256=f88f1f5efd0fe913eb3ceccc29dae258a0f8eb53dcb08862ece396ca2effd561 patch=0:36c9,382:f8ff +modulus line=5 sha256=3877991ef90084c1a7e1f3b671d6b0fd4b546ee8c5f9d605b997217dc053ceeb patch=0:36c9,384:010d +modulus line=6 sha256=8fc111ed729bbaf9748733fe56ede14995669f5a245dd10baef1a5e890fc28ab patch=0:36c9,766:12d0 +modulus line=7 sha256=035c17d15311499c4360ea929d4e23a1fdaba91b7e5b7b8b44a2e4dc31f36cf3 patch=0:36c9,384:ff0f +modulus line=8 sha256=c3a53d8a56dc33799235134c9b6a3dda4408dc42a4dc436aef2612b0b75cf3dd patch=0:36c9,766:f2ff +modulus line=9 sha256=9e6c134c77fd59768912318358e97283a1ec027a2f93bc7d4607cf51be950a50 patch=0:36c9,768:011d +modulus line=10 sha256=77326bf5e9cf810fd3586c8e08075e3cff77fd005f59748d5b94273fec143175 patch=0:36c9,1150:19d0 +modulus line=11 sha256=876a331d7493db89574c3c76af2f6f1c06d1830a3c0ff55e5a583be205aef075 patch=0:36c9,768:ff1f +modulus line=12 sha256=36f39a44ac05d70c542b6bdfa0864a437faba0f20cd8e418b48b2a9ad46ddd92 patch=0:36c9,1150:f9ff +modulus line=13 sha256=1548f43e54732a7c7eb90afbb2367bff67785d3dd8037996c73f9f2909529e7d patch=0:36c9,1152:01ad +modulus line=14 sha256=a8a0c1926a191a9814b665ecb246f02c4341cd99c9b01137a4ed4c6edb3a6c49 patch=0:36c9,1534:14d0 +modulus line=15 sha256=1a378ae2602211a702bfdd0ae13a5b504011628754733c49ed0d624a791e02e1 patch=0:36c9,1152:ffaf +modulus line=16 sha256=a5f41e8126594d628843fd61ba9b28bbb0bd728e6d8f67d363d57e984ac91b5f patch=0:36c9,1534:f4ff +modulus line=18 sha256=61e5200ca079af084322cf973c66d089c9c0da0ab72b5d5299dec25b9b91108f patch=0:3629d0 +modulus line=82 sha256=8837bcf89464a2ca476d92fd80169d63f8a5c6921697d94a5e5cce19f16104e8 patch=0:36c9,97:20d4 +modulus line=146 sha256=b27dd992fdddb27bc2655c4247ba4079fe673819685e8320d8749ec453858374 patch=0:36c9,194:d8 +modulus line=210 sha256=12a879afe6a3c8328006bdb2c12ae278bc70fffb4056a734c48ab14e620400bc patch=0:36c9,289:28dc +modulus line=274 sha256=f48ca5c828502d00e1a0051d0d52010e9f50cde6cb3b38edf248dd64503c13b3 patch=0:36c9,385:24e0 +modulus line=338 sha256=de031bd5a744809d78d1b7ffdc6e0fc9a0834c80552a1cccbc03b5c89c9a31b7 patch=0:36c9,482:e4 +modulus line=402 sha256=7762f870c1523fde36d97d42d2e5118c3ef8cbac39b000c07e85ddfef53d60eb patch=0:36c9,578:e8 +modulus line=466 sha256=cd57588865ba06432dd33b0b879ce695d77858d58e28cb39f3ff8dd153b899b6 patch=0:36c9,673:2bec +modulus line=530 sha256=2c96b884e9f3463fcfc3396a51675e0ad47095a4b81dbb625d547cfafe05cd0f patch=0:36c9,769:2cf0 +modulus line=594 sha256=eb8ff557ff2af8398d285cced00551d112229954f5e3cc81d20c27645ebc80f6 patch=0:36c9,866:f4 +modulus line=658 sha256=592aacd03b448cb27af6bc4da39337b2f51f3ec29930a7085074264c773ab430 patch=0:36c9,961:26f8 +modulus line=722 sha256=8768f5fb00ea07760ca16066987bcb23bf4bb4e52b9b87055f4c9fdb12d92bb7 patch=0:36c9,1057:2cfc +modulus line=786 sha256=be789e738c702231696a2107fdfb98b942273573fdf0c75305df0e1b70b4fbd5 patch=0:36c9,1153:37d0 +modulus line=850 sha256=3a5034c89dcd7810f595d95e8f6ac8db8873b0f5283b89182e8e7be78a5ecf8f patch=0:36c9,1249:35d4 +modulus line=914 sha256=c4a8392086da4f3e44d8e9579c5e42edceb7082aeb2661afa1d3e72c30c5a19f patch=0:36c9,1345:3bd8 +modulus line=978 sha256=129b0f331dac2d72ab98e6c4368eca0eb65e5cafd6541220bf1d5199fd805505 patch=0:36c9,1441:31dc diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 94f86fde7b..2aca741048 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -14,7 +14,6 @@ import platform import pdb import random -import re import shlex import shutil import subprocess @@ -28,7 +27,7 @@ from .blocktools import COINBASE_MATURITY from . import coverage from .p2p import NetworkThread -from .test_node import TestNode +from .test_node import TestNode, release_info from .util import ( MAX_NODES, PortSeed, @@ -74,10 +73,13 @@ class Binaries: bin_dir: An optional string containing a directory path to look for binaries, which takes precedence over the paths above, if specified. This is used by tests calling binaries from previous releases. + binary_prefix: The binary names used in bin_dir: "bitcoin" for + bitcoind, bitcoin-cli, ... or "qbit" for qbitd, qbit-cli, ... """ - def __init__(self, paths, bin_dir): + def __init__(self, paths, bin_dir, binary_prefix="bitcoin"): self.paths = paths self.bin_dir = bin_dir + self.binary_prefix = binary_prefix def node_argv(self, **kwargs): "Return argv array that should be used to invoke bitcoind" @@ -111,13 +113,14 @@ def _argv(self, command, bin_path, need_ipc=False): bin_dir is set (by tests calling binaries from previous releases) it always uses the direct path.""" if self.bin_dir is not None: + prefix = self.binary_prefix prev_release_binary = { - "node": "bitcoind", - "rpc": "bitcoin-cli", - "tx": "bitcoin-tx", - "util": "bitcoin-util", - "wallet": "bitcoin-wallet", - "chainstate": "bitcoin-chainstate", + "node": f"{prefix}d", + "rpc": f"{prefix}-cli", + "tx": f"{prefix}-tx", + "util": f"{prefix}-util", + "wallet": f"{prefix}-wallet", + "chainstate": f"{prefix}-chainstate", }.get(command, os.path.basename(bin_path)) if "." in os.path.basename(bin_path) and "." not in prev_release_binary: prev_release_binary = prev_release_binary + os.path.splitext(bin_path)[1] @@ -338,8 +341,8 @@ def get_binary_paths(self): paths.bitcoin_cmd = shlex.split(os.getenv("BITCOIN_CMD", "")) or None return paths - def get_binaries(self, bin_dir=None): - return Binaries(self.binary_paths, bin_dir) + def get_binaries(self, bin_dir=None, binary_prefix="bitcoin"): + return Binaries(self.binary_paths, bin_dir, binary_prefix) def setup(self): """Call this method to start up the test framework object with options set.""" @@ -540,27 +543,12 @@ def add_nodes(self, num_nodes: int, extra_args=None, *, rpchost=None, versions=N Should only be called once after the nodes have been specified in set_test_params().""" - def bin_dir_from_version(version): + def binaries_from_version(version): if not version: - return None - if version > 219999: - # Starting at client version 220000 the first two digits represent - # the major version, e.g. v22.0 instead of v0.22.0. - version *= 100 - return os.path.join( - self.options.previous_releases_path, - re.sub( - r'\.0$' if version <= 219999 else r'(\.0){1,2}$', - '', # Remove trailing dot for point releases, after 22.0 also remove double trailing dot. - 'v{}.{}.{}.{}'.format( - (version % 100000000) // 1000000, - (version % 1000000) // 10000, - (version % 10000) // 100, - (version % 100) // 1, - ), - ), - 'bin', - ) + return self.get_binaries() + release = release_info(version) + bin_dir = os.path.join(self.options.previous_releases_path, release.tag, 'bin') + return self.get_binaries(bin_dir, release.binary_prefix) if self.bind_to_localhost_only: extra_confs = [["bind=127.0.0.1"]] * num_nodes @@ -574,13 +562,12 @@ def bin_dir_from_version(version): extra_args[i] = extra_args[i] + ["-whitelist=noban,in,out@127.0.0.1"] if versions is None: versions = [None] * num_nodes - bin_dirs = [] + node_binaries = [] for v in versions: - bin_dir = bin_dir_from_version(v) + binaries = binaries_from_version(v) # Fail test if any of the needed release binaries is missing - for bin_path in (argv[0] for binaries in (self.get_binaries(bin_dir),) - for argv in (binaries.node_argv(), binaries.rpc_argv())): + for bin_path in (argv[0] for argv in (binaries.node_argv(), binaries.rpc_argv())): if shutil.which(bin_path) is None: self.log.error(f"Binary not found: {bin_path}") @@ -589,14 +576,14 @@ def bin_dir_from_version(version): raise AssertionError("At least one release binary is missing. " "Previous releases binaries can be downloaded via `test/get_previous_releases.py`.") - bin_dirs.append(bin_dir) + node_binaries.append(binaries) extra_init = [{}] * num_nodes if self.extra_init is None else self.extra_init # type: ignore[var-annotated] assert_equal(len(extra_init), num_nodes) assert_equal(len(extra_confs), num_nodes) assert_equal(len(extra_args), num_nodes) assert_equal(len(versions), num_nodes) - assert_equal(len(bin_dirs), num_nodes) + assert_equal(len(node_binaries), num_nodes) for i in range(num_nodes): args = list(extra_args[i]) init = dict( @@ -604,7 +591,7 @@ def bin_dir_from_version(version): rpchost=rpchost, timewait=self.rpc_timeout, timeout_factor=self.options.timeout_factor, - binaries=self.get_binaries(bin_dirs[i]), + binaries=node_binaries[i], version=versions[i], coverage_dir=self.options.coveragedir, cwd=self.options.tmpdir, diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index d7a5ff3c13..6edc0a99d7 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -5,6 +5,7 @@ """Class for bitcoind node under test""" import contextlib +from dataclasses import dataclass import decimal import errno from enum import Enum @@ -80,6 +81,66 @@ UNIX_PATH_MAX = 92 +@dataclass(frozen=True) +class ReleaseInfo: + """A previous release that a test can run through versions=[...].""" + family: str # "qbit" or "bitcoin" + client_version: int # the release's own CLIENT_VERSION + tag: str # directory under the previous-releases dir, e.g. "v1.0.0" + core_compat_version: int # the Bitcoin Core version whose options and RPCs it has + binary_prefix: str # "qbit" (qbitd, qbit-cli) or "bitcoin" (bitcoind, bitcoin-cli) + supports_v2_pq: bool # accepts -v2pqtransport + + +# qbit releases are listed explicitly: their client versions restart at 1.0.0, +# so neither the tag nor the Core interface can be derived from the number. +# test/get_previous_releases.py downloads each tag listed here. +QBIT_RELEASES = {r.client_version: r for r in ( + ReleaseInfo(family="qbit", client_version=10000, tag="v1.0.0", + core_compat_version=300200, binary_prefix="qbit", supports_v2_pq=False), +)} + + +def bitcoin_core_release_tag(version): + """Return the Bitcoin Core release tag for a Core client version, e.g. 280200 -> "v28.2".""" + if version > 219999: + # Starting at client version 220000 the first two digits represent + # the major version, e.g. v22.0 instead of v0.22.0. + version *= 100 + return re.sub( + r'\.0$' if version <= 219999 else r'(\.0){1,2}$', + '', # Remove trailing dot for point releases, after 22.0 also remove double trailing dot. + 'v{}.{}.{}.{}'.format( + (version % 100000000) // 1000000, + (version % 1000000) // 10000, + (version % 10000) // 100, + (version % 100) // 1, + ), + ) + + +# The oldest Bitcoin Core client version the previous-release tooling runs (v0.14). +# Lower client versions are qbit's, which must be listed in QBIT_RELEASES. +MIN_BITCOIN_CORE_VERSION = 140000 + + +def release_info(version): + """Return the ReleaseInfo for a previous release's client version. + + A version listed in QBIT_RELEASES is that qbit release. A version from + MIN_BITCOIN_CORE_VERSION up is the Bitcoin Core release with that client + version. Any other version is an unknown qbit release and is rejected rather + than read as a Core version.""" + if version in QBIT_RELEASES: + return QBIT_RELEASES[version] + if not isinstance(version, int) or version < MIN_BITCOIN_CORE_VERSION: + known = ", ".join(f"{r.client_version} ({r.tag})" for r in QBIT_RELEASES.values()) + raise ValueError(f"client version {version!r} is not a known qbit release ({known}) " + f"and is below the oldest Bitcoin Core release the tests run ({MIN_BITCOIN_CORE_VERSION})") + return ReleaseInfo(family="bitcoin", client_version=version, tag=bitcoin_core_release_tag(version), + core_compat_version=version, binary_prefix="bitcoin", supports_v2_pq=False) + + class FailedToStartError(Exception): """Raised when a node fails to start correctly.""" @@ -134,7 +195,12 @@ def __init__(self, i, datadir_path, *, chain, rpchost, timewait, timeout_factor, self.extra_args = extra_args self.running_args = list(extra_args or []) self.version = version - self.supports_p2mronly = self.version is None if supports_p2mronly is None else supports_p2mronly + # None for the build under test. + self.release = None if version is None else release_info(version) + self.supports_p2mronly = self.qbit_version_is_at_least(10000) if supports_p2mronly is None else supports_p2mronly + # qbit-only options such as -v2pqtransport go only to the build under test + # or to a release whose metadata advertises them. + self.supports_v2_pq = self.release is None or self.release.supports_v2_pq # Configuration for logging is set as command-line args rather than in the qbit.conf file. # This means that starting a bitcoind using the temp dir to debug a failed test won't # spam debug.log. @@ -566,7 +632,14 @@ def get_wallet_rpc(self, wallet_name): return self._rpc / wallet_path def version_is_at_least(self, ver): - return self.version is None or self.version >= ver + """Whether the node has the options and RPCs of Bitcoin Core client version ver. + + qbit releases are compared by the Core version they are based on.""" + return self.release is None or self.release.core_compat_version >= ver + + def qbit_version_is_at_least(self, ver): + """Whether the node is the build under test or a qbit release of at least client version ver.""" + return self.release is None or (self.release.family == "qbit" and self.release.client_version >= ver) def stop_node(self, expected_stderr='', *, wait=0, wait_until_stopped=True): """Stop the node.""" diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 9db8152b44..c926d3a005 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -135,6 +135,7 @@ 'feature_abortnode.py', 'wallet_address_types.py', 'p2p_orphan_handling.py', + 'p2p_v2_previous_release.py', 'wallet_basic.py', 'feature_maxtipage.py', 'wallet_multiwallet.py', @@ -394,6 +395,8 @@ 'rpc_mempool_info.py', 'rpc_help.py', 'tool_rpcauth.py', + 'tool_qbit_release_resolution.py', + 'tool_pq_canary.py', 'p2p_handshake.py', 'p2p_handshake.py --v2transport', 'feature_dirsymlinks.py', diff --git a/test/functional/tool_pq_canary.py b/test/functional/tool_pq_canary.py new file mode 100755 index 0000000000..4e2ebff9f2 --- /dev/null +++ b/test/functional/tool_pq_canary.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 The qbit developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test contrib/pq-canary against real nodes. + +Runs the evaluator's test suite, then the real pq-canary-sample.sh through +qbit-cli against the node under test and, when previous releases are present, +against a qbit v1.0.0 node, and checks what it records: NA, never 0, for +whatever a node does not report, a real 0 kept as 0, sample_ok=1 when the +required calls answered, and an unknown verdict from the evaluator for +evidence that does not cover the canary. +""" +import csv +import os +from pathlib import Path +import shutil +import subprocess +import sys + +from test_framework.authproxy import JSONRPCException +from test_framework.test_framework import BitcoinTestFramework +from test_framework.test_node import release_info +from test_framework.util import assert_equal, p2p_port, rpc_port + +V1_0_0 = 10000 +NA = "NA" +# Columns that come from getpqtransportinfo. +PQ_COLUMNS = ["pq_enabled", "instance_id", "arith_backend", "keccak_backend", "since", + "in_switched", "out_switched", "load_shedding_active", "in_ring_last_sequence", "out_ring_dropped"] + + +class PQCanaryToolTest(BitcoinTestFramework): + def set_test_params(self): + # A partial download can leave v1.0.0/bin without qbitd or qbit-cli. + release = release_info(V1_0_0) + binaries = self.get_binaries(os.path.join(self.options.previous_releases_path, release.tag, "bin"), release.binary_prefix) + self.with_v1_0_0 = self.options.prev_releases and all(shutil.which(argv[0]) is not None for argv in (binaries.node_argv(), binaries.rpc_argv())) + self.num_nodes = 2 if self.with_v1_0_0 else 1 + self.setup_clean_chain = True + + def skip_test_if_missing_module(self): + self.skip_if_platform_not_posix() # the sampler is a bash script + self.skip_if_no_cli() + + def setup_network(self): + # Unconnected, so the first samples see no peers at all. + self.add_nodes(self.num_nodes, versions=[None, V1_0_0][:self.num_nodes]) + self.start_nodes() + + def run_test(self): + self.tool_dir = Path(self.config["environment"]["SRCDIR"]) / "contrib" / "pq-canary" + self.out = Path(self.options.tmpdir) / "canary" + + self.log.info("The evaluator's test suite passes") + suite = subprocess.run([sys.executable, str(self.tool_dir / "test" / "test_pq_canary.py")], capture_output=True, text=True) + assert_equal(suite.returncode, 0) + + self.test_node_under_test() + if self.with_v1_0_0: + self.test_v1_0_0() + else: + self.log.info("Skipping the v1.0.0 sample: the v1.0.0 release is not downloaded") + self.test_rpc_outage() + self.test_evaluator_reads_the_samples() + + def cli_argv(self, node): + return node.binaries.rpc_argv() + [f"-datadir={node.datadir_path}"] + + def sample(self, host, argv, *, pinned=None, expect_status=0): + args = ["bash", str(self.tool_dir / "pq-canary-sample.sh"), "--host", host, "--out", str(self.out / host)] + if pinned is not None: + args += ["--pinned", pinned] + result = subprocess.run(args + ["--"] + argv, capture_output=True, text=True) + assert_equal(result.returncode, expect_status) + with open(self.out / host / "samples.csv", encoding="utf8", newline="") as samples: + return list(csv.DictReader(samples))[-1] + + def check_pq_columns(self, node, host, row): + """NA exactly when the node lacks getpqtransportinfo or connections_pq.""" + try: + node.getpqtransportinfo() + has_pq = True + except JSONRPCException as e: + assert_equal(e.error["code"], -32601) + has_pq = False + for column in PQ_COLUMNS: + assert_equal(row[column] == NA, not has_pq) + assert_equal(row["connections_pq"] == NA, "connections_pq" not in node.getnetworkinfo()) + assert_equal((self.out / host / "failures.jsonl").exists(), has_pq) + + def test_node_under_test(self): + node = self.nodes[0] + self.log.info("Sample the node under test: a real zero stays zero") + row = self.sample("current", self.cli_argv(node)) + assert_equal(row["sample_ok"], "1") + assert_equal(row["version"], str(node.getnetworkinfo()["version"])) + assert_equal(row["connections_in"], "0") + assert int(row["uptime"]) >= 0 + self.check_pq_columns(node, "current", row) + assert_equal(row["pinned_present"], NA) # not a pool node + + self.log.info("As a pool node, a pinned peer that is not connected is 0, not NA") + row = self.sample("current-pool", self.cli_argv(node), pinned=f"127.0.0.1:{p2p_port(1)}") + assert_equal((row["sample_ok"], row["pinned_present"], row["pinned_connection_type"]), ("1", "0", NA)) + + def test_v1_0_0(self): + self.log.info("Sample a v1.0.0 node: NA for the new fields, sample_ok=1") + old = self.nodes[1] + assert_equal(old.getnetworkinfo()["version"], V1_0_0) + row = self.sample("v1.0.0", self.cli_argv(old)) + assert_equal(row["sample_ok"], "1") + assert_equal(row["version"], str(V1_0_0)) + assert_equal(row["connections_in"], "0") + assert_equal(row["connections_pq"], NA) + for column in PQ_COLUMNS: + assert_equal(row[column], NA) + assert not (self.out / "v1.0.0" / "failures.jsonl").exists() + + self.log.info("The node under test pinned to v1.0.0 records the manual link without transport_pq") + node = self.nodes[0] + pinned = f"127.0.0.1:{p2p_port(1)}" + node.addnode(pinned, "add") + self.wait_until(lambda: any(peer["addr"] == pinned and peer["connection_type"] == "manual" for peer in node.getpeerinfo())) + row = self.sample("current-pool", self.cli_argv(node), pinned=pinned) + assert_equal((row["sample_ok"], row["pinned_present"], row["pinned_connection_type"]), ("1", "1", "manual")) + peer = next(peer for peer in node.getpeerinfo() if peer["addr"] == pinned) + assert_equal(row["pinned_transport_pq"] == NA, "transport_pq" not in peer) + + def test_rpc_outage(self): + self.log.info("An unreachable node gives sample_ok=0 and NA, never 0") + # No node listens on this RPC port. + argv = self.cli_argv(self.nodes[0]) + [f"-rpcport={rpc_port(5)}", "-rpcclienttimeout=5"] + row = self.sample("current", argv, expect_status=2) + assert_equal(row["sample_ok"], "0") + for column in ("uptime", "version", "connections_in", "connections_pq", "pq_enabled"): + assert_equal(row[column], NA) + + def test_evaluator_reads_the_samples(self): + self.log.info("The evaluator prints every figure, unknown for evidence that does not cover a canary") + evaluator = [sys.executable, str(self.tool_dir / "pq-canary-eval.py"), "--canary-start", "0"] + result = subprocess.run(evaluator + ["--pool", str(self.out / "current-pool" / "samples.csv")], capture_output=True, text=True) + assert_equal(result.returncode, 3) + figures = [line for line in result.stdout.splitlines() if line.split(" ", 1)[0] in ("pass", "fail", "unknown")] + assert_equal(len(figures), 4) + assert result.stdout.rstrip().endswith("canary verdict (figures 1, 2 and 4): unknown"), result.stdout + assert os.path.getsize(self.out / "current" / "samples.csv") > 0 + + self.log.info("The evaluator never compares a node with itself") + same = str(self.out / "current" / "samples.csv") + result = subprocess.run(evaluator + ["--archive", same, "--control", same], capture_output=True, text=True) + assert_equal(result.returncode, 2) + assert "--archive and --control are the same file" in result.stderr, result.stderr + + +if __name__ == '__main__': + PQCanaryToolTest(__file__).main() diff --git a/test/functional/tool_qbit_release_resolution.py b/test/functional/tool_qbit_release_resolution.py new file mode 100755 index 0000000000..efa8cb1ce6 --- /dev/null +++ b/test/functional/tool_qbit_release_resolution.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 The qbit developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the previous-release metadata (qbit_release_resolution). + +Checks, without downloading anything or starting a node, that a qbit client +version resolves to the right tag, download URL, archive, archive root, hash and +binary names on every host qbit publishes for, that get_previous_releases.py +detects each of those hosts, that the Bitcoin Core mappings +are unchanged, and that a qbit release gets Core-era options by its Core base +and qbit-only options only when it supports them. +""" +import argparse +import contextlib +import importlib +import io +import os +from pathlib import Path +import sys +from unittest import mock + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.test_node import MIN_BITCOIN_CORE_VERSION, QBIT_RELEASES, TestNode, release_info +from test_framework.util import assert_equal, assert_raises_message + +# The non-debug archive lines of https://github.com/Qbit-Org/qbit/releases/download/v1.0.0/SHA256SUMS, +# copied verbatim. get_previous_releases.py must register exactly these hashes. +QBIT_V1_0_0_SHA256SUMS = """ +a0242e5941eec64070f386e80c848220e2ec2292984a9b2ab1fd93f57436b31d qbit-1.0.0-aarch64-linux-gnu.tar.gz +5607d03b68d804934ae5918f7b8bdf4c6e0da3507e809bc05b97146de13156d7 qbit-1.0.0-arm-linux-gnueabihf.tar.gz +a98310136b57f49471bb19b7bbba1ccf3801da9a1d64202cf7dcf6756db0b3a1 qbit-1.0.0-arm64-apple-darwin-unsigned.tar.gz +3580f170ada2bc4230ff9b6c0000c8d23d1195c0ff9a178f99add1285f8f819b qbit-1.0.0-arm64-apple-darwin-unsigned.zip +acfc5098de60029e359fffda6438f29822e12a0ed5504b1a5940409c627f0384 qbit-1.0.0-powerpc64-linux-gnu.tar.gz +02455e62c33c0fac5ad65c3d8c8aa0c05be191cf7ad915a9d6dd4892c6b74cc4 qbit-1.0.0-riscv64-linux-gnu.tar.gz +93ca27facf73f736c1e1b152ad19f11eba0f075b800053bd07de9068fbe885b4 qbit-1.0.0-win64-unsigned.zip +e49b48469432e6f8bef26416b1660fc2a64809e0415f3b80cd6cb7b3f2109a2a qbit-1.0.0-x86_64-apple-darwin-unsigned.tar.gz +6f9de70a569df992218db792366cfb0b4d368a162ad53dccca03250d40517727 qbit-1.0.0-x86_64-apple-darwin-unsigned.zip +ae121af03263b55d530e3f3e8719a71362d0950cba82f54a2bc2d6c437a029b5 qbit-1.0.0-x86_64-linux-gnu.tar.gz +""" + +# Host triple -> the v1.0.0 archive holding bin/qbitd. macOS and Windows only have +# unsigned archives; the macOS "-unsigned.zip" is the GUI app bundle alone. +QBIT_V1_0_0_ARCHIVES = { + "x86_64-linux-gnu": "qbit-1.0.0-x86_64-linux-gnu.tar.gz", + "aarch64-linux-gnu": "qbit-1.0.0-aarch64-linux-gnu.tar.gz", + "arm-linux-gnueabihf": "qbit-1.0.0-arm-linux-gnueabihf.tar.gz", + "riscv64-linux-gnu": "qbit-1.0.0-riscv64-linux-gnu.tar.gz", + "powerpc64-linux-gnu": "qbit-1.0.0-powerpc64-linux-gnu.tar.gz", + "x86_64-apple-darwin": "qbit-1.0.0-x86_64-apple-darwin-unsigned.tar.gz", + "arm64-apple-darwin": "qbit-1.0.0-arm64-apple-darwin-unsigned.tar.gz", + "win64": "qbit-1.0.0-win64-unsigned.zip", +} + +# What set_host() picks for each depends/README.md triplet given in HOST. +DEPENDS_HOSTS = { + "x86_64-pc-linux-gnu": "x86_64-linux-gnu", + "aarch64-linux-gnu": "aarch64-linux-gnu", + "arm-linux-gnueabihf": "arm-linux-gnueabihf", + "riscv64-linux-gnu": "riscv64-linux-gnu", + "powerpc64-linux-gnu": "powerpc64-linux-gnu", + "x86_64-apple-darwin": "x86_64-apple-darwin", + "arm64-apple-darwin": "arm64-apple-darwin", +} + +# What set_host() picks for depends/config.guess output, newline included, or +# None when no archive runs on that host. +CONFIG_GUESS_HOSTS = { + "x86_64-pc-linux-gnu\n": "x86_64-linux-gnu", + "aarch64-unknown-linux-gnu\n": "aarch64-linux-gnu", + "armv7l-unknown-linux-gnueabihf\n": "arm-linux-gnueabihf", + "armv8l-unknown-linux-gnueabihf\n": "arm-linux-gnueabihf", + "riscv64-unknown-linux-gnu\n": "riscv64-linux-gnu", + "powerpc64-unknown-linux-gnu\n": "powerpc64-linux-gnu", + "powerpc64le-unknown-linux-gnu\n": "powerpc64le-linux-gnu", + "x86_64-apple-darwin23.6.0\n": "x86_64-apple-darwin", + "aarch64-apple-darwin23.6.0\n": "arm64-apple-darwin", + # No archive: ARMv6, soft-float or big-endian ARM (the ARM archives are + # little-endian ARMv7 hard-float binaries), and 32-bit POWER. + "armv6l-unknown-linux-gnueabihf\n": None, + "armv7l-unknown-linux-gnueabi\n": None, + "armv7b-unknown-linux-gnueabihf\n": None, + "powerpc-unknown-linux-gnu\n": None, +} + + +def core_client_version(tag): + """Bitcoin Core client version for a release tag, e.g. v0.20.1 -> 200100, v24.0.1 -> 240001.""" + parts = [int(p) for p in tag[1:].split(".")] + if parts[0] == 0: + return parts[1] * 10000 + parts[2] * 100 + return parts[0] * 10000 + parts[1] * 100 + (parts[2] if len(parts) > 2 else 0) + + +class QbitReleaseResolutionTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 0 # No node/datadir needed + + def setup_network(self): + pass + + def run_test(self): + sys.path.insert(0, os.path.join(self.config["environment"]["SRCDIR"], "test")) + self.releases = importlib.import_module("get_previous_releases") + self.exeext = self.config["environment"]["EXEEXT"] + + self.test_qbit_downloads() + self.test_set_host() + self.test_default_tags() + self.test_qbit_release_info() + self.test_core_mappings_unchanged() + self.test_node_options() + + def test_qbit_downloads(self): + self.log.info("qbit v1.0.0 resolves to its GitHub archive and SHA256SUMS hash on every host") + sums = {name: digest for digest, name in (line.split() for line in QBIT_V1_0_0_SHA256SUMS.split("\n") if line)} + assert_equal(set(self.releases.QBIT_RELEASES["v1.0.0"]["archives"]), set(QBIT_V1_0_0_ARCHIVES)) + for host, archive in QBIT_V1_0_0_ARCHIVES.items(): + download = self.releases.resolve_download("v1.0.0", host) + assert_equal(download.archive, archive) + assert_equal(download.url, f"https://github.com/Qbit-Org/qbit/releases/download/v1.0.0/{archive}") + assert_equal(download.archive_root, "qbit-1.0.0") + assert_equal(download.sha256, sums[archive]) + assert_equal(download.archive.endswith(".zip"), host == "win64") + assert_equal(download.adhoc_sign, host == "arm64-apple-darwin") + # No archive for a host qbit does not publish, rather than a guessed name. + assert_equal(self.releases.resolve_download("v1.0.0", "powerpc64le-linux-gnu"), None) + + self.log.info("Every qbit tag is both downloadable and known to the framework") + framework_tags = {r.tag for r in QBIT_RELEASES.values()} + assert_equal(framework_tags, set(self.releases.QBIT_RELEASES)) + # A qbit tag must not shadow a Bitcoin Core one. + core_tags = {v["tag"] for v in self.releases.SHA256_SUMS.values()} + assert_equal(framework_tags & core_tags, set()) + + def set_host(self, *, host=None, guess="", system="Linux", machine="x86_64"): + """Run set_host() as on a machine with this platform, HOST and + config.guess output. Return its status, chosen host and stderr.""" + args = argparse.Namespace() + fake_platform = mock.Mock(**{"system.return_value": system, "machine.return_value": machine}) + fake_subprocess = mock.Mock(**{"check_output.return_value": guess.encode()}) + err = io.StringIO() + with mock.patch.object(self.releases, "platform", fake_platform), \ + mock.patch.object(self.releases, "subprocess", fake_subprocess), \ + mock.patch.dict(os.environ), contextlib.redirect_stderr(err): + os.environ.pop("HOST", None) + if host is not None: + os.environ["HOST"] = host + ret = self.releases.set_host(args) + return ret, args.host, err.getvalue() + + def test_set_host(self): + self.log.info("set_host() finds every host qbit publishes for, from HOST or from config.guess") + assert_equal(set(DEPENDS_HOSTS.values()) | {"win64"}, set(QBIT_V1_0_0_ARCHIVES)) + assert set(QBIT_V1_0_0_ARCHIVES) - {"win64"} <= set(CONFIG_GUESS_HOSTS.values()) + for triplet, expected in DEPENDS_HOSTS.items(): + # config.guess runs even when HOST is set; HOST wins. + assert_equal(self.set_host(host=triplet, guess="powerpc64le-unknown-linux-gnu\n"), (0, expected, "")) + assert_equal(self.releases.resolve_download("v1.0.0", expected).archive, QBIT_V1_0_0_ARCHIVES[expected]) + for guess, expected in CONFIG_GUESS_HOSTS.items(): + system = "Darwin" if "-apple-darwin" in guess else "Linux" + if expected is None: + assert_equal(self.set_host(guess=guess, system=system), (1, "", f"Not sure which binary to download for {guess}\n")) + else: + assert_equal(self.set_host(guess=guess, system=system), (0, expected, "")) + assert_equal(self.set_host(host="x86_64-pc-linux-gnu", system="Windows", machine="AMD64"), (0, "win64", "")) + + def test_default_tags(self): + self.log.info("The default tags leave out qbit releases without an archive for the host") + core_tags = {v["tag"] for v in self.releases.SHA256_SUMS.values()} + assert_equal(self.releases.default_tags("x86_64-linux-gnu"), sorted(core_tags | {"v1.0.0"})) + # powerpc64le is a host set_host() recognizes, but v1.0.0 has no archive for it. + assert_equal(self.releases.default_tags("powerpc64le-linux-gnu"), sorted(core_tags)) + + self.log.info("Without tags nothing unavailable is fetched; an explicit unavailable tag is still an error") + target_dir = Path(self.options.tmpdir) / "previous_releases" + for tag in core_tags: + (target_dir / tag).mkdir(parents=True) # cached, so nothing is downloaded + + def set_powerpc64le_host(args): + # set_host() itself is upstream's and answers win64 on any Windows + # machine, so stand in for it: this checks main()'s use of the host. + args.host = "powerpc64le-linux-gnu" + return 0 + + def main(tags): + args = argparse.Namespace(target_dir=str(target_dir), remove_dir=False, tags=tags) + out, err = io.StringIO(), io.StringIO() + with mock.patch.object(self.releases, "set_host", set_powerpc64le_host), \ + contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + return self.releases.main(args), out.getvalue(), err.getvalue() + ret, out, err = main([]) + assert_equal((ret, err), (0, "")) + assert "Skipping v1.0.0: no qbit v1.0.0 archive for host powerpc64le-linux-gnu" in out, out + ret, out, err = main(["v1.0.0"]) + assert_equal((ret, err), (1, "No qbit v1.0.0 archive for host powerpc64le-linux-gnu\n")) + assert not (target_dir / "v1.0.0").exists() + + def test_qbit_release_info(self): + self.log.info("Client version 10000 is qbit v1.0.0, based on Core v30.2") + release = release_info(10000) + assert_equal(release.family, "qbit") + assert_equal(release.client_version, 10000) + assert_equal(release.tag, "v1.0.0") + assert_equal(release.core_compat_version, 300200) + assert_equal(release.binary_prefix, "qbit") + assert_equal(release.supports_v2_pq, False) + + bin_dir = os.path.join(self.options.previous_releases_path, release.tag, "bin") + binaries = self.get_binaries(bin_dir, release.binary_prefix) + assert_equal(binaries.node_argv()[0], os.path.join(bin_dir, "qbitd" + self.exeext)) + assert_equal(binaries.rpc_argv()[0], os.path.join(bin_dir, "qbit-cli" + self.exeext)) + + self.log.info("An unknown or too-low qbit version is rejected, not read as a Core version") + for version in (10001, 10100, 20000, 100, 1, 139999): + assert_raises_message(ValueError, f"client version {version} is not a known qbit release", release_info, version) + assert_equal(release_info(MIN_BITCOIN_CORE_VERSION).family, "bitcoin") + + def test_core_mappings_unchanged(self): + self.log.info("Bitcoin Core versions keep their tags, URLs, hashes and binary names") + for digest, entry in self.releases.SHA256_SUMS.items(): + tag, archive = entry["tag"], entry["archive"] + version = core_client_version(tag) + release = release_info(version) + assert_equal(release.family, "bitcoin") + assert_equal(release.tag, tag) + assert_equal(release.core_compat_version, version) + assert_equal(release.binary_prefix, "bitcoin") + assert_equal(release.supports_v2_pq, False) + + host = archive.removeprefix(f"bitcoin-{tag[1:]}-").removesuffix(".tar.gz").removesuffix(".zip") + if host == "osx64": + host = "x86_64-apple-darwin" + download = self.releases.resolve_download(tag, host) + assert_equal(download.archive, archive) + assert_equal(download.url, f"https://bitcoincore.org/bin/bitcoin-core-{tag[1:]}/{archive}") + assert_equal(download.archive_root, f"bitcoin-{tag[1:]}") + assert_equal(download.sha256, digest) + + # Release candidates, unregistered hashes and arm64 macOS self-signing. + download = self.releases.resolve_download("v0.20.0rc2", "x86_64-linux-gnu") + assert_equal(download.url, "https://bitcoincore.org/bin/bitcoin-core-0.20.0/test.rc2/bitcoin-0.20.0rc2-x86_64-linux-gnu.tar.gz") + assert_equal(download.sha256, None) + assert_equal(self.releases.resolve_download("v0.21.0", "arm64-apple-darwin").archive, "bitcoin-0.21.0-osx64.tar.gz") + assert_equal(self.releases.resolve_download("v24.0.1", "arm64-apple-darwin").adhoc_sign, True) + assert_equal(self.releases.resolve_download("v28.2", "arm64-apple-darwin").adhoc_sign, False) + assert_equal(self.releases.resolve_download("v24.0.1", "x86_64-apple-darwin").adhoc_sign, False) + + bin_dir = os.path.join(self.options.previous_releases_path, "v28.2", "bin") + binaries = self.get_binaries(bin_dir, release_info(280200).binary_prefix) + assert_equal(binaries.node_argv()[0], os.path.join(bin_dir, "bitcoind" + self.exeext)) + assert_equal(binaries.rpc_argv()[0], os.path.join(bin_dir, "bitcoin-cli" + self.exeext)) + + def make_node(self, version): + release = None if version is None else release_info(version) + bin_dir = None if release is None else os.path.join(self.options.previous_releases_path, release.tag, "bin") + return TestNode( + 0, + Path(self.options.tmpdir) / f"node_{version}", + chain=self.chain, + rpchost=None, + timewait=self.rpc_timeout, + timeout_factor=self.options.timeout_factor, + binaries=self.get_binaries(bin_dir, "bitcoin" if release is None else release.binary_prefix), + coverage_dir=None, + cwd=self.options.tmpdir, + version=version, + v2transport=True, + ) + + def test_node_options(self): + self.log.info("A v1.0.0 node gets -v2transport and the logging options through its Core base") + node = self.make_node(10000) + for arg in ("-v2transport=1", "-logthreadnames", "-logsourcelocations", "-loglevel=trace", "-nologratelimit", "-conf=qbit.conf"): + assert arg in node.args, arg + assert node.version_is_at_least(300200) + assert not node.version_is_at_least(300201) + + self.log.info("qbit-only options go to qbit releases that have them and to the build under test") + assert node.qbit_version_is_at_least(10000) + assert not node.qbit_version_is_at_least(10001) + assert_equal(node.supports_p2mronly, True) + assert_equal(node.supports_v2_pq, False) + + core = self.make_node(250000) + assert "-v2transport=1" not in core.args + assert not core.qbit_version_is_at_least(10000) + assert_equal(core.supports_p2mronly, False) + assert_equal(core.supports_v2_pq, False) + + current = self.make_node(None) + assert "-v2transport=1" in current.args + assert current.version_is_at_least(10**9) + assert current.qbit_version_is_at_least(10**9) + assert_equal(current.supports_p2mronly, True) + assert_equal(current.supports_v2_pq, True) + + +if __name__ == '__main__': + QbitReleaseResolutionTest(__file__).main() diff --git a/test/get_previous_releases.py b/test/get_previous_releases.py index ecb585c12d..408e2b2a22 100755 --- a/test/get_previous_releases.py +++ b/test/get_previous_releases.py @@ -10,6 +10,7 @@ import argparse import contextlib +from dataclasses import dataclass from fnmatch import fnmatch import hashlib import os @@ -93,6 +94,89 @@ "da0869639c323bbf6f264f1829083b9514e10179b90c34b09d8cbcab8a1897e3": {"tag": "v28.2", "archive": "bitcoin-28.2-win64.zip"}, } +# qbit releases are GitHub release assets. Each hash is copied verbatim from +# the release's SHA256SUMS. v1.0.0 has no signed macOS or Windows archives, so +# those hosts use the "-unsigned" ones; the macOS "-unsigned.zip" holds only the +# GUI app bundle, so the macOS tarball is used. The functional test framework +# maps client versions to these tags (QBIT_RELEASES in test_node.py). +QBIT_RELEASE_URL = "https://github.com/Qbit-Org/qbit/releases/download/{tag}/{archive}" +QBIT_RELEASES = { + "v1.0.0": { + "archive_root": "qbit-1.0.0", + "archives": { + "aarch64-linux-gnu": ("qbit-1.0.0-aarch64-linux-gnu.tar.gz", "a0242e5941eec64070f386e80c848220e2ec2292984a9b2ab1fd93f57436b31d"), + "arm-linux-gnueabihf": ("qbit-1.0.0-arm-linux-gnueabihf.tar.gz", "5607d03b68d804934ae5918f7b8bdf4c6e0da3507e809bc05b97146de13156d7"), + "arm64-apple-darwin": ("qbit-1.0.0-arm64-apple-darwin-unsigned.tar.gz", "a98310136b57f49471bb19b7bbba1ccf3801da9a1d64202cf7dcf6756db0b3a1"), + "powerpc64-linux-gnu": ("qbit-1.0.0-powerpc64-linux-gnu.tar.gz", "acfc5098de60029e359fffda6438f29822e12a0ed5504b1a5940409c627f0384"), + "riscv64-linux-gnu": ("qbit-1.0.0-riscv64-linux-gnu.tar.gz", "02455e62c33c0fac5ad65c3d8c8aa0c05be191cf7ad915a9d6dd4892c6b74cc4"), + "win64": ("qbit-1.0.0-win64-unsigned.zip", "93ca27facf73f736c1e1b152ad19f11eba0f075b800053bd07de9068fbe885b4"), + "x86_64-apple-darwin": ("qbit-1.0.0-x86_64-apple-darwin-unsigned.tar.gz", "e49b48469432e6f8bef26416b1660fc2a64809e0415f3b80cd6cb7b3f2109a2a"), + "x86_64-linux-gnu": ("qbit-1.0.0-x86_64-linux-gnu.tar.gz", "ae121af03263b55d530e3f3e8719a71362d0950cba82f54a2bc2d6c437a029b5"), + }, + }, +} + + +@dataclass(frozen=True) +class ReleaseDownload: + url: str + archive: str # file name; ends in .zip for win64, .tar.gz otherwise + archive_root: str # the single top-level directory in the archive + sha256: str | None # None when no hash is registered for this archive + adhoc_sign: bool # unsigned arm64 macOS binaries must be self-signed to run + + +def resolve_download(tag, host): + """Return the ReleaseDownload for a release tag on a host, or None if + that qbit release has no archive for the host.""" + if tag in QBIT_RELEASES: + release = QBIT_RELEASES[tag] + if host not in release["archives"]: + return None + archive, sha256 = release["archives"][host] + return ReleaseDownload( + url=QBIT_RELEASE_URL.format(tag=tag, archive=archive), + archive=archive, + archive_root=release["archive_root"], + sha256=sha256, + adhoc_sign=host == "arm64-apple-darwin", + ) + + bin_path = 'bin/bitcoin-core-{}'.format(tag[1:]) + + match = re.compile('v(.*)(rc[0-9]+)$').search(tag) + if match: + bin_path = 'bin/bitcoin-core-{}/test.{}'.format( + match.group(1), match.group(2)) + + archive_host = host + if tag < "v23" and host in ["x86_64-apple-darwin", "arm64-apple-darwin"]: + archive_host = "osx64" + + archive_format = 'tar.gz' + if archive_host == 'win64': + archive_format = 'zip' + + archive = f'bitcoin-{tag[1:]}-{archive_host}.{archive_format}' + sha256 = next((h for h, v in SHA256_SUMS.items() if v['archive'] == archive), None) + return ReleaseDownload( + url=f'https://bitcoincore.org/{bin_path}/{archive}', + archive=archive, + archive_root=f'bitcoin-{tag[1:]}', + sha256=sha256, + # Starting with v23 there are arm64 binaries for ARM (e.g. M1, M2) mac. + # Until v28.2 they had to be signed to run. + adhoc_sign=tag >= "v23" and tag < "v28.2" and host == "arm64-apple-darwin", + ) + + +def default_tags(host): + """Return the release tags the backwards compatibility tests use, without + the qbit releases that have no archive for the host.""" + core_tags = {v['tag'] for v in SHA256_SUMS.values()} + qbit_tags = {tag for tag in QBIT_RELEASES if resolve_download(tag, host) is not None} + return sorted(core_tags | qbit_tags) + @contextlib.contextmanager def pushd(new_dir) -> None: @@ -144,23 +228,12 @@ def download_binary(tag, args) -> int: return 0 shutil.rmtree(tag) - bin_path = 'bin/bitcoin-core-{}'.format(tag[1:]) - - match = re.compile('v(.*)(rc[0-9]+)$').search(tag) - if match: - bin_path = 'bin/bitcoin-core-{}/test.{}'.format( - match.group(1), match.group(2)) - - host = args.host - if tag < "v23" and host in ["x86_64-apple-darwin", "arm64-apple-darwin"]: - host = "osx64" - - archive_format = 'tar.gz' - if host == 'win64': - archive_format = 'zip' - - archive = f'bitcoin-{tag[1:]}-{host}.{archive_format}' - archive_url = f'https://bitcoincore.org/{bin_path}/{archive}' + download = resolve_download(tag, args.host) + if download is None: + print(f"No qbit {tag} archive for host {args.host}", file=sys.stderr) + return 1 + archive = download.archive + archive_url = download.url print(f'Fetching: {archive_url}') @@ -175,8 +248,8 @@ def download_binary(tag, args) -> int: hasher.update(afile.read()) archiveHash = hasher.hexdigest() - if archiveHash not in SHA256_SUMS or SHA256_SUMS[archiveHash]['archive'] != archive: - if archive in [v['archive'] for v in SHA256_SUMS.values()]: + if archiveHash != download.sha256: + if download.sha256 is not None: print(f"Checksum {archiveHash} did not match", file=sys.stderr) else: print("Checksum for given version doesn't exist", file=sys.stderr) @@ -186,7 +259,7 @@ def download_binary(tag, args) -> int: Path(tag).mkdir() # Extract archive - if host == 'win64': + if archive.endswith('.zip'): try: with zipfile.ZipFile(archive, 'r') as zip: zip.extractall(tag) @@ -204,16 +277,14 @@ def download_binary(tag, args) -> int: else: ret = subprocess.run(['tar', '-zxf', archive, '-C', tag, '--strip-components=1', - 'bitcoin-{tag}'.format(tag=tag[1:])]).returncode + download.archive_root]).returncode if ret != 0: print(f"Failed to extract the {tag} tarball", file=sys.stderr) return ret Path(archive).unlink() - if tag >= "v23" and tag < "v28.2" and args.host == "arm64-apple-darwin": - # Starting with v23 there are arm64 binaries for ARM (e.g. M1, M2) mac. - # Until v28.2 they had to be signed to run. + if download.adhoc_sign: binary_path = f'{os.getcwd()}/{tag}/bin/' for arm_binary in os.listdir(binary_path): @@ -251,13 +322,20 @@ def set_host(args) -> int: return 0 host = os.environ.get('HOST', subprocess.check_output( './depends/config.guess').decode()) + # A qbit host matches both its depends triplet (aarch64-linux-gnu) and + # config.guess output (aarch64-unknown-linux-gnu, newline included). platforms = { - 'aarch64-*-linux*': 'aarch64-linux-gnu', + 'aarch64-*linux*': 'aarch64-linux-gnu', + # The ARM archives are little-endian ARMv7 hard-float binaries. + 'arm-*linux-gnueabihf*': 'arm-linux-gnueabihf', + 'armv[78]l-*linux-gnueabihf*': 'arm-linux-gnueabihf', + 'powerpc64-*linux*': 'powerpc64-linux-gnu', 'powerpc64le-*-linux-*': 'powerpc64le-linux-gnu', - 'riscv64-*-linux*': 'riscv64-linux-gnu', + 'riscv64-*linux*': 'riscv64-linux-gnu', 'x86_64-*-linux*': 'x86_64-linux-gnu', 'x86_64-apple-darwin*': 'x86_64-apple-darwin', 'aarch64-apple-darwin*': 'arm64-apple-darwin', + 'arm64-apple-darwin*': 'arm64-apple-darwin', } args.host = '' for pattern, target in platforms.items(): @@ -275,8 +353,15 @@ def main(args) -> int: ret = set_host(args) if ret: return ret + tags = args.tags + if not tags: + # A tag that is asked for explicitly but has no archive for the host + # still fails in download_binary. + tags = default_tags(args.host) + for tag in sorted(set(QBIT_RELEASES) - set(tags)): + print(f"Skipping {tag}: no qbit {tag} archive for host {args.host}") with pushd(args.target_dir): - for tag in args.tags: + for tag in tags: ret = download_binary(tag, args) if ret: return ret @@ -295,11 +380,11 @@ def main(args) -> int: help='remove existing directory.') parser.add_argument('-t', '--target-dir', action='store', help='target directory.', default='releases') - all_tags = sorted([*set([v['tag'] for v in SHA256_SUMS.values()])]) - parser.add_argument('tags', nargs='*', default=all_tags, + parser.add_argument('tags', nargs='*', default=[], help='release tags. e.g.: v0.18.1 v0.20.0rc2 ' '(if not specified, the full list needed for ' - 'backwards compatibility tests will be used)' + 'backwards compatibility tests will be used, without ' + 'qbit releases that have no archive for the host)' ) args = parser.parse_args() sys.exit(main(args)) From 6fcdb4a2064d828717de9cb9d64424613d6d0628 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Wed, 21 Oct 2026 22:59:07 -0400 Subject: [PATCH 08/24] crypto: vendor mlkem-native and build ML-KEM-1024 with runtime backend selection (#225) * crypto: vendor pinned mlkem-native without source changes Vendor mlkem-native v2.0.0 (d1b2fe782888bdb761a50336012923180be7f502): the whole upstream mlkem/ directory and LICENSE, byte for byte, in src/mlkem-native. Nothing compiles it yet. contrib/devtools/mlkem-native.pin records the commit, tag, the tree id of mlkem/ and the LICENSE blob id. contrib/devtools/update-mlkem-native.sh reproduces the tree from upstream and rewrites the pin from the fetched object ids. test/lint/mlkem-native-check.sh, run by the subtree lint, proves offline that HEAD still holds exactly the pinned tree. Lints that scan src/ ignore the vendored tree, and doc/subtrees/mlkem-native.md is the runbook. Part of #184 (part 3, #190, D1). Co-Authored-By: Claude Opus 5.5 * crypto: share x86 feature primitives and detect usable AVX2 Move the XGETBV instruction out of sha256.cpp into compat/cpuid.h as XGetBV(), next to GetCPUID(). SHA256AutoDetect keeps its own selection logic unchanged; it calls the shared primitive only. Add compat/cpu_features.{h,cpp} with a standalone answer to "can this CPU and OS run AVX2 code": the supported leaves are queried first, XCR0 is read only once leaf 1 reports OSXSAVE and AVX, and AVX2 is usable only with XMM and YMM state enabled and the leaf 7 AVX2 bit set. Unlike the SHA256 logic it does not depend on SSE4 or SHA-NI. HasMlkemX86Native() adds the other extensions that mlkem-native's x86_64 assembly executes, found by auditing the assembled object: SSSE3, SSE4.1, POPCNT and BMI2. A virtual machine can show AVX2 while hiding those, and checking AVX2 alone would then end in SIGILL. Both checks detect once and cache the result, because the ML-KEM capability hook asks about a hundred times per operation. Non-x86 targets and compilers without report false. Part of #184 (part 3, #190, D2). Co-Authored-By: Claude Opus 5.5 * crypto: require SSSE3 for the SSE4 and SHA-NI SHA256 paths SHA256AutoDetect selects the SSE4 code whenever CPUID leaf 1 reports SSE4.1, and only under that check also the 4-way SSE4.1 code and the SHA-NI code. All three execute SSSE3 instructions. An audit of the built objects finds pshufb and palignr in the SSE4 and SHA-NI code and pshufb in the 4-way code, besides SSE4.1's pinsrd, pextrd and pblendw; no SSE4.2 instruction occurs. A virtual machine can report SSE4.1 while masking SSSE3, and the self-test then dies with SIGILL at startup. Require SSSE3 as well. Under qemu-x86_64, -cpu Haswell,-ssse3 and the SHA-NI model EPYC,-ssse3 die with SIGILL before this change and select the standard implementation after it. Unmasked, Haswell still selects sse4, sse41 and avx2, and EPYC and Denverton x86_shani. This is an upstream Bitcoin Core bug, so the change is kept minimal enough to go upstream. Part of #184 (part 3, #190). Co-Authored-By: Claude Opus 5.5 * crypto: add ML-KEM-1024 wrapper and runtime backend dispatch Build the vendored mlkem-native as ML-KEM-1024 into bitcoin_node only, behind a small C++ wrapper. Nothing in qbitd calls it yet. - src/crypto/mlkem_config.h configures the library (MLK_CONFIG_FILE): parameter set 1024, the qbit_mlkem1024 prefix, deterministic APIs only, zeroization through memory_cleanse, and a capability hook. - Native backends are chosen by qbit-owned backend headers instead of upstream's compile-time defaults, which enable x86_64 only for AVX2 compilers. No source is compiled with an ISA flag: every native entry point asks the hook, which reads only cached CPU detection and one atomic portable override. On x86_64 that detection requires every extension the assembly executes (AVX2, SSSE3, SSE4.1, POPCNT, BMI2), not only the AVX2 that upstream's capability name says. - Upstream's AArch64 single-lane Keccak entry point skips the hook, so the override could not reach it. qbit mirrors that header's dispatch glue with the check added; every other entry point checks itself. - mlkem::KeyGen, CheckPublicKey, Encaps and Decaps take fixed-size spans, map upstream's invalid-key results separately from internal failures, and clear their outputs on every failure. GetBackendNames() reports the implementation that runs. ForcePortableForTesting and InjectResultForTesting are test-only guards. - cmake/mlkem-native.cmake adds WITH_MLKEM_NATIVE=AUTO|ON|OFF and the configure summary line. The library gets core_interface's hardening flags but not qbit's C++ warnings, then APPEND_CPPFLAGS and APPEND_CFLAGS, so the MSan jobs' -U_FORTIFY_SOURCE applies to it as to the other C libraries. MSVC builds without assembly, MSan builds portable C, and AArch64 ELF targets build portable C because the upstream assembly has no BTI landing pads or GNU property note and would strip BTI/PAC marking from the binary (doc/subtrees has the evidence and the revisit trigger). ON fails where native is impossible. mlkem_tests covers NIST ACVP keygen, encapsulation, decapsulation and key checks, all 1040 C2SP CCTV modulus vectors and the 3328/3329 boundary, implicit rejection against an independent SHAKE256, injected failures, moves and wiping, and backend selection, each on the native backend and with portable C forced. ci/checks/test_mlkem_build_policy.py checks the WITH_MLKEM_NATIVE matrix, builds the library with each compiler found (core-checks requires the AArch64 cross compiler), and counts assembly calls to show that the portable override reaches every x86_64 routine. Part of #184 (part 3, #190, D3). Co-Authored-By: Claude Opus 5.5 * fuzz: exercise ML-KEM validation and backend equivalence Add two fuzz targets over the mlkem:: wrapper: - mlkem: honest key generation, encapsulation and decapsulation agree; an arbitrary encapsulation key is accepted exactly when an independent modulus check passes, and a rejected one leaves no output; any ciphertext of the right length decapsulates to a secret that differs from the honest one (implicit rejection is never proof of validity); an arbitrary decapsulation key is rejected exactly when an independent SHA3-256 hash check fails, and the secret is then cleared. - mlkem_backend_diff: the detected backend and forced portable C give equal statuses and outputs for the same inputs, and the override is restored afterwards. Both honor MLKEM_FORCE_PORTABLE=1. They join the required qbit corpus with nine generated seeds each (CCTV modulus key and 3328/3329/4095 coefficients, NIST ACVP decapsulation vectors, a ciphertext bit flip). CI replays them a second time with MLKEM_FORCE_PORTABLE=1 and includes them in the mutation phase; the PQ CI contract test checks that wiring. UBSan's -fsanitize=integer checks are suppressed for the vendored tree, as for the other external libraries. Part of #184 (part 3, #190, D5). Co-Authored-By: Claude Opus 5.5 * bench: measure ML-KEM operations by active backend Benchmark ML-KEM-1024 key generation, encapsulation and decapsulation twice each: on the detected backend and with portable C forced. Like the per-implementation SHA256 benchmarks, each is named after the backend that ran, so on an AVX2 host bench_qbit -filter='MLKEM.*' lists both, and the portable guard restores auto-detection afterwards. Part of #184 (part 3, #190, D5). Co-Authored-By: Claude Opus 5.5 * ci: add s390x and macOS arm64 PQ unit jobs Add two jobs to the PQ workflow, selectable by name like aarch64-unit: - s390x-unit cross-compiles only test_bitcoin for big-endian s390x on the x64 runner, with Ubuntu's g++-s390x-linux-gnu and the depends tree, and runs the suites through ctest with qemu-s390x as the crosscompiling emulator, so only the tests run emulated. It stays out of the merge gate. Its timeout is 180 minutes until runs are measured. - macos-arm64-unit runs natively on a GitHub-hosted macos-14 runner without Docker, asserts that uname -m is arm64, and shares the unit job's source checks and evidence report through YAML anchors. It is the job that runs native AArch64 ML-KEM, and configures with WITH_MLKEM_NATIVE=ON so that losing native code fails it. All three jobs now expect mlkem_tests besides bip324_tests and net_tests (CTEST_REGEX ^(mlkem_tests|bip324_tests|net_tests)$), and report the suites through ctest_evidence.py. The aarch64 job's description says that ML-KEM runs portable C on AArch64 Linux. The PQ CI contract test covers selecting each job, the s390x cross and emulator settings, the macOS architecture check and report, and that every expression uses a context GitHub makes available there (no runner in job env, no matrix in a job without one), which GitHub enforces when loading the file. Part of #184 (part 3, #190, C2 and the CI part of D3). Co-Authored-By: Claude Opus 5.5 * ci: gate ML-KEM and transport changes on the aarch64 PQ job The merge-profile classifier gains a pq_aarch64_required output, true when a change touches the ML-KEM library (src/mlkem-native, src/crypto/mlkem*, cmake/mlkem-native.cmake), the v2 transport (src/bip324*, src/net.h, src/net.cpp), the CPU feature checks, the PQ workflow or its env files, or the PQ test suites and vectors. A path it cannot read also requires the job. Full Validation calls ci-pq.yml with jobs: aarch64-unit on the change itself (a pull request's merge commit) when the output is true. The gate accepts a skipped job only when the classifier says it is not required; a required job that was skipped, failed or cancelled, or a missing or unreadable classifier output, fails it. The changed-file lists that feed the classifier, in Full Validation and in the Required Merge Gate, use --no-renames, so moving a file out of a gated path still names it. ci/checks/test_pq_merge_gate_contract.py runs the real classifier, job condition, ci-pq selection step and gate script: a pull request touching src/bip324.cpp requires and runs the job, while docs-only and other source pull requests skip it and pass, and a rename out of a gated path still requires it. The Required Merge Gate runs it with the other PQ CI contracts. Part of #184 (part 3, #190, C3). Co-Authored-By: Claude Opus 5.5 * ci: run the PQ macOS arm64 job on macos-15 GitHub is retiring the macos-14 runner image: brownouts start on 2026-10-05 and the image is removed on 2026-11-02. Move the macos-arm64-unit job to macos-15, which is also Apple Silicon. Co-Authored-By: Claude Opus 5.5 * build: probe the ML-KEM target with the appended flags APPEND_CPPFLAGS and APPEND_CFLAGS reach mlkem_native's C and assembly compiles, but the architecture probes ran without them. A build that appends a target-changing flag such as -m32 was detected as x86_64, enabled the native backend, and then failed in mlkem_config.h. Run the probes with the appended flags, and probe afresh on every configure, since those flags may change between configures. Co-Authored-By: Claude Opus 5.5 * ci: require the macOS arm64 and x86_64 PQ jobs on PQ changes No pull request ever built or ran ML-KEM's native code on macOS: the macos-arm64-unit job could only be dispatched, and nothing ran the x86_64 native code on an Intel Mac at all. Apple Silicon and Intel macOS are release targets with native code enabled. ci-pq.yml gains a macos-x86_64-unit job on GitHub's Intel image (macos-15-intel), sharing every step with macos-arm64-unit. Both build with WITH_MLKEM_NATIVE=ON, check the runner's architecture, and after the suites pass require the native backend by name from mlkem_tests/backend_selection, so a runner whose CPU lacks an extension cannot pass on portable C. Full Validation's PQ job, renamed pq-unit, now calls ci-pq.yml with aarch64-unit, macos-arm64-unit and macos-x86_64-unit, so all three are required exactly when the classifier says the change needs them. The classifier output is renamed pq_unit_required to match. On-demand runs accept the new job name; an empty jobs input runs it too. Co-Authored-By: Claude Opus 5.5 * test: check the portable override on AArch64 and per entry point The build-policy test counted assembly calls only for the x86_64 backend, so nothing in CI showed that the portable override reaches every AArch64 assembly routine. With upstream's x1 Keccak glue in place of qbit's mirror, 221 assembly calls ran with portable C forced, and every test passed: outputs agree, and the backend name comes from the hook. The counting harness now also runs on AArch64: a static binary built with aarch64-linux-gnu-g++, from a copy of mlkem_config.h without the AArch64 ELF refusal, and run under qemu-aarch64. A negative control builds upstream's x1 glue and requires the leaked calls to show. Each entry point also runs alone on a new thread while portable C is forced, and must make no assembly call: one that skipped BeginOperation() would run on the new thread's stale copy of the override. The earlier round trip could not show this, since KeyGen set the copy for the whole thread. The build smoke job installs g++-aarch64-linux-gnu and qemu-user and lists them as required, so the AArch64 case cannot skip. Co-Authored-By: Claude Opus 5.5 * test: cover each ML-KEM entry point's override and its per-thread copy Two wrapper mutations passed every test. An entry point without BeginOperation() went unnoticed because ForEachBackend calls GetBackendNames() on the same thread first, which takes the override for it. A shared (not thread_local) copy of the override went unnoticed because only one thread ran operations. every_entry_point_takes_the_override runs each entry point alone on a new thread with portable C forced, then asks the capability hooks what that thread's copy says. override_copy_is_per_thread lets one thread take the override set while another holds it unset, and checks that the first thread's copy did not change. override_changed_by_another_thread now runs operations on two threads at once, so a shared copy also gives wrong outputs on x86_64 and a data race under ThreadSanitizer. Co-Authored-By: Claude Opus 5.5 * fuzz: check that the ML-KEM portable replay runs portable C CI replays the mlkem and mlkem_backend_diff targets a second time with MLKEM_FORCE_PORTABLE=1, but nothing checked that the binary honored it: any other value, such as "true", silently selected native code, and the second replay would have repeated the first. Initialize now refuses any value but unset, empty, 0 or 1, prints the active backend once, and aborts when MLKEM_FORCE_PORTABLE=1 but the active backend is not portable/portable. A host without native code also shows in the log, where mlkem_backend_diff compares portable C with itself. Co-Authored-By: Claude Opus 5.5 * test: cover SHA256's SSSE3 requirement with synthetic CPUs SHA256AutoDetect requires SSSE3 besides SSE4.1 before it selects its SSE4, SSE4.1 or SHA-NI code, all of which execute SSSE3 instructions. Nothing tested that: dropping the SSSE3 term passed every test, because real SSE4.1 hardware always has SSSE3. An overload of SHA256AutoDetect takes the CPUID and XGETBV functions, as cpu_features::DetectX86Features does, and the existing one passes the real ones, so the queries and the selection are unchanged. A cpu_features_tests case runs it on synthetic CPUs derived from this one: unmasked it selects what this CPU gets, and with SSSE3 or SSE4.1 masked the standard code. Co-Authored-By: Claude Opus 5.5 * doc: say exactly where ML-KEM native code runs The wrapper's header said native code runs on "x86_64 with AVX2, AArch64", and the WITH_MLKEM_NATIVE help text spoke of "the x86_64 and AArch64 ML-KEM assembly". AArch64 runs native code only on macOS (AArch64 ELF builds stay portable to keep their BTI and PAC marking), and x86_64 also needs SSSE3, SSE4.1, POPCNT and BMI2, as cpu_features::HasMlkemX86Native checks. Co-Authored-By: Claude Opus 5.5 * contrib: keep the clone unshallow when vendoring mlkem-native update-mlkem-native.sh fetched the upstream release with --depth=1 into the qbit repository itself. That writes .git/shallow, so a full clone then reported itself as shallow to git and every tool that asks. Fetch into a temporary bare repository instead, and read the commit, the object ids and the archive from there. Staging and the byte-for-byte check still run in the qbit repository. Re-vendoring v2.0.0 by tag and by commit leaves no diff, and the clone stays complete. Co-Authored-By: Claude Opus 5.5 * doc: record the local x86_64 GNU property evidence for mlkem-native #190 asks for ELF property evidence that linking the x86_64 assembly keeps the CET marking. Record what a local Ubuntu build shows: the assembly and C objects and the test_qbit and fuzz binaries that link the native backend all carry IBT and SHSTK, the binaries need only baseline x86-64, and the assembly loses its note only when built with -fcf-protection=none. The Guix release evidence is still outstanding and is required before the post-quantum feature branch merges into 1.x.x. Co-Authored-By: Claude Opus 5.5 * test: name the ML-KEM entry point instead of capturing a binding every_entry_point_takes_the_override captured a structured binding in the thread's lambda, which compilers accept only from C++20 (P1091R3, P1381R1). Bind the label and the entry point to ordinary variables, so the test does not depend on that support. Co-Authored-By: Claude Opus 5.5 * ci: build the macOS PQ jobs' Boost and libevent through depends The first macOS arm64 run failed to compile txmempool.h: with NO_DEPENDS=1 the job used Homebrew's Boost, now 1.92, and Boost 1.91 stopped accepting a struct derived from multi_index::indexed_by as an index list. Bitcoin Core fixed the source in bitcoin/bitcoin#35175; that backport is handled on its own. Build Boost and libevent with depends, natively for the runner, so the jobs compile against the versions qbit releases with. Homebrew now installs GNU Make 4 for depends instead of the two libraries. A step sets HOST and reads depends' own build id (packages, recipes and compilers), which keys an actions/cache of depends/built. It is saved once the toolchain file exists, even if the suites fail. After the build, a step requires that depends built that id and that CMake took the toolchain, libevent and Boost at boost.mk's version from it. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .github/workflows/ci-pq.yml | 240 +- .github/workflows/ci.yml | 45 +- .github/workflows/core-checks.yml | 32 + .github/workflows/required-merge-gate.yml | 12 +- CMakeLists.txt | 7 + ci/checks/classify_merge_profile.py | 63 + ci/checks/test_ci_pq_contract.py | 467 ++- ci/checks/test_classify_merge_profile.py | 76 + ci/checks/test_mlkem_build_policy.py | 750 ++++ ci/checks/test_pq_merge_gate_contract.py | 327 ++ ci/test/00_setup_env_mac_native_pq.sh | 31 + ci/test/00_setup_env_native_aarch64_pq.sh | 2 +- ci/test/00_setup_env_s390x_unit.sh | 31 + ci/test/03_test_script.sh | 12 +- cmake/bitcoin-build-config.h.in | 7 + cmake/introspection.cmake | 12 + cmake/mlkem-native.cmake | 246 ++ contrib/devtools/mlkem-native.pin | 8 + contrib/devtools/update-mlkem-native.sh | 158 + doc/development/fuzzing.md | 13 +- doc/subtrees/mlkem-native.md | 332 ++ src/CMakeLists.txt | 3 + src/bench/CMakeLists.txt | 1 + src/bench/mlkem.cpp | 107 + src/compat/cpu_features.cpp | 53 + src/compat/cpu_features.h | 83 + src/compat/cpuid.h | 8 + src/crypto/mlkem.cpp | 195 + src/crypto/mlkem.h | 155 + src/crypto/mlkem_arith_backend.h | 29 + src/crypto/mlkem_backend.c | 29 + src/crypto/mlkem_config.h | 89 + src/crypto/mlkem_fips202_backend.h | 55 + src/crypto/mlkem_shim.h | 44 + src/crypto/sha256.cpp | 30 +- src/crypto/sha256.h | 12 + src/mlkem-native/LICENSE | 312 ++ src/mlkem-native/mlkem/README.md | 23 + src/mlkem-native/mlkem/mlkem_native.c | 692 ++++ src/mlkem-native/mlkem/mlkem_native.h | 464 +++ src/mlkem-native/mlkem/mlkem_native_asm.S | 716 ++++ src/mlkem-native/mlkem/mlkem_native_config.h | 683 ++++ src/mlkem-native/mlkem/src/cbmc.h | 222 ++ src/mlkem-native/mlkem/src/common.h | 296 ++ src/mlkem-native/mlkem/src/compress.c | 763 ++++ src/mlkem-native/mlkem/src/compress.h | 613 ++++ src/mlkem-native/mlkem/src/context.h | 51 + src/mlkem-native/mlkem/src/debug.c | 64 + src/mlkem-native/mlkem/src/debug.h | 121 + src/mlkem-native/mlkem/src/fips202/fips202.c | 249 ++ src/mlkem-native/mlkem/src/fips202/fips202.h | 144 + .../mlkem/src/fips202/fips202x4.c | 207 ++ .../mlkem/src/fips202/fips202x4.h | 81 + .../mlkem/src/fips202/keccakf1600.c | 499 +++ .../mlkem/src/fips202/keccakf1600.h | 98 + .../mlkem/src/fips202/native/aarch64/auto.h | 78 + .../aarch64/src/fips202_native_aarch64.h | 80 + .../src/keccak_f1600_x1_scalar_aarch64_asm.S | 377 ++ .../src/keccak_f1600_x1_v84a_aarch64_asm.S | 206 ++ .../src/keccak_f1600_x2_v84a_aarch64_asm.S | 261 ++ ...k_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S | 1079 ++++++ ...00_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S | 989 +++++ .../aarch64/src/keccakf1600_round_constants.c | 47 + .../src/fips202/native/aarch64/x1_scalar.h | 26 + .../src/fips202/native/aarch64/x1_v84a.h | 35 + .../src/fips202/native/aarch64/x2_v84a.h | 38 + .../fips202/native/aarch64/x4_v8a_scalar.h | 31 + .../native/aarch64/x4_v8a_v84a_scalar.h | 36 + .../mlkem/src/fips202/native/api.h | 117 + .../src/fips202/native/armv81m/README.md | 10 + .../mlkem/src/fips202/native/armv81m/mve.h | 65 + .../armv81m/src/fips202_native_armv81m.h | 36 + .../native/armv81m/src/keccak_f1600_x4_mve.S | 716 ++++ .../native/armv81m/src/keccak_f1600_x4_mve.c | 40 + .../keccak_f1600_x4_state_extract_bytes_mve.S | 334 ++ .../src/keccak_f1600_x4_state_xor_bytes_mve.S | 355 ++ .../armv81m/src/keccakf1600_round_constants.c | 52 + .../mlkem/src/fips202/native/auto.h | 29 + .../native/x86_64/keccak_f1600_x4_avx2.h | 33 + .../native/x86_64/src/fips202_native_x86_64.h | 44 + .../x86_64/src/keccak_f1600_x4_avx2_asm.S | 487 +++ .../native/x86_64/src/keccakf1600_constants.c | 52 + src/mlkem-native/mlkem/src/indcpa.c | 675 ++++ src/mlkem-native/mlkem/src/indcpa.h | 172 + src/mlkem-native/mlkem/src/kem.c | 474 +++ src/mlkem-native/mlkem/src/kem.h | 353 ++ .../mlkem/src/native/aarch64/README.md | 16 + .../mlkem/src/native/aarch64/meta.h | 166 + .../src/native/aarch64/src/aarch64_zetas.c | 184 + .../native/aarch64/src/arith_native_aarch64.h | 184 + .../aarch64/src/mlkem_intt_aarch64_asm.S | 635 ++++ .../aarch64/src/mlkem_ntt_aarch64_asm.S | 565 +++ .../mlkem_poly_mulcache_compute_aarch64_asm.S | 130 + .../src/mlkem_poly_reduce_aarch64_asm.S | 153 + .../src/mlkem_poly_tobytes_aarch64_asm.S | 124 + .../src/mlkem_poly_tomont_aarch64_asm.S | 102 + ...mul_acc_montgomery_cached_k2_aarch64_asm.S | 264 ++ ...mul_acc_montgomery_cached_k3_aarch64_asm.S | 317 ++ ...mul_acc_montgomery_cached_k4_aarch64_asm.S | 371 ++ .../src/mlkem_rej_uniform_aarch64_asm.S | 226 ++ .../native/aarch64/src/rej_uniform_table.c | 543 +++ src/mlkem-native/mlkem/src/native/api.h | 651 ++++ src/mlkem-native/mlkem/src/native/meta.h | 30 + .../mlkem/src/native/ppc64le/README.md | 6 + .../mlkem/src/native/ppc64le/meta.h | 81 + .../native/ppc64le/src/arith_native_ppc64le.h | 24 + .../mlkem/src/native/ppc64le/src/consts.c | 299 ++ .../mlkem/src/native/ppc64le/src/consts.h | 34 + .../native/ppc64le/src/mlkem_intt_ppc_asm.S | 3246 +++++++++++++++++ .../native/ppc64le/src/mlkem_ntt_ppc_asm.S | 1672 +++++++++ .../ppc64le/src/mlkem_poly_tomont_ppc_asm.S | 316 ++ .../native/ppc64le/src/mlkem_reduce_ppc_asm.S | 731 ++++ .../mlkem/src/native/riscv64/README.md | 11 + .../mlkem/src/native/riscv64/meta.h | 137 + .../native/riscv64/src/arith_native_riscv64.h | 49 + .../src/native/riscv64/src/rv64v_debug.c | 83 + .../src/native/riscv64/src/rv64v_debug.h | 140 + .../src/native/riscv64/src/rv64v_izetas.inc | 27 + .../mlkem/src/native/riscv64/src/rv64v_poly.c | 797 ++++ .../src/native/riscv64/src/rv64v_zetas.inc | 27 + .../riscv64/src/rv64v_zetas_basemul.inc | 39 + .../mlkem/src/native/x86_64/README.md | 4 + .../mlkem/src/native/x86_64/meta.h | 324 ++ .../native/x86_64/src/arith_native_x86_64.h | 327 ++ .../src/native/x86_64/src/compress_consts.c | 115 + .../src/native/x86_64/src/compress_consts.h | 53 + .../mlkem/src/native/x86_64/src/consts.c | 102 + .../mlkem/src/native/x86_64/src/consts.h | 25 + .../native/x86_64/src/mlkem_intt_avx2_asm.S | 743 ++++ .../native/x86_64/src/mlkem_ntt_avx2_asm.S | 661 ++++ .../x86_64/src/mlkem_nttfrombytes_avx2_asm.S | 217 ++ .../x86_64/src/mlkem_ntttobytes_avx2_asm.S | 205 ++ .../x86_64/src/mlkem_nttunpack_avx2_asm.S | 190 + .../src/mlkem_poly_compress_d10_avx2_asm.S | 413 +++ .../src/mlkem_poly_compress_d11_avx2_asm.S | 479 +++ .../src/mlkem_poly_compress_d4_avx2_asm.S | 194 + .../src/mlkem_poly_compress_d5_avx2_asm.S | 251 ++ .../src/mlkem_poly_decompress_d10_avx2_asm.S | 257 ++ .../src/mlkem_poly_decompress_d11_avx2_asm.S | 307 ++ .../src/mlkem_poly_decompress_d4_avx2_asm.S | 209 ++ .../src/mlkem_poly_decompress_d5_avx2_asm.S | 222 ++ .../mlkem_poly_mulcache_compute_avx2_asm.S | 118 + ...asemul_acc_montgomery_cached_k2_avx2_asm.S | 536 +++ ...asemul_acc_montgomery_cached_k3_avx2_asm.S | 784 ++++ ...asemul_acc_montgomery_cached_k4_avx2_asm.S | 1032 ++++++ .../native/x86_64/src/mlkem_reduce_avx2_asm.S | 234 ++ .../x86_64/src/mlkem_rej_uniform_avx2_asm.S | 136 + .../native/x86_64/src/mlkem_tomont_avx2_asm.S | 172 + .../src/native/x86_64/src/rej_uniform_table.c | 545 +++ src/mlkem-native/mlkem/src/params.h | 76 + src/mlkem-native/mlkem/src/poly.c | 581 +++ src/mlkem-native/mlkem/src/poly.h | 296 ++ src/mlkem-native/mlkem/src/poly_k.c | 522 +++ src/mlkem-native/mlkem/src/poly_k.h | 622 ++++ src/mlkem-native/mlkem/src/randombytes.h | 52 + src/mlkem-native/mlkem/src/sampling.c | 371 ++ src/mlkem-native/mlkem/src/sampling.h | 111 + src/mlkem-native/mlkem/src/symmetric.h | 70 + src/mlkem-native/mlkem/src/sys.h | 320 ++ src/mlkem-native/mlkem/src/verify.c | 20 + src/mlkem-native/mlkem/src/verify.h | 447 +++ src/mlkem-native/mlkem/src/zetas.inc | 30 + src/test/CMakeLists.txt | 3 + src/test/cpu_features_tests.cpp | 243 ++ src/test/data/mlkem1024_vectors.json | 144 + src/test/fuzz/CMakeLists.txt | 1 + src/test/fuzz/mlkem.cpp | 244 ++ src/test/mlkem_tests.cpp | 716 ++++ src/util/CMakeLists.txt | 1 + test/fuzz/qbit_corpora/MANIFEST.json | 130 + test/fuzz/qbit_corpora/README.md | 14 +- test/fuzz/qbit_corpora/generate_seeds.py | 95 + .../mlkem/acvp-modified-ciphertext | Bin 0 -> 6400 bytes .../qbit_corpora/mlkem/acvp-modified-key-hash | Bin 0 -> 6400 bytes .../mlkem/acvp-valid-decapsulation | Bin 0 -> 6400 bytes test/fuzz/qbit_corpora/mlkem/cctv-base-key | Bin 0 -> 1664 bytes test/fuzz/qbit_corpora/mlkem/coefficient-3328 | Bin 0 -> 1664 bytes test/fuzz/qbit_corpora/mlkem/coefficient-3329 | Bin 0 -> 1664 bytes .../mlkem/honest-ciphertext-bit-flip | Bin 0 -> 6403 bytes .../fuzz/qbit_corpora/mlkem/honest-round-trip | Bin 0 -> 96 bytes .../qbit_corpora/mlkem/last-coefficient-4095 | Bin 0 -> 1664 bytes .../acvp-modified-ciphertext | Bin 0 -> 6400 bytes .../mlkem_backend_diff/acvp-modified-key-hash | Bin 0 -> 6400 bytes .../acvp-valid-decapsulation | Bin 0 -> 6400 bytes .../mlkem_backend_diff/cctv-base-key | Bin 0 -> 1664 bytes .../mlkem_backend_diff/coefficient-3328 | Bin 0 -> 1664 bytes .../mlkem_backend_diff/coefficient-3329 | Bin 0 -> 1664 bytes .../honest-ciphertext-bit-flip | Bin 0 -> 6403 bytes .../mlkem_backend_diff/honest-round-trip | Bin 0 -> 96 bytes .../mlkem_backend_diff/last-coefficient-4095 | Bin 0 -> 1664 bytes test/fuzz/qbit_corpora/overlay.py | 2 + test/fuzz/test_qbit_fuzz_runner.py | 2 +- test/fuzz/test_runner.py | 2 + test/lint/README.md | 16 + test/lint/lint-locale-dependence.py | 1 + test/lint/lint_ignore_dirs.py | 1 + test/lint/mlkem-native-check.sh | 128 + test/lint/test_runner/src/main.rs | 8 + test/sanitizer_suppressions/ubsan | 5 + 199 files changed, 43846 insertions(+), 52 deletions(-) create mode 100755 ci/checks/test_mlkem_build_policy.py create mode 100755 ci/checks/test_pq_merge_gate_contract.py create mode 100755 ci/test/00_setup_env_mac_native_pq.sh create mode 100755 ci/test/00_setup_env_s390x_unit.sh create mode 100644 cmake/mlkem-native.cmake create mode 100644 contrib/devtools/mlkem-native.pin create mode 100755 contrib/devtools/update-mlkem-native.sh create mode 100644 doc/subtrees/mlkem-native.md create mode 100644 src/bench/mlkem.cpp create mode 100644 src/compat/cpu_features.cpp create mode 100644 src/compat/cpu_features.h create mode 100644 src/crypto/mlkem.cpp create mode 100644 src/crypto/mlkem.h create mode 100644 src/crypto/mlkem_arith_backend.h create mode 100644 src/crypto/mlkem_backend.c create mode 100644 src/crypto/mlkem_config.h create mode 100644 src/crypto/mlkem_fips202_backend.h create mode 100644 src/crypto/mlkem_shim.h create mode 100644 src/mlkem-native/LICENSE create mode 100644 src/mlkem-native/mlkem/README.md create mode 100644 src/mlkem-native/mlkem/mlkem_native.c create mode 100644 src/mlkem-native/mlkem/mlkem_native.h create mode 100644 src/mlkem-native/mlkem/mlkem_native_asm.S create mode 100644 src/mlkem-native/mlkem/mlkem_native_config.h create mode 100644 src/mlkem-native/mlkem/src/cbmc.h create mode 100644 src/mlkem-native/mlkem/src/common.h create mode 100644 src/mlkem-native/mlkem/src/compress.c create mode 100644 src/mlkem-native/mlkem/src/compress.h create mode 100644 src/mlkem-native/mlkem/src/context.h create mode 100644 src/mlkem-native/mlkem/src/debug.c create mode 100644 src/mlkem-native/mlkem/src/debug.h create mode 100644 src/mlkem-native/mlkem/src/fips202/fips202.c create mode 100644 src/mlkem-native/mlkem/src/fips202/fips202.h create mode 100644 src/mlkem-native/mlkem/src/fips202/fips202x4.c create mode 100644 src/mlkem-native/mlkem/src/fips202/fips202x4.h create mode 100644 src/mlkem-native/mlkem/src/fips202/keccakf1600.c create mode 100644 src/mlkem-native/mlkem/src/fips202/keccakf1600.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/auto.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/src/fips202_native_aarch64.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x1_scalar_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccakf1600_round_constants.c create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/x1_scalar.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/x1_v84a.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/x2_v84a.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/x4_v8a_scalar.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/x4_v8a_v84a_scalar.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/api.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/armv81m/README.md create mode 100644 src/mlkem-native/mlkem/src/fips202/native/armv81m/mve.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/armv81m/src/fips202_native_armv81m.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.S create mode 100644 src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.c create mode 100644 src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_state_extract_bytes_mve.S create mode 100644 src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_state_xor_bytes_mve.S create mode 100644 src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccakf1600_round_constants.c create mode 100644 src/mlkem-native/mlkem/src/fips202/native/auto.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/x86_64/keccak_f1600_x4_avx2.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/x86_64/src/fips202_native_x86_64.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/x86_64/src/keccak_f1600_x4_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/fips202/native/x86_64/src/keccakf1600_constants.c create mode 100644 src/mlkem-native/mlkem/src/indcpa.c create mode 100644 src/mlkem-native/mlkem/src/indcpa.h create mode 100644 src/mlkem-native/mlkem/src/kem.c create mode 100644 src/mlkem-native/mlkem/src/kem.h create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/README.md create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/meta.h create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/aarch64_zetas.c create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/arith_native_aarch64.h create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_intt_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_ntt_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_mulcache_compute_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_reduce_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_tobytes_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_tomont_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_rej_uniform_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/rej_uniform_table.c create mode 100644 src/mlkem-native/mlkem/src/native/api.h create mode 100644 src/mlkem-native/mlkem/src/native/meta.h create mode 100644 src/mlkem-native/mlkem/src/native/ppc64le/README.md create mode 100644 src/mlkem-native/mlkem/src/native/ppc64le/meta.h create mode 100644 src/mlkem-native/mlkem/src/native/ppc64le/src/arith_native_ppc64le.h create mode 100644 src/mlkem-native/mlkem/src/native/ppc64le/src/consts.c create mode 100644 src/mlkem-native/mlkem/src/native/ppc64le/src/consts.h create mode 100644 src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_intt_ppc_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_ntt_ppc_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_poly_tomont_ppc_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_reduce_ppc_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/riscv64/README.md create mode 100644 src/mlkem-native/mlkem/src/native/riscv64/meta.h create mode 100644 src/mlkem-native/mlkem/src/native/riscv64/src/arith_native_riscv64.h create mode 100644 src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_debug.c create mode 100644 src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_debug.h create mode 100644 src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_izetas.inc create mode 100644 src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_poly.c create mode 100644 src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_zetas.inc create mode 100644 src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_zetas_basemul.inc create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/README.md create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/meta.h create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/arith_native_x86_64.h create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/compress_consts.c create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/compress_consts.h create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/consts.c create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/consts.h create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_intt_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_ntt_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_nttfrombytes_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_ntttobytes_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_nttunpack_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d10_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d11_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d4_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d5_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d10_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d11_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d4_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d5_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_mulcache_compute_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_reduce_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_rej_uniform_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_tomont_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/rej_uniform_table.c create mode 100644 src/mlkem-native/mlkem/src/params.h create mode 100644 src/mlkem-native/mlkem/src/poly.c create mode 100644 src/mlkem-native/mlkem/src/poly.h create mode 100644 src/mlkem-native/mlkem/src/poly_k.c create mode 100644 src/mlkem-native/mlkem/src/poly_k.h create mode 100644 src/mlkem-native/mlkem/src/randombytes.h create mode 100644 src/mlkem-native/mlkem/src/sampling.c create mode 100644 src/mlkem-native/mlkem/src/sampling.h create mode 100644 src/mlkem-native/mlkem/src/symmetric.h create mode 100644 src/mlkem-native/mlkem/src/sys.h create mode 100644 src/mlkem-native/mlkem/src/verify.c create mode 100644 src/mlkem-native/mlkem/src/verify.h create mode 100644 src/mlkem-native/mlkem/src/zetas.inc create mode 100644 src/test/cpu_features_tests.cpp create mode 100644 src/test/data/mlkem1024_vectors.json create mode 100644 src/test/fuzz/mlkem.cpp create mode 100644 src/test/mlkem_tests.cpp create mode 100644 test/fuzz/qbit_corpora/mlkem/acvp-modified-ciphertext create mode 100644 test/fuzz/qbit_corpora/mlkem/acvp-modified-key-hash create mode 100644 test/fuzz/qbit_corpora/mlkem/acvp-valid-decapsulation create mode 100644 test/fuzz/qbit_corpora/mlkem/cctv-base-key create mode 100644 test/fuzz/qbit_corpora/mlkem/coefficient-3328 create mode 100644 test/fuzz/qbit_corpora/mlkem/coefficient-3329 create mode 100644 test/fuzz/qbit_corpora/mlkem/honest-ciphertext-bit-flip create mode 100644 test/fuzz/qbit_corpora/mlkem/honest-round-trip create mode 100644 test/fuzz/qbit_corpora/mlkem/last-coefficient-4095 create mode 100644 test/fuzz/qbit_corpora/mlkem_backend_diff/acvp-modified-ciphertext create mode 100644 test/fuzz/qbit_corpora/mlkem_backend_diff/acvp-modified-key-hash create mode 100644 test/fuzz/qbit_corpora/mlkem_backend_diff/acvp-valid-decapsulation create mode 100644 test/fuzz/qbit_corpora/mlkem_backend_diff/cctv-base-key create mode 100644 test/fuzz/qbit_corpora/mlkem_backend_diff/coefficient-3328 create mode 100644 test/fuzz/qbit_corpora/mlkem_backend_diff/coefficient-3329 create mode 100644 test/fuzz/qbit_corpora/mlkem_backend_diff/honest-ciphertext-bit-flip create mode 100644 test/fuzz/qbit_corpora/mlkem_backend_diff/honest-round-trip create mode 100644 test/fuzz/qbit_corpora/mlkem_backend_diff/last-coefficient-4095 create mode 100755 test/lint/mlkem-native-check.sh diff --git a/.github/workflows/ci-pq.yml b/.github/workflows/ci-pq.yml index c252804291..aaa17e9524 100644 --- a/.github/workflows/ci-pq.yml +++ b/.github/workflows/ci-pq.yml @@ -9,9 +9,14 @@ # - on-demand runs go through main's PQ caller (C4, part 2 of #184), which takes # workflow_dispatch inputs source_ref and jobs and calls ci-pq.yml@1.x.x, for # example gh workflow run ci-maintained-pq.yml -f source_ref= -f jobs=aarch64-unit. -# That caller lands on main after this part merges. +# That caller lands on main after this part merges; +# - Full Validation (ci.yml) calls it with aarch64-unit, macos-arm64-unit and +# macos-x86_64-unit for every change that ci/checks/classify_merge_profile.py +# says needs them. # The workflow_dispatch trigger below stays for when this file is on main itself; # until then GitHub rejects gh workflow run ci-pq.yml. +# Jobs: aarch64-unit, s390x-unit (big-endian, emulated), macos-arm64-unit and +# macos-x86_64-unit. # The source is resolved once to a commit, every job checks out and verifies # that commit, and the summaries record it. @@ -27,7 +32,7 @@ on: type: string default: '' jobs: - description: 'Comma-separated jobs to run (aarch64-unit); empty runs all' + description: 'Comma-separated jobs to run (aarch64-unit, s390x-unit, macos-arm64-unit, macos-x86_64-unit); empty runs all' required: false type: string default: '' @@ -39,7 +44,7 @@ on: type: string default: '' jobs: - description: 'Comma-separated jobs to run (aarch64-unit); empty runs all' + description: 'Comma-separated jobs to run (aarch64-unit, s390x-unit, macos-arm64-unit, macos-x86_64-unit); empty runs all' required: false type: string default: '' @@ -59,6 +64,8 @@ jobs: matrix: ${{ steps.select.outputs.matrix }} matrix_count: ${{ steps.select.outputs.matrix_count }} selected: ${{ steps.select.outputs.selected }} + run_macos_arm64_unit: ${{ steps.select.outputs.run_macos_arm64_unit }} + run_macos_x86_64_unit: ${{ steps.select.outputs.run_macos_x86_64_unit }} steps: - name: Resolve source id: resolve @@ -167,15 +174,23 @@ jobs: [ { "job": "aarch64-unit", - "name": "AArch64 unit tests (pre-library phase)", + "name": "AArch64 unit tests", "runs-on": "ubuntu-24.04-arm", "timeout-minutes": 120, "file-env": "./ci/test/00_setup_env_native_aarch64_pq.sh", - "phase": "Pre-library phase: bip324_tests and net_tests on AArch64. Part 3 (#190) adds mlkem_tests with the ML-KEM library." + "phase": "mlkem_tests, bip324_tests and net_tests on AArch64 Linux. ML-KEM runs portable C here: its AArch64 assembly has no BTI landing pads." + }, + { + "job": "s390x-unit", + "name": "s390x big-endian unit tests (emulated)", + "runs-on": "ubuntu-24.04", + "timeout-minutes": 180, + "file-env": "./ci/test/00_setup_env_s390x_unit.sh", + "phase": "mlkem_tests, bip324_tests and net_tests cross-built for big-endian s390x and run under qemu-s390x. The timeout stays 180 minutes until runs are measured, then becomes twice the measured time." } ] # Selectable jobs that are not matrix entries; each gets a run_ output. - NON_MATRIX_JOBS: '' + NON_MATRIX_JOBS: 'macos-arm64-unit macos-x86_64-unit' run: | # Turn the comma-separated jobs input into the matrix: empty selects # every job; an empty, unknown or repeated name fails the run. @@ -233,6 +248,8 @@ jobs: CI_FAILFAST_TEST_LEAVE_DANGLING: 1 DANGER_CI_ON_HOST_FOLDERS: 1 FILE_ENV: ${{ matrix.file-env }} + # Read by the steps the macOS jobs share, so it is set per job. + MATRIX_NAME: ${{ matrix.name }} TEST_RUNNER_FAILFAST: 'false' MAKEJOBS: '-j4' TEST_RUNNER_JOBS: '4' @@ -245,7 +262,8 @@ jobs: CI_BUILDKIT_DNS_NAMESERVERS: '1.1.1.1,8.8.8.8' CI_BUILDKIT_DNS_SEARCH_DOMAINS: '' steps: - - name: Require resolved source + - &REQUIRE_SOURCE + name: Require resolved source shell: bash env: RESOLVED_SHA: ${{ needs.resolve-source.outputs.resolved_sha }} @@ -258,20 +276,21 @@ jobs: || { echo "::error::resolved source '$RESOLVED_SHA' is not a full commit hash" >&2; exit 1; } [ -n "$REQUESTED_REF" ] || { echo "::error::requested ref is empty" >&2; exit 1; } - - name: Checkout + - &CHECKOUT_SOURCE + name: Checkout uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 with: ref: ${{ needs.resolve-source.outputs.resolved_sha }} fetch-depth: 0 - - name: Verify checked-out source + - &VERIFY_SOURCE + name: Verify checked-out source shell: bash env: SOURCE_MODE: ${{ needs.resolve-source.outputs.source_mode }} REQUESTED_REF: ${{ needs.resolve-source.outputs.requested_ref }} RESOLVED_SHA: ${{ needs.resolve-source.outputs.resolved_sha }} WORKFLOW_SHA: ${{ github.workflow_sha }} - MATRIX_NAME: ${{ matrix.name }} run: | set -euo pipefail actual_head="$(git rev-parse --verify 'HEAD^{commit}')" @@ -301,7 +320,8 @@ jobs: with: cache-provider: gha - - name: CI script + - &CI_SCRIPT + name: CI script run: ./ci/test_run_all.sh - name: Report unit-test evidence @@ -310,9 +330,8 @@ jobs: env: RESOLVED_SHA: ${{ needs.resolve-source.outputs.resolved_sha }} REQUESTED_REF: ${{ needs.resolve-source.outputs.requested_ref }} - MATRIX_NAME: ${{ matrix.name }} PHASE: ${{ matrix.phase }} - run: | + run: &REPORT_EVIDENCE | set -euo pipefail # ci/test/03_test_script.sh writes this after ctest_evidence.py has # checked that every expected suite was selected, executed and passed. @@ -333,3 +352,198 @@ jobs: - name: Save caches uses: ./.github/actions/save-caches + + # macOS runs natively on the host, without Docker, so each architecture is a + # job of its own rather than a matrix entry of unit. Both share unit's source + # checks, CI script and evidence report, and build with WITH_MLKEM_NATIVE=ON: + # these are the only jobs that run ML-KEM's native code on macOS. + macos-arm64-unit: + name: macOS arm64 unit tests + needs: resolve-source + if: ${{ needs.resolve-source.outputs.run_macos_arm64_unit == 'true' }} + runs-on: macos-15 + timeout-minutes: 120 + env: + DANGER_RUN_CI_ON_HOST: 1 + FILE_ENV: ./ci/test/00_setup_env_mac_native_pq.sh + MATRIX_NAME: macOS arm64 unit tests + BASE_ROOT_DIR: ${{ github.workspace }} + TEST_RUNNER_FAILFAST: 'false' + MAKEJOBS: '-j3' + TEST_RUNNER_JOBS: '3' + CTEST_JOBS: '3' + # Read by the steps the macOS jobs share: the runner's uname -m, the + # ML-KEM backend (arithmetic/Keccak) that must run, and the evidence line. + EXPECTED_MACHINE: arm64 + MLKEM_EXPECTED_BACKEND: aarch64-neon/aarch64 + PHASE: mlkem_tests, bip324_tests and net_tests natively on Apple Silicon, where ML-KEM runs its native AArch64 code. + steps: + - *REQUIRE_SOURCE + - *CHECKOUT_SOURCE + - *VERIFY_SOURCE + + - &REQUIRE_MACHINE + name: Require runner architecture + shell: bash + run: | + set -euo pipefail + machine="$(uname -m)" + if [ "$machine" != "$EXPECTED_MACHINE" ]; then + echo "::error::${GITHUB_JOB} must run on ${EXPECTED_MACHINE}, but uname -m reports '$machine'" >&2 + exit 1 + fi + echo "uname -m: $machine" + + # The runner context is not available in job-level env, so the build + # directory, which the later steps read, is set here. + - &SET_BUILD_DIR + name: Set build directory + shell: bash + run: echo "BASE_BUILD_DIR=${RUNNER_TEMP}/build" >> "$GITHUB_ENV" + + # Boost and libevent come from depends (see FILE_ENV), not Homebrew. + # depends needs GNU Make 4, which Homebrew installs as gmake; its gnubin + # directory makes it the make that 03_test_script.sh runs. + - &INSTALL_HOMEBREW_PACKAGES + name: Install Homebrew packages + env: + HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 + run: | + brew install --quiet coreutils ninja pkgconf gnu-getopt ccache make + echo "$(brew --prefix make)/libexec/gnubin" >> "$GITHUB_PATH" + + # 03_test_script.sh builds depends for HOST. Set HOST once here, and ask + # depends for the id of that build (its packages, recipes and compilers), + # so the cache key names exactly what the CI script builds. + - &IDENTIFY_DEPENDS + name: Identify depends build + shell: bash + run: | + set -euo pipefail + make_version="$(make --version | head -n 1)" + case "$make_version" in + "GNU Make "[4-9].*) echo "make: $make_version" ;; + *) echo "::error::depends needs GNU Make 4 or later, but make is '$make_version'" >&2; exit 1 ;; + esac + host="$("${BASE_ROOT_DIR}/depends/config.guess")" + # shellcheck source=/dev/null + source "$FILE_ENV" + # As in 03_test_script.sh, bash -c splits DEP_OPTS into make arguments. + id="$(bash -c "make -s -C '${BASE_ROOT_DIR}/depends' HOST=$host $DEP_OPTS print-final_build_id")" + id="${id#final_build_id=}" + printf '%s\n' "$id" | LC_ALL=C grep -Eq '^[0-9a-f]+$' \ + || { echo "::error::depends gave no build id for $host: '$id'" >&2; exit 1; } + echo "depends build $id for $host" + { + echo "HOST=$host" + echo "DEPENDS_BUILD_ID=$id" + } >> "$GITHUB_ENV" + + # Only the built packages are cached; depends rebuilds any whose id + # does not match. + - &RESTORE_DEPENDS + name: Restore depends cache + id: depends-cache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: depends/built + key: depends-${{ env.HOST }}-${{ env.DEPENDS_BUILD_ID }} + + - *CI_SCRIPT + + # Saved even when the suites fail, but only once depends wrote the + # toolchain file, which needs every package built. + - &SAVE_DEPENDS + name: Save depends cache + if: ${{ !cancelled() && steps.depends-cache.outputs.cache-hit != 'true' && hashFiles(format('depends/{0}/toolchain.cmake', env.HOST)) != '' }} + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: depends/built + key: ${{ steps.depends-cache.outputs.cache-primary-key }} + + # Which libraries did the suites run against? Require the depends build + # the cache key named, its toolchain, its libevent, and Boost at the + # version depends/packages/boost.mk pins. + - &REQUIRE_DEPENDS_BUILD + name: Require depends build + shell: bash + run: | + set -euo pipefail + fail() { echo "::error::$*" >&2; exit 1; } + prefix="${BASE_ROOT_DIR}/depends/${HOST}" + [ -f "${prefix}/.stamp_${DEPENDS_BUILD_ID}" ] \ + || fail "depends did not build ${DEPENDS_BUILD_ID} for ${HOST}, the build the cache key names" + boost_version="$(sed -n 's/^[$](package)_version *= *//p' "${BASE_ROOT_DIR}/depends/packages/boost.mk")" + [ -n "$boost_version" ] || fail "no Boost version in depends/packages/boost.mk" + # Compares the value of a CMakeCache.txt entry, whatever its type. + require_cache_entry() { + local actual + actual="$(sed -n "s/^$1:[A-Z]*=//p" "${BASE_BUILD_DIR}/CMakeCache.txt")" + [ "$actual" = "$2" ] || fail "$1 must be $2, but the build has: ${actual:-$1 unset}" + } + require_cache_entry CMAKE_TOOLCHAIN_FILE "${prefix}/toolchain.cmake" + require_cache_entry Boost_DIR "${prefix}/lib/cmake/Boost-${boost_version}" + require_cache_entry Libevent_DIR "${prefix}/lib/cmake/libevent" + echo "Built with Boost ${boost_version} and libevent from ${prefix}" + + # The suites passed, but on which code? A runner whose CPU lacks an + # extension the native code needs would pass them on portable C, so + # require the native backend by name. mlkem_tests/backend_selection logs + # the compiled and the active backend. + - &REQUIRE_NATIVE_MLKEM + name: Require native ML-KEM + shell: bash + run: | + set -euo pipefail + expected="compiled ${MLKEM_EXPECTED_BACKEND}, active ${MLKEM_EXPECTED_BACKEND}" + log="$("${BASE_BUILD_DIR}/bin/test_qbit" --run_test=mlkem_tests/backend_selection --log_level=message --color_output=no 2>&1)" \ + || { printf '%s\n' "$log"; echo "::error::mlkem_tests/backend_selection failed" >&2; exit 1; } + printf '%s\n' "$log" + if ! printf '%s\n' "$log" | grep -qxF "$expected"; then + echo "::error::${GITHUB_JOB} must run native ML-KEM, but the test log lacks '${expected}'" >&2 + sysctl machdep.cpu.brand_string machdep.cpu.features machdep.cpu.leaf7_features 2>/dev/null || true + exit 1 + fi + + - &MACOS_REPORT_EVIDENCE + name: Report unit-test evidence + if: ${{ always() }} + shell: bash + env: + RESOLVED_SHA: ${{ needs.resolve-source.outputs.resolved_sha }} + REQUESTED_REF: ${{ needs.resolve-source.outputs.requested_ref }} + run: *REPORT_EVIDENCE + + macos-x86_64-unit: + name: macOS x86_64 unit tests + needs: resolve-source + if: ${{ needs.resolve-source.outputs.run_macos_x86_64_unit == 'true' }} + # GitHub's Intel macOS image (4 CPUs); macos-15 itself is Apple Silicon. + runs-on: macos-15-intel + timeout-minutes: 120 + env: + DANGER_RUN_CI_ON_HOST: 1 + FILE_ENV: ./ci/test/00_setup_env_mac_native_pq.sh + MATRIX_NAME: macOS x86_64 unit tests + BASE_ROOT_DIR: ${{ github.workspace }} + TEST_RUNNER_FAILFAST: 'false' + MAKEJOBS: '-j4' + TEST_RUNNER_JOBS: '4' + CTEST_JOBS: '4' + EXPECTED_MACHINE: x86_64 + MLKEM_EXPECTED_BACKEND: x86_64-avx2/x86_64-avx2 + PHASE: mlkem_tests, bip324_tests and net_tests natively on an Intel Mac, where ML-KEM runs its native x86_64 code. + steps: + - *REQUIRE_SOURCE + - *CHECKOUT_SOURCE + - *VERIFY_SOURCE + - *REQUIRE_MACHINE + - *SET_BUILD_DIR + - *INSTALL_HOMEBREW_PACKAGES + - *IDENTIFY_DEPENDS + - *RESTORE_DEPENDS + - *CI_SCRIPT + - *SAVE_DEPENDS + - *REQUIRE_DEPENDS_BUILD + - *REQUIRE_NATIVE_MLKEM + - *MACOS_REPORT_EVIDENCE diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4db0ffa7b..0f439db648 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,6 +71,7 @@ jobs: profile: ${{ steps.classify.outputs.profile }} release_policy_only: ${{ steps.classify.outputs.release_policy_only }} source_validation_required: ${{ steps.classify.outputs.source_validation_required }} + pq_unit_required: ${{ steps.classify.outputs.pq_unit_required }} steps: - name: Checkout uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 @@ -96,7 +97,7 @@ jobs: echo "::error::Pull request base SHA is unavailable." exit 1 fi - git diff --name-only "${BASE_SHA}...HEAD" > "${changed_files}" + git diff --name-only --no-renames "${BASE_SHA}...HEAD" > "${changed_files}" elif [[ "${EVENT_NAME}" == "merge_group" ]]; then base_branch="${BASE_REF#refs/heads/}" if [[ -z "${base_branch}" ]]; then @@ -104,15 +105,15 @@ jobs: exit 1 fi git fetch --no-tags --prune origin "+refs/heads/${base_branch}:refs/remotes/origin/${base_branch}" - git diff --name-only "origin/${base_branch}...HEAD" > "${changed_files}" + git diff --name-only --no-renames "origin/${base_branch}...HEAD" > "${changed_files}" elif [[ "${EVENT_NAME}" == "push" ]]; then if [[ -n "${BEFORE_SHA}" && ! "${BEFORE_SHA}" =~ ^0+$ ]]; then - git diff --name-only "${BEFORE_SHA}..HEAD" > "${changed_files}" + git diff --name-only --no-renames "${BEFORE_SHA}..HEAD" > "${changed_files}" else git fetch --no-tags --prune origin "${DEFAULT_BRANCH}" merge_base="$(git merge-base "origin/${DEFAULT_BRANCH}" HEAD 2>/dev/null || true)" if [[ -n "${merge_base}" ]]; then - git diff --name-only "${merge_base}...HEAD" > "${changed_files}" + git diff --name-only --no-renames "${merge_base}...HEAD" > "${changed_files}" else git diff-tree --no-commit-id --name-only -r HEAD > "${changed_files}" fi @@ -1071,6 +1072,20 @@ jobs: path: | build/doc/rpc/** + # Changes to the ML-KEM library, the v2 transport, or their build and CI + # wiring (ci/checks/classify_merge_profile.py) must pass these jobs of the PQ + # workflow on the change itself (for a pull request, its merge commit), on + # GitHub-hosted runners: AArch64 Linux, and macOS on Apple Silicon and on + # Intel, the only jobs that build and run ML-KEM's native code on macOS. + pq-unit: + name: 'PQ unit tests' + needs: classify-changes + if: ${{ needs.classify-changes.outputs.pq_unit_required == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.draft != true) }} + uses: ./.github/workflows/ci-pq.yml + with: + source_ref: ${{ github.sha }} + jobs: aarch64-unit,macos-arm64-unit,macos-x86_64-unit + full-validation-gate: name: 'Full Validation Gate' runs-on: ubuntu-24.04 @@ -1084,6 +1099,7 @@ jobs: - lint - public-docs-lint - rpc-docs + - pq-unit if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.draft != true) }} steps: - name: Require trusted label for fork PRs @@ -1106,6 +1122,8 @@ jobs: LINT_RESULT: ${{ needs.lint.result }} PUBLIC_DOCS_LINT_RESULT: ${{ needs.public-docs-lint.result }} RPC_DOCS_RESULT: ${{ needs.rpc-docs.result }} + PQ_UNIT_REQUIRED: ${{ needs.classify-changes.outputs.pq_unit_required }} + PQ_UNIT_RESULT: ${{ needs.pq-unit.result }} run: | set -euo pipefail @@ -1124,6 +1142,25 @@ jobs: check_result "classify validation profile" "${CLASSIFY_CHANGES_RESULT}" echo "CI profile: ${CI_PROFILE:-unavailable}" + # The PQ unit jobs (AArch64 Linux, macOS arm64 and x86_64) may be + # skipped only when the classifier says this change does not need them. + case "${PQ_UNIT_REQUIRED}" in + true) + check_result "PQ unit tests" "${PQ_UNIT_RESULT}" + ;; + false) + echo "PQ unit tests: ${PQ_UNIT_RESULT} (not required for this change)" + if [[ "${PQ_UNIT_RESULT}" != "skipped" && "${PQ_UNIT_RESULT}" != "success" ]]; then + echo "::error title=Full Validation failed::PQ unit tests result was ${PQ_UNIT_RESULT}." + failed=1 + fi + ;; + *) + echo "::error title=Full Validation failed::The classifier did not report whether the PQ unit jobs are required (pq_unit_required='${PQ_UNIT_REQUIRED}')." + failed=1 + ;; + esac + if [[ "${SOURCE_VALIDATION_REQUIRED}" != "true" ]]; then echo "Validation profile ${VALIDATION_PROFILE} does not require full source validation." exit "${failed}" diff --git a/.github/workflows/core-checks.yml b/.github/workflows/core-checks.yml index fc69e7a364..f6badd35df 100644 --- a/.github/workflows/core-checks.yml +++ b/.github/workflows/core-checks.yml @@ -165,6 +165,38 @@ jobs: - name: Test integrated libbitcoinpqc build policy run: python3 ci/checks/test_libbitcoinpqc_build_policy.py + # Every compiler and tool the build-policy test uses, each listed as + # required so that none of its cases can skip: the AArch64 cross compiler + # proves that AArch64 ELF builds refuse native ML-KEM (no BTI landing + # pads), MinGW and Clang cover the Windows and Clang builds, and cc -m32 + # an x86 target without a backend. The AArch64 C++ cross compiler, its nm + # and qemu-aarch64 run the AArch64 backend's portable-override check: a + # static test binary under emulation. gcc-aarch64-linux-gnu only + # recommends its libc headers, so they are installed explicitly. For -m32, + # libc6-dev-i386 provides the 32-bit headers: gcc-multilib conflicts with + # the AArch64 cross compiler on Ubuntu 24.04. clang-21 comes from + # apt.llvm.org, as in ci/test/01_base_install.sh. + - name: Test integrated mlkem-native build policy + env: + MLKEM_POLICY_REQUIRED_COMPILERS: >- + cc cc-m32 clang-21 aarch64-linux-gnu-gcc aarch64-linux-gnu-g++ aarch64-linux-gnu-nm qemu-aarch64 + x86_64-w64-mingw32-gcc-posix + APT_LLVM_V: 21 + run: | + sudo install -d -m 0755 /etc/apt/keyrings + curl --fail --silent --show-error --location --retry 5 https://apt.llvm.org/llvm-snapshot.gpg.key \ + | sudo tee /etc/apt/keyrings/apt.llvm.org.asc > /dev/null + codename="$(. /etc/os-release && echo "${VERSION_CODENAME}")" + echo "deb [signed-by=/etc/apt/keyrings/apt.llvm.org.asc] http://apt.llvm.org/${codename}/ llvm-toolchain-${codename}-${APT_LLVM_V} main" \ + | sudo tee "/etc/apt/sources.list.d/llvm-toolchain-${codename}-${APT_LLVM_V}.list" > /dev/null + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross qemu-user \ + gcc-mingw-w64-x86-64-posix \ + libc6-dev-i386 \ + "clang-${APT_LLVM_V}" "llvm-${APT_LLVM_V}" + python3 ci/checks/test_mlkem_build_policy.py -v + - name: Test IBD timeout wiring run: python3 ci/checks/test_ibd_timeout_wiring.py diff --git a/.github/workflows/required-merge-gate.yml b/.github/workflows/required-merge-gate.yml index e9e4008c2a..b2b4bdff62 100644 --- a/.github/workflows/required-merge-gate.yml +++ b/.github/workflows/required-merge-gate.yml @@ -89,7 +89,7 @@ jobs: echo "::error::Pull request base SHA is unavailable." exit 1 fi - git diff --name-only "${BASE_SHA}...HEAD" > "${changed_files}" + git diff --name-only --no-renames "${BASE_SHA}...HEAD" > "${changed_files}" elif [[ "${EVENT_NAME}" == "merge_group" ]]; then base_branch="${BASE_REF#refs/heads/}" if [[ -z "${base_branch}" ]]; then @@ -97,15 +97,15 @@ jobs: exit 1 fi git fetch --no-tags --prune origin "+refs/heads/${base_branch}:refs/remotes/origin/${base_branch}" - git diff --name-only "origin/${base_branch}...HEAD" > "${changed_files}" + git diff --name-only --no-renames "origin/${base_branch}...HEAD" > "${changed_files}" elif [[ "${EVENT_NAME}" == "push" ]]; then if [[ -n "${BEFORE_SHA}" && ! "${BEFORE_SHA}" =~ ^0+$ ]]; then - git diff --name-only "${BEFORE_SHA}..HEAD" > "${changed_files}" + git diff --name-only --no-renames "${BEFORE_SHA}..HEAD" > "${changed_files}" else git fetch --no-tags --prune origin "${DEFAULT_BRANCH}" merge_base="$(git merge-base "origin/${DEFAULT_BRANCH}" HEAD 2>/dev/null || true)" if [[ -n "${merge_base}" ]]; then - git diff --name-only "${merge_base}...HEAD" > "${changed_files}" + git diff --name-only --no-renames "${merge_base}...HEAD" > "${changed_files}" else git diff-tree --no-commit-id --name-only -r HEAD > "${changed_files}" fi @@ -143,12 +143,14 @@ jobs: # Job selection, source resolution and unit-test evidence of the PQ and # nightly workflows, which run outside pull requests, the nightly - # workflow's scheduled source, and the PQ canary tools. + # workflow's scheduled source, the PQ canary tools, and the PQ unit jobs + # that Full Validation requires for ML-KEM and transport changes. - name: Test PQ CI workflow contract run: | /usr/bin/python3 ci/checks/test_ci_pq_contract.py -v /usr/bin/python3 ci/checks/test_scheduled_validation_contract.py -v /usr/bin/python3 contrib/pq-canary/test/test_pq_canary.py -v + /usr/bin/python3 ci/checks/test_pq_merge_gate_contract.py -v - name: Classify validation profile id: classify diff --git a/CMakeLists.txt b/CMakeLists.txt index 55e4117434..30f7cb1e77 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -136,6 +136,9 @@ if(WITH_USDT) find_package(USDT MODULE REQUIRED) endif() +set(WITH_MLKEM_NATIVE "AUTO" CACHE STRING "Build ML-KEM native code for x86_64 (GCC or Clang) and macOS arm64, used when the CPU allows it: AUTO, ON or OFF.") +set_property(CACHE WITH_MLKEM_NATIVE PROPERTY STRINGS AUTO ON OFF) + option(ENABLE_EXTERNAL_SIGNER "Enable external signer support." ON) cmake_dependent_option(WITH_QRENCODE "Enable QR code support." ON "BUILD_GUI" OFF) @@ -426,6 +429,9 @@ include(cmake/introspection.cmake) include(cmake/ccache.cmake) +include(cmake/mlkem-native.cmake) +mlkem_native_configure() + add_library(warn_interface INTERFACE) target_link_libraries(core_interface INTERFACE warn_interface) if(MSVC) @@ -689,6 +695,7 @@ message(" IPC ................................. ${ipc_status}") message(" USDT tracing ........................ ${WITH_USDT}") message(" QR code (GUI) ....................... ${WITH_QRENCODE}") message(" DBus (GUI) .......................... ${WITH_DBUS}") +message(" ML-KEM-1024 backend ................. ${MLKEM_NATIVE_SUMMARY}") message("Tests:") message(" test_qbit ........................... ${BUILD_TESTS}") message(" test_qbit-qt ........................ ${BUILD_GUI_TESTS}") diff --git a/ci/checks/classify_merge_profile.py b/ci/checks/classify_merge_profile.py index b1d32714a9..f00e503cbc 100755 --- a/ci/checks/classify_merge_profile.py +++ b/ci/checks/classify_merge_profile.py @@ -55,6 +55,58 @@ "contrib/keys/operator-keys/", "contrib/guix/repo-templates/qbit-guix.sigs/operator-keys/", ) +# Changes to the ML-KEM library, the v2 transport, or their build and CI wiring +# must pass the aarch64-unit, macos-arm64-unit and macos-x86_64-unit jobs of +# .github/workflows/ci-pq.yml before they merge (the pq-unit job of ci.yml). +# ci/checks/test_pq_merge_gate_contract.py fails if a CMake file, workflow or +# ci/ file that mentions ML-KEM is missing here. +PQ_UNIT_FILES = frozenset( + { + # The build: the WITH_MLKEM_NATIVE option, the library and its link + # into bitcoin_node, the run-time CPU detection, the configure header, + # and the tests, fuzz targets and benchmarks that exercise it. + "CMakeLists.txt", + "cmake/bitcoin-build-config.h.in", + "cmake/mlkem-native.cmake", + "src/CMakeLists.txt", + "src/bench/CMakeLists.txt", + "src/test/CMakeLists.txt", + "src/test/fuzz/CMakeLists.txt", + "src/util/CMakeLists.txt", + # The jobs and the gate: this classifier, the workflows that run it and + # the PQ jobs, the scripts and evidence check of those jobs, and the + # tests of that wiring. + ".github/workflows/ci-pq.yml", + ".github/workflows/ci.yml", + ".github/workflows/core-checks.yml", + ".github/workflows/required-merge-gate.yml", + "ci/checks/classify_merge_profile.py", + "ci/checks/ctest_evidence.py", + "ci/checks/test_ci_pq_contract.py", + "ci/checks/test_classify_merge_profile.py", + "ci/checks/test_mlkem_build_policy.py", + "ci/checks/test_pq_merge_gate_contract.py", + "ci/test/00_setup_env_mac_native_pq.sh", + "ci/test/00_setup_env_native_aarch64_pq.sh", + "ci/test/00_setup_env_s390x_unit.sh", + "ci/test/03_test_script.sh", + # The sources and tests. + "src/compat/cpuid.h", + "src/net.cpp", + "src/net.h", + "src/test/bip324_tests.cpp", + "src/test/data/mlkem1024_vectors.json", + "src/test/data/pq_transport_vectors.json", + "src/test/mlkem_tests.cpp", + "src/test/net_tests.cpp", + } +) +PQ_UNIT_PREFIXES = ( + "src/bip324", + "src/compat/cpu_features.", + "src/crypto/mlkem", + "src/mlkem-native/", +) @dataclass(frozen=True) @@ -105,6 +157,11 @@ def public_docs_only(self) -> bool: def github_metadata_only(self) -> bool: return self.profile == GITHUB_METADATA_PROFILE + @property + def pq_unit_required(self) -> bool: + """Whether the PQ unit jobs must pass. A path that cannot be read requires them.""" + return bool(self.invalid_paths) or any(is_pq_unit_path(path) for path in self.paths) + def normalize_path(path: str) -> str | None: path = path.strip() @@ -147,6 +204,10 @@ def is_github_metadata_path(path: str) -> bool: return path in GITHUB_METADATA_FILES or path.startswith(GITHUB_METADATA_PREFIXES) +def is_pq_unit_path(path: str) -> bool: + return path in PQ_UNIT_FILES or path.startswith(PQ_UNIT_PREFIXES) + + def classify_paths(paths: list[str] | tuple[str, ...]) -> Classification: normalized: list[str] = [] invalid: list[str] = [] @@ -223,6 +284,7 @@ def github_outputs(classification: Classification) -> dict[str, str]: "touched_rpc_docs": bool_output(bool(classification.rpc_docs_paths)), "touched_public_docs": bool_output(bool(classification.public_docs_paths)), "touched_github_metadata": bool_output(bool(classification.github_metadata_paths)), + "pq_unit_required": bool_output(classification.pq_unit_required), } @@ -236,6 +298,7 @@ def describe_classification(classification: Classification) -> str: lines = [ f"validation_profile={classification.profile}", f"changed_count={len(classification.paths)}", + f"pq_unit_required={bool_output(classification.pq_unit_required)}", ] if classification.release_policy_paths: diff --git a/ci/checks/test_ci_pq_contract.py b/ci/checks/test_ci_pq_contract.py index c54bbe596c..bc8a7f4315 100755 --- a/ci/checks/test_ci_pq_contract.py +++ b/ci/checks/test_ci_pq_contract.py @@ -2,7 +2,7 @@ # Copyright (c) 2026-present The qbit core developers # Distributed under the MIT software license, see the accompanying # file COPYING or https://opensource.org/license/mit/. -"""Executable contract for the PQ CI entry points (#184 part 2, work item C1). +"""Executable contract for the PQ CI entry points (#184 part 2, work item C1; part 3, C2). ``ci-pq.yml`` and ``ci-nightly-heavy.yml`` are reusable and job-selectable, and the PQ unit job proves which suites ran. These tests read the real workflow @@ -37,12 +37,14 @@ import json import os import re +import shutil import subprocess import sys import tempfile import unittest from pathlib import Path from typing import Any +from unittest import mock import ctest_evidence import test_scheduled_validation_contract as svc @@ -56,15 +58,35 @@ } GATE_WORKFLOW = svc.WORKFLOW_DIR / "required-merge-gate.yml" TEST_SCRIPT = REPO_ROOT / "ci" / "test" / "03_test_script.sh" +MLKEM_TESTS = REPO_ROOT / "src" / "test" / "mlkem_tests.cpp" +BOOST_RECIPE = REPO_ROOT / "depends" / "packages" / "boost.mk" +MLKEM_BACKEND_HEADERS = [REPO_ROOT / "src" / "crypto" / name for name in ("mlkem_arith_backend.h", "mlkem_fips202_backend.h")] RESOLVER_JOB = svc.RESOLVER_JOB TRIGGER_STEP = "Check how the run started" REQUESTED_STEP = "Resolve requested source" SHARED_RESOLVER = "resolve_source_ref" SELECT_STEP = "Select jobs" REPORT_STEP = "Report unit-test evidence" +DEPENDS_INSTALL_STEP = "Install Homebrew packages" +DEPENDS_IDENTIFY_STEP = "Identify depends build" +DEPENDS_RESTORE_STEP = "Restore depends cache" +DEPENDS_SAVE_STEP = "Save depends cache" +DEPENDS_REQUIRE_STEP = "Require depends build" PQ_UNIT_JOB = "unit" AARCH64_JOB = "aarch64-unit" -AARCH64_SUITES = ["bip324_tests", "net_tests"] +S390X_JOB = "s390x-unit" +MACOS_JOB = "macos-arm64-unit" +MACOS_X86_64_JOB = "macos-x86_64-unit" +# Each macOS job: (runner label, uname -m, the ML-KEM backend that must run, its +# word in the evidence line). The backend is read from the headers that name it. +MACOS_JOBS = { + MACOS_JOB: ("macos-15", "arm64", "aarch64", "native AArch64"), + MACOS_X86_64_JOB: ("macos-15-intel", "x86_64", "x86_64", "native x86_64"), +} +PQ_MATRIX_JOBS = [AARCH64_JOB, S390X_JOB] +ALL_PQ_JOBS = {PQ_UNIT_JOB: PQ_MATRIX_JOBS, MACOS_JOB: [], MACOS_X86_64_JOB: []} +PQ_SUITES = ["mlkem_tests", "bip324_tests", "net_tests"] +AARCH64_SUITES = PQ_SUITES NIGHTLY_MATRIX_JOBS = ["arm32-unit-1", "arm32-unit-2", "previous-releases", "fuzz"] NIGHTLY_ALL_JOBS: dict[str, list[str]] = {"nightly-matrix": NIGHTLY_MATRIX_JOBS, "scanner-readiness": []} NIGHTLY_VARS_OFF = {"CI_NIGHTLY_HEAVY_RUN_TEST_MATRIX": "false", "CI_NIGHTLY_HEAVY_RUN_SCANNERS": "false"} @@ -74,7 +96,7 @@ MATRIX_EXPRESSION = "${{ fromJSON(needs." + RESOLVER_JOB + ".outputs.matrix) }}" ANNOTATED_TAG = "v9.9.9-annotated" AMBIGUOUS_NAME = "dup" -SUITE_INVENTORY = ["addrman_tests", "bip324_tests", "bip324_tests_extra", "net_tests", "netbase_tests", "util_tests"] +SUITE_INVENTORY = ["addrman_tests", "bip324_tests", "bip324_tests_extra", "mlkem_tests", "net_tests", "netbase_tests", "util_tests"] # --------------------------------------------------------------------------- @@ -152,6 +174,18 @@ def running_jobs(workflow: dict[str, Any], context: dict[str, Any]) -> dict[str, return running +def mlkem_backend_names() -> dict[str, str]: + """The arithmetic/Keccak backend names GetBackendNames() reports, per architecture, read from the backend headers.""" + names: dict[str, list[str]] = {} + for header in MLKEM_BACKEND_HEADERS: + text = header.read_text(encoding="utf8") + for arch, name in re.findall(r'^#(?:el)?if defined\(MLK_SYS_(X86_64|AARCH64)\)\n#define QBIT_MLKEM_\w+_BACKEND_NAME "([^"]+)"', + text, re.M): + names.setdefault(arch.lower(), []).append(name) + assert all(len(pair) == 2 for pair in names.values()) and set(names) == {"x86_64", "aarch64"}, names + return {arch: "/".join(pair) for arch, pair in names.items()} + + def source_env(env_file: str, names: list[str], extra_env: dict[str, str] | None = None) -> dict[str, str]: """Source a ci/test env file in bash and return the effective values of names.""" script = f"set -e\nsource {env_file}\n" + "".join(f'printf "%s=%s\\n" {name} "${{{name}:-}}"\n' for name in names) @@ -161,6 +195,14 @@ def source_env(env_file: str, names: list[str], extra_env: dict[str, str] | None return svc.parse_key_values(completed.stdout) +def depends_print(host: str, dep_opts: str, *names: str) -> dict[str, str]: + """Values depends/Makefile computes for host with dep_opts, read with its print- rule (nothing is built).""" + targets = " ".join(f"print-{name}" for name in names) + completed = subprocess.run(["bash", "-c", f"make -s -C depends HOST={host} {dep_opts} {targets}"], cwd=REPO_ROOT, + text=True, capture_output=True, check=True) + return {name: value.strip() for name, value in svc.parse_key_values(completed.stdout).items()} + + def shell_block(text: str, start: str, end_line: str) -> str: """The lines of text from the line starting with start through the next line equal to end_line.""" lines = text.splitlines() @@ -455,7 +497,14 @@ def test_selection_scripts_are_identical(self) -> None: def test_named_dispatch_runs_only_selected_jobs(self) -> None: cases: list[tuple[str, dict[str, Any], dict[str, list[str]]]] = [ (PQ, {"jobs": AARCH64_JOB}, {PQ_UNIT_JOB: [AARCH64_JOB]}), - (PQ, {"jobs": ""}, {PQ_UNIT_JOB: [AARCH64_JOB]}), + (PQ, {"jobs": S390X_JOB}, {PQ_UNIT_JOB: [S390X_JOB]}), + (PQ, {"jobs": MACOS_JOB}, {MACOS_JOB: []}), + (PQ, {"jobs": MACOS_X86_64_JOB}, {MACOS_X86_64_JOB: []}), + (PQ, {"jobs": f" {MACOS_JOB} , {AARCH64_JOB} "}, {PQ_UNIT_JOB: [AARCH64_JOB], MACOS_JOB: []}), + # What Full Validation's pq-unit job passes. + (PQ, {"jobs": f"{AARCH64_JOB},{MACOS_JOB},{MACOS_X86_64_JOB}"}, + {PQ_UNIT_JOB: [AARCH64_JOB], MACOS_JOB: [], MACOS_X86_64_JOB: []}), + (PQ, {"jobs": ""}, ALL_PQ_JOBS), (NIGHTLY, {"run_test_matrix": True, "run_scanners": False, "jobs": "previous-releases"}, {"nightly-matrix": ["previous-releases"]}), (NIGHTLY, {"jobs": ""}, {"nightly-matrix": NIGHTLY_MATRIX_JOBS, "scanner-readiness": []}), @@ -488,7 +537,7 @@ def test_scheduled_call_from_main_runs_everything_on_the_maintained_branch(self) outputs = context["needs"][RESOLVER_JOB]["outputs"] self.assertEqual(outputs["resolved_sha"], self.fixture.maintained_sha) self.assertNotEqual(outputs["resolved_sha"], context["github"]["sha"]) - self.assertEqual(running_jobs(self.workflows[PQ], context), {PQ_UNIT_JOB: [AARCH64_JOB]}) + self.assertEqual(running_jobs(self.workflows[PQ], context), ALL_PQ_JOBS) # -- called and direct nightly runs --------------------------------------------- @@ -729,7 +778,8 @@ def group(context: dict[str, Any]) -> str: def test_rejects_unknown_repeated_and_empty_job_names(self) -> None: bad = { - PQ: ["aarch64", "AARCH64-UNIT", f"{AARCH64_JOB},{AARCH64_JOB}", f"{AARCH64_JOB},", ",", "s390x-unit"], + PQ: ["aarch64", "AARCH64-UNIT", f"{AARCH64_JOB},{AARCH64_JOB}", f"{AARCH64_JOB},", ",", "s390x", "macos-arm64", + f"{MACOS_JOB},{MACOS_JOB}", "macos-x86_64", "macos-intel"], NIGHTLY: ["previous_releases", "previous-releases,previous-releases", "previous-releases,,fuzz", "unit"], } for key, values in bad.items(): @@ -784,7 +834,12 @@ def test_every_pq_job_checks_out_the_resolved_commit(self) -> None: workflow = self.workflows[PQ] context = self.resolve(PQ, self.dispatch(PQ, source_ref=svc.MAINTAINED_BRANCH, jobs=AARCH64_JOB)) refs = svc.check_source_wiring(workflow, context) - self.assertEqual(refs, {PQ_UNIT_JOB: fixture.maintained_sha}) + self.assertEqual(refs, {job: fixture.maintained_sha for job in ALL_PQ_JOBS}) + # The macOS jobs share the unit job's guard, checkout and verification steps. + for macos_job in MACOS_JOBS: + for step_name in (svc.GUARD_STEP, svc.CHECKOUT_STEP, svc.VERIFY_STEP): + self.assertEqual(svc.find_step(workflow["jobs"][macos_job], step_name), + svc.find_step(workflow["jobs"][PQ_UNIT_JOB], step_name), (macos_job, step_name)) mutated = copy.deepcopy(workflow) checkout = svc.find_step(mutated["jobs"][PQ_UNIT_JOB], svc.CHECKOUT_STEP) @@ -812,7 +867,7 @@ def test_aarch64_job_runs_on_a_public_arm_runner(self) -> None: workflow = self.workflows[PQ] (entry,) = [entry for entry in catalog(workflow) if entry["job"] == AARCH64_JOB] self.assertEqual(entry["runs-on"], "ubuntu-24.04-arm") - self.assertIn("pre-library", entry["name"]) + self.assertIn("mlkem_tests", entry["phase"]) env_file = entry["file-env"] self.assertTrue(os.access(REPO_ROOT / env_file, os.X_OK), env_file) @@ -834,7 +889,7 @@ def test_aarch64_job_runs_on_a_public_arm_runner(self) -> None: self.assertEqual(effective["CI_IMAGE_NAME_TAG"], "docker.io/library/ubuntu:24.04") self.assertEqual(effective["RUN_FUNCTIONAL_TESTS"], "false") regex = re.compile(effective["CTEST_REGEX"]) - self.assertEqual([name for name in SUITE_INVENTORY if regex.search(name)], AARCH64_SUITES) + self.assertEqual([name for name in SUITE_INVENTORY if regex.search(name)], sorted(AARCH64_SUITES)) # 02_run_container.py passes only variables exported by some ci/test/00_setup_env*.sh into the container. exported = set() @@ -843,6 +898,320 @@ def test_aarch64_job_runs_on_a_public_arm_runner(self) -> None: for name in ("CI_BUILD_TARGET", "CTEST_REGEX", "CTEST_EXPECTED_SUITES", "CTEST_INCLUDE_RANGE"): self.assertIn(name, exported) + def test_s390x_job_cross_compiles_and_emulates_only_the_tests(self) -> None: + workflow = self.workflows[PQ] + (entry,) = [entry for entry in catalog(workflow) if entry["job"] == S390X_JOB] + self.assertEqual(entry["runs-on"], "ubuntu-24.04", "built on an x64 runner, not in an emulated container") + self.assertEqual(entry["timeout-minutes"], 180) + env_file = entry["file-env"] + self.assertTrue(os.access(REPO_ROOT / env_file, os.X_OK), env_file) + + context = dict(self.resolve(PQ, self.dispatch(PQ, jobs=S390X_JOB)), matrix=entry) + env = svc.job_env(workflow, workflow["jobs"][PQ_UNIT_JOB], context) + self.assertEqual(env["FILE_ENV"], env_file) + names = ["HOST", "PACKAGES", "DEP_OPTS", "NO_DEPENDS", "CI_IMAGE_PLATFORM", "CI_BUILD_TARGET", "CTEST_REGEX", + "CTEST_EXPECTED_SUITES", "CTEST_INCLUDE_RANGE", "RUN_FUNCTIONAL_TESTS", "BITCOIN_CONFIG"] + effective = source_env(env_file, names, {key: env[key] for key in ("CI_IMAGE_REGISTRY_PREFIX", "CI_ENFORCE_INTERNAL_REGISTRY")}) + self.assertEqual(effective["HOST"], "s390x-linux-gnu") + self.assertEqual(effective["NO_DEPENDS"], "", "the depends tree is built for s390x") + self.assertEqual(effective["CI_IMAGE_PLATFORM"], "", "only the tests run emulated") + self.assertIn("g++-s390x-linux-gnu", effective["PACKAGES"].split()) + self.assertIn("qemu-user", effective["PACKAGES"].split()) + self.assertEqual(effective["CI_BUILD_TARGET"], "test_bitcoin") + self.assertEqual(effective["CTEST_EXPECTED_SUITES"].split(), PQ_SUITES) + self.assertEqual(effective["CTEST_INCLUDE_RANGE"], "") + self.assertEqual(effective["RUN_FUNCTIONAL_TESTS"], "false") + regex = re.compile(effective["CTEST_REGEX"]) + self.assertEqual([name for name in SUITE_INVENTORY if regex.search(name)], sorted(PQ_SUITES)) + # ctest runs each test through the emulator, so test_qbit is never started directly. + self.assertIn("-DCMAKE_CROSSCOMPILING_EMULATOR='qemu-s390x;-L;/usr/s390x-linux-gnu'", effective["BITCOIN_CONFIG"]) + + def test_macos_jobs_run_natively_on_apple_silicon_and_intel(self) -> None: + workflow = self.workflows[PQ] + catalog_names = [entry["job"] for entry in catalog(workflow)] + # Both jobs run the same steps; only their runner and job env differ. + self.assertEqual(workflow["jobs"][MACOS_JOB]["steps"], workflow["jobs"][MACOS_X86_64_JOB]["steps"]) + backends = mlkem_backend_names() + for job_id, (runner, machine, arch, evidence_word) in MACOS_JOBS.items(): + with self.subTest(job=job_id): + self.assertNotIn(job_id, catalog_names) + self.check_macos_job(workflow, job_id, runner, machine, backends[arch], evidence_word) + + def check_macos_job(self, workflow: dict[str, Any], job_id: str, runner: str, machine: str, backend: str, + evidence_word: str) -> None: + fixture = self.fixture + job = workflow["jobs"][job_id] + self.assertEqual(job["runs-on"], runner) + context = self.resolve(PQ, self.dispatch(PQ, jobs=job_id)) + env = svc.job_env(workflow, job, context) + self.assertEqual(env["DANGER_RUN_CI_ON_HOST"], "1", "no Docker on the macOS host") + env_file = env["FILE_ENV"] + self.assertTrue(os.access(REPO_ROOT / env_file, os.X_OK), env_file) + self.assertEqual(env["MATRIX_NAME"], job["name"]) + self.assertEqual(env["EXPECTED_MACHINE"], machine) + self.assertEqual(env["MLKEM_EXPECTED_BACKEND"], backend) + names = ["CI_OS_NAME", "NO_DEPENDS", "CI_BUILD_TARGET", "CTEST_REGEX", "CTEST_EXPECTED_SUITES", "RUN_FUNCTIONAL_TESTS", + "BITCOIN_CONFIG"] + effective = source_env(env_file, names) + # Losing the native backend must fail the configure step, not fall back quietly. + self.assertIn("-DWITH_MLKEM_NATIVE=ON", effective["BITCOIN_CONFIG"].split()) + self.assertEqual(effective["CI_OS_NAME"], "macos") + self.assertEqual(effective["NO_DEPENDS"], "", "Boost and libevent come from depends, not Homebrew") + self.assertEqual(effective["CI_BUILD_TARGET"], "test_bitcoin") + self.assertEqual(effective["CTEST_EXPECTED_SUITES"].split(), PQ_SUITES) + self.assertEqual(effective["RUN_FUNCTIONAL_TESTS"], "false") + regex = re.compile(effective["CTEST_REGEX"]) + self.assertEqual([name for name in SUITE_INVENTORY if regex.search(name)], sorted(PQ_SUITES)) + github_env = svc.github_env_for(context, job_id, workflow["name"]) + + # The report and native checks read the build from BASE_BUILD_DIR, which + # a step sets from RUNNER_TEMP: the runner context is unavailable in job env. + self.assertNotIn("BASE_BUILD_DIR", job["env"]) + set_dir = svc.find_step(job, "Set build directory") + self.assertLess(svc.step_index(job, "Set build directory"), svc.step_index(job, "CI script")) + with tempfile.TemporaryDirectory(dir=fixture.root) as tmp: + github_env_file = Path(tmp) / "github-env" + github_env_file.touch() + step_env = svc.step_env(workflow, job, set_dir, context) + step_env.update({"RUNNER_TEMP": "/runner/temp", "GITHUB_ENV": str(github_env_file)}) + result = svc.run_step(fixture, set_dir, step_env, fixture.root, github_env) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(github_env_file.read_text(encoding="utf8"), "BASE_BUILD_DIR=/runner/temp/build\n") + + # The architecture check refuses any other machine. + step = svc.find_step(job, "Require runner architecture") + self.assertLess(svc.step_index(job, "Require runner architecture"), svc.step_index(job, "CI script")) + with tempfile.TemporaryDirectory(dir=fixture.root) as tmp: + stubs = Path(tmp) + (stubs / "uname").write_text('#!/bin/sh\necho "$STUB_MACHINE"\n', encoding="utf8") + (stubs / "uname").chmod(0o755) + for other in ("arm64", "x86_64"): + with self.subTest(machine=other): + step_env = svc.step_env(workflow, job, step, context) + step_env.update({"PATH": f"{stubs}:{os.environ['PATH']}", "STUB_MACHINE": other}) + result = svc.run_step(fixture, step, step_env, fixture.root, github_env) + self.assertEqual(result.returncode == 0, other == machine, result.stderr) + if other != machine: + self.assertIn(f"{job_id} must run on {machine}", result.stderr) + + # After the suites pass, the native backend must be the one that ran. + step = svc.find_step(job, "Require native ML-KEM") + self.assertLess(svc.step_index(job, "CI script"), svc.step_index(job, "Require native ML-KEM")) + self.assertLess(svc.step_index(job, "Require native ML-KEM"), svc.step_index(job, REPORT_STEP)) + native_line = f"compiled {backend}, active {backend}" + cases = [ + ("native", native_line, 0, True), + ("portable on a CPU without an extension", f"compiled {backend}, active portable/portable", 0, False), + ("portable build", "compiled portable/portable, active portable/portable", 0, False), + ("other native backend", "compiled other/other, active other/other", 0, False), + ("only part of the line", f"compiled {backend}, active {backend}x", 0, False), + ("test failed", native_line, 201, False), + ] + for label, line, exit_code, ok in cases: + with self.subTest(job=job_id, native_check=label), tempfile.TemporaryDirectory(dir=fixture.root) as tmp: + build = Path(tmp) + (build / "bin").mkdir() + stub = build / "bin" / "test_qbit" + stub.write_text("#!/bin/sh\n" + 'printf "%s\\n" "$*" > "$(dirname "$0")/args"\n' + f"echo 'Running 1 test case...'\necho '{line}'\nexit {exit_code}\n", encoding="utf8") + stub.chmod(0o755) + step_env = svc.step_env(workflow, job, step, context) + step_env["BASE_BUILD_DIR"] = str(build) + result = svc.run_step(fixture, step, step_env, fixture.root, github_env) + self.assertEqual(result.returncode == 0, ok, result.completed.stdout + result.stderr) + self.assertEqual((build / "bin" / "args").read_text(encoding="utf8").split(), + ["--run_test=mlkem_tests/backend_selection", "--log_level=message", "--color_output=no"]) + if not ok: + self.assertIn("::error::", result.stderr) + # The line the step looks for is the one backend_selection logs. + self.assertIn('BOOST_TEST_MESSAGE("compiled " << compiled_arith << "/" << compiled_keccak << ", active " << ' + 'active.arith << "/" << active.keccak);', MLKEM_TESTS.read_text(encoding="utf8")) + + # Evidence is reported exactly as for the matrix jobs. + report = svc.find_step(job, REPORT_STEP) + self.assertEqual(report.get("if"), "${{ always() }}") + self.assertEqual(report["run"], svc.find_step(workflow["jobs"][PQ_UNIT_JOB], REPORT_STEP)["run"]) + with tempfile.TemporaryDirectory(dir=fixture.root) as tmp: + build = Path(tmp) + (build / "ctest-evidence").mkdir() + (build / "ctest-evidence" / "summary.md").write_text("| mlkem_tests | passed | 0.5 |\n", encoding="utf8") + step_env = svc.step_env(workflow, job, report, context) + step_env["BASE_BUILD_DIR"] = str(build) + result = svc.run_step(fixture, report, step_env, fixture.root, github_env) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(f"## {job['name']}", result.summary) + self.assertIn(evidence_word, result.summary) + self.assertIn("| mlkem_tests | passed | 0.5 |", result.summary) + + def test_macos_jobs_build_boost_and_libevent_through_depends(self) -> None: + workflow = self.workflows[PQ] + for job_id, (_runner, _machine, arch, _evidence_word) in MACOS_JOBS.items(): + with self.subTest(job=job_id): + self.check_macos_depends(workflow, job_id, f"{arch}-apple-darwin24.6.0") + + def check_macos_depends(self, workflow: dict[str, Any], job_id: str, runner_host: str) -> None: + fixture = self.fixture + job = workflow["jobs"][job_id] + context = self.resolve(PQ, self.dispatch(PQ, jobs=job_id)) + github_env = svc.github_env_for(context, job_id, workflow["name"]) + env_file = svc.job_env(workflow, job, context)["FILE_ENV"] + dep_opts = source_env(env_file, ["DEP_OPTS"])["DEP_OPTS"] + + # For the runner's host, depends builds Boost and libevent and nothing else. + printed = depends_print(runner_host, dep_opts, "packages", "native_packages") + self.assertEqual(printed["packages"].split(), ["boost", "libevent"]) + self.assertEqual(printed["native_packages"].split(), []) + + order = [DEPENDS_INSTALL_STEP, DEPENDS_IDENTIFY_STEP, DEPENDS_RESTORE_STEP, "CI script", DEPENDS_SAVE_STEP, + DEPENDS_REQUIRE_STEP, "Require native ML-KEM", REPORT_STEP] + self.assertEqual([svc.step_index(job, name) for name in order], sorted(svc.step_index(job, name) for name in order)) + + # Homebrew provides GNU Make 4 as the make on PATH, but neither library. + step = svc.find_step(job, DEPENDS_INSTALL_STEP) + with tempfile.TemporaryDirectory(dir=fixture.root) as tmp: + stubs = Path(tmp) + (stubs / "brew").write_text('#!/bin/sh\nif [ "$1" = --prefix ]; then echo "/stub/opt/$2"; ' + 'else echo "$*" >> "$STUB_LOG"; fi\n', encoding="utf8") + (stubs / "brew").chmod(0o755) + step_env = svc.step_env(workflow, job, step, context) + step_env.update({"PATH": f"{stubs}:{os.environ['PATH']}", "STUB_LOG": str(stubs / "log"), + "GITHUB_PATH": str(stubs / "github-path")}) + result = svc.run_step(fixture, step, step_env, fixture.root, github_env) + self.assertEqual(result.returncode, 0, result.stderr) + installed = (stubs / "log").read_text(encoding="utf8").split() + self.assertEqual(installed[:2], ["install", "--quiet"]) + self.assertIn("make", installed) + self.assertNotIn("boost", installed) + self.assertNotIn("libevent", installed) + self.assertEqual((stubs / "github-path").read_text(encoding="utf8"), "/stub/opt/make/libexec/gnubin\n") + + # HOST and the cache key's id are what depends reports for this build. + step = svc.find_step(job, DEPENDS_IDENTIFY_STEP) + host = subprocess.run([str(REPO_ROOT / "depends" / "config.guess")], text=True, capture_output=True, + check=True).stdout.strip() + expected_id = depends_print(host, dep_opts, "final_build_id")["final_build_id"] + self.assertRegex(expected_id, r"^[0-9a-f]+$") + with tempfile.TemporaryDirectory(dir=fixture.root) as tmp: + # A working make that reports macOS's own GNU Make 3.81. + stubs = Path(tmp) / "stubs" + stubs.mkdir() + (stubs / "make").write_text("#!/bin/sh\n" + "if [ \"$1\" = --version ]; then echo 'GNU Make 3.81'; exit 0; fi\n" + f"exec {shutil.which('make')} \"$@\"\n", encoding="utf8") + (stubs / "make").chmod(0o755) + for label, path, ok in (("GNU Make 4", os.environ["PATH"], True), ("GNU Make 3.81", f"{stubs}:{os.environ['PATH']}", False)): + with self.subTest(job=job_id, make=label): + github_env_file = Path(tmp) / f"github-env-{ok}" + github_env_file.touch() + step_env = svc.step_env(workflow, job, step, context) + step_env.update({"PATH": path, "BASE_ROOT_DIR": str(REPO_ROOT), "GITHUB_ENV": str(github_env_file)}) + result = svc.run_step(fixture, step, step_env, REPO_ROOT, github_env) + self.assertEqual(result.returncode == 0, ok, result.completed.stdout + result.stderr) + if ok: + self.assertEqual(github_env_file.read_text(encoding="utf8"), f"HOST={host}\nDEPENDS_BUILD_ID={expected_id}\n") + else: + self.assertIn("::error::depends needs GNU Make 4 or later", result.stderr) + self.assertEqual(github_env_file.read_text(encoding="utf8"), "") + + # The cache holds depends/built under that key, and is saved under the + # same key once the CI script wrote the toolchain file. + restore = svc.find_step(job, DEPENDS_RESTORE_STEP) + save = svc.find_step(job, DEPENDS_SAVE_STEP) + self.assertEqual(restore["uses"].split("@"), ["actions/cache/restore", save["uses"].split("@")[1]]) + self.assertEqual(save["uses"].split("@")[0], "actions/cache/save") + self.assertEqual(restore["with"]["path"], "depends/built") + self.assertEqual(save["with"]["path"], restore["with"]["path"]) + self.assertEqual(save["with"]["key"], "${{ steps." + restore["id"] + ".outputs.cache-primary-key }}") + step_context = dict(context, env={"HOST": runner_host, "DEPENDS_BUILD_ID": "0123456789a"}) + self.assertEqual(svc.render(restore["with"]["key"], step_context), f"depends-{runner_host}-0123456789a") + toolchain = f"depends/{runner_host}/toolchain.cmake" + save_cases = [ + ("built on a miss", "", [toolchain], False, True), + ("restore-key match", "false", [toolchain], False, True), + ("exact hit", "true", [toolchain], False, False), + ("depends did not finish", "", [], False, False), + ("cancelled", "", [toolchain], True, False), + ] + for label, cache_hit, files, cancelled, saves in save_cases: + with self.subTest(job=job_id, save=label): + functions = {"cancelled": lambda cancelled=cancelled: cancelled, + "hashfiles": lambda pattern, files=files: "f00d" if pattern in files else ""} + save_context = dict(step_context, steps={restore["id"]: {"outputs": {"cache-hit": cache_hit}}}) + with mock.patch.dict(svc._FUNCTIONS, functions): + self.assertIs(svc._truthy(svc.render(save["if"], save_context)), saves) + + # Afterwards the build must have used that depends tree: its toolchain, + # libevent, and Boost at the version boost.mk pins. + step = svc.find_step(job, DEPENDS_REQUIRE_STEP) + pinned = re.search(r"^\$\(package\)_version *= *(\S+)$", BOOST_RECIPE.read_text(encoding="utf8"), re.M) + assert pinned is not None, BOOST_RECIPE + boost_version = pinned.group(1) + build_id = "0123456789a" + build_cases: list[tuple[str, dict[str, str | None], str, str | None]] = [ + ("depends", {}, build_id, None), + ("depends, other entry type", {"CMAKE_TOOLCHAIN_FILE:FILEPATH": None, + "CMAKE_TOOLCHAIN_FILE:UNINITIALIZED": "{prefix}/toolchain.cmake"}, build_id, None), + ("stale cache key", {}, "fedcba98765", "depends did not build 0123456789a"), + ("no depends", {"CMAKE_TOOLCHAIN_FILE:FILEPATH": None, + "Boost_DIR:PATH": "/opt/homebrew/Cellar/boost/1.92.0/lib/cmake/Boost-1.92.0"}, + build_id, "CMAKE_TOOLCHAIN_FILE unset"), + ("Homebrew Boost", {"Boost_DIR:PATH": "/opt/homebrew/Cellar/boost/1.92.0/lib/cmake/Boost-1.92.0"}, + build_id, "Boost_DIR must be {prefix}/lib/cmake/Boost-" + boost_version), + ("other depends Boost", {"Boost_DIR:PATH": "{prefix}/lib/cmake/Boost-1.0.0"}, build_id, "Boost-1.0.0"), + ("Homebrew libevent", {"Libevent_DIR:PATH": "/opt/homebrew/lib/cmake/libevent"}, build_id, "Libevent_DIR must be"), + ] + for label, changes, stamp_id, error in build_cases: + with self.subTest(job=job_id, depends_build=label), tempfile.TemporaryDirectory(dir=fixture.root) as tmp: + root = Path(tmp) / "workspace" + prefix = root / "depends" / runner_host + prefix.mkdir(parents=True) + (prefix / f".stamp_{stamp_id}").touch() + (root / "depends" / "packages").mkdir() + shutil.copy(BOOST_RECIPE, root / "depends" / "packages") + entries: dict[str, str | None] = { + "CMAKE_TOOLCHAIN_FILE:FILEPATH": "{prefix}/toolchain.cmake", + "Boost_DIR:PATH": "{prefix}/lib/cmake/Boost-" + boost_version, + "Libevent_DIR:PATH": "{prefix}/lib/cmake/libevent", + } + entries.update(changes) + build = Path(tmp) / "build" + build.mkdir() + (build / "CMakeCache.txt").write_text( + "".join(f"{key}={value.format(prefix=prefix)}\n" for key, value in entries.items() if value is not None), + encoding="utf8") + step_env = svc.step_env(workflow, job, step, context) + step_env.update({"BASE_ROOT_DIR": str(root), "BASE_BUILD_DIR": str(build), "HOST": runner_host, + "DEPENDS_BUILD_ID": build_id}) + result = svc.run_step(fixture, step, step_env, fixture.root, github_env) + self.assertEqual(result.returncode == 0, error is None, result.completed.stdout + result.stderr) + if error is None: + self.assertIn(f"Built with Boost {boost_version} and libevent from {prefix}", result.completed.stdout) + else: + self.assertIn("::error::", result.stderr) + self.assertIn(error.format(prefix=prefix), result.stderr) + + def test_contexts_are_available_where_used(self) -> None: + """Mirror actionlint's context-availability rule for the places these workflows use expressions. + + GitHub refuses to load a workflow that reads, say, runner in a job's env, + while this file's expression evaluator would render it as empty. + """ + expression = re.compile(r"\$\{\{(.*?)\}\}", re.S) + root = re.compile(r"(? None: block = shell_block(TEST_SCRIPT.read_text(encoding="utf8"), "CI_BUILD_TARGET=${CI_BUILD_TARGET:-all}", ")") for target, expected in (("test_bitcoin", ["test_bitcoin"]), ("", ["all", "install"])): @@ -858,7 +1227,7 @@ def test_build_target_reaches_both_build_commands(self) -> None: def test_ctest_selection_reaches_listing_run_and_evidence(self) -> None: block = shell_block(TEST_SCRIPT.read_text(encoding="utf8"), 'if [ "$RUN_UNIT_TESTS" = "true" ]; then', "fi") - regex = "^(bip324_tests|net_tests)$" + regex = "^(mlkem_tests|bip324_tests|net_tests)$" expected = " ".join(AARCH64_SUITES) run = run_test_script_block(block, {"CTEST_REGEX": regex, "CTEST_EXPECTED_SUITES": expected}) @@ -880,12 +1249,12 @@ def test_ctest_selection_reaches_listing_run_and_evidence(self) -> None: failures = { "skipped suite": ({"CTEST_REGEX": regex, "CTEST_EXPECTED_SUITES": expected, "STUB_OUTCOMES": json.dumps({"bip324_tests": "notrun"})}, "bip324_tests: skipped"), - "regex selects too little": ({"CTEST_REGEX": "^(net_tests)$", "CTEST_EXPECTED_SUITES": expected}, + "regex selects too little": ({"CTEST_REGEX": "^(mlkem_tests|net_tests)$", "CTEST_EXPECTED_SUITES": expected}, "did not select expected suite(s): bip324_tests"), - "regex selects too much": ({"CTEST_REGEX": "^bip324_tests|^net_tests$", "CTEST_EXPECTED_SUITES": expected}, + "regex selects too much": ({"CTEST_REGEX": "^mlkem_tests$|^bip324_tests|^net_tests$", "CTEST_EXPECTED_SUITES": expected}, "not expected: bip324_tests_extra"), - "suite missing from the build": ({"CTEST_REGEX": "^(bip324_tests|net_tests|mlkem_tests)$", - "CTEST_EXPECTED_SUITES": expected + " mlkem_tests"}, "mlkem_tests"), + "suite missing from the build": ({"CTEST_REGEX": "^(mlkem_tests|bip324_tests|net_tests|pq_records_tests)$", + "CTEST_EXPECTED_SUITES": expected + " pq_records_tests"}, "pq_records_tests"), "range and regex together": ({"CTEST_REGEX": regex, "CTEST_INCLUDE_RANGE": "1,2"}, "set only one"), "nothing selected": ({"CTEST_REGEX": "^nothing$"}, "No tests were found"), } @@ -929,7 +1298,7 @@ def test_report_step_publishes_source_and_suites(self) -> None: report = svc.run_step(fixture, step, env, fixture.root, github_env) self.assertEqual(report.returncode, 0, report.stderr) self.assertIn(f"- source: {fixture.manual_sha} (refs/heads/{fixture.manual_branch})", report.summary) - self.assertIn("Pre-library phase", report.summary) + self.assertIn("mlkem_tests, bip324_tests and net_tests on AArch64", report.summary) self.assertIn("| bip324_tests | passed | 0.5 |", report.summary) def test_required_gate_runs_the_contracts(self) -> None: @@ -1002,5 +1371,73 @@ def main() -> int: self.assertIn("unreadable ctest test listing", summary.read_text(encoding="utf8")) + +class FuzzReplayContractTest(unittest.TestCase): + """The fuzz phase of 03_test_script.sh replays the ML-KEM targets on portable C too (#190 D5).""" + + STUB_RUNNER = r'''#!/usr/bin/env python3 +import json, os, sys +with open(os.environ["STUB_LOG"], "a", encoding="utf8") as log: + log.write(json.dumps({"args": sys.argv[1:], "MLKEM_FORCE_PORTABLE": os.environ.get("MLKEM_FORCE_PORTABLE")}) + "\n") +''' + + @staticmethod + def fuzz_block() -> str: + """The last RUN_FUZZ_TESTS block of 03_test_script.sh: the replay and mutation phase.""" + lines = TEST_SCRIPT.read_text(encoding="utf8").splitlines() + first = max(index for index, line in enumerate(lines) if line == 'if [ "$RUN_FUZZ_TESTS" = "true" ]; then') + last = next(index for index in range(first + 1, len(lines)) if lines[index] == "fi") + return "\n".join(lines[first:last + 1]) + "\n" + + @staticmethod + def required_targets() -> list[str]: + """QBIT_REQUIRED_CORPUS_TARGETS from test/fuzz/test_runner.py, read without importing it.""" + import ast + tree = ast.parse((REPO_ROOT / "test" / "fuzz" / "test_runner.py").read_text(encoding="utf8")) + for node in tree.body: + if isinstance(node, ast.Assign) and any(getattr(t, "id", None) == "QBIT_REQUIRED_CORPUS_TARGETS" for t in node.targets): + return list(ast.literal_eval(node.value)) + raise AssertionError("QBIT_REQUIRED_CORPUS_TARGETS not found") + + def run_block(self, mutate_min_time: str) -> list[dict[str, Any]]: + with tempfile.TemporaryDirectory(prefix="ci-fuzz-") as tmp: + root = Path(tmp) + runner = root / "build" / "test" / "fuzz" / "test_runner.py" + runner.parent.mkdir(parents=True) + runner.write_text(self.STUB_RUNNER, encoding="utf8") + runner.chmod(0o755) + log = root / "calls.jsonl" + log.touch() + env = {"PATH": os.environ["PATH"], "HOME": tmp, "LC_ALL": "C", "RUN_FUZZ_TESTS": "true", + "BASE_BUILD_DIR": str(root / "build"), "DEPENDS_DIR": str(root / "depends"), "HOST": "x86_64-pc-linux-gnu", + "MAKEJOBS": "-j2", "DIR_FUZZ_IN": str(root / "corpora"), "FUZZ_TESTS_CONFIG": "", + "QBIT_FUZZ_MUTATE_MIN_TIME": mutate_min_time, "STUB_LOG": str(log)} + script = root / "block.sh" + script.write_text("set -ex\n" + self.fuzz_block(), encoding="utf8") + completed = subprocess.run(["bash", str(script)], env=env, text=True, capture_output=True) + self.assertEqual(completed.returncode, 0, completed.stderr) + return [json.loads(line) for line in log.read_text(encoding="utf8").splitlines()] + + def test_mlkem_targets_replay_on_portable_c(self) -> None: + replay, portable = self.run_block(mutate_min_time="") + self.assertIn("--require_qbit_corpus", replay["args"]) + self.assertIsNone(replay["MLKEM_FORCE_PORTABLE"]) + self.assertEqual(portable["MLKEM_FORCE_PORTABLE"], "1") + self.assertEqual(portable["args"][-2:], ["mlkem", "mlkem_backend_diff"]) + corpus = [arg for arg in replay["args"] if arg.endswith("/corpora")] + self.assertEqual(len(corpus), 1, replay["args"]) + self.assertIn(corpus[0], portable["args"], "the portable replay uses the same corpus") + + def test_mutation_phase_covers_every_required_target(self) -> None: + calls = self.run_block(mutate_min_time="60") + self.assertEqual(len(calls), 3) + mutate = calls[2] + self.assertIn("--mutate_min_time=60", mutate["args"]) + self.assertIsNone(mutate["MLKEM_FORCE_PORTABLE"]) + targets = self.required_targets() + self.assertIn("mlkem", targets) + self.assertEqual(mutate["args"][-len(targets):], targets) + + if __name__ == "__main__": unittest.main() diff --git a/ci/checks/test_classify_merge_profile.py b/ci/checks/test_classify_merge_profile.py index b2e3824651..917059be4f 100755 --- a/ci/checks/test_classify_merge_profile.py +++ b/ci/checks/test_classify_merge_profile.py @@ -242,6 +242,82 @@ def test_github_outputs_report_github_metadata_profile(self) -> None: self.assertEqual(outputs["source_validation_required"], "false") self.assertEqual(outputs["touched_github_metadata"], "true") + def test_pq_unit_jobs_required_for_mlkem_and_transport_paths(self) -> None: + required = [ + "src/mlkem-native/mlkem/src/kem.c", + "src/crypto/mlkem.cpp", + "src/crypto/mlkem_config.h", + "cmake/mlkem-native.cmake", + "src/bip324.cpp", + "src/bip324.h", + "src/bip324_pq.cpp", + "src/bip324_pq.h", + "src/net.h", + "src/net.cpp", + "src/compat/cpuid.h", + "src/compat/cpu_features.h", + "src/compat/cpu_features.cpp", + ".github/workflows/ci-pq.yml", + "ci/test/00_setup_env_native_aarch64_pq.sh", + "ci/test/00_setup_env_s390x_unit.sh", + "ci/test/00_setup_env_mac_native_pq.sh", + "src/test/mlkem_tests.cpp", + "src/test/bip324_tests.cpp", + "src/test/net_tests.cpp", + "src/test/data/pq_transport_vectors.json", + "src/test/data/mlkem1024_vectors.json", + # Build wiring. + "CMakeLists.txt", + "src/CMakeLists.txt", + "src/util/CMakeLists.txt", + "src/test/CMakeLists.txt", + "src/test/fuzz/CMakeLists.txt", + "src/bench/CMakeLists.txt", + "cmake/bitcoin-build-config.h.in", + # Gate and CI wiring, and its tests. + ".github/workflows/ci.yml", + ".github/workflows/core-checks.yml", + ".github/workflows/required-merge-gate.yml", + "ci/checks/classify_merge_profile.py", + "ci/checks/ctest_evidence.py", + "ci/checks/test_ci_pq_contract.py", + "ci/checks/test_classify_merge_profile.py", + "ci/checks/test_mlkem_build_policy.py", + "ci/checks/test_pq_merge_gate_contract.py", + "ci/test/03_test_script.sh", + ] + for path in required: + with self.subTest(path=path): + classification = self.classify([path]) + self.assertTrue(classification.pq_unit_required) + self.assertEqual(classify_merge_profile.github_outputs(classification)["pq_unit_required"], "true") + self.assertEqual(classification.profile, classify_merge_profile.SOURCE_PROFILE) + # One matching path among others is enough. + self.assertTrue(self.classify(["doc/user/README.md", "src/bip324.cpp"]).pq_unit_required) + + def test_pq_unit_jobs_not_required_elsewhere(self) -> None: + for paths in ( + ["src/wallet/wallet.cpp"], + ["src/netbase.cpp", "src/net_processing.cpp", "src/compat/compat.h"], + ["src/test/net_peer_eviction_tests.cpp"], + ["ci/test/00_setup_env_native_asan.sh", ".github/workflows/rpc-perf-manual.yml"], + # Wiring shared by every CI job, and ML-KEM tooling the PQ unit jobs never run. + ["ci/test/01_base_install.sh", "ci/test/02_run_container.py", ".github/actions/configure-docker/action.yml"], + ["src/crypto/CMakeLists.txt", "src/test/cpu_features_tests.cpp"], + ["test/lint/mlkem-native-check.sh", "contrib/devtools/mlkem-native.pin", "contrib/devtools/update-mlkem-native.sh"], + ["doc/user/README.md"], + RPC_DOCS_PATHS, + PUBLIC_DOCS_PATHS, + ): + with self.subTest(paths=paths): + classification = self.classify(paths) + self.assertFalse(classification.pq_unit_required) + self.assertEqual(classify_merge_profile.github_outputs(classification)["pq_unit_required"], "false") + + def test_pq_unit_jobs_required_for_unreadable_paths(self) -> None: + self.assertTrue(self.classify(["../src/bip324.cpp"]).pq_unit_required) + self.assertTrue(self.classify(["/abs/path"]).pq_unit_required) + def test_require_release_policy_only_cli_rejects_outside_path(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: changed_files = Path(tmpdir) / "changed-files.txt" diff --git a/ci/checks/test_mlkem_build_policy.py b/ci/checks/test_mlkem_build_policy.py new file mode 100755 index 0000000000..807afb1e34 --- /dev/null +++ b/ci/checks/test_mlkem_build_policy.py @@ -0,0 +1,750 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026-present The qbit core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. +"""Regression tests for qbit's mlkem-native build policy (cmake/mlkem-native.cmake). + +Three layers: +- the WITH_MLKEM_NATIVE decision across the architecture, toolchain and + sanitizer matrix, run in CMake script mode, so every target is covered on + any host; +- configure and build of the mlkem_native library with each compiler found on + this host (cross compilers included), checking which sources and flags are + used; +- the guards in src/crypto/mlkem_config.h that stop a build from compiling + native code where the policy forbids it. + +Cases that need a missing compiler are skipped, unless the compiler is listed +in MLKEM_POLICY_REQUIRED_COMPILERS (space-separated), as CI does for the ones +it installs: then a missing one fails. "cc-m32" stands for cc -m32 with 32-bit +headers. Failures print the signature, cause and fix. + +A fourth layer links qbit's glue and wrapper with every assembly routine +wrapped by a call counter, and checks that the portable override reaches each +of them: the x86_64 backend on x86_64 hosts, the AArch64 backend under +qemu-aarch64 (listed as qemu-aarch64 and aarch64-linux-gnu-g++ in +MLKEM_POLICY_REQUIRED_COMPILERS when required). +""" + +from __future__ import annotations + +import os +import re +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +MODULE = REPO_ROOT / "cmake" / "mlkem-native.cmake" +REQUIRED_COMPILERS = frozenset(os.environ.get("MLKEM_POLICY_REQUIRED_COMPILERS", "").split()) + + +def skip_missing_compiler(test: unittest.TestCase, executable: str, problem: str = "not found") -> None: + if executable in REQUIRED_COMPILERS: + test.fail(f"FAIL: {executable} {problem}\nCause: MLKEM_POLICY_REQUIRED_COMPILERS lists it\n" + f"Fix: install it, or drop it from MLKEM_POLICY_REQUIRED_COMPILERS") + test.skipTest(f"{executable} {problem}") + +SUPPORTED_ARCHS = ("x86_64", "aarch64-macho") +UNSUPPORTED_ARCHS = ( + "aarch64-elf", + "this target (riscv64)", + "this target (ppc64)", + "this target (powerpc64le)", + "this target (armv7)", + "this target (s390x)", + "this target (i686)", +) +SANITIZER_SETTINGS = ("", "address,undefined", "thread", "memory", "fuzzer,memory", "memory,undefined") + +POLICY_SCRIPT = r""" +include("${MODULE}") +mlkem_native_policy("${MODE}" "${ARCH}" "${MSVC}" "${SANITIZERS}") +message(NOTICE "RESULT enabled=${MLKEM_NATIVE_ENABLED} no_asm=${MLKEM_NATIVE_NO_ASM} reason=${MLKEM_NATIVE_REASON}") +""" + +FIXTURE_CMAKELISTS = r""" +cmake_minimum_required(VERSION 3.22) +set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) +project(qbit_mlkem_policy LANGUAGES NONE) + +if(NOT DEFINED QBIT_SOURCE_DIR) + message(FATAL_ERROR "QBIT_SOURCE_DIR is required") +endif() +set(PROJECT_SOURCE_DIR "${QBIT_SOURCE_DIR}") +set(WITH_MLKEM_NATIVE "AUTO" CACHE STRING "") + +# Stand-ins for qbit's interface libraries: the hardening marker must reach +# mlkem_native, the C++-only warning must not. +add_library(core_interface INTERFACE) +# Like core_interface's -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3, which the MSan +# jobs undo with APPEND_CPPFLAGS=-U_FORTIFY_SOURCE. +target_compile_options(core_interface INTERFACE -DQBIT_POLICY_CORE_FLAG -UQBIT_POLICY_FORTIFY -DQBIT_POLICY_FORTIFY=3) +add_library(warn_interface INTERFACE) +target_compile_options(warn_interface INTERFACE -Wsuggest-override) +target_link_libraries(core_interface INTERFACE warn_interface) +add_library(sanitize_interface INTERFACE) + +include("${QBIT_SOURCE_DIR}/cmake/mlkem-native.cmake") +mlkem_native_configure() +# Like CMakeLists.txt on MinGW, where the compiler accepts it (GCC's assembler +# does, Clang's integrated one does not). +if(MINGW AND CMAKE_C_COMPILER_ID STREQUAL "GNU") + target_compile_options(core_interface INTERFACE -Wa,-muse-unaligned-vector-move) + message(NOTICE "CORE_UNALIGNED_VECTOR_MOVE") +endif() +message(NOTICE "SUMMARY ${MLKEM_NATIVE_SUMMARY}") +message(NOTICE "ENABLE_MLKEM_NATIVE=${ENABLE_MLKEM_NATIVE}") +add_mlkem_native() +""" + +# Compiler setups for the configure-and-build layer: (name, executable, +# extra CMake definitions, target architecture, whether AUTO builds native code). +# A setup without extra definitions targets this host. AArch64 ELF is portable +# only (no BTI landing pads in the assembly); macOS arm64, which builds native +# code, needs an SDK and is covered by the CI job on Apple hardware. +COMPILERS = ( + ("host cc", "cc", (), "x86_64", True), + ("clang-21", "clang-21", (), "x86_64", True), + ("aarch64 gcc", "aarch64-linux-gnu-gcc", ("-DCMAKE_SYSTEM_NAME=Linux", "-DCMAKE_SYSTEM_PROCESSOR=aarch64"), "aarch64", False), + ("clang-21 --target=aarch64-linux-gnu", "clang-21", + ("-DCMAKE_SYSTEM_NAME=Linux", "-DCMAKE_SYSTEM_PROCESSOR=aarch64", "-DCMAKE_C_COMPILER_TARGET=aarch64-linux-gnu"), "aarch64", False), + ("mingw-w64 gcc", "x86_64-w64-mingw32-gcc-posix", ("-DCMAKE_SYSTEM_NAME=Windows", "-DCMAKE_SYSTEM_PROCESSOR=x86_64"), "x86_64", True), + ("clang-21 --target=x86_64-w64-mingw32", "clang-21", + ("-DCMAKE_SYSTEM_NAME=Windows", "-DCMAKE_SYSTEM_PROCESSOR=x86_64", "-DCMAKE_C_COMPILER_TARGET=x86_64-w64-mingw32"), "x86_64", True), +) + + +def run(command: list[str], **kwargs) -> subprocess.CompletedProcess[str]: + return subprocess.run(command, check=False, capture_output=True, text=True, **kwargs) + + +def host_arch() -> str: + machine = run(["uname", "-m"]).stdout.strip() + return {"amd64": "x86_64", "arm64": "aarch64"}.get(machine, machine) + + +class MlkemPolicyMatrixTest(unittest.TestCase): + """WITH_MLKEM_NATIVE decisions, in CMake script mode.""" + + def policy(self, mode: str, arch: str, msvc: bool, sanitizers: str) -> subprocess.CompletedProcess[str]: + cmake = shutil.which("cmake") + self.assertIsNotNone(cmake, "cmake is required") + assert cmake is not None + with tempfile.TemporaryDirectory() as tmp: + script = Path(tmp) / "policy.cmake" + script.write_text(POLICY_SCRIPT, encoding="utf8") + return run([ + cmake, + f"-DMODULE={MODULE}", + f"-DMODE={mode}", + f"-DARCH={arch}", + f"-DMSVC={'TRUE' if msvc else 'FALSE'}", + f"-DSANITIZERS={sanitizers}", + "-P", str(script), + ]) + + def expect(self, mode: str, arch: str, msvc: bool, sanitizers: str) -> tuple[bool, bool, str | None]: + """The policy as documented in cmake/mlkem-native.cmake: (enabled, no_asm, configure error).""" + no_asm = msvc + blocked = msvc or arch not in SUPPORTED_ARCHS or re.search(r"(^|,)memory($|,)", sanitizers) is not None + if mode.upper() == "OFF": + return False, no_asm, None + if blocked: + if mode.upper() == "ON": + return False, no_asm, "WITH_MLKEM_NATIVE=ON, but" + return False, no_asm, None + return True, no_asm, None + + def test_matrix(self) -> None: + for mode in ("AUTO", "ON", "OFF", "auto"): + for arch in SUPPORTED_ARCHS + UNSUPPORTED_ARCHS: + for msvc in (False, True): + for sanitizers in SANITIZER_SETTINGS: + with self.subTest(mode=mode, arch=arch, msvc=msvc, sanitizers=sanitizers): + result = self.policy(mode, arch, msvc, sanitizers) + combined = result.stdout + result.stderr + enabled, no_asm, error = self.expect(mode, arch, msvc, sanitizers) + if error: + self.assertNotEqual(result.returncode, 0, + f"FAIL: WITH_MLKEM_NATIVE={mode} configured for {arch}, msvc={msvc}, " + f"SANITIZERS={sanitizers!r}\nCause: ON must refuse targets without native code\n" + f"Fix: mlkem_native_policy in {MODULE}\n{combined}") + self.assertIn(error, combined) + continue + self.assertEqual(result.returncode, 0, combined) + match = re.search(r"RESULT enabled=(\w+) no_asm=(\w+) reason=(.*)", combined) + self.assertIsNotNone(match, combined) + assert match is not None + self.assertEqual(match.group(1) == "TRUE", enabled, + f"FAIL: native={match.group(1)} for {mode}/{arch}/msvc={msvc}/{sanitizers!r}\n" + f"Cause: policy disagrees with the documented matrix\nFix: mlkem_native_policy in {MODULE}") + self.assertEqual(match.group(2) == "TRUE", no_asm, combined) + self.assertTrue(match.group(3).strip(), "every decision states its reason") + + def test_invalid_mode_fails(self) -> None: + for mode in ("YES", "1", "native", ""): + with self.subTest(mode=mode): + result = self.policy(mode, "x86_64", False, "") + self.assertNotEqual(result.returncode, 0) + self.assertIn("WITH_MLKEM_NATIVE must be AUTO, ON or OFF", result.stdout + result.stderr) + + +class MlkemBuildTest(unittest.TestCase): + """Configure and build mlkem_native with the compilers on this host.""" + + def setUp(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.tmp_path = Path(self.tmp.name) + self.source = self.tmp_path / "source" + self.source.mkdir() + (self.source / "CMakeLists.txt").write_text(FIXTURE_CMAKELISTS, encoding="utf8") + self.cmake = shutil.which("cmake") + self.assertIsNotNone(self.cmake, "cmake is required") + self.assertIsNotNone(shutil.which("ninja"), "ninja is required") + + def configure(self, name: str, compiler: str, *definitions: str, + env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: + assert self.cmake is not None + return run([ + self.cmake, "-S", str(self.source), "-B", str(self.tmp_path / name), "-G", "Ninja", + f"-DQBIT_SOURCE_DIR={REPO_ROOT}", f"-DCMAKE_C_COMPILER={compiler}", "-DCMAKE_BUILD_TYPE=RelWithDebInfo", + *definitions, + ], env=env) + + def compile_flags(self, name: str) -> dict[str, list[str]]: + """The flags of mlkem_native's compile commands by source name, without the file arguments.""" + commands = run(["ninja", "-C", str(self.tmp_path / name), "-t", "commands", "mlkem_native"]) + self.assertEqual(commands.returncode, 0, commands.stdout + commands.stderr) + flags: dict[str, list[str]] = {} + for line in commands.stdout.splitlines(): + words = line.split() + if "-c" not in words: + continue + kept = [] + skip = False + for word in words[1:]: + if skip: + skip = False + elif word in ("-MT", "-MF", "-o", "-c"): + skip = True + elif word != "-MD": + kept.append(word) + flags[Path(words[-1]).name] = kept + return flags + + def build(self, name: str) -> subprocess.CompletedProcess[str]: + assert self.cmake is not None + return run([self.cmake, "--build", str(self.tmp_path / name), "--target", "mlkem_native", "--verbose"]) + + def check_build(self, label: str, compiler: str, extra: tuple[str, ...], mode: str, expect_native: bool) -> str: + name = re.sub(r"\W+", "_", f"{label}_{mode}") + configured = self.configure(name, compiler, f"-DWITH_MLKEM_NATIVE={mode}", + "-DAPPEND_CPPFLAGS=-UQBIT_POLICY_FORTIFY", "-DAPPEND_CFLAGS=-DQBIT_POLICY_APPEND_CFLAG", *extra) + self.assertEqual(configured.returncode, 0, configured.stdout + configured.stderr) + self.assertIn("SUMMARY native" if expect_native else "SUMMARY portable", configured.stdout + configured.stderr) + self.assertIn(f"ENABLE_MLKEM_NATIVE={'TRUE' if expect_native else 'FALSE'}", configured.stdout + configured.stderr) + unaligned_moves = "CORE_UNALIGNED_VECTOR_MOVE" in configured.stdout + configured.stderr + if "-DCMAKE_SYSTEM_NAME=Windows" in extra and Path(compiler).name.endswith(("gcc", "gcc-posix")): + self.assertTrue(unaligned_moves, "the fixture must pass -Wa,-muse-unaligned-vector-move for MinGW GCC, as CMakeLists.txt does") + built = self.build(name) + log = built.stdout + built.stderr + self.assertEqual(built.returncode, 0, + f"FAIL: mlkem_native does not build with {label}, WITH_MLKEM_NATIVE={mode}\n{log}") + compile_lines = [line for line in log.splitlines() if " -c " in line] + self.assertTrue(compile_lines, log) + for line in compile_lines: + self.assertNotRegex(line, r"(^|\s)-m(avx|avx2|arch=|cpu=)", + f"FAIL: an ISA flag reached an mlkem_native source with {label}\n" + f"Cause: native code must be selected at run time\nFix: remove the flag from {MODULE}\n{line}") + self.assertIn("-DQBIT_POLICY_CORE_FLAG", line, "core_interface's own options must apply to C and assembly") + self.assertNotIn("-Wsuggest-override", line, "qbit's C++ warning set must not reach the vendored C") + # APPEND_CPPFLAGS and APPEND_CFLAGS come after core_interface's options, so they can undo one + # (the MSan jobs pass -U_FORTIFY_SOURCE against core_interface's -D_FORTIFY_SOURCE=3). + words = line.split() + last_undefine = len(words) - 1 - words[::-1].index("-UQBIT_POLICY_FORTIFY") + self.assertGreater(last_undefine, words.index("-DQBIT_POLICY_FORTIFY=3"), + f"FAIL: APPEND_CPPFLAGS did not reach an mlkem_native compile after core_interface's options with {label}\n" + f"Cause: the C and assembly sources do not get the APPEND flags, or CMake de-duplicated them\n" + f"Fix: add_mlkem_native in {MODULE}\n{line}") + self.assertIn("-DQBIT_POLICY_APPEND_CFLAG", words, line) + asm_lines = [line for line in compile_lines if "mlkem_native_asm.S" in line] + if unaligned_moves: + # The flag would re-encode the proved assembly; the C sources keep it. + for line in compile_lines: + self.assertEqual("-Wa,-muse-unaligned-vector-move" in line.split(), line not in asm_lines, + f"FAIL: -Wa,-muse-unaligned-vector-move must reach mlkem_native's C sources and not its assembly ({label})\n" + f"Cause: gas would encode the assembly's vmovdqa as vmovdqu, which the HOL-Light proofs do not cover\n" + f"Fix: add_mlkem_native in {MODULE}\n{line}") + if expect_native: + self.assertEqual(len(asm_lines), 1, log) + for line in compile_lines: + self.assertIn("-DQBIT_MLKEM_NATIVE", line, line) + else: + self.assertEqual(asm_lines, [], log) + for line in compile_lines: + self.assertNotIn("-DQBIT_MLKEM_NATIVE", line, line) + return log + + def test_compilers(self) -> None: + found = 0 + for label, executable, extra, arch, native in COMPILERS: + compiler = shutil.which(executable) + with self.subTest(compiler=label): + if compiler is None: + skip_missing_compiler(self, executable) + if not extra and arch != host_arch(): + self.skipTest(f"{label} targets {arch}, not this host") + assert compiler is not None + found += 1 + self.check_build(label, compiler, extra, "OFF", False) + log = self.check_build(label, compiler, extra, "AUTO", native) + target = next((d.split("=", 1)[1] for d in extra if d.startswith("-DCMAKE_C_COMPILER_TARGET=")), None) + if native: + self.check_build(label, compiler, extra, "ON", True) + if target: + asm_line = next(line for line in log.splitlines() if "mlkem_native_asm.S" in line) + self.assertRegex(asm_line, rf"(--target=|-target ){re.escape(target)}\b", + "FAIL: assembly built without the C compiler's target\n" + "Cause: CMAKE_ASM_COMPILER_TARGET not propagated\nFix: mlkem_native_configure") + else: + on = self.configure(re.sub(r"\W+", "_", f"{label}_ON"), compiler, "-DWITH_MLKEM_NATIVE=ON", *extra) + combined = re.sub(r"\s+", " ", on.stdout + on.stderr) + self.assertNotEqual(on.returncode, 0, combined) + self.assertIn("WITH_MLKEM_NATIVE=ON, but", combined) + if arch == "aarch64": + self.assertIn("no BTI landing pads", combined) + self.assertGreater(found, 0, "no usable C compiler found") + + def test_append_flags_change_the_detected_target(self) -> None: + """APPEND_CPPFLAGS/APPEND_CFLAGS reach the library, so the target probes must see them: -m32 is not x86_64.""" + if host_arch() != "x86_64": + self.skipTest("needs an x86_64 host") + compiler = shutil.which("cc") + assert compiler is not None + if run([compiler, "-m32", "-x", "c", "-c", "-o", os.devnull, "-"], input="#include \nint x;\n").returncode != 0: + skip_missing_compiler(self, "cc-m32", "has no 32-bit headers") + name = "append_m32" + configured = self.configure(name, compiler, "-DWITH_MLKEM_NATIVE=AUTO", "-DAPPEND_CPPFLAGS=-m32", "-DAPPEND_CFLAGS=-m32") + output = configured.stdout + configured.stderr + self.assertEqual(configured.returncode, 0, output) + self.assertIn("SUMMARY portable", output, + f"FAIL: APPEND_CFLAGS=-m32 still detected x86_64\nCause: the target probes ran without the appended flags\nFix: mlkem_native_detect_arch in {MODULE}") + built = self.build(name) + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + # A reconfigure without the flag probes again and goes back to native. + configured = self.configure(name, compiler, "-DWITH_MLKEM_NATIVE=AUTO", "-DAPPEND_CPPFLAGS=", "-DAPPEND_CFLAGS=") + self.assertIn("SUMMARY native", configured.stdout + configured.stderr, configured.stdout + configured.stderr) + + def test_memory_sanitizer(self) -> None: + compiler = shutil.which("cc") + self.assertIsNotNone(compiler) + assert compiler is not None + auto = self.configure("msan_auto", compiler, "-DSANITIZERS=memory") + self.assertEqual(auto.returncode, 0, auto.stdout + auto.stderr) + self.assertIn("ENABLE_MLKEM_NATIVE=FALSE", auto.stdout + auto.stderr) + self.assertIn("MemorySanitizer", auto.stdout + auto.stderr) + built = self.build("msan_auto") + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + self.assertNotIn("mlkem_native_asm.S", built.stdout + built.stderr) + + on = self.configure("msan_on", compiler, "-DSANITIZERS=fuzzer,memory", "-DWITH_MLKEM_NATIVE=ON") + self.assertNotEqual(on.returncode, 0, on.stdout + on.stderr) + self.assertIn("WITH_MLKEM_NATIVE=ON, but MemorySanitizer", re.sub(r"\s+", " ", on.stdout + on.stderr)) + + def test_asm_flags_keep_explicit_configuration(self) -> None: + """The assembly gets the C flags, followed by the ASM flags the user, the + environment or the toolchain set; per configuration, only CMake's own + default gives way to the C flags.""" + if host_arch() != "x86_64": + self.skipTest("the host compiler builds no native ML-KEM code on this host") + compiler = shutil.which("cc") + self.assertIsNotNone(compiler) + assert compiler is not None + toolchain = self.tmp_path / "asm-toolchain.cmake" + toolchain.write_text('set(CMAKE_ASM_FLAGS_INIT "-DTOOLCHAIN_ASM_FLAG")\n' + 'set(CMAKE_ASM_FLAGS_RELWITHDEBINFO_INIT "-DTOOLCHAIN_ASM_RELWITHDEBINFO")\n', encoding="utf8") + c_flags = ("-DCMAKE_C_FLAGS=-DQBIT_POLICY_C_FLAG", "-DCMAKE_C_FLAGS_RELWITHDEBINFO=-O2 -g -DQBIT_POLICY_C_RELWITHDEBINFO") + clean_env = {key: value for key, value in os.environ.items() if key != "ASMFLAGS"} + # (name, definitions, ASMFLAGS, flags the assembly gets after the C flags, RelWithDebInfo flag) + cases = ( + ("default", (), None, (), "-DQBIT_POLICY_C_RELWITHDEBINFO"), + ("user", ("-DCMAKE_ASM_FLAGS=-DUSER_ASM_FLAG",), None, ("-DUSER_ASM_FLAG",), "-DQBIT_POLICY_C_RELWITHDEBINFO"), + ("environment", (), "-DENV_ASM_FLAG", ("-DENV_ASM_FLAG",), "-DQBIT_POLICY_C_RELWITHDEBINFO"), + ("toolchain", (f"-DCMAKE_TOOLCHAIN_FILE={toolchain}",), None, ("-DTOOLCHAIN_ASM_FLAG",), "-DTOOLCHAIN_ASM_RELWITHDEBINFO"), + ("user per-config", ("-DCMAKE_ASM_FLAGS_RELWITHDEBINFO=-DUSER_ASM_RELWITHDEBINFO",), None, (), "-DUSER_ASM_RELWITHDEBINFO"), + ) + for label, definitions, asmflags, asm_only, config_flag in cases: + with self.subTest(case=label): + name = re.sub(r"\W+", "_", f"asm_flags_{label}") + env = dict(clean_env, ASMFLAGS=asmflags) if asmflags else clean_env + configured = self.configure(name, compiler, *c_flags, *definitions, env=env) + self.assertEqual(configured.returncode, 0, configured.stdout + configured.stderr) + flags = self.compile_flags(name) + asm = flags["mlkem_native_asm.S"] + for flag in ("-DQBIT_POLICY_C_FLAG", *asm_only, config_flag): + self.assertEqual(asm.count(flag), 1, + f"FAIL: {flag} reaches mlkem_native_asm.S {asm.count(flag)} times ({label})\n" + f"Cause: mlkem_native_configure replaced or repeated the ASM flags\nFix: {MODULE}\n{asm}") + for flag in asm_only: + self.assertGreater(asm.index(flag), asm.index("-DQBIT_POLICY_C_FLAG"), + f"FAIL: {flag} comes before the C flags ({label})\n" + f"Cause: explicit ASM flags must win on conflict\nFix: {MODULE}\n{asm}") + if config_flag != "-DQBIT_POLICY_C_RELWITHDEBINFO": + self.assertNotIn("-DQBIT_POLICY_C_RELWITHDEBINFO", asm, + f"FAIL: the C RelWithDebInfo flags replaced an explicit ASM value ({label})\nFix: {MODULE}\n{asm}") + else: + # Apart from explicit ASM flags, the assembly compiles exactly like the C. + self.assertEqual([word for word in asm if word not in asm_only], flags["mlkem_native.c"]) + # A reconfigure, with or without the environment, must not accumulate flags. + for again in (env, clean_env): + reconfigured = self.configure(name, compiler, *c_flags, *definitions, env=again) + self.assertEqual(reconfigured.returncode, 0, reconfigured.stdout + reconfigured.stderr) + self.assertEqual(self.compile_flags(name), flags, + f"FAIL: a reconfigure changed the mlkem_native compile commands ({label})\n" + f"Cause: mlkem_native_configure wrote the ASM flags back to the cache\nFix: {MODULE}") + + def test_asm_flags_with_depends_toolchain(self) -> None: + """A toolchain rendered from depends/toolchain.cmake.in, as depends/Makefile + does, assembles with exactly the flags of the C sources.""" + # (name, C compiler, tool prefix, cross: (system name, arch, host), whether AUTO builds native code) + setups = ( + ("native", "cc", "", None, True), + ("mingw-w64", "x86_64-w64-mingw32-gcc-posix", "x86_64-w64-mingw32-", ("Windows", "x86_64", "x86_64-w64-mingw32"), True), + ("aarch64 linux", "aarch64-linux-gnu-gcc", "aarch64-linux-gnu-", ("Linux", "aarch64", "aarch64-linux-gnu"), False), + ) + template = (REPO_ROOT / "depends" / "toolchain.cmake.in").read_text(encoding="utf8") + clean_env = {key: value for key, value in os.environ.items() if key not in ("ASMFLAGS", "CFLAGS")} + for label, executable, prefix, cross, native in setups: + with self.subTest(toolchain=label): + compiler = shutil.which(executable) + if compiler is None: + skip_missing_compiler(self, executable) + assert compiler is not None + if cross is None and host_arch() != "x86_64": + self.skipTest("the host compiler builds no native ML-KEM code on this host") + system, arch, host = cross or ("", "", "") + values = { + "depends_crosscompiling": "TRUE" if cross else "FALSE", "host": host, + "host_system_name": system, "host_system_version": "", "host_arch": arch, + "CC": compiler, "CXX": compiler, "OSX_SDK": "", + "CFLAGS": "-pipe -DQBIT_DEPENDS_CFLAG", "CFLAGS_RELEASE": "-O2 -DQBIT_DEPENDS_RELEASE_CFLAG", + "CFLAGS_DEBUG": "-O1", "CXXFLAGS": "", "CXXFLAGS_RELEASE": "", "CXXFLAGS_DEBUG": "", + "CPPFLAGS": "", "CPPFLAGS_RELEASE": "", "CPPFLAGS_DEBUG": "", + "LDFLAGS": "", "LDFLAGS_RELEASE": "", "LDFLAGS_DEBUG": "", + **{tool.upper(): shutil.which(prefix + tool) or prefix + tool + for tool in ("ar", "ranlib", "strip", "objcopy", "objdump")}, + } + rendered = re.sub(r"@(\w+)@", lambda match: values.get(match.group(1), ""), template) + name = re.sub(r"\W+", "_", f"depends_{label}") + toolchain_dir = self.tmp_path / f"{name}_prefix" + toolchain_dir.mkdir() + (toolchain_dir / "toolchain.cmake").write_text(rendered, encoding="utf8") + configured = self.configure(name, compiler, f"-DCMAKE_TOOLCHAIN_FILE={toolchain_dir / 'toolchain.cmake'}", + env=clean_env) + self.assertEqual(configured.returncode, 0, configured.stdout + configured.stderr) + self.assertIn(f"ENABLE_MLKEM_NATIVE={'TRUE' if native else 'FALSE'}", configured.stdout + configured.stderr) + flags = self.compile_flags(name) + for flag in ("-DQBIT_DEPENDS_CFLAG", "-DQBIT_DEPENDS_RELEASE_CFLAG"): + self.assertIn(flag, flags["mlkem_native.c"], flags) + if not native: + self.assertNotIn("mlkem_native_asm.S", flags) + continue + # Except the MinGW flag that would re-encode the proved assembly (check_build). + c_flags = [flag for flag in flags["mlkem_native.c"] if flag != "-Wa,-muse-unaligned-vector-move"] + self.assertEqual(flags["mlkem_native_asm.S"], c_flags, + f"FAIL: with the {label} depends toolchain the assembly flags differ from the C flags\n" + f"Cause: mlkem_native_configure changed what depends builds pass to the assembler\nFix: {MODULE}") + + +class MlkemConfigHeaderTest(unittest.TestCase): + """src/crypto/mlkem_config.h refuses native code the policy forbids.""" + + def compile_backend(self, compiler: str, *flags: str) -> subprocess.CompletedProcess[str]: + return run([ + compiler, "-fsyntax-only", '-DMLK_CONFIG_FILE="crypto/mlkem_config.h"', + f"-I{REPO_ROOT / 'src'}", f"-I{REPO_ROOT / 'src' / 'mlkem-native'}", + *flags, str(REPO_ROOT / "src" / "crypto" / "mlkem_backend.c"), + ]) + + def test_guards(self) -> None: + cc = shutil.which("cc") + self.assertIsNotNone(cc) + assert cc is not None + self.assertEqual(self.compile_backend(cc).returncode, 0) + both = self.compile_backend(cc, "-DQBIT_MLKEM_NATIVE", "-DMLK_CONFIG_NO_ASM") + self.assertNotEqual(both.returncode, 0) + self.assertIn("QBIT_MLKEM_NATIVE and MLK_CONFIG_NO_ASM", both.stderr) + + cases = ( + ("clang-21", ("-fsanitize=memory", "-DQBIT_MLKEM_NATIVE"), "MemorySanitizer cannot see writes"), + ("aarch64-linux-gnu-gcc", ("-DQBIT_MLKEM_NATIVE",), "no BTI landing pads"), + ("cc", ("-m32", "-DQBIT_MLKEM_NATIVE"), "no native ML-KEM backend"), + ) + for executable, flags, message in cases: + compiler = shutil.which(executable) + with self.subTest(compiler=executable, flags=flags): + if compiler is None: + skip_missing_compiler(self, executable) + assert compiler is not None + if flags[0] == "-m32" and self.compile_backend(compiler, "-m32").returncode != 0: + skip_missing_compiler(self, "cc-m32", "has no 32-bit headers") + result = self.compile_backend(compiler, *flags) + self.assertNotEqual(result.returncode, 0, f"FAIL: {executable} {flags} compiled native code") + self.assertIn(message, result.stderr) + + + +COVERAGE_HARNESS = r""" +#include +#include +#include +#include +#include +#include +unsigned long AsmCalls(); +using namespace mlkem; +static std::array Seed() +{ + std::array seed{}; + for (size_t i = 0; i < seed.size(); ++i) seed[i] = uint8_t(i * 7 + 1); + return seed; +} +static const std::array COINS{}; +static std::vector RoundTrip() +{ + PublicKey ek; DecapsulationKey dk; Ciphertext ct; SharedSecret sent, received; + if (KeyGen(Seed(), ek, dk) != Error::NONE || CheckPublicKey(ek) != Error::NONE || + Encaps(ek, COINS, ct, sent) != Error::NONE || Decaps(dk, ct, received) != Error::NONE) return {}; + std::vector out(ek.begin(), ek.end()); + out.insert(out.end(), ct.begin(), ct.end()); + out.insert(out.end(), received.Bytes().begin(), received.Bytes().end()); + return out; +} +// The assembly calls of each entry point, run alone on a new thread, whose +// copy of the override is unset until the entry point takes it. +static std::array EachOperationOnAFreshThread(const PublicKey& ek, const DecapsulationKey& dk, const Ciphertext& ct) +{ + const std::function operations[4]{ + [] { PublicKey e; DecapsulationKey d; (void)KeyGen(Seed(), e, d); }, + [&] { (void)CheckPublicKey(ek); }, + [&] { Ciphertext c; SharedSecret s; (void)Encaps(ek, COINS, c, s); }, + [&] { SharedSecret s; (void)Decaps(dk, ct, s); }, + }; + std::array calls{}; + for (size_t i = 0; i < calls.size(); ++i) { + AsmCalls(); + std::thread{operations[i]}.join(); + calls[i] = AsmCalls(); + } + return calls; +} +int main() +{ + const auto detected{RoundTrip()}; + const unsigned long detected_calls{AsmCalls()}; + const auto active{GetBackendNames().arith}; + PublicKey ek; DecapsulationKey dk; Ciphertext ct; SharedSecret ss; + const bool inputs{KeyGen(Seed(), ek, dk) == Error::NONE && Encaps(ek, COINS, ct, ss) == Error::NONE}; + const auto fresh_detected{EachOperationOnAFreshThread(ek, dk, ct)}; + std::vector portable; + unsigned long portable_calls; + std::array fresh_portable; + { + ForcePortableForTesting force; + AsmCalls(); + portable = RoundTrip(); + portable_calls = AsmCalls(); + fresh_portable = EachOperationOnAFreshThread(ek, dk, ct); + } + std::printf("active=%.*s detected_calls=%lu portable_calls=%lu equal=%d ok=%d " + "fresh_detected=%lu,%lu,%lu,%lu fresh_portable=%lu,%lu,%lu,%lu\n", + int(active.size()), active.data(), detected_calls, portable_calls, int(detected == portable), + int(!detected.empty() && inputs), fresh_detected[0], fresh_detected[1], fresh_detected[2], fresh_detected[3], + fresh_portable[0], fresh_portable[1], fresh_portable[2], fresh_portable[3]); +} +""" + +# The entry points EachOperationOnAFreshThread runs, in its order. +OPERATIONS = ("KeyGen", "CheckPublicKey", "Encaps", "Decaps") + +# Test-only edits that let qbit's AArch64 glue build for AArch64 ELF, which +# release builds refuse (src/crypto/mlkem_config.h: no BTI landing pads), so +# that it can run under qemu-aarch64 on a Linux host. Each must match exactly once. +AARCH64_ELF_CONFIG_EDITS = ( + ("#if defined(MLK_SYS_AARCH64) && defined(__ELF__)\n" + "#error \"mlkem-native AArch64 assembly has no BTI landing pads or GNU property note; AArch64 ELF builds are portable only.\"\n" + "#endif\n", ""), + ("(defined(MLK_SYS_AARCH64) && defined(MLK_SYS_AARCH64_NEON) && defined(MLK_SYS_APPLE))", + "(defined(MLK_SYS_AARCH64) && defined(MLK_SYS_AARCH64_NEON))"), +) +# The negative control: upstream's single-lane Keccak glue, which runs its +# assembly without asking the capability hook, in place of qbit's mirror. +UPSTREAM_X1_EDIT = ( + re.compile(r"#define MLK_USE_NATIVE_FIPS202_X1\n.*?#endif // !__ASSEMBLER__\n", re.S), + "#include \n", +) + + +def edited(source: Path, edits: tuple) -> str: + text = source.read_text(encoding="utf8") + for old, new in edits: + count = len(old.findall(text)) if isinstance(old, re.Pattern) else text.count(old) + if count != 1: + raise AssertionError( + f"FAIL: test edit matched {count} times in {source}\n" + f"Cause: {source.name} changed, so the test can no longer derive its AArch64 ELF build\n" + f"Fix: update the edits in {Path(__file__).name} to the new text: {old if isinstance(old, str) else old.pattern!r}") + text = old.sub(new, text) if isinstance(old, re.Pattern) else text.replace(old, new) + return text + + +class MlkemOverrideCoverageTest(unittest.TestCase): + """The portable override reaches every assembly routine of each native backend. + + Every global symbol the assembly object defines is wrapped (ld --wrap) by a + counter. A missed entry point would still give identical outputs, so only + counting the calls can show it. Each wrapper entry point is also run alone + on a fresh thread, which shows one that runs the library without taking + the override first. + + x86_64 runs on x86_64 hosts. AArch64 is cross-built as a static Linux + binary and run under qemu-aarch64; release builds enable it only on macOS, + so the test builds a copy of src/crypto/mlkem_config.h without its AArch64 + ELF refusal. CI requires the AArch64 tools through + MLKEM_POLICY_REQUIRED_COMPILERS (aarch64-linux-gnu-g++ and qemu-aarch64). + """ + + def build_harness(self, out: Path, *, cc: str, cxx: str, nm: str, args: int, overrides: dict[str, str], + link_flags: tuple[str, ...] = ()) -> Path: + """Compile the library, qbit's glue and the counting harness; return the binary.""" + src = REPO_ROOT / "src" + include = out / "include" + for name, text in overrides.items(): + (include / "crypto").mkdir(parents=True, exist_ok=True) + (include / "crypto" / name).write_text(text, encoding="utf8") + # Overridden glue headers come first; qbit's own files fill in the rest. + flags = ['-DMLK_CONFIG_FILE="crypto/mlkem_config.h"', "-DQBIT_MLKEM_NATIVE", f"-I{include}", f"-I{src}", + f"-I{src / 'mlkem-native'}", "-O1"] + + def compile_(compiler: str, source: Path, *extra: str) -> Path: + obj = out / (source.name + ".o") + result = run([compiler, *flags, *extra, "-c", str(source), "-o", str(obj)]) + self.assertEqual(result.returncode, 0, result.stderr) + return obj + + objects = [ + compile_(cc, src / "mlkem-native" / "mlkem" / "mlkem_native.c"), + compile_(cc, src / "mlkem-native" / "mlkem" / "mlkem_native_asm.S"), + compile_(cc, src / "crypto" / "mlkem_backend.c"), + ] + asm_symbols = sorted(line.split()[2] for line in run([nm, "--defined-only", "-g", str(objects[1])]).stdout.splitlines() + if len(line.split()) == 3 and line.split()[1] == "T") + self.assertGreater(len(asm_symbols), 0, "no assembly routines: is the native backend compiled?") + # Every routine takes only integer or pointer arguments, at most as many + # as the ABI passes in registers (args), so a generic forwarder keeps them. + params = ", ".join("void*" for _ in range(args)) + named = ", ".join(f"void* a{i}" for i in range(args)) + forwarded = ", ".join(f"a{i}" for i in range(args)) + wrappers = ["#include ", "#include ", "static std::atomic g_calls{0};", 'extern "C" {'] + for symbol in asm_symbols: + wrappers.append(f"uint64_t __real_{symbol}({params});") + wrappers.append(f"uint64_t __wrap_{symbol}({named}) {{ ++g_calls; return __real_{symbol}({forwarded}); }}") + wrappers += ["}", "unsigned long AsmCalls() { return g_calls.exchange(0); }"] + (out / "wrap.cpp").write_text("\n".join(wrappers) + "\n", encoding="utf8") + (out / "harness.cpp").write_text(COVERAGE_HARNESS, encoding="utf8") + for source in (src / "crypto" / "mlkem.cpp", src / "compat" / "cpu_features.cpp", src / "support" / "cleanse.cpp", + out / "harness.cpp", out / "wrap.cpp"): + objects.append(compile_(cxx, source, "-std=c++20", "-pthread")) + binary = out / "harness" + link = run([cxx, "-pthread", *link_flags, "-o", str(binary), *map(str, objects), + *(f"-Wl,--wrap={symbol}" for symbol in asm_symbols)]) + self.assertEqual(link.returncode, 0, link.stderr) + return binary + + def run_harness(self, command: list[str]) -> dict[str, str]: + result = run(command) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + values = dict(item.split("=", 1) for item in result.stdout.split()) + self.assertEqual(values["ok"], "1", result.stdout) + self.assertEqual(values["equal"], "1", "the backends disagree") + return values + + def check_coverage(self, values: dict[str, str], native: str, mirror_hint: str) -> None: + self.assertEqual(values["active"], native, values) + self.assertGreater(int(values["detected_calls"]), 0, values) + self.assertEqual(values["portable_calls"], "0", + f"FAIL: assembly ran {values['portable_calls']} times with portable C forced\n" + "Cause: a native entry point does not ask mlk_sys_check_capability\n" + f"Fix: mirror its dispatch glue with a capability check, {mirror_hint}") + fresh_detected = [int(count) for count in values["fresh_detected"].split(",")] + fresh_portable = [int(count) for count in values["fresh_portable"].split(",")] + for operation, native_calls, portable_calls in zip(OPERATIONS, fresh_detected, fresh_portable): + # Without native calls of its own, an operation's portable count proves nothing. + self.assertGreater(native_calls, 0, f"{operation} runs no assembly natively: {values}") + self.assertEqual(portable_calls, 0, + f"FAIL: mlkem::{operation} ran assembly {portable_calls} times on a fresh thread with portable C forced\n" + "Cause: it enters the library without BeginOperation(), so this thread's copy of the override is stale\n" + "Fix: call BeginOperation() first in every entry point of src/crypto/mlkem.cpp") + + def test_x86_64_override_reaches_every_assembly_routine(self) -> None: + if host_arch() != "x86_64": + self.skipTest("the x86_64 backend runs only on x86_64 hosts") + cc, cxx, nm = shutil.which("cc"), shutil.which("c++"), shutil.which("nm") + if not (cc and cxx and nm): + self.skipTest("cc, c++ and nm are required") + assert cc is not None and cxx is not None and nm is not None + with tempfile.TemporaryDirectory() as tmp: + binary = self.build_harness(Path(tmp), cc=cc, cxx=cxx, nm=nm, args=6, overrides={}) + values = self.run_harness([str(binary)]) + if values["active"] != "x86_64-avx2": + # A CPU without the extensions runs portable C only: nothing to count. + self.assertEqual(values["detected_calls"], "0", values) + self.assertEqual(values["portable_calls"], "0", values) + self.skipTest("this CPU cannot run the x86_64 backend") + self.check_coverage(values, "x86_64-avx2", "as for AArch64's x1 Keccak in src/crypto/mlkem_fips202_backend.h") + + def aarch64_tools(self) -> tuple[str, str, str, str]: + tools = [] + for executable in ("aarch64-linux-gnu-gcc", "aarch64-linux-gnu-g++", "aarch64-linux-gnu-nm", "qemu-aarch64"): + path = shutil.which(executable) + if path is None: + skip_missing_compiler(self, executable) + assert path is not None + tools.append(path) + return tools[0], tools[1], tools[2], tools[3] + + def test_aarch64_override_reaches_every_assembly_routine(self) -> None: + cc, cxx, nm, qemu = self.aarch64_tools() + src = REPO_ROOT / "src" + config = edited(src / "crypto" / "mlkem_config.h", AARCH64_ELF_CONFIG_EDITS) + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "qbit" + out.mkdir() + binary = self.build_harness(out, cc=cc, cxx=cxx, nm=nm, args=8, overrides={"mlkem_config.h": config}, + link_flags=("-static",)) + values = self.run_harness([qemu, str(binary)]) + self.check_coverage(values, "aarch64-neon", + "as src/crypto/mlkem_fips202_backend.h does for upstream's x1_scalar.h") + + # Negative control: with upstream's x1 glue in place of qbit's + # mirror, forced portable runs leak assembly calls, and the count sees them. + upstream = Path(tmp) / "upstream-x1" + upstream.mkdir() + fips202 = edited(src / "crypto" / "mlkem_fips202_backend.h", (UPSTREAM_X1_EDIT,)) + binary = self.build_harness(upstream, cc=cc, cxx=cxx, nm=nm, args=8, + overrides={"mlkem_config.h": config, "mlkem_fips202_backend.h": fips202}, + link_flags=("-static",)) + control = self.run_harness([qemu, str(binary)]) + self.assertGreater(int(control["portable_calls"]), 0, + f"FAIL: upstream's x1 glue leaked no assembly calls with portable C forced: {control}\n" + "Cause: the counter no longer sees the x1 Keccak routine, so this test cannot catch a missing capability check\n" + "Fix: check that the --wrap list covers every global symbol of mlkem_native_asm.S") + + +if __name__ == "__main__": + unittest.main() diff --git a/ci/checks/test_pq_merge_gate_contract.py b/ci/checks/test_pq_merge_gate_contract.py new file mode 100755 index 0000000000..0895c9d159 --- /dev/null +++ b/ci/checks/test_pq_merge_gate_contract.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026-present The qbit core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. +"""Contract for the PQ unit jobs in Full Validation (#184 part 3, work item C3). + +Pull requests that touch the ML-KEM library, the v2 transport, or their build +and CI wiring must pass the aarch64-unit, macos-arm64-unit and +macos-x86_64-unit jobs of ci-pq.yml before they merge; others skip all three. +These tests run the real pieces of that chain: + +* ci/checks/classify_merge_profile.py, as the classify-changes job runs it, + reports pq_unit_required; +* the pq-unit job of ci.yml calls ci-pq.yml with those three jobs on the + change's own commit (a pull request's merge commit) exactly when that output + is true, and ci-pq.yml then runs exactly those three, so one output and one + call make each of them required exactly when the others are; +* full-validation-gate's shell accepts a skipped call only when the classifier + says it is not required, and fails on a missing classifier output; +* the changed-file lists that feed the classifier name both sides of a rename; +* every CMake file, workflow or ci/ file that mentions ML-KEM classifies as + requiring the job, so new wiring cannot be missed silently. + +GitHub itself is not called. +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from typing import Any + +import test_scheduled_validation_contract as svc + +REPO_ROOT = svc.REPO_ROOT +CI_WORKFLOW = svc.WORKFLOW_DIR / "ci.yml" +PQ_WORKFLOW = svc.WORKFLOW_DIR / "ci-pq.yml" +GATE_WORKFLOW = svc.WORKFLOW_DIR / "required-merge-gate.yml" +CLASSIFIER = REPO_ROOT / "ci" / "checks" / "classify_merge_profile.py" +CLASSIFY_JOB = "classify-changes" +PQ_JOB = "pq-unit" +GATE_JOB = "full-validation-gate" +GATE_STEP = "Check validation results" +AARCH64_JOB = "aarch64-unit" +# The ci-pq.yml jobs Full Validation requires: matrix entries of its unit job, +# and jobs of their own (the macOS ones), by the name the jobs input uses. +REQUIRED_PQ_JOBS = [AARCH64_JOB, "macos-arm64-unit", "macos-x86_64-unit"] +PQ_UNIT_MATRIX_JOB = "unit" +OUTPUT = "pq_unit_required" +MERGE_SHA = "1234567890abcdef1234567890abcdef12345678" +HEAD_SHA = "fedcba0987654321fedcba0987654321fedcba09" +CHANGED_FILES_STEP = "Write changed file list" +# Files that can wire ML-KEM into the build or CI: CMake files, workflows and +# composite actions, and everything under ci/. +WIRING_PATH_RE = re.compile(r"(^|/)CMakeLists[.]txt$|[.]cmake([.]in)?$|^cmake/|^[.]github/(workflows|actions)/|^ci/") +MLKEM_RE = re.compile(rb"ml[-_]?kem", re.IGNORECASE) + +MATCHING_PR = ["src/bip324.cpp"] +DOCS_ONLY_PR = ["doc/user/README.md"] +OTHER_SOURCE_PR = ["src/wallet/wallet.cpp"] + +# Every other job the gate checks, passing. +OTHER_GATE_RESULTS = { + "RUNNERS_RESULT": "success", + "WINDOWS_NATIVE_DLL_RESULT": "success", + "WINDOWS_CROSS_RESULT": "success", + "WINDOWS_NATIVE_TEST_RESULT": "success", + "CI_MATRIX_RESULT": "success", + "LINT_RESULT": "success", + "PUBLIC_DOCS_LINT_RESULT": "success", + "RPC_DOCS_RESULT": "success", +} + + +def classify(paths: list[str]) -> dict[str, str]: + """Run the classifier the way classify-changes does and return its GitHub outputs.""" + with tempfile.TemporaryDirectory() as tmp: + changed = Path(tmp) / "changed-files.txt" + changed.write_text("".join(f"{path}\n" for path in paths), encoding="utf8") + output = Path(tmp) / "github-output" + output.touch() + completed = subprocess.run( + [sys.executable, str(CLASSIFIER), "--changed-files", str(changed), "--github-output", str(output)], + cwd=REPO_ROOT, text=True, capture_output=True, check=False, + ) + assert completed.returncode == 0, completed.stderr + return svc.parse_key_values(output.read_text(encoding="utf8")) + + +def pull_request_context(outputs: dict[str, str], *, draft: bool = False) -> dict[str, Any]: + return { + "github": { + "event_name": "pull_request", + "sha": MERGE_SHA, + # The pull request's head differs from the merge commit the job must validate. + "event": {"pull_request": {"draft": draft, "head": {"sha": HEAD_SHA}}}, + }, + "needs": {CLASSIFY_JOB: {"outputs": outputs, "result": "success"}}, + } + + +class PQMergeGateContractTest(unittest.TestCase): + ci: dict[str, Any] + pq: dict[str, Any] + + @classmethod + def setUpClass(cls) -> None: + cls.ci = svc.load_workflow(CI_WORKFLOW) + cls.pq = svc.load_workflow(PQ_WORKFLOW) + + def job_runs(self, context: dict[str, Any]) -> bool: + return svc._truthy(svc.render(self.ci["jobs"][PQ_JOB]["if"], context)) + + def run_gate(self, **env: str) -> subprocess.CompletedProcess[str]: + step = svc.find_step(self.ci["jobs"][GATE_JOB], GATE_STEP) + full_env = {"PATH": os.environ["PATH"], "LC_ALL": "C", "CLASSIFY_CHANGES_RESULT": "success", + "VALIDATION_PROFILE": "source", "SOURCE_VALIDATION_REQUIRED": "true", "CI_PROFILE": "internal"} + full_env.update(OTHER_GATE_RESULTS) + full_env.update(env) + with tempfile.TemporaryDirectory() as tmp: + script = Path(tmp) / "gate.sh" + script.write_text(step["run"], encoding="utf8") + return subprocess.run(["bash", "--noprofile", "--norc", "-eo", "pipefail", str(script)], + env=full_env, text=True, capture_output=True, check=False) + + def select_pq_jobs(self, jobs: str) -> dict[str, str]: + """Run ci-pq.yml's selection step with the jobs input the caller passes.""" + step = svc.find_step(self.pq["jobs"][svc.RESOLVER_JOB], "Select jobs") + env = {key: str(value) for key, value in step["env"].items() if key != "JOBS_INPUT"} + env.update({"PATH": os.environ["PATH"], "JOBS_INPUT": jobs}) + with tempfile.TemporaryDirectory() as tmp: + script = Path(tmp) / "select.py" + script.write_text(step["run"], encoding="utf8") + output = Path(tmp) / "github-output" + output.touch() + env["GITHUB_OUTPUT"] = str(output) + env["GITHUB_STEP_SUMMARY"] = os.devnull + completed = subprocess.run([sys.executable, str(script)], env=env, text=True, capture_output=True, check=False) + self.assertEqual(completed.returncode, 0, completed.stderr) + return svc.parse_key_values(output.read_text(encoding="utf8")) + + # -- wiring ---------------------------------------------------------------------- + + def test_classify_changes_exposes_the_output(self) -> None: + job = self.ci["jobs"][CLASSIFY_JOB] + self.assertEqual(job["outputs"][OUTPUT], "${{ steps.classify.outputs." + OUTPUT + " }}") + classify_step = svc.find_step(job, "Classify validation profile") + self.assertIn("ci/checks/classify_merge_profile.py", classify_step["run"]) + self.assertIn('--github-output "$GITHUB_OUTPUT"', classify_step["run"]) + + def running_pq_jobs(self, outputs: dict[str, str]) -> dict[str, list[str]]: + """The ci-pq.yml jobs whose if passes with these selection outputs, with their matrix entries.""" + context = {"needs": {svc.RESOLVER_JOB: {"outputs": outputs, "result": "success"}}} + running: dict[str, list[str]] = {} + for job_id, job in self.pq["jobs"].items(): + if job_id == svc.RESOLVER_JOB or not svc._truthy(svc.render(job["if"], context)): + continue + matrix = (job.get("strategy") or {}).get("matrix") + running[job_id] = [entry["job"] for entry in json.loads(outputs["matrix"])["include"]] if matrix else [] + return running + + def test_pq_job_calls_the_required_jobs_on_the_change_itself(self) -> None: + job = self.ci["jobs"][PQ_JOB] + self.assertEqual(job["uses"], "./.github/workflows/ci-pq.yml") + self.assertEqual(job["needs"], CLASSIFY_JOB) + context = pull_request_context({OUTPUT: "true"}) + rendered = {key: svc._to_text(svc.render(value, context)) for key, value in job["with"].items()} + self.assertEqual(rendered, {"source_ref": MERGE_SHA, "jobs": ",".join(REQUIRED_PQ_JOBS)}) + self.assertEqual(set(job["with"]), set(self.pq["on"]["workflow_call"]["inputs"])) + + outputs = self.select_pq_jobs(rendered["jobs"]) + self.assertEqual(json.loads(outputs["selected"]), REQUIRED_PQ_JOBS) + # Exactly these run: aarch64-unit as the unit job's only entry, each + # macOS job as itself, and nothing else (s390x-unit stays on demand). + self.assertEqual(self.running_pq_jobs(outputs), + {PQ_UNIT_MATRIX_JOB: [AARCH64_JOB], "macos-arm64-unit": [], "macos-x86_64-unit": []}) + + def test_required_merge_gate_runs_this_contract(self) -> None: + gate = svc.load_workflow(GATE_WORKFLOW) + commands = "\n".join(step.get("run", "") for step in gate["jobs"][CLASSIFY_JOB]["steps"]) + self.assertIn(f"ci/checks/{Path(__file__).name}", commands) + + def test_changed_files_name_both_sides_of_a_rename(self) -> None: + """A rename out of a gated path must still require the job: git diff lists only the new path by default.""" + with tempfile.TemporaryDirectory() as tmp: + repo = Path(tmp) / "repo" + repo.mkdir() + git_env = {"PATH": os.environ["PATH"], "HOME": tmp, "GIT_CONFIG_NOSYSTEM": "1", + "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"} + + def git(*args: str) -> str: + return subprocess.run(["git", "-c", "commit.gpgsign=false", *args], cwd=repo, env=git_env, text=True, + capture_output=True, check=True).stdout.strip() + + git("init", "-q") + (repo / "src" / "crypto").mkdir(parents=True) + (repo / "src" / "crypto" / "mlkem.cpp").write_text("".join(f"line {i}\n" for i in range(50)), encoding="utf8") + git("add", "-A") + git("commit", "-q", "-m", "base") + base = git("rev-parse", "HEAD") + (repo / "src" / "crypto" / "kem").mkdir() + git("mv", "src/crypto/mlkem.cpp", "src/crypto/kem/kyber.cpp") + with open(repo / "src" / "crypto" / "kem" / "kyber.cpp", "a", encoding="utf8") as handle: + handle.write("edited\n") + git("commit", "-q", "-am", "rename") + self.assertEqual(git("diff", "--name-only", f"{base}...HEAD"), "src/crypto/kem/kyber.cpp", "git detects the rename") + + for workflow_path in (CI_WORKFLOW, GATE_WORKFLOW): + with self.subTest(workflow=workflow_path.name): + step = svc.find_step(svc.load_workflow(workflow_path)["jobs"][CLASSIFY_JOB], CHANGED_FILES_STEP) + output = Path(tmp) / "github-output" + output.write_text("", encoding="utf8") + env = dict(git_env, EVENT_NAME="pull_request", BASE_SHA=base, RUNNER_TEMP=tmp, GITHUB_OUTPUT=str(output)) + script = Path(tmp) / "step.sh" + script.write_text(step["run"], encoding="utf8") + completed = subprocess.run(["bash", "--noprofile", "--norc", "-eo", "pipefail", str(script)], + cwd=repo, env=env, text=True, capture_output=True, check=False) + self.assertEqual(completed.returncode, 0, completed.stderr) + changed = Path(svc.parse_key_values(output.read_text(encoding="utf8"))["changed_files"]) + paths = changed.read_text(encoding="utf8").split() + self.assertEqual(sorted(paths), ["src/crypto/kem/kyber.cpp", "src/crypto/mlkem.cpp"]) + self.assertEqual(classify(paths)[OUTPUT], "true") + + def test_gate_reads_the_job_and_the_classifier(self) -> None: + gate = self.ci["jobs"][GATE_JOB] + self.assertIn(PQ_JOB, gate["needs"]) + self.assertIn("always()", gate["if"]) + env = svc.find_step(gate, GATE_STEP)["env"] + self.assertEqual(env["PQ_UNIT_REQUIRED"], "${{ needs." + CLASSIFY_JOB + ".outputs." + OUTPUT + " }}") + self.assertEqual(env["PQ_UNIT_RESULT"], "${{ needs." + PQ_JOB + ".result }}") + + def test_job_condition(self) -> None: + self.assertTrue(self.job_runs(pull_request_context({OUTPUT: "true"}))) + self.assertFalse(self.job_runs(pull_request_context({OUTPUT: "false"}))) + self.assertFalse(self.job_runs(pull_request_context({})), "a missing output never starts the job") + self.assertFalse(self.job_runs(pull_request_context({OUTPUT: "true"}, draft=True))) + push = pull_request_context({OUTPUT: "true"}) + push["github"] = {"event_name": "push", "sha": MERGE_SHA, "event": {}} + self.assertTrue(self.job_runs(push)) + + # -- the gate ---------------------------------------------------------------------- + + def test_gate_outcomes(self) -> None: + cases = [ + ("required and passed", "true", "success", True, ""), + ("required but skipped", "true", "skipped", False, "PQ unit tests result was skipped"), + ("required but failed", "true", "failure", False, "PQ unit tests result was failure"), + ("required but cancelled", "true", "cancelled", False, "PQ unit tests result was cancelled"), + ("not required, skipped", "false", "skipped", True, ""), + ("not required, failed anyway", "false", "failure", False, "PQ unit tests result was failure"), + ("classifier output missing", "", "skipped", False, "did not report whether the PQ unit jobs are required"), + ("classifier output garbled", "yes", "success", False, "did not report whether the PQ unit jobs are required"), + ] + for label, required, result, ok, message in cases: + with self.subTest(label): + run = self.run_gate(PQ_UNIT_REQUIRED=required, PQ_UNIT_RESULT=result) + self.assertEqual(run.returncode == 0, ok, run.stdout + run.stderr) + if message: + self.assertIn(message, run.stdout) + + # A missing output fails even a change that needs no source validation. + run = self.run_gate(PQ_UNIT_REQUIRED="", PQ_UNIT_RESULT="skipped", + SOURCE_VALIDATION_REQUIRED="false", VALIDATION_PROFILE="public-docs") + self.assertNotEqual(run.returncode, 0) + + # -- end to end ---------------------------------------------------------------------- + + def gate_for_pull_request(self, paths: list[str]) -> tuple[bool, subprocess.CompletedProcess[str]]: + """Classify, decide whether the job runs (passing when it does), and run the gate.""" + outputs = classify(paths) + runs = self.job_runs(pull_request_context(outputs)) + source = outputs["source_validation_required"] + others = OTHER_GATE_RESULTS if source == "true" else {key: "skipped" for key in OTHER_GATE_RESULTS} + gate = self.run_gate(PQ_UNIT_REQUIRED=outputs[OUTPUT], PQ_UNIT_RESULT="success" if runs else "skipped", + SOURCE_VALIDATION_REQUIRED=source, VALIDATION_PROFILE=outputs["profile"], **others) + return runs, gate + + def test_matching_pull_request_requires_and_runs_the_job(self) -> None: + runs, gate = self.gate_for_pull_request(MATCHING_PR) + self.assertTrue(runs) + self.assertEqual(gate.returncode, 0, gate.stdout + gate.stderr) + self.assertIn("PQ unit tests: success", gate.stdout) + # Skipping it would fail the gate. + outputs = classify(MATCHING_PR) + self.assertNotEqual(self.run_gate(PQ_UNIT_REQUIRED=outputs[OUTPUT], PQ_UNIT_RESULT="skipped").returncode, 0) + + def test_other_pull_requests_skip_the_job_and_pass(self) -> None: + for paths in (DOCS_ONLY_PR, OTHER_SOURCE_PR): + with self.subTest(paths=paths): + runs, gate = self.gate_for_pull_request(paths) + self.assertFalse(runs) + self.assertEqual(gate.returncode, 0, gate.stdout + gate.stderr) + + +def repository_files() -> list[str]: + """Tracked files, and new ones git does not ignore.""" + completed = subprocess.run(["git", "ls-files", "-z", "--cached", "--others", "--exclude-standard"], + cwd=REPO_ROOT, capture_output=True, check=False) + assert completed.returncode == 0, completed.stderr.decode(errors="replace") + return sorted({path for path in completed.stdout.decode("utf8").split("\0") if (REPO_ROOT / path).is_file()}) + + +class MlkemWiringContractTest(unittest.TestCase): + """The classifier knows every file that wires ML-KEM into the build or CI.""" + + def test_wiring_that_mentions_mlkem_requires_the_job(self) -> None: + wiring = [path for path in repository_files() + if WIRING_PATH_RE.search(path) and MLKEM_RE.search((REPO_ROOT / path).read_bytes())] + # The scan proves nothing unless it finds the wiring known today. + for known in ("CMakeLists.txt", "src/CMakeLists.txt", "cmake/mlkem-native.cmake", ".github/workflows/ci.yml", + "ci/checks/classify_merge_profile.py", "ci/test/03_test_script.sh"): + self.assertIn(known, wiring) + missing = [path for path in wiring if classify([path])[OUTPUT] != "true"] + self.assertEqual(missing, [], + "FAIL: these files mention ML-KEM but do not require the PQ unit jobs\n" + "Cause: a change to only one of them would skip pq-unit, and Full Validation accepts the skip\n" + "Fix: add them to PQ_UNIT_FILES in ci/checks/classify_merge_profile.py") + + +if __name__ == "__main__": + unittest.main() diff --git a/ci/test/00_setup_env_mac_native_pq.sh b/ci/test/00_setup_env_mac_native_pq.sh new file mode 100755 index 0000000000..f2777eebea --- /dev/null +++ b/ci/test/00_setup_env_mac_native_pq.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2026-present The qbit core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +# The macos-arm64-unit and macos-x86_64-unit jobs of .github/workflows/ci-pq.yml: +# build only the unit-test binary natively on an Apple Silicon or Intel runner, +# without Docker, and run the post-quantum suites. These are the targets that +# run ML-KEM's native AArch64 and x86_64 code on macOS. +export CONTAINER_NAME="ci_mac_native_pq" # macos does not use a container, but the env var is needed for logging +export CMAKE_GENERATOR="Ninja" +export CI_OS_NAME="macos" +# Boost and libevent come from depends, built natively for the runner, so the +# jobs compile against the versions qbit releases with. Homebrew's Boost moves +# on its own: 1.92 stopped compiling the multi_index declarations in +# txmempool.h (fixed upstream by bitcoin/bitcoin#35175). +export DEP_OPTS="NO_QT=1 NO_QR=1 NO_ZMQ=1 NO_WALLET=1 NO_USDT=1 NO_IPC=1" +export OSX_SDK="" +export RUN_UNIT_TESTS=true +export RUN_FUNCTIONAL_TESTS=false +export CI_BUILD_TARGET=test_bitcoin +# The suites this job must run; ctest_evidence.py fails the job unless each was +# selected, executed and passed. CTEST_REGEX is derived from the same list. +export CTEST_EXPECTED_SUITES="mlkem_tests bip324_tests net_tests" +export CTEST_REGEX="^(${CTEST_EXPECTED_SUITES// /|})\$" +# WITH_MLKEM_NATIVE=ON makes losing the native backend a configure error instead +# of a quiet fall back to portable C. +export BITCOIN_CONFIG="-DBUILD_GUI=OFF -DENABLE_WALLET=OFF -DENABLE_IPC=OFF -DWITH_ZMQ=OFF -DREDUCE_EXPORTS=ON -DWITH_MLKEM_NATIVE=ON" diff --git a/ci/test/00_setup_env_native_aarch64_pq.sh b/ci/test/00_setup_env_native_aarch64_pq.sh index ddb42f03c7..2b50cf8a83 100755 --- a/ci/test/00_setup_env_native_aarch64_pq.sh +++ b/ci/test/00_setup_env_native_aarch64_pq.sh @@ -20,7 +20,7 @@ export RUN_FUNCTIONAL_TESTS=false export CI_BUILD_TARGET=test_bitcoin # The suites this job must run; ctest_evidence.py fails the job unless each was # selected, executed and passed. CTEST_REGEX is derived from the same list. -export CTEST_EXPECTED_SUITES="bip324_tests net_tests" +export CTEST_EXPECTED_SUITES="mlkem_tests bip324_tests net_tests" export CTEST_REGEX="^(${CTEST_EXPECTED_SUITES// /|})\$" # -Wno-psabi silences GCC's AArch64 "parameter passing ... changed" notes. export BITCOIN_CONFIG="\ diff --git a/ci/test/00_setup_env_s390x_unit.sh b/ci/test/00_setup_env_s390x_unit.sh new file mode 100755 index 0000000000..5f28461617 --- /dev/null +++ b/ci/test/00_setup_env_s390x_unit.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2026-present The qbit core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +# shellcheck source=ci/test/00_setup_env_base_image.sh +source "$( dirname "${BASH_SOURCE[0]}" )/00_setup_env_base_image.sh" + +# The s390x-unit job of .github/workflows/ci-pq.yml: cross-compile only the +# unit-test binary for big-endian s390x on the x64 runner, with depends, and run +# the post-quantum suites under qemu-s390x. ctest adds the emulator to each test +# command, so only the tests run emulated; test_qbit is never run directly. +export HOST=s390x-linux-gnu +export CONTAINER_NAME=ci_s390x_pq +ci_set_base_image_name_tag "ubuntu:24.04" +export PACKAGES="g++-s390x-linux-gnu qemu-user" +export DEP_OPTS="NO_QT=1 NO_QR=1 NO_ZMQ=1 NO_WALLET=1 NO_USDT=1 NO_IPC=1" +export RUN_UNIT_TESTS=true +export RUN_FUNCTIONAL_TESTS=false +export CI_BUILD_TARGET=test_bitcoin +# The suites this job must run; ctest_evidence.py fails the job unless each was +# selected, executed and passed. CTEST_REGEX is derived from the same list. +export CTEST_EXPECTED_SUITES="mlkem_tests bip324_tests net_tests" +export CTEST_REGEX="^(${CTEST_EXPECTED_SUITES// /|})\$" +export BITCOIN_CONFIG="\ + -DBUILD_GUI=OFF -DENABLE_WALLET=OFF -DENABLE_IPC=OFF -DWITH_ZMQ=OFF -DREDUCE_EXPORTS=ON \ + -DCMAKE_CROSSCOMPILING_EMULATOR='qemu-s390x;-L;/usr/s390x-linux-gnu' \ +" diff --git a/ci/test/03_test_script.sh b/ci/test/03_test_script.sh index 20dad10606..6399230d18 100755 --- a/ci/test/03_test_script.sh +++ b/ci/test/03_test_script.sh @@ -350,6 +350,16 @@ if [ "$RUN_FUZZ_TESTS" = "true" ]; then "${DIR_FUZZ_IN}" \ --empty_min_time=60 \ --require_qbit_corpus + # Replay the ML-KEM targets again on portable C, which they select at start-up. + # shellcheck disable=SC2086 + MLKEM_FORCE_PORTABLE=1 \ + LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" \ + "${BASE_BUILD_DIR}/test/fuzz/test_runner.py" \ + ${FUZZ_TESTS_CONFIG} \ + "${MAKEJOBS}" \ + -l DEBUG \ + "${DIR_FUZZ_IN}" \ + mlkem mlkem_backend_diff if [ -n "${QBIT_FUZZ_MUTATE_MIN_TIME}" ]; then # Seeded mutation phase over the qbit seed corpora, after the replay above. # shellcheck disable=SC2086 @@ -361,6 +371,6 @@ if [ "$RUN_FUZZ_TESTS" = "true" ]; then --require_qbit_corpus \ --mutate_min_time="${QBIT_FUZZ_MUTATE_MIN_TIME}" \ "${DIR_FUZZ_IN}" \ - asert_chain_transition asert_edge_cases asert_math auxpow p2mr_script pqc + asert_chain_transition asert_edge_cases asert_math auxpow mlkem mlkem_backend_diff p2mr_script pqc fi fi diff --git a/cmake/bitcoin-build-config.h.in b/cmake/bitcoin-build-config.h.in index cdc92893e7..eb9ee52b0c 100644 --- a/cmake/bitcoin-build-config.h.in +++ b/cmake/bitcoin-build-config.h.in @@ -39,6 +39,10 @@ /* Define to 1 to enable wallet functions. */ #cmakedefine ENABLE_WALLET 1 +/* Define to 1 if mlkem-native's native ML-KEM backends are compiled in, and to + 0 if not. */ +#cmakedefine01 ENABLE_MLKEM_NATIVE + /* Define to 1 if you have the declaration of `fork', and to 0 if you don't. */ #cmakedefine01 HAVE_DECL_FORK @@ -70,6 +74,9 @@ /* Define this symbol if you have malloc_info */ #cmakedefine HAVE_MALLOC_INFO 1 +/* Define this symbol if __builtin_cpu_supports links for the target */ +#cmakedefine HAVE_BUILTIN_CPU_SUPPORTS 1 + /* Define this symbol if you have mallopt with M_ARENA_MAX */ #cmakedefine HAVE_MALLOPT_ARENA_MAX 1 diff --git a/cmake/introspection.cmake b/cmake/introspection.cmake index d225763397..c15a0274f3 100644 --- a/cmake/introspection.cmake +++ b/cmake/introspection.cmake @@ -39,6 +39,18 @@ check_cxx_source_compiles(" " STRERROR_R_CHAR_P ) +# Check that __builtin_cpu_supports links for the target. It needs __cpu_model +# from the compiler's runtime library, which some cross toolchains (macOS from +# depends) don't provide. Tests use it only as an independent CPU check. +check_cxx_source_compiles(" + int main() + { + __builtin_cpu_init(); + return __builtin_cpu_supports(\"avx2\") ? 0 : 1; + } + " HAVE_BUILTIN_CPU_SUPPORTS +) + # Check for malloc_info (for memory statistics information in getmemoryinfo). check_cxx_symbol_exists(malloc_info "malloc.h" HAVE_MALLOC_INFO) diff --git a/cmake/mlkem-native.cmake b/cmake/mlkem-native.cmake new file mode 100644 index 0000000000..7fba4d7d54 --- /dev/null +++ b/cmake/mlkem-native.cmake @@ -0,0 +1,246 @@ +# Copyright (c) 2026-present The qbit core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. + +# Builds the vendored mlkem-native library (src/mlkem-native) as ML-KEM-1024. +# See doc/subtrees/mlkem-native.md and src/crypto/mlkem_config.h. +# +# Native backends (x86_64 AVX2 and AArch64 assembly) are selected at run time, +# so no source is ever compiled with an AVX2 flag and the portable C fallback +# runs on every CPU. WITH_MLKEM_NATIVE chooses whether they are built: +# AUTO build them where supported (the default); +# ON like AUTO, but a target that cannot build them is a configure error; +# OFF portable C only. + +include_guard(GLOBAL) + +# Decide whether native backends are built. This depends on its arguments only, +# so ci/checks/test_mlkem_build_policy.py can run it for any target in script +# mode. +# mode WITH_MLKEM_NATIVE: AUTO, ON or OFF +# arch from mlkem_native_detect_arch: x86_64, aarch64-macho, +# aarch64-elf, or a description of another target +# msvc true for MSVC-style toolchains, which cannot assemble GNU syntax +# sanitizers the SANITIZERS value +# +# AArch64 ELF targets (Linux and the BSDs) are portable only: mlkem-native's +# AArch64 assembly has no BTI landing pads and no GNU property note, so linking +# it would strip the BTI and PAC marking from the whole binary. Hardening wins; +# see doc/subtrees/mlkem-native.md for the evidence and when to revisit. +# Sets MLKEM_NATIVE_ENABLED, MLKEM_NATIVE_NO_ASM and MLKEM_NATIVE_REASON in the +# caller's scope. An invalid mode, or ON where native code is impossible, is a +# configure error. +function(mlkem_native_policy mode arch msvc sanitizers) + string(TOUPPER "${mode}" mode) + if(NOT mode MATCHES "^(AUTO|ON|OFF)$") + message(FATAL_ERROR "WITH_MLKEM_NATIVE must be AUTO, ON or OFF, not \"${mode}\".") + endif() + + set(no_asm FALSE) + set(blocker "") + if(msvc) + set(no_asm TRUE) + set(blocker "MSVC toolchains cannot assemble mlkem-native's GNU-syntax assembly") + elseif(arch STREQUAL "aarch64-elf") + set(blocker "mlkem-native AArch64 assembly has no BTI landing pads or GNU property note") + elseif(NOT arch STREQUAL "x86_64" AND NOT arch STREQUAL "aarch64-macho") + set(blocker "mlkem-native has no native backend qbit enables for ${arch}") + elseif(sanitizers MATCHES "(^|,)memory($|,)") + set(blocker "MemorySanitizer cannot see writes made by assembly") + endif() + + if(mode STREQUAL "OFF") + set(enabled FALSE) + set(reason "WITH_MLKEM_NATIVE=OFF") + elseif(blocker) + if(mode STREQUAL "ON") + message(FATAL_ERROR + "WITH_MLKEM_NATIVE=ON, but ${blocker}. " + "Configure with WITH_MLKEM_NATIVE=AUTO or OFF to build portable C." + ) + endif() + set(enabled FALSE) + set(reason "${blocker}") + else() + set(enabled TRUE) + set(reason "WITH_MLKEM_NATIVE=${mode} on ${arch}") + endif() + + set(MLKEM_NATIVE_ENABLED ${enabled} PARENT_SCOPE) + set(MLKEM_NATIVE_NO_ASM ${no_asm} PARENT_SCOPE) + set(MLKEM_NATIVE_REASON "${reason}" PARENT_SCOPE) +endfunction() + +# Name the target for mlkem_native_policy by asking the C compiler, with the +# flags in use, which architecture it generates code for. The tests mirror the +# conditions under which src/crypto/mlkem_config.h enables a backend. +function(mlkem_native_detect_arch var) + include(CheckCSourceCompiles) + set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) + # The library's compiles get APPEND_CPPFLAGS and APPEND_CFLAGS, which can + # change the target (-m32, say), so the probes must see them too. Probe + # afresh on every configure, as those flags may have changed. + string(STRIP "${CMAKE_REQUIRED_FLAGS} ${APPEND_CPPFLAGS} ${APPEND_CFLAGS}" CMAKE_REQUIRED_FLAGS) + unset(MLKEM_NATIVE_TARGET_X86_64 CACHE) + unset(MLKEM_NATIVE_TARGET_AARCH64_MACHO CACHE) + unset(MLKEM_NATIVE_TARGET_AARCH64_ELF CACHE) + check_c_source_compiles(" + #if !((defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64)) && !defined(__ILP32__)) + #error not LP64 x86_64 + #endif + int main(void) { return 0; } + " MLKEM_NATIVE_TARGET_X86_64 + ) + check_c_source_compiles(" + #if !((defined(__AARCH64EL__) || defined(_M_ARM64)) && defined(__ARM_NEON) && !defined(__ILP32__)) + #error not little-endian LP64 AArch64 with NEON + #endif + #if !defined(__APPLE__) + #error not Mach-O + #endif + int main(void) { return 0; } + " MLKEM_NATIVE_TARGET_AARCH64_MACHO + ) + check_c_source_compiles(" + #if !((defined(__AARCH64EL__) || defined(_M_ARM64)) && defined(__ELF__)) + #error not AArch64 ELF + #endif + int main(void) { return 0; } + " MLKEM_NATIVE_TARGET_AARCH64_ELF + ) + if(MLKEM_NATIVE_TARGET_X86_64) + set(${var} "x86_64" PARENT_SCOPE) + elseif(MLKEM_NATIVE_TARGET_AARCH64_MACHO) + set(${var} "aarch64-macho" PARENT_SCOPE) + elseif(MLKEM_NATIVE_TARGET_AARCH64_ELF) + set(${var} "aarch64-elf" PARENT_SCOPE) + else() + set(${var} "this target (${CMAKE_SYSTEM_PROCESSOR})" PARENT_SCOPE) + endif() +endfunction() + +# Apply WITH_MLKEM_NATIVE to this build. Call from the top-level directory, at +# file scope, after SANITIZERS is final. Sets MLKEM_NATIVE_ENABLED, +# MLKEM_NATIVE_NO_ASM, MLKEM_NATIVE_REASON and MLKEM_NATIVE_SUMMARY. A macro, +# because enable_language() must run at file scope. +macro(mlkem_native_configure) + enable_language(C) + if(MSVC) + set(_mlkem_native_arch "MSVC") + else() + mlkem_native_detect_arch(_mlkem_native_arch) + endif() + mlkem_native_policy("${WITH_MLKEM_NATIVE}" "${_mlkem_native_arch}" "${MSVC}" "${SANITIZERS}") + if(MLKEM_NATIVE_ENABLED) + # Assemble with exactly the C compiler invocation, including the arguments + # and target that cross toolchains (depends) give it. + set(CMAKE_ASM_COMPILER "${CMAKE_C_COMPILER}") + set(CMAKE_ASM_COMPILER_ARG1 "${CMAKE_C_COMPILER_ARG1}") + if(DEFINED CMAKE_C_COMPILER_TARGET) + set(CMAKE_ASM_COMPILER_TARGET "${CMAKE_C_COMPILER_TARGET}") + endif() + # enable_language(ASM) appends CMake's defaults to a toolchain's + # CMAKE_ASM_FLAGS__INIT, so note now which ones the toolchain set. + foreach(_mlkem_native_config IN ITEMS DEBUG RELEASE RELWITHDEBINFO MINSIZEREL) + set(_mlkem_native_init_${_mlkem_native_config} FALSE) + if(DEFINED CMAKE_ASM_FLAGS_${_mlkem_native_config}_INIT) + set(_mlkem_native_init_${_mlkem_native_config} TRUE) + endif() + endforeach() + enable_language(ASM) + # The C flags come first, then the ASM flags CMake initialized from + # -DCMAKE_ASM_FLAGS, the ASMFLAGS environment variable or the toolchain + # (target, sysroot or hardening flags), which win on conflict. Normal + # variables shadow the cache without changing it, so a reconfigure starts + # again from the same values. + string(STRIP "${CMAKE_C_FLAGS} ${CMAKE_ASM_FLAGS}" CMAKE_ASM_FLAGS) + # Per configuration, CMake's own ASM default (its _INIT value) gives way to + # the C flags; a value the user or the toolchain set is kept. + foreach(_mlkem_native_config IN ITEMS DEBUG RELEASE RELWITHDEBINFO MINSIZEREL) + string(STRIP "${CMAKE_ASM_FLAGS_${_mlkem_native_config}}" _mlkem_native_value) + string(STRIP "${CMAKE_ASM_FLAGS_${_mlkem_native_config}_INIT}" _mlkem_native_default) + if(NOT _mlkem_native_init_${_mlkem_native_config} AND _mlkem_native_value STREQUAL _mlkem_native_default) + set(CMAKE_ASM_FLAGS_${_mlkem_native_config} "${CMAKE_C_FLAGS_${_mlkem_native_config}}") + endif() + unset(_mlkem_native_init_${_mlkem_native_config}) + endforeach() + unset(_mlkem_native_value) + unset(_mlkem_native_default) + unset(_mlkem_native_config) + if(_mlkem_native_arch STREQUAL "x86_64") + set(MLKEM_NATIVE_SUMMARY "native x86_64-avx2 if the CPU has AVX2, SSSE3, SSE4.1, POPCNT and BMI2, else portable (${MLKEM_NATIVE_REASON})") + else() + set(MLKEM_NATIVE_SUMMARY "native aarch64-neon (${MLKEM_NATIVE_REASON})") + endif() + else() + set(MLKEM_NATIVE_SUMMARY "portable (${MLKEM_NATIVE_REASON})") + endif() + unset(_mlkem_native_arch) + # For bitcoin-build-config.h. + set(ENABLE_MLKEM_NATIVE ${MLKEM_NATIVE_ENABLED}) +endmacro() + +# Create the mlkem_native static library from the vendored tree in +# ${PROJECT_SOURCE_DIR}/src/mlkem-native, as configured by +# mlkem_native_configure(). Link it PRIVATE into bitcoin_node only; its +# interface lets bitcoin_node's ML-KEM wrapper include mlkem/mlkem_native.h. +# +# The library is C and assembly, so it takes core_interface's own compile +# options and definitions (hardening flags such as -fcf-protection and +# -mbranch-protection) and the sanitizer flags, but not qbit's C++ warning +# set: the vendored code is never edited to silence warnings. +function(add_mlkem_native) + set(vendor_dir ${PROJECT_SOURCE_DIR}/src/mlkem-native) + add_library(mlkem_native STATIC EXCLUDE_FROM_ALL + ${vendor_dir}/mlkem/mlkem_native.c + ${PROJECT_SOURCE_DIR}/src/crypto/mlkem_backend.c + ) + if(MLKEM_NATIVE_ENABLED) + target_sources(mlkem_native PRIVATE ${vendor_dir}/mlkem/mlkem_native_asm.S) + target_compile_definitions(mlkem_native PRIVATE QBIT_MLKEM_NATIVE) + endif() + if(MLKEM_NATIVE_NO_ASM) + target_compile_definitions(mlkem_native PRIVATE MLK_CONFIG_NO_ASM) + endif() + + # core_interface's own options and definitions, read directly: a + # $ expression would also pull in warn_interface. + get_target_property(core_options core_interface INTERFACE_COMPILE_OPTIONS) + get_target_property(core_definitions core_interface INTERFACE_COMPILE_DEFINITIONS) + target_compile_definitions(mlkem_native + PUBLIC + "MLK_CONFIG_FILE=\"crypto/mlkem_config.h\"" + ) + if(core_definitions) + target_compile_definitions(mlkem_native PRIVATE ${core_definitions}) + endif() + if(core_options) + # On MinGW, core_interface passes -Wa,-muse-unaligned-vector-move, which + # would make gas encode the assembly's aligned vector moves (vmovdqa) as + # unaligned ones: not the encoding the HOL-Light proofs cover. The assembly + # needs no such protection, because its aligned moves touch only 32-byte + # aligned data, so the flag applies to the C sources only. See + # doc/subtrees/mlkem-native.md. + list(TRANSFORM core_options REPLACE "^(-Wa,-muse-unaligned-vector-move)$" "$<$:\\1>") + target_compile_options(mlkem_native PRIVATE ${core_options}) + endif() + # APPEND_CPPFLAGS and APPEND_CFLAGS reach the C and assembly compiles last, + # after core_interface's options, as for the other C libraries + # (cmake/secp256k1.cmake). The MSan jobs rely on this to undo + # core_interface's -D_FORTIFY_SOURCE with -U_FORTIFY_SOURCE. One SHELL: + # group, because CMake drops an option that repeats an earlier one, and + # core_interface already passes -U_FORTIFY_SOURCE before its -D. + string(STRIP "${APPEND_CPPFLAGS} ${APPEND_CFLAGS}" append_flags) + if(append_flags) + target_compile_options(mlkem_native PRIVATE "SHELL:${append_flags}") + endif() + target_include_directories(mlkem_native + PUBLIC + $ + $ + ) + target_link_libraries(mlkem_native PRIVATE sanitize_interface) + set_target_properties(mlkem_native PROPERTIES + EXPORT_COMPILE_COMMANDS OFF + ) +endfunction() diff --git a/contrib/devtools/mlkem-native.pin b/contrib/devtools/mlkem-native.pin new file mode 100644 index 0000000000..5bef7bd4e1 --- /dev/null +++ b/contrib/devtools/mlkem-native.pin @@ -0,0 +1,8 @@ +# mlkem-native vendored into src/mlkem-native. Written by +# contrib/devtools/update-mlkem-native.sh; checked by +# test/lint/mlkem-native-check.sh. See doc/subtrees/mlkem-native.md. +url=https://github.com/pq-code-package/mlkem-native.git +tag=v2.0.0 +commit=d1b2fe782888bdb761a50336012923180be7f502 +mlkem_tree=d9d581290ea1e6fa37462bfbc61b9b4266a2e9e5 +license_blob=79c71b1fdcc53cc17b1df99a0f4d4d48121ac174 diff --git a/contrib/devtools/update-mlkem-native.sh b/contrib/devtools/update-mlkem-native.sh new file mode 100755 index 0000000000..34212b7c6e --- /dev/null +++ b/contrib/devtools/update-mlkem-native.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2026-present The qbit core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit. + +export LC_ALL=C +set -euo pipefail + +readonly PREFIX="src/mlkem-native" +readonly PIN_FILE="contrib/devtools/mlkem-native.pin" + +usage() { + cat <] + +Replace ${PREFIX} with the upstream mlkem/ directory and LICENSE of an +mlkem-native release, unmodified, rewrite ${PIN_FILE} from the fetched +upstream object ids, and stage both. + + tag or commit to vendor (default: REMOTE_REF, else the pinned tag) + +Environment overrides: + REMOTE_URL default: the url= entry of ${PIN_FILE} + REMOTE_REF default: the tag= entry of ${PIN_FILE} + +See doc/subtrees/mlkem-native.md. +EOF +} + +fail() { + local signature="$1" + local cause="$2" + local fix="$3" + echo "FAIL: ${signature}" >&2 + echo " Cause: ${cause}" >&2 + echo " Fix: ${fix}" >&2 + exit 1 +} + +pin_value() { + awk -F= -v key="$1" '$1 == key { print substr($0, length(key) + 2); exit }' "${PIN_FILE}" +} + +if [[ "${1-}" == "--help" || "${1-}" == "-h" ]]; then + usage + exit 0 +fi + +if [[ $# -gt 1 ]]; then + usage >&2 + exit 2 +fi + +if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then + fail "not inside a git work tree" \ + "the update stages the vendored tree with git" \ + "run it from the root of a qbit checkout" +fi +cd "$(git rev-parse --show-toplevel)" + +pinned_url="$(pin_value url)" +pinned_tag="$(pin_value tag)" +pinned_commit="$(pin_value commit)" + +REMOTE_URL="${REMOTE_URL:-${pinned_url}}" +REMOTE_REF="${1:-${REMOTE_REF:-${pinned_tag}}}" +if [[ -z "${REMOTE_URL}" || -z "${REMOTE_REF}" ]]; then + fail "no upstream url or ref" \ + "${PIN_FILE} lacks url= or tag= and no override was given" \ + "restore ${PIN_FILE} from git, or set REMOTE_URL and pass a tag" +fi + +# Fetch into a temporary repository: a depth-1 fetch into this one would +# write .git/shallow and turn it into a shallow clone. +upstream_git="$(mktemp -d "${TMPDIR:-/tmp}/update-mlkem-native.XXXXXX")" +trap 'rm -rf -- "${upstream_git}"' EXIT +git init --quiet --bare "${upstream_git}" +upstream() { + git --git-dir="${upstream_git}" "$@" +} + +echo "Fetching mlkem-native ${REMOTE_REF} from ${REMOTE_URL}" +if ! upstream fetch --quiet --depth=1 --no-tags "${REMOTE_URL}" "${REMOTE_REF}"; then + fail "git fetch ${REMOTE_URL} ${REMOTE_REF} failed" \ + "the ref does not exist upstream, or the network is unavailable" \ + "check the tag name at ${REMOTE_URL}, or retry with network access" +fi +commit="$(upstream rev-parse "FETCH_HEAD^{commit}")" + +# A tag that resolves to a different commit than the pin means upstream moved +# it; never follow that silently. +if [[ "${REMOTE_REF}" == "${pinned_tag}" && "${commit}" != "${pinned_commit}" ]]; then + fail "${REMOTE_REF} resolves to ${commit}, but ${PIN_FILE} pins ${pinned_commit}" \ + "the upstream tag was moved or ${REMOTE_URL} is not the pinned repository" \ + "stop and investigate upstream; to vendor a different commit deliberately, pass it explicitly" +fi + +tag="${REMOTE_REF}" +if [[ "${REMOTE_REF}" =~ ^[0-9a-f]{40}$ ]]; then + tag="$(git ls-remote --tags "${REMOTE_URL}" | + awk -v c="${commit}" '$1 == c { sub("^refs/tags/", "", $2); sub("\\^\\{\\}$", "", $2); print $2; exit }')" + if [[ -z "${tag}" ]]; then + echo "warning: ${commit} has no upstream tag; releases must vendor a tagged commit" >&2 + tag="untagged" + fi +fi + +upstream_tree="$(upstream rev-parse "${commit}:mlkem")" +upstream_license="$(upstream rev-parse "${commit}:LICENSE")" + +echo "Replacing ${PREFIX} with ${commit}:mlkem and ${commit}:LICENSE" +rm -rf -- "${PREFIX}" +mkdir -p -- "${PREFIX}" +upstream archive --format=tar "${commit}" mlkem LICENSE | tar -xf - -C "${PREFIX}" + +cat > "${PIN_FILE}" <` replaces the single replay with a mutation phase of that many seconds per target. The phase starts from the corpus, writes new inputs to a temporary directory and leaves the corpus unchanged. It fails on non-libFuzzer builds. The nightly native fuzz job sets `QBIT_FUZZ_MUTATE_MIN_TIME`, which makes `ci/test/03_test_script.sh` run -this phase for the six targets after the normal replay. +this phase for the eight targets after the normal replay. The budget is libFuzzer's `-max_total_time`, which counts time spent loading the seed corpus (libFuzzer's `INITED` line) as well as mutation. A run can end diff --git a/doc/subtrees/mlkem-native.md b/doc/subtrees/mlkem-native.md new file mode 100644 index 0000000000..a9e1d07a08 --- /dev/null +++ b/doc/subtrees/mlkem-native.md @@ -0,0 +1,332 @@ +# mlkem-native Vendoring Runbook + +This runbook defines how qbit vendors and updates +[mlkem-native](https://github.com/pq-code-package/mlkem-native), the ML-KEM +(FIPS 203) library behind the hybrid post-quantum BIP324 transport. + +mlkem-native's C code is CBMC-proved memory-safe, and its x86_64 and AArch64 +assembly is HOL-Light-proved correct and constant-time wherever it handles +secret data. Editing any vendored file voids those proofs, so qbit never edits +`src/mlkem-native`. All qbit-owned glue (configuration, backend selection, +the C++ wrapper and the build rules) lives outside it. + +## Current Pin + +- Source repository: `https://github.com/pq-code-package/mlkem-native.git` +- Source tag: `v2.0.0` +- Commit: `d1b2fe782888bdb761a50336012923180be7f502` +- Tree id of upstream `mlkem/`: `d9d581290ea1e6fa37462bfbc61b9b4266a2e9e5` +- Blob id of upstream `LICENSE`: `79c71b1fdcc53cc17b1df99a0f4d4d48121ac174` +- qbit path: `src/mlkem-native` (`mlkem/` and `LICENSE`, nothing else) + +The machine-readable pin is +[`contrib/devtools/mlkem-native.pin`](../../contrib/devtools/mlkem-native.pin). +Keep this section in sync with it. + +The whole upstream `mlkem/` directory is vendored, including backends qbit does +not build, so the identity check stays a single tree id and every file that +upstream's single-compilation-unit build (`mlkem_native.c`, +`mlkem_native_asm.S`) includes is present. + +## How qbit Builds It + +`cmake/mlkem-native.cmake` compiles upstream's single compilation unit +(`mlkem/mlkem_native.c`) and, when native code is enabled, its assembly unit +(`mlkem/mlkem_native_asm.S`) into the `mlkem_native` library, linked privately +into `bitcoin_node` only. Never into `bitcoin_crypto`, `bitcoin_common` or +`bitcoin_consensus`; tests, fuzzers and benchmarks reach it through their node +linkage. All qbit-owned glue lives outside the vendored tree: + +- `src/crypto/mlkem_config.h`: the configuration, selected with + `MLK_CONFIG_FILE`. ML-KEM-1024, the `qbit_mlkem1024` symbol prefix, + deterministic APIs only, zeroization through `memory_cleanse`, and the + capability hook. +- `src/crypto/mlkem_arith_backend.h` and `src/crypto/mlkem_fips202_backend.h`: + the native backends, chosen by qbit rather than by upstream's + compile-time defaults (which enable x86_64 only for AVX2 compilers). +- `src/crypto/mlkem_shim.h` and `src/crypto/mlkem_backend.c`: the C ABI + between the library and qbit's C++. +- `src/crypto/mlkem.{h,cpp}`: the `mlkem::` wrapper. + +Native code is selected at run time. No source is compiled with `-mavx2` or +another ISA flag: every native entry point asks the capability hook first, and +the hook reads only cached CPU detection plus the portable override. The +`WITH_MLKEM_NATIVE` CMake option (`AUTO`, `ON`, `OFF`) decides whether native +code is built at all, and the configure summary's `ML-KEM-1024 backend` line +says what was chosen and why: + +| Target | `AUTO` | `ON` | +|---|---|---| +| x86_64 with GCC or Clang (Linux, macOS, MinGW) | native x86_64 AVX2, used when the CPU has every extension it needs (below) | native | +| macOS arm64 | native AArch64 | native | +| AArch64 ELF (Linux, BSD) | portable (see below) | configure error | +| MSVC | portable, no assembly at all | configure error | +| `SANITIZERS` contains `memory` | portable (MSan cannot see assembly writes) | configure error | +| anything else (riscv64, powerpc64, armv7, s390x, i686, ...) | portable | configure error | + +`ci/checks/test_mlkem_build_policy.py` also links the glue with every +assembly routine wrapped by a call counter, and fails if any of them runs while +portable C is forced, or when an entry point run alone on a new thread uses +assembly although portable C is forced (it entered the library without taking +the override). It checks the x86_64 backend natively and the AArch64 backend +as a static binary under `qemu-aarch64`, built from a copy of +`src/crypto/mlkem_config.h` without its AArch64 ELF refusal. CI's build smoke +job requires both. + +### x86_64 instruction set + +The x86_64 backend runs only where the CPU has every extension its assembly +executes; `cpu_features::HasMlkemX86Native()` checks them once and caches the +answer, and the capability hook answers upstream's single +`MLK_SYS_CAP_X86_64_AVX2` query with it. Upstream annotates its routines +`Features: [AVX2]`, but an audit of the assembled `mlkem_native_asm.S.o` +(v2.0.0) found: + +| Extension | Instructions | Where | +|---|---|---| +| AVX2 (with AVX, and the OS saving YMM state) | 256-bit integer ops such as `vpmulhw`, `vperm2i128`, `vpblendd`, `vpsrlvq`, `vpbroadcastq`, `vinserti128` | every routine | +| AVX | `vmovdqa`, `vmovsldup`, `vmovhpd`, `vpinsrq`, `vpinsrw`, `vpextrw` | every routine | +| SSSE3 | `pshufb` | `rej_uniform` | +| SSE4.1 | `pinsrd`, `pinsrq`, `pblendw` | `rej_uniform` | +| POPCNT | `popcnt` | `rej_uniform` | +| BMI2 | `pext` | `rej_uniform` | +| baseline x86-64 | SSE2, `cmov`, `rep movsb`; `endbr64` is a no-op without CET | | + +No FMA, AVX-512, BMI1, LZCNT, MOVBE, AES, PCLMUL or SHA instruction occurs. +A virtual machine can show AVX2 while hiding the others; a check of AVX2 alone +would then die with SIGILL on the first ML-KEM operation. qbit's C code is +compiled for baseline x86-64 and uses only SSE2. + +Repeat the audit on every library update, with native code built: + +```bash +objdump -d --no-show-raw-insn build/src/CMakeFiles/mlkem_native.dir/mlkem-native/mlkem/mlkem_native_asm.S.o \ + | awk -F'\t' 'NF >= 2 && $1 ~ /:$/ { split($2, a, " "); print a[1] }' | sort | uniq -c +``` + +Classify every mnemonic, and extend `HasMlkemX86Native` and its tests for a +new extension. Then check masked CPUs: under +`qemu-x86_64 -cpu Haswell,-bmi2` (and `-popcnt`, `-sse4.1`, `-ssse3`) a +program using `mlkem::` must report portable and finish without SIGILL. +`test_qbit` itself runs under all four masks. + +### MinGW assembly encoding + +Windows builds run the x86_64 backend natively too: with GCC or Clang, +upstream declares its SysV-ABI routines `__attribute__((sysv_abi))`, and +`mlkem_native_asm.S` assembles all 15 of them. On MinGW, `core_interface` +passes `-Wa,-muse-unaligned-vector-move` (GCC bug 54412: on 64-bit Windows, +GCC does not keep the stack 32-byte aligned for AVX code). On the assembly, +that flag would make gas encode each of its 632 aligned vector moves +(`vmovdqa`) as an unaligned one (`vmovdqu`): object bytes the HOL-Light proofs +do not cover. `add_mlkem_native` therefore passes the flag to the C sources +only, and `ci/checks/test_mlkem_build_policy.py` checks that it does. + +The assembly needs no such protection, because its aligned moves touch only +32-byte aligned data: + +- `MLK_ALIGN` stack buffers, which MinGW GCC aligns by over-allocating and + rounding the address, since it cannot realign the stack pointer (14 sites in + `mlkem_native.c`, as many as the stack realignments on Linux); +- constant tables, in a 32-byte aligned `.rdata` section. + +No aligned move addresses the stack pointer or RIP-relative data, and the +vendored C is compiled without AVX, so it has no AVX spills. A misaligned +buffer would fault rather than give a wrong result. Recheck this on every +library update, by disassembling both objects built for MinGW. + +### AArch64 ELF builds are portable only + +mlkem-native v2.0.0's AArch64 assembly has no BTI landing pads (`bti c`) and +no GNU property note. A linker keeps an ELF binary's BTI and PAC marking only if +every input object carries it, so linking the native backend would strip the +marking from the whole binary. Hardening wins: AArch64 ELF targets build +portable C. Measured with Ubuntu's `aarch64-linux-gnu-gcc` 15.2.0 and +`-mbranch-protection=standard`, as qbit builds aarch64 Linux: + +```text +mlkem_native.c.o, qbit's C and C++ objects AArch64 feature: BTI, PAC, GCS +mlkem_native_asm.S.o (no GNU property note) +binary with portable ML-KEM AArch64 feature: BTI, PAC, GCS +same binary with the native backend (no AArch64 feature note) +ld -z force-bti: asm.o: warning: BTI is required by -z force-bti, but this + input object file lacks the necessary property note. +``` + +Revisit when an upstream release ships AArch64 assembly with BTI landing pads +and the GNU property note: then re-run the property comparison below on +aarch64 and, if no marking is lost, drop the `aarch64-elf` rule in +`cmake/mlkem-native.cmake` and `src/crypto/mlkem_config.h`. Never restore the +marking by force-marking objects that lack landing pads. macOS arm64 (Mach-O, +no GNU property notes) keeps the native backend. + +### x86_64 CET marking + +The x86_64 assembly keeps a binary's Intel CET marking. Upstream's `sys.h` +includes `` in the assembly whenever the compiler defines `__CET__`, so +with `-fcf-protection=full`, which qbit's hardening flags pass to the assembly +as to the C (`ci/checks/test_mlkem_build_policy.py` checks that the C flags +reach it), the object gets `endbr64` landing pads and the GNU property note. +Measured for the initial vendoring (v2.0.0) with Ubuntu's gcc 15.2.0 and GNU ld +2.46, `RelWithDebInfo` with qbit's default hardening, native x86_64 enabled: + +```text +mlkem_native_asm.S.o, mlkem_native.c.o, x86 feature: IBT, SHSTK +mlkem_backend.c.o, mlkem.cpp.o +test_qbit, fuzz (both link the native x86 feature: IBT, SHSTK +backend) x86 ISA needed: x86-64-baseline +mlkem_native_asm.S.o assembled with (no x86 feature note) +-fcf-protection=none +``` + +The release evidence is still outstanding for the initial vendoring: Guix +builds of every release host, reproduced, with this comparison for `qbitd`, +`qbit-cli`, `qbit-qt` and `test_qbit` (see the PR checklist below). It is +required before the post-quantum feature branch (#184) merges into `1.x.x`. + +### The AArch64 single-lane Keccak mirror + +Upstream's `fips202/native/aarch64/x1_scalar.h` runs its scalar assembly +without asking the capability hook, so qbit's portable override could not reach +it and `mlkem::GetBackendNames()` would report portable Keccak while assembly +ran. `src/crypto/mlkem_fips202_backend.h` therefore mirrors that header's +dispatch glue (the same macros and the same proved assembly routine) with a +capability check added. Every other native entry point in the x86_64 and +AArch64 backends checks the hook itself. + +The build-policy test's AArch64 case shows the mirror at work: the harness +makes 252 assembly calls natively and none with portable C forced. Its negative +control builds upstream's `x1_scalar.h` in place of the mirror, and then 221 +assembly calls leak into a forced-portable run. + +## Update Procedure + +Run in a qbit worktree: + +```bash +contrib/devtools/update-mlkem-native.sh # the pinned tag +contrib/devtools/update-mlkem-native.sh v2.0.1 # a new release +``` + +The script fetches the tag or commit from `REMOTE_URL` (default: the pinned +repository) into a temporary repository, so the qbit clone does not become +shallow, replaces `src/mlkem-native` with upstream's `mlkem/` and +`LICENSE` byte for byte, rewrites the pin from the object ids of the fetched +upstream commit, stages both, and checks that the staged tree has exactly those +ids. `REMOTE_REF` selects the ref when no argument is given. It refuses to +follow the pinned tag if upstream moved it to another commit. + +Vendor only tagged releases. Read the upstream release notes and changelog +before updating, then work through the PR checklist below: qbit's backend +headers include upstream paths, so an update can need qbit-side changes even +though no vendored file is edited. + +## Verify Integrity + +```bash +test/lint/mlkem-native-check.sh +``` + +The check runs offline as part of the lint runner's `subtree` lint. It prints +`GOOD` when `HEAD:src/mlkem-native/mlkem` has the pinned tree id, +`HEAD:src/mlkem-native/LICENSE` has the pinned blob id, `src/mlkem-native` +holds nothing else, and the work tree has no changes under it. Otherwise it +prints `FAIL` with the signature, cause and fix. + +## PR Checklist For Library Updates + +When a PR touches `src/mlkem-native` or `contrib/devtools/mlkem-native.pin`, +confirm: + +- [ ] The new commit is an upstream release tag, and its release notes and + security advisories were read. +- [ ] The tree was produced by `contrib/devtools/update-mlkem-native.sh`, and + this runbook's Current Pin matches the pin file. +- [ ] `test/lint/mlkem-native-check.sh` prints `GOOD`, and the full lint runner + passes. +- [ ] Running the update script again at the new tag leaves no diff. +- [ ] The options `src/crypto/mlkem_config.h` sets, the `MLK_SYS_*` macros it + reads and the upstream paths the backend headers include still exist with + the same meaning (upstream `mlkem/mlkem_native_config.h`, `mlkem/src/sys.h`, + `mlkem/src/native/*/meta.h`). +- [ ] Upstream's `mlkem/src/fips202/native/aarch64/x1_scalar.h` was diffed + against the mirror in `src/crypto/mlkem_fips202_backend.h`, and the mirror + still differs only by its capability check. +- [ ] Every native entry point compiled in still asks the capability hook: + `grep -n "static MLK_INLINE\|mlk_sys_check_capability"` over + `mlkem/src/native/x86_64/meta.h`, `mlkem/src/native/aarch64/meta.h`, + `mlkem/src/fips202/native/x86_64/keccak_f1600_x4_avx2.h` and + `mlkem/src/fips202/native/aarch64/x4_v8a_scalar.h` pairs each function with a + check. A new unchecked entry point needs a mirror like the x1 one. +- [ ] The x86_64 instruction-set audit above was repeated, and + `HasMlkemX86Native` still requires every extension the assembly executes. +- [ ] `test_qbit --run_test=mlkem_tests` passes (every case runs on the native + backend and with portable C forced), and so do + `ci/checks/test_mlkem_build_policy.py`, the `mlkem` and `mlkem_backend_diff` + fuzz targets, and `bench_qbit -filter='MLKEM.*'`. +- [ ] The GNU property notes of the release ELF binaries did not lose a + marking. Build the base and the change with the release hardening flags + (Guix for release evidence) and compare, for each binary (`qbitd`, + `qbit-cli`, `qbit-qt`, `test_qbit`, ...): + + ```bash + readelf -n | grep -E 'x86 feature|AArch64 feature' + ``` + + x86_64 must keep `IBT, SHSTK`; aarch64 must keep `BTI, PAC` (and `GCS` where + present). A lost marking blocks the update; see the AArch64 section above. +- [ ] Guix builds of every release host reproduce (two independent builds, + identical hashes). + +## Common Failures + +1. Signature: + `FAIL: src/mlkem-native/mlkem tree differs from pinned tree ` + (or the same for `LICENSE`) + Cause: + A vendored file was edited, added, removed or changed mode, or the pin was + changed without re-vendoring. + Fix: + Never edit `src/mlkem-native`. Restore it with + `contrib/devtools/update-mlkem-native.sh`. Changes belong upstream, or in + qbit's glue outside the vendored directory. + +2. Signature: + `FAIL: uncommitted changes under src/mlkem-native` + Cause: + The work tree or index has edits, new files or deletions under the vendored + directory. + Fix: + `git checkout HEAD -- src/mlkem-native && git clean -fdx -- src/mlkem-native`. + +3. Signature: + `FAIL: src/mlkem-native in HEAD holds entries other than LICENSE and mlkem` + Cause: + A qbit file was added next to the vendored tree. + Fix: + Move it out; qbit's ML-KEM glue lives in `src/crypto/mlkem*`. + +4. Signature: + `FAIL: resolves to , but contrib/devtools/mlkem-native.pin pins ` + Cause: + Upstream moved the tag, or `REMOTE_URL` points at a different repository. + Fix: + Stop and investigate upstream before vendoring anything. To vendor a + different commit deliberately, pass it to the script explicitly. + +5. Signature: + `FAIL: staged src/mlkem-native (...) differs from upstream (...)` + Cause: + A git attribute, filter or line-ending setting (for example + `core.autocrlf`) rewrote vendored bytes on staging. + Fix: + Remove the setting that applies to `src/mlkem-native` and re-run the update + script. + +## Review Cadence + +At least quarterly, check +[upstream releases](https://github.com/pq-code-package/mlkem-native/releases) +and [security advisories](https://github.com/pq-code-package/mlkem-native/security/advisories). +Update promptly for a security fix; otherwise update when a release brings a +proved improvement qbit uses. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c97d6651be..0e05808cdc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -27,6 +27,7 @@ include(../cmake/secp256k1.cmake) add_secp256k1(secp256k1) include(../cmake/libbitcoinpqc.cmake) add_libbitcoinpqc(libbitcoinpqc) +add_mlkem_native() # Set top-level target output locations. if(NOT CMAKE_RUNTIME_OUTPUT_DIRECTORY) @@ -193,6 +194,7 @@ add_library(bitcoin_node STATIC EXCLUDE_FROM_ALL blockencodings.cpp blockfilter.cpp consensus/tx_verify.cpp + crypto/mlkem.cpp dbwrapper.cpp deploymentstatus.cpp flatfile.cpp @@ -292,6 +294,7 @@ target_link_libraries(bitcoin_node $ leveldb minisketch + mlkem_native univalue Boost::headers $ diff --git a/src/bench/CMakeLists.txt b/src/bench/CMakeLists.txt index afa66f354a..52d45676be 100644 --- a/src/bench/CMakeLists.txt +++ b/src/bench/CMakeLists.txt @@ -38,6 +38,7 @@ add_executable(bench_bitcoin mempool_eviction.cpp mempool_stress.cpp merkle_root.cpp + mlkem.cpp obfuscation.cpp parse_hex.cpp peer_eviction.cpp diff --git a/src/bench/mlkem.cpp b/src/bench/mlkem.cpp new file mode 100644 index 0000000000..0be1a0d024 --- /dev/null +++ b/src/bench/mlkem.cpp @@ -0,0 +1,107 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#include +#include +#include + +#include +#include +#include +#include + +using namespace mlkem; + +namespace { + +enum class Backend { DETECTED, PORTABLE }; + +/** + * Run `body` on the detected backend, or with portable C forced, and name the + * benchmark after the backend that runs. Auto-detection is restored afterwards. + */ +template +void WithBackend(benchmark::Bench& bench, const char* operation, Backend backend, Body body) +{ + std::optional portable; + if (backend == Backend::PORTABLE) portable.emplace(); + const BackendNames names{GetBackendNames()}; + bench.name(strprintf("%s using the '%s/%s' ML-KEM-1024 backend", operation, names.arith, names.keccak)); + body(); +} + +std::array Seed() +{ + std::array seed; + for (size_t i{0}; i < seed.size(); ++i) seed[i] = uint8_t(i * 7 + 1); + return seed; +} + +void KeyGenBench(benchmark::Bench& bench, Backend backend) +{ + WithBackend(bench, "MLKEMKeyGen", backend, [&] { + auto seed{Seed()}; + PublicKey ek; + DecapsulationKey dk; + bench.batch(1).unit("keygen").run([&] { + ++seed[0]; + const Error error{KeyGen(seed, ek, dk)}; + assert(error == Error::NONE); + }); + }); +} + +void EncapsBench(benchmark::Bench& bench, Backend backend) +{ + WithBackend(bench, "MLKEMEncaps", backend, [&] { + PublicKey ek; + DecapsulationKey dk; + const Error keygen{KeyGen(Seed(), ek, dk)}; + assert(keygen == Error::NONE); + std::array coins{}; + Ciphertext ct; + SharedSecret ss; + bench.batch(1).unit("encaps").run([&] { + ++coins[0]; + const Error error{Encaps(ek, coins, ct, ss)}; + assert(error == Error::NONE); + }); + }); +} + +void DecapsBench(benchmark::Bench& bench, Backend backend) +{ + WithBackend(bench, "MLKEMDecaps", backend, [&] { + PublicKey ek; + DecapsulationKey dk; + const Error keygen{KeyGen(Seed(), ek, dk)}; + assert(keygen == Error::NONE); + const std::array coins{}; + Ciphertext ct; + SharedSecret sent; + const Error encaps{Encaps(ek, coins, ct, sent)}; + assert(encaps == Error::NONE); + SharedSecret received; + bench.batch(1).unit("decaps").run([&] { + const Error error{Decaps(dk, ct, received)}; + assert(error == Error::NONE); + }); + }); +} + +} // namespace + +static void MLKEM_KEYGEN_DETECTED(benchmark::Bench& bench) { KeyGenBench(bench, Backend::DETECTED); } +static void MLKEM_KEYGEN_PORTABLE(benchmark::Bench& bench) { KeyGenBench(bench, Backend::PORTABLE); } +static void MLKEM_ENCAPS_DETECTED(benchmark::Bench& bench) { EncapsBench(bench, Backend::DETECTED); } +static void MLKEM_ENCAPS_PORTABLE(benchmark::Bench& bench) { EncapsBench(bench, Backend::PORTABLE); } +static void MLKEM_DECAPS_DETECTED(benchmark::Bench& bench) { DecapsBench(bench, Backend::DETECTED); } +static void MLKEM_DECAPS_PORTABLE(benchmark::Bench& bench) { DecapsBench(bench, Backend::PORTABLE); } + +BENCHMARK(MLKEM_KEYGEN_DETECTED, benchmark::PriorityLevel::HIGH); +BENCHMARK(MLKEM_KEYGEN_PORTABLE, benchmark::PriorityLevel::HIGH); +BENCHMARK(MLKEM_ENCAPS_DETECTED, benchmark::PriorityLevel::HIGH); +BENCHMARK(MLKEM_ENCAPS_PORTABLE, benchmark::PriorityLevel::HIGH); +BENCHMARK(MLKEM_DECAPS_DETECTED, benchmark::PriorityLevel::HIGH); +BENCHMARK(MLKEM_DECAPS_PORTABLE, benchmark::PriorityLevel::HIGH); diff --git a/src/compat/cpu_features.cpp b/src/compat/cpu_features.cpp new file mode 100644 index 0000000000..ffe249b670 --- /dev/null +++ b/src/compat/cpu_features.cpp @@ -0,0 +1,53 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#include + +#include + +namespace cpu_features { + +X86Features DetectX86Features(CpuidFn cpuid, XGetBVFn xgetbv) noexcept +{ + X86Features f; + uint32_t a{0}, b{0}, c{0}, d{0}; + cpuid(0, 0, a, b, c, d); + f.max_basic_leaf = a; + if (f.max_basic_leaf >= 1) { + cpuid(1, 0, a, b, c, d); + f.leaf1_ecx = c; + } + if (f.max_basic_leaf >= 7) { + cpuid(7, 0, a, b, c, d); + f.leaf7_ebx = b; + } + // XGETBV faults unless the OS enabled it, which leaf 1 reports as OSXSAVE. + if ((f.leaf1_ecx & LEAF1_ECX_OSXSAVE) != 0 && (f.leaf1_ecx & LEAF1_ECX_AVX) != 0) { + f.xcr0 = xgetbv(0); + } + return f; +} + +X86Features DetectX86Features() noexcept +{ +#if defined(HAVE_GETCPUID) + return DetectX86Features(GetCPUID, XGetBV); +#else + return {}; +#endif +} + +bool HasUsableAVX2() noexcept +{ + static const bool usable{HasUsableAVX2(DetectX86Features())}; + return usable; +} + +bool HasMlkemX86Native() noexcept +{ + static const bool usable{HasMlkemX86Native(DetectX86Features())}; + return usable; +} + +} // namespace cpu_features diff --git a/src/compat/cpu_features.h b/src/compat/cpu_features.h new file mode 100644 index 0000000000..9855211bb9 --- /dev/null +++ b/src/compat/cpu_features.h @@ -0,0 +1,83 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#ifndef QBIT_COMPAT_CPU_FEATURES_H +#define QBIT_COMPAT_CPU_FEATURES_H + +#include + +/** + * Standalone x86 feature detection for code that dispatches at run time. + * + * This deliberately shares only the CPUID and XGETBV primitives with + * SHA256AutoDetect, whose selection logic is tuned for SHA256 (it skips leaf 7 + * without SSE4 and drops AVX2 when SHA-NI is present). + */ +namespace cpu_features { + +inline constexpr uint32_t LEAF1_ECX_SSSE3{uint32_t{1} << 9}; +inline constexpr uint32_t LEAF1_ECX_SSE41{uint32_t{1} << 19}; +inline constexpr uint32_t LEAF1_ECX_POPCNT{uint32_t{1} << 23}; +inline constexpr uint32_t LEAF1_ECX_OSXSAVE{uint32_t{1} << 27}; +inline constexpr uint32_t LEAF1_ECX_AVX{uint32_t{1} << 28}; +inline constexpr uint32_t LEAF7_EBX_AVX2{uint32_t{1} << 5}; +inline constexpr uint32_t LEAF7_EBX_BMI2{uint32_t{1} << 8}; +/** XCR0 bits for XMM (SSE) and YMM (AVX) register state saved by the OS. */ +inline constexpr uint64_t XCR0_XMM_YMM{0x6}; + +struct X86Features { + uint32_t max_basic_leaf{0}; + uint32_t leaf1_ecx{0}; + uint32_t leaf7_ebx{0}; + uint64_t xcr0{0}; +}; + +using CpuidFn = void (*)(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d); +using XGetBVFn = uint64_t (*)(uint32_t index); + +/** + * Query the supported leaves through the given primitives, then read XCR0 + * only if leaf 1 reports both OSXSAVE and AVX. Exposed so tests can supply + * synthetic primitives. + */ +X86Features DetectX86Features(CpuidFn cpuid, XGetBVFn xgetbv) noexcept; + +/** Query this CPU and OS. All fields are zero on non-x86 targets and unsupported compilers. */ +X86Features DetectX86Features() noexcept; + +/** Whether the CPU supports AVX2 and the OS saves YMM state, so AVX2 code can run. */ +constexpr bool HasUsableAVX2(const X86Features& f) noexcept +{ + return f.max_basic_leaf >= 7 && + (f.leaf1_ecx & LEAF1_ECX_OSXSAVE) != 0 && + (f.leaf1_ecx & LEAF1_ECX_AVX) != 0 && + (f.xcr0 & XCR0_XMM_YMM) == XCR0_XMM_YMM && + (f.leaf7_ebx & LEAF7_EBX_AVX2) != 0; +} + +/** HasUsableAVX2() for this CPU and OS, detected on first call and then cached. */ +bool HasUsableAVX2() noexcept; + +/** + * Whether this CPU and OS can run mlkem-native's x86_64 assembly. Besides + * AVX2 (with AVX and the OS saving YMM state), that assembly executes SSSE3, + * SSE4.1, POPCNT and BMI2 instructions, which a CPUID mask in a virtual + * machine can hide even when AVX2 is shown. The list comes from an audit of + * the assembled objects; doc/subtrees/mlkem-native.md says how to repeat it. + */ +constexpr bool HasMlkemX86Native(const X86Features& f) noexcept +{ + return HasUsableAVX2(f) && + (f.leaf1_ecx & LEAF1_ECX_SSSE3) != 0 && + (f.leaf1_ecx & LEAF1_ECX_SSE41) != 0 && + (f.leaf1_ecx & LEAF1_ECX_POPCNT) != 0 && + (f.leaf7_ebx & LEAF7_EBX_BMI2) != 0; +} + +/** HasMlkemX86Native() for this CPU and OS, detected on first call and then cached. */ +bool HasMlkemX86Native() noexcept; + +} // namespace cpu_features + +#endif // QBIT_COMPAT_CPU_FEATURES_H diff --git a/src/compat/cpuid.h b/src/compat/cpuid.h index 10a959df75..562e1d8b5a 100644 --- a/src/compat/cpuid.h +++ b/src/compat/cpuid.h @@ -22,5 +22,13 @@ void static inline GetCPUID(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32 #endif } +/** Read extended control register `index` (XCR0 for 0). Faults unless CPUID leaf 1 reports OSXSAVE. */ +static inline uint64_t XGetBV(uint32_t index) noexcept +{ + uint32_t a, d; + __asm__("xgetbv" : "=a"(a), "=d"(d) : "c"(index)); + return (uint64_t{d} << 32) | a; +} + #endif // defined(__x86_64__) || defined(__amd64__) || defined(__i386__) #endif // QBIT_COMPAT_CPUID_H diff --git a/src/crypto/mlkem.cpp b/src/crypto/mlkem.cpp new file mode 100644 index 0000000000..c96d693cb6 --- /dev/null +++ b/src/crypto/mlkem.cpp @@ -0,0 +1,195 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#include + +#include +#include +#include + +#include + +#include +#include + +static_assert(MLKEM1024_PUBLICKEYBYTES == mlkem::PUBLIC_KEY_BYTES); +static_assert(MLKEM1024_SECRETKEYBYTES == mlkem::SECRET_KEY_BYTES); +static_assert(MLKEM1024_CIPHERTEXTBYTES == mlkem::CIPHERTEXT_BYTES); +static_assert(MLKEM1024_BYTES == mlkem::SHARED_SECRET_BYTES); +static_assert(2 * MLKEM1024_SYMBYTES == mlkem::KEYGEN_SEED_BYTES); +static_assert(MLKEM1024_SYMBYTES == mlkem::ENCAPS_COINS_BYTES); +static_assert(MLK_ERR_FAIL == mlkem::upstream::ERR_FAIL); +static_assert(MLK_ERR_OUT_OF_MEMORY == mlkem::upstream::ERR_OUT_OF_MEMORY); +static_assert(MLK_ERR_RNG_FAIL == mlkem::upstream::ERR_RNG_FAIL); +static_assert(MLK_ERR_INVALID_PK == mlkem::upstream::ERR_INVALID_PK); +static_assert(MLK_ERR_INVALID_SK == mlkem::upstream::ERR_INVALID_SK); +static_assert(MLK_ERR_PCT_FAIL == mlkem::upstream::ERR_PCT_FAIL); + +namespace { + +constexpr std::string_view PORTABLE{"portable"}; + +//! When set, every operation that starts afterwards runs portable C only. +std::atomic_bool g_force_portable{false}; + +/** + * g_force_portable as the operation running on this thread read it at entry. + * The capability hooks read only this copy, so a change on another thread + * mid-operation cannot mix backends: the x86_64 backend keeps its own + * coefficient order, and a mixed operation would give a wrong result. + */ +thread_local bool t_force_portable{false}; + +//! Whether the CPU and OS can run the x86_64 backend. Written once, by DetectCapabilities(). +std::atomic_bool g_cpu_x86_native{false}; + +//! Results injected by InjectResultForTesting, indexed by Operation; 0 means none. +std::array g_injected_results{}; + +/** + * Detect CPU features on the first call. Every entry point calls this before + * entering the library, so no CPUID or XGETBV runs inside an operation, and + * the capability hooks read only the cached result. + */ +void DetectCapabilities() noexcept +{ + static const bool detected{[] { + g_cpu_x86_native.store(cpu_features::HasMlkemX86Native(), std::memory_order_relaxed); + return true; + }()}; + (void)detected; +} + +/** Call at every entry point, before the library runs: fixes the backend for this operation. */ +void BeginOperation() noexcept +{ + DetectCapabilities(); + t_force_portable = g_force_portable.load(std::memory_order_relaxed); +} + +int Injected(mlkem::Operation op, int result) noexcept +{ + const int injected{g_injected_results[static_cast(op)].load(std::memory_order_relaxed)}; + return injected != 0 ? injected : result; +} + +mlkem::Error MapResult(mlkem::Operation op, int result) noexcept +{ + if (result == 0) return mlkem::Error::NONE; + if (result == MLK_ERR_INVALID_PK && (op == mlkem::Operation::CHECK_PUBLIC_KEY || op == mlkem::Operation::ENCAPS)) { + return mlkem::Error::INVALID_PUBLIC_KEY; + } + if (result == MLK_ERR_INVALID_SK && op == mlkem::Operation::DECAPS) return mlkem::Error::INVALID_SECRET_KEY; + // Anything else, including codes this configuration never produces, is an internal failure. + return mlkem::Error::INTERNAL; +} + +/** The backend that runs for a compiled-in backend name, given the capability hooks' current answers. */ +std::string_view ActiveBackend(std::string_view compiled) noexcept +{ + if (compiled == "x86_64-avx2") return qbit_mlkem_has_avx2() ? compiled : PORTABLE; + if (compiled == "aarch64-neon" || compiled == "aarch64") return qbit_mlkem_has_neon() ? compiled : PORTABLE; + return PORTABLE; +} + +} // namespace + +extern "C" void qbit_mlkem_zeroize(void* ptr, size_t len) +{ + memory_cleanse(ptr, len); +} + +extern "C" int qbit_mlkem_has_avx2(void) +{ + return !t_force_portable && g_cpu_x86_native.load(std::memory_order_relaxed) ? 1 : 0; +} + +extern "C" int qbit_mlkem_has_neon(void) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + // NEON is mandatory on AArch64. + return !t_force_portable ? 1 : 0; +#else + return 0; +#endif +} + +namespace mlkem { + +Error KeyGen(std::span seed64, PublicKey& ek, DecapsulationKey& dk) noexcept +{ + BeginOperation(); + const int result{Injected(Operation::KEYGEN, qbit_mlkem1024_keypair_derand(ek.data(), dk.Bytes().data(), seed64.data()))}; + const Error error{MapResult(Operation::KEYGEN, result)}; + if (error != Error::NONE) { + dk.Clear(); + memory_cleanse(ek.data(), ek.size()); + } + return error; +} + +Error CheckPublicKey(std::span ek) noexcept +{ + BeginOperation(); + return MapResult(Operation::CHECK_PUBLIC_KEY, Injected(Operation::CHECK_PUBLIC_KEY, qbit_mlkem1024_check_pk(ek.data()))); +} + +Error Encaps(std::span ek, std::span coins32, + Ciphertext& ct, SharedSecret& ss) noexcept +{ + BeginOperation(); + const int result{Injected(Operation::ENCAPS, qbit_mlkem1024_enc_derand(ct.data(), ss.Bytes().data(), ek.data(), coins32.data()))}; + const Error error{MapResult(Operation::ENCAPS, result)}; + if (error != Error::NONE) { + ss.Clear(); + memory_cleanse(ct.data(), ct.size()); + } + return error; +} + +Error Decaps(const DecapsulationKey& dk, std::span ct, SharedSecret& ss) noexcept +{ + BeginOperation(); + const int result{Injected(Operation::DECAPS, qbit_mlkem1024_dec(ss.Bytes().data(), ct.data(), dk.Bytes().data()))}; + const Error error{MapResult(Operation::DECAPS, result)}; + if (error != Error::NONE) ss.Clear(); + return error; +} + +void InitializeRuntime(bool force_portable) noexcept +{ + DetectCapabilities(); + g_force_portable.store(force_portable); +} + +BackendNames GetBackendNames() noexcept +{ + BeginOperation(); + return { + .arith = ActiveBackend(qbit_mlkem_compiled_arith_backend()), + .keccak = ActiveBackend(qbit_mlkem_compiled_keccak_backend()), + }; +} + +ForcePortableForTesting::ForcePortableForTesting() noexcept + : m_previous{g_force_portable.exchange(true)} +{ +} + +ForcePortableForTesting::~ForcePortableForTesting() +{ + g_force_portable.store(m_previous); +} + +InjectResultForTesting::InjectResultForTesting(Operation op, int upstream_result) noexcept + : m_op{op}, m_previous{g_injected_results[static_cast(op)].exchange(upstream_result)} +{ +} + +InjectResultForTesting::~InjectResultForTesting() +{ + g_injected_results[static_cast(m_op)].store(m_previous); +} + +} // namespace mlkem diff --git a/src/crypto/mlkem.h b/src/crypto/mlkem.h new file mode 100644 index 0000000000..ab2cc03aaa --- /dev/null +++ b/src/crypto/mlkem.h @@ -0,0 +1,155 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#ifndef QBIT_CRYPTO_MLKEM_H +#define QBIT_CRYPTO_MLKEM_H + +#include + +#include +#include +#include +#include +#include + +/** + * ML-KEM-1024 (FIPS 203), backed by the vendored mlkem-native library. + * + * Native code runs where it is built and the CPU allows it: x86_64 with AVX2, + * SSSE3, SSE4.1, POPCNT and BMI2 (and the OS saving AVX state), and AArch64 on + * macOS. Everything else runs portable C, AArch64 ELF (Linux, BSD) included, + * whose builds leave the assembly out to keep their BTI and PAC marking. See + * doc/subtrees/mlkem-native.md. The library is built into bitcoin_node only. + * + * Fixed-size spans make wrong-sized calls impossible; callers check network + * lengths before constructing them. Entropy is the caller's job: key + * generation takes two independent 32-byte draws, encapsulation one. + */ +namespace mlkem { +inline constexpr size_t PUBLIC_KEY_BYTES = 1568; +inline constexpr size_t SECRET_KEY_BYTES = 3168; +inline constexpr size_t CIPHERTEXT_BYTES = 1568; +inline constexpr size_t SHARED_SECRET_BYTES = 32; +inline constexpr size_t KEYGEN_SEED_BYTES = 64; +inline constexpr size_t ENCAPS_COINS_BYTES = 32; + +/** Fixed-size secret bytes, wiped with memory_cleanse on destruction, Clear() and move-from. */ +template +class Secret +{ + std::array m_bytes{}; + +public: + Secret() noexcept = default; + ~Secret() { Clear(); } + Secret(const Secret&) = delete; + Secret& operator=(const Secret&) = delete; + Secret(Secret&& other) noexcept : m_bytes{other.m_bytes} { other.Clear(); } + Secret& operator=(Secret&& other) noexcept + { + if (this != &other) { + m_bytes = other.m_bytes; + other.Clear(); + } + return *this; + } + std::span Bytes() noexcept { return m_bytes; } + std::span Bytes() const noexcept { return m_bytes; } + void Clear() noexcept { memory_cleanse(m_bytes.data(), m_bytes.size()); } +}; + +using PublicKey = std::array; +using Ciphertext = std::array; +using DecapsulationKey = Secret; +using SharedSecret = Secret; + +enum class Error : uint8_t { NONE = 0, INVALID_PUBLIC_KEY = 1, INVALID_SECRET_KEY = 2, INTERNAL = 3 }; + +/** + * Deterministic key generation (FIPS 203 ML-KEM.KeyGen_internal) from + * seed64 = d || z. On failure, ek and dk are cleared. + */ +[[nodiscard]] Error KeyGen(std::span seed64, PublicKey& ek, DecapsulationKey& dk) noexcept; + +/** The FIPS 203 section 7.2 modulus check: INVALID_PUBLIC_KEY if a coefficient is not below q. */ +[[nodiscard]] Error CheckPublicKey(std::span ek) noexcept; + +/** + * Deterministic encapsulation (FIPS 203 ML-KEM.Encaps_internal), including the + * modulus check of ek. On failure, ct and ss are cleared. + */ +[[nodiscard]] Error Encaps(std::span ek, std::span coins32, + Ciphertext& ct, SharedSecret& ss) noexcept; + +/** + * Decapsulation (FIPS 203 ML-KEM.Decaps), including the section 7.3 hash + * check of dk (INVALID_SECRET_KEY). A corrupted ciphertext of the right + * length is not an error: it yields the implicit-rejection secret, which + * differs from the encapsulated one. On failure, ss is cleared. + */ +[[nodiscard]] Error Decaps(const DecapsulationKey& dk, std::span ct, + SharedSecret& ss) noexcept; + +struct BackendNames { + std::string_view arith; //!< "x86_64-avx2", "aarch64-neon" or "portable" + std::string_view keccak; //!< "x86_64-avx2", "aarch64" or "portable" +}; + +/** + * Detect CPU features and set the portable override. Call once at startup, + * before networking starts. Operations work without it, with auto-detection. + */ +void InitializeRuntime(bool force_portable) noexcept; + +/** The implementation that runs now: native only if compiled in, supported by the CPU and not overridden. */ +BackendNames GetBackendNames() noexcept; + +/** + * Forces portable C while alive, then restores the previous override (RAII). + * Each operation reads the override once, at entry, so one that is already + * running finishes on the backend it started with. Test code only. + */ +class ForcePortableForTesting +{ + bool m_previous; + +public: + ForcePortableForTesting() noexcept; + ~ForcePortableForTesting(); + ForcePortableForTesting(const ForcePortableForTesting&) = delete; + ForcePortableForTesting& operator=(const ForcePortableForTesting&) = delete; +}; + +/** mlkem-native's error codes (mlkem/mlkem_native.h), for InjectResultForTesting. */ +namespace upstream { +inline constexpr int ERR_FAIL{-1}; +inline constexpr int ERR_OUT_OF_MEMORY{-2}; +inline constexpr int ERR_RNG_FAIL{-3}; +inline constexpr int ERR_INVALID_PK{-4}; +inline constexpr int ERR_INVALID_SK{-5}; +inline constexpr int ERR_PCT_FAIL{-6}; +} // namespace upstream + +enum class Operation : uint8_t { KEYGEN, CHECK_PUBLIC_KEY, ENCAPS, DECAPS }; + +/** + * Test-only (RAII): while alive, `op` behaves as if mlkem-native had returned + * `upstream_result` (non-zero) after writing its outputs. This drives the + * wrapper's error mapping and output clearing, and lets callers test their + * handling of internal errors. Single-threaded test code only. + */ +class InjectResultForTesting +{ + Operation m_op; + int m_previous; + +public: + InjectResultForTesting(Operation op, int upstream_result) noexcept; + ~InjectResultForTesting(); + InjectResultForTesting(const InjectResultForTesting&) = delete; + InjectResultForTesting& operator=(const InjectResultForTesting&) = delete; +}; +} // namespace mlkem + +#endif // QBIT_CRYPTO_MLKEM_H diff --git a/src/crypto/mlkem_arith_backend.h b/src/crypto/mlkem_arith_backend.h new file mode 100644 index 0000000000..8d9a49625d --- /dev/null +++ b/src/crypto/mlkem_arith_backend.h @@ -0,0 +1,29 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#ifndef QBIT_CRYPTO_MLKEM_ARITH_BACKEND_H +#define QBIT_CRYPTO_MLKEM_ARITH_BACKEND_H + +// mlkem-native arithmetic backend (MLK_CONFIG_ARITH_BACKEND_FILE), used only +// when crypto/mlkem_config.h enables native code. +// +// Upstream's default selector, native/meta.h, picks the x86_64 backend only +// when the compiler itself targets AVX2, which baseline release builds do not. +// qbit includes the backend unconditionally instead: it is assembly plus +// constant tables, and every entry point asks mlk_sys_check_capability() +// before running, so CPUs without AVX2 take the portable C path. qbit answers +// MLK_SYS_CAP_X86_64_AVX2 for every extension the assembly executes (AVX2, +// SSSE3, SSE4.1, POPCNT, BMI2), not only the AVX2 its name says. + +#if defined(MLK_SYS_X86_64) +#define QBIT_MLKEM_ARITH_BACKEND_NAME "x86_64-avx2" +#include +#elif defined(MLK_SYS_AARCH64) +#define QBIT_MLKEM_ARITH_BACKEND_NAME "aarch64-neon" +#include +#else +#error "No mlkem-native arithmetic backend for this target." +#endif + +#endif // QBIT_CRYPTO_MLKEM_ARITH_BACKEND_H diff --git a/src/crypto/mlkem_backend.c b/src/crypto/mlkem_backend.c new file mode 100644 index 0000000000..933261e8ab --- /dev/null +++ b/src/crypto/mlkem_backend.c @@ -0,0 +1,29 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +// Reports which native backends mlkem-native was compiled with. This file is +// built with the same configuration and flags as mlkem-native's single +// compilation unit, so the backend headers it sees are the ones compiled in. + +#include + +#include + +const char* qbit_mlkem_compiled_arith_backend(void) +{ +#if defined(QBIT_MLKEM_ARITH_BACKEND_NAME) + return QBIT_MLKEM_ARITH_BACKEND_NAME; +#else + return "portable"; +#endif +} + +const char* qbit_mlkem_compiled_keccak_backend(void) +{ +#if defined(QBIT_MLKEM_KECCAK_BACKEND_NAME) + return QBIT_MLKEM_KECCAK_BACKEND_NAME; +#else + return "portable"; +#endif +} diff --git a/src/crypto/mlkem_config.h b/src/crypto/mlkem_config.h new file mode 100644 index 0000000000..488db496bf --- /dev/null +++ b/src/crypto/mlkem_config.h @@ -0,0 +1,89 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#ifndef QBIT_CRYPTO_MLKEM_CONFIG_H +#define QBIT_CRYPTO_MLKEM_CONFIG_H + +// qbit's configuration of the vendored mlkem-native library, selected with +// MLK_CONFIG_FILE in place of upstream's mlkem/mlkem_native_config.h, which +// documents every option used here. cmake/mlkem-native.cmake decides whether +// native code may be built and passes that decision in: +// +// QBIT_MLKEM_NATIVE build the native backends for this target (x86_64 with +// the SysV ABI, or AArch64 with NEON on Apple: AArch64 +// ELF targets stay portable because the assembly would +// strip their BTI and PAC marking); +// MLK_CONFIG_NO_ASM no assembly at all (MSVC and other toolchains without +// GNU assembler support). +// +// Neither set means portable C with inline-assembly value barriers. + +#define MLK_CONFIG_PARAMETER_SET 1024 +#define MLK_CONFIG_NAMESPACE_PREFIX qbit_mlkem1024 +// Only the deterministic APIs exist: callers supply all entropy. +#define MLK_CONFIG_NO_RANDOMIZED_API + +#if defined(MLK_BUILD_INTERNAL) + +#define MLK_CONFIG_CUSTOM_ZEROIZE +#define MLK_CONFIG_CUSTOM_CAPABILITY_FUNC + +// After the options sys.h reads, before the code that needs its definitions. +#include + +#if defined(QBIT_MLKEM_NATIVE) +#if defined(MLK_CONFIG_NO_ASM) +#error "QBIT_MLKEM_NATIVE and MLK_CONFIG_NO_ASM are both set; cmake/mlkem-native.cmake sets at most one." +#endif +#if defined(__has_feature) +#if __has_feature(memory_sanitizer) +#error "MemorySanitizer cannot see writes made by assembly; configure with WITH_MLKEM_NATIVE=AUTO or OFF." +#endif +#endif +#if defined(MLK_SYS_AARCH64) && defined(__ELF__) +#error "mlkem-native AArch64 assembly has no BTI landing pads or GNU property note; AArch64 ELF builds are portable only." +#endif +// Enable a backend only where qbit selects and tests one. Never define these +// to 0: upstream checks whether they are defined. +#if (defined(MLK_SYS_X86_64) && defined(MLK_SYSV_ABI_SUPPORTED)) || \ + (defined(MLK_SYS_AARCH64) && defined(MLK_SYS_AARCH64_NEON) && defined(MLK_SYS_APPLE)) +#define MLK_CONFIG_USE_NATIVE_BACKEND_ARITH +#define MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 +#define MLK_CONFIG_ARITH_BACKEND_FILE "crypto/mlkem_arith_backend.h" +#define MLK_CONFIG_FIPS202_BACKEND_FILE "crypto/mlkem_fips202_backend.h" +#else +#error "QBIT_MLKEM_NATIVE is set, but qbit has no native ML-KEM backend for this target; cmake/mlkem-native.cmake and this header disagree." +#endif +#endif // QBIT_MLKEM_NATIVE + +#if !defined(__ASSEMBLER__) +#include + +static MLK_INLINE void mlk_zeroize(void* ptr, size_t len) +{ + qbit_mlkem_zeroize(ptr, len); +} + +// Backends call this 80 to 100 times per operation, so it reads only state +// that crypto/mlkem.cpp cached before the operation started. Capabilities qbit +// does not use, MLK_SYS_CAP_AARCH64_SHA3 included, are never available. +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_sys_check_capability(mlk_sys_cap cap) +{ +#if defined(MLK_SYS_X86_64) + // Upstream gates its whole x86_64 backend on this one capability, so the + // answer covers every extension that assembly executes, not only AVX2. + if (cap == MLK_SYS_CAP_X86_64_AVX2) return qbit_mlkem_has_avx2() ? 1 : 0; +#endif +#if defined(MLK_SYS_AARCH64) + if (cap == MLK_SYS_CAP_AARCH64_NEON) return qbit_mlkem_has_neon() ? 1 : 0; +#endif + (void)cap; + return 0; +} +#endif // !__ASSEMBLER__ + +#endif // MLK_BUILD_INTERNAL + +#endif // QBIT_CRYPTO_MLKEM_CONFIG_H diff --git a/src/crypto/mlkem_fips202_backend.h b/src/crypto/mlkem_fips202_backend.h new file mode 100644 index 0000000000..26138ce5db --- /dev/null +++ b/src/crypto/mlkem_fips202_backend.h @@ -0,0 +1,55 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#ifndef QBIT_CRYPTO_MLKEM_FIPS202_BACKEND_H +#define QBIT_CRYPTO_MLKEM_FIPS202_BACKEND_H + +// mlkem-native Keccak backend (MLK_CONFIG_FIPS202_BACKEND_FILE), used only +// when crypto/mlkem_config.h enables native code. See +// crypto/mlkem_arith_backend.h for why the backend is chosen here rather than +// by upstream's fips202/native/auto.h. + +#if defined(MLK_SYS_X86_64) +#define QBIT_MLKEM_KECCAK_BACKEND_NAME "x86_64-avx2" +#include + +#elif defined(MLK_SYS_AARCH64) +#define QBIT_MLKEM_KECCAK_BACKEND_NAME "aarch64" +// The Keccak upstream picks for AArch64 without the SHA3 extension, used on +// Apple too even though Apple CPUs have it, so that every native AArch64 build +// runs the same code. (Native AArch64 is Apple-only for now; see +// crypto/mlkem_config.h.) +#include + +// Single-lane Keccak. This mirrors mlkem/src/fips202/native/aarch64/x1_scalar.h +// of mlkem-native v2.0.0 (d1b2fe782888bdb761a50336012923180be7f502): the same +// macros and the same proved assembly routine. The one difference is the +// capability check. Upstream's x1 entry point runs its scalar assembly +// unconditionally, so the portable override could not reach it, and +// mlkem::GetBackendNames() would claim portable Keccak while assembly ran. +// Here it falls back to C whenever the AArch64 capability is off. On every +// library update, diff upstream's x1_scalar.h against this block (see +// doc/subtrees/mlkem-native.md). +#define MLK_USE_NATIVE_FIPS202_X1 +#define MLK_FIPS202_AARCH64_NEED_X1_SCALAR +#if !defined(__ASSEMBLER__) +#include +#include +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x1_native(uint64_t* state) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON)) { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_keccak_f1600_x1_scalar_aarch64_asm(state, mlk_keccakf1600_round_constants); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif // !__ASSEMBLER__ + +#else +#error "No mlkem-native Keccak backend for this target." +#endif + +#endif // QBIT_CRYPTO_MLKEM_FIPS202_BACKEND_H diff --git a/src/crypto/mlkem_shim.h b/src/crypto/mlkem_shim.h new file mode 100644 index 0000000000..a7824fa339 --- /dev/null +++ b/src/crypto/mlkem_shim.h @@ -0,0 +1,44 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#ifndef QBIT_CRYPTO_MLKEM_SHIM_H +#define QBIT_CRYPTO_MLKEM_SHIM_H + +// C ABI between mlkem-native, which is C compiled with crypto/mlkem_config.h, +// and qbit's C++ code. mlkem-native's assembly includes the configuration too, +// so the declarations are hidden from the assembler. + +#if !defined(__ASSEMBLER__) +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// Implemented in C++, in crypto/mlkem.cpp. + +/** Wipe len bytes at ptr. Forwards to memory_cleanse. */ +void qbit_mlkem_zeroize(void* ptr, size_t len); +/** + * 1 if mlkem-native may run its x86_64 assembly now, else 0. Besides AVX2 that + * assembly needs SSSE3, SSE4.1, POPCNT and BMI2 (cpu_features::HasMlkemX86Native). + * Reads cached state only. + */ +int qbit_mlkem_has_avx2(void); +/** 1 if mlkem-native may run its AArch64 code now, else 0. Reads cached state only. */ +int qbit_mlkem_has_neon(void); + +// Implemented in C, in crypto/mlkem_backend.c, with mlkem-native's configuration. + +/** Name of the native arithmetic backend compiled in, or "portable". */ +const char* qbit_mlkem_compiled_arith_backend(void); +/** Name of the native Keccak backend compiled in, or "portable". */ +const char* qbit_mlkem_compiled_keccak_backend(void); + +#ifdef __cplusplus +} +#endif +#endif // !__ASSEMBLER__ + +#endif // QBIT_CRYPTO_MLKEM_SHIM_H diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index d75d01b6b7..40250e42ce 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -571,11 +571,9 @@ bool SelfTest() { #if !defined(DISABLE_OPTIMIZED_SHA256) #if (defined(__x86_64__) || defined(__amd64__) || defined(__i386__)) /** Check whether the OS has enabled AVX registers. */ -bool AVXEnabled() +bool AVXEnabled(sha256_implementation::XGetBVFn xgetbv) { - uint32_t a, d; - __asm__("xgetbv" : "=a"(a), "=d"(d) : "c"(0)); - return (a & 6) == 6; + return (xgetbv(0) & 6) == 6; } #endif #endif // DISABLE_OPTIMIZED_SHA256 @@ -583,6 +581,17 @@ bool AVXEnabled() std::string SHA256AutoDetect(sha256_implementation::UseImplementation use_implementation) +{ +#if defined(HAVE_GETCPUID) + return SHA256AutoDetect(use_implementation, GetCPUID, XGetBV); +#else + return SHA256AutoDetect(use_implementation, nullptr, nullptr); +#endif +} + +std::string SHA256AutoDetect(sha256_implementation::UseImplementation use_implementation, + [[maybe_unused]] sha256_implementation::CpuidFn cpuid, + [[maybe_unused]] sha256_implementation::XGetBVFn xgetbv) { std::string ret = "standard"; Transform = sha256::Transform; @@ -601,17 +610,22 @@ std::string SHA256AutoDetect(sha256_implementation::UseImplementation use_implem [[maybe_unused]] bool enabled_avx = false; uint32_t eax, ebx, ecx, edx; - GetCPUID(1, 0, eax, ebx, ecx, edx); + cpuid(1, 0, eax, ebx, ecx, edx); if (use_implementation & sha256_implementation::USE_SSE4) { - have_sse4 = (ecx >> 19) & 1; + // Fixes an upstream Bitcoin Core bug, which checks SSE4.1 only. The + // SSE4 code and the SSE4.1 and SHA-NI code chosen under this check + // also execute SSSE3 instructions (pshufb, palignr), which a CPU with + // SSE4.1 need not report: a virtual machine can mask SSSE3 alone, and + // the self-test below would then die with SIGILL. + have_sse4 = ((ecx >> 19) & 1) && ((ecx >> 9) & 1); } have_xsave = (ecx >> 27) & 1; have_avx = (ecx >> 28) & 1; if (have_xsave && have_avx) { - enabled_avx = AVXEnabled(); + enabled_avx = AVXEnabled(xgetbv); } if (have_sse4) { - GetCPUID(7, 0, eax, ebx, ecx, edx); + cpuid(7, 0, eax, ebx, ecx, edx); if (use_implementation & sha256_implementation::USE_AVX2) { have_avx2 = (ebx >> 5) & 1; } diff --git a/src/crypto/sha256.h b/src/crypto/sha256.h index f8a3b2c930..dc71b603cd 100644 --- a/src/crypto/sha256.h +++ b/src/crypto/sha256.h @@ -36,6 +36,10 @@ enum UseImplementation : uint8_t { USE_SSE4_AND_SHANI = USE_SSE4 | USE_SHANI, USE_ALL = USE_SSE4 | USE_AVX2 | USE_SHANI, }; + +/** The x86 CPUID and XGETBV queries, as compat/cpuid.h's GetCPUID and XGetBV. */ +using CpuidFn = void (*)(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d); +using XGetBVFn = uint64_t (*)(uint32_t index); } /** Autodetect the best available SHA256 implementation. @@ -43,6 +47,14 @@ enum UseImplementation : uint8_t { */ std::string SHA256AutoDetect(sha256_implementation::UseImplementation use_implementation = sha256_implementation::USE_ALL); +/** SHA256AutoDetect with the x86 CPUID and XGETBV queries answered by the + * given functions, so that tests can present synthetic CPUs; other targets + * ignore them. The selected code runs its self-test, so a synthetic CPU must + * not claim an extension this one lacks. + */ +std::string SHA256AutoDetect(sha256_implementation::UseImplementation use_implementation, + sha256_implementation::CpuidFn cpuid, sha256_implementation::XGetBVFn xgetbv); + /** Compute multiple double-SHA256's of 64-byte blobs. * output: pointer to a blocks*32 byte output buffer * input: pointer to a blocks*64 byte input buffer diff --git a/src/mlkem-native/LICENSE b/src/mlkem-native/LICENSE new file mode 100644 index 0000000000..79c71b1fdc --- /dev/null +++ b/src/mlkem-native/LICENSE @@ -0,0 +1,312 @@ +mlkem-native is a fork of the public domain Kyber reference +implementation, available on https://github.com/pq-crystals/kyber. + +All source code in mlkem/* and dev/* is licensed under your choice +of the Apache-2.0 license OR the ISC license OR the MIT license. +These licenses are reproduced at the bottom of this file. +The copyright holders are indicated at the top of each file. + +Files outside the library itself may carry different terms. Every file +states its own SPDX-License-Identifier, which determines the terms that +apply to it. In particular: + +The code in test/notrandombytes/*, and its copies in +examples/*/test_only_rng/* and scripts/notrandombytes, is derived from +https://cr.yp.to/papers.html#surf and licensed under +LicenseRef-PD-hp OR CC0-1.0 OR 0BSD OR MIT-0 OR MIT. +It is only used for testing purposes. + +The benchmarking code in test/hal/* carries the +MIT license. It is only used for testing purposes. + +The tiny_sha3 code in +examples/bring_your_own_fips202/custom_fips202/tiny_sha3/* and +examples/custom_backend/mlkem_native/src/fips202/native/custom/src/* +carries the MIT license. It is only used to demonstrate custom +FIPS-202 implementations. + +The proofs in proofs/* are in part derived from Amazon Web Services +verification infrastructure. Individual files carry, among others, +Apache-2.0 OR ISC OR MIT-0, MIT-0, and MIT-0 AND Apache-2.0. None of the +proofs are part of the library. + +Documentation is licensed under CC-BY-4.0. + +``` +Copyright (c) The mlkem-native project authors +Copyright (c) 2020 Dougall Johnson +Copyright (c) 2022 Arm Limited +SPDX-License-Identifier: MIT + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` + +The tiny_sha3 implementation in examples/custom_backend/mlkem_native/src/fips202/native/custom/src/ +carries the MIT license. It is only used for testing purposes. + +``` +SPDX-License-Identifier: MIT + +19-Nov-11 Markku-Juhani O. Saarinen +``` + +ISC license +----------- + +Copyright + +Permission to use, copy, modify, and/or distribute this software for any purpose +with or without fee is hereby granted, provided that the above copyright notice +and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF +THIS SOFTWARE. + + +MIT license +----------- + +Copyright + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the “Softwareâ€), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS ISâ€, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Apache-2.0 license +------------------ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/src/mlkem-native/mlkem/README.md b/src/mlkem-native/mlkem/README.md new file mode 100644 index 0000000000..e648c436a5 --- /dev/null +++ b/src/mlkem-native/mlkem/README.md @@ -0,0 +1,23 @@ +[//]: # (SPDX-License-Identifier: CC-BY-4.0) + +# mlkem-native source tree + +This is the main source tree of mlkem-native. + +## Building + +To build a mlkem-native for a fixed parameter set (ML-KEM-512/768/1024), build the compilation in units in `src/*` separately, and link to an RNG and your application. See [examples/basic](../examples/basic) for a simple example. + +Alternatively, you can use the auto-generated helper files [mlkem_native.c](mlkem_native.c) and [mlkem_native_asm.S](mlkem_native_asm.S), which bundle all *.c and *.S files together. See [examples/monolithic_build](../examples/monolithic_build) and [examples/monolithic_build_native](../examples/monolithic_build_native) for examples without and with native code. + +## Configuration + +The build is configured by [mlkem_native_config.h](mlkem_native_config.h), or by the file pointed to by `MLK_CONFIG_FILE`. Note in particular `MLK_CONFIG_PARAMETER_SET` and `MLK_CONFIG_NAMESPACE_PREFIX`, which set the parameter set and namespace prefix, respectively. + +## API + +The public API is defined in [mlkem_native.h](mlkem_native.h). + +## Supporting multiple parameter sets + +If you want to support multiple parameter sets, build the library once per parameter set you want to support. Set `MLK_CONFIG_MULTILEVEL_BUILD` for all of these builds; this is what appends the parameter set to the namespace prefix, keeping the level-specific symbols apart. To avoid duplicating shared functionality, additionally set `MLK_CONFIG_MULTILEVEL_WITH_SHARED` for exactly one of the builds, and `MLK_CONFIG_MULTILEVEL_NO_SHARED` for the others. Finally, link with RNG and your application as before. This is demonstrated in the examples [examples/multilevel_build](../examples/multilevel_build), [examples/multilevel_build_native](../examples/multilevel_build_native), [examples/monolithic_build_multilevel](../examples/monolithic_build_multilevel) and [examples/monolithic_build_multilevel_native](../examples/monolithic_build_multilevel_native). diff --git a/src/mlkem-native/mlkem/mlkem_native.c b/src/mlkem-native/mlkem/mlkem_native.c new file mode 100644 index 0000000000..66aed07ece --- /dev/null +++ b/src/mlkem-native/mlkem/mlkem_native.c @@ -0,0 +1,692 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +/****************************************************************************** + * + * Single compilation unit (SCU) for fixed-level build of mlkem-native + * + * This compilation unit bundles together all source files for a build + * of mlkem-native for a fixed security level (MLKEM-512/768/1024). + * + * # API + * + * The API exposed by this file is described in mlkem_native.h. + * + * # Multi-level build + * + * If you want an SCU build of mlkem-native with support for multiple security + * levels, you need to include this file multiple times, and set + * MLK_CONFIG_MULTILEVEL_WITH_SHARED and MLK_CONFIG_MULTILEVEL_NO_SHARED + * appropriately. This is exemplified in examples/monolithic_build_multilevel + * and examples/monolithic_build_multilevel_native. + * + * # Configuration + * + * The following options from the mlkem-native configuration are relevant: + * + * - MLK_CONFIG_FIPS202_CUSTOM_HEADER + * Set this option if you use a custom FIPS202 implementation. + * + * - MLK_CONFIG_USE_NATIVE_BACKEND_ARITH + * Set this option if you want to include the native arithmetic backends + * in your build. + * + * - MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 + * Set this option if you want to include the native FIPS202 backends + * in your build. + * + * - MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS + * Set this option if you want to keep the directives defined in + * level-independent headers. This is needed for a multi-level build. + */ + +/* If parts of the mlkem-native source tree are not used, + * consider reducing this header via `unifdef`. + * + * Example: + * ```bash + * unifdef -UMLK_CONFIG_USE_NATIVE_BACKEND_ARITH mlkem_native.c + * ``` + */ + +#include "src/common.h" + +#include "src/compress.c" +#include "src/debug.c" +#include "src/indcpa.c" +#include "src/kem.c" +#include "src/poly.c" +#include "src/poly_k.c" +#include "src/sampling.c" +#include "src/verify.c" + +#if !defined(MLK_CONFIG_FIPS202_CUSTOM_HEADER) +#include "src/fips202/fips202.c" +#include "src/fips202/fips202x4.c" +#include "src/fips202/keccakf1600.c" +#endif + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) +#if defined(MLK_SYS_AARCH64) +#include "src/native/aarch64/src/aarch64_zetas.c" +#include "src/native/aarch64/src/rej_uniform_table.c" +#endif +#if defined(MLK_SYS_X86_64) +#include "src/native/x86_64/src/compress_consts.c" +#include "src/native/x86_64/src/consts.c" +#include "src/native/x86_64/src/rej_uniform_table.c" +#endif +#if defined(MLK_SYS_RISCV64) +#include "src/native/riscv64/src/rv64v_debug.c" +#include "src/native/riscv64/src/rv64v_poly.c" +#endif +#if defined(MLK_SYS_PPC64LE) +#include "src/native/ppc64le/src/consts.c" +#endif +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_ARITH */ + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) +#if defined(MLK_SYS_AARCH64) +#include "src/fips202/native/aarch64/src/keccakf1600_round_constants.c" +#endif +#if defined(MLK_SYS_X86_64) +#include "src/fips202/native/x86_64/src/keccakf1600_constants.c" +#endif +#if defined(MLK_SYS_ARMV81M_MVE) +#include "src/fips202/native/armv81m/src/keccak_f1600_x4_mve.c" +#include "src/fips202/native/armv81m/src/keccakf1600_round_constants.c" +#endif +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ + +/* Macro #undef's + * + * The following undefines macros from headers + * included by the source files imported above. + * + * This is to allow building and linking multiple builds + * of mlkem-native for varying parameter sets through concatenation + * of this file, as if the files had been compiled separately. + * If this is not relevant to you, you may remove the following. + */ + +/* + * Undefine macros from MLK_CONFIG_PARAMETER_SET-specific files + */ +/* mlkem/mlkem_native.h */ +#undef MLKEM1024_BYTES +#undef MLKEM1024_CIPHERTEXTBYTES +#undef MLKEM1024_PUBLICKEYBYTES +#undef MLKEM1024_SECRETKEYBYTES +#undef MLKEM1024_SYMBYTES +#undef MLKEM512_BYTES +#undef MLKEM512_CIPHERTEXTBYTES +#undef MLKEM512_PUBLICKEYBYTES +#undef MLKEM512_SECRETKEYBYTES +#undef MLKEM512_SYMBYTES +#undef MLKEM768_BYTES +#undef MLKEM768_CIPHERTEXTBYTES +#undef MLKEM768_PUBLICKEYBYTES +#undef MLKEM768_SECRETKEYBYTES +#undef MLKEM768_SYMBYTES +#undef MLKEM_BYTES +#undef MLKEM_CIPHERTEXTBYTES +#undef MLKEM_CIPHERTEXTBYTES_ +#undef MLKEM_PUBLICKEYBYTES +#undef MLKEM_PUBLICKEYBYTES_ +#undef MLKEM_SECRETKEYBYTES +#undef MLKEM_SECRETKEYBYTES_ +#undef MLKEM_SYMBYTES +#undef MLK_API_CONCAT +#undef MLK_API_CONCAT_ +#undef MLK_API_CONCAT_UNDERSCORE +#undef MLK_API_MUST_CHECK_RETURN_VALUE +#undef MLK_API_NAMESPACE +#undef MLK_API_NAMESPACE_PREFIX +#undef MLK_API_QUALIFIER +#undef MLK_ERR_FAIL +#undef MLK_ERR_INVALID_PK +#undef MLK_ERR_INVALID_SK +#undef MLK_ERR_OUT_OF_MEMORY +#undef MLK_ERR_PCT_FAIL +#undef MLK_ERR_RNG_FAIL +#undef MLK_H +#undef MLK_MAX3_ +#undef MLK_TOTAL_ALLOC_1024 +#undef MLK_TOTAL_ALLOC_1024_DECAPS +#undef MLK_TOTAL_ALLOC_1024_ENCAPS +#undef MLK_TOTAL_ALLOC_1024_KEYPAIR +#undef MLK_TOTAL_ALLOC_1024_KEYPAIR_NO_PCT +#undef MLK_TOTAL_ALLOC_1024_KEYPAIR_PCT +#undef MLK_TOTAL_ALLOC_512 +#undef MLK_TOTAL_ALLOC_512_DECAPS +#undef MLK_TOTAL_ALLOC_512_ENCAPS +#undef MLK_TOTAL_ALLOC_512_KEYPAIR +#undef MLK_TOTAL_ALLOC_512_KEYPAIR_NO_PCT +#undef MLK_TOTAL_ALLOC_512_KEYPAIR_PCT +#undef MLK_TOTAL_ALLOC_768 +#undef MLK_TOTAL_ALLOC_768_DECAPS +#undef MLK_TOTAL_ALLOC_768_ENCAPS +#undef MLK_TOTAL_ALLOC_768_KEYPAIR +#undef MLK_TOTAL_ALLOC_768_KEYPAIR_NO_PCT +#undef MLK_TOTAL_ALLOC_768_KEYPAIR_PCT +/* mlkem/src/common.h */ +#undef MLK_ADD_PARAM_SET +#undef MLK_ALLOC +#undef MLK_APPLY +#undef MLK_ASM_FN_SIZE +#undef MLK_ASM_FN_SYMBOL +#undef MLK_ASM_NAMESPACE +#undef MLK_BUILD_INTERNAL +#undef MLK_COMMON_H +#undef MLK_CONCAT +#undef MLK_CONCAT_ +#undef MLK_EMPTY_CU +#undef MLK_ERR_FAIL +#undef MLK_ERR_INVALID_PK +#undef MLK_ERR_INVALID_SK +#undef MLK_ERR_OUT_OF_MEMORY +#undef MLK_ERR_PCT_FAIL +#undef MLK_ERR_RNG_FAIL +#undef MLK_EXTERNAL_API +#undef MLK_FIPS202X4_HEADER_FILE +#undef MLK_FIPS202_HEADER_FILE +#undef MLK_FREE +#undef MLK_INTERNAL_API +#undef MLK_INTERNAL_DATA_DECLARATION +#undef MLK_INTERNAL_DATA_DEFINITION +#undef MLK_NAMESPACE +#undef MLK_NAMESPACE_K +#undef MLK_NAMESPACE_PREFIX +#undef MLK_NAMESPACE_PREFIX_K +#undef mlk_memcpy +#undef mlk_memset +/* mlkem/src/indcpa.h */ +#undef MLK_INDCPA_H +#undef mlk_gen_matrix +#undef mlk_indcpa_dec +#undef mlk_indcpa_enc +#undef mlk_indcpa_keypair_derand +/* mlkem/src/kem.h */ +#undef MLK_KEM_H +#undef mlk_kem_check_pk +#undef mlk_kem_check_sk +#undef mlk_kem_dec +#undef mlk_kem_enc +#undef mlk_kem_enc_derand +#undef mlk_kem_keypair +#undef mlk_kem_keypair_derand +/* mlkem/src/params.h */ +#undef MLKEM_DU +#undef MLKEM_DV +#undef MLKEM_ETA1 +#undef MLKEM_ETA2 +#undef MLKEM_INDCCA_CIPHERTEXTBYTES +#undef MLKEM_INDCCA_PUBLICKEYBYTES +#undef MLKEM_INDCCA_SECRETKEYBYTES +#undef MLKEM_INDCPA_BYTES +#undef MLKEM_INDCPA_MSGBYTES +#undef MLKEM_INDCPA_PUBLICKEYBYTES +#undef MLKEM_INDCPA_SECRETKEYBYTES +#undef MLKEM_K +#undef MLKEM_N +#undef MLKEM_POLYBYTES +#undef MLKEM_POLYCOMPRESSEDBYTES_D10 +#undef MLKEM_POLYCOMPRESSEDBYTES_D11 +#undef MLKEM_POLYCOMPRESSEDBYTES_D4 +#undef MLKEM_POLYCOMPRESSEDBYTES_D5 +#undef MLKEM_POLYCOMPRESSEDBYTES_DU +#undef MLKEM_POLYCOMPRESSEDBYTES_DV +#undef MLKEM_POLYVECBYTES +#undef MLKEM_POLYVECCOMPRESSEDBYTES_DU +#undef MLKEM_Q +#undef MLKEM_Q_HALF +#undef MLKEM_SSBYTES +#undef MLKEM_SYMBYTES +#undef MLKEM_UINT12_LIMIT +#undef MLK_PARAMS_H +/* mlkem/src/poly_k.h */ +#undef MLK_POLY_K_H +#undef mlk_poly_compress_du +#undef mlk_poly_compress_dv +#undef mlk_poly_decompress_du +#undef mlk_poly_decompress_dv +#undef mlk_poly_getnoise_eta1122_4x +#undef mlk_poly_getnoise_eta1_4x +#undef mlk_poly_getnoise_eta2 +#undef mlk_poly_getnoise_eta2_4x +#undef mlk_polymat +#undef mlk_polyvec +#undef mlk_polyvec_add +#undef mlk_polyvec_basemul_acc_montgomery_cached +#undef mlk_polyvec_compress_du +#undef mlk_polyvec_decompress_du +#undef mlk_polyvec_frombytes +#undef mlk_polyvec_invntt_tomont +#undef mlk_polyvec_mulcache +#undef mlk_polyvec_mulcache_compute +#undef mlk_polyvec_ntt +#undef mlk_polyvec_reduce +#undef mlk_polyvec_tobytes +#undef mlk_polyvec_tomont + +#if !defined(MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS) +/* + * Undefine macros from MLK_CONFIG_PARAMETER_SET-generic files + */ +/* mlkem/src/compress.h */ +#undef MLK_COMPRESS_H +#undef mlk_poly_compress_d10 +#undef mlk_poly_compress_d11 +#undef mlk_poly_compress_d4 +#undef mlk_poly_compress_d5 +#undef mlk_poly_decompress_d10 +#undef mlk_poly_decompress_d11 +#undef mlk_poly_decompress_d4 +#undef mlk_poly_decompress_d5 +#undef mlk_poly_frombytes +#undef mlk_poly_frommsg +#undef mlk_poly_tobytes +#undef mlk_poly_tomsg +/* mlkem/src/context.h */ +#undef MLK_CONTEXT_H +#undef MLK_CONTEXT_PARAMETERS_0 +#undef MLK_CONTEXT_PARAMETERS_1 +#undef MLK_CONTEXT_PARAMETERS_2 +#undef MLK_CONTEXT_PARAMETERS_3 +#undef MLK_CONTEXT_PARAMETERS_4 +#undef MLK_CONTEXT_UNUSED +/* mlkem/src/debug.h */ +#undef MLK_DEBUG_H +#undef mlk_assert +#undef mlk_assert_abs_bound +#undef mlk_assert_abs_bound_2d +#undef mlk_assert_bound +#undef mlk_assert_bound_2d +#undef mlk_debug_check_assert +#undef mlk_debug_check_bounds +/* mlkem/src/poly.h */ +#undef MLK_INVNTT_BOUND +#undef MLK_NTT_BOUND +#undef MLK_POLY_H +#undef mlk_poly_add +#undef mlk_poly_invntt_tomont +#undef mlk_poly_mulcache_compute +#undef mlk_poly_ntt +#undef mlk_poly_reduce +#undef mlk_poly_sub +#undef mlk_poly_tomont +/* mlkem/src/randombytes.h */ +#undef MLK_RANDOMBYTES_H +/* mlkem/src/sampling.h */ +#undef MLK_SAMPLING_H +#undef mlk_poly_cbd2 +#undef mlk_poly_cbd3 +#undef mlk_poly_rej_uniform +#undef mlk_poly_rej_uniform_x4 +/* mlkem/src/symmetric.h */ +#undef MLK_SYMMETRIC_H +#undef MLK_XOF_RATE +#undef mlk_hash_g +#undef mlk_hash_h +#undef mlk_hash_j +#undef mlk_prf_eta +#undef mlk_prf_eta1 +#undef mlk_prf_eta1_x4 +#undef mlk_prf_eta2 +#undef mlk_xof_absorb +#undef mlk_xof_ctx +#undef mlk_xof_init +#undef mlk_xof_release +#undef mlk_xof_squeezeblocks +#undef mlk_xof_x4_absorb +#undef mlk_xof_x4_ctx +#undef mlk_xof_x4_init +#undef mlk_xof_x4_release +#undef mlk_xof_x4_squeezeblocks +/* mlkem/src/sys.h */ +#undef MLK_ALIGN +#undef MLK_ALIGN_UP +#undef MLK_ALWAYS_INLINE +#undef MLK_CET_ENDBR +#undef MLK_CT_TESTING_DECLASSIFY +#undef MLK_CT_TESTING_SECRET +#undef MLK_DEFAULT_ALIGN +#undef MLK_HAVE_INLINE_ASM +#undef MLK_INLINE +#undef MLK_MUST_CHECK_RETURN_VALUE +#undef MLK_NOINLINE +#undef MLK_RESTRICT +#undef MLK_STATIC_TESTABLE +#undef MLK_SYSV_ABI +#undef MLK_SYSV_ABI_SUPPORTED +#undef MLK_SYS_AARCH64 +#undef MLK_SYS_AARCH64_EB +#undef MLK_SYS_AARCH64_NEON +#undef MLK_SYS_APPLE +#undef MLK_SYS_ARMV81M_MVE +#undef MLK_SYS_BIG_ENDIAN +#undef MLK_SYS_H +#undef MLK_SYS_LINUX +#undef MLK_SYS_LITTLE_ENDIAN +#undef MLK_SYS_PPC64LE +#undef MLK_SYS_RISCV32 +#undef MLK_SYS_RISCV64 +#undef MLK_SYS_RISCV64_RVV +#undef MLK_SYS_WINDOWS +#undef MLK_SYS_X86_64 +#undef MLK_SYS_X86_64_AVX2 +/* mlkem/src/verify.h */ +#undef MLK_USE_ASM_VALUE_BARRIER +#undef MLK_VERIFY_H +#undef mlk_ct_opt_blocker_u64 +/* mlkem/src/cbmc.h */ +#undef MLK_CBMC_H +#undef __contract__ +#undef __loop__ + +#if !defined(MLK_CONFIG_FIPS202_CUSTOM_HEADER) +/* + * Undefine macros from FIPS-202 files + */ +/* mlkem/src/fips202/fips202.h */ +#undef FIPS202_X4_DEFAULT_IMPLEMENTATION +#undef MLK_FIPS202_FIPS202_H +#undef SHA3_256_HASHBYTES +#undef SHA3_256_RATE +#undef SHA3_384_RATE +#undef SHA3_512_HASHBYTES +#undef SHA3_512_RATE +#undef SHAKE128_RATE +#undef SHAKE256_RATE +#undef mlk_sha3_256 +#undef mlk_sha3_512 +#undef mlk_shake128_absorb_once +#undef mlk_shake128_init +#undef mlk_shake128_release +#undef mlk_shake128_squeezeblocks +#undef mlk_shake256 +/* mlkem/src/fips202/fips202x4.h */ +#undef MLK_FIPS202_FIPS202X4_H +#undef mlk_shake128x4_absorb_once +#undef mlk_shake128x4_init +#undef mlk_shake128x4_release +#undef mlk_shake128x4_squeezeblocks +#undef mlk_shake256x4 +/* mlkem/src/fips202/keccakf1600.h */ +#undef MLK_FIPS202_KECCAKF1600_H +#undef MLK_KECCAK_LANES +#undef MLK_KECCAK_WAY +#undef mlk_keccakf1600_extract_bytes +#undef mlk_keccakf1600_permute +#undef mlk_keccakf1600_xor_bytes +#undef mlk_keccakf1600x4_extract_bytes +#undef mlk_keccakf1600x4_permute +#undef mlk_keccakf1600x4_xor_bytes +#endif /* !MLK_CONFIG_FIPS202_CUSTOM_HEADER */ + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) +/* mlkem/src/fips202/native/api.h */ +#undef MLK_FIPS202_NATIVE_API_H +#undef MLK_NATIVE_FUNC_FALLBACK +#undef MLK_NATIVE_FUNC_SUCCESS +/* mlkem/src/fips202/native/auto.h */ +#undef MLK_FIPS202_NATIVE_AUTO_H +#if defined(MLK_SYS_AARCH64) +/* + * Undefine macros from native code (FIPS202, AArch64) + */ +/* mlkem/src/fips202/native/aarch64/auto.h */ +#undef MLK_FIPS202_NATIVE_AARCH64_AUTO_H +/* mlkem/src/fips202/native/aarch64/src/fips202_native_aarch64.h */ +#undef MLK_FIPS202_NATIVE_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H +#undef mlk_keccak_f1600_x1_scalar_aarch64_asm +#undef mlk_keccak_f1600_x1_v84a_aarch64_asm +#undef mlk_keccak_f1600_x2_v84a_aarch64_asm +#undef mlk_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm +#undef mlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm +#undef mlk_keccakf1600_round_constants +/* mlkem/src/fips202/native/aarch64/x1_scalar.h */ +#undef MLK_FIPS202_AARCH64_NEED_X1_SCALAR +#undef MLK_FIPS202_NATIVE_AARCH64_X1_SCALAR_H +#undef MLK_USE_NATIVE_FIPS202_X1 +/* mlkem/src/fips202/native/aarch64/x1_v84a.h */ +#undef MLK_FIPS202_AARCH64_NEED_X1_V84A +#undef MLK_FIPS202_NATIVE_AARCH64_X1_V84A_H +#undef MLK_USE_NATIVE_FIPS202_X1 +/* mlkem/src/fips202/native/aarch64/x2_v84a.h */ +#undef MLK_FIPS202_AARCH64_NEED_X2_V84A +#undef MLK_FIPS202_NATIVE_AARCH64_X2_V84A_H +#undef MLK_USE_NATIVE_FIPS202_X4 +/* mlkem/src/fips202/native/aarch64/x4_v8a_scalar.h */ +#undef MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID +#undef MLK_FIPS202_NATIVE_AARCH64_X4_V8A_SCALAR_H +#undef MLK_USE_NATIVE_FIPS202_X4 +/* mlkem/src/fips202/native/aarch64/x4_v8a_v84a_scalar.h */ +#undef MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID +#undef MLK_FIPS202_NATIVE_AARCH64_X4_V8A_V84A_SCALAR_H +#undef MLK_USE_NATIVE_FIPS202_X4 +#endif /* MLK_SYS_AARCH64 */ +#if defined(MLK_SYS_X86_64) +/* + * Undefine macros from native code (FIPS202, x86_64) + */ +/* mlkem/src/fips202/native/x86_64/keccak_f1600_x4_avx2.h */ +#undef MLK_FIPS202_NATIVE_X86_64_KECCAK_F1600_X4_AVX2_H +#undef MLK_FIPS202_X86_64_NEED_X4_AVX2 +#undef MLK_USE_NATIVE_FIPS202_X4 +/* mlkem/src/fips202/native/x86_64/src/fips202_native_x86_64.h */ +#undef MLK_FIPS202_NATIVE_X86_64_SRC_FIPS202_NATIVE_X86_64_H +#undef mlk_keccak_f1600_x4_avx2_asm +#undef mlk_keccak_rho56 +#undef mlk_keccak_rho8 +#undef mlk_keccakf1600_round_constants +#endif /* MLK_SYS_X86_64 */ +#if defined(MLK_SYS_ARMV81M_MVE) +/* + * Undefine macros from native code (FIPS202, Armv8.1-M) + */ +/* mlkem/src/fips202/native/armv81m/mve.h */ +#undef MLK_FIPS202_ARMV81M_NEED_X4 +#undef MLK_FIPS202_NATIVE_ARMV81M +#undef MLK_FIPS202_NATIVE_ARMV81M_MVE_H +#undef MLK_USE_NATIVE_FIPS202_X4 +#undef MLK_USE_NATIVE_FIPS202_X4_EXTRACT_BYTES +#undef MLK_USE_NATIVE_FIPS202_X4_XOR_BYTES +#undef mlk_keccak_f1600_x4_native_impl +/* mlkem/src/fips202/native/armv81m/src/fips202_native_armv81m.h */ +#undef MLK_FIPS202_NATIVE_ARMV81M_SRC_FIPS202_NATIVE_ARMV81M_H +#undef mlk_keccak_f1600_x4_mve_asm +#undef mlk_keccak_f1600_x4_state_extract_bytes_asm +#undef mlk_keccak_f1600_x4_state_xor_bytes_asm +#undef mlk_keccakf1600_round_constants +#endif /* MLK_SYS_ARMV81M_MVE */ +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) +/* mlkem/src/native/api.h */ +#undef MLK_INVNTT_BOUND +#undef MLK_NATIVE_API_H +#undef MLK_NATIVE_FUNC_FALLBACK +#undef MLK_NATIVE_FUNC_SUCCESS +#undef MLK_NTT_BOUND +/* mlkem/src/native/meta.h */ +#undef MLK_NATIVE_META_H +#if defined(MLK_SYS_AARCH64) +/* + * Undefine macros from native code (Arith, AArch64) + */ +/* mlkem/src/native/aarch64/meta.h */ +#undef MLK_ARITH_BACKEND_AARCH64 +#undef MLK_NATIVE_AARCH64_META_H +#undef MLK_USE_NATIVE_INTT +#undef MLK_USE_NATIVE_NTT +#undef MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED +#undef MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#undef MLK_USE_NATIVE_POLY_REDUCE +#undef MLK_USE_NATIVE_POLY_TOBYTES +#undef MLK_USE_NATIVE_POLY_TOMONT +#undef MLK_USE_NATIVE_REJ_UNIFORM +/* mlkem/src/native/aarch64/src/arith_native_aarch64.h */ +#undef MLK_NATIVE_AARCH64_SRC_ARITH_NATIVE_AARCH64_H +#undef mlk_aarch64_invntt_zetas_layer12345 +#undef mlk_aarch64_invntt_zetas_layer67 +#undef mlk_aarch64_ntt_zetas_layer12345 +#undef mlk_aarch64_ntt_zetas_layer67 +#undef mlk_aarch64_zetas_mulcache_native +#undef mlk_aarch64_zetas_mulcache_twisted_native +#undef mlk_intt_aarch64_asm +#undef mlk_ntt_aarch64_asm +#undef mlk_poly_mulcache_compute_aarch64_asm +#undef mlk_poly_reduce_aarch64_asm +#undef mlk_poly_tobytes_aarch64_asm +#undef mlk_poly_tomont_aarch64_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm +#undef mlk_rej_uniform_aarch64_asm +#undef mlk_rej_uniform_table +#endif /* MLK_SYS_AARCH64 */ +#if defined(MLK_SYS_X86_64) +/* + * Undefine macros from native code (Arith, X86_64) + */ +/* mlkem/src/native/x86_64/meta.h */ +#undef MLK_ARITH_BACKEND_X86_64_DEFAULT +#undef MLK_NATIVE_X86_64_META_H +#undef MLK_USE_NATIVE_INTT +#undef MLK_USE_NATIVE_NTT +#undef MLK_USE_NATIVE_NTT_CUSTOM_ORDER +#undef MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED +#undef MLK_USE_NATIVE_POLY_COMPRESS_D10 +#undef MLK_USE_NATIVE_POLY_COMPRESS_D11 +#undef MLK_USE_NATIVE_POLY_COMPRESS_D4 +#undef MLK_USE_NATIVE_POLY_COMPRESS_D5 +#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D10 +#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D11 +#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D4 +#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D5 +#undef MLK_USE_NATIVE_POLY_FROMBYTES +#undef MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#undef MLK_USE_NATIVE_POLY_REDUCE +#undef MLK_USE_NATIVE_POLY_TOBYTES +#undef MLK_USE_NATIVE_POLY_TOMONT +#undef MLK_USE_NATIVE_REJ_UNIFORM +/* mlkem/src/native/x86_64/src/arith_native_x86_64.h */ +#undef MLK_AVX2_REJ_UNIFORM_BUFLEN +#undef MLK_NATIVE_X86_64_SRC_ARITH_NATIVE_X86_64_H +#undef mlk_invntt_avx2_asm +#undef mlk_ntt_avx2_asm +#undef mlk_nttfrombytes_avx2_asm +#undef mlk_ntttobytes_avx2_asm +#undef mlk_nttunpack_avx2_asm +#undef mlk_poly_compress_d10_avx2_asm +#undef mlk_poly_compress_d11_avx2_asm +#undef mlk_poly_compress_d4_avx2_asm +#undef mlk_poly_compress_d5_avx2_asm +#undef mlk_poly_decompress_d10_avx2_asm +#undef mlk_poly_decompress_d11_avx2_asm +#undef mlk_poly_decompress_d4_avx2_asm +#undef mlk_poly_decompress_d5_avx2_asm +#undef mlk_poly_mulcache_compute_avx2_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm +#undef mlk_reduce_avx2_asm +#undef mlk_rej_uniform_avx2_asm +#undef mlk_rej_uniform_table +#undef mlk_tomont_avx2_asm +/* mlkem/src/native/x86_64/src/compress_consts.h */ +#undef MLK_NATIVE_X86_64_SRC_COMPRESS_CONSTS_H +#undef mlk_compress_d10_data +#undef mlk_compress_d11_data +#undef mlk_compress_d4_data +#undef mlk_compress_d5_data +#undef mlk_decompress_d10_data +#undef mlk_decompress_d11_data +#undef mlk_decompress_d4_data +#undef mlk_decompress_d5_data +/* mlkem/src/native/x86_64/src/consts.h */ +#undef MLK_AVX2_BACKEND_DATA_OFFSET_MULCACHE_TWIDDLES +#undef MLK_AVX2_BACKEND_DATA_OFFSET_REVIDXB +#undef MLK_AVX2_BACKEND_DATA_OFFSET_REVIDXD +#undef MLK_AVX2_BACKEND_DATA_OFFSET_ZETAS_EXP +#undef MLK_NATIVE_X86_64_SRC_CONSTS_H +#undef mlk_qdata +#endif /* MLK_SYS_X86_64 */ +#if defined(MLK_SYS_RISCV64) +/* + * Undefine macros from native code (Arith, RISC-V 64) + */ +/* mlkem/src/native/riscv64/meta.h */ +#undef MLK_ARITH_BACKEND_RISCV64 +#undef MLK_NATIVE_RISCV64_META_H +#undef MLK_USE_NATIVE_INTT +#undef MLK_USE_NATIVE_NTT +#undef MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED +#undef MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#undef MLK_USE_NATIVE_POLY_REDUCE +#undef MLK_USE_NATIVE_POLY_TOMONT +#undef MLK_USE_NATIVE_REJ_UNIFORM +/* mlkem/src/native/riscv64/src/arith_native_riscv64.h */ +#undef MLK_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H +#undef mlk_rv64v_poly_add +#undef mlk_rv64v_poly_basemul_mont_add_k2 +#undef mlk_rv64v_poly_basemul_mont_add_k3 +#undef mlk_rv64v_poly_basemul_mont_add_k4 +#undef mlk_rv64v_poly_invntt_tomont +#undef mlk_rv64v_poly_ntt +#undef mlk_rv64v_poly_reduce +#undef mlk_rv64v_poly_sub +#undef mlk_rv64v_poly_tomont +#undef mlk_rv64v_rej_uniform +/* mlkem/src/native/riscv64/src/rv64v_debug.h */ +#undef MLK_NATIVE_RISCV64_SRC_RV64V_DEBUG_H +#undef mlk_assert_abs_bound_int16m1 +#undef mlk_assert_abs_bound_int16m2 +#undef mlk_assert_bound_int16m1 +#undef mlk_assert_bound_int16m2 +#undef mlk_debug_check_bounds_int16m1 +#undef mlk_debug_check_bounds_int16m2 +#endif /* MLK_SYS_RISCV64 */ +#if defined(MLK_SYS_PPC64LE) +/* + * Undefine macros from native code (Arith, PPC64LE) + */ +/* mlkem/src/native/ppc64le/meta.h */ +#undef MLK_ARITH_BACKEND_NAME +#undef MLK_ARITH_BACKEND_PPC64LE_DEFAULT +#undef MLK_NATIVE_PPC64LE_META_H +#undef MLK_USE_NATIVE_INTT +#undef MLK_USE_NATIVE_NTT +#undef MLK_USE_NATIVE_POLY_REDUCE +#undef MLK_USE_NATIVE_POLY_TOMONT +/* mlkem/src/native/ppc64le/src/arith_native_ppc64le.h */ +#undef MLK_NATIVE_PPC64LE_SRC_ARITH_NATIVE_PPC64LE_H +#undef mlk_intt_ppc_asm +#undef mlk_ntt_ppc_asm +#undef mlk_poly_tomont_ppc_asm +#undef mlk_reduce_ppc_asm +/* mlkem/src/native/ppc64le/src/consts.h */ +#undef MLK_NATIVE_PPC64LE_SRC_CONSTS_H +#undef MLK_PPC_C20159_OFFSET +#undef MLK_PPC_NQ_OFFSET +#undef MLK_PPC_N_INV_OFFSET +#undef MLK_PPC_N_INV_TW_OFFSET +#undef MLK_PPC_Q_OFFSET +#undef MLK_PPC_TOMONT_OFFSET +#undef MLK_PPC_TOMONT_TW_OFFSET +#undef MLK_PPC_ZETA_INTT_OFFSET +#undef MLK_PPC_ZETA_INTT_TW_OFFSET +#undef MLK_PPC_ZETA_NTT_OFFSET +#undef MLK_PPC_ZETA_NTT_TW_OFFSET +#undef mlk_ppc_qdata +#endif /* MLK_SYS_PPC64LE */ +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_ARITH */ +#endif /* !MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS */ diff --git a/src/mlkem-native/mlkem/mlkem_native.h b/src/mlkem-native/mlkem/mlkem_native.h new file mode 100644 index 0000000000..fe8060aa8d --- /dev/null +++ b/src/mlkem-native/mlkem/mlkem_native.h @@ -0,0 +1,464 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS140_3_IG] + * Implementation Guidance for FIPS 140-3 and the Cryptographic Module + * Validation Program + * National Institute of Standards and Technology + * https://csrc.nist.gov/projects/cryptographic-module-validation-program/fips-140-3-ig-announcements + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_H +#define MLK_H + +/* + * Public API for mlkem-native. + * + * This header defines the public API of a single build of mlkem-native. + * + * Make sure the configuration file is in the include path + * (this is "mlkem_native_config.h" by default, or MLK_CONFIG_FILE if defined). + * + * # API conventions + * + * Conventions shared by all functions below (return values, pointer validity, + * output buffers on error) are documented in API-CONVENTIONS.md. + * + * # Multi-level builds + * + * This header specifies a build of mlkem-native for a fixed security level. + * If you need multiple security levels, leave the security level unspecified + * in the configuration file and include this header multiple times, setting + * MLK_CONFIG_PARAMETER_SET accordingly for each, and #undef'ing the MLK_H + * guard to allow multiple inclusions. + * + * In this case, the configuration file must also set + * MLK_CONFIG_MULTILEVEL_BUILD. Without it, the parameter set is not appended + * to the namespace prefix and all inclusions declare the same symbol names. + */ + +/******************************* Key sizes ************************************/ + +/* Sizes of cryptographic material, per parameter set */ +/* See mlkem/src/params.h for the arithmetic expressions giving rise to these */ +/* check-magic: off */ +#define MLKEM512_SECRETKEYBYTES 1632 +#define MLKEM512_PUBLICKEYBYTES 800 +#define MLKEM512_CIPHERTEXTBYTES 768 + +#define MLKEM768_SECRETKEYBYTES 2400 +#define MLKEM768_PUBLICKEYBYTES 1184 +#define MLKEM768_CIPHERTEXTBYTES 1088 + +#define MLKEM1024_SECRETKEYBYTES 3168 +#define MLKEM1024_PUBLICKEYBYTES 1568 +#define MLKEM1024_CIPHERTEXTBYTES 1568 +/* check-magic: on */ + +/* Size of randomness coins in bytes (level-independent) */ +#define MLKEM_SYMBYTES 32 +#define MLKEM512_SYMBYTES MLKEM_SYMBYTES +#define MLKEM768_SYMBYTES MLKEM_SYMBYTES +#define MLKEM1024_SYMBYTES MLKEM_SYMBYTES +/* Size of shared secret in bytes (level-independent) */ +#define MLKEM_BYTES 32 +#define MLKEM512_BYTES MLKEM_BYTES +#define MLKEM768_BYTES MLKEM_BYTES +#define MLKEM1024_BYTES MLKEM_BYTES + +/* Sizes of cryptographic material, as a function of LVL=512,768,1024 */ +#define MLKEM_SECRETKEYBYTES_(LVL) MLKEM##LVL##_SECRETKEYBYTES +#define MLKEM_PUBLICKEYBYTES_(LVL) MLKEM##LVL##_PUBLICKEYBYTES +#define MLKEM_CIPHERTEXTBYTES_(LVL) MLKEM##LVL##_CIPHERTEXTBYTES +#define MLKEM_SECRETKEYBYTES(LVL) MLKEM_SECRETKEYBYTES_(LVL) +#define MLKEM_PUBLICKEYBYTES(LVL) MLKEM_PUBLICKEYBYTES_(LVL) +#define MLKEM_CIPHERTEXTBYTES(LVL) MLKEM_CIPHERTEXTBYTES_(LVL) + +/****************************** Error codes ***********************************/ + +/* Generic failure condition. Currently not returned by any function; + * reserved for failures that no more specific code covers. */ +#define MLK_ERR_FAIL (-1) +/* An allocation failed. This can only happen if MLK_CONFIG_CUSTOM_ALLOC_FREE + * is defined and the provided MLK_CUSTOM_ALLOC can fail. */ +#define MLK_ERR_OUT_OF_MEMORY (-2) +/* An RNG failure occurred. Might be due to insufficient entropy or + * system misconfiguration. */ +#define MLK_ERR_RNG_FAIL (-3) +/* Public key validation failed: the @[FIPS203, Section 7.2, 'modulus check'] + * found a coefficient outside [0,q-1]. Returned by check_pk and by the + * encapsulation API. */ +#define MLK_ERR_INVALID_PK (-4) +/* Secret key validation failed: the @[FIPS203, Section 7.3, 'hash check'] + * found the embedded public key hash inconsistent. Returned by check_sk and + * by the decapsulation API. */ +#define MLK_ERR_INVALID_SK (-5) +/* The 'Pairwise Consistency Test' @[FIPS140_3_IG, p.87] and + * @[FIPS203, Section 7.1, Pairwise Consistency] failed. Only possible when + * MLK_CONFIG_KEYGEN_PCT is enabled; signals that the freshly generated key + * pair failed its encaps/decaps self-test. */ +#define MLK_ERR_PCT_FAIL (-6) + +/********************* Namespacing and Qualifiers *****************************/ + +#define MLK_API_CONCAT_(x, y) x##y +#define MLK_API_CONCAT(x, y) MLK_API_CONCAT_(x, y) +#define MLK_API_CONCAT_UNDERSCORE(x, y) MLK_API_CONCAT(MLK_API_CONCAT(x, _), y) + +/* You need to make sure the config file is in the include path. */ +#if defined(MLK_CONFIG_FILE) +#include MLK_CONFIG_FILE +#else +#include "mlkem_native_config.h" +#endif + +/* Namespace prefix for the public API symbols. For multi-level builds, the + * parameter set is appended to disambiguate the security levels. */ +#if defined(MLK_CONFIG_MULTILEVEL_BUILD) +#define MLK_API_NAMESPACE_PREFIX \ + MLK_API_CONCAT(MLK_CONFIG_NAMESPACE_PREFIX, MLK_CONFIG_PARAMETER_SET) +#else +#define MLK_API_NAMESPACE_PREFIX MLK_CONFIG_NAMESPACE_PREFIX +#endif + +#define MLK_API_NAMESPACE(sym) \ + MLK_API_CONCAT_UNDERSCORE(MLK_API_NAMESPACE_PREFIX, sym) + +#if defined(__GNUC__) || defined(__clang__) +#define MLK_API_MUST_CHECK_RETURN_VALUE __attribute__((warn_unused_result)) +#else +#define MLK_API_MUST_CHECK_RETURN_VALUE +#endif + +#if defined(MLK_CONFIG_EXTERNAL_API_QUALIFIER) +#define MLK_API_QUALIFIER MLK_CONFIG_EXTERNAL_API_QUALIFIER +#else +#define MLK_API_QUALIFIER +#endif + +/****************************** Function API **********************************/ + +#if !defined(MLK_CONFIG_CONSTANTS_ONLY) + +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +/** + * Generate a public/private keypair for the ML-KEM key encapsulation mechanism. + * + * @spec{Implements @[FIPS203, Algorithm 16, ML-KEM.KeyGen_Internal].} + * + * @param[out] pk Output public key, an array of + * MLKEM{512,768,1024}_PUBLICKEYBYTES bytes. + * @param[out] sk Output private key, an array of + * MLKEM{512,768,1024}_SECRETKEYBYTES bytes. + * @param[in] coins Input randomness, an array of 2*MLKEM_SYMBYTES uniformly + * random bytes. + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_PCT_FAIL MLK_CONFIG_KEYGEN_PCT enabled and PCT failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + * @retval MLK_ERR_RNG_FAIL MLK_CONFIG_KEYGEN_PCT enabled and random + * number generation failed within the PCT. + */ +MLK_API_QUALIFIER +MLK_API_MUST_CHECK_RETURN_VALUE +int MLK_API_NAMESPACE(keypair_derand)( + uint8_t pk[MLKEM_PUBLICKEYBYTES(MLK_CONFIG_PARAMETER_SET)], + uint8_t sk[MLKEM_SECRETKEYBYTES(MLK_CONFIG_PARAMETER_SET)], + const uint8_t coins[2 * MLKEM_SYMBYTES] +#ifdef MLK_CONFIG_CONTEXT_PARAMETER + , + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context +#endif +); + + +#if !defined(MLK_CONFIG_NO_RANDOMIZED_API) +/** + * Generate a public/private keypair for the ML-KEM key encapsulation mechanism. + * + * @spec{Implements @[FIPS203, Algorithm 19, ML-KEM.KeyGen].} + * + * @param[out] pk Output public key, an array of + * MLKEM{512,768,1024}_PUBLICKEYBYTES bytes. + * @param[out] sk Output private key, an array of + * MLKEM{512,768,1024}_SECRETKEYBYTES bytes. + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_PCT_FAIL MLK_CONFIG_KEYGEN_PCT enabled and PCT failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + * @retval MLK_ERR_RNG_FAIL Random number generation failed. + */ +MLK_API_QUALIFIER +MLK_API_MUST_CHECK_RETURN_VALUE +int MLK_API_NAMESPACE(keypair)( + uint8_t pk[MLKEM_PUBLICKEYBYTES(MLK_CONFIG_PARAMETER_SET)], + uint8_t sk[MLKEM_SECRETKEYBYTES(MLK_CONFIG_PARAMETER_SET)] +#ifdef MLK_CONFIG_CONTEXT_PARAMETER + , + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context +#endif +); +#endif /* !MLK_CONFIG_NO_RANDOMIZED_API */ +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) +/** + * Generate ciphertext and shared secret for a given public key. + * + * @spec{Implements @[FIPS203, Algorithm 17, ML-KEM.Encaps_Internal].} + * + * @param[out] ct Output ciphertext, an array of + * MLKEM{512,768,1024}_CIPHERTEXTBYTES bytes. + * @param[out] ss Output shared secret, an array of MLKEM_BYTES bytes. + * @param[in] pk Input public key, an array of + * MLKEM{512,768,1024}_PUBLICKEYBYTES bytes. + * @param[in] coins Input randomness, an array of MLKEM_SYMBYTES bytes. + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_INVALID_PK The 'modulus check' @[FIPS203, Section 7.2] + * for the public key failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +MLK_API_QUALIFIER +MLK_API_MUST_CHECK_RETURN_VALUE +int MLK_API_NAMESPACE(enc_derand)( + uint8_t ct[MLKEM_CIPHERTEXTBYTES(MLK_CONFIG_PARAMETER_SET)], + uint8_t ss[MLKEM_BYTES], + const uint8_t pk[MLKEM_PUBLICKEYBYTES(MLK_CONFIG_PARAMETER_SET)], + const uint8_t coins[MLKEM_SYMBYTES] +#ifdef MLK_CONFIG_CONTEXT_PARAMETER + , + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context +#endif +); + +#if !defined(MLK_CONFIG_NO_RANDOMIZED_API) +/** + * Generate ciphertext and shared secret for a given public key. + * + * @spec{Implements @[FIPS203, Algorithm 20, ML-KEM.Encaps].} + * + * @param[out] ct Output ciphertext, an array of + * MLKEM{512,768,1024}_CIPHERTEXTBYTES bytes. + * @param[out] ss Output shared secret, an array of MLKEM_BYTES bytes. + * @param[in] pk Input public key, an array of + * MLKEM{512,768,1024}_PUBLICKEYBYTES bytes. + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_INVALID_PK The 'modulus check' @[FIPS203, Section 7.2] + * for the public key failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + * @retval MLK_ERR_RNG_FAIL Random number generation failed. + */ +MLK_API_QUALIFIER +MLK_API_MUST_CHECK_RETURN_VALUE +int MLK_API_NAMESPACE(enc)( + uint8_t ct[MLKEM_CIPHERTEXTBYTES(MLK_CONFIG_PARAMETER_SET)], + uint8_t ss[MLKEM_BYTES], + const uint8_t pk[MLKEM_PUBLICKEYBYTES(MLK_CONFIG_PARAMETER_SET)] +#ifdef MLK_CONFIG_CONTEXT_PARAMETER + , + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context +#endif +); +#endif /* !MLK_CONFIG_NO_RANDOMIZED_API */ +#endif /* !MLK_CONFIG_NO_ENCAPS_API */ + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +/** + * Generate shared secret for a given ciphertext and private key. + * + * @spec{Implements @[FIPS203, Algorithm 21, ML-KEM.Decaps].} + * + * @param[out] ss Output shared secret, an array of MLKEM_BYTES bytes. + * @param[in] ct Input ciphertext, an array of + * MLKEM{512,768,1024}_CIPHERTEXTBYTES bytes. + * @param[in] sk Input private key, an array of + * MLKEM{512,768,1024}_SECRETKEYBYTES bytes. + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_INVALID_SK The 'hash check' @[FIPS203, Section 7.3] + * for the secret key failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +MLK_API_QUALIFIER +MLK_API_MUST_CHECK_RETURN_VALUE +int MLK_API_NAMESPACE(dec)( + uint8_t ss[MLKEM_BYTES], + const uint8_t ct[MLKEM_CIPHERTEXTBYTES(MLK_CONFIG_PARAMETER_SET)], + const uint8_t sk[MLKEM_SECRETKEYBYTES(MLK_CONFIG_PARAMETER_SET)] +#ifdef MLK_CONFIG_CONTEXT_PARAMETER + , + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context +#endif +); +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + + +/** + * Implements modulus check mandated by FIPS 203, i.e., ensures that + * coefficients are in [0,q-1]. + * + * @spec{Implements @[FIPS203, Section 7.2, 'modulus check'].} + * + * @param[in] pk Input public key, an array of + * MLKEM{512,768,1024}_PUBLICKEYBYTES bytes. + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_INVALID_PK Modulus check failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +#if !defined(MLK_CONFIG_NO_ENCAPS_API) +MLK_API_QUALIFIER +MLK_API_MUST_CHECK_RETURN_VALUE +int MLK_API_NAMESPACE(check_pk)( + const uint8_t pk[MLKEM_PUBLICKEYBYTES(MLK_CONFIG_PARAMETER_SET)] +#ifdef MLK_CONFIG_CONTEXT_PARAMETER + , + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context +#endif +); +#endif /* !MLK_CONFIG_NO_ENCAPS_API */ + +/** + * Implements public key hash check mandated by FIPS 203, i.e., ensures that + * sk[768ð‘˜+32 ∶ 768ð‘˜+64] = H(pk) = H(sk[384𑘠: 768ð‘˜+32]). + * + * @spec{Implements @[FIPS203, Section 7.3, 'hash check'].} + * + * @param[in] sk Input private key, an array of + * MLKEM{512,768,1024}_SECRETKEYBYTES bytes. + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_INVALID_SK Public key hash check failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +#if !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_API_QUALIFIER +MLK_API_MUST_CHECK_RETURN_VALUE +int MLK_API_NAMESPACE(check_sk)( + const uint8_t sk[MLKEM_SECRETKEYBYTES(MLK_CONFIG_PARAMETER_SET)] +#ifdef MLK_CONFIG_CONTEXT_PARAMETER + , + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context +#endif +); +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#ifdef __cplusplus +} +#endif + +#undef MLK_API_NAMESPACE_PREFIX + +#endif /* !MLK_CONFIG_CONSTANTS_ONLY */ + + +/***************************** Memory Usage **********************************/ + +/* + * By default mlkem-native performs all memory allocations on the stack. + * Alternatively, mlkem-native supports custom allocation of large structures + * through the `MLK_CONFIG_CUSTOM_ALLOC_FREE` configuration option. + * See mlkem_native_config.h for details. + * + * `MLK_TOTAL_ALLOC_{512,768,1024}_{KEYPAIR,ENCAPS,DECAPS}` indicates the + * maximum (accumulative) allocation via MLK_ALLOC for each parameter set and + * operation. Note that some stack allocation remains even when using custom + * allocators, so these values are lower than total stack usage with the default + * stack-only allocation. + * + * These constants may be used to implement custom allocations using a + * fixed-sized buffer and a simple allocator (e.g., bump allocator). + */ +/* check-magic: off */ +#define MLK_TOTAL_ALLOC_512_KEYPAIR_NO_PCT 5824 +#define MLK_TOTAL_ALLOC_512_KEYPAIR_PCT 10048 +#define MLK_TOTAL_ALLOC_512_ENCAPS 8384 +#define MLK_TOTAL_ALLOC_512_DECAPS 9152 +#define MLK_TOTAL_ALLOC_768_KEYPAIR_NO_PCT 10176 +#define MLK_TOTAL_ALLOC_768_KEYPAIR_PCT 15552 +#define MLK_TOTAL_ALLOC_768_ENCAPS 13248 +#define MLK_TOTAL_ALLOC_768_DECAPS 14336 +#define MLK_TOTAL_ALLOC_1024_KEYPAIR_NO_PCT 15552 +#define MLK_TOTAL_ALLOC_1024_KEYPAIR_PCT 22400 +#define MLK_TOTAL_ALLOC_1024_ENCAPS 19136 +#define MLK_TOTAL_ALLOC_1024_DECAPS 20704 +/* check-magic: on */ + +/* + * MLK_TOTAL_ALLOC_*_KEYPAIR adapts based on MLK_CONFIG_KEYGEN_PCT. + */ +#if defined(MLK_CONFIG_KEYGEN_PCT) +#define MLK_TOTAL_ALLOC_512_KEYPAIR MLK_TOTAL_ALLOC_512_KEYPAIR_PCT +#define MLK_TOTAL_ALLOC_768_KEYPAIR MLK_TOTAL_ALLOC_768_KEYPAIR_PCT +#define MLK_TOTAL_ALLOC_1024_KEYPAIR MLK_TOTAL_ALLOC_1024_KEYPAIR_PCT +#else +#define MLK_TOTAL_ALLOC_512_KEYPAIR MLK_TOTAL_ALLOC_512_KEYPAIR_NO_PCT +#define MLK_TOTAL_ALLOC_768_KEYPAIR MLK_TOTAL_ALLOC_768_KEYPAIR_NO_PCT +#define MLK_TOTAL_ALLOC_1024_KEYPAIR MLK_TOTAL_ALLOC_1024_KEYPAIR_NO_PCT +#endif + +#define MLK_MAX3_(a, b, c) \ + ((a) > (b) ? ((a) > (c) ? (a) : (c)) : ((b) > (c) ? (b) : (c))) + +/* + * `MLK_TOTAL_ALLOC_{512,768,1024}` is the maximum across all operations for + * each parameter set. + */ +#define MLK_TOTAL_ALLOC_512 \ + MLK_MAX3_(MLK_TOTAL_ALLOC_512_KEYPAIR, MLK_TOTAL_ALLOC_512_ENCAPS, \ + MLK_TOTAL_ALLOC_512_DECAPS) +#define MLK_TOTAL_ALLOC_768 \ + MLK_MAX3_(MLK_TOTAL_ALLOC_768_KEYPAIR, MLK_TOTAL_ALLOC_768_ENCAPS, \ + MLK_TOTAL_ALLOC_768_DECAPS) +#define MLK_TOTAL_ALLOC_1024 \ + MLK_MAX3_(MLK_TOTAL_ALLOC_1024_KEYPAIR, MLK_TOTAL_ALLOC_1024_ENCAPS, \ + MLK_TOTAL_ALLOC_1024_DECAPS) + +#endif /* !MLK_H */ diff --git a/src/mlkem-native/mlkem/mlkem_native_asm.S b/src/mlkem-native/mlkem/mlkem_native_asm.S new file mode 100644 index 0000000000..a2f1cf192a --- /dev/null +++ b/src/mlkem-native/mlkem/mlkem_native_asm.S @@ -0,0 +1,716 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +/****************************************************************************** + * + * Single assembly unit for fixed-level build of mlkem-native + * + * This assembly unit bundles together all assembly files for a build + * of mlkem-native for a fixed security level (MLKEM-512/768/1024). + * + * # Multi-level build + * + * If you want an SCU build of mlkem-native with support for multiple security + * levels, you should include this file once with + * MLK_CONFIG_MULTILEVEL_WITH_SHARED set. + * + * (You could also follow the same pattern as for mlkem_native.c + * and include it for every level, setting MLK_CONFIG_MULTILEVEL_NO_SHARED + * for all but one. For builds with MLK_CONFIG_MULTILEVEL_NO_SHARED, this + * file will then be ignored.) + * + * # Configuration + * + * The following options from the mlkem-native configuration are relevant: + * + * - MLK_CONFIG_FIPS202_CUSTOM_HEADER + * Set this option if you use a custom FIPS202 implementation. + * + * - MLK_CONFIG_USE_NATIVE_BACKEND_ARITH + * Set this option if you want to include the native arithmetic backends + * in your build. + * + * - MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 + * Set this option if you want to include the native FIPS202 backends + * in your build. + * + * - MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS + * Set this option if you want to keep the directives defined in + * level-independent headers. This is needed for a multi-level build. + */ + +/* If parts of the mlkem-native source tree are not used, + * consider reducing this header via `unifdef`. + * + * Example: + * ```bash + * unifdef -UMLK_CONFIG_USE_NATIVE_BACKEND_ARITH mlkem_native_asm.S + * ``` + */ + +#include "src/common.h" + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) +#if defined(MLK_SYS_AARCH64) +#include "src/native/aarch64/src/mlkem_intt_aarch64_asm.S" +#include "src/native/aarch64/src/mlkem_ntt_aarch64_asm.S" +#include "src/native/aarch64/src/mlkem_poly_mulcache_compute_aarch64_asm.S" +#include "src/native/aarch64/src/mlkem_poly_reduce_aarch64_asm.S" +#include "src/native/aarch64/src/mlkem_poly_tobytes_aarch64_asm.S" +#include "src/native/aarch64/src/mlkem_poly_tomont_aarch64_asm.S" +#include "src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.S" +#include "src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.S" +#include "src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.S" +#include "src/native/aarch64/src/mlkem_rej_uniform_aarch64_asm.S" +#endif /* MLK_SYS_AARCH64 */ +#if defined(MLK_SYS_X86_64) +#include "src/native/x86_64/src/mlkem_intt_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_ntt_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_nttfrombytes_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_ntttobytes_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_nttunpack_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_poly_compress_d10_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_poly_compress_d11_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_poly_compress_d4_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_poly_compress_d5_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_poly_decompress_d10_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_poly_decompress_d11_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_poly_decompress_d4_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_poly_decompress_d5_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_poly_mulcache_compute_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_reduce_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_rej_uniform_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_tomont_avx2_asm.S" +#endif /* MLK_SYS_X86_64 */ +#if defined(MLK_SYS_RISCV64) +#endif +#if defined(MLK_SYS_PPC64LE) +#include "src/native/ppc64le/src/mlkem_intt_ppc_asm.S" +#include "src/native/ppc64le/src/mlkem_ntt_ppc_asm.S" +#include "src/native/ppc64le/src/mlkem_poly_tomont_ppc_asm.S" +#include "src/native/ppc64le/src/mlkem_reduce_ppc_asm.S" +#endif /* MLK_SYS_PPC64LE */ +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_ARITH */ + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) +#if defined(MLK_SYS_AARCH64) +#include "src/fips202/native/aarch64/src/keccak_f1600_x1_scalar_aarch64_asm.S" +#include "src/fips202/native/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S" +#include "src/fips202/native/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S" +#include "src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S" +#include "src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S" +#endif /* MLK_SYS_AARCH64 */ +#if defined(MLK_SYS_X86_64) +#include "src/fips202/native/x86_64/src/keccak_f1600_x4_avx2_asm.S" +#endif +#if defined(MLK_SYS_ARMV81M_MVE) +#include "src/fips202/native/armv81m/src/keccak_f1600_x4_mve.S" +#include "src/fips202/native/armv81m/src/keccak_f1600_x4_state_extract_bytes_mve.S" +#include "src/fips202/native/armv81m/src/keccak_f1600_x4_state_xor_bytes_mve.S" +#endif +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ + + +/* Macro #undef's + * + * The following undefines macros from headers + * included by the source files imported above. + * + * This is to allow building and linking multiple builds + * of mlkem-native for varying parameter sets through concatenation + * of this file, as if the files had been compiled separately. + * If this is not relevant to you, you may remove the following. + * + * NOTE: This is not needed for the assembly SCU since, at present, + * there is no need to include it multiple times. + * We keep it for uniformity with mlkem_native.c only. + * + * NOTE: To avoid having to distinguish between which headers are included + * from the assembly files, we #undef the same set of directives + * as in mlkem_native.c + */ + +/* + * Undefine macros from MLK_CONFIG_PARAMETER_SET-specific files + */ +/* mlkem/mlkem_native.h */ +#undef MLKEM1024_BYTES +#undef MLKEM1024_CIPHERTEXTBYTES +#undef MLKEM1024_PUBLICKEYBYTES +#undef MLKEM1024_SECRETKEYBYTES +#undef MLKEM1024_SYMBYTES +#undef MLKEM512_BYTES +#undef MLKEM512_CIPHERTEXTBYTES +#undef MLKEM512_PUBLICKEYBYTES +#undef MLKEM512_SECRETKEYBYTES +#undef MLKEM512_SYMBYTES +#undef MLKEM768_BYTES +#undef MLKEM768_CIPHERTEXTBYTES +#undef MLKEM768_PUBLICKEYBYTES +#undef MLKEM768_SECRETKEYBYTES +#undef MLKEM768_SYMBYTES +#undef MLKEM_BYTES +#undef MLKEM_CIPHERTEXTBYTES +#undef MLKEM_CIPHERTEXTBYTES_ +#undef MLKEM_PUBLICKEYBYTES +#undef MLKEM_PUBLICKEYBYTES_ +#undef MLKEM_SECRETKEYBYTES +#undef MLKEM_SECRETKEYBYTES_ +#undef MLKEM_SYMBYTES +#undef MLK_API_CONCAT +#undef MLK_API_CONCAT_ +#undef MLK_API_CONCAT_UNDERSCORE +#undef MLK_API_MUST_CHECK_RETURN_VALUE +#undef MLK_API_NAMESPACE +#undef MLK_API_NAMESPACE_PREFIX +#undef MLK_API_QUALIFIER +#undef MLK_ERR_FAIL +#undef MLK_ERR_INVALID_PK +#undef MLK_ERR_INVALID_SK +#undef MLK_ERR_OUT_OF_MEMORY +#undef MLK_ERR_PCT_FAIL +#undef MLK_ERR_RNG_FAIL +#undef MLK_H +#undef MLK_MAX3_ +#undef MLK_TOTAL_ALLOC_1024 +#undef MLK_TOTAL_ALLOC_1024_DECAPS +#undef MLK_TOTAL_ALLOC_1024_ENCAPS +#undef MLK_TOTAL_ALLOC_1024_KEYPAIR +#undef MLK_TOTAL_ALLOC_1024_KEYPAIR_NO_PCT +#undef MLK_TOTAL_ALLOC_1024_KEYPAIR_PCT +#undef MLK_TOTAL_ALLOC_512 +#undef MLK_TOTAL_ALLOC_512_DECAPS +#undef MLK_TOTAL_ALLOC_512_ENCAPS +#undef MLK_TOTAL_ALLOC_512_KEYPAIR +#undef MLK_TOTAL_ALLOC_512_KEYPAIR_NO_PCT +#undef MLK_TOTAL_ALLOC_512_KEYPAIR_PCT +#undef MLK_TOTAL_ALLOC_768 +#undef MLK_TOTAL_ALLOC_768_DECAPS +#undef MLK_TOTAL_ALLOC_768_ENCAPS +#undef MLK_TOTAL_ALLOC_768_KEYPAIR +#undef MLK_TOTAL_ALLOC_768_KEYPAIR_NO_PCT +#undef MLK_TOTAL_ALLOC_768_KEYPAIR_PCT +/* mlkem/src/common.h */ +#undef MLK_ADD_PARAM_SET +#undef MLK_ALLOC +#undef MLK_APPLY +#undef MLK_ASM_FN_SIZE +#undef MLK_ASM_FN_SYMBOL +#undef MLK_ASM_NAMESPACE +#undef MLK_BUILD_INTERNAL +#undef MLK_COMMON_H +#undef MLK_CONCAT +#undef MLK_CONCAT_ +#undef MLK_EMPTY_CU +#undef MLK_ERR_FAIL +#undef MLK_ERR_INVALID_PK +#undef MLK_ERR_INVALID_SK +#undef MLK_ERR_OUT_OF_MEMORY +#undef MLK_ERR_PCT_FAIL +#undef MLK_ERR_RNG_FAIL +#undef MLK_EXTERNAL_API +#undef MLK_FIPS202X4_HEADER_FILE +#undef MLK_FIPS202_HEADER_FILE +#undef MLK_FREE +#undef MLK_INTERNAL_API +#undef MLK_INTERNAL_DATA_DECLARATION +#undef MLK_INTERNAL_DATA_DEFINITION +#undef MLK_NAMESPACE +#undef MLK_NAMESPACE_K +#undef MLK_NAMESPACE_PREFIX +#undef MLK_NAMESPACE_PREFIX_K +#undef mlk_memcpy +#undef mlk_memset +/* mlkem/src/indcpa.h */ +#undef MLK_INDCPA_H +#undef mlk_gen_matrix +#undef mlk_indcpa_dec +#undef mlk_indcpa_enc +#undef mlk_indcpa_keypair_derand +/* mlkem/src/kem.h */ +#undef MLK_KEM_H +#undef mlk_kem_check_pk +#undef mlk_kem_check_sk +#undef mlk_kem_dec +#undef mlk_kem_enc +#undef mlk_kem_enc_derand +#undef mlk_kem_keypair +#undef mlk_kem_keypair_derand +/* mlkem/src/params.h */ +#undef MLKEM_DU +#undef MLKEM_DV +#undef MLKEM_ETA1 +#undef MLKEM_ETA2 +#undef MLKEM_INDCCA_CIPHERTEXTBYTES +#undef MLKEM_INDCCA_PUBLICKEYBYTES +#undef MLKEM_INDCCA_SECRETKEYBYTES +#undef MLKEM_INDCPA_BYTES +#undef MLKEM_INDCPA_MSGBYTES +#undef MLKEM_INDCPA_PUBLICKEYBYTES +#undef MLKEM_INDCPA_SECRETKEYBYTES +#undef MLKEM_K +#undef MLKEM_N +#undef MLKEM_POLYBYTES +#undef MLKEM_POLYCOMPRESSEDBYTES_D10 +#undef MLKEM_POLYCOMPRESSEDBYTES_D11 +#undef MLKEM_POLYCOMPRESSEDBYTES_D4 +#undef MLKEM_POLYCOMPRESSEDBYTES_D5 +#undef MLKEM_POLYCOMPRESSEDBYTES_DU +#undef MLKEM_POLYCOMPRESSEDBYTES_DV +#undef MLKEM_POLYVECBYTES +#undef MLKEM_POLYVECCOMPRESSEDBYTES_DU +#undef MLKEM_Q +#undef MLKEM_Q_HALF +#undef MLKEM_SSBYTES +#undef MLKEM_SYMBYTES +#undef MLKEM_UINT12_LIMIT +#undef MLK_PARAMS_H +/* mlkem/src/poly_k.h */ +#undef MLK_POLY_K_H +#undef mlk_poly_compress_du +#undef mlk_poly_compress_dv +#undef mlk_poly_decompress_du +#undef mlk_poly_decompress_dv +#undef mlk_poly_getnoise_eta1122_4x +#undef mlk_poly_getnoise_eta1_4x +#undef mlk_poly_getnoise_eta2 +#undef mlk_poly_getnoise_eta2_4x +#undef mlk_polymat +#undef mlk_polyvec +#undef mlk_polyvec_add +#undef mlk_polyvec_basemul_acc_montgomery_cached +#undef mlk_polyvec_compress_du +#undef mlk_polyvec_decompress_du +#undef mlk_polyvec_frombytes +#undef mlk_polyvec_invntt_tomont +#undef mlk_polyvec_mulcache +#undef mlk_polyvec_mulcache_compute +#undef mlk_polyvec_ntt +#undef mlk_polyvec_reduce +#undef mlk_polyvec_tobytes +#undef mlk_polyvec_tomont + +#if !defined(MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS) +/* + * Undefine macros from MLK_CONFIG_PARAMETER_SET-generic files + */ +/* mlkem/src/compress.h */ +#undef MLK_COMPRESS_H +#undef mlk_poly_compress_d10 +#undef mlk_poly_compress_d11 +#undef mlk_poly_compress_d4 +#undef mlk_poly_compress_d5 +#undef mlk_poly_decompress_d10 +#undef mlk_poly_decompress_d11 +#undef mlk_poly_decompress_d4 +#undef mlk_poly_decompress_d5 +#undef mlk_poly_frombytes +#undef mlk_poly_frommsg +#undef mlk_poly_tobytes +#undef mlk_poly_tomsg +/* mlkem/src/context.h */ +#undef MLK_CONTEXT_H +#undef MLK_CONTEXT_PARAMETERS_0 +#undef MLK_CONTEXT_PARAMETERS_1 +#undef MLK_CONTEXT_PARAMETERS_2 +#undef MLK_CONTEXT_PARAMETERS_3 +#undef MLK_CONTEXT_PARAMETERS_4 +#undef MLK_CONTEXT_UNUSED +/* mlkem/src/debug.h */ +#undef MLK_DEBUG_H +#undef mlk_assert +#undef mlk_assert_abs_bound +#undef mlk_assert_abs_bound_2d +#undef mlk_assert_bound +#undef mlk_assert_bound_2d +#undef mlk_debug_check_assert +#undef mlk_debug_check_bounds +/* mlkem/src/poly.h */ +#undef MLK_INVNTT_BOUND +#undef MLK_NTT_BOUND +#undef MLK_POLY_H +#undef mlk_poly_add +#undef mlk_poly_invntt_tomont +#undef mlk_poly_mulcache_compute +#undef mlk_poly_ntt +#undef mlk_poly_reduce +#undef mlk_poly_sub +#undef mlk_poly_tomont +/* mlkem/src/randombytes.h */ +#undef MLK_RANDOMBYTES_H +/* mlkem/src/sampling.h */ +#undef MLK_SAMPLING_H +#undef mlk_poly_cbd2 +#undef mlk_poly_cbd3 +#undef mlk_poly_rej_uniform +#undef mlk_poly_rej_uniform_x4 +/* mlkem/src/symmetric.h */ +#undef MLK_SYMMETRIC_H +#undef MLK_XOF_RATE +#undef mlk_hash_g +#undef mlk_hash_h +#undef mlk_hash_j +#undef mlk_prf_eta +#undef mlk_prf_eta1 +#undef mlk_prf_eta1_x4 +#undef mlk_prf_eta2 +#undef mlk_xof_absorb +#undef mlk_xof_ctx +#undef mlk_xof_init +#undef mlk_xof_release +#undef mlk_xof_squeezeblocks +#undef mlk_xof_x4_absorb +#undef mlk_xof_x4_ctx +#undef mlk_xof_x4_init +#undef mlk_xof_x4_release +#undef mlk_xof_x4_squeezeblocks +/* mlkem/src/sys.h */ +#undef MLK_ALIGN +#undef MLK_ALIGN_UP +#undef MLK_ALWAYS_INLINE +#undef MLK_CET_ENDBR +#undef MLK_CT_TESTING_DECLASSIFY +#undef MLK_CT_TESTING_SECRET +#undef MLK_DEFAULT_ALIGN +#undef MLK_HAVE_INLINE_ASM +#undef MLK_INLINE +#undef MLK_MUST_CHECK_RETURN_VALUE +#undef MLK_NOINLINE +#undef MLK_RESTRICT +#undef MLK_STATIC_TESTABLE +#undef MLK_SYSV_ABI +#undef MLK_SYSV_ABI_SUPPORTED +#undef MLK_SYS_AARCH64 +#undef MLK_SYS_AARCH64_EB +#undef MLK_SYS_AARCH64_NEON +#undef MLK_SYS_APPLE +#undef MLK_SYS_ARMV81M_MVE +#undef MLK_SYS_BIG_ENDIAN +#undef MLK_SYS_H +#undef MLK_SYS_LINUX +#undef MLK_SYS_LITTLE_ENDIAN +#undef MLK_SYS_PPC64LE +#undef MLK_SYS_RISCV32 +#undef MLK_SYS_RISCV64 +#undef MLK_SYS_RISCV64_RVV +#undef MLK_SYS_WINDOWS +#undef MLK_SYS_X86_64 +#undef MLK_SYS_X86_64_AVX2 +/* mlkem/src/verify.h */ +#undef MLK_USE_ASM_VALUE_BARRIER +#undef MLK_VERIFY_H +#undef mlk_ct_opt_blocker_u64 +/* mlkem/src/cbmc.h */ +#undef MLK_CBMC_H +#undef __contract__ +#undef __loop__ + +#if !defined(MLK_CONFIG_FIPS202_CUSTOM_HEADER) +/* + * Undefine macros from FIPS-202 files + */ +/* mlkem/src/fips202/fips202.h */ +#undef FIPS202_X4_DEFAULT_IMPLEMENTATION +#undef MLK_FIPS202_FIPS202_H +#undef SHA3_256_HASHBYTES +#undef SHA3_256_RATE +#undef SHA3_384_RATE +#undef SHA3_512_HASHBYTES +#undef SHA3_512_RATE +#undef SHAKE128_RATE +#undef SHAKE256_RATE +#undef mlk_sha3_256 +#undef mlk_sha3_512 +#undef mlk_shake128_absorb_once +#undef mlk_shake128_init +#undef mlk_shake128_release +#undef mlk_shake128_squeezeblocks +#undef mlk_shake256 +/* mlkem/src/fips202/fips202x4.h */ +#undef MLK_FIPS202_FIPS202X4_H +#undef mlk_shake128x4_absorb_once +#undef mlk_shake128x4_init +#undef mlk_shake128x4_release +#undef mlk_shake128x4_squeezeblocks +#undef mlk_shake256x4 +/* mlkem/src/fips202/keccakf1600.h */ +#undef MLK_FIPS202_KECCAKF1600_H +#undef MLK_KECCAK_LANES +#undef MLK_KECCAK_WAY +#undef mlk_keccakf1600_extract_bytes +#undef mlk_keccakf1600_permute +#undef mlk_keccakf1600_xor_bytes +#undef mlk_keccakf1600x4_extract_bytes +#undef mlk_keccakf1600x4_permute +#undef mlk_keccakf1600x4_xor_bytes +#endif /* !MLK_CONFIG_FIPS202_CUSTOM_HEADER */ + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) +/* mlkem/src/fips202/native/api.h */ +#undef MLK_FIPS202_NATIVE_API_H +#undef MLK_NATIVE_FUNC_FALLBACK +#undef MLK_NATIVE_FUNC_SUCCESS +/* mlkem/src/fips202/native/auto.h */ +#undef MLK_FIPS202_NATIVE_AUTO_H +#if defined(MLK_SYS_AARCH64) +/* + * Undefine macros from native code (FIPS202, AArch64) + */ +/* mlkem/src/fips202/native/aarch64/auto.h */ +#undef MLK_FIPS202_NATIVE_AARCH64_AUTO_H +/* mlkem/src/fips202/native/aarch64/src/fips202_native_aarch64.h */ +#undef MLK_FIPS202_NATIVE_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H +#undef mlk_keccak_f1600_x1_scalar_aarch64_asm +#undef mlk_keccak_f1600_x1_v84a_aarch64_asm +#undef mlk_keccak_f1600_x2_v84a_aarch64_asm +#undef mlk_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm +#undef mlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm +#undef mlk_keccakf1600_round_constants +/* mlkem/src/fips202/native/aarch64/x1_scalar.h */ +#undef MLK_FIPS202_AARCH64_NEED_X1_SCALAR +#undef MLK_FIPS202_NATIVE_AARCH64_X1_SCALAR_H +#undef MLK_USE_NATIVE_FIPS202_X1 +/* mlkem/src/fips202/native/aarch64/x1_v84a.h */ +#undef MLK_FIPS202_AARCH64_NEED_X1_V84A +#undef MLK_FIPS202_NATIVE_AARCH64_X1_V84A_H +#undef MLK_USE_NATIVE_FIPS202_X1 +/* mlkem/src/fips202/native/aarch64/x2_v84a.h */ +#undef MLK_FIPS202_AARCH64_NEED_X2_V84A +#undef MLK_FIPS202_NATIVE_AARCH64_X2_V84A_H +#undef MLK_USE_NATIVE_FIPS202_X4 +/* mlkem/src/fips202/native/aarch64/x4_v8a_scalar.h */ +#undef MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID +#undef MLK_FIPS202_NATIVE_AARCH64_X4_V8A_SCALAR_H +#undef MLK_USE_NATIVE_FIPS202_X4 +/* mlkem/src/fips202/native/aarch64/x4_v8a_v84a_scalar.h */ +#undef MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID +#undef MLK_FIPS202_NATIVE_AARCH64_X4_V8A_V84A_SCALAR_H +#undef MLK_USE_NATIVE_FIPS202_X4 +#endif /* MLK_SYS_AARCH64 */ +#if defined(MLK_SYS_X86_64) +/* + * Undefine macros from native code (FIPS202, x86_64) + */ +/* mlkem/src/fips202/native/x86_64/keccak_f1600_x4_avx2.h */ +#undef MLK_FIPS202_NATIVE_X86_64_KECCAK_F1600_X4_AVX2_H +#undef MLK_FIPS202_X86_64_NEED_X4_AVX2 +#undef MLK_USE_NATIVE_FIPS202_X4 +/* mlkem/src/fips202/native/x86_64/src/fips202_native_x86_64.h */ +#undef MLK_FIPS202_NATIVE_X86_64_SRC_FIPS202_NATIVE_X86_64_H +#undef mlk_keccak_f1600_x4_avx2_asm +#undef mlk_keccak_rho56 +#undef mlk_keccak_rho8 +#undef mlk_keccakf1600_round_constants +#endif /* MLK_SYS_X86_64 */ +#if defined(MLK_SYS_ARMV81M_MVE) +/* + * Undefine macros from native code (FIPS202, Armv8.1-M) + */ +/* mlkem/src/fips202/native/armv81m/mve.h */ +#undef MLK_FIPS202_ARMV81M_NEED_X4 +#undef MLK_FIPS202_NATIVE_ARMV81M +#undef MLK_FIPS202_NATIVE_ARMV81M_MVE_H +#undef MLK_USE_NATIVE_FIPS202_X4 +#undef MLK_USE_NATIVE_FIPS202_X4_EXTRACT_BYTES +#undef MLK_USE_NATIVE_FIPS202_X4_XOR_BYTES +#undef mlk_keccak_f1600_x4_native_impl +/* mlkem/src/fips202/native/armv81m/src/fips202_native_armv81m.h */ +#undef MLK_FIPS202_NATIVE_ARMV81M_SRC_FIPS202_NATIVE_ARMV81M_H +#undef mlk_keccak_f1600_x4_mve_asm +#undef mlk_keccak_f1600_x4_state_extract_bytes_asm +#undef mlk_keccak_f1600_x4_state_xor_bytes_asm +#undef mlk_keccakf1600_round_constants +#endif /* MLK_SYS_ARMV81M_MVE */ +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) +/* mlkem/src/native/api.h */ +#undef MLK_INVNTT_BOUND +#undef MLK_NATIVE_API_H +#undef MLK_NATIVE_FUNC_FALLBACK +#undef MLK_NATIVE_FUNC_SUCCESS +#undef MLK_NTT_BOUND +/* mlkem/src/native/meta.h */ +#undef MLK_NATIVE_META_H +#if defined(MLK_SYS_AARCH64) +/* + * Undefine macros from native code (Arith, AArch64) + */ +/* mlkem/src/native/aarch64/meta.h */ +#undef MLK_ARITH_BACKEND_AARCH64 +#undef MLK_NATIVE_AARCH64_META_H +#undef MLK_USE_NATIVE_INTT +#undef MLK_USE_NATIVE_NTT +#undef MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED +#undef MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#undef MLK_USE_NATIVE_POLY_REDUCE +#undef MLK_USE_NATIVE_POLY_TOBYTES +#undef MLK_USE_NATIVE_POLY_TOMONT +#undef MLK_USE_NATIVE_REJ_UNIFORM +/* mlkem/src/native/aarch64/src/arith_native_aarch64.h */ +#undef MLK_NATIVE_AARCH64_SRC_ARITH_NATIVE_AARCH64_H +#undef mlk_aarch64_invntt_zetas_layer12345 +#undef mlk_aarch64_invntt_zetas_layer67 +#undef mlk_aarch64_ntt_zetas_layer12345 +#undef mlk_aarch64_ntt_zetas_layer67 +#undef mlk_aarch64_zetas_mulcache_native +#undef mlk_aarch64_zetas_mulcache_twisted_native +#undef mlk_intt_aarch64_asm +#undef mlk_ntt_aarch64_asm +#undef mlk_poly_mulcache_compute_aarch64_asm +#undef mlk_poly_reduce_aarch64_asm +#undef mlk_poly_tobytes_aarch64_asm +#undef mlk_poly_tomont_aarch64_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm +#undef mlk_rej_uniform_aarch64_asm +#undef mlk_rej_uniform_table +#endif /* MLK_SYS_AARCH64 */ +#if defined(MLK_SYS_X86_64) +/* + * Undefine macros from native code (Arith, X86_64) + */ +/* mlkem/src/native/x86_64/meta.h */ +#undef MLK_ARITH_BACKEND_X86_64_DEFAULT +#undef MLK_NATIVE_X86_64_META_H +#undef MLK_USE_NATIVE_INTT +#undef MLK_USE_NATIVE_NTT +#undef MLK_USE_NATIVE_NTT_CUSTOM_ORDER +#undef MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED +#undef MLK_USE_NATIVE_POLY_COMPRESS_D10 +#undef MLK_USE_NATIVE_POLY_COMPRESS_D11 +#undef MLK_USE_NATIVE_POLY_COMPRESS_D4 +#undef MLK_USE_NATIVE_POLY_COMPRESS_D5 +#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D10 +#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D11 +#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D4 +#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D5 +#undef MLK_USE_NATIVE_POLY_FROMBYTES +#undef MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#undef MLK_USE_NATIVE_POLY_REDUCE +#undef MLK_USE_NATIVE_POLY_TOBYTES +#undef MLK_USE_NATIVE_POLY_TOMONT +#undef MLK_USE_NATIVE_REJ_UNIFORM +/* mlkem/src/native/x86_64/src/arith_native_x86_64.h */ +#undef MLK_AVX2_REJ_UNIFORM_BUFLEN +#undef MLK_NATIVE_X86_64_SRC_ARITH_NATIVE_X86_64_H +#undef mlk_invntt_avx2_asm +#undef mlk_ntt_avx2_asm +#undef mlk_nttfrombytes_avx2_asm +#undef mlk_ntttobytes_avx2_asm +#undef mlk_nttunpack_avx2_asm +#undef mlk_poly_compress_d10_avx2_asm +#undef mlk_poly_compress_d11_avx2_asm +#undef mlk_poly_compress_d4_avx2_asm +#undef mlk_poly_compress_d5_avx2_asm +#undef mlk_poly_decompress_d10_avx2_asm +#undef mlk_poly_decompress_d11_avx2_asm +#undef mlk_poly_decompress_d4_avx2_asm +#undef mlk_poly_decompress_d5_avx2_asm +#undef mlk_poly_mulcache_compute_avx2_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm +#undef mlk_reduce_avx2_asm +#undef mlk_rej_uniform_avx2_asm +#undef mlk_rej_uniform_table +#undef mlk_tomont_avx2_asm +/* mlkem/src/native/x86_64/src/compress_consts.h */ +#undef MLK_NATIVE_X86_64_SRC_COMPRESS_CONSTS_H +#undef mlk_compress_d10_data +#undef mlk_compress_d11_data +#undef mlk_compress_d4_data +#undef mlk_compress_d5_data +#undef mlk_decompress_d10_data +#undef mlk_decompress_d11_data +#undef mlk_decompress_d4_data +#undef mlk_decompress_d5_data +/* mlkem/src/native/x86_64/src/consts.h */ +#undef MLK_AVX2_BACKEND_DATA_OFFSET_MULCACHE_TWIDDLES +#undef MLK_AVX2_BACKEND_DATA_OFFSET_REVIDXB +#undef MLK_AVX2_BACKEND_DATA_OFFSET_REVIDXD +#undef MLK_AVX2_BACKEND_DATA_OFFSET_ZETAS_EXP +#undef MLK_NATIVE_X86_64_SRC_CONSTS_H +#undef mlk_qdata +#endif /* MLK_SYS_X86_64 */ +#if defined(MLK_SYS_RISCV64) +/* + * Undefine macros from native code (Arith, RISC-V 64) + */ +/* mlkem/src/native/riscv64/meta.h */ +#undef MLK_ARITH_BACKEND_RISCV64 +#undef MLK_NATIVE_RISCV64_META_H +#undef MLK_USE_NATIVE_INTT +#undef MLK_USE_NATIVE_NTT +#undef MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED +#undef MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#undef MLK_USE_NATIVE_POLY_REDUCE +#undef MLK_USE_NATIVE_POLY_TOMONT +#undef MLK_USE_NATIVE_REJ_UNIFORM +/* mlkem/src/native/riscv64/src/arith_native_riscv64.h */ +#undef MLK_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H +#undef mlk_rv64v_poly_add +#undef mlk_rv64v_poly_basemul_mont_add_k2 +#undef mlk_rv64v_poly_basemul_mont_add_k3 +#undef mlk_rv64v_poly_basemul_mont_add_k4 +#undef mlk_rv64v_poly_invntt_tomont +#undef mlk_rv64v_poly_ntt +#undef mlk_rv64v_poly_reduce +#undef mlk_rv64v_poly_sub +#undef mlk_rv64v_poly_tomont +#undef mlk_rv64v_rej_uniform +/* mlkem/src/native/riscv64/src/rv64v_debug.h */ +#undef MLK_NATIVE_RISCV64_SRC_RV64V_DEBUG_H +#undef mlk_assert_abs_bound_int16m1 +#undef mlk_assert_abs_bound_int16m2 +#undef mlk_assert_bound_int16m1 +#undef mlk_assert_bound_int16m2 +#undef mlk_debug_check_bounds_int16m1 +#undef mlk_debug_check_bounds_int16m2 +#endif /* MLK_SYS_RISCV64 */ +#if defined(MLK_SYS_PPC64LE) +/* + * Undefine macros from native code (Arith, PPC64LE) + */ +/* mlkem/src/native/ppc64le/meta.h */ +#undef MLK_ARITH_BACKEND_NAME +#undef MLK_ARITH_BACKEND_PPC64LE_DEFAULT +#undef MLK_NATIVE_PPC64LE_META_H +#undef MLK_USE_NATIVE_INTT +#undef MLK_USE_NATIVE_NTT +#undef MLK_USE_NATIVE_POLY_REDUCE +#undef MLK_USE_NATIVE_POLY_TOMONT +/* mlkem/src/native/ppc64le/src/arith_native_ppc64le.h */ +#undef MLK_NATIVE_PPC64LE_SRC_ARITH_NATIVE_PPC64LE_H +#undef mlk_intt_ppc_asm +#undef mlk_ntt_ppc_asm +#undef mlk_poly_tomont_ppc_asm +#undef mlk_reduce_ppc_asm +/* mlkem/src/native/ppc64le/src/consts.h */ +#undef MLK_NATIVE_PPC64LE_SRC_CONSTS_H +#undef MLK_PPC_C20159_OFFSET +#undef MLK_PPC_NQ_OFFSET +#undef MLK_PPC_N_INV_OFFSET +#undef MLK_PPC_N_INV_TW_OFFSET +#undef MLK_PPC_Q_OFFSET +#undef MLK_PPC_TOMONT_OFFSET +#undef MLK_PPC_TOMONT_TW_OFFSET +#undef MLK_PPC_ZETA_INTT_OFFSET +#undef MLK_PPC_ZETA_INTT_TW_OFFSET +#undef MLK_PPC_ZETA_NTT_OFFSET +#undef MLK_PPC_ZETA_NTT_TW_OFFSET +#undef mlk_ppc_qdata +#endif /* MLK_SYS_PPC64LE */ +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_ARITH */ +#endif /* !MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS */ diff --git a/src/mlkem-native/mlkem/mlkem_native_config.h b/src/mlkem-native/mlkem/mlkem_native_config.h new file mode 100644 index 0000000000..a4b204b68e --- /dev/null +++ b/src/mlkem-native/mlkem/mlkem_native_config.h @@ -0,0 +1,683 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS140_3_IG] + * Implementation Guidance for FIPS 140-3 and the Cryptographic Module + * Validation Program + * National Institute of Standards and Technology + * https://csrc.nist.gov/projects/cryptographic-module-validation-program/fips-140-3-ig-announcements + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_CONFIG_H +#define MLK_CONFIG_H + +/** + * Specifies the parameter set for ML-KEM: + * - MLK_CONFIG_PARAMETER_SET=512 corresponds to ML-KEM-512 + * - MLK_CONFIG_PARAMETER_SET=768 corresponds to ML-KEM-768 + * - MLK_CONFIG_PARAMETER_SET=1024 corresponds to ML-KEM-1024 + * + * If you want to support multiple parameter sets, build the library multiple + * times and set MLK_CONFIG_MULTILEVEL_BUILD. See MLK_CONFIG_MULTILEVEL_BUILD + * for how to do this while minimizing code duplication. + * + * This can also be set using CFLAGS. + */ +#ifndef MLK_CONFIG_PARAMETER_SET +#define MLK_CONFIG_PARAMETER_SET \ + 768 /* Change this for different security strengths */ +#endif + +/** + * MLK_CONFIG_FILE + * + * If defined, this is a header that will be included instead of the default + * configuration file mlkem/mlkem_native_config.h. + * + * When you need to build mlkem-native in multiple configurations, using + * varying MLK_CONFIG_FILE can be more convenient than configuring everything + * through CFLAGS. + * + * To use, MLK_CONFIG_FILE _must_ be defined prior to the inclusion of any + * mlkem-native headers. For example, it can be set by passing + * `-DMLK_CONFIG_FILE="..."` on the command line. + */ +/* #define MLK_CONFIG_FILE "mlkem_native_config.h" */ + +/** + * The prefix to use to namespace global symbols from mlkem/. + * + * In a multi-level build, level-dependent symbols will additionally be + * prefixed with the parameter set (512/768/1024). + * + * This can also be set using CFLAGS. + */ +#if !defined(MLK_CONFIG_NAMESPACE_PREFIX) +#define MLK_CONFIG_NAMESPACE_PREFIX MLK_DEFAULT_NAMESPACE_PREFIX +#endif + +/** + * MLK_CONFIG_MULTILEVEL_BUILD + * + * Set this if the build is part of a multi-level build supporting multiple + * parameter sets. + * + * If you need only a single parameter set, keep this unset. + * + * To build mlkem-native with support for all parameter sets, build it three + * times -- once per parameter set -- and set the option + * MLK_CONFIG_MULTILEVEL_WITH_SHARED for exactly one of them, and + * MLK_CONFIG_MULTILEVEL_NO_SHARED for the others. + * MLK_CONFIG_MULTILEVEL_BUILD should be set for all of them. + * + * See examples/multilevel_build for an example. + * + * This can also be set using CFLAGS. + */ +/* #define MLK_CONFIG_MULTILEVEL_BUILD */ + +/** + * MLK_CONFIG_EXTERNAL_API_QUALIFIER + * + * If set, this option provides an additional function qualifier to be added + * to declarations of mlkem-native's public API. + * + * The primary use case for this option are single-CU builds where the public + * API exposed by mlkem-native is wrapped by another API in the consuming + * application. In this case, even mlkem-native's public API can be marked + * `static`. + */ +/* #define MLK_CONFIG_EXTERNAL_API_QUALIFIER */ + +/** + * MLK_CONFIG_NO_KEYPAIR_API + * + * By default, mlkem-native includes support for generating key pairs. + * If you don't need this, set MLK_CONFIG_NO_KEYPAIR_API to exclude + * keypair and keypair_derand, and all internal + * APIs only needed by those functions. + */ +/* #define MLK_CONFIG_NO_KEYPAIR_API */ + +/** + * MLK_CONFIG_NO_ENCAPS_API + * + * By default, mlkem-native includes support for encapsulation. If you + * don't need this, set MLK_CONFIG_NO_ENCAPS_API to exclude + * enc, enc_derand, check_pk, and + * all internal APIs only needed by those functions. + * + * @note Setting this option is incompatible with MLK_CONFIG_KEYGEN_PCT + * as the current PCT implementation requires enc(). + */ +/* #define MLK_CONFIG_NO_ENCAPS_API */ + +/** + * MLK_CONFIG_NO_DECAPS_API + * + * By default, mlkem-native includes support for decapsulation. If you + * don't need this, set MLK_CONFIG_NO_DECAPS_API to exclude + * dec, check_sk, and all internal APIs only + * needed by those functions. + * + * @note Setting this option is incompatible with MLK_CONFIG_KEYGEN_PCT + * as the current PCT implementation requires dec(). + */ +/* #define MLK_CONFIG_NO_DECAPS_API */ + +/** + * MLK_CONFIG_NO_RANDOMIZED_API + * + * If this option is set, mlkem-native will be built without the randomized + * API functions (keypair and enc). This allows users + * to build mlkem-native without providing a randombytes() implementation + * if they only need the deterministic API (keypair_derand, + * enc_derand, dec). + * + * @note This option is incompatible with MLK_CONFIG_KEYGEN_PCT as the + * current PCT implementation requires enc(). + */ +/* #define MLK_CONFIG_NO_RANDOMIZED_API */ + +/** + * MLK_CONFIG_CONSTANTS_ONLY + * + * If you only need the size constants (MLKEM_PUBLICKEYBYTES, etc.) but no + * function declarations, set MLK_CONFIG_CONSTANTS_ONLY. + * + * This only affects the public header mlkem_native.h, not the + * implementation. + */ +/* #define MLK_CONFIG_CONSTANTS_ONLY */ + +/****************************************************************************** + * + * Build-only configuration options + * + * The remaining configurations are build-options only. + * They do not affect the API described in mlkem_native.h. + * + *****************************************************************************/ + +#if defined(MLK_BUILD_INTERNAL) +/** + * MLK_CONFIG_MULTILEVEL_WITH_SHARED + * + * This is for multi-level builds of mlkem-native only. If you need only a + * single parameter set, keep this unset. + * + * If this is set, all MLK_CONFIG_PARAMETER_SET-independent code will be + * included in the build, including code needed only for other parameter + * sets. + * + * Example: mlk_poly_cbd3 is only needed for MLK_CONFIG_PARAMETER_SET == 512. + * Yet, if this option is set for a build with + * MLK_CONFIG_PARAMETER_SET == 768/1024, it would be included. + * + * To build mlkem-native with support for all parameter sets, build it three + * times -- once per parameter set -- and set the option + * MLK_CONFIG_MULTILEVEL_WITH_SHARED for exactly one of them, and + * MLK_CONFIG_MULTILEVEL_NO_SHARED for the others. + * MLK_CONFIG_MULTILEVEL_BUILD should be set for all of them. + * + * See examples/multilevel_build for an example. + * + * This can also be set using CFLAGS. + */ +/* #define MLK_CONFIG_MULTILEVEL_WITH_SHARED */ + +/** + * MLK_CONFIG_MULTILEVEL_NO_SHARED + * + * This is for multi-level builds of mlkem-native only. If you need only a + * single parameter set, keep this unset. + * + * If this is set, no MLK_CONFIG_PARAMETER_SET-independent code will be + * included in the build. + * + * To build mlkem-native with support for all parameter sets, build it three + * times -- once per parameter set -- and set the option + * MLK_CONFIG_MULTILEVEL_WITH_SHARED for exactly one of them, and + * MLK_CONFIG_MULTILEVEL_NO_SHARED for the others. + * MLK_CONFIG_MULTILEVEL_BUILD should be set for all of them. + * + * See examples/multilevel_build for an example. + * + * This can also be set using CFLAGS. + */ +/* #define MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +/** + * MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS + * + * This is only relevant for single compilation unit (SCU) builds of + * mlkem-native. In this case, it determines whether directives defined in + * parameter-set-independent headers should be #undef'ined or not at the + * end of the SCU file. This is needed in multilevel builds. + * + * See examples/multilevel_build_native for an example. + * + * This can also be set using CFLAGS. + */ +/* #define MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS */ + +/** + * MLK_CONFIG_USE_NATIVE_BACKEND_ARITH + * + * Determines whether a native arithmetic backend should be used. + * + * The arithmetic backend covers performance-critical functions such as the + * number-theoretic transform (NTT). + * + * If this option is unset, the C backend will be used. + * + * If this option is set, the arithmetic backend to be used is determined + * by MLK_CONFIG_ARITH_BACKEND_FILE: if the latter is unset, the default + * backend for the target architecture will be used. If set, it must be the + * name of a backend metadata file. + * + * This can also be set using CFLAGS. + */ +#if !defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) +/* #define MLK_CONFIG_USE_NATIVE_BACKEND_ARITH */ +#endif + +/** + * MLK_CONFIG_ARITH_BACKEND_FILE + * + * The arithmetic backend to use. + * + * If MLK_CONFIG_USE_NATIVE_BACKEND_ARITH is unset, this option is ignored. + * + * If MLK_CONFIG_USE_NATIVE_BACKEND_ARITH is set, this option must either + * be undefined or the filename of an arithmetic backend. If unset, the + * default backend will be used. + * + * This can be set using CFLAGS. + */ +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) && \ + !defined(MLK_CONFIG_ARITH_BACKEND_FILE) +#define MLK_CONFIG_ARITH_BACKEND_FILE "native/meta.h" +#endif + +/** + * MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 + * + * Determines whether a native FIPS202 backend should be used. + * + * The FIPS202 backend covers 1x/2x/4x-fold Keccak-f1600, which is the + * performance bottleneck of SHA3 and SHAKE. + * + * If this option is unset, the C backend will be used. + * + * If this option is set, the FIPS202 backend to be used is determined by + * MLK_CONFIG_FIPS202_BACKEND_FILE: if the latter is unset, the default + * backend for the target architecture will be used. If set, it must be + * the name of a backend metadata file. + * + * This can also be set using CFLAGS. + */ +#if !defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) +/* #define MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ +#endif + +/** + * MLK_CONFIG_FIPS202_BACKEND_FILE + * + * The FIPS-202 backend to use. + * + * If MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 is set, this option must either + * be undefined or the filename of a FIPS202 backend. If unset, the default + * backend will be used. + * + * This can be set using CFLAGS. + */ +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) && \ + !defined(MLK_CONFIG_FIPS202_BACKEND_FILE) +#define MLK_CONFIG_FIPS202_BACKEND_FILE "fips202/native/auto.h" +#endif + +/** + * MLK_CONFIG_FIPS202_CUSTOM_HEADER + * + * Custom header to use for FIPS-202. + * + * This should only be set if you intend to use a custom FIPS-202 + * implementation, different from the one shipped with mlkem-native. + * + * If set, it must be the name of a file serving as the replacement for + * mlkem/src/fips202/fips202.h, and exposing the same API (see FIPS202.md). + */ +/* #define MLK_CONFIG_FIPS202_CUSTOM_HEADER "SOME_FILE.h" */ + +/** + * MLK_CONFIG_FIPS202X4_CUSTOM_HEADER + * + * Custom header to use for FIPS-202-X4. + * + * This should only be set if you intend to use a custom FIPS-202 + * implementation, different from the one shipped with mlkem-native. + * + * If set, it must be the name of a file serving as the replacement for + * mlkem/src/fips202/fips202x4.h, and exposing the same API (see FIPS202.md). + */ +/* #define MLK_CONFIG_FIPS202X4_CUSTOM_HEADER "SOME_FILE.h" */ + +/** + * MLK_CONFIG_CUSTOM_ZEROIZE + * + * In compliance with @[FIPS203, Section 3.3], mlkem-native zeroizes + * intermediate buffers before returning from function calls. By default, + * those buffers are allocated from the stack; if MLK_CONFIG_CUSTOM_ALLOC_FREE + * is set, they are (mostly -- few exceptions remain at present) allocated from + * the configured custom allocator. + * + * mlkem-native also zeroizes caller-owned output buffers as needed to uphold + * the API convention that outputs be either unmodified or zeroized upon + * failure. + * + * Set this option and define `mlk_zeroize` if you want to use a custom + * method to zeroize intermediate and output buffers. + * + * The default implementation uses SecureZeroMemory on Windows and a + * memset + compiler barrier otherwise. If neither of those is available on + * the target platform, compilation will fail, and you will need to use + * MLK_CONFIG_CUSTOM_ZEROIZE to provide a custom implementation of + * `mlk_zeroize()`. + * + * @warning + * The zeroization conducted by mlkem-native reduces the likelihood of data + * leaking on the stack or custom allocators, but it does not eliminate it. + * For example, the C standard makes no guarantee about where a compiler + * allocates local structures and whether/where it makes copies of them. + * Also, in addition to entire structures, there may also be potentially + * exploitable leakage of individual values on the stack. If you need + * bullet-proof zeroization of the stack, you need to consider additional + * measures instead of what this feature provides. In this case, you can + * set mlk_zeroize to a no-op. Note that in this case you are also responsible + * for zeroizing output buffers upon failure. + */ +/* #define MLK_CONFIG_CUSTOM_ZEROIZE + #if !defined(__ASSEMBLER__) + #include + #include "src/sys.h" + static MLK_INLINE void mlk_zeroize(void *ptr, size_t len) + { + ... your implementation ... + } + #endif +*/ + +/** + * MLK_CONFIG_CUSTOM_RANDOMBYTES + * + * mlkem-native does not provide a secure randombytes implementation. Such + * an implementation has to be provided by the consumer. + * + * If this option is not set, mlkem-native expects a function + * int randombytes(uint8_t *out, size_t outlen). It is expected to return + * zero on success, and non-zero on failure. In case of failure, the + * top-level APIs will return an MLK_ERR_RNG_FAIL error code. + * + * Set this option and define `mlk_randombytes` (with the same signature + * and behaviour) if you want to use a custom method to sample randombytes + * with a different name or signature. + */ +/* #define MLK_CONFIG_CUSTOM_RANDOMBYTES + #if !defined(__ASSEMBLER__) + #include + #include "src/sys.h" + static MLK_INLINE int mlk_randombytes(uint8_t *ptr, size_t len) + { + ... your implementation ... + return 0; + } + #endif +*/ + +/** + * MLK_CONFIG_CUSTOM_CAPABILITY_FUNC + * + * mlkem-native backends may rely on specific hardware features. Those + * backends will only be included in an mlkem-native build if support for + * the respective features is enabled at compile-time. However, when + * building for a heterogeneous set of CPUs to run the resulting + * binary/library on, feature detection at _runtime_ is needed to decide + * whether a backend can be used or not. + * + * Set this option and define `mlk_sys_check_capability` if you want to + * use a custom method to dispatch between implementations. + * + * If this option is not set, mlkem-native uses compile-time feature + * detection only to decide which backend to use. + * + * If you compile mlkem-native on a system with different capabilities + * than the system that the resulting binary/library will be run on, you + * must use this option. + */ +/* #define MLK_CONFIG_CUSTOM_CAPABILITY_FUNC + static MLK_INLINE int mlk_sys_check_capability(mlk_sys_cap cap) + __contract__( + ensures(return_value == 0 || return_value == 1) + ) + { + ... your implementation ... + } +*/ + +/** + * MLK_CONFIG_CUSTOM_ALLOC_FREE [EXPERIMENTAL] + * + * Set this option and define `MLK_CUSTOM_ALLOC` and `MLK_CUSTOM_FREE` if + * you want to use custom allocation for large local structures or buffers. + * + * By default, all buffers/structures are allocated on the stack. If this + * option is set, most of them will be allocated via MLK_CUSTOM_ALLOC. + * + * Parameters to MLK_CUSTOM_ALLOC: + * - T* v: Target pointer to declare. + * - T: Type of structure to be allocated. + * - N: Number of elements to be allocated. + * + * Parameters to MLK_CUSTOM_FREE: + * - T* v: Target pointer to free. May be NULL. + * - T: Type of structure to be freed. + * - N: Number of elements to be freed. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. We expect a + * stable API in a future version. + * + * @note Even if this option is set, some allocations further down the call + * stack will still be made from the stack, consuming up to 3KB of + * stack space. Those will likely be added to the scope of this + * option in the future. + * + * @note MLK_CUSTOM_ALLOC need not guarantee a successful allocation nor + * include error handling. Upon failure, the target pointer should + * simply be set to NULL. The calling code will handle this case and + * invoke MLK_CUSTOM_FREE. + */ +/* #define MLK_CONFIG_CUSTOM_ALLOC_FREE + #if !defined(__ASSEMBLER__) + #include + #define MLK_CUSTOM_ALLOC(v, T, N) \ + T* (v) = (T *)aligned_alloc(MLK_DEFAULT_ALIGN, \ + MLK_ALIGN_UP(sizeof(T) * (N))) + #define MLK_CUSTOM_FREE(v, T, N) free(v) + #endif +*/ + +/** + * MLK_CONFIG_CUSTOM_MEMCPY + * + * Set this option and define `mlk_memcpy` if you want to use a custom + * method to copy memory instead of the standard library memcpy function. + * + * The custom implementation must have the same signature and behavior as + * the standard memcpy function: + * void *mlk_memcpy(void *dest, const void *src, size_t n) + */ +/* #define MLK_CONFIG_CUSTOM_MEMCPY + #if !defined(__ASSEMBLER__) + #include + #include "src/sys.h" + static MLK_INLINE void *mlk_memcpy(void *dest, const void *src, size_t n) + { + ... your implementation ... + } + #endif +*/ + +/** + * MLK_CONFIG_CUSTOM_MEMSET + * + * Set this option and define `mlk_memset` if you want to use a custom + * method to set memory instead of the standard library memset function. + * + * The custom implementation must have the same signature and behavior as + * the standard memset function: + * void *mlk_memset(void *s, int c, size_t n) + */ +/* #define MLK_CONFIG_CUSTOM_MEMSET + #if !defined(__ASSEMBLER__) + #include + #include "src/sys.h" + static MLK_INLINE void *mlk_memset(void *s, int c, size_t n) + { + ... your implementation ... + } + #endif +*/ + +/** + * MLK_CONFIG_INTERNAL_API_QUALIFIER + * + * If set, this option provides an additional qualifier to be added to + * declarations of internal API functions and data. + * + * The primary use case for this option are single-CU builds, in which case + * this option can be set to `static`. + */ +/* #define MLK_CONFIG_INTERNAL_API_QUALIFIER */ + +/** + * MLK_CONFIG_CT_TESTING_ENABLED + * + * If set, mlkem-native annotates data as secret/public using valgrind's + * annotations VALGRIND_MAKE_MEM_UNDEFINED and VALGRIND_MAKE_MEM_DEFINED, + * enabling various checks for secret-dependent control flow or + * variable-time execution (depending on the exact version of valgrind + * installed). + */ +/* #define MLK_CONFIG_CT_TESTING_ENABLED */ + +/** + * MLK_CONFIG_NO_ASM + * + * If this option is set, mlkem-native will be built without use of native + * code or inline assembly. + * + * By default, inline assembly is used to implement value barriers. Without + * inline assembly, mlkem-native will use a global volatile 'opt blocker' + * instead; see verify.h. + * + * Inline assembly is also used to implement a secure zeroization function + * on non-Windows platforms. If this option is set and the target platform + * is not Windows, you MUST set MLK_CONFIG_CUSTOM_ZEROIZE and provide a + * custom zeroization function. + * + * If this option is set, MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 and + * MLK_CONFIG_USE_NATIVE_BACKEND_ARITH will be ignored, and no native + * backends will be used. + */ +/* #define MLK_CONFIG_NO_ASM */ + +/** + * MLK_CONFIG_NO_ASM_VALUE_BARRIER + * + * If this option is set, mlkem-native will be built without use of native + * code or inline assembly for value barriers. + * + * By default, inline assembly (if available) is used to implement value + * barriers. Without inline assembly, mlkem-native will use a global + * volatile 'opt blocker' instead; see verify.h. + */ +/* #define MLK_CONFIG_NO_ASM_VALUE_BARRIER */ + +/** + * MLK_CONFIG_KEYGEN_PCT + * + * Compliance with @[FIPS140_3_IG, p.87] requires a Pairwise Consistency + * Test (PCT) to be carried out on a freshly generated keypair before it + * can be exported. + * + * Set this option if such a check should be implemented. In this case, + * keypair_derand and keypair will return + * MLK_ERR_PCT_FAIL if the PCT failed. + * + * @note This feature will drastically lower the performance of key + * generation. + */ +/* #define MLK_CONFIG_KEYGEN_PCT */ + +/** + * MLK_CONFIG_KEYGEN_PCT_BREAKAGE_TEST + * + * If this option is set, the user must provide a runtime function + * `static inline int mlk_break_pct() { ... }` to indicate whether the PCT + * should be made to fail. + * + * This option only has an effect if MLK_CONFIG_KEYGEN_PCT is set. + */ +/* #define MLK_CONFIG_KEYGEN_PCT_BREAKAGE_TEST + #if !defined(__ASSEMBLER__) + #include "src/sys.h" + static MLK_INLINE int mlk_break_pct(void) + { + ... return 0/1 depending on whether PCT should be broken ... + } + #endif +*/ + +/** + * MLK_CONFIG_SERIAL_FIPS202_ONLY + * + * Set this to use a FIPS202 implementation with global state that supports + * only one active Keccak computation at a time (e.g. some hardware + * accelerators). + * + * If this option is set, batched Keccak operations are disabled for + * rejection sampling during matrix generation. Instead, matrix entries + * will be generated one at a time. + * + * This allows offloading Keccak computations to a hardware accelerator + * that holds only a single Keccak state locally, rather than requiring + * support for batched (4x) Keccak states. + * + * @note Depending on the target CPU, disabling batched Keccak may reduce + * performance when using software FIPS202 implementations. Only + * enable this when you have to. + */ +/* #define MLK_CONFIG_SERIAL_FIPS202_ONLY */ + +/** + * MLK_CONFIG_CONTEXT_PARAMETER + * + * Set this to add a context parameter that is provided to public API + * functions and is then available in custom callbacks. + * + * The type of the context parameter is configured via + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + */ +/* #define MLK_CONFIG_CONTEXT_PARAMETER */ + +/** + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE + * + * Set this to define the type for the context parameter used by + * MLK_CONFIG_CONTEXT_PARAMETER. + * + * This is only relevant if MLK_CONFIG_CONTEXT_PARAMETER is set. + */ +/* #define MLK_CONFIG_CONTEXT_PARAMETER_TYPE void* */ + +/************************* Config internals ********************************/ + +#endif /* MLK_BUILD_INTERNAL */ + +/* Default namespace + * + * Don't change this. If you need a different namespace, re-define + * MLK_CONFIG_NAMESPACE_PREFIX above instead, and remove the following. + * + * The default MLKEM namespace is + * + * PQCP_MLKEM_NATIVE_MLKEM_ + * + * e.g., PQCP_MLKEM_NATIVE_MLKEM512_ + */ + +#if defined(MLK_CONFIG_MULTILEVEL_BUILD) +/* In a multi-level build the parameter set is appended by the namespacing + * machinery, so the default prefix must not embed it. */ +#define MLK_DEFAULT_NAMESPACE_PREFIX PQCP_MLKEM_NATIVE_MLKEM +#elif MLK_CONFIG_PARAMETER_SET == 512 +#define MLK_DEFAULT_NAMESPACE_PREFIX PQCP_MLKEM_NATIVE_MLKEM512 +#elif MLK_CONFIG_PARAMETER_SET == 768 +#define MLK_DEFAULT_NAMESPACE_PREFIX PQCP_MLKEM_NATIVE_MLKEM768 +#elif MLK_CONFIG_PARAMETER_SET == 1024 +#define MLK_DEFAULT_NAMESPACE_PREFIX PQCP_MLKEM_NATIVE_MLKEM1024 +#endif + +#endif /* !MLK_CONFIG_H */ diff --git a/src/mlkem-native/mlkem/src/cbmc.h b/src/mlkem-native/mlkem/src/cbmc.h new file mode 100644 index 0000000000..cb24bb52a6 --- /dev/null +++ b/src/mlkem-native/mlkem/src/cbmc.h @@ -0,0 +1,222 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_CBMC_H +#define MLK_CBMC_H +/*************************************************** + * Basic replacements for __CPROVER_XXX contracts + ***************************************************/ +/* + * The `__contract__` / `__loop__` annotation macros use a + * leading-double-underscore spelling in line with other CBMC macros. + * clang-tidy flags these as reserved identifiers; we suppress the diagnostic + * at each definition site (NOLINT) rather than disabling the check globally, + * so it stays active for the rest of the tree. + */ +#ifndef CBMC + +/* clang-format off */ +#define __contract__(x) /* NOLINT(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp) */ +#define __loop__(x) /* NOLINT(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp) */ +/* clang-format on */ + +#else /* !CBMC */ + + +/* clang-format off */ +#define __contract__(x) x /* NOLINT(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp) */ +#define __loop__(x) x /* NOLINT(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp) */ +/* clang-format on */ + +/* https://diffblue.github.io/cbmc/contracts-assigns.html */ +#define assigns(...) __CPROVER_assigns(__VA_ARGS__) + +/* https://diffblue.github.io/cbmc/contracts-requires-ensures.html */ +#define requires(...) __CPROVER_requires(__VA_ARGS__) +#define ensures(...) __CPROVER_ensures(__VA_ARGS__) +/* https://diffblue.github.io/cbmc/contracts-loops.html */ +#define invariant(...) __CPROVER_loop_invariant(__VA_ARGS__) +#define decreases(...) __CPROVER_decreases(__VA_ARGS__) +/* cassert to avoid confusion with in-built assert */ +#define cassert(x) __CPROVER_assert(x, "cbmc assertion failed") +#define assume(...) __CPROVER_assume(__VA_ARGS__) + +/*************************************************** + * Macros for "expression" forms that may appear + * _inside_ top-level contracts. + ***************************************************/ + +/* + * function return value - useful inside ensures + * https://diffblue.github.io/cbmc/contracts-functions.html + */ +#define return_value (__CPROVER_return_value) + +/* + * assigns l-value targets + * https://diffblue.github.io/cbmc/contracts-assigns.html + */ +#define object_whole(...) __CPROVER_object_whole(__VA_ARGS__) +#define memory_slice(...) __CPROVER_object_upto(__VA_ARGS__) + +/* + * Pointer-related predicates + * https://diffblue.github.io/cbmc/contracts-memory-predicates.html + */ +#define memory_no_alias(...) __CPROVER_is_fresh(__VA_ARGS__) +#define readable(...) __CPROVER_r_ok(__VA_ARGS__) +#define writeable(...) __CPROVER_w_ok(__VA_ARGS__) + +/* Maximum supported buffer size + * + * Larger buffers may be supported, but due to internal modeling constraints + * in CBMC, the proofs of memory- and type-safety won't be able to run. + * + * If you find yourself in need for a buffer size larger than this, + * please contact the maintainers, so we can prioritize work to relax + * this somewhat artificial bound. + */ +#define MLK_MAX_BUFFER_SIZE (SIZE_MAX >> 12) + +/* + * History variables + * https://diffblue.github.io/cbmc/contracts-history-variables.html + */ +#define old(...) __CPROVER_old(__VA_ARGS__) +#define loop_entry(...) __CPROVER_loop_entry(__VA_ARGS__) + +/* + * Quantifiers + * Note that the range on qvar is _exclusive_ between qvar_lb .. qvar_ub + * https://diffblue.github.io/cbmc/contracts-quantifiers.html + * + * The quantified variable is declared as uint32_t, so these macros + * quantify only over indices in [0, UINT32_MAX). Bounds larger than + * UINT32_MAX (4 GiB) are NOT supported: the explicit (uint32_t) casts + * on the bounds will trigger CBMC's conversion check if a wider bound + * (e.g. a size_t > UINT32_MAX) is passed. + * + * Quantifying over size_t (64-bit) was found to blow up SMT proof + * times, so we deliberately keep the index width at 32 bits. Callers + * dealing with size_t-typed buffers must add an explicit + * requires(len <= UINT32_MAX) + * precondition. + */ + +/* + * Prevent clang-format from corrupting CBMC's special ==> operator + */ +/* clang-format off */ +#define forall(qvar, qvar_lb, qvar_ub, predicate) \ + __CPROVER_forall \ + { \ + uint32_t qvar; \ + ((uint32_t) (qvar_lb) <= (qvar) && (qvar) < (uint32_t) (qvar_ub)) ==> (predicate) \ + } + +#define exists(qvar, qvar_lb, qvar_ub, predicate) \ + __CPROVER_exists \ + { \ + uint32_t qvar; \ + ((uint32_t) (qvar_lb) <= (qvar) && (qvar) < (uint32_t) (qvar_ub)) && (predicate) \ + } +/* clang-format on */ + +/*************************************************** + * Convenience macros for common contract patterns + ***************************************************/ + +/* + * Boolean-value predidate that asserts that "all values of array_var are in + * range value_lb (inclusive) .. value_ub (exclusive)" + * Example: + * array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q) + * expands to + * __CPROVER_forall { int k; (0 <= k && k <= MLKEM_N-1) ==> ( + * 0 <= a->coeffs[k]) && a->coeffs[k] < MLKEM_Q)) } + */ + +/* + * Prevent clang-format from corrupting CBMC's special ==> operator + */ +/* clang-format off */ +#define CBMC_CONCAT_(left, right) left##right +#define CBMC_CONCAT(left, right) CBMC_CONCAT_(left, right) + +#define array_bound_core(qvar, qvar_lb, qvar_ub, array_var, \ + value_lb, value_ub) \ + __CPROVER_forall \ + { \ + uint32_t qvar; \ + ((uint32_t) (qvar_lb) <= (qvar) && (qvar) < (uint32_t) (qvar_ub)) ==> \ + (((int)(value_lb) <= ((array_var)[(qvar)])) && \ + (((array_var)[(qvar)]) < (int)(value_ub))) \ + } + +#define array_bound(array_var, qvar_lb, qvar_ub, value_lb, value_ub) \ + array_bound_core(CBMC_CONCAT(_cbmc_idx, __COUNTER__), (qvar_lb), \ + (qvar_ub), (array_var), (value_lb), (value_ub)) + +#define array_unchanged_core(qvar, qvar_lb, qvar_ub, array_var) \ + __CPROVER_forall \ + { \ + uint32_t qvar; \ + ((uint32_t) (qvar_lb) <= (qvar) && (qvar) < (uint32_t) (qvar_ub)) ==> \ + ((array_var)[(qvar)]) == (old(* (int16_t (*)[(qvar_ub)])(array_var)))[(qvar)] \ + } + +#define array_unchanged(array_var, N) \ + array_unchanged_core(CBMC_CONCAT(_cbmc_idx, __COUNTER__), 0, (N), (array_var)) + +#define array_unchanged_u64_core(qvar, qvar_lb, qvar_ub, array_var) \ + __CPROVER_forall \ + { \ + uint32_t qvar; \ + ((uint32_t) (qvar_lb) <= (qvar) && (qvar) < (uint32_t) (qvar_ub)) ==> \ + ((array_var)[(qvar)]) == (old(* (uint64_t (*)[(qvar_ub)])(array_var)))[(qvar)] \ + } + +#define array_unchanged_u64(array_var, N) \ + array_unchanged_u64_core(CBMC_CONCAT(_cbmc_idx, __COUNTER__), 0, (N), (array_var)) + +#define array_unchanged_u8_core(qvar, array_var, N) \ + forall(qvar, 0, (N), \ + ((array_var)[(qvar)]) == (old(* (uint8_t (*)[(N)])(array_var)))[(qvar)]) + +#define array_unchanged_u8(array_var, N) \ + array_unchanged_u8_core(CBMC_CONCAT(_cbmc_idx, __COUNTER__), (array_var), (N)) + +#define array_zeroized_u8_core(qvar, array_var, N) \ + forall(qvar, 0, (N), ((array_var)[(qvar)]) == 0) + +#define array_zeroized_u8(array_var, N) \ + array_zeroized_u8_core(CBMC_CONCAT(_cbmc_idx, __COUNTER__), (array_var), (N)) +/* clang-format on */ + +/* + * Output-buffer discipline on failure, as documented in API-CONVENTIONS.md: + * when a function fails, each caller-owned output buffer is left either + * fully unchanged or fully zeroized -- never holding partially computed or + * stale data that could be mistaken for a valid result. + * + * Note the disjunction is over the buffer as a whole: it is not enough for + * each byte to be individually either unchanged or zero. + */ +#define array_unchanged_or_zeroized_u8(array_var, N) \ + (array_unchanged_u8((array_var), (N)) || array_zeroized_u8((array_var), (N))) + +/* Wrapper around array_bound operating on absolute values. + * + * The absolute value bound `k` is exclusive. + * + * Note that since the lower bound in array_bound is inclusive, we have to + * raise it by 1 here. + */ +#define array_abs_bound(arr, lb, ub, k) \ + array_bound((arr), (lb), (ub), -((int)(k)) + 1, (k)) + +#endif /* CBMC */ + +#endif /* !MLK_CBMC_H */ diff --git a/src/mlkem-native/mlkem/src/common.h b/src/mlkem-native/mlkem/src/common.h new file mode 100644 index 0000000000..96e0676fee --- /dev/null +++ b/src/mlkem-native/mlkem/src/common.h @@ -0,0 +1,296 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS140_3_IG] + * Implementation Guidance for FIPS 140-3 and the Cryptographic Module + * Validation Program + * National Institute of Standards and Technology + * https://csrc.nist.gov/projects/cryptographic-module-validation-program/fips-140-3-ig-announcements + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_COMMON_H +#define MLK_COMMON_H + +#ifndef __ASSEMBLER__ +#include +#endif + +#define MLK_BUILD_INTERNAL + +#if defined(MLK_CONFIG_FILE) +#include MLK_CONFIG_FILE +#else +#include "mlkem_native_config.h" +#endif + +#include "params.h" +#include "sys.h" + +/* Internal and public API have external linkage by default, but + * this can be overwritten by the user, e.g. for single-CU builds. */ +#if !defined(MLK_CONFIG_INTERNAL_API_QUALIFIER) +#define MLK_INTERNAL_API +#define MLK_INTERNAL_DATA_DECLARATION extern +#define MLK_INTERNAL_DATA_DEFINITION +#else +#define MLK_INTERNAL_API MLK_CONFIG_INTERNAL_API_QUALIFIER +#define MLK_INTERNAL_DATA_DECLARATION MLK_CONFIG_INTERNAL_API_QUALIFIER +#define MLK_INTERNAL_DATA_DEFINITION MLK_CONFIG_INTERNAL_API_QUALIFIER +#endif + +#if !defined(MLK_CONFIG_EXTERNAL_API_QUALIFIER) +#define MLK_EXTERNAL_API +#else +#define MLK_EXTERNAL_API MLK_CONFIG_EXTERNAL_API_QUALIFIER +#endif + +#define MLK_CONCAT_(x1, x2) x1##x2 +#define MLK_CONCAT(x1, x2) MLK_CONCAT_(x1, x2) + +#if (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || \ + defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)) +#define MLK_ADD_PARAM_SET(s) MLK_CONCAT(s, MLK_CONFIG_PARAMETER_SET) +#else +#define MLK_ADD_PARAM_SET(s) s +#endif + +#define MLK_NAMESPACE_PREFIX MLK_CONCAT(MLK_CONFIG_NAMESPACE_PREFIX, _) +#define MLK_NAMESPACE_PREFIX_K \ + MLK_CONCAT(MLK_ADD_PARAM_SET(MLK_CONFIG_NAMESPACE_PREFIX), _) + +/* Functions are prefixed by MLK_CONFIG_NAMESPACE_PREFIX. + * + * If multiple parameter sets are used, functions depending on the parameter + * set are additionally prefixed with 512/768/1024. See mlkem_native_config.h. + * + * Example: If MLK_CONFIG_NAMESPACE_PREFIX is mlkem, then + * MLK_NAMESPACE_K(enc) becomes mlkem512_enc/mlkem768_enc/mlkem1024_enc. + */ +#define MLK_NAMESPACE(s) MLK_CONCAT(MLK_NAMESPACE_PREFIX, s) +#define MLK_NAMESPACE_K(s) MLK_CONCAT(MLK_NAMESPACE_PREFIX_K, s) + +/* On Apple platforms, we need to emit leading underscore + * in front of assembly symbols. We thus introduce a separate + * namespace wrapper for ASM symbols. */ +#if !defined(__APPLE__) +#define MLK_ASM_NAMESPACE(sym) MLK_NAMESPACE(sym) +#else +#define MLK_ASM_NAMESPACE(sym) MLK_CONCAT(_, MLK_NAMESPACE(sym)) +#endif + +/* + * On X86_64 if control-flow protections (CET) are enabled (through + * -fcf-protection=), we add an endbr64 instruction at every global function + * label. See sys.h for more details + */ +#if defined(MLK_SYS_X86_64) +#define MLK_ASM_FN_SYMBOL(sym) MLK_ASM_NAMESPACE(sym) : MLK_CET_ENDBR +#elif defined(MLK_SYS_ARMV81M_MVE) +/* clang-format off */ +#define MLK_ASM_FN_SYMBOL(sym) \ + .type MLK_ASM_NAMESPACE(sym), %function; \ + MLK_ASM_NAMESPACE(sym) : +/* clang-format on */ +#else /* !MLK_SYS_X86_64 && MLK_SYS_ARMV81M_MVE */ +#define MLK_ASM_FN_SYMBOL(sym) MLK_ASM_NAMESPACE(sym) : +#endif /* !MLK_SYS_X86_64 && !MLK_SYS_ARMV81M_MVE */ + +/* + * Output the size of an assembly function. + */ +#if defined(__ELF__) +#define MLK_ASM_FN_SIZE(sym) \ + .size MLK_ASM_NAMESPACE(sym), .- MLK_ASM_NAMESPACE(sym) +#else +#define MLK_ASM_FN_SIZE(sym) +#endif + +/* We aim to simplify the user's life by supporting builds where + * all source files are included, even those that are not needed. + * Those files are appropriately guarded and will be empty when unneeded. + * The following is to avoid compilers complaining about this. */ +#define MLK_EMPTY_CU(s) extern int MLK_NAMESPACE_K(empty_cu_##s); + +/* MLK_CONFIG_NO_ASM takes precedence over MLK_USE_NATIVE_XXX */ +#if defined(MLK_CONFIG_NO_ASM) +#undef MLK_CONFIG_USE_NATIVE_BACKEND_ARITH +#undef MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 +#endif + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) && \ + !defined(MLK_CONFIG_ARITH_BACKEND_FILE) +#error Bad configuration: MLK_CONFIG_USE_NATIVE_BACKEND_ARITH is set, but MLK_CONFIG_ARITH_BACKEND_FILE is not. +#endif + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) && \ + !defined(MLK_CONFIG_FIPS202_BACKEND_FILE) +#error Bad configuration: MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 is set, but MLK_CONFIG_FIPS202_BACKEND_FILE is not. +#endif + +#if defined(MLK_CONFIG_NO_RANDOMIZED_API) && defined(MLK_CONFIG_KEYGEN_PCT) +#error Bad configuration: MLK_CONFIG_NO_RANDOMIZED_API is incompatible with MLK_CONFIG_KEYGEN_PCT as the current PCT implementation requires enc() +#endif + +#if defined(MLK_CONFIG_NO_ENCAPS_API) && defined(MLK_CONFIG_KEYGEN_PCT) +#error Bad configuration: MLK_CONFIG_NO_ENCAPS_API is incompatible with MLK_CONFIG_KEYGEN_PCT as the current PCT implementation requires enc() +#endif + +#if defined(MLK_CONFIG_NO_DECAPS_API) && defined(MLK_CONFIG_KEYGEN_PCT) +#error Bad configuration: MLK_CONFIG_NO_DECAPS_API is incompatible with MLK_CONFIG_KEYGEN_PCT as the current PCT implementation requires dec() +#endif + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) +#include MLK_CONFIG_ARITH_BACKEND_FILE +/* Include to enforce consistency of API and implementation, + * and conduct sanity checks on the backend. + * + * Keep this _after_ the inclusion of the backend; otherwise, + * the sanity checks won't have an effect. */ +#if defined(MLK_CHECK_APIS) && !defined(__ASSEMBLER__) +#include "native/api.h" +#endif +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_ARITH */ + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) +#include MLK_CONFIG_FIPS202_BACKEND_FILE +/* Include to enforce consistency of API and implementation, + * and conduct sanity checks on the backend. + * + * Keep this _after_ the inclusion of the backend; otherwise, + * the sanity checks won't have an effect. */ +#if defined(MLK_CHECK_APIS) && !defined(__ASSEMBLER__) +#include "fips202/native/api.h" +#endif +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ + +#if !defined(MLK_CONFIG_FIPS202_CUSTOM_HEADER) +#define MLK_FIPS202_HEADER_FILE "fips202/fips202.h" +#else +#define MLK_FIPS202_HEADER_FILE MLK_CONFIG_FIPS202_CUSTOM_HEADER +#endif + +#if !defined(MLK_CONFIG_FIPS202X4_CUSTOM_HEADER) +#define MLK_FIPS202X4_HEADER_FILE "fips202/fips202x4.h" +#else +#define MLK_FIPS202X4_HEADER_FILE MLK_CONFIG_FIPS202X4_CUSTOM_HEADER +#endif + +/* Standard library function replacements */ +#if !defined(__ASSEMBLER__) +#if !defined(MLK_CONFIG_CUSTOM_MEMCPY) +#include +#define mlk_memcpy memcpy +#endif + +#if !defined(MLK_CONFIG_CUSTOM_MEMSET) +#include +#define mlk_memset memset +#endif + + +/* Allocation macros for large local structures + * + * MLK_ALLOC(v, T, N) declares T *v and attempts to point it to an T[N] + * MLK_FREE(v, T, N) zeroizes and frees the allocation + * + * Default implementation uses stack allocation. + * Can be overridden by setting the config option MLK_CONFIG_CUSTOM_ALLOC_FREE + * and defining MLK_CUSTOM_ALLOC and MLK_CUSTOM_FREE. + */ +#if defined(MLK_CONFIG_CUSTOM_ALLOC_FREE) != \ + (defined(MLK_CUSTOM_ALLOC) && defined(MLK_CUSTOM_FREE)) +#error Bad configuration: MLK_CONFIG_CUSTOM_ALLOC_FREE must be set together with MLK_CUSTOM_ALLOC and MLK_CUSTOM_FREE +#endif + +/* Context-parameter machinery (MLK_CONTEXT_PARAMETERS_n and related config + * checks). Kept in a separate, level-generic header for readability; included + * here so it is available to the allocation macros below and to all consumers + * of common.h. */ +#include "context.h" + +#if !defined(MLK_CONFIG_CUSTOM_ALLOC_FREE) +/* Default: stack allocation */ + +/* This is a declaration macro, not an expression macro: T is a type and v is + * a declarator, neither of which can be wrapped in parentheses. The + * bugprone-macro-parentheses diagnostic is therefore a false positive here. */ +#define MLK_ALLOC(v, T, N, context) \ + MLK_ALIGN T mlk_alloc_##v[N]; \ + T *v = mlk_alloc_##v /* NOLINT(bugprone-macro-parentheses) */ + +/* The MLK_FREE macro body references mlk_zeroize(), which is declared in + * verify.h. We deliberately do NOT include verify.h here: doing so would + * create a circular dependency (verify.h includes common.h), and common.h + * itself never calls mlk_zeroize() -- only the macro expansion does. Each + * translation unit that uses MLK_FREE therefore includes verify.h directly. */ +#define MLK_FREE(v, T, N, context) \ + do \ + { \ + MLK_CONTEXT_UNUSED(context); \ + mlk_zeroize(mlk_alloc_##v, sizeof(mlk_alloc_##v)); \ + (v) = NULL; \ + } while (0) + +#else /* !MLK_CONFIG_CUSTOM_ALLOC_FREE */ + +/* Custom allocation */ + +/* + * The indirection here is necessary to use MLK_CONTEXT_PARAMETERS_3 here. + */ +#define MLK_APPLY(f, args) f args + +#define MLK_ALLOC(v, T, N, context) \ + MLK_APPLY(MLK_CUSTOM_ALLOC, MLK_CONTEXT_PARAMETERS_3(v, T, N, context)) + +#define MLK_FREE(v, T, N, context) \ + do \ + { \ + if (v != NULL) \ + { \ + mlk_zeroize(v, sizeof(T) * (N)); \ + MLK_APPLY(MLK_CUSTOM_FREE, MLK_CONTEXT_PARAMETERS_3(v, T, N, context)); \ + v = NULL; \ + } \ + } while (0) + +#endif /* MLK_CONFIG_CUSTOM_ALLOC_FREE */ + +/****************************** Error codes ***********************************/ + +/* Generic failure condition. Currently not returned by any function; + * reserved for failures that no more specific code covers. */ +#define MLK_ERR_FAIL (-1) +/* An allocation failed. This can only happen if MLK_CONFIG_CUSTOM_ALLOC_FREE + * is defined and the provided MLK_CUSTOM_ALLOC can fail. */ +#define MLK_ERR_OUT_OF_MEMORY (-2) +/* An RNG failure occurred. Might be due to insufficient entropy or + * system misconfiguration. */ +#define MLK_ERR_RNG_FAIL (-3) +/* Public key validation failed: the @[FIPS203, Section 7.2, 'modulus check'] + * found a coefficient outside [0,q-1]. Returned by check_pk and by the + * encapsulation API. */ +#define MLK_ERR_INVALID_PK (-4) +/* Secret key validation failed: the @[FIPS203, Section 7.3, 'hash check'] + * found the embedded public key hash inconsistent. Returned by check_sk and + * by the decapsulation API. */ +#define MLK_ERR_INVALID_SK (-5) +/* The 'Pairwise Consistency Test' @[FIPS140_3_IG, p.87] and + * @[FIPS203, Section 7.1, Pairwise Consistency] failed. Only possible when + * MLK_CONFIG_KEYGEN_PCT is enabled; signals that the freshly generated key + * pair failed its encaps/decaps self-test. */ +#define MLK_ERR_PCT_FAIL (-6) + +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_COMMON_H */ diff --git a/src/mlkem-native/mlkem/src/compress.c b/src/mlkem-native/mlkem/src/compress.c new file mode 100644 index 0000000000..b42c2584fa --- /dev/null +++ b/src/mlkem-native/mlkem/src/compress.c @@ -0,0 +1,763 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + */ + +#include "common.h" +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + + +#include "cbmc.h" +#include "compress.h" +#include "debug.h" +#include "verify.h" + +#if (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API)) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 || \ + MLKEM_K == 3) +/* Reference: `poly_compress()` in the reference implementation @[REF], + * for ML-KEM-{512,768}. + * - In contrast to the reference implementation, we assume + * unsigned canonical coefficients here. + * The reference implementation works with coefficients + * in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */ +MLK_STATIC_TESTABLE void mlk_poly_compress_d4_c( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D4], const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D4)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D4)) +) +{ + unsigned i; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + + for (i = 0; i < MLKEM_N / 8; i++) + __loop__(invariant(i <= MLKEM_N / 8) + decreases(MLKEM_N / 8 - i)) + { + unsigned j; + uint8_t t[8] = {0}; + for (j = 0; j < 8; j++) + __loop__( + invariant(i <= MLKEM_N / 8 && j <= 8) + invariant(array_bound(t, 0, j, 0, 16)) + decreases(8 - j)) + { + t[j] = mlk_scalar_compress_d4(a->coeffs[8 * i + j]); + } + + /* All t[i] are 4-bit wide, so the truncations don't alter the value. */ + r[i * 4] = (uint8_t)(t[0] | (t[1] << 4)); + r[i * 4 + 1] = (uint8_t)(t[2] | (t[3] << 4)); + r[i * 4 + 2] = (uint8_t)(t[4] | (t[5] << 4)); + r[i * 4 + 3] = (uint8_t)(t[6] | (t[7] << 4)); + } +} + +MLK_INTERNAL_API +void mlk_poly_compress_d4(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D4], + const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D4)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D4)) +) +{ +#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D4) + int ret; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + ret = mlk_poly_compress_d4_native(r, a->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D4 */ + + mlk_poly_compress_d4_c(r, a); +} + +/* Reference: Embedded into `polyvec_compress()` in the + * reference implementation, for ML-KEM-{512,768}. + * - In contrast to the reference implementation, we assume + * unsigned canonical coefficients here. + * The reference implementation works with coefficients + * in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */ +MLK_STATIC_TESTABLE void mlk_poly_compress_d10_c( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D10], const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D10)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D10)) +) +{ + unsigned j; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + for (j = 0; j < MLKEM_N / 4; j++) + __loop__(invariant(j <= MLKEM_N / 4) + decreases(MLKEM_N / 4 - j)) + { + unsigned k; + uint16_t t[4]; + for (k = 0; k < 4; k++) + __loop__( + invariant(k <= 4) + invariant(forall(r, 0, k, t[r] < (1u << 10))) + decreases(4 - k)) + { + t[k] = mlk_scalar_compress_d10(a->coeffs[4 * j + k]); + } + + /* + * Make all implicit truncation explicit. No data is being + * truncated for the LHS's since each t[i] is 10-bit in size. + */ + r[5 * j + 0] = (uint8_t)((t[0] >> 0) & 0xFF); + r[5 * j + 1] = (uint8_t)((t[0] >> 8) | ((t[1] << 2) & 0xFF)); + r[5 * j + 2] = (uint8_t)((t[1] >> 6) | ((t[2] << 4) & 0xFF)); + r[5 * j + 3] = (uint8_t)((t[2] >> 4) | ((t[3] << 6) & 0xFF)); + r[5 * j + 4] = (uint8_t)(t[3] >> 2); + } +} + +MLK_INTERNAL_API +void mlk_poly_compress_d10(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D10], + const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D10)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D10)) +) +{ +#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D10) + int ret; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + ret = mlk_poly_compress_d10_native(r, a->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D10 */ + + mlk_poly_compress_d10_c(r, a); +} + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +/* Reference: `poly_decompress()` in the reference implementation @[REF], + * for ML-KEM-{512,768}. */ +MLK_STATIC_TESTABLE void mlk_poly_decompress_d4_c( + mlk_poly *r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D4]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D4)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ + unsigned i; + for (i = 0; i < MLKEM_N / 2; i++) + __loop__( + invariant(i <= MLKEM_N / 2) + invariant(array_bound(r->coeffs, 0, 2 * i, 0, MLKEM_Q)) + decreases(MLKEM_N / 2 - i)) + { + r->coeffs[2 * i + 0] = mlk_scalar_decompress_d4((a[i] >> 0) & 0xF); + r->coeffs[2 * i + 1] = mlk_scalar_decompress_d4((a[i] >> 4) & 0xF); + } + + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); +} + +MLK_INTERNAL_API +void mlk_poly_decompress_d4(mlk_poly *r, + const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D4]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D4)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ +#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D4) + int ret; + ret = mlk_poly_decompress_d4_native(r->coeffs, a); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); + return; + } +#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D4 */ + + mlk_poly_decompress_d4_c(r, a); +} + +/* Reference: Embedded into `polyvec_decompress()` in the + * reference implementation, for ML-KEM-{512,768}. */ +MLK_STATIC_TESTABLE void mlk_poly_decompress_d10_c( + mlk_poly *r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D10]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D10)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ + unsigned j; + for (j = 0; j < MLKEM_N / 4; j++) + __loop__( + invariant(j <= MLKEM_N / 4) + invariant(array_bound(r->coeffs, 0, 4 * j, 0, MLKEM_Q)) + decreases(MLKEM_N / 4 - j)) + { + unsigned k; + uint16_t t[4]; + uint8_t const *base = &a[5 * j]; + + t[0] = 0x3FF & ((base[0] >> 0) | ((uint16_t)base[1] << 8)); + t[1] = 0x3FF & ((base[1] >> 2) | ((uint16_t)base[2] << 6)); + t[2] = 0x3FF & ((base[2] >> 4) | ((uint16_t)base[3] << 4)); + t[3] = 0x3FF & ((base[3] >> 6) | ((uint16_t)base[4] << 2)); + + for (k = 0; k < 4; k++) + __loop__( + invariant(k <= 4) + invariant(array_bound(r->coeffs, 0, 4 * j + k, 0, MLKEM_Q)) + decreases(4 - k)) + { + r->coeffs[4 * j + k] = mlk_scalar_decompress_d10(t[k]); + } + } + + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); +} + +MLK_INTERNAL_API +void mlk_poly_decompress_d10(mlk_poly *r, + const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D10]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D10)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ +#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D10) + int ret; + ret = mlk_poly_decompress_d10_native(r->coeffs, a); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); + return; + } +#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D10 */ + + mlk_poly_decompress_d10_c(r, a); +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ +#endif /* (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API) && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == 3) \ + */ + +#if (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API)) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) +/* Reference: `poly_compress()` in the reference implementation @[REF], + * for ML-KEM-1024. + * - In contrast to the reference implementation, we assume + * unsigned canonical coefficients here. + * The reference implementation works with coefficients + * in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */ +MLK_STATIC_TESTABLE void mlk_poly_compress_d5_c( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D5], const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D5)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D5)) +) +{ + unsigned i; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + + for (i = 0; i < MLKEM_N / 8; i++) + __loop__(invariant(i <= MLKEM_N / 8) + decreases(MLKEM_N / 8 - i)) + { + unsigned j; + uint8_t t[8] = {0}; + for (j = 0; j < 8; j++) + __loop__( + invariant(i <= MLKEM_N / 8 && j <= 8) + invariant(array_bound(t, 0, j, 0, 32)) + decreases(8 - j)) + { + t[j] = mlk_scalar_compress_d5(a->coeffs[8 * i + j]); + } + + r[i * 5] = (uint8_t)(0xFF & ((t[0] >> 0) | (t[1] << 5))); + r[i * 5 + 1] = (uint8_t)(0xFF & ((t[1] >> 3) | (t[2] << 2) | (t[3] << 7))); + r[i * 5 + 2] = (uint8_t)(0xFF & ((t[3] >> 1) | (t[4] << 4))); + r[i * 5 + 3] = (uint8_t)(0xFF & ((t[4] >> 4) | (t[5] << 1) | (t[6] << 6))); + r[i * 5 + 4] = (uint8_t)(0xFF & ((t[6] >> 2) | (t[7] << 3))); + } +} + +MLK_INTERNAL_API +void mlk_poly_compress_d5(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D5], + const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D5)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D5)) +) +{ +#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D5) + int ret; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + ret = mlk_poly_compress_d5_native(r, a->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D5 */ + + mlk_poly_compress_d5_c(r, a); +} + +/* Reference: Embedded into `polyvec_compress()` in the + * reference implementation, for ML-KEM-1024. + * - In contrast to the reference implementation, we assume + * unsigned canonical coefficients here. + * The reference implementation works with coefficients + * in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */ +MLK_STATIC_TESTABLE void mlk_poly_compress_d11_c( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D11], const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D11)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D11)) +) +{ + unsigned j; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + + for (j = 0; j < MLKEM_N / 8; j++) + __loop__(invariant(j <= MLKEM_N / 8) + decreases(MLKEM_N / 8 - j)) + { + unsigned k; + uint16_t t[8]; + for (k = 0; k < 8; k++) + __loop__( + invariant(k <= 8) + invariant(forall(r, 0, k, t[r] < (1u << 11))) + decreases(8 - k)) + { + t[k] = mlk_scalar_compress_d11(a->coeffs[8 * j + k]); + } + + /* + * Make all implicit truncation explicit. No data is being + * truncated for the LHS's since each t[i] is 11-bit in size. + */ + r[11 * j + 0] = (uint8_t)((t[0] >> 0) & 0xFF); + r[11 * j + 1] = (uint8_t)((t[0] >> 8) | ((t[1] << 3) & 0xFF)); + r[11 * j + 2] = (uint8_t)((t[1] >> 5) | ((t[2] << 6) & 0xFF)); + r[11 * j + 3] = (uint8_t)((t[2] >> 2) & 0xFF); + r[11 * j + 4] = (uint8_t)((t[2] >> 10) | ((t[3] << 1) & 0xFF)); + r[11 * j + 5] = (uint8_t)((t[3] >> 7) | ((t[4] << 4) & 0xFF)); + r[11 * j + 6] = (uint8_t)((t[4] >> 4) | ((t[5] << 7) & 0xFF)); + r[11 * j + 7] = (uint8_t)((t[5] >> 1) & 0xFF); + r[11 * j + 8] = (uint8_t)((t[5] >> 9) | ((t[6] << 2) & 0xFF)); + r[11 * j + 9] = (uint8_t)((t[6] >> 6) | ((t[7] << 5) & 0xFF)); + r[11 * j + 10] = (uint8_t)(t[7] >> 3); + } +} + +MLK_INTERNAL_API +void mlk_poly_compress_d11(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D11], + const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D11)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D11)) +) +{ +#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D11) + int ret; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + ret = mlk_poly_compress_d11_native(r, a->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D11 */ + + mlk_poly_compress_d11_c(r, a); +} + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +/* Reference: `poly_decompress()` in the reference implementation @[REF], + * for ML-KEM-1024. */ +MLK_STATIC_TESTABLE void mlk_poly_decompress_d5_c( + mlk_poly *r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D5]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D5)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ + unsigned i; + for (i = 0; i < MLKEM_N / 8; i++) + __loop__( + invariant(i <= MLKEM_N / 8) + invariant(array_bound(r->coeffs, 0, 8 * i, 0, MLKEM_Q)) + decreases(MLKEM_N / 8 - i)) + { + unsigned j; + uint8_t t[8]; + const unsigned offset = i * 5; + /* + * Explicitly truncate to avoid warning about + * implicit truncation in CBMC and unwind loop for ease + * of proof. + */ + + /* + * Decompress 5 8-bit bytes (so 40 bits) into + * 8 5-bit values stored in t[] + */ + t[0] = 0x1F & (a[offset + 0] >> 0); + t[1] = 0x1F & ((a[offset + 0] >> 5) | (a[offset + 1] << 3)); + t[2] = 0x1F & (a[offset + 1] >> 2); + t[3] = 0x1F & ((a[offset + 1] >> 7) | (a[offset + 2] << 1)); + t[4] = 0x1F & ((a[offset + 2] >> 4) | (a[offset + 3] << 4)); + t[5] = 0x1F & (a[offset + 3] >> 1); + t[6] = 0x1F & ((a[offset + 3] >> 6) | (a[offset + 4] << 2)); + t[7] = 0x1F & (a[offset + 4] >> 3); + + /* and copy to the correct slice in r[] */ + for (j = 0; j < 8; j++) + __loop__( + invariant(j <= 8 && i <= MLKEM_N / 8) + invariant(array_bound(r->coeffs, 0, 8 * i + j, 0, MLKEM_Q)) + decreases(8 - j)) + { + r->coeffs[8 * i + j] = mlk_scalar_decompress_d5(t[j]); + } + } + + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); +} + +MLK_INTERNAL_API +void mlk_poly_decompress_d5(mlk_poly *r, + const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D5]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D5)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ +#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D5) + int ret; + ret = mlk_poly_decompress_d5_native(r->coeffs, a); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); + return; + } +#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D5 */ + + mlk_poly_decompress_d5_c(r, a); +} + +/* Reference: Embedded into `polyvec_decompress()` in the + * reference implementation, for ML-KEM-1024. */ +MLK_STATIC_TESTABLE void mlk_poly_decompress_d11_c( + mlk_poly *r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D11]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D11)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ + unsigned j; + for (j = 0; j < MLKEM_N / 8; j++) + __loop__( + invariant(j <= MLKEM_N / 8) + invariant(array_bound(r->coeffs, 0, 8 * j, 0, MLKEM_Q)) + decreases(MLKEM_N / 8 - j)) + { + unsigned k; + uint16_t t[8]; + uint8_t const *base = &a[11 * j]; + t[0] = 0x7FF & ((base[0] >> 0) | ((uint16_t)base[1] << 8)); + t[1] = 0x7FF & ((base[1] >> 3) | ((uint16_t)base[2] << 5)); + t[2] = 0x7FF & ((base[2] >> 6) | ((uint16_t)base[3] << 2) | + ((uint16_t)base[4] << 10)); + t[3] = 0x7FF & ((base[4] >> 1) | ((uint16_t)base[5] << 7)); + t[4] = 0x7FF & ((base[5] >> 4) | ((uint16_t)base[6] << 4)); + t[5] = 0x7FF & ((base[6] >> 7) | ((uint16_t)base[7] << 1) | + ((uint16_t)base[8] << 9)); + t[6] = 0x7FF & ((base[8] >> 2) | ((uint16_t)base[9] << 6)); + t[7] = 0x7FF & ((base[9] >> 5) | ((uint16_t)base[10] << 3)); + + for (k = 0; k < 8; k++) + __loop__( + invariant(k <= 8) + invariant(array_bound(r->coeffs, 0, 8 * j + k, 0, MLKEM_Q)) + decreases(8 - k)) + { + r->coeffs[8 * j + k] = mlk_scalar_decompress_d11(t[k]); + } + } + + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); +} + +MLK_INTERNAL_API +void mlk_poly_decompress_d11(mlk_poly *r, + const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D11]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D11)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ +#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D11) + int ret; + ret = mlk_poly_decompress_d11_native(r->coeffs, a); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); + return; + } +#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D11 */ + + mlk_poly_decompress_d11_c(r, a); +} + +#endif /* !MLK_CONFIG_NO_DECAPS_API */ +#endif /* (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API) && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */ + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) || !defined(MLK_CONFIG_NO_ENCAPS_API) +/* Reference: `poly_tobytes()` in the reference implementation @[REF]. + * - In contrast to the reference implementation, we assume + * unsigned canonical coefficients here. + * The reference implementation works with coefficients + * in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */ +MLK_STATIC_TESTABLE void mlk_poly_tobytes_c(uint8_t r[MLKEM_POLYBYTES], + const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYBYTES)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYBYTES)) +) +{ + unsigned i; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + + for (i = 0; i < MLKEM_N / 2; i++) + __loop__(invariant(i <= MLKEM_N / 2) + decreases(MLKEM_N / 2 - i)) + { + /* The conversion to uint16_t is safe since we assume that + * the coefficients of `a` are non-negative. */ + const uint16_t t0 = (uint16_t)a->coeffs[2 * i]; + const uint16_t t1 = (uint16_t)a->coeffs[2 * i + 1]; + /* + * t0 and t1 are both < MLKEM_Q, so contain at most 12 bits each of + * significant data, so these can be packed into 24 bits or exactly + * 3 bytes, as follows. + */ + + /* Least significant bits 0 - 7 of t0. */ + r[3 * i + 0] = (uint8_t)(t0 & 0xFF); + + /* + * Most significant bits 8 - 11 of t0 become the least significant + * nibble of the second byte. The least significant 4 bits + * of t1 become the upper nibble of the second byte. + * + * The conversion to uint8_t does not alter the value. + */ + r[3 * i + 1] = (uint8_t)((t0 >> 8) | ((t1 << 4) & 0xF0)); + + /* Bits 4 - 11 of t1 become the third byte. The conversion to uint8_t + * does not alter the value because t1 is 12-bit wide. */ + r[3 * i + 2] = (uint8_t)(t1 >> 4); + } +} + +MLK_INTERNAL_API +void mlk_poly_tobytes(uint8_t r[MLKEM_POLYBYTES], const mlk_poly *a) +{ +#if defined(MLK_USE_NATIVE_POLY_TOBYTES) + int ret; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + ret = mlk_poly_tobytes_native(r, a->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_POLY_TOBYTES */ + + mlk_poly_tobytes_c(r, a); +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API || !MLK_CONFIG_NO_ENCAPS_API */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +/* Reference: `poly_frombytes()` in the reference implementation @[REF]. */ +MLK_STATIC_TESTABLE void mlk_poly_frombytes_c(mlk_poly *r, + const uint8_t a[MLKEM_POLYBYTES]) +__contract__( + requires(memory_no_alias(a, MLKEM_POLYBYTES)) + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT)) +) +{ + unsigned i; + for (i = 0; i < MLKEM_N / 2; i++) + __loop__( + invariant(i <= MLKEM_N / 2) + invariant(array_bound(r->coeffs, 0, 2 * i, 0, MLKEM_UINT12_LIMIT)) + decreases(MLKEM_N / 2 - i)) + { + const uint8_t t0 = a[3 * i + 0]; + const uint8_t t1 = a[3 * i + 1]; + const uint8_t t2 = a[3 * i + 2]; + /* Safety: + * - The explicit cast to uint16_t ensures that << 8 does + * not signed-overflow even on a 16-bit system. + * - The cast to int16_t is safe due to the explicit 0xFFF truncation. + */ + r->coeffs[2 * i + 0] = (int16_t)(t0 | (((uint16_t)t1 << 8) & 0xFFF)); + r->coeffs[2 * i + 1] = (int16_t)((t1 >> 4) | (t2 << 4)); + } + + /* Note that the coefficients are not canonical */ + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_UINT12_LIMIT); +} + +MLK_INTERNAL_API +void mlk_poly_frombytes(mlk_poly *r, const uint8_t a[MLKEM_POLYBYTES]) +{ +#if defined(MLK_USE_NATIVE_POLY_FROMBYTES) + int ret; + ret = mlk_poly_frombytes_native(r->coeffs, a); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_POLY_FROMBYTES */ + + mlk_poly_frombytes_c(r, a); +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +/* Reference: `poly_frommsg()` in the reference implementation @[REF]. + * - We use a value barrier around the bit-selection mask to + * reduce the risk of compiler-introduced branches. + * The reference implementation contains the expression + * `(msg[i] >> j) & 1` which the compiler can reason must + * be either 0 or 1. */ +MLK_INTERNAL_API +void mlk_poly_frommsg(mlk_poly *r, const uint8_t msg[MLKEM_INDCPA_MSGBYTES]) +{ + unsigned i; +#if (MLKEM_INDCPA_MSGBYTES != MLKEM_N / 8) +#error "MLKEM_INDCPA_MSGBYTES must be equal to MLKEM_N/8 bytes!" +#endif + + for (i = 0; i < MLKEM_N / 8; i++) + __loop__( + invariant(i <= MLKEM_N / 8) + invariant(array_bound(r->coeffs, 0, 8 * i, 0, MLKEM_Q)) + decreases(MLKEM_N / 8 - i)) + { + unsigned j; + for (j = 0; j < 8; j++) + __loop__( + invariant(i < MLKEM_N / 8 && j <= 8) + invariant(array_bound(r->coeffs, 0, 8 * i + j, 0, MLKEM_Q)) + decreases(8 - j)) + { + /* mlk_ct_sel_int16(MLKEM_Q_HALF, 0, b) is `Decompress_1(b != 0)` + * as per @[FIPS203, Eq (4.8)]. */ + + /* Prevent the compiler from recognizing this as a bit selection */ + uint8_t mask = mlk_value_barrier_u8((uint8_t)(1u << j)); + r->coeffs[8 * i + j] = mlk_ct_sel_int16(MLKEM_Q_HALF, 0, msg[i] & mask); + } + } + mlk_assert_abs_bound(r, MLKEM_N, MLKEM_Q); +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +/* Reference: `poly_tomsg()` in the reference implementation @[REF]. + * - In contrast to the reference implementation, we assume + * unsigned canonical coefficients here. + * The reference implementation works with coefficients + * in the range [-(MLKEM_Q-1), MLKEM_Q-1]. + */ +MLK_INTERNAL_API +void mlk_poly_tomsg(uint8_t msg[MLKEM_INDCPA_MSGBYTES], const mlk_poly *r) +{ + unsigned i; + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); + + for (i = 0; i < MLKEM_N / 8; i++) + __loop__(invariant(i <= MLKEM_N / 8) + decreases(MLKEM_N / 8 - i)) + { + unsigned j; + msg[i] = 0; + for (j = 0; j < 8; j++) + __loop__( + invariant(i <= MLKEM_N / 8 && j <= 8) + decreases(8 - j)) + { + uint32_t t = mlk_scalar_compress_d1(r->coeffs[8 * i + j]); + msg[i] |= (uint8_t)(t << j); + } + } +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(compress) + +#endif /* MLK_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/src/mlkem-native/mlkem/src/compress.h b/src/mlkem-native/mlkem/src/compress.h new file mode 100644 index 0000000000..19e1d402b2 --- /dev/null +++ b/src/mlkem-native/mlkem/src/compress.h @@ -0,0 +1,613 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + */ + +#ifndef MLK_COMPRESS_H +#define MLK_COMPRESS_H + + +#include "cbmc.h" +#include "common.h" +#include "debug.h" +#include "poly.h" +#include "verify.h" + +/* + * The multiplication in this routine will exceed UINT32_MAX + * and wrap around for large values of u. This is expected and required. + */ +#ifdef CBMC +#pragma CPROVER check push +#pragma CPROVER check disable "unsigned-overflow" +#endif + +/** + * Compute round(u * 2 / MLKEM_Q). + * + * @spec{Compress_1 from @[FIPS203, Eq (4.7)].} + * + * @reference{Part of poly_tomsg() in the reference implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo MLKEM_Q to be compressed. + * + * @return Compressed value. + */ +static MLK_INLINE uint8_t mlk_scalar_compress_d1(int16_t u) +__contract__( + requires(0 <= u && u <= MLKEM_Q - 1) + ensures(return_value < 2) + ensures(return_value == (((uint32_t)u * 2 + MLKEM_Q / 2) / MLKEM_Q) % 2) ) +{ + /* Compute as follows: + * ``` + * round(u * 2 / MLKEM_Q) + * = round(u * 2 * (2^31 / MLKEM_Q) / 2^31) + * ~= round(u * 2 * round(2^31 / MLKEM_Q) / 2^31) + * ``` + */ + /* check-magic: 1290168 == 2*round(2^31 / MLKEM_Q) */ + uint32_t d0 = (uint32_t)u * 1290168; + /* Unsigned shifting by 31 positions leaves only the top bit. */ + return (uint8_t)((d0 + ((uint32_t)1u << 30)) >> 31); +} +#ifdef CBMC +#pragma CPROVER check pop +#endif + +/* + * The multiplication in this routine will exceed UINT32_MAX + * and wrap around for large values of u. This is expected and required. + */ +#ifdef CBMC +#pragma CPROVER check push +#pragma CPROVER check disable "unsigned-overflow" +#endif + +/** + * Compute round(u * 16 / MLKEM_Q) % 16. + * + * @spec{Compress_4 from @[FIPS203, Eq (4.7)].} + * + * @reference{Embedded into `poly_compress()` in the reference + * implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo MLKEM_Q to be compressed. + * + * @return Compressed value. + */ +static MLK_INLINE uint8_t mlk_scalar_compress_d4(int16_t u) +__contract__( + requires(0 <= u && u <= MLKEM_Q - 1) + ensures(return_value < 16) + ensures(return_value == (((uint32_t)u * 16 + MLKEM_Q / 2) / MLKEM_Q) % 16)) +{ + /* Compute as follows: + * ``` + * round(u * 16 / MLKEM_Q) + * = round(u * 16 * (2^28 / MLKEM_Q) / 2^28) + * ~= round(u * 16 * round(2^28 / MLKEM_Q) / 2^28) + * ``` + */ + /* check-magic: 1290160 == 16 * round(2^28 / MLKEM_Q) */ + uint32_t d0 = (uint32_t)u * 1290160; + /* The return value is < 16, so not altered by the conversion to uint8_t. */ + return (uint8_t)((d0 + ((uint32_t)1u << 27)) >> 28); /* round(d0/2^28) */ +} +#ifdef CBMC +#pragma CPROVER check pop +#endif + +/** + * Compute round(u * MLKEM_Q / 16). + * + * @spec{Decompress_4 from @[FIPS203, Eq (4.8)].} + * + * @reference{Embedded into `poly_decompress()` in the reference + * implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo 16 to be decompressed. + * + * @return Decompressed value. + */ +static MLK_INLINE int16_t mlk_scalar_decompress_d4(uint8_t u) +__contract__( + requires(0 <= u && u < 16) + ensures(return_value <= (MLKEM_Q - 1)) +) +{ + /* The return value is in 0..MLKEM_Q-1, hence not altered by the + * conversion to int16_t. */ + return (int16_t)((((uint32_t)u * MLKEM_Q) + 8) >> 4); +} + +/* + * The multiplication in this routine will exceed UINT32_MAX + * and wrap around for large values of u. This is expected and required. + */ +#ifdef CBMC +#pragma CPROVER check push +#pragma CPROVER check disable "unsigned-overflow" +#endif + +/** + * Compute round(u * 32 / MLKEM_Q) % 32. + * + * @spec{Compress_5 from @[FIPS203, Eq (4.7)].} + * + * @reference{Embedded into `poly_compress()` in the reference + * implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo MLKEM_Q to be compressed. + * + * @return Compressed value. + */ +static MLK_INLINE uint8_t mlk_scalar_compress_d5(int16_t u) +__contract__( + requires(0 <= u && u <= MLKEM_Q - 1) + ensures(return_value < 32) + ensures(return_value == (((uint32_t)u * 32 + MLKEM_Q / 2) / MLKEM_Q) % 32) ) +{ + /* Compute as follows: + * ``` + * round(u * 32 / MLKEM_Q) + * = round(u * 32 * (2^27 / MLKEM_Q) / 2^27) + * ~= round(u * 32 * round(2^27 / MLKEM_Q) / 2^27) + * ``` + */ + /* check-magic: 1290176 == 2^5 * round(2^27 / MLKEM_Q) */ + uint32_t d0 = (uint32_t)u * 1290176; + /* The return value is < 32, so not altered by the conversion to uint8_t. */ + return (uint8_t)((d0 + ((uint32_t)1u << 26)) >> 27); /* round(d0/2^27) */ +} +#ifdef CBMC +#pragma CPROVER check pop +#endif + +/** + * Compute round(u * MLKEM_Q / 32). + * + * @spec{Decompress_5 from @[FIPS203, Eq (4.8)].} + * + * @reference{Embedded into `poly_decompress()` in the reference + * implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo 32 to be decompressed. + * + * @return Decompressed value. + */ +static MLK_INLINE int16_t mlk_scalar_decompress_d5(uint8_t u) +__contract__( + requires(0 <= u && u < 32) + ensures(0 <= return_value && return_value <= MLKEM_Q - 1) +) +{ + /* The return value is in 0..MLKEM_Q-1, hence not altered by the + * conversion to int16_t. */ + return (int16_t)((((uint32_t)u * MLKEM_Q) + 16) >> 5); +} + +/* + * The multiplication in this routine will exceed UINT32_MAX + * and wrap around for large values of u. This is expected and required. + */ +#ifdef CBMC +#pragma CPROVER check push +#pragma CPROVER check disable "unsigned-overflow" +#endif + +/** + * Compute round(u * 2**10 / MLKEM_Q) % 2**10. + * + * @spec{Compress_10 from @[FIPS203, Eq (4.7)].} + * + * @reference{Embedded into `polyvec_compress()` in the reference + * implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo MLKEM_Q to be compressed. + * + * @return Compressed value. + */ +static MLK_INLINE uint16_t mlk_scalar_compress_d10(int16_t u) +__contract__( + requires(0 <= u && u <= MLKEM_Q - 1) + ensures(return_value < (1u << 10)) + ensures(return_value == (((uint32_t)u * (1u << 10) + MLKEM_Q / 2) / MLKEM_Q) % (1 << 10))) +{ + /* Compute as follows: + * ``` + * round(u * 1024 / MLKEM_Q) + * = round(u * 1024 * (2^33 / MLKEM_Q) / 2^33) + * ~= round(u * 1024 * round(2^33 / MLKEM_Q) / 2^33) + * ``` + */ + /* check-magic: 2642263040 == 2^10 * round(2^33 / MLKEM_Q) */ + uint64_t d0 = (uint64_t)u * 2642263040; + d0 = (d0 + ((uint64_t)1u << 32)) >> 33; /* round(d0/2^33) */ + return (d0 & 0x3FF); +} +#ifdef CBMC +#pragma CPROVER check pop +#endif + +/** + * Compute round(u * MLKEM_Q / 1024). + * + * @spec{Decompress_10 from @[FIPS203, Eq (4.8)].} + * + * @reference{Embedded into `polyvec_decompress()` in the reference + * implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo 1024 to be decompressed. + * + * @return Decompressed value. + */ +static MLK_INLINE int16_t mlk_scalar_decompress_d10(uint16_t u) +__contract__( + requires(0 <= u && u < 1024) + ensures(0 <= return_value && return_value <= (MLKEM_Q - 1)) +) +{ + /* The return value is in 0..MLKEM_Q-1, hence not altered by the + * conversion to int16_t. */ + return (int16_t)((((uint32_t)u * MLKEM_Q) + 512) >> 10); +} + +/* + * The multiplication in this routine will exceed UINT32_MAX + * and wrap around for large values of u. This is expected and required. + */ +#ifdef CBMC +#pragma CPROVER check push +#pragma CPROVER check disable "unsigned-overflow" +#endif + +/** + * Compute round(u * 2**11 / MLKEM_Q) % 2**11. + * + * @spec{Compress_11 from @[FIPS203, Eq (4.7)].} + * + * @reference{Embedded into `polyvec_compress()` in the reference + * implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo MLKEM_Q to be compressed. + * + * @return Compressed value. + */ +static MLK_INLINE uint16_t mlk_scalar_compress_d11(int16_t u) +__contract__( + requires(0 <= u && u <= MLKEM_Q - 1) + ensures(return_value < (1u << 11)) + ensures(return_value == (((uint32_t)u * (1u << 11) + MLKEM_Q / 2) / MLKEM_Q) % (1 << 11))) +{ + /* Compute as follows: + * ``` + * round(u * 2048 / MLKEM_Q) + * = round(u * 2048 * (2^33 / MLKEM_Q) / 2^33) + * ~= round(u * 2048 * round(2^33 / MLKEM_Q) / 2^33) + * ``` + */ + /* check-magic: 5284526080 == 2^11 * round(2^33 / MLKEM_Q) */ + uint64_t d0 = (uint64_t)u * 5284526080; + d0 = (d0 + ((uint64_t)1u << 32)) >> 33; /* round(d0/2^33) */ + return (d0 & 0x7FF); +} +#ifdef CBMC +#pragma CPROVER check pop +#endif + +/** + * Compute round(u * MLKEM_Q / 2048). + * + * @spec{Decompress_11 from @[FIPS203, Eq (4.8)].} + * + * @reference{Embedded into `polyvec_decompress()` in the reference + * implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo 2048 to be decompressed. + * + * @return Decompressed value. + */ +static MLK_INLINE int16_t mlk_scalar_decompress_d11(uint16_t u) +__contract__( + requires(0 <= u && u < 2048) + ensures(0 <= return_value && return_value <= (MLKEM_Q - 1)) +) +{ + /* The return value is in 0..MLKEM_Q-1, hence not altered by the + * conversion to int16_t. */ + return (int16_t)((((uint32_t)u * MLKEM_Q) + 1024) >> 11); +} + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || (MLKEM_K == 2 || MLKEM_K == 3) +#define mlk_poly_compress_d4 MLK_NAMESPACE(poly_compress_d4) +/** + * Compression (4 bits) and subsequent serialization of a polynomial. + * + * @spec{Implements `ByteEncode_4 (Compress_4 (a))`: ByteEncode_d + * @[FIPS203, Algorithm 5], Compress_d @[FIPS203, Eq (4.7)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `ByteEncode_{d_v} (Compress_{d_v} (v))` appears in @[FIPS203, Algorithm + * 14 (K-PKE.Encrypt), L23], where `d_v=4` for ML-KEM-{512,768} @[FIPS203, + * Table 2].} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D4 + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + */ +MLK_INTERNAL_API +void mlk_poly_compress_d4(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D4], + const mlk_poly *a); + +#define mlk_poly_compress_d10 MLK_NAMESPACE(poly_compress_d10) +/** + * Compression (10 bits) and subsequent serialization of a polynomial. + * + * @spec{Implements `ByteEncode_10 (Compress_10 (a))`: ByteEncode_d + * @[FIPS203, Algorithm 5], Compress_d @[FIPS203, Eq (4.7)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `ByteEncode_{d_u} (Compress_{d_u} (u))` appears in @[FIPS203, Algorithm + * 14 (K-PKE.Encrypt), L22], where `d_u=10` for ML-KEM-{512,768} @[FIPS203, + * Table 2].} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D10 + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + */ +MLK_INTERNAL_API +void mlk_poly_compress_d10(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D10], + const mlk_poly *a); + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_poly_decompress_d4 MLK_NAMESPACE(poly_decompress_d4) +/** + * De-serialization and subsequent decompression (4 bits) of a polynomial; + * approximate inverse of mlk_poly_compress_d4. + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @spec{Implements `Decompress_4 (ByteDecode_4 (a))`: ByteDecode_d + * @[FIPS203, Algorithm 6], Decompress_d @[FIPS203, Eq (4.8)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `Decompress_{d_v} (ByteDecode_{d_v} (v))` appears in @[FIPS203, Algorithm + * 15 (K-PKE.Decrypt), L4], where `d_v=4` for ML-KEM-{512,768} @[FIPS203, + * Table 2].} + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D4 + * bytes). + */ +MLK_INTERNAL_API +void mlk_poly_decompress_d4(mlk_poly *r, + const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D4]); + +#define mlk_poly_decompress_d10 MLK_NAMESPACE(poly_decompress_d10) +/** + * De-serialization and subsequent decompression (10 bits) of a polynomial; + * approximate inverse of mlk_poly_compress_d10. + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @spec{Implements `Decompress_10 (ByteDecode_10 (a))`: ByteDecode_d + * @[FIPS203, Algorithm 6], Decompress_d @[FIPS203, Eq (4.8)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `Decompress_{d_u} (ByteDecode_{d_u} (u))` appears in @[FIPS203, Algorithm + * 15 (K-PKE.Decrypt), L3], where `d_u=10` for ML-KEM-{512,768} @[FIPS203, + * Table 2].} + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D10 + * bytes). + */ +MLK_INTERNAL_API +void mlk_poly_decompress_d10(mlk_poly *r, + const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D10]); +#endif /* !MLK_CONFIG_NO_DECAPS_API */ +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == 3 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4 +#define mlk_poly_compress_d5 MLK_NAMESPACE(poly_compress_d5) +/** + * Compression (5 bits) and subsequent serialization of a polynomial. + * + * @spec{Implements `ByteEncode_5 (Compress_5 (a))`: ByteEncode_d + * @[FIPS203, Algorithm 5], Compress_d @[FIPS203, Eq (4.7)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `ByteEncode_{d_v} (Compress_{d_v} (v))` appears in @[FIPS203, Algorithm + * 14 (K-PKE.Encrypt), L23], where `d_v=5` for ML-KEM-1024 @[FIPS203, + * Table 2].} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D5 + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + */ +MLK_INTERNAL_API +void mlk_poly_compress_d5(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D5], + const mlk_poly *a); + +#define mlk_poly_compress_d11 MLK_NAMESPACE(poly_compress_d11) +/** + * Compression (11 bits) and subsequent serialization of a polynomial. + * + * @spec{`ByteEncode_11 (Compress_11 (a))`: ByteEncode_d @[FIPS203, + * Algorithm 5], Compress_d @[FIPS203, Eq (4.7)], extended to vectors as + * per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `ByteEncode_{d_u} (Compress_{d_u} (u))` appears in @[FIPS203, Algorithm + * 14 (K-PKE.Encrypt), L22], where `d_u=11` for ML-KEM-1024 @[FIPS203, + * Table 2].} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D11 + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + */ +MLK_INTERNAL_API +void mlk_poly_compress_d11(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D11], + const mlk_poly *a); + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_poly_decompress_d5 MLK_NAMESPACE(poly_decompress_d5) +/** + * De-serialization and subsequent decompression (5 bits) of a polynomial; + * approximate inverse of mlk_poly_compress_d5. + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @spec{Implements `Decompress_5 (ByteDecode_5 (a))`: ByteDecode_d + * @[FIPS203, Algorithm 6], Decompress_d @[FIPS203, Eq (4.8)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `Decompress_{d_v} (ByteDecode_{d_v} (v))` appears in @[FIPS203, Algorithm + * 15 (K-PKE.Decrypt), L4], where `d_v=5` for ML-KEM-1024 @[FIPS203, + * Table 2].} + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D5 + * bytes). + */ +MLK_INTERNAL_API +void mlk_poly_decompress_d5(mlk_poly *r, + const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D5]); + +#define mlk_poly_decompress_d11 MLK_NAMESPACE(poly_decompress_d11) +/** + * De-serialization and subsequent decompression (11 bits) of a polynomial; + * approximate inverse of mlk_poly_compress_d11. + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @spec{Implements `Decompress_11 (ByteDecode_11 (a))`: ByteDecode_d + * @[FIPS203, Algorithm 6], Decompress_d @[FIPS203, Eq (4.8)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `Decompress_{d_u} (ByteDecode_{d_u} (u))` appears in @[FIPS203, Algorithm + * 15 (K-PKE.Decrypt), L3], where `d_u=11` for ML-KEM-1024 @[FIPS203, + * Table 2].} + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D11 + * bytes). + */ +MLK_INTERNAL_API +void mlk_poly_decompress_d11(mlk_poly *r, + const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D11]); +#endif /* !MLK_CONFIG_NO_DECAPS_API */ +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */ +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) || !defined(MLK_CONFIG_NO_ENCAPS_API) +#define mlk_poly_tobytes MLK_NAMESPACE(poly_tobytes) +/** + * Serialization of a polynomial. Signed coefficients are converted to + * unsigned form before serialization. + * + * @spec{Implements ByteEncode_12 @[FIPS203, Algorithm 5]. Extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays].} + * + * @param[out] r Output byte array (of MLKEM_POLYBYTES bytes). + * @param[in] a Input polynomial, with each coefficient in the range + * [0,1,..,MLKEM_Q-1]. + */ +MLK_INTERNAL_API +void mlk_poly_tobytes(uint8_t r[MLKEM_POLYBYTES], const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYBYTES)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYBYTES)) +); +#endif /* !MLK_CONFIG_NO_KEYPAIR_API || !MLK_CONFIG_NO_ENCAPS_API */ + + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_poly_frombytes MLK_NAMESPACE(poly_frombytes) +/** + * De-serialization of a polynomial. + * + * @spec{Implements ByteDecode_12 @[FIPS203, Algorithm 6]. Extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays].} + * + * @param[out] r Output polynomial, with each coefficient unsigned and in + * the range 0..4095. + * @param[in] a Input byte array (of MLKEM_POLYBYTES bytes). + */ +MLK_INTERNAL_API +void mlk_poly_frombytes(mlk_poly *r, const uint8_t a[MLKEM_POLYBYTES]) +__contract__( + requires(memory_no_alias(a, MLKEM_POLYBYTES)) + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT)) +); +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_poly_frommsg MLK_NAMESPACE(poly_frommsg) +/** + * Convert a 32-byte message to a polynomial. + * + * @spec{Implements `Decompress_1 (ByteDecode_1 (a))`: ByteDecode_d + * @[FIPS203, Algorithm 6], Decompress_d @[FIPS203, Eq (4.8)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `Decompress_1 (ByteDecode_1 (w))` appears in @[FIPS203, Algorithm 15 + * (K-PKE.Encrypt), L20].} + * + * @param[out] r Output polynomial. + * @param[in] msg Input message. + */ +MLK_INTERNAL_API +void mlk_poly_frommsg(mlk_poly *r, const uint8_t msg[MLKEM_INDCPA_MSGBYTES]) +__contract__( + requires(memory_no_alias(msg, MLKEM_INDCPA_MSGBYTES)) + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +); +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_poly_tomsg MLK_NAMESPACE(poly_tomsg) +/** + * Convert a polynomial to a 32-byte message. + * + * @spec{Implements `ByteEncode_1 (Compress_1 (a))`: ByteEncode_d + * @[FIPS203, Algorithm 5], Compress_d @[FIPS203, Eq (4.7)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `ByteEncode_1 (Compress_1 (w))` appears in @[FIPS203, Algorithm 14 + * (K-PKE.Decrypt), L7].} + * + * @param[out] msg Output message. + * @param[in] r Input polynomial. Coefficients must be unsigned canonical. + */ +MLK_INTERNAL_API +void mlk_poly_tomsg(uint8_t msg[MLKEM_INDCPA_MSGBYTES], const mlk_poly *r) +__contract__( + requires(memory_no_alias(msg, MLKEM_INDCPA_MSGBYTES)) + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(msg, MLKEM_INDCPA_MSGBYTES)) +); +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#endif /* !MLK_COMPRESS_H */ diff --git a/src/mlkem-native/mlkem/src/context.h b/src/mlkem-native/mlkem/src/context.h new file mode 100644 index 0000000000..9a07ef2846 --- /dev/null +++ b/src/mlkem-native/mlkem/src/context.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_CONTEXT_H +#define MLK_CONTEXT_H + +/* This header is included by common.h once the configuration has been pulled + * in; it is not meant to be included directly. */ + +/* + * If the integration wants to provide a context parameter for use in + * platform-specific hooks, then it should define this parameter. + * + * The MLK_CONTEXT_PARAMETERS_n macros are intended to be used with macros + * defining the function names and expand to either pass or discard the context + * argument as required by the current build. If there is no context parameter + * requested then these are removed from the prototypes and from all calls. + */ +#ifdef MLK_CONFIG_CONTEXT_PARAMETER +#define MLK_CONTEXT_PARAMETERS_0(context) (context) +#define MLK_CONTEXT_PARAMETERS_1(arg0, context) (arg0, context) +#define MLK_CONTEXT_PARAMETERS_2(arg0, arg1, context) (arg0, arg1, context) +#define MLK_CONTEXT_PARAMETERS_3(arg0, arg1, arg2, context) \ + (arg0, arg1, arg2, context) +#define MLK_CONTEXT_PARAMETERS_4(arg0, arg1, arg2, arg3, context) \ + (arg0, arg1, arg2, arg3, context) +#else /* MLK_CONFIG_CONTEXT_PARAMETER */ +#define MLK_CONTEXT_PARAMETERS_0(context) () +#define MLK_CONTEXT_PARAMETERS_1(arg0, context) (arg0) +#define MLK_CONTEXT_PARAMETERS_2(arg0, arg1, context) (arg0, arg1) +#define MLK_CONTEXT_PARAMETERS_3(arg0, arg1, arg2, context) (arg0, arg1, arg2) +#define MLK_CONTEXT_PARAMETERS_4(arg0, arg1, arg2, arg3, context) \ + (arg0, arg1, arg2, arg3) +#endif /* !MLK_CONFIG_CONTEXT_PARAMETER */ + +/* Consume a context parameter carried only for the integration's benefit, + * avoiding -Wunused-parameter; expands to nothing when no context is + * configured. */ +#if defined(MLK_CONFIG_CONTEXT_PARAMETER) +#define MLK_CONTEXT_UNUSED(context) ((void)(context)) +#else +#define MLK_CONTEXT_UNUSED(context) ((void)0) +#endif + +#if defined(MLK_CONFIG_CONTEXT_PARAMETER_TYPE) != \ + defined(MLK_CONFIG_CONTEXT_PARAMETER) +#error MLK_CONFIG_CONTEXT_PARAMETER_TYPE must be defined if and only if MLK_CONFIG_CONTEXT_PARAMETER is defined +#endif + +#endif /* !MLK_CONTEXT_H */ diff --git a/src/mlkem-native/mlkem/src/debug.c b/src/mlkem-native/mlkem/src/debug.c new file mode 100644 index 0000000000..386f5264ff --- /dev/null +++ b/src/mlkem-native/mlkem/src/debug.c @@ -0,0 +1,64 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* NOTE: You can remove this file unless you compile with MLKEM_DEBUG. */ + +#include "common.h" + +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(MLKEM_DEBUG) + + +#include +#include +#include "debug.h" + +#define MLK_DEBUG_ERROR_HEADER "[ERROR:%s:%04d] " + +void mlk_debug_check_assert(const char *file, int line, const int val) +{ + if (val == 0) + { + fprintf(stderr, MLK_DEBUG_ERROR_HEADER "Assertion failed (value %d)\n", + file, line, val); + exit(1); + } +} + +void mlk_debug_check_bounds(const char *file, int line, const int16_t *ptr, + unsigned len, int lower_bound_exclusive, + int upper_bound_exclusive) +{ + int err = 0; + unsigned i; + for (i = 0; i < len; i++) + { + int16_t val = ptr[i]; + if (!(val > lower_bound_exclusive && val < upper_bound_exclusive)) + { + fprintf( + stderr, + MLK_DEBUG_ERROR_HEADER + "Bounds assertion failed: Index %u, value %d out of bounds (%d,%d)\n", + file, line, i, (int)val, lower_bound_exclusive, + upper_bound_exclusive); + err = 1; + } + } + + if (err == 1) + { + exit(1); + } +} + +#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED && MLKEM_DEBUG */ + +MLK_EMPTY_CU(debug) + +#endif /* !(!MLK_CONFIG_MULTILEVEL_NO_SHARED && MLKEM_DEBUG) */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef MLK_DEBUG_ERROR_HEADER diff --git a/src/mlkem-native/mlkem/src/debug.h b/src/mlkem-native/mlkem/src/debug.h new file mode 100644 index 0000000000..ce4a33a373 --- /dev/null +++ b/src/mlkem-native/mlkem/src/debug.h @@ -0,0 +1,121 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_DEBUG_H +#define MLK_DEBUG_H +#include "common.h" + +#if defined(MLKEM_DEBUG) + +/** + * Check debug assertion. + * + * Prints an error message to stderr and calls exit(1) on failure. + * + * @param[in] file Filename. + * @param line Line number. + * @param val Value asserted to be non-zero. + */ +#define mlk_debug_check_assert MLK_NAMESPACE(mlkem_debug_assert) +void mlk_debug_check_assert(const char *file, int line, const int val); + +/** + * Check whether values in an array of int16_t are within specified bounds. + * + * Prints an error message to stderr and calls exit(1) on failure. + * + * @param[in] file Filename. + * @param line Line number. + * @param[in] ptr Base of array to be checked. + * @param len Number of int16_t in @p ptr. + * @param lower_bound_exclusive Exclusive lower bound. + * @param upper_bound_exclusive Exclusive upper bound. + */ +#define mlk_debug_check_bounds MLK_NAMESPACE(mlkem_debug_check_bounds) +void mlk_debug_check_bounds(const char *file, int line, const int16_t *ptr, + unsigned len, int lower_bound_exclusive, + int upper_bound_exclusive); + +/* Check assertion, calling exit() upon failure + * + * val: Value that's asserted to be non-zero + */ +#define mlk_assert(val) mlk_debug_check_assert(__FILE__, __LINE__, (val)) + +/* Check bounds in array of int16_t's + * ptr: Base of int16_t array; will be explicitly cast to int16_t*, + * so you may pass a byte-compatible type such as mlk_poly or mlk_polyvec. + * len: Number of int16_t in array + * value_lb: Inclusive lower value bound + * value_ub: Exclusive upper value bound */ +#define mlk_assert_bound(ptr, len, value_lb, value_ub) \ + mlk_debug_check_bounds(__FILE__, __LINE__, (const int16_t *)(ptr), (len), \ + (value_lb) - 1, (value_ub)) + +/* Check absolute bounds in array of int16_t's + * ptr: Base of array, expression of type int16_t* + * len: Number of int16_t in array + * value_abs_bd: Exclusive absolute upper bound */ +#define mlk_assert_abs_bound(ptr, len, value_abs_bd) \ + mlk_assert_bound((ptr), (len), (-(value_abs_bd) + 1), (value_abs_bd)) + +/* Version of bounds assertions for 2-dimensional arrays */ +#define mlk_assert_bound_2d(ptr, len0, len1, value_lb, value_ub) \ + mlk_assert_bound((ptr), ((len0) * (len1)), (value_lb), (value_ub)) + +#define mlk_assert_abs_bound_2d(ptr, len0, len1, value_abs_bd) \ + mlk_assert_abs_bound((ptr), ((len0) * (len1)), (value_abs_bd)) + +/* When running CBMC, convert debug assertions into proof obligations */ +#elif defined(CBMC) +#include "cbmc.h" + +#define mlk_assert(val) cassert(val) + +#define mlk_assert_bound(ptr, len, value_lb, value_ub) \ + cassert(array_bound(((int16_t *)(ptr)), 0, (len), (value_lb), (value_ub))) + +#define mlk_assert_abs_bound(ptr, len, value_abs_bd) \ + cassert(array_abs_bound(((int16_t *)(ptr)), 0, (len), (value_abs_bd))) + +/* Because of https://github.com/diffblue/cbmc/issues/8570, we can't + * just use a single flattened array_bound(...) here. */ +#define mlk_assert_bound_2d(ptr, M, N, value_lb, value_ub) \ + cassert(forall(kN, 0, (M), \ + array_bound(&((int16_t (*)[(N)])(ptr))[kN][0], 0, (N), \ + (value_lb), (value_ub)))) + +#define mlk_assert_abs_bound_2d(ptr, M, N, value_abs_bd) \ + cassert(forall(kN, 0, (M), \ + array_abs_bound(&((int16_t (*)[(N)])(ptr))[kN][0], 0, (N), \ + (value_abs_bd)))) + +#else /* !MLKEM_DEBUG && CBMC */ + +#define mlk_assert(val) \ + do \ + { \ + } while (0) +#define mlk_assert_bound(ptr, len, value_lb, value_ub) \ + do \ + { \ + } while (0) +#define mlk_assert_abs_bound(ptr, len, value_abs_bd) \ + do \ + { \ + } while (0) + +#define mlk_assert_bound_2d(ptr, len0, len1, value_lb, value_ub) \ + do \ + { \ + } while (0) + +#define mlk_assert_abs_bound_2d(ptr, len0, len1, value_abs_bd) \ + do \ + { \ + } while (0) + + +#endif /* !MLKEM_DEBUG && !CBMC */ +#endif /* !MLK_DEBUG_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/fips202.c b/src/mlkem-native/mlkem/src/fips202/fips202.c new file mode 100644 index 0000000000..33883aa9f8 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/fips202.c @@ -0,0 +1,249 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [mupq] + * Common files for pqm4, pqm3, pqriscv + * Kannwischer, Petri, Rijneveld, Schwabe, Stoffelen + * https://github.com/mupq/mupq + * + * - [supercop] + * SUPERCOP benchmarking framework + * Daniel J. Bernstein + * http://bench.cr.yp.to/supercop.html + * + * - [tweetfips] + * 'tweetfips202' FIPS202 implementation + * Van Assche, Bernstein, Schwabe + * https://keccak.team/2015/tweetfips202.html + */ + +/* Based on the CC0 implementation from @[mupq] and the public domain + * implementation @[supercop, crypto_hash/keccakc512/simple/] + * by Ronny Van Keer, and the public domain @[tweetfips] implementation. */ + +#include "../common.h" +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + + +#include "../verify.h" +#include "fips202.h" +#include "keccakf1600.h" + +/** + * Absorb step of Keccak; non-incremental, starts by zeroeing the state. + * + * @warning Must only be called once. + * + * @param[out] s Pointer to (uninitialized) output Keccak state. + * @param r Rate in bytes (e.g., 168 for SHAKE128). + * @param[in] m Input to be absorbed into @p s. + * @param mlen Length of input in bytes. + * @param p Domain-separation byte for different Keccak-derived + * functions. + */ +static void mlk_keccak_absorb_once(uint64_t *s, unsigned r, const uint8_t *m, + size_t mlen, uint8_t p) +__contract__( + requires(mlen <= MLK_MAX_BUFFER_SIZE) + requires(r > 0) + requires(r <= sizeof(uint64_t) * MLK_KECCAK_LANES) + requires(memory_no_alias(s, sizeof(uint64_t) * MLK_KECCAK_LANES)) + requires(memory_no_alias(m, mlen)) + assigns(memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES))) +{ + /* Initialize state */ + size_t i; + for (i = 0; i < 25; ++i) + __loop__(invariant(i <= 25) + decreases(25 - i)) + { + s[i] = 0; + } + + while (mlen >= r) + __loop__( + assigns(mlen, m, memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES)) + invariant(mlen <= loop_entry(mlen)) + invariant(m == loop_entry(m) + (loop_entry(mlen) - mlen)) + decreases(mlen)) + { + mlk_keccakf1600_xor_bytes(s, m, 0, r); + mlk_keccakf1600_permute(s); + mlen -= r; + m += r; + } + + /* At this point, mlen < r, so the truncations to unsigned are safe below. */ + + if (mlen > 0) + { + mlk_keccakf1600_xor_bytes(s, m, 0, (unsigned int)mlen); + } + + if (mlen == r - 1) + { + p |= 128; + mlk_keccakf1600_xor_bytes(s, &p, (unsigned int)mlen, 1); + } + else + { + mlk_keccakf1600_xor_bytes(s, &p, (unsigned int)mlen, 1); + p = 128; + mlk_keccakf1600_xor_bytes(s, &p, r - 1, 1); + } +} + +/** + * Block-level Keccak squeeze. + * + * @param[out] h Output bytes. + * @param nblocks Number of blocks to be squeezed. + * @param[in,out] s Input/output state. + * @param r Rate in bytes (e.g., 168 for SHAKE128). + */ +static void mlk_keccak_squeezeblocks(uint8_t *h, size_t nblocks, uint64_t *s, + unsigned r) +__contract__( + requires(r <= sizeof(uint64_t) * MLK_KECCAK_LANES) + requires(nblocks <= 8 /* somewhat arbitrary bound */) + requires(memory_no_alias(s, sizeof(uint64_t) * MLK_KECCAK_LANES)) + requires(memory_no_alias(h, nblocks * r)) + assigns(memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES)) + assigns(memory_slice(h, nblocks * r))) +{ + while (nblocks > 0) + __loop__( + assigns(h, nblocks, + memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES), + memory_slice(h, nblocks * r)) + invariant(nblocks <= loop_entry(nblocks) && + h == loop_entry(h) + r * (loop_entry(nblocks) - nblocks)) + decreases(nblocks)) + { + mlk_keccakf1600_permute(s); + mlk_keccakf1600_extract_bytes(s, h, 0, r); + h += r; + nblocks--; + } +} + +/** + * Keccak squeeze; can be called on byte-level. + * + * @warning Must only be called once. + * + * @param[out] h Output bytes. + * @param outlen Number of bytes to be squeezed. + * @param[in,out] s Keccak state. + * @param r Rate in bytes (e.g., 168 for SHAKE128). + */ +static void mlk_keccak_squeeze_once(uint8_t *h, size_t outlen, uint64_t *s, + unsigned r) +__contract__( + requires(outlen <= MLK_MAX_BUFFER_SIZE) + requires(r > 0) + requires(r <= sizeof(uint64_t) * MLK_KECCAK_LANES) + requires(memory_no_alias(s, sizeof(uint64_t) * MLK_KECCAK_LANES)) + requires(memory_no_alias(h, outlen)) + assigns(memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES)) + assigns(memory_slice(h, outlen))) +{ + size_t len; + while (outlen > 0) + __loop__( + assigns(len, h, outlen, + memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES), + memory_slice(h, outlen)) + invariant(outlen <= loop_entry(outlen) && + h == loop_entry(h) + (loop_entry(outlen) - outlen)) + decreases(outlen)) + { + mlk_keccakf1600_permute(s); + + if (outlen < r) + { + len = outlen; + } + else + { + len = r; + } + mlk_keccakf1600_extract_bytes(s, h, 0, (unsigned int)len); + h += len; + outlen -= len; + } +} + +void mlk_shake128_absorb_once(mlk_shake128ctx *state, const uint8_t *input, + size_t inlen) +{ + mlk_keccak_absorb_once(state->ctx, SHAKE128_RATE, input, inlen, 0x1F); +} + +void mlk_shake128_squeezeblocks(uint8_t *output, size_t nblocks, + mlk_shake128ctx *state) +{ + mlk_keccak_squeezeblocks(output, nblocks, state->ctx, SHAKE128_RATE); +} + +void mlk_shake128_init(mlk_shake128ctx *state) { (void)state; } +void mlk_shake128_release(mlk_shake128ctx *state) +{ + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(state, sizeof(mlk_shake128ctx)); +} + +typedef mlk_shake128ctx mlk_shake256ctx; +void mlk_shake256(uint8_t *output, size_t outlen, const uint8_t *input, + size_t inlen) +{ + mlk_shake256ctx state; + /* Absorb input */ + mlk_keccak_absorb_once(state.ctx, SHAKE256_RATE, input, inlen, 0x1F); + /* Squeeze output */ + mlk_keccak_squeeze_once(output, outlen, state.ctx, SHAKE256_RATE); + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(&state, sizeof(state)); +} + +void mlk_sha3_256(uint8_t *output, const uint8_t *input, size_t inlen) +{ + uint64_t ctx[25]; + /* Absorb input */ + mlk_keccak_absorb_once(ctx, SHA3_256_RATE, input, inlen, 0x06); + /* Squeeze output */ + mlk_keccak_squeeze_once(output, 32, ctx, SHA3_256_RATE); + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(ctx, sizeof(ctx)); +} + +void mlk_sha3_512(uint8_t *output, const uint8_t *input, size_t inlen) +{ + uint64_t ctx[25]; + /* Absorb input */ + mlk_keccak_absorb_once(ctx, SHA3_512_RATE, input, inlen, 0x06); + /* Squeeze output */ + mlk_keccak_squeeze_once(output, 64, ctx, SHA3_512_RATE); + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(ctx, sizeof(ctx)); +} + +#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(fips202) + +#endif /* MLK_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/src/mlkem-native/mlkem/src/fips202/fips202.h b/src/mlkem-native/mlkem/src/fips202/fips202.h new file mode 100644 index 0000000000..99eaccc0fe --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/fips202.h @@ -0,0 +1,144 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_FIPS202_FIPS202_H +#define MLK_FIPS202_FIPS202_H + +#include "../cbmc.h" +#include "../common.h" + +#define SHAKE128_RATE 168 +#define SHAKE256_RATE 136 +#define SHA3_256_RATE 136 +#define SHA3_384_RATE 104 +#define SHA3_512_RATE 72 + +/** Context for the non-incremental SHAKE128 API. */ +typedef struct +{ + uint64_t ctx[25]; /**< Keccak state. */ +} MLK_ALIGN mlk_shake128ctx; + +#define mlk_shake128_absorb_once MLK_NAMESPACE(shake128_absorb_once) +/** + * One-shot absorb step of the SHAKE128 XOF. + * + * For call-sites (in mlkem-native): + * - This function MUST ONLY be called straight after mlk_shake128_init(). + * - This function MUST ONLY be called once. + * + * Consequently, for providers of custom FIPS202 code to be used with + * mlkem-native: + * - You may assume that the input context is freshly initialized via + * mlk_shake128_init(). + * - You may assume that this function is called exactly once. + * + * @param[in,out] state SHAKE128 context. + * @param[in] input Input to be absorbed into the state. + * @param inlen Length of input in bytes. + */ +void mlk_shake128_absorb_once(mlk_shake128ctx *state, const uint8_t *input, + size_t inlen) +__contract__( + requires(inlen <= MLK_MAX_BUFFER_SIZE) + requires(memory_no_alias(state, sizeof(mlk_shake128ctx))) + requires(memory_no_alias(input, inlen)) + assigns(memory_slice(state, sizeof(mlk_shake128ctx))) +); + +#define mlk_shake128_squeezeblocks MLK_NAMESPACE(shake128_squeezeblocks) +/** + * Squeeze step of SHAKE128 XOF. Squeezes full blocks of SHAKE128_RATE bytes + * each. Modifies the state. Can be called multiple times to keep squeezing, + * i.e., is incremental. + * + * @param[out] output Output blocks. + * @param nblocks Number of blocks to be squeezed (written to output). + * @param[in,out] state Keccak state. + */ +void mlk_shake128_squeezeblocks(uint8_t *output, size_t nblocks, + mlk_shake128ctx *state) +__contract__( + requires(nblocks <= 8 /* somewhat arbitrary bound */) + requires(memory_no_alias(state, sizeof(mlk_shake128ctx))) + requires(memory_no_alias(output, nblocks * SHAKE128_RATE)) + assigns(memory_slice(output, nblocks * SHAKE128_RATE), memory_slice(state, sizeof(mlk_shake128ctx))) +); + +#define mlk_shake128_init MLK_NAMESPACE(shake128_init) +void mlk_shake128_init(mlk_shake128ctx *state); + +#define mlk_shake128_release MLK_NAMESPACE(shake128_release) +void mlk_shake128_release(mlk_shake128ctx *state); + +/* One-stop SHAKE256 call. Aliasing between input and + * output is not permitted */ +#define mlk_shake256 MLK_NAMESPACE(shake256) +/** + * SHAKE256 XOF with non-incremental API. + * + * @param[out] output Output buffer. + * @param outlen Requested output length in bytes. + * @param[in] input Input buffer. + * @param inlen Length of input in bytes. + */ +void mlk_shake256(uint8_t *output, size_t outlen, const uint8_t *input, + size_t inlen) +__contract__( + requires(inlen <= MLK_MAX_BUFFER_SIZE) + requires(outlen <= MLK_MAX_BUFFER_SIZE) + requires(memory_no_alias(input, inlen)) + requires(memory_no_alias(output, outlen)) + assigns(memory_slice(output, outlen)) +); + +/* One-stop SHA3_256 call. Aliasing between input and + * output is not permitted */ +#define SHA3_256_HASHBYTES 32 +#define mlk_sha3_256 MLK_NAMESPACE(sha3_256) +/** + * SHA3-256 with non-incremental API. + * + * @param[out] output Output buffer. + * @param[in] input Input buffer. + * @param inlen Length of input in bytes. + */ +void mlk_sha3_256(uint8_t *output, const uint8_t *input, size_t inlen) +__contract__( + requires(inlen <= MLK_MAX_BUFFER_SIZE) + requires(memory_no_alias(input, inlen)) + requires(memory_no_alias(output, SHA3_256_HASHBYTES)) + assigns(memory_slice(output, SHA3_256_HASHBYTES)) +); + +/* One-stop SHA3_512 call. Aliasing between input and + * output is not permitted */ +#define SHA3_512_HASHBYTES 64 +#define mlk_sha3_512 MLK_NAMESPACE(sha3_512) +/** + * SHA3-512 with non-incremental API. + * + * @param[out] output Output buffer. + * @param[in] input Input buffer. + * @param inlen Length of input in bytes. + */ +void mlk_sha3_512(uint8_t *output, const uint8_t *input, size_t inlen) +__contract__( + requires(inlen <= MLK_MAX_BUFFER_SIZE) + requires(memory_no_alias(input, inlen)) + requires(memory_no_alias(output, SHA3_512_HASHBYTES)) + assigns(memory_slice(output, SHA3_512_HASHBYTES)) +); + +#if !defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) || \ + !defined(MLK_USE_NATIVE_FIPS202_X4) +/* If you provide your own FIPS-202 implementation where the x4- + * Keccak-f1600-x4 implementation falls back to 4-fold Keccak-f1600, + * set this to gain a small speedup. */ +#define FIPS202_X4_DEFAULT_IMPLEMENTATION +#endif /* !MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 || !MLK_USE_NATIVE_FIPS202_X4 \ + */ + + +#endif /* !MLK_FIPS202_FIPS202_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/fips202x4.c b/src/mlkem-native/mlkem/src/fips202/fips202x4.c new file mode 100644 index 0000000000..400ca07109 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/fips202x4.c @@ -0,0 +1,207 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#include "../common.h" +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "../verify.h" +#include "fips202.h" +#include "fips202x4.h" +#include "keccakf1600.h" + +typedef mlk_shake128x4ctx mlk_shake256x4_ctx; + +static void mlk_keccak_absorb_once_x4(uint64_t *s, unsigned r, + const uint8_t *in0, const uint8_t *in1, + const uint8_t *in2, const uint8_t *in3, + size_t inlen, uint8_t p) +__contract__( + requires(inlen <= MLK_MAX_BUFFER_SIZE) + requires(memory_no_alias(s, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + requires(r > 0) + requires(r <= sizeof(uint64_t) * MLK_KECCAK_LANES) + requires(memory_no_alias(in0, inlen)) + requires(memory_no_alias(in1, inlen)) + requires(memory_no_alias(in2, inlen)) + requires(memory_no_alias(in3, inlen)) + assigns(memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY))) +{ + while (inlen >= r) + __loop__( + assigns(inlen, in0, in1, in2, in3, memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + invariant(inlen <= loop_entry(inlen)) + invariant(in0 == loop_entry(in0) + (loop_entry(inlen) - inlen)) + invariant(in1 == loop_entry(in1) + (loop_entry(inlen) - inlen)) + invariant(in2 == loop_entry(in2) + (loop_entry(inlen) - inlen)) + invariant(in3 == loop_entry(in3) + (loop_entry(inlen) - inlen)) + decreases(inlen)) + { + mlk_keccakf1600x4_xor_bytes(s, in0, in1, in2, in3, 0, r); + mlk_keccakf1600x4_permute(s); + + in0 += r; + in1 += r; + in2 += r; + in3 += r; + inlen -= r; + } + + /* At this point, inlen < r, so the truncations to unsigned are safe below. */ + + if (inlen > 0) + { + mlk_keccakf1600x4_xor_bytes(s, in0, in1, in2, in3, 0, (unsigned int)inlen); + } + + if (inlen == r - 1) + { + p |= 128; + mlk_keccakf1600x4_xor_bytes(s, &p, &p, &p, &p, (unsigned int)inlen, 1); + } + else + { + mlk_keccakf1600x4_xor_bytes(s, &p, &p, &p, &p, (unsigned int)inlen, 1); + p = 128; + mlk_keccakf1600x4_xor_bytes(s, &p, &p, &p, &p, r - 1, 1); + } +} + +static void mlk_keccak_squeezeblocks_x4(uint8_t *out0, uint8_t *out1, + uint8_t *out2, uint8_t *out3, + size_t nblocks, uint64_t *s, unsigned r) +__contract__( + requires(r <= sizeof(uint64_t) * MLK_KECCAK_LANES) + requires(r == SHAKE128_RATE || r == SHAKE256_RATE) + requires(nblocks <= (MLK_MAX_BUFFER_SIZE / SHAKE256_RATE)) + requires(memory_no_alias(s, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + requires(memory_no_alias(out0, nblocks * r)) + requires(memory_no_alias(out1, nblocks * r)) + requires(memory_no_alias(out2, nblocks * r)) + requires(memory_no_alias(out3, nblocks * r)) + assigns(memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + assigns(memory_slice(out0, nblocks * r)) + assigns(memory_slice(out1, nblocks * r)) + assigns(memory_slice(out2, nblocks * r)) + assigns(memory_slice(out3, nblocks * r))) +{ + size_t current_offset = 0; + while (nblocks > 0) + __loop__( + assigns(nblocks, current_offset, + memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY), + memory_slice(out0, nblocks * r), + memory_slice(out1, nblocks * r), + memory_slice(out2, nblocks * r), + memory_slice(out3, nblocks * r)) + invariant(nblocks <= loop_entry(nblocks)) + invariant(current_offset == (loop_entry(nblocks) - nblocks) * r) + decreases(nblocks)) + { + mlk_keccakf1600x4_permute(s); + mlk_keccakf1600x4_extract_bytes( + s, &out0[current_offset], &out1[current_offset], &out2[current_offset], + &out3[current_offset], 0, r); + current_offset += r; + nblocks--; + } +} + +void mlk_shake128x4_absorb_once(mlk_shake128x4ctx *state, const uint8_t *in0, + const uint8_t *in1, const uint8_t *in2, + const uint8_t *in3, size_t inlen) +{ + mlk_memset(state, 0, sizeof(mlk_shake128x4ctx)); + mlk_keccak_absorb_once_x4(state->ctx, SHAKE128_RATE, in0, in1, in2, in3, + inlen, 0x1F); +} + +void mlk_shake128x4_squeezeblocks(uint8_t *out0, uint8_t *out1, uint8_t *out2, + uint8_t *out3, size_t nblocks, + mlk_shake128x4ctx *state) +{ + mlk_keccak_squeezeblocks_x4(out0, out1, out2, out3, nblocks, state->ctx, + SHAKE128_RATE); +} + +void mlk_shake128x4_init(mlk_shake128x4ctx *state) { (void)state; } +void mlk_shake128x4_release(mlk_shake128x4ctx *state) +{ + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(state, sizeof(mlk_shake128x4ctx)); +} + +static void mlk_shake256x4_absorb_once(mlk_shake256x4_ctx *state, + const uint8_t *in0, const uint8_t *in1, + const uint8_t *in2, const uint8_t *in3, + size_t inlen) +{ + mlk_memset(state, 0, sizeof(mlk_shake128x4ctx)); + mlk_keccak_absorb_once_x4(state->ctx, SHAKE256_RATE, in0, in1, in2, in3, + inlen, 0x1F); +} + +static void mlk_shake256x4_squeezeblocks(uint8_t *out0, uint8_t *out1, + uint8_t *out2, uint8_t *out3, + size_t nblocks, + mlk_shake256x4_ctx *state) +{ + mlk_keccak_squeezeblocks_x4(out0, out1, out2, out3, nblocks, state->ctx, + SHAKE256_RATE); +} + +void mlk_shake256x4(uint8_t *out0, uint8_t *out1, uint8_t *out2, uint8_t *out3, + size_t outlen, const uint8_t *in0, const uint8_t *in1, + const uint8_t *in2, const uint8_t *in3, size_t inlen) +{ + mlk_shake256x4_ctx statex; + size_t nblocks = outlen / SHAKE256_RATE; + uint8_t tmp0[SHAKE256_RATE]; + uint8_t tmp1[SHAKE256_RATE]; + uint8_t tmp2[SHAKE256_RATE]; + uint8_t tmp3[SHAKE256_RATE]; + + mlk_shake256x4_absorb_once(&statex, in0, in1, in2, in3, inlen); + mlk_shake256x4_squeezeblocks(out0, out1, out2, out3, nblocks, &statex); + + out0 += nblocks * SHAKE256_RATE; + out1 += nblocks * SHAKE256_RATE; + out2 += nblocks * SHAKE256_RATE; + out3 += nblocks * SHAKE256_RATE; + + outlen -= nblocks * SHAKE256_RATE; + + if (outlen) + { + mlk_shake256x4_squeezeblocks(tmp0, tmp1, tmp2, tmp3, 1, &statex); + mlk_memcpy(out0, tmp0, outlen); + mlk_memcpy(out1, tmp1, outlen); + mlk_memcpy(out2, tmp2, outlen); + mlk_memcpy(out3, tmp3, outlen); + } + + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(&statex, sizeof(statex)); + mlk_zeroize(tmp0, sizeof(tmp0)); + mlk_zeroize(tmp1, sizeof(tmp1)); + mlk_zeroize(tmp2, sizeof(tmp2)); + mlk_zeroize(tmp3, sizeof(tmp3)); +} + +#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(fips202x4) + +#endif /* MLK_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/src/mlkem-native/mlkem/src/fips202/fips202x4.h b/src/mlkem-native/mlkem/src/fips202/fips202x4.h new file mode 100644 index 0000000000..a0490a0974 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/fips202x4.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_FIPS202_FIPS202X4_H +#define MLK_FIPS202_FIPS202X4_H + + +#include "../cbmc.h" +#include "../common.h" + +#include "fips202.h" +#include "keccakf1600.h" + +/** Context for the non-incremental 4-way SHAKE128 API. */ +typedef struct +{ + uint64_t ctx[MLK_KECCAK_LANES * + MLK_KECCAK_WAY]; /**< 4-way Keccak state, stored sequentially. */ +} MLK_ALIGN mlk_shake128x4ctx; + +#define mlk_shake128x4_absorb_once MLK_NAMESPACE(shake128x4_absorb_once) +void mlk_shake128x4_absorb_once(mlk_shake128x4ctx *state, const uint8_t *in0, + const uint8_t *in1, const uint8_t *in2, + const uint8_t *in3, size_t inlen) +__contract__( + requires(inlen <= MLK_MAX_BUFFER_SIZE) + requires(memory_no_alias(state, sizeof(mlk_shake128x4ctx))) + requires(memory_no_alias(in0, inlen)) + requires(memory_no_alias(in1, inlen)) + requires(memory_no_alias(in2, inlen)) + requires(memory_no_alias(in3, inlen)) + assigns(memory_slice(state, sizeof(mlk_shake128x4ctx))) +); + +#define mlk_shake128x4_squeezeblocks MLK_NAMESPACE(shake128x4_squeezeblocks) +void mlk_shake128x4_squeezeblocks(uint8_t *out0, uint8_t *out1, uint8_t *out2, + uint8_t *out3, size_t nblocks, + mlk_shake128x4ctx *state) +__contract__( + requires(nblocks <= 8 /* somewhat arbitrary bound */) + requires(memory_no_alias(state, sizeof(mlk_shake128x4ctx))) + requires(memory_no_alias(out0, nblocks * SHAKE128_RATE)) + requires(memory_no_alias(out1, nblocks * SHAKE128_RATE)) + requires(memory_no_alias(out2, nblocks * SHAKE128_RATE)) + requires(memory_no_alias(out3, nblocks * SHAKE128_RATE)) + assigns(memory_slice(out0, nblocks * SHAKE128_RATE), + memory_slice(out1, nblocks * SHAKE128_RATE), + memory_slice(out2, nblocks * SHAKE128_RATE), + memory_slice(out3, nblocks * SHAKE128_RATE), + memory_slice(state, sizeof(mlk_shake128x4ctx))) +); + +#define mlk_shake128x4_init MLK_NAMESPACE(shake128x4_init) +void mlk_shake128x4_init(mlk_shake128x4ctx *state); + +#define mlk_shake128x4_release MLK_NAMESPACE(shake128x4_release) +void mlk_shake128x4_release(mlk_shake128x4ctx *state); + +#define mlk_shake256x4 MLK_NAMESPACE(shake256x4) +void mlk_shake256x4(uint8_t *out0, uint8_t *out1, uint8_t *out2, uint8_t *out3, + size_t outlen, const uint8_t *in0, const uint8_t *in1, + const uint8_t *in2, const uint8_t *in3, size_t inlen) +__contract__( + requires(inlen <= MLK_MAX_BUFFER_SIZE) + requires(outlen <= MLK_MAX_BUFFER_SIZE) + requires(memory_no_alias(in0, inlen)) + requires(memory_no_alias(in1, inlen)) + requires(memory_no_alias(in2, inlen)) + requires(memory_no_alias(in3, inlen)) + requires(memory_no_alias(out0, outlen)) + requires(memory_no_alias(out1, outlen)) + requires(memory_no_alias(out2, outlen)) + requires(memory_no_alias(out3, outlen)) + assigns(memory_slice(out0, outlen)) + assigns(memory_slice(out1, outlen)) + assigns(memory_slice(out2, outlen)) + assigns(memory_slice(out3, outlen)) +); + +#endif /* !MLK_FIPS202_FIPS202X4_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/keccakf1600.c b/src/mlkem-native/mlkem/src/fips202/keccakf1600.c new file mode 100644 index 0000000000..b3cec2eaba --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/keccakf1600.c @@ -0,0 +1,499 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [mupq] + * Common files for pqm4, pqm3, pqriscv + * Kannwischer, Petri, Rijneveld, Schwabe, Stoffelen + * https://github.com/mupq/mupq + * + * - [supercop] + * SUPERCOP benchmarking framework + * Daniel J. Bernstein + * http://bench.cr.yp.to/supercop.html + * + * - [tweetfips] + * 'tweetfips202' FIPS202 implementation + * Van Assche, Bernstein, Schwabe + * https://keccak.team/2015/tweetfips202.html + */ + +/* Based on the CC0 implementation from @[mupq] and the public domain + * implementation @[supercop, crypto_hash/keccakc512/simple/] + * by Ronny Van Keer, and the public domain @[tweetfips] implementation. */ + + +#include "keccakf1600.h" +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#define MLK_KECCAK_NROUNDS 24 +#define MLK_KECCAK_ROL(a, offset) (((a) << (offset)) ^ ((a) >> (64 - (offset)))) + +void mlk_keccakf1600_extract_bytes(uint64_t *state, unsigned char *data, + unsigned offset, unsigned length) +{ + unsigned i; +#if defined(MLK_SYS_LITTLE_ENDIAN) + uint8_t *state_ptr = (uint8_t *)state + offset; + for (i = 0; i < length; i++) + __loop__(invariant(i <= length) + decreases(length - i)) + { + data[i] = state_ptr[i]; + } +#else /* MLK_SYS_LITTLE_ENDIAN */ + /* Portable version */ + for (i = 0; i < length; i++) + __loop__(invariant(i <= length) + decreases(length - i)) + { + data[i] = (state[(offset + i) >> 3] >> (8 * ((offset + i) & 0x07))) & 0xFF; + } +#endif /* !MLK_SYS_LITTLE_ENDIAN */ +} + +void mlk_keccakf1600_xor_bytes(uint64_t *state, const unsigned char *data, + unsigned offset, unsigned length) +{ + unsigned i; +#if defined(MLK_SYS_LITTLE_ENDIAN) + uint8_t *state_ptr = (uint8_t *)state + offset; + for (i = 0; i < length; i++) + __loop__(invariant(i <= length) + decreases(length - i)) + { + state_ptr[i] ^= data[i]; + } +#else /* MLK_SYS_LITTLE_ENDIAN */ + /* Portable version */ + for (i = 0; i < length; i++) + __loop__(invariant(i <= length) + decreases(length - i)) + { + state[(offset + i) >> 3] ^= (uint64_t)data[i] + << (8 * ((offset + i) & 0x07)); + } +#endif /* !MLK_SYS_LITTLE_ENDIAN */ +} + +static void mlk_keccakf1600x4_extract_bytes_c(uint64_t *state, + unsigned char *data0, + unsigned char *data1, + unsigned char *data2, + unsigned char *data3, + unsigned offset, unsigned length) +__contract__( + requires(0 <= offset && offset <= MLK_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLK_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + requires(memory_no_alias(data0, length)) + requires(memory_no_alias(data1, length)) + requires(memory_no_alias(data2, length)) + requires(memory_no_alias(data3, length)) + assigns(memory_slice(data0, length)) + assigns(memory_slice(data1, length)) + assigns(memory_slice(data2, length)) + assigns(memory_slice(data3, length)) +) +{ + mlk_keccakf1600_extract_bytes(state + MLK_KECCAK_LANES * 0, data0, offset, + length); + mlk_keccakf1600_extract_bytes(state + MLK_KECCAK_LANES * 1, data1, offset, + length); + mlk_keccakf1600_extract_bytes(state + MLK_KECCAK_LANES * 2, data2, offset, + length); + mlk_keccakf1600_extract_bytes(state + MLK_KECCAK_LANES * 3, data3, offset, + length); +} + +void mlk_keccakf1600x4_extract_bytes(uint64_t *state, unsigned char *data0, + unsigned char *data1, unsigned char *data2, + unsigned char *data3, unsigned offset, + unsigned length) +{ +#if defined(MLK_USE_NATIVE_FIPS202_X4_EXTRACT_BYTES) + if (mlk_keccakf1600_extract_bytes_x4_native(state, data0, data1, data2, data3, + offset, length) == + MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_FIPS202_X4_EXTRACT_BYTES */ + mlk_keccakf1600x4_extract_bytes_c(state, data0, data1, data2, data3, offset, + length); +} + +static void mlk_keccakf1600x4_xor_bytes_c(uint64_t *state, + const unsigned char *data0, + const unsigned char *data1, + const unsigned char *data2, + const unsigned char *data3, + unsigned offset, unsigned length) +__contract__( + requires(0 <= offset && offset <= MLK_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLK_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + requires(memory_no_alias(data0, length)) + /* Case 1: all input buffers are distinct; Case 2: All input buffers are the same */ + requires((data0 == data1 && + data0 == data2 && + data0 == data3) || + (memory_no_alias(data1, length) && + memory_no_alias(data2, length) && + memory_no_alias(data3, length))) + assigns(memory_slice(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) +) +{ + mlk_keccakf1600_xor_bytes(state + MLK_KECCAK_LANES * 0, data0, offset, + length); + mlk_keccakf1600_xor_bytes(state + MLK_KECCAK_LANES * 1, data1, offset, + length); + mlk_keccakf1600_xor_bytes(state + MLK_KECCAK_LANES * 2, data2, offset, + length); + mlk_keccakf1600_xor_bytes(state + MLK_KECCAK_LANES * 3, data3, offset, + length); +} + +void mlk_keccakf1600x4_xor_bytes(uint64_t *state, const unsigned char *data0, + const unsigned char *data1, + const unsigned char *data2, + const unsigned char *data3, unsigned offset, + unsigned length) +{ +#if defined(MLK_USE_NATIVE_FIPS202_X4_XOR_BYTES) + if (mlk_keccakf1600_xor_bytes_x4_native(state, data0, data1, data2, data3, + offset, + length) == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_FIPS202_X4_XOR_BYTES */ + mlk_keccakf1600x4_xor_bytes_c(state, data0, data1, data2, data3, offset, + length); +} + +void mlk_keccakf1600x4_permute(uint64_t *state) +{ +#if defined(MLK_USE_NATIVE_FIPS202_X4) + if (mlk_keccak_f1600_x4_native(state) == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_FIPS202_X4 */ + mlk_keccakf1600_permute(state + MLK_KECCAK_LANES * 0); + mlk_keccakf1600_permute(state + MLK_KECCAK_LANES * 1); + mlk_keccakf1600_permute(state + MLK_KECCAK_LANES * 2); + mlk_keccakf1600_permute(state + MLK_KECCAK_LANES * 3); +} + +static const uint64_t mlk_KeccakF_RoundConstants[MLK_KECCAK_NROUNDS] = { + (uint64_t)0x0000000000000001ULL, (uint64_t)0x0000000000008082ULL, + (uint64_t)0x800000000000808aULL, (uint64_t)0x8000000080008000ULL, + (uint64_t)0x000000000000808bULL, (uint64_t)0x0000000080000001ULL, + (uint64_t)0x8000000080008081ULL, (uint64_t)0x8000000000008009ULL, + (uint64_t)0x000000000000008aULL, (uint64_t)0x0000000000000088ULL, + (uint64_t)0x0000000080008009ULL, (uint64_t)0x000000008000000aULL, + (uint64_t)0x000000008000808bULL, (uint64_t)0x800000000000008bULL, + (uint64_t)0x8000000000008089ULL, (uint64_t)0x8000000000008003ULL, + (uint64_t)0x8000000000008002ULL, (uint64_t)0x8000000000000080ULL, + (uint64_t)0x000000000000800aULL, (uint64_t)0x800000008000000aULL, + (uint64_t)0x8000000080008081ULL, (uint64_t)0x8000000000008080ULL, + (uint64_t)0x0000000080000001ULL, (uint64_t)0x8000000080008008ULL}; + +MLK_STATIC_TESTABLE +void mlk_keccakf1600_permute_c(uint64_t *state) +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES)) + assigns(memory_slice(state, sizeof(uint64_t) * MLK_KECCAK_LANES)) +) +{ + unsigned round; + + uint64_t Aba, Abe, Abi, Abo, Abu; + uint64_t Aga, Age, Agi, Ago, Agu; + uint64_t Aka, Ake, Aki, Ako, Aku; + uint64_t Ama, Ame, Ami, Amo, Amu; + uint64_t Asa, Ase, Asi, Aso, Asu; + uint64_t BCa, BCe, BCi, BCo, BCu; + uint64_t Da, De, Di, Do, Du; + uint64_t Eba, Ebe, Ebi, Ebo, Ebu; + uint64_t Ega, Ege, Egi, Ego, Egu; + uint64_t Eka, Eke, Eki, Eko, Eku; + uint64_t Ema, Eme, Emi, Emo, Emu; + uint64_t Esa, Ese, Esi, Eso, Esu; + + /* copyFromState(A, state) */ + Aba = state[0]; + Abe = state[1]; + Abi = state[2]; + Abo = state[3]; + Abu = state[4]; + Aga = state[5]; + Age = state[6]; + Agi = state[7]; + Ago = state[8]; + Agu = state[9]; + Aka = state[10]; + Ake = state[11]; + Aki = state[12]; + Ako = state[13]; + Aku = state[14]; + Ama = state[15]; + Ame = state[16]; + Ami = state[17]; + Amo = state[18]; + Amu = state[19]; + Asa = state[20]; + Ase = state[21]; + Asi = state[22]; + Aso = state[23]; + Asu = state[24]; + + for (round = 0; round < MLK_KECCAK_NROUNDS; round += 2) + __loop__(invariant(round <= MLK_KECCAK_NROUNDS && round % 2 == 0) + decreases(MLK_KECCAK_NROUNDS - round)) + { + /* prepareTheta */ + BCa = Aba ^ Aga ^ Aka ^ Ama ^ Asa; + BCe = Abe ^ Age ^ Ake ^ Ame ^ Ase; + BCi = Abi ^ Agi ^ Aki ^ Ami ^ Asi; + BCo = Abo ^ Ago ^ Ako ^ Amo ^ Aso; + BCu = Abu ^ Agu ^ Aku ^ Amu ^ Asu; + + /* thetaRhoPiChiIotaPrepareTheta(round, A, E) */ + Da = BCu ^ MLK_KECCAK_ROL(BCe, 1); + De = BCa ^ MLK_KECCAK_ROL(BCi, 1); + Di = BCe ^ MLK_KECCAK_ROL(BCo, 1); + Do = BCi ^ MLK_KECCAK_ROL(BCu, 1); + Du = BCo ^ MLK_KECCAK_ROL(BCa, 1); + + Aba ^= Da; + BCa = Aba; + Age ^= De; + BCe = MLK_KECCAK_ROL(Age, 44); + Aki ^= Di; + BCi = MLK_KECCAK_ROL(Aki, 43); + Amo ^= Do; + BCo = MLK_KECCAK_ROL(Amo, 21); + Asu ^= Du; + BCu = MLK_KECCAK_ROL(Asu, 14); + Eba = BCa ^ ((~BCe) & BCi); + Eba ^= (uint64_t)mlk_KeccakF_RoundConstants[round]; + Ebe = BCe ^ ((~BCi) & BCo); + Ebi = BCi ^ ((~BCo) & BCu); + Ebo = BCo ^ ((~BCu) & BCa); + Ebu = BCu ^ ((~BCa) & BCe); + + Abo ^= Do; + BCa = MLK_KECCAK_ROL(Abo, 28); + Agu ^= Du; + BCe = MLK_KECCAK_ROL(Agu, 20); + Aka ^= Da; + BCi = MLK_KECCAK_ROL(Aka, 3); + Ame ^= De; + BCo = MLK_KECCAK_ROL(Ame, 45); + Asi ^= Di; + BCu = MLK_KECCAK_ROL(Asi, 61); + Ega = BCa ^ ((~BCe) & BCi); + Ege = BCe ^ ((~BCi) & BCo); + Egi = BCi ^ ((~BCo) & BCu); + Ego = BCo ^ ((~BCu) & BCa); + Egu = BCu ^ ((~BCa) & BCe); + + Abe ^= De; + BCa = MLK_KECCAK_ROL(Abe, 1); + Agi ^= Di; + BCe = MLK_KECCAK_ROL(Agi, 6); + Ako ^= Do; + BCi = MLK_KECCAK_ROL(Ako, 25); + Amu ^= Du; + BCo = MLK_KECCAK_ROL(Amu, 8); + Asa ^= Da; + BCu = MLK_KECCAK_ROL(Asa, 18); + Eka = BCa ^ ((~BCe) & BCi); + Eke = BCe ^ ((~BCi) & BCo); + Eki = BCi ^ ((~BCo) & BCu); + Eko = BCo ^ ((~BCu) & BCa); + Eku = BCu ^ ((~BCa) & BCe); + + Abu ^= Du; + BCa = MLK_KECCAK_ROL(Abu, 27); + Aga ^= Da; + BCe = MLK_KECCAK_ROL(Aga, 36); + Ake ^= De; + BCi = MLK_KECCAK_ROL(Ake, 10); + Ami ^= Di; + BCo = MLK_KECCAK_ROL(Ami, 15); + Aso ^= Do; + BCu = MLK_KECCAK_ROL(Aso, 56); + Ema = BCa ^ ((~BCe) & BCi); + Eme = BCe ^ ((~BCi) & BCo); + Emi = BCi ^ ((~BCo) & BCu); + Emo = BCo ^ ((~BCu) & BCa); + Emu = BCu ^ ((~BCa) & BCe); + + Abi ^= Di; + BCa = MLK_KECCAK_ROL(Abi, 62); + Ago ^= Do; + BCe = MLK_KECCAK_ROL(Ago, 55); + Aku ^= Du; + BCi = MLK_KECCAK_ROL(Aku, 39); + Ama ^= Da; + BCo = MLK_KECCAK_ROL(Ama, 41); + Ase ^= De; + BCu = MLK_KECCAK_ROL(Ase, 2); + Esa = BCa ^ ((~BCe) & BCi); + Ese = BCe ^ ((~BCi) & BCo); + Esi = BCi ^ ((~BCo) & BCu); + Eso = BCo ^ ((~BCu) & BCa); + Esu = BCu ^ ((~BCa) & BCe); + + /* prepareTheta */ + BCa = Eba ^ Ega ^ Eka ^ Ema ^ Esa; + BCe = Ebe ^ Ege ^ Eke ^ Eme ^ Ese; + BCi = Ebi ^ Egi ^ Eki ^ Emi ^ Esi; + BCo = Ebo ^ Ego ^ Eko ^ Emo ^ Eso; + BCu = Ebu ^ Egu ^ Eku ^ Emu ^ Esu; + + /* thetaRhoPiChiIotaPrepareTheta(round+1, E, A) */ + Da = BCu ^ MLK_KECCAK_ROL(BCe, 1); + De = BCa ^ MLK_KECCAK_ROL(BCi, 1); + Di = BCe ^ MLK_KECCAK_ROL(BCo, 1); + Do = BCi ^ MLK_KECCAK_ROL(BCu, 1); + Du = BCo ^ MLK_KECCAK_ROL(BCa, 1); + + Eba ^= Da; + BCa = Eba; + Ege ^= De; + BCe = MLK_KECCAK_ROL(Ege, 44); + Eki ^= Di; + BCi = MLK_KECCAK_ROL(Eki, 43); + Emo ^= Do; + BCo = MLK_KECCAK_ROL(Emo, 21); + Esu ^= Du; + BCu = MLK_KECCAK_ROL(Esu, 14); + Aba = BCa ^ ((~BCe) & BCi); + Aba ^= (uint64_t)mlk_KeccakF_RoundConstants[round + 1]; + Abe = BCe ^ ((~BCi) & BCo); + Abi = BCi ^ ((~BCo) & BCu); + Abo = BCo ^ ((~BCu) & BCa); + Abu = BCu ^ ((~BCa) & BCe); + + Ebo ^= Do; + BCa = MLK_KECCAK_ROL(Ebo, 28); + Egu ^= Du; + BCe = MLK_KECCAK_ROL(Egu, 20); + Eka ^= Da; + BCi = MLK_KECCAK_ROL(Eka, 3); + Eme ^= De; + BCo = MLK_KECCAK_ROL(Eme, 45); + Esi ^= Di; + BCu = MLK_KECCAK_ROL(Esi, 61); + Aga = BCa ^ ((~BCe) & BCi); + Age = BCe ^ ((~BCi) & BCo); + Agi = BCi ^ ((~BCo) & BCu); + Ago = BCo ^ ((~BCu) & BCa); + Agu = BCu ^ ((~BCa) & BCe); + + Ebe ^= De; + BCa = MLK_KECCAK_ROL(Ebe, 1); + Egi ^= Di; + BCe = MLK_KECCAK_ROL(Egi, 6); + Eko ^= Do; + BCi = MLK_KECCAK_ROL(Eko, 25); + Emu ^= Du; + BCo = MLK_KECCAK_ROL(Emu, 8); + Esa ^= Da; + BCu = MLK_KECCAK_ROL(Esa, 18); + Aka = BCa ^ ((~BCe) & BCi); + Ake = BCe ^ ((~BCi) & BCo); + Aki = BCi ^ ((~BCo) & BCu); + Ako = BCo ^ ((~BCu) & BCa); + Aku = BCu ^ ((~BCa) & BCe); + + Ebu ^= Du; + BCa = MLK_KECCAK_ROL(Ebu, 27); + Ega ^= Da; + BCe = MLK_KECCAK_ROL(Ega, 36); + Eke ^= De; + BCi = MLK_KECCAK_ROL(Eke, 10); + Emi ^= Di; + BCo = MLK_KECCAK_ROL(Emi, 15); + Eso ^= Do; + BCu = MLK_KECCAK_ROL(Eso, 56); + Ama = BCa ^ ((~BCe) & BCi); + Ame = BCe ^ ((~BCi) & BCo); + Ami = BCi ^ ((~BCo) & BCu); + Amo = BCo ^ ((~BCu) & BCa); + Amu = BCu ^ ((~BCa) & BCe); + + Ebi ^= Di; + BCa = MLK_KECCAK_ROL(Ebi, 62); + Ego ^= Do; + BCe = MLK_KECCAK_ROL(Ego, 55); + Eku ^= Du; + BCi = MLK_KECCAK_ROL(Eku, 39); + Ema ^= Da; + BCo = MLK_KECCAK_ROL(Ema, 41); + Ese ^= De; + BCu = MLK_KECCAK_ROL(Ese, 2); + Asa = BCa ^ ((~BCe) & BCi); + Ase = BCe ^ ((~BCi) & BCo); + Asi = BCi ^ ((~BCo) & BCu); + Aso = BCo ^ ((~BCu) & BCa); + Asu = BCu ^ ((~BCa) & BCe); + } + + /* copyToState(state, A) */ + state[0] = Aba; + state[1] = Abe; + state[2] = Abi; + state[3] = Abo; + state[4] = Abu; + state[5] = Aga; + state[6] = Age; + state[7] = Agi; + state[8] = Ago; + state[9] = Agu; + state[10] = Aka; + state[11] = Ake; + state[12] = Aki; + state[13] = Ako; + state[14] = Aku; + state[15] = Ama; + state[16] = Ame; + state[17] = Ami; + state[18] = Amo; + state[19] = Amu; + state[20] = Asa; + state[21] = Ase; + state[22] = Asi; + state[23] = Aso; + state[24] = Asu; +} + +void mlk_keccakf1600_permute(uint64_t *state) +{ +#if defined(MLK_USE_NATIVE_FIPS202_X1) + if (mlk_keccak_f1600_x1_native(state) == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_FIPS202_X1 */ + mlk_keccakf1600_permute_c(state); +} + +#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(keccakf1600) + +#endif /* MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef MLK_KECCAK_NROUNDS +#undef MLK_KECCAK_ROL diff --git a/src/mlkem-native/mlkem/src/fips202/keccakf1600.h b/src/mlkem-native/mlkem/src/fips202/keccakf1600.h new file mode 100644 index 0000000000..c26c36a084 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/keccakf1600.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_FIPS202_KECCAKF1600_H +#define MLK_FIPS202_KECCAKF1600_H +#include "../cbmc.h" +#include "../common.h" + +#define MLK_KECCAK_LANES 25 +#define MLK_KECCAK_WAY 4 + +/* + * WARNING: + * The contents of this structure, including the placement + * and interleaving of Keccak lanes, are IMPLEMENTATION-DEFINED. + * The struct is only exposed here to allow its construction on the stack. + */ + +#define mlk_keccakf1600_extract_bytes MLK_NAMESPACE(keccakf1600_extract_bytes) +void mlk_keccakf1600_extract_bytes(uint64_t *state, unsigned char *data, + unsigned offset, unsigned length) +__contract__( + requires(0 <= offset && offset <= MLK_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLK_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES)) + requires(memory_no_alias(data, length)) + assigns(memory_slice(data, length)) +); + +#define mlk_keccakf1600_xor_bytes MLK_NAMESPACE(keccakf1600_xor_bytes) +void mlk_keccakf1600_xor_bytes(uint64_t *state, const unsigned char *data, + unsigned offset, unsigned length) +__contract__( + requires(0 <= offset && offset <= MLK_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLK_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES)) + requires(memory_no_alias(data, length)) + assigns(memory_slice(state, sizeof(uint64_t) * MLK_KECCAK_LANES)) +); + +#define mlk_keccakf1600x4_extract_bytes \ + MLK_NAMESPACE(keccakf1600x4_extract_bytes) +void mlk_keccakf1600x4_extract_bytes(uint64_t *state, unsigned char *data0, + unsigned char *data1, unsigned char *data2, + unsigned char *data3, unsigned offset, + unsigned length) +__contract__( + requires(0 <= offset && offset <= MLK_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLK_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + requires(memory_no_alias(data0, length)) + requires(memory_no_alias(data1, length)) + requires(memory_no_alias(data2, length)) + requires(memory_no_alias(data3, length)) + assigns(memory_slice(data0, length)) + assigns(memory_slice(data1, length)) + assigns(memory_slice(data2, length)) + assigns(memory_slice(data3, length)) +); + +#define mlk_keccakf1600x4_xor_bytes MLK_NAMESPACE(keccakf1600x4_xor_bytes) +void mlk_keccakf1600x4_xor_bytes(uint64_t *state, const unsigned char *data0, + const unsigned char *data1, + const unsigned char *data2, + const unsigned char *data3, unsigned offset, + unsigned length) +__contract__( + requires(0 <= offset && offset <= MLK_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLK_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + requires(memory_no_alias(data0, length)) + /* Case 1: all input buffers are distinct; Case 2: All input buffers are the same */ + requires((data0 == data1 && + data0 == data2 && + data0 == data3) || + (memory_no_alias(data1, length) && + memory_no_alias(data2, length) && + memory_no_alias(data3, length))) + assigns(memory_slice(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) +); + + +#define mlk_keccakf1600x4_permute MLK_NAMESPACE(keccakf1600x4_permute) +void mlk_keccakf1600x4_permute(uint64_t *state) +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + assigns(memory_slice(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) +); + +#define mlk_keccakf1600_permute MLK_NAMESPACE(keccakf1600_permute) +void mlk_keccakf1600_permute(uint64_t *state) +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES)) + assigns(memory_slice(state, sizeof(uint64_t) * MLK_KECCAK_LANES)) +); + +#endif /* !MLK_FIPS202_KECCAKF1600_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/auto.h b/src/mlkem-native/mlkem/src/fips202/native/aarch64/auto.h new file mode 100644 index 0000000000..b22bfe41cb --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/auto.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [HYBRID] + * Hybrid scalar/vector implementations of Keccak and SPHINCS+ on AArch64 + * Becker, Kannwischer + * https://eprint.iacr.org/2022/1243 + */ + +#ifndef MLK_FIPS202_NATIVE_AARCH64_AUTO_H +#define MLK_FIPS202_NATIVE_AARCH64_AUTO_H +/* Default FIPS202 assembly profile for AArch64 systems */ + +/* + * Default logic to decide which implementation to use. + * + */ + +/* + * Keccak-f1600 + * + * - On Arm-based Apple CPUs, or if MLK_SYS_AARCH64_FAST_SHA3 is set, + * we pick a pure Neon implementation. + * - Otherwise, unless MLK_SYS_AARCH64_SLOW_BARREL_SHIFTER is set, + * we use lazy-rotation scalar assembly from @[HYBRID]. + * - Otherwise, if MLK_SYS_AARCH64_SLOW_BARREL_SHIFTER is set, we + * fall back to the standard C implementation. + */ +#if defined(__ARM_FEATURE_SHA3) && \ + (defined(__APPLE__) || defined(MLK_SYS_AARCH64_FAST_SHA3)) +#include "x1_v84a.h" +#elif !defined(MLK_SYS_AARCH64_SLOW_BARREL_SHIFTER) +#include "x1_scalar.h" +#endif + +/* Batched, SIMD-based Keccak-f1600 implementations. */ +#if defined(MLK_SYS_AARCH64_NEON) + +/* + * Keccak-f1600x2/x4 + * + * The optimal implementation is highly CPU-specific; see @[HYBRID]. + * + * For now, if v8.4-A is not implemented, we fall back to Keccak-f1600. + * If v8.4-A is implemented and we are on an Apple CPU or + * MLK_SYS_AARCH64_FAST_SHA3 is set, we use a plain Neon-based + * implementation. + * Otherwise, if v8.4-A is implemented, we use a scalar/Neon/Neon hybrid. + * The reason for this distinction is that Apple CPUs (and CPUs flagged with + * MLK_SYS_AARCH64_FAST_SHA3) implement the SHA3 instructions on all SIMD + * units, while Arm CPUs prior to Cortex-X4 don't, and ordinary Neon + * instructions are still needed. + */ +#if defined(__ARM_FEATURE_SHA3) +/* + * For Apple-M cores (and CPUs flagged with MLK_SYS_AARCH64_FAST_SHA3), we + * use a plain implementation leveraging SHA3 instructions only. + */ +#if defined(__APPLE__) || defined(MLK_SYS_AARCH64_FAST_SHA3) +#include "x2_v84a.h" +#else +#include "x4_v8a_v84a_scalar.h" +#endif + +#else /* __ARM_FEATURE_SHA3 */ + +#include "x4_v8a_scalar.h" + +#endif /* !__ARM_FEATURE_SHA3 */ + +#endif /* MLK_SYS_AARCH64_NEON */ + +#endif /* !MLK_FIPS202_NATIVE_AARCH64_AUTO_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/fips202_native_aarch64.h b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/fips202_native_aarch64.h new file mode 100644 index 0000000000..08b105b212 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/fips202_native_aarch64.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_FIPS202_NATIVE_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H +#define MLK_FIPS202_NATIVE_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H + +#include "../../../../cbmc.h" +#include "../../../../common.h" + +#define mlk_keccakf1600_round_constants \ + MLK_NAMESPACE(keccakf1600_round_constants) +MLK_INTERNAL_DATA_DECLARATION const uint64_t + mlk_keccakf1600_round_constants[24]; + +#define mlk_keccak_f1600_x1_scalar_aarch64_asm \ + MLK_NAMESPACE(keccak_f1600_x1_scalar_aarch64_asm) +void mlk_keccak_f1600_x1_scalar_aarch64_asm(uint64_t state[25], + const uint64_t rc[24]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/keccak_f1600_x1_scalar_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 1)) + requires(rc == mlk_keccakf1600_round_constants) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 1)) +); + +#define mlk_keccak_f1600_x1_v84a_aarch64_asm \ + MLK_NAMESPACE(keccak_f1600_x1_v84a_aarch64_asm) +void mlk_keccak_f1600_x1_v84a_aarch64_asm(uint64_t state[25], + const uint64_t rc[24]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/keccak_f1600_x1_v84a_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 1)) + requires(rc == mlk_keccakf1600_round_constants) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 1)) +); + +#define mlk_keccak_f1600_x2_v84a_aarch64_asm \ + MLK_NAMESPACE(keccak_f1600_x2_v84a_aarch64_asm) +void mlk_keccak_f1600_x2_v84a_aarch64_asm(uint64_t state[50], + const uint64_t rc[24]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/keccak_f1600_x2_v84a_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 2)) + requires(rc == mlk_keccakf1600_round_constants) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 2)) +); + +#define mlk_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm \ + MLK_NAMESPACE(keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm) +void mlk_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm(uint64_t state[100], + const uint64_t rc[24]) +/* This must be kept in sync with the HOL-Light specification + * in + * proofs/hol_light/aarch64/proofs/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.ml + */ +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 4)) + requires(rc == mlk_keccakf1600_round_constants) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 4)) +); + +#define mlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm \ + MLK_NAMESPACE(keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm) +void mlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm( + uint64_t state[100], const uint64_t rc[24]) +/* This must be kept in sync with the HOL-Light specification + * in + * proofs/hol_light/aarch64/proofs/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.ml + */ +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 4)) + requires(rc == mlk_keccakf1600_round_constants) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 4)) +); + +#endif /* !MLK_FIPS202_NATIVE_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x1_scalar_aarch64_asm.S b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x1_scalar_aarch64_asm.S new file mode 100644 index 0000000000..108e17f83f --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x1_scalar_aarch64_asm.S @@ -0,0 +1,377 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// Author: Hanno Becker +// Author: Matthias Kannwischer + +/*yaml + Name: keccak_f1600_x1_scalar_aarch64_asm + Description: AArch64 scalar implementation of Keccak-f[1600] permutation for single state + Signature: void mlk_keccak_f1600_x1_scalar_aarch64_asm(uint64_t state[25], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 200 + permissions: read/write + c_parameter: uint64_t state[25] + description: Keccak state (25 x uint64_t) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: uint64_t const *rc + description: Round constants (24 x uint64_t) + Stack: + bytes: 128 + description: register preservation and temporary storage +*/ + +#include "../../../../common.h" +#if defined(MLK_FIPS202_AARCH64_NEED_X1_SCALAR) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/aarch64/src/keccak_f1600_x1_scalar_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x1_scalar_aarch64_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x1_scalar_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x80 + .cfi_adjust_cfa_offset 0x80 + stp x19, x20, [sp, #0x20] + .cfi_rel_offset x19, 0x20 + .cfi_rel_offset x20, 0x28 + stp x21, x22, [sp, #0x30] + .cfi_rel_offset x21, 0x30 + .cfi_rel_offset x22, 0x38 + stp x23, x24, [sp, #0x40] + .cfi_rel_offset x23, 0x40 + .cfi_rel_offset x24, 0x48 + stp x25, x26, [sp, #0x50] + .cfi_rel_offset x25, 0x50 + .cfi_rel_offset x26, 0x58 + stp x27, x28, [sp, #0x60] + .cfi_rel_offset x27, 0x60 + .cfi_rel_offset x28, 0x68 + stp x29, x30, [sp, #0x70] + .cfi_rel_offset x29, 0x70 + .cfi_rel_offset x30, 0x78 + +Lmlk_keccak_f1600_x1_scalar_initial: + mov x26, x1 + str x1, [sp, #0x8] + ldp x1, x6, [x0] + ldp x11, x16, [x0, #0x10] + ldp x21, x2, [x0, #0x20] + ldp x7, x12, [x0, #0x30] + ldp x17, x22, [x0, #0x40] + ldp x3, x8, [x0, #0x50] + ldp x13, x28, [x0, #0x60] + ldp x23, x4, [x0, #0x70] + ldp x9, x14, [x0, #0x80] + ldp x19, x24, [x0, #0x90] + ldp x5, x10, [x0, #0xa0] + ldp x15, x20, [x0, #0xb0] + ldr x25, [x0, #0xc0] + str x0, [sp] + eor x30, x24, x25 + eor x27, x9, x10 + eor x0, x30, x21 + eor x26, x27, x6 + eor x27, x26, x7 + eor x29, x0, x22 + eor x26, x29, x23 + eor x29, x4, x5 + eor x30, x29, x1 + eor x0, x27, x8 + eor x29, x30, x2 + eor x30, x19, x20 + eor x30, x30, x16 + eor x27, x26, x0, ror #63 + eor x4, x4, x27 + eor x30, x30, x17 + eor x30, x30, x28 + eor x29, x29, x3 + eor x0, x0, x30, ror #63 + eor x30, x30, x29, ror #63 + eor x22, x22, x30 + eor x23, x23, x30 + str x23, [sp, #0x18] + eor x23, x14, x15 + eor x14, x14, x0 + eor x23, x23, x11 + eor x15, x15, x0 + eor x1, x1, x27 + eor x23, x23, x12 + eor x23, x23, x13 + eor x11, x11, x0 + eor x29, x29, x23, ror #63 + eor x23, x23, x26, ror #63 + eor x26, x13, x0 + eor x13, x28, x23 + eor x28, x24, x30 + eor x24, x16, x23 + eor x16, x21, x30 + eor x21, x25, x30 + eor x30, x19, x23 + eor x19, x20, x23 + eor x20, x17, x23 + eor x17, x12, x0 + eor x0, x2, x27 + eor x2, x6, x29 + eor x6, x8, x29 + bic x8, x28, x13, ror #47 + eor x12, x3, x27 + bic x3, x13, x17, ror #19 + eor x5, x5, x27 + ldr x27, [sp, #0x18] + bic x25, x17, x2, ror #5 + eor x9, x9, x29 + eor x23, x25, x5, ror #52 + eor x3, x3, x2, ror #24 + eor x8, x8, x17, ror #2 + eor x17, x10, x29 + bic x25, x12, x22, ror #47 + eor x29, x7, x29 + bic x10, x4, x27, ror #2 + bic x7, x5, x28, ror #10 + eor x10, x10, x20, ror #50 + eor x13, x7, x13, ror #57 + bic x7, x2, x5, ror #47 + eor x2, x25, x24, ror #39 + bic x25, x20, x11, ror #57 + bic x5, x17, x4, ror #25 + eor x25, x25, x17, ror #53 + bic x17, x11, x17, ror #60 + eor x28, x7, x28, ror #57 + bic x7, x9, x12, ror #42 + eor x7, x7, x22, ror #25 + bic x22, x22, x24, ror #56 + bic x24, x24, x15, ror #31 + eor x22, x22, x15, ror #23 + bic x20, x27, x20, ror #48 + bic x15, x15, x9, ror #16 + eor x12, x15, x12, ror #58 + eor x15, x5, x27, ror #27 + eor x5, x20, x11, ror #41 + ldr x11, [sp, #0x8] + eor x20, x17, x4, ror #21 + eor x17, x24, x9, ror #47 + mov x24, #0x1 // =1 + bic x9, x0, x16, ror #9 + str x24, [sp, #0x10] + bic x24, x29, x1, ror #44 + bic x27, x1, x21, ror #50 + bic x4, x26, x29, ror #63 + eor x1, x1, x4, ror #21 + ldr x11, [x11] + bic x4, x21, x30, ror #57 + eor x21, x24, x21, ror #30 + eor x24, x9, x19, ror #44 + bic x9, x14, x6, ror #5 + eor x9, x9, x0, ror #43 + bic x0, x6, x0, ror #38 + eor x1, x1, x11 + eor x11, x4, x26, ror #35 + eor x4, x0, x16, ror #47 + bic x0, x16, x19, ror #35 + eor x16, x27, x30, ror #43 + bic x27, x30, x26, ror #42 + bic x26, x19, x14, ror #41 + eor x19, x0, x14, ror #12 + eor x14, x26, x6, ror #46 + eor x6, x27, x29, ror #41 + +Lmlk_keccak_f1600_x1_scalar_loop: + eor x0, x15, x11, ror #52 + eor x0, x0, x13, ror #48 + eor x26, x8, x9, ror #57 + eor x27, x0, x14, ror #10 + eor x29, x16, x28, ror #63 + eor x26, x26, x6, ror #51 + eor x30, x23, x22, ror #50 + eor x0, x26, x10, ror #31 + eor x29, x29, x19, ror #37 + eor x27, x27, x12, ror #5 + eor x30, x30, x24, ror #34 + eor x0, x0, x7, ror #27 + eor x26, x30, x21, ror #26 + eor x26, x26, x25, ror #15 + ror x30, x27, #0x3e + eor x30, x30, x26, ror #57 + ror x26, x26, #0x3a + eor x16, x30, x16 + eor x28, x30, x28, ror #63 + str x28, [sp, #0x18] + eor x29, x29, x17, ror #36 + eor x28, x1, x2, ror #61 + eor x19, x30, x19, ror #37 + eor x29, x29, x20, ror #2 + eor x28, x28, x4, ror #54 + eor x26, x26, x0, ror #55 + eor x28, x28, x3, ror #39 + eor x28, x28, x5, ror #25 + ror x0, x0, #0x38 + eor x0, x0, x29, ror #63 + eor x27, x28, x27, ror #61 + eor x13, x0, x13, ror #46 + eor x28, x29, x28, ror #63 + eor x29, x30, x20, ror #2 + eor x20, x26, x3, ror #39 + eor x11, x0, x11, ror #50 + eor x25, x28, x25, ror #9 + eor x3, x28, x21, ror #20 + eor x21, x26, x1 + eor x9, x27, x9, ror #49 + eor x24, x28, x24, ror #28 + eor x1, x30, x17, ror #36 + eor x14, x0, x14, ror #8 + eor x22, x28, x22, ror #44 + eor x8, x27, x8, ror #56 + eor x17, x27, x7, ror #19 + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + eor x4, x26, x4, ror #54 + eor x0, x0, x12, ror #3 + eor x28, x28, x23, ror #58 + eor x23, x26, x2, ror #61 + eor x26, x26, x5, ror #25 + eor x2, x7, x16, ror #39 + bic x7, x9, x20, ror #42 + bic x30, x15, x9, ror #16 + eor x7, x7, x22, ror #25 + eor x12, x30, x20, ror #58 + bic x20, x22, x16, ror #56 + eor x30, x27, x6, ror #43 + eor x22, x20, x15, ror #23 + bic x6, x19, x13, ror #42 + eor x6, x6, x17, ror #41 + bic x5, x13, x17, ror #63 + eor x5, x21, x5, ror #21 + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + bic x21, x21, x25, ror #50 + bic x20, x27, x4, ror #25 + bic x10, x16, x15, ror #31 + eor x16, x21, x19, ror #43 + eor x21, x17, x25, ror #30 + bic x19, x25, x19, ror #57 + ldr x25, [sp, #0x10] + eor x17, x10, x9, ror #47 + ldr x9, [sp, #0x8] + eor x15, x20, x28, ror #27 + bic x20, x4, x28, ror #2 + eor x10, x20, x1, ror #50 + bic x20, x11, x27, ror #60 + eor x20, x20, x4, ror #21 + bic x4, x28, x1, ror #48 + bic x1, x1, x11, ror #57 + ldr x28, [x9, x25, lsl #3] + ldr x9, [sp, #0x18] + add x25, x25, #0x1 + str x25, [sp, #0x10] + cmp x25, #0x17 + eor x25, x1, x27, ror #53 + bic x27, x30, x26, ror #47 + eor x1, x5, x28 + eor x5, x4, x11, ror #41 + eor x11, x19, x13, ror #35 + bic x13, x26, x24, ror #10 + eor x28, x27, x24, ror #57 + bic x27, x24, x9, ror #47 + bic x19, x23, x3, ror #9 + bic x4, x29, x14, ror #41 + eor x24, x19, x29, ror #44 + bic x29, x3, x29, ror #35 + eor x13, x13, x9, ror #57 + eor x19, x29, x14, ror #12 + bic x29, x9, x0, ror #19 + bic x14, x14, x8, ror #5 + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 + b.le Lmlk_keccak_f1600_x1_scalar_loop + ror x6, x6, #0x2b + ror x11, x11, #0x32 + ror x21, x21, #0x14 + ror x2, x2, #0x3d + ror x7, x7, #0x13 + ror x12, x12, #0x3 + ror x17, x17, #0x24 + ror x22, x22, #0x2c + ror x3, x3, #0x27 + ror x8, x8, #0x38 + ror x13, x13, #0x2e + ror x28, x28, #0x3f + ror x23, x23, #0x3a + ror x4, x4, #0x36 + ror x9, x9, #0x31 + ror x14, x14, #0x8 + ror x19, x19, #0x25 + ror x24, x24, #0x1c + ror x5, x5, #0x19 + ror x10, x10, #0x17 + ror x15, x15, #0x3e + ror x20, x20, #0x2 + ror x25, x25, #0x9 + ldr x0, [sp] + stp x1, x6, [x0] + stp x11, x16, [x0, #0x10] + stp x21, x2, [x0, #0x20] + stp x7, x12, [x0, #0x30] + stp x17, x22, [x0, #0x40] + stp x3, x8, [x0, #0x50] + stp x13, x28, [x0, #0x60] + stp x23, x4, [x0, #0x70] + stp x9, x14, [x0, #0x80] + stp x19, x24, [x0, #0x90] + stp x5, x10, [x0, #0xa0] + stp x15, x20, [x0, #0xb0] + str x25, [x0, #0xc0] + ldp x19, x20, [sp, #0x20] + .cfi_restore x19 + .cfi_restore x20 + ldp x21, x22, [sp, #0x30] + .cfi_restore x21 + .cfi_restore x22 + ldp x23, x24, [sp, #0x40] + .cfi_restore x23 + .cfi_restore x24 + ldp x25, x26, [sp, #0x50] + .cfi_restore x25 + .cfi_restore x26 + ldp x27, x28, [sp, #0x60] + .cfi_restore x27 + .cfi_restore x28 + ldp x29, x30, [sp, #0x70] + .cfi_restore x29 + .cfi_restore x30 + add sp, sp, #0x80 + .cfi_adjust_cfa_offset -0x80 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(keccak_f1600_x1_scalar_aarch64_asm) + +#endif /* MLK_FIPS202_AARCH64_NEED_X1_SCALAR && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S new file mode 100644 index 0000000000..cb27677691 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S @@ -0,0 +1,206 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [HYBRID] + * Hybrid scalar/vector implementations of Keccak and SPHINCS+ on AArch64 + * Becker, Kannwischer + * https://eprint.iacr.org/2022/1243 + */ + +/*yaml + Name: keccak_f1600_x1_v84a_aarch64_asm + Description: AArch64 ARMv8.4-A implementation of Keccak-f[1600] permutation for single state + Signature: void mlk_keccak_f1600_x1_v84a_aarch64_asm(uint64_t state[25], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON, SHA3] + x0: + type: buffer + size_bytes: 200 + permissions: read/write + c_parameter: uint64_t state[25] + description: Keccak state (25 x uint64_t) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: const uint64_t rc[24] + description: Round constants (24 x uint64_t) + Stack: + bytes: 64 + description: register preservation +*/ + +// +// Author: Hanno Becker +// Author: Matthias Kannwischer +// +// This implementation is essentially from the paper @[HYBRID]. +// The only difference is interleaving/deinterleaving of Keccak state +// during load and store, so that the caller need not do this. +// + +#include "../../../../common.h" +#if defined(MLK_FIPS202_AARCH64_NEED_X1_V84A) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#if defined(__ARM_FEATURE_SHA3) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x1_v84a_aarch64_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x1_v84a_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x40 + .cfi_adjust_cfa_offset 0x40 + stp d8, d9, [sp] + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + stp d10, d11, [sp, #0x10] + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + stp d12, d13, [sp, #0x20] + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + stp d14, d15, [sp, #0x30] + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + ldp d0, d1, [x0] + ldp d2, d3, [x0, #0x10] + ldp d4, d5, [x0, #0x20] + ldp d6, d7, [x0, #0x30] + ldp d8, d9, [x0, #0x40] + ldp d10, d11, [x0, #0x50] + ldp d12, d13, [x0, #0x60] + ldp d14, d15, [x0, #0x70] + ldp d16, d17, [x0, #0x80] + ldp d18, d19, [x0, #0x90] + ldp d20, d21, [x0, #0xa0] + ldp d22, d23, [x0, #0xb0] + ldr d24, [x0, #0xc0] + mov x2, #0x18 // =24 + +Lmlk_keccak_f1600_x1_v84a_loop: + eor3 v30.16b, v0.16b, v5.16b, v10.16b + eor3 v29.16b, v1.16b, v6.16b, v11.16b + eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor3 v27.16b, v3.16b, v8.16b, v13.16b + eor3 v26.16b, v4.16b, v9.16b, v14.16b + eor3 v30.16b, v30.16b, v15.16b, v20.16b + eor3 v29.16b, v29.16b, v16.16b, v21.16b + eor3 v28.16b, v28.16b, v17.16b, v22.16b + eor3 v27.16b, v27.16b, v18.16b, v23.16b + eor3 v26.16b, v26.16b, v19.16b, v24.16b + rax1 v25.2d, v30.2d, v28.2d + rax1 v28.2d, v28.2d, v26.2d + rax1 v26.2d, v26.2d, v29.2d + rax1 v29.2d, v29.2d, v27.2d + rax1 v27.2d, v27.2d, v30.2d + eor v30.16b, v0.16b, v26.16b + xar v0.2d, v2.2d, v29.2d, #0x2 + xar v2.2d, v12.2d, v29.2d, #0x15 + xar v12.2d, v13.2d, v28.2d, #0x27 + xar v13.2d, v19.2d, v27.2d, #0x38 + xar v19.2d, v23.2d, v28.2d, #0x8 + xar v23.2d, v15.2d, v26.2d, #0x17 + xar v15.2d, v1.2d, v25.2d, #0x3f + xar v1.2d, v8.2d, v28.2d, #0x9 + xar v8.2d, v16.2d, v25.2d, #0x13 + xar v16.2d, v7.2d, v29.2d, #0x3a + xar v7.2d, v10.2d, v26.2d, #0x3d + xar v10.2d, v3.2d, v28.2d, #0x24 + xar v3.2d, v18.2d, v28.2d, #0x2b + xar v18.2d, v17.2d, v29.2d, #0x31 + xar v17.2d, v11.2d, v25.2d, #0x36 + xar v11.2d, v9.2d, v27.2d, #0x2c + xar v9.2d, v22.2d, v29.2d, #0x3 + xar v22.2d, v14.2d, v27.2d, #0x19 + xar v14.2d, v20.2d, v26.2d, #0x2e + xar v20.2d, v4.2d, v27.2d, #0x25 + xar v4.2d, v24.2d, v27.2d, #0x32 + xar v24.2d, v21.2d, v25.2d, #0x3e + xar v21.2d, v5.2d, v26.2d, #0x1c + xar v27.2d, v6.2d, v25.2d, #0x14 + ld1r { v31.2d }, [x1], #8 + bcax v5.16b, v10.16b, v7.16b, v11.16b + bcax v6.16b, v11.16b, v8.16b, v7.16b + bcax v7.16b, v7.16b, v9.16b, v8.16b + bcax v8.16b, v8.16b, v10.16b, v9.16b + bcax v9.16b, v9.16b, v11.16b, v10.16b + bcax v10.16b, v15.16b, v12.16b, v16.16b + bcax v11.16b, v16.16b, v13.16b, v12.16b + bcax v12.16b, v12.16b, v14.16b, v13.16b + bcax v13.16b, v13.16b, v15.16b, v14.16b + bcax v14.16b, v14.16b, v16.16b, v15.16b + bcax v15.16b, v20.16b, v17.16b, v21.16b + bcax v16.16b, v21.16b, v18.16b, v17.16b + bcax v17.16b, v17.16b, v19.16b, v18.16b + bcax v18.16b, v18.16b, v20.16b, v19.16b + bcax v19.16b, v19.16b, v21.16b, v20.16b + bcax v20.16b, v0.16b, v22.16b, v1.16b + bcax v21.16b, v1.16b, v23.16b, v22.16b + bcax v22.16b, v22.16b, v24.16b, v23.16b + bcax v23.16b, v23.16b, v0.16b, v24.16b + bcax v24.16b, v24.16b, v1.16b, v0.16b + bcax v0.16b, v30.16b, v2.16b, v27.16b + bcax v1.16b, v27.16b, v3.16b, v2.16b + bcax v2.16b, v2.16b, v4.16b, v3.16b + bcax v3.16b, v3.16b, v30.16b, v4.16b + bcax v4.16b, v4.16b, v27.16b, v30.16b + eor v0.16b, v0.16b, v31.16b + sub x2, x2, #0x1 + cbnz x2, Lmlk_keccak_f1600_x1_v84a_loop + stp d0, d1, [x0] + stp d2, d3, [x0, #0x10] + stp d4, d5, [x0, #0x20] + stp d6, d7, [x0, #0x30] + stp d8, d9, [x0, #0x40] + stp d10, d11, [x0, #0x50] + stp d12, d13, [x0, #0x60] + stp d14, d15, [x0, #0x70] + stp d16, d17, [x0, #0x80] + stp d18, d19, [x0, #0x90] + stp d20, d21, [x0, #0xa0] + stp d22, d23, [x0, #0xb0] + str d24, [x0, #0xc0] + ldp d8, d9, [sp] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0x10] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0x20] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0x30] + .cfi_restore d14 + .cfi_restore d15 + add sp, sp, #0x40 + .cfi_adjust_cfa_offset -0x40 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(keccak_f1600_x1_v84a_aarch64_asm) + +#endif /* __ARM_FEATURE_SHA3 */ + +#endif /* MLK_FIPS202_AARCH64_NEED_X1_V84A && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S new file mode 100644 index 0000000000..36ebe1d32f --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S @@ -0,0 +1,261 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [HYBRID] + * Hybrid scalar/vector implementations of Keccak and SPHINCS+ on AArch64 + * Becker, Kannwischer + * https://eprint.iacr.org/2022/1243 + */ + +/*yaml + Name: keccak_f1600_x2_v84a_aarch64_asm + Description: AArch64 ARMv8.4-A implementation of Keccak-f[1600] permutation for two sequential states + Signature: void mlk_keccak_f1600_x2_v84a_aarch64_asm(uint64_t state[50], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON, SHA3] + x0: + type: buffer + size_bytes: 400 + permissions: read/write + c_parameter: uint64_t state[50] + description: Two sequential Keccak states (state0[25], state1[25]) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: const uint64_t rc[24] + description: Round constants (24 x uint64_t) + Stack: + bytes: 64 + description: register preservation +*/ + +// +// Author: Hanno Becker +// Author: Matthias Kannwischer +// +// This implementation is essentially from the paper @[HYBRID]. +// The only difference is interleaving/deinterleaving of Keccak state +// during load and store, so that the caller need not do this. +// + +#include "../../../../common.h" +#if defined(MLK_FIPS202_AARCH64_NEED_X2_V84A) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#if defined(__ARM_FEATURE_SHA3) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x2_v84a_aarch64_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x2_v84a_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x40 + .cfi_adjust_cfa_offset 0x40 + stp d8, d9, [sp] + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + stp d10, d11, [sp, #0x10] + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + stp d12, d13, [sp, #0x20] + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + stp d14, d15, [sp, #0x30] + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + add x2, x0, #0xc8 + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x2], #32 + trn1 v0.2d, v25.2d, v27.2d + trn2 v1.2d, v25.2d, v27.2d + trn1 v2.2d, v26.2d, v28.2d + trn2 v3.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x2], #32 + trn1 v4.2d, v25.2d, v27.2d + trn2 v5.2d, v25.2d, v27.2d + trn1 v6.2d, v26.2d, v28.2d + trn2 v7.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x2], #32 + trn1 v8.2d, v25.2d, v27.2d + trn2 v9.2d, v25.2d, v27.2d + trn1 v10.2d, v26.2d, v28.2d + trn2 v11.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x2], #32 + trn1 v12.2d, v25.2d, v27.2d + trn2 v13.2d, v25.2d, v27.2d + trn1 v14.2d, v26.2d, v28.2d + trn2 v15.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x2], #32 + trn1 v16.2d, v25.2d, v27.2d + trn2 v17.2d, v25.2d, v27.2d + trn1 v18.2d, v26.2d, v28.2d + trn2 v19.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x2], #32 + trn1 v20.2d, v25.2d, v27.2d + trn2 v21.2d, v25.2d, v27.2d + trn1 v22.2d, v26.2d, v28.2d + trn2 v23.2d, v26.2d, v28.2d + ldr d25, [x0] + ldr d27, [x2] + trn1 v24.2d, v25.2d, v27.2d + mov x2, #0x18 // =24 + +Lmlk_keccak_f1600_x2_v84a_loop: + eor3 v30.16b, v0.16b, v5.16b, v10.16b + eor3 v29.16b, v1.16b, v6.16b, v11.16b + eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor3 v27.16b, v3.16b, v8.16b, v13.16b + eor3 v26.16b, v4.16b, v9.16b, v14.16b + eor3 v30.16b, v30.16b, v15.16b, v20.16b + eor3 v29.16b, v29.16b, v16.16b, v21.16b + eor3 v28.16b, v28.16b, v17.16b, v22.16b + eor3 v27.16b, v27.16b, v18.16b, v23.16b + eor3 v26.16b, v26.16b, v19.16b, v24.16b + rax1 v25.2d, v30.2d, v28.2d + rax1 v28.2d, v28.2d, v26.2d + rax1 v26.2d, v26.2d, v29.2d + rax1 v29.2d, v29.2d, v27.2d + rax1 v27.2d, v27.2d, v30.2d + eor v30.16b, v0.16b, v26.16b + xar v0.2d, v2.2d, v29.2d, #0x2 + xar v2.2d, v12.2d, v29.2d, #0x15 + xar v12.2d, v13.2d, v28.2d, #0x27 + xar v13.2d, v19.2d, v27.2d, #0x38 + xar v19.2d, v23.2d, v28.2d, #0x8 + xar v23.2d, v15.2d, v26.2d, #0x17 + xar v15.2d, v1.2d, v25.2d, #0x3f + xar v1.2d, v8.2d, v28.2d, #0x9 + xar v8.2d, v16.2d, v25.2d, #0x13 + xar v16.2d, v7.2d, v29.2d, #0x3a + xar v7.2d, v10.2d, v26.2d, #0x3d + xar v10.2d, v3.2d, v28.2d, #0x24 + xar v3.2d, v18.2d, v28.2d, #0x2b + xar v18.2d, v17.2d, v29.2d, #0x31 + xar v17.2d, v11.2d, v25.2d, #0x36 + xar v11.2d, v9.2d, v27.2d, #0x2c + xar v9.2d, v22.2d, v29.2d, #0x3 + xar v22.2d, v14.2d, v27.2d, #0x19 + xar v14.2d, v20.2d, v26.2d, #0x2e + xar v20.2d, v4.2d, v27.2d, #0x25 + xar v4.2d, v24.2d, v27.2d, #0x32 + xar v24.2d, v21.2d, v25.2d, #0x3e + xar v21.2d, v5.2d, v26.2d, #0x1c + xar v27.2d, v6.2d, v25.2d, #0x14 + ld1r { v31.2d }, [x1], #8 + bcax v5.16b, v10.16b, v7.16b, v11.16b + bcax v6.16b, v11.16b, v8.16b, v7.16b + bcax v7.16b, v7.16b, v9.16b, v8.16b + bcax v8.16b, v8.16b, v10.16b, v9.16b + bcax v9.16b, v9.16b, v11.16b, v10.16b + bcax v10.16b, v15.16b, v12.16b, v16.16b + bcax v11.16b, v16.16b, v13.16b, v12.16b + bcax v12.16b, v12.16b, v14.16b, v13.16b + bcax v13.16b, v13.16b, v15.16b, v14.16b + bcax v14.16b, v14.16b, v16.16b, v15.16b + bcax v15.16b, v20.16b, v17.16b, v21.16b + bcax v16.16b, v21.16b, v18.16b, v17.16b + bcax v17.16b, v17.16b, v19.16b, v18.16b + bcax v18.16b, v18.16b, v20.16b, v19.16b + bcax v19.16b, v19.16b, v21.16b, v20.16b + bcax v20.16b, v0.16b, v22.16b, v1.16b + bcax v21.16b, v1.16b, v23.16b, v22.16b + bcax v22.16b, v22.16b, v24.16b, v23.16b + bcax v23.16b, v23.16b, v0.16b, v24.16b + bcax v24.16b, v24.16b, v1.16b, v0.16b + bcax v0.16b, v30.16b, v2.16b, v27.16b + bcax v1.16b, v27.16b, v3.16b, v2.16b + bcax v2.16b, v2.16b, v4.16b, v3.16b + bcax v3.16b, v3.16b, v30.16b, v4.16b + bcax v4.16b, v4.16b, v27.16b, v30.16b + eor v0.16b, v0.16b, v31.16b + sub x2, x2, #0x1 + cbnz x2, Lmlk_keccak_f1600_x2_v84a_loop + sub x0, x0, #0xc0 + add x2, x0, #0xc8 + trn1 v25.2d, v0.2d, v1.2d + trn1 v26.2d, v2.2d, v3.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v0.2d, v1.2d + trn2 v28.2d, v2.2d, v3.2d + st1 { v27.2d, v28.2d }, [x2], #32 + trn1 v25.2d, v4.2d, v5.2d + trn1 v26.2d, v6.2d, v7.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v4.2d, v5.2d + trn2 v28.2d, v6.2d, v7.2d + st1 { v27.2d, v28.2d }, [x2], #32 + trn1 v25.2d, v8.2d, v9.2d + trn1 v26.2d, v10.2d, v11.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v8.2d, v9.2d + trn2 v28.2d, v10.2d, v11.2d + st1 { v27.2d, v28.2d }, [x2], #32 + trn1 v25.2d, v12.2d, v13.2d + trn1 v26.2d, v14.2d, v15.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v12.2d, v13.2d + trn2 v28.2d, v14.2d, v15.2d + st1 { v27.2d, v28.2d }, [x2], #32 + trn1 v25.2d, v16.2d, v17.2d + trn1 v26.2d, v18.2d, v19.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v16.2d, v17.2d + trn2 v28.2d, v18.2d, v19.2d + st1 { v27.2d, v28.2d }, [x2], #32 + trn1 v25.2d, v20.2d, v21.2d + trn1 v26.2d, v22.2d, v23.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v20.2d, v21.2d + trn2 v28.2d, v22.2d, v23.2d + st1 { v27.2d, v28.2d }, [x2], #32 + str d24, [x0] + trn2 v25.2d, v24.2d, v24.2d + str d25, [x2] + ldp d8, d9, [sp] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0x10] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0x20] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0x30] + .cfi_restore d14 + .cfi_restore d15 + add sp, sp, #0x40 + .cfi_adjust_cfa_offset -0x40 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(keccak_f1600_x2_v84a_aarch64_asm) + +#endif /* __ARM_FEATURE_SHA3 */ + +#endif /* MLK_FIPS202_AARCH64_NEED_X2_V84A && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S new file mode 100644 index 0000000000..7f0f78b340 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S @@ -0,0 +1,1079 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// Author: Hanno Becker +// Author: Matthias Kannwischer + +/*yaml + Name: keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm + Description: AArch64 hybrid scalar/vector implementation of Keccak-f[1600] permutation for four sequential states + Signature: void mlk_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm(uint64_t state[100], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON] + x0: + type: buffer + size_bytes: 800 + permissions: read/write + c_parameter: uint64_t state[100] + description: Four sequential Keccak states (state0[25], state1[25], state2[25], state3[25]) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: const uint64_t rc[24] + description: Round constants (24 x uint64_t) + Stack: + bytes: 224 + description: register preservation and temporary storage +*/ + +#include "../../../../common.h" +#if defined(MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0xe0 + .cfi_adjust_cfa_offset 0xe0 + stp x19, x20, [sp, #0x30] + .cfi_rel_offset x19, 0x30 + .cfi_rel_offset x20, 0x38 + stp x21, x22, [sp, #0x40] + .cfi_rel_offset x21, 0x40 + .cfi_rel_offset x22, 0x48 + stp x23, x24, [sp, #0x50] + .cfi_rel_offset x23, 0x50 + .cfi_rel_offset x24, 0x58 + stp x25, x26, [sp, #0x60] + .cfi_rel_offset x25, 0x60 + .cfi_rel_offset x26, 0x68 + stp x27, x28, [sp, #0x70] + .cfi_rel_offset x27, 0x70 + .cfi_rel_offset x28, 0x78 + stp x29, x30, [sp, #0x80] + .cfi_rel_offset x29, 0x80 + .cfi_rel_offset x30, 0x88 + stp d8, d9, [sp, #0x90] + .cfi_rel_offset d8, 0x90 + .cfi_rel_offset d9, 0x98 + stp d10, d11, [sp, #0xa0] + .cfi_rel_offset d10, 0xa0 + .cfi_rel_offset d11, 0xa8 + stp d12, d13, [sp, #0xb0] + .cfi_rel_offset d12, 0xb0 + .cfi_rel_offset d13, 0xb8 + stp d14, d15, [sp, #0xc0] + .cfi_rel_offset d14, 0xc0 + .cfi_rel_offset d15, 0xc8 + mov x29, x1 + mov x30, #0x0 // =0 + str x30, [sp, #0x20] + str x29, [sp, #0x8] + str x29, [sp, #0x10] + str x0, [sp] + add x4, x0, #0xc8 + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v0.2d, v25.2d, v27.2d + trn2 v1.2d, v25.2d, v27.2d + trn1 v2.2d, v26.2d, v28.2d + trn2 v3.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v4.2d, v25.2d, v27.2d + trn2 v5.2d, v25.2d, v27.2d + trn1 v6.2d, v26.2d, v28.2d + trn2 v7.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v8.2d, v25.2d, v27.2d + trn2 v9.2d, v25.2d, v27.2d + trn1 v10.2d, v26.2d, v28.2d + trn2 v11.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v12.2d, v25.2d, v27.2d + trn2 v13.2d, v25.2d, v27.2d + trn1 v14.2d, v26.2d, v28.2d + trn2 v15.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v16.2d, v25.2d, v27.2d + trn2 v17.2d, v25.2d, v27.2d + trn1 v18.2d, v26.2d, v28.2d + trn2 v19.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v20.2d, v25.2d, v27.2d + trn2 v21.2d, v25.2d, v27.2d + trn1 v22.2d, v26.2d, v28.2d + trn2 v23.2d, v26.2d, v28.2d + ldr d25, [x0] + ldr d27, [x4] + trn1 v24.2d, v25.2d, v27.2d + sub x0, x0, #0xc0 + add x0, x0, #0x190 + ldp x1, x6, [x0] + ldp x11, x16, [x0, #0x10] + ldp x21, x2, [x0, #0x20] + ldp x7, x12, [x0, #0x30] + ldp x17, x22, [x0, #0x40] + ldp x3, x8, [x0, #0x50] + ldp x13, x28, [x0, #0x60] + ldp x23, x4, [x0, #0x70] + ldp x9, x14, [x0, #0x80] + ldp x19, x24, [x0, #0x90] + ldp x5, x10, [x0, #0xa0] + ldp x15, x20, [x0, #0xb0] + ldr x25, [x0, #0xc0] + sub x0, x0, #0x190 + +Lmlk_keccak_f1600_x4_v8a_scalar_hybrid_initial: + eor x30, x24, x25 + eor x27, x9, x10 + eor v30.16b, v0.16b, v5.16b + eor v30.16b, v30.16b, v10.16b + eor x0, x30, x21 + eor v30.16b, v30.16b, v15.16b + eor x26, x27, x6 + eor x27, x26, x7 + eor v30.16b, v30.16b, v20.16b + eor x29, x0, x22 + eor v29.16b, v1.16b, v6.16b + eor x26, x29, x23 + eor v29.16b, v29.16b, v11.16b + eor x29, x4, x5 + eor x30, x29, x1 + eor v29.16b, v29.16b, v16.16b + eor x0, x27, x8 + eor v29.16b, v29.16b, v21.16b + eor x29, x30, x2 + eor v28.16b, v2.16b, v7.16b + eor x30, x19, x20 + eor x30, x30, x16 + eor v28.16b, v28.16b, v12.16b + eor x27, x26, x0, ror #63 + eor v28.16b, v28.16b, v17.16b + eor x4, x4, x27 + eor v28.16b, v28.16b, v22.16b + eor x30, x30, x17 + eor x30, x30, x28 + eor v27.16b, v3.16b, v8.16b + eor x29, x29, x3 + eor v27.16b, v27.16b, v13.16b + eor x0, x0, x30, ror #63 + eor v27.16b, v27.16b, v18.16b + eor x30, x30, x29, ror #63 + eor x22, x22, x30 + eor v27.16b, v27.16b, v23.16b + eor x23, x23, x30 + eor v26.16b, v4.16b, v9.16b + str x23, [sp, #0xd0] + eor v26.16b, v26.16b, v14.16b + eor x23, x14, x15 + eor x14, x14, x0 + eor v26.16b, v26.16b, v19.16b + eor x23, x23, x11 + eor v26.16b, v26.16b, v24.16b + eor x15, x15, x0 + eor x1, x1, x27 + add v31.2d, v28.2d, v28.2d + eor x23, x23, x12 + sri v31.2d, v28.2d, #0x3f + eor x23, x23, x13 + eor v25.16b, v31.16b, v30.16b + eor x11, x11, x0 + eor x29, x29, x23, ror #63 + add v31.2d, v26.2d, v26.2d + eor x23, x23, x26, ror #63 + sri v31.2d, v26.2d, #0x3f + eor x26, x13, x0 + eor v28.16b, v31.16b, v28.16b + eor x13, x28, x23 + eor x28, x24, x30 + add v31.2d, v29.2d, v29.2d + eor x24, x16, x23 + sri v31.2d, v29.2d, #0x3f + eor x16, x21, x30 + eor v26.16b, v31.16b, v26.16b + eor x21, x25, x30 + eor x30, x19, x23 + add v31.2d, v27.2d, v27.2d + eor x19, x20, x23 + sri v31.2d, v27.2d, #0x3f + eor x20, x17, x23 + eor v29.16b, v31.16b, v29.16b + eor x17, x12, x0 + eor x0, x2, x27 + add v31.2d, v30.2d, v30.2d + eor x2, x6, x29 + sri v31.2d, v30.2d, #0x3f + eor x6, x8, x29 + eor v27.16b, v31.16b, v27.16b + bic x8, x28, x13, ror #47 + eor x12, x3, x27 + eor v30.16b, v0.16b, v26.16b + bic x3, x13, x17, ror #19 + eor v31.16b, v2.16b, v29.16b + eor x5, x5, x27 + ldr x27, [sp, #0xd0] + shl v0.2d, v31.2d, #0x3e + bic x25, x17, x2, ror #5 + sri v0.2d, v31.2d, #0x2 + eor x9, x9, x29 + eor v31.16b, v12.16b, v29.16b + eor x23, x25, x5, ror #52 + eor x3, x3, x2, ror #24 + shl v2.2d, v31.2d, #0x2b + eor x8, x8, x17, ror #2 + sri v2.2d, v31.2d, #0x15 + eor x17, x10, x29 + eor v31.16b, v13.16b, v28.16b + bic x25, x12, x22, ror #47 + eor x29, x7, x29 + shl v12.2d, v31.2d, #0x19 + bic x10, x4, x27, ror #2 + sri v12.2d, v31.2d, #0x27 + bic x7, x5, x28, ror #10 + eor v31.16b, v19.16b, v27.16b + eor x10, x10, x20, ror #50 + eor x13, x7, x13, ror #57 + shl v13.2d, v31.2d, #0x8 + bic x7, x2, x5, ror #47 + sri v13.2d, v31.2d, #0x38 + eor x2, x25, x24, ror #39 + eor v31.16b, v23.16b, v28.16b + bic x25, x20, x11, ror #57 + bic x5, x17, x4, ror #25 + shl v19.2d, v31.2d, #0x38 + eor x25, x25, x17, ror #53 + sri v19.2d, v31.2d, #0x8 + bic x17, x11, x17, ror #60 + eor v31.16b, v15.16b, v26.16b + eor x28, x7, x28, ror #57 + bic x7, x9, x12, ror #42 + shl v23.2d, v31.2d, #0x29 + eor x7, x7, x22, ror #25 + sri v23.2d, v31.2d, #0x17 + bic x22, x22, x24, ror #56 + bic x24, x24, x15, ror #31 + eor v31.16b, v1.16b, v25.16b + eor x22, x22, x15, ror #23 + shl v15.2d, v31.2d, #0x1 + bic x20, x27, x20, ror #48 + sri v15.2d, v31.2d, #0x3f + bic x15, x15, x9, ror #16 + eor x12, x15, x12, ror #58 + eor v31.16b, v8.16b, v28.16b + eor x15, x5, x27, ror #27 + shl v1.2d, v31.2d, #0x37 + eor x5, x20, x11, ror #41 + sri v1.2d, v31.2d, #0x9 + ldr x11, [sp, #0x8] + eor x20, x17, x4, ror #21 + eor v31.16b, v16.16b, v25.16b + eor x17, x24, x9, ror #47 + shl v8.2d, v31.2d, #0x2d + mov x24, #0x1 // =1 + sri v8.2d, v31.2d, #0x13 + bic x9, x0, x16, ror #9 + str x24, [sp, #0x18] + eor v31.16b, v7.16b, v29.16b + bic x24, x29, x1, ror #44 + shl v16.2d, v31.2d, #0x6 + bic x27, x1, x21, ror #50 + sri v16.2d, v31.2d, #0x3a + bic x4, x26, x29, ror #63 + eor x1, x1, x4, ror #21 + eor v31.16b, v10.16b, v26.16b + ldr x11, [x11] + shl v7.2d, v31.2d, #0x3 + bic x4, x21, x30, ror #57 + sri v7.2d, v31.2d, #0x3d + eor x21, x24, x21, ror #30 + eor x24, x9, x19, ror #44 + eor v31.16b, v3.16b, v28.16b + bic x9, x14, x6, ror #5 + shl v10.2d, v31.2d, #0x1c + eor x9, x9, x0, ror #43 + sri v10.2d, v31.2d, #0x24 + bic x0, x6, x0, ror #38 + eor x1, x1, x11 + eor v31.16b, v18.16b, v28.16b + eor x11, x4, x26, ror #35 + shl v3.2d, v31.2d, #0x15 + eor x4, x0, x16, ror #47 + bic x0, x16, x19, ror #35 + sri v3.2d, v31.2d, #0x2b + eor x16, x27, x30, ror #43 + eor v31.16b, v17.16b, v29.16b + bic x27, x30, x26, ror #42 + shl v18.2d, v31.2d, #0xf + bic x26, x19, x14, ror #41 + eor x19, x0, x14, ror #12 + sri v18.2d, v31.2d, #0x31 + eor x14, x26, x6, ror #46 + eor v31.16b, v11.16b, v25.16b + eor x6, x27, x29, ror #41 + shl v17.2d, v31.2d, #0xa + eor x0, x15, x11, ror #52 + eor x0, x0, x13, ror #48 + sri v17.2d, v31.2d, #0x36 + eor x26, x8, x9, ror #57 + eor v31.16b, v9.16b, v27.16b + eor x27, x0, x14, ror #10 + shl v11.2d, v31.2d, #0x14 + eor x29, x16, x28, ror #63 + eor x26, x26, x6, ror #51 + sri v11.2d, v31.2d, #0x2c + eor x30, x23, x22, ror #50 + eor v31.16b, v22.16b, v29.16b + eor x0, x26, x10, ror #31 + shl v9.2d, v31.2d, #0x3d + eor x29, x29, x19, ror #37 + eor x27, x27, x12, ror #5 + sri v9.2d, v31.2d, #0x3 + eor x30, x30, x24, ror #34 + eor v31.16b, v14.16b, v27.16b + eor x0, x0, x7, ror #27 + shl v22.2d, v31.2d, #0x27 + eor x26, x30, x21, ror #26 + eor x26, x26, x25, ror #15 + sri v22.2d, v31.2d, #0x19 + ror x30, x27, #0x3e + eor v31.16b, v20.16b, v26.16b + eor x30, x30, x26, ror #57 + ror x26, x26, #0x3a + shl v14.2d, v31.2d, #0x12 + eor x16, x30, x16 + sri v14.2d, v31.2d, #0x2e + eor x28, x30, x28, ror #63 + eor v31.16b, v4.16b, v27.16b + str x28, [sp, #0xd0] + eor x29, x29, x17, ror #36 + shl v20.2d, v31.2d, #0x1b + eor x28, x1, x2, ror #61 + sri v20.2d, v31.2d, #0x25 + eor x19, x30, x19, ror #37 + eor v31.16b, v24.16b, v27.16b + eor x29, x29, x20, ror #2 + eor x28, x28, x4, ror #54 + shl v4.2d, v31.2d, #0xe + eor x26, x26, x0, ror #55 + sri v4.2d, v31.2d, #0x32 + eor x28, x28, x3, ror #39 + eor v31.16b, v21.16b, v25.16b + eor x28, x28, x5, ror #25 + ror x0, x0, #0x38 + shl v24.2d, v31.2d, #0x2 + eor x0, x0, x29, ror #63 + sri v24.2d, v31.2d, #0x3e + eor x27, x28, x27, ror #61 + eor v31.16b, v5.16b, v26.16b + eor x13, x0, x13, ror #46 + eor x28, x29, x28, ror #63 + shl v21.2d, v31.2d, #0x24 + eor x29, x30, x20, ror #2 + sri v21.2d, v31.2d, #0x1c + eor x20, x26, x3, ror #39 + eor v31.16b, v6.16b, v25.16b + eor x11, x0, x11, ror #50 + eor x25, x28, x25, ror #9 + shl v27.2d, v31.2d, #0x2c + eor x3, x28, x21, ror #20 + sri v27.2d, v31.2d, #0x14 + eor x21, x26, x1 + eor x9, x27, x9, ror #49 + bic v31.16b, v7.16b, v11.16b + eor x24, x28, x24, ror #28 + eor v5.16b, v31.16b, v10.16b + eor x1, x30, x17, ror #36 + bic v31.16b, v8.16b, v7.16b + eor x14, x0, x14, ror #8 + eor x22, x28, x22, ror #44 + eor v6.16b, v31.16b, v11.16b + eor x8, x27, x8, ror #56 + bic v31.16b, v9.16b, v8.16b + eor x17, x27, x7, ror #19 + eor v7.16b, v31.16b, v7.16b + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + bic v31.16b, v10.16b, v9.16b + eor x4, x26, x4, ror #54 + eor v8.16b, v31.16b, v8.16b + eor x0, x0, x12, ror #3 + bic v31.16b, v11.16b, v10.16b + eor x28, x28, x23, ror #58 + eor x23, x26, x2, ror #61 + eor v9.16b, v31.16b, v9.16b + eor x26, x26, x5, ror #25 + bic v31.16b, v12.16b, v16.16b + eor x2, x7, x16, ror #39 + eor v10.16b, v31.16b, v15.16b + bic x7, x9, x20, ror #42 + bic x30, x15, x9, ror #16 + bic v31.16b, v13.16b, v12.16b + eor x7, x7, x22, ror #25 + eor v11.16b, v31.16b, v16.16b + eor x12, x30, x20, ror #58 + bic v31.16b, v14.16b, v13.16b + bic x20, x22, x16, ror #56 + eor x30, x27, x6, ror #43 + eor v12.16b, v31.16b, v12.16b + eor x22, x20, x15, ror #23 + bic v31.16b, v15.16b, v14.16b + bic x6, x19, x13, ror #42 + eor v13.16b, v31.16b, v13.16b + eor x6, x6, x17, ror #41 + bic x5, x13, x17, ror #63 + bic v31.16b, v16.16b, v15.16b + eor x5, x21, x5, ror #21 + eor v14.16b, v31.16b, v14.16b + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + bic v31.16b, v17.16b, v21.16b + bic x21, x21, x25, ror #50 + eor v15.16b, v31.16b, v20.16b + bic x20, x27, x4, ror #25 + bic v31.16b, v18.16b, v17.16b + bic x10, x16, x15, ror #31 + eor x16, x21, x19, ror #43 + eor v16.16b, v31.16b, v21.16b + eor x21, x17, x25, ror #30 + bic v31.16b, v19.16b, v18.16b + bic x19, x25, x19, ror #57 + eor v17.16b, v31.16b, v17.16b + ldr x25, [sp, #0x18] + eor x17, x10, x9, ror #47 + bic v31.16b, v20.16b, v19.16b + ldr x9, [sp, #0x8] + eor v18.16b, v31.16b, v18.16b + eor x15, x20, x28, ror #27 + bic v31.16b, v21.16b, v20.16b + bic x20, x4, x28, ror #2 + eor x10, x20, x1, ror #50 + eor v19.16b, v31.16b, v19.16b + bic x20, x11, x27, ror #60 + bic v31.16b, v22.16b, v1.16b + eor x20, x20, x4, ror #21 + eor v20.16b, v31.16b, v0.16b + bic x4, x28, x1, ror #48 + bic x1, x1, x11, ror #57 + bic v31.16b, v23.16b, v22.16b + ldr x28, [x9, x25, lsl #3] + eor v21.16b, v31.16b, v1.16b + ldr x9, [sp, #0xd0] + bic v31.16b, v24.16b, v23.16b + add x25, x25, #0x1 + str x25, [sp, #0x18] + eor v22.16b, v31.16b, v22.16b + cmp x25, #0x17 + bic v31.16b, v0.16b, v24.16b + eor x25, x1, x27, ror #53 + bic x27, x30, x26, ror #47 + eor v23.16b, v31.16b, v23.16b + eor x1, x5, x28 + bic v31.16b, v1.16b, v0.16b + eor x5, x4, x11, ror #41 + eor v24.16b, v31.16b, v24.16b + eor x11, x19, x13, ror #35 + bic x13, x26, x24, ror #10 + bic v31.16b, v2.16b, v27.16b + eor x28, x27, x24, ror #57 + eor v0.16b, v31.16b, v30.16b + bic x27, x24, x9, ror #47 + bic v31.16b, v3.16b, v2.16b + bic x19, x23, x3, ror #9 + bic x4, x29, x14, ror #41 + eor v1.16b, v31.16b, v27.16b + eor x24, x19, x29, ror #44 + bic v31.16b, v4.16b, v3.16b + bic x29, x3, x29, ror #35 + eor v2.16b, v31.16b, v2.16b + eor x13, x13, x9, ror #57 + eor x19, x29, x14, ror #12 + bic v31.16b, v30.16b, v4.16b + bic x29, x9, x0, ror #19 + eor v3.16b, v31.16b, v3.16b + bic x14, x14, x8, ror #5 + bic v31.16b, v27.16b, v30.16b + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + eor v4.16b, v31.16b, v4.16b + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 + ldr x30, [sp, #0x10] + ld1r { v28.2d }, [x30], #8 + str x30, [sp, #0x10] + eor v0.16b, v0.16b, v28.16b + +Lmlk_keccak_f1600_x4_v8a_scalar_hybrid_loop: + eor x0, x15, x11, ror #52 + eor x0, x0, x13, ror #48 + eor v30.16b, v0.16b, v5.16b + eor v30.16b, v30.16b, v10.16b + eor x26, x8, x9, ror #57 + eor v30.16b, v30.16b, v15.16b + eor x27, x0, x14, ror #10 + eor x29, x16, x28, ror #63 + eor v30.16b, v30.16b, v20.16b + eor x26, x26, x6, ror #51 + eor v29.16b, v1.16b, v6.16b + eor x30, x23, x22, ror #50 + eor v29.16b, v29.16b, v11.16b + eor x0, x26, x10, ror #31 + eor x29, x29, x19, ror #37 + eor v29.16b, v29.16b, v16.16b + eor x27, x27, x12, ror #5 + eor v29.16b, v29.16b, v21.16b + eor x30, x30, x24, ror #34 + eor x0, x0, x7, ror #27 + eor v28.16b, v2.16b, v7.16b + eor x26, x30, x21, ror #26 + eor v28.16b, v28.16b, v12.16b + eor x26, x26, x25, ror #15 + eor v28.16b, v28.16b, v17.16b + ror x30, x27, #0x3e + eor x30, x30, x26, ror #57 + eor v28.16b, v28.16b, v22.16b + ror x26, x26, #0x3a + eor v27.16b, v3.16b, v8.16b + eor x16, x30, x16 + eor v27.16b, v27.16b, v13.16b + eor x28, x30, x28, ror #63 + str x28, [sp, #0xd0] + eor v27.16b, v27.16b, v18.16b + eor x29, x29, x17, ror #36 + eor v27.16b, v27.16b, v23.16b + eor x28, x1, x2, ror #61 + eor x19, x30, x19, ror #37 + eor v26.16b, v4.16b, v9.16b + eor x29, x29, x20, ror #2 + eor v26.16b, v26.16b, v14.16b + eor x28, x28, x4, ror #54 + eor v26.16b, v26.16b, v19.16b + eor x26, x26, x0, ror #55 + eor x28, x28, x3, ror #39 + eor v26.16b, v26.16b, v24.16b + eor x28, x28, x5, ror #25 + add v31.2d, v28.2d, v28.2d + ror x0, x0, #0x38 + eor x0, x0, x29, ror #63 + sri v31.2d, v28.2d, #0x3f + eor x27, x28, x27, ror #61 + eor v25.16b, v31.16b, v30.16b + eor x13, x0, x13, ror #46 + add v31.2d, v26.2d, v26.2d + eor x28, x29, x28, ror #63 + eor x29, x30, x20, ror #2 + sri v31.2d, v26.2d, #0x3f + eor x20, x26, x3, ror #39 + eor v28.16b, v31.16b, v28.16b + eor x11, x0, x11, ror #50 + add v31.2d, v29.2d, v29.2d + eor x25, x28, x25, ror #9 + eor x3, x28, x21, ror #20 + sri v31.2d, v29.2d, #0x3f + eor x21, x26, x1 + eor v26.16b, v31.16b, v26.16b + eor x9, x27, x9, ror #49 + eor x24, x28, x24, ror #28 + add v31.2d, v27.2d, v27.2d + eor x1, x30, x17, ror #36 + sri v31.2d, v27.2d, #0x3f + eor x14, x0, x14, ror #8 + eor v29.16b, v31.16b, v29.16b + eor x22, x28, x22, ror #44 + eor x8, x27, x8, ror #56 + add v31.2d, v30.2d, v30.2d + eor x17, x27, x7, ror #19 + sri v31.2d, v30.2d, #0x3f + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + eor v27.16b, v31.16b, v27.16b + eor x4, x26, x4, ror #54 + eor v30.16b, v0.16b, v26.16b + eor x0, x0, x12, ror #3 + eor v31.16b, v2.16b, v29.16b + eor x28, x28, x23, ror #58 + eor x23, x26, x2, ror #61 + shl v0.2d, v31.2d, #0x3e + eor x26, x26, x5, ror #25 + sri v0.2d, v31.2d, #0x2 + eor x2, x7, x16, ror #39 + eor v31.16b, v12.16b, v29.16b + bic x7, x9, x20, ror #42 + bic x30, x15, x9, ror #16 + shl v2.2d, v31.2d, #0x2b + eor x7, x7, x22, ror #25 + sri v2.2d, v31.2d, #0x15 + eor x12, x30, x20, ror #58 + bic x20, x22, x16, ror #56 + eor v31.16b, v13.16b, v28.16b + eor x30, x27, x6, ror #43 + shl v12.2d, v31.2d, #0x19 + eor x22, x20, x15, ror #23 + sri v12.2d, v31.2d, #0x27 + bic x6, x19, x13, ror #42 + eor x6, x6, x17, ror #41 + eor v31.16b, v19.16b, v27.16b + bic x5, x13, x17, ror #63 + shl v13.2d, v31.2d, #0x8 + eor x5, x21, x5, ror #21 + sri v13.2d, v31.2d, #0x38 + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + eor v31.16b, v23.16b, v28.16b + bic x21, x21, x25, ror #50 + shl v19.2d, v31.2d, #0x38 + bic x20, x27, x4, ror #25 + bic x10, x16, x15, ror #31 + sri v19.2d, v31.2d, #0x8 + eor x16, x21, x19, ror #43 + eor v31.16b, v15.16b, v26.16b + eor x21, x17, x25, ror #30 + shl v23.2d, v31.2d, #0x29 + bic x19, x25, x19, ror #57 + ldr x25, [sp, #0x18] + sri v23.2d, v31.2d, #0x17 + eor x17, x10, x9, ror #47 + eor v31.16b, v1.16b, v25.16b + ldr x9, [sp, #0x8] + eor x15, x20, x28, ror #27 + shl v15.2d, v31.2d, #0x1 + bic x20, x4, x28, ror #2 + sri v15.2d, v31.2d, #0x3f + eor x10, x20, x1, ror #50 + eor v31.16b, v8.16b, v28.16b + bic x20, x11, x27, ror #60 + eor x20, x20, x4, ror #21 + shl v1.2d, v31.2d, #0x37 + bic x4, x28, x1, ror #48 + sri v1.2d, v31.2d, #0x9 + bic x1, x1, x11, ror #57 + eor v31.16b, v16.16b, v25.16b + ldr x28, [x9, x25, lsl #3] + ldr x9, [sp, #0xd0] + shl v8.2d, v31.2d, #0x2d + add x25, x25, #0x1 + sri v8.2d, v31.2d, #0x13 + str x25, [sp, #0x18] + cmp x25, #0x17 + eor v31.16b, v7.16b, v29.16b + eor x25, x1, x27, ror #53 + shl v16.2d, v31.2d, #0x6 + bic x27, x30, x26, ror #47 + sri v16.2d, v31.2d, #0x3a + eor x1, x5, x28 + eor x5, x4, x11, ror #41 + eor v31.16b, v10.16b, v26.16b + eor x11, x19, x13, ror #35 + shl v7.2d, v31.2d, #0x3 + bic x13, x26, x24, ror #10 + eor x28, x27, x24, ror #57 + sri v7.2d, v31.2d, #0x3d + bic x27, x24, x9, ror #47 + eor v31.16b, v3.16b, v28.16b + bic x19, x23, x3, ror #9 + shl v10.2d, v31.2d, #0x1c + bic x4, x29, x14, ror #41 + eor x24, x19, x29, ror #44 + sri v10.2d, v31.2d, #0x24 + bic x29, x3, x29, ror #35 + eor v31.16b, v18.16b, v28.16b + eor x13, x13, x9, ror #57 + shl v3.2d, v31.2d, #0x15 + eor x19, x29, x14, ror #12 + bic x29, x9, x0, ror #19 + sri v3.2d, v31.2d, #0x2b + bic x14, x14, x8, ror #5 + eor v31.16b, v17.16b, v29.16b + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + shl v18.2d, v31.2d, #0xf + bic x23, x8, x23, ror #38 + sri v18.2d, v31.2d, #0x31 + eor x8, x27, x0, ror #2 + eor v31.16b, v11.16b, v25.16b + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + shl v17.2d, v31.2d, #0xa + eor x23, x3, x26, ror #52 + sri v17.2d, v31.2d, #0x36 + eor x3, x29, x30, ror #24 + eor x0, x15, x11, ror #52 + eor v31.16b, v9.16b, v27.16b + eor x0, x0, x13, ror #48 + shl v11.2d, v31.2d, #0x14 + eor x26, x8, x9, ror #57 + sri v11.2d, v31.2d, #0x2c + eor x27, x0, x14, ror #10 + eor x29, x16, x28, ror #63 + eor v31.16b, v22.16b, v29.16b + eor x26, x26, x6, ror #51 + shl v9.2d, v31.2d, #0x3d + eor x30, x23, x22, ror #50 + sri v9.2d, v31.2d, #0x3 + eor x0, x26, x10, ror #31 + eor x29, x29, x19, ror #37 + eor v31.16b, v14.16b, v27.16b + eor x27, x27, x12, ror #5 + shl v22.2d, v31.2d, #0x27 + eor x30, x30, x24, ror #34 + eor x0, x0, x7, ror #27 + sri v22.2d, v31.2d, #0x19 + eor x26, x30, x21, ror #26 + eor v31.16b, v20.16b, v26.16b + eor x26, x26, x25, ror #15 + shl v14.2d, v31.2d, #0x12 + ror x30, x27, #0x3e + eor x30, x30, x26, ror #57 + sri v14.2d, v31.2d, #0x2e + ror x26, x26, #0x3a + eor v31.16b, v4.16b, v27.16b + eor x16, x30, x16 + shl v20.2d, v31.2d, #0x1b + eor x28, x30, x28, ror #63 + str x28, [sp, #0xd0] + sri v20.2d, v31.2d, #0x25 + eor x29, x29, x17, ror #36 + eor v31.16b, v24.16b, v27.16b + eor x28, x1, x2, ror #61 + eor x19, x30, x19, ror #37 + shl v4.2d, v31.2d, #0xe + eor x29, x29, x20, ror #2 + sri v4.2d, v31.2d, #0x32 + eor x28, x28, x4, ror #54 + eor v31.16b, v21.16b, v25.16b + eor x26, x26, x0, ror #55 + eor x28, x28, x3, ror #39 + shl v24.2d, v31.2d, #0x2 + eor x28, x28, x5, ror #25 + sri v24.2d, v31.2d, #0x3e + ror x0, x0, #0x38 + eor x0, x0, x29, ror #63 + eor v31.16b, v5.16b, v26.16b + eor x27, x28, x27, ror #61 + shl v21.2d, v31.2d, #0x24 + eor x13, x0, x13, ror #46 + sri v21.2d, v31.2d, #0x1c + eor x28, x29, x28, ror #63 + eor x29, x30, x20, ror #2 + eor v31.16b, v6.16b, v25.16b + eor x20, x26, x3, ror #39 + shl v27.2d, v31.2d, #0x2c + eor x11, x0, x11, ror #50 + sri v27.2d, v31.2d, #0x14 + eor x25, x28, x25, ror #9 + eor x3, x28, x21, ror #20 + bic v31.16b, v7.16b, v11.16b + eor x21, x26, x1 + eor v5.16b, v31.16b, v10.16b + eor x9, x27, x9, ror #49 + eor x24, x28, x24, ror #28 + bic v31.16b, v8.16b, v7.16b + eor x1, x30, x17, ror #36 + eor v6.16b, v31.16b, v11.16b + eor x14, x0, x14, ror #8 + bic v31.16b, v9.16b, v8.16b + eor x22, x28, x22, ror #44 + eor x8, x27, x8, ror #56 + eor v7.16b, v31.16b, v7.16b + eor x17, x27, x7, ror #19 + bic v31.16b, v10.16b, v9.16b + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + eor v8.16b, v31.16b, v8.16b + eor x4, x26, x4, ror #54 + bic v31.16b, v11.16b, v10.16b + eor x0, x0, x12, ror #3 + eor v9.16b, v31.16b, v9.16b + eor x28, x28, x23, ror #58 + eor x23, x26, x2, ror #61 + bic v31.16b, v12.16b, v16.16b + eor x26, x26, x5, ror #25 + eor v10.16b, v31.16b, v15.16b + eor x2, x7, x16, ror #39 + bic v31.16b, v13.16b, v12.16b + bic x7, x9, x20, ror #42 + bic x30, x15, x9, ror #16 + eor v11.16b, v31.16b, v16.16b + eor x7, x7, x22, ror #25 + bic v31.16b, v14.16b, v13.16b + eor x12, x30, x20, ror #58 + bic x20, x22, x16, ror #56 + eor v12.16b, v31.16b, v12.16b + eor x30, x27, x6, ror #43 + bic v31.16b, v15.16b, v14.16b + eor x22, x20, x15, ror #23 + eor v13.16b, v31.16b, v13.16b + bic x6, x19, x13, ror #42 + eor x6, x6, x17, ror #41 + bic v31.16b, v16.16b, v15.16b + bic x5, x13, x17, ror #63 + eor v14.16b, v31.16b, v14.16b + eor x5, x21, x5, ror #21 + bic v31.16b, v17.16b, v21.16b + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + eor v15.16b, v31.16b, v20.16b + bic x21, x21, x25, ror #50 + bic v31.16b, v18.16b, v17.16b + bic x20, x27, x4, ror #25 + bic x10, x16, x15, ror #31 + eor v16.16b, v31.16b, v21.16b + eor x16, x21, x19, ror #43 + bic v31.16b, v19.16b, v18.16b + eor x21, x17, x25, ror #30 + eor v17.16b, v31.16b, v17.16b + bic x19, x25, x19, ror #57 + ldr x25, [sp, #0x18] + bic v31.16b, v20.16b, v19.16b + eor x17, x10, x9, ror #47 + eor v18.16b, v31.16b, v18.16b + ldr x9, [sp, #0x8] + eor x15, x20, x28, ror #27 + bic v31.16b, v21.16b, v20.16b + bic x20, x4, x28, ror #2 + eor v19.16b, v31.16b, v19.16b + eor x10, x20, x1, ror #50 + bic v31.16b, v22.16b, v1.16b + bic x20, x11, x27, ror #60 + eor x20, x20, x4, ror #21 + eor v20.16b, v31.16b, v0.16b + bic x4, x28, x1, ror #48 + bic v31.16b, v23.16b, v22.16b + bic x1, x1, x11, ror #57 + eor v21.16b, v31.16b, v1.16b + ldr x28, [x9, x25, lsl #3] + ldr x9, [sp, #0xd0] + bic v31.16b, v24.16b, v23.16b + add x25, x25, #0x1 + eor v22.16b, v31.16b, v22.16b + str x25, [sp, #0x18] + cmp x25, #0x17 + bic v31.16b, v0.16b, v24.16b + eor x25, x1, x27, ror #53 + eor v23.16b, v31.16b, v23.16b + bic x27, x30, x26, ror #47 + bic v31.16b, v1.16b, v0.16b + eor x1, x5, x28 + eor x5, x4, x11, ror #41 + eor v24.16b, v31.16b, v24.16b + eor x11, x19, x13, ror #35 + bic v31.16b, v2.16b, v27.16b + bic x13, x26, x24, ror #10 + eor x28, x27, x24, ror #57 + eor v0.16b, v31.16b, v30.16b + bic x27, x24, x9, ror #47 + bic v31.16b, v3.16b, v2.16b + bic x19, x23, x3, ror #9 + eor v1.16b, v31.16b, v27.16b + bic x4, x29, x14, ror #41 + eor x24, x19, x29, ror #44 + bic v31.16b, v4.16b, v3.16b + bic x29, x3, x29, ror #35 + eor v2.16b, v31.16b, v2.16b + eor x13, x13, x9, ror #57 + bic v31.16b, v30.16b, v4.16b + eor x19, x29, x14, ror #12 + bic x29, x9, x0, ror #19 + eor v3.16b, v31.16b, v3.16b + bic x14, x14, x8, ror #5 + bic v31.16b, v27.16b, v30.16b + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + eor v4.16b, v31.16b, v4.16b + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 + ldr x30, [sp, #0x10] + ld1r { v28.2d }, [x30], #8 + str x30, [sp, #0x10] + eor v0.16b, v0.16b, v28.16b + +Lmlk_keccak_f1600_x4_v8a_scalar_hybrid_loop_end: + b.le Lmlk_keccak_f1600_x4_v8a_scalar_hybrid_loop + ror x2, x2, #0x3d + ror x3, x3, #0x27 + ror x4, x4, #0x36 + ror x5, x5, #0x19 + ror x6, x6, #0x2b + ror x7, x7, #0x13 + ror x8, x8, #0x38 + ror x9, x9, #0x31 + ror x10, x10, #0x17 + ror x11, x11, #0x32 + ror x12, x12, #0x3 + ror x13, x13, #0x2e + ror x14, x14, #0x8 + ror x15, x15, #0x3e + ror x17, x17, #0x24 + ror x28, x28, #0x3f + ror x19, x19, #0x25 + ror x20, x20, #0x2 + ror x21, x21, #0x14 + ror x22, x22, #0x2c + ror x23, x23, #0x3a + ror x24, x24, #0x1c + ror x25, x25, #0x9 + ldr x30, [sp, #0x20] + cmp x30, #0x1 + b.eq Lmlk_keccak_f1600_x4_v8a_scalar_hybrid_done + mov x30, #0x1 // =1 + str x30, [sp, #0x20] + ldr x0, [sp] + add x0, x0, #0x190 + stp x1, x6, [x0] + stp x11, x16, [x0, #0x10] + stp x21, x2, [x0, #0x20] + stp x7, x12, [x0, #0x30] + stp x17, x22, [x0, #0x40] + stp x3, x8, [x0, #0x50] + stp x13, x28, [x0, #0x60] + stp x23, x4, [x0, #0x70] + stp x9, x14, [x0, #0x80] + stp x19, x24, [x0, #0x90] + stp x5, x10, [x0, #0xa0] + stp x15, x20, [x0, #0xb0] + str x25, [x0, #0xc0] + sub x0, x0, #0x190 + add x0, x0, #0x258 + ldp x1, x6, [x0] + ldp x11, x16, [x0, #0x10] + ldp x21, x2, [x0, #0x20] + ldp x7, x12, [x0, #0x30] + ldp x17, x22, [x0, #0x40] + ldp x3, x8, [x0, #0x50] + ldp x13, x28, [x0, #0x60] + ldp x23, x4, [x0, #0x70] + ldp x9, x14, [x0, #0x80] + ldp x19, x24, [x0, #0x90] + ldp x5, x10, [x0, #0xa0] + ldp x15, x20, [x0, #0xb0] + ldr x25, [x0, #0xc0] + sub x0, x0, #0x258 + b Lmlk_keccak_f1600_x4_v8a_scalar_hybrid_initial + +Lmlk_keccak_f1600_x4_v8a_scalar_hybrid_done: + ldr x0, [sp] + add x0, x0, #0x258 + stp x1, x6, [x0] + stp x11, x16, [x0, #0x10] + stp x21, x2, [x0, #0x20] + stp x7, x12, [x0, #0x30] + stp x17, x22, [x0, #0x40] + stp x3, x8, [x0, #0x50] + stp x13, x28, [x0, #0x60] + stp x23, x4, [x0, #0x70] + stp x9, x14, [x0, #0x80] + stp x19, x24, [x0, #0x90] + stp x5, x10, [x0, #0xa0] + stp x15, x20, [x0, #0xb0] + str x25, [x0, #0xc0] + sub x0, x0, #0x258 + add x4, x0, #0xc8 + trn1 v25.2d, v0.2d, v1.2d + trn1 v26.2d, v2.2d, v3.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v0.2d, v1.2d + trn2 v28.2d, v2.2d, v3.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v4.2d, v5.2d + trn1 v26.2d, v6.2d, v7.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v4.2d, v5.2d + trn2 v28.2d, v6.2d, v7.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v8.2d, v9.2d + trn1 v26.2d, v10.2d, v11.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v8.2d, v9.2d + trn2 v28.2d, v10.2d, v11.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v12.2d, v13.2d + trn1 v26.2d, v14.2d, v15.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v12.2d, v13.2d + trn2 v28.2d, v14.2d, v15.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v16.2d, v17.2d + trn1 v26.2d, v18.2d, v19.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v16.2d, v17.2d + trn2 v28.2d, v18.2d, v19.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v20.2d, v21.2d + trn1 v26.2d, v22.2d, v23.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v20.2d, v21.2d + trn2 v28.2d, v22.2d, v23.2d + st1 { v27.2d, v28.2d }, [x4], #32 + str d24, [x0] + trn2 v25.2d, v24.2d, v24.2d + str d25, [x4] + ldp d8, d9, [sp, #0x90] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0xa0] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0xb0] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0xc0] + .cfi_restore d14 + .cfi_restore d15 + ldp x19, x20, [sp, #0x30] + .cfi_restore x19 + .cfi_restore x20 + ldp x21, x22, [sp, #0x40] + .cfi_restore x21 + .cfi_restore x22 + ldp x23, x24, [sp, #0x50] + .cfi_restore x23 + .cfi_restore x24 + ldp x25, x26, [sp, #0x60] + .cfi_restore x25 + .cfi_restore x26 + ldp x27, x28, [sp, #0x70] + .cfi_restore x27 + .cfi_restore x28 + ldp x29, x30, [sp, #0x80] + .cfi_restore x29 + .cfi_restore x30 + add sp, sp, #0xe0 + .cfi_adjust_cfa_offset -0xe0 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm) + +#endif /* MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S new file mode 100644 index 0000000000..c17267c605 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S @@ -0,0 +1,989 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// Author: Hanno Becker +// Author: Matthias Kannwischer + +/*yaml + Name: keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm + Description: AArch64 hybrid scalar/vector implementation of Keccak-f[1600] permutation for four sequential states with ARMv8.4-A optimizations + Signature: void mlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm(uint64_t state[100], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON, SHA3] + x0: + type: buffer + size_bytes: 800 + permissions: read/write + c_parameter: uint64_t state[100] + description: Four sequential Keccak states (state0[25], state1[25], state2[25], state3[25]) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: const uint64_t rc[24] + description: Round constants (24 x uint64_t) + Stack: + bytes: 224 + description: register preservation and temporary storage +*/ + +#include "../../../../common.h" +#if defined(MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#if defined(__ARM_FEATURE_SHA3) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0xe0 + .cfi_adjust_cfa_offset 0xe0 + stp x19, x20, [sp, #0x30] + .cfi_rel_offset x19, 0x30 + .cfi_rel_offset x20, 0x38 + stp x21, x22, [sp, #0x40] + .cfi_rel_offset x21, 0x40 + .cfi_rel_offset x22, 0x48 + stp x23, x24, [sp, #0x50] + .cfi_rel_offset x23, 0x50 + .cfi_rel_offset x24, 0x58 + stp x25, x26, [sp, #0x60] + .cfi_rel_offset x25, 0x60 + .cfi_rel_offset x26, 0x68 + stp x27, x28, [sp, #0x70] + .cfi_rel_offset x27, 0x70 + .cfi_rel_offset x28, 0x78 + stp x29, x30, [sp, #0x80] + .cfi_rel_offset x29, 0x80 + .cfi_rel_offset x30, 0x88 + stp d8, d9, [sp, #0x90] + .cfi_rel_offset d8, 0x90 + .cfi_rel_offset d9, 0x98 + stp d10, d11, [sp, #0xa0] + .cfi_rel_offset d10, 0xa0 + .cfi_rel_offset d11, 0xa8 + stp d12, d13, [sp, #0xb0] + .cfi_rel_offset d12, 0xb0 + .cfi_rel_offset d13, 0xb8 + stp d14, d15, [sp, #0xc0] + .cfi_rel_offset d14, 0xc0 + .cfi_rel_offset d15, 0xc8 + mov x29, x1 + mov x30, #0x0 // =0 + str x30, [sp, #0x20] + str x29, [sp, #0x8] + str x29, [sp, #0x10] + str x0, [sp] + add x4, x0, #0xc8 + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v0.2d, v25.2d, v27.2d + trn2 v1.2d, v25.2d, v27.2d + trn1 v2.2d, v26.2d, v28.2d + trn2 v3.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v4.2d, v25.2d, v27.2d + trn2 v5.2d, v25.2d, v27.2d + trn1 v6.2d, v26.2d, v28.2d + trn2 v7.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v8.2d, v25.2d, v27.2d + trn2 v9.2d, v25.2d, v27.2d + trn1 v10.2d, v26.2d, v28.2d + trn2 v11.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v12.2d, v25.2d, v27.2d + trn2 v13.2d, v25.2d, v27.2d + trn1 v14.2d, v26.2d, v28.2d + trn2 v15.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v16.2d, v25.2d, v27.2d + trn2 v17.2d, v25.2d, v27.2d + trn1 v18.2d, v26.2d, v28.2d + trn2 v19.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v20.2d, v25.2d, v27.2d + trn2 v21.2d, v25.2d, v27.2d + trn1 v22.2d, v26.2d, v28.2d + trn2 v23.2d, v26.2d, v28.2d + ldr d25, [x0] + ldr d27, [x4] + trn1 v24.2d, v25.2d, v27.2d + sub x0, x0, #0xc0 + add x0, x0, #0x190 + ldp x1, x6, [x0] + ldp x11, x16, [x0, #0x10] + ldp x21, x2, [x0, #0x20] + ldp x7, x12, [x0, #0x30] + ldp x17, x22, [x0, #0x40] + ldp x3, x8, [x0, #0x50] + ldp x13, x28, [x0, #0x60] + ldp x23, x4, [x0, #0x70] + ldp x9, x14, [x0, #0x80] + ldp x19, x24, [x0, #0x90] + ldp x5, x10, [x0, #0xa0] + ldp x15, x20, [x0, #0xb0] + ldr x25, [x0, #0xc0] + sub x0, x0, #0x190 + +Lmlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_initial: + eor x30, x24, x25 + eor x27, x9, x10 + eor3 v30.16b, v0.16b, v5.16b, v10.16b + eor v30.16b, v30.16b, v15.16b + eor x0, x30, x21 + eor x26, x27, x6 + eor v30.16b, v30.16b, v20.16b + eor x27, x26, x7 + eor x29, x0, x22 + eor3 v29.16b, v1.16b, v6.16b, v11.16b + eor x26, x29, x23 + eor x29, x4, x5 + eor v29.16b, v29.16b, v16.16b + eor x30, x29, x1 + eor x0, x27, x8 + eor v29.16b, v29.16b, v21.16b + eor x29, x30, x2 + eor x30, x19, x20 + eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor x30, x30, x16 + eor x27, x26, x0, ror #63 + eor v28.16b, v28.16b, v17.16b + eor x4, x4, x27 + eor x30, x30, x17 + eor v28.16b, v28.16b, v22.16b + eor x30, x30, x28 + eor x29, x29, x3 + eor3 v27.16b, v3.16b, v8.16b, v13.16b + eor x0, x0, x30, ror #63 + eor x30, x30, x29, ror #63 + eor v27.16b, v27.16b, v18.16b + eor x22, x22, x30 + eor v27.16b, v27.16b, v23.16b + eor x23, x23, x30 + str x23, [sp, #0xd0] + eor3 v26.16b, v4.16b, v9.16b, v14.16b + eor x23, x14, x15 + eor x14, x14, x0 + eor v26.16b, v26.16b, v19.16b + eor x23, x23, x11 + eor x15, x15, x0 + eor v26.16b, v26.16b, v24.16b + eor x1, x1, x27 + eor x23, x23, x12 + rax1 v25.2d, v30.2d, v28.2d + eor x23, x23, x13 + eor x11, x11, x0 + add v31.2d, v26.2d, v26.2d + eor x29, x29, x23, ror #63 + eor x23, x23, x26, ror #63 + sri v31.2d, v26.2d, #0x3f + eor x26, x13, x0 + eor x13, x28, x23 + eor v28.16b, v31.16b, v28.16b + eor x28, x24, x30 + eor x24, x16, x23 + rax1 v26.2d, v26.2d, v29.2d + eor x16, x21, x30 + eor x21, x25, x30 + add v31.2d, v27.2d, v27.2d + eor x30, x19, x23 + sri v31.2d, v27.2d, #0x3f + eor x19, x20, x23 + eor x20, x17, x23 + eor v29.16b, v31.16b, v29.16b + eor x17, x12, x0 + eor x0, x2, x27 + rax1 v27.2d, v27.2d, v30.2d + eor x2, x6, x29 + eor x6, x8, x29 + eor v30.16b, v0.16b, v26.16b + bic x8, x28, x13, ror #47 + eor x12, x3, x27 + eor v31.16b, v2.16b, v29.16b + bic x3, x13, x17, ror #19 + eor x5, x5, x27 + shl v0.2d, v31.2d, #0x3e + ldr x27, [sp, #0xd0] + bic x25, x17, x2, ror #5 + sri v0.2d, v31.2d, #0x2 + eor x9, x9, x29 + eor x23, x25, x5, ror #52 + xar v2.2d, v12.2d, v29.2d, #0x15 + eor x3, x3, x2, ror #24 + eor x8, x8, x17, ror #2 + eor v31.16b, v13.16b, v28.16b + eor x17, x10, x29 + bic x25, x12, x22, ror #47 + shl v12.2d, v31.2d, #0x19 + eor x29, x7, x29 + bic x10, x4, x27, ror #2 + sri v12.2d, v31.2d, #0x27 + bic x7, x5, x28, ror #10 + xar v13.2d, v19.2d, v27.2d, #0x38 + eor x10, x10, x20, ror #50 + eor x13, x7, x13, ror #57 + eor v31.16b, v23.16b, v28.16b + bic x7, x2, x5, ror #47 + eor x2, x25, x24, ror #39 + shl v19.2d, v31.2d, #0x38 + bic x25, x20, x11, ror #57 + bic x5, x17, x4, ror #25 + sri v19.2d, v31.2d, #0x8 + eor x25, x25, x17, ror #53 + bic x17, x11, x17, ror #60 + xar v23.2d, v15.2d, v26.2d, #0x17 + eor x28, x7, x28, ror #57 + bic x7, x9, x12, ror #42 + eor v31.16b, v1.16b, v25.16b + eor x7, x7, x22, ror #25 + bic x22, x22, x24, ror #56 + shl v15.2d, v31.2d, #0x1 + bic x24, x24, x15, ror #31 + eor x22, x22, x15, ror #23 + sri v15.2d, v31.2d, #0x3f + bic x20, x27, x20, ror #48 + bic x15, x15, x9, ror #16 + xar v1.2d, v8.2d, v28.2d, #0x9 + eor x12, x15, x12, ror #58 + eor x15, x5, x27, ror #27 + eor v31.16b, v16.16b, v25.16b + eor x5, x20, x11, ror #41 + shl v8.2d, v31.2d, #0x2d + ldr x11, [sp, #0x8] + eor x20, x17, x4, ror #21 + sri v8.2d, v31.2d, #0x13 + eor x17, x24, x9, ror #47 + mov x24, #0x1 // =1 + xar v16.2d, v7.2d, v29.2d, #0x3a + bic x9, x0, x16, ror #9 + str x24, [sp, #0x18] + eor v31.16b, v10.16b, v26.16b + bic x24, x29, x1, ror #44 + bic x27, x1, x21, ror #50 + shl v7.2d, v31.2d, #0x3 + bic x4, x26, x29, ror #63 + eor x1, x1, x4, ror #21 + sri v7.2d, v31.2d, #0x3d + ldr x11, [x11] + bic x4, x21, x30, ror #57 + xar v10.2d, v3.2d, v28.2d, #0x24 + eor x21, x24, x21, ror #30 + eor x24, x9, x19, ror #44 + eor v31.16b, v18.16b, v28.16b + bic x9, x14, x6, ror #5 + eor x9, x9, x0, ror #43 + shl v3.2d, v31.2d, #0x15 + bic x0, x6, x0, ror #38 + eor x1, x1, x11 + sri v3.2d, v31.2d, #0x2b + eor x11, x4, x26, ror #35 + eor x4, x0, x16, ror #47 + xar v18.2d, v17.2d, v29.2d, #0x31 + bic x0, x16, x19, ror #35 + eor v31.16b, v11.16b, v25.16b + eor x16, x27, x30, ror #43 + bic x27, x30, x26, ror #42 + shl v17.2d, v31.2d, #0xa + bic x26, x19, x14, ror #41 + eor x19, x0, x14, ror #12 + sri v17.2d, v31.2d, #0x36 + eor x14, x26, x6, ror #46 + eor x6, x27, x29, ror #41 + xar v11.2d, v9.2d, v27.2d, #0x2c + eor x0, x15, x11, ror #52 + eor x0, x0, x13, ror #48 + eor v31.16b, v22.16b, v29.16b + eor x26, x8, x9, ror #57 + eor x27, x0, x14, ror #10 + shl v9.2d, v31.2d, #0x3d + eor x29, x16, x28, ror #63 + eor x26, x26, x6, ror #51 + sri v9.2d, v31.2d, #0x3 + eor x30, x23, x22, ror #50 + eor x0, x26, x10, ror #31 + xar v22.2d, v14.2d, v27.2d, #0x19 + eor x29, x29, x19, ror #37 + eor x27, x27, x12, ror #5 + eor v31.16b, v20.16b, v26.16b + eor x30, x30, x24, ror #34 + eor x0, x0, x7, ror #27 + shl v14.2d, v31.2d, #0x12 + eor x26, x30, x21, ror #26 + sri v14.2d, v31.2d, #0x2e + eor x26, x26, x25, ror #15 + ror x30, x27, #0x3e + xar v20.2d, v4.2d, v27.2d, #0x25 + eor x30, x30, x26, ror #57 + ror x26, x26, #0x3a + eor v31.16b, v24.16b, v27.16b + eor x16, x30, x16 + eor x28, x30, x28, ror #63 + shl v4.2d, v31.2d, #0xe + str x28, [sp, #0xd0] + eor x29, x29, x17, ror #36 + sri v4.2d, v31.2d, #0x32 + eor x28, x1, x2, ror #61 + eor x19, x30, x19, ror #37 + xar v24.2d, v21.2d, v25.2d, #0x3e + eor x29, x29, x20, ror #2 + eor x28, x28, x4, ror #54 + eor v31.16b, v5.16b, v26.16b + eor x26, x26, x0, ror #55 + eor x28, x28, x3, ror #39 + shl v21.2d, v31.2d, #0x24 + eor x28, x28, x5, ror #25 + ror x0, x0, #0x38 + sri v21.2d, v31.2d, #0x1c + eor x0, x0, x29, ror #63 + eor x27, x28, x27, ror #61 + xar v27.2d, v6.2d, v25.2d, #0x14 + eor x13, x0, x13, ror #46 + eor x28, x29, x28, ror #63 + bic v31.16b, v7.16b, v11.16b + eor x29, x30, x20, ror #2 + eor v5.16b, v31.16b, v10.16b + eor x20, x26, x3, ror #39 + eor x11, x0, x11, ror #50 + bcax v6.16b, v11.16b, v8.16b, v7.16b + eor x25, x28, x25, ror #9 + eor x3, x28, x21, ror #20 + bic v31.16b, v9.16b, v8.16b + eor x21, x26, x1 + eor x9, x27, x9, ror #49 + eor v7.16b, v31.16b, v7.16b + eor x24, x28, x24, ror #28 + eor x1, x30, x17, ror #36 + bcax v8.16b, v8.16b, v10.16b, v9.16b + eor x14, x0, x14, ror #8 + eor x22, x28, x22, ror #44 + bic v31.16b, v11.16b, v10.16b + eor x8, x27, x8, ror #56 + eor x17, x27, x7, ror #19 + eor v9.16b, v31.16b, v9.16b + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + bcax v10.16b, v15.16b, v12.16b, v16.16b + eor x4, x26, x4, ror #54 + eor x0, x0, x12, ror #3 + bic v31.16b, v13.16b, v12.16b + eor x28, x28, x23, ror #58 + eor x23, x26, x2, ror #61 + eor v11.16b, v31.16b, v16.16b + eor x26, x26, x5, ror #25 + bcax v12.16b, v12.16b, v14.16b, v13.16b + eor x2, x7, x16, ror #39 + bic x7, x9, x20, ror #42 + bic v31.16b, v15.16b, v14.16b + bic x30, x15, x9, ror #16 + eor x7, x7, x22, ror #25 + eor v13.16b, v31.16b, v13.16b + eor x12, x30, x20, ror #58 + bic x20, x22, x16, ror #56 + bic v31.16b, v16.16b, v15.16b + eor x30, x27, x6, ror #43 + eor x22, x20, x15, ror #23 + eor v14.16b, v31.16b, v14.16b + bic x6, x19, x13, ror #42 + eor x6, x6, x17, ror #41 + bcax v15.16b, v20.16b, v17.16b, v21.16b + bic x5, x13, x17, ror #63 + eor x5, x21, x5, ror #21 + bic v31.16b, v18.16b, v17.16b + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + eor v16.16b, v31.16b, v21.16b + bic x21, x21, x25, ror #50 + bic x20, x27, x4, ror #25 + bcax v17.16b, v17.16b, v19.16b, v18.16b + bic x10, x16, x15, ror #31 + eor x16, x21, x19, ror #43 + bic v31.16b, v20.16b, v19.16b + eor x21, x17, x25, ror #30 + bic x19, x25, x19, ror #57 + eor v18.16b, v31.16b, v18.16b + ldr x25, [sp, #0x18] + bcax v19.16b, v19.16b, v21.16b, v20.16b + eor x17, x10, x9, ror #47 + ldr x9, [sp, #0x8] + bic v31.16b, v22.16b, v1.16b + eor x15, x20, x28, ror #27 + bic x20, x4, x28, ror #2 + eor v20.16b, v31.16b, v0.16b + eor x10, x20, x1, ror #50 + bic x20, x11, x27, ror #60 + bcax v21.16b, v1.16b, v23.16b, v22.16b + eor x20, x20, x4, ror #21 + bic x4, x28, x1, ror #48 + bic v31.16b, v24.16b, v23.16b + bic x1, x1, x11, ror #57 + ldr x28, [x9, x25, lsl #3] + eor v22.16b, v31.16b, v22.16b + ldr x9, [sp, #0xd0] + add x25, x25, #0x1 + bcax v23.16b, v23.16b, v0.16b, v24.16b + str x25, [sp, #0x18] + cmp x25, #0x17 + bic v31.16b, v1.16b, v0.16b + eor x25, x1, x27, ror #53 + bic x27, x30, x26, ror #47 + eor v24.16b, v31.16b, v24.16b + eor x1, x5, x28 + eor x5, x4, x11, ror #41 + bcax v0.16b, v30.16b, v2.16b, v27.16b + eor x11, x19, x13, ror #35 + bic v31.16b, v3.16b, v2.16b + bic x13, x26, x24, ror #10 + eor x28, x27, x24, ror #57 + eor v1.16b, v31.16b, v27.16b + bic x27, x24, x9, ror #47 + bic x19, x23, x3, ror #9 + bcax v2.16b, v2.16b, v4.16b, v3.16b + bic x4, x29, x14, ror #41 + eor x24, x19, x29, ror #44 + bic v31.16b, v30.16b, v4.16b + bic x29, x3, x29, ror #35 + eor x13, x13, x9, ror #57 + eor v3.16b, v31.16b, v3.16b + eor x19, x29, x14, ror #12 + bic x29, x9, x0, ror #19 + bcax v4.16b, v4.16b, v27.16b, v30.16b + bic x14, x14, x8, ror #5 + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 + ldr x30, [sp, #0x10] + ld1r { v28.2d }, [x30], #8 + str x30, [sp, #0x10] + eor v0.16b, v0.16b, v28.16b + +Lmlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_loop: + eor x0, x15, x11, ror #52 + eor x0, x0, x13, ror #48 + eor3 v30.16b, v0.16b, v5.16b, v10.16b + eor v30.16b, v30.16b, v15.16b + eor x26, x8, x9, ror #57 + eor x27, x0, x14, ror #10 + eor v30.16b, v30.16b, v20.16b + eor x29, x16, x28, ror #63 + eor x26, x26, x6, ror #51 + eor3 v29.16b, v1.16b, v6.16b, v11.16b + eor x30, x23, x22, ror #50 + eor x0, x26, x10, ror #31 + eor v29.16b, v29.16b, v16.16b + eor x29, x29, x19, ror #37 + eor x27, x27, x12, ror #5 + eor v29.16b, v29.16b, v21.16b + eor x30, x30, x24, ror #34 + eor x0, x0, x7, ror #27 + eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor x26, x30, x21, ror #26 + eor x26, x26, x25, ror #15 + eor v28.16b, v28.16b, v17.16b + ror x30, x27, #0x3e + eor x30, x30, x26, ror #57 + eor v28.16b, v28.16b, v22.16b + ror x26, x26, #0x3a + eor x16, x30, x16 + eor3 v27.16b, v3.16b, v8.16b, v13.16b + eor x28, x30, x28, ror #63 + str x28, [sp, #0xd0] + eor v27.16b, v27.16b, v18.16b + eor x29, x29, x17, ror #36 + eor x28, x1, x2, ror #61 + eor v27.16b, v27.16b, v23.16b + eor x19, x30, x19, ror #37 + eor x29, x29, x20, ror #2 + eor3 v26.16b, v4.16b, v9.16b, v14.16b + eor x28, x28, x4, ror #54 + eor x26, x26, x0, ror #55 + eor v26.16b, v26.16b, v19.16b + eor x28, x28, x3, ror #39 + eor x28, x28, x5, ror #25 + eor v26.16b, v26.16b, v24.16b + ror x0, x0, #0x38 + eor x0, x0, x29, ror #63 + rax1 v25.2d, v30.2d, v28.2d + eor x27, x28, x27, ror #61 + eor x13, x0, x13, ror #46 + add v31.2d, v26.2d, v26.2d + eor x28, x29, x28, ror #63 + eor x29, x30, x20, ror #2 + sri v31.2d, v26.2d, #0x3f + eor x20, x26, x3, ror #39 + eor x11, x0, x11, ror #50 + eor v28.16b, v31.16b, v28.16b + eor x25, x28, x25, ror #9 + eor x3, x28, x21, ror #20 + rax1 v26.2d, v26.2d, v29.2d + eor x21, x26, x1 + add v31.2d, v27.2d, v27.2d + eor x9, x27, x9, ror #49 + eor x24, x28, x24, ror #28 + sri v31.2d, v27.2d, #0x3f + eor x1, x30, x17, ror #36 + eor x14, x0, x14, ror #8 + eor v29.16b, v31.16b, v29.16b + eor x22, x28, x22, ror #44 + eor x8, x27, x8, ror #56 + rax1 v27.2d, v27.2d, v30.2d + eor x17, x27, x7, ror #19 + eor x15, x0, x15, ror #62 + eor v30.16b, v0.16b, v26.16b + bic x7, x20, x22, ror #47 + eor x4, x26, x4, ror #54 + eor v31.16b, v2.16b, v29.16b + eor x0, x0, x12, ror #3 + eor x28, x28, x23, ror #58 + shl v0.2d, v31.2d, #0x3e + eor x23, x26, x2, ror #61 + eor x26, x26, x5, ror #25 + sri v0.2d, v31.2d, #0x2 + eor x2, x7, x16, ror #39 + bic x7, x9, x20, ror #42 + xar v2.2d, v12.2d, v29.2d, #0x15 + bic x30, x15, x9, ror #16 + eor x7, x7, x22, ror #25 + eor v31.16b, v13.16b, v28.16b + eor x12, x30, x20, ror #58 + bic x20, x22, x16, ror #56 + shl v12.2d, v31.2d, #0x19 + eor x30, x27, x6, ror #43 + eor x22, x20, x15, ror #23 + sri v12.2d, v31.2d, #0x27 + bic x6, x19, x13, ror #42 + eor x6, x6, x17, ror #41 + xar v13.2d, v19.2d, v27.2d, #0x38 + bic x5, x13, x17, ror #63 + eor x5, x21, x5, ror #21 + eor v31.16b, v23.16b, v28.16b + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + shl v19.2d, v31.2d, #0x38 + bic x21, x21, x25, ror #50 + bic x20, x27, x4, ror #25 + sri v19.2d, v31.2d, #0x8 + bic x10, x16, x15, ror #31 + eor x16, x21, x19, ror #43 + xar v23.2d, v15.2d, v26.2d, #0x17 + eor x21, x17, x25, ror #30 + bic x19, x25, x19, ror #57 + eor v31.16b, v1.16b, v25.16b + ldr x25, [sp, #0x18] + eor x17, x10, x9, ror #47 + shl v15.2d, v31.2d, #0x1 + ldr x9, [sp, #0x8] + sri v15.2d, v31.2d, #0x3f + eor x15, x20, x28, ror #27 + bic x20, x4, x28, ror #2 + xar v1.2d, v8.2d, v28.2d, #0x9 + eor x10, x20, x1, ror #50 + bic x20, x11, x27, ror #60 + eor v31.16b, v16.16b, v25.16b + eor x20, x20, x4, ror #21 + bic x4, x28, x1, ror #48 + shl v8.2d, v31.2d, #0x2d + bic x1, x1, x11, ror #57 + ldr x28, [x9, x25, lsl #3] + sri v8.2d, v31.2d, #0x13 + ldr x9, [sp, #0xd0] + add x25, x25, #0x1 + xar v16.2d, v7.2d, v29.2d, #0x3a + str x25, [sp, #0x18] + cmp x25, #0x17 + eor v31.16b, v10.16b, v26.16b + eor x25, x1, x27, ror #53 + bic x27, x30, x26, ror #47 + shl v7.2d, v31.2d, #0x3 + eor x1, x5, x28 + eor x5, x4, x11, ror #41 + sri v7.2d, v31.2d, #0x3d + eor x11, x19, x13, ror #35 + bic x13, x26, x24, ror #10 + xar v10.2d, v3.2d, v28.2d, #0x24 + eor x28, x27, x24, ror #57 + bic x27, x24, x9, ror #47 + eor v31.16b, v18.16b, v28.16b + bic x19, x23, x3, ror #9 + bic x4, x29, x14, ror #41 + shl v3.2d, v31.2d, #0x15 + eor x24, x19, x29, ror #44 + bic x29, x3, x29, ror #35 + sri v3.2d, v31.2d, #0x2b + eor x13, x13, x9, ror #57 + eor x19, x29, x14, ror #12 + xar v18.2d, v17.2d, v29.2d, #0x31 + bic x29, x9, x0, ror #19 + bic x14, x14, x8, ror #5 + eor v31.16b, v11.16b, v25.16b + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + shl v17.2d, v31.2d, #0xa + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + sri v17.2d, v31.2d, #0x36 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + xar v11.2d, v9.2d, v27.2d, #0x2c + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 + eor v31.16b, v22.16b, v29.16b + eor x0, x15, x11, ror #52 + shl v9.2d, v31.2d, #0x3d + eor x0, x0, x13, ror #48 + eor x26, x8, x9, ror #57 + sri v9.2d, v31.2d, #0x3 + eor x27, x0, x14, ror #10 + eor x29, x16, x28, ror #63 + xar v22.2d, v14.2d, v27.2d, #0x19 + eor x26, x26, x6, ror #51 + eor x30, x23, x22, ror #50 + eor v31.16b, v20.16b, v26.16b + eor x0, x26, x10, ror #31 + eor x29, x29, x19, ror #37 + shl v14.2d, v31.2d, #0x12 + eor x27, x27, x12, ror #5 + eor x30, x30, x24, ror #34 + sri v14.2d, v31.2d, #0x2e + eor x0, x0, x7, ror #27 + eor x26, x30, x21, ror #26 + xar v20.2d, v4.2d, v27.2d, #0x25 + eor x26, x26, x25, ror #15 + ror x30, x27, #0x3e + eor v31.16b, v24.16b, v27.16b + eor x30, x30, x26, ror #57 + ror x26, x26, #0x3a + shl v4.2d, v31.2d, #0xe + eor x16, x30, x16 + eor x28, x30, x28, ror #63 + sri v4.2d, v31.2d, #0x32 + str x28, [sp, #0xd0] + eor x29, x29, x17, ror #36 + xar v24.2d, v21.2d, v25.2d, #0x3e + eor x28, x1, x2, ror #61 + eor x19, x30, x19, ror #37 + eor v31.16b, v5.16b, v26.16b + eor x29, x29, x20, ror #2 + eor x28, x28, x4, ror #54 + shl v21.2d, v31.2d, #0x24 + eor x26, x26, x0, ror #55 + eor x28, x28, x3, ror #39 + sri v21.2d, v31.2d, #0x1c + eor x28, x28, x5, ror #25 + ror x0, x0, #0x38 + xar v27.2d, v6.2d, v25.2d, #0x14 + eor x0, x0, x29, ror #63 + eor x27, x28, x27, ror #61 + bic v31.16b, v7.16b, v11.16b + eor x13, x0, x13, ror #46 + eor x28, x29, x28, ror #63 + eor v5.16b, v31.16b, v10.16b + eor x29, x30, x20, ror #2 + eor x20, x26, x3, ror #39 + bcax v6.16b, v11.16b, v8.16b, v7.16b + eor x11, x0, x11, ror #50 + eor x25, x28, x25, ror #9 + bic v31.16b, v9.16b, v8.16b + eor x3, x28, x21, ror #20 + eor v7.16b, v31.16b, v7.16b + eor x21, x26, x1 + eor x9, x27, x9, ror #49 + bcax v8.16b, v8.16b, v10.16b, v9.16b + eor x24, x28, x24, ror #28 + eor x1, x30, x17, ror #36 + bic v31.16b, v11.16b, v10.16b + eor x14, x0, x14, ror #8 + eor x22, x28, x22, ror #44 + eor v9.16b, v31.16b, v9.16b + eor x8, x27, x8, ror #56 + eor x17, x27, x7, ror #19 + bcax v10.16b, v15.16b, v12.16b, v16.16b + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + bic v31.16b, v13.16b, v12.16b + eor x4, x26, x4, ror #54 + eor x0, x0, x12, ror #3 + eor v11.16b, v31.16b, v16.16b + eor x28, x28, x23, ror #58 + eor x23, x26, x2, ror #61 + bcax v12.16b, v12.16b, v14.16b, v13.16b + eor x26, x26, x5, ror #25 + eor x2, x7, x16, ror #39 + bic v31.16b, v15.16b, v14.16b + bic x7, x9, x20, ror #42 + bic x30, x15, x9, ror #16 + eor v13.16b, v31.16b, v13.16b + eor x7, x7, x22, ror #25 + eor x12, x30, x20, ror #58 + bic v31.16b, v16.16b, v15.16b + bic x20, x22, x16, ror #56 + eor x30, x27, x6, ror #43 + eor v14.16b, v31.16b, v14.16b + eor x22, x20, x15, ror #23 + bic x6, x19, x13, ror #42 + bcax v15.16b, v20.16b, v17.16b, v21.16b + eor x6, x6, x17, ror #41 + bic x5, x13, x17, ror #63 + bic v31.16b, v18.16b, v17.16b + eor x5, x21, x5, ror #21 + bic x17, x17, x21, ror #44 + eor v16.16b, v31.16b, v21.16b + eor x27, x27, x10, ror #23 + bic x21, x21, x25, ror #50 + bcax v17.16b, v17.16b, v19.16b, v18.16b + bic x20, x27, x4, ror #25 + bic x10, x16, x15, ror #31 + bic v31.16b, v20.16b, v19.16b + eor x16, x21, x19, ror #43 + eor x21, x17, x25, ror #30 + eor v18.16b, v31.16b, v18.16b + bic x19, x25, x19, ror #57 + ldr x25, [sp, #0x18] + bcax v19.16b, v19.16b, v21.16b, v20.16b + eor x17, x10, x9, ror #47 + bic v31.16b, v22.16b, v1.16b + ldr x9, [sp, #0x8] + eor x15, x20, x28, ror #27 + eor v20.16b, v31.16b, v0.16b + bic x20, x4, x28, ror #2 + eor x10, x20, x1, ror #50 + bcax v21.16b, v1.16b, v23.16b, v22.16b + bic x20, x11, x27, ror #60 + eor x20, x20, x4, ror #21 + bic v31.16b, v24.16b, v23.16b + bic x4, x28, x1, ror #48 + bic x1, x1, x11, ror #57 + eor v22.16b, v31.16b, v22.16b + ldr x28, [x9, x25, lsl #3] + ldr x9, [sp, #0xd0] + bcax v23.16b, v23.16b, v0.16b, v24.16b + add x25, x25, #0x1 + str x25, [sp, #0x18] + bic v31.16b, v1.16b, v0.16b + cmp x25, #0x17 + eor x25, x1, x27, ror #53 + eor v24.16b, v31.16b, v24.16b + bic x27, x30, x26, ror #47 + eor x1, x5, x28 + bcax v0.16b, v30.16b, v2.16b, v27.16b + eor x5, x4, x11, ror #41 + eor x11, x19, x13, ror #35 + bic v31.16b, v3.16b, v2.16b + bic x13, x26, x24, ror #10 + eor x28, x27, x24, ror #57 + eor v1.16b, v31.16b, v27.16b + bic x27, x24, x9, ror #47 + bic x19, x23, x3, ror #9 + bcax v2.16b, v2.16b, v4.16b, v3.16b + bic x4, x29, x14, ror #41 + eor x24, x19, x29, ror #44 + bic v31.16b, v30.16b, v4.16b + bic x29, x3, x29, ror #35 + eor x13, x13, x9, ror #57 + eor v3.16b, v31.16b, v3.16b + eor x19, x29, x14, ror #12 + bic x29, x9, x0, ror #19 + bcax v4.16b, v4.16b, v27.16b, v30.16b + bic x14, x14, x8, ror #5 + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 + ldr x30, [sp, #0x10] + ld1r { v28.2d }, [x30], #8 + str x30, [sp, #0x10] + eor v0.16b, v0.16b, v28.16b + +Lmlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_loop_end: + b.le Lmlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_loop + ror x2, x2, #0x3d + ror x3, x3, #0x27 + ror x4, x4, #0x36 + ror x5, x5, #0x19 + ror x6, x6, #0x2b + ror x7, x7, #0x13 + ror x8, x8, #0x38 + ror x9, x9, #0x31 + ror x10, x10, #0x17 + ror x11, x11, #0x32 + ror x12, x12, #0x3 + ror x13, x13, #0x2e + ror x14, x14, #0x8 + ror x15, x15, #0x3e + ror x17, x17, #0x24 + ror x28, x28, #0x3f + ror x19, x19, #0x25 + ror x20, x20, #0x2 + ror x21, x21, #0x14 + ror x22, x22, #0x2c + ror x23, x23, #0x3a + ror x24, x24, #0x1c + ror x25, x25, #0x9 + ldr x30, [sp, #0x20] + cmp x30, #0x1 + b.eq Lmlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_done + mov x30, #0x1 // =1 + str x30, [sp, #0x20] + ldr x0, [sp] + add x0, x0, #0x190 + stp x1, x6, [x0] + stp x11, x16, [x0, #0x10] + stp x21, x2, [x0, #0x20] + stp x7, x12, [x0, #0x30] + stp x17, x22, [x0, #0x40] + stp x3, x8, [x0, #0x50] + stp x13, x28, [x0, #0x60] + stp x23, x4, [x0, #0x70] + stp x9, x14, [x0, #0x80] + stp x19, x24, [x0, #0x90] + stp x5, x10, [x0, #0xa0] + stp x15, x20, [x0, #0xb0] + str x25, [x0, #0xc0] + sub x0, x0, #0x190 + add x0, x0, #0x258 + ldp x1, x6, [x0] + ldp x11, x16, [x0, #0x10] + ldp x21, x2, [x0, #0x20] + ldp x7, x12, [x0, #0x30] + ldp x17, x22, [x0, #0x40] + ldp x3, x8, [x0, #0x50] + ldp x13, x28, [x0, #0x60] + ldp x23, x4, [x0, #0x70] + ldp x9, x14, [x0, #0x80] + ldp x19, x24, [x0, #0x90] + ldp x5, x10, [x0, #0xa0] + ldp x15, x20, [x0, #0xb0] + ldr x25, [x0, #0xc0] + sub x0, x0, #0x258 + b Lmlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_initial + +Lmlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_done: + ldr x0, [sp] + add x0, x0, #0x258 + stp x1, x6, [x0] + stp x11, x16, [x0, #0x10] + stp x21, x2, [x0, #0x20] + stp x7, x12, [x0, #0x30] + stp x17, x22, [x0, #0x40] + stp x3, x8, [x0, #0x50] + stp x13, x28, [x0, #0x60] + stp x23, x4, [x0, #0x70] + stp x9, x14, [x0, #0x80] + stp x19, x24, [x0, #0x90] + stp x5, x10, [x0, #0xa0] + stp x15, x20, [x0, #0xb0] + str x25, [x0, #0xc0] + sub x0, x0, #0x258 + add x4, x0, #0xc8 + trn1 v25.2d, v0.2d, v1.2d + trn1 v26.2d, v2.2d, v3.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v0.2d, v1.2d + trn2 v28.2d, v2.2d, v3.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v4.2d, v5.2d + trn1 v26.2d, v6.2d, v7.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v4.2d, v5.2d + trn2 v28.2d, v6.2d, v7.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v8.2d, v9.2d + trn1 v26.2d, v10.2d, v11.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v8.2d, v9.2d + trn2 v28.2d, v10.2d, v11.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v12.2d, v13.2d + trn1 v26.2d, v14.2d, v15.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v12.2d, v13.2d + trn2 v28.2d, v14.2d, v15.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v16.2d, v17.2d + trn1 v26.2d, v18.2d, v19.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v16.2d, v17.2d + trn2 v28.2d, v18.2d, v19.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v20.2d, v21.2d + trn1 v26.2d, v22.2d, v23.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v20.2d, v21.2d + trn2 v28.2d, v22.2d, v23.2d + st1 { v27.2d, v28.2d }, [x4], #32 + str d24, [x0] + trn2 v25.2d, v24.2d, v24.2d + str d25, [x4] + ldp d8, d9, [sp, #0x90] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0xa0] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0xb0] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0xc0] + .cfi_restore d14 + .cfi_restore d15 + ldp x19, x20, [sp, #0x30] + .cfi_restore x19 + .cfi_restore x20 + ldp x21, x22, [sp, #0x40] + .cfi_restore x21 + .cfi_restore x22 + ldp x23, x24, [sp, #0x50] + .cfi_restore x23 + .cfi_restore x24 + ldp x25, x26, [sp, #0x60] + .cfi_restore x25 + .cfi_restore x26 + ldp x27, x28, [sp, #0x70] + .cfi_restore x27 + .cfi_restore x28 + ldp x29, x30, [sp, #0x80] + .cfi_restore x29 + .cfi_restore x30 + add sp, sp, #0xe0 + .cfi_adjust_cfa_offset -0xe0 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm) + +#endif /* __ARM_FEATURE_SHA3 */ + +#endif /* MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccakf1600_round_constants.c b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccakf1600_round_constants.c new file mode 100644 index 0000000000..12a897f04b --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccakf1600_round_constants.c @@ -0,0 +1,47 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "../../../../common.h" + +#if (defined(MLK_FIPS202_AARCH64_NEED_X1_SCALAR) || \ + defined(MLK_FIPS202_AARCH64_NEED_X1_V84A) || \ + defined(MLK_FIPS202_AARCH64_NEED_X2_V84A) || \ + defined(MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID) || \ + defined(MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID)) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "fips202_native_aarch64.h" + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint64_t + mlk_keccakf1600_round_constants[24] = { + 0x0000000000000001, 0x0000000000008082, 0x800000000000808a, + 0x8000000080008000, 0x000000000000808b, 0x0000000080000001, + 0x8000000080008081, 0x8000000000008009, 0x000000000000008a, + 0x0000000000000088, 0x0000000080008009, 0x000000008000000a, + 0x000000008000808b, 0x800000000000008b, 0x8000000000008089, + 0x8000000000008003, 0x8000000000008002, 0x8000000000000080, + 0x000000000000800a, 0x800000008000000a, 0x8000000080008081, + 0x8000000000008080, 0x0000000080000001, 0x8000000080008008, +}; + +#else /* (MLK_FIPS202_AARCH64_NEED_X1_SCALAR || \ + MLK_FIPS202_AARCH64_NEED_X1_V84A || MLK_FIPS202_AARCH64_NEED_X2_V84A \ + || MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \ + MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(fips202_aarch64_round_constants) + +#endif /* !((MLK_FIPS202_AARCH64_NEED_X1_SCALAR || \ + MLK_FIPS202_AARCH64_NEED_X1_V84A || MLK_FIPS202_AARCH64_NEED_X2_V84A \ + || MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \ + MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/x1_scalar.h b/src/mlkem-native/mlkem/src/fips202/native/aarch64/x1_scalar.h new file mode 100644 index 0000000000..ccf805b002 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/x1_scalar.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_AARCH64_X1_SCALAR_H +#define MLK_FIPS202_NATIVE_AARCH64_X1_SCALAR_H + +/* Part of backend API */ +#define MLK_USE_NATIVE_FIPS202_X1 +/* Guard for assembly file */ +#define MLK_FIPS202_AARCH64_NEED_X1_SCALAR + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_aarch64.h" +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x1_native(uint64_t *state) +{ + mlk_keccak_f1600_x1_scalar_aarch64_asm(state, + mlk_keccakf1600_round_constants); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_FIPS202_NATIVE_AARCH64_X1_SCALAR_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/x1_v84a.h b/src/mlkem-native/mlkem/src/fips202/native/aarch64/x1_v84a.h new file mode 100644 index 0000000000..72d538a66b --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/x1_v84a.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_AARCH64_X1_V84A_H +#define MLK_FIPS202_NATIVE_AARCH64_X1_V84A_H + +#if !defined(__ARM_FEATURE_SHA3) +#error This backend can only be used if SHA3 extensions are available. +#endif + +/* Part of backend API */ +#define MLK_USE_NATIVE_FIPS202_X1 +/* Guard for assembly file */ +#define MLK_FIPS202_AARCH64_NEED_X1_V84A + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_aarch64.h" +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x1_native(uint64_t *state) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON) || + !mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_SHA3)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_keccak_f1600_x1_v84a_aarch64_asm(state, mlk_keccakf1600_round_constants); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_FIPS202_NATIVE_AARCH64_X1_V84A_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/x2_v84a.h b/src/mlkem-native/mlkem/src/fips202/native/aarch64/x2_v84a.h new file mode 100644 index 0000000000..5848454828 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/x2_v84a.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_AARCH64_X2_V84A_H +#define MLK_FIPS202_NATIVE_AARCH64_X2_V84A_H + +#if !defined(__ARM_FEATURE_SHA3) +#error This backend can only be used if SHA3 extensions are available. +#endif + +/* Part of backend API */ +#define MLK_USE_NATIVE_FIPS202_X4 +/* Guard for assembly file */ +#define MLK_FIPS202_AARCH64_NEED_X2_V84A + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_aarch64.h" +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x4_native(uint64_t *state) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON) || + !mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_SHA3)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_keccak_f1600_x2_v84a_aarch64_asm(state + 0 * 25, + mlk_keccakf1600_round_constants); + mlk_keccak_f1600_x2_v84a_aarch64_asm(state + 2 * 25, + mlk_keccakf1600_round_constants); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_FIPS202_NATIVE_AARCH64_X2_V84A_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/x4_v8a_scalar.h b/src/mlkem-native/mlkem/src/fips202/native/aarch64/x4_v8a_scalar.h new file mode 100644 index 0000000000..e3f84ad52d --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/x4_v8a_scalar.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_AARCH64_X4_V8A_SCALAR_H +#define MLK_FIPS202_NATIVE_AARCH64_X4_V8A_SCALAR_H + +/* Part of backend API */ +#define MLK_USE_NATIVE_FIPS202_X4 +/* Guard for assembly file */ +#define MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_aarch64.h" +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x4_native(uint64_t *state) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm( + state, mlk_keccakf1600_round_constants); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_FIPS202_NATIVE_AARCH64_X4_V8A_SCALAR_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/x4_v8a_v84a_scalar.h b/src/mlkem-native/mlkem/src/fips202/native/aarch64/x4_v8a_v84a_scalar.h new file mode 100644 index 0000000000..bd112cc5d9 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/x4_v8a_v84a_scalar.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_AARCH64_X4_V8A_V84A_SCALAR_H +#define MLK_FIPS202_NATIVE_AARCH64_X4_V8A_V84A_SCALAR_H + +#if !defined(__ARM_FEATURE_SHA3) +#error This backend can only be used if SHA3 extensions are available. +#endif + +/* Part of backend API */ +#define MLK_USE_NATIVE_FIPS202_X4 +/* Guard for assembly file */ +#define MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_aarch64.h" +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x4_native(uint64_t *state) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON) || + !mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_SHA3)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm( + state, mlk_keccakf1600_round_constants); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_FIPS202_NATIVE_AARCH64_X4_V8A_V84A_SCALAR_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/api.h b/src/mlkem-native/mlkem/src/fips202/native/api.h new file mode 100644 index 0000000000..8ef7383e25 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/api.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_API_H +#define MLK_FIPS202_NATIVE_API_H +/* + * FIPS-202 native interface + * + * This header is primarily for documentation purposes. + * It should not be included by backend implementations. + */ + +#include "../../cbmc.h" + +/* Backends must return MLK_NATIVE_FUNC_SUCCESS upon success. */ +#define MLK_NATIVE_FUNC_SUCCESS (0) +/* Backends may return MLK_NATIVE_FUNC_FALLBACK to signal to the frontend that + * the target/parameters are unsupported; typically, this would be because of + * dependencies on CPU features not detected on the host CPU. In this case, + * the frontend falls back to the default C implementation. */ +#define MLK_NATIVE_FUNC_FALLBACK (-1) + +/* + * This is the C<->native interface allowing for the drop-in + * of custom Keccak-F1600 implementations. + * + * A _backend_ is a specific implementation of parts of this interface. + * + * You can replace 1-fold or 4-fold batched Keccak-F1600. + * To enable, set MLK_USE_NATIVE_FIPS202_X1 or MLK_USE_NATIVE_FIPS202_X4 + * in your backend, and define the inline wrappers mlk_keccak_f1600_x1_native() + * and/or mlk_keccak_f1600_x4_native(), respectively, to forward to your + * implementation. + */ + +#if defined(MLK_USE_NATIVE_FIPS202_X1) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x1_native(uint64_t *state) +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 1)) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 1)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged_u64(state, 25 * 1))); +#endif /* MLK_USE_NATIVE_FIPS202_X1 */ +#if defined(MLK_USE_NATIVE_FIPS202_X4) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x4_native(uint64_t *state) +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 4)) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 4)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged_u64(state, 25 * 4))); +#endif /* MLK_USE_NATIVE_FIPS202_X4 */ + +/* + * Native x4 XOR bytes and extract bytes interface. + * + * These functions allow backends to provide optimized implementations for + * XORing input data into the state and extracting output data from the state. + * This is particularly useful for backends that use a different internal state + * representation (e.g., bit-interleaved), as conversion can happen during + * XOR/extract rather than before/after each permutation. + * + * NOTE: We assume that the custom representation of the zero state is the + * all-zero state. + * + * MLK_USE_NATIVE_FIPS202_X4_XOR_BYTES: Backend provides native XOR bytes + * MLK_USE_NATIVE_FIPS202_X4_EXTRACT_BYTES: Backend provides native extract + * bytes + */ + +#if defined(MLK_USE_NATIVE_FIPS202_X4_XOR_BYTES) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccakf1600_xor_bytes_x4_native( + uint64_t *state, const unsigned char *data0, const unsigned char *data1, + const unsigned char *data2, const unsigned char *data3, unsigned offset, + unsigned length) +__contract__( + requires(0 <= offset && offset <= 25 * sizeof(uint64_t) && + 0 <= length && length <= 25 * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 4)) + requires(memory_no_alias(data0, length)) + requires((data0 == data1 && + data0 == data2 && + data0 == data3) || + (memory_no_alias(data1, length) && + memory_no_alias(data2, length) && + memory_no_alias(data3, length))) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 4)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged_u64(state, 25 * 4))); +#endif /* MLK_USE_NATIVE_FIPS202_X4_XOR_BYTES */ + +#if defined(MLK_USE_NATIVE_FIPS202_X4_EXTRACT_BYTES) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccakf1600_extract_bytes_x4_native( + uint64_t *state, unsigned char *data0, unsigned char *data1, + unsigned char *data2, unsigned char *data3, unsigned offset, + unsigned length) +__contract__( + requires(0 <= offset && offset <= 25 * sizeof(uint64_t) && + 0 <= length && length <= 25 * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 4)) + requires(memory_no_alias(data0, length)) + requires(memory_no_alias(data1, length)) + requires(memory_no_alias(data2, length)) + requires(memory_no_alias(data3, length)) + assigns(memory_slice(data0, length)) + assigns(memory_slice(data1, length)) + assigns(memory_slice(data2, length)) + assigns(memory_slice(data3, length)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS)); +#endif /* MLK_USE_NATIVE_FIPS202_X4_EXTRACT_BYTES */ + +#endif /* !MLK_FIPS202_NATIVE_API_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/armv81m/README.md b/src/mlkem-native/mlkem/src/fips202/native/armv81m/README.md new file mode 100644 index 0000000000..9c2c0d26ea --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/armv81m/README.md @@ -0,0 +1,10 @@ +[//]: # (SPDX-License-Identifier: CC-BY-4.0) + +# FIPS202 backend for Armv8.1-M + MVE + +This directory contains the source code for a FIPS202 backend targeting +the Armv8.1-M + MVE/Helium architecture. It is automatically derived from +the respective development source in [dev/fips202/armv81m](../../../../../dev/fips202/armv81m). + +**Warning:** This backend is still in active development and has not yet undergone +the same level of review as the rest of the code. Use at your own risk! diff --git a/src/mlkem-native/mlkem/src/fips202/native/armv81m/mve.h b/src/mlkem-native/mlkem/src/fips202/native/armv81m/mve.h new file mode 100644 index 0000000000..0f87fad5e3 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/armv81m/mve.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_ARMV81M_MVE_H +#define MLK_FIPS202_NATIVE_ARMV81M_MVE_H + +#define MLK_FIPS202_NATIVE_ARMV81M + +/* Part of backend API */ +#define MLK_USE_NATIVE_FIPS202_X4 +#define MLK_USE_NATIVE_FIPS202_X4_XOR_BYTES +#define MLK_USE_NATIVE_FIPS202_X4_EXTRACT_BYTES +/* Guard for assembly file */ +#define MLK_FIPS202_ARMV81M_NEED_X4 + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_armv81m.h" + +/* + * Native x4 permutation + * State is kept in bit-interleaved format. + */ +#define mlk_keccak_f1600_x4_native_impl \ + MLK_NAMESPACE(keccak_f1600_x4_native_impl) +int mlk_keccak_f1600_x4_native_impl(uint64_t *state); + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x4_native(uint64_t *state) +{ + return mlk_keccak_f1600_x4_native_impl(state); +} + +/* + * Native x4 XOR bytes (with on-the-fly bit interleaving) + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccakf1600_xor_bytes_x4_native( + uint64_t *state, const uint8_t *data0, const uint8_t *data1, + const uint8_t *data2, const uint8_t *data3, unsigned offset, + unsigned length) +{ + mlk_keccak_f1600_x4_state_xor_bytes_asm(state, data0, data1, data2, data3, + offset, length); + return MLK_NATIVE_FUNC_SUCCESS; +} + +/* + * Native x4 extract bytes (with on-the-fly bit de-interleaving) + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccakf1600_extract_bytes_x4_native( + uint64_t *state, uint8_t *data0, uint8_t *data1, uint8_t *data2, + uint8_t *data3, unsigned offset, unsigned length) +{ + mlk_keccak_f1600_x4_state_extract_bytes_asm(state, data0, data1, data2, data3, + offset, length); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_FIPS202_NATIVE_ARMV81M_MVE_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/fips202_native_armv81m.h b/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/fips202_native_armv81m.h new file mode 100644 index 0000000000..7888f337f2 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/fips202_native_armv81m.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_FIPS202_NATIVE_ARMV81M_SRC_FIPS202_NATIVE_ARMV81M_H +#define MLK_FIPS202_NATIVE_ARMV81M_SRC_FIPS202_NATIVE_ARMV81M_H + +#include "../../../../common.h" + +/* Keccak round constants in bit-interleaved form */ +#define mlk_keccakf1600_round_constants \ + MLK_NAMESPACE(keccakf1600_round_constants) +MLK_INTERNAL_DATA_DECLARATION const uint32_t + mlk_keccakf1600_round_constants[48]; + +#define mlk_keccak_f1600_x4_mve_asm MLK_NAMESPACE(keccak_f1600_x4_mve_asm) +void mlk_keccak_f1600_x4_mve_asm(uint64_t state[100], uint64_t tmpstate[100], + const uint32_t rc[48]); + +#define mlk_keccak_f1600_x4_state_xor_bytes_asm \ + MLK_NAMESPACE(keccak_f1600_x4_state_xor_bytes_asm) +void mlk_keccak_f1600_x4_state_xor_bytes_asm(void *state, const uint8_t *d0, + const uint8_t *d1, + const uint8_t *d2, + const uint8_t *d3, unsigned offset, + unsigned length); + +#define mlk_keccak_f1600_x4_state_extract_bytes_asm \ + MLK_NAMESPACE(keccak_f1600_x4_state_extract_bytes_asm) +void mlk_keccak_f1600_x4_state_extract_bytes_asm(void *state, uint8_t *data0, + uint8_t *data1, uint8_t *data2, + uint8_t *data3, + unsigned offset, + unsigned length); + +#endif /* !MLK_FIPS202_NATIVE_ARMV81M_SRC_FIPS202_NATIVE_ARMV81M_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.S b/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.S new file mode 100644 index 0000000000..2bf6c3ec18 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.S @@ -0,0 +1,716 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) 2025 Arm Limited + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: keccak_f1600_x4_mve_asm + Description: Armv8.1-M MVE implementation of batched (x4) Keccak-f[1600] permutation using bit-interleaved state + Signature: void mlk_keccak_f1600_x4_mve_asm(void *state, void *tmpstate, const uint32_t *rc) + ABI: + Architecture: armv81m + CallingConvention: AAPCS32 + Features: [MVE] + r0: + type: buffer + size_bytes: 800 + permissions: read/write + c_parameter: void *state + description: Bit-interleaved state for 4 Keccak instances (even halves followed by odd halves) + r1: + type: buffer + size_bytes: 800 + permissions: read/write + c_parameter: void *tmpstate + description: Temporary storage for intermediate state + r2: + type: buffer + size_bytes: 192 + permissions: read + c_parameter: const uint32_t *rc + description: Keccak round constants in bit-interleaved form (24 pairs of 32-bit words) + Stack: + bytes: 228 + description: register preservation (36) + SIMD registers (64) + temporary storage (128) +*/ + +// --------------------------------------------------------------------------- +// Bit-interleaving background +// --------------------------------------------------------------------------- +// Each 64-bit Keccak lane is stored as two 32-bit words: +// even half -- bits 0, 2, 4, ..., 62 of the lane +// odd half -- bits 1, 3, 5, ..., 63 of the lane +// This representation allows 64-bit lane rotations (used in the Keccak +// round function) to be implemented as pairs of 32-bit rotations. +// +// Batched (x4) processing: +// Four Keccak instances are processed as a batch. Their states are +// stored interleaved in a single 800-byte buffer: first the even +// halves of all 25 lanes (400 bytes), then the odd halves (400 bytes). +// Within each 16-byte row, the four u32 words correspond to +// instances 0..3 of the same lane, enabling SIMD-parallel operations +// across all four instances. +// +// State memory layout (25 lanes x 4 instances x 2 halves): +// S[i][l]_even/odd = even/odd half of lane l, instance i (u32) +// Each row is 16 bytes (one Q-register). +// Offset Contents +// 0 S[0][ 0]_even, S[1][ 0]_even, S[2][ 0]_even, S[3][ 0]_even +// 16 S[0][ 1]_even, S[1][ 1]_even, S[2][ 1]_even, S[3][ 1]_even +// ... +// 384 S[0][24]_even, S[1][24]_even, S[2][24]_even, S[3][24]_even +// 400 S[0][ 0]_odd, S[1][ 0]_odd, S[2][ 0]_odd, S[3][ 0]_odd +// 416 S[0][ 1]_odd, S[1][ 1]_odd, S[2][ 1]_odd, S[3][ 1]_odd +// ... +// 784 S[0][24]_odd, S[1][24]_odd, S[2][24]_odd, S[3][24]_odd + +#include "../../../../common.h" +#if defined(MLK_FIPS202_ARMV81M_NEED_X4) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/armv81m/src/keccak_f1600_x4_mve.S using scripts/simpasm. Do not modify it directly. + */ + +.thumb +.syntax unified + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x4_mve_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x4_mve_asm) + + .cfi_startproc + push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr} + .cfi_adjust_cfa_offset 0x24 + .cfi_rel_offset r4, 0x0 + .cfi_rel_offset r5, 0x4 + .cfi_rel_offset r6, 0x8 + .cfi_rel_offset r7, 0xc + .cfi_rel_offset r8, 0x10 + .cfi_rel_offset r9, 0x14 + .cfi_rel_offset r10, 0x18 + .cfi_rel_offset r11, 0x1c + .cfi_rel_offset lr, 0x20 + vpush {d8, d9, d10, d11, d12, d13, d14, d15} + .cfi_adjust_cfa_offset 0x40 + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + sub sp, #0x80 + .cfi_adjust_cfa_offset 0x80 + mov r6, r2 + mov.w lr, #0x18 + mov r2, r0 + mov r4, r1 + add.w r3, r2, #0x190 + vldrw.u32 q0, [r3] + vldrw.u32 q1, [r2] + vldrw.u32 q2, [r2, #32] + wls lr, lr, Lmlk_keccak_f1600_x4_mve_asm_roundend @ imm = #0x8c0 + +Lmlk_keccak_f1600_x4_mve_asm_roundstart: + vldrw.u32 q6, [r2, #112] + veor q7, q6, q2 + vldrw.u32 q2, [r2, #80] + veor q1, q2, q1 + add.w r5, r2, #0x190 + vldrw.u32 q5, [r5, #80] + veor q4, q5, q0 + vldrw.u32 q0, [r2, #192] + veor q3, q7, q0 + vldrw.u32 q0, [r2, #160] + veor q1, q1, q0 + vldrw.u32 q0, [r5, #160] + veor q0, q4, q0 + vldrw.u32 q6, [r2, #272] + veor q2, q3, q6 + vldrw.u32 q7, [r2, #240] + veor q5, q1, q7 + vldrw.u32 q4, [r5, #240] + veor q4, q0, q4 + vldrw.u32 q6, [r2, #352] + veor q3, q2, q6 + vldrw.u32 q0, [r2, #320] + veor q2, q5, q0 + vldrw.u32 q1, [r5, #320] + veor q5, q4, q1 + vldrw.u32 q4, [r5, #32] + veor q0, q3, q5 + vldrw.u32 q1, [r5, #16] + veor q6, q1, q0 + vstrw.32 q5, [sp] + vshr.u32 q7, q6, #0x1f + add.w r10, r4, #0x190 + vsli.32 q7, q6, #0x1 + vldrw.u32 q6, [r5, #112] + veor q6, q4, q6 + vldrw.u32 q4, [r5, #192] + veor q4, q6, q4 + vldrw.u32 q6, [r5, #272] + veor q4, q4, q6 + vldrw.u32 q6, [r5, #352] + veor q5, q4, q6 + vstrw.32 q7, [r4, #160] + vshr.u32 q4, q5, #0x1f + vsli.32 q4, q5, #0x1 + vldrw.u32 q6, [r2, #16] + veor q7, q4, q2 + veor q1, q6, q7 + vldrw.u32 q6, [r5, #96] + veor q6, q6, q0 + vstrw.32 q1, [r10, #160] + vshr.u32 q1, q6, #0xa + vsli.32 q1, q6, #0x16 + vldrw.u32 q6, [r2, #96] + veor q4, q6, q7 + vstrw.32 q1, [r10, #16] + vshr.u32 q6, q4, #0xa + vsli.32 q6, q4, #0x16 + vldrw.u32 q1, [r5, #336] + veor q4, q1, q0 + vldrw.u32 q1, [r2, #176] + veor q1, q1, q7 + vstrw.32 q6, [r4, #16] + vshr.u32 q6, q1, #0x1b + vsli.32 q6, q1, #0x5 + vldrw.u32 q1, [r2, #256] + veor q1, q1, q7 + vstrw.32 q6, [r4, #272] + vshr.u32 q6, q1, #0xa + vsli.32 q6, q1, #0x16 + vldrw.u32 q1, [r2, #336] + veor q1, q1, q7 + vstrw.32 q6, [r10, #128] + vshr.u32 q6, q1, #0x1f + vsli.32 q6, q1, #0x1 + vldrw.u32 q7, [r5, #176] + veor q7, q7, q0 + vstrw.32 q6, [r4, #384] + vshr.u32 q1, q7, #0x1b + vsli.32 q1, q7, #0x5 + vldrw.u32 q6, [r5, #256] + veor q0, q6, q0 + vstrw.32 q1, [r10, #272] + vshr.u32 q1, q4, #0x1f + vldrw.u32 q7, [r5, #64] + vsli.32 q1, q4, #0x1 + vldrw.u32 q4, [r5, #144] + vshr.u32 q6, q0, #0x9 + vstrw.32 q1, [r10, #384] + vsli.32 q6, q0, #0x17 + veor q7, q7, q4 + vldrw.u32 q1, [r5, #224] + veor q4, q7, q1 + vldrw.u32 q7, [r5, #304] + veor q1, q4, q7 + vldrw.u32 q0, [r5, #384] + veor q7, q1, q0 + vstrw.32 q6, [r4, #128] + vshr.u32 q1, q7, #0x1f + vsli.32 q1, q7, #0x1 + vldrw.u32 q6, [r2, #144] + veor q0, q1, q3 + vldrw.u32 q3, [r2, #64] + veor q1, q3, q6 + vldrw.u32 q6, [r2, #224] + veor q1, q1, q6 + vldrw.u32 q3, [r2, #304] + veor q6, q1, q3 + vldrw.u32 q4, [r2, #384] + veor q3, q6, q4 + vldrw.u32 q4, [r2, #48] + veor q5, q3, q5 + vldrw.u32 q1, [r5, #48] + veor q1, q1, q5 + vshr.u32 q6, q1, #0x12 + vsli.32 q6, q1, #0xe + vldrw.u32 q1, [r2, #128] + veor q1, q1, q0 + vstrw.32 q6, [r10, #80] + vshr.u32 q6, q1, #0x5 + vsli.32 q6, q1, #0x1b + vldrw.u32 q1, [r5, #128] + veor q1, q1, q5 + vstrw.32 q6, [r10, #336] + vshr.u32 q6, q1, #0x4 + vsli.32 q6, q1, #0x1c + veor q1, q4, q0 + vstrw.32 q6, [r4, #336] + vshr.u32 q4, q1, #0x12 + vsli.32 q4, q1, #0xe + vldrw.u32 q6, [r2, #208] + veor q6, q6, q0 + vstrw.32 q4, [r4, #80] + vshr.u32 q1, q6, #0x14 + vsli.32 q1, q6, #0xc + vldrw.u32 q4, [r2, #288] + veor q4, q4, q0 + vldrw.u32 q6, [r2, #368] + veor q0, q6, q0 + vshr.u32 q6, q0, #0x4 + vstrw.32 q1, [r10, #192] + vsli.32 q6, q0, #0x1c + vshr.u32 q0, q4, #0x16 + vldrw.u32 q1, [r5, #368] + vsli.32 q0, q4, #0xa + vstrw.32 q6, [r4, #304] + veor q4, q1, q5 + vstrw.32 q0, [r10, #48] + vshr.u32 q1, q4, #0x4 + vsli.32 q1, q4, #0x1c + vldrw.u32 q6, [r5, #208] + veor q6, q6, q5 + vldrw.u32 q0, [r5, #288] + veor q5, q0, q5 + vstrw.32 q1, [r10, #304] + vshr.u32 q0, q6, #0x13 + vsli.32 q0, q6, #0xd + vldrw.u32 q1, [r5, #96] + vshr.u32 q6, q5, #0x15 + vldrw.u32 q4, [r5, #16] + vsli.32 q6, q5, #0xb + vldrw.u32 q5, [r5, #176] + veor q1, q4, q1 + vldrw.u32 q4, [r5, #256] + veor q5, q1, q5 + vldrw.u32 q1, [r5, #336] + veor q5, q5, q4 + vstrw.32 q0, [r4, #192] + veor q0, q5, q1 + vstrw.32 q6, [r4, #48] + vshr.u32 q5, q0, #0x1f + vsli.32 q5, q0, #0x1 + vldrw.u32 q4, [r2, #16] + veor q3, q5, q3 + vldrw.u32 q6, [r2, #96] + veor q4, q4, q6 + vldrw.u32 q1, [r2, #176] + veor q5, q4, q1 + vldrw.u32 q6, [r2, #256] + veor q6, q5, q6 + vldrw.u32 q4, [r2, #336] + veor q5, q6, q4 + vldrw.u32 q1, [r5] + veor q7, q5, q7 + vldrw.u32 q4, [r2] + veor q1, q1, q7 + veor q4, q4, q3 + vshr.u32 q6, q1, #0x20 + vsli.32 q6, q1, #0x0 + vldrw.u32 q1, [r2, #80] + veor q1, q1, q3 + vstrw.32 q6, [r10] + vshr.u32 q6, q4, #0x20 + vsli.32 q6, q4, #0x0 + vldrw.u32 q4, [r5, #80] + veor q4, q4, q7 + vstrw.32 q6, [r4] + vshr.u32 q6, q1, #0xe + vsli.32 q6, q1, #0x12 + vldrw.u32 q1, [r2, #160] + veor q1, q1, q3 + vstrw.32 q6, [r4, #256] + vshr.u32 q6, q4, #0xe + vsli.32 q6, q4, #0x12 + vldrw.u32 q4, [r2, #240] + veor q4, q4, q3 + vstrw.32 q6, [r10, #256] + vshr.u32 q6, q1, #0x1f + vsli.32 q6, q1, #0x1 + vldrw.u32 q1, [r2, #320] + veor q1, q1, q3 + vstrw.32 q6, [r10, #112] + vshr.u32 q6, q4, #0xc + vsli.32 q6, q4, #0x14 + vldrw.u32 q3, [r5, #240] + veor q3, q3, q7 + vstrw.32 q6, [r10, #368] + vshr.u32 q4, q3, #0xb + vsli.32 q4, q3, #0x15 + vldrw.u32 q3, [r5, #160] + veor q6, q3, q7 + vstrw.32 q4, [r4, #368] + vshr.u32 q3, q6, #0x1e + vsli.32 q3, q6, #0x2 + vldrw.u32 q6, [r5, #320] + veor q7, q6, q7 + vldrw.u32 q4, [r2, #368] + vshr.u32 q6, q1, #0x17 + vstrw.32 q3, [r4, #112] + vsli.32 q6, q1, #0x9 + vshr.u32 q1, q7, #0x17 + vldrw.u32 q3, [r2, #48] + vsli.32 q1, q7, #0x9 + vldrw.u32 q7, [r2, #128] + veor q3, q3, q7 + vldrw.u32 q7, [r2, #208] + veor q7, q3, q7 + vldrw.u32 q3, [r2, #288] + veor q3, q7, q3 + vldrw.u32 q7, [r5, #128] + veor q3, q3, q4 + vldrw.u32 q4, [r5, #48] + veor q0, q3, q0 + veor q4, q4, q7 + vldrw.u32 q7, [r5, #208] + veor q4, q4, q7 + vldrw.u32 q7, [r5, #288] + veor q4, q4, q7 + vldrw.u32 q7, [r5, #368] + veor q7, q4, q7 + vstrw.32 q6, [r4, #224] + vshr.u32 q4, q7, #0x1f + vstrw.32 q1, [r10, #224] + vsli.32 q4, q7, #0x1 + veor q5, q4, q5 + vldrw.u32 q6, [r2, #192] + veor q1, q6, q5 + vldrw.u32 q4, [r5, #112] + veor q7, q2, q7 + vldrw.u32 q6, [r5, #32] + vshr.u32 q2, q1, #0xb + vsli.32 q2, q1, #0x15 + veor q1, q6, q0 + vstrw.32 q2, [r10, #32] + vshr.u32 q6, q1, #0x1 + vsli.32 q6, q1, #0x1f + vldrw.u32 q2, [r2, #112] + veor q2, q2, q5 + vstrw.32 q6, [r10, #320] + vshr.u32 q1, q2, #0x1d + vsli.32 q1, q2, #0x3 + vldrw.u32 q6, [r2, #32] + veor q4, q4, q0 + vstrw.32 q1, [r4, #176] + veor q2, q6, q5 + vshr.u32 q6, q2, #0x1 + vldrw.u32 q1, [r5, #352] + vsli.32 q6, q2, #0x1f + veor q1, q1, q0 + vstrw.32 q6, [r4, #320] + vshr.u32 q6, q1, #0x1 + vsli.32 q6, q1, #0x1f + vldrw.u32 q2, [r5, #192] + vshr.u32 q1, q4, #0x1d + vstrw.32 q6, [r4, #144] + vsli.32 q1, q4, #0x3 + veor q2, q2, q0 + vldrw.u32 q6, [r5, #272] + veor q0, q6, q0 + vldrw.u32 q4, [r2, #352] + veor q6, q4, q5 + vldrw.u32 q4, [r2, #272] + veor q4, q4, q5 + vstrw.32 q1, [r10, #176] + vshr.u32 q1, q2, #0xa + vsli.32 q1, q2, #0x16 + vldrw.u32 q5, [sp] + vshr.u32 q2, q0, #0x18 + vstrw.32 q1, [r4, #32] + vsli.32 q2, q0, #0x8 + vshr.u32 q1, q6, #0x2 + vstrw.32 q2, [r4, #288] + vsli.32 q1, q6, #0x1e + vshr.u32 q6, q4, #0x19 + vstrw.32 q1, [r10, #144] + vsli.32 q6, q4, #0x7 + vshr.u32 q0, q5, #0x1f + vstrw.32 q6, [r10, #288] + vsli.32 q0, q5, #0x1 + veor q5, q0, q3 + vldrw.u32 q6, [r2, #64] + veor q3, q6, q5 + vldrw.u32 q1, [r5, #64] + vshr.u32 q4, q3, #0x13 + vldrw.u32 q2, [r2, #384] + vsli.32 q4, q3, #0xd + vldrw.u32 q0, [r5, #224] + veor q6, q1, q7 + vstrw.32 q4, [r10, #240] + veor q2, q2, q5 + veor q3, q0, q7 + vldrw.u32 q0, [r2, #224] + vshr.u32 q4, q6, #0x12 + vldrw.u32 q1, [r5, #384] + vsli.32 q4, q6, #0xe + vshr.u32 q6, q2, #0x19 + vstrw.32 q4, [r4, #240] + vsli.32 q6, q2, #0x7 + vshr.u32 q2, q3, #0xc + vstrw.32 q6, [r4, #64] + vsli.32 q2, q3, #0x14 + veor q0, q0, q5 + vldrw.u32 q6, [r2, #144] + veor q4, q1, q7 + veor q6, q6, q5 + vstrw.32 q2, [r4, #352] + vshr.u32 q2, q4, #0x19 + vsli.32 q2, q4, #0x7 + vldrw.u32 q1, [r2, #304] + veor q5, q1, q5 + vldrw.u32 q1, [r5, #144] + veor q4, q1, q7 + vldrw.u32 q3, [r5, #304] + veor q1, q3, q7 + vstrw.32 q2, [r10, #64] + vshr.u32 q3, q0, #0xd + vsli.32 q3, q0, #0x13 + vldrw.u32 q7, [r4, #80] + vshr.u32 q0, q6, #0x16 + vstrw.32 q3, [r10, #352] + vsli.32 q0, q6, #0xa + vshr.u32 q2, q5, #0x1c + vsli.32 q2, q5, #0x4 + vldrw.u32 q5, [r4, #112] + vshr.u32 q3, q1, #0x1c + vsli.32 q3, q1, #0x4 + vldrw.u32 q1, [r4, #128] + vbic q6, q5, q0 + vstrw.32 q3, [r10, #208] + vbic q3, q1, q5 + veor q3, q0, q3 + vstrw.32 q3, [r2, #96] + vbic q3, q0, q7 + veor q0, q7, q6 + vldrw.u32 q6, [r4, #144] + vbic q7, q7, q6 + vstrw.32 q0, [r2, #80] + veor q3, q6, q3 + vstrw.32 q3, [r2, #144] + veor q0, q1, q7 + vstrw.32 q0, [r2, #128] + vbic q1, q6, q1 + vshr.u32 q6, q4, #0x16 + vldrw.u32 q3, [r10, #112] + vsli.32 q6, q4, #0xa + vldrw.u32 q4, [r10, #80] + veor q1, q5, q1 + vldrw.u32 q0, [r10, #144] + vbic q7, q4, q0 + vldrw.u32 q5, [r10, #128] + veor q7, q5, q7 + vstrw.32 q1, [r2, #112] + vbic q1, q0, q5 + vstrw.32 q7, [r5, #128] + veor q7, q3, q1 + vstrw.32 q7, [r5, #112] + vbic q7, q5, q3 + vbic q1, q3, q6 + vldrw.u32 q3, [r4, #176] + veor q5, q4, q1 + vbic q4, q6, q4 + vldrw.u32 q1, [r4, #160] + veor q0, q0, q4 + vldrw.u32 q4, [r4, #224] + veor q7, q6, q7 + vstrw.32 q0, [r5, #144] + vbic q0, q1, q4 + vstrw.32 q7, [r5, #96] + veor q0, q2, q0 + vstrw.32 q0, [r2, #208] + vbic q6, q3, q1 + vstrw.32 q5, [r5, #80] + vbic q7, q4, q2 + vldrw.u32 q0, [r10, #160] + veor q6, q4, q6 + vldrw.u32 q5, [r4, #192] + vbic q4, q2, q5 + vldrw.u32 q2, [r10, #224] + veor q4, q3, q4 + vstrw.32 q4, [r2, #176] + vbic q4, q5, q3 + vstrw.32 q6, [r2, #224] + veor q4, q1, q4 + vldrw.u32 q1, [r10, #208] + veor q3, q5, q7 + vldrw.u32 q5, [r10, #192] + vbic q6, q1, q5 + vldrw.u32 q7, [r10, #176] + veor q6, q7, q6 + vstrw.32 q3, [r2, #192] + vbic q3, q0, q2 + vstrw.32 q6, [r5, #176] + veor q3, q1, q3 + vstrw.32 q3, [r5, #208] + vbic q3, q5, q7 + vstrw.32 q4, [r2, #160] + veor q3, q0, q3 + vstrw.32 q3, [r5, #160] + vbic q6, q2, q1 + vldrw.u32 q1, [r4, #288] + vbic q7, q7, q0 + vldrw.u32 q3, [r4, #272] + veor q0, q5, q6 + vldrw.u32 q4, [r4, #304] + veor q6, q2, q7 + vldrw.u32 q7, [r4, #256] + vbic q5, q4, q1 + vstrw.32 q0, [r5, #192] + veor q5, q3, q5 + vstrw.32 q6, [r5, #224] + vbic q0, q3, q7 + vstrw.32 q5, [r2, #272] + vbic q6, q1, q3 + veor q5, q7, q6 + vldrw.u32 q3, [r4, #240] + veor q6, q3, q0 + vldrw.u32 q2, [r10, #288] + vbic q0, q3, q4 + vstrw.32 q6, [r2, #240] + vbic q7, q7, q3 + vstrw.32 q5, [r2, #256] + veor q7, q4, q7 + vstrw.32 q7, [r2, #304] + veor q7, q1, q0 + vstrw.32 q7, [r2, #288] + vldrw.u32 q5, [r10, #304] + vbic q7, q5, q2 + vldrw.u32 q3, [r10, #272] + veor q1, q3, q7 + vldrw.u32 q7, [r4, #336] + vbic q4, q2, q3 + vldrw.u32 q6, [r10, #256] + vbic q3, q3, q6 + vldrw.u32 q0, [r10, #240] + veor q3, q0, q3 + vstrw.32 q1, [r5, #272] + vbic q1, q0, q5 + vstrw.32 q3, [r5, #240] + veor q1, q2, q1 + vldrw.u32 q3, [r4, #384] + vbic q2, q6, q0 + vldrw.u32 q0, [r4, #320] + veor q2, q5, q2 + vldrw.u32 q5, [r4, #352] + veor q4, q6, q4 + vstrw.32 q2, [r5, #304] + vbic q2, q7, q0 + vstrw.32 q1, [r5, #288] + veor q1, q3, q2 + vstrw.32 q1, [r2, #384] + vbic q2, q5, q7 + vstrw.32 q4, [r5, #256] + veor q4, q0, q2 + vstrw.32 q4, [r2, #320] + vbic q2, q0, q3 + vldrw.u32 q4, [r4, #368] + vbic q3, q3, q4 + vldrw.u32 q0, [r10, #320] + veor q1, q5, q3 + vldrw.u32 q6, [r10, #336] + vbic q5, q4, q5 + vstrw.32 q1, [r2, #352] + veor q5, q7, q5 + vstrw.32 q5, [r2, #336] + veor q3, q4, q2 + vstrw.32 q3, [r2, #368] + vbic q7, q6, q0 + vldrw.u32 q5, [r10, #352] + vbic q3, q5, q6 + vldrw.u32 q1, [r10, #368] + vbic q4, q1, q5 + vldrw.u32 q2, [r4, #16] + veor q6, q6, q4 + vldrw.u32 q4, [r10, #384] + veor q3, q0, q3 + vstrw.32 q3, [r5, #320] + veor q3, q4, q7 + vstrw.32 q3, [r5, #384] + vbic q0, q0, q4 + vstrw.32 q6, [r5, #336] + veor q3, q1, q0 + vstrw.32 q3, [r5, #368] + vbic q7, q4, q1 + veor q5, q5, q7 + vldrw.u32 q6, [r4, #32] + vbic q3, q6, q2 + vldrw.u32 q4, [r4, #48] + vbic q0, q4, q6 + vldrw.u32 q1, [r4] + veor q0, q2, q0 + vldrw.u32 q7, [r4, #64] + veor q3, q1, q3 + vstrw.32 q5, [r5, #352] + vbic q5, q1, q7 + vstrw.32 q0, [r2, #16] + veor q0, q4, q5 + vstrw.32 q0, [r2, #48] + vbic q5, q2, q1 + veor q2, q7, q5 + vldrw.u32 q0, [r10, #16] + vbic q5, q7, q4 + vldrw.u32 q4, [r10] + vbic q1, q0, q4 + vldrw.u32 q7, [r10, #64] + veor q1, q7, q1 + vstrw.32 q2, [r2, #64] + veor q2, q6, q5 + vbic q6, q4, q7 + vldrw.u32 q5, [r10, #48] + veor q6, q5, q6 + ldrd r7, r8, [r6] + vbic q7, q7, q5 + vstrw.32 q1, [r5, #64] + vdup.32 q1, r7 + veor q1, q3, q1 + vldrw.u32 q3, [r10, #32] + veor q7, q3, q7 + add.w r6, r6, #0x8 + vbic q5, q5, q3 + vstrw.32 q6, [r5, #48] + vbic q6, q3, q0 + vstrw.32 q1, [r2] + veor q5, q0, q5 + vstrw.32 q7, [r5, #32] + veor q4, q4, q6 + vstrw.32 q5, [r5, #16] + vdup.32 q6, r8 + vstrw.32 q2, [r2, #32] + veor q0, q4, q6 + vstrw.32 q0, [r5] + +Lmlk_keccak_f1600_x4_mve_asm_roundend_pre: + le lr, Lmlk_keccak_f1600_x4_mve_asm_roundstart @ imm = #-0x8c0 + +Lmlk_keccak_f1600_x4_mve_asm_roundend: + add sp, #0x80 + .cfi_adjust_cfa_offset -0x80 + vpop {d8, d9, d10, d11, d12, d13, d14, d15} + .cfi_restore d8 + .cfi_restore d9 + .cfi_restore d10 + .cfi_restore d11 + .cfi_restore d12 + .cfi_restore d13 + .cfi_restore d14 + .cfi_restore d15 + .cfi_adjust_cfa_offset -0x40 + pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc} + .cfi_restore r4 + .cfi_restore r5 + .cfi_restore r6 + .cfi_restore r7 + .cfi_restore r8 + .cfi_restore r9 + .cfi_restore r10 + .cfi_restore r11 + .cfi_restore lr + .cfi_adjust_cfa_offset -0x24 + .cfi_endproc + nop + +MLK_ASM_FN_SIZE(keccak_f1600_x4_mve_asm) + +#endif /* MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.c b/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.c new file mode 100644 index 0000000000..d0c06b5324 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.c @@ -0,0 +1,40 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#include "../../../../common.h" +#include "../../../../verify.h" + +#if defined(MLK_FIPS202_ARMV81M_NEED_X4) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "fips202_native_armv81m.h" + + +/* + * Keccak-f1600 x4 permutation (on bit-interleaved state) + * State is expected to already be in bit-interleaved format. + */ +#define mlk_keccak_f1600_x4_native_impl \ + MLK_NAMESPACE(keccak_f1600_x4_native_impl) +int mlk_keccak_f1600_x4_native_impl(uint64_t *state) +{ + MLK_ALIGN uint64_t state_tmp[100]; + mlk_keccak_f1600_x4_mve_asm(state, state_tmp, + mlk_keccakf1600_round_constants); + mlk_zeroize(state_tmp, sizeof(state_tmp)); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#else /* MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(keccak_f1600_x4_mve) + +#endif /* !(MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED) \ + */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +/* Some macros are kept because they are also defined in a header. */ +/* Keep: mlk_keccak_f1600_x4_native_impl (mve.h) */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_state_extract_bytes_mve.S b/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_state_extract_bytes_mve.S new file mode 100644 index 0000000000..43aaa95c53 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_state_extract_bytes_mve.S @@ -0,0 +1,334 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * Copyright (c) 2026 Arm Limited + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// --------------------------------------------------------------------------- +// Overview +// --------------------------------------------------------------------------- +// MVE/Helium implementation of KeccakF1600x4_StateExtractBytes +// (inverse of keccak_f1600_x4_state_xor_bytes_mve.S). +// +// void KeccakF1600x4_StateExtractBytes(state, d0, d1, d2, d3, offset, length) +// +// Reads 'length' bytes from the bit-interleaved Keccak state starting at +// byte 'offset', recombines the even and odd halves of each lane back +// into plain bytes, and writes them to four output buffers (d0..d3). +// +// --------------------------------------------------------------------------- +// Bit-interleaving background +// --------------------------------------------------------------------------- +// Each 64-bit Keccak lane is stored as two 32-bit words: +// even half -- bits 0, 2, 4, ..., 62 of the lane +// odd half -- bits 1, 3, 5, ..., 63 of the lane +// This representation allows 64-bit lane rotations (used in the Keccak +// round function) to be implemented as pairs of 32-bit rotations. +// +// Batched (x4) processing: +// Four Keccak instances are processed as a batch. Their states are +// stored interleaved in a single 800-byte buffer: first the even +// halves of all 25 lanes (400 bytes), then the odd halves (400 bytes). +// Within each 16-byte row, the four u32 words correspond to +// instances 0..3 of the same lane, enabling SIMD-parallel operations +// across all four instances. +// +// State memory layout (25 lanes x 4 instances x 2 halves): +// S[i][l]_even/odd = even/odd half of lane l, instance i (u32) +// Each row is 16 bytes (one Q-register). +// Offset Contents +// 0 S[0][ 0]_even, S[1][ 0]_even, S[2][ 0]_even, S[3][ 0]_even +// 16 S[0][ 1]_even, S[1][ 1]_even, S[2][ 1]_even, S[3][ 1]_even +// ... +// 384 S[0][24]_even, S[1][24]_even, S[2][24]_even, S[3][24]_even +// 400 S[0][ 0]_odd, S[1][ 0]_odd, S[2][ 0]_odd, S[3][ 0]_odd +// 416 S[0][ 1]_odd, S[1][ 1]_odd, S[2][ 1]_odd, S[3][ 1]_odd +// ... +// 784 S[0][24]_odd, S[1][24]_odd, S[2][24]_odd, S[3][24]_odd +// +// --------------------------------------------------------------------------- +// Three-phase structure +// --------------------------------------------------------------------------- +// Prologue -- if offset is not 8-byte aligned, extract +// min(length, 8-(offset%8)) bytes via predicated byte stores. +// Main -- process full 8-byte groups: load even/odd lane pair, +// de-interleave, scatter-store to output buffers. +// Tail -- extract remaining <8 bytes via predicated byte stores. + +#include "../../../../common.h" +#if defined(MLK_FIPS202_ARMV81M_NEED_X4) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/armv81m/src/keccak_f1600_x4_state_extract_bytes_mve.S using scripts/simpasm. Do not modify it directly. + */ + +.thumb +.syntax unified + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x4_state_extract_bytes_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x4_state_extract_bytes_asm) + + .cfi_startproc + push.w {r4, r5, r6, r7, r8, r9, r10, r11, r12, lr} + .cfi_adjust_cfa_offset 0x28 + .cfi_rel_offset r4, 0x0 + .cfi_rel_offset r5, 0x4 + .cfi_rel_offset r6, 0x8 + .cfi_rel_offset r7, 0xc + .cfi_rel_offset r8, 0x10 + .cfi_rel_offset r9, 0x14 + .cfi_rel_offset r10, 0x18 + .cfi_rel_offset r11, 0x1c + .cfi_rel_offset lr, 0x24 + vpush {d8, d9, d10, d11, d12, d13, d14, d15} + .cfi_adjust_cfa_offset 0x40 + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + ldr r4, [sp, #0x68] + ldr.w r10, [sp, #0x6c] + ldr r6, [sp, #0x70] + cmp r6, #0x0 + beq.w Lmlk_keccak_f1600_x4_state_extract_bytes_asm_exit @ imm = #0x2ea + and r5, r10, #0x7 + bic r9, r10, #0x7 + add.w r8, r0, r9, lsl #1 + add.w r7, r8, #0x190 + cmp r5, #0x0 + beq.w Lmlk_keccak_f1600_x4_state_extract_bytes_asm_pre_main @ imm = #0x112 + vldrw.u32 q0, [r8], #16 + vldrw.u32 q1, [r7], #16 + vrev32.16 q2, q0 + vrev32.16 q3, q1 + vsli.32 q0, q0, #0x8 + vsli.16 q0, q0, #0x4 + vsli.8 q0, q0, #0x1 + vshr.u8 q4, q0, #0x3 + vsli.8 q0, q4, #0x4 + vshr.u8 q4, q0, #0x5 + vsli.8 q0, q4, #0x6 + vsli.32 q1, q1, #0x8 + vsli.16 q1, q1, #0x4 + vsli.8 q1, q1, #0x1 + vshr.u8 q4, q1, #0x3 + vsli.8 q1, q4, #0x4 + vshr.u8 q4, q1, #0x5 + vsli.8 q1, q4, #0x6 + mov.w r0, #0x55 + vdup.8 q4, r0 + vand q0, q0, q4 + vand q1, q1, q4 + vshl.i32 q1, q1, #0x1 + vorr q0, q0, q1 + vsli.32 q2, q2, #0x8 + vsli.16 q2, q2, #0x4 + vsli.8 q2, q2, #0x1 + vshr.u8 q1, q2, #0x3 + vsli.8 q2, q1, #0x4 + vshr.u8 q1, q2, #0x5 + vsli.8 q2, q1, #0x6 + vsli.32 q3, q3, #0x8 + vsli.16 q3, q3, #0x4 + vsli.8 q3, q3, #0x1 + vshr.u8 q1, q3, #0x3 + vsli.8 q3, q1, #0x4 + vshr.u8 q1, q3, #0x5 + vsli.8 q3, q1, #0x6 + vand q1, q2, q4 + vand q3, q3, q4 + vshl.i32 q3, q3, #0x1 + vorr q1, q1, q3 + vrev64.32 q2, q0 + vrev64.32 q3, q1 + movw r0, #0xf0f + vmsr p0, r0 + vpsel q0, q0, q3 + vpsel q1, q2, q1 + vmov.f64 d4, d1 + vmov.f64 d6, d3 + rsb.w lr, r5, #0x8 + cmp r6, lr + it ls + movls lr, r6 + vctp.8 lr + vmrs r11, p0 + lsl.w r11, r11, r5 + vmsr p0, r11 + subs r1, r1, r5 + subs r2, r2, r5 + subs r3, r3, r5 + subs r4, r4, r5 + vpstttt + vstrbt.8 q0, [r1], #4 + vstrbt.8 q1, [r2], #4 + vstrbt.8 q2, [r3], #4 + vstrbt.8 q3, [r4], #4 + subs.w r6, r6, lr + cmp r6, #0x0 + beq.w Lmlk_keccak_f1600_x4_state_extract_bytes_asm_exit @ imm = #0x1cc + vmov q7[2], q7[0], r1, r3 + vmov q7[3], q7[1], r2, r4 + b Lmlk_keccak_f1600_x4_state_extract_bytes_asm_main_body @ imm = #0xe + +Lmlk_keccak_f1600_x4_state_extract_bytes_asm_pre_main: + vmov q7[2], q7[0], r1, r3 + vmov q7[3], q7[1], r2, r4 + mov.w r12, #0x4 + vsub.i32 q7, q7, r12 + +Lmlk_keccak_f1600_x4_state_extract_bytes_asm_main_body: + lsr.w lr, r6, #0x3 + wls lr, lr, Lmlk_keccak_f1600_x4_state_extract_bytes_asm_main_loop_end @ imm = #0xb4 + +Lmlk_keccak_f1600_x4_state_extract_bytes_asm_main_loop_start: + vldrw.u32 q0, [r8], #16 + vldrw.u32 q1, [r7], #16 + vrev32.16 q2, q0 + vrev32.16 q3, q1 + vsli.32 q0, q0, #0x8 + vsli.16 q0, q0, #0x4 + vsli.8 q0, q0, #0x1 + vshr.u8 q4, q0, #0x3 + vsli.8 q0, q4, #0x4 + vshr.u8 q4, q0, #0x5 + vsli.8 q0, q4, #0x6 + vsli.32 q1, q1, #0x8 + vsli.16 q1, q1, #0x4 + vsli.8 q1, q1, #0x1 + vshr.u8 q4, q1, #0x3 + vsli.8 q1, q4, #0x4 + vshr.u8 q4, q1, #0x5 + vsli.8 q1, q4, #0x6 + mov.w r0, #0x55 + vdup.8 q4, r0 + vand q0, q0, q4 + vand q1, q1, q4 + vshl.i32 q1, q1, #0x1 + vorr q0, q0, q1 + vsli.32 q2, q2, #0x8 + vsli.16 q2, q2, #0x4 + vsli.8 q2, q2, #0x1 + vshr.u8 q1, q2, #0x3 + vsli.8 q2, q1, #0x4 + vshr.u8 q1, q2, #0x5 + vsli.8 q2, q1, #0x6 + vsli.32 q3, q3, #0x8 + vsli.16 q3, q3, #0x4 + vsli.8 q3, q3, #0x1 + vshr.u8 q1, q3, #0x3 + vsli.8 q3, q1, #0x4 + vshr.u8 q1, q3, #0x5 + vsli.8 q3, q1, #0x6 + vand q1, q2, q4 + vand q3, q3, q4 + vshl.i32 q3, q3, #0x1 + vorr q1, q1, q3 + vstrw.32 q0, [q7, #4]! + vstrw.32 q1, [q7, #4]! + le lr, Lmlk_keccak_f1600_x4_state_extract_bytes_asm_main_loop_start @ imm = #-0xb4 + +Lmlk_keccak_f1600_x4_state_extract_bytes_asm_main_loop_end: + ands r6, r6, #0x7 + beq Lmlk_keccak_f1600_x4_state_extract_bytes_asm_exit @ imm = #0xee + mov.w r12, #0x4 + vadd.i32 q7, q7, r12 + vmov r1, r3, q7[2], q7[0] + vmov r2, r4, q7[3], q7[1] + vldrw.u32 q0, [r8], #16 + vldrw.u32 q1, [r7], #16 + vrev32.16 q2, q0 + vrev32.16 q3, q1 + vsli.32 q0, q0, #0x8 + vsli.16 q0, q0, #0x4 + vsli.8 q0, q0, #0x1 + vshr.u8 q4, q0, #0x3 + vsli.8 q0, q4, #0x4 + vshr.u8 q4, q0, #0x5 + vsli.8 q0, q4, #0x6 + vsli.32 q1, q1, #0x8 + vsli.16 q1, q1, #0x4 + vsli.8 q1, q1, #0x1 + vshr.u8 q4, q1, #0x3 + vsli.8 q1, q4, #0x4 + vshr.u8 q4, q1, #0x5 + vsli.8 q1, q4, #0x6 + mov.w r0, #0x55 + vdup.8 q4, r0 + vand q0, q0, q4 + vand q1, q1, q4 + vshl.i32 q1, q1, #0x1 + vorr q0, q0, q1 + vsli.32 q2, q2, #0x8 + vsli.16 q2, q2, #0x4 + vsli.8 q2, q2, #0x1 + vshr.u8 q1, q2, #0x3 + vsli.8 q2, q1, #0x4 + vshr.u8 q1, q2, #0x5 + vsli.8 q2, q1, #0x6 + vsli.32 q3, q3, #0x8 + vsli.16 q3, q3, #0x4 + vsli.8 q3, q3, #0x1 + vshr.u8 q1, q3, #0x3 + vsli.8 q3, q1, #0x4 + vshr.u8 q1, q3, #0x5 + vsli.8 q3, q1, #0x6 + vand q1, q2, q4 + vand q3, q3, q4 + vshl.i32 q3, q3, #0x1 + vorr q1, q1, q3 + vrev64.32 q2, q0 + vrev64.32 q3, q1 + movw r0, #0xf0f + vmsr p0, r0 + vpsel q0, q0, q3 + vpsel q1, q2, q1 + vmov.f64 d4, d1 + vmov.f64 d6, d3 + vctp.8 r6 + vpstttt + vstrbt.8 q0, [r1], #4 + vstrbt.8 q1, [r2], #4 + vstrbt.8 q2, [r3], #4 + vstrbt.8 q3, [r4], #4 + +Lmlk_keccak_f1600_x4_state_extract_bytes_asm_exit: + vpop {d8, d9, d10, d11, d12, d13, d14, d15} + .cfi_restore d8 + .cfi_restore d9 + .cfi_restore d10 + .cfi_restore d11 + .cfi_restore d12 + .cfi_restore d13 + .cfi_restore d14 + .cfi_restore d15 + .cfi_adjust_cfa_offset -0x40 + pop.w {r4, r5, r6, r7, r8, r9, r10, r11, r12, pc} + .cfi_restore r4 + .cfi_restore r5 + .cfi_restore r6 + .cfi_restore r7 + .cfi_restore r8 + .cfi_restore r9 + .cfi_restore r10 + .cfi_restore r11 + .cfi_restore lr + .cfi_adjust_cfa_offset -0x28 + .cfi_endproc + +MLK_ASM_FN_SIZE(keccak_f1600_x4_state_extract_bytes_asm) + +#endif /* MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_state_xor_bytes_mve.S b/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_state_xor_bytes_mve.S new file mode 100644 index 0000000000..fd58e7cbe6 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_state_xor_bytes_mve.S @@ -0,0 +1,355 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * Copyright (c) 2026 Arm Limited + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// --------------------------------------------------------------------------- +// Overview +// --------------------------------------------------------------------------- +// MVE/Helium implementation of KeccakF1600x4_StateXORBytes. +// +// void KeccakF1600x4_StateXORBytes(state, d0, d1, d2, d3, offset, length) +// +// Reads 'length' plain bytes from each of four input buffers (d0..d3), +// splits every byte into its even and odd bits (bit-interleaving), and +// XORs the result into the Keccak state starting at byte 'offset'. +// +// --------------------------------------------------------------------------- +// Bit-interleaving background +// --------------------------------------------------------------------------- +// Each 64-bit Keccak lane is stored as two 32-bit words: +// even half -- bits 0, 2, 4, ..., 62 of the lane +// odd half -- bits 1, 3, 5, ..., 63 of the lane +// This representation allows 64-bit lane rotations (used in the Keccak +// round function) to be implemented as pairs of 32-bit rotations. +// +// Batched (x4) processing: +// Four Keccak instances are processed as a batch. Their states are +// stored interleaved in a single 800-byte buffer: first the even +// halves of all 25 lanes (400 bytes), then the odd halves (400 bytes). +// Within each 16-byte row, the four u32 words correspond to +// instances 0..3 of the same lane, enabling SIMD-parallel operations +// across all four instances. +// +// State memory layout (25 lanes x 4 instances x 2 halves): +// S[i][l]_even/odd = even/odd half of lane l, instance i (u32) +// Each row is 16 bytes (one Q-register). +// Offset Contents +// 0 S[0][ 0]_even, S[1][ 0]_even, S[2][ 0]_even, S[3][ 0]_even +// 16 S[0][ 1]_even, S[1][ 1]_even, S[2][ 1]_even, S[3][ 1]_even +// ... +// 384 S[0][24]_even, S[1][24]_even, S[2][24]_even, S[3][24]_even +// 400 S[0][ 0]_odd, S[1][ 0]_odd, S[2][ 0]_odd, S[3][ 0]_odd +// 416 S[0][ 1]_odd, S[1][ 1]_odd, S[2][ 1]_odd, S[3][ 1]_odd +// ... +// 784 S[0][24]_odd, S[1][24]_odd, S[2][24]_odd, S[3][24]_odd +// +// --------------------------------------------------------------------------- +// Three-phase structure +// --------------------------------------------------------------------------- +// Prologue -- if offset is not 8-byte aligned, absorb +// min(length, 8-(offset%8)) bytes via predicated byte loads. +// Main -- process full 8-byte groups via word-level gather loads, +// bit-interleave, then VEOR into even/odd state halves. +// Tail -- absorb remaining <8 bytes via predicated byte loads. + +#include "../../../../common.h" +#if defined(MLK_FIPS202_ARMV81M_NEED_X4) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/armv81m/src/keccak_f1600_x4_state_xor_bytes_mve.S using scripts/simpasm. Do not modify it directly. + */ + +.thumb +.syntax unified + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x4_state_xor_bytes_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x4_state_xor_bytes_asm) + + .cfi_startproc + push.w {r4, r5, r6, r7, r8, r9, r10, r11, r12, lr} + .cfi_adjust_cfa_offset 0x28 + .cfi_rel_offset r4, 0x0 + .cfi_rel_offset r5, 0x4 + .cfi_rel_offset r6, 0x8 + .cfi_rel_offset r7, 0xc + .cfi_rel_offset r8, 0x10 + .cfi_rel_offset r9, 0x14 + .cfi_rel_offset r10, 0x18 + .cfi_rel_offset r11, 0x1c + .cfi_rel_offset lr, 0x24 + vpush {d8, d9, d10, d11, d12, d13, d14, d15} + .cfi_adjust_cfa_offset 0x40 + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + ldr r4, [sp, #0x68] + ldr.w r10, [sp, #0x6c] + ldr r6, [sp, #0x70] + cmp r6, #0x0 + beq.w Lmlk_keccak_f1600_x4_state_xor_bytes_asm_exit @ imm = #0x346 + and r5, r10, #0x7 + bic r9, r10, #0x7 + add.w r8, r0, r9, lsl #1 + add.w r7, r8, #0x190 + cmp r5, #0x0 + beq.w Lmlk_keccak_f1600_x4_state_xor_bytes_asm_pre_main @ imm = #0x12c + subs r1, r1, r5 + subs r2, r2, r5 + subs r3, r3, r5 + subs r4, r4, r5 + rsb.w lr, r5, #0x8 + cmp r6, lr + it ls + movls lr, r6 + subs.w r6, r6, lr + vctp.8 lr + vmrs r11, p0 + lsl.w r11, r11, r5 + vmsr p0, r11 + vpstttt + vldrbt.u8 q0, [r1], #4 + vldrbt.u8 q1, [r2], #4 + vldrbt.u8 q2, [r3], #4 + vldrbt.u8 q3, [r4], #4 + vmov.f64 d1, d4 + vmov.f64 d3, d6 + vrev64.32 q2, q0 + vrev64.32 q3, q1 + movw r0, #0xf0f + vmsr p0, r0 + vpsel q0, q0, q3 + vpsel q1, q2, q1 + vmov q2, q0 + vmov q3, q1 + vshr.u8 q4, q0, #0x2 + vsli.8 q0, q4, #0x1 + vshr.u8 q4, q0, #0x3 + vsli.8 q0, q4, #0x2 + vshr.u8 q4, q0, #0x4 + vsli.8 q0, q4, #0x3 + vshr.u16 q4, q0, #0x8 + vsli.8 q0, q4, #0x4 + vshr.u32 q4, q0, #0x10 + vsli.16 q0, q4, #0x8 + vshr.u8 q4, q3, #0x2 + vsli.8 q3, q4, #0x1 + vshr.u8 q4, q3, #0x3 + vsli.8 q3, q4, #0x2 + vshr.u8 q4, q3, #0x4 + vsli.8 q3, q4, #0x3 + vshr.u16 q4, q3, #0x8 + vsli.8 q3, q4, #0x4 + vshr.u32 q4, q3, #0x10 + vsli.16 q3, q4, #0x8 + vsli.32 q0, q3, #0x10 + vshl.i8 q4, q2, #0x2 + vsri.8 q2, q4, #0x1 + vshl.i8 q4, q2, #0x3 + vsri.8 q2, q4, #0x2 + vshl.i8 q4, q2, #0x4 + vsri.8 q2, q4, #0x3 + vshl.i16 q4, q2, #0x8 + vsri.8 q2, q4, #0x4 + vshl.i32 q4, q2, #0x10 + vsri.16 q2, q4, #0x8 + vshl.i8 q4, q1, #0x2 + vsri.8 q1, q4, #0x1 + vshl.i8 q4, q1, #0x3 + vsri.8 q1, q4, #0x2 + vshl.i8 q4, q1, #0x4 + vsri.8 q1, q4, #0x3 + vshl.i16 q4, q1, #0x8 + vsri.8 q1, q4, #0x4 + vshl.i32 q4, q1, #0x10 + vsri.16 q1, q4, #0x8 + vsri.32 q1, q2, #0x10 + vldrw.u32 q4, [r8] + vldrw.u32 q5, [r7] + veor q4, q4, q0 + veor q5, q5, q1 + vstrw.32 q4, [r8], #16 + vstrw.32 q5, [r7], #16 + vmov q7[2], q7[0], r1, r3 + vmov q7[3], q7[1], r2, r4 + b Lmlk_keccak_f1600_x4_state_xor_bytes_asm_main_body @ imm = #0xe + +Lmlk_keccak_f1600_x4_state_xor_bytes_asm_pre_main: + vmov q7[2], q7[0], r1, r3 + vmov q7[3], q7[1], r2, r4 + mov.w r0, #0x4 + vsub.i32 q7, q7, r0 + +Lmlk_keccak_f1600_x4_state_xor_bytes_asm_main_body: + lsr.w lr, r6, #0x3 + wls lr, lr, Lmlk_keccak_f1600_x4_state_xor_bytes_asm_main_loop_end @ imm = #0xd4 + +Lmlk_keccak_f1600_x4_state_xor_bytes_asm_main_loop_start: + vldrw.u32 q0, [q7, #4]! + vldrw.u32 q1, [q7, #4]! + vmov q2, q0 + vmov q3, q1 + vshr.u8 q4, q0, #0x2 + vsli.8 q0, q4, #0x1 + vshr.u8 q4, q0, #0x3 + vsli.8 q0, q4, #0x2 + vshr.u8 q4, q0, #0x4 + vsli.8 q0, q4, #0x3 + vshr.u16 q4, q0, #0x8 + vsli.8 q0, q4, #0x4 + vshr.u32 q4, q0, #0x10 + vsli.16 q0, q4, #0x8 + vshr.u8 q4, q3, #0x2 + vsli.8 q3, q4, #0x1 + vshr.u8 q4, q3, #0x3 + vsli.8 q3, q4, #0x2 + vshr.u8 q4, q3, #0x4 + vsli.8 q3, q4, #0x3 + vshr.u16 q4, q3, #0x8 + vsli.8 q3, q4, #0x4 + vshr.u32 q4, q3, #0x10 + vsli.16 q3, q4, #0x8 + vsli.32 q0, q3, #0x10 + vshl.i8 q4, q2, #0x2 + vsri.8 q2, q4, #0x1 + vshl.i8 q4, q2, #0x3 + vsri.8 q2, q4, #0x2 + vshl.i8 q4, q2, #0x4 + vsri.8 q2, q4, #0x3 + vshl.i16 q4, q2, #0x8 + vsri.8 q2, q4, #0x4 + vshl.i32 q4, q2, #0x10 + vsri.16 q2, q4, #0x8 + vshl.i8 q4, q1, #0x2 + vsri.8 q1, q4, #0x1 + vshl.i8 q4, q1, #0x3 + vsri.8 q1, q4, #0x2 + vshl.i8 q4, q1, #0x4 + vsri.8 q1, q4, #0x3 + vshl.i16 q4, q1, #0x8 + vsri.8 q1, q4, #0x4 + vshl.i32 q4, q1, #0x10 + vsri.16 q1, q4, #0x8 + vsri.32 q1, q2, #0x10 + vldrw.u32 q4, [r8] + vldrw.u32 q5, [r7] + veor q4, q4, q0 + veor q5, q5, q1 + vstrw.32 q4, [r8], #16 + vstrw.32 q5, [r7], #16 + le lr, Lmlk_keccak_f1600_x4_state_xor_bytes_asm_main_loop_start @ imm = #-0xd4 + +Lmlk_keccak_f1600_x4_state_xor_bytes_asm_main_loop_end: + ands r6, r6, #0x7 + beq.w Lmlk_keccak_f1600_x4_state_xor_bytes_asm_exit @ imm = #0x110 + mov.w r0, #0x4 + vadd.i32 q7, q7, r0 + vmov r1, r3, q7[2], q7[0] + vmov r2, r4, q7[3], q7[1] + vctp.8 r6 + vpstttt + vldrbt.u8 q0, [r1] + vldrbt.u8 q1, [r2] + vldrbt.u8 q2, [r3] + vldrbt.u8 q3, [r4] + vmov.f64 d1, d4 + vmov.f64 d3, d6 + vrev64.32 q2, q0 + vrev64.32 q3, q1 + movw r0, #0xf0f + vmsr p0, r0 + vpsel q0, q0, q3 + vpsel q1, q2, q1 + vmov q2, q0 + vmov q3, q1 + vshr.u8 q4, q0, #0x2 + vsli.8 q0, q4, #0x1 + vshr.u8 q4, q0, #0x3 + vsli.8 q0, q4, #0x2 + vshr.u8 q4, q0, #0x4 + vsli.8 q0, q4, #0x3 + vshr.u16 q4, q0, #0x8 + vsli.8 q0, q4, #0x4 + vshr.u32 q4, q0, #0x10 + vsli.16 q0, q4, #0x8 + vshr.u8 q4, q3, #0x2 + vsli.8 q3, q4, #0x1 + vshr.u8 q4, q3, #0x3 + vsli.8 q3, q4, #0x2 + vshr.u8 q4, q3, #0x4 + vsli.8 q3, q4, #0x3 + vshr.u16 q4, q3, #0x8 + vsli.8 q3, q4, #0x4 + vshr.u32 q4, q3, #0x10 + vsli.16 q3, q4, #0x8 + vsli.32 q0, q3, #0x10 + vshl.i8 q4, q2, #0x2 + vsri.8 q2, q4, #0x1 + vshl.i8 q4, q2, #0x3 + vsri.8 q2, q4, #0x2 + vshl.i8 q4, q2, #0x4 + vsri.8 q2, q4, #0x3 + vshl.i16 q4, q2, #0x8 + vsri.8 q2, q4, #0x4 + vshl.i32 q4, q2, #0x10 + vsri.16 q2, q4, #0x8 + vshl.i8 q4, q1, #0x2 + vsri.8 q1, q4, #0x1 + vshl.i8 q4, q1, #0x3 + vsri.8 q1, q4, #0x2 + vshl.i8 q4, q1, #0x4 + vsri.8 q1, q4, #0x3 + vshl.i16 q4, q1, #0x8 + vsri.8 q1, q4, #0x4 + vshl.i32 q4, q1, #0x10 + vsri.16 q1, q4, #0x8 + vsri.32 q1, q2, #0x10 + vldrw.u32 q4, [r8] + vldrw.u32 q5, [r7] + veor q4, q4, q0 + veor q5, q5, q1 + vstrw.32 q4, [r8], #16 + vstrw.32 q5, [r7], #16 + +Lmlk_keccak_f1600_x4_state_xor_bytes_asm_exit: + vpop {d8, d9, d10, d11, d12, d13, d14, d15} + .cfi_restore d8 + .cfi_restore d9 + .cfi_restore d10 + .cfi_restore d11 + .cfi_restore d12 + .cfi_restore d13 + .cfi_restore d14 + .cfi_restore d15 + .cfi_adjust_cfa_offset -0x40 + pop.w {r4, r5, r6, r7, r8, r9, r10, r11, r12, pc} + .cfi_restore r4 + .cfi_restore r5 + .cfi_restore r6 + .cfi_restore r7 + .cfi_restore r8 + .cfi_restore r9 + .cfi_restore r10 + .cfi_restore r11 + .cfi_restore lr + .cfi_adjust_cfa_offset -0x28 + .cfi_endproc + +MLK_ASM_FN_SIZE(keccak_f1600_x4_state_xor_bytes_asm) + +#endif /* MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccakf1600_round_constants.c b/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccakf1600_round_constants.c new file mode 100644 index 0000000000..b19bf33271 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccakf1600_round_constants.c @@ -0,0 +1,52 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#include "../../../../common.h" + +#if defined(MLK_FIPS202_ARMV81M_NEED_X4) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "fips202_native_armv81m.h" + +/* + * Keccak round constants in bit-interleaved form. + * Each 64-bit constant is split into two 32-bit words: + * - low word contains even-indexed bits + * - high word contains odd-indexed bits + */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint32_t + mlk_keccakf1600_round_constants[48] = { + 0x00000001, 0x00000000, /* RC0 */ + 0x00000000, 0x00000089, /* RC1 */ + 0x00000000, 0x8000008b, /* RC2 */ + 0x00000000, 0x80008080, /* RC3 */ + 0x00000001, 0x0000008b, /* RC4 */ + 0x00000001, 0x00008000, /* RC5 */ + 0x00000001, 0x80008088, /* RC6 */ + 0x00000001, 0x80000082, /* RC7 */ + 0x00000000, 0x0000000b, /* RC8 */ + 0x00000000, 0x0000000a, /* RC9 */ + 0x00000001, 0x00008082, /* RC10 */ + 0x00000000, 0x00008003, /* RC11 */ + 0x00000001, 0x0000808b, /* RC12 */ + 0x00000001, 0x8000000b, /* RC13 */ + 0x00000001, 0x8000008a, /* RC14 */ + 0x00000001, 0x80000081, /* RC15 */ + 0x00000000, 0x80000081, /* RC16 */ + 0x00000000, 0x80000008, /* RC17 */ + 0x00000000, 0x00000083, /* RC18 */ + 0x00000000, 0x80008003, /* RC19 */ + 0x00000001, 0x80008088, /* RC20 */ + 0x00000000, 0x80000088, /* RC21 */ + 0x00000001, 0x00008000, /* RC22 */ + 0x00000000, 0x80008082, /* RC23 */ +}; + +#else /* MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(fips202_armv81m_round_constants) + +#endif /* !(MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED) \ + */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/auto.h b/src/mlkem-native/mlkem/src/fips202/native/auto.h new file mode 100644 index 0000000000..e5a6819402 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/auto.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_AUTO_H +#define MLK_FIPS202_NATIVE_AUTO_H + +/* + * Default FIPS202 backend + */ +#include "../../sys.h" + +#if defined(MLK_SYS_AARCH64) +#include "aarch64/auto.h" +#endif + +/* The x86_64 backend requires toolchain support for the SysV ABI */ +#if defined(MLK_SYS_X86_64_AVX2) && defined(MLK_SYSV_ABI_SUPPORTED) +#include "x86_64/keccak_f1600_x4_avx2.h" +#endif + +/* We do not yet include the FIPS202 backend for Armv8.1-M+MVE by default + * as it is still experimental and undergoing review. */ +/* #if defined(MLK_SYS_ARMV81M_MVE) */ +/* #include "armv81m/mve.h" */ +/* #endif */ + +#endif /* !MLK_FIPS202_NATIVE_AUTO_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/x86_64/keccak_f1600_x4_avx2.h b/src/mlkem-native/mlkem/src/fips202/native/x86_64/keccak_f1600_x4_avx2.h new file mode 100644 index 0000000000..0a16b3782d --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/x86_64/keccak_f1600_x4_avx2.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_X86_64_KECCAK_F1600_X4_AVX2_H +#define MLK_FIPS202_NATIVE_X86_64_KECCAK_F1600_X4_AVX2_H + +#include "../../../common.h" + +#define MLK_FIPS202_X86_64_NEED_X4_AVX2 + +/* Part of backend API */ +#define MLK_USE_NATIVE_FIPS202_X4 + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_x86_64.h" +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x4_native(uint64_t *state) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_keccak_f1600_x4_avx2_asm(state, mlk_keccakf1600_round_constants, + mlk_keccak_rho8, mlk_keccak_rho56); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_FIPS202_NATIVE_X86_64_KECCAK_F1600_X4_AVX2_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/x86_64/src/fips202_native_x86_64.h b/src/mlkem-native/mlkem/src/fips202/native/x86_64/src/fips202_native_x86_64.h new file mode 100644 index 0000000000..69ce1749ac --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/x86_64/src/fips202_native_x86_64.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_X86_64_SRC_FIPS202_NATIVE_X86_64_H +#define MLK_FIPS202_NATIVE_X86_64_SRC_FIPS202_NATIVE_X86_64_H + +#include "../../../../cbmc.h" +#include "../../../../common.h" + +/* TODO: Reconsider whether this check is needed -- x86_64 is always + * little-endian, so the backend selection already implies this. */ +#ifndef MLK_SYS_LITTLE_ENDIAN +#error Expecting a little-endian platform +#endif + +#define mlk_keccakf1600_round_constants \ + MLK_NAMESPACE(keccakf1600_round_constants) +MLK_INTERNAL_DATA_DECLARATION const uint64_t + mlk_keccakf1600_round_constants[24]; + +#define mlk_keccak_rho8 MLK_NAMESPACE(keccak_rho8) +MLK_INTERNAL_DATA_DECLARATION const uint64_t mlk_keccak_rho8[4]; + +#define mlk_keccak_rho56 MLK_NAMESPACE(keccak_rho56) +MLK_INTERNAL_DATA_DECLARATION const uint64_t mlk_keccak_rho56[4]; + +#define mlk_keccak_f1600_x4_avx2_asm MLK_NAMESPACE(keccak_f1600_x4_avx2_asm) +MLK_SYSV_ABI +void mlk_keccak_f1600_x4_avx2_asm(uint64_t states[100], const uint64_t rc[24], + const uint64_t rho8[4], + const uint64_t rho56[4]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/keccak_f1600_x4_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(states, sizeof(uint64_t) * 25 * 4)) + requires(rc == mlk_keccakf1600_round_constants) + requires(rho8 == mlk_keccak_rho8) + requires(rho56 == mlk_keccak_rho56) + assigns(memory_slice(states, sizeof(uint64_t) * 25 * 4)) +); + +#endif /* !MLK_FIPS202_NATIVE_X86_64_SRC_FIPS202_NATIVE_X86_64_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/x86_64/src/keccak_f1600_x4_avx2_asm.S b/src/mlkem-native/mlkem/src/fips202/native/x86_64/src/keccak_f1600_x4_avx2_asm.S new file mode 100644 index 0000000000..c449ed279b --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/x86_64/src/keccak_f1600_x4_avx2_asm.S @@ -0,0 +1,487 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#include "../../../../common.h" + +#if defined(MLK_FIPS202_X86_64_NEED_X4_AVX2) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/*yaml + Name: keccak_f1600_x4_avx2_asm + Description: x86_64 AVX2 Keccak-f[1600] permutation for four sequential states + Signature: void mlk_keccak_f1600_x4_avx2_asm(uint64_t states[100], const uint64_t rc[24], const uint64_t rho8[4], const uint64_t rho56[4]) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 800 + permissions: read/write + c_parameter: uint64_t states[100] + description: Four sequential Keccak states (4 x 25 x uint64_t) + rsi: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: const uint64_t rc[24] + description: Round constants (24 x uint64_t) + rdx: + type: buffer + size_bytes: 32 + permissions: read-only + c_parameter: const uint64_t rho8[4] + description: Rotation constant rho8 (4 x uint64_t) + rcx: + type: buffer + size_bytes: 32 + permissions: read-only + c_parameter: const uint64_t rho56[4] + description: Rotation constant rho56 (4 x uint64_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/x86_64/src/keccak_f1600_x4_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x4_avx2_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x4_avx2_asm) + + .cfi_startproc + movq %rsp, %r11 + .cfi_def_cfa_register %r11 + andq $-0x20, %rsp + subq $0x300, %rsp # imm = 0x300 + vmovdqu (%rdi), %ymm0 + vmovdqu 0xc8(%rdi), %ymm3 + vmovdqu 0x190(%rdi), %ymm1 + vmovdqu 0x258(%rdi), %ymm4 + vpunpcklqdq %ymm3, %ymm0, %ymm2 # ymm2 = ymm0[0],ymm3[0],ymm0[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3] + vpunpcklqdq %ymm4, %ymm1, %ymm3 # ymm3 = ymm1[0],ymm4[0],ymm1[2],ymm4[2] + vperm2i128 $0x20, %ymm3, %ymm2, %ymm7 # ymm7 = ymm2[0,1],ymm3[0,1] + vpunpckhqdq %ymm4, %ymm1, %ymm1 # ymm1 = ymm1[1],ymm4[1],ymm1[3],ymm4[3] + vperm2i128 $0x31, %ymm3, %ymm2, %ymm3 # ymm3 = ymm2[2,3],ymm3[2,3] + vmovdqu 0x278(%rdi), %ymm4 + vmovdqu %ymm3, 0x40(%rsp) + vperm2i128 $0x31, %ymm1, %ymm0, %ymm3 # ymm3 = ymm0[2,3],ymm1[2,3] + vmovdqu %ymm7, (%rsp) + vperm2i128 $0x20, %ymm1, %ymm0, %ymm7 # ymm7 = ymm0[0,1],ymm1[0,1] + vmovdqu 0x20(%rdi), %ymm0 + vmovdqu 0x1b0(%rdi), %ymm1 + vmovdqu %ymm3, 0x60(%rsp) + vmovdqu 0xe8(%rdi), %ymm3 + vmovdqu %ymm7, 0x20(%rsp) + vpunpcklqdq %ymm3, %ymm0, %ymm2 # ymm2 = ymm0[0],ymm3[0],ymm0[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3] + vpunpcklqdq %ymm4, %ymm1, %ymm3 # ymm3 = ymm1[0],ymm4[0],ymm1[2],ymm4[2] + vperm2i128 $0x20, %ymm3, %ymm2, %ymm7 # ymm7 = ymm2[0,1],ymm3[0,1] + vpunpckhqdq %ymm4, %ymm1, %ymm1 # ymm1 = ymm1[1],ymm4[1],ymm1[3],ymm4[3] + vperm2i128 $0x31, %ymm3, %ymm2, %ymm3 # ymm3 = ymm2[2,3],ymm3[2,3] + vmovdqu 0x298(%rdi), %ymm4 + vperm2i128 $0x31, %ymm1, %ymm0, %ymm14 # ymm14 = ymm0[2,3],ymm1[2,3] + vmovdqu %ymm7, 0x80(%rsp) + vperm2i128 $0x20, %ymm1, %ymm0, %ymm7 # ymm7 = ymm0[0,1],ymm1[0,1] + vmovdqu 0x40(%rdi), %ymm0 + vmovdqu 0x1d0(%rdi), %ymm1 + vmovdqu %ymm3, 0xc0(%rsp) + vmovdqu 0x108(%rdi), %ymm3 + vmovdqu %ymm14, %ymm10 + vmovdqu %ymm7, 0xa0(%rsp) + vpunpcklqdq %ymm3, %ymm0, %ymm2 # ymm2 = ymm0[0],ymm3[0],ymm0[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3] + vpunpcklqdq %ymm4, %ymm1, %ymm3 # ymm3 = ymm1[0],ymm4[0],ymm1[2],ymm4[2] + vpunpckhqdq %ymm4, %ymm1, %ymm1 # ymm1 = ymm1[1],ymm4[1],ymm1[3],ymm4[3] + vperm2i128 $0x20, %ymm3, %ymm2, %ymm11 # ymm11 = ymm2[0,1],ymm3[0,1] + vperm2i128 $0x31, %ymm3, %ymm2, %ymm3 # ymm3 = ymm2[2,3],ymm3[2,3] + vperm2i128 $0x20, %ymm1, %ymm0, %ymm7 # ymm7 = ymm0[0,1],ymm1[0,1] + vmovdqu %ymm3, 0x100(%rsp) + vperm2i128 $0x31, %ymm1, %ymm0, %ymm8 # ymm8 = ymm0[2,3],ymm1[2,3] + vmovdqu 0x128(%rdi), %ymm3 + vmovdqu 0x60(%rdi), %ymm0 + vmovdqu 0x1f0(%rdi), %ymm1 + vmovdqu %ymm7, 0xe0(%rsp) + vmovdqu %ymm11, %ymm14 + vmovdqu 0x2b8(%rdi), %ymm4 + vmovdqu 0x2f8(%rdi), %ymm5 + vpunpcklqdq %ymm3, %ymm0, %ymm2 # ymm2 = ymm0[0],ymm3[0],ymm0[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3] + vpunpcklqdq %ymm4, %ymm1, %ymm3 # ymm3 = ymm1[0],ymm4[0],ymm1[2],ymm4[2] + vpunpckhqdq %ymm4, %ymm1, %ymm1 # ymm1 = ymm1[1],ymm4[1],ymm1[3],ymm4[3] + vmovdqu 0x2d8(%rdi), %ymm4 + vperm2i128 $0x20, %ymm3, %ymm2, %ymm15 # ymm15 = ymm2[0,1],ymm3[0,1] + vperm2i128 $0x31, %ymm3, %ymm2, %ymm3 # ymm3 = ymm2[2,3],ymm3[2,3] + vperm2i128 $0x20, %ymm1, %ymm0, %ymm7 # ymm7 = ymm0[0,1],ymm1[0,1] + vperm2i128 $0x31, %ymm1, %ymm0, %ymm9 # ymm9 = ymm0[2,3],ymm1[2,3] + vmovdqu %ymm3, 0x140(%rsp) + vmovdqu 0x80(%rdi), %ymm0 + vmovdqu 0x148(%rdi), %ymm3 + vmovdqu 0x210(%rdi), %ymm1 + vmovdqu %ymm7, 0x120(%rsp) + vpunpcklqdq %ymm3, %ymm0, %ymm2 # ymm2 = ymm0[0],ymm3[0],ymm0[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3] + vpunpcklqdq %ymm4, %ymm1, %ymm3 # ymm3 = ymm1[0],ymm4[0],ymm1[2],ymm4[2] + vpunpckhqdq %ymm4, %ymm1, %ymm1 # ymm1 = ymm1[1],ymm4[1],ymm1[3],ymm4[3] + vperm2i128 $0x20, %ymm3, %ymm2, %ymm7 # ymm7 = ymm2[0,1],ymm3[0,1] + vperm2i128 $0x31, %ymm3, %ymm2, %ymm13 # ymm13 = ymm2[2,3],ymm3[2,3] + vperm2i128 $0x31, %ymm1, %ymm0, %ymm3 # ymm3 = ymm0[2,3],ymm1[2,3] + vmovdqu %ymm7, 0x160(%rsp) + vperm2i128 $0x20, %ymm1, %ymm0, %ymm7 # ymm7 = ymm0[0,1],ymm1[0,1] + vmovdqu 0xa0(%rdi), %ymm0 + vmovdqu 0x230(%rdi), %ymm1 + vmovdqu %ymm3, 0x1a0(%rsp) + vmovdqu 0x168(%rdi), %ymm3 + vpunpcklqdq %ymm5, %ymm1, %ymm4 # ymm4 = ymm1[0],ymm5[0],ymm1[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm1, %ymm1 # ymm1 = ymm1[1],ymm5[1],ymm1[3],ymm5[3] + vmovdqu %ymm7, 0x180(%rsp) + vpunpcklqdq %ymm3, %ymm0, %ymm2 # ymm2 = ymm0[0],ymm3[0],ymm0[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3] + vperm2i128 $0x20, %ymm4, %ymm2, %ymm12 # ymm12 = ymm2[0,1],ymm4[0,1] + vperm2i128 $0x20, %ymm1, %ymm0, %ymm3 # ymm3 = ymm0[0,1],ymm1[0,1] + vperm2i128 $0x31, %ymm4, %ymm2, %ymm7 # ymm7 = ymm2[2,3],ymm4[2,3] + vperm2i128 $0x31, %ymm1, %ymm0, %ymm4 # ymm4 = ymm0[2,3],ymm1[2,3] + vmovq 0x250(%rdi), %xmm0 + vmovq 0xc0(%rdi), %xmm1 + vmovdqu %ymm12, 0x1c0(%rsp) + vmovdqu %ymm4, 0x1e0(%rsp) + vpinsrq $0x1, 0x318(%rdi), %xmm0, %xmm0 + vpinsrq $0x1, 0x188(%rdi), %xmm1, %xmm1 + vinserti128 $0x1, %xmm0, %ymm1, %ymm2 + movq $0x0, %r10 + +LLmlk_keccak_f1600_x4_avx2: + vmovdqu 0xa0(%rsp), %ymm4 + vpxor 0x1c0(%rsp), %ymm9, %ymm0 + vmovdqu %ymm9, 0x200(%rsp) + vmovdqu %ymm10, %ymm9 + vmovdqu 0xc0(%rsp), %ymm11 + vmovdqu 0x160(%rsp), %ymm12 + vmovdqu %ymm3, 0x240(%rsp) + vpxor 0x100(%rsp), %ymm4, %ymm1 + vmovdqu 0x40(%rsp), %ymm10 + vmovdqu %ymm4, 0x220(%rsp) + vpxor %ymm3, %ymm12, %ymm12 + vmovdqu 0x20(%rsp), %ymm6 + vmovdqu 0x140(%rsp), %ymm4 + vmovdqu %ymm14, 0x2a0(%rsp) + vpxor %ymm1, %ymm0, %ymm0 + vpxor %ymm8, %ymm11, %ymm1 + vpxor 0x180(%rsp), %ymm7, %ymm11 + vmovdqu %ymm10, 0x280(%rsp) + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm15, %ymm9, %ymm1 + vmovdqu 0xe0(%rsp), %ymm3 + vmovdqu %ymm8, 0x260(%rsp) + vpxor %ymm1, %ymm11, %ymm11 + vpxor 0x120(%rsp), %ymm14, %ymm1 + vpxor %ymm6, %ymm12, %ymm12 + vmovdqu 0x60(%rsp), %ymm8 + vpxor %ymm10, %ymm11, %ymm11 + vpxor 0x1e0(%rsp), %ymm13, %ymm10 + vpxor %ymm4, %ymm3, %ymm3 + vmovdqu %ymm4, 0x2c0(%rsp) + vpsrlq $0x3f, %ymm12, %ymm4 + vpsrlq $0x3f, %ymm11, %ymm5 + vpxor (%rsp), %ymm0, %ymm0 + vpxor %ymm1, %ymm10, %ymm10 + vmovdqu 0x80(%rsp), %ymm1 + vpxor %ymm8, %ymm10, %ymm10 + vmovdqu %ymm1, %ymm14 + vpxor 0x1a0(%rsp), %ymm2, %ymm1 + vmovdqu %ymm14, 0x2e0(%rsp) + vpxor %ymm3, %ymm1, %ymm1 + vpsllq $0x1, %ymm12, %ymm3 + vpor %ymm4, %ymm3, %ymm3 + vpsllq $0x1, %ymm11, %ymm4 + vpxor %ymm14, %ymm1, %ymm1 + vpor %ymm5, %ymm4, %ymm4 + vpsrlq $0x3f, %ymm10, %ymm14 + vpxor %ymm1, %ymm3, %ymm3 + vpsllq $0x1, %ymm10, %ymm5 + vpxor %ymm0, %ymm4, %ymm4 + vpor %ymm14, %ymm5, %ymm5 + vpxor %ymm6, %ymm4, %ymm6 + vpxor %ymm12, %ymm5, %ymm5 + vpsrlq $0x3f, %ymm1, %ymm12 + vpsllq $0x1, %ymm1, %ymm1 + vpxor %ymm7, %ymm5, %ymm7 + vpxor %ymm9, %ymm5, %ymm9 + vpor %ymm12, %ymm1, %ymm1 + vpxor (%rsp), %ymm3, %ymm12 + vpxor %ymm11, %ymm1, %ymm1 + vpsrlq $0x3f, %ymm0, %ymm11 + vpsllq $0x1, %ymm0, %ymm0 + vpxor %ymm13, %ymm1, %ymm13 + vpxor %ymm8, %ymm1, %ymm8 + vpor %ymm11, %ymm0, %ymm0 + vpxor %ymm10, %ymm0, %ymm0 + vpxor 0xc0(%rsp), %ymm4, %ymm10 + vpxor %ymm2, %ymm0, %ymm2 + vpsrlq $0x14, %ymm10, %ymm11 + vpsllq $0x2c, %ymm10, %ymm10 + vpor %ymm11, %ymm10, %ymm10 + vpxor %ymm15, %ymm5, %ymm11 + vpbroadcastq (%rsi), %ymm15 + vpsrlq $0x15, %ymm11, %ymm14 + vpsllq $0x2b, %ymm11, %ymm11 + vpor %ymm14, %ymm11, %ymm11 + vpandn %ymm11, %ymm10, %ymm14 + vpxor %ymm15, %ymm14, %ymm14 + vpxor %ymm12, %ymm14, %ymm15 + vpsrlq $0x2b, %ymm13, %ymm14 + vpsllq $0x15, %ymm13, %ymm13 + vmovdqu %ymm15, (%rsp) + vpor %ymm14, %ymm13, %ymm13 + vpandn %ymm13, %ymm11, %ymm14 + vpxor %ymm10, %ymm14, %ymm15 + vpsrlq $0x32, %ymm2, %ymm14 + vpsllq $0xe, %ymm2, %ymm2 + vmovdqu %ymm15, 0x20(%rsp) + vpor %ymm14, %ymm2, %ymm2 + vpandn %ymm2, %ymm13, %ymm14 + vpxor %ymm11, %ymm14, %ymm11 + vmovdqu %ymm11, 0x40(%rsp) + vpandn %ymm12, %ymm2, %ymm11 + vpandn %ymm10, %ymm12, %ymm12 + vpxor %ymm13, %ymm11, %ymm11 + vmovdqu %ymm11, 0x60(%rsp) + vpxor %ymm2, %ymm12, %ymm11 + vpsrlq $0x24, %ymm8, %ymm2 + vpsllq $0x1c, %ymm8, %ymm8 + vmovdqu %ymm11, 0x80(%rsp) + vpor %ymm2, %ymm8, %ymm8 + vpxor 0xe0(%rsp), %ymm0, %ymm2 + vpsrlq $0x2c, %ymm2, %ymm10 + vpsllq $0x14, %ymm2, %ymm2 + vpor %ymm10, %ymm2, %ymm2 + vpxor 0x100(%rsp), %ymm3, %ymm10 + vpsrlq $0x3d, %ymm10, %ymm11 + vpsllq $0x3, %ymm10, %ymm10 + vpor %ymm11, %ymm10, %ymm10 + vpandn %ymm10, %ymm2, %ymm11 + vpxor %ymm8, %ymm11, %ymm11 + vmovdqu %ymm11, 0xa0(%rsp) + vpxor 0x160(%rsp), %ymm4, %ymm11 + vpsrlq $0x13, %ymm11, %ymm12 + vpsllq $0x2d, %ymm11, %ymm11 + vpor %ymm12, %ymm11, %ymm11 + vpandn %ymm11, %ymm10, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vmovdqu %ymm12, 0xc0(%rsp) + vpsrlq $0x3, %ymm7, %ymm12 + vpsllq $0x3d, %ymm7, %ymm7 + vpor %ymm12, %ymm7, %ymm7 + vpandn %ymm7, %ymm11, %ymm12 + vpxor %ymm10, %ymm12, %ymm10 + vpandn %ymm8, %ymm7, %ymm12 + vpandn %ymm2, %ymm8, %ymm8 + vpsrlq $0x3f, %ymm6, %ymm2 + vpsllq $0x1, %ymm6, %ymm6 + vpxor %ymm11, %ymm12, %ymm14 + vpor %ymm2, %ymm6, %ymm6 + vpsrlq $0x3a, %ymm9, %ymm2 + vpxor %ymm7, %ymm8, %ymm12 + vpsllq $0x6, %ymm9, %ymm9 + vmovdqu %ymm12, 0xe0(%rsp) + vpxor 0x1a0(%rsp), %ymm0, %ymm7 + vpor %ymm2, %ymm9, %ymm9 + vpxor 0x120(%rsp), %ymm1, %ymm2 + vpshufb (%rdx), %ymm7, %ymm7 + vpsrlq $0x27, %ymm2, %ymm11 + vpsllq $0x19, %ymm2, %ymm2 + vpor %ymm2, %ymm11, %ymm11 + vpandn %ymm11, %ymm9, %ymm2 + vpandn %ymm7, %ymm11, %ymm8 + vpxor %ymm6, %ymm2, %ymm12 + vpxor 0x1c0(%rsp), %ymm3, %ymm2 + vpxor %ymm9, %ymm8, %ymm8 + vmovdqu %ymm12, 0x100(%rsp) + vpsrlq $0x2e, %ymm2, %ymm12 + vpsllq $0x12, %ymm2, %ymm2 + vpor %ymm2, %ymm12, %ymm2 + vpandn %ymm2, %ymm7, %ymm12 + vpxor %ymm11, %ymm12, %ymm15 + vpandn %ymm6, %ymm2, %ymm11 + vpandn %ymm9, %ymm6, %ymm6 + vpxor %ymm7, %ymm11, %ymm12 + vmovdqu %ymm12, 0x120(%rsp) + vpxor %ymm2, %ymm6, %ymm12 + vpxor 0x2e0(%rsp), %ymm0, %ymm6 + vpxor 0x2c0(%rsp), %ymm0, %ymm0 + vmovdqu %ymm12, 0x140(%rsp) + vpsrlq $0x25, %ymm6, %ymm2 + vpsllq $0x1b, %ymm6, %ymm6 + vpor %ymm6, %ymm2, %ymm2 + vpxor 0x220(%rsp), %ymm3, %ymm6 + vpxor 0x200(%rsp), %ymm3, %ymm3 + vpsrlq $0x1c, %ymm6, %ymm7 + vpsllq $0x24, %ymm6, %ymm6 + vpor %ymm6, %ymm7, %ymm7 + vpxor 0x260(%rsp), %ymm4, %ymm6 + vpxor 0x240(%rsp), %ymm4, %ymm4 + vpsrlq $0x36, %ymm6, %ymm12 + vpsllq $0xa, %ymm6, %ymm6 + vpor %ymm6, %ymm12, %ymm12 + vpxor 0x180(%rsp), %ymm5, %ymm6 + vpxor 0x280(%rsp), %ymm5, %ymm5 + vpandn %ymm12, %ymm7, %ymm9 + vpsrlq $0x31, %ymm6, %ymm11 + vpsllq $0xf, %ymm6, %ymm6 + vpxor %ymm2, %ymm9, %ymm9 + vpor %ymm6, %ymm11, %ymm11 + vpandn %ymm11, %ymm12, %ymm6 + vpxor %ymm7, %ymm6, %ymm6 + vmovdqu %ymm6, 0x160(%rsp) + vpxor 0x1e0(%rsp), %ymm1, %ymm6 + vpxor 0x2a0(%rsp), %ymm1, %ymm1 + vpshufb (%rcx), %ymm6, %ymm6 + vpandn %ymm6, %ymm11, %ymm13 + vpxor %ymm12, %ymm13, %ymm13 + vmovdqu %ymm13, 0x180(%rsp) + vpandn %ymm2, %ymm6, %ymm13 + vpandn %ymm7, %ymm2, %ymm2 + vpxor %ymm6, %ymm2, %ymm2 + vpsrlq $0x3e, %ymm4, %ymm6 + vpxor %ymm11, %ymm13, %ymm13 + vmovdqu %ymm2, 0x1a0(%rsp) + vpsrlq $0x2, %ymm5, %ymm2 + vpsllq $0x3e, %ymm5, %ymm5 + vpor %ymm5, %ymm2, %ymm2 + vpsrlq $0x9, %ymm1, %ymm5 + vpsllq $0x37, %ymm1, %ymm1 + vpsllq $0x2, %ymm4, %ymm4 + vpor %ymm1, %ymm5, %ymm1 + vpsrlq $0x19, %ymm0, %ymm5 + vpor %ymm4, %ymm6, %ymm4 + vpsllq $0x27, %ymm0, %ymm0 + vpor %ymm0, %ymm5, %ymm5 + vpandn %ymm5, %ymm1, %ymm0 + vpxor %ymm2, %ymm0, %ymm0 + vmovdqu %ymm0, 0x1c0(%rsp) + vpsrlq $0x17, %ymm3, %ymm0 + vpsllq $0x29, %ymm3, %ymm3 + vpor %ymm3, %ymm0, %ymm0 + vpandn %ymm4, %ymm0, %ymm7 + vpandn %ymm0, %ymm5, %ymm3 + vpxor %ymm5, %ymm7, %ymm7 + vpandn %ymm2, %ymm4, %ymm5 + vpandn %ymm1, %ymm2, %ymm2 + vpxor %ymm0, %ymm5, %ymm5 + vpxor %ymm1, %ymm3, %ymm3 + vpxor %ymm4, %ymm2, %ymm2 + vmovdqu %ymm5, 0x1e0(%rsp) + addq $0x8, %rsi + addq $0x1, %r10 + cmpq $0x18, %r10 + jne LLmlk_keccak_f1600_x4_avx2 + vmovdqu (%rsp), %ymm4 + vmovdqu 0x40(%rsp), %ymm5 + vmovdqu 0x20(%rsp), %ymm0 + vmovdqu 0x60(%rsp), %ymm1 + vmovdqu 0x1c0(%rsp), %ymm12 + vmovdqu %ymm2, 0x1c0(%rsp) + vpunpcklqdq %ymm0, %ymm4, %ymm2 # ymm2 = ymm4[0],ymm0[0],ymm4[2],ymm0[2] + vpunpckhqdq %ymm0, %ymm4, %ymm0 # ymm0 = ymm4[1],ymm0[1],ymm4[3],ymm0[3] + vpunpcklqdq %ymm1, %ymm5, %ymm4 # ymm4 = ymm5[0],ymm1[0],ymm5[2],ymm1[2] + vpunpckhqdq %ymm1, %ymm5, %ymm1 # ymm1 = ymm5[1],ymm1[1],ymm5[3],ymm1[3] + vperm2i128 $0x20, %ymm4, %ymm2, %ymm6 # ymm6 = ymm2[0,1],ymm4[0,1] + vperm2i128 $0x31, %ymm4, %ymm2, %ymm2 # ymm2 = ymm2[2,3],ymm4[2,3] + vmovdqu 0x80(%rsp), %ymm4 + vperm2i128 $0x20, %ymm1, %ymm0, %ymm5 # ymm5 = ymm0[0,1],ymm1[0,1] + vperm2i128 $0x31, %ymm1, %ymm0, %ymm0 # ymm0 = ymm0[2,3],ymm1[2,3] + vmovdqu %ymm6, (%rdi) + vmovdqu %ymm5, 0xc8(%rdi) + vmovdqu %ymm2, 0x190(%rdi) + vmovdqu %ymm0, 0x258(%rdi) + vmovdqu 0xa0(%rsp), %ymm0 + vpunpcklqdq %ymm0, %ymm4, %ymm2 # ymm2 = ymm4[0],ymm0[0],ymm4[2],ymm0[2] + vpunpckhqdq %ymm0, %ymm4, %ymm1 # ymm1 = ymm4[1],ymm0[1],ymm4[3],ymm0[3] + vmovdqu 0xc0(%rsp), %ymm0 + vpunpcklqdq %ymm10, %ymm0, %ymm4 # ymm4 = ymm0[0],ymm10[0],ymm0[2],ymm10[2] + vpunpckhqdq %ymm10, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm10[1],ymm0[3],ymm10[3] + vperm2i128 $0x20, %ymm4, %ymm2, %ymm6 # ymm6 = ymm2[0,1],ymm4[0,1] + vperm2i128 $0x20, %ymm0, %ymm1, %ymm5 # ymm5 = ymm1[0,1],ymm0[0,1] + vperm2i128 $0x31, %ymm4, %ymm2, %ymm2 # ymm2 = ymm2[2,3],ymm4[2,3] + vmovdqu 0xe0(%rsp), %ymm4 + vperm2i128 $0x31, %ymm0, %ymm1, %ymm1 # ymm1 = ymm1[2,3],ymm0[2,3] + vmovdqu 0x100(%rsp), %ymm0 + vmovdqu %ymm2, 0x1b0(%rdi) + vmovdqu %ymm1, 0x278(%rdi) + vpunpcklqdq %ymm4, %ymm14, %ymm2 # ymm2 = ymm14[0],ymm4[0],ymm14[2],ymm4[2] + vpunpckhqdq %ymm4, %ymm14, %ymm1 # ymm1 = ymm14[1],ymm4[1],ymm14[3],ymm4[3] + vpunpcklqdq %ymm8, %ymm0, %ymm4 # ymm4 = ymm0[0],ymm8[0],ymm0[2],ymm8[2] + vpunpckhqdq %ymm8, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm8[1],ymm0[3],ymm8[3] + vmovdqu %ymm6, 0x20(%rdi) + vmovdqu %ymm5, 0xe8(%rdi) + vperm2i128 $0x20, %ymm4, %ymm2, %ymm6 # ymm6 = ymm2[0,1],ymm4[0,1] + vperm2i128 $0x20, %ymm0, %ymm1, %ymm5 # ymm5 = ymm1[0,1],ymm0[0,1] + vperm2i128 $0x31, %ymm4, %ymm2, %ymm2 # ymm2 = ymm2[2,3],ymm4[2,3] + vperm2i128 $0x31, %ymm0, %ymm1, %ymm1 # ymm1 = ymm1[2,3],ymm0[2,3] + vmovdqu 0x120(%rsp), %ymm4 + vmovdqu 0x140(%rsp), %ymm0 + vmovdqu %ymm2, 0x1d0(%rdi) + vmovdqu %ymm1, 0x298(%rdi) + vpunpcklqdq %ymm4, %ymm15, %ymm2 # ymm2 = ymm15[0],ymm4[0],ymm15[2],ymm4[2] + vpunpckhqdq %ymm4, %ymm15, %ymm1 # ymm1 = ymm15[1],ymm4[1],ymm15[3],ymm4[3] + vpunpcklqdq %ymm9, %ymm0, %ymm4 # ymm4 = ymm0[0],ymm9[0],ymm0[2],ymm9[2] + vmovdqu %ymm5, 0x108(%rdi) + vpunpckhqdq %ymm9, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm9[1],ymm0[3],ymm9[3] + vmovdqu %ymm6, 0x40(%rdi) + vperm2i128 $0x20, %ymm4, %ymm2, %ymm6 # ymm6 = ymm2[0,1],ymm4[0,1] + vperm2i128 $0x31, %ymm4, %ymm2, %ymm2 # ymm2 = ymm2[2,3],ymm4[2,3] + vperm2i128 $0x20, %ymm0, %ymm1, %ymm5 # ymm5 = ymm1[0,1],ymm0[0,1] + vmovdqu 0x160(%rsp), %ymm4 + vperm2i128 $0x31, %ymm0, %ymm1, %ymm1 # ymm1 = ymm1[2,3],ymm0[2,3] + vmovdqu 0x180(%rsp), %ymm0 + vmovdqu %ymm5, 0x128(%rdi) + vmovdqu 0x1a0(%rsp), %ymm5 + vmovdqu %ymm2, 0x1f0(%rdi) + vpunpcklqdq %ymm0, %ymm4, %ymm2 # ymm2 = ymm4[0],ymm0[0],ymm4[2],ymm0[2] + vpunpckhqdq %ymm0, %ymm4, %ymm0 # ymm0 = ymm4[1],ymm0[1],ymm4[3],ymm0[3] + vpunpcklqdq %ymm5, %ymm13, %ymm4 # ymm4 = ymm13[0],ymm5[0],ymm13[2],ymm5[2] + vmovdqu %ymm6, 0x60(%rdi) + vperm2i128 $0x20, %ymm4, %ymm2, %ymm6 # ymm6 = ymm2[0,1],ymm4[0,1] + vmovdqu %ymm1, 0x2b8(%rdi) + vperm2i128 $0x31, %ymm4, %ymm2, %ymm2 # ymm2 = ymm2[2,3],ymm4[2,3] + vpunpckhqdq %ymm5, %ymm13, %ymm1 # ymm1 = ymm13[1],ymm5[1],ymm13[3],ymm5[3] + vmovdqu %ymm6, 0x80(%rdi) + vmovdqu 0x1e0(%rsp), %ymm4 + vperm2i128 $0x20, %ymm1, %ymm0, %ymm5 # ymm5 = ymm0[0,1],ymm1[0,1] + vperm2i128 $0x31, %ymm1, %ymm0, %ymm0 # ymm0 = ymm0[2,3],ymm1[2,3] + vmovdqu %ymm2, 0x210(%rdi) + vpunpcklqdq %ymm3, %ymm12, %ymm2 # ymm2 = ymm12[0],ymm3[0],ymm12[2],ymm3[2] + vmovdqu %ymm0, 0x2d8(%rdi) + vpunpckhqdq %ymm3, %ymm12, %ymm0 # ymm0 = ymm12[1],ymm3[1],ymm12[3],ymm3[3] + vpunpcklqdq %ymm4, %ymm7, %ymm3 # ymm3 = ymm7[0],ymm4[0],ymm7[2],ymm4[2] + vpunpckhqdq %ymm4, %ymm7, %ymm1 # ymm1 = ymm7[1],ymm4[1],ymm7[3],ymm4[3] + vmovdqu %ymm5, 0x148(%rdi) + vperm2i128 $0x20, %ymm3, %ymm2, %ymm5 # ymm5 = ymm2[0,1],ymm3[0,1] + vperm2i128 $0x31, %ymm3, %ymm2, %ymm2 # ymm2 = ymm2[2,3],ymm3[2,3] + vmovdqu 0x1c0(%rsp), %ymm3 + vperm2i128 $0x20, %ymm1, %ymm0, %ymm4 # ymm4 = ymm0[0,1],ymm1[0,1] + vperm2i128 $0x31, %ymm1, %ymm0, %ymm0 # ymm0 = ymm0[2,3],ymm1[2,3] + vmovdqu %ymm5, 0xa0(%rdi) + vextracti128 $0x1, %ymm3, %xmm15 + vmovdqu %ymm4, 0x168(%rdi) + vmovdqu %ymm2, 0x230(%rdi) + vmovdqu %ymm0, 0x2f8(%rdi) + vmovq %xmm3, 0xc0(%rdi) + vmovhpd %xmm3, 0x188(%rdi) + vmovq %xmm15, 0x250(%rdi) + vmovhpd %xmm15, 0x318(%rdi) + movq %r11, %rsp + .cfi_def_cfa_register %rsp + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(keccak_f1600_x4_avx2_asm) + +#endif /* MLK_FIPS202_X86_64_NEED_X4_AVX2 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/fips202/native/x86_64/src/keccakf1600_constants.c b/src/mlkem-native/mlkem/src/fips202/native/x86_64/src/keccakf1600_constants.c new file mode 100644 index 0000000000..1fb58915f0 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/x86_64/src/keccakf1600_constants.c @@ -0,0 +1,52 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "../../../../common.h" +#if defined(MLK_FIPS202_X86_64_NEED_X4_AVX2) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include + +#include "fips202_native_x86_64.h" + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint64_t + mlk_keccakf1600_round_constants[24] = { + 0x0000000000000001, 0x0000000000008082, 0x800000000000808a, + 0x8000000080008000, 0x000000000000808b, 0x0000000080000001, + 0x8000000080008081, 0x8000000000008009, 0x000000000000008a, + 0x0000000000000088, 0x0000000080008009, 0x000000008000000a, + 0x000000008000808b, 0x800000000000008b, 0x8000000000008089, + 0x8000000000008003, 0x8000000000008002, 0x8000000000000080, + 0x000000000000800a, 0x800000008000000a, 0x8000000080008081, + 0x8000000000008080, 0x0000000080000001, 0x8000000080008008, +}; + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint64_t mlk_keccak_rho8[4] = { + 0x0605040302010007, + 0x0e0d0c0b0a09080f, + 0x1615141312111017, + 0x1e1d1c1b1a19181f, +}; + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint64_t mlk_keccak_rho56[4] = { + 0x0007060504030201, + 0x080f0e0d0c0b0a09, + 0x1017161514131211, + 0x181f1e1d1c1b1a19, +}; + +#else /* MLK_FIPS202_X86_64_NEED_X4_AVX2 && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +MLK_EMPTY_CU(fips202_x86_64_constants) + +#endif /* !(MLK_FIPS202_X86_64_NEED_X4_AVX2 && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/src/mlkem-native/mlkem/src/indcpa.c b/src/mlkem-native/mlkem/src/indcpa.c new file mode 100644 index 0000000000..98060e3742 --- /dev/null +++ b/src/mlkem-native/mlkem/src/indcpa.c @@ -0,0 +1,675 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + */ + +#include "indcpa.h" + +#include "debug.h" +#include "randombytes.h" +#include "sampling.h" +#include "symmetric.h" +#include "verify.h" + +/* Parameter set namespacing + * This is to facilitate building multiple instances + * of mlkem-native (e.g. with varying parameter sets) + * within a single compilation unit. */ +#define mlk_pack_pk MLK_ADD_PARAM_SET(mlk_pack_pk) +#define mlk_unpack_pk MLK_ADD_PARAM_SET(mlk_unpack_pk) +#define mlk_pack_sk MLK_ADD_PARAM_SET(mlk_pack_sk) +#define mlk_unpack_sk MLK_ADD_PARAM_SET(mlk_unpack_sk) +#define mlk_pack_ciphertext MLK_ADD_PARAM_SET(mlk_pack_ciphertext) +#define mlk_unpack_ciphertext MLK_ADD_PARAM_SET(mlk_unpack_ciphertext) +#define mlk_matvec_mul MLK_ADD_PARAM_SET(mlk_matvec_mul) +#define mlk_polyvec_permute_bitrev_to_custom \ + MLK_ADD_PARAM_SET(mlk_polyvec_permute_bitrev_to_custom) +#define mlk_polymat_permute_bitrev_to_custom \ + MLK_ADD_PARAM_SET(mlk_polymat_permute_bitrev_to_custom) +#define mlk_keypair_getnoise_eta1 MLK_ADD_PARAM_SET(mlk_keypair_getnoise_eta1) +#define mlk_enc_getnoise_eta1_eta2 MLK_ADD_PARAM_SET(mlk_enc_getnoise_eta1_eta2) +/* End of parameter set namespacing */ + +/** + * Serialize the public key as the concatenation of the serialized vector of + * polynomials pk and the public seed used to generate the matrix A. + * + * @spec{Implements @[FIPS203, Algorithm 13 (K-PKE.KeyGen), L19].} + * + * @param[out] r Output serialized public key. + * @param[in] pk Input public-key polyvec. Must have coefficients within + * [0,..,MLKEM_Q-1]. + * @param[in] seed Input public seed. + */ +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +static void mlk_pack_pk(uint8_t r[MLKEM_INDCPA_PUBLICKEYBYTES], + const mlk_polyvec *pk, + const uint8_t seed[MLKEM_SYMBYTES]) +{ + mlk_assert_bound_2d(pk->vec, MLKEM_K, MLKEM_N, 0, MLKEM_Q); + mlk_polyvec_tobytes(r, pk); + mlk_memcpy(r + MLKEM_POLYVECBYTES, seed, MLKEM_SYMBYTES); +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +/** + * De-serialize public key from a byte array; approximate inverse of + * mlk_pack_pk. + * + * @spec{Implements @[FIPS203, Algorithm 14 (K-PKE.Encrypt), L2-3].} + * + * @param[out] pk Output public-key polynomial vector. Coefficients + * will be normalized to [0,1,..,MLKEM_Q-1]. + * @param[out] seed Output seed to generate matrix A. + * @param[in] packedpk Input serialized public key. + */ +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +static void mlk_unpack_pk(mlk_polyvec *pk, uint8_t seed[MLKEM_SYMBYTES], + const uint8_t packedpk[MLKEM_INDCPA_PUBLICKEYBYTES]) +{ + mlk_polyvec_frombytes(pk, packedpk); + mlk_memcpy(seed, packedpk + MLKEM_POLYVECBYTES, MLKEM_SYMBYTES); + + /* NOTE: If a modulus check was conducted on the PK, we know at this + * point that the coefficients of `pk` are unsigned canonical. The + * specifications and proofs, however, do _not_ assume this, and instead + * work with the easily provable bound by MLKEM_UINT12_LIMIT. */ +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +/** + * Serialize the secret key. + * + * @spec{Implements @[FIPS203, Algorithm 13 (K-PKE.KeyGen), L20].} + * + * @param[out] r Output serialized secret key. + * @param[in] sk Input vector of polynomials (secret key). + */ +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +static void mlk_pack_sk(uint8_t r[MLKEM_INDCPA_SECRETKEYBYTES], + const mlk_polyvec *sk) +{ + mlk_assert_bound_2d(sk->vec, MLKEM_K, MLKEM_N, 0, MLKEM_Q); + mlk_polyvec_tobytes(r, sk); +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +/** + * De-serialize the secret key; inverse of mlk_pack_sk. + * + * @spec{Implements @[FIPS203, Algorithm 15 (K-PKE.Decrypt), L5].} + * + * @param[out] sk Output vector of polynomials (secret key). + * @param[in] packedsk Input serialized secret key. + */ +#if !defined(MLK_CONFIG_NO_DECAPS_API) +static void mlk_unpack_sk(mlk_polyvec *sk, + const uint8_t packedsk[MLKEM_INDCPA_SECRETKEYBYTES]) +{ + mlk_polyvec_frombytes(sk, packedsk); +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +/** + * Serialize the ciphertext as the concatenation of the compressed and + * serialized vector of polynomials b and the compressed and serialized + * polynomial v. + * + * @spec{Implements @[FIPS203, Algorithm 14 (K-PKE.Encrypt), L22-23].} + * + * @param[out] r Output serialized ciphertext. + * @param[in] b Input vector of polynomials b. + * @param[in] v Input polynomial v. + */ +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +static void mlk_pack_ciphertext(uint8_t r[MLKEM_INDCPA_BYTES], + const mlk_polyvec *b, mlk_poly *v) +{ + mlk_polyvec_compress_du(r, b); + mlk_poly_compress_dv(r + MLKEM_POLYVECCOMPRESSEDBYTES_DU, v); +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +/** + * De-serialize and decompress ciphertext from a byte array; approximate + * inverse of mlk_pack_ciphertext. + * + * @spec{Implements @[FIPS203, Algorithm 15 (K-PKE.Decrypt), L1-4].} + * + * @param[out] b Output vector of polynomials b. + * @param[out] v Output polynomial v. + * @param[in] c Input serialized ciphertext. + */ +#if !defined(MLK_CONFIG_NO_DECAPS_API) +static void mlk_unpack_ciphertext(mlk_polyvec *b, mlk_poly *v, + const uint8_t c[MLKEM_INDCPA_BYTES]) +{ + mlk_polyvec_decompress_du(b, c); + mlk_poly_decompress_dv(v, c + MLKEM_POLYVECCOMPRESSEDBYTES_DU); +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +/* Helper function to ensure that the polynomial entries in the output + * of gen_matrix use the standard (bitreversed) ordering of coefficients. + * No-op unless a native backend with a custom ordering is used. + * + * We don't inline this into gen_matrix to avoid having to split the CBMC + * proof for gen_matrix based on MLK_USE_NATIVE_NTT_CUSTOM_ORDER. */ +static void mlk_polyvec_permute_bitrev_to_custom(mlk_polyvec *v) +__contract__( + /* We don't specify that this should be a permutation, but only + * that it does not change the bound established at the end of mlk_gen_matrix. */ + requires(memory_no_alias(v, sizeof(mlk_polyvec))) + requires(forall(x, 0, MLKEM_K, + array_bound(v->vec[x].coeffs, 0, MLKEM_N, 0, MLKEM_Q))) + assigns(memory_slice(v, sizeof(mlk_polyvec))) + ensures(forall(x, 0, MLKEM_K, + array_bound(v->vec[x].coeffs, 0, MLKEM_N, 0, MLKEM_Q)))) +{ +#if defined(MLK_USE_NATIVE_NTT_CUSTOM_ORDER) + unsigned i; + for (i = 0; i < MLKEM_K; i++) + __loop__( + assigns(i, memory_slice(v, sizeof(mlk_polyvec))) + invariant(i <= MLKEM_K) + invariant(forall(x, 0, MLKEM_K, + array_bound(v->vec[x].coeffs, 0, MLKEM_N, 0, MLKEM_Q))) + decreases(MLKEM_K - i)) + { + mlk_poly_permute_bitrev_to_custom(v->vec[i].coeffs); + } +#else /* MLK_USE_NATIVE_NTT_CUSTOM_ORDER */ + /* Nothing to do */ + (void)v; +#endif /* !MLK_USE_NATIVE_NTT_CUSTOM_ORDER */ +} + +static void mlk_polymat_permute_bitrev_to_custom(mlk_polymat *a) +__contract__( + /* We don't specify that this should be a permutation, but only + * that it does not change the bound established at the end of mlk_gen_matrix. */ + requires(memory_no_alias(a, sizeof(mlk_polymat))) + requires(forall(x, 0, MLKEM_K, forall(y, 0, MLKEM_K, + array_bound(a->vec[x].vec[y].coeffs, 0, MLKEM_N, 0, MLKEM_Q)))) + assigns(memory_slice(a, sizeof(mlk_polymat))) + ensures(forall(x, 0, MLKEM_K, forall(y, 0, MLKEM_K, + array_bound(a->vec[x].vec[y].coeffs, 0, MLKEM_N, 0, MLKEM_Q))))) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + __loop__( + assigns(i, memory_slice(a, sizeof(mlk_polymat))) + invariant(i <= MLKEM_K) + invariant(forall(x, 0, MLKEM_K, forall(y, 0, MLKEM_K, + array_bound(a->vec[x].vec[y].coeffs, 0, MLKEM_N, 0, MLKEM_Q)))) + decreases(MLKEM_K - i)) + { + mlk_polyvec_permute_bitrev_to_custom(&a->vec[i]); + } +} + +/* Reference: `gen_matrix()` in the reference implementation @[REF]. + * - We use a special subroutine to generate 4 polynomials + * at a time, to be able to leverage batched Keccak-f1600 + * implementations. The reference implementation generates + * one matrix entry a time. + * + * Not static for benchmarking */ +MLK_INTERNAL_API +void mlk_gen_matrix(mlk_polymat *a, const uint8_t seed[MLKEM_SYMBYTES], + int transposed) +{ + unsigned i, j; + MLK_ALIGN uint8_t seed_ext[4][MLK_ALIGN_UP(MLKEM_SYMBYTES + 2)]; + + for (j = 0; j < 4; j++) + { + mlk_memcpy(seed_ext[j], seed, MLKEM_SYMBYTES); + } + +#if !defined(MLK_CONFIG_SERIAL_FIPS202_ONLY) + /* Sample 4 matrix entries a time. */ + for (i = 0; i < (MLKEM_K * MLKEM_K / 4) * 4; i += 4) + { + for (j = 0; j < 4; j++) + { + uint8_t x, y; + /* MLKEM_K <= 4, so the values fit in uint8_t. */ + x = (uint8_t)((i + j) / MLKEM_K); + y = (uint8_t)((i + j) % MLKEM_K); + if (transposed) + { + seed_ext[j][MLKEM_SYMBYTES + 0] = x; + seed_ext[j][MLKEM_SYMBYTES + 1] = y; + } + else + { + seed_ext[j][MLKEM_SYMBYTES + 0] = y; + seed_ext[j][MLKEM_SYMBYTES + 1] = x; + } + } + + mlk_poly_rej_uniform_x4(&a->vec[i / MLKEM_K].vec[i % MLKEM_K], + &a->vec[(i + 1) / MLKEM_K].vec[(i + 1) % MLKEM_K], + &a->vec[(i + 2) / MLKEM_K].vec[(i + 2) % MLKEM_K], + &a->vec[(i + 3) / MLKEM_K].vec[(i + 3) % MLKEM_K], + seed_ext); + } +#else /* !MLK_CONFIG_SERIAL_FIPS202_ONLY */ + /* When using serial FIPS202, sample all entries individually. */ + i = 0; +#endif /* MLK_CONFIG_SERIAL_FIPS202_ONLY */ + + /* For MLKEM_K == 3, sample the last entry individually. + * When MLK_CONFIG_SERIAL_FIPS202_ONLY is set, sample all entries + * individually. */ + for (; i < MLKEM_K * MLKEM_K; i++) + { + uint8_t x, y; + /* MLKEM_K <= 4, so the values fit in uint8_t. */ + x = (uint8_t)(i / MLKEM_K); + y = (uint8_t)(i % MLKEM_K); + + if (transposed) + { + seed_ext[0][MLKEM_SYMBYTES + 0] = x; + seed_ext[0][MLKEM_SYMBYTES + 1] = y; + } + else + { + seed_ext[0][MLKEM_SYMBYTES + 0] = y; + seed_ext[0][MLKEM_SYMBYTES + 1] = x; + } + + mlk_poly_rej_uniform(&a->vec[i / MLKEM_K].vec[i % MLKEM_K], seed_ext[0]); + } + + mlk_assert(i == MLKEM_K * MLKEM_K); + + /* + * The public matrix is generated in NTT domain. If the native backend + * uses a custom order in NTT domain, permute A accordingly. + */ + mlk_polymat_permute_bitrev_to_custom(a); + + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(seed_ext, sizeof(seed_ext)); +} + +/** + * Compute matrix-vector product in NTT domain, via Montgomery multiplication. + * + * @spec{Implements @[FIPS203, Section 2.4.7, Eq (2.12), (2.13)].} + * + * @param[out] out Output polynomial vector. + * @param[in] a Input matrix. Must be in NTT domain and have coefficients + * of absolute value < 4096. + * @param[in] v Input polynomial vector. Must be in NTT domain. + * @param[in] vc Mulcache for @p v, computed via + * mlk_polyvec_mulcache_compute(). + */ +static void mlk_matvec_mul(mlk_polyvec *out, const mlk_polymat *a, + const mlk_polyvec *v, const mlk_polyvec_mulcache *vc) +__contract__( + requires(memory_no_alias(out, sizeof(mlk_polyvec))) + requires(memory_no_alias(a, sizeof(mlk_polymat))) + requires(memory_no_alias(v, sizeof(mlk_polyvec))) + requires(memory_no_alias(vc, sizeof(mlk_polyvec_mulcache))) + requires(forall(k0, 0, MLKEM_K, + forall(k1, 0, MLKEM_K, + array_bound(a->vec[k0].vec[k1].coeffs, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT)))) + assigns(memory_slice(out, sizeof(mlk_polyvec)))) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + __loop__( + assigns(i, memory_slice(out, sizeof(mlk_polyvec))) + invariant(i <= MLKEM_K) + decreases(MLKEM_K - i)) + { + mlk_polyvec_basemul_acc_montgomery_cached(&out->vec[i], &a->vec[i], v, vc); + } +} + +/** + * Compute and fill the pv and e polyvec structures needed by + * mlk_keypair_derand(). Uses x4-batched versions of `poly_getnoise` to + * leverage batched Keccak-f1600. + * + * @spec{Implements @[FIPS203, Algorithm 13 (K-PKE.KeyGen)] steps 8-15.} + * + * @param[out] pv Output polynomial vector. + * @param[out] e Output polynomial vector. + * @param[in] seed Seed bytes for sampling. + */ +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +static void mlk_keypair_getnoise_eta1(mlk_polyvec *pv, mlk_polyvec *e, + const uint8_t seed[MLKEM_SYMBYTES]) +__contract__( + requires(memory_no_alias(pv, sizeof(mlk_polyvec))) + requires(memory_no_alias(e, sizeof(mlk_polyvec))) + requires(memory_no_alias(seed, MLKEM_SYMBYTES)) + assigns(memory_slice(pv, sizeof(mlk_polyvec))) + assigns(memory_slice(e, sizeof(mlk_polyvec))) + ensures(forall(k0, 0, MLKEM_K, array_abs_bound(pv->vec[k0].coeffs, 0, MLKEM_N, MLKEM_ETA1 + 1))) + ensures(forall(k1, 0, MLKEM_K, array_abs_bound(e->vec[k1].coeffs, 0, MLKEM_N, MLKEM_ETA1 + 1))) +) +{ +#if MLKEM_K == 2 + mlk_poly_getnoise_eta1_4x(&pv->vec[0], &pv->vec[1], /* Fill elements of pv */ + &e->vec[0], &e->vec[1], /* and two elements of e */ + seed, 0, 1, 2, 3); +#elif MLKEM_K == 3 + /* + * Only the first three output buffers are needed, so we pass NULL as + * the fourth parameter, and 0xFF as its dummy nonce. + */ + mlk_poly_getnoise_eta1_4x(&pv->vec[0], &pv->vec[1], &pv->vec[2], NULL, seed, + 0, 1, 2, 0xFF); + /* Same here */ + mlk_poly_getnoise_eta1_4x(&e->vec[0], &e->vec[1], &e->vec[2], NULL, seed, 3, + 4, 5, 0xFF); +#elif MLKEM_K == 4 + mlk_poly_getnoise_eta1_4x(&pv->vec[0], &pv->vec[1], &pv->vec[2], &pv->vec[3], + seed, 0, 1, 2, 3); + mlk_poly_getnoise_eta1_4x(&e->vec[0], &e->vec[1], &e->vec[2], &e->vec[3], + seed, 4, 5, 6, 7); +#endif /* MLKEM_K == 4 */ +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +/** + * Compute and fill the sp, ep, and epp polynomial structures needed by + * mlk_indcpa_enc(). Uses x4-batched versions of `poly_getnoise` to leverage + * batched Keccak-f1600. + * + * @spec{Implements @[FIPS203, Algorithm 14 (K-PKE.Encrypt)] steps 9-16.} + * + * @param[out] sp Output polynomial vector. + * @param[out] ep Output polynomial vector. + * @param[out] epp Output polynomial. + * @param[in] coins Seed bytes for sampling. + */ +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +static void mlk_enc_getnoise_eta1_eta2(mlk_polyvec *sp, mlk_polyvec *ep, + mlk_poly *epp, + const uint8_t coins[MLKEM_SYMBYTES]) +__contract__( + requires(memory_no_alias(sp, sizeof(mlk_polyvec))) + requires(memory_no_alias(ep, sizeof(mlk_polyvec))) + requires(memory_no_alias(epp, sizeof(mlk_poly))) + requires(memory_no_alias(coins, MLKEM_SYMBYTES)) + assigns(memory_slice(sp, sizeof(mlk_polyvec))) + assigns(memory_slice(ep, sizeof(mlk_polyvec))) + assigns(memory_slice(epp, sizeof(mlk_poly))) + ensures(forall(k0, 0, MLKEM_K, array_abs_bound(sp->vec[k0].coeffs, 0, MLKEM_N, MLKEM_ETA1 + 1))) + ensures(forall(k1, 0, MLKEM_K, array_abs_bound(ep->vec[k1].coeffs, 0, MLKEM_N, MLKEM_ETA2 + 1))) + ensures(array_abs_bound(epp->coeffs, 0, MLKEM_N, MLKEM_ETA2 + 1)) +) +{ +#if MLKEM_K == 2 + mlk_poly_getnoise_eta1122_4x(&sp->vec[0], &sp->vec[1], &ep->vec[0], + &ep->vec[1], coins, 0, 1, 2, 3); + mlk_poly_getnoise_eta2(epp, coins, 4); +#elif MLKEM_K == 3 + /* + * In this call, only the first three output buffers are needed, + * so we pass NULL as the fourth parameter, and 0xFF as its dummy nonce. + */ + mlk_poly_getnoise_eta1_4x(&sp->vec[0], &sp->vec[1], &sp->vec[2], NULL, coins, + 0, 1, 2, 0xFF /* irrelevant */); + /* The fourth output buffer in this call _is_ used. */ + mlk_poly_getnoise_eta2_4x(&ep->vec[0], &ep->vec[1], &ep->vec[2], epp, coins, + 3, 4, 5, 6); +#elif MLKEM_K == 4 + mlk_poly_getnoise_eta1_4x(&sp->vec[0], &sp->vec[1], &sp->vec[2], &sp->vec[3], + coins, 0, 1, 2, 3); + mlk_poly_getnoise_eta2_4x(&ep->vec[0], &ep->vec[1], &ep->vec[2], &ep->vec[3], + coins, 4, 5, 6, 7); + mlk_poly_getnoise_eta2(epp, coins, 8); +#endif /* MLKEM_K == 4 */ +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + + +/* Reference: `indcpa_keypair_derand()` in the reference implementation @[REF]. + * - We use a different implementation of `gen_matrix()` which + * uses x4-batched Keccak-f1600 (see `mlk_gen_matrix()` above). + * - We use a mulcache to speed up matrix-vector multiplication. + * - We include buffer zeroization. + */ +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +MLK_INTERNAL_API +int mlk_indcpa_keypair_derand(uint8_t pk[MLKEM_INDCPA_PUBLICKEYBYTES], + uint8_t sk[MLKEM_INDCPA_SECRETKEYBYTES], + const uint8_t coins[MLKEM_SYMBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + const uint8_t *publicseed; + const uint8_t *noiseseed; + MLK_ALLOC(buf, uint8_t, 2 * MLKEM_SYMBYTES, context); + MLK_ALLOC(coins_with_domain_separator, uint8_t, MLKEM_SYMBYTES + 1, context); + MLK_ALLOC(a, mlk_polymat, 1, context); + MLK_ALLOC(e, mlk_polyvec, 1, context); + MLK_ALLOC(pkpv, mlk_polyvec, 1, context); + MLK_ALLOC(skpv, mlk_polyvec, 1, context); + MLK_ALLOC(skpv_cache, mlk_polyvec_mulcache, 1, context); + + if (buf == NULL || coins_with_domain_separator == NULL || a == NULL || + e == NULL || pkpv == NULL || skpv == NULL || skpv_cache == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + publicseed = buf; + noiseseed = buf + MLKEM_SYMBYTES; + + /* Concatenate coins with MLKEM_K for domain separation of security levels */ + mlk_memcpy(coins_with_domain_separator, coins, MLKEM_SYMBYTES); + coins_with_domain_separator[MLKEM_SYMBYTES] = MLKEM_K; + + mlk_hash_g(buf, coins_with_domain_separator, MLKEM_SYMBYTES + 1); + + /* + * Declassify the public seed. + * Required to use it in conditional-branches in rejection sampling. + * This is needed because all output of randombytes is marked as secret + * (=undefined) + */ + MLK_CT_TESTING_DECLASSIFY(publicseed, MLKEM_SYMBYTES); + + mlk_gen_matrix(a, publicseed, 0 /* no transpose */); + + mlk_keypair_getnoise_eta1(skpv, e, noiseseed); + + mlk_polyvec_ntt(skpv); + mlk_polyvec_ntt(e); + + mlk_polyvec_mulcache_compute(skpv_cache, skpv); + mlk_matvec_mul(pkpv, a, skpv, skpv_cache); + mlk_polyvec_tomont(pkpv); + + mlk_polyvec_add(pkpv, e); + mlk_polyvec_reduce(pkpv); + mlk_polyvec_reduce(skpv); + + mlk_pack_sk(sk, skpv); + mlk_pack_pk(pk, pkpv, publicseed); + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(skpv_cache, mlk_polyvec_mulcache, 1, context); + MLK_FREE(skpv, mlk_polyvec, 1, context); + MLK_FREE(pkpv, mlk_polyvec, 1, context); + MLK_FREE(e, mlk_polyvec, 1, context); + MLK_FREE(a, mlk_polymat, 1, context); + MLK_FREE(coins_with_domain_separator, uint8_t, MLKEM_SYMBYTES + 1, context); + MLK_FREE(buf, uint8_t, 2 * MLKEM_SYMBYTES, context); + return ret; +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +/* Reference: `indcpa_enc()` in the reference implementation @[REF]. + * - We use x4-batched versions of `poly_getnoise` to leverage + * batched x4-batched Keccak-f1600. + * - We use a different implementation of `gen_matrix()` which + * uses x4-batched Keccak-f1600 (see `mlk_gen_matrix()` above). + * - We use a mulcache to speed up matrix-vector multiplication. + * - We include buffer zeroization. + */ +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_INTERNAL_API +int mlk_indcpa_enc(uint8_t c[MLKEM_INDCPA_BYTES], + const uint8_t m[MLKEM_INDCPA_MSGBYTES], + const uint8_t pk[MLKEM_INDCPA_PUBLICKEYBYTES], + const uint8_t coins[MLKEM_SYMBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + MLK_ALLOC(seed, uint8_t, MLKEM_SYMBYTES, context); + MLK_ALLOC(at, mlk_polymat, 1, context); + MLK_ALLOC(sp, mlk_polyvec, 1, context); + MLK_ALLOC(pkpv, mlk_polyvec, 1, context); + MLK_ALLOC(ep, mlk_polyvec, 1, context); + MLK_ALLOC(b, mlk_polyvec, 1, context); + MLK_ALLOC(v, mlk_poly, 1, context); + MLK_ALLOC(k, mlk_poly, 1, context); + MLK_ALLOC(epp, mlk_poly, 1, context); + MLK_ALLOC(sp_cache, mlk_polyvec_mulcache, 1, context); + + if (seed == NULL || at == NULL || sp == NULL || pkpv == NULL || ep == NULL || + b == NULL || v == NULL || k == NULL || epp == NULL || sp_cache == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + mlk_unpack_pk(pkpv, seed, pk); + mlk_poly_frommsg(k, m); + + /* + * Declassify the public seed. + * Required to use it in conditional-branches in rejection sampling. + * This is needed because in re-encryption the publicseed originated from sk + * which is marked undefined. + */ + MLK_CT_TESTING_DECLASSIFY(seed, MLKEM_SYMBYTES); + + mlk_gen_matrix(at, seed, 1 /* transpose */); + + mlk_enc_getnoise_eta1_eta2(sp, ep, epp, coins); + + mlk_polyvec_ntt(sp); + + mlk_polyvec_mulcache_compute(sp_cache, sp); + mlk_matvec_mul(b, at, sp, sp_cache); + mlk_polyvec_basemul_acc_montgomery_cached(v, pkpv, sp, sp_cache); + + mlk_polyvec_invntt_tomont(b); + mlk_poly_invntt_tomont(v); + + mlk_polyvec_add(b, ep); + mlk_poly_add(v, epp); + mlk_poly_add(v, k); + + mlk_polyvec_reduce(b); + mlk_poly_reduce(v); + + mlk_pack_ciphertext(c, b, v); + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(sp_cache, mlk_polyvec_mulcache, 1, context); + MLK_FREE(epp, mlk_poly, 1, context); + MLK_FREE(k, mlk_poly, 1, context); + MLK_FREE(v, mlk_poly, 1, context); + MLK_FREE(b, mlk_polyvec, 1, context); + MLK_FREE(ep, mlk_polyvec, 1, context); + MLK_FREE(pkpv, mlk_polyvec, 1, context); + MLK_FREE(sp, mlk_polyvec, 1, context); + MLK_FREE(at, mlk_polymat, 1, context); + MLK_FREE(seed, uint8_t, MLKEM_SYMBYTES, context); + return ret; +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +/* Reference: `indcpa_dec()` in the reference implementation @[REF]. + * - We use a mulcache for the scalar product. + * - We include buffer zeroization. */ +#if !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_INTERNAL_API +int mlk_indcpa_dec(uint8_t m[MLKEM_INDCPA_MSGBYTES], + const uint8_t c[MLKEM_INDCPA_BYTES], + const uint8_t sk[MLKEM_INDCPA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + MLK_ALLOC(b, mlk_polyvec, 1, context); + MLK_ALLOC(skpv, mlk_polyvec, 1, context); + MLK_ALLOC(v, mlk_poly, 1, context); + MLK_ALLOC(sb, mlk_poly, 1, context); + MLK_ALLOC(b_cache, mlk_polyvec_mulcache, 1, context); + + if (b == NULL || skpv == NULL || v == NULL || sb == NULL || b_cache == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + mlk_unpack_ciphertext(b, v, c); + mlk_unpack_sk(skpv, sk); + + mlk_polyvec_ntt(b); + mlk_polyvec_mulcache_compute(b_cache, b); + mlk_polyvec_basemul_acc_montgomery_cached(sb, skpv, b, b_cache); + mlk_poly_invntt_tomont(sb); + + mlk_poly_sub(v, sb); + mlk_poly_reduce(v); + + mlk_poly_tomsg(m, v); + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(b_cache, mlk_polyvec_mulcache, 1, context); + MLK_FREE(sb, mlk_poly, 1, context); + MLK_FREE(v, mlk_poly, 1, context); + MLK_FREE(skpv, mlk_polyvec, 1, context); + MLK_FREE(b, mlk_polyvec, 1, context); + return ret; +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef mlk_pack_pk +#undef mlk_unpack_pk +#undef mlk_pack_sk +#undef mlk_unpack_sk +#undef mlk_pack_ciphertext +#undef mlk_unpack_ciphertext +#undef mlk_matvec_mul +#undef mlk_polyvec_permute_bitrev_to_custom +#undef mlk_polymat_permute_bitrev_to_custom +#undef mlk_keypair_getnoise_eta1 +#undef mlk_enc_getnoise_eta1_eta2 diff --git a/src/mlkem-native/mlkem/src/indcpa.h b/src/mlkem-native/mlkem/src/indcpa.h new file mode 100644 index 0000000000..82315c2ef2 --- /dev/null +++ b/src/mlkem-native/mlkem/src/indcpa.h @@ -0,0 +1,172 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_INDCPA_H +#define MLK_INDCPA_H + +#include "cbmc.h" +#include "common.h" +#include "poly_k.h" + +#define mlk_gen_matrix MLK_NAMESPACE_K(gen_matrix) +/** + * Deterministically generate matrix A (or the transpose of A) from a seed. + * Entries of the matrix are polynomials that look uniformly random. + * Performs rejection sampling on the output of an XOF. + * + * @spec{Implements @[FIPS203, Algorithm 13 (K-PKE.KeyGen), L3-7] and + * @[FIPS203, Algorithm 14 (K-PKE.Encrypt), L4-8]. The @p transposed + * parameter only affects internal presentation.} + * + * @param[out] a Output matrix A. + * @param[in] seed Input seed. + * @param transposed Boolean deciding whether A or A^T is generated. + */ +MLK_INTERNAL_API +void mlk_gen_matrix(mlk_polymat *a, const uint8_t seed[MLKEM_SYMBYTES], + int transposed) +__contract__( + requires(memory_no_alias(a, sizeof(mlk_polymat))) + requires(memory_no_alias(seed, MLKEM_SYMBYTES)) + requires(transposed == 0 || transposed == 1) + assigns(memory_slice(a, sizeof(mlk_polymat))) + ensures(forall(x, 0, MLKEM_K, forall(y, 0, MLKEM_K, + array_bound(a->vec[x].vec[y].coeffs, 0, MLKEM_N, 0, MLKEM_Q)))) +); + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +#define mlk_indcpa_keypair_derand \ + MLK_NAMESPACE_K(indcpa_keypair_derand) MLK_CONTEXT_PARAMETERS_3 +/** + * Generate public and private key for the CPA-secure public-key encryption + * scheme underlying ML-KEM. + * + * @spec{Implements @[FIPS203, Algorithm 13 (K-PKE.KeyGen)].} + * + * @param[out] pk Output public key + * (length MLKEM_INDCPA_PUBLICKEYBYTES bytes). + * @param[out] sk Output private key + * (length MLKEM_INDCPA_SECRETKEYBYTES bytes). + * @param[in] coins Input randomness (length MLKEM_SYMBYTES bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +MLK_INTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_indcpa_keypair_derand(uint8_t pk[MLKEM_INDCPA_PUBLICKEYBYTES], + uint8_t sk[MLKEM_INDCPA_SECRETKEYBYTES], + const uint8_t coins[MLKEM_SYMBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(pk, MLKEM_INDCPA_PUBLICKEYBYTES)) + requires(memory_no_alias(sk, MLKEM_INDCPA_SECRETKEYBYTES)) + requires(memory_no_alias(coins, MLKEM_SYMBYTES)) + assigns(memory_slice(pk, MLKEM_INDCPA_PUBLICKEYBYTES)) + assigns(memory_slice(sk, MLKEM_INDCPA_SECRETKEYBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_OUT_OF_MEMORY) + /* Output buffers on error, per API-CONVENTIONS.md */ + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(pk, MLKEM_INDCPA_PUBLICKEYBYTES)) + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(sk, MLKEM_INDCPA_SECRETKEYBYTES)) +); +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_indcpa_enc MLK_NAMESPACE_K(indcpa_enc) MLK_CONTEXT_PARAMETERS_4 +/** + * Encryption function of the CPA-secure public-key encryption scheme + * underlying ML-KEM. + * + * @spec{Implements @[FIPS203, Algorithm 14 (K-PKE.Encrypt)].} + * + * @param[out] c Output ciphertext (length MLKEM_INDCPA_BYTES bytes). + * @param[in] m Input message (length MLKEM_INDCPA_MSGBYTES bytes). + * @param[in] pk Input public key + * (length MLKEM_INDCPA_PUBLICKEYBYTES bytes). + * @param[in] coins Input random coins used as seed (length MLKEM_SYMBYTES + * bytes) to deterministically generate all randomness. + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +MLK_INTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_indcpa_enc(uint8_t c[MLKEM_INDCPA_BYTES], + const uint8_t m[MLKEM_INDCPA_MSGBYTES], + const uint8_t pk[MLKEM_INDCPA_PUBLICKEYBYTES], + const uint8_t coins[MLKEM_SYMBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(c, MLKEM_INDCPA_BYTES)) + requires(memory_no_alias(m, MLKEM_INDCPA_MSGBYTES)) + requires(memory_no_alias(pk, MLKEM_INDCPA_PUBLICKEYBYTES)) + requires(memory_no_alias(coins, MLKEM_SYMBYTES)) + assigns(memory_slice(c, MLKEM_INDCPA_BYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_OUT_OF_MEMORY) + /* Output buffers on error, per API-CONVENTIONS.md */ + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(c, MLKEM_INDCPA_BYTES)) +); +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_indcpa_dec MLK_NAMESPACE_K(indcpa_dec) MLK_CONTEXT_PARAMETERS_3 +/** + * Decryption function of the CPA-secure public-key encryption scheme + * underlying ML-KEM. + * + * @spec{Implements @[FIPS203, Algorithm 15 (K-PKE.Decrypt)].} + * + * @param[out] m Output decrypted message + * (length MLKEM_INDCPA_MSGBYTES bytes). + * @param[in] c Input ciphertext (length MLKEM_INDCPA_BYTES bytes). + * @param[in] sk Input secret key + * (length MLKEM_INDCPA_SECRETKEYBYTES bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +MLK_INTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_indcpa_dec(uint8_t m[MLKEM_INDCPA_MSGBYTES], + const uint8_t c[MLKEM_INDCPA_BYTES], + const uint8_t sk[MLKEM_INDCPA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(c, MLKEM_INDCPA_BYTES)) + requires(memory_no_alias(m, MLKEM_INDCPA_MSGBYTES)) + requires(memory_no_alias(sk, MLKEM_INDCPA_SECRETKEYBYTES)) + assigns(memory_slice(m, MLKEM_INDCPA_MSGBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_OUT_OF_MEMORY) + /* Output buffers on error, per API-CONVENTIONS.md */ + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(m, MLKEM_INDCPA_MSGBYTES)) +); +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#endif /* !MLK_INDCPA_H */ diff --git a/src/mlkem-native/mlkem/src/kem.c b/src/mlkem-native/mlkem/src/kem.c new file mode 100644 index 0000000000..57e767fd51 --- /dev/null +++ b/src/mlkem-native/mlkem/src/kem.c @@ -0,0 +1,474 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS140_3_IG] + * Implementation Guidance for FIPS 140-3 and the Cryptographic Module + * Validation Program + * National Institute of Standards and Technology + * https://csrc.nist.gov/projects/cryptographic-module-validation-program/fips-140-3-ig-announcements + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + */ + +#include "kem.h" + +#include "indcpa.h" +#include "randombytes.h" +#include "symmetric.h" +#include "verify.h" + +/* Parameter set namespacing + * This is to facilitate building multiple instances + * of mlkem-native (e.g. with varying security levels) + * within a single compilation unit. */ +#define mlk_check_pct MLK_ADD_PARAM_SET(mlk_check_pct) MLK_CONTEXT_PARAMETERS_2 +/* End of parameter set namespacing */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) +/* Reference: Not implemented in the reference implementation @[REF]. */ +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_check_pk(const uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + MLK_ALLOC(p, mlk_polyvec, 1, context); + MLK_ALLOC(p_reencoded, uint8_t, MLKEM_POLYVECBYTES, context); + + if (p == NULL || p_reencoded == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + mlk_polyvec_frombytes(p, pk); + mlk_polyvec_reduce(p); + mlk_polyvec_tobytes(p_reencoded, p); + + /* We use a constant-time memcmp here to avoid having to + * declassify the PK before the PCT has succeeded. */ + ret = mlk_ct_memcmp(pk, p_reencoded, MLKEM_POLYVECBYTES) ? MLK_ERR_INVALID_PK + : 0; + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(p_reencoded, uint8_t, MLKEM_POLYVECBYTES, context); + MLK_FREE(p, mlk_polyvec, 1, context); + return ret; +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API */ + + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +/* Reference: Not implemented in the reference implementation @[REF]. */ +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_check_sk(const uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + MLK_ALLOC(test, uint8_t, MLKEM_SYMBYTES, context); + + if (test == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + /* + * The parts of `sk` being hashed and compared here are public, so + * no public information is leaked through the runtime or the return value + * of this function. + */ + + /* Declassify the public part of the secret key */ + MLK_CT_TESTING_DECLASSIFY(sk + MLKEM_INDCPA_SECRETKEYBYTES, + MLKEM_INDCCA_PUBLICKEYBYTES); + MLK_CT_TESTING_DECLASSIFY( + sk + MLKEM_INDCCA_SECRETKEYBYTES - 2 * MLKEM_SYMBYTES, MLKEM_SYMBYTES); + + mlk_hash_h(test, sk + MLKEM_INDCPA_SECRETKEYBYTES, + MLKEM_INDCCA_PUBLICKEYBYTES); + /* This doesn't have to be a constant-time memcmp, but it's the only place + * in the library where a normal memcmp would be used otherwise, so for sake + * of minimizing stdlib dependency, we use our constant-time one anyway. */ + ret = mlk_ct_memcmp(sk + MLKEM_INDCCA_SECRETKEYBYTES - 2 * MLKEM_SYMBYTES, + test, MLKEM_SYMBYTES) + ? MLK_ERR_INVALID_SK + : 0; + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(test, uint8_t, MLKEM_SYMBYTES, context); + return ret; +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +MLK_MUST_CHECK_RETURN_VALUE +static int mlk_check_pct(uint8_t const pk[MLKEM_INDCCA_PUBLICKEYBYTES], + uint8_t const sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + requires(memory_no_alias(sk, MLKEM_INDCCA_SECRETKEYBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_OUT_OF_MEMORY || + return_value == MLK_ERR_RNG_FAIL || + return_value == MLK_ERR_PCT_FAIL) +); + +#if defined(MLK_CONFIG_KEYGEN_PCT) +/* Specification: + * Partially implements 'Pairwise Consistency Test' @[FIPS140_3_IG, p.87] and + * @[FIPS203, Section 7.1, Pairwise Consistency]. */ + +/* Reference: Not implemented in the reference implementation @[REF]. + * + * @retval 0 Success. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + * @retval MLK_ERR_RNG_FAIL Random number generation failed. + * @retval MLK_ERR_PCT_FAIL The consistency check failed. */ +MLK_MUST_CHECK_RETURN_VALUE +static int mlk_check_pct(uint8_t const pk[MLKEM_INDCCA_PUBLICKEYBYTES], + uint8_t const sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + MLK_ALLOC(ct, uint8_t, MLKEM_INDCCA_CIPHERTEXTBYTES, context); + MLK_ALLOC(ss_enc, uint8_t, MLKEM_SSBYTES, context); + MLK_ALLOC(ss_dec, uint8_t, MLKEM_SSBYTES, context); + + if (ct == NULL || ss_enc == NULL || ss_dec == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + ret = mlk_kem_enc(ct, ss_enc, pk, context); + if (ret != 0) + { + goto cleanup; + } + + ret = mlk_kem_dec(ss_dec, ct, sk, context); + if (ret != 0) + { + goto cleanup; + } + +#if defined(MLK_CONFIG_KEYGEN_PCT_BREAKAGE_TEST) + /* Deliberately break PCT for testing purposes */ + if (mlk_break_pct()) + { + ss_enc[0] = ~ss_enc[0]; + } +#endif /* MLK_CONFIG_KEYGEN_PCT_BREAKAGE_TEST */ + + ret = mlk_ct_memcmp(ss_enc, ss_dec, MLKEM_SSBYTES); + /* The result of the PCT is public. */ + MLK_CT_TESTING_DECLASSIFY(&ret, sizeof(ret)); + + if (ret != 0) + { + ret = MLK_ERR_PCT_FAIL; + } + +cleanup: + + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(ss_dec, uint8_t, MLKEM_SSBYTES, context); + MLK_FREE(ss_enc, uint8_t, MLKEM_SSBYTES, context); + MLK_FREE(ct, uint8_t, MLKEM_INDCCA_CIPHERTEXTBYTES, context); + + /* The key pair being tested was just generated by this library, so a key + * check rejecting it hints at a faulty implementation rather than at bad + * input. Report it as a PCT failure. */ + if (ret == MLK_ERR_INVALID_PK || ret == MLK_ERR_INVALID_SK) + { + ret = MLK_ERR_PCT_FAIL; + } + + /* Other error codes, e.g. platform failures like out of memory or + * randomness failure, are passed on unmodified. */ + + return ret; +} +#else /* MLK_CONFIG_KEYGEN_PCT */ +MLK_MUST_CHECK_RETURN_VALUE +static int mlk_check_pct(uint8_t const pk[MLKEM_INDCCA_PUBLICKEYBYTES], + uint8_t const sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + /* Skip PCT */ + ((void)pk); + ((void)sk); + MLK_CONTEXT_UNUSED(context); + return 0; +} +#endif /* !MLK_CONFIG_KEYGEN_PCT */ + +/* Reference: `crypto_kem_keypair_derand()` in the reference implementation + * @[REF]. + * - We optionally include PCT which is not present in + * the reference code. */ +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_keypair_derand(uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES], + const uint8_t coins[2 * MLKEM_SYMBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret; + + ret = mlk_indcpa_keypair_derand(pk, sk, coins, context); + if (ret != 0) + { + goto cleanup; + } + + mlk_memcpy(sk + MLKEM_INDCPA_SECRETKEYBYTES, pk, MLKEM_INDCCA_PUBLICKEYBYTES); + mlk_hash_h(sk + MLKEM_INDCCA_SECRETKEYBYTES - 2 * MLKEM_SYMBYTES, pk, + MLKEM_INDCCA_PUBLICKEYBYTES); + /* Value z for pseudo-random output on reject */ + mlk_memcpy(sk + MLKEM_INDCCA_SECRETKEYBYTES - MLKEM_SYMBYTES, + coins + MLKEM_SYMBYTES, MLKEM_SYMBYTES); + + /* Declassify public key */ + MLK_CT_TESTING_DECLASSIFY(pk, MLKEM_INDCCA_PUBLICKEYBYTES); + + /* Pairwise Consistency Test (PCT) @[FIPS140_3_IG, p.87] */ + ret = mlk_check_pct(pk, sk, context); + if (ret != 0) + { + goto cleanup; + } + +cleanup: + if (ret != 0) + { + mlk_zeroize(pk, MLKEM_INDCCA_PUBLICKEYBYTES); + mlk_zeroize(sk, MLKEM_INDCCA_SECRETKEYBYTES); + } + + return ret; +} + +#if !defined(MLK_CONFIG_NO_RANDOMIZED_API) +/* Reference: `crypto_kem_keypair()` in the reference implementation @[REF] + * - We zeroize the stack buffer */ +MLK_EXTERNAL_API +int mlk_kem_keypair(uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + MLK_ALLOC(coins, uint8_t, 2 * MLKEM_SYMBYTES, context); + + if (coins == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + /* Acquire necessary randomness, and mark it as secret. */ + if (mlk_randombytes(coins, 2 * MLKEM_SYMBYTES) != 0) + { + ret = MLK_ERR_RNG_FAIL; + goto cleanup; + } + + MLK_CT_TESTING_SECRET(coins, 2 * MLKEM_SYMBYTES); + + ret = mlk_kem_keypair_derand(pk, sk, coins, context); + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(coins, uint8_t, 2 * MLKEM_SYMBYTES, context); + return ret; +} +#endif /* !MLK_CONFIG_NO_RANDOMIZED_API */ +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) +/* Reference: `crypto_kem_enc_derand()` in the reference implementation @[REF] + * - We include public key check + * - We include stack buffer zeroization */ +MLK_EXTERNAL_API +int mlk_kem_enc_derand(uint8_t ct[MLKEM_INDCCA_CIPHERTEXTBYTES], + uint8_t ss[MLKEM_SSBYTES], + const uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + const uint8_t coins[MLKEM_SYMBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + MLK_ALLOC(buf, uint8_t, 2 * MLKEM_SYMBYTES, context); + MLK_ALLOC(kr, uint8_t, 2 * MLKEM_SYMBYTES, context); + + if (buf == NULL || kr == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + /* Specification: Implements @[FIPS203, Section 7.2, Modulus check] */ + ret = mlk_kem_check_pk(pk, context); + if (ret != 0) + { + goto cleanup; + } + + mlk_memcpy(buf, coins, MLKEM_SYMBYTES); + + /* Multitarget countermeasure for coins + contributory KEM */ + mlk_hash_h(buf + MLKEM_SYMBYTES, pk, MLKEM_INDCCA_PUBLICKEYBYTES); + mlk_hash_g(kr, buf, 2 * MLKEM_SYMBYTES); + + /* coins are in kr+MLKEM_SYMBYTES */ + ret = mlk_indcpa_enc(ct, buf, pk, kr + MLKEM_SYMBYTES, context); + if (ret != 0) + { + goto cleanup; + } + + mlk_memcpy(ss, kr, MLKEM_SYMBYTES); + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(kr, uint8_t, 2 * MLKEM_SYMBYTES, context); + MLK_FREE(buf, uint8_t, 2 * MLKEM_SYMBYTES, context); + return ret; +} + +#if !defined(MLK_CONFIG_NO_RANDOMIZED_API) +/* Reference: `crypto_kem_enc()` in the reference implementation @[REF] + * - We include stack buffer zeroization */ +MLK_EXTERNAL_API +int mlk_kem_enc(uint8_t ct[MLKEM_INDCCA_CIPHERTEXTBYTES], + uint8_t ss[MLKEM_SSBYTES], + const uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + MLK_ALLOC(coins, uint8_t, MLKEM_SYMBYTES, context); + + if (coins == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + if (mlk_randombytes(coins, MLKEM_SYMBYTES) != 0) + { + ret = MLK_ERR_RNG_FAIL; + goto cleanup; + } + + MLK_CT_TESTING_SECRET(coins, MLKEM_SYMBYTES); + + ret = mlk_kem_enc_derand(ct, ss, pk, coins, context); + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(coins, uint8_t, MLKEM_SYMBYTES, context); + return ret; +} +#endif /* !MLK_CONFIG_NO_RANDOMIZED_API */ +#endif /* !MLK_CONFIG_NO_ENCAPS_API */ + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +/* Reference: `crypto_kem_dec()` in the reference implementation @[REF] + * - We include secret key check + * - We include stack buffer zeroization */ +MLK_EXTERNAL_API +int mlk_kem_dec(uint8_t ss[MLKEM_SSBYTES], + const uint8_t ct[MLKEM_INDCCA_CIPHERTEXTBYTES], + const uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + uint8_t fail; + const uint8_t *pk = sk + MLKEM_INDCPA_SECRETKEYBYTES; + MLK_ALLOC(buf, uint8_t, 2 * MLKEM_SYMBYTES, context); + MLK_ALLOC(kr, uint8_t, 2 * MLKEM_SYMBYTES, context); + MLK_ALLOC(tmp, uint8_t, MLKEM_SYMBYTES + MLKEM_INDCCA_CIPHERTEXTBYTES, + context); + + if (buf == NULL || kr == NULL || tmp == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + /* Specification: Implements @[FIPS203, Section 7.3, Hash check] */ + ret = mlk_kem_check_sk(sk, context); + if (ret != 0) + { + goto cleanup; + } + + ret = mlk_indcpa_dec(buf, ct, sk, context); + if (ret != 0) + { + goto cleanup; + } + + /* Multitarget countermeasure for coins + contributory KEM */ + mlk_memcpy(buf + MLKEM_SYMBYTES, + sk + MLKEM_INDCCA_SECRETKEYBYTES - 2 * MLKEM_SYMBYTES, + MLKEM_SYMBYTES); + mlk_hash_g(kr, buf, 2 * MLKEM_SYMBYTES); + + /* Recompute and compare ciphertext */ + /* coins are in kr+MLKEM_SYMBYTES */ + ret = mlk_indcpa_enc(tmp, buf, pk, kr + MLKEM_SYMBYTES, context); + if (ret != 0) + { + goto cleanup; + } + + fail = mlk_ct_memcmp(ct, tmp, MLKEM_INDCCA_CIPHERTEXTBYTES); + + /* Compute rejection key */ + mlk_memcpy(tmp, sk + MLKEM_INDCCA_SECRETKEYBYTES - MLKEM_SYMBYTES, + MLKEM_SYMBYTES); + mlk_memcpy(tmp + MLKEM_SYMBYTES, ct, MLKEM_INDCCA_CIPHERTEXTBYTES); + mlk_hash_j(ss, tmp, MLKEM_SYMBYTES + MLKEM_INDCCA_CIPHERTEXTBYTES); + + /* Copy true key to return buffer if fail is 0 */ + mlk_ct_cmov_zero(ss, kr, MLKEM_SYMBYTES, fail); + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(tmp, uint8_t, MLKEM_SYMBYTES + MLKEM_INDCCA_CIPHERTEXTBYTES, + context); + MLK_FREE(kr, uint8_t, 2 * MLKEM_SYMBYTES, context); + MLK_FREE(buf, uint8_t, 2 * MLKEM_SYMBYTES, context); + + return ret; +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef mlk_check_pct diff --git a/src/mlkem-native/mlkem/src/kem.h b/src/mlkem-native/mlkem/src/kem.h new file mode 100644 index 0000000000..2de960c9c0 --- /dev/null +++ b/src/mlkem-native/mlkem/src/kem.h @@ -0,0 +1,353 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + */ + +#ifndef MLK_KEM_H +#define MLK_KEM_H + +#include "cbmc.h" +#include "common.h" +#include "sys.h" + +#if defined(MLK_CHECK_APIS) +/* Include to ensure consistency between internal kem.h + * and external mlkem_native.h. */ +#include "mlkem_native.h" + +#if MLKEM_INDCCA_SECRETKEYBYTES != \ + MLKEM_SECRETKEYBYTES(MLK_CONFIG_PARAMETER_SET) +#error Mismatch for SECRETKEYBYTES between kem.h and mlkem_native.h +#endif + +#if MLKEM_INDCCA_PUBLICKEYBYTES != \ + MLKEM_PUBLICKEYBYTES(MLK_CONFIG_PARAMETER_SET) +#error Mismatch for PUBLICKEYBYTES between kem.h and mlkem_native.h +#endif + +#if MLKEM_INDCCA_CIPHERTEXTBYTES != \ + MLKEM_CIPHERTEXTBYTES(MLK_CONFIG_PARAMETER_SET) +#error Mismatch for CIPHERTEXTBYTES between kem.h and mlkem_native.h +#endif + +#endif /* MLK_CHECK_APIS */ + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +#define mlk_kem_keypair_derand \ + MLK_NAMESPACE_K(keypair_derand) MLK_CONTEXT_PARAMETERS_3 +#if !defined(MLK_CONFIG_NO_RANDOMIZED_API) +#define mlk_kem_keypair MLK_NAMESPACE_K(keypair) MLK_CONTEXT_PARAMETERS_2 +#endif +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ +#if !defined(MLK_CONFIG_NO_ENCAPS_API) +#define mlk_kem_enc_derand MLK_NAMESPACE_K(enc_derand) MLK_CONTEXT_PARAMETERS_4 +#if !defined(MLK_CONFIG_NO_RANDOMIZED_API) +#define mlk_kem_enc MLK_NAMESPACE_K(enc) MLK_CONTEXT_PARAMETERS_3 +#endif +#define mlk_kem_check_pk MLK_NAMESPACE_K(check_pk) MLK_CONTEXT_PARAMETERS_1 +#endif /* !MLK_CONFIG_NO_ENCAPS_API */ +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_kem_dec MLK_NAMESPACE_K(dec) MLK_CONTEXT_PARAMETERS_3 +#define mlk_kem_check_sk MLK_NAMESPACE_K(check_sk) MLK_CONTEXT_PARAMETERS_1 +#endif + +/** + * Implements modulus check mandated by FIPS 203, i.e., ensures that + * coefficients are in [0,q-1]. + * + * @spec{Implements @[FIPS203, Section 7.2, 'modulus check'].} + * + * @reference{Not implemented in the reference implementation @[REF].} + * + * @param[in] pk Input public key (an already allocated array of + * MLKEM_INDCCA_PUBLICKEYBYTES bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_INVALID_PK Modulus check failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +#if !defined(MLK_CONFIG_NO_ENCAPS_API) +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_check_pk(const uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_INVALID_PK || + return_value == MLK_ERR_OUT_OF_MEMORY) +); +#endif /* !MLK_CONFIG_NO_ENCAPS_API */ + + +/** + * Implements public key hash check mandated by FIPS 203, i.e., ensures that + * sk[768ð‘˜+32 ∶ 768ð‘˜+64] = H(pk) = H(sk[384𑘠: 768ð‘˜+32]). + * + * @spec{Implements @[FIPS203, Section 7.3, 'hash check'].} + * + * @reference{Not implemented in the reference implementation @[REF].} + * + * @param[in] sk Input private key (an already allocated array of + * MLKEM_INDCCA_SECRETKEYBYTES bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_INVALID_SK Public key hash check failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +#if !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_check_sk(const uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(sk, MLKEM_INDCCA_SECRETKEYBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_INVALID_SK || + return_value == MLK_ERR_OUT_OF_MEMORY) +); +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +/** + * Generate a public/private keypair for the ML-KEM key encapsulation mechanism. + * + * @spec{Implements @[FIPS203, Algorithm 16, ML-KEM.KeyGen_Internal].} + * + * @param[out] pk Output public key (an already allocated array of + * MLKEM_INDCCA_PUBLICKEYBYTES bytes). + * @param[out] sk Output private key (an already allocated array of + * MLKEM_INDCCA_SECRETKEYBYTES bytes). + * @param[in] coins Input randomness (an already allocated array filled + * with 2*MLKEM_SYMBYTES random bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_PCT_FAIL MLK_CONFIG_KEYGEN_PCT enabled and PCT failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + * @retval MLK_ERR_RNG_FAIL MLK_CONFIG_KEYGEN_PCT enabled and random + * number generation failed within the PCT. + */ +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_keypair_derand(uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES], + const uint8_t coins[2 * MLKEM_SYMBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + requires(memory_no_alias(sk, MLKEM_INDCCA_SECRETKEYBYTES)) + requires(memory_no_alias(coins, 2 * MLKEM_SYMBYTES)) + assigns(memory_slice(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + assigns(memory_slice(sk, MLKEM_INDCCA_SECRETKEYBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_PCT_FAIL || + return_value == MLK_ERR_OUT_OF_MEMORY || + return_value == MLK_ERR_RNG_FAIL) + /* Output buffers on error, per API-CONVENTIONS.md */ + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(sk, MLKEM_INDCCA_SECRETKEYBYTES)) +); + +#if !defined(MLK_CONFIG_NO_RANDOMIZED_API) +/** + * Generate a public/private keypair for the ML-KEM key encapsulation mechanism. + * + * @spec{Implements @[FIPS203, Algorithm 19, ML-KEM.KeyGen].} + * + * @param[out] pk Output public key (an already allocated array of + * MLKEM_INDCCA_PUBLICKEYBYTES bytes). + * @param[out] sk Output private key (an already allocated array of + * MLKEM_INDCCA_SECRETKEYBYTES bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + * @retval MLK_ERR_RNG_FAIL Random number generation failed. + * @retval MLK_ERR_PCT_FAIL MLK_CONFIG_KEYGEN_PCT enabled and PCT failed. + */ +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_keypair(uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + requires(memory_no_alias(sk, MLKEM_INDCCA_SECRETKEYBYTES)) + assigns(memory_slice(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + assigns(memory_slice(sk, MLKEM_INDCCA_SECRETKEYBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_PCT_FAIL || + return_value == MLK_ERR_OUT_OF_MEMORY || + return_value == MLK_ERR_RNG_FAIL) + /* Output buffers on error, per API-CONVENTIONS.md */ + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(sk, MLKEM_INDCCA_SECRETKEYBYTES)) +); +#endif /* !MLK_CONFIG_NO_RANDOMIZED_API */ +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +/** + * Generate ciphertext and shared secret for a given public key. + * + * @spec{Implements @[FIPS203, Algorithm 17, ML-KEM.Encaps_Internal].} + * + * @param[out] ct Output ciphertext (an already allocated array of + * MLKEM_INDCCA_CIPHERTEXTBYTES bytes). + * @param[out] ss Output shared secret (an already allocated array of + * MLKEM_SSBYTES bytes). + * @param[in] pk Input public key (an already allocated array of + * MLKEM_INDCCA_PUBLICKEYBYTES bytes). + * @param[in] coins Input randomness (an already allocated array filled + * with MLKEM_SYMBYTES random bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_INVALID_PK The 'modulus check' @[FIPS203, Section 7.2] + * for the public key failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +#if !defined(MLK_CONFIG_NO_ENCAPS_API) +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_enc_derand(uint8_t ct[MLKEM_INDCCA_CIPHERTEXTBYTES], + uint8_t ss[MLKEM_SSBYTES], + const uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + const uint8_t coins[MLKEM_SYMBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(ct, MLKEM_INDCCA_CIPHERTEXTBYTES)) + requires(memory_no_alias(ss, MLKEM_SSBYTES)) + requires(memory_no_alias(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + requires(memory_no_alias(coins, MLKEM_SYMBYTES)) + assigns(memory_slice(ct, MLKEM_INDCCA_CIPHERTEXTBYTES)) + assigns(memory_slice(ss, MLKEM_SSBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_INVALID_PK || + return_value == MLK_ERR_OUT_OF_MEMORY) + /* Output buffers on error, per API-CONVENTIONS.md */ + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(ct, MLKEM_INDCCA_CIPHERTEXTBYTES)) + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(ss, MLKEM_SSBYTES)) +); + +#if !defined(MLK_CONFIG_NO_RANDOMIZED_API) +/** + * Generate ciphertext and shared secret for a given public key. + * + * @spec{Implements @[FIPS203, Algorithm 20, ML-KEM.Encaps].} + * + * @param[out] ct Output ciphertext (an already allocated array of + * MLKEM_INDCCA_CIPHERTEXTBYTES bytes). + * @param[out] ss Output shared secret (an already allocated array of + * MLKEM_SSBYTES bytes). + * @param[in] pk Input public key (an already allocated array of + * MLKEM_INDCCA_PUBLICKEYBYTES bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + * @retval MLK_ERR_RNG_FAIL Random number generation failed. + * @retval MLK_ERR_INVALID_PK The 'modulus check' @[FIPS203, Section 7.2] + * for the public key failed. + */ +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_enc(uint8_t ct[MLKEM_INDCCA_CIPHERTEXTBYTES], + uint8_t ss[MLKEM_SSBYTES], + const uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(ct, MLKEM_INDCCA_CIPHERTEXTBYTES)) + requires(memory_no_alias(ss, MLKEM_SSBYTES)) + requires(memory_no_alias(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + assigns(memory_slice(ct, MLKEM_INDCCA_CIPHERTEXTBYTES)) + assigns(memory_slice(ss, MLKEM_SSBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_INVALID_PK || + return_value == MLK_ERR_OUT_OF_MEMORY || + return_value == MLK_ERR_RNG_FAIL) + /* Output buffers on error, per API-CONVENTIONS.md */ + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(ct, MLKEM_INDCCA_CIPHERTEXTBYTES)) + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(ss, MLKEM_SSBYTES)) +); +#endif /* !MLK_CONFIG_NO_RANDOMIZED_API */ +#endif /* !MLK_CONFIG_NO_ENCAPS_API */ + +/** + * Generate shared secret for a given ciphertext and private key. + * + * @spec{Implements @[FIPS203, Algorithm 21, ML-KEM.Decaps].} + * + * @param[out] ss Output shared secret (an already allocated array of + * MLKEM_SSBYTES bytes). + * @param[in] ct Input ciphertext (an already allocated array of + * MLKEM_INDCCA_CIPHERTEXTBYTES bytes). + * @param[in] sk Input private key (an already allocated array of + * MLKEM_INDCCA_SECRETKEYBYTES bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_INVALID_SK The 'hash check' @[FIPS203, Section 7.3] + * for the secret key failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +#if !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_dec(uint8_t ss[MLKEM_SSBYTES], + const uint8_t ct[MLKEM_INDCCA_CIPHERTEXTBYTES], + const uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(ss, MLKEM_SSBYTES)) + requires(memory_no_alias(ct, MLKEM_INDCCA_CIPHERTEXTBYTES)) + requires(memory_no_alias(sk, MLKEM_INDCCA_SECRETKEYBYTES)) + assigns(memory_slice(ss, MLKEM_SSBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_INVALID_SK || + return_value == MLK_ERR_OUT_OF_MEMORY) + /* Output buffers on error, per API-CONVENTIONS.md */ + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(ss, MLKEM_SSBYTES)) +); +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#endif /* !MLK_KEM_H */ diff --git a/src/mlkem-native/mlkem/src/native/aarch64/README.md b/src/mlkem-native/mlkem/src/native/aarch64/README.md new file mode 100644 index 0000000000..0d56871d6a --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/README.md @@ -0,0 +1,16 @@ +[//]: # (SPDX-License-Identifier: CC-BY-4.0) + +# AArch64 backend (little endian) + +This directory contains a native backend for little endian AArch64 systems. It is derived from [^NeonNTT] [^SLOTHY_Paper]. + +The code in this directory is auto-generated from the 'clean' assembly in [dev/aarch64_clean](../../../../dev/aarch64_clean) +in a two-step fashion: First, it is superoptimized using the [SLOTHY](https://github.com/slothy-optimizer/slothy) superoptimizer, +giving the assembly in [dev/aarch64_opt](../../../../dev/aarch64_opt). Then, it is stripped of remaining register aliases, macros +and most preprocessor directives by [`scripts/simpasm`](../../../../scripts/simpasm). + +If you want to understand how the assembly works, and/or make changes to it, consult [dev/](../../../../dev). + + +[^NeonNTT]: Becker, Hwang, Kannwischer, Yang, Yang: Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1, [https://eprint.iacr.org/2021/986](https://eprint.iacr.org/2021/986) +[^SLOTHY_Paper]: Abdulrahman, Becker, Kannwischer, Klein: Fast and Clean: Auditable high-performance assembly via constraint solving, [https://eprint.iacr.org/2022/1303](https://eprint.iacr.org/2022/1303) diff --git a/src/mlkem-native/mlkem/src/native/aarch64/meta.h b/src/mlkem-native/mlkem/src/native/aarch64/meta.h new file mode 100644 index 0000000000..6e8b9dc9e8 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/meta.h @@ -0,0 +1,166 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_NATIVE_AARCH64_META_H +#define MLK_NATIVE_AARCH64_META_H + +/* Set of primitives that this backend replaces */ +#define MLK_USE_NATIVE_NTT +#define MLK_USE_NATIVE_INTT +#define MLK_USE_NATIVE_POLY_REDUCE +#define MLK_USE_NATIVE_POLY_TOMONT +#define MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#define MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED +#define MLK_USE_NATIVE_POLY_TOBYTES +#define MLK_USE_NATIVE_REJ_UNIFORM + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLK_ARITH_BACKEND_AARCH64 + + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/arith_native_aarch64.h" + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_ntt_native(int16_t data[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + mlk_ntt_aarch64_asm(data, mlk_aarch64_ntt_zetas_layer12345, + mlk_aarch64_ntt_zetas_layer67); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_intt_native(int16_t data[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + mlk_intt_aarch64_asm(data, mlk_aarch64_invntt_zetas_layer12345, + mlk_aarch64_invntt_zetas_layer67); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_reduce_native(int16_t data[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + mlk_poly_reduce_aarch64_asm(data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_tomont_native(int16_t data[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + mlk_poly_tomont_aarch64_asm(data); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_mulcache_compute_native(int16_t x[MLKEM_N / 2], + const int16_t y[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + mlk_poly_mulcache_compute_aarch64_asm( + x, y, mlk_aarch64_zetas_mulcache_native, + mlk_aarch64_zetas_mulcache_twisted_native); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k2_native( + int16_t r[MLKEM_N], const int16_t a[2 * MLKEM_N], + const int16_t b[2 * MLKEM_N], const int16_t b_cache[2 * (MLKEM_N / 2)]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm(r, a, b, b_cache); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k3_native( + int16_t r[MLKEM_N], const int16_t a[3 * MLKEM_N], + const int16_t b[3 * MLKEM_N], const int16_t b_cache[3 * (MLKEM_N / 2)]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm(r, a, b, b_cache); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k4_native( + int16_t r[MLKEM_N], const int16_t a[4 * MLKEM_N], + const int16_t b[4 * MLKEM_N], const int16_t b_cache[4 * (MLKEM_N / 2)]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm(r, a, b, b_cache); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */ + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) || !defined(MLK_CONFIG_NO_ENCAPS_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_tobytes_native(uint8_t r[MLKEM_POLYBYTES], + const int16_t a[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + mlk_poly_tobytes_aarch64_asm(r, a); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API || !MLK_CONFIG_NO_ENCAPS_API */ + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_rej_uniform_native(int16_t *r, unsigned len, + const uint8_t *buf, + unsigned buflen) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON) || len != MLKEM_N || + buflen % 24 != 0) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + return (int)mlk_rej_uniform_aarch64_asm(r, buf, buflen, + mlk_rej_uniform_table); +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_NATIVE_AARCH64_META_H */ diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/aarch64_zetas.c b/src/mlkem-native/mlkem/src/native/aarch64/src/aarch64_zetas.c new file mode 100644 index 0000000000..c743d92f18 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/aarch64_zetas.c @@ -0,0 +1,184 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_AARCH64) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "arith_native_aarch64.h" + +/* + * Table of zeta values used in the AArch64 forward NTT + * See autogen for details. + */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t + mlk_aarch64_ntt_zetas_layer12345[80] = { + -1600, -15749, -749, -7373, -40, -394, -687, -6762, 630, 6201, + -1432, -14095, 848, 8347, 0, 0, 1062, 10453, 296, 2914, + -882, -8682, 0, 0, -1410, -13879, 1339, 13180, 1476, 14529, + 0, 0, 193, 1900, -283, -2786, 56, 551, 0, 0, + 797, 7845, -1089, -10719, 1333, 13121, 0, 0, -543, -5345, + 1426, 14036, -1235, -12156, 0, 0, -69, -679, 535, 5266, + -447, -4400, 0, 0, 569, 5601, -936, -9213, -450, -4429, + 0, 0, -1583, -15582, -1355, -13338, 821, 8081, 0, 0, +}; + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t + mlk_aarch64_ntt_zetas_layer67[384] = { + 289, 289, 331, 331, -76, -76, -1573, -1573, 2845, + 2845, 3258, 3258, -748, -748, -15483, -15483, 17, 17, + 583, 583, 1637, 1637, -1041, -1041, 167, 167, 5739, + 5739, 16113, 16113, -10247, -10247, -568, -568, -680, -680, + 723, 723, 1100, 1100, -5591, -5591, -6693, -6693, 7117, + 7117, 10828, 10828, 1197, 1197, -1025, -1025, -1052, -1052, + -1274, -1274, 11782, 11782, -10089, -10089, -10355, -10355, -12540, + -12540, 1409, 1409, -48, -48, 756, 756, -314, -314, + 13869, 13869, -472, -472, 7441, 7441, -3091, -3091, -667, + -667, 233, 233, -1173, -1173, -279, -279, -6565, -6565, + 2293, 2293, -11546, -11546, -2746, -2746, 650, 650, -1352, + -1352, -816, -816, 632, 632, 6398, 6398, -13308, -13308, + -8032, -8032, 6221, 6221, -1626, -1626, -540, -540, -1482, + -1482, 1461, 1461, -16005, -16005, -5315, -5315, -14588, -14588, + 14381, 14381, 1651, 1651, -1540, -1540, 952, 952, -642, + -642, 16251, 16251, -15159, -15159, 9371, 9371, -6319, -6319, + -464, -464, 33, 33, 1320, 1320, -1414, -1414, -4567, + -4567, 325, 325, 12993, 12993, -13918, -13918, 939, 939, + -892, -892, 733, 733, 268, 268, 9243, 9243, -8780, + -8780, 7215, 7215, 2638, 2638, -1021, -1021, -941, -941, + -992, -992, 641, 641, -10050, -10050, -9262, -9262, -9764, + -9764, 6309, 6309, -1010, -1010, 1435, 1435, 807, 807, + 452, 452, -9942, -9942, 14125, 14125, 7943, 7943, 4449, + 4449, 1584, 1584, -1292, -1292, 375, 375, -1239, -1239, + 15592, 15592, -12717, -12717, 3691, 3691, -12196, -12196, -1031, + -1031, -109, -109, -780, -780, 1645, 1645, -10148, -10148, + -1073, -1073, -7678, -7678, 16192, 16192, 1438, 1438, -461, + -461, 1534, 1534, -927, -927, 14155, 14155, -4538, -4538, + 15099, 15099, -9125, -9125, 1063, 1063, -556, -556, -1230, + -1230, -863, -863, 10463, 10463, -5473, -5473, -12107, -12107, + -8495, -8495, 319, 319, 757, 757, 561, 561, -735, + -735, 3140, 3140, 7451, 7451, 5522, 5522, -7235, -7235, + -682, -682, -712, -712, 1481, 1481, 648, 648, -6713, + -6713, -7008, -7008, 14578, 14578, 6378, 6378, -525, -525, + 403, 403, 1143, 1143, -554, -554, -5168, -5168, 3967, + 3967, 11251, 11251, -5453, -5453, 1092, 1092, 1026, 1026, + -1179, -1179, 886, 886, 10749, 10749, 10099, 10099, -11605, + -11605, 8721, 8721, -855, -855, -219, -219, 1227, 1227, + 910, 910, -8416, -8416, -2156, -2156, 12078, 12078, 8957, + 8957, -1607, -1607, -1455, -1455, -1219, -1219, 885, 885, + -15818, -15818, -14322, -14322, -11999, -11999, 8711, 8711, 1212, + 1212, 1029, 1029, -394, -394, -1175, -1175, 11930, 11930, + 10129, 10129, -3878, -3878, -11566, -11566, +}; + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t + mlk_aarch64_invntt_zetas_layer12345[80] = { + 1583, 15582, -821, -8081, 1355, 13338, 0, 0, -569, + -5601, 450, 4429, 936, 9213, 0, 0, 69, 679, + 447, 4400, -535, -5266, 0, 0, 543, 5345, 1235, + 12156, -1426, -14036, 0, 0, -797, -7845, -1333, -13121, + 1089, 10719, 0, 0, -193, -1900, -56, -551, 283, + 2786, 0, 0, 1410, 13879, -1476, -14529, -1339, -13180, + 0, 0, -1062, -10453, 882, 8682, -296, -2914, 0, + 0, 1600, 15749, 40, 394, 749, 7373, -848, -8347, + 1432, 14095, -630, -6201, 687, 6762, 0, 0, +}; + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t + mlk_aarch64_invntt_zetas_layer67[384] = { + -910, -910, -1227, -1227, 219, 219, 855, 855, -8957, + -8957, -12078, -12078, 2156, 2156, 8416, 8416, 1175, 1175, + 394, 394, -1029, -1029, -1212, -1212, 11566, 11566, 3878, + 3878, -10129, -10129, -11930, -11930, -885, -885, 1219, 1219, + 1455, 1455, 1607, 1607, -8711, -8711, 11999, 11999, 14322, + 14322, 15818, 15818, -648, -648, -1481, -1481, 712, 712, + 682, 682, -6378, -6378, -14578, -14578, 7008, 7008, 6713, + 6713, -886, -886, 1179, 1179, -1026, -1026, -1092, -1092, + -8721, -8721, 11605, 11605, -10099, -10099, -10749, -10749, 554, + 554, -1143, -1143, -403, -403, 525, 525, 5453, 5453, + -11251, -11251, -3967, -3967, 5168, 5168, 927, 927, -1534, + -1534, 461, 461, -1438, -1438, 9125, 9125, -15099, -15099, + 4538, 4538, -14155, -14155, 735, 735, -561, -561, -757, + -757, -319, -319, 7235, 7235, -5522, -5522, -7451, -7451, + -3140, -3140, 863, 863, 1230, 1230, 556, 556, -1063, + -1063, 8495, 8495, 12107, 12107, 5473, 5473, -10463, -10463, + -452, -452, -807, -807, -1435, -1435, 1010, 1010, -4449, + -4449, -7943, -7943, -14125, -14125, 9942, 9942, -1645, -1645, + 780, 780, 109, 109, 1031, 1031, -16192, -16192, 7678, + 7678, 1073, 1073, 10148, 10148, 1239, 1239, -375, -375, + 1292, 1292, -1584, -1584, 12196, 12196, -3691, -3691, 12717, + 12717, -15592, -15592, 1414, 1414, -1320, -1320, -33, -33, + 464, 464, 13918, 13918, -12993, -12993, -325, -325, 4567, + 4567, -641, -641, 992, 992, 941, 941, 1021, 1021, + -6309, -6309, 9764, 9764, 9262, 9262, 10050, 10050, -268, + -268, -733, -733, 892, 892, -939, -939, -2638, -2638, + -7215, -7215, 8780, 8780, -9243, -9243, -632, -632, 816, + 816, 1352, 1352, -650, -650, -6221, -6221, 8032, 8032, + 13308, 13308, -6398, -6398, 642, 642, -952, -952, 1540, + 1540, -1651, -1651, 6319, 6319, -9371, -9371, 15159, 15159, + -16251, -16251, -1461, -1461, 1482, 1482, 540, 540, 1626, + 1626, -14381, -14381, 14588, 14588, 5315, 5315, 16005, 16005, + 1274, 1274, 1052, 1052, 1025, 1025, -1197, -1197, 12540, + 12540, 10355, 10355, 10089, 10089, -11782, -11782, 279, 279, + 1173, 1173, -233, -233, 667, 667, 2746, 2746, 11546, + 11546, -2293, -2293, 6565, 6565, 314, 314, -756, -756, + 48, 48, -1409, -1409, 3091, 3091, -7441, -7441, 472, + 472, -13869, -13869, 1573, 1573, 76, 76, -331, -331, + -289, -289, 15483, 15483, 748, 748, -3258, -3258, -2845, + -2845, -1100, -1100, -723, -723, 680, 680, 568, 568, + -10828, -10828, -7117, -7117, 6693, 6693, 5591, 5591, 1041, + 1041, -1637, -1637, -583, -583, -17, -17, 10247, 10247, + -16113, -16113, -5739, -5739, -167, -167, +}; +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t + mlk_aarch64_zetas_mulcache_native[128] = { + 17, -17, -568, 568, 583, -583, -680, 680, 1637, -1637, + 723, -723, -1041, 1041, 1100, -1100, 1409, -1409, -667, 667, + -48, 48, 233, -233, 756, -756, -1173, 1173, -314, 314, + -279, 279, -1626, 1626, 1651, -1651, -540, 540, -1540, 1540, + -1482, 1482, 952, -952, 1461, -1461, -642, 642, 939, -939, + -1021, 1021, -892, 892, -941, 941, 733, -733, -992, 992, + 268, -268, 641, -641, 1584, -1584, -1031, 1031, -1292, 1292, + -109, 109, 375, -375, -780, 780, -1239, 1239, 1645, -1645, + 1063, -1063, 319, -319, -556, 556, 757, -757, -1230, 1230, + 561, -561, -863, 863, -735, 735, -525, 525, 1092, -1092, + 403, -403, 1026, -1026, 1143, -1143, -1179, 1179, -554, 554, + 886, -886, -1607, 1607, 1212, -1212, -1455, 1455, 1029, -1029, + -1219, 1219, -394, 394, 885, -885, -1175, 1175, +}; + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t + mlk_aarch64_zetas_mulcache_twisted_native[128] = { + 167, -167, -5591, 5591, 5739, -5739, -6693, 6693, 16113, + -16113, 7117, -7117, -10247, 10247, 10828, -10828, 13869, -13869, + -6565, 6565, -472, 472, 2293, -2293, 7441, -7441, -11546, + 11546, -3091, 3091, -2746, 2746, -16005, 16005, 16251, -16251, + -5315, 5315, -15159, 15159, -14588, 14588, 9371, -9371, 14381, + -14381, -6319, 6319, 9243, -9243, -10050, 10050, -8780, 8780, + -9262, 9262, 7215, -7215, -9764, 9764, 2638, -2638, 6309, + -6309, 15592, -15592, -10148, 10148, -12717, 12717, -1073, 1073, + 3691, -3691, -7678, 7678, -12196, 12196, 16192, -16192, 10463, + -10463, 3140, -3140, -5473, 5473, 7451, -7451, -12107, 12107, + 5522, -5522, -8495, 8495, -7235, 7235, -5168, 5168, 10749, + -10749, 3967, -3967, 10099, -10099, 11251, -11251, -11605, 11605, + -5453, 5453, 8721, -8721, -15818, 15818, 11930, -11930, -14322, + 14322, 10129, -10129, -11999, 11999, -3878, 3878, 8711, -8711, + -11566, 11566, +}; + +#else /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(aarch64_zetas) + +#endif /* !(MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/arith_native_aarch64.h b/src/mlkem-native/mlkem/src/native/aarch64/src/arith_native_aarch64.h new file mode 100644 index 0000000000..d7c3d48e5f --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/arith_native_aarch64.h @@ -0,0 +1,184 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_NATIVE_AARCH64_SRC_ARITH_NATIVE_AARCH64_H +#define MLK_NATIVE_AARCH64_SRC_ARITH_NATIVE_AARCH64_H + +#include "../../../cbmc.h" +#include "../../../common.h" + +#define mlk_aarch64_ntt_zetas_layer12345 \ + MLK_NAMESPACE(aarch64_ntt_zetas_layer12345) +#define mlk_aarch64_ntt_zetas_layer67 MLK_NAMESPACE(aarch64_ntt_zetas_layer67) +#define mlk_aarch64_invntt_zetas_layer12345 \ + MLK_NAMESPACE(aarch64_invntt_zetas_layer12345) +#define mlk_aarch64_invntt_zetas_layer67 \ + MLK_NAMESPACE(aarch64_invntt_zetas_layer67) +#define mlk_aarch64_zetas_mulcache_native \ + MLK_NAMESPACE(aarch64_zetas_mulcache_native) +#define mlk_aarch64_zetas_mulcache_twisted_native \ + MLK_NAMESPACE(aarch64_zetas_mulcache_twisted_native) +#define mlk_rej_uniform_table MLK_NAMESPACE(rej_uniform_table) + +MLK_INTERNAL_DATA_DECLARATION const int16_t + mlk_aarch64_ntt_zetas_layer12345[80]; +MLK_INTERNAL_DATA_DECLARATION const int16_t mlk_aarch64_ntt_zetas_layer67[384]; +MLK_INTERNAL_DATA_DECLARATION const int16_t + mlk_aarch64_invntt_zetas_layer12345[80]; +MLK_INTERNAL_DATA_DECLARATION const int16_t + mlk_aarch64_invntt_zetas_layer67[384]; +MLK_INTERNAL_DATA_DECLARATION const int16_t + mlk_aarch64_zetas_mulcache_native[128]; +MLK_INTERNAL_DATA_DECLARATION const int16_t + mlk_aarch64_zetas_mulcache_twisted_native[128]; +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_rej_uniform_table[4096]; + +#define mlk_ntt_aarch64_asm MLK_NAMESPACE(ntt_aarch64_asm) +void mlk_ntt_aarch64_asm(int16_t p[256], const int16_t twiddles12345[80], + const int16_t twiddles56[384]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/mlkem_ntt_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + requires(array_abs_bound(p, 0, MLKEM_N, 8192)) + requires(twiddles12345 == mlk_aarch64_ntt_zetas_layer12345) + requires(twiddles56 == mlk_aarch64_ntt_zetas_layer67) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + /* check-magic: off */ + ensures(array_abs_bound(p, 0, MLKEM_N, 23595)) + /* check-magic: on */ +); + +#define mlk_intt_aarch64_asm MLK_NAMESPACE(intt_aarch64_asm) +void mlk_intt_aarch64_asm(int16_t p[256], const int16_t twiddles12345[80], + const int16_t twiddles56[384]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/mlkem_intt_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + requires(twiddles12345 == mlk_aarch64_invntt_zetas_layer12345) + requires(twiddles56 == mlk_aarch64_invntt_zetas_layer67) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + /* check-magic: off */ + ensures(array_abs_bound(p, 0, MLKEM_N, 26625)) + /* check-magic: on */ +); + +#define mlk_poly_reduce_aarch64_asm MLK_NAMESPACE(poly_reduce_aarch64_asm) +void mlk_poly_reduce_aarch64_asm(int16_t p[256]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/mlkem_poly_reduce_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + ensures(array_bound(p, 0, MLKEM_N, 0, MLKEM_Q)) +); + +#define mlk_poly_tomont_aarch64_asm MLK_NAMESPACE(poly_tomont_aarch64_asm) +void mlk_poly_tomont_aarch64_asm(int16_t p[256]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/mlkem_poly_tomont_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + ensures(array_abs_bound(p, 0, MLKEM_N, MLKEM_Q)) +); + +#define mlk_poly_mulcache_compute_aarch64_asm \ + MLK_NAMESPACE(poly_mulcache_compute_aarch64_asm) +void mlk_poly_mulcache_compute_aarch64_asm(int16_t cache[128], + const int16_t mlk_poly[256], + const int16_t zetas[128], + const int16_t zetas_twisted[128]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/mlkem_poly_mulcache_compute_aarch64_asm.ml + */ +__contract__( + requires(memory_no_alias(cache, sizeof(int16_t) * (MLKEM_N / 2))) + requires(memory_no_alias(mlk_poly, sizeof(int16_t) * MLKEM_N)) + requires(zetas == mlk_aarch64_zetas_mulcache_native) + requires(zetas_twisted == mlk_aarch64_zetas_mulcache_twisted_native) + assigns(memory_slice(cache, sizeof(int16_t) * (MLKEM_N / 2))) + ensures(array_abs_bound(cache, 0, MLKEM_N/2, MLKEM_Q)) +); + +#define mlk_poly_tobytes_aarch64_asm MLK_NAMESPACE(poly_tobytes_aarch64_asm) +void mlk_poly_tobytes_aarch64_asm(uint8_t r[384], const int16_t a[256]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/mlkem_poly_tobytes_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(r, MLKEM_POLYBYTES)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT)) + assigns(memory_slice(r, MLKEM_POLYBYTES)) +); + +#define mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm \ + MLK_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm) +void mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm( + int16_t r[256], const int16_t a[512], const int16_t b[512], + const int16_t b_cache[256]) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/aarch64/proofs/mlkem_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 2 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 2 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 2 * (MLKEM_N / 2))) + requires(array_abs_bound(a, 0, 2 * MLKEM_N, MLKEM_UINT12_LIMIT + 1)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) +); + +#define mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm \ + MLK_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm) +void mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm( + int16_t r[256], const int16_t a[768], const int16_t b[768], + const int16_t b_cache[384]) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/aarch64/proofs/mlkem_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 3 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 3 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 3 * (MLKEM_N / 2))) + requires(array_abs_bound(a, 0, 3 * MLKEM_N, MLKEM_UINT12_LIMIT + 1)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) +); + +#define mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm \ + MLK_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm) +void mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm( + int16_t r[256], const int16_t a[1024], const int16_t b[1024], + const int16_t b_cache[512]) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/aarch64/proofs/mlkem_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 4 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 4 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 4 * (MLKEM_N / 2))) + requires(array_abs_bound(a, 0, 4 * MLKEM_N, MLKEM_UINT12_LIMIT + 1)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) +); + +#define mlk_rej_uniform_aarch64_asm MLK_NAMESPACE(rej_uniform_aarch64_asm) +MLK_MUST_CHECK_RETURN_VALUE +uint64_t mlk_rej_uniform_aarch64_asm(int16_t r[256], const uint8_t *buf, + unsigned buflen, const uint8_t table[4096]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/mlkem_rej_uniform_aarch64_asm.ml. */ +__contract__( + requires(buflen % 24 == 0) + requires(memory_no_alias(buf, buflen)) + requires(table == mlk_rej_uniform_table) + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value <= MLKEM_N) + ensures(array_bound(r, 0, (unsigned) return_value, 0, MLKEM_Q)) +); + +#endif /* !MLK_NATIVE_AARCH64_SRC_ARITH_NATIVE_AARCH64_H */ diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_intt_aarch64_asm.S b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_intt_aarch64_asm.S new file mode 100644 index 0000000000..0901957893 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_intt_aarch64_asm.S @@ -0,0 +1,635 @@ +/* Copyright (c) 2022 Arm Limited + * Copyright (c) 2022 Hanno Becker + * Copyright (c) 2023 Amin Abdulrahman, Matthias Kannwischer + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [NeonNTT] + * Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1 + * Becker, Hwang, Kannwischer, Yang, Yang + * https://eprint.iacr.org/2021/986 + * + * - [SLOTHY_Paper] + * Fast and Clean: Auditable high-performance assembly via constraint solving + * Abdulrahman, Becker, Kannwischer, Klein + * https://eprint.iacr.org/2022/1303 + */ + +/*yaml + Name: intt_aarch64_asm + Description: AArch64 ML-KEM inverse NTT following @[NeonNTT] and @[SLOTHY_Paper] + Signature: void mlk_intt_aarch64_asm(int16_t p[256], const int16_t twiddles12345[80], const int16_t twiddles56[384]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON] + x0: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t p[256] + description: Input/output polynomial + x1: + type: buffer + size_bytes: 160 + permissions: read-only + c_parameter: const int16_t twiddles12345[80] + description: Twiddle factors for layers 1-5 + x2: + type: buffer + size_bytes: 768 + permissions: read-only + c_parameter: const int16_t twiddles56[384] + description: Twiddle factors for layers 6-7 + Stack: + bytes: 64 + description: saving callee-saved Neon registers +*/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API)) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/mlkem_intt_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(intt_aarch64_asm) +MLK_ASM_FN_SYMBOL(intt_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x40 + .cfi_adjust_cfa_offset 0x40 + stp d8, d9, [sp] + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + stp d10, d11, [sp, #0x10] + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + stp d12, d13, [sp, #0x20] + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + stp d14, d15, [sp, #0x30] + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + mov w5, #0xd01 // =3329 + mov v7.h[0], w5 + mov w5, #0x4ebf // =20159 + mov v7.h[1], w5 + mov w5, #0x200 // =512 + dup v29.8h, w5 + mov w5, #0x13b0 // =5040 + dup v30.8h, w5 + mov x3, x0 + mov x4, #0x8 // =8 + ldr q13, [x3, #0x20] + ldr q8, [x3, #0x30] + ldr q6, [x3] + ldr q16, [x3, #0x10] + ldr q4, [x3, #0x50] + ldr q11, [x3, #0x40] + ldr q3, [x3, #0x70] + trn1 v23.4s, v13.4s, v8.4s + ldr q0, [x3, #0x60] + trn2 v19.4s, v6.4s, v16.4s + trn2 v21.4s, v13.4s, v8.4s + trn1 v6.4s, v6.4s, v16.4s + ldr q24, [x2, #0x20] + trn1 v10.2d, v19.2d, v21.2d + ldr q16, [x2], #0x60 + trn1 v5.2d, v6.2d, v23.2d + trn1 v28.4s, v0.4s, v3.4s + trn2 v18.2d, v6.2d, v23.2d + mul v31.8h, v10.8h, v29.8h + trn2 v13.4s, v0.4s, v3.4s + ldur q14, [x2, #-0x50] + sqrdmulh v26.8h, v18.8h, v30.8h + ldur q20, [x2, #-0x20] + mul v17.8h, v18.8h, v29.8h + trn2 v18.2d, v19.2d, v21.2d + mul v9.8h, v18.8h, v29.8h + trn1 v12.4s, v11.4s, v4.4s + sqrdmulh v22.8h, v18.8h, v30.8h + sqrdmulh v3.8h, v10.8h, v30.8h + sqrdmulh v25.8h, v5.8h, v30.8h + mls v9.8h, v22.8h, v7.h[0] + mls v17.8h, v26.8h, v7.h[0] + trn2 v26.4s, v11.4s, v4.4s + mul v8.8h, v5.8h, v29.8h + trn1 v10.2d, v26.2d, v13.2d + ldur q11, [x2, #-0x10] + mls v31.8h, v3.8h, v7.h[0] + trn1 v6.2d, v12.2d, v28.2d + trn2 v3.2d, v26.2d, v13.2d + ldur q4, [x2, #-0x30] + mls v8.8h, v25.8h, v7.h[0] + sub v19.8h, v17.8h, v9.8h + trn2 v13.2d, v12.2d, v28.2d + sqrdmulh v1.8h, v3.8h, v30.8h + add v9.8h, v17.8h, v9.8h + mul v18.8h, v19.8h, v20.8h + add v28.8h, v8.8h, v31.8h + sqrdmulh v20.8h, v19.8h, v11.8h + sub v12.8h, v28.8h, v9.8h + sub v23.8h, v8.8h, v31.8h + sqrdmulh v11.8h, v13.8h, v30.8h + sqrdmulh v5.8h, v23.8h, v4.8h + mul v0.8h, v23.8h, v24.8h + mul v2.8h, v13.8h, v29.8h + mls v0.8h, v5.8h, v7.h[0] + add v24.8h, v28.8h, v9.8h + mls v18.8h, v20.8h, v7.h[0] + sqrdmulh v15.8h, v6.8h, v30.8h + sqrdmulh v25.8h, v12.8h, v14.8h + mul v21.8h, v12.8h, v16.8h + sub v23.8h, v0.8h, v18.8h + sqrdmulh v8.8h, v23.8h, v14.8h + mul v23.8h, v23.8h, v16.8h + mls v21.8h, v25.8h, v7.h[0] + mls v23.8h, v8.8h, v7.h[0] + mul v14.8h, v3.8h, v29.8h + add v3.8h, v0.8h, v18.8h + trn2 v4.4s, v24.4s, v3.4s + mls v14.8h, v1.8h, v7.h[0] + trn1 v9.4s, v24.4s, v3.4s + trn2 v12.4s, v21.4s, v23.4s + mls v2.8h, v11.8h, v7.h[0] + trn1 v28.4s, v21.4s, v23.4s + ldr q11, [x1], #0x10 + mul v31.8h, v10.8h, v29.8h + trn1 v25.2d, v4.2d, v12.2d + trn1 v20.2d, v9.2d, v28.2d + ldr q23, [x2, #0x50] + trn2 v13.2d, v4.2d, v12.2d + sqrdmulh v21.8h, v10.8h, v30.8h + trn2 v4.2d, v9.2d, v28.2d + ldr q9, [x2, #0x40] + mul v27.8h, v6.8h, v29.8h + add v26.8h, v20.8h, v25.8h + sub v3.8h, v2.8h, v14.8h + sqdmulh v12.8h, v26.8h, v7.h[1] + add v5.8h, v4.8h, v13.8h + sub v8.8h, v4.8h, v13.8h + add v10.8h, v2.8h, v14.8h + sqdmulh v6.8h, v5.8h, v7.h[1] + ldr q2, [x2, #0x10] + mls v27.8h, v15.8h, v7.h[0] + ldr q15, [x2, #0x20] + srshr v12.8h, v12.8h, #0xb + mls v31.8h, v21.8h, v7.h[0] + srshr v6.8h, v6.8h, #0xb + sqrdmulh v23.8h, v3.8h, v23.8h + mls v26.8h, v12.8h, v7.h[0] + add v21.8h, v27.8h, v31.8h + mls v5.8h, v6.8h, v7.h[0] + sub v6.8h, v27.8h, v31.8h + sub v14.8h, v21.8h, v10.8h + ldr q27, [x2], #0x60 + mul v3.8h, v3.8h, v9.8h + mls v3.8h, v23.8h, v7.h[0] + ldur q13, [x2, #-0x30] + sub v12.8h, v26.8h, v5.8h + add v5.8h, v26.8h, v5.8h + sqrdmulh v31.8h, v8.8h, v11.h[5] + sqrdmulh v19.8h, v12.8h, v11.h[1] + mul v24.8h, v12.8h, v11.h[0] + sqrdmulh v13.8h, v6.8h, v13.8h + mls v24.8h, v19.8h, v7.h[0] + sub x4, x4, #0x2 + +Lmlk_intt_layer4567_start: + add v16.8h, v21.8h, v10.8h + mul v18.8h, v6.8h, v15.8h + sub v19.8h, v20.8h, v25.8h + ldr q21, [x3, #0xa0] + str q5, [x3], #0x40 + mls v18.8h, v13.8h, v7.h[0] + sqrdmulh v15.8h, v14.8h, v2.8h + ldr q10, [x3, #0x50] + ldr q12, [x3, #0x40] + stur q24, [x3, #-0x20] + mul v5.8h, v8.8h, v11.h[4] + sub v0.8h, v18.8h, v3.8h + ldr q24, [x3, #0x70] + mls v5.8h, v31.8h, v7.h[0] + ldr q26, [x2, #0x50] + trn2 v1.4s, v12.4s, v10.4s + add v6.8h, v18.8h, v3.8h + sqrdmulh v20.8h, v0.8h, v2.8h + trn1 v13.4s, v12.4s, v10.4s + trn1 v18.4s, v16.4s, v6.4s + mul v22.8h, v0.8h, v27.8h + trn1 v17.4s, v21.4s, v24.4s + sqrdmulh v0.8h, v19.8h, v11.h[3] + trn1 v25.2d, v13.2d, v17.2d + mls v22.8h, v20.8h, v7.h[0] + trn2 v21.4s, v21.4s, v24.4s + mul v24.8h, v25.8h, v29.8h + trn2 v28.2d, v13.2d, v17.2d + sqrdmulh v4.8h, v25.8h, v30.8h + trn2 v3.2d, v1.2d, v21.2d + mul v17.8h, v28.8h, v29.8h + sqrdmulh v31.8h, v28.8h, v30.8h + ldr q2, [x2, #0x10] + mls v24.8h, v4.8h, v7.h[0] + mul v4.8h, v19.8h, v11.h[2] + ldr q19, [x2, #0x40] + mls v4.8h, v0.8h, v7.h[0] + mul v0.8h, v14.8h, v27.8h + mls v0.8h, v15.8h, v7.h[0] + sub v8.8h, v4.8h, v5.8h + mul v12.8h, v3.8h, v29.8h + mul v23.8h, v8.8h, v11.h[0] + trn2 v28.4s, v16.4s, v6.4s + sqrdmulh v10.8h, v8.8h, v11.h[1] + trn1 v9.4s, v0.4s, v22.4s + trn2 v22.4s, v0.4s, v22.4s + ldr q11, [x1], #0x10 + mls v17.8h, v31.8h, v7.h[0] + trn1 v20.2d, v18.2d, v9.2d + trn2 v14.2d, v18.2d, v9.2d + ldr q15, [x2, #0x20] + trn1 v6.2d, v1.2d, v21.2d + sqrdmulh v9.8h, v3.8h, v30.8h + trn1 v25.2d, v28.2d, v22.2d + trn2 v16.2d, v28.2d, v22.2d + mls v23.8h, v10.8h, v7.h[0] + add v1.8h, v20.8h, v25.8h + sqrdmulh v21.8h, v6.8h, v30.8h + add v8.8h, v14.8h, v16.8h + ldr q27, [x2], #0x60 + sqdmulh v28.8h, v8.8h, v7.h[1] + mls v12.8h, v9.8h, v7.h[0] + sqdmulh v31.8h, v1.8h, v7.h[1] + mul v0.8h, v6.8h, v29.8h + sub v10.8h, v17.8h, v12.8h + mls v0.8h, v21.8h, v7.h[0] + srshr v21.8h, v28.8h, #0xb + srshr v13.8h, v31.8h, #0xb + sqrdmulh v22.8h, v10.8h, v26.8h + mls v8.8h, v21.8h, v7.h[0] + mls v1.8h, v13.8h, v7.h[0] + add v21.8h, v24.8h, v0.8h + stur q23, [x3, #-0x10] + sub v6.8h, v24.8h, v0.8h + mul v3.8h, v10.8h, v19.8h + add v0.8h, v4.8h, v5.8h + sqdmulh v13.8h, v0.8h, v7.h[1] + ldur q10, [x2, #-0x30] + add v5.8h, v1.8h, v8.8h + mls v3.8h, v22.8h, v7.h[0] + sub v8.8h, v1.8h, v8.8h + mul v24.8h, v8.8h, v11.h[0] + sqrdmulh v8.8h, v8.8h, v11.h[1] + srshr v1.8h, v13.8h, #0xb + sqrdmulh v13.8h, v6.8h, v10.8h + mls v0.8h, v1.8h, v7.h[0] + add v10.8h, v17.8h, v12.8h + mls v24.8h, v8.8h, v7.h[0] + sub v8.8h, v14.8h, v16.8h + sqrdmulh v31.8h, v8.8h, v11.h[5] + sub v14.8h, v21.8h, v10.8h + stur q0, [x3, #-0x30] + sub x4, x4, #0x1 + cbnz x4, Lmlk_intt_layer4567_start + mul v15.8h, v6.8h, v15.8h + sub v22.8h, v20.8h, v25.8h + add v4.8h, v21.8h, v10.8h + str q24, [x3, #0x20] + mls v15.8h, v13.8h, v7.h[0] + str q5, [x3], #0x40 + ldr q9, [x1], #0x10 + sqrdmulh v28.8h, v14.8h, v2.8h + mul v16.8h, v14.8h, v27.8h + sub v18.8h, v15.8h, v3.8h + add v15.8h, v15.8h, v3.8h + sqrdmulh v0.8h, v18.8h, v2.8h + trn2 v24.4s, v4.4s, v15.4s + trn1 v2.4s, v4.4s, v15.4s + mul v18.8h, v18.8h, v27.8h + mls v16.8h, v28.8h, v7.h[0] + mls v18.8h, v0.8h, v7.h[0] + mul v23.8h, v8.8h, v11.h[4] + sqrdmulh v12.8h, v22.8h, v11.h[3] + trn1 v17.4s, v16.4s, v18.4s + trn2 v4.4s, v16.4s, v18.4s + mls v23.8h, v31.8h, v7.h[0] + trn2 v3.2d, v2.2d, v17.2d + trn2 v6.2d, v24.2d, v4.2d + mul v26.8h, v22.8h, v11.h[2] + trn1 v28.2d, v2.2d, v17.2d + mls v26.8h, v12.8h, v7.h[0] + add v25.8h, v3.8h, v6.8h + sub v18.8h, v3.8h, v6.8h + trn1 v24.2d, v24.2d, v4.2d + sqdmulh v1.8h, v25.8h, v7.h[1] + sub v27.8h, v28.8h, v24.8h + sqrdmulh v2.8h, v18.8h, v9.h[5] + add v28.8h, v28.8h, v24.8h + mul v24.8h, v27.8h, v9.h[2] + sqdmulh v12.8h, v28.8h, v7.h[1] + mul v20.8h, v18.8h, v9.h[4] + mls v20.8h, v2.8h, v7.h[0] + srshr v1.8h, v1.8h, #0xb + sqrdmulh v19.8h, v27.8h, v9.h[3] + srshr v15.8h, v12.8h, #0xb + mls v25.8h, v1.8h, v7.h[0] + add v8.8h, v26.8h, v23.8h + sub v4.8h, v26.8h, v23.8h + mls v28.8h, v15.8h, v7.h[0] + mls v24.8h, v19.8h, v7.h[0] + mul v2.8h, v4.8h, v11.h[0] + sub v19.8h, v28.8h, v25.8h + sqrdmulh v15.8h, v4.8h, v11.h[1] + add v25.8h, v28.8h, v25.8h + sub v10.8h, v24.8h, v20.8h + str q25, [x3], #0x40 + sqrdmulh v22.8h, v19.8h, v9.h[1] + add v28.8h, v24.8h, v20.8h + sqrdmulh v25.8h, v10.8h, v9.h[1] + mul v27.8h, v19.8h, v9.h[0] + mul v26.8h, v10.8h, v9.h[0] + sqdmulh v20.8h, v28.8h, v7.h[1] + sqdmulh v16.8h, v8.8h, v7.h[1] + mls v26.8h, v25.8h, v7.h[0] + mls v2.8h, v15.8h, v7.h[0] + srshr v15.8h, v20.8h, #0xb + srshr v1.8h, v16.8h, #0xb + mls v27.8h, v22.8h, v7.h[0] + mls v28.8h, v15.8h, v7.h[0] + mls v8.8h, v1.8h, v7.h[0] + stur q27, [x3, #-0x20] + stur q2, [x3, #-0x50] + stur q28, [x3, #-0x30] + stur q26, [x3, #-0x10] + stur q8, [x3, #-0x70] + mov x4, #0x4 // =4 + ldr q0, [x1], #0x20 + ldur q1, [x1, #-0x10] + ldr q26, [x0] + ldr q13, [x0, #0x40] + ldr q28, [x0, #0xc0] + ldr q2, [x0, #0x140] + ldr q6, [x0, #0x80] + ldr q9, [x0, #0x100] + ldr q29, [x0, #0x1c0] + ldr q23, [x0, #0x180] + sub v17.8h, v26.8h, v13.8h + add v4.8h, v26.8h, v13.8h + ldr q25, [x0, #0xd0] + ldr q24, [x0, #0x50] + add v5.8h, v6.8h, v28.8h + mul v19.8h, v17.8h, v0.h[6] + sub v10.8h, v6.8h, v28.8h + ldr q30, [x0, #0x150] + sqrdmulh v12.8h, v17.8h, v0.h[7] + add v17.8h, v9.8h, v2.8h + sub v28.8h, v9.8h, v2.8h + ldr q2, [x0, #0x90] + sub v26.8h, v23.8h, v29.8h + sqrdmulh v31.8h, v10.8h, v1.h[1] + add v22.8h, v23.8h, v29.8h + ldr q3, [x0, #0x110] + sqrdmulh v9.8h, v28.8h, v1.h[3] + sub v20.8h, v4.8h, v5.8h + sub v27.8h, v17.8h, v22.8h + ldr q29, [x0, #0x10] + add v16.8h, v4.8h, v5.8h + sqrdmulh v4.8h, v26.8h, v1.h[5] + add v6.8h, v17.8h, v22.8h + ldr q22, [x0, #0x1d0] + mul v8.8h, v28.8h, v1.h[2] + sub v21.8h, v2.8h, v25.8h + sub v5.8h, v16.8h, v6.8h + mul v17.8h, v26.8h, v1.h[4] + mul v26.8h, v10.8h, v1.h[0] + mls v26.8h, v31.8h, v7.h[0] + mls v17.8h, v4.8h, v7.h[0] + mls v19.8h, v12.8h, v7.h[0] + mls v8.8h, v9.8h, v7.h[0] + sqrdmulh v10.8h, v27.8h, v0.h[5] + sub v12.8h, v19.8h, v26.8h + add v9.8h, v19.8h, v26.8h + sqrdmulh v26.8h, v20.8h, v0.h[3] + sub v11.8h, v8.8h, v17.8h + add v14.8h, v8.8h, v17.8h + sqrdmulh v13.8h, v12.8h, v0.h[3] + add v23.8h, v9.8h, v14.8h + sqrdmulh v28.8h, v11.8h, v0.h[5] + sub v19.8h, v9.8h, v14.8h + mul v17.8h, v27.8h, v0.h[4] + str q23, [x0, #0x40] + mul v14.8h, v20.8h, v0.h[2] + mul v8.8h, v11.8h, v0.h[4] + mul v4.8h, v12.8h, v0.h[2] + mls v14.8h, v26.8h, v7.h[0] + mls v17.8h, v10.8h, v7.h[0] + mls v8.8h, v28.8h, v7.h[0] + mls v4.8h, v13.8h, v7.h[0] + sub v10.8h, v14.8h, v17.8h + add v20.8h, v14.8h, v17.8h + sqrdmulh v28.8h, v5.8h, v0.h[1] + mul v18.8h, v5.8h, v0.h[0] + str q20, [x0, #0x80] + sub v13.8h, v4.8h, v8.8h + mul v23.8h, v10.8h, v0.h[0] + mul v17.8h, v19.8h, v0.h[0] + sqrdmulh v9.8h, v13.8h, v0.h[1] + mls v18.8h, v28.8h, v7.h[0] + sqrdmulh v10.8h, v10.8h, v0.h[1] + sub x4, x4, #0x2 + +Lmlk_intt_layer123_start: + sub v12.8h, v3.8h, v30.8h + mul v11.8h, v21.8h, v1.h[0] + add v28.8h, v4.8h, v8.8h + ldr q20, [x0, #0x190] + add v27.8h, v16.8h, v6.8h + sqrdmulh v8.8h, v12.8h, v1.h[3] + add v16.8h, v29.8h, v24.8h + str q28, [x0, #0xc0] + mls v23.8h, v10.8h, v7.h[0] + str q27, [x0], #0x10 + add v15.8h, v20.8h, v22.8h + str q18, [x0, #0xf0] + mul v14.8h, v13.8h, v0.h[0] + add v2.8h, v2.8h, v25.8h + sub v26.8h, v20.8h, v22.8h + mul v4.8h, v12.8h, v1.h[2] + sub v5.8h, v16.8h, v2.8h + str q23, [x0, #0x170] + add v20.8h, v3.8h, v30.8h + sqrdmulh v27.8h, v26.8h, v1.h[5] + add v16.8h, v16.8h, v2.8h + mul v18.8h, v26.8h, v1.h[4] + sub v31.8h, v20.8h, v15.8h + mls v4.8h, v8.8h, v7.h[0] + sub v28.8h, v29.8h, v24.8h + mls v18.8h, v27.8h, v7.h[0] + ldr q22, [x0, #0x1d0] + mul v26.8h, v28.8h, v0.h[6] + mul v2.8h, v5.8h, v0.h[2] + sub v12.8h, v4.8h, v18.8h + sqrdmulh v24.8h, v28.8h, v0.h[7] + mls v14.8h, v9.8h, v7.h[0] + sqrdmulh v10.8h, v12.8h, v0.h[5] + mls v26.8h, v24.8h, v7.h[0] + ldr q24, [x0, #0x50] + mul v8.8h, v12.8h, v0.h[4] + str q14, [x0, #0x1b0] + add v28.8h, v4.8h, v18.8h + sqrdmulh v5.8h, v5.8h, v0.h[3] + add v6.8h, v20.8h, v15.8h + sqrdmulh v3.8h, v19.8h, v0.h[1] + sub v13.8h, v16.8h, v6.8h + sqrdmulh v12.8h, v21.8h, v1.h[1] + sqrdmulh v21.8h, v13.8h, v0.h[1] + sqrdmulh v27.8h, v31.8h, v0.h[5] + ldr q25, [x0, #0xd0] + mls v11.8h, v12.8h, v7.h[0] + mul v23.8h, v31.8h, v0.h[4] + mul v18.8h, v13.8h, v0.h[0] + add v30.8h, v26.8h, v11.8h + sub v13.8h, v26.8h, v11.8h + mls v23.8h, v27.8h, v7.h[0] + add v12.8h, v30.8h, v28.8h + sub v19.8h, v30.8h, v28.8h + mls v2.8h, v5.8h, v7.h[0] + str q12, [x0, #0x40] + sqrdmulh v26.8h, v13.8h, v0.h[3] + mls v8.8h, v10.8h, v7.h[0] + ldr q30, [x0, #0x150] + sub v20.8h, v2.8h, v23.8h + mul v4.8h, v13.8h, v0.h[2] + add v13.8h, v2.8h, v23.8h + mls v4.8h, v26.8h, v7.h[0] + ldr q2, [x0, #0x90] + mul v23.8h, v20.8h, v0.h[0] + ldr q29, [x0, #0x10] + sqrdmulh v10.8h, v20.8h, v0.h[1] + str q13, [x0, #0x80] + sub v13.8h, v4.8h, v8.8h + mls v17.8h, v3.8h, v7.h[0] + ldr q3, [x0, #0x110] + mls v18.8h, v21.8h, v7.h[0] + sub v21.8h, v2.8h, v25.8h + sqrdmulh v9.8h, v13.8h, v0.h[1] + str q17, [x0, #0x130] + mul v17.8h, v19.8h, v0.h[0] + sub x4, x4, #0x1 + cbnz x4, Lmlk_intt_layer123_start + mls v23.8h, v10.8h, v7.h[0] + ldr q11, [x0, #0x190] + str q18, [x0, #0x100] + add v27.8h, v3.8h, v30.8h + mul v13.8h, v13.8h, v0.h[0] + sub v5.8h, v29.8h, v24.8h + add v14.8h, v16.8h, v6.8h + mls v13.8h, v9.8h, v7.h[0] + add v10.8h, v11.8h, v22.8h + str q23, [x0, #0x180] + sub v20.8h, v11.8h, v22.8h + sub v23.8h, v27.8h, v10.8h + sqrdmulh v16.8h, v21.8h, v1.h[1] + sqrdmulh v31.8h, v23.8h, v0.h[5] + str q13, [x0, #0x1c0] + add v13.8h, v4.8h, v8.8h + mul v18.8h, v21.8h, v1.h[0] + str q13, [x0, #0xc0] + sqrdmulh v13.8h, v19.8h, v0.h[1] + sqrdmulh v28.8h, v20.8h, v1.h[5] + str q14, [x0], #0x10 + mul v4.8h, v20.8h, v1.h[4] + mls v17.8h, v13.8h, v7.h[0] + sub v13.8h, v3.8h, v30.8h + sqrdmulh v8.8h, v13.8h, v1.h[3] + mul v12.8h, v13.8h, v1.h[2] + mls v4.8h, v28.8h, v7.h[0] + mls v12.8h, v8.8h, v7.h[0] + mls v18.8h, v16.8h, v7.h[0] + str q17, [x0, #0x130] + sqrdmulh v15.8h, v5.8h, v0.h[7] + add v11.8h, v27.8h, v10.8h + mul v16.8h, v5.8h, v0.h[6] + sub v8.8h, v12.8h, v4.8h + sqrdmulh v28.8h, v8.8h, v0.h[5] + add v13.8h, v2.8h, v25.8h + mls v16.8h, v15.8h, v7.h[0] + add v26.8h, v12.8h, v4.8h + mul v8.8h, v8.8h, v0.h[4] + add v4.8h, v29.8h, v24.8h + mls v8.8h, v28.8h, v7.h[0] + sub v20.8h, v4.8h, v13.8h + add v14.8h, v4.8h, v13.8h + add v12.8h, v16.8h, v18.8h + sqrdmulh v22.8h, v20.8h, v0.h[3] + add v27.8h, v14.8h, v11.8h + sub v13.8h, v16.8h, v18.8h + mul v4.8h, v20.8h, v0.h[2] + str q27, [x0], #0x10 + sub v24.8h, v12.8h, v26.8h + sqrdmulh v3.8h, v13.8h, v0.h[3] + mul v13.8h, v13.8h, v0.h[2] + sqrdmulh v27.8h, v24.8h, v0.h[1] + mls v13.8h, v3.8h, v7.h[0] + mul v9.8h, v24.8h, v0.h[0] + mls v9.8h, v27.8h, v7.h[0] + add v30.8h, v13.8h, v8.8h + sub v13.8h, v13.8h, v8.8h + mls v4.8h, v22.8h, v7.h[0] + str q30, [x0, #0xb0] + sqrdmulh v16.8h, v13.8h, v0.h[1] + str q9, [x0, #0x130] + mul v9.8h, v13.8h, v0.h[0] + add v13.8h, v12.8h, v26.8h + str q13, [x0, #0x30] + mul v13.8h, v23.8h, v0.h[4] + sub v23.8h, v14.8h, v11.8h + mls v13.8h, v31.8h, v7.h[0] + mls v9.8h, v16.8h, v7.h[0] + mul v30.8h, v23.8h, v0.h[0] + sub v24.8h, v4.8h, v13.8h + add v13.8h, v4.8h, v13.8h + sqrdmulh v23.8h, v23.8h, v0.h[1] + str q9, [x0, #0x1b0] + str q13, [x0, #0x70] + sqrdmulh v13.8h, v24.8h, v0.h[1] + mul v21.8h, v24.8h, v0.h[0] + mls v30.8h, v23.8h, v7.h[0] + mls v21.8h, v13.8h, v7.h[0] + str q30, [x0, #0xf0] + str q21, [x0, #0x170] + ldp d8, d9, [sp] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0x10] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0x20] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0x30] + .cfi_restore d14 + .cfi_restore d15 + add sp, sp, #0x40 + .cfi_adjust_cfa_offset -0x40 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(intt_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_ntt_aarch64_asm.S b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_ntt_aarch64_asm.S new file mode 100644 index 0000000000..2383c0f05e --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_ntt_aarch64_asm.S @@ -0,0 +1,565 @@ +/* Copyright (c) 2022 Arm Limited + * Copyright (c) 2022 Hanno Becker + * Copyright (c) 2023 Amin Abdulrahman, Matthias Kannwischer + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [NeonNTT] + * Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1 + * Becker, Hwang, Kannwischer, Yang, Yang + * https://eprint.iacr.org/2021/986 + * + * - [SLOTHY_Paper] + * Fast and Clean: Auditable high-performance assembly via constraint solving + * Abdulrahman, Becker, Kannwischer, Klein + * https://eprint.iacr.org/2022/1303 + */ + +/*yaml + Name: ntt_aarch64_asm + Description: AArch64 ML-KEM forward NTT following @[NeonNTT] and @[SLOTHY_Paper] + Signature: void mlk_ntt_aarch64_asm(int16_t p[256], const int16_t twiddles12345[80], const int16_t twiddles56[384]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON] + x0: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t p[256] + description: Input/output polynomial + x1: + type: buffer + size_bytes: 160 + permissions: read-only + c_parameter: const int16_t twiddles12345[80] + description: Twiddle factors for layers 1-5 + x2: + type: buffer + size_bytes: 768 + permissions: read-only + c_parameter: const int16_t twiddles56[384] + description: Twiddle factors for layers 6-7 + Stack: + bytes: 64 + description: saving callee-saved Neon registers +*/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/mlkem_ntt_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(ntt_aarch64_asm) +MLK_ASM_FN_SYMBOL(ntt_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x40 + .cfi_adjust_cfa_offset 0x40 + stp d8, d9, [sp] + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + stp d10, d11, [sp, #0x10] + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + stp d12, d13, [sp, #0x20] + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + stp d14, d15, [sp, #0x30] + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + mov w5, #0xd01 // =3329 + mov v7.h[0], w5 + mov w5, #0x4ebf // =20159 + mov v7.h[1], w5 + mov x3, x0 + mov x4, #0x4 // =4 + ldr q0, [x1], #0x20 + ldur q1, [x1, #-0x10] + ldr q21, [x0, #0x40] + ldr q5, [x0, #0x1c0] + ldr q30, [x0, #0x110] + ldr q24, [x0, #0x140] + ldr q12, [x0, #0x80] + sqrdmulh v9.8h, v5.8h, v0.h[1] + mul v23.8h, v5.8h, v0.h[0] + sqrdmulh v17.8h, v24.8h, v0.h[1] + ldr q13, [x0, #0xc0] + mls v23.8h, v9.8h, v7.h[0] + mul v8.8h, v24.8h, v0.h[0] + mls v8.8h, v17.8h, v7.h[0] + add v9.8h, v13.8h, v23.8h + sub v10.8h, v13.8h, v23.8h + mul v11.8h, v30.8h, v0.h[0] + ldr q13, [x0, #0x180] + sqrdmulh v28.8h, v9.8h, v0.h[3] + sub v29.8h, v21.8h, v8.8h + mul v26.8h, v9.8h, v0.h[2] + add v8.8h, v21.8h, v8.8h + mul v2.8h, v13.8h, v0.h[0] + mls v26.8h, v28.8h, v7.h[0] + mul v28.8h, v10.8h, v0.h[4] + sqrdmulh v23.8h, v10.8h, v0.h[5] + add v22.8h, v8.8h, v26.8h + sqrdmulh v10.8h, v13.8h, v0.h[1] + sqrdmulh v21.8h, v22.8h, v0.h[7] + ldr q13, [x0, #0x100] + mul v16.8h, v22.8h, v0.h[6] + mls v28.8h, v23.8h, v7.h[0] + mls v2.8h, v10.8h, v7.h[0] + sqrdmulh v23.8h, v13.8h, v0.h[1] + sub v10.8h, v29.8h, v28.8h + add v17.8h, v29.8h, v28.8h + mls v16.8h, v21.8h, v7.h[0] + sub v18.8h, v12.8h, v2.8h + ldr q29, [x0] + sqrdmulh v14.8h, v17.8h, v1.h[3] + add v22.8h, v12.8h, v2.8h + sqrdmulh v9.8h, v18.8h, v0.h[5] + mul v21.8h, v13.8h, v0.h[0] + ldr q13, [x0, #0x150] + mul v5.8h, v18.8h, v0.h[4] + mls v5.8h, v9.8h, v7.h[0] + mul v18.8h, v13.8h, v0.h[0] + mls v21.8h, v23.8h, v7.h[0] + sqrdmulh v2.8h, v13.8h, v0.h[1] + mul v13.8h, v17.8h, v1.h[2] + sub v4.8h, v29.8h, v21.8h + mls v13.8h, v14.8h, v7.h[0] + add v25.8h, v29.8h, v21.8h + add v6.8h, v4.8h, v5.8h + sqrdmulh v15.8h, v22.8h, v0.h[3] + sub v21.8h, v4.8h, v5.8h + sub v5.8h, v8.8h, v26.8h + mul v23.8h, v22.8h, v0.h[2] + add v28.8h, v6.8h, v13.8h + sub v13.8h, v6.8h, v13.8h + mul v4.8h, v5.8h, v1.h[0] + sub x4, x4, #0x2 + +Lmlk_ntt_layer123_start: + mls v23.8h, v15.8h, v7.h[0] + ldr q6, [x0, #0x190] + ldr q15, [x0, #0x90] + ldr q19, [x0, #0x10] + mul v22.8h, v10.8h, v1.h[4] + ldr q24, [x0, #0x50] + str q13, [x0, #0x140] + sqrdmulh v13.8h, v6.8h, v0.h[1] + sub v20.8h, v25.8h, v23.8h + sqrdmulh v3.8h, v30.8h, v0.h[1] + str q28, [x0, #0x100] + ldr q30, [x0, #0x120] + mul v8.8h, v6.8h, v0.h[0] + sqrdmulh v27.8h, v10.8h, v1.h[5] + mls v11.8h, v3.8h, v7.h[0] + mls v18.8h, v2.8h, v7.h[0] + ldr q31, [x0, #0x160] + sqrdmulh v10.8h, v5.8h, v1.h[1] + mls v8.8h, v13.8h, v7.h[0] + ldr q13, [x0, #0x1d0] + sub v14.8h, v24.8h, v18.8h + add v9.8h, v24.8h, v18.8h + sqrdmulh v2.8h, v31.8h, v0.h[1] + mls v4.8h, v10.8h, v7.h[0] + add v10.8h, v25.8h, v23.8h + sub v24.8h, v19.8h, v11.8h + add v25.8h, v19.8h, v11.8h + sqrdmulh v28.8h, v13.8h, v0.h[1] + mul v11.8h, v30.8h, v0.h[0] + mul v17.8h, v13.8h, v0.h[0] + sub v13.8h, v10.8h, v16.8h + sub v6.8h, v15.8h, v8.8h + mls v17.8h, v28.8h, v7.h[0] + str q13, [x0, #0x40] + mls v22.8h, v27.8h, v7.h[0] + ldr q13, [x0, #0xd0] + add v26.8h, v20.8h, v4.8h + mul v18.8h, v31.8h, v0.h[0] + add v27.8h, v10.8h, v16.8h + str q26, [x0, #0x80] + sqrdmulh v31.8h, v6.8h, v0.h[5] + add v3.8h, v21.8h, v22.8h + str q27, [x0], #0x10 + mul v26.8h, v6.8h, v0.h[4] + add v6.8h, v13.8h, v17.8h + sub v5.8h, v13.8h, v17.8h + str q3, [x0, #0x170] + sub v17.8h, v21.8h, v22.8h + sqrdmulh v10.8h, v6.8h, v0.h[3] + sub v13.8h, v20.8h, v4.8h + add v20.8h, v15.8h, v8.8h + sqrdmulh v12.8h, v5.8h, v0.h[5] + str q13, [x0, #0xb0] + mul v8.8h, v6.8h, v0.h[2] + str q17, [x0, #0x1b0] + mls v8.8h, v10.8h, v7.h[0] + mul v29.8h, v5.8h, v0.h[4] + mls v29.8h, v12.8h, v7.h[0] + sub v5.8h, v9.8h, v8.8h + add v3.8h, v9.8h, v8.8h + sqrdmulh v15.8h, v20.8h, v0.h[3] + mul v4.8h, v5.8h, v1.h[0] + add v6.8h, v14.8h, v29.8h + sqrdmulh v9.8h, v3.8h, v0.h[7] + sqrdmulh v12.8h, v6.8h, v1.h[3] + sub v10.8h, v14.8h, v29.8h + mul v23.8h, v6.8h, v1.h[2] + mls v26.8h, v31.8h, v7.h[0] + mls v23.8h, v12.8h, v7.h[0] + mul v16.8h, v3.8h, v0.h[6] + add v13.8h, v24.8h, v26.8h + sub v21.8h, v24.8h, v26.8h + mls v16.8h, v9.8h, v7.h[0] + add v28.8h, v13.8h, v23.8h + sub v13.8h, v13.8h, v23.8h + mul v23.8h, v20.8h, v0.h[2] + sub x4, x4, #0x1 + cbnz x4, Lmlk_ntt_layer123_start + sqrdmulh v3.8h, v5.8h, v1.h[1] + mls v23.8h, v15.8h, v7.h[0] + ldr q5, [x0, #0x190] + mul v29.8h, v10.8h, v1.h[4] + mls v4.8h, v3.8h, v7.h[0] + sub v19.8h, v25.8h, v23.8h + sqrdmulh v31.8h, v5.8h, v0.h[1] + sqrdmulh v6.8h, v30.8h, v0.h[1] + sub v3.8h, v19.8h, v4.8h + mul v5.8h, v5.8h, v0.h[0] + str q3, [x0, #0xc0] + sqrdmulh v12.8h, v10.8h, v1.h[5] + mls v18.8h, v2.8h, v7.h[0] + ldr q3, [x0, #0x1d0] + mls v5.8h, v31.8h, v7.h[0] + sqrdmulh v10.8h, v3.8h, v0.h[1] + mls v11.8h, v6.8h, v7.h[0] + ldr q31, [x0, #0x90] + mul v30.8h, v3.8h, v0.h[0] + mls v30.8h, v10.8h, v7.h[0] + sub v10.8h, v31.8h, v5.8h + mls v29.8h, v12.8h, v7.h[0] + ldr q6, [x0, #0xd0] + sqrdmulh v15.8h, v10.8h, v0.h[5] + mul v17.8h, v10.8h, v0.h[4] + add v10.8h, v6.8h, v30.8h + sub v6.8h, v6.8h, v30.8h + sqrdmulh v12.8h, v10.8h, v0.h[3] + sub v27.8h, v21.8h, v29.8h + sqrdmulh v3.8h, v6.8h, v0.h[5] + mul v10.8h, v10.8h, v0.h[2] + ldr q20, [x0, #0x50] + mls v10.8h, v12.8h, v7.h[0] + mul v2.8h, v6.8h, v0.h[4] + add v6.8h, v20.8h, v18.8h + add v5.8h, v31.8h, v5.8h + mls v2.8h, v3.8h, v7.h[0] + sub v31.8h, v6.8h, v10.8h + sqrdmulh v12.8h, v5.8h, v0.h[3] + sub v22.8h, v20.8h, v18.8h + add v6.8h, v6.8h, v10.8h + mul v20.8h, v31.8h, v1.h[0] + add v30.8h, v22.8h, v2.8h + sqrdmulh v3.8h, v6.8h, v0.h[7] + sqrdmulh v10.8h, v30.8h, v1.h[3] + mul v9.8h, v30.8h, v1.h[2] + ldr q30, [x0, #0x10] + mls v17.8h, v15.8h, v7.h[0] + mls v9.8h, v10.8h, v7.h[0] + mul v15.8h, v6.8h, v0.h[6] + add v24.8h, v30.8h, v11.8h + sub v10.8h, v22.8h, v2.8h + mls v15.8h, v3.8h, v7.h[0] + add v6.8h, v19.8h, v4.8h + add v22.8h, v25.8h, v23.8h + sqrdmulh v3.8h, v10.8h, v1.h[5] + str q13, [x0, #0x140] + sub v19.8h, v30.8h, v11.8h + add v25.8h, v22.8h, v16.8h + mul v5.8h, v5.8h, v0.h[2] + sub v13.8h, v22.8h, v16.8h + str q28, [x0, #0x100] + mls v5.8h, v12.8h, v7.h[0] + str q13, [x0, #0x40] + str q6, [x0, #0x80] + add v21.8h, v21.8h, v29.8h + sqrdmulh v13.8h, v31.8h, v1.h[1] + str q25, [x0], #0x10 + add v12.8h, v19.8h, v17.8h + sub v31.8h, v19.8h, v17.8h + mul v30.8h, v10.8h, v1.h[4] + str q21, [x0, #0x170] + add v21.8h, v24.8h, v5.8h + add v6.8h, v12.8h, v9.8h + mls v30.8h, v3.8h, v7.h[0] + str q27, [x0, #0x1b0] + sub v10.8h, v21.8h, v15.8h + sub v12.8h, v12.8h, v9.8h + mls v20.8h, v13.8h, v7.h[0] + str q6, [x0, #0x100] + str q10, [x0, #0x40] + sub v13.8h, v24.8h, v5.8h + add v3.8h, v21.8h, v15.8h + str q12, [x0, #0x140] + sub v10.8h, v31.8h, v30.8h + add v21.8h, v31.8h, v30.8h + str q3, [x0], #0x10 + add v12.8h, v13.8h, v20.8h + sub v13.8h, v13.8h, v20.8h + str q21, [x0, #0x170] + str q10, [x0, #0x1b0] + str q12, [x0, #0x70] + str q13, [x0, #0xb0] + mov x0, x3 + mov x4, #0x8 // =8 + ldr q2, [x0, #0x20] + ldr q13, [x1], #0x10 + ldr q30, [x0, #0x30] + ldr q25, [x2, #0x40] + ldr q5, [x0] + ldr q18, [x0, #0x60] + ldr q12, [x0, #0x70] + sqrdmulh v17.8h, v2.8h, v13.h[1] + ldr q4, [x1], #0x10 + ldr q23, [x0, #0x10] + sqrdmulh v21.8h, v30.8h, v13.h[1] + ldr q24, [x2, #0x20] + ldr q9, [x2], #0x60 + mul v10.8h, v30.8h, v13.h[0] + mul v11.8h, v2.8h, v13.h[0] + mls v10.8h, v21.8h, v7.h[0] + sqrdmulh v29.8h, v12.8h, v4.h[1] + mul v1.8h, v12.8h, v4.h[0] + add v21.8h, v23.8h, v10.8h + sub v10.8h, v23.8h, v10.8h + mul v8.8h, v18.8h, v4.h[0] + sqrdmulh v23.8h, v21.8h, v13.h[3] + mul v2.8h, v21.8h, v13.h[2] + mls v1.8h, v29.8h, v7.h[0] + mls v2.8h, v23.8h, v7.h[0] + ldur q15, [x2, #-0x50] + sqrdmulh v0.8h, v10.8h, v13.h[5] + mls v11.8h, v17.8h, v7.h[0] + ldr q29, [x0, #0x50] + mul v23.8h, v10.8h, v13.h[4] + mls v23.8h, v0.8h, v7.h[0] + sub v16.8h, v29.8h, v1.8h + add v3.8h, v5.8h, v11.8h + sub v31.8h, v5.8h, v11.8h + sqrdmulh v22.8h, v16.8h, v4.h[5] + add v30.8h, v3.8h, v2.8h + sub v0.8h, v3.8h, v2.8h + sqrdmulh v28.8h, v18.8h, v4.h[1] + add v21.8h, v31.8h, v23.8h + sub v19.8h, v31.8h, v23.8h + mul v26.8h, v16.8h, v4.h[4] + trn2 v3.4s, v30.4s, v0.4s + ldur q23, [x2, #-0x10] + trn2 v18.4s, v21.4s, v19.4s + mls v26.8h, v22.8h, v7.h[0] + trn1 v13.4s, v30.4s, v0.4s + mls v8.8h, v28.8h, v7.h[0] + trn2 v31.2d, v3.2d, v18.2d + trn1 v11.4s, v21.4s, v19.4s + add v27.8h, v29.8h, v1.8h + sqrdmulh v6.8h, v31.8h, v15.8h + trn1 v2.2d, v13.2d, v11.2d + trn2 v13.2d, v13.2d, v11.2d + mul v1.8h, v31.8h, v9.8h + ldr q11, [x0, #0x40] + sqrdmulh v29.8h, v13.8h, v15.8h + mls v1.8h, v6.8h, v7.h[0] + trn1 v6.2d, v3.2d, v18.2d + mul v17.8h, v13.8h, v9.8h + sub v13.8h, v11.8h, v8.8h + sqrdmulh v10.8h, v27.8h, v4.h[3] + sub v12.8h, v13.8h, v26.8h + sub v18.8h, v6.8h, v1.8h + mls v17.8h, v29.8h, v7.h[0] + add v30.8h, v6.8h, v1.8h + add v6.8h, v13.8h, v26.8h + ldur q13, [x2, #-0x30] + sqrdmulh v16.8h, v18.8h, v23.8h + trn1 v28.4s, v6.4s, v12.4s + mul v23.8h, v18.8h, v25.8h + ldr q25, [x2, #0x10] + add v20.8h, v2.8h, v17.8h + mul v0.8h, v30.8h, v24.8h + sqrdmulh v29.8h, v30.8h, v13.8h + sub v30.8h, v2.8h, v17.8h + mls v23.8h, v16.8h, v7.h[0] + sub x4, x4, #0x2 + +Lmlk_ntt_layer4567_start: + ldr q19, [x2, #0x50] + sub v31.8h, v30.8h, v23.8h + mls v0.8h, v29.8h, v7.h[0] + add v16.8h, v11.8h, v8.8h + ldr q18, [x0, #0xa0] + trn2 v14.4s, v6.4s, v12.4s + mul v26.8h, v27.8h, v4.h[2] + ldr q4, [x1], #0x10 + ldr q24, [x2, #0x40] + ldr q21, [x0, #0xb0] + mls v26.8h, v10.8h, v7.h[0] + add v23.8h, v30.8h, v23.8h + sub v15.8h, v20.8h, v0.8h + ldr q9, [x0, #0x90] + add v10.8h, v20.8h, v0.8h + mul v8.8h, v18.8h, v4.h[0] + ldr q1, [x2], #0x60 + trn1 v27.4s, v23.4s, v31.4s + sqrdmulh v12.8h, v18.8h, v4.h[1] + trn1 v5.4s, v10.4s, v15.4s + sub v30.8h, v16.8h, v26.8h + trn2 v13.2d, v5.2d, v27.2d + sqrdmulh v2.8h, v21.8h, v4.h[1] + add v29.8h, v16.8h, v26.8h + mul v0.8h, v21.8h, v4.h[0] + str q13, [x0, #0x20] + trn1 v11.4s, v29.4s, v30.4s + mls v8.8h, v12.8h, v7.h[0] + trn2 v26.4s, v29.4s, v30.4s + trn2 v6.2d, v11.2d, v28.2d + mls v0.8h, v2.8h, v7.h[0] + trn2 v16.2d, v26.2d, v14.2d + trn1 v26.2d, v26.2d, v14.2d + trn1 v20.2d, v5.2d, v27.2d + sqrdmulh v29.8h, v6.8h, v25.8h + trn2 v15.4s, v10.4s, v15.4s + sqrdmulh v13.8h, v16.8h, v25.8h + str q20, [x0], #0x40 + sub v30.8h, v9.8h, v0.8h + add v27.8h, v9.8h, v0.8h + mul v17.8h, v6.8h, v1.8h + sqrdmulh v22.8h, v30.8h, v4.h[5] + mul v18.8h, v16.8h, v1.8h + mls v18.8h, v13.8h, v7.h[0] + mul v2.8h, v30.8h, v4.h[4] + mls v2.8h, v22.8h, v7.h[0] + trn2 v22.4s, v23.4s, v31.4s + sub v3.8h, v26.8h, v18.8h + ldur q25, [x2, #-0x30] + mls v17.8h, v29.8h, v7.h[0] + trn2 v31.2d, v15.2d, v22.2d + trn1 v20.2d, v15.2d, v22.2d + add v16.8h, v26.8h, v18.8h + sqrdmulh v26.8h, v3.8h, v19.8h + trn1 v21.2d, v11.2d, v28.2d + ldr q11, [x0, #0x40] + sqrdmulh v29.8h, v16.8h, v25.8h + stur q20, [x0, #-0x30] + add v20.8h, v21.8h, v17.8h + stur q31, [x0, #-0x10] + mul v23.8h, v3.8h, v24.8h + ldr q25, [x2, #0x10] + sub v13.8h, v11.8h, v8.8h + mls v23.8h, v26.8h, v7.h[0] + ldur q1, [x2, #-0x40] + sub v12.8h, v13.8h, v2.8h + add v6.8h, v13.8h, v2.8h + sqrdmulh v10.8h, v27.8h, v4.h[3] + sub v30.8h, v21.8h, v17.8h + mul v0.8h, v16.8h, v1.8h + trn1 v28.4s, v6.4s, v12.4s + sub x4, x4, #0x1 + cbnz x4, Lmlk_ntt_layer4567_start + add v22.8h, v11.8h, v8.8h + mul v27.8h, v27.8h, v4.h[2] + trn2 v17.4s, v6.4s, v12.4s + ldr q15, [x2], #0x60 + mls v27.8h, v10.8h, v7.h[0] + add v4.8h, v30.8h, v23.8h + sub v18.8h, v30.8h, v23.8h + ldur q6, [x2, #-0x30] + mls v0.8h, v29.8h, v7.h[0] + ldur q12, [x2, #-0x40] + ldur q24, [x2, #-0x20] + ldur q2, [x2, #-0x10] + trn1 v9.4s, v4.4s, v18.4s + add v10.8h, v22.8h, v27.8h + sub v13.8h, v22.8h, v27.8h + sub v1.8h, v20.8h, v0.8h + trn2 v21.4s, v10.4s, v13.4s + add v27.8h, v20.8h, v0.8h + trn2 v3.2d, v21.2d, v17.2d + trn1 v13.4s, v10.4s, v13.4s + trn1 v31.4s, v27.4s, v1.4s + sqrdmulh v10.8h, v3.8h, v25.8h + trn2 v5.2d, v13.2d, v28.2d + trn1 v13.2d, v13.2d, v28.2d + trn1 v21.2d, v21.2d, v17.2d + sqrdmulh v17.8h, v5.8h, v25.8h + trn2 v30.2d, v31.2d, v9.2d + mul v25.8h, v3.8h, v15.8h + str q30, [x0, #0x20] + trn2 v30.4s, v4.4s, v18.4s + mls v25.8h, v10.8h, v7.h[0] + trn2 v3.4s, v27.4s, v1.4s + mul v20.8h, v5.8h, v15.8h + trn2 v10.2d, v3.2d, v30.2d + mls v20.8h, v17.8h, v7.h[0] + str q10, [x0, #0x30] + sub v18.8h, v21.8h, v25.8h + add v10.8h, v21.8h, v25.8h + trn1 v3.2d, v3.2d, v30.2d + sqrdmulh v30.8h, v18.8h, v2.8h + mul v12.8h, v10.8h, v12.8h + sqrdmulh v6.8h, v10.8h, v6.8h + str q3, [x0, #0x10] + add v21.8h, v13.8h, v20.8h + mul v10.8h, v18.8h, v24.8h + sub v13.8h, v13.8h, v20.8h + mls v10.8h, v30.8h, v7.h[0] + mls v12.8h, v6.8h, v7.h[0] + trn1 v30.2d, v31.2d, v9.2d + sub v3.8h, v13.8h, v10.8h + add v6.8h, v13.8h, v10.8h + add v10.8h, v21.8h, v12.8h + sub v21.8h, v21.8h, v12.8h + trn2 v13.4s, v6.4s, v3.4s + trn1 v12.4s, v10.4s, v21.4s + trn2 v21.4s, v10.4s, v21.4s + trn1 v3.4s, v6.4s, v3.4s + str q30, [x0], #0x40 + trn2 v10.2d, v21.2d, v13.2d + trn1 v13.2d, v21.2d, v13.2d + trn2 v21.2d, v12.2d, v3.2d + trn1 v3.2d, v12.2d, v3.2d + str q10, [x0, #0x30] + str q13, [x0, #0x10] + str q3, [x0], #0x40 + stur q21, [x0, #-0x20] + ldp d8, d9, [sp] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0x10] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0x20] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0x30] + .cfi_restore d14 + .cfi_restore d15 + add sp, sp, #0x40 + .cfi_adjust_cfa_offset -0x40 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(ntt_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_mulcache_compute_aarch64_asm.S b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_mulcache_compute_aarch64_asm.S new file mode 100644 index 0000000000..4b869f8c92 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_mulcache_compute_aarch64_asm.S @@ -0,0 +1,130 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_mulcache_compute_aarch64_asm + Description: Compute multiplication cache for polynomial + Signature: void mlk_poly_mulcache_compute_aarch64_asm(int16_t cache[128], const int16_t mlk_poly[256], const int16_t zetas[128], const int16_t zetas_twisted[128]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON] + x0: + type: buffer + size_bytes: 256 + permissions: write-only + c_parameter: int16_t cache[128] + description: Output cache + x1: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t mlk_poly[256] + description: Input polynomial + x2: + type: buffer + size_bytes: 256 + permissions: read-only + c_parameter: const int16_t zetas[128] + description: Zeta values + x3: + type: buffer + size_bytes: 256 + permissions: read-only + c_parameter: const int16_t zetas_twisted[128] + description: Twisted zeta values + Stack: + bytes: 0 +*/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/mlkem_poly_mulcache_compute_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_mulcache_compute_aarch64_asm) +MLK_ASM_FN_SYMBOL(poly_mulcache_compute_aarch64_asm) + + .cfi_startproc + mov w5, #0xd01 // =3329 + dup v6.8h, w5 + mov w5, #0x4ebf // =20159 + dup v7.8h, w5 + mov x4, #0x10 // =16 + ldr q0, [x1], #0x20 + ldur q2, [x1, #-0x10] + ldr q19, [x1], #0x20 + ldr q29, [x3], #0x10 + ldur q16, [x1, #-0x10] + ldr q18, [x2], #0x10 + ldr q26, [x1], #0x20 + ldr q25, [x2], #0x10 + uzp2 v5.8h, v0.8h, v2.8h + ldr q28, [x3], #0x10 + ldur q7, [x1, #-0x10] + ldr q2, [x1], #0x20 + uzp2 v27.8h, v19.8h, v16.8h + sqrdmulh v16.8h, v5.8h, v29.8h + ldr q17, [x3], #0x10 + ldr q19, [x3], #0x10 + mul v5.8h, v5.8h, v18.8h + uzp2 v29.8h, v26.8h, v7.8h + mul v26.8h, v27.8h, v25.8h + sqrdmulh v4.8h, v27.8h, v28.8h + mls v5.8h, v16.8h, v6.h[0] + lsr x4, x4, #1 + sub x4, x4, #0x2 + +Lmlk_poly_mulcache_compute_loop_start: + str q5, [x0], #0x10 + sqrdmulh v22.8h, v29.8h, v17.8h + ldr q28, [x2], #0x10 + ldur q24, [x1, #-0x10] + ldr q0, [x1], #0x20 + mls v26.8h, v4.8h, v6.h[0] + ldur q16, [x1, #-0x10] + ldr q17, [x3], #0x10 + mul v5.8h, v29.8h, v28.8h + uzp2 v23.8h, v2.8h, v24.8h + ldr q18, [x2], #0x10 + mls v5.8h, v22.8h, v6.h[0] + uzp2 v29.8h, v0.8h, v16.8h + sqrdmulh v4.8h, v23.8h, v19.8h + ldr q2, [x1], #0x20 + ldr q19, [x3], #0x10 + str q26, [x0], #0x10 + mul v26.8h, v23.8h, v18.8h + subs x4, x4, #0x1 + cbnz x4, Lmlk_poly_mulcache_compute_loop_start + mls v26.8h, v4.8h, v6.h[0] + str q5, [x0], #0x10 + ldr q5, [x2], #0x10 + ldur q4, [x1, #-0x10] + sqrdmulh v16.8h, v29.8h, v17.8h + ldr q0, [x2], #0x10 + mul v29.8h, v29.8h, v5.8h + uzp2 v18.8h, v2.8h, v4.8h + str q26, [x0], #0x10 + sqrdmulh v17.8h, v18.8h, v19.8h + mls v29.8h, v16.8h, v6.h[0] + mul v26.8h, v18.8h, v0.8h + mls v26.8h, v17.8h, v6.h[0] + str q29, [x0], #0x10 + str q26, [x0], #0x10 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_mulcache_compute_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_reduce_aarch64_asm.S b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_reduce_aarch64_asm.S new file mode 100644 index 0000000000..0244ac6af3 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_reduce_aarch64_asm.S @@ -0,0 +1,153 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_reduce_aarch64_asm + Description: Barrett reduction of polynomial coefficients + Signature: void mlk_poly_reduce_aarch64_asm(int16_t p[256]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON] + x0: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t p[256] + description: Input/output polynomial + Stack: + bytes: 0 +*/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/mlkem_poly_reduce_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_reduce_aarch64_asm) +MLK_ASM_FN_SYMBOL(poly_reduce_aarch64_asm) + + .cfi_startproc + mov w2, #0xd01 // =3329 + dup v3.8h, w2 + mov w2, #0x4ebf // =20159 + dup v4.8h, w2 + mov x1, #0x8 // =8 + ldr q21, [x0], #0x40 + ldur q18, [x0, #-0x20] + ldur q0, [x0, #-0x30] + ldur q5, [x0, #-0x10] + ldr q26, [x0], #0x40 + sqdmulh v17.8h, v21.8h, v4.h[0] + sqdmulh v27.8h, v18.8h, v4.h[0] + sqdmulh v22.8h, v0.8h, v4.h[0] + srshr v17.8h, v17.8h, #0xb + sqdmulh v23.8h, v5.8h, v4.h[0] + srshr v29.8h, v27.8h, #0xb + mls v21.8h, v17.8h, v3.h[0] + srshr v17.8h, v22.8h, #0xb + mls v18.8h, v29.8h, v3.h[0] + srshr v22.8h, v23.8h, #0xb + mls v0.8h, v17.8h, v3.h[0] + sshr v2.8h, v21.8h, #0xf + mls v5.8h, v22.8h, v3.h[0] + sshr v29.8h, v18.8h, #0xf + and v19.16b, v3.16b, v2.16b + sqdmulh v2.8h, v26.8h, v4.h[0] + sshr v31.8h, v0.8h, #0xf + add v17.8h, v21.8h, v19.8h + and v21.16b, v3.16b, v29.16b + and v31.16b, v3.16b, v31.16b + sub x1, x1, #0x2 + +Lmlk_poly_reduce_loop_start: + add v21.8h, v18.8h, v21.8h + ldur q18, [x0, #-0x20] + add v25.8h, v0.8h, v31.8h + ldur q0, [x0, #-0x30] + stur q21, [x0, #-0x60] + sshr v28.8h, v5.8h, #0xf + stur q17, [x0, #-0x80] + srshr v23.8h, v2.8h, #0xb + sqdmulh v30.8h, v18.8h, v4.h[0] + stur q25, [x0, #-0x70] + and v22.16b, v3.16b, v28.16b + sqdmulh v7.8h, v0.8h, v4.h[0] + add v16.8h, v5.8h, v22.8h + ldur q5, [x0, #-0x10] + mls v26.8h, v23.8h, v3.h[0] + stur q16, [x0, #-0x50] + srshr v6.8h, v30.8h, #0xb + srshr v1.8h, v7.8h, #0xb + sqdmulh v19.8h, v5.8h, v4.h[0] + mls v18.8h, v6.8h, v3.h[0] + sshr v24.8h, v26.8h, #0xf + mls v0.8h, v1.8h, v3.h[0] + and v27.16b, v3.16b, v24.16b + srshr v29.8h, v19.8h, #0xb + add v17.8h, v26.8h, v27.8h + ldr q26, [x0], #0x40 + sshr v1.8h, v18.8h, #0xf + mls v5.8h, v29.8h, v3.h[0] + sshr v20.8h, v0.8h, #0xf + and v21.16b, v3.16b, v1.16b + and v31.16b, v3.16b, v20.16b + sqdmulh v2.8h, v26.8h, v4.h[0] + subs x1, x1, #0x1 + cbnz x1, Lmlk_poly_reduce_loop_start + add v28.8h, v0.8h, v31.8h + ldur q29, [x0, #-0x10] + add v21.8h, v18.8h, v21.8h + srshr v18.8h, v2.8h, #0xb + sshr v2.8h, v5.8h, #0xf + ldur q16, [x0, #-0x20] + stur q17, [x0, #-0x80] + ldur q0, [x0, #-0x30] + and v2.16b, v3.16b, v2.16b + sqdmulh v24.8h, v29.8h, v4.h[0] + stur q28, [x0, #-0x70] + stur q21, [x0, #-0x60] + add v31.8h, v5.8h, v2.8h + sqdmulh v6.8h, v16.8h, v4.h[0] + stur q31, [x0, #-0x50] + sqdmulh v17.8h, v0.8h, v4.h[0] + srshr v22.8h, v24.8h, #0xb + mls v26.8h, v18.8h, v3.h[0] + srshr v31.8h, v6.8h, #0xb + mls v29.8h, v22.8h, v3.h[0] + srshr v19.8h, v17.8h, #0xb + mls v16.8h, v31.8h, v3.h[0] + sshr v7.8h, v26.8h, #0xf + mls v0.8h, v19.8h, v3.h[0] + and v5.16b, v3.16b, v7.16b + sshr v22.8h, v29.8h, #0xf + add v27.8h, v26.8h, v5.8h + and v26.16b, v3.16b, v22.16b + sshr v20.8h, v16.8h, #0xf + stur q27, [x0, #-0x40] + and v2.16b, v3.16b, v20.16b + sshr v23.8h, v0.8h, #0xf + add v18.8h, v29.8h, v26.8h + add v31.8h, v16.8h, v2.8h + and v29.16b, v3.16b, v23.16b + stur q18, [x0, #-0x10] + add v25.8h, v0.8h, v29.8h + stur q31, [x0, #-0x20] + stur q25, [x0, #-0x30] + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_reduce_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_tobytes_aarch64_asm.S b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_tobytes_aarch64_asm.S new file mode 100644 index 0000000000..57b798c171 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_tobytes_aarch64_asm.S @@ -0,0 +1,124 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_tobytes_aarch64_asm + Description: Convert polynomial to byte representation + Signature: void mlk_poly_tobytes_aarch64_asm(uint8_t r[384], const int16_t a[256]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON] + x0: + type: buffer + size_bytes: 384 + permissions: write-only + c_parameter: uint8_t r[384] + description: Output byte array + x1: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t a[256] + description: Input polynomial + Stack: + bytes: 0 +*/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (!defined(MLK_CONFIG_NO_KEYPAIR_API) || \ + !defined(MLK_CONFIG_NO_ENCAPS_API)) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/mlkem_poly_tobytes_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_tobytes_aarch64_asm) +MLK_ASM_FN_SYMBOL(poly_tobytes_aarch64_asm) + + .cfi_startproc + mov x2, #0x10 // =16 + ldr q5, [x1, #0x10] + ldr q3, [x1], #0x20 + ldr q29, [x1], #0x20 + ldur q2, [x1, #-0x10] + ldr q27, [x1, #0x10] + ldr q23, [x1, #0x30] + ldr q17, [x1], #0x20 + ldr q16, [x1], #0x20 + uzp2 v26.8h, v3.8h, v5.8h + uzp1 v19.8h, v3.8h, v5.8h + uzp2 v0.8h, v29.8h, v2.8h + uzp1 v1.8h, v29.8h, v2.8h + xtn v5.8b, v26.8h + shrn v3.8b, v19.8h, #0x8 + shrn v4.8b, v26.8h, #0x4 + xtn v18.8b, v0.8h + shrn v30.8b, v0.8h, #0x4 + xtn v28.8b, v1.8h + shrn v29.8b, v1.8h, #0x8 + sli v3.8b, v5.8b, #0x4 + xtn v2.8b, v19.8h + sli v29.8b, v18.8b, #0x4 + lsr x2, x2, #1 + sub x2, x2, #0x2 + +Lmlk_poly_tobytes_loop_start: + uzp1 v25.8h, v17.8h, v27.8h + uzp2 v31.8h, v17.8h, v27.8h + uzp1 v24.8h, v16.8h, v23.8h + uzp2 v6.8h, v16.8h, v23.8h + st3 { v2.8b, v3.8b, v4.8b }, [x0], #24 + shrn v3.8b, v25.8h, #0x8 + ldr q17, [x1], #0x20 + shrn v4.8b, v31.8h, #0x4 + xtn v21.8b, v6.8h + ldr q23, [x1, #0x10] + st3 { v28.8b, v29.8b, v30.8b }, [x0], #24 + shrn v29.8b, v24.8h, #0x8 + ldur q27, [x1, #-0x10] + xtn v20.8b, v31.8h + ldr q16, [x1], #0x20 + sli v29.8b, v21.8b, #0x4 + xtn v2.8b, v25.8h + sli v3.8b, v20.8b, #0x4 + xtn v28.8b, v24.8h + shrn v30.8b, v6.8h, #0x4 + subs x2, x2, #0x1 + cbnz x2, Lmlk_poly_tobytes_loop_start + uzp2 v7.8h, v17.8h, v27.8h + uzp1 v25.8h, v17.8h, v27.8h + uzp2 v0.8h, v16.8h, v23.8h + st3 { v2.8b, v3.8b, v4.8b }, [x0], #24 + st3 { v28.8b, v29.8b, v30.8b }, [x0], #24 + shrn v21.8b, v25.8h, #0x8 + uzp1 v2.8h, v16.8h, v23.8h + shrn v22.8b, v7.8h, #0x4 + shrn v4.8b, v0.8h, #0x4 + xtn v28.8b, v7.8h + xtn v27.8b, v0.8h + shrn v3.8b, v2.8h, #0x8 + sli v21.8b, v28.8b, #0x4 + xtn v2.8b, v2.8h + sli v3.8b, v27.8b, #0x4 + xtn v20.8b, v25.8h + st3 { v20.8b, v21.8b, v22.8b }, [x0], #24 + st3 { v2.8b, v3.8b, v4.8b }, [x0], #24 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_tobytes_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + (!MLK_CONFIG_NO_KEYPAIR_API || !MLK_CONFIG_NO_ENCAPS_API) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_tomont_aarch64_asm.S b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_tomont_aarch64_asm.S new file mode 100644 index 0000000000..9179e3ff94 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_tomont_aarch64_asm.S @@ -0,0 +1,102 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_tomont_aarch64_asm + Description: Convert polynomial to Montgomery domain + Signature: void mlk_poly_tomont_aarch64_asm(int16_t p[256]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON] + x0: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t p[256] + description: Input/output polynomial + Stack: + bytes: 0 +*/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + !defined(MLK_CONFIG_NO_KEYPAIR_API) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/mlkem_poly_tomont_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_tomont_aarch64_asm) +MLK_ASM_FN_SYMBOL(poly_tomont_aarch64_asm) + + .cfi_startproc + mov w2, #0xd01 // =3329 + dup v4.8h, w2 + mov w2, #-0x414 // =-1044 + dup v2.8h, w2 + mov w2, #-0x2824 // =-10276 + dup v3.8h, w2 + mov x1, #0x8 // =8 + ldr q18, [x0, #0x20] + ldr q0, [x0, #0x10] + ldr q16, [x0], #0x40 + sqrdmulh v23.8h, v0.8h, v3.8h + mul v26.8h, v0.8h, v2.8h + sqrdmulh v19.8h, v16.8h, v3.8h + mls v26.8h, v23.8h, v4.h[0] + mul v29.8h, v16.8h, v2.8h + ldur q16, [x0, #-0x10] + mls v29.8h, v19.8h, v4.h[0] + stur q26, [x0, #-0x30] + sqrdmulh v26.8h, v18.8h, v3.8h + mul v18.8h, v18.8h, v2.8h + stur q29, [x0, #-0x40] + sqrdmulh v29.8h, v16.8h, v3.8h + mls v18.8h, v26.8h, v4.h[0] + sub x1, x1, #0x1 + +Lmlk_poly_tomont_loop: + ldr q19, [x0, #0x10] + mul v26.8h, v16.8h, v2.8h + ldr q23, [x0, #0x20] + ldr q17, [x0], #0x40 + mls v26.8h, v29.8h, v4.h[0] + ldur q16, [x0, #-0x10] + sqrdmulh v28.8h, v19.8h, v3.8h + stur q18, [x0, #-0x60] + mul v0.8h, v19.8h, v2.8h + stur q26, [x0, #-0x50] + sqrdmulh v24.8h, v23.8h, v3.8h + mul v18.8h, v23.8h, v2.8h + sqrdmulh v22.8h, v17.8h, v3.8h + mul v26.8h, v17.8h, v2.8h + mls v0.8h, v28.8h, v4.h[0] + mls v26.8h, v22.8h, v4.h[0] + sqrdmulh v29.8h, v16.8h, v3.8h + stur q0, [x0, #-0x30] + mls v18.8h, v24.8h, v4.h[0] + stur q26, [x0, #-0x40] + sub x1, x1, #0x1 + cbnz x1, Lmlk_poly_tomont_loop + mul v16.8h, v16.8h, v2.8h + stur q18, [x0, #-0x20] + mls v16.8h, v29.8h, v4.h[0] + stur q16, [x0, #-0x10] + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_tomont_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + !MLK_CONFIG_NO_KEYPAIR_API */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.S b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.S new file mode 100644 index 0000000000..41bd1581ed --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.S @@ -0,0 +1,264 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [NeonNTT] + * Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1 + * Becker, Hwang, Kannwischer, Yang, Yang + * https://eprint.iacr.org/2021/986 + */ + +/*yaml + Name: polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm + Description: Re-implementation of asymmetric base multiplication following @[NeonNTT] for k=2 + Signature: void mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm(int16_t r[256], const int16_t a[512], const int16_t b[512], const int16_t b_cache[256]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON] + x0: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t r[256] + description: Output polynomial + x1: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int16_t a[512] + description: Input polynomial vector a + x2: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int16_t b[512] + description: Input polynomial vector b + x3: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t b_cache[256] + description: Cached values for b + Stack: + bytes: 64 + description: saving callee-saved Neon registers +*/ + +/* Re-implementation of asymmetric base multiplication following @[NeonNTT] */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/mlkem_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm) +MLK_ASM_FN_SYMBOL(polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x40 + .cfi_adjust_cfa_offset 0x40 + stp d8, d9, [sp] + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + stp d10, d11, [sp, #0x10] + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + stp d12, d13, [sp, #0x20] + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + stp d14, d15, [sp, #0x30] + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + mov w14, #0xd01 // =3329 + dup v0.8h, w14 + mov w14, #0xcff // =3327 + dup v2.8h, w14 + add x4, x1, #0x200 + add x5, x2, #0x200 + add x6, x3, #0x100 + mov x13, #0x10 // =16 + ldr q12, [x1], #0x20 + ldur q9, [x1, #-0x10] + ldr q22, [x2], #0x20 + ldur q30, [x2, #-0x10] + ldr q6, [x5], #0x20 + ldr q7, [x4, #0x10] + ldr q8, [x4], #0x20 + ldur q23, [x5, #-0x10] + uzp1 v16.8h, v12.8h, v9.8h + uzp2 v14.8h, v12.8h, v9.8h + uzp2 v13.8h, v22.8h, v30.8h + uzp1 v18.8h, v22.8h, v30.8h + ld1 { v27.8h }, [x3], #16 + ld1 { v17.8h }, [x6], #16 + smull2 v4.4s, v16.8h, v18.8h + ldr q31, [x1, #0x10] + smull v19.4s, v16.4h, v13.4h + ldr q24, [x1], #0x20 + smlal v19.4s, v14.4h, v18.4h + ldr q22, [x2], #0x20 + smlal2 v4.4s, v14.8h, v27.8h + uzp2 v5.8h, v6.8h, v23.8h + smull2 v29.4s, v16.8h, v13.8h + uzp2 v26.8h, v8.8h, v7.8h + smlal2 v29.4s, v14.8h, v18.8h + uzp1 v30.8h, v24.8h, v31.8h + uzp1 v8.8h, v8.8h, v7.8h + smull v11.4s, v16.4h, v18.4h + smlal v11.4s, v14.4h, v27.4h + ldur q1, [x2, #-0x10] + uzp1 v28.8h, v6.8h, v23.8h + smlal2 v29.4s, v8.8h, v5.8h + ldr q25, [x5], #0x20 + smlal v19.4s, v8.4h, v5.4h + ldr q3, [x4, #0x10] + smlal2 v29.4s, v26.8h, v28.8h + uzp1 v27.8h, v22.8h, v1.8h + smlal v19.4s, v26.4h, v28.4h + ldr q12, [x4], #0x20 + smlal2 v4.4s, v8.8h, v28.8h + ldur q21, [x5, #-0x10] + smlal2 v4.4s, v26.8h, v17.8h + smlal v11.4s, v8.4h, v28.4h + ld1 { v15.8h }, [x6], #16 + smlal v11.4s, v26.4h, v17.4h + ld1 { v20.8h }, [x3], #16 + uzp1 v28.8h, v19.8h, v29.8h + smull2 v23.4s, v30.8h, v27.8h + smull v26.4s, v30.4h, v27.4h + uzp2 v16.8h, v22.8h, v1.8h + mul v28.8h, v28.8h, v2.8h + uzp1 v10.8h, v11.8h, v4.8h + smull2 v8.4s, v30.8h, v16.8h + mul v13.8h, v10.8h, v2.8h + smlal v19.4s, v28.4h, v0.4h + smlal2 v29.4s, v28.8h, v0.8h + smull v18.4s, v30.4h, v16.4h + uzp1 v30.8h, v25.8h, v21.8h + smlal v11.4s, v13.4h, v0.4h + uzp2 v6.8h, v24.8h, v31.8h + uzp1 v16.8h, v12.8h, v3.8h + smlal2 v4.4s, v13.8h, v0.8h + uzp2 v17.8h, v25.8h, v21.8h + smlal2 v8.4s, v6.8h, v27.8h + uzp2 v12.8h, v12.8h, v3.8h + smlal v18.4s, v6.4h, v27.4h + uzp2 v9.8h, v19.8h, v29.8h + smlal2 v8.4s, v16.8h, v17.8h + smlal2 v8.4s, v12.8h, v30.8h + uzp2 v19.8h, v11.8h, v4.8h + sub x13, x13, #0x2 + +Lmlk_polyvec_basemul_acc_montgomery_cached_k2_loop_start: + smlal v18.4s, v16.4h, v17.4h + ldr q7, [x4], #0x20 + ldr q10, [x2, #0x10] + smlal v18.4s, v12.4h, v30.4h + smlal2 v23.4s, v6.8h, v20.8h + ldr q14, [x2], #0x20 + smlal2 v23.4s, v16.8h, v30.8h + zip1 v25.8h, v19.8h, v9.8h + zip2 v3.8h, v19.8h, v9.8h + smlal2 v23.4s, v12.8h, v15.8h + smlal v26.4s, v6.4h, v20.4h + uzp1 v5.8h, v18.8h, v8.8h + uzp2 v21.8h, v14.8h, v10.8h + smlal v26.4s, v16.4h, v30.4h + str q25, [x0], #0x20 + mul v29.8h, v5.8h, v2.8h + uzp1 v24.8h, v14.8h, v10.8h + stur q3, [x0, #-0x10] + smlal v26.4s, v12.4h, v15.4h + ld1 { v15.8h }, [x6], #16 + ldr q28, [x1, #0x10] + ldr q11, [x1], #0x20 + ldr q13, [x5], #0x20 + ldur q27, [x4, #-0x10] + smlal2 v8.4s, v29.8h, v0.8h + ldur q22, [x5, #-0x10] + smlal v18.4s, v29.4h, v0.4h + uzp1 v4.8h, v26.8h, v23.8h + uzp1 v1.8h, v11.8h, v28.8h + uzp2 v6.8h, v11.8h, v28.8h + uzp1 v16.8h, v7.8h, v27.8h + mul v31.8h, v4.8h, v2.8h + uzp2 v17.8h, v13.8h, v22.8h + ld1 { v20.8h }, [x3], #16 + uzp2 v9.8h, v18.8h, v8.8h + smull2 v8.4s, v1.8h, v21.8h + uzp1 v30.8h, v13.8h, v22.8h + smlal2 v8.4s, v6.8h, v24.8h + smlal2 v8.4s, v16.8h, v17.8h + uzp2 v12.8h, v7.8h, v27.8h + smlal v26.4s, v31.4h, v0.4h + smlal2 v23.4s, v31.8h, v0.8h + smull v18.4s, v1.4h, v21.4h + smlal v18.4s, v6.4h, v24.4h + smlal2 v8.4s, v12.8h, v30.8h + uzp2 v19.8h, v26.8h, v23.8h + smull2 v23.4s, v1.8h, v24.8h + smull v26.4s, v1.4h, v24.4h + subs x13, x13, #0x1 + cbnz x13, Lmlk_polyvec_basemul_acc_montgomery_cached_k2_loop_start + smlal v26.4s, v6.4h, v20.4h + smlal2 v23.4s, v6.8h, v20.8h + smlal v26.4s, v16.4h, v30.4h + smlal2 v23.4s, v16.8h, v30.8h + smlal v26.4s, v12.4h, v15.4h + smlal2 v23.4s, v12.8h, v15.8h + smlal v18.4s, v16.4h, v17.4h + smlal v18.4s, v12.4h, v30.4h + zip1 v12.8h, v19.8h, v9.8h + str q12, [x0], #0x20 + uzp1 v12.8h, v26.8h, v23.8h + mul v6.8h, v12.8h, v2.8h + uzp1 v12.8h, v18.8h, v8.8h + mul v12.8h, v12.8h, v2.8h + smlal v26.4s, v6.4h, v0.4h + smlal2 v23.4s, v6.8h, v0.8h + smlal2 v8.4s, v12.8h, v0.8h + smlal v18.4s, v12.4h, v0.4h + zip2 v12.8h, v19.8h, v9.8h + uzp2 v6.8h, v26.8h, v23.8h + stur q12, [x0, #-0x10] + uzp2 v12.8h, v18.8h, v8.8h + zip2 v1.8h, v6.8h, v12.8h + zip1 v12.8h, v6.8h, v12.8h + str q1, [x0, #0x10] + str q12, [x0], #0x20 + ldp d8, d9, [sp] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0x10] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0x20] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0x30] + .cfi_restore d14 + .cfi_restore d15 + add sp, sp, #0x40 + .cfi_adjust_cfa_offset -0x40 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm) + + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.S b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.S new file mode 100644 index 0000000000..7dcc8aead2 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.S @@ -0,0 +1,317 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [NeonNTT] + * Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1 + * Becker, Hwang, Kannwischer, Yang, Yang + * https://eprint.iacr.org/2021/986 + */ + +/*yaml + Name: polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm + Description: Re-implementation of asymmetric base multiplication following @[NeonNTT] for k=3 + Signature: void mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm(int16_t r[256], const int16_t a[768], const int16_t b[768], const int16_t b_cache[384]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON] + x0: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t r[256] + description: Output polynomial + x1: + type: buffer + size_bytes: 1536 + permissions: read-only + c_parameter: const int16_t a[768] + description: Input polynomial vector a + x2: + type: buffer + size_bytes: 1536 + permissions: read-only + c_parameter: const int16_t b[768] + description: Input polynomial vector b + x3: + type: buffer + size_bytes: 768 + permissions: read-only + c_parameter: const int16_t b_cache[384] + description: Cached values for b + Stack: + bytes: 64 + description: saving callee-saved Neon registers +*/ + +/* Re-implementation of asymmetric base multiplication following @[NeonNTT] */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/mlkem_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm) +MLK_ASM_FN_SYMBOL(polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x40 + .cfi_adjust_cfa_offset 0x40 + stp d8, d9, [sp] + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + stp d10, d11, [sp, #0x10] + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + stp d12, d13, [sp, #0x20] + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + stp d14, d15, [sp, #0x30] + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + mov w14, #0xd01 // =3329 + dup v0.8h, w14 + mov w14, #0xcff // =3327 + dup v2.8h, w14 + add x4, x1, #0x200 + add x5, x2, #0x200 + add x6, x3, #0x100 + add x7, x1, #0x400 + add x8, x2, #0x400 + add x9, x3, #0x200 + mov x13, #0x10 // =16 + ldr q6, [x7], #0x20 + ldr q19, [x2, #0x10] + ldr q23, [x1], #0x20 + ldur q14, [x1, #-0x10] + ldr q17, [x2], #0x20 + ldr q11, [x4, #0x10] + ldur q28, [x7, #-0x10] + ld1 { v30.8h }, [x3], #16 + ldr q26, [x4], #0x20 + ldr q16, [x8, #0x10] + uzp1 v8.8h, v23.8h, v14.8h + ldr q22, [x5, #0x10] + ldr q18, [x5], #0x20 + uzp1 v20.8h, v17.8h, v19.8h + uzp2 v24.8h, v23.8h, v14.8h + ldr q31, [x8], #0x20 + smull2 v4.4s, v8.8h, v20.8h + uzp1 v25.8h, v26.8h, v11.8h + smull v13.4s, v8.4h, v20.4h + ld1 { v23.8h }, [x6], #16 + uzp1 v1.8h, v18.8h, v22.8h + smlal v13.4s, v24.4h, v30.4h + smlal2 v4.4s, v24.8h, v30.8h + uzp2 v5.8h, v26.8h, v11.8h + smlal2 v4.4s, v25.8h, v1.8h + uzp1 v29.8h, v6.8h, v28.8h + smlal2 v4.4s, v5.8h, v23.8h + ld1 { v7.8h }, [x9], #16 + smlal v13.4s, v25.4h, v1.4h + uzp2 v17.8h, v17.8h, v19.8h + uzp1 v27.8h, v31.8h, v16.8h + smlal v13.4s, v5.4h, v23.4h + uzp2 v22.8h, v18.8h, v22.8h + smull v18.4s, v8.4h, v17.4h + uzp2 v28.8h, v6.8h, v28.8h + smlal v13.4s, v29.4h, v27.4h + smlal2 v4.4s, v29.8h, v27.8h + uzp2 v26.8h, v31.8h, v16.8h + smlal2 v4.4s, v28.8h, v7.8h + ldr q3, [x7, #0x10] + smlal v13.4s, v28.4h, v7.4h + ldr q7, [x1], #0x20 + smlal v18.4s, v24.4h, v20.4h + ldr q15, [x2], #0x20 + smlal v18.4s, v25.4h, v22.4h + smull2 v8.4s, v8.8h, v17.8h + ldur q17, [x1, #-0x10] + uzp1 v23.8h, v13.8h, v4.8h + smlal v18.4s, v5.4h, v1.4h + smlal2 v8.4s, v24.8h, v20.8h + ld1 { v16.8h }, [x3], #16 + mul v23.8h, v23.8h, v2.8h + ldr q19, [x5, #0x10] + ldr q14, [x4, #0x10] + ldr q11, [x4], #0x20 + ldur q20, [x2, #-0x10] + smlal2 v8.4s, v25.8h, v22.8h + smlal2 v8.4s, v5.8h, v1.8h + ldr q22, [x5], #0x20 + uzp1 v1.8h, v7.8h, v17.8h + smlal v18.4s, v29.4h, v26.4h + smlal v13.4s, v23.4h, v0.4h + uzp2 v31.8h, v11.8h, v14.8h + uzp1 v21.8h, v15.8h, v20.8h + smlal2 v4.4s, v23.8h, v0.8h + ld1 { v9.8h }, [x6], #16 + smlal v18.4s, v28.4h, v27.4h + smlal2 v8.4s, v29.8h, v26.8h + ldr q25, [x7], #0x20 + smull v26.4s, v1.4h, v21.4h + uzp1 v24.8h, v22.8h, v19.8h + smlal2 v8.4s, v28.8h, v27.8h + uzp2 v28.8h, v7.8h, v17.8h + uzp1 v29.8h, v11.8h, v14.8h + smull2 v23.4s, v1.8h, v21.8h + ldr q27, [x8], #0x20 + smlal2 v23.4s, v28.8h, v16.8h + ldur q11, [x8, #-0x10] + smlal2 v23.4s, v29.8h, v24.8h + uzp2 v7.8h, v13.8h, v4.8h + uzp2 v19.8h, v22.8h, v19.8h + ld1 { v4.8h }, [x9], #16 + smlal2 v23.4s, v31.8h, v9.8h + uzp1 v13.8h, v25.8h, v3.8h + uzp1 v14.8h, v18.8h, v8.8h + smlal v26.4s, v28.4h, v16.4h + uzp2 v17.8h, v27.8h, v11.8h + uzp2 v20.8h, v15.8h, v20.8h + mul v14.8h, v14.8h, v2.8h + sub x13, x13, #0x2 + +Lmlk_polyvec_basemul_acc_montgomery_cached_k3_loop_start: + uzp1 v6.8h, v27.8h, v11.8h + smlal v26.4s, v29.4h, v24.4h + uzp2 v16.8h, v25.8h, v3.8h + smlal v26.4s, v31.4h, v9.4h + ldr q3, [x7, #0x10] + smlal v26.4s, v13.4h, v6.4h + smlal2 v8.4s, v14.8h, v0.8h + ldr q27, [x8], #0x20 + smlal v18.4s, v14.4h, v0.4h + ldr q25, [x7], #0x20 + smlal2 v23.4s, v13.8h, v6.8h + ldr q11, [x1], #0x20 + smlal2 v23.4s, v16.8h, v4.8h + smlal v26.4s, v16.4h, v4.4h + ldur q22, [x1, #-0x10] + uzp2 v30.8h, v18.8h, v8.8h + smull v18.4s, v1.4h, v20.4h + smlal v18.4s, v28.4h, v21.4h + ldr q14, [x2], #0x20 + smlal v18.4s, v29.4h, v19.4h + zip1 v5.8h, v7.8h, v30.8h + uzp1 v4.8h, v26.8h, v23.8h + smull2 v8.4s, v1.8h, v20.8h + zip2 v10.8h, v7.8h, v30.8h + smlal v18.4s, v31.4h, v24.4h + mul v12.8h, v4.8h, v2.8h + ldr q4, [x5, #0x10] + ldr q20, [x4, #0x10] + ldr q1, [x4], #0x20 + ldur q30, [x2, #-0x10] + smlal2 v8.4s, v28.8h, v21.8h + smlal2 v8.4s, v29.8h, v19.8h + ldr q19, [x5], #0x20 + smlal2 v8.4s, v31.8h, v24.8h + ld1 { v15.8h }, [x3], #16 + uzp2 v31.8h, v1.8h, v20.8h + smlal v26.4s, v12.4h, v0.4h + smlal2 v23.4s, v12.8h, v0.8h + uzp1 v21.8h, v14.8h, v30.8h + uzp1 v29.8h, v1.8h, v20.8h + uzp1 v1.8h, v11.8h, v22.8h + smlal2 v8.4s, v13.8h, v17.8h + ld1 { v9.8h }, [x6], #16 + smlal v18.4s, v13.4h, v17.4h + uzp1 v24.8h, v19.8h, v4.8h + uzp2 v7.8h, v26.8h, v23.8h + smull v26.4s, v1.4h, v21.4h + smlal v18.4s, v16.4h, v6.4h + uzp2 v19.8h, v19.8h, v4.8h + smlal2 v8.4s, v16.8h, v6.8h + uzp2 v28.8h, v11.8h, v22.8h + smull2 v23.4s, v1.8h, v21.8h + uzp1 v13.8h, v25.8h, v3.8h + smlal2 v23.4s, v28.8h, v15.8h + ldur q11, [x8, #-0x10] + smlal2 v23.4s, v29.8h, v24.8h + ld1 { v4.8h }, [x9], #16 + smlal2 v23.4s, v31.8h, v9.8h + uzp1 v12.8h, v18.8h, v8.8h + uzp2 v20.8h, v14.8h, v30.8h + smlal v26.4s, v28.4h, v15.4h + str q5, [x0], #0x20 + mul v14.8h, v12.8h, v2.8h + stur q10, [x0, #-0x10] + uzp2 v17.8h, v27.8h, v11.8h + subs x13, x13, #0x1 + cbnz x13, Lmlk_polyvec_basemul_acc_montgomery_cached_k3_loop_start + uzp2 v3.8h, v25.8h, v3.8h + smull2 v16.4s, v1.8h, v20.8h + smull v25.4s, v1.4h, v20.4h + uzp1 v22.8h, v27.8h, v11.8h + smlal2 v16.4s, v28.8h, v21.8h + smlal v25.4s, v28.4h, v21.4h + smlal2 v16.4s, v29.8h, v19.8h + smlal v25.4s, v29.4h, v19.4h + smlal2 v16.4s, v31.8h, v24.8h + smlal v25.4s, v31.4h, v24.4h + smlal v25.4s, v13.4h, v17.4h + smlal2 v16.4s, v13.8h, v17.8h + smlal2 v16.4s, v3.8h, v22.8h + smlal v25.4s, v3.4h, v22.4h + smlal2 v23.4s, v13.8h, v22.8h + smlal v26.4s, v29.4h, v24.4h + smlal v26.4s, v31.4h, v9.4h + smlal v26.4s, v13.4h, v22.4h + uzp1 v10.8h, v25.8h, v16.8h + smlal2 v23.4s, v3.8h, v4.8h + smlal v26.4s, v3.4h, v4.4h + mul v13.8h, v10.8h, v2.8h + smlal v18.4s, v14.4h, v0.4h + smlal2 v8.4s, v14.8h, v0.8h + uzp1 v3.8h, v26.8h, v23.8h + mul v24.8h, v3.8h, v2.8h + uzp2 v17.8h, v18.8h, v8.8h + smlal v25.4s, v13.4h, v0.4h + smlal2 v16.4s, v13.8h, v0.8h + zip1 v21.8h, v7.8h, v17.8h + zip2 v20.8h, v7.8h, v17.8h + smlal2 v23.4s, v24.8h, v0.8h + str q21, [x0], #0x20 + smlal v26.4s, v24.4h, v0.4h + uzp2 v13.8h, v25.8h, v16.8h + stur q20, [x0, #-0x10] + uzp2 v23.8h, v26.8h, v23.8h + zip1 v18.8h, v23.8h, v13.8h + zip2 v13.8h, v23.8h, v13.8h + str q18, [x0], #0x20 + stur q13, [x0, #-0x10] + ldp d8, d9, [sp] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0x10] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0x20] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0x30] + .cfi_restore d14 + .cfi_restore d15 + add sp, sp, #0x40 + .cfi_adjust_cfa_offset -0x40 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.S b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.S new file mode 100644 index 0000000000..5d1ff00e94 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.S @@ -0,0 +1,371 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [NeonNTT] + * Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1 + * Becker, Hwang, Kannwischer, Yang, Yang + * https://eprint.iacr.org/2021/986 + */ + +/*yaml + Name: polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm + Description: Re-implementation of asymmetric base multiplication following @[NeonNTT] for k=4 + Signature: void mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm(int16_t r[256], const int16_t a[1024], const int16_t b[1024], const int16_t b_cache[512]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON] + x0: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t r[256] + description: Output polynomial + x1: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const int16_t a[1024] + description: Input polynomial vector a + x2: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const int16_t b[1024] + description: Input polynomial vector b + x3: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int16_t b_cache[512] + description: Cached values for b + Stack: + bytes: 64 + description: saving callee-saved Neon registers +*/ + +/* Re-implementation of asymmetric base multiplication following @[NeonNTT] */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/mlkem_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm) +MLK_ASM_FN_SYMBOL(polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x40 + .cfi_adjust_cfa_offset 0x40 + stp d8, d9, [sp] + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + stp d10, d11, [sp, #0x10] + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + stp d12, d13, [sp, #0x20] + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + stp d14, d15, [sp, #0x30] + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + mov w14, #0xd01 // =3329 + dup v0.8h, w14 + mov w14, #0xcff // =3327 + dup v2.8h, w14 + add x4, x1, #0x200 + add x5, x2, #0x200 + add x6, x3, #0x100 + add x7, x1, #0x400 + add x8, x2, #0x400 + add x9, x3, #0x200 + add x10, x1, #0x600 + add x11, x2, #0x600 + add x12, x3, #0x300 + mov x13, #0x10 // =16 + ldr q28, [x1], #0x20 + ldur q5, [x1, #-0x10] + ldr q31, [x2], #0x20 + ldur q27, [x2, #-0x10] + ldr q7, [x5], #0x20 + ldr q10, [x4], #0x20 + ldur q18, [x5, #-0x10] + ldur q9, [x4, #-0x10] + uzp1 v11.8h, v28.8h, v5.8h + uzp2 v19.8h, v28.8h, v5.8h + uzp2 v4.8h, v31.8h, v27.8h + uzp1 v1.8h, v31.8h, v27.8h + ldr q29, [x7], #0x20 + ldr q28, [x8, #0x10] + uzp1 v24.8h, v10.8h, v9.8h + uzp1 v17.8h, v7.8h, v18.8h + uzp2 v7.8h, v7.8h, v18.8h + ldr q21, [x8], #0x20 + uzp2 v27.8h, v10.8h, v9.8h + ldur q6, [x7, #-0x10] + smull v18.4s, v11.4h, v4.4h + ld1 { v9.8h }, [x3], #16 + smull2 v8.4s, v11.8h, v4.8h + ldr q16, [x11], #0x20 + smlal2 v8.4s, v19.8h, v1.8h + ldur q14, [x11, #-0x10] + smlal v18.4s, v19.4h, v1.4h + uzp1 v10.8h, v21.8h, v28.8h + smlal v18.4s, v24.4h, v7.4h + ldr q4, [x10], #0x20 + smlal2 v8.4s, v24.8h, v7.8h + ld1 { v12.8h }, [x6], #16 + smull2 v23.4s, v11.8h, v1.8h + uzp2 v13.8h, v29.8h, v6.8h + smull v26.4s, v11.4h, v1.4h + uzp1 v29.8h, v29.8h, v6.8h + smlal v26.4s, v19.4h, v9.4h + ldur q15, [x10, #-0x10] + smlal2 v23.4s, v19.8h, v9.8h + uzp2 v9.8h, v21.8h, v28.8h + smlal v18.4s, v27.4h, v17.4h + uzp2 v6.8h, v16.8h, v14.8h + uzp1 v21.8h, v16.8h, v14.8h + smlal2 v8.4s, v27.8h, v17.8h + smlal2 v8.4s, v29.8h, v9.8h + uzp1 v30.8h, v4.8h, v15.8h + uzp2 v16.8h, v4.8h, v15.8h + smlal v18.4s, v29.4h, v9.4h + smlal2 v8.4s, v13.8h, v10.8h + ld1 { v15.8h }, [x9], #16 + smlal v18.4s, v13.4h, v10.4h + ldr q11, [x4], #0x20 + smlal v18.4s, v30.4h, v6.4h + ldr q7, [x2], #0x20 + smlal2 v8.4s, v30.8h, v6.8h + ld1 { v9.8h }, [x12], #16 + smlal2 v23.4s, v24.8h, v17.8h + ldur q4, [x2, #-0x10] + smlal v26.4s, v24.4h, v17.4h + ldur q25, [x4, #-0x10] + smlal2 v8.4s, v16.8h, v21.8h + ldr q5, [x5], #0x20 + smlal v18.4s, v16.4h, v21.4h + ldur q22, [x5, #-0x10] + smlal v26.4s, v27.4h, v12.4h + ldr q19, [x1, #0x10] + smlal v26.4s, v29.4h, v10.4h + ld1 { v20.8h }, [x3], #16 + smlal v26.4s, v13.4h, v15.4h + uzp1 v24.8h, v7.8h, v4.8h + smlal2 v23.4s, v27.8h, v12.8h + uzp1 v28.8h, v18.8h, v8.8h + smlal v26.4s, v30.4h, v21.4h + uzp2 v27.8h, v11.8h, v25.8h + smlal2 v23.4s, v29.8h, v10.8h + uzp2 v31.8h, v7.8h, v4.8h + smlal2 v23.4s, v13.8h, v15.8h + uzp1 v14.8h, v5.8h, v22.8h + uzp1 v17.8h, v11.8h, v25.8h + smlal v26.4s, v16.4h, v9.4h + mul v29.8h, v28.8h, v2.8h + sub x13, x13, #0x2 + +Lmlk_polyvec_basemul_acc_montgomery_cached_k4_loop_start: + smlal2 v23.4s, v30.8h, v21.8h + ldr q11, [x1], #0x20 + uzp2 v15.8h, v5.8h, v22.8h + smlal v18.4s, v29.4h, v0.4h + ldr q12, [x7], #0x20 + smlal2 v8.4s, v29.8h, v0.8h + ldur q3, [x7, #-0x10] + ldr q21, [x8], #0x20 + uzp1 v29.8h, v11.8h, v19.8h + ldur q13, [x8, #-0x10] + uzp2 v5.8h, v11.8h, v19.8h + smlal2 v23.4s, v16.8h, v9.8h + uzp2 v28.8h, v18.8h, v8.8h + smull2 v8.4s, v29.8h, v31.8h + smlal2 v8.4s, v5.8h, v24.8h + uzp1 v7.8h, v12.8h, v3.8h + smlal2 v8.4s, v17.8h, v15.8h + uzp2 v11.8h, v21.8h, v13.8h + uzp1 v4.8h, v26.8h, v23.8h + smlal2 v8.4s, v27.8h, v14.8h + smlal2 v8.4s, v7.8h, v11.8h + mul v6.8h, v4.8h, v2.8h + ldr q19, [x11], #0x20 + uzp2 v25.8h, v12.8h, v3.8h + ldr q12, [x10], #0x20 + smull v18.4s, v29.4h, v31.4h + ldur q3, [x10, #-0x10] + smlal v18.4s, v5.4h, v24.4h + uzp1 v4.8h, v21.8h, v13.8h + smlal v18.4s, v17.4h, v15.4h + ldur q13, [x11, #-0x10] + ld1 { v1.8h }, [x6], #16 + smlal v26.4s, v6.4h, v0.4h + smlal2 v23.4s, v6.8h, v0.8h + ld1 { v10.8h }, [x9], #16 + smlal v18.4s, v27.4h, v14.4h + uzp1 v30.8h, v12.8h, v3.8h + smlal2 v8.4s, v25.8h, v4.8h + uzp2 v31.8h, v19.8h, v13.8h + smlal v18.4s, v7.4h, v11.4h + ld1 { v9.8h }, [x12], #16 + smlal v18.4s, v25.4h, v4.4h + uzp1 v21.8h, v19.8h, v13.8h + uzp2 v16.8h, v12.8h, v3.8h + smlal v18.4s, v30.4h, v31.4h + smlal2 v8.4s, v30.8h, v31.8h + uzp2 v31.8h, v26.8h, v23.8h + smlal2 v8.4s, v16.8h, v21.8h + smlal v18.4s, v16.4h, v21.4h + zip1 v15.8h, v31.8h, v28.8h + ldr q19, [x1, #0x10] + smull2 v23.4s, v29.8h, v24.8h + smull v26.4s, v29.4h, v24.4h + ldr q3, [x2, #0x10] + smlal v26.4s, v5.4h, v20.4h + ldr q11, [x2], #0x20 + uzp1 v6.8h, v18.8h, v8.8h + smlal v26.4s, v17.4h, v14.4h + smlal v26.4s, v27.4h, v1.4h + zip2 v13.8h, v31.8h, v28.8h + smlal v26.4s, v7.4h, v4.4h + str q15, [x0], #0x20 + smlal v26.4s, v25.4h, v10.4h + stur q13, [x0, #-0x10] + mul v29.8h, v6.8h, v2.8h + uzp1 v24.8h, v11.8h, v3.8h + uzp2 v31.8h, v11.8h, v3.8h + ldr q11, [x4], #0x20 + smlal2 v23.4s, v5.8h, v20.8h + ldur q28, [x4, #-0x10] + smlal2 v23.4s, v17.8h, v14.8h + ldr q5, [x5], #0x20 + smlal2 v23.4s, v27.8h, v1.8h + ldur q22, [x5, #-0x10] + smlal v26.4s, v30.4h, v21.4h + ld1 { v20.8h }, [x3], #16 + smlal v26.4s, v16.4h, v9.4h + uzp1 v17.8h, v11.8h, v28.8h + smlal2 v23.4s, v7.8h, v4.8h + uzp2 v27.8h, v11.8h, v28.8h + smlal2 v23.4s, v25.8h, v10.8h + uzp1 v14.8h, v5.8h, v22.8h + subs x13, x13, #0x1 + cbnz x13, Lmlk_polyvec_basemul_acc_montgomery_cached_k4_loop_start + smlal v18.4s, v29.4h, v0.4h + ldr q11, [x1], #0x20 + uzp2 v28.8h, v5.8h, v22.8h + smlal2 v23.4s, v30.8h, v21.8h + smlal2 v8.4s, v29.8h, v0.8h + ldr q15, [x8, #0x10] + smlal2 v23.4s, v16.8h, v9.8h + ldr q21, [x8], #0x20 + uzp1 v22.8h, v11.8h, v19.8h + uzp2 v12.8h, v11.8h, v19.8h + ldr q1, [x7, #0x10] + ld1 { v6.8h }, [x6], #16 + uzp2 v3.8h, v18.8h, v8.8h + smull v9.4s, v22.4h, v31.4h + smull2 v18.4s, v22.8h, v31.8h + ldr q16, [x7], #0x20 + smull v19.4s, v22.4h, v24.4h + uzp1 v30.8h, v21.8h, v15.8h + uzp2 v25.8h, v21.8h, v15.8h + smull2 v8.4s, v22.8h, v24.8h + smlal v19.4s, v12.4h, v20.4h + ldr q13, [x10, #0x10] + smlal2 v8.4s, v12.8h, v20.8h + uzp1 v29.8h, v16.8h, v1.8h + smlal2 v18.4s, v12.8h, v24.8h + ldr q5, [x10], #0x20 + smlal v9.4s, v12.4h, v24.4h + ldr q4, [x11], #0x20 + smlal v9.4s, v17.4h, v28.4h + ldur q22, [x11, #-0x10] + smlal2 v18.4s, v17.8h, v28.8h + uzp2 v16.8h, v16.8h, v1.8h + smlal v19.4s, v17.4h, v14.4h + ld1 { v28.8h }, [x9], #16 + smlal2 v8.4s, v17.8h, v14.8h + uzp1 v7.8h, v5.8h, v13.8h + smlal v9.4s, v27.4h, v14.4h + uzp1 v17.8h, v4.8h, v22.8h + smlal2 v18.4s, v27.8h, v14.8h + uzp2 v12.8h, v5.8h, v13.8h + uzp2 v21.8h, v4.8h, v22.8h + smlal v19.4s, v27.4h, v6.4h + smlal2 v8.4s, v27.8h, v6.8h + ld1 { v15.8h }, [x12], #16 + smlal v19.4s, v29.4h, v30.4h + uzp1 v20.8h, v26.8h, v23.8h + smlal v9.4s, v29.4h, v25.4h + smlal2 v18.4s, v29.8h, v25.8h + smlal2 v8.4s, v29.8h, v30.8h + smlal v19.4s, v16.4h, v28.4h + smlal2 v8.4s, v16.8h, v28.8h + smlal2 v18.4s, v16.8h, v30.8h + smlal v9.4s, v16.4h, v30.4h + smlal v9.4s, v7.4h, v21.4h + smlal2 v18.4s, v7.8h, v21.8h + smlal2 v8.4s, v7.8h, v17.8h + smlal v19.4s, v7.4h, v17.4h + smlal v19.4s, v12.4h, v15.4h + smlal2 v8.4s, v12.8h, v15.8h + smlal2 v18.4s, v12.8h, v17.8h + smlal v9.4s, v12.4h, v17.4h + mul v6.8h, v20.8h, v2.8h + uzp1 v4.8h, v19.8h, v8.8h + mul v17.8h, v4.8h, v2.8h + uzp1 v12.8h, v9.8h, v18.8h + smlal v26.4s, v6.4h, v0.4h + mul v21.8h, v12.8h, v2.8h + smlal2 v23.4s, v6.8h, v0.8h + smlal2 v8.4s, v17.8h, v0.8h + smlal v19.4s, v17.4h, v0.4h + smlal2 v18.4s, v21.8h, v0.8h + uzp2 v23.8h, v26.8h, v23.8h + smlal v9.4s, v21.4h, v0.4h + zip2 v12.8h, v23.8h, v3.8h + zip1 v22.8h, v23.8h, v3.8h + uzp2 v14.8h, v19.8h, v8.8h + uzp2 v18.8h, v9.8h, v18.8h + str q12, [x0, #0x10] + str q22, [x0], #0x20 + zip2 v24.8h, v14.8h, v18.8h + zip1 v21.8h, v14.8h, v18.8h + str q24, [x0, #0x10] + str q21, [x0], #0x20 + ldp d8, d9, [sp] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0x10] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0x20] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0x30] + .cfi_restore d14 + .cfi_restore d15 + add sp, sp, #0x40 + .cfi_adjust_cfa_offset -0x40 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_rej_uniform_aarch64_asm.S b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_rej_uniform_aarch64_asm.S new file mode 100644 index 0000000000..2cd1ca9fec --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_rej_uniform_aarch64_asm.S @@ -0,0 +1,226 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: rej_uniform_aarch64_asm + Description: Run rejection sampling on uniform random bytes to generate uniform random integers mod q + Signature: uint64_t mlk_rej_uniform_aarch64_asm(int16_t r[256], const uint8_t *buf, unsigned buflen, const uint8_t table[4096]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON] + x0: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t r[256] + description: Output buffer + x1: + type: buffer + size_bytes: x2 + permissions: read-only + c_parameter: const uint8_t *buf + description: Input buffer + x2: + type: scalar + c_parameter: unsigned buflen + description: Length of input buffer (must be multiple of 24) + test_with: 504 # MLKEM_GEN_MATRIX_NBLOCKS * MLK_XOF_RATE + x3: + type: buffer + size_bytes: 4096 + permissions: read-only + c_parameter: const uint8_t table[4096] + description: Lookup table + Stack: + bytes: 576 + description: register preservation and temporary storage +*/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/mlkem_rej_uniform_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(rej_uniform_aarch64_asm) +MLK_ASM_FN_SYMBOL(rej_uniform_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x240 + .cfi_adjust_cfa_offset 0x240 + mov x7, #0x1 // =1 + movk x7, #0x2, lsl #16 + movk x7, #0x4, lsl #32 + movk x7, #0x8, lsl #48 + mov v31.d[0], x7 + mov x7, #0x10 // =16 + movk x7, #0x20, lsl #16 + movk x7, #0x40, lsl #32 + movk x7, #0x80, lsl #48 + mov v31.d[1], x7 + mov w11, #0xd01 // =3329 + dup v30.8h, w11 + mov x8, sp + mov x7, x8 + mov x11, #0x0 // =0 + eor v16.16b, v16.16b, v16.16b + +Lmlk_rej_uniform_initial_zero: + str q16, [x7], #0x40 + stur q16, [x7, #-0x30] + stur q16, [x7, #-0x20] + stur q16, [x7, #-0x10] + add x11, x11, #0x20 + cmp x11, #0x100 + b.lt Lmlk_rej_uniform_initial_zero + mov x7, x8 + mov x9, #0x0 // =0 + mov x4, #0x100 // =256 + cmp x2, #0x30 + b.lo Lmlk_rej_uniform_loop48_end + +Lmlk_rej_uniform_loop48: + cmp x9, x4 + b.hs Lmlk_rej_uniform_memory_copy + sub x2, x2, #0x30 + ld3 { v0.16b, v1.16b, v2.16b }, [x1], #48 + zip1 v4.16b, v0.16b, v1.16b + zip2 v5.16b, v0.16b, v1.16b + zip1 v6.16b, v1.16b, v2.16b + zip2 v7.16b, v1.16b, v2.16b + bic v4.8h, #0xf0, lsl #8 + bic v5.8h, #0xf0, lsl #8 + ushr v6.8h, v6.8h, #0x4 + ushr v7.8h, v7.8h, #0x4 + zip1 v16.8h, v4.8h, v6.8h + zip2 v17.8h, v4.8h, v6.8h + zip1 v18.8h, v5.8h, v7.8h + zip2 v19.8h, v5.8h, v7.8h + cmhi v4.8h, v30.8h, v16.8h + cmhi v5.8h, v30.8h, v17.8h + cmhi v6.8h, v30.8h, v18.8h + cmhi v7.8h, v30.8h, v19.8h + and v4.16b, v4.16b, v31.16b + and v5.16b, v5.16b, v31.16b + and v6.16b, v6.16b, v31.16b + and v7.16b, v7.16b, v31.16b + uaddlv s20, v4.8h + uaddlv s21, v5.8h + uaddlv s22, v6.8h + uaddlv s23, v7.8h + fmov w12, s20 + fmov w13, s21 + fmov w14, s22 + fmov w15, s23 + ldr q24, [x3, x12, lsl #4] + ldr q25, [x3, x13, lsl #4] + ldr q26, [x3, x14, lsl #4] + ldr q27, [x3, x15, lsl #4] + cnt v4.16b, v4.16b + cnt v5.16b, v5.16b + cnt v6.16b, v6.16b + cnt v7.16b, v7.16b + uaddlv s20, v4.8h + uaddlv s21, v5.8h + uaddlv s22, v6.8h + uaddlv s23, v7.8h + fmov w12, s20 + fmov w13, s21 + fmov w14, s22 + fmov w15, s23 + tbl v16.16b, { v16.16b }, v24.16b + tbl v17.16b, { v17.16b }, v25.16b + tbl v18.16b, { v18.16b }, v26.16b + tbl v19.16b, { v19.16b }, v27.16b + st1 { v16.8h }, [x7] + add x7, x7, x12, lsl #1 + st1 { v17.8h }, [x7] + add x7, x7, x13, lsl #1 + st1 { v18.8h }, [x7] + add x7, x7, x14, lsl #1 + st1 { v19.8h }, [x7] + add x7, x7, x15, lsl #1 + add x12, x12, x13 + add x14, x14, x15 + add x9, x9, x12 + add x9, x9, x14 + cmp x2, #0x30 + b.hs Lmlk_rej_uniform_loop48 + +Lmlk_rej_uniform_loop48_end: + cmp x9, x4 + b.hs Lmlk_rej_uniform_memory_copy + cmp x2, #0x18 + b.lo Lmlk_rej_uniform_memory_copy + ld3 { v0.8b, v1.8b, v2.8b }, [x1], #24 + zip1 v4.16b, v0.16b, v1.16b + zip1 v5.16b, v1.16b, v2.16b + bic v4.8h, #0xf0, lsl #8 + ushr v5.8h, v5.8h, #0x4 + zip1 v16.8h, v4.8h, v5.8h + zip2 v17.8h, v4.8h, v5.8h + cmhi v4.8h, v30.8h, v16.8h + cmhi v5.8h, v30.8h, v17.8h + and v4.16b, v4.16b, v31.16b + and v5.16b, v5.16b, v31.16b + uaddlv s20, v4.8h + uaddlv s21, v5.8h + fmov w12, s20 + fmov w13, s21 + ldr q24, [x3, x12, lsl #4] + ldr q25, [x3, x13, lsl #4] + cnt v4.16b, v4.16b + cnt v5.16b, v5.16b + uaddlv s20, v4.8h + uaddlv s21, v5.8h + fmov w12, s20 + fmov w13, s21 + tbl v16.16b, { v16.16b }, v24.16b + tbl v17.16b, { v17.16b }, v25.16b + st1 { v16.8h }, [x7] + add x7, x7, x12, lsl #1 + st1 { v17.8h }, [x7] + add x9, x9, x12 + add x9, x9, x13 + +Lmlk_rej_uniform_memory_copy: + cmp x9, x4 + csel x9, x9, x4, lo + mov x11, #0x0 // =0 + mov x7, x8 + +Lmlk_rej_uniform_final_copy: + ldr q16, [x7], #0x40 + ldur q17, [x7, #-0x30] + ldur q18, [x7, #-0x20] + ldur q19, [x7, #-0x10] + str q16, [x0], #0x40 + stur q17, [x0, #-0x30] + stur q18, [x0, #-0x20] + stur q19, [x0, #-0x10] + add x11, x11, #0x20 + cmp x11, #0x100 + b.lt Lmlk_rej_uniform_final_copy + mov x0, x9 + +Lmlk_rej_uniform_return: + add sp, sp, #0x240 + .cfi_adjust_cfa_offset -0x240 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(rej_uniform_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/rej_uniform_table.c b/src/mlkem-native/mlkem/src/native/aarch64/src/rej_uniform_table.c new file mode 100644 index 0000000000..e1f48dc77d --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/rej_uniform_table.c @@ -0,0 +1,543 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_AARCH64) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "arith_native_aarch64.h" + +/* + * Lookup table used by rejection sampling of the public matrix. + * See autogen for details. + */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_rej_uniform_table[4096] = { + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 0 */, + 0, 1, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 1 */, + 2, 3, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 2 */, + 0, 1, 2, 3, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 3 */, + 4, 5, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 4 */, + 0, 1, 4, 5, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 5 */, + 2, 3, 4, 5, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 6 */, + 0, 1, 2, 3, 4, 5, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 7 */, + 6, 7, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 8 */, + 0, 1, 6, 7, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 9 */, + 2, 3, 6, 7, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 10 */, + 0, 1, 2, 3, 6, 7, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 11 */, + 4, 5, 6, 7, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 12 */, + 0, 1, 4, 5, 6, 7, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 13 */, + 2, 3, 4, 5, 6, 7, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 14 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 255, 255, 255, 255, 255, 255, 255, 255 /* 15 */, + 8, 9, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 16 */, + 0, 1, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 17 */, + 2, 3, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 18 */, + 0, 1, 2, 3, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 19 */, + 4, 5, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 20 */, + 0, 1, 4, 5, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 21 */, + 2, 3, 4, 5, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 22 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 255, 255, 255, 255, 255, 255, 255, 255 /* 23 */, + 6, 7, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 24 */, + 0, 1, 6, 7, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 25 */, + 2, 3, 6, 7, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 26 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 255, 255, 255, 255, 255, 255, 255, 255 /* 27 */, + 4, 5, 6, 7, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 28 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 255, 255, 255, 255, 255, 255, 255, 255 /* 29 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 255, 255, 255, 255, 255, 255, 255, 255 /* 30 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 255, 255, 255, 255, 255, 255 /* 31 */, + 10, 11, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 32 */, + 0, 1, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 33 */, + 2, 3, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 34 */, + 0, 1, 2, 3, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 35 */, + 4, 5, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 36 */, + 0, 1, 4, 5, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 37 */, + 2, 3, 4, 5, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 38 */, + 0, 1, 2, 3, 4, 5, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 39 */, + 6, 7, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 40 */, + 0, 1, 6, 7, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 41 */, + 2, 3, 6, 7, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 42 */, + 0, 1, 2, 3, 6, 7, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 43 */, + 4, 5, 6, 7, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 44 */, + 0, 1, 4, 5, 6, 7, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 45 */, + 2, 3, 4, 5, 6, 7, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 46 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 255, 255, 255, 255, 255, 255 /* 47 */, + 8, 9, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 48 */, + 0, 1, 8, 9, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 49 */, + 2, 3, 8, 9, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 50 */, + 0, 1, 2, 3, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 51 */, + 4, 5, 8, 9, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 52 */, + 0, 1, 4, 5, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 53 */, + 2, 3, 4, 5, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 54 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 255, 255, 255, 255, 255, 255 /* 55 */, + 6, 7, 8, 9, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 56 */, + 0, 1, 6, 7, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 57 */, + 2, 3, 6, 7, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 58 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 255, 255, 255, 255, 255, 255 /* 59 */, + 4, 5, 6, 7, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 60 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 255, 255, 255, 255, 255, 255 /* 61 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 255, 255, 255, 255, 255, 255 /* 62 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 255, 255, 255, 255 /* 63 */, + 12, 13, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 64 */, + 0, 1, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 65 */, + 2, 3, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 66 */, + 0, 1, 2, 3, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 67 */, + 4, 5, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 68 */, + 0, 1, 4, 5, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 69 */, + 2, 3, 4, 5, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 70 */, + 0, 1, 2, 3, 4, 5, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 71 */, + 6, 7, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 72 */, + 0, 1, 6, 7, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 73 */, + 2, 3, 6, 7, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 74 */, + 0, 1, 2, 3, 6, 7, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 75 */, + 4, 5, 6, 7, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 76 */, + 0, 1, 4, 5, 6, 7, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 77 */, + 2, 3, 4, 5, 6, 7, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 78 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 12, 13, 255, 255, 255, 255, 255, 255 /* 79 */, + 8, 9, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 80 */, + 0, 1, 8, 9, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 81 */, + 2, 3, 8, 9, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 82 */, + 0, 1, 2, 3, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 83 */, + 4, 5, 8, 9, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 84 */, + 0, 1, 4, 5, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 85 */, + 2, 3, 4, 5, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 86 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 12, 13, 255, 255, 255, 255, 255, 255 /* 87 */, + 6, 7, 8, 9, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 88 */, + 0, 1, 6, 7, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 89 */, + 2, 3, 6, 7, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 90 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 12, 13, 255, 255, 255, 255, 255, 255 /* 91 */, + 4, 5, 6, 7, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 92 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 12, 13, 255, 255, 255, 255, 255, 255 /* 93 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 12, 13, 255, 255, 255, 255, 255, 255 /* 94 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 12, 13, 255, 255, 255, 255 /* 95 */, + 10, 11, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 96 */, + 0, 1, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 97 */, + 2, 3, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 98 */, + 0, 1, 2, 3, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 99 */, + 4, 5, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 100 */, + 0, 1, 4, 5, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 101 */, + 2, 3, 4, 5, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 102 */, + 0, 1, 2, 3, 4, 5, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 103 */, + 6, 7, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 104 */, + 0, 1, 6, 7, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 105 */, + 2, 3, 6, 7, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 106 */, + 0, 1, 2, 3, 6, 7, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 107 */, + 4, 5, 6, 7, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 108 */, + 0, 1, 4, 5, 6, 7, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 109 */, + 2, 3, 4, 5, 6, 7, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 110 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 12, 13, 255, 255, 255, 255 /* 111 */, + 8, 9, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 112 */, + 0, 1, 8, 9, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 113 */, + 2, 3, 8, 9, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 114 */, + 0, 1, 2, 3, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 115 */, + 4, 5, 8, 9, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 116 */, + 0, 1, 4, 5, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 117 */, + 2, 3, 4, 5, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 118 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 12, 13, 255, 255, 255, 255 /* 119 */, + 6, 7, 8, 9, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 120 */, + 0, 1, 6, 7, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 121 */, + 2, 3, 6, 7, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 122 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 12, 13, 255, 255, 255, 255 /* 123 */, + 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 124 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 255, 255, 255, 255 /* 125 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 255, 255, 255, 255 /* 126 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 255, 255 /* 127 */, + 14, 15, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 128 */, + 0, 1, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 129 */, + 2, 3, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 130 */, + 0, 1, 2, 3, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 131 */, + 4, 5, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 132 */, + 0, 1, 4, 5, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 133 */, + 2, 3, 4, 5, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 134 */, + 0, 1, 2, 3, 4, 5, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 135 */, + 6, 7, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 136 */, + 0, 1, 6, 7, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 137 */, + 2, 3, 6, 7, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 138 */, + 0, 1, 2, 3, 6, 7, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 139 */, + 4, 5, 6, 7, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 140 */, + 0, 1, 4, 5, 6, 7, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 141 */, + 2, 3, 4, 5, 6, 7, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 142 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 14, 15, 255, 255, 255, 255, 255, 255 /* 143 */, + 8, 9, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 144 */, + 0, 1, 8, 9, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 145 */, + 2, 3, 8, 9, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 146 */, + 0, 1, 2, 3, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 147 */, + 4, 5, 8, 9, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 148 */, + 0, 1, 4, 5, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 149 */, + 2, 3, 4, 5, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 150 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 14, 15, 255, 255, 255, 255, 255, 255 /* 151 */, + 6, 7, 8, 9, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 152 */, + 0, 1, 6, 7, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 153 */, + 2, 3, 6, 7, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 154 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 14, 15, 255, 255, 255, 255, 255, 255 /* 155 */, + 4, 5, 6, 7, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 156 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 14, 15, 255, 255, 255, 255, 255, 255 /* 157 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 14, 15, 255, 255, 255, 255, 255, 255 /* 158 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 14, 15, 255, 255, 255, 255 /* 159 */, + 10, 11, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 160 */, + 0, 1, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 161 */, + 2, 3, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 162 */, + 0, 1, 2, 3, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 163 */, + 4, 5, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 164 */, + 0, 1, 4, 5, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 165 */, + 2, 3, 4, 5, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 166 */, + 0, 1, 2, 3, 4, 5, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 167 */, + 6, 7, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 168 */, + 0, 1, 6, 7, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 169 */, + 2, 3, 6, 7, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 170 */, + 0, 1, 2, 3, 6, 7, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 171 */, + 4, 5, 6, 7, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 172 */, + 0, 1, 4, 5, 6, 7, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 173 */, + 2, 3, 4, 5, 6, 7, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 174 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 14, 15, 255, 255, 255, 255 /* 175 */, + 8, 9, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 176 */, + 0, 1, 8, 9, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 177 */, + 2, 3, 8, 9, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 178 */, + 0, 1, 2, 3, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 179 */, + 4, 5, 8, 9, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 180 */, + 0, 1, 4, 5, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 181 */, + 2, 3, 4, 5, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 182 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 14, 15, 255, 255, 255, 255 /* 183 */, + 6, 7, 8, 9, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 184 */, + 0, 1, 6, 7, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 185 */, + 2, 3, 6, 7, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 186 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 14, 15, 255, 255, 255, 255 /* 187 */, + 4, 5, 6, 7, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 188 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 14, 15, 255, 255, 255, 255 /* 189 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 14, 15, 255, 255, 255, 255 /* 190 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 14, 15, 255, 255 /* 191 */, + 12, 13, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 192 */, + 0, 1, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 193 */, + 2, 3, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 194 */, + 0, 1, 2, 3, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 195 */, + 4, 5, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 196 */, + 0, 1, 4, 5, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 197 */, + 2, 3, 4, 5, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 198 */, + 0, 1, 2, 3, 4, 5, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 199 */, + 6, 7, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 200 */, + 0, 1, 6, 7, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 201 */, + 2, 3, 6, 7, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 202 */, + 0, 1, 2, 3, 6, 7, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 203 */, + 4, 5, 6, 7, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 204 */, + 0, 1, 4, 5, 6, 7, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 205 */, + 2, 3, 4, 5, 6, 7, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 206 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 12, 13, 14, 15, 255, 255, 255, 255 /* 207 */, + 8, 9, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 208 */, + 0, 1, 8, 9, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 209 */, + 2, 3, 8, 9, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 210 */, + 0, 1, 2, 3, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 211 */, + 4, 5, 8, 9, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 212 */, + 0, 1, 4, 5, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 213 */, + 2, 3, 4, 5, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 214 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 12, 13, 14, 15, 255, 255, 255, 255 /* 215 */, + 6, 7, 8, 9, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 216 */, + 0, 1, 6, 7, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 217 */, + 2, 3, 6, 7, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 218 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 12, 13, 14, 15, 255, 255, 255, 255 /* 219 */, + 4, 5, 6, 7, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 220 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 12, 13, 14, 15, 255, 255, 255, 255 /* 221 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 12, 13, 14, 15, 255, 255, 255, 255 /* 222 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 12, 13, 14, 15, 255, 255 /* 223 */, + 10, 11, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 224 */, + 0, 1, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 225 */, + 2, 3, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 226 */, + 0, 1, 2, 3, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 227 */, + 4, 5, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 228 */, + 0, 1, 4, 5, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 229 */, + 2, 3, 4, 5, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 230 */, + 0, 1, 2, 3, 4, 5, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 231 */, + 6, 7, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 232 */, + 0, 1, 6, 7, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 233 */, + 2, 3, 6, 7, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 234 */, + 0, 1, 2, 3, 6, 7, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 235 */, + 4, 5, 6, 7, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 236 */, + 0, 1, 4, 5, 6, 7, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 237 */, + 2, 3, 4, 5, 6, 7, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 238 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 12, 13, 14, 15, 255, 255 /* 239 */, + 8, 9, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 240 */, + 0, 1, 8, 9, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 241 */, + 2, 3, 8, 9, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 242 */, + 0, 1, 2, 3, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 243 */, + 4, 5, 8, 9, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 244 */, + 0, 1, 4, 5, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 245 */, + 2, 3, 4, 5, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 246 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 12, 13, 14, 15, 255, 255 /* 247 */, + 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 248 */, + 0, 1, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 249 */, + 2, 3, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 250 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 255, 255 /* 251 */, + 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 252 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 255, 255 /* 253 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 255, 255 /* 254 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15 /* 255 */, +}; + +#else /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(aarch64_rej_uniform_table) + +#endif /* !(MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/src/mlkem-native/mlkem/src/native/api.h b/src/mlkem-native/mlkem/src/native/api.h new file mode 100644 index 0000000000..f8578763b6 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/api.h @@ -0,0 +1,651 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_NATIVE_API_H +#define MLK_NATIVE_API_H +/* + * Native arithmetic interface + * + * This header is primarily for documentation purposes. + * It should not be included by backend implementations. + * + * To ensure consistency with backends, the header will be + * included automatically after inclusion of the active + * backend, to ensure consistency of function signatures, + * and run sanity checks. + */ + +#include "../cbmc.h" +#include "../common.h" + +/* Backends must return MLK_NATIVE_FUNC_SUCCESS upon success. */ +#define MLK_NATIVE_FUNC_SUCCESS (0) +/* Backends may return MLK_NATIVE_FUNC_FALLBACK to signal to the frontend that + * the target/parameters are unsupported; typically, this would be because of + * dependencies on CPU features not detected on the host CPU. In this case, + * the frontend falls back to the default C implementation. */ +#define MLK_NATIVE_FUNC_FALLBACK (-1) + + +/* Absolute exclusive upper bound for the output of the inverse NTT + * + * NOTE: This is the same bound as in poly.h and has to be kept + * in sync. */ +#define MLK_INVNTT_BOUND (8 * MLKEM_Q) + +/* Absolute exclusive upper bound for the output of the forward NTT + * + * NOTE: This is the same bound as in poly.h and has to be kept + * in sync. */ +#define MLK_NTT_BOUND (8 * MLKEM_Q) + +/* + * This is the C<->native interface allowing for the drop-in of + * native code for performance critical arithmetic components of ML-KEM. + * + * A _backend_ is a specific implementation of (part of) this interface. + * + * To add a function to a backend, define MLK_USE_NATIVE_XXX and + * implement `static inline xxx(...)` in the profile header. + * + * The only exception is MLK_USE_NATIVE_NTT_CUSTOM_ORDER. This option can + * be set if there are native implementations for all of NTT, invNTT, and + * base multiplication, and allows the native implementation to use a + * custom order of polynomial coefficients in NTT domain -- the use of such + * custom order is not an implementation-detail since the public matrix + * is generated in NTT domain. In this case, a permutation function + * mlk_poly_permute_bitrev_to_custom() needs to be provided that permutes + * polynomials in NTT domain from bitreversed to the custom order. + */ + +/* + * Those functions are meant to be trivial wrappers around the chosen native + * implementation. The are static inline to avoid unnecessary calls. + * The macro before each declaration controls whether a native + * implementation is present. + */ + +#if defined(MLK_USE_NATIVE_NTT) +/** + * Compute the negacyclic number-theoretic transform (NTT) of a polynomial + * in place. + * + * The input polynomial is assumed to be in normal order. The output + * polynomial is in bitreversed order, or of a custom order if + * MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set. See the documentation of + * MLK_USE_NATIVE_NTT_CUSTOM_ORDER for more information. + * + * @param[in,out] p Input/output polynomial. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_ntt_native(int16_t p[MLKEM_N]) +__contract__( + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + requires(array_abs_bound(p, 0, MLKEM_N, MLKEM_Q)) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_abs_bound(p, 0, MLKEM_N, MLK_NTT_BOUND)) + ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_abs_bound(p, 0, MLKEM_N, MLKEM_Q)) + ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged(p, MLKEM_N)) +); +#endif /* MLK_USE_NATIVE_NTT */ + +#if defined(MLK_USE_NATIVE_NTT_CUSTOM_ORDER) +/* + * This must only be set if NTT, invNTT, basemul, mulcache, and + * to/from byte stream conversions all have native implementations + * that are adapted to the custom order. + */ +#if !defined(MLK_USE_NATIVE_NTT) || !defined(MLK_USE_NATIVE_INTT) || \ + !defined(MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE) || \ + !defined(MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED) || \ + !defined(MLK_USE_NATIVE_POLY_TOBYTES) || \ + !defined(MLK_USE_NATIVE_POLY_FROMBYTES) +#error \ + "Invalid native profile: MLK_USE_NATIVE_NTT_CUSTOM_ORDER can only be \ +set if there are native implementations for NTT, invNTT, mulcache, basemul, \ +and to/from bytes conversions." +#endif /* !MLK_USE_NATIVE_NTT || !MLK_USE_NATIVE_INTT || \ + !MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE || \ + !MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED || \ + !MLK_USE_NATIVE_POLY_TOBYTES || !MLK_USE_NATIVE_POLY_FROMBYTES */ + +/** + * When MLK_USE_NATIVE_NTT_CUSTOM_ORDER is defined, convert a polynomial in + * NTT domain from bitreversed order to the custom order output by the + * native NTT. + * + * This must only be defined if there is native code for all of (a) NTT, + * (b) invNTT, (c) basemul, (d) mulcache. + * + * @param[in,out] p Input/output polynomial. + */ +static MLK_INLINE void mlk_poly_permute_bitrev_to_custom(int16_t p[MLKEM_N]) +__contract__( + /* We don't specify that this should be a permutation, but only + * that it does not change the bound established at the end of mlk_gen_matrix. */ + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(p, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + ensures(array_bound(p, 0, MLKEM_N, 0, MLKEM_Q))); +#endif /* MLK_USE_NATIVE_NTT_CUSTOM_ORDER */ + +#if defined(MLK_USE_NATIVE_INTT) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API)) +/** + * Compute the inverse negacyclic number-theoretic transform (NTT) of a + * polynomial in place. + * + * The input polynomial is in bitreversed order, or of a custom order if + * MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set. See the documentation of + * MLK_USE_NATIVE_NTT_CUSTOM_ORDER for more information. The output + * polynomial is assumed to be in normal order. + * + * @param[in,out] p Input/output polynomial. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_intt_native(int16_t p[MLKEM_N]) +__contract__( + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_abs_bound(p, 0, MLKEM_N, MLK_INVNTT_BOUND)) + ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged(p, MLKEM_N)) +); +#endif /* MLK_USE_NATIVE_INTT && (!MLK_CONFIG_NO_ENCAPS_API || \ + !MLK_CONFIG_NO_DECAPS_API) */ + +#if defined(MLK_USE_NATIVE_POLY_REDUCE) +/** + * Apply modular reduction to all coefficients of a polynomial, mapping them + * to unsigned canonical representatives in [0,..,MLKEM_Q-1]. + * + * @param[in,out] p Input/output polynomial. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_reduce_native(int16_t p[MLKEM_N]) +__contract__( + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_bound(p, 0, MLKEM_N, 0, MLKEM_Q)) + ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged(p, MLKEM_N)) +); +#endif /* MLK_USE_NATIVE_POLY_REDUCE */ + +#if defined(MLK_USE_NATIVE_POLY_TOMONT) && !defined(MLK_CONFIG_NO_KEYPAIR_API) +/** + * In-place conversion of all coefficients of a polynomial from the normal + * domain to the Montgomery domain. + * + * @param[in,out] p Input/output polynomial. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_tomont_native(int16_t p[MLKEM_N]) +__contract__( + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_abs_bound(p, 0, MLKEM_N, MLKEM_Q)) + ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged(p, MLKEM_N)) +); +#endif /* MLK_USE_NATIVE_POLY_TOMONT && !MLK_CONFIG_NO_KEYPAIR_API */ + +#if defined(MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE) +/** + * Compute multiplication cache for a polynomial in NTT domain. + * + * The purpose of the multiplication cache is to cache repeated computations + * required during a base multiplication of polynomials in NTT domain. The + * structure of the multiplication-cache is implementation defined. + * + * @param[out] cache Multiplication cache. + * @param[in] mlk_poly Input polynomial. Must be in NTT domain and in + * bitreversed order, or of a custom order if + * MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set. See the + * documentation of MLK_USE_NATIVE_NTT_CUSTOM_ORDER for + * more information. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_mulcache_compute_native( + int16_t cache[MLKEM_N / 2], const int16_t mlk_poly[MLKEM_N]) +__contract__( + requires(memory_no_alias(cache, sizeof(int16_t) * (MLKEM_N / 2))) + requires(memory_no_alias(mlk_poly, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(cache, sizeof(int16_t) * (MLKEM_N / 2))) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_abs_bound(cache, 0, MLKEM_N/2, MLKEM_Q)) +); +#endif /* MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE */ + +#if defined(MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED) +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 +/** + * Compute scalar product of length-2 polynomial vectors in NTT domain. + * + * @param[out] r Result of the scalar product. Again in NTT domain, of + * the same ordering as @p a and @p b. + * @param[in] a First polynomial vector operand. Must be in NTT + * domain and in bitreversed order, or of a custom order + * if MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set. See the + * documentation of MLK_USE_NATIVE_NTT_CUSTOM_ORDER for + * more information. + * @param[in] b Second polynomial vector operand. As for @p a. + * @param[in] b_cache Multiplication-cache for @p b. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k2_native( + int16_t r[MLKEM_N], const int16_t a[2 * MLKEM_N], + const int16_t b[2 * MLKEM_N], const int16_t b_cache[2 * (MLKEM_N / 2)]) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 2 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 2 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 2 * (MLKEM_N / 2))) + requires(array_bound(a, 0, 2 * MLKEM_N, 0, MLKEM_UINT12_LIMIT)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) +); +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3 +/** + * Compute scalar product of length-3 polynomial vectors in NTT domain. + * + * @param[out] r Result of the scalar product. Again in NTT domain, of + * the same ordering as @p a and @p b. + * @param[in] a First polynomial vector operand. Must be in NTT + * domain and in bitreversed order, or of a custom order + * if MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set. See the + * documentation of MLK_USE_NATIVE_NTT_CUSTOM_ORDER for + * more information. + * @param[in] b Second polynomial vector operand. As for @p a. + * @param[in] b_cache Multiplication-cache for @p b. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k3_native( + int16_t r[MLKEM_N], const int16_t a[3 * MLKEM_N], + const int16_t b[3 * MLKEM_N], const int16_t b_cache[3 * (MLKEM_N / 2)]) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 3 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 3 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 3 * (MLKEM_N / 2))) + requires(array_bound(a, 0, 3 * MLKEM_N, 0, MLKEM_UINT12_LIMIT)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) +); +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4 +/** + * Compute scalar product of length-4 polynomial vectors in NTT domain. + * + * @param[out] r Result of the scalar product. Again in NTT domain, of + * the same ordering as @p a and @p b. + * @param[in] a First polynomial vector operand. Must be in NTT + * domain and in bitreversed order, or of a custom order + * if MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set. See the + * documentation of MLK_USE_NATIVE_NTT_CUSTOM_ORDER for + * more information. + * @param[in] b Second polynomial vector operand. As for @p a. + * @param[in] b_cache Multiplication-cache for @p b. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k4_native( + int16_t r[MLKEM_N], const int16_t a[4 * MLKEM_N], + const int16_t b[4 * MLKEM_N], const int16_t b_cache[4 * (MLKEM_N / 2)]) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 4 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 4 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 4 * (MLKEM_N / 2))) + requires(array_bound(a, 0, 4 * MLKEM_N, 0, MLKEM_UINT12_LIMIT)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) +); +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */ +#endif /* MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED */ + +#if defined(MLK_USE_NATIVE_POLY_TOBYTES) && \ + (!defined(MLK_CONFIG_NO_KEYPAIR_API) || \ + !defined(MLK_CONFIG_NO_ENCAPS_API)) +/** + * Serialization of a polynomial with unsigned canonical coefficients. + * + * @spec{Implements ByteEncode_12 from @[FIPS203, Algorithm 5].} + * + * @param[out] r Output byte array (of MLKEM_POLYBYTES bytes). + * @param[in] a Input polynomial, with each coefficient in [0,..,MLKEM_Q-1]. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_tobytes_native(uint8_t r[MLKEM_POLYBYTES], + const int16_t a[MLKEM_N]) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYBYTES)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYBYTES)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK) +); +#endif /* MLK_USE_NATIVE_POLY_TOBYTES && (!MLK_CONFIG_NO_KEYPAIR_API || \ + !MLK_CONFIG_NO_ENCAPS_API) */ + +#if defined(MLK_USE_NATIVE_POLY_FROMBYTES) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API)) +/** + * Deserialization of a polynomial. + * + * @spec{Implements ByteDecode_12 from @[FIPS203, Algorithm 6].} + * + * @param[out] a Output polynomial in NTT domain. + * @param[in] r Input byte array (of MLKEM_POLYBYTES bytes). + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_frombytes_native( + int16_t a[MLKEM_N], const uint8_t r[MLKEM_POLYBYTES]) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYBYTES)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(a, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_bound(a, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT)) +); +#endif /* MLK_USE_NATIVE_POLY_FROMBYTES && (!MLK_CONFIG_NO_ENCAPS_API || \ + !MLK_CONFIG_NO_DECAPS_API) */ + +#if defined(MLK_USE_NATIVE_REJ_UNIFORM) +/** + * Run rejection sampling on uniformly random bytes to generate uniformly random + * integers mod MLKEM_Q, represented in [0,..,MLKEM_Q-1]. + * + * @param[out] r Output buffer. + * @param len Requested number of 16-bit integers (uniform mod MLKEM_Q). + * @param[in] buf Input buffer (assumed to be uniform random bytes). + * @param buflen Length of input buffer in bytes. + * + * @retval MLK_NATIVE_FUNC_FALLBACK Native implementation does not support + * the input lengths. + * @retval other Non-negative number of sampled 16-bit + * integers (at most @p len). + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_rej_uniform_native(int16_t *r, unsigned len, + const uint8_t *buf, + unsigned buflen) +__contract__( + requires(len <= 4096 && buflen <= 4096 && buflen % 3 == 0) + requires(memory_no_alias(r, sizeof(int16_t) * len)) + requires(memory_no_alias(buf, buflen)) + assigns(memory_slice(r, sizeof(int16_t) * len)) + ensures(return_value != MLK_NATIVE_FUNC_FALLBACK + ==> (0 <= return_value && return_value <= len)) + ensures(return_value != MLK_NATIVE_FUNC_FALLBACK + ==> array_bound(r, 0, (unsigned) return_value, 0, MLKEM_Q)) +); +#endif /* MLK_USE_NATIVE_REJ_UNIFORM */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || (MLKEM_K == 2 || MLKEM_K == 3) +#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D4) +/** + * Compression (4 bits) and subsequent serialization of a polynomial. + * + * @spec{Compress_4 from @[FIPS203, Eq (4.7)].} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D4 + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_compress_d4_native( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D4], const int16_t a[MLKEM_N]) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D4)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D4)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK)); +#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D4 */ + +#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D10) +/** + * Compression (10 bits) and subsequent serialization of a polynomial. + * + * @spec{Compress_10 from @[FIPS203, Eq (4.7)].} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D10 + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_compress_d10_native( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D10], const int16_t a[MLKEM_N]) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D10)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D10)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK)); +#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D10 */ + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D4) +/** + * De-serialization and subsequent decompression (4 bits) of a polynomial; + * approximate inverse of mlk_poly_compress_d4. + * + * @spec{Decompress_4 from @[FIPS203, Eq (4.8)].} + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D4 + * bytes). + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_decompress_d4_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D4]) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D4)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_bound(r, 0, MLKEM_N, 0, MLKEM_Q))); +#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D4 */ + +#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D10) +/** + * De-serialization and subsequent decompression (10 bits) of a polynomial; + * approximate inverse of mlk_poly_compress_d10. + * + * @spec{Decompress_10 from @[FIPS203, Eq (4.8)].} + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D10 + * bytes). + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_decompress_d10_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D10]) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D10)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_bound(r, 0, MLKEM_N, 0, MLKEM_Q))); +#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D10 */ +#endif /* !MLK_CONFIG_NO_DECAPS_API */ +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == 3 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4 +#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D5) +/** + * Compression (5 bits) and subsequent serialization of a polynomial. + * + * @spec{Compress_5 from @[FIPS203, Eq (4.7)].} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D5 + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_compress_d5_native( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D5], const int16_t a[MLKEM_N]) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D5)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D5)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK)); +#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D5 */ + +#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D11) +/** + * Compression (11 bits) and subsequent serialization of a polynomial. + * + * @spec{Compress_11 from @[FIPS203, Eq (4.7)].} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D11 + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_compress_d11_native( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D11], const int16_t a[MLKEM_N]) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D11)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D11)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK)); +#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D11 */ + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D5) +/** + * De-serialization and subsequent decompression (5 bits) of a polynomial; + * approximate inverse of mlk_poly_compress_d5. + * + * @spec{Decompress_5 from @[FIPS203, Eq (4.8)].} + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D5 + * bytes). + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_decompress_d5_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D5]) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D5)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_bound(r, 0, MLKEM_N, 0, MLKEM_Q))); +#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D5 */ + +#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D11) +/** + * De-serialization and subsequent decompression (11 bits) of a polynomial; + * approximate inverse of mlk_poly_compress_d11. + * + * @spec{Decompress_11 from @[FIPS203, Eq (4.8)].} + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D11 + * bytes). + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_decompress_d11_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D11]) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D11)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_bound(r, 0, MLKEM_N, 0, MLKEM_Q))); +#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D11 */ +#endif /* !MLK_CONFIG_NO_DECAPS_API */ +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */ +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#endif /* !MLK_NATIVE_API_H */ diff --git a/src/mlkem-native/mlkem/src/native/meta.h b/src/mlkem-native/mlkem/src/native/meta.h new file mode 100644 index 0000000000..dbd07a5bda --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/meta.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_NATIVE_META_H +#define MLK_NATIVE_META_H + +/* + * Default arithmetic backend + */ +#include "../sys.h" + +#ifdef MLK_SYS_AARCH64_NEON +#include "aarch64/meta.h" +#endif + +/* The x86_64 backend requires toolchain support for the SysV ABI */ +#if defined(MLK_SYS_X86_64_AVX2) && defined(MLK_SYSV_ABI_SUPPORTED) +#include "x86_64/meta.h" +#endif + +#if defined(MLK_SYS_RISCV64_RVV) +#include "riscv64/meta.h" +#endif + +#ifdef MLK_SYS_PPC64LE +#include "ppc64le/meta.h" +#endif + +#endif /* !MLK_NATIVE_META_H */ diff --git a/src/mlkem-native/mlkem/src/native/ppc64le/README.md b/src/mlkem-native/mlkem/src/native/ppc64le/README.md new file mode 100644 index 0000000000..733e32e113 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/ppc64le/README.md @@ -0,0 +1,6 @@ +[//]: # (SPDX-License-Identifier: CC-BY-4.0) + +# ppc64le backend (little endian) + +This directory contains a native backend for little endian POWER 9 (ppc64le) and above systems. +Or, Power systems supporting ISA 2.07 and above. diff --git a/src/mlkem-native/mlkem/src/native/ppc64le/meta.h b/src/mlkem-native/mlkem/src/native/ppc64le/meta.h new file mode 100644 index 0000000000..2f292432df --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/ppc64le/meta.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_NATIVE_PPC64LE_META_H +#define MLK_NATIVE_PPC64LE_META_H + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLK_ARITH_BACKEND_PPC64LE_DEFAULT + +#define MLK_ARITH_BACKEND_NAME PPC64LE_DEFAULT + +/* Set of primitives that this backend replaces */ +#define MLK_USE_NATIVE_NTT +#define MLK_USE_NATIVE_INTT +#define MLK_USE_NATIVE_POLY_REDUCE +#define MLK_USE_NATIVE_POLY_TOMONT + +#if !defined(__ASSEMBLER__) +#include +#include "../../common.h" +#include "../../params.h" +#include "../api.h" +#include "src/arith_native_ppc64le.h" + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_ntt_native(int16_t data[MLKEM_N]) +{ +#if defined(__POWER8_VECTOR__) + mlk_ntt_ppc_asm(data, mlk_ppc_qdata); + return MLK_NATIVE_FUNC_SUCCESS; +#else + (void)data; + return MLK_NATIVE_FUNC_FALLBACK; +#endif +} + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_intt_native(int16_t data[MLKEM_N]) +{ +#if defined(__POWER8_VECTOR__) + mlk_intt_ppc_asm(data, mlk_ppc_qdata); + return MLK_NATIVE_FUNC_SUCCESS; +#else + (void)data; + return MLK_NATIVE_FUNC_FALLBACK; +#endif +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_reduce_native(int16_t data[MLKEM_N]) +{ +#if defined(__POWER8_VECTOR__) + mlk_reduce_ppc_asm(data, mlk_ppc_qdata); + return MLK_NATIVE_FUNC_SUCCESS; +#else + (void)data; + return MLK_NATIVE_FUNC_FALLBACK; +#endif +} + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_tomont_native(int16_t data[MLKEM_N]) +{ +#if defined(__POWER8_VECTOR__) + mlk_poly_tomont_ppc_asm(data, mlk_ppc_qdata); + return MLK_NATIVE_FUNC_SUCCESS; +#else + (void)data; + return MLK_NATIVE_FUNC_FALLBACK; +#endif +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_NATIVE_PPC64LE_META_H */ diff --git a/src/mlkem-native/mlkem/src/native/ppc64le/src/arith_native_ppc64le.h b/src/mlkem-native/mlkem/src/native/ppc64le/src/arith_native_ppc64le.h new file mode 100644 index 0000000000..3bd47ebd76 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/ppc64le/src/arith_native_ppc64le.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_NATIVE_PPC64LE_SRC_ARITH_NATIVE_PPC64LE_H +#define MLK_NATIVE_PPC64LE_SRC_ARITH_NATIVE_PPC64LE_H + +#include +#include "../../../common.h" +#include "consts.h" + +#define mlk_ntt_ppc_asm MLK_NAMESPACE(ntt_ppc_asm) +void mlk_ntt_ppc_asm(int16_t *, const int16_t *); + +#define mlk_intt_ppc_asm MLK_NAMESPACE(intt_ppc_asm) +void mlk_intt_ppc_asm(int16_t *, const int16_t *); + +#define mlk_reduce_ppc_asm MLK_NAMESPACE(reduce_ppc_asm) +void mlk_reduce_ppc_asm(int16_t *r, const int16_t *); + +#define mlk_poly_tomont_ppc_asm MLK_NAMESPACE(poly_tomont_ppc_asm) +void mlk_poly_tomont_ppc_asm(int16_t *, const int16_t *); + +#endif /* !MLK_NATIVE_PPC64LE_SRC_ARITH_NATIVE_PPC64LE_H */ diff --git a/src/mlkem-native/mlkem/src/native/ppc64le/src/consts.c b/src/mlkem-native/mlkem/src/native/ppc64le/src/consts.c new file mode 100644 index 0000000000..f6e1abd5b9 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/ppc64le/src/consts.c @@ -0,0 +1,299 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_PPC64LE_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(__POWER8_VECTOR__) + +#include "consts.h" + +/* + * Constant table for the PPC64LE NTT/invNTT. + * For the layout, see the offset macros in consts.h. + */ +/* clang-format off */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t mlk_ppc_qdata[2072] = { + /* -Q */ + -3329, -3329, -3329, -3329, -3329, -3329, -3329, -3329, + /* Q */ + 3329, 3329, 3329, 3329, 3329, 3329, 3329, 3329, + /* round(2^26 / MLKEM_Q), Barrett reduction constant */ + 20159, 20159, 20159, 20159, 20159, 20159, 20159, 20159, + /* N^-1 in Montgomery form */ + 512, 512, 512, 512, 512, 512, 512, 512, + /* Barrett twist of N^-1 */ + 5040, 5040, 5040, 5040, 5040, 5040, 5040, 5040, + /* 2^16 mod MLKEM_Q (R), Barrett tomont factor */ + -1044, -1044, -1044, -1044, -1044, -1044, -1044, -1044, + /* Barrett twist of R for poly_tomont */ + -10276, -10276, -10276, -10276, -10276, -10276, -10276, -10276, + /* Twiddle factors for the forward NTT */ + -1600, -1600, -1600, -1600, -1600, -1600, -1600, -1600, + -749, -749, -749, -749, -749, -749, -749, -749, + -40, -40, -40, -40, -40, -40, -40, -40, + -687, -687, -687, -687, -687, -687, -687, -687, + 630, 630, 630, 630, 630, 630, 630, 630, + -1432, -1432, -1432, -1432, -1432, -1432, -1432, -1432, + 848, 848, 848, 848, 848, 848, 848, 848, + 1062, 1062, 1062, 1062, 1062, 1062, 1062, 1062, + -1410, -1410, -1410, -1410, -1410, -1410, -1410, -1410, + 193, 193, 193, 193, 193, 193, 193, 193, + 797, 797, 797, 797, 797, 797, 797, 797, + -543, -543, -543, -543, -543, -543, -543, -543, + -69, -69, -69, -69, -69, -69, -69, -69, + 569, 569, 569, 569, 569, 569, 569, 569, + -1583, -1583, -1583, -1583, -1583, -1583, -1583, -1583, + 296, 296, 296, 296, 296, 296, 296, 296, + -882, -882, -882, -882, -882, -882, -882, -882, + 1339, 1339, 1339, 1339, 1339, 1339, 1339, 1339, + 1476, 1476, 1476, 1476, 1476, 1476, 1476, 1476, + -283, -283, -283, -283, -283, -283, -283, -283, + 56, 56, 56, 56, 56, 56, 56, 56, + -1089, -1089, -1089, -1089, -1089, -1089, -1089, -1089, + 1333, 1333, 1333, 1333, 1333, 1333, 1333, 1333, + 1426, 1426, 1426, 1426, 1426, 1426, 1426, 1426, + -1235, -1235, -1235, -1235, -1235, -1235, -1235, -1235, + 535, 535, 535, 535, 535, 535, 535, 535, + -447, -447, -447, -447, -447, -447, -447, -447, + -936, -936, -936, -936, -936, -936, -936, -936, + -450, -450, -450, -450, -450, -450, -450, -450, + -1355, -1355, -1355, -1355, -1355, -1355, -1355, -1355, + 821, 821, 821, 821, 821, 821, 821, 821, + 331, 331, 331, 331, 289, 289, 289, 289, + -1573, -1573, -1573, -1573, -76, -76, -76, -76, + -1025, -1025, -1025, -1025, 1197, 1197, 1197, 1197, + -1274, -1274, -1274, -1274, -1052, -1052, -1052, -1052, + -1352, -1352, -1352, -1352, 650, 650, 650, 650, + 632, 632, 632, 632, -816, -816, -816, -816, + 33, 33, 33, 33, -464, -464, -464, -464, + -1414, -1414, -1414, -1414, 1320, 1320, 1320, 1320, + 1435, 1435, 1435, 1435, -1010, -1010, -1010, -1010, + 452, 452, 452, 452, 807, 807, 807, 807, + -461, -461, -461, -461, 1438, 1438, 1438, 1438, + -927, -927, -927, -927, 1534, 1534, 1534, 1534, + -712, -712, -712, -712, -682, -682, -682, -682, + 648, 648, 648, 648, 1481, 1481, 1481, 1481, + -219, -219, -219, -219, -855, -855, -855, -855, + 910, 910, 910, 910, 1227, 1227, 1227, 1227, + 583, 583, 17, 17, -680, -680, -568, -568, + -1041, -1041, 1637, 1637, 1100, 1100, 723, 723, + -48, -48, 1409, 1409, 233, 233, -667, -667, + -314, -314, 756, 756, -279, -279, -1173, -1173, + -540, -540, -1626, -1626, -1540, -1540, 1651, 1651, + 1461, 1461, -1482, -1482, -642, -642, 952, 952, + -892, -892, 939, 939, -941, -941, -1021, -1021, + 268, 268, 733, 733, 641, 641, -992, -992, + -1292, -1292, 1584, 1584, -109, -109, -1031, -1031, + -1239, -1239, 375, 375, 1645, 1645, -780, -780, + -556, -556, 1063, 1063, 757, 757, 319, 319, + -863, -863, -1230, -1230, -735, -735, 561, 561, + 403, 403, -525, -525, 1026, 1026, 1092, 1092, + -554, -554, 1143, 1143, 886, 886, -1179, -1179, + -1455, -1455, -1607, -1607, 1029, 1029, 1212, 1212, + 885, 885, -1219, -1219, -1175, -1175, -394, -394, + /* Twiddle factors for the inverse NTT */ + -394, -394, -1175, -1175, -1219, -1219, 885, 885, + 1212, 1212, 1029, 1029, -1607, -1607, -1455, -1455, + -1179, -1179, 886, 886, 1143, 1143, -554, -554, + 1092, 1092, 1026, 1026, -525, -525, 403, 403, + 561, 561, -735, -735, -1230, -1230, -863, -863, + 319, 319, 757, 757, 1063, 1063, -556, -556, + -780, -780, 1645, 1645, 375, 375, -1239, -1239, + -1031, -1031, -109, -109, 1584, 1584, -1292, -1292, + -992, -992, 641, 641, 733, 733, 268, 268, + -1021, -1021, -941, -941, 939, 939, -892, -892, + 952, 952, -642, -642, -1482, -1482, 1461, 1461, + 1651, 1651, -1540, -1540, -1626, -1626, -540, -540, + -1173, -1173, -279, -279, 756, 756, -314, -314, + -667, -667, 233, 233, 1409, 1409, -48, -48, + 723, 723, 1100, 1100, 1637, 1637, -1041, -1041, + -568, -568, -680, -680, 17, 17, 583, 583, + 1227, 1227, 1227, 1227, 910, 910, 910, 910, + -855, -855, -855, -855, -219, -219, -219, -219, + 1481, 1481, 1481, 1481, 648, 648, 648, 648, + -682, -682, -682, -682, -712, -712, -712, -712, + 1534, 1534, 1534, 1534, -927, -927, -927, -927, + 1438, 1438, 1438, 1438, -461, -461, -461, -461, + 807, 807, 807, 807, 452, 452, 452, 452, + -1010, -1010, -1010, -1010, 1435, 1435, 1435, 1435, + 1320, 1320, 1320, 1320, -1414, -1414, -1414, -1414, + -464, -464, -464, -464, 33, 33, 33, 33, + -816, -816, -816, -816, 632, 632, 632, 632, + 650, 650, 650, 650, -1352, -1352, -1352, -1352, + -1052, -1052, -1052, -1052, -1274, -1274, -1274, -1274, + 1197, 1197, 1197, 1197, -1025, -1025, -1025, -1025, + -76, -76, -76, -76, -1573, -1573, -1573, -1573, + 289, 289, 289, 289, 331, 331, 331, 331, + 821, 821, 821, 821, 821, 821, 821, 821, + -1355, -1355, -1355, -1355, -1355, -1355, -1355, -1355, + -450, -450, -450, -450, -450, -450, -450, -450, + -936, -936, -936, -936, -936, -936, -936, -936, + -447, -447, -447, -447, -447, -447, -447, -447, + 535, 535, 535, 535, 535, 535, 535, 535, + -1235, -1235, -1235, -1235, -1235, -1235, -1235, -1235, + 1426, 1426, 1426, 1426, 1426, 1426, 1426, 1426, + 1333, 1333, 1333, 1333, 1333, 1333, 1333, 1333, + -1089, -1089, -1089, -1089, -1089, -1089, -1089, -1089, + 56, 56, 56, 56, 56, 56, 56, 56, + -283, -283, -283, -283, -283, -283, -283, -283, + 1476, 1476, 1476, 1476, 1476, 1476, 1476, 1476, + 1339, 1339, 1339, 1339, 1339, 1339, 1339, 1339, + -882, -882, -882, -882, -882, -882, -882, -882, + 296, 296, 296, 296, 296, 296, 296, 296, + -1583, -1583, -1583, -1583, -1583, -1583, -1583, -1583, + 569, 569, 569, 569, 569, 569, 569, 569, + -69, -69, -69, -69, -69, -69, -69, -69, + -543, -543, -543, -543, -543, -543, -543, -543, + 797, 797, 797, 797, 797, 797, 797, 797, + 193, 193, 193, 193, 193, 193, 193, 193, + -1410, -1410, -1410, -1410, -1410, -1410, -1410, -1410, + 1062, 1062, 1062, 1062, 1062, 1062, 1062, 1062, + 848, 848, 848, 848, 848, 848, 848, 848, + -1432, -1432, -1432, -1432, -1432, -1432, -1432, -1432, + 630, 630, 630, 630, 630, 630, 630, 630, + -687, -687, -687, -687, -687, -687, -687, -687, + -40, -40, -40, -40, -40, -40, -40, -40, + -749, -749, -749, -749, -749, -749, -749, -749, + -1600, -1600, -1600, -1600, -1600, -1600, -1600, -1600, + /* Twisted twiddle factors for the forward NTT */ + -15749, -15749, -15749, -15749, -15749, -15749, -15749, -15749, + -7373, -7373, -7373, -7373, -7373, -7373, -7373, -7373, + -394, -394, -394, -394, -394, -394, -394, -394, + -6762, -6762, -6762, -6762, -6762, -6762, -6762, -6762, + 6201, 6201, 6201, 6201, 6201, 6201, 6201, 6201, + -14095, -14095, -14095, -14095, -14095, -14095, -14095, -14095, + 8347, 8347, 8347, 8347, 8347, 8347, 8347, 8347, + 10453, 10453, 10453, 10453, 10453, 10453, 10453, 10453, + -13879, -13879, -13879, -13879, -13879, -13879, -13879, -13879, + 1900, 1900, 1900, 1900, 1900, 1900, 1900, 1900, + 7845, 7845, 7845, 7845, 7845, 7845, 7845, 7845, + -5345, -5345, -5345, -5345, -5345, -5345, -5345, -5345, + -679, -679, -679, -679, -679, -679, -679, -679, + 5601, 5601, 5601, 5601, 5601, 5601, 5601, 5601, + -15582, -15582, -15582, -15582, -15582, -15582, -15582, -15582, + 2914, 2914, 2914, 2914, 2914, 2914, 2914, 2914, + -8682, -8682, -8682, -8682, -8682, -8682, -8682, -8682, + 13180, 13180, 13180, 13180, 13180, 13180, 13180, 13180, + 14529, 14529, 14529, 14529, 14529, 14529, 14529, 14529, + -2786, -2786, -2786, -2786, -2786, -2786, -2786, -2786, + 551, 551, 551, 551, 551, 551, 551, 551, + -10719, -10719, -10719, -10719, -10719, -10719, -10719, -10719, + 13121, 13121, 13121, 13121, 13121, 13121, 13121, 13121, + 14036, 14036, 14036, 14036, 14036, 14036, 14036, 14036, + -12156, -12156, -12156, -12156, -12156, -12156, -12156, -12156, + 5266, 5266, 5266, 5266, 5266, 5266, 5266, 5266, + -4400, -4400, -4400, -4400, -4400, -4400, -4400, -4400, + -9213, -9213, -9213, -9213, -9213, -9213, -9213, -9213, + -4429, -4429, -4429, -4429, -4429, -4429, -4429, -4429, + -13338, -13338, -13338, -13338, -13338, -13338, -13338, -13338, + 8081, 8081, 8081, 8081, 8081, 8081, 8081, 8081, + 3258, 3258, 3258, 3258, 2845, 2845, 2845, 2845, + -15483, -15483, -15483, -15483, -748, -748, -748, -748, + -10089, -10089, -10089, -10089, 11782, 11782, 11782, 11782, + -12540, -12540, -12540, -12540, -10355, -10355, -10355, -10355, + -13308, -13308, -13308, -13308, 6398, 6398, 6398, 6398, + 6221, 6221, 6221, 6221, -8032, -8032, -8032, -8032, + 325, 325, 325, 325, -4567, -4567, -4567, -4567, + -13918, -13918, -13918, -13918, 12993, 12993, 12993, 12993, + 14125, 14125, 14125, 14125, -9942, -9942, -9942, -9942, + 4449, 4449, 4449, 4449, 7943, 7943, 7943, 7943, + -4538, -4538, -4538, -4538, 14155, 14155, 14155, 14155, + -9125, -9125, -9125, -9125, 15099, 15099, 15099, 15099, + -7008, -7008, -7008, -7008, -6713, -6713, -6713, -6713, + 6378, 6378, 6378, 6378, 14578, 14578, 14578, 14578, + -2156, -2156, -2156, -2156, -8416, -8416, -8416, -8416, + 8957, 8957, 8957, 8957, 12078, 12078, 12078, 12078, + 5739, 5739, 167, 167, -6693, -6693, -5591, -5591, + -10247, -10247, 16113, 16113, 10828, 10828, 7117, 7117, + -472, -472, 13869, 13869, 2293, 2293, -6565, -6565, + -3091, -3091, 7441, 7441, -2746, -2746, -11546, -11546, + -5315, -5315, -16005, -16005, -15159, -15159, 16251, 16251, + 14381, 14381, -14588, -14588, -6319, -6319, 9371, 9371, + -8780, -8780, 9243, 9243, -9262, -9262, -10050, -10050, + 2638, 2638, 7215, 7215, 6309, 6309, -9764, -9764, + -12717, -12717, 15592, 15592, -1073, -1073, -10148, -10148, + -12196, -12196, 3691, 3691, 16192, 16192, -7678, -7678, + -5473, -5473, 10463, 10463, 7451, 7451, 3140, 3140, + -8495, -8495, -12107, -12107, -7235, -7235, 5522, 5522, + 3967, 3967, -5168, -5168, 10099, 10099, 10749, 10749, + -5453, -5453, 11251, 11251, 8721, 8721, -11605, -11605, + -14322, -14322, -15818, -15818, 10129, 10129, 11930, 11930, + 8711, 8711, -11999, -11999, -11566, -11566, -3878, -3878, + /* Twisted twiddle factors for the inverse NTT */ + -3878, -3878, -11566, -11566, -11999, -11999, 8711, 8711, + 11930, 11930, 10129, 10129, -15818, -15818, -14322, -14322, + -11605, -11605, 8721, 8721, 11251, 11251, -5453, -5453, + 10749, 10749, 10099, 10099, -5168, -5168, 3967, 3967, + 5522, 5522, -7235, -7235, -12107, -12107, -8495, -8495, + 3140, 3140, 7451, 7451, 10463, 10463, -5473, -5473, + -7678, -7678, 16192, 16192, 3691, 3691, -12196, -12196, + -10148, -10148, -1073, -1073, 15592, 15592, -12717, -12717, + -9764, -9764, 6309, 6309, 7215, 7215, 2638, 2638, + -10050, -10050, -9262, -9262, 9243, 9243, -8780, -8780, + 9371, 9371, -6319, -6319, -14588, -14588, 14381, 14381, + 16251, 16251, -15159, -15159, -16005, -16005, -5315, -5315, + -11546, -11546, -2746, -2746, 7441, 7441, -3091, -3091, + -6565, -6565, 2293, 2293, 13869, 13869, -472, -472, + 7117, 7117, 10828, 10828, 16113, 16113, -10247, -10247, + -5591, -5591, -6693, -6693, 167, 167, 5739, 5739, + 12078, 12078, 12078, 12078, 8957, 8957, 8957, 8957, + -8416, -8416, -8416, -8416, -2156, -2156, -2156, -2156, + 14578, 14578, 14578, 14578, 6378, 6378, 6378, 6378, + -6713, -6713, -6713, -6713, -7008, -7008, -7008, -7008, + 15099, 15099, 15099, 15099, -9125, -9125, -9125, -9125, + 14155, 14155, 14155, 14155, -4538, -4538, -4538, -4538, + 7943, 7943, 7943, 7943, 4449, 4449, 4449, 4449, + -9942, -9942, -9942, -9942, 14125, 14125, 14125, 14125, + 12993, 12993, 12993, 12993, -13918, -13918, -13918, -13918, + -4567, -4567, -4567, -4567, 325, 325, 325, 325, + -8032, -8032, -8032, -8032, 6221, 6221, 6221, 6221, + 6398, 6398, 6398, 6398, -13308, -13308, -13308, -13308, + -10355, -10355, -10355, -10355, -12540, -12540, -12540, -12540, + 11782, 11782, 11782, 11782, -10089, -10089, -10089, -10089, + -748, -748, -748, -748, -15483, -15483, -15483, -15483, + 2845, 2845, 2845, 2845, 3258, 3258, 3258, 3258, + 8081, 8081, 8081, 8081, 8081, 8081, 8081, 8081, + -13338, -13338, -13338, -13338, -13338, -13338, -13338, -13338, + -4429, -4429, -4429, -4429, -4429, -4429, -4429, -4429, + -9213, -9213, -9213, -9213, -9213, -9213, -9213, -9213, + -4400, -4400, -4400, -4400, -4400, -4400, -4400, -4400, + 5266, 5266, 5266, 5266, 5266, 5266, 5266, 5266, + -12156, -12156, -12156, -12156, -12156, -12156, -12156, -12156, + 14036, 14036, 14036, 14036, 14036, 14036, 14036, 14036, + 13121, 13121, 13121, 13121, 13121, 13121, 13121, 13121, + -10719, -10719, -10719, -10719, -10719, -10719, -10719, -10719, + 551, 551, 551, 551, 551, 551, 551, 551, + -2786, -2786, -2786, -2786, -2786, -2786, -2786, -2786, + 14529, 14529, 14529, 14529, 14529, 14529, 14529, 14529, + 13180, 13180, 13180, 13180, 13180, 13180, 13180, 13180, + -8682, -8682, -8682, -8682, -8682, -8682, -8682, -8682, + 2914, 2914, 2914, 2914, 2914, 2914, 2914, 2914, + -15582, -15582, -15582, -15582, -15582, -15582, -15582, -15582, + 5601, 5601, 5601, 5601, 5601, 5601, 5601, 5601, + -679, -679, -679, -679, -679, -679, -679, -679, + -5345, -5345, -5345, -5345, -5345, -5345, -5345, -5345, + 7845, 7845, 7845, 7845, 7845, 7845, 7845, 7845, + 1900, 1900, 1900, 1900, 1900, 1900, 1900, 1900, + -13879, -13879, -13879, -13879, -13879, -13879, -13879, -13879, + 10453, 10453, 10453, 10453, 10453, 10453, 10453, 10453, + 8347, 8347, 8347, 8347, 8347, 8347, 8347, 8347, + -14095, -14095, -14095, -14095, -14095, -14095, -14095, -14095, + 6201, 6201, 6201, 6201, 6201, 6201, 6201, 6201, + -6762, -6762, -6762, -6762, -6762, -6762, -6762, -6762, + -394, -394, -394, -394, -394, -394, -394, -394, + -7373, -7373, -7373, -7373, -7373, -7373, -7373, -7373, + -15749, -15749, -15749, -15749, -15749, -15749, -15749, -15749, +}; +/* clang-format on */ + +#endif /* MLK_ARITH_BACKEND_PPC64LE_DEFAULT && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED && __POWER8_VECTOR__ */ diff --git a/src/mlkem-native/mlkem/src/native/ppc64le/src/consts.h b/src/mlkem-native/mlkem/src/native/ppc64le/src/consts.h new file mode 100644 index 0000000000..acac168b60 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/ppc64le/src/consts.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#ifndef MLK_NATIVE_PPC64LE_SRC_CONSTS_H +#define MLK_NATIVE_PPC64LE_SRC_CONSTS_H +#include "../../../common.h" + +/* Byte offsets into the constant table */ +#define MLK_PPC_NQ_OFFSET 0 +#define MLK_PPC_Q_OFFSET 16 +#define MLK_PPC_C20159_OFFSET 32 +#define MLK_PPC_N_INV_OFFSET 48 +#define MLK_PPC_N_INV_TW_OFFSET 64 +#define MLK_PPC_TOMONT_OFFSET 80 +#define MLK_PPC_TOMONT_TW_OFFSET 96 +#define MLK_PPC_ZETA_NTT_OFFSET 112 +#define MLK_PPC_ZETA_INTT_OFFSET 1120 +#define MLK_PPC_ZETA_NTT_TW_OFFSET 2128 +#define MLK_PPC_ZETA_INTT_TW_OFFSET 3136 + +#ifndef __ASSEMBLER__ +#define mlk_ppc_qdata MLK_NAMESPACE(ppc_qdata) +MLK_INTERNAL_DATA_DECLARATION const int16_t mlk_ppc_qdata[2072]; +#endif + +#endif /* !MLK_NATIVE_PPC64LE_SRC_CONSTS_H */ diff --git a/src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_intt_ppc_asm.S b/src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_intt_ppc_asm.S new file mode 100644 index 0000000000..cd6929f6ac --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_intt_ppc_asm.S @@ -0,0 +1,3246 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) IBM Corp. 2025, 2026 + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + * + * Written by Danny Tsen + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_PPC64LE_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(__POWER8_VECTOR__) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API)) +/*yaml + Name: intt_ppc_asm + Description: PowerPC64 VSX inverse NTT + Signature: void mlk_intt_ppc_asm(int16_t *r, const int16_t *qdata) + ABI: + Architecture: powerpc64le + CallingConvention: ELFv2 + Features: [VSX] + r3: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) + r4: + type: buffer + size_bytes: 4144 + permissions: read-only + c_parameter: const int16_t *qdata + description: Precomputed constants (2072 x int16_t) +*/ + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/ppc64le/src/mlkem_intt_ppc_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(intt_ppc_asm) +MLK_ASM_FN_SYMBOL(intt_ppc_asm) + + stdu 1, -352(1) + mflr 0 + std 14, 56(1) + std 15, 64(1) + std 16, 72(1) + std 17, 80(1) + std 18, 88(1) + std 19, 96(1) + std 20, 104(1) + std 21, 112(1) + std 22, 120(1) + li 10, 128 + li 11, 144 + li 12, 160 + li 14, 176 + li 15, 192 + li 16, 208 + stxvd2x 52, 10, 1 + stxvd2x 53, 11, 1 + stxvd2x 54, 12, 1 + stxvd2x 55, 14, 1 + stxvd2x 56, 15, 1 + stxvd2x 57, 16, 1 + li 10, 224 + li 11, 240 + li 12, 256 + li 14, 272 + li 15, 288 + li 16, 304 + stxvd2x 58, 10, 1 + stxvd2x 59, 11, 1 + stxvd2x 60, 12, 1 + stxvd2x 61, 14, 1 + stxvd2x 62, 15, 1 + stxvd2x 63, 16, 1 + lxvd2x 0, 0, 4 + xxlxor 35, 35, 35 + xxlor 3, 35, 35 + li 10, 16 + li 11, 32 + lxvd2x 6, 10, 4 + lxvd2x 32, 11, 4 + vspltisw 8, 13 + vadduwm 8, 8, 8 + xxlor 8, 40, 40 + vspltisw 9, 1 + vsubuwm 10, 8, 9 + vslw 9, 9, 10 + xxlor 7, 41, 41 + li 10, 16 + li 11, 32 + li 12, 48 + li 15, 64 + li 16, 80 + li 17, 96 + li 18, 112 + addi 14, 4, 48 + lvx 10, 0, 14 + addi 14, 4, 64 + lvx 11, 0, 14 + li 8, 4 + mtctr 8 + xxlor 37, 0, 0 + +mlk_intt_ppc_asm_Loopf: + lxvd2x 57, 0, 3 + lxvd2x 58, 10, 3 + lxvd2x 62, 11, 3 + lxvd2x 63, 12, 3 + addi 3, 3, 64 + vmhraddshs 14, 25, 11, 3 + vmhraddshs 19, 26, 11, 3 + vmhraddshs 24, 30, 11, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 6, 25, 10, 3 + vmladduhm 7, 26, 10, 3 + vmladduhm 8, 30, 10, 3 + vmladduhm 9, 31, 10, 3 + vmladduhm 6, 14, 5, 6 + vmladduhm 7, 19, 5, 7 + vmladduhm 8, 24, 5, 8 + vmladduhm 9, 29, 5, 9 + lxvd2x 57, 0, 3 + lxvd2x 58, 10, 3 + lxvd2x 62, 11, 3 + lxvd2x 63, 12, 3 + addi 3, 3, 64 + vmhraddshs 14, 25, 11, 3 + vmhraddshs 19, 26, 11, 3 + vmhraddshs 24, 30, 11, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + addi 3, 3, -128 + stxvd2x 38, 0, 3 + stxvd2x 39, 10, 3 + stxvd2x 40, 11, 3 + stxvd2x 41, 12, 3 + stxvd2x 45, 15, 3 + stxvd2x 50, 16, 3 + stxvd2x 55, 17, 3 + stxvd2x 60, 18, 3 + addi 3, 3, 128 + bdnz mlk_intt_ppc_asm_Loopf + addi 3, 3, -512 + nop + nop + ori 2, 2, 0 + addi 14, 4, 1120 + addi 22, 4, 3136 + li 7, 4 + mr 5, 3 + lxvd2x 57, 0, 5 + lxvd2x 58, 10, 5 + vmrgew 8, 25, 26 + vmrgow 21, 25, 26 + lxvd2x 57, 11, 5 + lxvd2x 58, 12, 5 + vmrgew 12, 25, 26 + vmrgow 22, 25, 26 + lxvd2x 57, 15, 5 + lxvd2x 58, 16, 5 + vmrgew 16, 25, 26 + vmrgow 23, 25, 26 + lxvd2x 57, 17, 5 + lxvd2x 58, 18, 5 + vmrgew 20, 25, 26 + vmrgow 24, 25, 26 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + xxlor 10, 36, 36 + xxlor 11, 41, 41 + xxlor 12, 45, 45 + xxlor 13, 49, 49 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + xxlor 46, 10, 10 + xxlor 51, 11, 11 + xxlor 56, 12, 12 + xxlor 61, 13, 13 + vmrgew 10, 13, 14 + vmrgow 11, 13, 14 + vmrgew 12, 18, 19 + vmrgow 13, 18, 19 + vmrgew 14, 23, 24 + vmrgow 15, 23, 24 + vmrgew 16, 28, 29 + vmrgow 17, 28, 29 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + lxvd2x 57, 0, 5 + lxvd2x 58, 10, 5 + vmrgew 8, 25, 26 + vmrgow 21, 25, 26 + lxvd2x 57, 11, 5 + lxvd2x 58, 12, 5 + vmrgew 12, 25, 26 + vmrgow 22, 25, 26 + lxvd2x 57, 15, 5 + lxvd2x 58, 16, 5 + vmrgew 16, 25, 26 + vmrgow 23, 25, 26 + lxvd2x 57, 17, 5 + lxvd2x 58, 18, 5 + vmrgew 20, 25, 26 + vmrgow 24, 25, 26 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + xxlor 10, 36, 36 + xxlor 11, 41, 41 + xxlor 12, 45, 45 + xxlor 13, 49, 49 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + xxlor 46, 10, 10 + xxlor 51, 11, 11 + xxlor 56, 12, 12 + xxlor 61, 13, 13 + vmrgew 10, 13, 14 + vmrgow 11, 13, 14 + vmrgew 12, 18, 19 + vmrgow 13, 18, 19 + vmrgew 14, 23, 24 + vmrgow 15, 23, 24 + vmrgew 16, 28, 29 + vmrgow 17, 28, 29 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + lxvd2x 57, 0, 5 + lxvd2x 58, 10, 5 + vmrgew 8, 25, 26 + vmrgow 21, 25, 26 + lxvd2x 57, 11, 5 + lxvd2x 58, 12, 5 + vmrgew 12, 25, 26 + vmrgow 22, 25, 26 + lxvd2x 57, 15, 5 + lxvd2x 58, 16, 5 + vmrgew 16, 25, 26 + vmrgow 23, 25, 26 + lxvd2x 57, 17, 5 + lxvd2x 58, 18, 5 + vmrgew 20, 25, 26 + vmrgow 24, 25, 26 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + xxlor 10, 36, 36 + xxlor 11, 41, 41 + xxlor 12, 45, 45 + xxlor 13, 49, 49 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + xxlor 46, 10, 10 + xxlor 51, 11, 11 + xxlor 56, 12, 12 + xxlor 61, 13, 13 + vmrgew 10, 13, 14 + vmrgow 11, 13, 14 + vmrgew 12, 18, 19 + vmrgow 13, 18, 19 + vmrgew 14, 23, 24 + vmrgow 15, 23, 24 + vmrgew 16, 28, 29 + vmrgow 17, 28, 29 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + lxvd2x 57, 0, 5 + lxvd2x 58, 10, 5 + vmrgew 8, 25, 26 + vmrgow 21, 25, 26 + lxvd2x 57, 11, 5 + lxvd2x 58, 12, 5 + vmrgew 12, 25, 26 + vmrgow 22, 25, 26 + lxvd2x 57, 15, 5 + lxvd2x 58, 16, 5 + vmrgew 16, 25, 26 + vmrgow 23, 25, 26 + lxvd2x 57, 17, 5 + lxvd2x 58, 18, 5 + vmrgew 20, 25, 26 + vmrgow 24, 25, 26 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + xxlor 10, 36, 36 + xxlor 11, 41, 41 + xxlor 12, 45, 45 + xxlor 13, 49, 49 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + xxlor 46, 10, 10 + xxlor 51, 11, 11 + xxlor 56, 12, 12 + xxlor 61, 13, 13 + vmrgew 10, 13, 14 + vmrgow 11, 13, 14 + vmrgew 12, 18, 19 + vmrgow 13, 18, 19 + vmrgew 14, 23, 24 + vmrgow 15, 23, 24 + vmrgew 16, 28, 29 + vmrgow 17, 28, 29 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + mr 5, 3 + li 7, 8 + lxvd2x 10, 0, 5 + lxvd2x 11, 10, 5 + xxmrgld 40, 11, 10 + xxmrghd 53, 11, 10 + lxvd2x 10, 11, 5 + lxvd2x 11, 12, 5 + xxmrgld 44, 11, 10 + xxmrghd 54, 11, 10 + lxvd2x 10, 15, 5 + lxvd2x 11, 16, 5 + xxmrgld 48, 11, 10 + xxmrghd 55, 11, 10 + lxvd2x 10, 17, 5 + lxvd2x 11, 18, 5 + xxmrgld 52, 11, 10 + xxmrghd 56, 11, 10 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + xxlor 10, 36, 36 + xxlor 11, 41, 41 + xxlor 12, 45, 45 + xxlor 13, 49, 49 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + xxlor 46, 10, 10 + xxlor 51, 11, 11 + xxlor 56, 12, 12 + xxlor 61, 13, 13 + xxmrgld 42, 46, 45 + xxmrghd 43, 46, 45 + xxmrgld 44, 51, 50 + xxmrghd 45, 51, 50 + xxmrgld 46, 56, 55 + xxmrghd 47, 56, 55 + xxmrgld 48, 61, 60 + xxmrghd 49, 61, 60 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + lxvd2x 10, 0, 5 + lxvd2x 11, 10, 5 + xxmrgld 40, 11, 10 + xxmrghd 53, 11, 10 + lxvd2x 10, 11, 5 + lxvd2x 11, 12, 5 + xxmrgld 44, 11, 10 + xxmrghd 54, 11, 10 + lxvd2x 10, 15, 5 + lxvd2x 11, 16, 5 + xxmrgld 48, 11, 10 + xxmrghd 55, 11, 10 + lxvd2x 10, 17, 5 + lxvd2x 11, 18, 5 + xxmrgld 52, 11, 10 + xxmrghd 56, 11, 10 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + xxlor 10, 36, 36 + xxlor 11, 41, 41 + xxlor 12, 45, 45 + xxlor 13, 49, 49 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + xxlor 46, 10, 10 + xxlor 51, 11, 11 + xxlor 56, 12, 12 + xxlor 61, 13, 13 + xxmrgld 42, 46, 45 + xxmrghd 43, 46, 45 + xxmrgld 44, 51, 50 + xxmrghd 45, 51, 50 + xxmrgld 46, 56, 55 + xxmrghd 47, 56, 55 + xxmrgld 48, 61, 60 + xxmrghd 49, 61, 60 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + lxvd2x 10, 0, 5 + lxvd2x 11, 10, 5 + xxmrgld 40, 11, 10 + xxmrghd 53, 11, 10 + lxvd2x 10, 11, 5 + lxvd2x 11, 12, 5 + xxmrgld 44, 11, 10 + xxmrghd 54, 11, 10 + lxvd2x 10, 15, 5 + lxvd2x 11, 16, 5 + xxmrgld 48, 11, 10 + xxmrghd 55, 11, 10 + lxvd2x 10, 17, 5 + lxvd2x 11, 18, 5 + xxmrgld 52, 11, 10 + xxmrghd 56, 11, 10 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + xxlor 10, 36, 36 + xxlor 11, 41, 41 + xxlor 12, 45, 45 + xxlor 13, 49, 49 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + xxlor 46, 10, 10 + xxlor 51, 11, 11 + xxlor 56, 12, 12 + xxlor 61, 13, 13 + xxmrgld 42, 46, 45 + xxmrghd 43, 46, 45 + xxmrgld 44, 51, 50 + xxmrghd 45, 51, 50 + xxmrgld 46, 56, 55 + xxmrghd 47, 56, 55 + xxmrgld 48, 61, 60 + xxmrghd 49, 61, 60 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + lxvd2x 10, 0, 5 + lxvd2x 11, 10, 5 + xxmrgld 40, 11, 10 + xxmrghd 53, 11, 10 + lxvd2x 10, 11, 5 + lxvd2x 11, 12, 5 + xxmrgld 44, 11, 10 + xxmrghd 54, 11, 10 + lxvd2x 10, 15, 5 + lxvd2x 11, 16, 5 + xxmrgld 48, 11, 10 + xxmrghd 55, 11, 10 + lxvd2x 10, 17, 5 + lxvd2x 11, 18, 5 + xxmrgld 52, 11, 10 + xxmrghd 56, 11, 10 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + xxlor 10, 36, 36 + xxlor 11, 41, 41 + xxlor 12, 45, 45 + xxlor 13, 49, 49 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + xxlor 46, 10, 10 + xxlor 51, 11, 11 + xxlor 56, 12, 12 + xxlor 61, 13, 13 + xxmrgld 42, 46, 45 + xxmrghd 43, 46, 45 + xxmrgld 44, 51, 50 + xxmrghd 45, 51, 50 + xxmrgld 46, 56, 55 + xxmrghd 47, 56, 55 + xxmrgld 48, 61, 60 + xxmrghd 49, 61, 60 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + nop + ori 2, 2, 0 + li 7, 16 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 32 + addi 17, 10, 32 + addi 18, 16, 32 + addi 19, 17, 32 + addi 20, 18, 32 + addi 21, 19, 32 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 128 + add 10, 7, 9 + addi 16, 9, 32 + addi 17, 10, 32 + addi 18, 16, 32 + addi 19, 17, 32 + addi 20, 18, 32 + addi 21, 19, 32 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 256 + add 10, 7, 9 + addi 16, 9, 32 + addi 17, 10, 32 + addi 18, 16, 32 + addi 19, 17, 32 + addi 20, 18, 32 + addi 21, 19, 32 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 384 + add 10, 7, 9 + addi 16, 9, 32 + addi 17, 10, 32 + addi 18, 16, 32 + addi 19, 17, 32 + addi 20, 18, 32 + addi 21, 19, 32 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + nop + nop + ori 2, 2, 0 + li 7, 32 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 64 + addi 17, 10, 64 + addi 18, 16, 64 + addi 19, 17, 64 + addi 20, 18, 64 + addi 21, 19, 64 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + addi 14, 14, -64 + addi 22, 22, -64 + li 9, 16 + add 10, 7, 9 + addi 16, 9, 64 + addi 17, 10, 64 + addi 18, 16, 64 + addi 19, 17, 64 + addi 20, 18, 64 + addi 21, 19, 64 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 256 + add 10, 7, 9 + addi 16, 9, 64 + addi 17, 10, 64 + addi 18, 16, 64 + addi 19, 17, 64 + addi 20, 18, 64 + addi 21, 19, 64 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + addi 14, 14, -64 + addi 22, 22, -64 + li 9, 272 + add 10, 7, 9 + addi 16, 9, 64 + addi 17, 10, 64 + addi 18, 16, 64 + addi 19, 17, 64 + addi 20, 18, 64 + addi 21, 19, 64 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + nop + nop + ori 2, 2, 0 + li 7, 64 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 128 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 256 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 384 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + addi 14, 14, 16 + addi 22, 22, 16 + nop + nop + ori 2, 2, 0 + li 7, 128 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 64 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 256 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 320 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + addi 14, 14, 16 + addi 22, 22, 16 + nop + nop + ori 2, 2, 0 + li 7, 256 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 64 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 128 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 192 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 10, 128 + li 11, 144 + li 12, 160 + li 14, 176 + li 15, 192 + li 16, 208 + lxvd2x 52, 10, 1 + lxvd2x 53, 11, 1 + lxvd2x 54, 12, 1 + lxvd2x 55, 14, 1 + lxvd2x 56, 15, 1 + lxvd2x 57, 16, 1 + li 10, 224 + li 11, 240 + li 12, 256 + li 14, 272 + li 15, 288 + li 16, 304 + lxvd2x 58, 10, 1 + lxvd2x 59, 11, 1 + lxvd2x 60, 12, 1 + lxvd2x 61, 14, 1 + lxvd2x 62, 15, 1 + lxvd2x 63, 16, 1 + ld 14, 56(1) + ld 15, 64(1) + ld 16, 72(1) + ld 17, 80(1) + ld 18, 88(1) + ld 19, 96(1) + ld 20, 104(1) + ld 21, 112(1) + ld 22, 120(1) + mtlr 0 + addi 1, 1, 352 + blr + +MLK_ASM_FN_SIZE(intt_ppc_asm) + +#endif /* MLK_ARITH_BACKEND_PPC64LE_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && __POWER8_VECTOR__ && (!MLK_CONFIG_NO_ENCAPS_API || \ + !MLK_CONFIG_NO_DECAPS_API) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_ntt_ppc_asm.S b/src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_ntt_ppc_asm.S new file mode 100644 index 0000000000..09ec84a966 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_ntt_ppc_asm.S @@ -0,0 +1,1672 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) IBM Corp. 2025, 2026 + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + * + * Written by Danny Tsen + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_PPC64LE_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(__POWER8_VECTOR__) +/*yaml + Name: ntt_ppc_asm + Description: PowerPC64 VSX forward NTT + Signature: void mlk_ntt_ppc_asm(int16_t *r, const int16_t *qdata) + ABI: + Architecture: powerpc64le + CallingConvention: ELFv2 + Features: [VSX] + r3: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) + r4: + type: buffer + size_bytes: 4144 + permissions: read-only + c_parameter: const int16_t *qdata + description: Precomputed constants (2072 x int16_t) +*/ + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/ppc64le/src/mlkem_ntt_ppc_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(ntt_ppc_asm) +MLK_ASM_FN_SYMBOL(ntt_ppc_asm) + + stdu 1, -352(1) + mflr 0 + std 14, 56(1) + std 15, 64(1) + std 16, 72(1) + std 17, 80(1) + std 18, 88(1) + std 19, 96(1) + std 20, 104(1) + std 21, 112(1) + std 22, 120(1) + li 10, 128 + li 11, 144 + li 12, 160 + li 14, 176 + li 15, 192 + li 16, 208 + stxvd2x 52, 10, 1 + stxvd2x 53, 11, 1 + stxvd2x 54, 12, 1 + stxvd2x 55, 14, 1 + stxvd2x 56, 15, 1 + stxvd2x 57, 16, 1 + li 10, 224 + li 11, 240 + li 12, 256 + li 14, 272 + li 15, 288 + li 16, 304 + stxvd2x 58, 10, 1 + stxvd2x 59, 11, 1 + stxvd2x 60, 12, 1 + stxvd2x 61, 14, 1 + stxvd2x 62, 15, 1 + stxvd2x 63, 16, 1 + lvx 5, 0, 4 + addi 14, 4, 112 + addi 22, 4, 2128 + vxor 3, 3, 3 + ori 2, 2, 0 + li 7, 256 + lvx 10, 0, 14 + lvx 2, 0, 22 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 9, 64 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 9, 128 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 9, 192 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + nop + nop + ori 2, 2, 0 + li 7, 128 + lvx 10, 0, 14 + lvx 2, 0, 22 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 9, 64 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + lvx 10, 0, 14 + lvx 2, 0, 22 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 256 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 9, 320 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + nop + nop + ori 2, 2, 0 + li 7, 64 + lvx 10, 0, 14 + lvx 2, 0, 22 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + lvx 10, 0, 14 + lvx 2, 0, 22 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 128 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + lvx 10, 0, 14 + lvx 2, 0, 22 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 256 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + lvx 10, 0, 14 + lvx 2, 0, 22 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 384 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + nop + nop + ori 2, 2, 0 + li 7, 32 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 64 + addi 17, 10, 64 + addi 18, 16, 64 + addi 19, 17, 64 + addi 20, 18, 64 + addi 21, 19, 64 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 9, 16 + add 10, 7, 9 + addi 16, 9, 64 + addi 17, 10, 64 + addi 18, 16, 64 + addi 19, 17, 64 + addi 20, 18, 64 + addi 21, 19, 64 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + li 9, 256 + add 10, 7, 9 + addi 16, 9, 64 + addi 17, 10, 64 + addi 18, 16, 64 + addi 19, 17, 64 + addi 20, 18, 64 + addi 21, 19, 64 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 9, 272 + add 10, 7, 9 + addi 16, 9, 64 + addi 17, 10, 64 + addi 18, 16, 64 + addi 19, 17, 64 + addi 20, 18, 64 + addi 21, 19, 64 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + ori 2, 2, 0 + li 7, 16 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 32 + addi 17, 10, 32 + addi 18, 16, 32 + addi 19, 17, 32 + addi 20, 18, 32 + addi 21, 19, 32 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + li 9, 128 + add 10, 7, 9 + addi 16, 9, 32 + addi 17, 10, 32 + addi 18, 16, 32 + addi 19, 17, 32 + addi 20, 18, 32 + addi 21, 19, 32 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + li 9, 256 + add 10, 7, 9 + addi 16, 9, 32 + addi 17, 10, 32 + addi 18, 16, 32 + addi 19, 17, 32 + addi 20, 18, 32 + addi 21, 19, 32 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + li 9, 384 + add 10, 7, 9 + addi 16, 9, 32 + addi 17, 10, 32 + addi 18, 16, 32 + addi 19, 17, 32 + addi 20, 18, 32 + addi 21, 19, 32 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + mr 5, 3 + li 7, 8 + li 10, 16 + li 11, 32 + li 12, 48 + li 15, 64 + li 16, 80 + li 17, 96 + li 18, 112 + nop + ori 2, 2, 0 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + lxvd2x 1, 0, 5 + lxvd2x 2, 10, 5 + xxmrgld 45, 2, 1 + xxmrghd 44, 2, 1 + lxvd2x 3, 11, 5 + lxvd2x 4, 12, 5 + xxmrgld 50, 4, 3 + xxmrghd 49, 4, 3 + lxvd2x 1, 15, 5 + lxvd2x 2, 16, 5 + xxmrgld 55, 2, 1 + xxmrghd 54, 2, 1 + lxvd2x 3, 17, 5 + lxvd2x 4, 18, 5 + xxmrgld 60, 4, 3 + xxmrghd 59, 4, 3 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + xxmrgld 0, 47, 48 + xxmrghd 1, 47, 48 + xxmrgld 2, 52, 53 + xxmrghd 3, 52, 53 + xxmrgld 4, 57, 58 + xxmrghd 5, 57, 58 + xxmrgld 6, 62, 63 + xxmrghd 7, 62, 63 + stxvd2x 0, 0, 5 + stxvd2x 1, 10, 5 + stxvd2x 2, 11, 5 + stxvd2x 3, 12, 5 + stxvd2x 4, 15, 5 + stxvd2x 5, 16, 5 + stxvd2x 6, 17, 5 + stxvd2x 7, 18, 5 + addi 5, 5, 128 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + lxvd2x 1, 0, 5 + lxvd2x 2, 10, 5 + xxmrgld 45, 2, 1 + xxmrghd 44, 2, 1 + lxvd2x 3, 11, 5 + lxvd2x 4, 12, 5 + xxmrgld 50, 4, 3 + xxmrghd 49, 4, 3 + lxvd2x 1, 15, 5 + lxvd2x 2, 16, 5 + xxmrgld 55, 2, 1 + xxmrghd 54, 2, 1 + lxvd2x 3, 17, 5 + lxvd2x 4, 18, 5 + xxmrgld 60, 4, 3 + xxmrghd 59, 4, 3 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + xxmrgld 0, 47, 48 + xxmrghd 1, 47, 48 + xxmrgld 2, 52, 53 + xxmrghd 3, 52, 53 + xxmrgld 4, 57, 58 + xxmrghd 5, 57, 58 + xxmrgld 6, 62, 63 + xxmrghd 7, 62, 63 + stxvd2x 0, 0, 5 + stxvd2x 1, 10, 5 + stxvd2x 2, 11, 5 + stxvd2x 3, 12, 5 + stxvd2x 4, 15, 5 + stxvd2x 5, 16, 5 + stxvd2x 6, 17, 5 + stxvd2x 7, 18, 5 + addi 5, 5, 128 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + lxvd2x 1, 0, 5 + lxvd2x 2, 10, 5 + xxmrgld 45, 2, 1 + xxmrghd 44, 2, 1 + lxvd2x 3, 11, 5 + lxvd2x 4, 12, 5 + xxmrgld 50, 4, 3 + xxmrghd 49, 4, 3 + lxvd2x 1, 15, 5 + lxvd2x 2, 16, 5 + xxmrgld 55, 2, 1 + xxmrghd 54, 2, 1 + lxvd2x 3, 17, 5 + lxvd2x 4, 18, 5 + xxmrgld 60, 4, 3 + xxmrghd 59, 4, 3 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + xxmrgld 0, 47, 48 + xxmrghd 1, 47, 48 + xxmrgld 2, 52, 53 + xxmrghd 3, 52, 53 + xxmrgld 4, 57, 58 + xxmrghd 5, 57, 58 + xxmrgld 6, 62, 63 + xxmrghd 7, 62, 63 + stxvd2x 0, 0, 5 + stxvd2x 1, 10, 5 + stxvd2x 2, 11, 5 + stxvd2x 3, 12, 5 + stxvd2x 4, 15, 5 + stxvd2x 5, 16, 5 + stxvd2x 6, 17, 5 + stxvd2x 7, 18, 5 + addi 5, 5, 128 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + lxvd2x 1, 0, 5 + lxvd2x 2, 10, 5 + xxmrgld 45, 2, 1 + xxmrghd 44, 2, 1 + lxvd2x 3, 11, 5 + lxvd2x 4, 12, 5 + xxmrgld 50, 4, 3 + xxmrghd 49, 4, 3 + lxvd2x 1, 15, 5 + lxvd2x 2, 16, 5 + xxmrgld 55, 2, 1 + xxmrghd 54, 2, 1 + lxvd2x 3, 17, 5 + lxvd2x 4, 18, 5 + xxmrgld 60, 4, 3 + xxmrghd 59, 4, 3 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + xxmrgld 0, 47, 48 + xxmrghd 1, 47, 48 + xxmrgld 2, 52, 53 + xxmrghd 3, 52, 53 + xxmrgld 4, 57, 58 + xxmrghd 5, 57, 58 + xxmrgld 6, 62, 63 + xxmrghd 7, 62, 63 + stxvd2x 0, 0, 5 + stxvd2x 1, 10, 5 + stxvd2x 2, 11, 5 + stxvd2x 3, 12, 5 + stxvd2x 4, 15, 5 + stxvd2x 5, 16, 5 + stxvd2x 6, 17, 5 + stxvd2x 7, 18, 5 + addi 5, 5, 128 + mr 5, 3 + li 7, 4 + nop + ori 2, 2, 0 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + lxvd2x 57, 0, 5 + lxvd2x 58, 10, 5 + vmrgew 13, 25, 26 + vmrgow 12, 25, 26 + lxvd2x 57, 11, 5 + lxvd2x 58, 12, 5 + vmrgew 18, 25, 26 + vmrgow 17, 25, 26 + lxvd2x 57, 15, 5 + lxvd2x 58, 16, 5 + vmrgew 23, 25, 26 + vmrgow 22, 25, 26 + lxvd2x 57, 17, 5 + lxvd2x 58, 18, 5 + vmrgew 28, 25, 26 + vmrgow 27, 25, 26 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + vmrgew 10, 16, 15 + vmrgow 11, 16, 15 + vmrgew 12, 21, 20 + vmrgow 13, 21, 20 + vmrgew 14, 26, 25 + vmrgow 15, 26, 25 + vmrgew 16, 31, 30 + vmrgow 17, 31, 30 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + lxvd2x 57, 0, 5 + lxvd2x 58, 10, 5 + vmrgew 13, 25, 26 + vmrgow 12, 25, 26 + lxvd2x 57, 11, 5 + lxvd2x 58, 12, 5 + vmrgew 18, 25, 26 + vmrgow 17, 25, 26 + lxvd2x 57, 15, 5 + lxvd2x 58, 16, 5 + vmrgew 23, 25, 26 + vmrgow 22, 25, 26 + lxvd2x 57, 17, 5 + lxvd2x 58, 18, 5 + vmrgew 28, 25, 26 + vmrgow 27, 25, 26 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + vmrgew 10, 16, 15 + vmrgow 11, 16, 15 + vmrgew 12, 21, 20 + vmrgow 13, 21, 20 + vmrgew 14, 26, 25 + vmrgow 15, 26, 25 + vmrgew 16, 31, 30 + vmrgow 17, 31, 30 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + lxvd2x 57, 0, 5 + lxvd2x 58, 10, 5 + vmrgew 13, 25, 26 + vmrgow 12, 25, 26 + lxvd2x 57, 11, 5 + lxvd2x 58, 12, 5 + vmrgew 18, 25, 26 + vmrgow 17, 25, 26 + lxvd2x 57, 15, 5 + lxvd2x 58, 16, 5 + vmrgew 23, 25, 26 + vmrgow 22, 25, 26 + lxvd2x 57, 17, 5 + lxvd2x 58, 18, 5 + vmrgew 28, 25, 26 + vmrgow 27, 25, 26 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + vmrgew 10, 16, 15 + vmrgow 11, 16, 15 + vmrgew 12, 21, 20 + vmrgow 13, 21, 20 + vmrgew 14, 26, 25 + vmrgow 15, 26, 25 + vmrgew 16, 31, 30 + vmrgow 17, 31, 30 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + lxvd2x 57, 0, 5 + lxvd2x 58, 10, 5 + vmrgew 13, 25, 26 + vmrgow 12, 25, 26 + lxvd2x 57, 11, 5 + lxvd2x 58, 12, 5 + vmrgew 18, 25, 26 + vmrgow 17, 25, 26 + lxvd2x 57, 15, 5 + lxvd2x 58, 16, 5 + vmrgew 23, 25, 26 + vmrgow 22, 25, 26 + lxvd2x 57, 17, 5 + lxvd2x 58, 18, 5 + vmrgew 28, 25, 26 + vmrgow 27, 25, 26 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + vmrgew 10, 16, 15 + vmrgow 11, 16, 15 + vmrgew 12, 21, 20 + vmrgow 13, 21, 20 + vmrgew 14, 26, 25 + vmrgow 15, 26, 25 + vmrgew 16, 31, 30 + vmrgow 17, 31, 30 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + li 10, 128 + li 11, 144 + li 12, 160 + li 14, 176 + li 15, 192 + li 16, 208 + lxvd2x 52, 10, 1 + lxvd2x 53, 11, 1 + lxvd2x 54, 12, 1 + lxvd2x 55, 14, 1 + lxvd2x 56, 15, 1 + lxvd2x 57, 16, 1 + li 10, 224 + li 11, 240 + li 12, 256 + li 14, 272 + li 15, 288 + li 16, 304 + lxvd2x 58, 10, 1 + lxvd2x 59, 11, 1 + lxvd2x 60, 12, 1 + lxvd2x 61, 14, 1 + lxvd2x 62, 15, 1 + lxvd2x 63, 16, 1 + ld 14, 56(1) + ld 15, 64(1) + ld 16, 72(1) + ld 17, 80(1) + ld 18, 88(1) + ld 19, 96(1) + ld 20, 104(1) + ld 21, 112(1) + ld 22, 120(1) + mtlr 0 + addi 1, 1, 352 + blr + +MLK_ASM_FN_SIZE(ntt_ppc_asm) + +#endif /* MLK_ARITH_BACKEND_PPC64LE_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && __POWER8_VECTOR__ */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_poly_tomont_ppc_asm.S b/src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_poly_tomont_ppc_asm.S new file mode 100644 index 0000000000..3e430638d3 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_poly_tomont_ppc_asm.S @@ -0,0 +1,316 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) IBM Corp. 2025, 2026 + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + * + * Written by Danny Tsen + */ + +/* + * Poly_tomont: Inplace conversion of all coefficients of a polynomial + * from normal domain to Montgomery domain + * + * Arguments:*r: pointer to input/output polynomial + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_PPC64LE_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(__POWER8_VECTOR__) && \ + !defined(MLK_CONFIG_NO_KEYPAIR_API) +/*yaml + Name: poly_tomont_ppc_asm + Description: PowerPC64 VSX conversion to Montgomery form + Signature: void mlk_poly_tomont_ppc_asm(int16_t *r, const int16_t *qdata) + ABI: + Architecture: powerpc64le + CallingConvention: ELFv2 + Features: [VSX] + r3: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) + r4: + type: buffer + size_bytes: 4144 + permissions: read-only + c_parameter: const int16_t *qdata + description: Precomputed constants (2072 x int16_t) +*/ + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/ppc64le/src/mlkem_poly_tomont_ppc_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_tomont_ppc_asm) +MLK_ASM_FN_SYMBOL(poly_tomont_ppc_asm) + + stdu 1, -320(1) + mflr 0 + li 6, 128 + li 7, 144 + li 8, 160 + li 9, 176 + li 10, 192 + li 11, 208 + li 12, 224 + stxvd2x 52, 6, 1 + stxvd2x 53, 7, 1 + stxvd2x 54, 8, 1 + stxvd2x 55, 9, 1 + stxvd2x 56, 10, 1 + stxvd2x 57, 11, 1 + stxvd2x 58, 12, 1 + li 6, 240 + li 7, 256 + li 8, 272 + li 9, 288 + stxvd2x 59, 6, 1 + stxvd2x 60, 7, 1 + stxvd2x 61, 8, 1 + stxvd2x 62, 9, 1 + li 6, 0 + li 7, 80 + li 8, 96 + lxvd2x 37, 6, 4 + lxvd2x 32, 7, 4 + lxvd2x 34, 8, 4 + vxor 3, 3, 3 + li 4, -128 + li 5, -112 + li 6, -96 + li 7, -80 + li 8, -64 + li 9, -48 + li 10, -32 + li 11, -16 + lxvd2x 45, 0, 3 + addi 3, 3, 16 + lxvd2x 50, 0, 3 + addi 3, 3, 16 + lxvd2x 55, 0, 3 + addi 3, 3, 16 + lxvd2x 39, 0, 3 + addi 3, 3, 16 + vmhraddshs 14, 13, 2, 3 + vmhraddshs 19, 18, 2, 3 + vmhraddshs 24, 23, 2, 3 + vmhraddshs 8, 7, 2, 3 + vmladduhm 13, 13, 0, 3 + vmladduhm 18, 18, 0, 3 + vmladduhm 23, 23, 0, 3 + vmladduhm 7, 7, 0, 3 + vmladduhm 27, 14, 5, 13 + vmladduhm 28, 19, 5, 18 + vmladduhm 29, 24, 5, 23 + vmladduhm 30, 8, 5, 7 + lxvd2x 45, 0, 3 + addi 3, 3, 16 + lxvd2x 50, 0, 3 + addi 3, 3, 16 + lxvd2x 55, 0, 3 + addi 3, 3, 16 + lxvd2x 39, 0, 3 + addi 3, 3, 16 + vmhraddshs 14, 13, 2, 3 + vmhraddshs 19, 18, 2, 3 + vmhraddshs 24, 23, 2, 3 + vmhraddshs 8, 7, 2, 3 + vmladduhm 13, 13, 0, 3 + vmladduhm 18, 18, 0, 3 + vmladduhm 23, 23, 0, 3 + vmladduhm 7, 7, 0, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 7, 8, 5, 7 + stxvd2x 59, 4, 3 + stxvd2x 60, 5, 3 + stxvd2x 61, 6, 3 + stxvd2x 62, 7, 3 + stxvd2x 45, 8, 3 + stxvd2x 50, 9, 3 + stxvd2x 55, 10, 3 + stxvd2x 39, 11, 3 + lxvd2x 45, 0, 3 + addi 3, 3, 16 + lxvd2x 50, 0, 3 + addi 3, 3, 16 + lxvd2x 55, 0, 3 + addi 3, 3, 16 + lxvd2x 39, 0, 3 + addi 3, 3, 16 + vmhraddshs 14, 13, 2, 3 + vmhraddshs 19, 18, 2, 3 + vmhraddshs 24, 23, 2, 3 + vmhraddshs 8, 7, 2, 3 + vmladduhm 13, 13, 0, 3 + vmladduhm 18, 18, 0, 3 + vmladduhm 23, 23, 0, 3 + vmladduhm 7, 7, 0, 3 + vmladduhm 27, 14, 5, 13 + vmladduhm 28, 19, 5, 18 + vmladduhm 29, 24, 5, 23 + vmladduhm 30, 8, 5, 7 + lxvd2x 45, 0, 3 + addi 3, 3, 16 + lxvd2x 50, 0, 3 + addi 3, 3, 16 + lxvd2x 55, 0, 3 + addi 3, 3, 16 + lxvd2x 39, 0, 3 + addi 3, 3, 16 + vmhraddshs 14, 13, 2, 3 + vmhraddshs 19, 18, 2, 3 + vmhraddshs 24, 23, 2, 3 + vmhraddshs 8, 7, 2, 3 + vmladduhm 13, 13, 0, 3 + vmladduhm 18, 18, 0, 3 + vmladduhm 23, 23, 0, 3 + vmladduhm 7, 7, 0, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 7, 8, 5, 7 + stxvd2x 59, 4, 3 + stxvd2x 60, 5, 3 + stxvd2x 61, 6, 3 + stxvd2x 62, 7, 3 + stxvd2x 45, 8, 3 + stxvd2x 50, 9, 3 + stxvd2x 55, 10, 3 + stxvd2x 39, 11, 3 + lxvd2x 45, 0, 3 + addi 3, 3, 16 + lxvd2x 50, 0, 3 + addi 3, 3, 16 + lxvd2x 55, 0, 3 + addi 3, 3, 16 + lxvd2x 39, 0, 3 + addi 3, 3, 16 + vmhraddshs 14, 13, 2, 3 + vmhraddshs 19, 18, 2, 3 + vmhraddshs 24, 23, 2, 3 + vmhraddshs 8, 7, 2, 3 + vmladduhm 13, 13, 0, 3 + vmladduhm 18, 18, 0, 3 + vmladduhm 23, 23, 0, 3 + vmladduhm 7, 7, 0, 3 + vmladduhm 27, 14, 5, 13 + vmladduhm 28, 19, 5, 18 + vmladduhm 29, 24, 5, 23 + vmladduhm 30, 8, 5, 7 + lxvd2x 45, 0, 3 + addi 3, 3, 16 + lxvd2x 50, 0, 3 + addi 3, 3, 16 + lxvd2x 55, 0, 3 + addi 3, 3, 16 + lxvd2x 39, 0, 3 + addi 3, 3, 16 + vmhraddshs 14, 13, 2, 3 + vmhraddshs 19, 18, 2, 3 + vmhraddshs 24, 23, 2, 3 + vmhraddshs 8, 7, 2, 3 + vmladduhm 13, 13, 0, 3 + vmladduhm 18, 18, 0, 3 + vmladduhm 23, 23, 0, 3 + vmladduhm 7, 7, 0, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 7, 8, 5, 7 + stxvd2x 59, 4, 3 + stxvd2x 60, 5, 3 + stxvd2x 61, 6, 3 + stxvd2x 62, 7, 3 + stxvd2x 45, 8, 3 + stxvd2x 50, 9, 3 + stxvd2x 55, 10, 3 + stxvd2x 39, 11, 3 + lxvd2x 45, 0, 3 + addi 3, 3, 16 + lxvd2x 50, 0, 3 + addi 3, 3, 16 + lxvd2x 55, 0, 3 + addi 3, 3, 16 + lxvd2x 39, 0, 3 + addi 3, 3, 16 + vmhraddshs 14, 13, 2, 3 + vmhraddshs 19, 18, 2, 3 + vmhraddshs 24, 23, 2, 3 + vmhraddshs 8, 7, 2, 3 + vmladduhm 13, 13, 0, 3 + vmladduhm 18, 18, 0, 3 + vmladduhm 23, 23, 0, 3 + vmladduhm 7, 7, 0, 3 + vmladduhm 27, 14, 5, 13 + vmladduhm 28, 19, 5, 18 + vmladduhm 29, 24, 5, 23 + vmladduhm 30, 8, 5, 7 + lxvd2x 45, 0, 3 + addi 3, 3, 16 + lxvd2x 50, 0, 3 + addi 3, 3, 16 + lxvd2x 55, 0, 3 + addi 3, 3, 16 + lxvd2x 39, 0, 3 + addi 3, 3, 16 + vmhraddshs 14, 13, 2, 3 + vmhraddshs 19, 18, 2, 3 + vmhraddshs 24, 23, 2, 3 + vmhraddshs 8, 7, 2, 3 + vmladduhm 13, 13, 0, 3 + vmladduhm 18, 18, 0, 3 + vmladduhm 23, 23, 0, 3 + vmladduhm 7, 7, 0, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 7, 8, 5, 7 + stxvd2x 59, 4, 3 + stxvd2x 60, 5, 3 + stxvd2x 61, 6, 3 + stxvd2x 62, 7, 3 + stxvd2x 45, 8, 3 + stxvd2x 50, 9, 3 + stxvd2x 55, 10, 3 + stxvd2x 39, 11, 3 + li 6, 128 + li 7, 144 + li 8, 160 + li 9, 176 + li 10, 192 + li 11, 208 + li 12, 224 + lxvd2x 52, 6, 1 + lxvd2x 53, 7, 1 + lxvd2x 54, 8, 1 + lxvd2x 55, 9, 1 + lxvd2x 56, 10, 1 + lxvd2x 57, 11, 1 + lxvd2x 58, 12, 1 + li 6, 240 + li 7, 256 + li 8, 272 + li 9, 288 + lxvd2x 59, 6, 1 + lxvd2x 60, 7, 1 + lxvd2x 61, 8, 1 + lxvd2x 62, 9, 1 + mtlr 0 + addi 1, 1, 320 + blr + +MLK_ASM_FN_SIZE(poly_tomont_ppc_asm) + +#endif /* MLK_ARITH_BACKEND_PPC64LE_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && __POWER8_VECTOR__ && !MLK_CONFIG_NO_KEYPAIR_API */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_reduce_ppc_asm.S b/src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_reduce_ppc_asm.S new file mode 100644 index 0000000000..e82376c74b --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_reduce_ppc_asm.S @@ -0,0 +1,731 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) IBM Corp. 2025, 2026 + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + * + * Written by Danny Tsen + */ +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_PPC64LE_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(__POWER8_VECTOR__) +/*yaml + Name: reduce_ppc_asm + Description: PowerPC64 VSX modular reduction + Signature: void mlk_reduce_ppc_asm(int16_t *r, const int16_t *qdata) + ABI: + Architecture: powerpc64le + CallingConvention: ELFv2 + Features: [VSX] + r3: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) + r4: + type: buffer + size_bytes: 4144 + permissions: read-only + c_parameter: const int16_t *qdata + description: Precomputed constants (2072 x int16_t) +*/ + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/ppc64le/src/mlkem_reduce_ppc_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(reduce_ppc_asm) +MLK_ASM_FN_SYMBOL(reduce_ppc_asm) + + stdu 1, -224(1) + mflr 0 + std 14, 96(1) + std 15, 104(1) + std 16, 112(1) + li 6, 128 + li 7, 144 + li 8, 160 + li 9, 176 + li 10, 192 + stxvd2x 52, 6, 1 + stxvd2x 53, 7, 1 + stxvd2x 54, 8, 1 + stxvd2x 55, 9, 1 + stxvd2x 56, 10, 1 + vxor 7, 7, 7 + li 6, 16 + li 7, 32 + lxvd2x 35, 6, 4 + lxvd2x 32, 7, 4 + vspltisw 2, 13 + vadduwm 2, 2, 2 + vspltisw 4, 1 + vsubuwm 5, 2, 4 + vslw 1, 4, 5 + li 4, -128 + li 5, -112 + li 6, -96 + li 7, -80 + li 8, -64 + li 9, -48 + li 10, -32 + li 11, -16 + li 14, 16 + li 15, 32 + li 16, 48 + lxvd2x 40, 0, 3 + lxvd2x 44, 14, 3 + lxvd2x 48, 15, 3 + lxvd2x 52, 16, 3 + addi 3, 3, 64 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 21, 4, 3, 8 + vmladduhm 22, 9, 3, 12 + vmladduhm 23, 13, 3, 16 + vmladduhm 24, 17, 3, 20 + lxvd2x 40, 0, 3 + lxvd2x 44, 14, 3 + lxvd2x 48, 15, 3 + lxvd2x 52, 16, 3 + addi 3, 3, 64 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 53, 4, 3 + stxvd2x 54, 5, 3 + stxvd2x 55, 6, 3 + stxvd2x 56, 7, 3 + stxvd2x 36, 8, 3 + stxvd2x 41, 9, 3 + stxvd2x 45, 10, 3 + stxvd2x 49, 11, 3 + lxvd2x 40, 0, 3 + lxvd2x 44, 14, 3 + lxvd2x 48, 15, 3 + lxvd2x 52, 16, 3 + addi 3, 3, 64 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 21, 4, 3, 8 + vmladduhm 22, 9, 3, 12 + vmladduhm 23, 13, 3, 16 + vmladduhm 24, 17, 3, 20 + lxvd2x 40, 0, 3 + lxvd2x 44, 14, 3 + lxvd2x 48, 15, 3 + lxvd2x 52, 16, 3 + addi 3, 3, 64 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 53, 4, 3 + stxvd2x 54, 5, 3 + stxvd2x 55, 6, 3 + stxvd2x 56, 7, 3 + stxvd2x 36, 8, 3 + stxvd2x 41, 9, 3 + stxvd2x 45, 10, 3 + stxvd2x 49, 11, 3 + lxvd2x 40, 0, 3 + lxvd2x 44, 14, 3 + lxvd2x 48, 15, 3 + lxvd2x 52, 16, 3 + addi 3, 3, 64 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 21, 4, 3, 8 + vmladduhm 22, 9, 3, 12 + vmladduhm 23, 13, 3, 16 + vmladduhm 24, 17, 3, 20 + lxvd2x 40, 0, 3 + lxvd2x 44, 14, 3 + lxvd2x 48, 15, 3 + lxvd2x 52, 16, 3 + addi 3, 3, 64 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 53, 4, 3 + stxvd2x 54, 5, 3 + stxvd2x 55, 6, 3 + stxvd2x 56, 7, 3 + stxvd2x 36, 8, 3 + stxvd2x 41, 9, 3 + stxvd2x 45, 10, 3 + stxvd2x 49, 11, 3 + lxvd2x 40, 0, 3 + lxvd2x 44, 14, 3 + lxvd2x 48, 15, 3 + lxvd2x 52, 16, 3 + addi 3, 3, 64 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 21, 4, 3, 8 + vmladduhm 22, 9, 3, 12 + vmladduhm 23, 13, 3, 16 + vmladduhm 24, 17, 3, 20 + lxvd2x 40, 0, 3 + lxvd2x 44, 14, 3 + lxvd2x 48, 15, 3 + lxvd2x 52, 16, 3 + addi 3, 3, 64 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 53, 4, 3 + stxvd2x 54, 5, 3 + stxvd2x 55, 6, 3 + stxvd2x 56, 7, 3 + stxvd2x 36, 8, 3 + stxvd2x 41, 9, 3 + stxvd2x 45, 10, 3 + stxvd2x 49, 11, 3 + addi 3, 3, -512 + vxor 9, 9, 9 + vspltish 10, 15 + vmr 11, 3 + lxvd2x 44, 0, 3 + lxvd2x 45, 14, 3 + lxvd2x 46, 15, 3 + lxvd2x 47, 16, 3 + addi 3, 3, 64 + vsrh 1, 12, 10 + vsrh 0, 13, 10 + vsrh 3, 14, 10 + vsrh 2, 15, 10 + vadduhm 7, 12, 11 + vadduhm 8, 13, 11 + vadduhm 5, 14, 11 + vadduhm 6, 15, 11 + vcmpequh 1, 1, 9 + vcmpequh 0, 0, 9 + vcmpequh 3, 3, 9 + vcmpequh 2, 2, 9 + xxsel 33, 39, 44, 33 + xxsel 32, 40, 45, 32 + xxsel 35, 37, 46, 35 + xxsel 34, 38, 47, 34 + stxvd2x 35, 10, 3 + stxvd2x 34, 11, 3 + stxvd2x 33, 8, 3 + stxvd2x 32, 9, 3 + lxvd2x 44, 0, 3 + lxvd2x 45, 14, 3 + lxvd2x 46, 15, 3 + lxvd2x 47, 16, 3 + addi 3, 3, 64 + vsrh 1, 12, 10 + vsrh 0, 13, 10 + vsrh 3, 14, 10 + vsrh 2, 15, 10 + vadduhm 7, 12, 11 + vadduhm 8, 13, 11 + vadduhm 5, 14, 11 + vadduhm 6, 15, 11 + vcmpequh 1, 1, 9 + vcmpequh 0, 0, 9 + vcmpequh 3, 3, 9 + vcmpequh 2, 2, 9 + xxsel 33, 39, 44, 33 + xxsel 32, 40, 45, 32 + xxsel 35, 37, 46, 35 + xxsel 34, 38, 47, 34 + stxvd2x 35, 10, 3 + stxvd2x 34, 11, 3 + stxvd2x 33, 8, 3 + stxvd2x 32, 9, 3 + lxvd2x 44, 0, 3 + lxvd2x 45, 14, 3 + lxvd2x 46, 15, 3 + lxvd2x 47, 16, 3 + addi 3, 3, 64 + vsrh 1, 12, 10 + vsrh 0, 13, 10 + vsrh 3, 14, 10 + vsrh 2, 15, 10 + vadduhm 7, 12, 11 + vadduhm 8, 13, 11 + vadduhm 5, 14, 11 + vadduhm 6, 15, 11 + vcmpequh 1, 1, 9 + vcmpequh 0, 0, 9 + vcmpequh 3, 3, 9 + vcmpequh 2, 2, 9 + xxsel 33, 39, 44, 33 + xxsel 32, 40, 45, 32 + xxsel 35, 37, 46, 35 + xxsel 34, 38, 47, 34 + stxvd2x 35, 10, 3 + stxvd2x 34, 11, 3 + stxvd2x 33, 8, 3 + stxvd2x 32, 9, 3 + lxvd2x 44, 0, 3 + lxvd2x 45, 14, 3 + lxvd2x 46, 15, 3 + lxvd2x 47, 16, 3 + addi 3, 3, 64 + vsrh 1, 12, 10 + vsrh 0, 13, 10 + vsrh 3, 14, 10 + vsrh 2, 15, 10 + vadduhm 7, 12, 11 + vadduhm 8, 13, 11 + vadduhm 5, 14, 11 + vadduhm 6, 15, 11 + vcmpequh 1, 1, 9 + vcmpequh 0, 0, 9 + vcmpequh 3, 3, 9 + vcmpequh 2, 2, 9 + xxsel 33, 39, 44, 33 + xxsel 32, 40, 45, 32 + xxsel 35, 37, 46, 35 + xxsel 34, 38, 47, 34 + stxvd2x 35, 10, 3 + stxvd2x 34, 11, 3 + stxvd2x 33, 8, 3 + stxvd2x 32, 9, 3 + lxvd2x 44, 0, 3 + lxvd2x 45, 14, 3 + lxvd2x 46, 15, 3 + lxvd2x 47, 16, 3 + addi 3, 3, 64 + vsrh 1, 12, 10 + vsrh 0, 13, 10 + vsrh 3, 14, 10 + vsrh 2, 15, 10 + vadduhm 7, 12, 11 + vadduhm 8, 13, 11 + vadduhm 5, 14, 11 + vadduhm 6, 15, 11 + vcmpequh 1, 1, 9 + vcmpequh 0, 0, 9 + vcmpequh 3, 3, 9 + vcmpequh 2, 2, 9 + xxsel 33, 39, 44, 33 + xxsel 32, 40, 45, 32 + xxsel 35, 37, 46, 35 + xxsel 34, 38, 47, 34 + stxvd2x 35, 10, 3 + stxvd2x 34, 11, 3 + stxvd2x 33, 8, 3 + stxvd2x 32, 9, 3 + lxvd2x 44, 0, 3 + lxvd2x 45, 14, 3 + lxvd2x 46, 15, 3 + lxvd2x 47, 16, 3 + addi 3, 3, 64 + vsrh 1, 12, 10 + vsrh 0, 13, 10 + vsrh 3, 14, 10 + vsrh 2, 15, 10 + vadduhm 7, 12, 11 + vadduhm 8, 13, 11 + vadduhm 5, 14, 11 + vadduhm 6, 15, 11 + vcmpequh 1, 1, 9 + vcmpequh 0, 0, 9 + vcmpequh 3, 3, 9 + vcmpequh 2, 2, 9 + xxsel 33, 39, 44, 33 + xxsel 32, 40, 45, 32 + xxsel 35, 37, 46, 35 + xxsel 34, 38, 47, 34 + stxvd2x 35, 10, 3 + stxvd2x 34, 11, 3 + stxvd2x 33, 8, 3 + stxvd2x 32, 9, 3 + lxvd2x 44, 0, 3 + lxvd2x 45, 14, 3 + lxvd2x 46, 15, 3 + lxvd2x 47, 16, 3 + addi 3, 3, 64 + vsrh 1, 12, 10 + vsrh 0, 13, 10 + vsrh 3, 14, 10 + vsrh 2, 15, 10 + vadduhm 7, 12, 11 + vadduhm 8, 13, 11 + vadduhm 5, 14, 11 + vadduhm 6, 15, 11 + vcmpequh 1, 1, 9 + vcmpequh 0, 0, 9 + vcmpequh 3, 3, 9 + vcmpequh 2, 2, 9 + xxsel 33, 39, 44, 33 + xxsel 32, 40, 45, 32 + xxsel 35, 37, 46, 35 + xxsel 34, 38, 47, 34 + stxvd2x 35, 10, 3 + stxvd2x 34, 11, 3 + stxvd2x 33, 8, 3 + stxvd2x 32, 9, 3 + lxvd2x 44, 0, 3 + lxvd2x 45, 14, 3 + lxvd2x 46, 15, 3 + lxvd2x 47, 16, 3 + addi 3, 3, 64 + vsrh 1, 12, 10 + vsrh 0, 13, 10 + vsrh 3, 14, 10 + vsrh 2, 15, 10 + vadduhm 7, 12, 11 + vadduhm 8, 13, 11 + vadduhm 5, 14, 11 + vadduhm 6, 15, 11 + vcmpequh 1, 1, 9 + vcmpequh 0, 0, 9 + vcmpequh 3, 3, 9 + vcmpequh 2, 2, 9 + xxsel 33, 39, 44, 33 + xxsel 32, 40, 45, 32 + xxsel 35, 37, 46, 35 + xxsel 34, 38, 47, 34 + stxvd2x 35, 10, 3 + stxvd2x 34, 11, 3 + stxvd2x 33, 8, 3 + stxvd2x 32, 9, 3 + ld 14, 96(1) + ld 15, 104(1) + ld 16, 112(1) + li 6, 128 + li 7, 144 + li 8, 160 + li 9, 176 + li 10, 192 + lxvd2x 52, 6, 1 + lxvd2x 53, 7, 1 + lxvd2x 54, 8, 1 + lxvd2x 55, 9, 1 + lxvd2x 56, 10, 1 + mtlr 0 + addi 1, 1, 224 + blr + +MLK_ASM_FN_SIZE(reduce_ppc_asm) + +#endif /* MLK_ARITH_BACKEND_PPC64LE_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && __POWER8_VECTOR__ */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/riscv64/README.md b/src/mlkem-native/mlkem/src/native/riscv64/README.md new file mode 100644 index 0000000000..56ebe937e4 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/riscv64/README.md @@ -0,0 +1,11 @@ +[//]: # (SPDX-License-Identifier: CC-BY-4.0) + +# RISC-V Vector Extension Backend + +This is an arithmetic backend for CPUs implementing the RISC-V Vector Extension. The backend is functional for all physical `VLEN`, but the NTT and inverse NTT are so far only implemented for VLEN=256, falling back to the default C implementations for other VLENs. + +## Requirements + +- RISC-V 64-bit architecture +- Vector extension (RVV) version 1.0 +- Standard "gc" extensions (integer and compressed instructions) diff --git a/src/mlkem-native/mlkem/src/native/riscv64/meta.h b/src/mlkem-native/mlkem/src/native/riscv64/meta.h new file mode 100644 index 0000000000..1dc637615b --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/riscv64/meta.h @@ -0,0 +1,137 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_NATIVE_RISCV64_META_H +#define MLK_NATIVE_RISCV64_META_H + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLK_ARITH_BACKEND_RISCV64 + +/* Set of primitives that this backend replaces */ +#define MLK_USE_NATIVE_NTT +#define MLK_USE_NATIVE_INTT +#define MLK_USE_NATIVE_POLY_TOMONT +#define MLK_USE_NATIVE_REJ_UNIFORM +#define MLK_USE_NATIVE_POLY_REDUCE +#define MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#define MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED + +#include "../../common.h" + +#if !defined(__ASSEMBLER__) +#include + +#include "../api.h" +#include "src/arith_native_riscv64.h" + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_ntt_native(int16_t data[MLKEM_N]) +{ + /* VLEN = 256 only for now */ + if (__riscv_vsetvlmax_e16m1() != 16) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_rv64v_poly_ntt(data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_intt_native(int16_t data[MLKEM_N]) +{ + /* VLEN = 256 only for now */ + if (__riscv_vsetvlmax_e16m1() != 16) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_rv64v_poly_invntt_tomont(data); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_tomont_native(int16_t data[MLKEM_N]) +{ + mlk_rv64v_poly_tomont(data); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_rej_uniform_native(int16_t *r, unsigned len, + const uint8_t *buf, + unsigned buflen) +{ + /* mlk_rv64v_rej_uniform requires buflen to be a multiple of 12 */ + if (buflen % 12 != 0) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + /* The cast from unsigned to signed integer is safe + * because the return value is <= len, which we asssume + * to be bound by 4096 and hence <= INT_MAX. */ + return (int)mlk_rv64v_rej_uniform(r, len, buf, buflen); +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_reduce_native(int16_t data[MLKEM_N]) +{ + mlk_rv64v_poly_reduce(data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_mulcache_compute_native(int16_t x[MLKEM_N / 2], + const int16_t y[MLKEM_N]) +{ + (void)x; /* not using the cache at the moment */ + (void)y; + return MLK_NATIVE_FUNC_SUCCESS; +} + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k2_native( + int16_t r[MLKEM_N], const int16_t a[2 * MLKEM_N], + const int16_t b[2 * MLKEM_N], const int16_t b_cache[2 * (MLKEM_N / 2)]) +{ + (void)b_cache; + mlk_rv64v_poly_basemul_mont_add_k2(r, a, b); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k3_native( + int16_t r[MLKEM_N], const int16_t a[3 * MLKEM_N], + const int16_t b[3 * MLKEM_N], const int16_t b_cache[3 * (MLKEM_N / 2)]) +{ + (void)b_cache; + mlk_rv64v_poly_basemul_mont_add_k3(r, a, b); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k4_native( + int16_t r[MLKEM_N], const int16_t a[4 * MLKEM_N], + const int16_t b[4 * MLKEM_N], const int16_t b_cache[4 * (MLKEM_N / 2)]) +{ + (void)b_cache; + mlk_rv64v_poly_basemul_mont_add_k4(r, a, b); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */ + +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_NATIVE_RISCV64_META_H */ diff --git a/src/mlkem-native/mlkem/src/native/riscv64/src/arith_native_riscv64.h b/src/mlkem-native/mlkem/src/native/riscv64/src/arith_native_riscv64.h new file mode 100644 index 0000000000..ab97ca9eb2 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/riscv64/src/arith_native_riscv64.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H +#define MLK_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H + +#include "../../../common.h" + +#define mlk_rv64v_poly_ntt MLK_NAMESPACE(ntt_riscv64) +void mlk_rv64v_poly_ntt(int16_t *); + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_rv64v_poly_invntt_tomont MLK_NAMESPACE(intt_riscv64) +void mlk_rv64v_poly_invntt_tomont(int16_t *r); +#endif + +#define mlk_rv64v_poly_basemul_mont_add_k2 MLK_NAMESPACE(basemul_add_k2_riscv64) +void mlk_rv64v_poly_basemul_mont_add_k2(int16_t *r, const int16_t *a, + const int16_t *b); + +#define mlk_rv64v_poly_basemul_mont_add_k3 MLK_NAMESPACE(basemul_add_k3_riscv64) +void mlk_rv64v_poly_basemul_mont_add_k3(int16_t *r, const int16_t *a, + const int16_t *b); + +#define mlk_rv64v_poly_basemul_mont_add_k4 MLK_NAMESPACE(basemul_add_k4_riscv64) +void mlk_rv64v_poly_basemul_mont_add_k4(int16_t *r, const int16_t *a, + const int16_t *b); + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +#define mlk_rv64v_poly_tomont MLK_NAMESPACE(tomont_riscv64) +void mlk_rv64v_poly_tomont(int16_t *r); +#endif + +#define mlk_rv64v_poly_reduce MLK_NAMESPACE(reduce_riscv64) +void mlk_rv64v_poly_reduce(int16_t *r); + +#define mlk_rv64v_poly_add MLK_NAMESPACE(poly_add_riscv64) +void mlk_rv64v_poly_add(int16_t *r, const int16_t *a, const int16_t *b); + +#define mlk_rv64v_poly_sub MLK_NAMESPACE(poly_sub_riscv64) +void mlk_rv64v_poly_sub(int16_t *r, const int16_t *a, const int16_t *b); + +#define mlk_rv64v_rej_uniform MLK_NAMESPACE(rj_uniform_riscv64) +MLK_MUST_CHECK_RETURN_VALUE +unsigned int mlk_rv64v_rej_uniform(int16_t *r, unsigned int len, + const uint8_t *buf, unsigned int buflen); + +#endif /* !MLK_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H */ diff --git a/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_debug.c b/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_debug.c new file mode 100644 index 0000000000..46baa07998 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_debug.c @@ -0,0 +1,83 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* NOTE: You can remove this file unless you compile with MLKEM_DEBUG. */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_RISCV64) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(MLKEM_DEBUG) + +#include +#include + +#include "../../../debug.h" +#include "rv64v_debug.h" + +#define MLK_DEBUG_ERROR_HEADER "[ERROR:%s:%04d] " + +/** + * Check whether values in a vint16m1_t vector are within specified bounds. + * + * @param[in] file Filename. + * @param line Line number. + * @param vec RISC-V vector to be checked. + * @param vl Vector length (number of active elements). + * @param lower_bound_exclusive Exclusive lower bound. + * @param upper_bound_exclusive Exclusive upper bound. + */ +void mlk_debug_check_bounds_int16m1(const char *file, int line, vint16m1_t vec, + size_t vl, int lower_bound_exclusive, + int upper_bound_exclusive) +{ + /* Allocate temporary array to store vector elements + * We use the maximum possible vector length to be safe */ + int16_t temp_array[64]; + + /* Store vector elements to temporary array for inspection */ + __riscv_vse16_v_i16m1(temp_array, vec, vl); + + /* Reuse existing array bounds checking function */ + mlk_debug_check_bounds(file, line, temp_array, (unsigned)vl, + lower_bound_exclusive, upper_bound_exclusive); +} + +/** + * Check whether values in a vint16m2_t vector are within specified bounds. + * + * @param[in] file Filename. + * @param line Line number. + * @param vec RISC-V vector to be checked. + * @param vl Vector length (active elements per m1 half). + * @param lower_bound_exclusive Exclusive lower bound. + * @param upper_bound_exclusive Exclusive upper bound. + */ +void mlk_debug_check_bounds_int16m2(const char *file, int line, vint16m2_t vec, + size_t vl, int lower_bound_exclusive, + int upper_bound_exclusive) +{ + /* Allocate temporary array to store vector elements + * m2 vectors hold 2x the elements of m1 vectors */ + int16_t temp_array[2 * 64]; + + /* Store vector elements to temporary array for inspection */ + __riscv_vse16_v_i16m2(temp_array, vec, 2 * vl); + + /* Reuse existing array bounds checking function for all elements */ + mlk_debug_check_bounds(file, line, temp_array, (unsigned)(2 * vl), + lower_bound_exclusive, upper_bound_exclusive); +} + +#else /* MLK_ARITH_BACKEND_RISCV64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + MLKEM_DEBUG */ + +MLK_EMPTY_CU(rv64v_debug) + +#endif /* !(MLK_ARITH_BACKEND_RISCV64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + MLKEM_DEBUG) */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef MLK_DEBUG_ERROR_HEADER diff --git a/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_debug.h b/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_debug.h new file mode 100644 index 0000000000..18d4b123f8 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_debug.h @@ -0,0 +1,140 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_NATIVE_RISCV64_SRC_RV64V_DEBUG_H +#define MLK_NATIVE_RISCV64_SRC_RV64V_DEBUG_H + +#include "../../../debug.h" + +#include + +/************************************************* + * RISC-V Vector Bounds Assertion Macros + * + * These macros provide runtime bounds checking for RISC-V vector types + * vint16m1_t and vint16m2_t, following the same pattern as the scalar + * bounds assertions in debug.h + * + * The macros are only active when MLKEM_DEBUG is defined, otherwise they + * compile to no-ops for zero runtime overhead in release builds. + **************************************************/ + +#if defined(MLKEM_DEBUG) + +/** + * Check whether values in a vint16m1_t vector are within specified bounds. + * + * @param[in] file Filename. + * @param line Line number. + * @param vec RISC-V vector to be checked. + * @param vl Vector length (number of active elements). + * @param lower_bound_exclusive Exclusive lower bound. + * @param upper_bound_exclusive Exclusive upper bound. + */ +#define mlk_debug_check_bounds_int16m1 \ + MLK_NAMESPACE(mlkem_debug_check_bounds_int16m1) +void mlk_debug_check_bounds_int16m1(const char *file, int line, vint16m1_t vec, + size_t vl, int lower_bound_exclusive, + int upper_bound_exclusive); + +/** + * Check whether values in a vint16m2_t vector are within specified bounds + * by splitting into m1 vectors. + * + * @param[in] file Filename. + * @param line Line number. + * @param vec RISC-V vector to be checked. + * @param vl Vector length (active elements per m1 half). + * @param lower_bound_exclusive Exclusive lower bound. + * @param upper_bound_exclusive Exclusive upper bound. + */ +#define mlk_debug_check_bounds_int16m2 \ + MLK_NAMESPACE(mlkem_debug_check_bounds_int16m2) +void mlk_debug_check_bounds_int16m2(const char *file, int line, vint16m2_t vec, + size_t vl, int lower_bound_exclusive, + int upper_bound_exclusive); + +/* Check bounds in vint16m1_t vector + * vec: RISC-V vector of type vint16m1_t + * vl: Vector length (number of active elements) + * value_lb: Inclusive lower value bound + * value_ub: Exclusive upper value bound */ +#define mlk_assert_bound_int16m1(vec, vl, value_lb, value_ub) \ + mlk_debug_check_bounds_int16m1(__FILE__, __LINE__, (vec), (vl), \ + (value_lb) - 1, (value_ub)) + +/* Check absolute bounds in vint16m1_t vector + * vec: RISC-V vector of type vint16m1_t + * vl: Vector length (number of active elements) + * value_abs_bd: Exclusive absolute upper bound */ +#define mlk_assert_abs_bound_int16m1(vec, vl, value_abs_bd) \ + mlk_assert_bound_int16m1((vec), (vl), (-(value_abs_bd) + 1), (value_abs_bd)) + +/* Check bounds in vint16m2_t vector + * vec: RISC-V vector of type vint16m2_t + * vl: Vector length (number of active elements per m1 half) + * value_lb: Inclusive lower value bound + * value_ub: Exclusive upper value bound */ +#define mlk_assert_bound_int16m2(vec, vl, value_lb, value_ub) \ + mlk_debug_check_bounds_int16m2(__FILE__, __LINE__, (vec), (vl), \ + (value_lb) - 1, (value_ub)) + +/* Check absolute bounds in vint16m2_t vector + * vec: RISC-V vector of type vint16m2_t + * vl: Vector length (number of active elements per m1 half) + * value_abs_bd: Exclusive absolute upper bound */ +#define mlk_assert_abs_bound_int16m2(vec, vl, value_abs_bd) \ + mlk_assert_bound_int16m2((vec), (vl), (-(value_abs_bd) + 1), (value_abs_bd)) + +#elif defined(CBMC) + +/* For CBMC, we would need to implement vector bounds checking using CBMC + * primitives This is complex and would require extracting vector elements, so + * for now we provide empty implementations that could be extended later */ +#define mlk_assert_bound_int16m1(vec, vl, value_lb, value_ub) \ + do \ + { \ + } while (0) + +#define mlk_assert_abs_bound_int16m1(vec, vl, value_abs_bd) \ + do \ + { \ + } while (0) + +#define mlk_assert_bound_int16m2(vec, vl, value_lb, value_ub) \ + do \ + { \ + } while (0) + +#define mlk_assert_abs_bound_int16m2(vec, vl, value_abs_bd) \ + do \ + { \ + } while (0) + +#else /* !MLKEM_DEBUG && CBMC */ + +/* When debugging is disabled, all assertions become no-ops */ +#define mlk_assert_bound_int16m1(vec, vl, value_lb, value_ub) \ + do \ + { \ + } while (0) + +#define mlk_assert_abs_bound_int16m1(vec, vl, value_abs_bd) \ + do \ + { \ + } while (0) + +#define mlk_assert_bound_int16m2(vec, vl, value_lb, value_ub) \ + do \ + { \ + } while (0) + +#define mlk_assert_abs_bound_int16m2(vec, vl, value_abs_bd) \ + do \ + { \ + } while (0) + +#endif /* !MLKEM_DEBUG && !CBMC */ + +#endif /* !MLK_NATIVE_RISCV64_SRC_RV64V_DEBUG_H */ diff --git a/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_izetas.inc b/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_izetas.inc new file mode 100644 index 0000000000..e24a9abe08 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_izetas.inc @@ -0,0 +1,27 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "arith_native_riscv64.h" + +const int16_t izeta[] = { + -1044, 758, 1571, 205, 1275, -677, 1065, -448, -1628, -1522, 1460, + -958, -991, -996, 308, 108, 1517, 359, -411, 1542, 725, 1508, + -961, 398, -478, 870, 854, 1510, -794, 1278, 1530, 1185, -202, + -287, -608, -732, 951, 247, 1421, -107, 1659, 1187, -220, 874, + 1335, -1218, 136, 1215, -1422, -1493, -1017, 681, -830, 271, 90, + 853, -384, 1465, 1285, -1322, -610, -603, -1097, -817, -1468, 1474, + 130, 1602, -1469, -126, 1162, 1618, 75, 156, -329, -418, -349, + 872, -644, 1590, 1202, -962, -1458, 829, 666, 320, 8, -516, + -1119, 602, -1483, 777, 147, -1159, -778, 246, -182, -1577, -383, + -264, 1544, 282, -1491, 1293, -1653, -1574, 460, 291, 235, -177, + -587, -422, -622, 171, 1325, -573, -1015, 552, -652, -1223, -105, + -1550, -871, 1251, -843, -555, -430, 1103, +}; diff --git a/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_poly.c b/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_poly.c new file mode 100644 index 0000000000..101bdab5dc --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_poly.c @@ -0,0 +1,797 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* === ML-KEM NTT using RISC-V Vector intrinstics */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_RISCV64) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include + +#include "arith_native_riscv64.h" +#include "rv64v_debug.h" + +/* Montgomery reduction constants */ +/* check-magic: -3327 == signed_mod(pow(MLKEM_Q,-1,2^16), 2^16) */ +#define MLK_RVV_QI -3327 + +/* check-magic: 2285 == unsigned_mod(2^16, MLKEM_Q) */ +#define MLK_RVV_MONT_R1 2285 + +/* check-magic: 1353 == pow(2, 32, MLKEM_Q) */ +#define MLK_RVV_MONT_R2 1353 + +/* check-magic: 1441 == pow(2,32-7,MLKEM_Q) */ +#define MLK_RVV_MONT_NR 1441 + +static MLK_INLINE vint16m1_t fq_redc(vint16m1_t rh, vint16m1_t rl, size_t vl) +{ + vint16m1_t t; + + t = __riscv_vmul_vx_i16m1(rl, MLK_RVV_QI, vl); /* t = l * Q^-1 */ + t = __riscv_vmulh_vx_i16m1(t, MLKEM_Q, vl); /* t = (t*Q) / R */ + t = __riscv_vsub_vv_i16m1(rh, t, vl); /* t = h - t */ + + return t; +} + +/* Narrowing reduction */ + +static MLK_INLINE vint16m1_t fq_redc2(vint32m2_t z, size_t vl) +{ + vint16m1_t t; + + t = __riscv_vmul_vx_i16m1(__riscv_vncvt_x_x_w_i16m1(z, vl), MLK_RVV_QI, + vl); /* t = l * Q^-1 */ + z = __riscv_vsub_vv_i32m2(z, __riscv_vwmul_vx_i32m2(t, MLKEM_Q, vl), + vl); /* x = (x - (t*Q)) */ + t = __riscv_vnsra_wx_i16m1(z, 16, vl); + + return t; +} + +/* Narrowing Barrett */ + +static MLK_INLINE vint16m1_t fq_barrett(vint16m1_t a, size_t vl) +{ + vint16m1_t t; + const int16_t v = ((1 << 26) + MLKEM_Q / 2) / MLKEM_Q; + + t = __riscv_vmulh_vx_i16m1(a, v, vl); + t = __riscv_vadd_vx_i16m1(t, 1 << (25 - 16), vl); + t = __riscv_vsra_vx_i16m1(t, 26 - 16, vl); + t = __riscv_vmul_vx_i16m1(t, MLKEM_Q, vl); + t = __riscv_vsub_vv_i16m1(a, t, vl); + + mlk_assert_abs_bound_int16m1(t, vl, MLKEM_Q_HALF); + return t; +} + +/* Conditionally add Q (if negative). + * + * Constant-time note: We deliberately avoid a `vmslt` + masked `vadd_mu` + * here. RVV 1.0 does not mandate mask-population-independent latency for + * masked ops, and Zvkt's DIEL guarantee does not cover the v0 mask + * register. Using an arithmetic-shift sign mask keeps all operands in the + * Zvkt-covered set and matches the idiom used in the portable C + * (mlk_ct_cmask_neg_i16, verify.h) and AArch64 + * (mlkem_poly_reduce_aarch64_asm.S) implementations. */ + +static MLK_INLINE vint16m1_t fq_cadd(vint16m1_t rx, size_t vl) +{ + vint16m1_t m; + + m = __riscv_vsra_vx_i16m1(rx, 15, vl); /* m = (x < 0) ? -1 : 0 */ + m = __riscv_vand_vx_i16m1(m, MLKEM_Q, vl); /* m = (x < 0) ? Q : 0 */ + rx = __riscv_vadd_vv_i16m1(rx, m, vl); /* x += m */ + return rx; +} + +/* Montgomery multiply: vector-vector */ + +static MLK_INLINE vint16m1_t fq_mul_vv(vint16m1_t rx, vint16m1_t ry, size_t vl) +{ + vint16m1_t rl, rh; + + rh = __riscv_vmulh_vv_i16m1(rx, ry, vl); /* h = (x * y) / R */ + rl = __riscv_vmul_vv_i16m1(rx, ry, vl); /* l = (x * y) % R */ + return fq_redc(rh, rl, vl); +} + +/* Montgomery multiply: vector-scalar */ + +static MLK_INLINE vint16m1_t fq_mul_vx(vint16m1_t rx, int16_t ry, size_t vl) +{ + vint16m1_t rl, rh; + + rh = __riscv_vmulh_vx_i16m1(rx, ry, vl); /* h = (x * y) / R */ + rl = __riscv_vmul_vx_i16m1(rx, ry, vl); /* l = (x * y) % R */ + return fq_redc(rh, rl, vl); +} + +/* full normalization */ + +static MLK_INLINE vint16m1_t fq_mulq_vx(vint16m1_t rx, int16_t ry, size_t vl) +{ + vint16m1_t result; + + result = fq_mul_vx(rx, ry, vl); + + mlk_assert_abs_bound_int16m1(result, vl, MLKEM_Q); + return result; +} + +/* create a permutation for swapping index bits a and b, a < b */ + +static vuint16m2_t bitswap_perm(unsigned a, unsigned b, size_t vl) +{ + const vuint16m2_t v2id = __riscv_vid_v_u16m2(vl); + + vuint16m2_t xa, xb; + xa = __riscv_vsrl_vx_u16m2(v2id, b - a, vl); + xa = __riscv_vxor_vv_u16m2(xa, v2id, vl); + xa = __riscv_vand_vx_u16m2(xa, (1 << a), vl); + xb = __riscv_vsll_vx_u16m2(xa, b - a, vl); + xa = __riscv_vxor_vv_u16m2(xa, xb, vl); + xa = __riscv_vxor_vv_u16m2(v2id, xa, vl); + return xa; +} + +/** + * Compute negacyclic number-theoretic transform (NTT) of a polynomial in + * place; input assumed to be in normal order, output in bitreversed order. + * + * @param[in,out] r Input/output polynomial. + */ + +/* Forward / Cooley-Tukey butterfly operation */ + +#define MLK_RVV_CT_BFLY_FX(u0, u1, ut, uc, vl, layer) \ + { \ + mlk_assert_abs_bound(&uc, 1, MLKEM_Q_HALF); \ + \ + ut = fq_mul_vx(u1, uc, vl); \ + mlk_assert_abs_bound_int16m1(ut, vl, MLKEM_Q); \ + \ + u1 = __riscv_vsub_vv_i16m1(u0, ut, vl); \ + u0 = __riscv_vadd_vv_i16m1(u0, ut, vl); \ + mlk_assert_abs_bound_int16m1(u0, vl, (layer + 1) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(u1, vl, (layer + 1) * MLKEM_Q); \ + } + +#define MLK_RVV_CT_BFLY_FV(u0, u1, ut, uc, vl, layer) \ + { \ + mlk_assert_abs_bound_int16m1(uc, vl, MLKEM_Q_HALF); \ + \ + ut = fq_mul_vv(u1, uc, vl); \ + mlk_assert_abs_bound_int16m1(ut, vl, MLKEM_Q); \ + \ + u1 = __riscv_vsub_vv_i16m1(u0, ut, vl); \ + u0 = __riscv_vadd_vv_i16m1(u0, ut, vl); \ + mlk_assert_abs_bound_int16m1(u0, vl, (layer + 1) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(u1, vl, (layer + 1) * MLKEM_Q); \ + } + +static vint16m2_t mlk_rv64v_ntt2(vint16m2_t vp, vint16m1_t cz) +{ + size_t vl = 16; /* We work with 256-bit vectors of 16x16-bit elements */ + size_t vl2 = 2 * vl; + + const vuint16m2_t v2p8 = bitswap_perm(3, 4, vl2); + const vuint16m2_t v2p4 = bitswap_perm(2, 4, vl2); + const vuint16m2_t v2p2 = bitswap_perm(1, 4, vl2); + + /* p1 = p8(p4(p2)) */ + const vuint16m2_t v2p1 = __riscv_vrgather_vv_u16m2( + __riscv_vrgather_vv_u16m2(v2p2, v2p4, vl2), v2p8, vl2); + + const vuint16m1_t vid = __riscv_vid_v_u16m1(vl); + const vuint16m1_t cs8 = + __riscv_vadd_vx_u16m1(__riscv_vsrl_vx_u16m1(vid, 3, vl), 2, vl); + const vuint16m1_t cs4 = + __riscv_vadd_vx_u16m1(__riscv_vsrl_vx_u16m1(vid, 2, vl), 2 + 2, vl); + const vuint16m1_t cs2 = + __riscv_vadd_vx_u16m1(__riscv_vsrl_vx_u16m1(vid, 1, vl), 2 + 2 + 4, vl); + + vint16m1_t vt, c0, t0, t1; + + /* swap 8 */ + vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p8, vl2); + t0 = __riscv_vget_v_i16m2_i16m1(vp, 0); + t1 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + c0 = __riscv_vrgather_vv_i16m1(cz, cs8, vl); + MLK_RVV_CT_BFLY_FV(t0, t1, vt, c0, vl, 5); + + /* swap 4 */ + vp = __riscv_vcreate_v_i16m1_i16m2(t0, t1); + vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p4, vl2); + t0 = __riscv_vget_v_i16m2_i16m1(vp, 0); + t1 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + c0 = __riscv_vrgather_vv_i16m1(cz, cs4, vl); + MLK_RVV_CT_BFLY_FV(t0, t1, vt, c0, vl, 6); + + /* swap 2 */ + vp = __riscv_vcreate_v_i16m1_i16m2(t0, t1); + vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p2, vl2); + t0 = __riscv_vget_v_i16m2_i16m1(vp, 0); + t1 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + c0 = __riscv_vrgather_vv_i16m1(cz, cs2, vl); + MLK_RVV_CT_BFLY_FV(t0, t1, vt, c0, vl, 7); + + /* reorganize */ + vp = __riscv_vcreate_v_i16m1_i16m2(t0, t1); + vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p1, vl2); + + return vp; +} + +/* Only for VLEN=256 for now */ +void mlk_rv64v_poly_ntt(int16_t *r) +{ +/* zetas can be compiled into vector constants; don't pass as a pointer */ +#include "rv64v_zetas.inc" + + size_t vl = 16; /* We work with 256-bit vectors of 16x16-bit elements */ + size_t vl2 = 2 * vl; + + vint16m1_t vt; + vint16m1_t v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf; + + const vint16m1_t z0 = __riscv_vle16_v_i16m1(&zeta[0x00], vl); + const vint16m1_t z2 = __riscv_vle16_v_i16m1(&zeta[0x10], vl); + const vint16m1_t z4 = __riscv_vle16_v_i16m1(&zeta[0x20], vl); + const vint16m1_t z6 = __riscv_vle16_v_i16m1(&zeta[0x30], vl); + const vint16m1_t z8 = __riscv_vle16_v_i16m1(&zeta[0x40], vl); + const vint16m1_t za = __riscv_vle16_v_i16m1(&zeta[0x50], vl); + const vint16m1_t zc = __riscv_vle16_v_i16m1(&zeta[0x60], vl); + const vint16m1_t ze = __riscv_vle16_v_i16m1(&zeta[0x70], vl); + + v0 = __riscv_vle16_v_i16m1(&r[0x00], vl); + v1 = __riscv_vle16_v_i16m1(&r[0x10], vl); + v2 = __riscv_vle16_v_i16m1(&r[0x20], vl); + v3 = __riscv_vle16_v_i16m1(&r[0x30], vl); + v4 = __riscv_vle16_v_i16m1(&r[0x40], vl); + v5 = __riscv_vle16_v_i16m1(&r[0x50], vl); + v6 = __riscv_vle16_v_i16m1(&r[0x60], vl); + v7 = __riscv_vle16_v_i16m1(&r[0x70], vl); + v8 = __riscv_vle16_v_i16m1(&r[0x80], vl); + v9 = __riscv_vle16_v_i16m1(&r[0x90], vl); + va = __riscv_vle16_v_i16m1(&r[0xa0], vl); + vb = __riscv_vle16_v_i16m1(&r[0xb0], vl); + vc = __riscv_vle16_v_i16m1(&r[0xc0], vl); + vd = __riscv_vle16_v_i16m1(&r[0xd0], vl); + ve = __riscv_vle16_v_i16m1(&r[0xe0], vl); + vf = __riscv_vle16_v_i16m1(&r[0xf0], vl); + + MLK_RVV_CT_BFLY_FX(v0, v8, vt, zeta[0x01], vl, 1); + MLK_RVV_CT_BFLY_FX(v1, v9, vt, zeta[0x01], vl, 1); + MLK_RVV_CT_BFLY_FX(v2, va, vt, zeta[0x01], vl, 1); + MLK_RVV_CT_BFLY_FX(v3, vb, vt, zeta[0x01], vl, 1); + MLK_RVV_CT_BFLY_FX(v4, vc, vt, zeta[0x01], vl, 1); + MLK_RVV_CT_BFLY_FX(v5, vd, vt, zeta[0x01], vl, 1); + MLK_RVV_CT_BFLY_FX(v6, ve, vt, zeta[0x01], vl, 1); + MLK_RVV_CT_BFLY_FX(v7, vf, vt, zeta[0x01], vl, 1); + + MLK_RVV_CT_BFLY_FX(v0, v4, vt, zeta[0x10], vl, 2); + MLK_RVV_CT_BFLY_FX(v1, v5, vt, zeta[0x10], vl, 2); + MLK_RVV_CT_BFLY_FX(v2, v6, vt, zeta[0x10], vl, 2); + MLK_RVV_CT_BFLY_FX(v3, v7, vt, zeta[0x10], vl, 2); + MLK_RVV_CT_BFLY_FX(v8, vc, vt, zeta[0x11], vl, 2); + MLK_RVV_CT_BFLY_FX(v9, vd, vt, zeta[0x11], vl, 2); + MLK_RVV_CT_BFLY_FX(va, ve, vt, zeta[0x11], vl, 2); + MLK_RVV_CT_BFLY_FX(vb, vf, vt, zeta[0x11], vl, 2); + + MLK_RVV_CT_BFLY_FX(v0, v2, vt, zeta[0x20], vl, 3); + MLK_RVV_CT_BFLY_FX(v1, v3, vt, zeta[0x20], vl, 3); + MLK_RVV_CT_BFLY_FX(v4, v6, vt, zeta[0x21], vl, 3); + MLK_RVV_CT_BFLY_FX(v5, v7, vt, zeta[0x21], vl, 3); + MLK_RVV_CT_BFLY_FX(v8, va, vt, zeta[0x30], vl, 3); + MLK_RVV_CT_BFLY_FX(v9, vb, vt, zeta[0x30], vl, 3); + MLK_RVV_CT_BFLY_FX(vc, ve, vt, zeta[0x31], vl, 3); + MLK_RVV_CT_BFLY_FX(vd, vf, vt, zeta[0x31], vl, 3); + + MLK_RVV_CT_BFLY_FX(v0, v1, vt, zeta[0x40], vl, 4); + MLK_RVV_CT_BFLY_FX(v2, v3, vt, zeta[0x41], vl, 4); + MLK_RVV_CT_BFLY_FX(v4, v5, vt, zeta[0x50], vl, 4); + MLK_RVV_CT_BFLY_FX(v6, v7, vt, zeta[0x51], vl, 4); + MLK_RVV_CT_BFLY_FX(v8, v9, vt, zeta[0x60], vl, 4); + MLK_RVV_CT_BFLY_FX(va, vb, vt, zeta[0x61], vl, 4); + MLK_RVV_CT_BFLY_FX(vc, vd, vt, zeta[0x70], vl, 4); + MLK_RVV_CT_BFLY_FX(ve, vf, vt, zeta[0x71], vl, 4); + + __riscv_vse16_v_i16m2( + &r[0x00], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(v0, v1), z0), vl2); + __riscv_vse16_v_i16m2( + &r[0x20], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(v2, v3), z2), vl2); + __riscv_vse16_v_i16m2( + &r[0x40], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(v4, v5), z4), vl2); + __riscv_vse16_v_i16m2( + &r[0x60], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(v6, v7), z6), vl2); + __riscv_vse16_v_i16m2( + &r[0x80], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(v8, v9), z8), vl2); + __riscv_vse16_v_i16m2( + &r[0xa0], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(va, vb), za), vl2); + __riscv_vse16_v_i16m2( + &r[0xc0], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(vc, vd), zc), vl2); + __riscv_vse16_v_i16m2( + &r[0xe0], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(ve, vf), ze), vl2); +} + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +/* Reverse / Gentleman-Sande butterfly operation */ + +#define MLK_RVV_GS_BFLY_RX(u0, u1, ut, uc, vl) \ + { \ + ut = __riscv_vsub_vv_i16m1(u0, u1, vl); \ + u0 = __riscv_vadd_vv_i16m1(u0, u1, vl); \ + u1 = fq_mul_vx(ut, uc, vl); \ + } + +#define MLK_RVV_GS_BFLY_RV(u0, u1, ut, uc, vl) \ + { \ + ut = __riscv_vsub_vv_i16m1(u0, u1, vl); \ + u0 = __riscv_vadd_vv_i16m1(u0, u1, vl); \ + u1 = fq_mul_vv(ut, uc, vl); \ + } + +static vint16m2_t mlk_rv64v_intt2(vint16m2_t vp, vint16m1_t cz) +{ + size_t vl = 16; /* We work with 256-bit vectors of 16x16-bit elements */ + size_t vl2 = 2 * vl; + + const vuint16m2_t v2p8 = bitswap_perm(3, 4, vl2); + const vuint16m2_t v2p4 = bitswap_perm(2, 4, vl2); + const vuint16m2_t v2p2 = bitswap_perm(1, 4, vl2); + + /* p0 = p2(p4(p8)) */ + const vuint16m2_t v2p0 = __riscv_vrgather_vv_u16m2( + __riscv_vrgather_vv_u16m2(v2p8, v2p4, vl2), v2p2, vl2); + + const vuint16m1_t vid = __riscv_vid_v_u16m1(vl); + const vuint16m1_t cs8 = + __riscv_vadd_vx_u16m1(__riscv_vsrl_vx_u16m1(vid, 3, vl), 2, vl); + const vuint16m1_t cs4 = + __riscv_vadd_vx_u16m1(__riscv_vsrl_vx_u16m1(vid, 2, vl), 2 + 2, vl); + const vuint16m1_t cs2 = + __riscv_vadd_vx_u16m1(__riscv_vsrl_vx_u16m1(vid, 1, vl), 2 + 2 + 4, vl); + + vint16m1_t t0, t1, c0, vt; + + /* initial permute */ + vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p0, vl2); + t0 = __riscv_vget_v_i16m2_i16m1(vp, 0); + t1 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + /* pre-scale */ + t0 = fq_mulq_vx(t0, MLK_RVV_MONT_NR, vl); + t1 = fq_mulq_vx(t1, MLK_RVV_MONT_NR, vl); + + /* absolute bounds: < t0 < q, t1 < q */ + mlk_assert_abs_bound_int16m1(t0, vl, MLKEM_Q); + mlk_assert_abs_bound_int16m1(t1, vl, MLKEM_Q); + + c0 = __riscv_vrgather_vv_i16m1(cz, cs2, vl); + MLK_RVV_GS_BFLY_RV(t0, t1, vt, c0, vl); + + /* absolute bounds: < t0 < 2*q, t1 < q */ + mlk_assert_abs_bound_int16m1(t0, vl, 2 * MLKEM_Q); + mlk_assert_abs_bound_int16m1(t1, vl, MLKEM_Q); + + /* swap 2 */ + vp = __riscv_vcreate_v_i16m1_i16m2(t0, t1); + vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p2, vl2); + t0 = __riscv_vget_v_i16m2_i16m1(vp, 0); + t1 = __riscv_vget_v_i16m2_i16m1(vp, 1); + c0 = __riscv_vrgather_vv_i16m1(cz, cs4, vl); + MLK_RVV_GS_BFLY_RV(t0, t1, vt, c0, vl); + + /* absolute bounds: t0 < 4*q, t1 < q */ + mlk_assert_abs_bound_int16m1(t0, vl, 4 * MLKEM_Q); + mlk_assert_abs_bound_int16m1(t1, vl, MLKEM_Q); + + /* swap 4 */ + vp = __riscv_vcreate_v_i16m1_i16m2(t0, t1); + vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p4, vl2); + t0 = __riscv_vget_v_i16m2_i16m1(vp, 0); + t1 = __riscv_vget_v_i16m2_i16m1(vp, 1); + c0 = __riscv_vrgather_vv_i16m1(cz, cs8, vl); + MLK_RVV_GS_BFLY_RV(t0, t1, vt, c0, vl); + + /* absolute bounds: < 8*q */ + mlk_assert_abs_bound_int16m1(t0, vl, 8 * MLKEM_Q); + mlk_assert_abs_bound_int16m1(t1, vl, MLKEM_Q); + + t0 = fq_mulq_vx(t0, MLK_RVV_MONT_R1, vl); + + /* absolute bounds: < q */ + mlk_assert_abs_bound_int16m1(t0, vl, MLKEM_Q); + mlk_assert_abs_bound_int16m1(t1, vl, MLKEM_Q); + + /* swap 8 */ + vp = __riscv_vcreate_v_i16m1_i16m2(t0, t1); + vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p8, vl2); + + return vp; +} + +#define MLK_RV64V_ABS_BOUNDS16(vl, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, \ + vb, vc, vd, ve, vf, b0, b1, b2, b3, b4, b5, b6, \ + b7, b8, b9, ba, bb, bc, bd, be, bf) \ + do \ + { \ + mlk_assert_abs_bound_int16m1(v0, vl, (b0) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v1, vl, (b1) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v2, vl, (b2) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v3, vl, (b3) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v4, vl, (b4) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v5, vl, (b5) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v6, vl, (b6) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v7, vl, (b7) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v8, vl, (b8) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v9, vl, (b9) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(va, vl, (ba) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(vb, vl, (bb) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(vc, vl, (bc) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(vd, vl, (bd) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(ve, vl, (be) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(vf, vl, (bf) * MLKEM_Q); \ + } while (0) + + +/** + * Compute the inverse negacyclic number-theoretic transform (NTT) of a + * polynomial in place; input assumed to be in bitreversed order, output in + * normal order. + * + * @param[in,out] r Input/output polynomial. + */ +/* Only for VLEN=256 for now */ +void mlk_rv64v_poly_invntt_tomont(int16_t *r) +{ +/* zetas can be compiled into vector constants; don't pass as a pointer */ +#include "rv64v_izetas.inc" + + size_t vl = 16; /* We work with 256-bit vectors of 16x16-bit elements */ + size_t vl2 = 2 * vl; + + const vint16m1_t z0 = __riscv_vle16_v_i16m1(&izeta[0x00], vl); + const vint16m1_t z2 = __riscv_vle16_v_i16m1(&izeta[0x10], vl); + const vint16m1_t z4 = __riscv_vle16_v_i16m1(&izeta[0x20], vl); + const vint16m1_t z6 = __riscv_vle16_v_i16m1(&izeta[0x30], vl); + const vint16m1_t z8 = __riscv_vle16_v_i16m1(&izeta[0x40], vl); + const vint16m1_t za = __riscv_vle16_v_i16m1(&izeta[0x50], vl); + const vint16m1_t zc = __riscv_vle16_v_i16m1(&izeta[0x60], vl); + const vint16m1_t ze = __riscv_vle16_v_i16m1(&izeta[0x70], vl); + + vint16m1_t vt; + vint16m1_t v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf; + vint16m2_t vp; + + vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0x00], vl2), z0); + v0 = __riscv_vget_v_i16m2_i16m1(vp, 0); + v1 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0x20], vl2), z2); + v2 = __riscv_vget_v_i16m2_i16m1(vp, 0); + v3 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0x40], vl2), z4); + v4 = __riscv_vget_v_i16m2_i16m1(vp, 0); + v5 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0x60], vl2), z6); + v6 = __riscv_vget_v_i16m2_i16m1(vp, 0); + v7 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0x80], vl2), z8); + v8 = __riscv_vget_v_i16m2_i16m1(vp, 0); + v9 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0xa0], vl2), za); + va = __riscv_vget_v_i16m2_i16m1(vp, 0); + vb = __riscv_vget_v_i16m2_i16m1(vp, 1); + + vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0xc0], vl2), zc); + vc = __riscv_vget_v_i16m2_i16m1(vp, 0); + vd = __riscv_vget_v_i16m2_i16m1(vp, 1); + + vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0xe0], vl2), ze); + ve = __riscv_vget_v_i16m2_i16m1(vp, 0); + vf = __riscv_vget_v_i16m2_i16m1(vp, 1); + + /* absolute bounds < q (see mlk_rv64v_intt2) */ + MLK_RV64V_ABS_BOUNDS16(vl, + v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + + MLK_RVV_GS_BFLY_RX(v0, v1, vt, izeta[0x40], vl); + MLK_RVV_GS_BFLY_RX(v2, v3, vt, izeta[0x41], vl); + MLK_RVV_GS_BFLY_RX(v4, v5, vt, izeta[0x50], vl); + MLK_RVV_GS_BFLY_RX(v6, v7, vt, izeta[0x51], vl); + MLK_RVV_GS_BFLY_RX(v8, v9, vt, izeta[0x60], vl); + MLK_RVV_GS_BFLY_RX(va, vb, vt, izeta[0x61], vl); + MLK_RVV_GS_BFLY_RX(vc, vd, vt, izeta[0x70], vl); + MLK_RVV_GS_BFLY_RX(ve, vf, vt, izeta[0x71], vl); + + /* absolute bounds: + * - v{0,2,4,6,8,a,c,e}: < 2*q + * - v{1,3,5,7,9,b,d,f}: < 1*q + */ + MLK_RV64V_ABS_BOUNDS16(vl, + v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf, + 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1); + + MLK_RVV_GS_BFLY_RX(v0, v2, vt, izeta[0x20], vl); + MLK_RVV_GS_BFLY_RX(v1, v3, vt, izeta[0x20], vl); + MLK_RVV_GS_BFLY_RX(v4, v6, vt, izeta[0x21], vl); + MLK_RVV_GS_BFLY_RX(v5, v7, vt, izeta[0x21], vl); + MLK_RVV_GS_BFLY_RX(v8, va, vt, izeta[0x30], vl); + MLK_RVV_GS_BFLY_RX(v9, vb, vt, izeta[0x30], vl); + MLK_RVV_GS_BFLY_RX(vc, ve, vt, izeta[0x31], vl); + MLK_RVV_GS_BFLY_RX(vd, vf, vt, izeta[0x31], vl); + + /* absolute bounds: + * - v{0,4,8,c}: < 4*q + * - v{1,5,9,d}: < 2*q + * - v{2,3,6,7,a,b,e,f}: < 1*q + */ + MLK_RV64V_ABS_BOUNDS16(vl, + v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf, + 4, 2, 1, 1, 4, 2, 1, 1, 4, 2, 1, 1, 4, 2, 1, 1); + + MLK_RVV_GS_BFLY_RX(v0, v4, vt, izeta[0x10], vl); + MLK_RVV_GS_BFLY_RX(v1, v5, vt, izeta[0x10], vl); + MLK_RVV_GS_BFLY_RX(v2, v6, vt, izeta[0x10], vl); + MLK_RVV_GS_BFLY_RX(v3, v7, vt, izeta[0x10], vl); + MLK_RVV_GS_BFLY_RX(v8, vc, vt, izeta[0x11], vl); + MLK_RVV_GS_BFLY_RX(v9, vd, vt, izeta[0x11], vl); + MLK_RVV_GS_BFLY_RX(va, ve, vt, izeta[0x11], vl); + MLK_RVV_GS_BFLY_RX(vb, vf, vt, izeta[0x11], vl); + + /* absolute bounds: + * - v{0,8}: < 8*q + * - v{1,9}: < 4*q + * - v{2,3,a,b}: < 2*q + * - v{4,5,6,7,c,d,e,f}: < 1*q + */ + MLK_RV64V_ABS_BOUNDS16(vl, + v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf, + 8, 4, 2, 2, 1, 1, 1, 1, 8, 4, 2, 2, 1, 1, 1, 1); + + /* Reduce v0, v8 to avoid overflow */ + v0 = fq_mulq_vx(v0, MLK_RVV_MONT_R1, vl); + v8 = fq_mulq_vx(v8, MLK_RVV_MONT_R1, vl); + + /* absolute bounds: < 4*q */ + MLK_RV64V_ABS_BOUNDS16(vl, + v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4); + + MLK_RVV_GS_BFLY_RX(v0, v8, vt, izeta[0x01], vl); + MLK_RVV_GS_BFLY_RX(v1, v9, vt, izeta[0x01], vl); + MLK_RVV_GS_BFLY_RX(v2, va, vt, izeta[0x01], vl); + MLK_RVV_GS_BFLY_RX(v3, vb, vt, izeta[0x01], vl); + MLK_RVV_GS_BFLY_RX(v4, vc, vt, izeta[0x01], vl); + MLK_RVV_GS_BFLY_RX(v5, vd, vt, izeta[0x01], vl); + MLK_RVV_GS_BFLY_RX(v6, ve, vt, izeta[0x01], vl); + MLK_RVV_GS_BFLY_RX(v7, vf, vt, izeta[0x01], vl); + + /* absolute bounds: < 8*q */ + MLK_RV64V_ABS_BOUNDS16(vl, + v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8); + + __riscv_vse16_v_i16m1(&r[0x00], v0, vl); + __riscv_vse16_v_i16m1(&r[0x10], v1, vl); + __riscv_vse16_v_i16m1(&r[0x20], v2, vl); + __riscv_vse16_v_i16m1(&r[0x30], v3, vl); + __riscv_vse16_v_i16m1(&r[0x40], v4, vl); + __riscv_vse16_v_i16m1(&r[0x50], v5, vl); + __riscv_vse16_v_i16m1(&r[0x60], v6, vl); + __riscv_vse16_v_i16m1(&r[0x70], v7, vl); + __riscv_vse16_v_i16m1(&r[0x80], v8, vl); + __riscv_vse16_v_i16m1(&r[0x90], v9, vl); + __riscv_vse16_v_i16m1(&r[0xa0], va, vl); + __riscv_vse16_v_i16m1(&r[0xb0], vb, vl); + __riscv_vse16_v_i16m1(&r[0xc0], vc, vl); + __riscv_vse16_v_i16m1(&r[0xd0], vd, vl); + __riscv_vse16_v_i16m1(&r[0xe0], ve, vl); + __riscv_vse16_v_i16m1(&r[0xf0], vf, vl); +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +/* ML-KEM's middle field GF(3329)[X]/(X^2) multiplication */ + +static MLK_INLINE void mlk_rv64v_poly_basemul_mont_add_k(int16_t *r, + const int16_t *a, + const int16_t *b, + unsigned kn) +{ +#include "rv64v_zetas_basemul.inc" + + size_t vl = __riscv_vsetvl_e16m1(MLKEM_N); + size_t i, j; + + const vuint16m1_t sw0 = __riscv_vxor_vx_u16m1(__riscv_vid_v_u16m1(vl), 1, vl); + const vbool16_t sb0 = __riscv_vmseq_vx_u16m1_b16( + __riscv_vand_vx_u16m1(__riscv_vid_v_u16m1(vl), 1, vl), 0, vl); + + vint16m1_t vt, vu; + vint32m2_t wa, wb, ws; + + for (i = 0; i < MLKEM_N; i += vl) + { + const vint16m1_t vz = __riscv_vle16_v_i16m1(&roots[i], vl); + + for (j = 0; j < kn; j += MLKEM_N) + { + vt = __riscv_vle16_v_i16m1(&a[i + j], vl); + vu = __riscv_vle16_v_i16m1(&b[i + j], vl); + + wa = __riscv_vwmul_vv_i32m2(vz, fq_mul_vv(vt, vu, vl), vl); + wb = __riscv_vwmul_vv_i32m2(vt, __riscv_vrgather_vv_i16m1(vu, sw0, vl), + vl); + + wa = + __riscv_vadd_vv_i32m2(wa, __riscv_vslidedown_vx_i32m2(wa, 1, vl), vl); + wb = __riscv_vadd_vv_i32m2(wb, __riscv_vslideup_vx_i32m2(wb, wb, 1, vl), + vl); + + wa = __riscv_vmerge_vvm_i32m2(wb, wa, sb0, vl); + + if (j == 0) + { + ws = wa; + } + else + { + ws = __riscv_vadd_vv_i32m2(ws, wa, vl); + } + } + /* the idea is to keep 32-bit intermediate result, reduce in the end */ + __riscv_vse16_v_i16m1(&r[i], fq_redc2(ws, vl), vl); + } +} + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 +void mlk_rv64v_poly_basemul_mont_add_k2(int16_t *r, const int16_t *a, + const int16_t *b) +{ + mlk_rv64v_poly_basemul_mont_add_k(r, a, b, 2 * MLKEM_N); +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3 +void mlk_rv64v_poly_basemul_mont_add_k3(int16_t *r, const int16_t *a, + const int16_t *b) +{ + mlk_rv64v_poly_basemul_mont_add_k(r, a, b, 3 * MLKEM_N); +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4 +void mlk_rv64v_poly_basemul_mont_add_k4(int16_t *r, const int16_t *a, + const int16_t *b) +{ + mlk_rv64v_poly_basemul_mont_add_k(r, a, b, 4 * MLKEM_N); +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */ + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +/** + * In-place conversion of all coefficients of a polynomial from the normal + * domain to the Montgomery domain. + * + * @param[in,out] r Input/output polynomial. + */ +void mlk_rv64v_poly_tomont(int16_t *r) +{ + size_t vl = __riscv_vsetvl_e16m1(MLKEM_N); + size_t i; + + for (i = 0; i < MLKEM_N; i += vl) + { + vint16m1_t vec = __riscv_vle16_v_i16m1(&r[i], vl); + vec = fq_mul_vx(vec, MLK_RVV_MONT_R2, vl); + __riscv_vse16_v_i16m1(&r[i], vec, vl); + } +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +/** + * Apply Barrett reduction to all coefficients of a polynomial. For details + * of the Barrett reduction see the comments in poly.c. + * + * @param[in,out] r Input/output polynomial. + */ +void mlk_rv64v_poly_reduce(int16_t *r) +{ + size_t vl = __riscv_vsetvl_e16m1(MLKEM_N); + vint16m1_t vt; + size_t i; + + for (i = 0; i < MLKEM_N; i += vl) + { + vt = __riscv_vle16_v_i16m1(&r[i], vl); + vt = fq_barrett(vt, vl); + vt = fq_cadd(vt, vl); + __riscv_vse16_v_i16m1(&r[i], vt, vl); + } +} + +/* Run rejection sampling to get uniform random integers mod q. + * buflen must be a multiple of 12. */ + +unsigned int mlk_rv64v_rej_uniform(int16_t *r, unsigned int len, + const uint8_t *buf, unsigned int buflen) +{ + unsigned n, ctr, pos; + vuint16m1_t x, y; + vbool16_t lt; + + pos = 0; + ctr = 0; + + while (ctr < len && pos < buflen) + { + const unsigned vl = (unsigned)__riscv_vsetvl_e16m1((buflen - pos) * 8 / 12); + const unsigned vl23 = (vl * 24) / 32; + + const vuint16m1_t vid = __riscv_vid_v_u16m1(vl); + const vuint16m1_t srl12v = __riscv_vmul_vx_u16m1(vid, 12, vl); + const vuint16m1_t sel12v = __riscv_vsrl_vx_u16m1(srl12v, 4, vl); + const vuint16m1_t sll12v = __riscv_vsll_vx_u16m1(vid, 2, vl); + + /* Functionally, this loop is not necessary, but it avoids re-evaluating + * the VL too many times. In particular, in the first outer iteration, + * the inner loop will process the bulk of the data with fixed VL. */ + while (ctr < len && vl23 * 2 <= buflen - pos) + { + x = __riscv_vle16_v_u16m1((uint16_t *)&buf[pos], vl23); + pos += vl23 * 2; + x = __riscv_vrgather_vv_u16m1(x, sel12v, vl); + x = __riscv_vor_vv_u16m1( + __riscv_vsrl_vv_u16m1(x, srl12v, vl), + __riscv_vsll_vv_u16m1(__riscv_vslidedown(x, 1, vl), sll12v, vl), vl); + x = __riscv_vand_vx_u16m1(x, 0xFFF, vl); + + lt = __riscv_vmsltu_vx_u16m1_b16(x, MLKEM_Q, vl); + y = __riscv_vcompress_vm_u16m1(x, lt, vl); + n = (unsigned)__riscv_vcpop_m_b16(lt, vl); + + if (ctr + n > len) + { + n = len - ctr; + } + __riscv_vse16_v_u16m1((uint16_t *)&r[ctr], y, n); + ctr += n; + } + } + + return ctr; +} + +#else /* MLK_ARITH_BACKEND_RISCV64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(rv64v_poly) + +#endif /* !(MLK_ARITH_BACKEND_RISCV64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED) */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef MLK_RVV_QI +#undef MLK_RVV_MONT_R1 +#undef MLK_RVV_MONT_R2 +#undef MLK_RVV_MONT_NR +#undef MLK_RVV_CT_BFLY_FX +#undef MLK_RVV_CT_BFLY_FV +#undef MLK_RVV_GS_BFLY_RX +#undef MLK_RVV_GS_BFLY_RV +#undef MLK_RV64V_ABS_BOUNDS16 diff --git a/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_zetas.inc b/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_zetas.inc new file mode 100644 index 0000000000..9deb95900b --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_zetas.inc @@ -0,0 +1,27 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "arith_native_riscv64.h" + +const int16_t zeta[] = { + -1044, -758, 573, -1325, 1223, 652, -552, 1015, -1103, 430, 555, + 843, -1251, 871, 1550, 105, -359, -1517, 264, 383, -1293, 1491, + -282, -1544, 422, 587, 177, -235, -291, -460, 1574, 1653, 1493, + 1422, -829, 1458, 516, -8, -320, -666, -246, 778, 1159, -147, + -777, 1483, -602, 1119, 287, 202, -1602, -130, -1618, -1162, 126, + 1469, -1590, 644, -872, 349, 418, 329, -156, -75, -171, 622, + -681, 1017, -853, -90, -271, 830, 817, 1097, 603, 610, 1322, + -1285, -1465, 384, 1577, 182, 732, 608, 107, -1421, -247, -951, + -1215, -136, 1218, -1335, -874, 220, -1187, -1659, 962, -1202, -1542, + 411, -398, 961, -1508, -725, -1185, -1530, -1278, 794, -1510, -854, + -870, 478, -1474, 1468, -205, -1571, 448, -1065, 677, -1275, -108, + -308, 996, 991, 958, -1460, 1522, 1628, +}; diff --git a/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_zetas_basemul.inc b/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_zetas_basemul.inc new file mode 100644 index 0000000000..ee4033c6ee --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_zetas_basemul.inc @@ -0,0 +1,39 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "arith_native_riscv64.h" + +const int16_t roots[] = { + -1044, -1103, -1044, 1103, -1044, 430, -1044, -430, -1044, 555, -1044, + -555, -1044, 843, -1044, -843, -1044, -1251, -1044, 1251, -1044, 871, + -1044, -871, -1044, 1550, -1044, -1550, -1044, 105, -1044, -105, -1044, + 422, -1044, -422, -1044, 587, -1044, -587, -1044, 177, -1044, -177, + -1044, -235, -1044, 235, -1044, -291, -1044, 291, -1044, -460, -1044, + 460, -1044, 1574, -1044, -1574, -1044, 1653, -1044, -1653, -1044, -246, + -1044, 246, -1044, 778, -1044, -778, -1044, 1159, -1044, -1159, -1044, + -147, -1044, 147, -1044, -777, -1044, 777, -1044, 1483, -1044, -1483, + -1044, -602, -1044, 602, -1044, 1119, -1044, -1119, -1044, -1590, -1044, + 1590, -1044, 644, -1044, -644, -1044, -872, -1044, 872, -1044, 349, + -1044, -349, -1044, 418, -1044, -418, -1044, 329, -1044, -329, -1044, + -156, -1044, 156, -1044, -75, -1044, 75, -1044, 817, -1044, -817, + -1044, 1097, -1044, -1097, -1044, 603, -1044, -603, -1044, 610, -1044, + -610, -1044, 1322, -1044, -1322, -1044, -1285, -1044, 1285, -1044, -1465, + -1044, 1465, -1044, 384, -1044, -384, -1044, -1215, -1044, 1215, -1044, + -136, -1044, 136, -1044, 1218, -1044, -1218, -1044, -1335, -1044, 1335, + -1044, -874, -1044, 874, -1044, 220, -1044, -220, -1044, -1187, -1044, + 1187, -1044, -1659, -1044, 1659, -1044, -1185, -1044, 1185, -1044, -1530, + -1044, 1530, -1044, -1278, -1044, 1278, -1044, 794, -1044, -794, -1044, + -1510, -1044, 1510, -1044, -854, -1044, 854, -1044, -870, -1044, 870, + -1044, 478, -1044, -478, -1044, -108, -1044, 108, -1044, -308, -1044, + 308, -1044, 996, -1044, -996, -1044, 991, -1044, -991, -1044, 958, + -1044, -958, -1044, -1460, -1044, 1460, -1044, 1522, -1044, -1522, -1044, + 1628, -1044, -1628, +}; diff --git a/src/mlkem-native/mlkem/src/native/x86_64/README.md b/src/mlkem-native/mlkem/src/native/x86_64/README.md new file mode 100644 index 0000000000..2073425c3b --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/README.md @@ -0,0 +1,4 @@ +[//]: # (SPDX-License-Identifier: CC-BY-4.0) + +This directory contains the native x86_64 arithmetic backend for ML-KEM provided by the official [AVX2 +implementation](https://github.com/pq-crystals/kyber/tree/main/avx2) of the Kyber team. diff --git a/src/mlkem-native/mlkem/src/native/x86_64/meta.h b/src/mlkem-native/mlkem/src/native/x86_64/meta.h new file mode 100644 index 0000000000..073b233df0 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/meta.h @@ -0,0 +1,324 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_NATIVE_X86_64_META_H +#define MLK_NATIVE_X86_64_META_H + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLK_ARITH_BACKEND_X86_64_DEFAULT + +#define MLK_USE_NATIVE_NTT_CUSTOM_ORDER +#define MLK_USE_NATIVE_REJ_UNIFORM +#define MLK_USE_NATIVE_NTT +#define MLK_USE_NATIVE_INTT +#define MLK_USE_NATIVE_POLY_REDUCE +#define MLK_USE_NATIVE_POLY_TOMONT +#define MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED +#define MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#define MLK_USE_NATIVE_POLY_TOBYTES +#define MLK_USE_NATIVE_POLY_FROMBYTES +#define MLK_USE_NATIVE_POLY_COMPRESS_D4 +#define MLK_USE_NATIVE_POLY_COMPRESS_D5 +#define MLK_USE_NATIVE_POLY_COMPRESS_D10 +#define MLK_USE_NATIVE_POLY_COMPRESS_D11 +#define MLK_USE_NATIVE_POLY_DECOMPRESS_D4 +#define MLK_USE_NATIVE_POLY_DECOMPRESS_D5 +#define MLK_USE_NATIVE_POLY_DECOMPRESS_D10 +#define MLK_USE_NATIVE_POLY_DECOMPRESS_D11 + +#if !defined(__ASSEMBLER__) +#include "../../common.h" +#include "../api.h" +#include "src/arith_native_x86_64.h" +#include "src/compress_consts.h" + +static MLK_INLINE void mlk_poly_permute_bitrev_to_custom(int16_t data[MLKEM_N]) +{ + if (mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + mlk_nttunpack_avx2_asm(data); + } +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_rej_uniform_native(int16_t *r, unsigned len, + const uint8_t *buf, + unsigned buflen) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2) || len != MLKEM_N || + buflen % 12 != 0) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + return (int)mlk_rej_uniform_avx2_asm(r, buf, buflen, mlk_rej_uniform_table); +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_ntt_native(int16_t data[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_ntt_avx2_asm(data, mlk_qdata); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_intt_native(int16_t data[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_invntt_avx2_asm(data, mlk_qdata); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_reduce_native(int16_t data[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_reduce_avx2_asm(data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_tomont_native(int16_t data[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_tomont_avx2_asm(data); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_mulcache_compute_native(int16_t x[MLKEM_N / 2], + const int16_t y[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_mulcache_compute_avx2_asm(x, y, mlk_qdata); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k2_native( + int16_t r[MLKEM_N], const int16_t a[2 * MLKEM_N], + const int16_t b[2 * MLKEM_N], const int16_t b_cache[2 * (MLKEM_N / 2)]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm(r, a, b, b_cache); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k3_native( + int16_t r[MLKEM_N], const int16_t a[3 * MLKEM_N], + const int16_t b[3 * MLKEM_N], const int16_t b_cache[3 * (MLKEM_N / 2)]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm(r, a, b, b_cache); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k4_native( + int16_t r[MLKEM_N], const int16_t a[4 * MLKEM_N], + const int16_t b[4 * MLKEM_N], const int16_t b_cache[4 * (MLKEM_N / 2)]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm(r, a, b, b_cache); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */ + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) || !defined(MLK_CONFIG_NO_ENCAPS_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_tobytes_native(uint8_t r[MLKEM_POLYBYTES], + const int16_t a[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_ntttobytes_avx2_asm(r, a); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API || !MLK_CONFIG_NO_ENCAPS_API */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_frombytes_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYBYTES]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_nttfrombytes_avx2_asm(r, a); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#if (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API)) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 || \ + MLKEM_K == 3) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_compress_d4_native( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D4], const int16_t a[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_compress_d4_avx2_asm(r, a, mlk_compress_d4_data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_compress_d10_native( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D10], const int16_t a[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_compress_d10_avx2_asm(r, a, mlk_compress_d10_data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_decompress_d4_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D4]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_decompress_d4_avx2_asm(r, a, mlk_decompress_d4_data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_decompress_d10_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D10]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_decompress_d10_avx2_asm(r, a, mlk_decompress_d10_data); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ +#endif /* (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API) && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == 3) \ + */ + +#if (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API)) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_compress_d5_native( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D5], const int16_t a[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_compress_d5_avx2_asm(r, a, mlk_compress_d5_data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_compress_d11_native( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D11], const int16_t a[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_compress_d11_avx2_asm(r, a, mlk_compress_d11_data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_decompress_d5_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D5]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_decompress_d5_avx2_asm(r, a, mlk_decompress_d5_data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_decompress_d11_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D11]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_decompress_d11_avx2_asm(r, a, mlk_decompress_d11_data); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ +#endif /* (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API) && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */ + +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_NATIVE_X86_64_META_H */ diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/arith_native_x86_64.h b/src/mlkem-native/mlkem/src/native/x86_64/src/arith_native_x86_64.h new file mode 100644 index 0000000000..b4423a3981 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/arith_native_x86_64.h @@ -0,0 +1,327 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_NATIVE_X86_64_SRC_ARITH_NATIVE_X86_64_H +#define MLK_NATIVE_X86_64_SRC_ARITH_NATIVE_X86_64_H + +#include "../../../common.h" + +#include +#include "compress_consts.h" +#include "consts.h" + +#define MLK_AVX2_REJ_UNIFORM_BUFLEN \ + (3 * 168) /* REJ_UNIFORM_NBLOCKS * SHAKE128_RATE */ + +#define mlk_rej_uniform_table MLK_NAMESPACE(rej_uniform_table) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_rej_uniform_table[4096]; + +#define mlk_rej_uniform_avx2_asm MLK_NAMESPACE(rej_uniform_avx2_asm) +MLK_MUST_CHECK_RETURN_VALUE MLK_SYSV_ABI +uint64_t mlk_rej_uniform_avx2_asm(int16_t *r, const uint8_t *buf, + unsigned buflen, const uint8_t *table) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/mlkem_rej_uniform_avx2_asm.ml. */ +__contract__( + requires(buflen % 12 == 0) + requires(memory_no_alias(buf, buflen)) + requires(table == mlk_rej_uniform_table) + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value <= MLKEM_N) + ensures(array_bound(r, 0, (unsigned) return_value, 0, MLKEM_Q)) +); + +#define mlk_ntt_avx2_asm MLK_NAMESPACE(ntt_avx2_asm) +MLK_SYSV_ABI +void mlk_ntt_avx2_asm(int16_t *r, const int16_t *qdata) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/mlkem_ntt_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(array_abs_bound(r, 0, MLKEM_N, 8192)) + requires(qdata == mlk_qdata) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + /* check-magic: off */ + ensures(array_abs_bound(r, 0, MLKEM_N, 23595)) + /* check-magic: on */ +); + +#define mlk_invntt_avx2_asm MLK_NAMESPACE(invntt_avx2_asm) +MLK_SYSV_ABI +void mlk_invntt_avx2_asm(int16_t *r, const int16_t *qdata) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/mlkem_intt_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(qdata == mlk_qdata) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + /* check-magic: off */ + ensures(array_abs_bound(r, 0, MLKEM_N, 26632)) + /* check-magic: on */ +); + +#define mlk_nttunpack_avx2_asm MLK_NAMESPACE(nttunpack_avx2_asm) +MLK_SYSV_ABI +void mlk_nttunpack_avx2_asm(int16_t *r) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/mlkem_nttunpack_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(r, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + /* Output is a permutation of input: every output coefficient + * is some input coefficient */ + ensures(forall(i, 0, MLKEM_N, exists(j, 0, MLKEM_N, + r[i] == old(*(int16_t (*)[MLKEM_N])r)[j]))) +); + +#define mlk_reduce_avx2_asm MLK_NAMESPACE(reduce_avx2_asm) +MLK_SYSV_ABI +void mlk_reduce_avx2_asm(int16_t *r) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/mlkem_reduce_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_bound(r, 0, MLKEM_N, 0, MLKEM_Q)) +); + +#define mlk_poly_mulcache_compute_avx2_asm \ + MLK_NAMESPACE(poly_mulcache_compute_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_mulcache_compute_avx2_asm(int16_t *out, const int16_t *in, + const int16_t *qdata) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/mlkem_poly_mulcache_compute_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(out, sizeof(int16_t) * (MLKEM_N / 2))) + requires(memory_no_alias(in, sizeof(int16_t) * MLKEM_N)) + requires(qdata == mlk_qdata) + assigns(memory_slice(out, sizeof(int16_t) * (MLKEM_N / 2))) + ensures(array_abs_bound(out, 0, MLKEM_N/2, MLKEM_Q)) +); + +#define mlk_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm \ + MLK_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k2_avx2_asm) +MLK_SYSV_ABI +void mlk_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm( + int16_t *r, const int16_t *a, const int16_t *b, const int16_t *b_cache) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 2 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 2 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 2 * (MLKEM_N / 2))) + requires(array_abs_bound(a, 0, 2 * MLKEM_N, MLKEM_UINT12_LIMIT + 1)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) +); + +#define mlk_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm \ + MLK_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k3_avx2_asm) +MLK_SYSV_ABI +void mlk_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm( + int16_t *r, const int16_t *a, const int16_t *b, const int16_t *b_cache) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 3 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 3 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 3 * (MLKEM_N / 2))) + requires(array_abs_bound(a, 0, 3 * MLKEM_N, MLKEM_UINT12_LIMIT + 1)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) +); + +#define mlk_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm \ + MLK_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k4_avx2_asm) +MLK_SYSV_ABI +void mlk_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm( + int16_t *r, const int16_t *a, const int16_t *b, const int16_t *b_cache) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 4 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 4 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 4 * (MLKEM_N / 2))) + requires(array_abs_bound(a, 0, 4 * MLKEM_N, MLKEM_UINT12_LIMIT + 1)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) +); + +#define mlk_ntttobytes_avx2_asm MLK_NAMESPACE(ntttobytes_avx2_asm) +MLK_SYSV_ABI +void mlk_ntttobytes_avx2_asm(uint8_t *r, const int16_t *a) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_ntttobytes_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, MLKEM_POLYBYTES)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYBYTES)) +); + +#define mlk_nttfrombytes_avx2_asm MLK_NAMESPACE(nttfrombytes_avx2_asm) +MLK_SYSV_ABI +void mlk_nttfrombytes_avx2_asm(int16_t *r, const uint8_t *a) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_nttfrombytes_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(a, MLKEM_POLYBYTES)) + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_bound(r, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT)) +); + +#define mlk_tomont_avx2_asm MLK_NAMESPACE(tomont_avx2_asm) +MLK_SYSV_ABI +void mlk_tomont_avx2_asm(int16_t *r) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_tomont_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_abs_bound(r, 0, MLKEM_N, MLKEM_Q)) +); + +#define mlk_poly_compress_d4_avx2_asm MLK_NAMESPACE(poly_compress_d4_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_compress_d4_avx2_asm(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D4], + const int16_t *MLK_RESTRICT a, + const uint8_t *data) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_poly_compress_d4_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D4)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + requires(data == mlk_compress_d4_data) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D4)) +); + +#define mlk_poly_decompress_d4_avx2_asm \ + MLK_NAMESPACE(poly_decompress_d4_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_decompress_d4_avx2_asm( + int16_t *MLK_RESTRICT r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D4], + const uint8_t *data) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_poly_decompress_d4_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D4)) + requires(data == mlk_decompress_d4_data) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_bound(r, 0, MLKEM_N, 0, MLKEM_Q)) +); + +#define mlk_poly_compress_d10_avx2_asm MLK_NAMESPACE(poly_compress_d10_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_compress_d10_avx2_asm(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D10], + const int16_t *MLK_RESTRICT a, + const uint8_t *data) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_poly_compress_d10_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D10)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + requires(data == mlk_compress_d10_data) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D10)) +); + +#define mlk_poly_decompress_d10_avx2_asm \ + MLK_NAMESPACE(poly_decompress_d10_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_decompress_d10_avx2_asm( + int16_t *MLK_RESTRICT r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D10], + const uint8_t *data) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_poly_decompress_d10_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D10)) + requires(data == mlk_decompress_d10_data) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_bound(r, 0, MLKEM_N, 0, MLKEM_Q)) +); + +#define mlk_poly_compress_d5_avx2_asm MLK_NAMESPACE(poly_compress_d5_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_compress_d5_avx2_asm(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D5], + const int16_t *MLK_RESTRICT a, + const uint8_t *data) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_poly_compress_d5_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D5)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + requires(data == mlk_compress_d5_data) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D5)) +); + +#define mlk_poly_decompress_d5_avx2_asm \ + MLK_NAMESPACE(poly_decompress_d5_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_decompress_d5_avx2_asm( + int16_t *MLK_RESTRICT r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D5], + const uint8_t *data) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_poly_decompress_d5_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D5)) + requires(data == mlk_decompress_d5_data) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_bound(r, 0, MLKEM_N, 0, MLKEM_Q)) +); + +#define mlk_poly_compress_d11_avx2_asm MLK_NAMESPACE(poly_compress_d11_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_compress_d11_avx2_asm(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D11], + const int16_t *MLK_RESTRICT a, + const uint8_t *data) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_poly_compress_d11_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D11)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + requires(data == mlk_compress_d11_data) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D11)) +); + +#define mlk_poly_decompress_d11_avx2_asm \ + MLK_NAMESPACE(poly_decompress_d11_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_decompress_d11_avx2_asm( + int16_t *MLK_RESTRICT r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D11], + const uint8_t *data) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_poly_decompress_d11_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D11)) + requires(data == mlk_decompress_d11_data) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_bound(r, 0, MLKEM_N, 0, MLKEM_Q)) +); + +#endif /* !MLK_NATIVE_X86_64_SRC_ARITH_NATIVE_X86_64_H */ diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/compress_consts.c b/src/mlkem-native/mlkem/src/native/x86_64/src/compress_consts.c new file mode 100644 index 0000000000..1645fca573 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/compress_consts.c @@ -0,0 +1,115 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API)) + +#include "compress_consts.h" + +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 || \ + MLKEM_K == 3) + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_compress_d4_data[32] = { + 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, + 2, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0, /* permdidx */ +}; + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_decompress_d4_data[32] = { + 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, + 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, /* shufbidx */ +}; +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_compress_d10_data[32] = { + 0, 1, 2, 3, 4, 8, 9, 10, 11, 12, 255, + 255, 255, 255, 255, 255, 9, 10, 11, 12, 255, 255, + 255, 255, 255, 255, 0, 1, 2, 3, 4, 8, /* shufbidx */ +}; + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_decompress_d10_data[32] = { + 0, 1, 1, 2, 2, 3, 3, 4, 5, 6, 6, 7, 7, 8, 8, 9, + 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 8, 9, 9, 10, 10, 11, /* shufbidx */ +}; +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#endif /* !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == 3) \ + */ + +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_compress_d5_data[32] = { + 0, 1, 2, 3, 4, 255, 255, 255, 255, 255, 8, + 9, 10, 11, 12, 255, 9, 10, 11, 12, 255, 0, + 1, 2, 3, 4, 255, 255, 255, 255, 255, 8, /* shufbidx */ +}; + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +/* shufbidx[0:32], mask[32:64], shift[64:96] */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_decompress_d5_data[96] = { + 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, + 5, 6, 6, 6, 6, 7, 7, 8, 8, 8, 8, 9, 9, 9, /* shufbidx */ + 31, 0, 224, 3, 124, 0, 128, 15, 240, 1, 62, 0, 192, 7, 248, 0, 31, 0, + 224, 3, 124, 0, 128, 15, 240, 1, 62, 0, 192, 7, 248, 0, /* mask */ + 0, 4, 32, 0, 0, 1, 8, 0, 64, 0, 0, 2, 16, 0, 128, 0, 0, 4, + 32, 0, 0, 1, 8, 0, 64, 0, 0, 2, 16, 0, 128, 0, /* shift */ +}; +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +/* srlvqidx[0:32], shufbidx[32:64] */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_compress_d11_data[64] = + { + 10, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, + 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, + 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, /* srlvqidx */ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 255, 255, 255, 255, 255, 5, 6, 7, 8, 9, 10, + 255, 255, 255, 255, 0, 0, 1, 2, 3, 4, /* shufbidx */ +}; + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +/* shufbidx[0:32], srlvdidx[32:64], srlvqidx[64:96], shift[96:128] */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_decompress_d11_data[128] = { + 0, 1, 1, 2, 2, 3, 4, 5, 5, 6, 6, 7, 8, 9, 9, 10, + 3, 4, 4, 5, 5, 6, 7, 8, 8, 9, 9, 10, 11, 12, 12, 13, /* shufbidx */ + 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* srlvdidx */ + 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, /* srlvqidx */ + 32, 0, 4, 0, 1, 0, 32, 0, 8, 0, 1, 0, 32, 0, 4, 0, + 32, 0, 4, 0, 1, 0, 32, 0, 8, 0, 1, 0, 32, 0, 4, 0, /* shift */ +}; +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#endif /* !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */ + +#else /* MLK_ARITH_BACKEND_X86_64_DEFAULT && (!MLK_CONFIG_NO_ENCAPS_API || \ + !MLK_CONFIG_NO_DECAPS_API) */ + +MLK_EMPTY_CU(avx2_compress_consts) + +#endif /* !(MLK_ARITH_BACKEND_X86_64_DEFAULT && (!MLK_CONFIG_NO_ENCAPS_API || \ + !MLK_CONFIG_NO_DECAPS_API)) */ diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/compress_consts.h b/src/mlkem-native/mlkem/src/native/x86_64/src/compress_consts.h new file mode 100644 index 0000000000..79182b0052 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/compress_consts.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#ifndef MLK_NATIVE_X86_64_SRC_COMPRESS_CONSTS_H +#define MLK_NATIVE_X86_64_SRC_COMPRESS_CONSTS_H + +#include "../../../common.h" + +#ifndef __ASSEMBLER__ + +#define mlk_compress_d4_data MLK_NAMESPACE(compress_d4_data) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_compress_d4_data[32]; + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_decompress_d4_data MLK_NAMESPACE(decompress_d4_data) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_decompress_d4_data[32]; +#endif + +#define mlk_compress_d10_data MLK_NAMESPACE(compress_d10_data) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_compress_d10_data[32]; + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_decompress_d10_data MLK_NAMESPACE(decompress_d10_data) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_decompress_d10_data[32]; +#endif + +#define mlk_compress_d5_data MLK_NAMESPACE(compress_d5_data) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_compress_d5_data[32]; + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_decompress_d5_data MLK_NAMESPACE(decompress_d5_data) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_decompress_d5_data[96]; +#endif + +#define mlk_compress_d11_data MLK_NAMESPACE(compress_d11_data) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_compress_d11_data[64]; + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_decompress_d11_data MLK_NAMESPACE(decompress_d11_data) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_decompress_d11_data[128]; +#endif + +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_NATIVE_X86_64_SRC_COMPRESS_CONSTS_H */ diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/consts.c b/src/mlkem-native/mlkem/src/native/x86_64/src/consts.c new file mode 100644 index 0000000000..d0acfe0f9c --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/consts.c @@ -0,0 +1,102 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "consts.h" + +/* + * Table of zeta values used in the AVX2 NTTs + * See autogen for details. + */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t mlk_qdata[624] = { + 3854, 3340, 2826, 2312, 1798, 1284, 770, 256, 3854, + 3340, 2826, 2312, 1798, 1284, 770, 256, 7, 0, + 6, 0, 5, 0, 4, 0, 3, 0, 2, + 0, 1, 0, 0, 0, 31498, 31498, 31498, 31498, + -758, -758, -758, -758, 0, 0, 0, 0, 0, + 0, 0, 0, 14745, 14745, 14745, 14745, 14745, 14745, + 14745, 14745, 14745, 14745, 14745, 14745, 14745, 14745, 14745, + 14745, -359, -359, -359, -359, -359, -359, -359, -359, + -359, -359, -359, -359, -359, -359, -359, -359, 13525, + 13525, 13525, 13525, 13525, 13525, 13525, 13525, -12402, -12402, + -12402, -12402, -12402, -12402, -12402, -12402, 1493, 1493, 1493, + 1493, 1493, 1493, 1493, 1493, 1422, 1422, 1422, 1422, + 1422, 1422, 1422, 1422, -20907, -20907, -20907, -20907, 27758, + 27758, 27758, 27758, -3799, -3799, -3799, -3799, -15690, -15690, + -15690, -15690, -171, -171, -171, -171, 622, 622, 622, + 622, 1577, 1577, 1577, 1577, 182, 182, 182, 182, + -5827, -5827, 17363, 17363, -26360, -26360, -29057, -29057, 5571, + 5571, -1102, -1102, 21438, 21438, -26242, -26242, 573, 573, + -1325, -1325, 264, 264, 383, 383, -829, -829, 1458, + 1458, -1602, -1602, -130, -130, -5689, -6516, 1496, 30967, + -23565, 20179, 20710, 25080, -12796, 26616, 16064, -12442, 9134, + -650, -25986, 27837, 1223, 652, -552, 1015, -1293, 1491, + -282, -1544, 516, -8, -320, -666, -1618, -1162, 126, + 1469, -335, -11477, -32227, 20494, -27738, 945, -14883, 6182, + 32010, 10631, 29175, -28762, -18486, 17560, -14430, -5276, -1103, + 555, -1251, 1550, 422, 177, -291, 1574, -246, 1159, + -777, -602, -1590, -872, 418, -156, 11182, 13387, -14233, + -21655, 13131, -4587, 23092, 5493, -32502, 30317, -18741, 12639, + 20100, 18525, 19529, -12619, 430, 843, 871, 105, 587, + -235, -460, 1653, 778, -147, 1483, 1119, 644, 349, + 329, -75, 787, 787, 787, 787, 787, 787, 787, + 787, 787, 787, 787, 787, 787, 787, 787, 787, + -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, + -1517, -1517, -1517, -1517, -1517, -1517, -1517, 28191, 28191, + 28191, 28191, 28191, 28191, 28191, 28191, -16694, -16694, -16694, + -16694, -16694, -16694, -16694, -16694, 287, 287, 287, 287, + 287, 287, 287, 287, 202, 202, 202, 202, 202, + 202, 202, 202, 10690, 10690, 10690, 10690, 1358, 1358, + 1358, 1358, -11202, -11202, -11202, -11202, 31164, 31164, 31164, + 31164, 962, 962, 962, 962, -1202, -1202, -1202, -1202, + -1474, -1474, -1474, -1474, 1468, 1468, 1468, 1468, -28073, + -28073, 24313, 24313, -10532, -10532, 8800, 8800, 18426, 18426, + 8859, 8859, 26675, 26675, -16163, -16163, -681, -681, 1017, + 1017, 732, 732, 608, 608, -1542, -1542, 411, 411, + -205, -205, -1571, -1571, 19883, -28250, -15887, -8898, -28309, + 9075, -30199, 18249, 13426, 14017, -29156, -12757, 16832, 4311, + -24155, -17915, -853, -90, -271, 830, 107, -1421, -247, + -951, -398, 961, -1508, -725, 448, -1065, 677, -1275, + -31183, 25435, -7382, 24391, -20927, 10946, 24214, 16989, 10335, + -7934, -22502, 10906, 31636, 28644, 23998, -17422, 817, 603, + 1322, -1465, -1215, 1218, -874, -1187, -1185, -1278, -1510, + -870, -108, 996, 958, 1522, 20297, 2146, 15355, -32384, + -6280, -14903, -11044, 14469, -21498, -20198, 23210, -17442, -23860, + -20257, 7756, 23132, 1097, 610, -1285, 384, -136, -1335, + 220, -1659, -1530, 794, -854, 478, -308, 991, -1460, + 1628, -1103, 555, -1251, 1550, 422, 177, -291, 1574, + -246, 1159, -777, -602, -1590, -872, 418, -156, 430, + 843, 871, 105, 587, -235, -460, 1653, 778, -147, + 1483, 1119, 644, 349, 329, -75, 817, 603, 1322, + -1465, -1215, 1218, -874, -1187, -1185, -1278, -1510, -870, + -108, 996, 958, 1522, 1097, 610, -1285, 384, -136, + -1335, 220, -1659, -1530, 794, -854, 478, -308, 991, + -1460, 1628, -335, -11477, -32227, 20494, -27738, 945, -14883, + 6182, 32010, 10631, 29175, -28762, -18486, 17560, -14430, -5276, + 11182, 13387, -14233, -21655, 13131, -4587, 23092, 5493, -32502, + 30317, -18741, 12639, 20100, 18525, 19529, -12619, -31183, 25435, + -7382, 24391, -20927, 10946, 24214, 16989, 10335, -7934, -22502, + 10906, 31636, 28644, 23998, -17422, 20297, 2146, 15355, -32384, + -6280, -14903, -11044, 14469, -21498, -20198, 23210, -17442, -23860, + -20257, 7756, 23132, +}; + +#else /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +MLK_EMPTY_CU(avx2_consts) + +#endif /* !(MLK_ARITH_BACKEND_X86_64_DEFAULT && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/consts.h b/src/mlkem-native/mlkem/src/native/x86_64/src/consts.h new file mode 100644 index 0000000000..591529d471 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/consts.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#ifndef MLK_NATIVE_X86_64_SRC_CONSTS_H +#define MLK_NATIVE_X86_64_SRC_CONSTS_H +#include "../../../common.h" +#define MLK_AVX2_BACKEND_DATA_OFFSET_REVIDXB 0 +#define MLK_AVX2_BACKEND_DATA_OFFSET_REVIDXD 16 +#define MLK_AVX2_BACKEND_DATA_OFFSET_ZETAS_EXP 32 +#define MLK_AVX2_BACKEND_DATA_OFFSET_MULCACHE_TWIDDLES 496 + +#ifndef __ASSEMBLER__ +#define mlk_qdata MLK_NAMESPACE(qdata) +MLK_INTERNAL_DATA_DECLARATION const int16_t mlk_qdata[624]; +#endif + +#endif /* !MLK_NATIVE_X86_64_SRC_CONSTS_H */ diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_intt_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_intt_avx2_asm.S new file mode 100644 index 0000000000..ca76612184 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_intt_avx2_asm.S @@ -0,0 +1,743 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [AVX2_NTT] + * Faster AVX2 optimized NTT multiplication for Ring-LWE lattice cryptography. + * Gregor Seiler + * https://eprint.iacr.org/2018/039 + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + * + * The core ideas behind the implementation are described in @[AVX2_NTT]. + * + * Changes: + * - Different placement of modular reductions to simplify + * reasoning of non-overflow + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API)) +/*yaml + Name: invntt_avx2_asm + Description: x86_64 AVX2 inverse NTT + Signature: void mlk_invntt_avx2_asm(int16_t *r, const int16_t *qdata) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 1248 + permissions: read-only + c_parameter: const int16_t *qdata + description: Precomputed constants (624 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_intt_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(invntt_avx2_asm) +MLK_ASM_FN_SYMBOL(invntt_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0xd8a1d8a1, %eax # imm = 0xD8A1D8A1 + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + movl $0x5a105a1, %eax # imm = 0x5A105A1 + vmovd %eax, %xmm3 + vpbroadcastd %xmm3, %ymm3 + vmovdqa (%rdi), %ymm4 + vmovdqa 0x40(%rdi), %ymm6 + vmovdqa 0x20(%rdi), %ymm5 + vmovdqa 0x60(%rdi), %ymm7 + vpmullw %ymm2, %ymm4, %ymm12 + vpmulhw %ymm3, %ymm4, %ymm4 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm4, %ymm4 + vpmullw %ymm2, %ymm6, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm6, %ymm6 + vpmullw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm5, %ymm5 + vpmullw %ymm2, %ymm7, %ymm12 + vpmulhw %ymm3, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm7, %ymm7 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xc0(%rdi), %ymm10 + vmovdqa 0xa0(%rdi), %ymm9 + vmovdqa 0xe0(%rdi), %ymm11 + vpmullw %ymm2, %ymm8, %ymm12 + vpmulhw %ymm3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm8, %ymm8 + vpmullw %ymm2, %ymm10, %ymm12 + vpmulhw %ymm3, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm10, %ymm10 + vpmullw %ymm2, %ymm9, %ymm12 + vpmulhw %ymm3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm9, %ymm9 + vpmullw %ymm2, %ymm11, %ymm12 + vpmulhw %ymm3, %ymm11, %ymm11 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm11, %ymm11 + vpermq $0x4e, 0x3a0(%rsi), %ymm15 # ymm15 = mem[2,3,0,1] + vpermq $0x4e, 0x360(%rsi), %ymm1 # ymm1 = mem[2,3,0,1] + vpermq $0x4e, 0x3c0(%rsi), %ymm2 # ymm2 = mem[2,3,0,1] + vpermq $0x4e, 0x380(%rsi), %ymm3 # ymm3 = mem[2,3,0,1] + vmovdqa (%rsi), %ymm12 + vpshufb %ymm12, %ymm15, %ymm15 + vpshufb %ymm12, %ymm1, %ymm1 + vpshufb %ymm12, %ymm2, %ymm2 + vpshufb %ymm12, %ymm3, %ymm3 + vpsubw %ymm4, %ymm6, %ymm12 + vpaddw %ymm6, %ymm4, %ymm4 + vpsubw %ymm5, %ymm7, %ymm13 + vpmullw %ymm15, %ymm12, %ymm6 + vpaddw %ymm7, %ymm5, %ymm5 + vpsubw %ymm8, %ymm10, %ymm14 + vpmullw %ymm15, %ymm13, %ymm7 + vpaddw %ymm10, %ymm8, %ymm8 + vpsubw %ymm9, %ymm11, %ymm15 + vpmullw %ymm1, %ymm14, %ymm10 + vpaddw %ymm11, %ymm9, %ymm9 + vpmullw %ymm1, %ymm15, %ymm11 + vpmulhw %ymm2, %ymm12, %ymm12 + vpmulhw %ymm2, %ymm13, %ymm13 + vpmulhw %ymm3, %ymm14, %ymm14 + vpmulhw %ymm3, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm6, %ymm12, %ymm6 + vpsubw %ymm7, %ymm13, %ymm7 + vpsubw %ymm10, %ymm14, %ymm10 + vpsubw %ymm11, %ymm15, %ymm11 + vpermq $0x4e, 0x320(%rsi), %ymm2 # ymm2 = mem[2,3,0,1] + vpermq $0x4e, 0x340(%rsi), %ymm3 # ymm3 = mem[2,3,0,1] + vmovdqa (%rsi), %ymm1 + vpshufb %ymm1, %ymm2, %ymm2 + vpshufb %ymm1, %ymm3, %ymm3 + vpsubw %ymm4, %ymm8, %ymm12 + vpaddw %ymm8, %ymm4, %ymm4 + vpsubw %ymm5, %ymm9, %ymm13 + vpmullw %ymm2, %ymm12, %ymm8 + vpaddw %ymm9, %ymm5, %ymm5 + vpsubw %ymm6, %ymm10, %ymm14 + vpmullw %ymm2, %ymm13, %ymm9 + vpaddw %ymm10, %ymm6, %ymm6 + vpsubw %ymm7, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm10 + vpaddw %ymm11, %ymm7, %ymm7 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm3, %ymm12, %ymm12 + vpmulhw %ymm3, %ymm13, %ymm13 + vpmulhw %ymm3, %ymm14, %ymm14 + vpmulhw %ymm3, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm8, %ymm12, %ymm8 + vpsubw %ymm9, %ymm13, %ymm9 + vpsubw %ymm10, %ymm14, %ymm10 + vpsubw %ymm11, %ymm15, %ymm11 + vpslld $0x10, %ymm5, %ymm3 + vpblendw $0xaa, %ymm3, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7],ymm4[8],ymm3[9],ymm4[10],ymm3[11],ymm4[12],ymm3[13],ymm4[14],ymm3[15] + vpsrld $0x10, %ymm4, %ymm4 + vpblendw $0xaa, %ymm5, %ymm4, %ymm5 # ymm5 = ymm4[0],ymm5[1],ymm4[2],ymm5[3],ymm4[4],ymm5[5],ymm4[6],ymm5[7],ymm4[8],ymm5[9],ymm4[10],ymm5[11],ymm4[12],ymm5[13],ymm4[14],ymm5[15] + vpslld $0x10, %ymm7, %ymm4 + vpblendw $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7],ymm6[8],ymm4[9],ymm6[10],ymm4[11],ymm6[12],ymm4[13],ymm6[14],ymm4[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm7, %ymm6, %ymm7 # ymm7 = ymm6[0],ymm7[1],ymm6[2],ymm7[3],ymm6[4],ymm7[5],ymm6[6],ymm7[7],ymm6[8],ymm7[9],ymm6[10],ymm7[11],ymm6[12],ymm7[13],ymm6[14],ymm7[15] + vpslld $0x10, %ymm9, %ymm6 + vpblendw $0xaa, %ymm6, %ymm8, %ymm6 # ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7],ymm8[8],ymm6[9],ymm8[10],ymm6[11],ymm8[12],ymm6[13],ymm8[14],ymm6[15] + vpsrld $0x10, %ymm8, %ymm8 + vpblendw $0xaa, %ymm9, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7],ymm8[8],ymm9[9],ymm8[10],ymm9[11],ymm8[12],ymm9[13],ymm8[14],ymm9[15] + vpslld $0x10, %ymm11, %ymm8 + vpblendw $0xaa, %ymm8, %ymm10, %ymm8 # ymm8 = ymm10[0],ymm8[1],ymm10[2],ymm8[3],ymm10[4],ymm8[5],ymm10[6],ymm8[7],ymm10[8],ymm8[9],ymm10[10],ymm8[11],ymm10[12],ymm8[13],ymm10[14],ymm8[15] + vpsrld $0x10, %ymm10, %ymm10 + vpblendw $0xaa, %ymm11, %ymm10, %ymm11 # ymm11 = ymm10[0],ymm11[1],ymm10[2],ymm11[3],ymm10[4],ymm11[5],ymm10[6],ymm11[7],ymm10[8],ymm11[9],ymm10[10],ymm11[11],ymm10[12],ymm11[13],ymm10[14],ymm11[15] + vmovdqa 0x20(%rsi), %ymm12 + vpermd 0x2e0(%rsi), %ymm12, %ymm2 + vpermd 0x300(%rsi), %ymm12, %ymm10 + vpsubw %ymm3, %ymm5, %ymm12 + vpaddw %ymm5, %ymm3, %ymm3 + vpsubw %ymm4, %ymm7, %ymm13 + vpmullw %ymm2, %ymm12, %ymm5 + vpaddw %ymm7, %ymm4, %ymm4 + vpsubw %ymm6, %ymm9, %ymm14 + vpmullw %ymm2, %ymm13, %ymm7 + vpaddw %ymm9, %ymm6, %ymm6 + vpsubw %ymm8, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm9 + vpaddw %ymm11, %ymm8, %ymm8 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm10, %ymm12, %ymm12 + vpmulhw %ymm10, %ymm13, %ymm13 + vpmulhw %ymm10, %ymm14, %ymm14 + vpmulhw %ymm10, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm5, %ymm12, %ymm5 + vpsubw %ymm7, %ymm13, %ymm7 + vpsubw %ymm9, %ymm14, %ymm9 + vpsubw %ymm11, %ymm15, %ymm11 + movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + vpmulhw %ymm1, %ymm3, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm3, %ymm3 + vmovsldup %ymm4, %ymm10 # ymm10 = ymm4[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm10, %ymm3, %ymm10 # ymm10 = ymm3[0],ymm10[1],ymm3[2],ymm10[3],ymm3[4],ymm10[5],ymm3[6],ymm10[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm4, %ymm3, %ymm4 # ymm4 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7] + vmovsldup %ymm8, %ymm3 # ymm3 = ymm8[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm3, %ymm6, %ymm3 # ymm3 = ymm6[0],ymm3[1],ymm6[2],ymm3[3],ymm6[4],ymm3[5],ymm6[6],ymm3[7] + vpsrlq $0x20, %ymm6, %ymm6 + vpblendd $0xaa, %ymm8, %ymm6, %ymm8 # ymm8 = ymm6[0],ymm8[1],ymm6[2],ymm8[3],ymm6[4],ymm8[5],ymm6[6],ymm8[7] + vmovsldup %ymm7, %ymm6 # ymm6 = ymm7[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm6, %ymm5, %ymm6 # ymm6 = ymm5[0],ymm6[1],ymm5[2],ymm6[3],ymm5[4],ymm6[5],ymm5[6],ymm6[7] + vpsrlq $0x20, %ymm5, %ymm5 + vpblendd $0xaa, %ymm7, %ymm5, %ymm7 # ymm7 = ymm5[0],ymm7[1],ymm5[2],ymm7[3],ymm5[4],ymm7[5],ymm5[6],ymm7[7] + vmovsldup %ymm11, %ymm5 # ymm5 = ymm11[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm5, %ymm9, %ymm5 # ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7] + vpsrlq $0x20, %ymm9, %ymm9 + vpblendd $0xaa, %ymm11, %ymm9, %ymm11 # ymm11 = ymm9[0],ymm11[1],ymm9[2],ymm11[3],ymm9[4],ymm11[5],ymm9[6],ymm11[7] + vpermq $0x1b, 0x2a0(%rsi), %ymm2 # ymm2 = mem[3,2,1,0] + vpermq $0x1b, 0x2c0(%rsi), %ymm9 # ymm9 = mem[3,2,1,0] + vpsubw %ymm10, %ymm4, %ymm12 + vpaddw %ymm4, %ymm10, %ymm10 + vpsubw %ymm3, %ymm8, %ymm13 + vpmullw %ymm2, %ymm12, %ymm4 + vpaddw %ymm8, %ymm3, %ymm3 + vpsubw %ymm6, %ymm7, %ymm14 + vpmullw %ymm2, %ymm13, %ymm8 + vpaddw %ymm7, %ymm6, %ymm6 + vpsubw %ymm5, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm7 + vpaddw %ymm11, %ymm5, %ymm5 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm9, %ymm12, %ymm12 + vpmulhw %ymm9, %ymm13, %ymm13 + vpmulhw %ymm9, %ymm14, %ymm14 + vpmulhw %ymm9, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm4, %ymm4 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm4, %ymm12, %ymm4 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm7, %ymm14, %ymm7 + vpsubw %ymm11, %ymm15, %ymm11 + vpmulhw %ymm1, %ymm10, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm10, %ymm10 + vpunpcklqdq %ymm3, %ymm10, %ymm9 # ymm9 = ymm10[0],ymm3[0],ymm10[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm10, %ymm3 # ymm3 = ymm10[1],ymm3[1],ymm10[3],ymm3[3] + vpunpcklqdq %ymm5, %ymm6, %ymm10 # ymm10 = ymm6[0],ymm5[0],ymm6[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[1],ymm5[1],ymm6[3],ymm5[3] + vpunpcklqdq %ymm8, %ymm4, %ymm6 # ymm6 = ymm4[0],ymm8[0],ymm4[2],ymm8[2] + vpunpckhqdq %ymm8, %ymm4, %ymm8 # ymm8 = ymm4[1],ymm8[1],ymm4[3],ymm8[3] + vpunpcklqdq %ymm11, %ymm7, %ymm4 # ymm4 = ymm7[0],ymm11[0],ymm7[2],ymm11[2] + vpunpckhqdq %ymm11, %ymm7, %ymm11 # ymm11 = ymm7[1],ymm11[1],ymm7[3],ymm11[3] + vpermq $0x4e, 0x260(%rsi), %ymm2 # ymm2 = mem[2,3,0,1] + vpermq $0x4e, 0x280(%rsi), %ymm7 # ymm7 = mem[2,3,0,1] + vpsubw %ymm9, %ymm3, %ymm12 + vpaddw %ymm3, %ymm9, %ymm9 + vpsubw %ymm10, %ymm5, %ymm13 + vpmullw %ymm2, %ymm12, %ymm3 + vpaddw %ymm5, %ymm10, %ymm10 + vpsubw %ymm6, %ymm8, %ymm14 + vpmullw %ymm2, %ymm13, %ymm5 + vpaddw %ymm8, %ymm6, %ymm6 + vpsubw %ymm4, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm8 + vpaddw %ymm11, %ymm4, %ymm4 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm7, %ymm12, %ymm12 + vpmulhw %ymm7, %ymm13, %ymm13 + vpmulhw %ymm7, %ymm14, %ymm14 + vpmulhw %ymm7, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm3, %ymm3 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm3, %ymm12, %ymm3 + vpsubw %ymm5, %ymm13, %ymm5 + vpsubw %ymm8, %ymm14, %ymm8 + vpsubw %ymm11, %ymm15, %ymm11 + vpmulhw %ymm1, %ymm9, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm9, %ymm9 + vperm2i128 $0x20, %ymm10, %ymm9, %ymm7 # ymm7 = ymm9[0,1],ymm10[0,1] + vperm2i128 $0x31, %ymm10, %ymm9, %ymm10 # ymm10 = ymm9[2,3],ymm10[2,3] + vperm2i128 $0x20, %ymm4, %ymm6, %ymm9 # ymm9 = ymm6[0,1],ymm4[0,1] + vperm2i128 $0x31, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[2,3],ymm4[2,3] + vperm2i128 $0x20, %ymm5, %ymm3, %ymm6 # ymm6 = ymm3[0,1],ymm5[0,1] + vperm2i128 $0x31, %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[2,3],ymm5[2,3] + vperm2i128 $0x20, %ymm11, %ymm8, %ymm3 # ymm3 = ymm8[0,1],ymm11[0,1] + vperm2i128 $0x31, %ymm11, %ymm8, %ymm11 # ymm11 = ymm8[2,3],ymm11[2,3] + vmovdqa 0x220(%rsi), %ymm2 + vmovdqa 0x240(%rsi), %ymm8 + vpsubw %ymm7, %ymm10, %ymm12 + vpaddw %ymm10, %ymm7, %ymm7 + vpsubw %ymm9, %ymm4, %ymm13 + vpmullw %ymm2, %ymm12, %ymm10 + vpaddw %ymm4, %ymm9, %ymm9 + vpsubw %ymm6, %ymm5, %ymm14 + vpmullw %ymm2, %ymm13, %ymm4 + vpaddw %ymm5, %ymm6, %ymm6 + vpsubw %ymm3, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm5 + vpaddw %ymm11, %ymm3, %ymm3 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm8, %ymm12, %ymm12 + vpmulhw %ymm8, %ymm13, %ymm13 + vpmulhw %ymm8, %ymm14, %ymm14 + vpmulhw %ymm8, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm4, %ymm4 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm10, %ymm12, %ymm10 + vpsubw %ymm4, %ymm13, %ymm4 + vpsubw %ymm5, %ymm14, %ymm5 + vpsubw %ymm11, %ymm15, %ymm11 + vpmulhw %ymm1, %ymm7, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm7, %ymm7 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa %ymm6, 0x40(%rdi) + vmovdqa %ymm3, 0x60(%rdi) + vmovdqa %ymm10, 0x80(%rdi) + vmovdqa %ymm4, 0xa0(%rdi) + vmovdqa %ymm5, 0xc0(%rdi) + vmovdqa %ymm11, 0xe0(%rdi) + movl $0xd8a1d8a1, %eax # imm = 0xD8A1D8A1 + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + movl $0x5a105a1, %eax # imm = 0x5A105A1 + vmovd %eax, %xmm3 + vpbroadcastd %xmm3, %ymm3 + vmovdqa 0x100(%rdi), %ymm4 + vmovdqa 0x140(%rdi), %ymm6 + vmovdqa 0x120(%rdi), %ymm5 + vmovdqa 0x160(%rdi), %ymm7 + vpmullw %ymm2, %ymm4, %ymm12 + vpmulhw %ymm3, %ymm4, %ymm4 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm4, %ymm4 + vpmullw %ymm2, %ymm6, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm6, %ymm6 + vpmullw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm5, %ymm5 + vpmullw %ymm2, %ymm7, %ymm12 + vpmulhw %ymm3, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm7, %ymm7 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1c0(%rdi), %ymm10 + vmovdqa 0x1a0(%rdi), %ymm9 + vmovdqa 0x1e0(%rdi), %ymm11 + vpmullw %ymm2, %ymm8, %ymm12 + vpmulhw %ymm3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm8, %ymm8 + vpmullw %ymm2, %ymm10, %ymm12 + vpmulhw %ymm3, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm10, %ymm10 + vpmullw %ymm2, %ymm9, %ymm12 + vpmulhw %ymm3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm9, %ymm9 + vpmullw %ymm2, %ymm11, %ymm12 + vpmulhw %ymm3, %ymm11, %ymm11 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm11, %ymm11 + vpermq $0x4e, 0x1e0(%rsi), %ymm15 # ymm15 = mem[2,3,0,1] + vpermq $0x4e, 0x1a0(%rsi), %ymm1 # ymm1 = mem[2,3,0,1] + vpermq $0x4e, 0x200(%rsi), %ymm2 # ymm2 = mem[2,3,0,1] + vpermq $0x4e, 0x1c0(%rsi), %ymm3 # ymm3 = mem[2,3,0,1] + vmovdqa (%rsi), %ymm12 + vpshufb %ymm12, %ymm15, %ymm15 + vpshufb %ymm12, %ymm1, %ymm1 + vpshufb %ymm12, %ymm2, %ymm2 + vpshufb %ymm12, %ymm3, %ymm3 + vpsubw %ymm4, %ymm6, %ymm12 + vpaddw %ymm6, %ymm4, %ymm4 + vpsubw %ymm5, %ymm7, %ymm13 + vpmullw %ymm15, %ymm12, %ymm6 + vpaddw %ymm7, %ymm5, %ymm5 + vpsubw %ymm8, %ymm10, %ymm14 + vpmullw %ymm15, %ymm13, %ymm7 + vpaddw %ymm10, %ymm8, %ymm8 + vpsubw %ymm9, %ymm11, %ymm15 + vpmullw %ymm1, %ymm14, %ymm10 + vpaddw %ymm11, %ymm9, %ymm9 + vpmullw %ymm1, %ymm15, %ymm11 + vpmulhw %ymm2, %ymm12, %ymm12 + vpmulhw %ymm2, %ymm13, %ymm13 + vpmulhw %ymm3, %ymm14, %ymm14 + vpmulhw %ymm3, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm6, %ymm12, %ymm6 + vpsubw %ymm7, %ymm13, %ymm7 + vpsubw %ymm10, %ymm14, %ymm10 + vpsubw %ymm11, %ymm15, %ymm11 + vpermq $0x4e, 0x160(%rsi), %ymm2 # ymm2 = mem[2,3,0,1] + vpermq $0x4e, 0x180(%rsi), %ymm3 # ymm3 = mem[2,3,0,1] + vmovdqa (%rsi), %ymm1 + vpshufb %ymm1, %ymm2, %ymm2 + vpshufb %ymm1, %ymm3, %ymm3 + vpsubw %ymm4, %ymm8, %ymm12 + vpaddw %ymm8, %ymm4, %ymm4 + vpsubw %ymm5, %ymm9, %ymm13 + vpmullw %ymm2, %ymm12, %ymm8 + vpaddw %ymm9, %ymm5, %ymm5 + vpsubw %ymm6, %ymm10, %ymm14 + vpmullw %ymm2, %ymm13, %ymm9 + vpaddw %ymm10, %ymm6, %ymm6 + vpsubw %ymm7, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm10 + vpaddw %ymm11, %ymm7, %ymm7 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm3, %ymm12, %ymm12 + vpmulhw %ymm3, %ymm13, %ymm13 + vpmulhw %ymm3, %ymm14, %ymm14 + vpmulhw %ymm3, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm8, %ymm12, %ymm8 + vpsubw %ymm9, %ymm13, %ymm9 + vpsubw %ymm10, %ymm14, %ymm10 + vpsubw %ymm11, %ymm15, %ymm11 + vpslld $0x10, %ymm5, %ymm3 + vpblendw $0xaa, %ymm3, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7],ymm4[8],ymm3[9],ymm4[10],ymm3[11],ymm4[12],ymm3[13],ymm4[14],ymm3[15] + vpsrld $0x10, %ymm4, %ymm4 + vpblendw $0xaa, %ymm5, %ymm4, %ymm5 # ymm5 = ymm4[0],ymm5[1],ymm4[2],ymm5[3],ymm4[4],ymm5[5],ymm4[6],ymm5[7],ymm4[8],ymm5[9],ymm4[10],ymm5[11],ymm4[12],ymm5[13],ymm4[14],ymm5[15] + vpslld $0x10, %ymm7, %ymm4 + vpblendw $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7],ymm6[8],ymm4[9],ymm6[10],ymm4[11],ymm6[12],ymm4[13],ymm6[14],ymm4[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm7, %ymm6, %ymm7 # ymm7 = ymm6[0],ymm7[1],ymm6[2],ymm7[3],ymm6[4],ymm7[5],ymm6[6],ymm7[7],ymm6[8],ymm7[9],ymm6[10],ymm7[11],ymm6[12],ymm7[13],ymm6[14],ymm7[15] + vpslld $0x10, %ymm9, %ymm6 + vpblendw $0xaa, %ymm6, %ymm8, %ymm6 # ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7],ymm8[8],ymm6[9],ymm8[10],ymm6[11],ymm8[12],ymm6[13],ymm8[14],ymm6[15] + vpsrld $0x10, %ymm8, %ymm8 + vpblendw $0xaa, %ymm9, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7],ymm8[8],ymm9[9],ymm8[10],ymm9[11],ymm8[12],ymm9[13],ymm8[14],ymm9[15] + vpslld $0x10, %ymm11, %ymm8 + vpblendw $0xaa, %ymm8, %ymm10, %ymm8 # ymm8 = ymm10[0],ymm8[1],ymm10[2],ymm8[3],ymm10[4],ymm8[5],ymm10[6],ymm8[7],ymm10[8],ymm8[9],ymm10[10],ymm8[11],ymm10[12],ymm8[13],ymm10[14],ymm8[15] + vpsrld $0x10, %ymm10, %ymm10 + vpblendw $0xaa, %ymm11, %ymm10, %ymm11 # ymm11 = ymm10[0],ymm11[1],ymm10[2],ymm11[3],ymm10[4],ymm11[5],ymm10[6],ymm11[7],ymm10[8],ymm11[9],ymm10[10],ymm11[11],ymm10[12],ymm11[13],ymm10[14],ymm11[15] + vmovdqa 0x20(%rsi), %ymm12 + vpermd 0x120(%rsi), %ymm12, %ymm2 + vpermd 0x140(%rsi), %ymm12, %ymm10 + vpsubw %ymm3, %ymm5, %ymm12 + vpaddw %ymm5, %ymm3, %ymm3 + vpsubw %ymm4, %ymm7, %ymm13 + vpmullw %ymm2, %ymm12, %ymm5 + vpaddw %ymm7, %ymm4, %ymm4 + vpsubw %ymm6, %ymm9, %ymm14 + vpmullw %ymm2, %ymm13, %ymm7 + vpaddw %ymm9, %ymm6, %ymm6 + vpsubw %ymm8, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm9 + vpaddw %ymm11, %ymm8, %ymm8 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm10, %ymm12, %ymm12 + vpmulhw %ymm10, %ymm13, %ymm13 + vpmulhw %ymm10, %ymm14, %ymm14 + vpmulhw %ymm10, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm5, %ymm12, %ymm5 + vpsubw %ymm7, %ymm13, %ymm7 + vpsubw %ymm9, %ymm14, %ymm9 + vpsubw %ymm11, %ymm15, %ymm11 + movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + vpmulhw %ymm1, %ymm3, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm3, %ymm3 + vmovsldup %ymm4, %ymm10 # ymm10 = ymm4[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm10, %ymm3, %ymm10 # ymm10 = ymm3[0],ymm10[1],ymm3[2],ymm10[3],ymm3[4],ymm10[5],ymm3[6],ymm10[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm4, %ymm3, %ymm4 # ymm4 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7] + vmovsldup %ymm8, %ymm3 # ymm3 = ymm8[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm3, %ymm6, %ymm3 # ymm3 = ymm6[0],ymm3[1],ymm6[2],ymm3[3],ymm6[4],ymm3[5],ymm6[6],ymm3[7] + vpsrlq $0x20, %ymm6, %ymm6 + vpblendd $0xaa, %ymm8, %ymm6, %ymm8 # ymm8 = ymm6[0],ymm8[1],ymm6[2],ymm8[3],ymm6[4],ymm8[5],ymm6[6],ymm8[7] + vmovsldup %ymm7, %ymm6 # ymm6 = ymm7[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm6, %ymm5, %ymm6 # ymm6 = ymm5[0],ymm6[1],ymm5[2],ymm6[3],ymm5[4],ymm6[5],ymm5[6],ymm6[7] + vpsrlq $0x20, %ymm5, %ymm5 + vpblendd $0xaa, %ymm7, %ymm5, %ymm7 # ymm7 = ymm5[0],ymm7[1],ymm5[2],ymm7[3],ymm5[4],ymm7[5],ymm5[6],ymm7[7] + vmovsldup %ymm11, %ymm5 # ymm5 = ymm11[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm5, %ymm9, %ymm5 # ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7] + vpsrlq $0x20, %ymm9, %ymm9 + vpblendd $0xaa, %ymm11, %ymm9, %ymm11 # ymm11 = ymm9[0],ymm11[1],ymm9[2],ymm11[3],ymm9[4],ymm11[5],ymm9[6],ymm11[7] + vpermq $0x1b, 0xe0(%rsi), %ymm2 # ymm2 = mem[3,2,1,0] + vpermq $0x1b, 0x100(%rsi), %ymm9 # ymm9 = mem[3,2,1,0] + vpsubw %ymm10, %ymm4, %ymm12 + vpaddw %ymm4, %ymm10, %ymm10 + vpsubw %ymm3, %ymm8, %ymm13 + vpmullw %ymm2, %ymm12, %ymm4 + vpaddw %ymm8, %ymm3, %ymm3 + vpsubw %ymm6, %ymm7, %ymm14 + vpmullw %ymm2, %ymm13, %ymm8 + vpaddw %ymm7, %ymm6, %ymm6 + vpsubw %ymm5, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm7 + vpaddw %ymm11, %ymm5, %ymm5 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm9, %ymm12, %ymm12 + vpmulhw %ymm9, %ymm13, %ymm13 + vpmulhw %ymm9, %ymm14, %ymm14 + vpmulhw %ymm9, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm4, %ymm4 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm4, %ymm12, %ymm4 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm7, %ymm14, %ymm7 + vpsubw %ymm11, %ymm15, %ymm11 + vpmulhw %ymm1, %ymm10, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm10, %ymm10 + vpunpcklqdq %ymm3, %ymm10, %ymm9 # ymm9 = ymm10[0],ymm3[0],ymm10[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm10, %ymm3 # ymm3 = ymm10[1],ymm3[1],ymm10[3],ymm3[3] + vpunpcklqdq %ymm5, %ymm6, %ymm10 # ymm10 = ymm6[0],ymm5[0],ymm6[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[1],ymm5[1],ymm6[3],ymm5[3] + vpunpcklqdq %ymm8, %ymm4, %ymm6 # ymm6 = ymm4[0],ymm8[0],ymm4[2],ymm8[2] + vpunpckhqdq %ymm8, %ymm4, %ymm8 # ymm8 = ymm4[1],ymm8[1],ymm4[3],ymm8[3] + vpunpcklqdq %ymm11, %ymm7, %ymm4 # ymm4 = ymm7[0],ymm11[0],ymm7[2],ymm11[2] + vpunpckhqdq %ymm11, %ymm7, %ymm11 # ymm11 = ymm7[1],ymm11[1],ymm7[3],ymm11[3] + vpermq $0x4e, 0xa0(%rsi), %ymm2 # ymm2 = mem[2,3,0,1] + vpermq $0x4e, 0xc0(%rsi), %ymm7 # ymm7 = mem[2,3,0,1] + vpsubw %ymm9, %ymm3, %ymm12 + vpaddw %ymm3, %ymm9, %ymm9 + vpsubw %ymm10, %ymm5, %ymm13 + vpmullw %ymm2, %ymm12, %ymm3 + vpaddw %ymm5, %ymm10, %ymm10 + vpsubw %ymm6, %ymm8, %ymm14 + vpmullw %ymm2, %ymm13, %ymm5 + vpaddw %ymm8, %ymm6, %ymm6 + vpsubw %ymm4, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm8 + vpaddw %ymm11, %ymm4, %ymm4 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm7, %ymm12, %ymm12 + vpmulhw %ymm7, %ymm13, %ymm13 + vpmulhw %ymm7, %ymm14, %ymm14 + vpmulhw %ymm7, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm3, %ymm3 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm3, %ymm12, %ymm3 + vpsubw %ymm5, %ymm13, %ymm5 + vpsubw %ymm8, %ymm14, %ymm8 + vpsubw %ymm11, %ymm15, %ymm11 + vpmulhw %ymm1, %ymm9, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm9, %ymm9 + vperm2i128 $0x20, %ymm10, %ymm9, %ymm7 # ymm7 = ymm9[0,1],ymm10[0,1] + vperm2i128 $0x31, %ymm10, %ymm9, %ymm10 # ymm10 = ymm9[2,3],ymm10[2,3] + vperm2i128 $0x20, %ymm4, %ymm6, %ymm9 # ymm9 = ymm6[0,1],ymm4[0,1] + vperm2i128 $0x31, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[2,3],ymm4[2,3] + vperm2i128 $0x20, %ymm5, %ymm3, %ymm6 # ymm6 = ymm3[0,1],ymm5[0,1] + vperm2i128 $0x31, %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[2,3],ymm5[2,3] + vperm2i128 $0x20, %ymm11, %ymm8, %ymm3 # ymm3 = ymm8[0,1],ymm11[0,1] + vperm2i128 $0x31, %ymm11, %ymm8, %ymm11 # ymm11 = ymm8[2,3],ymm11[2,3] + vmovdqa 0x60(%rsi), %ymm2 + vmovdqa 0x80(%rsi), %ymm8 + vpsubw %ymm7, %ymm10, %ymm12 + vpaddw %ymm10, %ymm7, %ymm7 + vpsubw %ymm9, %ymm4, %ymm13 + vpmullw %ymm2, %ymm12, %ymm10 + vpaddw %ymm4, %ymm9, %ymm9 + vpsubw %ymm6, %ymm5, %ymm14 + vpmullw %ymm2, %ymm13, %ymm4 + vpaddw %ymm5, %ymm6, %ymm6 + vpsubw %ymm3, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm5 + vpaddw %ymm11, %ymm3, %ymm3 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm8, %ymm12, %ymm12 + vpmulhw %ymm8, %ymm13, %ymm13 + vpmulhw %ymm8, %ymm14, %ymm14 + vpmulhw %ymm8, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm4, %ymm4 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm10, %ymm12, %ymm10 + vpsubw %ymm4, %ymm13, %ymm4 + vpsubw %ymm5, %ymm14, %ymm5 + vpsubw %ymm11, %ymm15, %ymm11 + vpmulhw %ymm1, %ymm7, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm7, %ymm7 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa %ymm6, 0x140(%rdi) + vmovdqa %ymm3, 0x160(%rdi) + vmovdqa %ymm10, 0x180(%rdi) + vmovdqa %ymm4, 0x1a0(%rdi) + vmovdqa %ymm5, 0x1c0(%rdi) + vmovdqa %ymm11, 0x1e0(%rdi) + vmovdqa (%rdi), %ymm4 + vmovdqa 0x100(%rdi), %ymm8 + vmovdqa 0x20(%rdi), %ymm5 + vmovdqa 0x120(%rdi), %ymm9 + vpbroadcastq 0x40(%rsi), %ymm2 + vmovdqa 0x40(%rdi), %ymm6 + vmovdqa 0x140(%rdi), %ymm10 + vmovdqa 0x60(%rdi), %ymm7 + vmovdqa 0x160(%rdi), %ymm11 + vpbroadcastq 0x48(%rsi), %ymm3 + vpsubw %ymm4, %ymm8, %ymm12 + vpaddw %ymm8, %ymm4, %ymm4 + vpsubw %ymm5, %ymm9, %ymm13 + vpmullw %ymm2, %ymm12, %ymm8 + vpaddw %ymm9, %ymm5, %ymm5 + vpsubw %ymm6, %ymm10, %ymm14 + vpmullw %ymm2, %ymm13, %ymm9 + vpaddw %ymm10, %ymm6, %ymm6 + vpsubw %ymm7, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm10 + vpaddw %ymm11, %ymm7, %ymm7 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm3, %ymm12, %ymm12 + vpmulhw %ymm3, %ymm13, %ymm13 + vpmulhw %ymm3, %ymm14, %ymm14 + vpmulhw %ymm3, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm8, %ymm12, %ymm8 + vpsubw %ymm9, %ymm13, %ymm9 + vpsubw %ymm10, %ymm14, %ymm10 + vpsubw %ymm11, %ymm15, %ymm11 + vmovdqa %ymm4, (%rdi) + vmovdqa %ymm5, 0x20(%rdi) + vmovdqa %ymm6, 0x40(%rdi) + vmovdqa %ymm7, 0x60(%rdi) + vmovdqa %ymm8, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa %ymm10, 0x140(%rdi) + vmovdqa %ymm11, 0x160(%rdi) + vmovdqa 0x80(%rdi), %ymm4 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm5 + vmovdqa 0x1a0(%rdi), %ymm9 + vpbroadcastq 0x40(%rsi), %ymm2 + vmovdqa 0xc0(%rdi), %ymm6 + vmovdqa 0x1c0(%rdi), %ymm10 + vmovdqa 0xe0(%rdi), %ymm7 + vmovdqa 0x1e0(%rdi), %ymm11 + vpbroadcastq 0x48(%rsi), %ymm3 + vpsubw %ymm4, %ymm8, %ymm12 + vpaddw %ymm8, %ymm4, %ymm4 + vpsubw %ymm5, %ymm9, %ymm13 + vpmullw %ymm2, %ymm12, %ymm8 + vpaddw %ymm9, %ymm5, %ymm5 + vpsubw %ymm6, %ymm10, %ymm14 + vpmullw %ymm2, %ymm13, %ymm9 + vpaddw %ymm10, %ymm6, %ymm6 + vpsubw %ymm7, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm10 + vpaddw %ymm11, %ymm7, %ymm7 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm3, %ymm12, %ymm12 + vpmulhw %ymm3, %ymm13, %ymm13 + vpmulhw %ymm3, %ymm14, %ymm14 + vpmulhw %ymm3, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm8, %ymm12, %ymm8 + vpsubw %ymm9, %ymm13, %ymm9 + vpsubw %ymm10, %ymm14, %ymm10 + vpsubw %ymm11, %ymm15, %ymm11 + vmovdqa %ymm4, 0x80(%rdi) + vmovdqa %ymm5, 0xa0(%rdi) + vmovdqa %ymm6, 0xc0(%rdi) + vmovdqa %ymm7, 0xe0(%rdi) + vmovdqa %ymm8, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa %ymm10, 0x1c0(%rdi) + vmovdqa %ymm11, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(invntt_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_ntt_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_ntt_avx2_asm.S new file mode 100644 index 0000000000..4e7894a411 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_ntt_avx2_asm.S @@ -0,0 +1,661 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [AVX2_NTT] + * Faster AVX2 optimized NTT multiplication for Ring-LWE lattice cryptography. + * Gregor Seiler + * https://eprint.iacr.org/2018/039 + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + * + * The core ideas behind the implementation are described in @[AVX2_NTT]. + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/*yaml + Name: ntt_avx2_asm + Description: x86_64 AVX2 forward NTT + Signature: void mlk_ntt_avx2_asm(int16_t *r, const int16_t *qdata) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 1248 + permissions: read-only + c_parameter: const int16_t *qdata + description: Precomputed constants (624 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_ntt_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(ntt_avx2_asm) +MLK_ASM_FN_SYMBOL(ntt_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + vpbroadcastq 0x40(%rsi), %ymm15 + vmovdqa 0x100(%rdi), %ymm8 + vmovdqa 0x120(%rdi), %ymm9 + vmovdqa 0x140(%rdi), %ymm10 + vmovdqa 0x160(%rdi), %ymm11 + vpbroadcastq 0x48(%rsi), %ymm2 + vpmullw %ymm15, %ymm8, %ymm12 + vpmullw %ymm15, %ymm9, %ymm13 + vpmullw %ymm15, %ymm10, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm2, %ymm11, %ymm11 + vmovdqa (%rdi), %ymm4 + vmovdqa 0x20(%rdi), %ymm5 + vmovdqa 0x40(%rdi), %ymm6 + vmovdqa 0x60(%rdi), %ymm7 + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm8, %ymm4, %ymm3 + vpsubw %ymm8, %ymm4, %ymm8 + vpaddw %ymm9, %ymm5, %ymm4 + vpsubw %ymm9, %ymm5, %ymm9 + vpaddw %ymm10, %ymm6, %ymm5 + vpsubw %ymm10, %ymm6, %ymm10 + vpaddw %ymm11, %ymm7, %ymm6 + vpsubw %ymm11, %ymm7, %ymm11 + vpsubw %ymm12, %ymm3, %ymm3 + vpaddw %ymm12, %ymm8, %ymm8 + vpsubw %ymm13, %ymm4, %ymm4 + vpaddw %ymm13, %ymm9, %ymm9 + vpsubw %ymm14, %ymm5, %ymm5 + vpaddw %ymm14, %ymm10, %ymm10 + vpsubw %ymm15, %ymm6, %ymm6 + vpaddw %ymm15, %ymm11, %ymm11 + vmovdqa %ymm3, (%rdi) + vmovdqa %ymm4, 0x20(%rdi) + vmovdqa %ymm5, 0x40(%rdi) + vmovdqa %ymm6, 0x60(%rdi) + vmovdqa %ymm8, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa %ymm10, 0x140(%rdi) + vmovdqa %ymm11, 0x160(%rdi) + vpbroadcastq 0x40(%rsi), %ymm15 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm9 + vmovdqa 0x1c0(%rdi), %ymm10 + vmovdqa 0x1e0(%rdi), %ymm11 + vpbroadcastq 0x48(%rsi), %ymm2 + vpmullw %ymm15, %ymm8, %ymm12 + vpmullw %ymm15, %ymm9, %ymm13 + vpmullw %ymm15, %ymm10, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm2, %ymm11, %ymm11 + vmovdqa 0x80(%rdi), %ymm4 + vmovdqa 0xa0(%rdi), %ymm5 + vmovdqa 0xc0(%rdi), %ymm6 + vmovdqa 0xe0(%rdi), %ymm7 + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm8, %ymm4, %ymm3 + vpsubw %ymm8, %ymm4, %ymm8 + vpaddw %ymm9, %ymm5, %ymm4 + vpsubw %ymm9, %ymm5, %ymm9 + vpaddw %ymm10, %ymm6, %ymm5 + vpsubw %ymm10, %ymm6, %ymm10 + vpaddw %ymm11, %ymm7, %ymm6 + vpsubw %ymm11, %ymm7, %ymm11 + vpsubw %ymm12, %ymm3, %ymm3 + vpaddw %ymm12, %ymm8, %ymm8 + vpsubw %ymm13, %ymm4, %ymm4 + vpaddw %ymm13, %ymm9, %ymm9 + vpsubw %ymm14, %ymm5, %ymm5 + vpaddw %ymm14, %ymm10, %ymm10 + vpsubw %ymm15, %ymm6, %ymm6 + vpaddw %ymm15, %ymm11, %ymm11 + vmovdqa %ymm3, 0x80(%rdi) + vmovdqa %ymm4, 0xa0(%rdi) + vmovdqa %ymm5, 0xc0(%rdi) + vmovdqa %ymm6, 0xe0(%rdi) + vmovdqa %ymm8, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa %ymm10, 0x1c0(%rdi) + vmovdqa %ymm11, 0x1e0(%rdi) + vmovdqa 0x60(%rsi), %ymm15 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm9 + vmovdqa 0xc0(%rdi), %ymm10 + vmovdqa 0xe0(%rdi), %ymm11 + vmovdqa 0x80(%rsi), %ymm2 + vpmullw %ymm15, %ymm8, %ymm12 + vpmullw %ymm15, %ymm9, %ymm13 + vpmullw %ymm15, %ymm10, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm2, %ymm11, %ymm11 + vmovdqa (%rdi), %ymm4 + vmovdqa 0x20(%rdi), %ymm5 + vmovdqa 0x40(%rdi), %ymm6 + vmovdqa 0x60(%rdi), %ymm7 + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm8, %ymm4, %ymm3 + vpsubw %ymm8, %ymm4, %ymm8 + vpaddw %ymm9, %ymm5, %ymm4 + vpsubw %ymm9, %ymm5, %ymm9 + vpaddw %ymm10, %ymm6, %ymm5 + vpsubw %ymm10, %ymm6, %ymm10 + vpaddw %ymm11, %ymm7, %ymm6 + vpsubw %ymm11, %ymm7, %ymm11 + vpsubw %ymm12, %ymm3, %ymm3 + vpaddw %ymm12, %ymm8, %ymm8 + vpsubw %ymm13, %ymm4, %ymm4 + vpaddw %ymm13, %ymm9, %ymm9 + vpsubw %ymm14, %ymm5, %ymm5 + vpaddw %ymm14, %ymm10, %ymm10 + vpsubw %ymm15, %ymm6, %ymm6 + vpaddw %ymm15, %ymm11, %ymm11 + vperm2i128 $0x20, %ymm10, %ymm5, %ymm7 # ymm7 = ymm5[0,1],ymm10[0,1] + vperm2i128 $0x31, %ymm10, %ymm5, %ymm10 # ymm10 = ymm5[2,3],ymm10[2,3] + vperm2i128 $0x20, %ymm11, %ymm6, %ymm5 # ymm5 = ymm6[0,1],ymm11[0,1] + vperm2i128 $0x31, %ymm11, %ymm6, %ymm11 # ymm11 = ymm6[2,3],ymm11[2,3] + vmovdqa 0xa0(%rsi), %ymm15 + vmovdqa 0xc0(%rsi), %ymm2 + vpmullw %ymm15, %ymm7, %ymm12 + vpmullw %ymm15, %ymm10, %ymm13 + vpmullw %ymm15, %ymm5, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm7, %ymm7 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm2, %ymm5, %ymm5 + vpmulhw %ymm2, %ymm11, %ymm11 + vperm2i128 $0x20, %ymm8, %ymm3, %ymm6 # ymm6 = ymm3[0,1],ymm8[0,1] + vperm2i128 $0x31, %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[2,3],ymm8[2,3] + vperm2i128 $0x20, %ymm9, %ymm4, %ymm3 # ymm3 = ymm4[0,1],ymm9[0,1] + vperm2i128 $0x31, %ymm9, %ymm4, %ymm9 # ymm9 = ymm4[2,3],ymm9[2,3] + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm7, %ymm6, %ymm4 + vpsubw %ymm7, %ymm6, %ymm7 + vpaddw %ymm10, %ymm8, %ymm6 + vpsubw %ymm10, %ymm8, %ymm10 + vpaddw %ymm5, %ymm3, %ymm8 + vpsubw %ymm5, %ymm3, %ymm5 + vpaddw %ymm11, %ymm9, %ymm3 + vpsubw %ymm11, %ymm9, %ymm11 + vpsubw %ymm12, %ymm4, %ymm4 + vpaddw %ymm12, %ymm7, %ymm7 + vpsubw %ymm13, %ymm6, %ymm6 + vpaddw %ymm13, %ymm10, %ymm10 + vpsubw %ymm14, %ymm8, %ymm8 + vpaddw %ymm14, %ymm5, %ymm5 + vpsubw %ymm15, %ymm3, %ymm3 + vpaddw %ymm15, %ymm11, %ymm11 + vpunpcklqdq %ymm5, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm5[0],ymm8[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm8, %ymm5 # ymm5 = ymm8[1],ymm5[1],ymm8[3],ymm5[3] + vpunpcklqdq %ymm11, %ymm3, %ymm8 # ymm8 = ymm3[0],ymm11[0],ymm3[2],ymm11[2] + vpunpckhqdq %ymm11, %ymm3, %ymm11 # ymm11 = ymm3[1],ymm11[1],ymm3[3],ymm11[3] + vmovdqa 0xe0(%rsi), %ymm15 + vmovdqa 0x100(%rsi), %ymm2 + vpmullw %ymm15, %ymm9, %ymm12 + vpmullw %ymm15, %ymm5, %ymm13 + vpmullw %ymm15, %ymm8, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm5, %ymm5 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm2, %ymm11, %ymm11 + vpunpcklqdq %ymm7, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm7[0],ymm4[2],ymm7[2] + vpunpckhqdq %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[1],ymm7[1],ymm4[3],ymm7[3] + vpunpcklqdq %ymm10, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm10[0],ymm6[2],ymm10[2] + vpunpckhqdq %ymm10, %ymm6, %ymm10 # ymm10 = ymm6[1],ymm10[1],ymm6[3],ymm10[3] + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm9, %ymm3, %ymm6 + vpsubw %ymm9, %ymm3, %ymm9 + vpaddw %ymm5, %ymm7, %ymm3 + vpsubw %ymm5, %ymm7, %ymm5 + vpaddw %ymm8, %ymm4, %ymm7 + vpsubw %ymm8, %ymm4, %ymm8 + vpaddw %ymm11, %ymm10, %ymm4 + vpsubw %ymm11, %ymm10, %ymm11 + vpsubw %ymm12, %ymm6, %ymm6 + vpaddw %ymm12, %ymm9, %ymm9 + vpsubw %ymm13, %ymm3, %ymm3 + vpaddw %ymm13, %ymm5, %ymm5 + vpsubw %ymm14, %ymm7, %ymm7 + vpaddw %ymm14, %ymm8, %ymm8 + vpsubw %ymm15, %ymm4, %ymm4 + vpaddw %ymm15, %ymm11, %ymm11 + vmovsldup %ymm8, %ymm10 # ymm10 = ymm8[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm10, %ymm7, %ymm10 # ymm10 = ymm7[0],ymm10[1],ymm7[2],ymm10[3],ymm7[4],ymm10[5],ymm7[6],ymm10[7] + vpsrlq $0x20, %ymm7, %ymm7 + vpblendd $0xaa, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[0],ymm8[1],ymm7[2],ymm8[3],ymm7[4],ymm8[5],ymm7[6],ymm8[7] + vmovsldup %ymm11, %ymm7 # ymm7 = ymm11[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[0],ymm7[1],ymm4[2],ymm7[3],ymm4[4],ymm7[5],ymm4[6],ymm7[7] + vpsrlq $0x20, %ymm4, %ymm4 + vpblendd $0xaa, %ymm11, %ymm4, %ymm11 # ymm11 = ymm4[0],ymm11[1],ymm4[2],ymm11[3],ymm4[4],ymm11[5],ymm4[6],ymm11[7] + vmovdqa 0x120(%rsi), %ymm15 + vmovdqa 0x140(%rsi), %ymm2 + vpmullw %ymm15, %ymm10, %ymm12 + vpmullw %ymm15, %ymm8, %ymm13 + vpmullw %ymm15, %ymm7, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm2, %ymm7, %ymm7 + vpmulhw %ymm2, %ymm11, %ymm11 + vmovsldup %ymm9, %ymm4 # ymm4 = ymm9[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7] + vpsrlq $0x20, %ymm6, %ymm6 + vpblendd $0xaa, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[0],ymm9[1],ymm6[2],ymm9[3],ymm6[4],ymm9[5],ymm6[6],ymm9[7] + vmovsldup %ymm5, %ymm6 # ymm6 = ymm5[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm6, %ymm3, %ymm6 # ymm6 = ymm3[0],ymm6[1],ymm3[2],ymm6[3],ymm3[4],ymm6[5],ymm3[6],ymm6[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[0],ymm5[1],ymm3[2],ymm5[3],ymm3[4],ymm5[5],ymm3[6],ymm5[7] + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm10, %ymm4, %ymm3 + vpsubw %ymm10, %ymm4, %ymm10 + vpaddw %ymm8, %ymm9, %ymm4 + vpsubw %ymm8, %ymm9, %ymm8 + vpaddw %ymm7, %ymm6, %ymm9 + vpsubw %ymm7, %ymm6, %ymm7 + vpaddw %ymm11, %ymm5, %ymm6 + vpsubw %ymm11, %ymm5, %ymm11 + vpsubw %ymm12, %ymm3, %ymm3 + vpaddw %ymm12, %ymm10, %ymm10 + vpsubw %ymm13, %ymm4, %ymm4 + vpaddw %ymm13, %ymm8, %ymm8 + vpsubw %ymm14, %ymm9, %ymm9 + vpaddw %ymm14, %ymm7, %ymm7 + vpsubw %ymm15, %ymm6, %ymm6 + vpaddw %ymm15, %ymm11, %ymm11 + vpslld $0x10, %ymm7, %ymm5 + vpblendw $0xaa, %ymm5, %ymm9, %ymm5 # ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7],ymm9[8],ymm5[9],ymm9[10],ymm5[11],ymm9[12],ymm5[13],ymm9[14],ymm5[15] + vpsrld $0x10, %ymm9, %ymm9 + vpblendw $0xaa, %ymm7, %ymm9, %ymm7 # ymm7 = ymm9[0],ymm7[1],ymm9[2],ymm7[3],ymm9[4],ymm7[5],ymm9[6],ymm7[7],ymm9[8],ymm7[9],ymm9[10],ymm7[11],ymm9[12],ymm7[13],ymm9[14],ymm7[15] + vpslld $0x10, %ymm11, %ymm9 + vpblendw $0xaa, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[0],ymm9[1],ymm6[2],ymm9[3],ymm6[4],ymm9[5],ymm6[6],ymm9[7],ymm6[8],ymm9[9],ymm6[10],ymm9[11],ymm6[12],ymm9[13],ymm6[14],ymm9[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm11, %ymm6, %ymm11 # ymm11 = ymm6[0],ymm11[1],ymm6[2],ymm11[3],ymm6[4],ymm11[5],ymm6[6],ymm11[7],ymm6[8],ymm11[9],ymm6[10],ymm11[11],ymm6[12],ymm11[13],ymm6[14],ymm11[15] + vmovdqa 0x160(%rsi), %ymm15 + vmovdqa 0x180(%rsi), %ymm2 + vpmullw %ymm15, %ymm5, %ymm12 + vpmullw %ymm15, %ymm7, %ymm13 + vpmullw %ymm15, %ymm9, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm5, %ymm5 + vpmulhw %ymm2, %ymm7, %ymm7 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm11, %ymm11 + vpslld $0x10, %ymm10, %ymm6 + vpblendw $0xaa, %ymm6, %ymm3, %ymm6 # ymm6 = ymm3[0],ymm6[1],ymm3[2],ymm6[3],ymm3[4],ymm6[5],ymm3[6],ymm6[7],ymm3[8],ymm6[9],ymm3[10],ymm6[11],ymm3[12],ymm6[13],ymm3[14],ymm6[15] + vpsrld $0x10, %ymm3, %ymm3 + vpblendw $0xaa, %ymm10, %ymm3, %ymm10 # ymm10 = ymm3[0],ymm10[1],ymm3[2],ymm10[3],ymm3[4],ymm10[5],ymm3[6],ymm10[7],ymm3[8],ymm10[9],ymm3[10],ymm10[11],ymm3[12],ymm10[13],ymm3[14],ymm10[15] + vpslld $0x10, %ymm8, %ymm3 + vpblendw $0xaa, %ymm3, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7],ymm4[8],ymm3[9],ymm4[10],ymm3[11],ymm4[12],ymm3[13],ymm4[14],ymm3[15] + vpsrld $0x10, %ymm4, %ymm4 + vpblendw $0xaa, %ymm8, %ymm4, %ymm8 # ymm8 = ymm4[0],ymm8[1],ymm4[2],ymm8[3],ymm4[4],ymm8[5],ymm4[6],ymm8[7],ymm4[8],ymm8[9],ymm4[10],ymm8[11],ymm4[12],ymm8[13],ymm4[14],ymm8[15] + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm5, %ymm6, %ymm4 + vpsubw %ymm5, %ymm6, %ymm5 + vpaddw %ymm7, %ymm10, %ymm6 + vpsubw %ymm7, %ymm10, %ymm7 + vpaddw %ymm9, %ymm3, %ymm10 + vpsubw %ymm9, %ymm3, %ymm9 + vpaddw %ymm11, %ymm8, %ymm3 + vpsubw %ymm11, %ymm8, %ymm11 + vpsubw %ymm12, %ymm4, %ymm4 + vpaddw %ymm12, %ymm5, %ymm5 + vpsubw %ymm13, %ymm6, %ymm6 + vpaddw %ymm13, %ymm7, %ymm7 + vpsubw %ymm14, %ymm10, %ymm10 + vpaddw %ymm14, %ymm9, %ymm9 + vpsubw %ymm15, %ymm3, %ymm3 + vpaddw %ymm15, %ymm11, %ymm11 + vmovdqa 0x1a0(%rsi), %ymm14 + vmovdqa 0x1e0(%rsi), %ymm15 + vmovdqa 0x1c0(%rsi), %ymm8 + vmovdqa 0x200(%rsi), %ymm2 + vpmullw %ymm14, %ymm10, %ymm12 + vpmullw %ymm14, %ymm3, %ymm13 + vpmullw %ymm15, %ymm9, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm8, %ymm10, %ymm10 + vpmulhw %ymm8, %ymm3, %ymm3 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm11, %ymm11 + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm10, %ymm4, %ymm8 + vpsubw %ymm10, %ymm4, %ymm10 + vpaddw %ymm3, %ymm6, %ymm4 + vpsubw %ymm3, %ymm6, %ymm3 + vpaddw %ymm9, %ymm5, %ymm6 + vpsubw %ymm9, %ymm5, %ymm9 + vpaddw %ymm11, %ymm7, %ymm5 + vpsubw %ymm11, %ymm7, %ymm11 + vpsubw %ymm12, %ymm8, %ymm8 + vpaddw %ymm12, %ymm10, %ymm10 + vpsubw %ymm13, %ymm4, %ymm4 + vpaddw %ymm13, %ymm3, %ymm3 + vpsubw %ymm14, %ymm6, %ymm6 + vpaddw %ymm14, %ymm9, %ymm9 + vpsubw %ymm15, %ymm5, %ymm5 + vpaddw %ymm15, %ymm11, %ymm11 + vmovdqa %ymm8, (%rdi) + vmovdqa %ymm4, 0x20(%rdi) + vmovdqa %ymm10, 0x40(%rdi) + vmovdqa %ymm3, 0x60(%rdi) + vmovdqa %ymm6, 0x80(%rdi) + vmovdqa %ymm5, 0xa0(%rdi) + vmovdqa %ymm9, 0xc0(%rdi) + vmovdqa %ymm11, 0xe0(%rdi) + vmovdqa 0x220(%rsi), %ymm15 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm9 + vmovdqa 0x1c0(%rdi), %ymm10 + vmovdqa 0x1e0(%rdi), %ymm11 + vmovdqa 0x240(%rsi), %ymm2 + vpmullw %ymm15, %ymm8, %ymm12 + vpmullw %ymm15, %ymm9, %ymm13 + vpmullw %ymm15, %ymm10, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm2, %ymm11, %ymm11 + vmovdqa 0x100(%rdi), %ymm4 + vmovdqa 0x120(%rdi), %ymm5 + vmovdqa 0x140(%rdi), %ymm6 + vmovdqa 0x160(%rdi), %ymm7 + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm8, %ymm4, %ymm3 + vpsubw %ymm8, %ymm4, %ymm8 + vpaddw %ymm9, %ymm5, %ymm4 + vpsubw %ymm9, %ymm5, %ymm9 + vpaddw %ymm10, %ymm6, %ymm5 + vpsubw %ymm10, %ymm6, %ymm10 + vpaddw %ymm11, %ymm7, %ymm6 + vpsubw %ymm11, %ymm7, %ymm11 + vpsubw %ymm12, %ymm3, %ymm3 + vpaddw %ymm12, %ymm8, %ymm8 + vpsubw %ymm13, %ymm4, %ymm4 + vpaddw %ymm13, %ymm9, %ymm9 + vpsubw %ymm14, %ymm5, %ymm5 + vpaddw %ymm14, %ymm10, %ymm10 + vpsubw %ymm15, %ymm6, %ymm6 + vpaddw %ymm15, %ymm11, %ymm11 + vperm2i128 $0x20, %ymm10, %ymm5, %ymm7 # ymm7 = ymm5[0,1],ymm10[0,1] + vperm2i128 $0x31, %ymm10, %ymm5, %ymm10 # ymm10 = ymm5[2,3],ymm10[2,3] + vperm2i128 $0x20, %ymm11, %ymm6, %ymm5 # ymm5 = ymm6[0,1],ymm11[0,1] + vperm2i128 $0x31, %ymm11, %ymm6, %ymm11 # ymm11 = ymm6[2,3],ymm11[2,3] + vmovdqa 0x260(%rsi), %ymm15 + vmovdqa 0x280(%rsi), %ymm2 + vpmullw %ymm15, %ymm7, %ymm12 + vpmullw %ymm15, %ymm10, %ymm13 + vpmullw %ymm15, %ymm5, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm7, %ymm7 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm2, %ymm5, %ymm5 + vpmulhw %ymm2, %ymm11, %ymm11 + vperm2i128 $0x20, %ymm8, %ymm3, %ymm6 # ymm6 = ymm3[0,1],ymm8[0,1] + vperm2i128 $0x31, %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[2,3],ymm8[2,3] + vperm2i128 $0x20, %ymm9, %ymm4, %ymm3 # ymm3 = ymm4[0,1],ymm9[0,1] + vperm2i128 $0x31, %ymm9, %ymm4, %ymm9 # ymm9 = ymm4[2,3],ymm9[2,3] + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm7, %ymm6, %ymm4 + vpsubw %ymm7, %ymm6, %ymm7 + vpaddw %ymm10, %ymm8, %ymm6 + vpsubw %ymm10, %ymm8, %ymm10 + vpaddw %ymm5, %ymm3, %ymm8 + vpsubw %ymm5, %ymm3, %ymm5 + vpaddw %ymm11, %ymm9, %ymm3 + vpsubw %ymm11, %ymm9, %ymm11 + vpsubw %ymm12, %ymm4, %ymm4 + vpaddw %ymm12, %ymm7, %ymm7 + vpsubw %ymm13, %ymm6, %ymm6 + vpaddw %ymm13, %ymm10, %ymm10 + vpsubw %ymm14, %ymm8, %ymm8 + vpaddw %ymm14, %ymm5, %ymm5 + vpsubw %ymm15, %ymm3, %ymm3 + vpaddw %ymm15, %ymm11, %ymm11 + vpunpcklqdq %ymm5, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm5[0],ymm8[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm8, %ymm5 # ymm5 = ymm8[1],ymm5[1],ymm8[3],ymm5[3] + vpunpcklqdq %ymm11, %ymm3, %ymm8 # ymm8 = ymm3[0],ymm11[0],ymm3[2],ymm11[2] + vpunpckhqdq %ymm11, %ymm3, %ymm11 # ymm11 = ymm3[1],ymm11[1],ymm3[3],ymm11[3] + vmovdqa 0x2a0(%rsi), %ymm15 + vmovdqa 0x2c0(%rsi), %ymm2 + vpmullw %ymm15, %ymm9, %ymm12 + vpmullw %ymm15, %ymm5, %ymm13 + vpmullw %ymm15, %ymm8, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm5, %ymm5 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm2, %ymm11, %ymm11 + vpunpcklqdq %ymm7, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm7[0],ymm4[2],ymm7[2] + vpunpckhqdq %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[1],ymm7[1],ymm4[3],ymm7[3] + vpunpcklqdq %ymm10, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm10[0],ymm6[2],ymm10[2] + vpunpckhqdq %ymm10, %ymm6, %ymm10 # ymm10 = ymm6[1],ymm10[1],ymm6[3],ymm10[3] + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm9, %ymm3, %ymm6 + vpsubw %ymm9, %ymm3, %ymm9 + vpaddw %ymm5, %ymm7, %ymm3 + vpsubw %ymm5, %ymm7, %ymm5 + vpaddw %ymm8, %ymm4, %ymm7 + vpsubw %ymm8, %ymm4, %ymm8 + vpaddw %ymm11, %ymm10, %ymm4 + vpsubw %ymm11, %ymm10, %ymm11 + vpsubw %ymm12, %ymm6, %ymm6 + vpaddw %ymm12, %ymm9, %ymm9 + vpsubw %ymm13, %ymm3, %ymm3 + vpaddw %ymm13, %ymm5, %ymm5 + vpsubw %ymm14, %ymm7, %ymm7 + vpaddw %ymm14, %ymm8, %ymm8 + vpsubw %ymm15, %ymm4, %ymm4 + vpaddw %ymm15, %ymm11, %ymm11 + vmovsldup %ymm8, %ymm10 # ymm10 = ymm8[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm10, %ymm7, %ymm10 # ymm10 = ymm7[0],ymm10[1],ymm7[2],ymm10[3],ymm7[4],ymm10[5],ymm7[6],ymm10[7] + vpsrlq $0x20, %ymm7, %ymm7 + vpblendd $0xaa, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[0],ymm8[1],ymm7[2],ymm8[3],ymm7[4],ymm8[5],ymm7[6],ymm8[7] + vmovsldup %ymm11, %ymm7 # ymm7 = ymm11[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[0],ymm7[1],ymm4[2],ymm7[3],ymm4[4],ymm7[5],ymm4[6],ymm7[7] + vpsrlq $0x20, %ymm4, %ymm4 + vpblendd $0xaa, %ymm11, %ymm4, %ymm11 # ymm11 = ymm4[0],ymm11[1],ymm4[2],ymm11[3],ymm4[4],ymm11[5],ymm4[6],ymm11[7] + vmovdqa 0x2e0(%rsi), %ymm15 + vmovdqa 0x300(%rsi), %ymm2 + vpmullw %ymm15, %ymm10, %ymm12 + vpmullw %ymm15, %ymm8, %ymm13 + vpmullw %ymm15, %ymm7, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm2, %ymm7, %ymm7 + vpmulhw %ymm2, %ymm11, %ymm11 + vmovsldup %ymm9, %ymm4 # ymm4 = ymm9[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7] + vpsrlq $0x20, %ymm6, %ymm6 + vpblendd $0xaa, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[0],ymm9[1],ymm6[2],ymm9[3],ymm6[4],ymm9[5],ymm6[6],ymm9[7] + vmovsldup %ymm5, %ymm6 # ymm6 = ymm5[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm6, %ymm3, %ymm6 # ymm6 = ymm3[0],ymm6[1],ymm3[2],ymm6[3],ymm3[4],ymm6[5],ymm3[6],ymm6[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[0],ymm5[1],ymm3[2],ymm5[3],ymm3[4],ymm5[5],ymm3[6],ymm5[7] + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm10, %ymm4, %ymm3 + vpsubw %ymm10, %ymm4, %ymm10 + vpaddw %ymm8, %ymm9, %ymm4 + vpsubw %ymm8, %ymm9, %ymm8 + vpaddw %ymm7, %ymm6, %ymm9 + vpsubw %ymm7, %ymm6, %ymm7 + vpaddw %ymm11, %ymm5, %ymm6 + vpsubw %ymm11, %ymm5, %ymm11 + vpsubw %ymm12, %ymm3, %ymm3 + vpaddw %ymm12, %ymm10, %ymm10 + vpsubw %ymm13, %ymm4, %ymm4 + vpaddw %ymm13, %ymm8, %ymm8 + vpsubw %ymm14, %ymm9, %ymm9 + vpaddw %ymm14, %ymm7, %ymm7 + vpsubw %ymm15, %ymm6, %ymm6 + vpaddw %ymm15, %ymm11, %ymm11 + vpslld $0x10, %ymm7, %ymm5 + vpblendw $0xaa, %ymm5, %ymm9, %ymm5 # ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7],ymm9[8],ymm5[9],ymm9[10],ymm5[11],ymm9[12],ymm5[13],ymm9[14],ymm5[15] + vpsrld $0x10, %ymm9, %ymm9 + vpblendw $0xaa, %ymm7, %ymm9, %ymm7 # ymm7 = ymm9[0],ymm7[1],ymm9[2],ymm7[3],ymm9[4],ymm7[5],ymm9[6],ymm7[7],ymm9[8],ymm7[9],ymm9[10],ymm7[11],ymm9[12],ymm7[13],ymm9[14],ymm7[15] + vpslld $0x10, %ymm11, %ymm9 + vpblendw $0xaa, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[0],ymm9[1],ymm6[2],ymm9[3],ymm6[4],ymm9[5],ymm6[6],ymm9[7],ymm6[8],ymm9[9],ymm6[10],ymm9[11],ymm6[12],ymm9[13],ymm6[14],ymm9[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm11, %ymm6, %ymm11 # ymm11 = ymm6[0],ymm11[1],ymm6[2],ymm11[3],ymm6[4],ymm11[5],ymm6[6],ymm11[7],ymm6[8],ymm11[9],ymm6[10],ymm11[11],ymm6[12],ymm11[13],ymm6[14],ymm11[15] + vmovdqa 0x320(%rsi), %ymm15 + vmovdqa 0x340(%rsi), %ymm2 + vpmullw %ymm15, %ymm5, %ymm12 + vpmullw %ymm15, %ymm7, %ymm13 + vpmullw %ymm15, %ymm9, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm5, %ymm5 + vpmulhw %ymm2, %ymm7, %ymm7 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm11, %ymm11 + vpslld $0x10, %ymm10, %ymm6 + vpblendw $0xaa, %ymm6, %ymm3, %ymm6 # ymm6 = ymm3[0],ymm6[1],ymm3[2],ymm6[3],ymm3[4],ymm6[5],ymm3[6],ymm6[7],ymm3[8],ymm6[9],ymm3[10],ymm6[11],ymm3[12],ymm6[13],ymm3[14],ymm6[15] + vpsrld $0x10, %ymm3, %ymm3 + vpblendw $0xaa, %ymm10, %ymm3, %ymm10 # ymm10 = ymm3[0],ymm10[1],ymm3[2],ymm10[3],ymm3[4],ymm10[5],ymm3[6],ymm10[7],ymm3[8],ymm10[9],ymm3[10],ymm10[11],ymm3[12],ymm10[13],ymm3[14],ymm10[15] + vpslld $0x10, %ymm8, %ymm3 + vpblendw $0xaa, %ymm3, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7],ymm4[8],ymm3[9],ymm4[10],ymm3[11],ymm4[12],ymm3[13],ymm4[14],ymm3[15] + vpsrld $0x10, %ymm4, %ymm4 + vpblendw $0xaa, %ymm8, %ymm4, %ymm8 # ymm8 = ymm4[0],ymm8[1],ymm4[2],ymm8[3],ymm4[4],ymm8[5],ymm4[6],ymm8[7],ymm4[8],ymm8[9],ymm4[10],ymm8[11],ymm4[12],ymm8[13],ymm4[14],ymm8[15] + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm5, %ymm6, %ymm4 + vpsubw %ymm5, %ymm6, %ymm5 + vpaddw %ymm7, %ymm10, %ymm6 + vpsubw %ymm7, %ymm10, %ymm7 + vpaddw %ymm9, %ymm3, %ymm10 + vpsubw %ymm9, %ymm3, %ymm9 + vpaddw %ymm11, %ymm8, %ymm3 + vpsubw %ymm11, %ymm8, %ymm11 + vpsubw %ymm12, %ymm4, %ymm4 + vpaddw %ymm12, %ymm5, %ymm5 + vpsubw %ymm13, %ymm6, %ymm6 + vpaddw %ymm13, %ymm7, %ymm7 + vpsubw %ymm14, %ymm10, %ymm10 + vpaddw %ymm14, %ymm9, %ymm9 + vpsubw %ymm15, %ymm3, %ymm3 + vpaddw %ymm15, %ymm11, %ymm11 + vmovdqa 0x360(%rsi), %ymm14 + vmovdqa 0x3a0(%rsi), %ymm15 + vmovdqa 0x380(%rsi), %ymm8 + vmovdqa 0x3c0(%rsi), %ymm2 + vpmullw %ymm14, %ymm10, %ymm12 + vpmullw %ymm14, %ymm3, %ymm13 + vpmullw %ymm15, %ymm9, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm8, %ymm10, %ymm10 + vpmulhw %ymm8, %ymm3, %ymm3 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm11, %ymm11 + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm10, %ymm4, %ymm8 + vpsubw %ymm10, %ymm4, %ymm10 + vpaddw %ymm3, %ymm6, %ymm4 + vpsubw %ymm3, %ymm6, %ymm3 + vpaddw %ymm9, %ymm5, %ymm6 + vpsubw %ymm9, %ymm5, %ymm9 + vpaddw %ymm11, %ymm7, %ymm5 + vpsubw %ymm11, %ymm7, %ymm11 + vpsubw %ymm12, %ymm8, %ymm8 + vpaddw %ymm12, %ymm10, %ymm10 + vpsubw %ymm13, %ymm4, %ymm4 + vpaddw %ymm13, %ymm3, %ymm3 + vpsubw %ymm14, %ymm6, %ymm6 + vpaddw %ymm14, %ymm9, %ymm9 + vpsubw %ymm15, %ymm5, %ymm5 + vpaddw %ymm15, %ymm11, %ymm11 + vmovdqa %ymm8, 0x100(%rdi) + vmovdqa %ymm4, 0x120(%rdi) + vmovdqa %ymm10, 0x140(%rdi) + vmovdqa %ymm3, 0x160(%rdi) + vmovdqa %ymm6, 0x180(%rdi) + vmovdqa %ymm5, 0x1a0(%rdi) + vmovdqa %ymm9, 0x1c0(%rdi) + vmovdqa %ymm11, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(ntt_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_nttfrombytes_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_nttfrombytes_avx2_asm.S new file mode 100644 index 0000000000..78b3b66d9a --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_nttfrombytes_avx2_asm.S @@ -0,0 +1,217 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API)) +/*yaml + Name: nttfrombytes_avx2_asm + Description: x86_64 AVX2 polynomial deserialization in NTT domain + Signature: void mlk_nttfrombytes_avx2_asm(int16_t *r, const uint8_t *a) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t *r + description: Output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 384 + permissions: read-only + c_parameter: const uint8_t *a + description: Input byte array (MLKEM_POLYBYTES = 384) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_nttfrombytes_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(nttfrombytes_avx2_asm) +MLK_ASM_FN_SYMBOL(nttfrombytes_avx2_asm) + + .cfi_startproc + movl $0xfff0fff, %eax # imm = 0xFFF0FFF + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + vmovdqu (%rsi), %ymm4 + vmovdqu 0x20(%rsi), %ymm5 + vmovdqu 0x40(%rsi), %ymm6 + vmovdqu 0x60(%rsi), %ymm7 + vmovdqu 0x80(%rsi), %ymm8 + vmovdqu 0xa0(%rsi), %ymm9 + vperm2i128 $0x20, %ymm7, %ymm4, %ymm3 # ymm3 = ymm4[0,1],ymm7[0,1] + vperm2i128 $0x31, %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[2,3],ymm7[2,3] + vperm2i128 $0x20, %ymm8, %ymm5, %ymm4 # ymm4 = ymm5[0,1],ymm8[0,1] + vperm2i128 $0x31, %ymm8, %ymm5, %ymm8 # ymm8 = ymm5[2,3],ymm8[2,3] + vperm2i128 $0x20, %ymm9, %ymm6, %ymm5 # ymm5 = ymm6[0,1],ymm9[0,1] + vperm2i128 $0x31, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[2,3],ymm9[2,3] + vpunpcklqdq %ymm8, %ymm3, %ymm6 # ymm6 = ymm3[0],ymm8[0],ymm3[2],ymm8[2] + vpunpckhqdq %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[1],ymm8[1],ymm3[3],ymm8[3] + vpunpcklqdq %ymm5, %ymm7, %ymm3 # ymm3 = ymm7[0],ymm5[0],ymm7[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm7, %ymm5 # ymm5 = ymm7[1],ymm5[1],ymm7[3],ymm5[3] + vpunpcklqdq %ymm9, %ymm4, %ymm7 # ymm7 = ymm4[0],ymm9[0],ymm4[2],ymm9[2] + vpunpckhqdq %ymm9, %ymm4, %ymm9 # ymm9 = ymm4[1],ymm9[1],ymm4[3],ymm9[3] + vmovsldup %ymm5, %ymm4 # ymm4 = ymm5[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7] + vpsrlq $0x20, %ymm6, %ymm6 + vpblendd $0xaa, %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7] + vmovsldup %ymm7, %ymm6 # ymm6 = ymm7[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm6, %ymm8, %ymm6 # ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7] + vpsrlq $0x20, %ymm8, %ymm8 + vpblendd $0xaa, %ymm7, %ymm8, %ymm7 # ymm7 = ymm8[0],ymm7[1],ymm8[2],ymm7[3],ymm8[4],ymm7[5],ymm8[6],ymm7[7] + vmovsldup %ymm9, %ymm8 # ymm8 = ymm9[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[0],ymm8[1],ymm3[2],ymm8[3],ymm3[4],ymm8[5],ymm3[6],ymm8[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm9, %ymm3, %ymm9 # ymm9 = ymm3[0],ymm9[1],ymm3[2],ymm9[3],ymm3[4],ymm9[5],ymm3[6],ymm9[7] + vpslld $0x10, %ymm7, %ymm10 + vpblendw $0xaa, %ymm10, %ymm4, %ymm10 # ymm10 = ymm4[0],ymm10[1],ymm4[2],ymm10[3],ymm4[4],ymm10[5],ymm4[6],ymm10[7],ymm4[8],ymm10[9],ymm4[10],ymm10[11],ymm4[12],ymm10[13],ymm4[14],ymm10[15] + vpsrld $0x10, %ymm4, %ymm4 + vpblendw $0xaa, %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[0],ymm7[1],ymm4[2],ymm7[3],ymm4[4],ymm7[5],ymm4[6],ymm7[7],ymm4[8],ymm7[9],ymm4[10],ymm7[11],ymm4[12],ymm7[13],ymm4[14],ymm7[15] + vpslld $0x10, %ymm8, %ymm4 + vpblendw $0xaa, %ymm4, %ymm5, %ymm4 # ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4],ymm4[5],ymm5[6],ymm4[7],ymm5[8],ymm4[9],ymm5[10],ymm4[11],ymm5[12],ymm4[13],ymm5[14],ymm4[15] + vpsrld $0x10, %ymm5, %ymm5 + vpblendw $0xaa, %ymm8, %ymm5, %ymm8 # ymm8 = ymm5[0],ymm8[1],ymm5[2],ymm8[3],ymm5[4],ymm8[5],ymm5[6],ymm8[7],ymm5[8],ymm8[9],ymm5[10],ymm8[11],ymm5[12],ymm8[13],ymm5[14],ymm8[15] + vpslld $0x10, %ymm9, %ymm5 + vpblendw $0xaa, %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7],ymm6[8],ymm5[9],ymm6[10],ymm5[11],ymm6[12],ymm5[13],ymm6[14],ymm5[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[0],ymm9[1],ymm6[2],ymm9[3],ymm6[4],ymm9[5],ymm6[6],ymm9[7],ymm6[8],ymm9[9],ymm6[10],ymm9[11],ymm6[12],ymm9[13],ymm6[14],ymm9[15] + vpsrlw $0xc, %ymm10, %ymm11 + vpsllw $0x4, %ymm7, %ymm12 + vpor %ymm11, %ymm12, %ymm11 + vpand %ymm0, %ymm10, %ymm10 + vpand %ymm0, %ymm11, %ymm11 + vpsrlw $0x8, %ymm7, %ymm12 + vpsllw $0x8, %ymm4, %ymm13 + vpor %ymm12, %ymm13, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpsrlw $0x4, %ymm4, %ymm13 + vpand %ymm0, %ymm13, %ymm13 + vpsrlw $0xc, %ymm8, %ymm14 + vpsllw $0x4, %ymm5, %ymm15 + vpor %ymm14, %ymm15, %ymm14 + vpand %ymm0, %ymm8, %ymm8 + vpand %ymm0, %ymm14, %ymm14 + vpsrlw $0x8, %ymm5, %ymm15 + vpsllw $0x8, %ymm9, %ymm1 + vpor %ymm15, %ymm1, %ymm15 + vpand %ymm0, %ymm15, %ymm15 + vpsrlw $0x4, %ymm9, %ymm1 + vpand %ymm0, %ymm1, %ymm1 + vmovdqa %ymm10, (%rdi) + vmovdqa %ymm11, 0x20(%rdi) + vmovdqa %ymm12, 0x40(%rdi) + vmovdqa %ymm13, 0x60(%rdi) + vmovdqa %ymm8, 0x80(%rdi) + vmovdqa %ymm14, 0xa0(%rdi) + vmovdqa %ymm15, 0xc0(%rdi) + vmovdqa %ymm1, 0xe0(%rdi) + vmovdqu 0xc0(%rsi), %ymm4 + vmovdqu 0xe0(%rsi), %ymm5 + vmovdqu 0x100(%rsi), %ymm6 + vmovdqu 0x120(%rsi), %ymm7 + vmovdqu 0x140(%rsi), %ymm8 + vmovdqu 0x160(%rsi), %ymm9 + vperm2i128 $0x20, %ymm7, %ymm4, %ymm3 # ymm3 = ymm4[0,1],ymm7[0,1] + vperm2i128 $0x31, %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[2,3],ymm7[2,3] + vperm2i128 $0x20, %ymm8, %ymm5, %ymm4 # ymm4 = ymm5[0,1],ymm8[0,1] + vperm2i128 $0x31, %ymm8, %ymm5, %ymm8 # ymm8 = ymm5[2,3],ymm8[2,3] + vperm2i128 $0x20, %ymm9, %ymm6, %ymm5 # ymm5 = ymm6[0,1],ymm9[0,1] + vperm2i128 $0x31, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[2,3],ymm9[2,3] + vpunpcklqdq %ymm8, %ymm3, %ymm6 # ymm6 = ymm3[0],ymm8[0],ymm3[2],ymm8[2] + vpunpckhqdq %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[1],ymm8[1],ymm3[3],ymm8[3] + vpunpcklqdq %ymm5, %ymm7, %ymm3 # ymm3 = ymm7[0],ymm5[0],ymm7[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm7, %ymm5 # ymm5 = ymm7[1],ymm5[1],ymm7[3],ymm5[3] + vpunpcklqdq %ymm9, %ymm4, %ymm7 # ymm7 = ymm4[0],ymm9[0],ymm4[2],ymm9[2] + vpunpckhqdq %ymm9, %ymm4, %ymm9 # ymm9 = ymm4[1],ymm9[1],ymm4[3],ymm9[3] + vmovsldup %ymm5, %ymm4 # ymm4 = ymm5[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7] + vpsrlq $0x20, %ymm6, %ymm6 + vpblendd $0xaa, %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7] + vmovsldup %ymm7, %ymm6 # ymm6 = ymm7[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm6, %ymm8, %ymm6 # ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7] + vpsrlq $0x20, %ymm8, %ymm8 + vpblendd $0xaa, %ymm7, %ymm8, %ymm7 # ymm7 = ymm8[0],ymm7[1],ymm8[2],ymm7[3],ymm8[4],ymm7[5],ymm8[6],ymm7[7] + vmovsldup %ymm9, %ymm8 # ymm8 = ymm9[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[0],ymm8[1],ymm3[2],ymm8[3],ymm3[4],ymm8[5],ymm3[6],ymm8[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm9, %ymm3, %ymm9 # ymm9 = ymm3[0],ymm9[1],ymm3[2],ymm9[3],ymm3[4],ymm9[5],ymm3[6],ymm9[7] + vpslld $0x10, %ymm7, %ymm10 + vpblendw $0xaa, %ymm10, %ymm4, %ymm10 # ymm10 = ymm4[0],ymm10[1],ymm4[2],ymm10[3],ymm4[4],ymm10[5],ymm4[6],ymm10[7],ymm4[8],ymm10[9],ymm4[10],ymm10[11],ymm4[12],ymm10[13],ymm4[14],ymm10[15] + vpsrld $0x10, %ymm4, %ymm4 + vpblendw $0xaa, %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[0],ymm7[1],ymm4[2],ymm7[3],ymm4[4],ymm7[5],ymm4[6],ymm7[7],ymm4[8],ymm7[9],ymm4[10],ymm7[11],ymm4[12],ymm7[13],ymm4[14],ymm7[15] + vpslld $0x10, %ymm8, %ymm4 + vpblendw $0xaa, %ymm4, %ymm5, %ymm4 # ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4],ymm4[5],ymm5[6],ymm4[7],ymm5[8],ymm4[9],ymm5[10],ymm4[11],ymm5[12],ymm4[13],ymm5[14],ymm4[15] + vpsrld $0x10, %ymm5, %ymm5 + vpblendw $0xaa, %ymm8, %ymm5, %ymm8 # ymm8 = ymm5[0],ymm8[1],ymm5[2],ymm8[3],ymm5[4],ymm8[5],ymm5[6],ymm8[7],ymm5[8],ymm8[9],ymm5[10],ymm8[11],ymm5[12],ymm8[13],ymm5[14],ymm8[15] + vpslld $0x10, %ymm9, %ymm5 + vpblendw $0xaa, %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7],ymm6[8],ymm5[9],ymm6[10],ymm5[11],ymm6[12],ymm5[13],ymm6[14],ymm5[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[0],ymm9[1],ymm6[2],ymm9[3],ymm6[4],ymm9[5],ymm6[6],ymm9[7],ymm6[8],ymm9[9],ymm6[10],ymm9[11],ymm6[12],ymm9[13],ymm6[14],ymm9[15] + vpsrlw $0xc, %ymm10, %ymm11 + vpsllw $0x4, %ymm7, %ymm12 + vpor %ymm11, %ymm12, %ymm11 + vpand %ymm0, %ymm10, %ymm10 + vpand %ymm0, %ymm11, %ymm11 + vpsrlw $0x8, %ymm7, %ymm12 + vpsllw $0x8, %ymm4, %ymm13 + vpor %ymm12, %ymm13, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpsrlw $0x4, %ymm4, %ymm13 + vpand %ymm0, %ymm13, %ymm13 + vpsrlw $0xc, %ymm8, %ymm14 + vpsllw $0x4, %ymm5, %ymm15 + vpor %ymm14, %ymm15, %ymm14 + vpand %ymm0, %ymm8, %ymm8 + vpand %ymm0, %ymm14, %ymm14 + vpsrlw $0x8, %ymm5, %ymm15 + vpsllw $0x8, %ymm9, %ymm1 + vpor %ymm15, %ymm1, %ymm15 + vpand %ymm0, %ymm15, %ymm15 + vpsrlw $0x4, %ymm9, %ymm1 + vpand %ymm0, %ymm1, %ymm1 + vmovdqa %ymm10, 0x100(%rdi) + vmovdqa %ymm11, 0x120(%rdi) + vmovdqa %ymm12, 0x140(%rdi) + vmovdqa %ymm13, 0x160(%rdi) + vmovdqa %ymm8, 0x180(%rdi) + vmovdqa %ymm14, 0x1a0(%rdi) + vmovdqa %ymm15, 0x1c0(%rdi) + vmovdqa %ymm1, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(nttfrombytes_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_ntttobytes_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_ntttobytes_avx2_asm.S new file mode 100644 index 0000000000..e2776b6ef2 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_ntttobytes_avx2_asm.S @@ -0,0 +1,205 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (!defined(MLK_CONFIG_NO_KEYPAIR_API) || \ + !defined(MLK_CONFIG_NO_ENCAPS_API)) +/*yaml + Name: ntttobytes_avx2_asm + Description: x86_64 AVX2 polynomial serialization in NTT domain + Signature: void mlk_ntttobytes_avx2_asm(uint8_t *r, const int16_t *a) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 384 + permissions: write-only + c_parameter: uint8_t *r + description: Output byte array (MLKEM_POLYBYTES = 384) + rsi: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t *a + description: Input polynomial (256 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_ntttobytes_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(ntttobytes_avx2_asm) +MLK_ASM_FN_SYMBOL(ntttobytes_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + vmovdqa (%rsi), %ymm5 + vmovdqa 0x20(%rsi), %ymm6 + vmovdqa 0x40(%rsi), %ymm7 + vmovdqa 0x60(%rsi), %ymm8 + vmovdqa 0x80(%rsi), %ymm9 + vmovdqa 0xa0(%rsi), %ymm10 + vmovdqa 0xc0(%rsi), %ymm11 + vmovdqa 0xe0(%rsi), %ymm12 + vpsllw $0xc, %ymm6, %ymm4 + vpor %ymm4, %ymm5, %ymm4 + vpsrlw $0x4, %ymm6, %ymm5 + vpsllw $0x8, %ymm7, %ymm6 + vpor %ymm5, %ymm6, %ymm5 + vpsrlw $0x8, %ymm7, %ymm6 + vpsllw $0x4, %ymm8, %ymm7 + vpor %ymm6, %ymm7, %ymm6 + vpsllw $0xc, %ymm10, %ymm7 + vpor %ymm7, %ymm9, %ymm7 + vpsrlw $0x4, %ymm10, %ymm8 + vpsllw $0x8, %ymm11, %ymm9 + vpor %ymm8, %ymm9, %ymm8 + vpsrlw $0x8, %ymm11, %ymm9 + vpsllw $0x4, %ymm12, %ymm10 + vpor %ymm9, %ymm10, %ymm9 + vpslld $0x10, %ymm5, %ymm3 + vpblendw $0xaa, %ymm3, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7],ymm4[8],ymm3[9],ymm4[10],ymm3[11],ymm4[12],ymm3[13],ymm4[14],ymm3[15] + vpsrld $0x10, %ymm4, %ymm4 + vpblendw $0xaa, %ymm5, %ymm4, %ymm5 # ymm5 = ymm4[0],ymm5[1],ymm4[2],ymm5[3],ymm4[4],ymm5[5],ymm4[6],ymm5[7],ymm4[8],ymm5[9],ymm4[10],ymm5[11],ymm4[12],ymm5[13],ymm4[14],ymm5[15] + vpslld $0x10, %ymm7, %ymm4 + vpblendw $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7],ymm6[8],ymm4[9],ymm6[10],ymm4[11],ymm6[12],ymm4[13],ymm6[14],ymm4[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm7, %ymm6, %ymm7 # ymm7 = ymm6[0],ymm7[1],ymm6[2],ymm7[3],ymm6[4],ymm7[5],ymm6[6],ymm7[7],ymm6[8],ymm7[9],ymm6[10],ymm7[11],ymm6[12],ymm7[13],ymm6[14],ymm7[15] + vpslld $0x10, %ymm9, %ymm6 + vpblendw $0xaa, %ymm6, %ymm8, %ymm6 # ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7],ymm8[8],ymm6[9],ymm8[10],ymm6[11],ymm8[12],ymm6[13],ymm8[14],ymm6[15] + vpsrld $0x10, %ymm8, %ymm8 + vpblendw $0xaa, %ymm9, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7],ymm8[8],ymm9[9],ymm8[10],ymm9[11],ymm8[12],ymm9[13],ymm8[14],ymm9[15] + vmovsldup %ymm4, %ymm8 # ymm8 = ymm4[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[0],ymm8[1],ymm3[2],ymm8[3],ymm3[4],ymm8[5],ymm3[6],ymm8[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm4, %ymm3, %ymm4 # ymm4 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7] + vmovsldup %ymm5, %ymm3 # ymm3 = ymm5[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm3, %ymm6, %ymm3 # ymm3 = ymm6[0],ymm3[1],ymm6[2],ymm3[3],ymm6[4],ymm3[5],ymm6[6],ymm3[7] + vpsrlq $0x20, %ymm6, %ymm6 + vpblendd $0xaa, %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7] + vmovsldup %ymm9, %ymm6 # ymm6 = ymm9[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm6, %ymm7, %ymm6 # ymm6 = ymm7[0],ymm6[1],ymm7[2],ymm6[3],ymm7[4],ymm6[5],ymm7[6],ymm6[7] + vpsrlq $0x20, %ymm7, %ymm7 + vpblendd $0xaa, %ymm9, %ymm7, %ymm9 # ymm9 = ymm7[0],ymm9[1],ymm7[2],ymm9[3],ymm7[4],ymm9[5],ymm7[6],ymm9[7] + vpunpcklqdq %ymm3, %ymm8, %ymm7 # ymm7 = ymm8[0],ymm3[0],ymm8[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm8, %ymm3 # ymm3 = ymm8[1],ymm3[1],ymm8[3],ymm3[3] + vpunpcklqdq %ymm4, %ymm6, %ymm8 # ymm8 = ymm6[0],ymm4[0],ymm6[2],ymm4[2] + vpunpckhqdq %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[1],ymm4[1],ymm6[3],ymm4[3] + vpunpcklqdq %ymm9, %ymm5, %ymm6 # ymm6 = ymm5[0],ymm9[0],ymm5[2],ymm9[2] + vpunpckhqdq %ymm9, %ymm5, %ymm9 # ymm9 = ymm5[1],ymm9[1],ymm5[3],ymm9[3] + vperm2i128 $0x20, %ymm8, %ymm7, %ymm5 # ymm5 = ymm7[0,1],ymm8[0,1] + vperm2i128 $0x31, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[2,3],ymm8[2,3] + vperm2i128 $0x20, %ymm3, %ymm6, %ymm7 # ymm7 = ymm6[0,1],ymm3[0,1] + vperm2i128 $0x31, %ymm3, %ymm6, %ymm3 # ymm3 = ymm6[2,3],ymm3[2,3] + vperm2i128 $0x20, %ymm9, %ymm4, %ymm6 # ymm6 = ymm4[0,1],ymm9[0,1] + vperm2i128 $0x31, %ymm9, %ymm4, %ymm9 # ymm9 = ymm4[2,3],ymm9[2,3] + vmovdqu %ymm5, (%rdi) + vmovdqu %ymm7, 0x20(%rdi) + vmovdqu %ymm6, 0x40(%rdi) + vmovdqu %ymm8, 0x60(%rdi) + vmovdqu %ymm3, 0x80(%rdi) + vmovdqu %ymm9, 0xa0(%rdi) + vmovdqa 0x100(%rsi), %ymm5 + vmovdqa 0x120(%rsi), %ymm6 + vmovdqa 0x140(%rsi), %ymm7 + vmovdqa 0x160(%rsi), %ymm8 + vmovdqa 0x180(%rsi), %ymm9 + vmovdqa 0x1a0(%rsi), %ymm10 + vmovdqa 0x1c0(%rsi), %ymm11 + vmovdqa 0x1e0(%rsi), %ymm12 + vpsllw $0xc, %ymm6, %ymm4 + vpor %ymm4, %ymm5, %ymm4 + vpsrlw $0x4, %ymm6, %ymm5 + vpsllw $0x8, %ymm7, %ymm6 + vpor %ymm5, %ymm6, %ymm5 + vpsrlw $0x8, %ymm7, %ymm6 + vpsllw $0x4, %ymm8, %ymm7 + vpor %ymm6, %ymm7, %ymm6 + vpsllw $0xc, %ymm10, %ymm7 + vpor %ymm7, %ymm9, %ymm7 + vpsrlw $0x4, %ymm10, %ymm8 + vpsllw $0x8, %ymm11, %ymm9 + vpor %ymm8, %ymm9, %ymm8 + vpsrlw $0x8, %ymm11, %ymm9 + vpsllw $0x4, %ymm12, %ymm10 + vpor %ymm9, %ymm10, %ymm9 + vpslld $0x10, %ymm5, %ymm3 + vpblendw $0xaa, %ymm3, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7],ymm4[8],ymm3[9],ymm4[10],ymm3[11],ymm4[12],ymm3[13],ymm4[14],ymm3[15] + vpsrld $0x10, %ymm4, %ymm4 + vpblendw $0xaa, %ymm5, %ymm4, %ymm5 # ymm5 = ymm4[0],ymm5[1],ymm4[2],ymm5[3],ymm4[4],ymm5[5],ymm4[6],ymm5[7],ymm4[8],ymm5[9],ymm4[10],ymm5[11],ymm4[12],ymm5[13],ymm4[14],ymm5[15] + vpslld $0x10, %ymm7, %ymm4 + vpblendw $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7],ymm6[8],ymm4[9],ymm6[10],ymm4[11],ymm6[12],ymm4[13],ymm6[14],ymm4[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm7, %ymm6, %ymm7 # ymm7 = ymm6[0],ymm7[1],ymm6[2],ymm7[3],ymm6[4],ymm7[5],ymm6[6],ymm7[7],ymm6[8],ymm7[9],ymm6[10],ymm7[11],ymm6[12],ymm7[13],ymm6[14],ymm7[15] + vpslld $0x10, %ymm9, %ymm6 + vpblendw $0xaa, %ymm6, %ymm8, %ymm6 # ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7],ymm8[8],ymm6[9],ymm8[10],ymm6[11],ymm8[12],ymm6[13],ymm8[14],ymm6[15] + vpsrld $0x10, %ymm8, %ymm8 + vpblendw $0xaa, %ymm9, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7],ymm8[8],ymm9[9],ymm8[10],ymm9[11],ymm8[12],ymm9[13],ymm8[14],ymm9[15] + vmovsldup %ymm4, %ymm8 # ymm8 = ymm4[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[0],ymm8[1],ymm3[2],ymm8[3],ymm3[4],ymm8[5],ymm3[6],ymm8[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm4, %ymm3, %ymm4 # ymm4 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7] + vmovsldup %ymm5, %ymm3 # ymm3 = ymm5[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm3, %ymm6, %ymm3 # ymm3 = ymm6[0],ymm3[1],ymm6[2],ymm3[3],ymm6[4],ymm3[5],ymm6[6],ymm3[7] + vpsrlq $0x20, %ymm6, %ymm6 + vpblendd $0xaa, %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7] + vmovsldup %ymm9, %ymm6 # ymm6 = ymm9[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm6, %ymm7, %ymm6 # ymm6 = ymm7[0],ymm6[1],ymm7[2],ymm6[3],ymm7[4],ymm6[5],ymm7[6],ymm6[7] + vpsrlq $0x20, %ymm7, %ymm7 + vpblendd $0xaa, %ymm9, %ymm7, %ymm9 # ymm9 = ymm7[0],ymm9[1],ymm7[2],ymm9[3],ymm7[4],ymm9[5],ymm7[6],ymm9[7] + vpunpcklqdq %ymm3, %ymm8, %ymm7 # ymm7 = ymm8[0],ymm3[0],ymm8[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm8, %ymm3 # ymm3 = ymm8[1],ymm3[1],ymm8[3],ymm3[3] + vpunpcklqdq %ymm4, %ymm6, %ymm8 # ymm8 = ymm6[0],ymm4[0],ymm6[2],ymm4[2] + vpunpckhqdq %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[1],ymm4[1],ymm6[3],ymm4[3] + vpunpcklqdq %ymm9, %ymm5, %ymm6 # ymm6 = ymm5[0],ymm9[0],ymm5[2],ymm9[2] + vpunpckhqdq %ymm9, %ymm5, %ymm9 # ymm9 = ymm5[1],ymm9[1],ymm5[3],ymm9[3] + vperm2i128 $0x20, %ymm8, %ymm7, %ymm5 # ymm5 = ymm7[0,1],ymm8[0,1] + vperm2i128 $0x31, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[2,3],ymm8[2,3] + vperm2i128 $0x20, %ymm3, %ymm6, %ymm7 # ymm7 = ymm6[0,1],ymm3[0,1] + vperm2i128 $0x31, %ymm3, %ymm6, %ymm3 # ymm3 = ymm6[2,3],ymm3[2,3] + vperm2i128 $0x20, %ymm9, %ymm4, %ymm6 # ymm6 = ymm4[0,1],ymm9[0,1] + vperm2i128 $0x31, %ymm9, %ymm4, %ymm9 # ymm9 = ymm4[2,3],ymm9[2,3] + vmovdqu %ymm5, 0xc0(%rdi) + vmovdqu %ymm7, 0xe0(%rdi) + vmovdqu %ymm6, 0x100(%rdi) + vmovdqu %ymm8, 0x120(%rdi) + vmovdqu %ymm3, 0x140(%rdi) + vmovdqu %ymm9, 0x160(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(ntttobytes_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (!MLK_CONFIG_NO_KEYPAIR_API || !MLK_CONFIG_NO_ENCAPS_API) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_nttunpack_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_nttunpack_avx2_asm.S new file mode 100644 index 0000000000..015d43d9c7 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_nttunpack_avx2_asm.S @@ -0,0 +1,190 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/*yaml + Name: nttunpack_avx2_asm + Description: x86_64 AVX2 NTT unpack from custom to bitreversed order + Signature: void mlk_nttunpack_avx2_asm(int16_t *r) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_nttunpack_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(nttunpack_avx2_asm) +MLK_ASM_FN_SYMBOL(nttunpack_avx2_asm) + + .cfi_startproc + vmovdqa (%rdi), %ymm4 + vmovdqa 0x20(%rdi), %ymm5 + vmovdqa 0x40(%rdi), %ymm6 + vmovdqa 0x60(%rdi), %ymm7 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm9 + vmovdqa 0xc0(%rdi), %ymm10 + vmovdqa 0xe0(%rdi), %ymm11 + vperm2i128 $0x20, %ymm8, %ymm4, %ymm3 # ymm3 = ymm4[0,1],ymm8[0,1] + vperm2i128 $0x31, %ymm8, %ymm4, %ymm8 # ymm8 = ymm4[2,3],ymm8[2,3] + vperm2i128 $0x20, %ymm9, %ymm5, %ymm4 # ymm4 = ymm5[0,1],ymm9[0,1] + vperm2i128 $0x31, %ymm9, %ymm5, %ymm9 # ymm9 = ymm5[2,3],ymm9[2,3] + vperm2i128 $0x20, %ymm10, %ymm6, %ymm5 # ymm5 = ymm6[0,1],ymm10[0,1] + vperm2i128 $0x31, %ymm10, %ymm6, %ymm10 # ymm10 = ymm6[2,3],ymm10[2,3] + vperm2i128 $0x20, %ymm11, %ymm7, %ymm6 # ymm6 = ymm7[0,1],ymm11[0,1] + vperm2i128 $0x31, %ymm11, %ymm7, %ymm11 # ymm11 = ymm7[2,3],ymm11[2,3] + vpunpcklqdq %ymm5, %ymm3, %ymm7 # ymm7 = ymm3[0],ymm5[0],ymm3[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[1],ymm5[1],ymm3[3],ymm5[3] + vpunpcklqdq %ymm10, %ymm8, %ymm3 # ymm3 = ymm8[0],ymm10[0],ymm8[2],ymm10[2] + vpunpckhqdq %ymm10, %ymm8, %ymm10 # ymm10 = ymm8[1],ymm10[1],ymm8[3],ymm10[3] + vpunpcklqdq %ymm6, %ymm4, %ymm8 # ymm8 = ymm4[0],ymm6[0],ymm4[2],ymm6[2] + vpunpckhqdq %ymm6, %ymm4, %ymm6 # ymm6 = ymm4[1],ymm6[1],ymm4[3],ymm6[3] + vpunpcklqdq %ymm11, %ymm9, %ymm4 # ymm4 = ymm9[0],ymm11[0],ymm9[2],ymm11[2] + vpunpckhqdq %ymm11, %ymm9, %ymm11 # ymm11 = ymm9[1],ymm11[1],ymm9[3],ymm11[3] + vmovsldup %ymm8, %ymm9 # ymm9 = ymm8[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm9, %ymm7, %ymm9 # ymm9 = ymm7[0],ymm9[1],ymm7[2],ymm9[3],ymm7[4],ymm9[5],ymm7[6],ymm9[7] + vpsrlq $0x20, %ymm7, %ymm7 + vpblendd $0xaa, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[0],ymm8[1],ymm7[2],ymm8[3],ymm7[4],ymm8[5],ymm7[6],ymm8[7] + vmovsldup %ymm6, %ymm7 # ymm7 = ymm6[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm7, %ymm5, %ymm7 # ymm7 = ymm5[0],ymm7[1],ymm5[2],ymm7[3],ymm5[4],ymm7[5],ymm5[6],ymm7[7] + vpsrlq $0x20, %ymm5, %ymm5 + vpblendd $0xaa, %ymm6, %ymm5, %ymm6 # ymm6 = ymm5[0],ymm6[1],ymm5[2],ymm6[3],ymm5[4],ymm6[5],ymm5[6],ymm6[7] + vmovsldup %ymm4, %ymm5 # ymm5 = ymm4[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[0],ymm5[1],ymm3[2],ymm5[3],ymm3[4],ymm5[5],ymm3[6],ymm5[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm4, %ymm3, %ymm4 # ymm4 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7] + vmovsldup %ymm11, %ymm3 # ymm3 = ymm11[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm3, %ymm10, %ymm3 # ymm3 = ymm10[0],ymm3[1],ymm10[2],ymm3[3],ymm10[4],ymm3[5],ymm10[6],ymm3[7] + vpsrlq $0x20, %ymm10, %ymm10 + vpblendd $0xaa, %ymm11, %ymm10, %ymm11 # ymm11 = ymm10[0],ymm11[1],ymm10[2],ymm11[3],ymm10[4],ymm11[5],ymm10[6],ymm11[7] + vpslld $0x10, %ymm5, %ymm10 + vpblendw $0xaa, %ymm10, %ymm9, %ymm10 # ymm10 = ymm9[0],ymm10[1],ymm9[2],ymm10[3],ymm9[4],ymm10[5],ymm9[6],ymm10[7],ymm9[8],ymm10[9],ymm9[10],ymm10[11],ymm9[12],ymm10[13],ymm9[14],ymm10[15] + vpsrld $0x10, %ymm9, %ymm9 + vpblendw $0xaa, %ymm5, %ymm9, %ymm5 # ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7],ymm9[8],ymm5[9],ymm9[10],ymm5[11],ymm9[12],ymm5[13],ymm9[14],ymm5[15] + vpslld $0x10, %ymm4, %ymm9 + vpblendw $0xaa, %ymm9, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7],ymm8[8],ymm9[9],ymm8[10],ymm9[11],ymm8[12],ymm9[13],ymm8[14],ymm9[15] + vpsrld $0x10, %ymm8, %ymm8 + vpblendw $0xaa, %ymm4, %ymm8, %ymm4 # ymm4 = ymm8[0],ymm4[1],ymm8[2],ymm4[3],ymm8[4],ymm4[5],ymm8[6],ymm4[7],ymm8[8],ymm4[9],ymm8[10],ymm4[11],ymm8[12],ymm4[13],ymm8[14],ymm4[15] + vpslld $0x10, %ymm3, %ymm8 + vpblendw $0xaa, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[0],ymm8[1],ymm7[2],ymm8[3],ymm7[4],ymm8[5],ymm7[6],ymm8[7],ymm7[8],ymm8[9],ymm7[10],ymm8[11],ymm7[12],ymm8[13],ymm7[14],ymm8[15] + vpsrld $0x10, %ymm7, %ymm7 + vpblendw $0xaa, %ymm3, %ymm7, %ymm3 # ymm3 = ymm7[0],ymm3[1],ymm7[2],ymm3[3],ymm7[4],ymm3[5],ymm7[6],ymm3[7],ymm7[8],ymm3[9],ymm7[10],ymm3[11],ymm7[12],ymm3[13],ymm7[14],ymm3[15] + vpslld $0x10, %ymm11, %ymm7 + vpblendw $0xaa, %ymm7, %ymm6, %ymm7 # ymm7 = ymm6[0],ymm7[1],ymm6[2],ymm7[3],ymm6[4],ymm7[5],ymm6[6],ymm7[7],ymm6[8],ymm7[9],ymm6[10],ymm7[11],ymm6[12],ymm7[13],ymm6[14],ymm7[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm11, %ymm6, %ymm11 # ymm11 = ymm6[0],ymm11[1],ymm6[2],ymm11[3],ymm6[4],ymm11[5],ymm6[6],ymm11[7],ymm6[8],ymm11[9],ymm6[10],ymm11[11],ymm6[12],ymm11[13],ymm6[14],ymm11[15] + vmovdqa %ymm10, (%rdi) + vmovdqa %ymm5, 0x20(%rdi) + vmovdqa %ymm9, 0x40(%rdi) + vmovdqa %ymm4, 0x60(%rdi) + vmovdqa %ymm8, 0x80(%rdi) + vmovdqa %ymm3, 0xa0(%rdi) + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm11, 0xe0(%rdi) + vmovdqa 0x100(%rdi), %ymm4 + vmovdqa 0x120(%rdi), %ymm5 + vmovdqa 0x140(%rdi), %ymm6 + vmovdqa 0x160(%rdi), %ymm7 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm9 + vmovdqa 0x1c0(%rdi), %ymm10 + vmovdqa 0x1e0(%rdi), %ymm11 + vperm2i128 $0x20, %ymm8, %ymm4, %ymm3 # ymm3 = ymm4[0,1],ymm8[0,1] + vperm2i128 $0x31, %ymm8, %ymm4, %ymm8 # ymm8 = ymm4[2,3],ymm8[2,3] + vperm2i128 $0x20, %ymm9, %ymm5, %ymm4 # ymm4 = ymm5[0,1],ymm9[0,1] + vperm2i128 $0x31, %ymm9, %ymm5, %ymm9 # ymm9 = ymm5[2,3],ymm9[2,3] + vperm2i128 $0x20, %ymm10, %ymm6, %ymm5 # ymm5 = ymm6[0,1],ymm10[0,1] + vperm2i128 $0x31, %ymm10, %ymm6, %ymm10 # ymm10 = ymm6[2,3],ymm10[2,3] + vperm2i128 $0x20, %ymm11, %ymm7, %ymm6 # ymm6 = ymm7[0,1],ymm11[0,1] + vperm2i128 $0x31, %ymm11, %ymm7, %ymm11 # ymm11 = ymm7[2,3],ymm11[2,3] + vpunpcklqdq %ymm5, %ymm3, %ymm7 # ymm7 = ymm3[0],ymm5[0],ymm3[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[1],ymm5[1],ymm3[3],ymm5[3] + vpunpcklqdq %ymm10, %ymm8, %ymm3 # ymm3 = ymm8[0],ymm10[0],ymm8[2],ymm10[2] + vpunpckhqdq %ymm10, %ymm8, %ymm10 # ymm10 = ymm8[1],ymm10[1],ymm8[3],ymm10[3] + vpunpcklqdq %ymm6, %ymm4, %ymm8 # ymm8 = ymm4[0],ymm6[0],ymm4[2],ymm6[2] + vpunpckhqdq %ymm6, %ymm4, %ymm6 # ymm6 = ymm4[1],ymm6[1],ymm4[3],ymm6[3] + vpunpcklqdq %ymm11, %ymm9, %ymm4 # ymm4 = ymm9[0],ymm11[0],ymm9[2],ymm11[2] + vpunpckhqdq %ymm11, %ymm9, %ymm11 # ymm11 = ymm9[1],ymm11[1],ymm9[3],ymm11[3] + vmovsldup %ymm8, %ymm9 # ymm9 = ymm8[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm9, %ymm7, %ymm9 # ymm9 = ymm7[0],ymm9[1],ymm7[2],ymm9[3],ymm7[4],ymm9[5],ymm7[6],ymm9[7] + vpsrlq $0x20, %ymm7, %ymm7 + vpblendd $0xaa, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[0],ymm8[1],ymm7[2],ymm8[3],ymm7[4],ymm8[5],ymm7[6],ymm8[7] + vmovsldup %ymm6, %ymm7 # ymm7 = ymm6[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm7, %ymm5, %ymm7 # ymm7 = ymm5[0],ymm7[1],ymm5[2],ymm7[3],ymm5[4],ymm7[5],ymm5[6],ymm7[7] + vpsrlq $0x20, %ymm5, %ymm5 + vpblendd $0xaa, %ymm6, %ymm5, %ymm6 # ymm6 = ymm5[0],ymm6[1],ymm5[2],ymm6[3],ymm5[4],ymm6[5],ymm5[6],ymm6[7] + vmovsldup %ymm4, %ymm5 # ymm5 = ymm4[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[0],ymm5[1],ymm3[2],ymm5[3],ymm3[4],ymm5[5],ymm3[6],ymm5[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm4, %ymm3, %ymm4 # ymm4 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7] + vmovsldup %ymm11, %ymm3 # ymm3 = ymm11[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm3, %ymm10, %ymm3 # ymm3 = ymm10[0],ymm3[1],ymm10[2],ymm3[3],ymm10[4],ymm3[5],ymm10[6],ymm3[7] + vpsrlq $0x20, %ymm10, %ymm10 + vpblendd $0xaa, %ymm11, %ymm10, %ymm11 # ymm11 = ymm10[0],ymm11[1],ymm10[2],ymm11[3],ymm10[4],ymm11[5],ymm10[6],ymm11[7] + vpslld $0x10, %ymm5, %ymm10 + vpblendw $0xaa, %ymm10, %ymm9, %ymm10 # ymm10 = ymm9[0],ymm10[1],ymm9[2],ymm10[3],ymm9[4],ymm10[5],ymm9[6],ymm10[7],ymm9[8],ymm10[9],ymm9[10],ymm10[11],ymm9[12],ymm10[13],ymm9[14],ymm10[15] + vpsrld $0x10, %ymm9, %ymm9 + vpblendw $0xaa, %ymm5, %ymm9, %ymm5 # ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7],ymm9[8],ymm5[9],ymm9[10],ymm5[11],ymm9[12],ymm5[13],ymm9[14],ymm5[15] + vpslld $0x10, %ymm4, %ymm9 + vpblendw $0xaa, %ymm9, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7],ymm8[8],ymm9[9],ymm8[10],ymm9[11],ymm8[12],ymm9[13],ymm8[14],ymm9[15] + vpsrld $0x10, %ymm8, %ymm8 + vpblendw $0xaa, %ymm4, %ymm8, %ymm4 # ymm4 = ymm8[0],ymm4[1],ymm8[2],ymm4[3],ymm8[4],ymm4[5],ymm8[6],ymm4[7],ymm8[8],ymm4[9],ymm8[10],ymm4[11],ymm8[12],ymm4[13],ymm8[14],ymm4[15] + vpslld $0x10, %ymm3, %ymm8 + vpblendw $0xaa, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[0],ymm8[1],ymm7[2],ymm8[3],ymm7[4],ymm8[5],ymm7[6],ymm8[7],ymm7[8],ymm8[9],ymm7[10],ymm8[11],ymm7[12],ymm8[13],ymm7[14],ymm8[15] + vpsrld $0x10, %ymm7, %ymm7 + vpblendw $0xaa, %ymm3, %ymm7, %ymm3 # ymm3 = ymm7[0],ymm3[1],ymm7[2],ymm3[3],ymm7[4],ymm3[5],ymm7[6],ymm3[7],ymm7[8],ymm3[9],ymm7[10],ymm3[11],ymm7[12],ymm3[13],ymm7[14],ymm3[15] + vpslld $0x10, %ymm11, %ymm7 + vpblendw $0xaa, %ymm7, %ymm6, %ymm7 # ymm7 = ymm6[0],ymm7[1],ymm6[2],ymm7[3],ymm6[4],ymm7[5],ymm6[6],ymm7[7],ymm6[8],ymm7[9],ymm6[10],ymm7[11],ymm6[12],ymm7[13],ymm6[14],ymm7[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm11, %ymm6, %ymm11 # ymm11 = ymm6[0],ymm11[1],ymm6[2],ymm11[3],ymm6[4],ymm11[5],ymm6[6],ymm11[7],ymm6[8],ymm11[9],ymm6[10],ymm11[11],ymm6[12],ymm11[13],ymm6[14],ymm11[15] + vmovdqa %ymm10, 0x100(%rdi) + vmovdqa %ymm5, 0x120(%rdi) + vmovdqa %ymm9, 0x140(%rdi) + vmovdqa %ymm4, 0x160(%rdi) + vmovdqa %ymm8, 0x180(%rdi) + vmovdqa %ymm3, 0x1a0(%rdi) + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm11, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(nttunpack_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d10_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d10_avx2_asm.S new file mode 100644 index 0000000000..af8cecfd9c --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d10_avx2_asm.S @@ -0,0 +1,413 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +/************************************************* + * Name: mlk_poly_compress_d10_avx2_asm + * + * Description: Compression of a polynomial to 10 bits per coefficient. + * + * Arguments: - uint8_t *r: pointer to output byte array + * (of length MLKEM_POLYCOMPRESSEDBYTES_D10) + * - const int16_t *a: pointer to input polynomial + * - const uint8_t *data: pointer to shufbidx constant + **************************************************/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API)) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 || MLKEM_K == 3) +/*yaml + Name: poly_compress_d10_avx2_asm + Description: x86_64 AVX2 polynomial compression (d=10) + Signature: void mlk_poly_compress_d10_avx2_asm(uint8_t *r, const int16_t *a, const uint8_t *data) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 320 + permissions: write-only + c_parameter: uint8_t *r + description: Output compressed polynomial + rsi: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t *a + description: Input polynomial (256 x int16_t) + rdx: + type: buffer + size_bytes: 32 + permissions: read-only + c_parameter: const uint8_t *data + description: Precomputed compression constants +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_poly_compress_d10_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_compress_d10_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_compress_d10_avx2_asm) + + .cfi_startproc + movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + vpsllw $0x3, %ymm0, %ymm1 + movl $0xf000f, %eax # imm = 0xF000F + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + movl $0x10001000, %eax # imm = 0x10001000 + vmovd %eax, %xmm3 + vpbroadcastd %xmm3, %ymm3 + movl $0x3ff03ff, %eax # imm = 0x3FF03FF + vmovd %eax, %xmm4 + vpbroadcastd %xmm4, %ymm4 + movabsq $0x400000104000001, %rax # imm = 0x400000104000001 + vmovq %rax, %xmm5 + vpbroadcastq %xmm5, %ymm5 + movl $0xc, %eax + vmovq %rax, %xmm6 + vpbroadcastq %xmm6, %ymm6 + vmovdqa (%rdx), %ymm7 + vmovdqa (%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, (%rdi) + vmovd %xmm9, 0x10(%rdi) + vmovdqa 0x20(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x14(%rdi) + vmovd %xmm9, 0x24(%rdi) + vmovdqa 0x40(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x28(%rdi) + vmovd %xmm9, 0x38(%rdi) + vmovdqa 0x60(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x3c(%rdi) + vmovd %xmm9, 0x4c(%rdi) + vmovdqa 0x80(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x50(%rdi) + vmovd %xmm9, 0x60(%rdi) + vmovdqa 0xa0(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x64(%rdi) + vmovd %xmm9, 0x74(%rdi) + vmovdqa 0xc0(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x78(%rdi) + vmovd %xmm9, 0x88(%rdi) + vmovdqa 0xe0(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x8c(%rdi) + vmovd %xmm9, 0x9c(%rdi) + vmovdqa 0x100(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0xa0(%rdi) + vmovd %xmm9, 0xb0(%rdi) + vmovdqa 0x120(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0xb4(%rdi) + vmovd %xmm9, 0xc4(%rdi) + vmovdqa 0x140(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0xc8(%rdi) + vmovd %xmm9, 0xd8(%rdi) + vmovdqa 0x160(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0xdc(%rdi) + vmovd %xmm9, 0xec(%rdi) + vmovdqa 0x180(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0xf0(%rdi) + vmovd %xmm9, 0x100(%rdi) + vmovdqa 0x1a0(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x104(%rdi) + vmovd %xmm9, 0x114(%rdi) + vmovdqa 0x1c0(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x118(%rdi) + vmovd %xmm9, 0x128(%rdi) + vmovdqa 0x1e0(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x12c(%rdi) + vmovd %xmm9, 0x13c(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_compress_d10_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API) && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == 3) \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d11_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d11_avx2_asm.S new file mode 100644 index 0000000000..9442315c3b --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d11_avx2_asm.S @@ -0,0 +1,479 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +/************************************************* + * Name: mlk_poly_compress_d11_avx2_asm + * + * Description: Compression of a polynomial to 11 bits per coefficient. + * + * Arguments: - uint8_t *r: pointer to output byte array + * (of length MLKEM_POLYCOMPRESSEDBYTES_D11) + * - const int16_t *a: pointer to input polynomial + * - const uint8_t *data: pointer to constants + * (srlvqidx[0:32], shufbidx[32:64]) + **************************************************/ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API)) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) +/*yaml + Name: poly_compress_d11_avx2_asm + Description: x86_64 AVX2 polynomial compression (d=11) + Signature: void mlk_poly_compress_d11_avx2_asm(uint8_t *r, const int16_t *a, const uint8_t *data) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 352 + permissions: write-only + c_parameter: uint8_t *r + description: Output compressed polynomial + rsi: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t *a + description: Input polynomial (256 x int16_t) + rdx: + type: buffer + size_bytes: 64 + permissions: read-only + c_parameter: const uint8_t *data + description: Precomputed compression constants +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_poly_compress_d11_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_compress_d11_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_compress_d11_avx2_asm) + + .cfi_startproc + movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + vpsllw $0x3, %ymm0, %ymm1 + movl $0x240024, %eax # imm = 0x240024 + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + movl $0x20002000, %eax # imm = 0x20002000 + vmovd %eax, %xmm3 + vpbroadcastd %xmm3, %ymm3 + movl $0x7ff07ff, %eax # imm = 0x7FF07FF + vmovd %eax, %xmm4 + vpbroadcastd %xmm4, %ymm4 + movabsq $0x800000108000001, %rax # imm = 0x800000108000001 + vmovq %rax, %xmm5 + vpbroadcastq %xmm5, %ymm5 + movl $0xa, %eax + vmovq %rax, %xmm6 + vpbroadcastq %xmm6, %ymm6 + vmovdqa (%rdx), %ymm7 + vmovdqa 0x20(%rdx), %ymm8 + vmovdqa (%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, (%rdi) + vmovd %xmm10, 0x10(%rdi) + vpextrw $0x2, %xmm10, 0x14(%rdi) + vmovdqa 0x20(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x16(%rdi) + vmovd %xmm10, 0x26(%rdi) + vpextrw $0x2, %xmm10, 0x2a(%rdi) + vmovdqa 0x40(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x2c(%rdi) + vmovd %xmm10, 0x3c(%rdi) + vpextrw $0x2, %xmm10, 0x40(%rdi) + vmovdqa 0x60(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x42(%rdi) + vmovd %xmm10, 0x52(%rdi) + vpextrw $0x2, %xmm10, 0x56(%rdi) + vmovdqa 0x80(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x58(%rdi) + vmovd %xmm10, 0x68(%rdi) + vpextrw $0x2, %xmm10, 0x6c(%rdi) + vmovdqa 0xa0(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x6e(%rdi) + vmovd %xmm10, 0x7e(%rdi) + vpextrw $0x2, %xmm10, 0x82(%rdi) + vmovdqa 0xc0(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x84(%rdi) + vmovd %xmm10, 0x94(%rdi) + vpextrw $0x2, %xmm10, 0x98(%rdi) + vmovdqa 0xe0(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x9a(%rdi) + vmovd %xmm10, 0xaa(%rdi) + vpextrw $0x2, %xmm10, 0xae(%rdi) + vmovdqa 0x100(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0xb0(%rdi) + vmovd %xmm10, 0xc0(%rdi) + vpextrw $0x2, %xmm10, 0xc4(%rdi) + vmovdqa 0x120(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0xc6(%rdi) + vmovd %xmm10, 0xd6(%rdi) + vpextrw $0x2, %xmm10, 0xda(%rdi) + vmovdqa 0x140(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0xdc(%rdi) + vmovd %xmm10, 0xec(%rdi) + vpextrw $0x2, %xmm10, 0xf0(%rdi) + vmovdqa 0x160(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0xf2(%rdi) + vmovd %xmm10, 0x102(%rdi) + vpextrw $0x2, %xmm10, 0x106(%rdi) + vmovdqa 0x180(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x108(%rdi) + vmovd %xmm10, 0x118(%rdi) + vpextrw $0x2, %xmm10, 0x11c(%rdi) + vmovdqa 0x1a0(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x11e(%rdi) + vmovd %xmm10, 0x12e(%rdi) + vpextrw $0x2, %xmm10, 0x132(%rdi) + vmovdqa 0x1c0(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x134(%rdi) + vmovd %xmm10, 0x144(%rdi) + vpextrw $0x2, %xmm10, 0x148(%rdi) + vmovdqa 0x1e0(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x14a(%rdi) + vmovd %xmm10, 0x15a(%rdi) + vpextrw $0x2, %xmm10, 0x15e(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_compress_d11_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API) && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d4_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d4_avx2_asm.S new file mode 100644 index 0000000000..428fda5b42 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d4_avx2_asm.S @@ -0,0 +1,194 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +/************************************************* + * Name: mlk_poly_compress_d4_avx2_asm + * + * Description: Compression of a polynomial to 4 bits per coefficient. + * + * Arguments: - uint8_t *r: pointer to output byte array + * (of length MLKEM_POLYCOMPRESSEDBYTES_D4) + * - const int16_t *a: pointer to input polynomial + * - const uint8_t *data: pointer to permdidx constant + **************************************************/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API)) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 || MLKEM_K == 3) +/*yaml + Name: poly_compress_d4_avx2_asm + Description: x86_64 AVX2 polynomial compression (d=4) + Signature: void mlk_poly_compress_d4_avx2_asm(uint8_t *r, const int16_t *a, const uint8_t *data) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 128 + permissions: write-only + c_parameter: uint8_t *r + description: Output compressed polynomial + rsi: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t *a + description: Input polynomial (256 x int16_t) + rdx: + type: buffer + size_bytes: 32 + permissions: read-only + c_parameter: const uint8_t *data + description: Precomputed compression constants +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_poly_compress_d4_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_compress_d4_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_compress_d4_avx2_asm) + + .cfi_startproc + movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0x2000200, %eax # imm = 0x2000200 + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + movl $0xf000f, %eax # imm = 0xF000F + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + movl $0x10011001, %eax # imm = 0x10011001 + vmovd %eax, %xmm3 + vpbroadcastd %xmm3, %ymm3 + vmovdqa (%rdx), %ymm4 + vmovdqa (%rsi), %ymm5 + vmovdqa 0x20(%rsi), %ymm6 + vmovdqa 0x40(%rsi), %ymm7 + vmovdqa 0x60(%rsi), %ymm8 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm5, %ymm5 + vpmulhrsw %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm5, %ymm5 + vpand %ymm2, %ymm6, %ymm6 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm6, %ymm5, %ymm5 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm5, %ymm5 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpackuswb %ymm7, %ymm5, %ymm5 + vpermd %ymm5, %ymm4, %ymm5 + vmovdqu %ymm5, (%rdi) + vmovdqa 0x80(%rsi), %ymm5 + vmovdqa 0xa0(%rsi), %ymm6 + vmovdqa 0xc0(%rsi), %ymm7 + vmovdqa 0xe0(%rsi), %ymm8 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm5, %ymm5 + vpmulhrsw %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm5, %ymm5 + vpand %ymm2, %ymm6, %ymm6 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm6, %ymm5, %ymm5 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm5, %ymm5 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpackuswb %ymm7, %ymm5, %ymm5 + vpermd %ymm5, %ymm4, %ymm5 + vmovdqu %ymm5, 0x20(%rdi) + vmovdqa 0x100(%rsi), %ymm5 + vmovdqa 0x120(%rsi), %ymm6 + vmovdqa 0x140(%rsi), %ymm7 + vmovdqa 0x160(%rsi), %ymm8 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm5, %ymm5 + vpmulhrsw %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm5, %ymm5 + vpand %ymm2, %ymm6, %ymm6 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm6, %ymm5, %ymm5 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm5, %ymm5 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpackuswb %ymm7, %ymm5, %ymm5 + vpermd %ymm5, %ymm4, %ymm5 + vmovdqu %ymm5, 0x40(%rdi) + vmovdqa 0x180(%rsi), %ymm5 + vmovdqa 0x1a0(%rsi), %ymm6 + vmovdqa 0x1c0(%rsi), %ymm7 + vmovdqa 0x1e0(%rsi), %ymm8 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm5, %ymm5 + vpmulhrsw %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm5, %ymm5 + vpand %ymm2, %ymm6, %ymm6 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm6, %ymm5, %ymm5 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm5, %ymm5 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpackuswb %ymm7, %ymm5, %ymm5 + vpermd %ymm5, %ymm4, %ymm5 + vmovdqu %ymm5, 0x60(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_compress_d4_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API) && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == 3) \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d5_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d5_avx2_asm.S new file mode 100644 index 0000000000..41e3c679d7 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d5_avx2_asm.S @@ -0,0 +1,251 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +/************************************************* + * Name: mlk_poly_compress_d5_avx2_asm + * + * Description: Compression of a polynomial to 5 bits per coefficient. + * + * Arguments: - uint8_t *r: pointer to output byte array + * (of length MLKEM_POLYCOMPRESSEDBYTES_D5) + * - const int16_t *a: pointer to input polynomial + * - const uint8_t *data: pointer to shufbidx constant + **************************************************/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API)) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) +/*yaml + Name: poly_compress_d5_avx2_asm + Description: x86_64 AVX2 polynomial compression (d=5) + Signature: void mlk_poly_compress_d5_avx2_asm(uint8_t *r, const int16_t *a, const uint8_t *data) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 160 + permissions: write-only + c_parameter: uint8_t *r + description: Output compressed polynomial + rsi: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t *a + description: Input polynomial (256 x int16_t) + rdx: + type: buffer + size_bytes: 32 + permissions: read-only + c_parameter: const uint8_t *data + description: Precomputed compression constants +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_poly_compress_d5_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_compress_d5_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_compress_d5_avx2_asm) + + .cfi_startproc + movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0x4000400, %eax # imm = 0x4000400 + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + movl $0x1f001f, %eax # imm = 0x1F001F + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + movl $0x20012001, %eax # imm = 0x20012001 + vmovd %eax, %xmm3 + vpbroadcastd %xmm3, %ymm3 + movl $0x4000001, %eax # imm = 0x4000001 + vmovd %eax, %xmm4 + vpbroadcastd %xmm4, %ymm4 + movl $0xc, %eax + vmovq %rax, %xmm5 + vpbroadcastq %xmm5, %ymm5 + vmovdqa (%rdx), %ymm6 + vmovdqa (%rsi), %ymm7 + vmovdqa 0x20(%rsi), %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpmaddwd %ymm4, %ymm7, %ymm7 + vpsllvd %ymm5, %ymm7, %ymm7 + vpsrlvq %ymm5, %ymm7, %ymm7 + vpshufb %ymm6, %ymm7, %ymm7 + vextracti128 $0x1, %ymm7, %xmm8 + vpblendvb %xmm6, %xmm8, %xmm7, %xmm7 + vmovdqu %xmm7, (%rdi) + vmovd %xmm8, 0x10(%rdi) + vmovdqa 0x40(%rsi), %ymm7 + vmovdqa 0x60(%rsi), %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpmaddwd %ymm4, %ymm7, %ymm7 + vpsllvd %ymm5, %ymm7, %ymm7 + vpsrlvq %ymm5, %ymm7, %ymm7 + vpshufb %ymm6, %ymm7, %ymm7 + vextracti128 $0x1, %ymm7, %xmm8 + vpblendvb %xmm6, %xmm8, %xmm7, %xmm7 + vmovdqu %xmm7, 0x14(%rdi) + vmovd %xmm8, 0x24(%rdi) + vmovdqa 0x80(%rsi), %ymm7 + vmovdqa 0xa0(%rsi), %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpmaddwd %ymm4, %ymm7, %ymm7 + vpsllvd %ymm5, %ymm7, %ymm7 + vpsrlvq %ymm5, %ymm7, %ymm7 + vpshufb %ymm6, %ymm7, %ymm7 + vextracti128 $0x1, %ymm7, %xmm8 + vpblendvb %xmm6, %xmm8, %xmm7, %xmm7 + vmovdqu %xmm7, 0x28(%rdi) + vmovd %xmm8, 0x38(%rdi) + vmovdqa 0xc0(%rsi), %ymm7 + vmovdqa 0xe0(%rsi), %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpmaddwd %ymm4, %ymm7, %ymm7 + vpsllvd %ymm5, %ymm7, %ymm7 + vpsrlvq %ymm5, %ymm7, %ymm7 + vpshufb %ymm6, %ymm7, %ymm7 + vextracti128 $0x1, %ymm7, %xmm8 + vpblendvb %xmm6, %xmm8, %xmm7, %xmm7 + vmovdqu %xmm7, 0x3c(%rdi) + vmovd %xmm8, 0x4c(%rdi) + vmovdqa 0x100(%rsi), %ymm7 + vmovdqa 0x120(%rsi), %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpmaddwd %ymm4, %ymm7, %ymm7 + vpsllvd %ymm5, %ymm7, %ymm7 + vpsrlvq %ymm5, %ymm7, %ymm7 + vpshufb %ymm6, %ymm7, %ymm7 + vextracti128 $0x1, %ymm7, %xmm8 + vpblendvb %xmm6, %xmm8, %xmm7, %xmm7 + vmovdqu %xmm7, 0x50(%rdi) + vmovd %xmm8, 0x60(%rdi) + vmovdqa 0x140(%rsi), %ymm7 + vmovdqa 0x160(%rsi), %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpmaddwd %ymm4, %ymm7, %ymm7 + vpsllvd %ymm5, %ymm7, %ymm7 + vpsrlvq %ymm5, %ymm7, %ymm7 + vpshufb %ymm6, %ymm7, %ymm7 + vextracti128 $0x1, %ymm7, %xmm8 + vpblendvb %xmm6, %xmm8, %xmm7, %xmm7 + vmovdqu %xmm7, 0x64(%rdi) + vmovd %xmm8, 0x74(%rdi) + vmovdqa 0x180(%rsi), %ymm7 + vmovdqa 0x1a0(%rsi), %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpmaddwd %ymm4, %ymm7, %ymm7 + vpsllvd %ymm5, %ymm7, %ymm7 + vpsrlvq %ymm5, %ymm7, %ymm7 + vpshufb %ymm6, %ymm7, %ymm7 + vextracti128 $0x1, %ymm7, %xmm8 + vpblendvb %xmm6, %xmm8, %xmm7, %xmm7 + vmovdqu %xmm7, 0x78(%rdi) + vmovd %xmm8, 0x88(%rdi) + vmovdqa 0x1c0(%rsi), %ymm7 + vmovdqa 0x1e0(%rsi), %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpmaddwd %ymm4, %ymm7, %ymm7 + vpsllvd %ymm5, %ymm7, %ymm7 + vpsrlvq %ymm5, %ymm7, %ymm7 + vpshufb %ymm6, %ymm7, %ymm7 + vextracti128 $0x1, %ymm7, %xmm8 + vpblendvb %xmm6, %xmm8, %xmm7, %xmm7 + vmovdqu %xmm7, 0x8c(%rdi) + vmovd %xmm8, 0x9c(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_compress_d5_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API) && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d10_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d10_avx2_asm.S new file mode 100644 index 0000000000..656955555a --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d10_avx2_asm.S @@ -0,0 +1,257 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +/************************************************* + * Name: mlk_poly_decompress_d10_avx2_asm + * + * Description: Decompression of a polynomial from 10 bits per coefficient. + * + * Arguments: - int16_t *r: pointer to output polynomial + * - const uint8_t *a: pointer to input byte array + * (of length MLKEM_POLYCOMPRESSEDBYTES_D10) + * - const uint8_t *data: pointer to shufbidx constant + **************************************************/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + !defined(MLK_CONFIG_NO_DECAPS_API) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 || MLKEM_K == 3) +/*yaml + Name: poly_decompress_d10_avx2_asm + Description: x86_64 AVX2 polynomial decompression (d=10) + Signature: void mlk_poly_decompress_d10_avx2_asm(int16_t *r, const uint8_t *a, const uint8_t *data) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t *r + description: Output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 320 + permissions: read-only + c_parameter: const uint8_t *a + description: Input compressed polynomial + rdx: + type: buffer + size_bytes: 32 + permissions: read-only + c_parameter: const uint8_t *data + description: Precomputed decompression constants +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_poly_decompress_d10_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_decompress_d10_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_decompress_d10_avx2_asm) + + .cfi_startproc + movl $0xd013404, %eax # imm = 0xD013404 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0x4, %eax + vmovq %rax, %xmm1 + vpbroadcastq %xmm1, %ymm1 + movl $0x7fe01ff8, %eax # imm = 0x7FE01FF8 + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + vmovdqa (%rdx), %ymm3 + vmovdqu (%rsi), %xmm4 + vmovd 0x10(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, (%rdi) + vmovdqu 0x14(%rsi), %xmm4 + vmovd 0x24(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x20(%rdi) + vmovdqu 0x28(%rsi), %xmm4 + vmovd 0x38(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x40(%rdi) + vmovdqu 0x3c(%rsi), %xmm4 + vmovd 0x4c(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x60(%rdi) + vmovdqu 0x50(%rsi), %xmm4 + vmovd 0x60(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x80(%rdi) + vmovdqu 0x64(%rsi), %xmm4 + vmovd 0x74(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xa0(%rdi) + vmovdqu 0x78(%rsi), %xmm4 + vmovd 0x88(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xc0(%rdi) + vmovdqu 0x8c(%rsi), %xmm4 + vmovd 0x9c(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xe0(%rdi) + vmovdqu 0xa0(%rsi), %xmm4 + vmovd 0xb0(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x100(%rdi) + vmovdqu 0xb4(%rsi), %xmm4 + vmovd 0xc4(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x120(%rdi) + vmovdqu 0xc8(%rsi), %xmm4 + vmovd 0xd8(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x140(%rdi) + vmovdqu 0xdc(%rsi), %xmm4 + vmovd 0xec(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x160(%rdi) + vmovdqu 0xf0(%rsi), %xmm4 + vmovd 0x100(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x180(%rdi) + vmovdqu 0x104(%rsi), %xmm4 + vmovd 0x114(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1a0(%rdi) + vmovdqu 0x118(%rsi), %xmm4 + vmovd 0x128(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1c0(%rdi) + vmovdqu 0x12c(%rsi), %xmm4 + vmovd 0x13c(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_decompress_d10_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && !MLK_CONFIG_NO_DECAPS_API && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || \ + MLKEM_K == 2 || MLKEM_K == 3) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d11_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d11_avx2_asm.S new file mode 100644 index 0000000000..d5c42312fc --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d11_avx2_asm.S @@ -0,0 +1,307 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +/************************************************* + * Name: mlk_poly_decompress_d11_avx2_asm + * + * Description: Decompression of a polynomial from 11 bits per coefficient. + * + * Arguments: - int16_t *r: pointer to output polynomial + * - const uint8_t *a: pointer to input byte array + * (of length MLKEM_POLYCOMPRESSEDBYTES_D11) + * - const uint8_t *data: pointer to constants + * (shufbidx[0:32], srlvdidx[32:64], + * srlvqidx[64:96], shift[96:128]) + **************************************************/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + !defined(MLK_CONFIG_NO_DECAPS_API) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) +/*yaml + Name: poly_decompress_d11_avx2_asm + Description: x86_64 AVX2 polynomial decompression (d=11) + Signature: void mlk_poly_decompress_d11_avx2_asm(int16_t *r, const uint8_t *a, const uint8_t *data) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t *r + description: Output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 352 + permissions: read-only + c_parameter: const uint8_t *a + description: Input compressed polynomial + rdx: + type: buffer + size_bytes: 128 + permissions: read-only + c_parameter: const uint8_t *data + description: Precomputed decompression constants +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_poly_decompress_d11_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_decompress_d11_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_decompress_d11_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0x7ff07ff0, %eax # imm = 0x7FF07FF0 + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + vmovdqa (%rdx), %ymm2 + vmovdqa 0x20(%rdx), %ymm3 + vmovdqa 0x40(%rdx), %ymm4 + vmovdqa 0x60(%rdx), %ymm5 + vmovdqu (%rsi), %xmm6 + vmovd 0x10(%rsi), %xmm7 + vpinsrw $0x2, 0x14(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, (%rdi) + vmovdqu 0x16(%rsi), %xmm6 + vmovd 0x26(%rsi), %xmm7 + vpinsrw $0x2, 0x2a(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x20(%rdi) + vmovdqu 0x2c(%rsi), %xmm6 + vmovd 0x3c(%rsi), %xmm7 + vpinsrw $0x2, 0x40(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x40(%rdi) + vmovdqu 0x42(%rsi), %xmm6 + vmovd 0x52(%rsi), %xmm7 + vpinsrw $0x2, 0x56(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x60(%rdi) + vmovdqu 0x58(%rsi), %xmm6 + vmovd 0x68(%rsi), %xmm7 + vpinsrw $0x2, 0x6c(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x80(%rdi) + vmovdqu 0x6e(%rsi), %xmm6 + vmovd 0x7e(%rsi), %xmm7 + vpinsrw $0x2, 0x82(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0xa0(%rdi) + vmovdqu 0x84(%rsi), %xmm6 + vmovd 0x94(%rsi), %xmm7 + vpinsrw $0x2, 0x98(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0xc0(%rdi) + vmovdqu 0x9a(%rsi), %xmm6 + vmovd 0xaa(%rsi), %xmm7 + vpinsrw $0x2, 0xae(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0xe0(%rdi) + vmovdqu 0xb0(%rsi), %xmm6 + vmovd 0xc0(%rsi), %xmm7 + vpinsrw $0x2, 0xc4(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x100(%rdi) + vmovdqu 0xc6(%rsi), %xmm6 + vmovd 0xd6(%rsi), %xmm7 + vpinsrw $0x2, 0xda(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x120(%rdi) + vmovdqu 0xdc(%rsi), %xmm6 + vmovd 0xec(%rsi), %xmm7 + vpinsrw $0x2, 0xf0(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x140(%rdi) + vmovdqu 0xf2(%rsi), %xmm6 + vmovd 0x102(%rsi), %xmm7 + vpinsrw $0x2, 0x106(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x160(%rdi) + vmovdqu 0x108(%rsi), %xmm6 + vmovd 0x118(%rsi), %xmm7 + vpinsrw $0x2, 0x11c(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x180(%rdi) + vmovdqu 0x11e(%rsi), %xmm6 + vmovd 0x12e(%rsi), %xmm7 + vpinsrw $0x2, 0x132(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x1a0(%rdi) + vmovdqu 0x134(%rsi), %xmm6 + vmovd 0x144(%rsi), %xmm7 + vpinsrw $0x2, 0x148(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x1c0(%rdi) + vmovdqu 0x14a(%rsi), %xmm6 + vmovd 0x15a(%rsi), %xmm7 + vpinsrw $0x2, 0x15e(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_decompress_d11_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && !MLK_CONFIG_NO_DECAPS_API && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || \ + MLKEM_K == 4) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d4_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d4_avx2_asm.S new file mode 100644 index 0000000000..fd9bbc87e1 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d4_avx2_asm.S @@ -0,0 +1,209 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +/************************************************* + * Name: mlk_poly_decompress_d4_avx2_asm + * + * Description: Decompression of a polynomial from 4 bits per coefficient. + * + * Arguments: - int16_t *r: pointer to output polynomial + * - const uint8_t *a: pointer to input byte array + * (of length MLKEM_POLYCOMPRESSEDBYTES_D4) + * - const int8_t *data: pointer to shufbidx constant + **************************************************/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + !defined(MLK_CONFIG_NO_DECAPS_API) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 || MLKEM_K == 3) +/*yaml + Name: poly_decompress_d4_avx2_asm + Description: x86_64 AVX2 polynomial decompression (d=4) + Signature: void mlk_poly_decompress_d4_avx2_asm(int16_t *r, const uint8_t *a, const uint8_t *data) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t *r + description: Output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 128 + permissions: read-only + c_parameter: const uint8_t *a + description: Input compressed polynomial + rdx: + type: buffer + size_bytes: 32 + permissions: read-only + c_parameter: const uint8_t *data + description: Precomputed decompression constants +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_poly_decompress_d4_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_decompress_d4_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_decompress_d4_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0xf0000f, %eax # imm = 0xF0000F + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + movl $0x800800, %eax # imm = 0x800800 + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + vmovdqa (%rdx), %ymm3 + vmovq (%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, (%rdi) + vmovq 0x8(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x20(%rdi) + vmovq 0x10(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x40(%rdi) + vmovq 0x18(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x60(%rdi) + vmovq 0x20(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x80(%rdi) + vmovq 0x28(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xa0(%rdi) + vmovq 0x30(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xc0(%rdi) + vmovq 0x38(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xe0(%rdi) + vmovq 0x40(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x100(%rdi) + vmovq 0x48(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x120(%rdi) + vmovq 0x50(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x140(%rdi) + vmovq 0x58(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x160(%rdi) + vmovq 0x60(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x180(%rdi) + vmovq 0x68(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1a0(%rdi) + vmovq 0x70(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1c0(%rdi) + vmovq 0x78(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_decompress_d4_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && !MLK_CONFIG_NO_DECAPS_API && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || \ + MLKEM_K == 2 || MLKEM_K == 3) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d5_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d5_avx2_asm.S new file mode 100644 index 0000000000..46aa87c8ec --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d5_avx2_asm.S @@ -0,0 +1,222 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +/************************************************* + * Name: mlk_poly_decompress_d5_avx2_asm + * + * Description: Decompression of a polynomial from 5 bits per coefficient. + * + * Arguments: - int16_t *r: pointer to output polynomial + * - const uint8_t *a: pointer to input byte array + * (of length MLKEM_POLYCOMPRESSEDBYTES_D5) + * - const uint8_t *data: pointer to constants + * (shufbidx[0:32], mask[32:64], shift[64:96]) + **************************************************/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + !defined(MLK_CONFIG_NO_DECAPS_API) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) +/*yaml + Name: poly_decompress_d5_avx2_asm + Description: x86_64 AVX2 polynomial decompression (d=5) + Signature: void mlk_poly_decompress_d5_avx2_asm(int16_t *r, const uint8_t *a, const uint8_t *data) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t *r + description: Output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 160 + permissions: read-only + c_parameter: const uint8_t *a + description: Input compressed polynomial + rdx: + type: buffer + size_bytes: 96 + permissions: read-only + c_parameter: const uint8_t *data + description: Precomputed decompression constants +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_poly_decompress_d5_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_decompress_d5_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_decompress_d5_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + vmovdqa (%rdx), %ymm1 + vmovdqa 0x20(%rdx), %ymm2 + vmovdqa 0x40(%rdx), %ymm3 + vmovq (%rsi), %xmm4 + vpinsrw $0x4, 0x8(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, (%rdi) + vmovq 0xa(%rsi), %xmm4 + vpinsrw $0x4, 0x12(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x20(%rdi) + vmovq 0x14(%rsi), %xmm4 + vpinsrw $0x4, 0x1c(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x40(%rdi) + vmovq 0x1e(%rsi), %xmm4 + vpinsrw $0x4, 0x26(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x60(%rdi) + vmovq 0x28(%rsi), %xmm4 + vpinsrw $0x4, 0x30(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x80(%rdi) + vmovq 0x32(%rsi), %xmm4 + vpinsrw $0x4, 0x3a(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xa0(%rdi) + vmovq 0x3c(%rsi), %xmm4 + vpinsrw $0x4, 0x44(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xc0(%rdi) + vmovq 0x46(%rsi), %xmm4 + vpinsrw $0x4, 0x4e(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xe0(%rdi) + vmovq 0x50(%rsi), %xmm4 + vpinsrw $0x4, 0x58(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x100(%rdi) + vmovq 0x5a(%rsi), %xmm4 + vpinsrw $0x4, 0x62(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x120(%rdi) + vmovq 0x64(%rsi), %xmm4 + vpinsrw $0x4, 0x6c(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x140(%rdi) + vmovq 0x6e(%rsi), %xmm4 + vpinsrw $0x4, 0x76(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x160(%rdi) + vmovq 0x78(%rsi), %xmm4 + vpinsrw $0x4, 0x80(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x180(%rdi) + vmovq 0x82(%rsi), %xmm4 + vpinsrw $0x4, 0x8a(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1a0(%rdi) + vmovq 0x8c(%rsi), %xmm4 + vpinsrw $0x4, 0x94(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1c0(%rdi) + vmovq 0x96(%rsi), %xmm4 + vpinsrw $0x4, 0x9e(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_decompress_d5_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && !MLK_CONFIG_NO_DECAPS_API && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || \ + MLKEM_K == 4) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_mulcache_compute_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_mulcache_compute_avx2_asm.S new file mode 100644 index 0000000000..23a095a732 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_mulcache_compute_avx2_asm.S @@ -0,0 +1,118 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/*yaml + Name: poly_mulcache_compute_avx2_asm + Description: x86_64 AVX2 mulcache computation + Signature: void mlk_poly_mulcache_compute_avx2_asm(int16_t *out, const int16_t *in, const int16_t *qdata) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 256 + permissions: write-only + c_parameter: int16_t *out + description: Output mulcache (128 x int16_t) + rsi: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t *in + description: Input polynomial (256 x int16_t) + rdx: + type: buffer + size_bytes: 1248 + permissions: read-only + c_parameter: const int16_t *qdata + description: Precomputed constants (624 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_poly_mulcache_compute_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_mulcache_compute_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_mulcache_compute_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + vmovdqa 0x20(%rsi), %ymm2 + vmovdqa 0x60(%rsi), %ymm3 + vmovdqa 0x3e0(%rdx), %ymm4 + vmovdqa 0x460(%rdx), %ymm1 + vpmullw %ymm2, %ymm1, %ymm5 + vpmullw %ymm3, %ymm1, %ymm6 + vpmulhw %ymm2, %ymm4, %ymm7 + vpmulhw %ymm3, %ymm4, %ymm8 + vpmulhw %ymm5, %ymm0, %ymm9 + vpmulhw %ymm6, %ymm0, %ymm10 + vpsubw %ymm9, %ymm7, %ymm7 + vpsubw %ymm10, %ymm8, %ymm8 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm8, 0x20(%rdi) + vmovdqa 0xa0(%rsi), %ymm2 + vmovdqa 0xe0(%rsi), %ymm3 + vmovdqa 0x400(%rdx), %ymm4 + vmovdqa 0x480(%rdx), %ymm1 + vpmullw %ymm2, %ymm1, %ymm5 + vpmullw %ymm3, %ymm1, %ymm6 + vpmulhw %ymm2, %ymm4, %ymm7 + vpmulhw %ymm3, %ymm4, %ymm8 + vpmulhw %ymm5, %ymm0, %ymm9 + vpmulhw %ymm6, %ymm0, %ymm10 + vpsubw %ymm9, %ymm7, %ymm7 + vpsubw %ymm10, %ymm8, %ymm8 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm8, 0x60(%rdi) + vmovdqa 0x120(%rsi), %ymm2 + vmovdqa 0x160(%rsi), %ymm3 + vmovdqa 0x420(%rdx), %ymm4 + vmovdqa 0x4a0(%rdx), %ymm1 + vpmullw %ymm2, %ymm1, %ymm5 + vpmullw %ymm3, %ymm1, %ymm6 + vpmulhw %ymm2, %ymm4, %ymm7 + vpmulhw %ymm3, %ymm4, %ymm8 + vpmulhw %ymm5, %ymm0, %ymm9 + vpmulhw %ymm6, %ymm0, %ymm10 + vpsubw %ymm9, %ymm7, %ymm7 + vpsubw %ymm10, %ymm8, %ymm8 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm8, 0xa0(%rdi) + vmovdqa 0x1a0(%rsi), %ymm2 + vmovdqa 0x1e0(%rsi), %ymm3 + vmovdqa 0x440(%rdx), %ymm4 + vmovdqa 0x4c0(%rdx), %ymm1 + vpmullw %ymm2, %ymm1, %ymm5 + vpmullw %ymm3, %ymm1, %ymm6 + vpmulhw %ymm2, %ymm4, %ymm7 + vpmulhw %ymm3, %ymm4, %ymm8 + vpmulhw %ymm5, %ymm0, %ymm9 + vpmulhw %ymm6, %ymm0, %ymm10 + vpsubw %ymm9, %ymm7, %ymm7 + vpsubw %ymm10, %ymm8, %ymm8 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm8, 0xe0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_mulcache_compute_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm.S new file mode 100644 index 0000000000..5f4b812684 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm.S @@ -0,0 +1,536 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2) +/*yaml + Name: polyvec_basemul_acc_montgomery_cached_k2_avx2_asm + Description: x86_64 AVX2 base multiplication with accumulation (k=2) + Signature: void mlk_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm(int16_t *r, const int16_t *a, const int16_t *b, const int16_t *b_cache) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int16_t *a + description: Input polyvec a (2 x 256 x int16_t) + rdx: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int16_t *b + description: Input polyvec b (2 x 256 x int16_t) + rcx: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t *b_cache + description: Mulcache for b (2 x 128 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k2_avx2_asm) +MLK_ASM_FN_SYMBOL(polyvec_basemul_acc_montgomery_cached_k2_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0xf301f301, %eax # imm = 0xF301F301 + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + vmovdqa (%rsi), %ymm2 + vmovdqa 0x20(%rsi), %ymm3 + vmovdqa (%rdx), %ymm4 + vmovdqa 0x20(%rdx), %ymm5 + vmovdqa (%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x40(%rsi), %ymm2 + vmovdqa 0x60(%rsi), %ymm3 + vmovdqa 0x40(%rdx), %ymm4 + vmovdqa 0x60(%rdx), %ymm5 + vmovdqa 0x20(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x80(%rsi), %ymm2 + vmovdqa 0xa0(%rsi), %ymm3 + vmovdqa 0x80(%rdx), %ymm4 + vmovdqa 0xa0(%rdx), %ymm5 + vmovdqa 0x40(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0xc0(%rsi), %ymm2 + vmovdqa 0xe0(%rsi), %ymm3 + vmovdqa 0xc0(%rdx), %ymm4 + vmovdqa 0xe0(%rdx), %ymm5 + vmovdqa 0x60(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x100(%rsi), %ymm2 + vmovdqa 0x120(%rsi), %ymm3 + vmovdqa 0x100(%rdx), %ymm4 + vmovdqa 0x120(%rdx), %ymm5 + vmovdqa 0x80(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x140(%rsi), %ymm2 + vmovdqa 0x160(%rsi), %ymm3 + vmovdqa 0x140(%rdx), %ymm4 + vmovdqa 0x160(%rdx), %ymm5 + vmovdqa 0xa0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x180(%rsi), %ymm2 + vmovdqa 0x1a0(%rsi), %ymm3 + vmovdqa 0x180(%rdx), %ymm4 + vmovdqa 0x1a0(%rdx), %ymm5 + vmovdqa 0xc0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x1c0(%rsi), %ymm2 + vmovdqa 0x1e0(%rsi), %ymm3 + vmovdqa 0x1c0(%rdx), %ymm4 + vmovdqa 0x1e0(%rdx), %ymm5 + vmovdqa 0xe0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + vmovdqa 0x200(%rsi), %ymm2 + vmovdqa 0x220(%rsi), %ymm3 + vmovdqa 0x200(%rdx), %ymm4 + vmovdqa 0x220(%rdx), %ymm5 + vmovdqa 0x100(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa (%rdi), %ymm8 + vmovdqa 0x20(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x240(%rsi), %ymm2 + vmovdqa 0x260(%rsi), %ymm3 + vmovdqa 0x240(%rdx), %ymm4 + vmovdqa 0x260(%rdx), %ymm5 + vmovdqa 0x120(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x40(%rdi), %ymm8 + vmovdqa 0x60(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x280(%rsi), %ymm2 + vmovdqa 0x2a0(%rsi), %ymm3 + vmovdqa 0x280(%rdx), %ymm4 + vmovdqa 0x2a0(%rdx), %ymm5 + vmovdqa 0x140(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0x2c0(%rsi), %ymm2 + vmovdqa 0x2e0(%rsi), %ymm3 + vmovdqa 0x2c0(%rdx), %ymm4 + vmovdqa 0x2e0(%rdx), %ymm5 + vmovdqa 0x160(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0xc0(%rdi), %ymm8 + vmovdqa 0xe0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x300(%rsi), %ymm2 + vmovdqa 0x320(%rsi), %ymm3 + vmovdqa 0x300(%rdx), %ymm4 + vmovdqa 0x320(%rdx), %ymm5 + vmovdqa 0x180(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x100(%rdi), %ymm8 + vmovdqa 0x120(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x340(%rsi), %ymm2 + vmovdqa 0x360(%rsi), %ymm3 + vmovdqa 0x340(%rdx), %ymm4 + vmovdqa 0x360(%rdx), %ymm5 + vmovdqa 0x1a0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x140(%rdi), %ymm8 + vmovdqa 0x160(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x380(%rsi), %ymm2 + vmovdqa 0x3a0(%rsi), %ymm3 + vmovdqa 0x380(%rdx), %ymm4 + vmovdqa 0x3a0(%rdx), %ymm5 + vmovdqa 0x1c0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x3c0(%rsi), %ymm2 + vmovdqa 0x3e0(%rsi), %ymm3 + vmovdqa 0x3c0(%rdx), %ymm4 + vmovdqa 0x3e0(%rdx), %ymm5 + vmovdqa 0x1e0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x1c0(%rdi), %ymm8 + vmovdqa 0x1e0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(polyvec_basemul_acc_montgomery_cached_k2_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm.S new file mode 100644 index 0000000000..838eb16bed --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm.S @@ -0,0 +1,784 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3) +/*yaml + Name: polyvec_basemul_acc_montgomery_cached_k3_avx2_asm + Description: x86_64 AVX2 base multiplication with accumulation (k=3) + Signature: void mlk_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm(int16_t *r, const int16_t *a, const int16_t *b, const int16_t *b_cache) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 1536 + permissions: read-only + c_parameter: const int16_t *a + description: Input polyvec a (3 x 256 x int16_t) + rdx: + type: buffer + size_bytes: 1536 + permissions: read-only + c_parameter: const int16_t *b + description: Input polyvec b (3 x 256 x int16_t) + rcx: + type: buffer + size_bytes: 768 + permissions: read-only + c_parameter: const int16_t *b_cache + description: Mulcache for b (3 x 128 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k3_avx2_asm) +MLK_ASM_FN_SYMBOL(polyvec_basemul_acc_montgomery_cached_k3_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0xf301f301, %eax # imm = 0xF301F301 + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + vmovdqa (%rsi), %ymm2 + vmovdqa 0x20(%rsi), %ymm3 + vmovdqa (%rdx), %ymm4 + vmovdqa 0x20(%rdx), %ymm5 + vmovdqa (%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x40(%rsi), %ymm2 + vmovdqa 0x60(%rsi), %ymm3 + vmovdqa 0x40(%rdx), %ymm4 + vmovdqa 0x60(%rdx), %ymm5 + vmovdqa 0x20(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x80(%rsi), %ymm2 + vmovdqa 0xa0(%rsi), %ymm3 + vmovdqa 0x80(%rdx), %ymm4 + vmovdqa 0xa0(%rdx), %ymm5 + vmovdqa 0x40(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0xc0(%rsi), %ymm2 + vmovdqa 0xe0(%rsi), %ymm3 + vmovdqa 0xc0(%rdx), %ymm4 + vmovdqa 0xe0(%rdx), %ymm5 + vmovdqa 0x60(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x100(%rsi), %ymm2 + vmovdqa 0x120(%rsi), %ymm3 + vmovdqa 0x100(%rdx), %ymm4 + vmovdqa 0x120(%rdx), %ymm5 + vmovdqa 0x80(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x140(%rsi), %ymm2 + vmovdqa 0x160(%rsi), %ymm3 + vmovdqa 0x140(%rdx), %ymm4 + vmovdqa 0x160(%rdx), %ymm5 + vmovdqa 0xa0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x180(%rsi), %ymm2 + vmovdqa 0x1a0(%rsi), %ymm3 + vmovdqa 0x180(%rdx), %ymm4 + vmovdqa 0x1a0(%rdx), %ymm5 + vmovdqa 0xc0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x1c0(%rsi), %ymm2 + vmovdqa 0x1e0(%rsi), %ymm3 + vmovdqa 0x1c0(%rdx), %ymm4 + vmovdqa 0x1e0(%rdx), %ymm5 + vmovdqa 0xe0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + vmovdqa 0x200(%rsi), %ymm2 + vmovdqa 0x220(%rsi), %ymm3 + vmovdqa 0x200(%rdx), %ymm4 + vmovdqa 0x220(%rdx), %ymm5 + vmovdqa 0x100(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa (%rdi), %ymm8 + vmovdqa 0x20(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x240(%rsi), %ymm2 + vmovdqa 0x260(%rsi), %ymm3 + vmovdqa 0x240(%rdx), %ymm4 + vmovdqa 0x260(%rdx), %ymm5 + vmovdqa 0x120(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x40(%rdi), %ymm8 + vmovdqa 0x60(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x280(%rsi), %ymm2 + vmovdqa 0x2a0(%rsi), %ymm3 + vmovdqa 0x280(%rdx), %ymm4 + vmovdqa 0x2a0(%rdx), %ymm5 + vmovdqa 0x140(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0x2c0(%rsi), %ymm2 + vmovdqa 0x2e0(%rsi), %ymm3 + vmovdqa 0x2c0(%rdx), %ymm4 + vmovdqa 0x2e0(%rdx), %ymm5 + vmovdqa 0x160(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0xc0(%rdi), %ymm8 + vmovdqa 0xe0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x300(%rsi), %ymm2 + vmovdqa 0x320(%rsi), %ymm3 + vmovdqa 0x300(%rdx), %ymm4 + vmovdqa 0x320(%rdx), %ymm5 + vmovdqa 0x180(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x100(%rdi), %ymm8 + vmovdqa 0x120(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x340(%rsi), %ymm2 + vmovdqa 0x360(%rsi), %ymm3 + vmovdqa 0x340(%rdx), %ymm4 + vmovdqa 0x360(%rdx), %ymm5 + vmovdqa 0x1a0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x140(%rdi), %ymm8 + vmovdqa 0x160(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x380(%rsi), %ymm2 + vmovdqa 0x3a0(%rsi), %ymm3 + vmovdqa 0x380(%rdx), %ymm4 + vmovdqa 0x3a0(%rdx), %ymm5 + vmovdqa 0x1c0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x3c0(%rsi), %ymm2 + vmovdqa 0x3e0(%rsi), %ymm3 + vmovdqa 0x3c0(%rdx), %ymm4 + vmovdqa 0x3e0(%rdx), %ymm5 + vmovdqa 0x1e0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x1c0(%rdi), %ymm8 + vmovdqa 0x1e0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + vmovdqa 0x400(%rsi), %ymm2 + vmovdqa 0x420(%rsi), %ymm3 + vmovdqa 0x400(%rdx), %ymm4 + vmovdqa 0x420(%rdx), %ymm5 + vmovdqa 0x200(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa (%rdi), %ymm8 + vmovdqa 0x20(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x440(%rsi), %ymm2 + vmovdqa 0x460(%rsi), %ymm3 + vmovdqa 0x440(%rdx), %ymm4 + vmovdqa 0x460(%rdx), %ymm5 + vmovdqa 0x220(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x40(%rdi), %ymm8 + vmovdqa 0x60(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x480(%rsi), %ymm2 + vmovdqa 0x4a0(%rsi), %ymm3 + vmovdqa 0x480(%rdx), %ymm4 + vmovdqa 0x4a0(%rdx), %ymm5 + vmovdqa 0x240(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0x4c0(%rsi), %ymm2 + vmovdqa 0x4e0(%rsi), %ymm3 + vmovdqa 0x4c0(%rdx), %ymm4 + vmovdqa 0x4e0(%rdx), %ymm5 + vmovdqa 0x260(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0xc0(%rdi), %ymm8 + vmovdqa 0xe0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x500(%rsi), %ymm2 + vmovdqa 0x520(%rsi), %ymm3 + vmovdqa 0x500(%rdx), %ymm4 + vmovdqa 0x520(%rdx), %ymm5 + vmovdqa 0x280(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x100(%rdi), %ymm8 + vmovdqa 0x120(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x540(%rsi), %ymm2 + vmovdqa 0x560(%rsi), %ymm3 + vmovdqa 0x540(%rdx), %ymm4 + vmovdqa 0x560(%rdx), %ymm5 + vmovdqa 0x2a0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x140(%rdi), %ymm8 + vmovdqa 0x160(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x580(%rsi), %ymm2 + vmovdqa 0x5a0(%rsi), %ymm3 + vmovdqa 0x580(%rdx), %ymm4 + vmovdqa 0x5a0(%rdx), %ymm5 + vmovdqa 0x2c0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x5c0(%rsi), %ymm2 + vmovdqa 0x5e0(%rsi), %ymm3 + vmovdqa 0x5c0(%rdx), %ymm4 + vmovdqa 0x5e0(%rdx), %ymm5 + vmovdqa 0x2e0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x1c0(%rdi), %ymm8 + vmovdqa 0x1e0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(polyvec_basemul_acc_montgomery_cached_k3_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm.S new file mode 100644 index 0000000000..314d938f75 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm.S @@ -0,0 +1,1032 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) +/*yaml + Name: polyvec_basemul_acc_montgomery_cached_k4_avx2_asm + Description: x86_64 AVX2 base multiplication with accumulation (k=4) + Signature: void mlk_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm(int16_t *r, const int16_t *a, const int16_t *b, const int16_t *b_cache) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const int16_t *a + description: Input polyvec a (4 x 256 x int16_t) + rdx: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const int16_t *b + description: Input polyvec b (4 x 256 x int16_t) + rcx: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int16_t *b_cache + description: Mulcache for b (4 x 128 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k4_avx2_asm) +MLK_ASM_FN_SYMBOL(polyvec_basemul_acc_montgomery_cached_k4_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0xf301f301, %eax # imm = 0xF301F301 + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + vmovdqa (%rsi), %ymm2 + vmovdqa 0x20(%rsi), %ymm3 + vmovdqa (%rdx), %ymm4 + vmovdqa 0x20(%rdx), %ymm5 + vmovdqa (%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x40(%rsi), %ymm2 + vmovdqa 0x60(%rsi), %ymm3 + vmovdqa 0x40(%rdx), %ymm4 + vmovdqa 0x60(%rdx), %ymm5 + vmovdqa 0x20(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x80(%rsi), %ymm2 + vmovdqa 0xa0(%rsi), %ymm3 + vmovdqa 0x80(%rdx), %ymm4 + vmovdqa 0xa0(%rdx), %ymm5 + vmovdqa 0x40(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0xc0(%rsi), %ymm2 + vmovdqa 0xe0(%rsi), %ymm3 + vmovdqa 0xc0(%rdx), %ymm4 + vmovdqa 0xe0(%rdx), %ymm5 + vmovdqa 0x60(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x100(%rsi), %ymm2 + vmovdqa 0x120(%rsi), %ymm3 + vmovdqa 0x100(%rdx), %ymm4 + vmovdqa 0x120(%rdx), %ymm5 + vmovdqa 0x80(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x140(%rsi), %ymm2 + vmovdqa 0x160(%rsi), %ymm3 + vmovdqa 0x140(%rdx), %ymm4 + vmovdqa 0x160(%rdx), %ymm5 + vmovdqa 0xa0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x180(%rsi), %ymm2 + vmovdqa 0x1a0(%rsi), %ymm3 + vmovdqa 0x180(%rdx), %ymm4 + vmovdqa 0x1a0(%rdx), %ymm5 + vmovdqa 0xc0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x1c0(%rsi), %ymm2 + vmovdqa 0x1e0(%rsi), %ymm3 + vmovdqa 0x1c0(%rdx), %ymm4 + vmovdqa 0x1e0(%rdx), %ymm5 + vmovdqa 0xe0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + vmovdqa 0x200(%rsi), %ymm2 + vmovdqa 0x220(%rsi), %ymm3 + vmovdqa 0x200(%rdx), %ymm4 + vmovdqa 0x220(%rdx), %ymm5 + vmovdqa 0x100(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa (%rdi), %ymm8 + vmovdqa 0x20(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x240(%rsi), %ymm2 + vmovdqa 0x260(%rsi), %ymm3 + vmovdqa 0x240(%rdx), %ymm4 + vmovdqa 0x260(%rdx), %ymm5 + vmovdqa 0x120(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x40(%rdi), %ymm8 + vmovdqa 0x60(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x280(%rsi), %ymm2 + vmovdqa 0x2a0(%rsi), %ymm3 + vmovdqa 0x280(%rdx), %ymm4 + vmovdqa 0x2a0(%rdx), %ymm5 + vmovdqa 0x140(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0x2c0(%rsi), %ymm2 + vmovdqa 0x2e0(%rsi), %ymm3 + vmovdqa 0x2c0(%rdx), %ymm4 + vmovdqa 0x2e0(%rdx), %ymm5 + vmovdqa 0x160(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0xc0(%rdi), %ymm8 + vmovdqa 0xe0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x300(%rsi), %ymm2 + vmovdqa 0x320(%rsi), %ymm3 + vmovdqa 0x300(%rdx), %ymm4 + vmovdqa 0x320(%rdx), %ymm5 + vmovdqa 0x180(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x100(%rdi), %ymm8 + vmovdqa 0x120(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x340(%rsi), %ymm2 + vmovdqa 0x360(%rsi), %ymm3 + vmovdqa 0x340(%rdx), %ymm4 + vmovdqa 0x360(%rdx), %ymm5 + vmovdqa 0x1a0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x140(%rdi), %ymm8 + vmovdqa 0x160(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x380(%rsi), %ymm2 + vmovdqa 0x3a0(%rsi), %ymm3 + vmovdqa 0x380(%rdx), %ymm4 + vmovdqa 0x3a0(%rdx), %ymm5 + vmovdqa 0x1c0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x3c0(%rsi), %ymm2 + vmovdqa 0x3e0(%rsi), %ymm3 + vmovdqa 0x3c0(%rdx), %ymm4 + vmovdqa 0x3e0(%rdx), %ymm5 + vmovdqa 0x1e0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x1c0(%rdi), %ymm8 + vmovdqa 0x1e0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + vmovdqa 0x400(%rsi), %ymm2 + vmovdqa 0x420(%rsi), %ymm3 + vmovdqa 0x400(%rdx), %ymm4 + vmovdqa 0x420(%rdx), %ymm5 + vmovdqa 0x200(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa (%rdi), %ymm8 + vmovdqa 0x20(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x440(%rsi), %ymm2 + vmovdqa 0x460(%rsi), %ymm3 + vmovdqa 0x440(%rdx), %ymm4 + vmovdqa 0x460(%rdx), %ymm5 + vmovdqa 0x220(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x40(%rdi), %ymm8 + vmovdqa 0x60(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x480(%rsi), %ymm2 + vmovdqa 0x4a0(%rsi), %ymm3 + vmovdqa 0x480(%rdx), %ymm4 + vmovdqa 0x4a0(%rdx), %ymm5 + vmovdqa 0x240(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0x4c0(%rsi), %ymm2 + vmovdqa 0x4e0(%rsi), %ymm3 + vmovdqa 0x4c0(%rdx), %ymm4 + vmovdqa 0x4e0(%rdx), %ymm5 + vmovdqa 0x260(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0xc0(%rdi), %ymm8 + vmovdqa 0xe0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x500(%rsi), %ymm2 + vmovdqa 0x520(%rsi), %ymm3 + vmovdqa 0x500(%rdx), %ymm4 + vmovdqa 0x520(%rdx), %ymm5 + vmovdqa 0x280(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x100(%rdi), %ymm8 + vmovdqa 0x120(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x540(%rsi), %ymm2 + vmovdqa 0x560(%rsi), %ymm3 + vmovdqa 0x540(%rdx), %ymm4 + vmovdqa 0x560(%rdx), %ymm5 + vmovdqa 0x2a0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x140(%rdi), %ymm8 + vmovdqa 0x160(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x580(%rsi), %ymm2 + vmovdqa 0x5a0(%rsi), %ymm3 + vmovdqa 0x580(%rdx), %ymm4 + vmovdqa 0x5a0(%rdx), %ymm5 + vmovdqa 0x2c0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x5c0(%rsi), %ymm2 + vmovdqa 0x5e0(%rsi), %ymm3 + vmovdqa 0x5c0(%rdx), %ymm4 + vmovdqa 0x5e0(%rdx), %ymm5 + vmovdqa 0x2e0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x1c0(%rdi), %ymm8 + vmovdqa 0x1e0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + vmovdqa 0x600(%rsi), %ymm2 + vmovdqa 0x620(%rsi), %ymm3 + vmovdqa 0x600(%rdx), %ymm4 + vmovdqa 0x620(%rdx), %ymm5 + vmovdqa 0x300(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa (%rdi), %ymm8 + vmovdqa 0x20(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x640(%rsi), %ymm2 + vmovdqa 0x660(%rsi), %ymm3 + vmovdqa 0x640(%rdx), %ymm4 + vmovdqa 0x660(%rdx), %ymm5 + vmovdqa 0x320(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x40(%rdi), %ymm8 + vmovdqa 0x60(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x680(%rsi), %ymm2 + vmovdqa 0x6a0(%rsi), %ymm3 + vmovdqa 0x680(%rdx), %ymm4 + vmovdqa 0x6a0(%rdx), %ymm5 + vmovdqa 0x340(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0x6c0(%rsi), %ymm2 + vmovdqa 0x6e0(%rsi), %ymm3 + vmovdqa 0x6c0(%rdx), %ymm4 + vmovdqa 0x6e0(%rdx), %ymm5 + vmovdqa 0x360(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0xc0(%rdi), %ymm8 + vmovdqa 0xe0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x700(%rsi), %ymm2 + vmovdqa 0x720(%rsi), %ymm3 + vmovdqa 0x700(%rdx), %ymm4 + vmovdqa 0x720(%rdx), %ymm5 + vmovdqa 0x380(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x100(%rdi), %ymm8 + vmovdqa 0x120(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x740(%rsi), %ymm2 + vmovdqa 0x760(%rsi), %ymm3 + vmovdqa 0x740(%rdx), %ymm4 + vmovdqa 0x760(%rdx), %ymm5 + vmovdqa 0x3a0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x140(%rdi), %ymm8 + vmovdqa 0x160(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x780(%rsi), %ymm2 + vmovdqa 0x7a0(%rsi), %ymm3 + vmovdqa 0x780(%rdx), %ymm4 + vmovdqa 0x7a0(%rdx), %ymm5 + vmovdqa 0x3c0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x7c0(%rsi), %ymm2 + vmovdqa 0x7e0(%rsi), %ymm3 + vmovdqa 0x7c0(%rdx), %ymm4 + vmovdqa 0x7e0(%rdx), %ymm5 + vmovdqa 0x3e0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x1c0(%rdi), %ymm8 + vmovdqa 0x1e0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(polyvec_basemul_acc_montgomery_cached_k4_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_reduce_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_reduce_avx2_asm.S new file mode 100644 index 0000000000..4b5edf0b92 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_reduce_avx2_asm.S @@ -0,0 +1,234 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + * + * Changes: + * - Add call to csub in reduce128_avx to produce outputs + * in [0,1,...,q-1] rather than [0,1,...,q], matching the + * semantics of mlk_poly_reduce(), + * - Use a macro instead of a local function call. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/*yaml + Name: reduce_avx2_asm + Description: x86_64 AVX2 modular reduction + Signature: void mlk_reduce_avx2_asm(int16_t *r) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_reduce_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(reduce_avx2_asm) +MLK_ASM_FN_SYMBOL(reduce_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + vmovdqa (%rdi), %ymm2 + vmovdqa 0x20(%rdi), %ymm3 + vmovdqa 0x40(%rdi), %ymm4 + vmovdqa 0x60(%rdi), %ymm5 + vmovdqa 0x80(%rdi), %ymm6 + vmovdqa 0xa0(%rdi), %ymm7 + vmovdqa 0xc0(%rdi), %ymm8 + vmovdqa 0xe0(%rdi), %ymm9 + vpmulhw %ymm1, %ymm2, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm2, %ymm2 + vpmulhw %ymm1, %ymm3, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm3, %ymm3 + vpmulhw %ymm1, %ymm4, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm4, %ymm4 + vpmulhw %ymm1, %ymm5, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm5, %ymm5 + vpmulhw %ymm1, %ymm6, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm6, %ymm6 + vpmulhw %ymm1, %ymm7, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm7, %ymm7 + vpmulhw %ymm1, %ymm8, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm8, %ymm8 + vpmulhw %ymm1, %ymm9, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm9, %ymm9 + vpsubw %ymm0, %ymm2, %ymm2 + vpsraw $0xf, %ymm2, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm2, %ymm2 + vpsubw %ymm0, %ymm3, %ymm3 + vpsraw $0xf, %ymm3, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm3, %ymm3 + vpsubw %ymm0, %ymm4, %ymm4 + vpsraw $0xf, %ymm4, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm4, %ymm4 + vpsubw %ymm0, %ymm5, %ymm5 + vpsraw $0xf, %ymm5, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm5, %ymm5 + vpsubw %ymm0, %ymm6, %ymm6 + vpsraw $0xf, %ymm6, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm6, %ymm6 + vpsubw %ymm0, %ymm7, %ymm7 + vpsraw $0xf, %ymm7, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm7, %ymm7 + vpsubw %ymm0, %ymm8, %ymm8 + vpsraw $0xf, %ymm8, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm8, %ymm8 + vpsubw %ymm0, %ymm9, %ymm9 + vpsraw $0xf, %ymm9, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm9, %ymm9 + vmovdqa %ymm2, (%rdi) + vmovdqa %ymm3, 0x20(%rdi) + vmovdqa %ymm4, 0x40(%rdi) + vmovdqa %ymm5, 0x60(%rdi) + vmovdqa %ymm6, 0x80(%rdi) + vmovdqa %ymm7, 0xa0(%rdi) + vmovdqa %ymm8, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x100(%rdi), %ymm2 + vmovdqa 0x120(%rdi), %ymm3 + vmovdqa 0x140(%rdi), %ymm4 + vmovdqa 0x160(%rdi), %ymm5 + vmovdqa 0x180(%rdi), %ymm6 + vmovdqa 0x1a0(%rdi), %ymm7 + vmovdqa 0x1c0(%rdi), %ymm8 + vmovdqa 0x1e0(%rdi), %ymm9 + vpmulhw %ymm1, %ymm2, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm2, %ymm2 + vpmulhw %ymm1, %ymm3, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm3, %ymm3 + vpmulhw %ymm1, %ymm4, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm4, %ymm4 + vpmulhw %ymm1, %ymm5, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm5, %ymm5 + vpmulhw %ymm1, %ymm6, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm6, %ymm6 + vpmulhw %ymm1, %ymm7, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm7, %ymm7 + vpmulhw %ymm1, %ymm8, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm8, %ymm8 + vpmulhw %ymm1, %ymm9, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm9, %ymm9 + vpsubw %ymm0, %ymm2, %ymm2 + vpsraw $0xf, %ymm2, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm2, %ymm2 + vpsubw %ymm0, %ymm3, %ymm3 + vpsraw $0xf, %ymm3, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm3, %ymm3 + vpsubw %ymm0, %ymm4, %ymm4 + vpsraw $0xf, %ymm4, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm4, %ymm4 + vpsubw %ymm0, %ymm5, %ymm5 + vpsraw $0xf, %ymm5, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm5, %ymm5 + vpsubw %ymm0, %ymm6, %ymm6 + vpsraw $0xf, %ymm6, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm6, %ymm6 + vpsubw %ymm0, %ymm7, %ymm7 + vpsraw $0xf, %ymm7, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm7, %ymm7 + vpsubw %ymm0, %ymm8, %ymm8 + vpsraw $0xf, %ymm8, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm8, %ymm8 + vpsubw %ymm0, %ymm9, %ymm9 + vpsraw $0xf, %ymm9, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm9, %ymm9 + vmovdqa %ymm2, 0x100(%rdi) + vmovdqa %ymm3, 0x120(%rdi) + vmovdqa %ymm4, 0x140(%rdi) + vmovdqa %ymm5, 0x160(%rdi) + vmovdqa %ymm6, 0x180(%rdi) + vmovdqa %ymm7, 0x1a0(%rdi) + vmovdqa %ymm8, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(reduce_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_rej_uniform_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_rej_uniform_avx2_asm.S new file mode 100644 index 0000000000..789c876e1a --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_rej_uniform_avx2_asm.S @@ -0,0 +1,136 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/************************************************* + * Name: mlk_rej_uniform_avx2_asm + * + * Description: Run rejection sampling on uniform random bytes to generate + * uniform random integers mod q + * + * Arguments: - int16_t *r: pointer to output buffer of MLKEM_N + * 16-bit coefficients. + * - const uint8_t *buf: pointer to input buffer + * (assumed to be uniform random bytes) + * - unsigned buflen: length of input buffer in bytes. + * Must be a multiple of 12. + * + * Returns number of sampled 16-bit integers (at most MLKEM_N). + **************************************************/ +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/*yaml + Name: rej_uniform_avx2_asm + Description: x86_64 AVX2 rejection sampling + Signature: uint64_t mlk_rej_uniform_avx2_asm(int16_t *r, const uint8_t *buf, unsigned buflen, const uint8_t *table) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t *r + description: Output buffer (256 x int16_t) + rsi: + type: buffer + size_bytes: rdx + permissions: read-only + c_parameter: const uint8_t *buf + description: Input buffer + rdx: + type: scalar + c_parameter: unsigned buflen + description: Length of input buffer (must be multiple of 12) + test_with: 504 # MLKEM_GEN_MATRIX_NBLOCKS * MLK_XOF_RATE + rcx: + type: buffer + size_bytes: 4096 + permissions: read-only + c_parameter: const uint8_t *table + description: Lookup table +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_rej_uniform_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(rej_uniform_avx2_asm) +MLK_ASM_FN_SYMBOL(rej_uniform_avx2_asm) + + .cfi_startproc + subq $0x210, %rsp # imm = 0x210 + .cfi_adjust_cfa_offset 0x210 + xorl %eax, %eax + testq %rdx, %rdx + je Lmlk_rej_uniform_asm_end + movabsq $0xd010d010d010d01, %rax # imm = 0xD010D010D010D01 + movq %rax, %xmm0 + pinsrq $0x1, %rax, %xmm0 + movabsq $0xfff0fff0fff0fff, %rax # imm = 0xFFF0FFF0FFF0FFF + movq %rax, %xmm5 + pinsrq $0x1, %rax, %xmm5 + movabsq $0x504040302010100, %rax # imm = 0x504040302010100 + movq %rax, %xmm4 + movabsq $0xb0a0a0908070706, %rax # imm = 0xB0A0A0908070706 + pinsrq $0x1, %rax, %xmm4 + movq $0x0, %rax + movq $0x0, %r8 + movq $0x5555, %r9 # imm = 0x5555 + +Lmlk_rej_uniform_asm_loop_start: + movq (%rsi,%r8), %xmm2 + pinsrd $0x2, 0x8(%rsi,%r8), %xmm2 + pshufb %xmm4, %xmm2 + movdqa %xmm2, %xmm3 + psrlw $0x4, %xmm3 + pblendw $0xaa, %xmm3, %xmm2 # xmm2 = xmm2[0],xmm3[1],xmm2[2],xmm3[3],xmm2[4],xmm3[5],xmm2[6],xmm3[7] + pand %xmm5, %xmm2 + movdqa %xmm0, %xmm1 + pcmpgtw %xmm2, %xmm1 + pmovmskb %xmm1, %r11d + pextq %r9, %r11, %r11 + movq %r11, %r10 + shlq $0x4, %r10 + movdqu (%rcx,%r10), %xmm3 + pshufb %xmm3, %xmm2 + movdqu %xmm2, (%rsp,%rax,2) + popcntq %r11, %r11 + addq %r11, %rax + cmpq $0x100, %rax # imm = 0x100 + jae Lmlk_rej_uniform_asm_final_copy + addq $0xc, %r8 + cmpq %r8, %rdx + ja Lmlk_rej_uniform_asm_loop_start + +Lmlk_rej_uniform_asm_final_copy: + movq $0x100, %rcx # imm = 0x100 + cmpq $0x100, %rax # imm = 0x100 + cmovaq %rcx, %rax + movq %rsp, %rsi + movq %rax, %rcx + shlq %rcx + rep movsb (%rsi), %es:(%rdi) + +Lmlk_rej_uniform_asm_end: + addq $0x210, %rsp # imm = 0x210 + .cfi_adjust_cfa_offset -0x210 + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(rej_uniform_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_tomont_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_tomont_avx2_asm.S new file mode 100644 index 0000000000..cf6268ba01 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_tomont_avx2_asm.S @@ -0,0 +1,172 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * Implementation from Kyber reference repository @[REF_AVX2] + * + * Changes: + * - Add call to csub in reduce128_avx to produce outputs + * in [0,1,...,q-1] rather than [0,1,...,q], matching the + * semantics of mlk_poly_reduce(), + * - Use a macro instead of a local function call. + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + !defined(MLK_CONFIG_NO_KEYPAIR_API) +/*yaml + Name: tomont_avx2_asm + Description: x86_64 AVX2 Montgomery conversion + Signature: void mlk_tomont_avx2_asm(int16_t *r) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_tomont_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(tomont_avx2_asm) +MLK_ASM_FN_SYMBOL(tomont_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0x50495049, %eax # imm = 0x50495049 + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + movl $0x5490549, %eax # imm = 0x5490549 + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + vmovdqa (%rdi), %ymm3 + vmovdqa 0x20(%rdi), %ymm4 + vmovdqa 0x40(%rdi), %ymm5 + vmovdqa 0x60(%rdi), %ymm6 + vmovdqa 0x80(%rdi), %ymm7 + vmovdqa 0xa0(%rdi), %ymm8 + vmovdqa 0xc0(%rdi), %ymm9 + vmovdqa 0xe0(%rdi), %ymm10 + vpmullw %ymm1, %ymm3, %ymm11 + vpmulhw %ymm2, %ymm3, %ymm3 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm11, %ymm3, %ymm3 + vpmullw %ymm1, %ymm4, %ymm12 + vpmulhw %ymm2, %ymm4, %ymm4 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm4, %ymm4 + vpmullw %ymm1, %ymm5, %ymm13 + vpmulhw %ymm2, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm13, %ymm13 + vpsubw %ymm13, %ymm5, %ymm5 + vpmullw %ymm1, %ymm6, %ymm14 + vpmulhw %ymm2, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm14, %ymm14 + vpsubw %ymm14, %ymm6, %ymm6 + vpmullw %ymm1, %ymm7, %ymm15 + vpmulhw %ymm2, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm15, %ymm15 + vpsubw %ymm15, %ymm7, %ymm7 + vpmullw %ymm1, %ymm8, %ymm11 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm11, %ymm8, %ymm8 + vpmullw %ymm1, %ymm9, %ymm12 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm9, %ymm9 + vpmullw %ymm1, %ymm10, %ymm13 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm13, %ymm13 + vpsubw %ymm13, %ymm10, %ymm10 + vmovdqa %ymm3, (%rdi) + vmovdqa %ymm4, 0x20(%rdi) + vmovdqa %ymm5, 0x40(%rdi) + vmovdqa %ymm6, 0x60(%rdi) + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm8, 0xa0(%rdi) + vmovdqa %ymm9, 0xc0(%rdi) + vmovdqa %ymm10, 0xe0(%rdi) + vmovdqa 0x100(%rdi), %ymm3 + vmovdqa 0x120(%rdi), %ymm4 + vmovdqa 0x140(%rdi), %ymm5 + vmovdqa 0x160(%rdi), %ymm6 + vmovdqa 0x180(%rdi), %ymm7 + vmovdqa 0x1a0(%rdi), %ymm8 + vmovdqa 0x1c0(%rdi), %ymm9 + vmovdqa 0x1e0(%rdi), %ymm10 + vpmullw %ymm1, %ymm3, %ymm11 + vpmulhw %ymm2, %ymm3, %ymm3 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm11, %ymm3, %ymm3 + vpmullw %ymm1, %ymm4, %ymm12 + vpmulhw %ymm2, %ymm4, %ymm4 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm4, %ymm4 + vpmullw %ymm1, %ymm5, %ymm13 + vpmulhw %ymm2, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm13, %ymm13 + vpsubw %ymm13, %ymm5, %ymm5 + vpmullw %ymm1, %ymm6, %ymm14 + vpmulhw %ymm2, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm14, %ymm14 + vpsubw %ymm14, %ymm6, %ymm6 + vpmullw %ymm1, %ymm7, %ymm15 + vpmulhw %ymm2, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm15, %ymm15 + vpsubw %ymm15, %ymm7, %ymm7 + vpmullw %ymm1, %ymm8, %ymm11 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm11, %ymm8, %ymm8 + vpmullw %ymm1, %ymm9, %ymm12 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm9, %ymm9 + vpmullw %ymm1, %ymm10, %ymm13 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm13, %ymm13 + vpsubw %ymm13, %ymm10, %ymm10 + vmovdqa %ymm3, 0x100(%rdi) + vmovdqa %ymm4, 0x120(%rdi) + vmovdqa %ymm5, 0x140(%rdi) + vmovdqa %ymm6, 0x160(%rdi) + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm8, 0x1a0(%rdi) + vmovdqa %ymm9, 0x1c0(%rdi) + vmovdqa %ymm10, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(tomont_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && !MLK_CONFIG_NO_KEYPAIR_API */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/rej_uniform_table.c b/src/mlkem-native/mlkem/src/native/x86_64/src/rej_uniform_table.c new file mode 100644 index 0000000000..d62256af5a --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/rej_uniform_table.c @@ -0,0 +1,545 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "arith_native_x86_64.h" + +/* + * Lookup table used by rejection sampling of the public matrix. + * See autogen for details. + */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_rej_uniform_table[4096] = { + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 0 */, + 0, 1, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 1 */, + 2, 3, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 2 */, + 0, 1, 2, 3, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 3 */, + 4, 5, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 4 */, + 0, 1, 4, 5, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 5 */, + 2, 3, 4, 5, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 6 */, + 0, 1, 2, 3, 4, 5, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 7 */, + 6, 7, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 8 */, + 0, 1, 6, 7, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 9 */, + 2, 3, 6, 7, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 10 */, + 0, 1, 2, 3, 6, 7, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 11 */, + 4, 5, 6, 7, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 12 */, + 0, 1, 4, 5, 6, 7, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 13 */, + 2, 3, 4, 5, 6, 7, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 14 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 255, 255, 255, 255, 255, 255, 255, 255 /* 15 */, + 8, 9, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 16 */, + 0, 1, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 17 */, + 2, 3, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 18 */, + 0, 1, 2, 3, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 19 */, + 4, 5, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 20 */, + 0, 1, 4, 5, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 21 */, + 2, 3, 4, 5, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 22 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 255, 255, 255, 255, 255, 255, 255, 255 /* 23 */, + 6, 7, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 24 */, + 0, 1, 6, 7, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 25 */, + 2, 3, 6, 7, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 26 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 255, 255, 255, 255, 255, 255, 255, 255 /* 27 */, + 4, 5, 6, 7, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 28 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 255, 255, 255, 255, 255, 255, 255, 255 /* 29 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 255, 255, 255, 255, 255, 255, 255, 255 /* 30 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 255, 255, 255, 255, 255, 255 /* 31 */, + 10, 11, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 32 */, + 0, 1, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 33 */, + 2, 3, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 34 */, + 0, 1, 2, 3, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 35 */, + 4, 5, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 36 */, + 0, 1, 4, 5, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 37 */, + 2, 3, 4, 5, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 38 */, + 0, 1, 2, 3, 4, 5, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 39 */, + 6, 7, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 40 */, + 0, 1, 6, 7, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 41 */, + 2, 3, 6, 7, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 42 */, + 0, 1, 2, 3, 6, 7, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 43 */, + 4, 5, 6, 7, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 44 */, + 0, 1, 4, 5, 6, 7, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 45 */, + 2, 3, 4, 5, 6, 7, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 46 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 255, 255, 255, 255, 255, 255 /* 47 */, + 8, 9, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 48 */, + 0, 1, 8, 9, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 49 */, + 2, 3, 8, 9, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 50 */, + 0, 1, 2, 3, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 51 */, + 4, 5, 8, 9, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 52 */, + 0, 1, 4, 5, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 53 */, + 2, 3, 4, 5, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 54 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 255, 255, 255, 255, 255, 255 /* 55 */, + 6, 7, 8, 9, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 56 */, + 0, 1, 6, 7, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 57 */, + 2, 3, 6, 7, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 58 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 255, 255, 255, 255, 255, 255 /* 59 */, + 4, 5, 6, 7, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 60 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 255, 255, 255, 255, 255, 255 /* 61 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 255, 255, 255, 255, 255, 255 /* 62 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 255, 255, 255, 255 /* 63 */, + 12, 13, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 64 */, + 0, 1, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 65 */, + 2, 3, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 66 */, + 0, 1, 2, 3, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 67 */, + 4, 5, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 68 */, + 0, 1, 4, 5, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 69 */, + 2, 3, 4, 5, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 70 */, + 0, 1, 2, 3, 4, 5, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 71 */, + 6, 7, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 72 */, + 0, 1, 6, 7, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 73 */, + 2, 3, 6, 7, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 74 */, + 0, 1, 2, 3, 6, 7, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 75 */, + 4, 5, 6, 7, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 76 */, + 0, 1, 4, 5, 6, 7, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 77 */, + 2, 3, 4, 5, 6, 7, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 78 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 12, 13, 255, 255, 255, 255, 255, 255 /* 79 */, + 8, 9, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 80 */, + 0, 1, 8, 9, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 81 */, + 2, 3, 8, 9, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 82 */, + 0, 1, 2, 3, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 83 */, + 4, 5, 8, 9, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 84 */, + 0, 1, 4, 5, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 85 */, + 2, 3, 4, 5, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 86 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 12, 13, 255, 255, 255, 255, 255, 255 /* 87 */, + 6, 7, 8, 9, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 88 */, + 0, 1, 6, 7, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 89 */, + 2, 3, 6, 7, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 90 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 12, 13, 255, 255, 255, 255, 255, 255 /* 91 */, + 4, 5, 6, 7, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 92 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 12, 13, 255, 255, 255, 255, 255, 255 /* 93 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 12, 13, 255, 255, 255, 255, 255, 255 /* 94 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 12, 13, 255, 255, 255, 255 /* 95 */, + 10, 11, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 96 */, + 0, 1, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 97 */, + 2, 3, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 98 */, + 0, 1, 2, 3, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 99 */, + 4, 5, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 100 */, + 0, 1, 4, 5, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 101 */, + 2, 3, 4, 5, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 102 */, + 0, 1, 2, 3, 4, 5, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 103 */, + 6, 7, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 104 */, + 0, 1, 6, 7, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 105 */, + 2, 3, 6, 7, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 106 */, + 0, 1, 2, 3, 6, 7, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 107 */, + 4, 5, 6, 7, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 108 */, + 0, 1, 4, 5, 6, 7, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 109 */, + 2, 3, 4, 5, 6, 7, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 110 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 12, 13, 255, 255, 255, 255 /* 111 */, + 8, 9, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 112 */, + 0, 1, 8, 9, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 113 */, + 2, 3, 8, 9, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 114 */, + 0, 1, 2, 3, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 115 */, + 4, 5, 8, 9, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 116 */, + 0, 1, 4, 5, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 117 */, + 2, 3, 4, 5, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 118 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 12, 13, 255, 255, 255, 255 /* 119 */, + 6, 7, 8, 9, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 120 */, + 0, 1, 6, 7, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 121 */, + 2, 3, 6, 7, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 122 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 12, 13, 255, 255, 255, 255 /* 123 */, + 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 124 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 255, 255, 255, 255 /* 125 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 255, 255, 255, 255 /* 126 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 255, 255 /* 127 */, + 14, 15, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 128 */, + 0, 1, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 129 */, + 2, 3, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 130 */, + 0, 1, 2, 3, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 131 */, + 4, 5, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 132 */, + 0, 1, 4, 5, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 133 */, + 2, 3, 4, 5, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 134 */, + 0, 1, 2, 3, 4, 5, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 135 */, + 6, 7, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 136 */, + 0, 1, 6, 7, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 137 */, + 2, 3, 6, 7, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 138 */, + 0, 1, 2, 3, 6, 7, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 139 */, + 4, 5, 6, 7, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 140 */, + 0, 1, 4, 5, 6, 7, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 141 */, + 2, 3, 4, 5, 6, 7, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 142 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 14, 15, 255, 255, 255, 255, 255, 255 /* 143 */, + 8, 9, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 144 */, + 0, 1, 8, 9, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 145 */, + 2, 3, 8, 9, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 146 */, + 0, 1, 2, 3, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 147 */, + 4, 5, 8, 9, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 148 */, + 0, 1, 4, 5, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 149 */, + 2, 3, 4, 5, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 150 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 14, 15, 255, 255, 255, 255, 255, 255 /* 151 */, + 6, 7, 8, 9, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 152 */, + 0, 1, 6, 7, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 153 */, + 2, 3, 6, 7, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 154 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 14, 15, 255, 255, 255, 255, 255, 255 /* 155 */, + 4, 5, 6, 7, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 156 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 14, 15, 255, 255, 255, 255, 255, 255 /* 157 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 14, 15, 255, 255, 255, 255, 255, 255 /* 158 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 14, 15, 255, 255, 255, 255 /* 159 */, + 10, 11, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 160 */, + 0, 1, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 161 */, + 2, 3, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 162 */, + 0, 1, 2, 3, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 163 */, + 4, 5, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 164 */, + 0, 1, 4, 5, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 165 */, + 2, 3, 4, 5, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 166 */, + 0, 1, 2, 3, 4, 5, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 167 */, + 6, 7, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 168 */, + 0, 1, 6, 7, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 169 */, + 2, 3, 6, 7, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 170 */, + 0, 1, 2, 3, 6, 7, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 171 */, + 4, 5, 6, 7, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 172 */, + 0, 1, 4, 5, 6, 7, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 173 */, + 2, 3, 4, 5, 6, 7, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 174 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 14, 15, 255, 255, 255, 255 /* 175 */, + 8, 9, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 176 */, + 0, 1, 8, 9, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 177 */, + 2, 3, 8, 9, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 178 */, + 0, 1, 2, 3, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 179 */, + 4, 5, 8, 9, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 180 */, + 0, 1, 4, 5, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 181 */, + 2, 3, 4, 5, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 182 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 14, 15, 255, 255, 255, 255 /* 183 */, + 6, 7, 8, 9, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 184 */, + 0, 1, 6, 7, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 185 */, + 2, 3, 6, 7, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 186 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 14, 15, 255, 255, 255, 255 /* 187 */, + 4, 5, 6, 7, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 188 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 14, 15, 255, 255, 255, 255 /* 189 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 14, 15, 255, 255, 255, 255 /* 190 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 14, 15, 255, 255 /* 191 */, + 12, 13, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 192 */, + 0, 1, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 193 */, + 2, 3, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 194 */, + 0, 1, 2, 3, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 195 */, + 4, 5, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 196 */, + 0, 1, 4, 5, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 197 */, + 2, 3, 4, 5, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 198 */, + 0, 1, 2, 3, 4, 5, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 199 */, + 6, 7, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 200 */, + 0, 1, 6, 7, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 201 */, + 2, 3, 6, 7, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 202 */, + 0, 1, 2, 3, 6, 7, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 203 */, + 4, 5, 6, 7, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 204 */, + 0, 1, 4, 5, 6, 7, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 205 */, + 2, 3, 4, 5, 6, 7, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 206 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 12, 13, 14, 15, 255, 255, 255, 255 /* 207 */, + 8, 9, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 208 */, + 0, 1, 8, 9, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 209 */, + 2, 3, 8, 9, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 210 */, + 0, 1, 2, 3, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 211 */, + 4, 5, 8, 9, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 212 */, + 0, 1, 4, 5, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 213 */, + 2, 3, 4, 5, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 214 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 12, 13, 14, 15, 255, 255, 255, 255 /* 215 */, + 6, 7, 8, 9, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 216 */, + 0, 1, 6, 7, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 217 */, + 2, 3, 6, 7, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 218 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 12, 13, 14, 15, 255, 255, 255, 255 /* 219 */, + 4, 5, 6, 7, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 220 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 12, 13, 14, 15, 255, 255, 255, 255 /* 221 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 12, 13, 14, 15, 255, 255, 255, 255 /* 222 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 12, 13, 14, 15, 255, 255 /* 223 */, + 10, 11, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 224 */, + 0, 1, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 225 */, + 2, 3, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 226 */, + 0, 1, 2, 3, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 227 */, + 4, 5, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 228 */, + 0, 1, 4, 5, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 229 */, + 2, 3, 4, 5, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 230 */, + 0, 1, 2, 3, 4, 5, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 231 */, + 6, 7, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 232 */, + 0, 1, 6, 7, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 233 */, + 2, 3, 6, 7, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 234 */, + 0, 1, 2, 3, 6, 7, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 235 */, + 4, 5, 6, 7, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 236 */, + 0, 1, 4, 5, 6, 7, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 237 */, + 2, 3, 4, 5, 6, 7, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 238 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 12, 13, 14, 15, 255, 255 /* 239 */, + 8, 9, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 240 */, + 0, 1, 8, 9, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 241 */, + 2, 3, 8, 9, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 242 */, + 0, 1, 2, 3, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 243 */, + 4, 5, 8, 9, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 244 */, + 0, 1, 4, 5, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 245 */, + 2, 3, 4, 5, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 246 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 12, 13, 14, 15, 255, 255 /* 247 */, + 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 248 */, + 0, 1, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 249 */, + 2, 3, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 250 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 255, 255 /* 251 */, + 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 252 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 255, 255 /* 253 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 255, 255 /* 254 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15 /* 255 */, +}; + +#else /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +MLK_EMPTY_CU(avx2_rej_uniform_table) + +#endif /* !(MLK_ARITH_BACKEND_X86_64_DEFAULT && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/src/mlkem-native/mlkem/src/params.h b/src/mlkem-native/mlkem/src/params.h new file mode 100644 index 0000000000..04598539c4 --- /dev/null +++ b/src/mlkem-native/mlkem/src/params.h @@ -0,0 +1,76 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_PARAMS_H +#define MLK_PARAMS_H + +#if !defined(MLK_CONFIG_PARAMETER_SET) +#error MLK_CONFIG_PARAMETER_SET is not defined +#endif + +#if MLK_CONFIG_PARAMETER_SET == 512 +#define MLKEM_K 2 +#elif MLK_CONFIG_PARAMETER_SET == 768 +#define MLKEM_K 3 +#elif MLK_CONFIG_PARAMETER_SET == 1024 +#define MLKEM_K 4 +#else +#error Invalid value for MLK_CONFIG_PARAMETER_SET. Must be 512, 768, or 1024. +#endif + +#define MLKEM_N 256 +#define MLKEM_Q 3329 +#define MLKEM_Q_HALF ((MLKEM_Q + 1) / 2) /* 1665 */ +#define MLKEM_UINT12_LIMIT 4096 + +#define MLKEM_SYMBYTES 32 /* size in bytes of hashes, and seeds */ +#define MLKEM_SSBYTES 32 /* size in bytes of shared key */ + +#define MLKEM_POLYBYTES 384 +#define MLKEM_POLYVECBYTES (MLKEM_K * MLKEM_POLYBYTES) + +#define MLKEM_POLYCOMPRESSEDBYTES_D4 128 +#define MLKEM_POLYCOMPRESSEDBYTES_D5 160 +#define MLKEM_POLYCOMPRESSEDBYTES_D10 320 +#define MLKEM_POLYCOMPRESSEDBYTES_D11 352 + +#if MLKEM_K == 2 +#define MLKEM_ETA1 3 +#define MLKEM_DU 10 +#define MLKEM_DV 4 +#define MLKEM_POLYCOMPRESSEDBYTES_DV MLKEM_POLYCOMPRESSEDBYTES_D4 +#define MLKEM_POLYCOMPRESSEDBYTES_DU MLKEM_POLYCOMPRESSEDBYTES_D10 +#define MLKEM_POLYVECCOMPRESSEDBYTES_DU (MLKEM_K * MLKEM_POLYCOMPRESSEDBYTES_DU) +#elif MLKEM_K == 3 +#define MLKEM_ETA1 2 +#define MLKEM_DU 10 +#define MLKEM_DV 4 +#define MLKEM_POLYCOMPRESSEDBYTES_DV MLKEM_POLYCOMPRESSEDBYTES_D4 +#define MLKEM_POLYCOMPRESSEDBYTES_DU MLKEM_POLYCOMPRESSEDBYTES_D10 +#define MLKEM_POLYVECCOMPRESSEDBYTES_DU (MLKEM_K * MLKEM_POLYCOMPRESSEDBYTES_DU) +#elif MLKEM_K == 4 +#define MLKEM_ETA1 2 +#define MLKEM_DU 11 +#define MLKEM_DV 5 +#define MLKEM_POLYCOMPRESSEDBYTES_DV MLKEM_POLYCOMPRESSEDBYTES_D5 +#define MLKEM_POLYCOMPRESSEDBYTES_DU MLKEM_POLYCOMPRESSEDBYTES_D11 +#define MLKEM_POLYVECCOMPRESSEDBYTES_DU (MLKEM_K * MLKEM_POLYCOMPRESSEDBYTES_DU) +#endif /* MLKEM_K == 4 */ + +#define MLKEM_ETA2 2 + +#define MLKEM_INDCPA_MSGBYTES (MLKEM_SYMBYTES) +#define MLKEM_INDCPA_PUBLICKEYBYTES (MLKEM_POLYVECBYTES + MLKEM_SYMBYTES) +#define MLKEM_INDCPA_SECRETKEYBYTES (MLKEM_POLYVECBYTES) +#define MLKEM_INDCPA_BYTES \ + (MLKEM_POLYVECCOMPRESSEDBYTES_DU + MLKEM_POLYCOMPRESSEDBYTES_DV) + +#define MLKEM_INDCCA_PUBLICKEYBYTES (MLKEM_INDCPA_PUBLICKEYBYTES) +/* 32 bytes of additional space to save H(pk) */ +#define MLKEM_INDCCA_SECRETKEYBYTES \ + (MLKEM_INDCPA_SECRETKEYBYTES + MLKEM_INDCPA_PUBLICKEYBYTES + \ + 2 * MLKEM_SYMBYTES) +#define MLKEM_INDCCA_CIPHERTEXTBYTES (MLKEM_INDCPA_BYTES) + +#endif /* !MLK_PARAMS_H */ diff --git a/src/mlkem-native/mlkem/src/poly.c b/src/mlkem-native/mlkem/src/poly.c new file mode 100644 index 0000000000..edc17a2b18 --- /dev/null +++ b/src/mlkem-native/mlkem/src/poly.c @@ -0,0 +1,581 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [NeonNTT] + * Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1 + * Becker, Hwang, Kannwischer, Yang, Yang + * https://eprint.iacr.org/2021/986 + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + */ + +#include "common.h" +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + + +#include "cbmc.h" +#include "debug.h" +#include "poly.h" +#include "sampling.h" +#include "symmetric.h" +#include "verify.h" + +/** + * Montgomery multiplication modulo MLKEM_Q. + * + * @reference{`fqmul()` in the reference implementation @[REF].} + * + * @param a First factor. Can be any int16_t. + * @param b Second factor. Must be signed canonical + * (abs value < (MLKEM_Q+1)/2). + * + * @return 16-bit integer congruent to a*b*R^{-1} mod MLKEM_Q, and + * smaller than MLKEM_Q in absolute value. + */ +static MLK_INLINE int16_t mlk_fqmul(int16_t a, int16_t b) +__contract__( + requires(b > -MLKEM_Q_HALF && b < MLKEM_Q_HALF) + ensures(return_value > -MLKEM_Q && return_value < MLKEM_Q) +) +{ + int16_t res; + mlk_assert_abs_bound(&b, 1, MLKEM_Q_HALF); + + res = mlk_montgomery_reduce((int32_t)a * (int32_t)b); + /* Bounds: + * |res| <= ceil(|a| * |b| / 2^16) + (MLKEM_Q + 1) / 2 + * <= ceil(2^15 * ((MLKEM_Q - 1)/2) / 2^16) + (MLKEM_Q + 1) / 2 + * <= ceil((MLKEM_Q - 1) / 4) + (MLKEM_Q + 1) / 2 + * < MLKEM_Q + */ + + mlk_assert_abs_bound(&res, 1, MLKEM_Q); + return res; +} + +/** + * Barrett reduction; given a 16-bit integer a, computes the centered + * representative congruent to a mod MLKEM_Q in [-(MLKEM_Q-1)/2, (MLKEM_Q-1)/2]. + * + * @reference{`barrett_reduce()` in the reference implementation @[REF].} + * + * @param a Input integer to be reduced. + * + * @return Integer in [-(MLKEM_Q-1)/2, (MLKEM_Q-1)/2] congruent to @p a modulo + * MLKEM_Q. + */ +static MLK_INLINE int16_t mlk_barrett_reduce(int16_t a) +__contract__( + ensures(return_value > -MLKEM_Q_HALF && return_value < MLKEM_Q_HALF) +) +{ + /* Barrett reduction approximates + * ``` + * round(a/MLKEM_Q) + * = round(a*(2^N/MLKEM_Q))/2^N) + * ~= round(a*round(2^N/MLKEM_Q)/2^N) + * ``` + * Here, we pick N=26. + */ + const int32_t magic = 20159; /* check-magic: 20159 == round(2^26 / MLKEM_Q) */ + + /* + * PORTABILITY: Right-shift on a signed integer is + * implementation-defined for negative left argument. + * Here, we assume it's sign-preserving "arithmetic" shift right. + * See (C99 6.5.7 (5)) + */ + const int32_t t = (magic * a + ((int32_t)1 << 25)) >> 26; + + /* + * t is in -10 .. +10, so we need 32-bit math to + * evaluate t * MLKEM_Q and the subsequent subtraction + */ + int16_t res = (int16_t)(a - t * MLKEM_Q); + + mlk_assert_abs_bound(&res, 1, MLKEM_Q_HALF); + return res; +} + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +/* Reference: `poly_tomont()` in the reference implementation @[REF]. */ +MLK_STATIC_TESTABLE void mlk_poly_tomont_c(mlk_poly *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_abs_bound(r->coeffs, 0, MLKEM_N, MLKEM_Q)) +) +{ + unsigned i; + const int16_t f = 1353; /* check-magic: 1353 == signed_mod(2^32, MLKEM_Q) */ + for (i = 0; i < MLKEM_N; i++) + __loop__( + invariant(i <= MLKEM_N) + invariant(array_abs_bound(r->coeffs, 0, i, MLKEM_Q)) + decreases(MLKEM_N - i)) + { + r->coeffs[i] = mlk_fqmul(r->coeffs[i], f); + } + + mlk_assert_abs_bound(r, MLKEM_N, MLKEM_Q); +} + +MLK_INTERNAL_API +void mlk_poly_tomont(mlk_poly *r) +{ +#if defined(MLK_USE_NATIVE_POLY_TOMONT) + int ret; + ret = mlk_poly_tomont_native(r->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + mlk_assert_abs_bound(r, MLKEM_N, MLKEM_Q); + return; + } +#endif /* MLK_USE_NATIVE_POLY_TOMONT */ + + mlk_poly_tomont_c(r); +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +/** + * Constant-time conversion of signed representatives modulo MLKEM_Q within + * range [-(MLKEM_Q-1), MLKEM_Q-1] into unsigned representatives within + * range [0, MLKEM_Q-1]. + * + * @reference{Not present in the reference implementation @[REF]. Used here + * to implement different semantics of `poly_reduce()`; see below. In the + * reference implementation @[REF] this logic is part of all compression + * functions (see `compress.c`).} + * + * @param c Signed coefficient to be converted. + * + * @return Unsigned representative in [0, MLKEM_Q). + */ +static MLK_INLINE int16_t mlk_scalar_signed_to_unsigned_q(int16_t c) +__contract__( + requires(c > -MLKEM_Q && c < MLKEM_Q) + ensures(return_value >= 0 && return_value < MLKEM_Q) + ensures(return_value == (int32_t)c + (((int32_t)c < 0) * MLKEM_Q))) +{ + mlk_assert_abs_bound(&c, 1, MLKEM_Q); + + /* Add MLKEM_Q if c is negative, but in constant time. + * + * Note that c + MLKEM_Q does not overflow in int16_t, + * so the cast to uint16_t is safe. */ + c = mlk_ct_sel_int16((int16_t)(c + MLKEM_Q), c, mlk_ct_cmask_neg_i16(c)); + + mlk_assert_bound(&c, 1, 0, MLKEM_Q); + return c; +} + +/* Reference: `poly_reduce()` in the reference implementation @[REF] + * - We use _unsigned_ canonical outputs, while the reference + * implementation uses _signed_ canonical outputs. + * Accordingly, we need a conditional addition of MLKEM_Q + * here to go from signed to unsigned representatives. + * This conditional addition is then dropped from all + * polynomial compression functions instead (see `compress.c`). */ +MLK_STATIC_TESTABLE void mlk_poly_reduce_c(mlk_poly *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ + unsigned i; + + for (i = 0; i < MLKEM_N; i++) + __loop__( + invariant(i <= MLKEM_N) + invariant(array_bound(r->coeffs, 0, i, 0, MLKEM_Q)) + decreases(MLKEM_N - i)) + { + /* Barrett reduction, giving signed canonical representative */ + int16_t t = mlk_barrett_reduce(r->coeffs[i]); + /* Conditional addition to get unsigned canonical representative */ + r->coeffs[i] = mlk_scalar_signed_to_unsigned_q(t); + } + + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); +} + +MLK_INTERNAL_API +void mlk_poly_reduce(mlk_poly *r) +{ +#if defined(MLK_USE_NATIVE_POLY_REDUCE) + int ret; + ret = mlk_poly_reduce_native(r->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); + return; + } +#endif /* MLK_USE_NATIVE_POLY_REDUCE */ + + mlk_poly_reduce_c(r); +} + +/* Reference: `poly_add()` in the reference implementation @[REF]. + * - We use destructive version (output=first input) to avoid + * reasoning about aliasing in the CBMC specification */ +MLK_INTERNAL_API +void mlk_poly_add(mlk_poly *r, const mlk_poly *b) +{ + unsigned i; + for (i = 0; i < MLKEM_N; i++) + __loop__( + invariant(i <= MLKEM_N) + invariant(forall(k0, i, MLKEM_N, r->coeffs[k0] == loop_entry(*r).coeffs[k0])) + invariant(forall(k1, 0, i, r->coeffs[k1] == loop_entry(*r).coeffs[k1] + b->coeffs[k1])) + decreases(MLKEM_N - i)) + { + /* The preconditions imply that the addition stays within int16_t. */ + r->coeffs[i] = (int16_t)(r->coeffs[i] + b->coeffs[i]); + } +} + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +/* Reference: `poly_sub()` in the reference implementation @[REF]. + * - We use destructive version (output=first input) to avoid + * reasoning about aliasing in the CBMC specification */ +MLK_INTERNAL_API +void mlk_poly_sub(mlk_poly *r, const mlk_poly *b) +{ + unsigned i; + for (i = 0; i < MLKEM_N; i++) + __loop__( + invariant(i <= MLKEM_N) + invariant(forall(k0, i, MLKEM_N, r->coeffs[k0] == loop_entry(*r).coeffs[k0])) + invariant(forall(k1, 0, i, r->coeffs[k1] == loop_entry(*r).coeffs[k1] - b->coeffs[k1])) + decreases(MLKEM_N - i)) + { + /* The preconditions imply that the subtraction stays within int16_t. */ + r->coeffs[i] = (int16_t)(r->coeffs[i] - b->coeffs[i]); + } +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#include "zetas.inc" + +/* Reference: Does not exist in the reference implementation @[REF]. + * - The reference implementation does not use a + * multiplication cache ('mulcache'). This idea originates + * from @[NeonNTT] and is used at the C level here. */ +MLK_STATIC_TESTABLE void mlk_poly_mulcache_compute_c(mlk_poly_mulcache *x, + const mlk_poly *a) +__contract__( + requires(memory_no_alias(x, sizeof(mlk_poly_mulcache))) + requires(memory_no_alias(a, sizeof(mlk_poly))) + assigns(memory_slice(x, sizeof(mlk_poly_mulcache))) +) +{ + unsigned i; + for (i = 0; i < MLKEM_N / 4; i++) + __loop__( + invariant(i <= MLKEM_N / 4) + invariant(array_abs_bound(x->coeffs, 0, 2 * i, MLKEM_Q)) + decreases(MLKEM_N / 4 - i)) + { + x->coeffs[2 * i + 0] = mlk_fqmul(a->coeffs[4 * i + 1], mlk_zetas[64 + i]); + /* The values in zeta table are <= MLKEM_Q in absolute value, + * so the negation in int16_t is safe. */ + x->coeffs[2 * i + 1] = + mlk_fqmul(a->coeffs[4 * i + 3], (int16_t)(-mlk_zetas[64 + i])); + } + + /* + * This bound is true for the C implementation, but not needed + * in the higher level bounds reasoning. It is thus omitted + * from the spec to not unnecessarily constrain native + * implementations, but checked here nonetheless. + */ + mlk_assert_abs_bound(x, MLKEM_N / 2, MLKEM_Q); +} + +MLK_INTERNAL_API +void mlk_poly_mulcache_compute(mlk_poly_mulcache *x, const mlk_poly *a) +{ +#if defined(MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE) + int ret; + ret = mlk_poly_mulcache_compute_native(x->coeffs, a->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE */ + + mlk_poly_mulcache_compute_c(x, a); +} + +/* + * Computes a block CT butterflies with a fixed twiddle factor, + * using Montgomery multiplication. + * Parameters: + * - r: Pointer to base of polynomial (_not_ the base of butterfly block) + * - root: Twiddle factor to use for the butterfly. This must be in + * Montgomery form and signed canonical. + * - start: Offset to the beginning of the butterfly block + * - len: Index difference between coefficients subject to a butterfly + * - bound: Ghost variable describing coefficient bound: Prior to `start`, + * coefficients must be bound by `bound + MLKEM_Q`. Post `start`, + * they must be bound by `bound`. + * When this function returns, output coefficients in the index range + * [start, start+2*len) have bound bumped to `bound + MLKEM_Q`. + * Example: + * - start=8, len=4 + * This would compute the following four butterflies + * 8 -- 12 + * 9 -- 13 + * 10 -- 14 + * 11 -- 15 + * - start=4, len=2 + * This would compute the following two butterflies + * 4 -- 6 + * 5 -- 7 + */ + +/* Reference: Embedded in `ntt()` in the reference implementation @[REF]. */ +static void mlk_ntt_butterfly_block(int16_t r[MLKEM_N], int16_t zeta, + unsigned start, unsigned len, + unsigned bound) +__contract__( + requires(start < MLKEM_N) + requires(1 <= len && len <= MLKEM_N / 2 && start + 2 * len <= MLKEM_N) + requires(0 <= bound && bound < INT16_MAX - MLKEM_Q) + requires(-MLKEM_Q_HALF < zeta && zeta < MLKEM_Q_HALF) + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(array_abs_bound(r, 0, start, bound + MLKEM_Q)) + requires(array_abs_bound(r, start, MLKEM_N, bound)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_abs_bound(r, 0, start + 2*len, bound + MLKEM_Q)) + ensures(array_abs_bound(r, start + 2 * len, MLKEM_N, bound))) +{ + /* `bound` is a ghost variable only needed in the CBMC specification */ + unsigned j; + ((void)bound); + for (j = start; j < start + len; j++) + __loop__( + invariant(start <= j && j <= start + len) + /* + * Coefficients are updated in strided pairs, so the bounds for the + * intermediate states alternate twice between the old and new bound + */ + invariant(array_abs_bound(r, 0, j, bound + MLKEM_Q)) + invariant(array_abs_bound(r, j, start + len, bound)) + invariant(array_abs_bound(r, start + len, j + len, bound + MLKEM_Q)) + invariant(array_abs_bound(r, j + len, MLKEM_N, bound)) + decreases(start + len - j)) + { + int16_t t; + t = mlk_fqmul(r[j + len], zeta); + /* The precondition implies that the arithmetic does not overflow. */ + r[j + len] = (int16_t)(r[j] - t); + r[j] = (int16_t)(r[j] + t); + } +} + +/* + * Compute one layer of forward NTT + * Parameters: + * - r: Pointer to base of polynomial + * - layer: Variable indicating which layer is being applied. + */ + +/* Reference: Embedded in `ntt()` in the reference implementation @[REF]. */ +static void mlk_ntt_layer(int16_t r[MLKEM_N], unsigned layer) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(1 <= layer && layer <= 7) + requires(array_abs_bound(r, 0, MLKEM_N, layer * MLKEM_Q)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_abs_bound(r, 0, MLKEM_N, (layer + 1) * MLKEM_Q))) +{ + unsigned start, k, len; + /* Twiddle factors for layer n are at indices 2^(n-1)..2^n-1. */ + k = 1u << (layer - 1); + len = (unsigned)MLKEM_N >> layer; + for (start = 0; start < MLKEM_N; start += 2 * len) + __loop__( + invariant(start < MLKEM_N + 2 * len) + invariant(k <= MLKEM_N / 2 && 2 * len * k == start + MLKEM_N) + invariant(array_abs_bound(r, 0, start, layer * MLKEM_Q + MLKEM_Q)) + invariant(array_abs_bound(r, start, MLKEM_N, layer * MLKEM_Q)) + decreases(MLKEM_N - start)) + { + int16_t zeta = mlk_zetas[k++]; + mlk_ntt_butterfly_block(r, zeta, start, len, layer * MLKEM_Q); + } +} + +/* + * Compute full forward NTT + * NOTE: This particular implementation satisfies a much tighter + * bound on the output coefficients (5*q) than the contractual one (8*q), + * but this is not needed in the calling code. Should we change the + * base multiplication strategy to require smaller NTT output bounds, + * the proof may need strengthening. + */ + +/* Reference: `ntt()` in the reference implementation @[REF]. + * - Iterate over `layer` instead of `len` in the outer loop + * to simplify computation of zeta index. */ +MLK_STATIC_TESTABLE void mlk_poly_ntt_c(mlk_poly *p) +__contract__( + requires(memory_no_alias(p, sizeof(mlk_poly))) + requires(array_abs_bound(p->coeffs, 0, MLKEM_N, MLKEM_Q)) + assigns(memory_slice(p, sizeof(mlk_poly))) + ensures(array_abs_bound(p->coeffs, 0, MLKEM_N, MLK_NTT_BOUND)) +) +{ + unsigned layer; + int16_t *r; + + mlk_assert_abs_bound(p, MLKEM_N, MLKEM_Q); + + r = p->coeffs; + + for (layer = 1; layer <= 7; layer++) + __loop__( + invariant(1 <= layer && layer <= 8) + invariant(array_abs_bound(r, 0, MLKEM_N, layer * MLKEM_Q)) + decreases(8 - layer)) + { + mlk_ntt_layer(r, layer); + } + + /* Check the stronger bound */ + mlk_assert_abs_bound(p, MLKEM_N, MLK_NTT_BOUND); +} + +MLK_INTERNAL_API +void mlk_poly_ntt(mlk_poly *r) +{ +#if defined(MLK_USE_NATIVE_NTT) + int ret; + mlk_assert_abs_bound(r, MLKEM_N, MLKEM_Q); + ret = mlk_ntt_native(r->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + mlk_assert_abs_bound(r, MLKEM_N, MLK_NTT_BOUND); + return; + } +#endif /* MLK_USE_NATIVE_NTT */ + + mlk_poly_ntt_c(r); +} + + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +/* Compute one layer of inverse NTT */ + +/* Reference: Embedded into `invntt()` in the reference implementation @[REF] */ +static void mlk_invntt_layer(int16_t *r, unsigned layer) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(1 <= layer && layer <= 7) + requires(array_abs_bound(r, 0, MLKEM_N, MLKEM_Q)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_abs_bound(r, 0, MLKEM_N, MLKEM_Q))) +{ + unsigned start, k, len; + len = (unsigned)MLKEM_N >> layer; + k = (1u << layer) - 1; + for (start = 0; start < MLKEM_N; start += 2 * len) + __loop__( + invariant(array_abs_bound(r, 0, MLKEM_N, MLKEM_Q)) + invariant(start <= MLKEM_N && k <= 127) + /* Normalised form of k == MLKEM_N / len - 1 - start / (2 * len) */ + invariant(2 * len * k + start == 2 * MLKEM_N - 2 * len) + decreases(MLKEM_N - start)) + { + unsigned j; + int16_t zeta = mlk_zetas[k--]; + for (j = start; j < start + len; j++) + __loop__( + invariant(start <= j && j <= start + len) + invariant(start <= MLKEM_N && k <= 127) + invariant(array_abs_bound(r, 0, MLKEM_N, MLKEM_Q)) + decreases(start + len - j)) + { + int16_t t = r[j]; + /* The preconditions imply that the arithmetic does not overflow. */ + r[j] = mlk_barrett_reduce((int16_t)(t + r[j + len])); + r[j + len] = (int16_t)(r[j + len] - t); + r[j + len] = mlk_fqmul(r[j + len], zeta); + } + } +} + +/* Reference: `invntt()` in the reference implementation @[REF] + * - We normalize at the beginning of the inverse NTT, + * while the reference implementation normalizes at + * the end. This allows us to drop a call to `poly_reduce()` + * from the base multiplication. */ +MLK_STATIC_TESTABLE void mlk_poly_invntt_tomont_c(mlk_poly *p) +__contract__( + requires(memory_no_alias(p, sizeof(mlk_poly))) + assigns(memory_slice(p, sizeof(mlk_poly))) + ensures(array_abs_bound(p->coeffs, 0, MLKEM_N, MLK_INVNTT_BOUND)) +) +{ + unsigned j, layer; + const int16_t f = 1441; /* check-magic: 1441 == pow(2,32 - 7,MLKEM_Q) */ + int16_t *r = p->coeffs; + + /* + * Scale input polynomial to account for Montgomery factor + * and NTT twist. This also brings coefficients down to + * absolute value < MLKEM_Q. + */ + for (j = 0; j < MLKEM_N; j++) + __loop__( + invariant(j <= MLKEM_N) + invariant(array_abs_bound(r, 0, j, MLKEM_Q)) + decreases(MLKEM_N - j)) + { + r[j] = mlk_fqmul(r[j], f); + } + + /* Run the invNTT layers */ + for (layer = 7; layer > 0; layer--) + __loop__( + invariant(0 <= layer && layer < 8) + invariant(array_abs_bound(r, 0, MLKEM_N, MLKEM_Q)) + decreases(layer)) + { + mlk_invntt_layer(r, layer); + } + + mlk_assert_abs_bound(p, MLKEM_N, MLK_INVNTT_BOUND); +} + +MLK_INTERNAL_API +void mlk_poly_invntt_tomont(mlk_poly *r) +{ +#if defined(MLK_USE_NATIVE_INTT) + int ret; + ret = mlk_intt_native(r->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + mlk_assert_abs_bound(r, MLKEM_N, MLK_INVNTT_BOUND); + return; + } +#endif /* MLK_USE_NATIVE_INTT */ + + mlk_poly_invntt_tomont_c(r); +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ +#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(mlk_poly) + +#endif /* MLK_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/src/mlkem-native/mlkem/src/poly.h b/src/mlkem-native/mlkem/src/poly.h new file mode 100644 index 0000000000..3def4394f1 --- /dev/null +++ b/src/mlkem-native/mlkem/src/poly.h @@ -0,0 +1,296 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_POLY_H +#define MLK_POLY_H + + +#include "cbmc.h" +#include "common.h" +#include "debug.h" +#include "verify.h" + +/* Absolute exclusive upper bound for the output of the inverse NTT */ +#define MLK_INVNTT_BOUND (8 * MLKEM_Q) + +/* Absolute exclusive upper bound for the output of the forward NTT */ +#define MLK_NTT_BOUND (8 * MLKEM_Q) + +/** + * Element of R_q = Z_q[X]/(X^n + 1). Represents polynomial + * coeffs[0] + X*coeffs[1] + X^2*coeffs[2] + ... + X^{n-1}*coeffs[n-1]. + */ +typedef struct +{ + int16_t coeffs[MLKEM_N]; /**< Polynomial coefficients. */ +} MLK_ALIGN mlk_poly; + +/** + * INTERNAL representation of precomputed data speeding up + * the base multiplication of two polynomials in NTT domain. + */ +typedef struct +{ + int16_t coeffs[MLKEM_N >> 1]; /**< Cached coefficients. */ +} MLK_ALIGN mlk_poly_mulcache; + +/** + * Generic Montgomery reduction; given a 32-bit integer a, computes a 16-bit + * integer congruent to a * R^-1 mod MLKEM_Q, where R=2^16. + * + * @param a Input integer to be reduced, of absolute value smaller or equal + * to INT32_MAX - 2^15 * MLKEM_Q. + * + * @return Integer congruent to a * R^-1 modulo MLKEM_Q, with absolute value + * <= ceil(|a| / 2^16) + (MLKEM_Q + 1)/2. + */ +static MLK_ALWAYS_INLINE int16_t mlk_montgomery_reduce(int32_t a) +__contract__( + requires(a < +(INT32_MAX - (((int32_t)1 << 15) * MLKEM_Q)) && + a > -(INT32_MAX - (((int32_t)1 << 15) * MLKEM_Q))) + /* We don't attempt to express an input-dependent output bound + * as the post-condition here. There are two call-sites for this + * function: + * - The base multiplication: Here, we need no output bound. + * - mlk_fqmul: Here, we inline this function and prove another spec + * for mlk_fqmul which does have a post-condition bound. */ +) +{ + /* check-magic: 62209 == unsigned_mod(pow(MLKEM_Q, -1, 2^16), 2^16) */ + const uint32_t QINV = 62209; + + /* Compute a*q^{-1} mod 2^16 in unsigned representatives. */ + const uint16_t a_reduced = mlk_cast_int32_to_uint16(a); + const uint16_t a_inverted = (a_reduced * QINV) & UINT16_MAX; + + /* Lift to signed canonical representative mod 2^16. */ + const int16_t t = mlk_cast_uint16_to_int16(a_inverted); + + int32_t r; + + mlk_assert(a < +(INT32_MAX - (((int32_t)1 << 15) * MLKEM_Q)) && + a > -(INT32_MAX - (((int32_t)1 << 15) * MLKEM_Q))); + + r = a - ((int32_t)t * MLKEM_Q); + + /* + * PORTABILITY: Right-shift on a signed integer is, strictly-speaking, + * implementation-defined for negative left argument. Here, + * we assume it's sign-preserving "arithmetic" shift right. (C99 6.5.7 (5)) + */ + r = r >> 16; + /* Bounds: |r >> 16| <= ceil(|r| / 2^16) + * <= ceil(|a| / 2^16 + MLKEM_Q / 2) + * <= ceil(|a| / 2^16) + (MLKEM_Q + 1) / 2 + * + * (Note that |a >> n| = ceil(|a| / 2^16) for negative a) + */ + return (int16_t)r; +} + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +#define mlk_poly_tomont MLK_NAMESPACE(poly_tomont) +/** + * In-place conversion of all coefficients of a polynomial from the normal + * domain to the Montgomery domain. + * + * Bounds: output < MLKEM_Q in absolute value. + * + * @spec{Internal normalization required in `mlk_indcpa_keypair_derand` as + * part of matrix-vector multiplication @[FIPS203, Algorithm 13, K-PKE.KeyGen, + * L18].} + * + * @param[in,out] r Input/output polynomial. + */ +MLK_INTERNAL_API +void mlk_poly_tomont(mlk_poly *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_abs_bound(r->coeffs, 0, MLKEM_N, MLKEM_Q)) +); +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +#define mlk_poly_mulcache_compute MLK_NAMESPACE(poly_mulcache_compute) +/** + * Compute the mulcache for a polynomial in NTT domain. + * + * The mulcache of a degree-2 polynomial b := b0 + b1*X in Fq[X]/(X^2-zeta) + * is the value b1*zeta, needed when computing products of b in + * Fq[X]/(X^2-zeta). + * + * The mulcache of a polynomial in NTT domain -- which is a 128-tuple of + * degree-2 polynomials in Fq[X]/(X^2-zeta), for varying zeta, is the + * 128-tuple of mulcaches of those polynomials. + * + * @spec{Caches `b_1 * \gamma` in @[FIPS203, Algorithm 12, BaseCaseMultiply, + * L1].} + * + * @param[out] x Mulcache to be populated. + * @param[in] a Input polynomial. + */ +/* + * NOTE: The default C implementation of this function populates + * the mulcache with values in (-q,q), but this is not needed for the + * higher level safety proofs, and thus not part of the spec. + */ +MLK_INTERNAL_API +void mlk_poly_mulcache_compute(mlk_poly_mulcache *x, const mlk_poly *a) +__contract__( + requires(memory_no_alias(x, sizeof(mlk_poly_mulcache))) + requires(memory_no_alias(a, sizeof(mlk_poly))) + assigns(memory_slice(x, sizeof(mlk_poly_mulcache))) +); + +#define mlk_poly_reduce MLK_NAMESPACE(poly_reduce) +/** + * Convert a polynomial to unsigned canonical representatives. + * + * The input coefficients can be arbitrary integers in int16_t. The output + * coefficients are in [0,1,..,MLKEM_Q-1]. + * + * @spec{Normalizes on unsigned canonical representatives ahead of calling + * @[FIPS203, Compress_d, Eq (4.7)]. This is not made explicit in FIPS 203.} + * + * @param[in,out] r Input/output polynomial. + */ +/* + * NOTE: The semantics of mlk_poly_reduce() is different in + * the reference implementation, which requires + * signed canonical output data. Unsigned canonical + * outputs are better suited to the only remaining + * use of mlk_poly_reduce() in the context of (de)serialization. + */ +MLK_INTERNAL_API +void mlk_poly_reduce(mlk_poly *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +); + +#define mlk_poly_add MLK_NAMESPACE(poly_add) +/** + * Add two polynomials in place. + * + * The coefficients of @p r and @p b must be such that the addition does not + * overflow. Otherwise, the behaviour of this function is undefined. + * + * @spec{@[FIPS203, 2.4.5, Arithmetic With Polynomials and NTT + * Representations]. Used in @[FIPS203, Algorithm 14 (K-PKE.Encrypt), L21].} + * + * @param[in,out] r Input-output polynomial to be added to. + * @param[in] b Input polynomial that should be added to @p r. Must be + * disjoint from @p r. + */ +/* + * NOTE: The reference implementation uses a 3-argument mlk_poly_add. + * We specialize to the accumulator form to avoid reasoning about aliasing. + */ +MLK_INTERNAL_API +void mlk_poly_add(mlk_poly *r, const mlk_poly *b) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(b, sizeof(mlk_poly))) + requires(forall(k0, 0, MLKEM_N, (int32_t) r->coeffs[k0] + b->coeffs[k0] <= INT16_MAX)) + requires(forall(k1, 0, MLKEM_N, (int32_t) r->coeffs[k1] + b->coeffs[k1] >= INT16_MIN)) + ensures(forall(k, 0, MLKEM_N, r->coeffs[k] == old(*r).coeffs[k] + b->coeffs[k])) + assigns(memory_slice(r, sizeof(mlk_poly))) +); + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_poly_sub MLK_NAMESPACE(poly_sub) +/** + * Subtract two polynomials; no modular reduction is performed. + * + * @spec{@[FIPS203, 2.4.5, Arithmetic With Polynomials and NTT + * Representations]. Used in @[FIPS203, Algorithm 15, K-PKE.Decrypt, L6].} + * + * @param[in,out] r Input-output polynomial to be subtracted from. + * @param[in] b Second input polynomial. + */ +/* + * NOTE: The reference implementation uses a 3-argument mlk_poly_sub. + * We specialize to the accumulator form to avoid reasoning about aliasing. + */ +MLK_INTERNAL_API +void mlk_poly_sub(mlk_poly *r, const mlk_poly *b) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(b, sizeof(mlk_poly))) + requires(forall(k0, 0, MLKEM_N, (int32_t) r->coeffs[k0] - b->coeffs[k0] <= INT16_MAX)) + requires(forall(k1, 0, MLKEM_N, (int32_t) r->coeffs[k1] - b->coeffs[k1] >= INT16_MIN)) + ensures(forall(k, 0, MLKEM_N, r->coeffs[k] == old(*r).coeffs[k] - b->coeffs[k])) + assigns(memory_slice(r, sizeof(mlk_poly))) +); +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#define mlk_poly_ntt MLK_NAMESPACE(poly_ntt) +/** + * Compute the negacyclic number-theoretic transform (NTT) of a polynomial + * in place. + * + * The input is assumed to be in normal order and coefficient-wise bound by + * MLKEM_Q in absolute value. + * + * The output polynomial is in bitreversed order, or of a custom order if + * MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set, and coefficient-wise bound + * by MLK_NTT_BOUND in absolute value. + * + * (NOTE: Sometimes the input to the NTT is actually smaller, which gives + * better bounds.) + * + * @spec{Implements @[FIPS203, Algorithm 9, NTT].} + * + * @param[in,out] r Input/output polynomial. + */ +MLK_INTERNAL_API +void mlk_poly_ntt(mlk_poly *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(array_abs_bound(r->coeffs, 0, MLKEM_N, MLKEM_Q)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_abs_bound(r->coeffs, 0, MLKEM_N, MLK_NTT_BOUND)) +); + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_poly_invntt_tomont MLK_NAMESPACE(poly_invntt_tomont) +/** + * Compute the inverse negacyclic number-theoretic transform (NTT) of a + * polynomial in place; input assumed to be in bitreversed order, output in + * normal order. + * + * The input is assumed to be in bitreversed order, or of a custom order if + * MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set, and can have arbitrary + * coefficients in int16_t. + * + * The output polynomial is in normal order, and coefficient-wise bound by + * MLK_INVNTT_BOUND in absolute value. + * + * @spec{Implements composition of @[FIPS203, Algorithm 10, NTT^{-1}] and + * elementwise modular multiplication with a suitable Montgomery factor + * introduced during the base multiplication.} + * + * @param[in,out] r Input/output polynomial. + */ +MLK_INTERNAL_API +void mlk_poly_invntt_tomont(mlk_poly *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_abs_bound(r->coeffs, 0, MLKEM_N, MLK_INVNTT_BOUND)) +); +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#endif /* !MLK_POLY_H */ diff --git a/src/mlkem-native/mlkem/src/poly_k.c b/src/mlkem-native/mlkem/src/poly_k.c new file mode 100644 index 0000000000..60380e34f3 --- /dev/null +++ b/src/mlkem-native/mlkem/src/poly_k.c @@ -0,0 +1,522 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [NeonNTT] + * Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1 + * Becker, Hwang, Kannwischer, Yang, Yang + * https://eprint.iacr.org/2021/986 + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + */ + +#include "poly_k.h" + +#include "debug.h" +#include "sampling.h" +#include "symmetric.h" +#include "verify.h" + +/* Parameter set namespacing + * This is to facilitate building multiple instances + * of mlkem-native (e.g. with varying parameter sets) + * within a single compilation unit. */ +#define mlk_poly_cbd_eta1 MLK_ADD_PARAM_SET(mlk_poly_cbd_eta1) +#define mlk_poly_cbd_eta2 MLK_ADD_PARAM_SET(mlk_poly_cbd_eta2) +#define mlk_polyvec_basemul_acc_montgomery_cached_c \ + MLK_ADD_PARAM_SET(mlk_polyvec_basemul_acc_montgomery_cached_c) +/* End of parameter set namespacing */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +/* Reference: `polyvec_compress()` in the reference implementation @[REF] + * - In contrast to the reference implementation, we assume + * unsigned canonical coefficients here. + * The reference implementation works with coefficients + * in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */ +MLK_INTERNAL_API +void mlk_polyvec_compress_du(uint8_t r[MLKEM_POLYVECCOMPRESSEDBYTES_DU], + const mlk_polyvec *a) +{ + unsigned i; + mlk_assert_bound_2d(a->vec, MLKEM_K, MLKEM_N, 0, MLKEM_Q); + + for (i = 0; i < MLKEM_K; i++) + { + mlk_poly_compress_du(r + i * MLKEM_POLYCOMPRESSEDBYTES_DU, &a->vec[i]); + } +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +/* Reference: `polyvec_decompress()` in the reference implementation @[REF]. */ +MLK_INTERNAL_API +void mlk_polyvec_decompress_du(mlk_polyvec *r, + const uint8_t a[MLKEM_POLYVECCOMPRESSEDBYTES_DU]) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + { + mlk_poly_decompress_du(&r->vec[i], a + i * MLKEM_POLYCOMPRESSEDBYTES_DU); + } + + mlk_assert_bound_2d(r->vec, MLKEM_K, MLKEM_N, 0, MLKEM_Q); +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) || !defined(MLK_CONFIG_NO_ENCAPS_API) +/* Reference: `polyvec_tobytes()` in the reference implementation @[REF]. + * - In contrast to the reference implementation, we assume + * unsigned canonical coefficients here. + * The reference implementation works with coefficients + * in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */ +MLK_INTERNAL_API +void mlk_polyvec_tobytes(uint8_t r[MLKEM_POLYVECBYTES], const mlk_polyvec *a) +{ + unsigned i; + mlk_assert_bound_2d(a->vec, MLKEM_K, MLKEM_N, 0, MLKEM_Q); + + for (i = 0; i < MLKEM_K; i++) + __loop__( + assigns(i, memory_slice(r, MLKEM_POLYVECBYTES)) + invariant(i <= MLKEM_K) + decreases(MLKEM_K - i) + ) + { + mlk_poly_tobytes(&r[i * MLKEM_POLYBYTES], &a->vec[i]); + } +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API || !MLK_CONFIG_NO_ENCAPS_API */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +/* Reference: `polyvec_frombytes()` in the reference implementation @[REF]. */ +MLK_INTERNAL_API +void mlk_polyvec_frombytes(mlk_polyvec *r, const uint8_t a[MLKEM_POLYVECBYTES]) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + { + mlk_poly_frombytes(&r->vec[i], a + i * MLKEM_POLYBYTES); + } + + mlk_assert_bound_2d(r->vec, MLKEM_K, MLKEM_N, 0, MLKEM_UINT12_LIMIT); +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +/* Reference: `polyvec_ntt()` in the reference implementation @[REF]. */ +MLK_INTERNAL_API +void mlk_polyvec_ntt(mlk_polyvec *r) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + { + mlk_poly_ntt(&r->vec[i]); + } + + mlk_assert_abs_bound_2d(r->vec, MLKEM_K, MLKEM_N, MLK_NTT_BOUND); +} + +/* Reference: `polyvec_invntt_tomont()` in the reference implementation @[REF]. + * - We normalize at the beginning of the inverse NTT, + * while the reference implementation normalizes at + * the end. This allows us to drop a call to `poly_reduce()` + * from the base multiplication. */ +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_INTERNAL_API +void mlk_polyvec_invntt_tomont(mlk_polyvec *r) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + { + mlk_poly_invntt_tomont(&r->vec[i]); + } + + mlk_assert_abs_bound_2d(r->vec, MLKEM_K, MLKEM_N, MLK_INVNTT_BOUND); +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +/* Reference: `polyvec_basemul_acc_montgomery()` in the + * reference implementation @[REF]. + * - We use a multiplication cache ('mulcache') here + * which is not present in the reference implementation @[REF]. + * This idea originates from @[NeonNTT] and is used + * at the C level here. + * - We compute the coefficients of the scalar product in 32-bit + * coefficients and perform only a single modular reduction + * at the end. The reference implementation uses 2 * MLKEM_K + * more modular reductions since it reduces after every modular + * multiplication. */ +MLK_STATIC_TESTABLE void mlk_polyvec_basemul_acc_montgomery_cached_c( + mlk_poly *r, const mlk_polyvec *a, const mlk_polyvec *b, + const mlk_polyvec_mulcache *b_cache) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, sizeof(mlk_polyvec))) + requires(memory_no_alias(b, sizeof(mlk_polyvec))) + requires(memory_no_alias(b_cache, sizeof(mlk_polyvec_mulcache))) + requires(forall(k1, 0, MLKEM_K, + array_bound(a->vec[k1].coeffs, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT))) + assigns(memory_slice(r, sizeof(mlk_poly))) +) +{ + unsigned i; + mlk_assert_bound_2d(a->vec, MLKEM_K, MLKEM_N, 0, MLKEM_UINT12_LIMIT); + + for (i = 0; i < MLKEM_N / 2; i++) + __loop__(invariant(i <= MLKEM_N / 2) + decreases(MLKEM_N / 2 - i)) + { + unsigned k; + int32_t t[2] = {0}; + for (k = 0; k < MLKEM_K; k++) + __loop__( + invariant(k <= MLKEM_K && + t[0] <= (int32_t) k * 2 * MLKEM_UINT12_LIMIT * 32768 && + t[0] >= - ((int32_t) k * 2 * MLKEM_UINT12_LIMIT * 32768) && + t[1] <= ((int32_t) k * 2 * MLKEM_UINT12_LIMIT * 32768) && + t[1] >= - ((int32_t) k * 2 * MLKEM_UINT12_LIMIT * 32768)) + decreases(MLKEM_K - k)) + { + t[0] += (int32_t)a->vec[k].coeffs[2 * i + 1] * b_cache->vec[k].coeffs[i]; + t[0] += (int32_t)a->vec[k].coeffs[2 * i] * b->vec[k].coeffs[2 * i]; + t[1] += (int32_t)a->vec[k].coeffs[2 * i] * b->vec[k].coeffs[2 * i + 1]; + t[1] += (int32_t)a->vec[k].coeffs[2 * i + 1] * b->vec[k].coeffs[2 * i]; + } + r->coeffs[2 * i + 0] = mlk_montgomery_reduce(t[0]); + r->coeffs[2 * i + 1] = mlk_montgomery_reduce(t[1]); + } +} + +MLK_INTERNAL_API +void mlk_polyvec_basemul_acc_montgomery_cached( + mlk_poly *r, const mlk_polyvec *a, const mlk_polyvec *b, + const mlk_polyvec_mulcache *b_cache) +{ +#if defined(MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED) + { + int ret; + mlk_assert_bound_2d(a->vec, MLKEM_K, MLKEM_N, 0, MLKEM_UINT12_LIMIT); +#if MLKEM_K == 2 + ret = mlk_polyvec_basemul_acc_montgomery_cached_k2_native( + r->coeffs, (const int16_t *)a, (const int16_t *)b, + (const int16_t *)b_cache); +#elif MLKEM_K == 3 + ret = mlk_polyvec_basemul_acc_montgomery_cached_k3_native( + r->coeffs, (const int16_t *)a, (const int16_t *)b, + (const int16_t *)b_cache); +#elif MLKEM_K == 4 + ret = mlk_polyvec_basemul_acc_montgomery_cached_k4_native( + r->coeffs, (const int16_t *)a, (const int16_t *)b, + (const int16_t *)b_cache); +#endif + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } + } +#endif /* MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED */ + + mlk_polyvec_basemul_acc_montgomery_cached_c(r, a, b, b_cache); +} + +/* Reference: Does not exist in the reference implementation @[REF]. + * - The reference implementation does not use a + * multiplication cache ('mulcache'). This idea originates + * from @[NeonNTT] and is used at the C level here. */ +MLK_INTERNAL_API +void mlk_polyvec_mulcache_compute(mlk_polyvec_mulcache *x, const mlk_polyvec *a) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + { + mlk_poly_mulcache_compute(&x->vec[i], &a->vec[i]); + } +} + +/* Reference: `polyvec_reduce()` in the reference implementation @[REF]. + * - We use _unsigned_ canonical outputs, while the reference + * implementation uses _signed_ canonical outputs. + * Accordingly, we need a conditional addition of MLKEM_Q + * here to go from signed to unsigned representatives. + * This conditional addition is then dropped from all + * polynomial compression functions instead (see `compress.c`). */ +MLK_INTERNAL_API +void mlk_polyvec_reduce(mlk_polyvec *r) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + { + mlk_poly_reduce(&r->vec[i]); + } + + mlk_assert_bound_2d(r->vec, MLKEM_K, MLKEM_N, 0, MLKEM_Q); +} + +/* Reference: `polyvec_add()` in the reference implementation @[REF]. + * - We use destructive version (output=first input) to avoid + * reasoning about aliasing in the CBMC specification */ +MLK_INTERNAL_API +void mlk_polyvec_add(mlk_polyvec *r, const mlk_polyvec *b) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + __loop__( + assigns(i, memory_slice(r, sizeof(mlk_polyvec))) + invariant(i <= MLKEM_K) + invariant(forall(j0, i, MLKEM_K, + forall(k0, 0, MLKEM_N, + ((int32_t)r->vec[j0].coeffs[k0] + b->vec[j0].coeffs[k0] <= INT16_MAX) && + ((int32_t)r->vec[j0].coeffs[k0] + b->vec[j0].coeffs[k0] >= INT16_MIN)))) + invariant(forall(j2, 0, i, + forall(k2, 0, MLKEM_N, + (r->vec[j2].coeffs[k2] <= INT16_MAX) && + (r->vec[j2].coeffs[k2] >= INT16_MIN)))) + decreases(MLKEM_K - i) + ) + { + mlk_poly_add(&r->vec[i], &b->vec[i]); + } +} + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +/* Reference: `polyvec_tomont()` in the reference implementation @[REF]. */ +MLK_INTERNAL_API +void mlk_polyvec_tomont(mlk_polyvec *r) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + { + mlk_poly_tomont(&r->vec[i]); + } + + mlk_assert_abs_bound_2d(r->vec, MLKEM_K, MLKEM_N, MLKEM_Q); +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + + +/** + * Given an array of uniformly random bytes, compute a polynomial with + * coefficients distributed according to a centered binomial distribution + * with parameter MLKEM_ETA1. + * + * @spec{Implements @[FIPS203, Algorithm 8, SamplePolyCBD_eta1], where eta1 + * is specified per parameter set in @[FIPS203, Table 2] and represented as + * MLKEM_ETA1 here.} + * + * @reference{`poly_cbd_eta1` in the reference implementation @[REF].} + * + * @param[out] r Output polynomial. + * @param[in] buf Input byte array. + */ +static MLK_INLINE void mlk_poly_cbd_eta1( + mlk_poly *r, const uint8_t buf[MLKEM_ETA1 * MLKEM_N / 4]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(buf, MLKEM_ETA1 * MLKEM_N / 4)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_abs_bound(r->coeffs, 0, MLKEM_N, MLKEM_ETA1 + 1)) +) +{ +#if MLKEM_ETA1 == 2 + mlk_poly_cbd2(r, buf); +#elif MLKEM_ETA1 == 3 + mlk_poly_cbd3(r, buf); +#else +#error "Invalid value of MLKEM_ETA1" +#endif +} + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) || \ + (MLKEM_ETA1 == MLKEM_ETA2 && (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API))) +/* Reference: Does not exist in the reference implementation @[REF]. + * - This implements a x4-batched version of `poly_getnoise_eta1()` + * from the reference implementation, to leverage + * batched Keccak-f1600.*/ +MLK_INTERNAL_API +void mlk_poly_getnoise_eta1_4x(mlk_poly *r0, mlk_poly *r1, mlk_poly *r2, + mlk_poly *r3, const uint8_t seed[MLKEM_SYMBYTES], + uint8_t nonce0, uint8_t nonce1, uint8_t nonce2, + uint8_t nonce3) +{ + MLK_ALIGN uint8_t buf[4][MLK_ALIGN_UP(MLKEM_ETA1 * MLKEM_N / 4)]; + MLK_ALIGN uint8_t extkey[4][MLK_ALIGN_UP(MLKEM_SYMBYTES + 1)]; + mlk_memcpy(extkey[0], seed, MLKEM_SYMBYTES); + mlk_memcpy(extkey[1], seed, MLKEM_SYMBYTES); + mlk_memcpy(extkey[2], seed, MLKEM_SYMBYTES); + mlk_memcpy(extkey[3], seed, MLKEM_SYMBYTES); + extkey[0][MLKEM_SYMBYTES] = nonce0; + extkey[1][MLKEM_SYMBYTES] = nonce1; + extkey[2][MLKEM_SYMBYTES] = nonce2; + extkey[3][MLKEM_SYMBYTES] = nonce3; + +#if !defined(FIPS202_X4_DEFAULT_IMPLEMENTATION) && \ + !defined(MLK_CONFIG_SERIAL_FIPS202_ONLY) + mlk_prf_eta1_x4(buf, extkey); +#else + mlk_prf_eta1(buf[0], extkey[0]); + mlk_prf_eta1(buf[1], extkey[1]); + mlk_prf_eta1(buf[2], extkey[2]); + if (r3 != NULL) + { + mlk_prf_eta1(buf[3], extkey[3]); + } +#endif /* !(!FIPS202_X4_DEFAULT_IMPLEMENTATION && \ + !MLK_CONFIG_SERIAL_FIPS202_ONLY) */ + + mlk_poly_cbd_eta1(r0, buf[0]); + mlk_poly_cbd_eta1(r1, buf[1]); + mlk_poly_cbd_eta1(r2, buf[2]); + if (r3 != NULL) + { + mlk_poly_cbd_eta1(r3, buf[3]); + mlk_assert_abs_bound(r3, MLKEM_N, MLKEM_ETA1 + 1); + } + + mlk_assert_abs_bound(r0, MLKEM_N, MLKEM_ETA1 + 1); + mlk_assert_abs_bound(r1, MLKEM_N, MLKEM_ETA1 + 1); + mlk_assert_abs_bound(r2, MLKEM_N, MLKEM_ETA1 + 1); + + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(buf, sizeof(buf)); + mlk_zeroize(extkey, sizeof(extkey)); +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API || (MLKEM_ETA1 == MLKEM_ETA2 && \ + (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API)) */ + +#if (MLKEM_K == 2 || MLKEM_K == 4) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API)) +/** + * Given an array of uniformly random bytes, compute a polynomial with + * coefficients distributed according to a centered binomial distribution + * with parameter MLKEM_ETA2. + * + * @spec{Implements @[FIPS203, Algorithm 8, SamplePolyCBD_eta2], where eta2 + * is specified per parameter set in @[FIPS203, Table 2] and represented as + * MLKEM_ETA2 here.} + * + * @reference{`poly_cbd_eta2` in the reference implementation @[REF].} + * + * @param[out] r Output polynomial. + * @param[in] buf Input byte array. + */ +static MLK_INLINE void mlk_poly_cbd_eta2( + mlk_poly *r, const uint8_t buf[MLKEM_ETA2 * MLKEM_N / 4]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(buf, MLKEM_ETA2 * MLKEM_N / 4)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_abs_bound(r->coeffs, 0, MLKEM_N, MLKEM_ETA2 + 1))) +{ +#if MLKEM_ETA2 == 2 + mlk_poly_cbd2(r, buf); +#else +#error "Invalid value of MLKEM_ETA2" +#endif +} + +/* Reference: `poly_getnoise_eta2()` in the reference implementation @[REF]. + * - We include buffer zeroization. */ +MLK_INTERNAL_API +void mlk_poly_getnoise_eta2(mlk_poly *r, const uint8_t seed[MLKEM_SYMBYTES], + uint8_t nonce) +{ + MLK_ALIGN uint8_t buf[MLKEM_ETA2 * MLKEM_N / 4]; + MLK_ALIGN uint8_t extkey[MLKEM_SYMBYTES + 1]; + + mlk_memcpy(extkey, seed, MLKEM_SYMBYTES); + extkey[MLKEM_SYMBYTES] = nonce; + mlk_prf_eta2(buf, extkey); + + mlk_poly_cbd_eta2(r, buf); + + mlk_assert_abs_bound(r, MLKEM_N, MLKEM_ETA2 + 1); + + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(buf, sizeof(buf)); + mlk_zeroize(extkey, sizeof(extkey)); +} +#endif /* (MLKEM_K == 2 || MLKEM_K == 4) && (!MLK_CONFIG_NO_ENCAPS_API || \ + !MLK_CONFIG_NO_DECAPS_API) */ + +#if MLKEM_K == 2 && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API)) +/* Reference: Does not exist in the reference implementation @[REF]. + * - This implements a x4-batched version of `poly_getnoise_eta1()` + * and `poly_getnoise_eta2()` from the reference implementation, + * leveraging batched Keccak-f1600. + * - If a x4-batched Keccak-f1600 is available, we squeeze + * more random data than needed for the eta2 calls, to be + * be able to use a x4-batched Keccak-f1600. */ +MLK_INTERNAL_API +void mlk_poly_getnoise_eta1122_4x(mlk_poly *r0, mlk_poly *r1, mlk_poly *r2, + mlk_poly *r3, + const uint8_t seed[MLKEM_SYMBYTES], + uint8_t nonce0, uint8_t nonce1, + uint8_t nonce2, uint8_t nonce3) +{ +#if MLKEM_ETA2 >= MLKEM_ETA1 +#error mlk_poly_getnoise_eta1122_4x assumes MLKEM_ETA1 > MLKEM_ETA2 +#endif + MLK_ALIGN uint8_t buf[4][MLK_ALIGN_UP(MLKEM_ETA1 * MLKEM_N / 4)]; + MLK_ALIGN uint8_t extkey[4][MLK_ALIGN_UP(MLKEM_SYMBYTES + 1)]; + + mlk_memcpy(extkey[0], seed, MLKEM_SYMBYTES); + mlk_memcpy(extkey[1], seed, MLKEM_SYMBYTES); + mlk_memcpy(extkey[2], seed, MLKEM_SYMBYTES); + mlk_memcpy(extkey[3], seed, MLKEM_SYMBYTES); + extkey[0][MLKEM_SYMBYTES] = nonce0; + extkey[1][MLKEM_SYMBYTES] = nonce1; + extkey[2][MLKEM_SYMBYTES] = nonce2; + extkey[3][MLKEM_SYMBYTES] = nonce3; + + /* On systems with fast batched Keccak, we use 4-fold batched PRF, + * even though that means generating more random data in buf[2] and buf[3] + * than necessary. */ +#if !defined(FIPS202_X4_DEFAULT_IMPLEMENTATION) && \ + !defined(MLK_CONFIG_SERIAL_FIPS202_ONLY) + mlk_prf_eta1_x4(buf, extkey); +#else + mlk_prf_eta1(buf[0], extkey[0]); + mlk_prf_eta1(buf[1], extkey[1]); + mlk_prf_eta2(buf[2], extkey[2]); + mlk_prf_eta2(buf[3], extkey[3]); +#endif /* !(!FIPS202_X4_DEFAULT_IMPLEMENTATION && \ + !MLK_CONFIG_SERIAL_FIPS202_ONLY) */ + + mlk_poly_cbd_eta1(r0, buf[0]); + mlk_poly_cbd_eta1(r1, buf[1]); + mlk_poly_cbd_eta2(r2, buf[2]); + mlk_poly_cbd_eta2(r3, buf[3]); + + mlk_assert_abs_bound(r0, MLKEM_N, MLKEM_ETA1 + 1); + mlk_assert_abs_bound(r1, MLKEM_N, MLKEM_ETA1 + 1); + mlk_assert_abs_bound(r2, MLKEM_N, MLKEM_ETA2 + 1); + mlk_assert_abs_bound(r3, MLKEM_N, MLKEM_ETA2 + 1); + + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(buf, sizeof(buf)); + mlk_zeroize(extkey, sizeof(extkey)); +} +#endif /* MLKEM_K == 2 && (!MLK_CONFIG_NO_ENCAPS_API || \ + !MLK_CONFIG_NO_DECAPS_API) */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef mlk_poly_cbd_eta1 +#undef mlk_poly_cbd_eta2 +#undef mlk_polyvec_basemul_acc_montgomery_cached_c diff --git a/src/mlkem-native/mlkem/src/poly_k.h b/src/mlkem-native/mlkem/src/poly_k.h new file mode 100644 index 0000000000..1eba9fea67 --- /dev/null +++ b/src/mlkem-native/mlkem/src/poly_k.h @@ -0,0 +1,622 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_POLY_K_H +#define MLK_POLY_K_H + +#include "common.h" +#include "compress.h" +#include "poly.h" + +/* Parameter set namespacing + * This is to facilitate building multiple instances + * of mlkem-native (e.g. with varying parameter sets) + * within a single compilation unit. */ +#define mlk_polyvec MLK_ADD_PARAM_SET(mlk_polyvec) +#define mlk_polymat MLK_ADD_PARAM_SET(mlk_polymat) +#define mlk_polyvec_mulcache MLK_ADD_PARAM_SET(mlk_polyvec_mulcache) +/* End of parameter set namespacing */ + +/** Vector of MLKEM_K polynomials. */ +typedef struct +{ + mlk_poly vec[MLKEM_K]; /**< Component polynomials. */ +} MLK_ALIGN mlk_polyvec; + +/** MLKEM_K x MLKEM_K matrix of polynomials. */ +typedef struct +{ + mlk_polyvec vec[MLKEM_K]; /**< Rows of the matrix. */ +} MLK_ALIGN mlk_polymat; + +/** Vector of MLKEM_K mlk_poly_mulcache entries. */ +typedef struct +{ + mlk_poly_mulcache vec[MLKEM_K]; /**< Per-component caches. */ +} MLK_ALIGN mlk_polyvec_mulcache; + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_poly_compress_du MLK_NAMESPACE_K(poly_compress_du) +/** + * Compression (du bits) and subsequent serialization of a polynomial. + * + * @spec{Implements `ByteEncode_{d_u} (Compress_{d_u} (u))` in @[FIPS203, + * Algorithm 14 (K-PKE.Encrypt), L22], with level-specific d_u defined in + * @[FIPS203, Table 2], and given by MLKEM_DU here.} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_DU + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + */ +static MLK_INLINE void mlk_poly_compress_du( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_DU], const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_DU)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_DU))) +{ +#if MLKEM_DU == 10 + mlk_poly_compress_d10(r, a); +#elif MLKEM_DU == 11 + mlk_poly_compress_d11(r, a); +#else +#error "Invalid value of MLKEM_DU" +#endif +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_poly_decompress_du MLK_NAMESPACE_K(poly_decompress_du) +/** + * De-serialization and subsequent decompression (du bits) of a polynomial; + * approximate inverse of mlk_poly_compress_du. + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @spec{Implements `Decompress_{d_u} (ByteDecode_{d_u} (u))` in @[FIPS203, + * Algorithm 15 (K-PKE.Decrypt), L3], with level-specific d_u defined in + * @[FIPS203, Table 2], and given by MLKEM_DU here.} + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_DU + * bytes). + */ +static MLK_INLINE void mlk_poly_decompress_du( + mlk_poly *r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_DU]) +__contract__( + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_DU)) + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q))) +{ +#if MLKEM_DU == 10 + mlk_poly_decompress_d10(r, a); +#elif MLKEM_DU == 11 + mlk_poly_decompress_d11(r, a); +#else +#error "Invalid value of MLKEM_DU" +#endif +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_poly_compress_dv MLK_NAMESPACE_K(poly_compress_dv) +/** + * Compression (dv bits) and subsequent serialization of a polynomial. + * + * @spec{Implements `ByteEncode_{d_v} (Compress_{d_v} (v))` in @[FIPS203, + * Algorithm 14 (K-PKE.Encrypt), L23], with level-specific d_v defined in + * @[FIPS203, Table 2], and given by MLKEM_DV here.} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_DV + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + */ +static MLK_INLINE void mlk_poly_compress_dv( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_DV], const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_DV)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_DV))) +{ +#if MLKEM_DV == 4 + mlk_poly_compress_d4(r, a); +#elif MLKEM_DV == 5 + mlk_poly_compress_d5(r, a); +#else +#error "Invalid value of MLKEM_DV" +#endif +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_poly_decompress_dv MLK_NAMESPACE_K(poly_decompress_dv) +/** + * De-serialization and subsequent decompression (dv bits) of a polynomial; + * approximate inverse of mlk_poly_compress_dv. + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @spec{Implements `Decompress_{d_v} (ByteDecode_{d_v} (v))` in @[FIPS203, + * Algorithm 15 (K-PKE.Decrypt), L4], with level-specific d_v defined in + * @[FIPS203, Table 2], and given by MLKEM_DV here.} + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_DV + * bytes). + */ +static MLK_INLINE void mlk_poly_decompress_dv( + mlk_poly *r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_DV]) +__contract__( + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_DV)) + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q))) +{ +#if MLKEM_DV == 4 + mlk_poly_decompress_d4(r, a); +#elif MLKEM_DV == 5 + mlk_poly_decompress_d5(r, a); +#else +#error "Invalid value of MLKEM_DV" +#endif +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_polyvec_compress_du MLK_NAMESPACE_K(polyvec_compress_du) +/** + * Compress and serialize a vector of polynomials. + * + * @spec{Implements `ByteEncode_{d_u} (Compress_{d_u} (u))` in @[FIPS203, + * Algorithm 14 (K-PKE.Encrypt), L22], with level-specific d_u defined in + * @[FIPS203, Table 2], and given by MLKEM_DU here.} + * + * @param[out] r Output byte array (needs space for + * MLKEM_POLYVECCOMPRESSEDBYTES_DU bytes). + * @param[in] a Input vector of polynomials. Coefficients must be unsigned + * canonical, i.e. in [0,1,..,MLKEM_Q-1]. + */ +MLK_INTERNAL_API +void mlk_polyvec_compress_du(uint8_t r[MLKEM_POLYVECCOMPRESSEDBYTES_DU], + const mlk_polyvec *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYVECCOMPRESSEDBYTES_DU)) + requires(memory_no_alias(a, sizeof(mlk_polyvec))) + requires(forall(k0, 0, MLKEM_K, + array_bound(a->vec[k0].coeffs, 0, MLKEM_N, 0, MLKEM_Q))) + assigns(memory_slice(r, MLKEM_POLYVECCOMPRESSEDBYTES_DU)) +); +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_polyvec_decompress_du MLK_NAMESPACE_K(polyvec_decompress_du) +/** + * De-serialize and decompress a vector of polynomials; approximate inverse + * of mlk_polyvec_compress_du. + * + * @spec{Implements `Decompress_{d_u} (ByteDecode_{d_u} (u))` in @[FIPS203, + * Algorithm 15 (K-PKE.Decrypt), L3], with level-specific d_u defined in + * @[FIPS203, Table 2], and given by MLKEM_DU here.} + * + * @param[out] r Output vector of polynomials. Coefficients are normalized + * to [0,1,..,MLKEM_Q-1]. + * @param[in] a Input byte array (of length MLKEM_POLYVECCOMPRESSEDBYTES_DU + * bytes). + */ +MLK_INTERNAL_API +void mlk_polyvec_decompress_du(mlk_polyvec *r, + const uint8_t a[MLKEM_POLYVECCOMPRESSEDBYTES_DU]) +__contract__( + requires(memory_no_alias(a, MLKEM_POLYVECCOMPRESSEDBYTES_DU)) + requires(memory_no_alias(r, sizeof(mlk_polyvec))) + assigns(memory_slice(r, sizeof(mlk_polyvec))) + ensures(forall(k0, 0, MLKEM_K, + array_bound(r->vec[k0].coeffs, 0, MLKEM_N, 0, MLKEM_Q))) +); +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) || !defined(MLK_CONFIG_NO_ENCAPS_API) +#define mlk_polyvec_tobytes MLK_NAMESPACE_K(polyvec_tobytes) +/** + * Serialize a vector of polynomials. + * + * @spec{Implements ByteEncode_12 @[FIPS203, Algorithm 5]. Extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays] and + * @[FIPS203, 2.4.6, Matrices and Vectors].} + * + * @param[out] r Output byte array (needs space for MLKEM_POLYVECBYTES bytes). + * @param[in] a Input vector of polynomials. Each polynomial must have + * coefficients in [0,1,..,MLKEM_Q-1]. + */ +MLK_INTERNAL_API +void mlk_polyvec_tobytes(uint8_t r[MLKEM_POLYVECBYTES], const mlk_polyvec *a) +__contract__( + requires(memory_no_alias(a, sizeof(mlk_polyvec))) + requires(memory_no_alias(r, MLKEM_POLYVECBYTES)) + requires(forall(k0, 0, MLKEM_K, + array_bound(a->vec[k0].coeffs, 0, MLKEM_N, 0, MLKEM_Q))) + assigns(memory_slice(r, MLKEM_POLYVECBYTES)) +); +#endif /* !MLK_CONFIG_NO_KEYPAIR_API || !MLK_CONFIG_NO_ENCAPS_API */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_polyvec_frombytes MLK_NAMESPACE_K(polyvec_frombytes) +/** + * De-serialize a vector of polynomials; inverse of mlk_polyvec_tobytes. + * + * @spec{Implements ByteDecode_12 @[FIPS203, Algorithm 6]. Extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays] and + * @[FIPS203, 2.4.6, Matrices and Vectors].} + * + * @param[out] r Output vector of polynomials. Coefficients will be + * normalized in [0,1,..,4095]. + * @param[in] a Input byte array (of length MLKEM_POLYVECBYTES bytes). + */ +MLK_INTERNAL_API +void mlk_polyvec_frombytes(mlk_polyvec *r, const uint8_t a[MLKEM_POLYVECBYTES]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_polyvec))) + requires(memory_no_alias(a, MLKEM_POLYVECBYTES)) + assigns(memory_slice(r, sizeof(mlk_polyvec))) + ensures(forall(k0, 0, MLKEM_K, + array_bound(r->vec[k0].coeffs, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT))) +); +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#define mlk_polyvec_ntt MLK_NAMESPACE_K(polyvec_ntt) +/** + * Apply forward NTT to all elements of a vector of polynomials. + * + * The input is assumed to be in normal order and coefficient-wise bound by + * MLKEM_Q in absolute value. + * + * The output polynomial is in bitreversed order, and coefficient-wise bound + * by MLK_NTT_BOUND in absolute value. + * + * @spec{Implements @[FIPS203, Algorithm 9, NTT]. Extended to vectors as per + * @[FIPS203, 2.4.6, Matrices and Vectors].} + * + * @param[in,out] r Input/output vector of polynomials. + */ +MLK_INTERNAL_API +void mlk_polyvec_ntt(mlk_polyvec *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_polyvec))) + requires(forall(j, 0, MLKEM_K, + array_abs_bound(r->vec[j].coeffs, 0, MLKEM_N, MLKEM_Q))) + assigns(memory_slice(r, sizeof(mlk_polyvec))) + ensures(forall(j, 0, MLKEM_K, + array_abs_bound(r->vec[j].coeffs, 0, MLKEM_N, MLK_NTT_BOUND))) +); + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_polyvec_invntt_tomont MLK_NAMESPACE_K(polyvec_invntt_tomont) +/** + * Apply inverse NTT to all elements of a vector of polynomials and multiply + * by Montgomery factor 2^16. + * + * The input is assumed to be in bitreversed order, and can have arbitrary + * coefficients in int16_t. + * + * The output polynomial is in normal order, and coefficient-wise bound by + * MLK_INVNTT_BOUND in absolute value. + * + * @spec{Implements @[FIPS203, Algorithm 10, NTT^{-1}]. Extended to vectors + * as per @[FIPS203, 2.4.6, Matrices and Vectors].} + * + * @param[in,out] r Input/output vector of polynomials. + */ +MLK_INTERNAL_API +void mlk_polyvec_invntt_tomont(mlk_polyvec *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_polyvec))) + assigns(memory_slice(r, sizeof(mlk_polyvec))) + ensures(forall(j, 0, MLKEM_K, + array_abs_bound(r->vec[j].coeffs, 0, MLKEM_N, MLK_INVNTT_BOUND))) +); +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#define mlk_polyvec_basemul_acc_montgomery_cached \ + MLK_NAMESPACE_K(polyvec_basemul_acc_montgomery_cached) +/** + * Scalar product of two vectors of polynomials in NTT domain, using + * mulcache for the second operand. + * + * Bounds: every coefficient of @p a is assumed to be in [0,1,..,4095]. No + * bounds guarantees for the coefficients in the result. + * + * @spec{Implements @[FIPS203, Section 2.4.7, Eq (2.14)], @[FIPS203, + * Algorithm 11, MultiplyNTTs], and @[FIPS203, Algorithm 12, + * BaseCaseMultiply].} + * + * @param[out] r Output polynomial. + * @param[in] a First input polynomial vector. + * @param[in] b Second input polynomial vector. + * @param[in] b_cache Mulcache for the second input polynomial vector. Can + * be computed via mlk_polyvec_mulcache_compute(). + */ +MLK_INTERNAL_API +void mlk_polyvec_basemul_acc_montgomery_cached( + mlk_poly *r, const mlk_polyvec *a, const mlk_polyvec *b, + const mlk_polyvec_mulcache *b_cache) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, sizeof(mlk_polyvec))) + requires(memory_no_alias(b, sizeof(mlk_polyvec))) + requires(memory_no_alias(b_cache, sizeof(mlk_polyvec_mulcache))) + requires(forall(k1, 0, MLKEM_K, + array_bound(a->vec[k1].coeffs, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT))) + assigns(memory_slice(r, sizeof(mlk_poly))) +); + +#define mlk_polyvec_mulcache_compute MLK_NAMESPACE_K(polyvec_mulcache_compute) +/** + * Compute the mulcache for a vector of polynomials in NTT domain. + * + * The mulcache of a degree-2 polynomial b := b0 + b1*X in Fq[X]/(X^2-zeta) + * is the value b1*zeta, needed when computing products of b in + * Fq[X]/(X^2-zeta). + * + * The mulcache of a polynomial in NTT domain -- which is a 128-tuple of + * degree-2 polynomials in Fq[X]/(X^2-zeta), for varying zeta, is the + * 128-tuple of mulcaches of those polynomials. + * + * The mulcache of a vector of polynomials is the vector of mulcaches of + * its entries. + * + * @spec{Caches `b_1 * \gamma` in @[FIPS203, Algorithm 12, BaseCaseMultiply, + * L1].} + * + * @param[out] x Mulcache to be populated. + * @param[in] a Input polynomial vector. + */ +/* + * NOTE: The default C implementation of this function populates + * the mulcache with values in (-q,q), but this is not needed for the + * higher level safety proofs, and thus not part of the spec. + */ +MLK_INTERNAL_API +void mlk_polyvec_mulcache_compute(mlk_polyvec_mulcache *x, const mlk_polyvec *a) +__contract__( + requires(memory_no_alias(x, sizeof(mlk_polyvec_mulcache))) + requires(memory_no_alias(a, sizeof(mlk_polyvec))) + assigns(memory_slice(x, sizeof(mlk_polyvec_mulcache))) +); + +#define mlk_polyvec_reduce MLK_NAMESPACE_K(polyvec_reduce) +/** + * Apply Barrett reduction to each coefficient of each element of a vector + * of polynomials. For details of the Barrett reduction see comments in + * poly.c. + * + * @spec{Normalizes on unsigned canonical representatives ahead of calling + * @[FIPS203, Compress_d, Eq (4.7)]. This is not made explicit in FIPS 203.} + * + * @param[in,out] r Input/output polynomial vector. + */ +/* + * NOTE: The semantics of mlk_polyvec_reduce() is different in + * the reference implementation, which requires + * signed canonical output data. Unsigned canonical + * outputs are better suited to the only remaining + * use of mlk_poly_reduce() in the context of (de)serialization. + */ +MLK_INTERNAL_API +void mlk_polyvec_reduce(mlk_polyvec *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_polyvec))) + assigns(memory_slice(r, sizeof(mlk_polyvec))) + ensures(forall(k0, 0, MLKEM_K, + array_bound(r->vec[k0].coeffs, 0, MLKEM_N, 0, MLKEM_Q))) +); + +#define mlk_polyvec_add MLK_NAMESPACE_K(polyvec_add) +/** + * Add vectors of polynomials. + * + * The coefficients of @p r and @p b must be such that the addition does + * not overflow. Otherwise, the behaviour of this function is undefined. + * + * The coefficients returned in @p *r are in int16_t which is sufficient to + * prove type-safety of calling units. Therefore, no stronger ensures clause + * is required on this function. + * + * @spec{@[FIPS203, 2.4.5, Arithmetic With Polynomials and NTT + * Representations]. Used in @[FIPS203, Algorithm 14 (K-PKE.Encrypt), L19].} + * + * @param[in,out] r Input-output vector of polynomials to be added to. + * @param[in] b Second input vector of polynomials. + */ +MLK_INTERNAL_API +void mlk_polyvec_add(mlk_polyvec *r, const mlk_polyvec *b) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_polyvec))) + requires(memory_no_alias(b, sizeof(mlk_polyvec))) + requires(forall(j0, 0, MLKEM_K, + forall(k0, 0, MLKEM_N, + (int32_t)r->vec[j0].coeffs[k0] + b->vec[j0].coeffs[k0] <= INT16_MAX))) + requires(forall(j1, 0, MLKEM_K, + forall(k1, 0, MLKEM_N, + (int32_t)r->vec[j1].coeffs[k1] + b->vec[j1].coeffs[k1] >= INT16_MIN))) + assigns(memory_slice(r, sizeof(mlk_polyvec))) +); + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +#define mlk_polyvec_tomont MLK_NAMESPACE_K(polyvec_tomont) +/** + * In-place conversion of all coefficients of a polynomial vector from the + * normal domain to the Montgomery domain. + * + * Bounds: output < MLKEM_Q in absolute value. + * + * @spec{Internal normalization required in `mlk_indcpa_keypair_derand` as + * part of matrix-vector multiplication @[FIPS203, Algorithm 13, K-PKE.KeyGen, + * L18].} + * + * @param[in,out] r Input/output polynomial vector. + */ +MLK_INTERNAL_API +void mlk_polyvec_tomont(mlk_polyvec *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_polyvec))) + assigns(memory_slice(r, sizeof(mlk_polyvec))) + ensures(forall(j, 0, MLKEM_K, + array_abs_bound(r->vec[j].coeffs, 0, MLKEM_N, MLKEM_Q))) +); +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) || \ + (MLKEM_ETA1 == MLKEM_ETA2 && (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API))) +#define mlk_poly_getnoise_eta1_4x MLK_NAMESPACE_K(poly_getnoise_eta1_4x) +/** + * Batch sample four polynomials deterministically from a seed and nonces, + * with output polynomials close to centered binomial distribution with + * parameter MLKEM_ETA1. + * + * @spec{Implements 4x `SamplePolyCBD_{eta1} (PRF_{eta1} (sigma, N))`: + * @[FIPS203, Algorithm 8, SamplePolyCBD_eta] and @[FIPS203, Eq (4.3), + * PRF_eta]. `SamplePolyCBD_{eta1} (PRF_{eta1} (sigma, N))` appears in + * @[FIPS203, Algorithm 13, K-PKE.KeyGen, L{9, 13}] and @[FIPS203, + * Algorithm 14, K-PKE.Encrypt, L10].} + * + * @param[out] r0 Output polynomial. + * @param[out] r1 Output polynomial. + * @param[out] r2 Output polynomial. + * @param[out] r3 Output polynomial. May be NULL. + * @param[in] seed Input seed (of length MLKEM_SYMBYTES bytes). + * @param nonce0 One-byte input nonce. + * @param nonce1 One-byte input nonce. + * @param nonce2 One-byte input nonce. + * @param nonce3 One-byte input nonce. + */ +MLK_INTERNAL_API +void mlk_poly_getnoise_eta1_4x(mlk_poly *r0, mlk_poly *r1, mlk_poly *r2, + mlk_poly *r3, const uint8_t seed[MLKEM_SYMBYTES], + uint8_t nonce0, uint8_t nonce1, uint8_t nonce2, + uint8_t nonce3) +__contract__( + requires(memory_no_alias(seed, MLKEM_SYMBYTES)) + requires(memory_no_alias(r0, sizeof(mlk_poly))) + requires(memory_no_alias(r1, sizeof(mlk_poly))) + requires(memory_no_alias(r2, sizeof(mlk_poly))) + requires(r3 == NULL || memory_no_alias(r3, sizeof(mlk_poly))) + assigns(memory_slice(r0, sizeof(mlk_poly))) + assigns(memory_slice(r1, sizeof(mlk_poly))) + assigns(memory_slice(r2, sizeof(mlk_poly))) + assigns(r3 != NULL: memory_slice(r3, sizeof(mlk_poly))) + ensures(array_abs_bound(r0->coeffs,0, MLKEM_N, MLKEM_ETA1 + 1)) + ensures(array_abs_bound(r1->coeffs,0, MLKEM_N, MLKEM_ETA1 + 1)) + ensures(array_abs_bound(r2->coeffs,0, MLKEM_N, MLKEM_ETA1 + 1)) + ensures(r3 != NULL ==> array_abs_bound(r3->coeffs,0, MLKEM_N, MLKEM_ETA1 + 1)) +); + +#if MLKEM_ETA1 == MLKEM_ETA2 +/* + * We only require mlk_poly_getnoise_eta2_4x for ml-kem-768 and ml-kem-1024 + * where MLKEM_ETA2 = MLKEM_ETA1 = 2. + * For ml-kem-512, mlk_poly_getnoise_eta1122_4x is used instead. + */ +#define mlk_poly_getnoise_eta2_4x mlk_poly_getnoise_eta1_4x +#endif /* MLKEM_ETA1 == MLKEM_ETA2 */ +#endif /* !MLK_CONFIG_NO_KEYPAIR_API || (MLKEM_ETA1 == MLKEM_ETA2 && \ + (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API)) */ + +#if (MLKEM_K == 2 || MLKEM_K == 4) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API)) +#define mlk_poly_getnoise_eta2 MLK_NAMESPACE_K(poly_getnoise_eta2) +/** + * Sample a polynomial deterministically from a seed and a nonce, with + * output polynomial close to centered binomial distribution with parameter + * MLKEM_ETA2. + * + * @spec{Implements `SamplePolyCBD_{eta2} (PRF_{eta2} (sigma, N))`: + * @[FIPS203, Algorithm 8, SamplePolyCBD_eta] and @[FIPS203, Eq (4.3), + * PRF_eta]. `SamplePolyCBD_{eta2} (PRF_{eta2} (sigma, N))` appears in + * @[FIPS203, Algorithm 14, K-PKE.Encrypt, L14].} + * + * @param[out] r Output polynomial. + * @param[in] seed Input seed (of length MLKEM_SYMBYTES bytes). + * @param nonce One-byte input nonce. + */ +MLK_INTERNAL_API +void mlk_poly_getnoise_eta2(mlk_poly *r, const uint8_t seed[MLKEM_SYMBYTES], + uint8_t nonce) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(seed, MLKEM_SYMBYTES)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_abs_bound(r->coeffs, 0, MLKEM_N, MLKEM_ETA2 + 1)) +); +#endif /* (MLKEM_K == 2 || MLKEM_K == 4) && (!MLK_CONFIG_NO_ENCAPS_API || \ + !MLK_CONFIG_NO_DECAPS_API) */ + +#if MLKEM_K == 2 && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API)) +#define mlk_poly_getnoise_eta1122_4x MLK_NAMESPACE_K(poly_getnoise_eta1122_4x) +/** + * Batch sample four polynomials deterministically from a seed and nonces, + * with output polynomials close to centered binomial distribution with + * parameter MLKEM_ETA1 and MLKEM_ETA2. + * + * @spec{Implements two instances each of + * `SamplePolyCBD_{eta1} (PRF_{eta1} (sigma, N))` and + * `SamplePolyCBD_{eta2} (PRF_{eta2} (sigma, N))`: + * @[FIPS203, Algorithm 8, SamplePolyCBD_eta] and @[FIPS203, Eq (4.3), + * PRF_eta]. `SamplePolyCBD_{eta2} (PRF_{eta2} (sigma, N))` appears in + * @[FIPS203, Algorithm 14, K-PKE.Encrypt, L14].} + * + * @param[out] r0 Output polynomial. + * @param[out] r1 Output polynomial. + * @param[out] r2 Output polynomial. + * @param[out] r3 Output polynomial. + * @param[in] seed Input seed (of length MLKEM_SYMBYTES bytes). + * @param nonce0 One-byte input nonce. + * @param nonce1 One-byte input nonce. + * @param nonce2 One-byte input nonce. + * @param nonce3 One-byte input nonce. + */ +MLK_INTERNAL_API +void mlk_poly_getnoise_eta1122_4x(mlk_poly *r0, mlk_poly *r1, mlk_poly *r2, + mlk_poly *r3, + const uint8_t seed[MLKEM_SYMBYTES], + uint8_t nonce0, uint8_t nonce1, + uint8_t nonce2, uint8_t nonce3) +__contract__( + requires(memory_no_alias(r0, sizeof(mlk_poly))) + requires(memory_no_alias(r1, sizeof(mlk_poly))) + requires(memory_no_alias(r2, sizeof(mlk_poly))) + requires(memory_no_alias(r3, sizeof(mlk_poly))) + requires(memory_no_alias(seed, MLKEM_SYMBYTES)) + assigns(memory_slice(r0, sizeof(mlk_poly))) + assigns(memory_slice(r1, sizeof(mlk_poly))) + assigns(memory_slice(r2, sizeof(mlk_poly))) + assigns(memory_slice(r3, sizeof(mlk_poly))) + ensures(array_abs_bound(r0->coeffs,0, MLKEM_N, MLKEM_ETA1 + 1) + && array_abs_bound(r1->coeffs,0, MLKEM_N, MLKEM_ETA1 + 1) + && array_abs_bound(r2->coeffs,0, MLKEM_N, MLKEM_ETA2 + 1) + && array_abs_bound(r3->coeffs,0, MLKEM_N, MLKEM_ETA2 + 1)) +); +#endif /* MLKEM_K == 2 && (!MLK_CONFIG_NO_ENCAPS_API || \ + !MLK_CONFIG_NO_DECAPS_API) */ + +#endif /* !MLK_POLY_K_H */ diff --git a/src/mlkem-native/mlkem/src/randombytes.h b/src/mlkem-native/mlkem/src/randombytes.h new file mode 100644 index 0000000000..46274422b0 --- /dev/null +++ b/src/mlkem-native/mlkem/src/randombytes.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_RANDOMBYTES_H +#define MLK_RANDOMBYTES_H + + +#include "cbmc.h" +#include "common.h" + +#if !defined(MLK_CONFIG_NO_RANDOMIZED_API) +#if !defined(MLK_CONFIG_CUSTOM_RANDOMBYTES) +/** + * Fill a buffer with cryptographically secure random bytes. + * + * mlkem-native does not provide an implementation of this function. + * It must be provided by the consumer. + * + * To use a custom random byte source with a different name or signature, + * set MLK_CONFIG_CUSTOM_RANDOMBYTES and define mlk_randombytes directly. + * + * @param[out] out Output buffer. + * @param outlen Number of random bytes to write. + * + * @retval 0 Success. + * @retval other Failure; top-level APIs propagate this as MLK_ERR_RNG_FAIL. + */ +int randombytes(uint8_t *out, size_t outlen); + +/** + * Internal wrapper around randombytes(). + * + * Fills a buffer with cryptographically secure random bytes. + * + * This function can be replaced by setting MLK_CONFIG_CUSTOM_RANDOMBYTES + * and defining mlk_randombytes directly. + * + * @param[out] out Output buffer. + * @param outlen Number of random bytes to write. + * + * @retval 0 Success. + * @retval other Failure; top-level APIs propagate this as MLK_ERR_RNG_FAIL. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_randombytes(uint8_t *out, size_t outlen) +__contract__( + requires(memory_no_alias(out, outlen)) + assigns(memory_slice(out, outlen))) { return randombytes(out, outlen); } +#endif /* !MLK_CONFIG_CUSTOM_RANDOMBYTES */ +#endif /* !MLK_CONFIG_NO_RANDOMIZED_API */ +#endif /* !MLK_RANDOMBYTES_H */ diff --git a/src/mlkem-native/mlkem/src/sampling.c b/src/mlkem-native/mlkem/src/sampling.c new file mode 100644 index 0000000000..b01ac5619e --- /dev/null +++ b/src/mlkem-native/mlkem/src/sampling.c @@ -0,0 +1,371 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + */ + +#include "common.h" +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "debug.h" +#include "sampling.h" +#include "symmetric.h" +#include "verify.h" + +/* Reference: `rej_uniform()` in the reference implementation @[REF]. + * - Our signature differs from the reference implementation + * in that it adds the offset and always expects the base of the + * target buffer. This avoids shifting the buffer base in the + * caller, which appears tricky to reason about. */ +MLK_STATIC_TESTABLE unsigned mlk_rej_uniform_c(int16_t *r, unsigned target, + unsigned offset, + const uint8_t *buf, + unsigned buflen) +__contract__( + requires(offset <= target && target <= 4096 && buflen <= 4096 && buflen % 3 == 0) + requires(memory_no_alias(r, sizeof(int16_t) * target)) + requires(memory_no_alias(buf, buflen)) + requires(array_bound(r, 0, offset, 0, MLKEM_Q)) + assigns(memory_slice(r, sizeof(int16_t) * target)) + ensures(offset <= return_value && return_value <= target) + ensures(array_bound(r, 0, return_value, 0, MLKEM_Q))) +{ + unsigned ctr, pos; + int16_t val0, val1; + + mlk_assert_bound(r, offset, 0, MLKEM_Q); + + ctr = offset; + pos = 0; + /* pos + 3 cannot overflow due to the assumption buflen <= 4096 */ + while (ctr < target && pos + 3 <= buflen) + __loop__( + invariant(offset <= ctr && ctr <= target && pos <= buflen) + invariant(array_bound(r, 0, ctr, 0, MLKEM_Q)) + decreases(buflen - pos)) + { + /* Safety: + * - The explicit cast to uint16_t ensures that << 8 does + * not signed-overflow even on a 16-bit system. + * - The conversion to int16_t is safe due to the explicit 0xFFF + * truncation. + */ + val0 = (int16_t)(((buf[pos + 0] >> 0) | ((uint16_t)buf[pos + 1] << 8)) & + 0xFFF); + val1 = (int16_t)(((buf[pos + 1] >> 4) | (buf[pos + 2] << 4)) & 0xFFF); + pos += 3; + + if (val0 < MLKEM_Q) + { + r[ctr++] = val0; + } + if (ctr < target && val1 < MLKEM_Q) + { + r[ctr++] = val1; + } + } + + mlk_assert_bound(r, ctr, 0, MLKEM_Q); + return ctr; +} + +/** + * Run rejection sampling on uniform random bytes to generate uniform random + * integers mod MLKEM_Q. + * + * @reference{`rej_uniform()` in the reference implementation @[REF]. Our + * signature differs from the reference in that it adds the offset and always + * expects the base of the target buffer; this avoids shifting the buffer + * base in the caller, which is tricky to reason about. Has an optional + * fallback to a native implementation.} + * + * @param[out] r Output buffer. + * @param target Requested number of 16-bit integers (uniform mod MLKEM_Q). + * Must be <= 4096. + * @param offset Number of 16-bit integers that have already been + * sampled. Must be <= @p target. + * @param[in] buf Input buffer (assumed to be uniform random bytes). + * @param buflen Length of input buffer in bytes. Must be <= 4096 and a + * multiple of 3. + * + * @note Strictly speaking, only a few values of @p buflen near UINT_MAX need + * excluding. The limit of 4096 is somewhat arbitrary but sufficient + * for all uses of this function. Similarly, the actual limit for + * @p target is UINT_MAX/2. + * + * @return New offset of sampled 16-bit integers, at most @p target and at + * least the initial @p offset. If the new offset is strictly less + * than @p target, the entire input buffer is guaranteed to have been + * consumed; otherwise no information is provided on how many bytes + * of the input buffer have been consumed. + */ +static unsigned mlk_rej_uniform(int16_t *r, unsigned target, unsigned offset, + const uint8_t *buf, unsigned buflen) +__contract__( + requires(offset <= target && target <= 4096 && buflen <= 4096 && buflen % 3 == 0) + requires(memory_no_alias(r, sizeof(int16_t) * target)) + requires(memory_no_alias(buf, buflen)) + requires(array_bound(r, 0, offset, 0, MLKEM_Q)) + assigns(memory_slice(r, sizeof(int16_t) * target)) + ensures(offset <= return_value && return_value <= target) + ensures(array_bound(r, 0, return_value, 0, MLKEM_Q)) +) +{ +#if defined(MLK_USE_NATIVE_REJ_UNIFORM) + if (offset == 0) + { + int ret; + ret = mlk_rej_uniform_native(r, target, buf, buflen); + if (ret != MLK_NATIVE_FUNC_FALLBACK) + { + unsigned res = (unsigned)ret; + mlk_assert_bound(r, res, 0, MLKEM_Q); + return res; + } + } +#endif /* MLK_USE_NATIVE_REJ_UNIFORM */ + + return mlk_rej_uniform_c(r, target, offset, buf, buflen); +} + +#ifndef MLKEM_GEN_MATRIX_NBLOCKS +#define MLKEM_GEN_MATRIX_NBLOCKS \ + ((12 * MLKEM_N / 8 * ((uint32_t)1 << 12) / MLKEM_Q + MLK_XOF_RATE) / \ + MLK_XOF_RATE) +#endif + +#if !defined(MLK_CONFIG_SERIAL_FIPS202_ONLY) +/* Reference: Does not exist in the reference implementation @[REF]. + * - x4-batched version of `rej_uniform()` from the + * reference implementation, leveraging x4-batched Keccak-f1600. */ +MLK_INTERNAL_API +void mlk_poly_rej_uniform_x4(mlk_poly *vec0, mlk_poly *vec1, mlk_poly *vec2, + mlk_poly *vec3, + uint8_t seed[4][MLK_ALIGN_UP(MLKEM_SYMBYTES + 2)]) +{ + /* Temporary buffers for XOF output before rejection sampling */ + MLK_ALIGN uint8_t + buf[4][MLK_ALIGN_UP(MLKEM_GEN_MATRIX_NBLOCKS * MLK_XOF_RATE)]; + + /* Tracks the number of coefficients we have already sampled */ + unsigned ctr[4]; + mlk_xof_x4_ctx statex; + unsigned buflen; + + mlk_xof_x4_init(&statex); + mlk_xof_x4_absorb(&statex, seed, MLKEM_SYMBYTES + 2); + + /* + * Initially, squeeze heuristic number of MLKEM_GEN_MATRIX_NBLOCKS. + * This should generate the matrix entries with high probability. + */ + mlk_xof_x4_squeezeblocks(buf, MLKEM_GEN_MATRIX_NBLOCKS, &statex); + buflen = MLKEM_GEN_MATRIX_NBLOCKS * MLK_XOF_RATE; + ctr[0] = mlk_rej_uniform(vec0->coeffs, MLKEM_N, 0, buf[0], buflen); + ctr[1] = mlk_rej_uniform(vec1->coeffs, MLKEM_N, 0, buf[1], buflen); + ctr[2] = mlk_rej_uniform(vec2->coeffs, MLKEM_N, 0, buf[2], buflen); + ctr[3] = mlk_rej_uniform(vec3->coeffs, MLKEM_N, 0, buf[3], buflen); + + /* + * So long as not all matrix entries have been generated, squeeze + * one more block a time until we're done. + */ + buflen = MLK_XOF_RATE; + while (ctr[0] < MLKEM_N || ctr[1] < MLKEM_N || ctr[2] < MLKEM_N || + ctr[3] < MLKEM_N) + __loop__( + assigns(ctr, statex, + memory_slice(vec0, sizeof(mlk_poly)), + memory_slice(vec1, sizeof(mlk_poly)), + memory_slice(vec2, sizeof(mlk_poly)), + memory_slice(vec3, sizeof(mlk_poly)), + object_whole(buf)) + invariant(ctr[0] <= MLKEM_N && ctr[1] <= MLKEM_N) + invariant(ctr[2] <= MLKEM_N && ctr[3] <= MLKEM_N) + invariant(array_bound(vec0->coeffs, 0, ctr[0], 0, MLKEM_Q)) + invariant(array_bound(vec1->coeffs, 0, ctr[1], 0, MLKEM_Q)) + invariant(array_bound(vec2->coeffs, 0, ctr[2], 0, MLKEM_Q)) + invariant(array_bound(vec3->coeffs, 0, ctr[3], 0, MLKEM_Q))) + { + mlk_xof_x4_squeezeblocks(buf, 1, &statex); + ctr[0] = mlk_rej_uniform(vec0->coeffs, MLKEM_N, ctr[0], buf[0], buflen); + ctr[1] = mlk_rej_uniform(vec1->coeffs, MLKEM_N, ctr[1], buf[1], buflen); + ctr[2] = mlk_rej_uniform(vec2->coeffs, MLKEM_N, ctr[2], buf[2], buflen); + ctr[3] = mlk_rej_uniform(vec3->coeffs, MLKEM_N, ctr[3], buf[3], buflen); + } + + mlk_xof_x4_release(&statex); + + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(buf, sizeof(buf)); +} +#endif /* !MLK_CONFIG_SERIAL_FIPS202_ONLY */ + +MLK_INTERNAL_API +void mlk_poly_rej_uniform(mlk_poly *entry, uint8_t seed[MLKEM_SYMBYTES + 2]) +{ + mlk_xof_ctx state; + MLK_ALIGN uint8_t buf[MLKEM_GEN_MATRIX_NBLOCKS * MLK_XOF_RATE]; + unsigned ctr, buflen; + + mlk_xof_init(&state); + mlk_xof_absorb(&state, seed, MLKEM_SYMBYTES + 2); + + /* Initially, squeeze + sample heuristic number of MLKEM_GEN_MATRIX_NBLOCKS. + */ + /* This should generate the matrix entry with high probability. */ + mlk_xof_squeezeblocks(buf, MLKEM_GEN_MATRIX_NBLOCKS, &state); + buflen = MLKEM_GEN_MATRIX_NBLOCKS * MLK_XOF_RATE; + ctr = mlk_rej_uniform(entry->coeffs, MLKEM_N, 0, buf, buflen); + + /* Squeeze + sample one more block a time until we're done */ + buflen = MLK_XOF_RATE; + while (ctr < MLKEM_N) + __loop__( + assigns(ctr, state, memory_slice(entry, sizeof(mlk_poly)), object_whole(buf)) + invariant(ctr <= MLKEM_N) + invariant(array_bound(entry->coeffs, 0, ctr, 0, MLKEM_Q))) + { + mlk_xof_squeezeblocks(buf, 1, &state); + ctr = mlk_rej_uniform(entry->coeffs, MLKEM_N, ctr, buf, buflen); + } + + mlk_xof_release(&state); + + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(buf, sizeof(buf)); +} + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_ETA1 == 2 || \ + !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +/** + * Load 4 bytes into a 32-bit integer in little-endian order. + * + * @reference{`load32_littleendian()` in the reference implementation @[REF].} + * + * @param[in] x Input byte array. + * + * @return 32-bit unsigned integer loaded from @p x. + */ +static uint32_t mlk_load32_littleendian(const uint8_t x[4]) +{ + uint32_t r; + r = (uint32_t)x[0]; + r |= (uint32_t)x[1] << 8; + r |= (uint32_t)x[2] << 16; + r |= (uint32_t)x[3] << 24; + return r; +} + +/* Reference: `cbd2()` in the reference implementation @[REF]. */ +MLK_INTERNAL_API +void mlk_poly_cbd2(mlk_poly *r, const uint8_t buf[2 * MLKEM_N / 4]) +{ + unsigned i; + for (i = 0; i < MLKEM_N / 8; i++) + __loop__( + invariant(i <= MLKEM_N / 8) + invariant(array_abs_bound(r->coeffs, 0, 8 * i, 3)) + decreases(MLKEM_N / 8 - i)) + { + unsigned j; + uint32_t t = mlk_load32_littleendian(buf + 4 * i); + uint32_t d = t & 0x55555555; + d += (t >> 1) & 0x55555555; + + for (j = 0; j < 8; j++) + __loop__( + invariant(i <= MLKEM_N / 8 && j <= 8) + invariant(array_abs_bound(r->coeffs, 0, 8 * i + j, 3)) + decreases(8 - j)) + { + /* Safety: The & 0x3 masks each value to 2 bits (range [0, 3]), so the + * truncation and subsequent subtraction in int16_t is lossless. */ + const int16_t a = (int16_t)((d >> (4 * j + 0)) & 0x3); + const int16_t b = (int16_t)((d >> (4 * j + 2)) & 0x3); + r->coeffs[8 * i + j] = (int16_t)(a - b); + } + } +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_ETA1 == 2 || \ + !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_ETA1 == 3 +/** + * Load 3 bytes into a 32-bit integer in little-endian order. + * + * This function is only needed for ML-KEM-512. + * + * @reference{`load24_littleendian()` in the reference implementation @[REF].} + * + * @param[in] x Input byte array. + * + * @return 32-bit unsigned integer loaded from @p x (most significant byte + * is zero). + */ +static uint32_t mlk_load24_littleendian(const uint8_t x[3]) +{ + uint32_t r; + r = (uint32_t)x[0]; + r |= (uint32_t)x[1] << 8; + r |= (uint32_t)x[2] << 16; + return r; +} + +/* Reference: `cbd3()` in the reference implementation @[REF]. */ +MLK_INTERNAL_API +void mlk_poly_cbd3(mlk_poly *r, const uint8_t buf[3 * MLKEM_N / 4]) +{ + unsigned i; + for (i = 0; i < MLKEM_N / 4; i++) + __loop__( + invariant(i <= MLKEM_N / 4) + invariant(array_abs_bound(r->coeffs, 0, 4 * i, 4)) + decreases(MLKEM_N / 4 - i)) + { + unsigned j; + const uint32_t t = mlk_load24_littleendian(buf + 3 * i); + uint32_t d = t & 0x00249249; + d += (t >> 1) & 0x00249249; + d += (t >> 2) & 0x00249249; + + for (j = 0; j < 4; j++) + __loop__( + invariant(i <= MLKEM_N / 4 && j <= 4) + invariant(array_abs_bound(r->coeffs, 0, 4 * i + j, 4)) + decreases(4 - j)) + { + /* Safety: The & 0x7 masks each value to 3 bits (range [0, 7]), so the + * truncation and subsequent subtraction in int16_t is lossless. */ + const int16_t a = (int16_t)((d >> (6 * j + 0)) & 0x7); + const int16_t b = (int16_t)((d >> (6 * j + 3)) & 0x7); + r->coeffs[4 * i + j] = (int16_t)(a - b); + } + } +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_ETA1 == 3 */ + +#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(sampling) + +#endif /* MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef MLKEM_GEN_MATRIX_NBLOCKS diff --git a/src/mlkem-native/mlkem/src/sampling.h b/src/mlkem-native/mlkem/src/sampling.h new file mode 100644 index 0000000000..24462aeb03 --- /dev/null +++ b/src/mlkem-native/mlkem/src/sampling.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_SAMPLING_H +#define MLK_SAMPLING_H + +#include "cbmc.h" +#include "common.h" +#include "poly.h" + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_ETA1 == 2 || \ + !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_poly_cbd2 MLK_NAMESPACE(poly_cbd2) +/** + * Given an array of uniformly random bytes, compute a polynomial with + * coefficients distributed according to a centered binomial distribution + * with parameter eta=2. + * + * @spec{Implements @[FIPS203, Algorithm 8, SamplePolyCBD_2].} + * + * @param[out] r Output polynomial. + * @param[in] buf Input byte array. + */ +MLK_INTERNAL_API +void mlk_poly_cbd2(mlk_poly *r, const uint8_t buf[2 * MLKEM_N / 4]); +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_ETA1 == 2 || \ + !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_ETA1 == 3 +#define mlk_poly_cbd3 MLK_NAMESPACE(poly_cbd3) +/** + * Given an array of uniformly random bytes, compute a polynomial with + * coefficients distributed according to a centered binomial distribution + * with parameter eta=3. + * + * This function is only needed for ML-KEM-512. + * + * @spec{Implements @[FIPS203, Algorithm 8, SamplePolyCBD_3].} + * + * @param[out] r Output polynomial. + * @param[in] buf Input byte array. + */ +MLK_INTERNAL_API +void mlk_poly_cbd3(mlk_poly *r, const uint8_t buf[3 * MLKEM_N / 4]); +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_ETA1 == 3 */ + +#if !defined(MLK_CONFIG_SERIAL_FIPS202_ONLY) +#define mlk_poly_rej_uniform_x4 MLK_NAMESPACE(poly_rej_uniform_x4) +/** + * Generate four polynomials using rejection sampling on (pseudo-)uniformly + * random bytes sampled from a seed. + * + * @spec{Implements @[FIPS203, Algorithm 7, SampleNTT].} + * + * @param[out] vec0 Polynomial to be sampled. + * @param[out] vec1 Polynomial to be sampled. + * @param[out] vec2 Polynomial to be sampled. + * @param[out] vec3 Polynomial to be sampled. + * @param[in] seed Consecutive array of 4 seed buffers of size + * MLKEM_SYMBYTES + 2 each, plus padding for alignment. + */ +MLK_INTERNAL_API +void mlk_poly_rej_uniform_x4(mlk_poly *vec0, mlk_poly *vec1, mlk_poly *vec2, + mlk_poly *vec3, + uint8_t seed[4][MLK_ALIGN_UP(MLKEM_SYMBYTES + 2)]) +__contract__( + requires(memory_no_alias(vec0, sizeof(mlk_poly))) + requires(memory_no_alias(vec1, sizeof(mlk_poly))) + requires(memory_no_alias(vec2, sizeof(mlk_poly))) + requires(memory_no_alias(vec3, sizeof(mlk_poly))) + requires(memory_no_alias(seed, 4 * MLK_ALIGN_UP(MLKEM_SYMBYTES + 2))) + assigns(memory_slice(vec0, sizeof(mlk_poly))) + assigns(memory_slice(vec1, sizeof(mlk_poly))) + assigns(memory_slice(vec2, sizeof(mlk_poly))) + assigns(memory_slice(vec3, sizeof(mlk_poly))) + ensures(array_bound(vec0->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + ensures(array_bound(vec1->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + ensures(array_bound(vec2->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + ensures(array_bound(vec3->coeffs, 0, MLKEM_N, 0, MLKEM_Q))); +#endif /* !MLK_CONFIG_SERIAL_FIPS202_ONLY */ + +#define mlk_poly_rej_uniform MLK_NAMESPACE(poly_rej_uniform) +/** + * Generate a polynomial using rejection sampling on (pseudo-)uniformly + * random bytes sampled from a seed. + * + * @spec{Implements @[FIPS203, Algorithm 7, SampleNTT].} + * + * @param[out] entry Polynomial to be sampled. + * @param[in] seed Seed buffer of size MLKEM_SYMBYTES + 2. + */ +MLK_INTERNAL_API +void mlk_poly_rej_uniform(mlk_poly *entry, uint8_t seed[MLKEM_SYMBYTES + 2]) +__contract__( + requires(memory_no_alias(entry, sizeof(mlk_poly))) + requires(memory_no_alias(seed, MLKEM_SYMBYTES + 2)) + assigns(memory_slice(entry, sizeof(mlk_poly))) + ensures(array_bound(entry->coeffs, 0, MLKEM_N, 0, MLKEM_Q))); + +#endif /* !MLK_SAMPLING_H */ diff --git a/src/mlkem-native/mlkem/src/symmetric.h b/src/mlkem-native/mlkem/src/symmetric.h new file mode 100644 index 0000000000..68d7e1a0cd --- /dev/null +++ b/src/mlkem-native/mlkem/src/symmetric.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_SYMMETRIC_H +#define MLK_SYMMETRIC_H + + +#include "cbmc.h" +#include "common.h" +#include MLK_FIPS202_HEADER_FILE +#if !defined(MLK_CONFIG_SERIAL_FIPS202_ONLY) +#include MLK_FIPS202X4_HEADER_FILE +#endif + +/* Macros denoting FIPS 203 specific Hash functions */ + +/* Hash function H, @[FIPS203, Section 4.1, Eq (4.4)] */ +#define mlk_hash_h(OUT, IN, INBYTES) mlk_sha3_256(OUT, IN, INBYTES) + +/* Hash function G, @[FIPS203, Section 4.1, Eq (4.5)] */ +#define mlk_hash_g(OUT, IN, INBYTES) mlk_sha3_512(OUT, IN, INBYTES) + +/* Hash function J, @[FIPS203, Section 4.1, Eq (4.4)] */ +#define mlk_hash_j(OUT, IN, INBYTES) \ + mlk_shake256(OUT, MLKEM_SYMBYTES, IN, INBYTES) + +/* PRF function, @[FIPS203, Section 4.1, Eq (4.3)] + * Referring to (eq 4.3), `OUT` is assumed to contain `s || b`. */ +#define mlk_prf_eta(ETA, OUT, IN) \ + mlk_shake256(OUT, (ETA) * MLKEM_N / 4, IN, MLKEM_SYMBYTES + 1) +#define mlk_prf_eta1(OUT, IN) mlk_prf_eta(MLKEM_ETA1, OUT, IN) +#define mlk_prf_eta2(OUT, IN) mlk_prf_eta(MLKEM_ETA2, OUT, IN) +#define mlk_prf_eta1_x4(OUT, IN) \ + mlk_shake256x4((OUT)[0], (OUT)[1], (OUT)[2], (OUT)[3], \ + (MLKEM_ETA1 * MLKEM_N / 4), (IN)[0], (IN)[1], (IN)[2], \ + (IN)[3], MLKEM_SYMBYTES + 1) + +/* XOF function, FIPS 203 4.1 */ +#define mlk_xof_ctx mlk_shake128ctx +#define mlk_xof_x4_ctx mlk_shake128x4ctx +#define mlk_xof_init(CTX) mlk_shake128_init((CTX)) +#define mlk_xof_absorb(CTX, IN, INBYTES) \ + mlk_shake128_absorb_once((CTX), (IN), (INBYTES)) +#define mlk_xof_squeezeblocks(BUF, NBLOCKS, CTX) \ + mlk_shake128_squeezeblocks((BUF), (NBLOCKS), (CTX)) +#define mlk_xof_release(CTX) mlk_shake128_release((CTX)) + +#define mlk_xof_x4_init(CTX) mlk_shake128x4_init((CTX)) +#define mlk_xof_x4_absorb(CTX, IN, INBYTES) \ + mlk_shake128x4_absorb_once((CTX), (IN)[0], (IN)[1], (IN)[2], (IN)[3], \ + (INBYTES)) +#define mlk_xof_x4_squeezeblocks(BUF, NBLOCKS, CTX) \ + mlk_shake128x4_squeezeblocks((BUF)[0], (BUF)[1], (BUF)[2], (BUF)[3], \ + (NBLOCKS), (CTX)) +#define mlk_xof_x4_release(CTX) mlk_shake128x4_release((CTX)) + +#define MLK_XOF_RATE SHAKE128_RATE + +#endif /* !MLK_SYMMETRIC_H */ diff --git a/src/mlkem-native/mlkem/src/sys.h b/src/mlkem-native/mlkem/src/sys.h new file mode 100644 index 0000000000..c430f77b0d --- /dev/null +++ b/src/mlkem-native/mlkem/src/sys.h @@ -0,0 +1,320 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_SYS_H +#define MLK_SYS_H + +#if !defined(MLK_CONFIG_NO_ASM) && (defined(__GNUC__) || defined(__clang__)) +#define MLK_HAVE_INLINE_ASM +#endif + +/* Try to find endianness, if not forced through CFLAGS already */ +#if !defined(MLK_SYS_LITTLE_ENDIAN) && !defined(MLK_SYS_BIG_ENDIAN) +#if defined(__BYTE_ORDER__) +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define MLK_SYS_LITTLE_ENDIAN +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#define MLK_SYS_BIG_ENDIAN +#else +#error "__BYTE_ORDER__ defined, but don't recognize value." +#endif +#endif /* __BYTE_ORDER__ */ + +/* MSVC does not define __BYTE_ORDER__. However, MSVC only supports + * little endian x86, x86_64, and AArch64. It is, hence, safe to assume + * little endian. */ +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_AMD64) || \ + defined(_M_IX86) || defined(_M_ARM64)) +#define MLK_SYS_LITTLE_ENDIAN +#endif + +#endif /* !MLK_SYS_LITTLE_ENDIAN && !MLK_SYS_BIG_ENDIAN */ + +/* Check if we're running on an AArch64 little endian system. _M_ARM64 is set by + * MSVC. */ +#if defined(__AARCH64EL__) || defined(_M_ARM64) +#define MLK_SYS_AARCH64 +#endif + +/* Check if the AArch64 compilation target supports NEON (Advanced SIMD). + * + * Some compilers also define __ARM_NEON__, but __ARM_NEON is the most reliable + * signal. Specifically, clang on Apple appears to keep __ARM_NEON__ set even if + * -march=armv8-a+nosimd is set. + * + * gcc 4.8 -- the first gcc version introducing Neon support -- sets neither + * __ARM_NEON nor __ARM_NEON__; in fact, there is no preprocessor signal that + * Neon is enabled. If you use gcc 4.8, you should set __ARM_NEON manually. + * gcc 4.9 onwards do set __ARM_NEON. + */ +#if defined(MLK_SYS_AARCH64) && defined(__ARM_NEON) +#define MLK_SYS_AARCH64_NEON +#endif + +/* Check if we're running on an AArch64 big endian system. */ +#if defined(__AARCH64EB__) +#define MLK_SYS_AARCH64_EB +#endif + +/* Check if we're running on an Armv8.1-M system with MVE */ +#if defined(__ARM_ARCH_8_1M_MAIN__) || defined(__ARM_FEATURE_MVE) +#define MLK_SYS_ARMV81M_MVE +#endif + +/* Check if we're running on an x86_64 system. */ +#if defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64) +#define MLK_SYS_X86_64 +#if defined(__AVX2__) +#define MLK_SYS_X86_64_AVX2 +#endif +#endif /* __x86_64__ || _M_X64 || _M_AMD64 */ + +#if defined(MLK_SYS_LITTLE_ENDIAN) && defined(__powerpc64__) +#define MLK_SYS_PPC64LE +#endif + +#if defined(__riscv) && defined(__riscv_xlen) && __riscv_xlen == 64 +#define MLK_SYS_RISCV64 +#endif + +#if defined(MLK_SYS_RISCV64) && defined(__riscv_vector) && \ + defined(__riscv_v_intrinsic) +#define MLK_SYS_RISCV64_RVV +#endif + +#if defined(__riscv) && defined(__riscv_xlen) && __riscv_xlen == 32 +#define MLK_SYS_RISCV32 +#endif + +#if defined(_WIN32) +#define MLK_SYS_WINDOWS +#endif + +#if defined(__linux__) +#define MLK_SYS_LINUX +#endif + +#if defined(__APPLE__) +#define MLK_SYS_APPLE +#endif + +#if defined(MLK_FORCE_AARCH64) && !defined(MLK_SYS_AARCH64) +#error "MLK_FORCE_AARCH64 is set, but we don't seem to be on an AArch64 system." +#endif + +#if defined(MLK_FORCE_AARCH64_EB) && !defined(MLK_SYS_AARCH64_EB) +#error \ + "MLK_FORCE_AARCH64_EB is set, but we don't seem to be on an AArch64 system." +#endif + +#if defined(MLK_FORCE_X86_64) && !defined(MLK_SYS_X86_64) +#error "MLK_FORCE_X86_64 is set, but we don't seem to be on an X86_64 system." +#endif + +#if defined(MLK_FORCE_PPC64LE) && !defined(MLK_SYS_PPC64LE) +#error "MLK_FORCE_PPC64LE is set, but we don't seem to be on a PPC64LE system." +#endif + +#if defined(MLK_FORCE_RISCV64) && !defined(MLK_SYS_RISCV64) +#error "MLK_FORCE_RISCV64 is set, but we don't seem to be on a RISCV64 system." +#endif + +#if defined(MLK_FORCE_RISCV32) && !defined(MLK_SYS_RISCV32) +#error "MLK_FORCE_RISCV32 is set, but we don't seem to be on a RISCV32 system." +#endif + +/* + * MLK_INLINE: Hint for inlining. + * - MSVC: __inline + * - C99+: inline + * - GCC/Clang C90: __attribute__((unused)) to silence warnings + * - Other C90: empty + */ +#if !defined(MLK_INLINE) +#if defined(_MSC_VER) +#define MLK_INLINE __inline +#elif defined(inline) || \ + (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) +#define MLK_INLINE inline +#elif defined(__GNUC__) || defined(__clang__) +#define MLK_INLINE __attribute__((unused)) +#else +#define MLK_INLINE +#endif +#endif /* !MLK_INLINE */ + +/* + * MLK_ALWAYS_INLINE: Force inlining. + * - MSVC: __forceinline + * - GCC/Clang C99+: MLK_INLINE __attribute__((always_inline)) + * - Other: MLK_INLINE (no forced inlining) + */ +#if !defined(MLK_ALWAYS_INLINE) +#if defined(_MSC_VER) +#define MLK_ALWAYS_INLINE __forceinline +#elif (defined(__GNUC__) || defined(__clang__)) && \ + (defined(inline) || \ + (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L)) +#define MLK_ALWAYS_INLINE MLK_INLINE __attribute__((always_inline)) +#else +#define MLK_ALWAYS_INLINE MLK_INLINE +#endif +#endif /* !MLK_ALWAYS_INLINE */ + +/* + * MLK_NOINLINE: Prevent inlining. + * - MSVC: __declspec(noinline) + * - GCC/Clang: __attribute__((noinline)) + * - Other: empty + */ +#if !defined(MLK_NOINLINE) +#if defined(_MSC_VER) +#define MLK_NOINLINE __declspec(noinline) +#elif defined(__GNUC__) || defined(__clang__) +#define MLK_NOINLINE __attribute__((noinline)) +#else +#define MLK_NOINLINE +#endif +#endif /* !MLK_NOINLINE */ + +#ifndef MLK_STATIC_TESTABLE +#define MLK_STATIC_TESTABLE static +#endif + +/* + * C90 does not have the restrict compiler directive yet. + * We don't use it in C90 builds. + */ +#if !defined(restrict) +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L +#define MLK_RESTRICT restrict +#else +#define MLK_RESTRICT +#endif + +#else /* !restrict */ + +#define MLK_RESTRICT restrict +#endif /* restrict */ + +#define MLK_DEFAULT_ALIGN 32 +#define MLK_ALIGN_UP(N) \ + ((((N) + (MLK_DEFAULT_ALIGN - 1)) / MLK_DEFAULT_ALIGN) * MLK_DEFAULT_ALIGN) +#if defined(__GNUC__) +#define MLK_ALIGN __attribute__((aligned(MLK_DEFAULT_ALIGN))) +#elif defined(_MSC_VER) +#define MLK_ALIGN __declspec(align(MLK_DEFAULT_ALIGN)) +#else +#define MLK_ALIGN /* No known support for alignment constraints */ +#endif + + +/* New X86_64 CPUs support control-flow protection using the CET instructions. + * When enabled (through -fcf-protection=), all compilation units (including + * empty ones) need to support CET for this to work. + * For assembly, this means that source files need to signal support for + * CET by setting the appropriate note.gnu.property section. + * This can be achieved by including the header in all assembly file. + * This file also provides the _CET_ENDBR macro which needs to be placed at + * every potential target of an indirect branch. + * If CET is enabled _CET_ENDBR maps to the endbr64 instruction, otherwise + * it is empty. + * In case the compiler does not support CET (e.g., +#define MLK_CET_ENDBR _CET_ENDBR +#else +#define MLK_CET_ENDBR +#endif +#endif /* MLK_SYS_X86_64 */ + +#if defined(MLK_CONFIG_CT_TESTING_ENABLED) && !defined(__ASSEMBLER__) +#include +#define MLK_CT_TESTING_SECRET(ptr, len) \ + VALGRIND_MAKE_MEM_UNDEFINED((ptr), (len)) +#define MLK_CT_TESTING_DECLASSIFY(ptr, len) \ + VALGRIND_MAKE_MEM_DEFINED((ptr), (len)) +#else /* MLK_CONFIG_CT_TESTING_ENABLED && !__ASSEMBLER__ */ +#define MLK_CT_TESTING_SECRET(ptr, len) \ + do \ + { \ + } while (0) +#define MLK_CT_TESTING_DECLASSIFY(ptr, len) \ + do \ + { \ + } while (0) +#endif /* !(MLK_CONFIG_CT_TESTING_ENABLED && !__ASSEMBLER__) */ + +#if defined(__GNUC__) || defined(__clang__) +#define MLK_MUST_CHECK_RETURN_VALUE __attribute__((warn_unused_result)) +#else +#define MLK_MUST_CHECK_RETURN_VALUE +#endif + +/* The x86_64 assembly backend uses the SysV calling convention. On Windows, + * where the Microsoft x64 calling convention is the default, it can still be + * used with compilers that allow choosing the calling convention per + * function: GCC and Clang support __attribute__((sysv_abi)), which makes + * calls to the annotated function follow the SysV calling convention. + * + * MLK_SYSV_ABI_SUPPORTED signals that the toolchain can call SysV assembly + * routines; the x86_64 assembly backend is only enabled if it is defined. + * MLK_SYSV_ABI is the attribute carried by declarations of x86_64 assembly + * routines. Both macros can be set externally for toolchains offering an + * equivalent mechanism that is not recognized here. */ +#if defined(MLK_SYS_X86_64) && !defined(MLK_SYSV_ABI_SUPPORTED) +#if !defined(MLK_SYS_WINDOWS) || defined(__GNUC__) || defined(__clang__) +#define MLK_SYSV_ABI_SUPPORTED +#endif +#endif + +#if !defined(MLK_SYSV_ABI) +#if defined(MLK_SYS_WINDOWS) && defined(MLK_SYSV_ABI_SUPPORTED) +#define MLK_SYSV_ABI __attribute__((sysv_abi)) +#else +#define MLK_SYSV_ABI +#endif +#endif /* !MLK_SYSV_ABI */ + +#if !defined(__ASSEMBLER__) +/* System capability enumeration */ +typedef enum +{ + /* x86_64 */ + MLK_SYS_CAP_X86_64_AVX2, + /* AArch64 */ + MLK_SYS_CAP_AARCH64_NEON, + MLK_SYS_CAP_AARCH64_SHA3, + /* Armv8.1-M */ + MLK_SYS_CAP_ARMV81M_MVE +} mlk_sys_cap; + +#if !defined(MLK_CONFIG_CUSTOM_CAPABILITY_FUNC) +#include "cbmc.h" + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_sys_check_capability(mlk_sys_cap cap) +__contract__( + ensures(return_value == 0 || return_value == 1) +) +{ + /* By default, we rely on compile-time feature detection/specification: + * If a feature is enabled at compile-time, we assume it is supported by + * the host that the resulting library/binary will be built on. + * If this assumption is not true, you MUST overwrite this function. + * See the documentation of MLK_CONFIG_CUSTOM_CAPABILITY_FUNC in + * mlkem_native_config.h for more information. */ + (void)cap; + return 1; +} +#endif /* !MLK_CONFIG_CUSTOM_CAPABILITY_FUNC */ +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_SYS_H */ diff --git a/src/mlkem-native/mlkem/src/verify.c b/src/mlkem-native/mlkem/src/verify.c new file mode 100644 index 0000000000..db760f7271 --- /dev/null +++ b/src/mlkem-native/mlkem/src/verify.c @@ -0,0 +1,20 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#include "verify.h" + +#if !defined(MLK_USE_ASM_VALUE_BARRIER) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/* + * Masking value used in constant-time functions from + * verify.h to block the compiler's range analysis and + * thereby reduce the risk of compiler-introduced branches. + */ +volatile uint64_t mlk_ct_opt_blocker_u64 = 0; + +#else /* !MLK_USE_ASM_VALUE_BARRIER && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(verify) + +#endif /* !(!MLK_USE_ASM_VALUE_BARRIER && !MLK_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/src/mlkem-native/mlkem/src/verify.h b/src/mlkem-native/mlkem/src/verify.h new file mode 100644 index 0000000000..8302a1c41b --- /dev/null +++ b/src/mlkem-native/mlkem/src/verify.h @@ -0,0 +1,447 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + * + * - [libmceliece] + * libmceliece implementation of Classic McEliece + * Bernstein, Chou + * https://lib.mceliece.org/ + * + * - [optblocker] + * PQC forum post on opt-blockers using volatile globals + * Daniel J. Bernstein + * https://groups.google.com/a/list.nist.gov/g/pqc-forum/c/hqbtIGFKIpU/m/H14H0wOlBgAJ + */ + +#ifndef MLK_VERIFY_H +#define MLK_VERIFY_H + + +#include "cbmc.h" +#include "common.h" + +/* Constant-time comparisons and conditional operations + + We reduce the risk for compilation into variable-time code + through the use of 'value barriers'. + + Functionally, a value barrier is a no-op. To the compiler, however, + it constitutes an arbitrary modification of its input, and therefore + harden's value propagation and range analysis. + + We consider two approaches to implement a value barrier: + - An empty inline asm block which marks the target value as clobbered. + - XOR'ing with the value of a volatile global that's set to 0; + see @[optblocker] for a discussion of this idea, and + @[libmceliece, inttypes/crypto_intN.h] for an implementation. + + The first approach is cheap because it only prevents the compiler + from reasoning about the value of the variable past the barrier, + but does not directly generate additional instructions. + + The second approach generates redundant loads and XOR operations + and therefore comes at a higher runtime cost. However, it appears + more robust towards optimization, as compilers should never drop + a volatile load. + + We use the empty-ASM value barrier for GCC and clang, and fall + back to the global volatile barrier otherwise. + + The global value barrier can be forced by setting + MLK_CONFIG_NO_ASM_VALUE_BARRIER. + +*/ + +#if defined(MLK_HAVE_INLINE_ASM) && !defined(MLK_CONFIG_NO_ASM_VALUE_BARRIER) +#define MLK_USE_ASM_VALUE_BARRIER +#endif + +#if !defined(MLK_USE_ASM_VALUE_BARRIER) + +/* + * Declaration of global volatile that the global value barrier + * is loading from and masking with. + */ +#define mlk_ct_opt_blocker_u64 MLK_NAMESPACE(ct_opt_blocker_u64) +extern volatile uint64_t mlk_ct_opt_blocker_u64; + +/* Helper functions for obtaining global masks of various sizes */ + +/* This contract is not proved but treated as an axiom. + * + * Its validity relies on the assumption that the global opt-blocker + * constant mlk_ct_opt_blocker_u64 is not modified. + */ +static MLK_INLINE uint64_t mlk_ct_get_optblocker_u64(void) +__contract__(ensures(return_value == 0)) { return mlk_ct_opt_blocker_u64; } + +static MLK_INLINE uint8_t mlk_ct_get_optblocker_u8(void) +__contract__(ensures(return_value == 0)) { return (uint8_t)mlk_ct_get_optblocker_u64(); } + +static MLK_INLINE uint32_t mlk_ct_get_optblocker_u32(void) +__contract__(ensures(return_value == 0)) { return (uint32_t)mlk_ct_get_optblocker_u64(); } + +static MLK_INLINE int32_t mlk_ct_get_optblocker_i32(void) +__contract__(ensures(return_value == 0)) { return (int32_t)mlk_ct_get_optblocker_u64(); } + +/* Opt-blocker based implementation of value barriers */ +static MLK_INLINE uint32_t mlk_value_barrier_u32(uint32_t b) +__contract__(ensures(return_value == b)) { return (b ^ mlk_ct_get_optblocker_u32()); } + +static MLK_INLINE int32_t mlk_value_barrier_i32(int32_t b) +__contract__(ensures(return_value == b)) { return (b ^ mlk_ct_get_optblocker_i32()); } + +static MLK_INLINE uint8_t mlk_value_barrier_u8(uint8_t b) +__contract__(ensures(return_value == b)) { return (b ^ mlk_ct_get_optblocker_u8()); } + +#else /* !MLK_USE_ASM_VALUE_BARRIER */ + +static MLK_INLINE uint32_t mlk_value_barrier_u32(uint32_t b) +__contract__(ensures(return_value == b)) +{ + __asm__ volatile("" : "+r"(b)); + return b; +} + +static MLK_INLINE int32_t mlk_value_barrier_i32(int32_t b) +__contract__(ensures(return_value == b)) +{ + __asm__ volatile("" : "+r"(b)); + return b; +} + +static MLK_INLINE uint8_t mlk_value_barrier_u8(uint8_t b) +__contract__(ensures(return_value == b)) +{ + __asm__ volatile("" : "+r"(b)); + return b; +} + +#endif /* MLK_USE_ASM_VALUE_BARRIER */ + +#ifdef CBMC +#pragma CPROVER check push +#pragma CPROVER check disable "conversion" +#endif +/** + * Cast uint16 value to int16. + * + * @param x Input value. + * + * @return For uint16_t x, the unique y in int16_t so that x == y mod 2^16. + * Concretely: + * - x < 32768: returns x + * - x >= 32768: returns x - 65536 + */ +static MLK_ALWAYS_INLINE int16_t mlk_cast_uint16_to_int16(uint16_t x) +{ + /* + * PORTABILITY: This relies on uint16_t -> int16_t + * being implemented as the inverse of int16_t -> uint16_t, + * which is implementation-defined (C99 6.3.1.3 (3)) + * CBMC (correctly) fails to prove this conversion is OK, + * so we have to suppress that check here + */ + return (int16_t)x; +} +#ifdef CBMC +#pragma CPROVER check pop +#endif + +/** + * Cast int32 value to uint16 as per C standard. + * + * @param x Input value. + * + * @return For int32_t x, the unique y in uint16_t so that x == y mod 2^16. + */ +static MLK_ALWAYS_INLINE uint16_t mlk_cast_int32_to_uint16(int32_t x) +{ + return (uint16_t)(x & (int32_t)UINT16_MAX); +} + +/** + * Cast int16 value to uint16 as per C standard. + * + * @param x Input value. + * + * @return For int16_t x, the unique y in uint16_t so that x == y mod 2^16. + */ +static MLK_ALWAYS_INLINE uint16_t mlk_cast_int16_to_uint16(int32_t x) +{ + return mlk_cast_int32_to_uint16(x); +} + +/** + * Return 0 if input is non-negative, and -1 otherwise. + * + * @reference{Embedded in the polynomial compression function in the + * reference implementation @[REF]. Used as part of signed->unsigned + * conversion for modular representatives to detect whether the input is + * negative. This happens in `mlk_poly_reduce()` here, and as part of + * polynomial compression functions in the reference implementation. See + * `mlk_poly_reduce()`. We use value barriers to reduce the risk of + * compiler-introduced branches.} + * + * @param x Value to be converted into a mask. + * + * @return Mask value (0 or 0xFFFF). + */ +static MLK_INLINE uint16_t mlk_ct_cmask_neg_i16(int16_t x) +__contract__(ensures(return_value == ((x < 0) ? 0xFFFF : 0))) +{ + int32_t tmp = mlk_value_barrier_i32((int32_t)x); + /* + * PORTABILITY: Right-shift on a signed integer is + * implementation-defined for negative left argument. + * Here, we assume it's sign-preserving "arithmetic" shift right. + * See (C99 6.5.7 (5)) + */ + tmp >>= 16; + return mlk_cast_int32_to_uint16(tmp); +} + +/** + * Return 0 if input is zero, and -1 otherwise. + * + * @reference{Embedded in `cmov_int16()` in the reference implementation + * @[REF]. Uses a value barrier and shift instead of `b = -b` to convert + * condition into mask.} + * + * @param x Value to be converted into a mask. + * + * @return Mask value (0 or 0xFFFF). + */ +static MLK_INLINE uint16_t mlk_ct_cmask_nonzero_u16(uint16_t x) +__contract__(ensures(return_value == ((x == 0) ? 0 : 0xFFFF))) +{ + int32_t tmp = mlk_value_barrier_i32(-((int32_t)x)); + /* + * PORTABILITY: Right-shift on a signed integer is + * implementation-defined for negative left argument. + * Here, we assume it's sign-preserving "arithmetic" shift right. + * See (C99 6.5.7 (5)) + */ + tmp >>= 16; + return mlk_cast_int32_to_uint16(tmp); +} + +/** + * Return 0 if input is zero, and -1 otherwise. + * + * @reference{Embedded in `verify()` and `cmov()` in the reference + * implementation @[REF]. We include a value barrier not present in the + * reference implementation, to prevent the compiler from realizing that + * this function returns a mask.} + * + * @param x Value to be converted into a mask. + * + * @return Mask value (0 or 0xFF). + */ +static MLK_INLINE uint8_t mlk_ct_cmask_nonzero_u8(uint8_t x) +__contract__(ensures(return_value == ((x == 0) ? 0 : 0xFF))) +{ + uint16_t mask = mlk_ct_cmask_nonzero_u16((uint16_t)x); + return (uint8_t)(mask & 0xFF); +} + +/** + * Functionally equivalent to cond ? a : b, but implemented with guards + * against compiler-introduced branches. + * + * @spec{With `a = MLKEM_Q_HALF` and `b=0`, this essentially implements + * `Decompress_1` @[FIPS203, Eq (4.8)] in `mlk_poly_frommsg()`. With + * `a = x + MLKEM_Q`, `b = x`, and `cond` indicating whether `x` is negative, + * implements signed->unsigned conversion of modular representatives. + * Questions of representation are not considered in the specification + * @[FIPS203, Section 2.4.1, "The pseudocode is agnostic regarding how an + * integer modulo 𑚠is represented in actual implementations"].} + * + * @reference{Embedded in the polynomial compression function in the + * reference implementation @[REF]. Used as part of signed->unsigned + * conversion for modular representatives. This happens in `mlk_poly_reduce()` + * here, and as part of polynomial compression functions in @[REF]. See + * `mlk_poly_reduce()`. Barrier to reduce the risk of compiler-introduced + * branches. For `a = MLKEM_Q_HALF` and `b=0`, also embedded in + * `poly_frommsg()` from the reference implementation, which uses + * `cmov_int16()` instead.} + * + * @param a First alternative. + * @param b Second alternative. + * @param cond Condition variable. + * + * @return @p a if @p cond != 0, else @p b. + */ +static MLK_INLINE int16_t mlk_ct_sel_int16(int16_t a, int16_t b, uint16_t cond) +__contract__(ensures(return_value == (cond ? a : b))) +{ + uint16_t au = mlk_cast_int16_to_uint16(a); + uint16_t bu = mlk_cast_int16_to_uint16(b); + uint16_t res = bu ^ (mlk_ct_cmask_nonzero_u16(cond) & (au ^ bu)); + return mlk_cast_uint16_to_int16(res); +} + +/** + * Functionally equivalent to cond ? a : b, but implemented with guards + * against compiler-introduced branches. + * + * @reference{Embedded into `cmov()` in the reference implementation @[REF]. + * Uses a value barrier to get mask from condition value.} + * + * @param a First alternative. + * @param b Second alternative. + * @param cond Condition variable. + * + * @return @p a if @p cond != 0, else @p b. + */ +static MLK_INLINE uint8_t mlk_ct_sel_uint8(uint8_t a, uint8_t b, uint8_t cond) +__contract__(ensures(return_value == (cond ? a : b))) +{ + return b ^ (mlk_ct_cmask_nonzero_u8(cond) & (a ^ b)); +} + +/** + * Compare two arrays for equality in constant time. + * + * @spec{Used to securely compute conditional move in @[FIPS203, Algorithm + * 18 (ML-KEM.Decaps_Internal, L9-11].} + * + * @reference{`cmov()` in the reference implementation @[REF]. We return + * `uint8_t`, not `int`. We use an additional XOR-accumulator in the + * comparison loop which prevents early abort if the OR-accumulator is 0xFF. + * We use a value barrier to convert the OR-accumulator into a mask; the + * reference implementation uses a shift which the compiler can argue to + * result in either 0 or 0xFF..FF.} + * + * @param[in] a First byte array. + * @param[in] b Second byte array. + * @param len Length of the byte arrays, upper-bounded to UINT16_MAX to + * control proof complexity only. + * + * @retval 0 The byte arrays are equal. + * @retval 0xFF The byte arrays are not equal. + */ +static MLK_INLINE uint8_t mlk_ct_memcmp(const uint8_t *a, const uint8_t *b, + const size_t len) +__contract__( + requires(len <= UINT16_MAX) + requires(memory_no_alias(a, len)) + requires(memory_no_alias(b, len)) + ensures((return_value == 0) || (return_value == 0xFF)) + ensures((return_value == 0) == forall(i, 0, len, (a[i] == b[i])))) +{ + uint8_t r = 0, s = 0; + unsigned i; + + for (i = 0; i < len; i++) + __loop__( + invariant(i <= len) + invariant((r == 0) == (forall(k, 0, i, (a[k] == b[k])))) + decreases(len - i)) + { + r |= a[i] ^ b[i]; + /* s is useless, but prevents the loop from being aborted once r=0xff. */ + s ^= a[i] ^ b[i]; + } + + /* + * - Convert r into a mask; this may not be necessary, but is an additional + * safeguard + * towards leaking information about a and b. + * - XOR twice with s, separated by a value barrier, to prevent the compile + * from dropping the s computation in the loop. + */ + return (mlk_value_barrier_u8(mlk_ct_cmask_nonzero_u8(r) ^ s) ^ s); +} + +/** + * Copy len bytes from x to r if b is zero; don't modify x if b is non-zero. + * Assumes two's complement representation of negative integers. Runs in + * constant time. + * + * @spec{Used to securely compute conditional move in @[FIPS203, Algorithm + * 18 (ML-KEM.Decaps_Internal, L9-11].} + * + * @reference{`cmov()` in the reference implementation @[REF]. We move if + * condition value is `0`, not `1`. We use `mlk_ct_sel_uint8` for + * constant-time selection.} + * + * @param[out] r Output byte array. + * @param[in] x Input byte array. + * @param len Number of bytes to be copied. + * @param b Condition value. + */ +static MLK_INLINE void mlk_ct_cmov_zero(uint8_t *r, const uint8_t *x, + size_t len, uint8_t b) +__contract__( + requires(len <= UINT32_MAX) + requires(memory_no_alias(r, len)) + requires(memory_no_alias(x, len)) + assigns(memory_slice(r, len)) + ensures(forall(i, 0, len, (r[i] == (b == 0 ? x[i] : old(r)[i]))))) +{ + size_t i; + for (i = 0; i < len; i++) + __loop__( + invariant(i <= len) + invariant(forall(k, 0, i, r[k] == (b == 0 ? x[k] : loop_entry(r)[k]))) + decreases(len - i)) + { + r[i] = mlk_ct_sel_uint8(r[i], x[i], b); + } +} + +/** + * Force-zeroize a buffer. + * + * @spec{Used to implement @[FIPS203, Section 3.3, Destruction of + * intermediate values].} + * + * @reference{Not present in the reference implementation @[REF].} + * + * @param[out] ptr Buffer to be zeroed. + * @param len Number of bytes to be zeroed. + */ +#if !defined(MLK_CONFIG_CUSTOM_ZEROIZE) +#if defined(MLK_SYS_WINDOWS) +#include +#elif !defined(MLK_HAVE_INLINE_ASM) +#error No plausibly-secure implementation of mlk_zeroize available. Please provide your own using MLK_CONFIG_CUSTOM_ZEROIZE. +#endif + +static MLK_INLINE void mlk_zeroize(void *ptr, size_t len) +__contract__( + requires(len <= UINT32_MAX) + requires(memory_no_alias(ptr, len)) + assigns(memory_slice(ptr, len)) + ensures(array_zeroized_u8((uint8_t *)ptr, len))) +{ +#if defined(MLK_SYS_WINDOWS) + SecureZeroMemory(ptr, len); +#else + mlk_memset(ptr, 0, len); + /* This follows OpenSSL and seems sufficient to prevent the compiler + * from optimizing away the memset. + * + * If there was a reliable way to detect availability of memset_s(), + * that would be preferred. */ + __asm__ volatile("" : : "r"(ptr) : "memory"); +#endif /* !MLK_SYS_WINDOWS */ +} +#endif /* !MLK_CONFIG_CUSTOM_ZEROIZE */ + +#endif /* !MLK_VERIFY_H */ diff --git a/src/mlkem-native/mlkem/src/zetas.inc b/src/mlkem-native/mlkem/src/zetas.inc new file mode 100644 index 0000000000..00316daf67 --- /dev/null +++ b/src/mlkem-native/mlkem/src/zetas.inc @@ -0,0 +1,30 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + + +/* + * Table of zeta values used in the reference NTT and inverse NTT. + * See autogen for details. + */ +static MLK_ALIGN const int16_t mlk_zetas[128] = { + -1044, -758, -359, -1517, 1493, 1422, 287, 202, -171, 622, 1577, + 182, 962, -1202, -1474, 1468, 573, -1325, 264, 383, -829, 1458, + -1602, -130, -681, 1017, 732, 608, -1542, 411, -205, -1571, 1223, + 652, -552, 1015, -1293, 1491, -282, -1544, 516, -8, -320, -666, + -1618, -1162, 126, 1469, -853, -90, -271, 830, 107, -1421, -247, + -951, -398, 961, -1508, -725, 448, -1065, 677, -1275, -1103, 430, + 555, 843, -1251, 871, 1550, 105, 422, 587, 177, -235, -291, + -460, 1574, 1653, -246, 778, 1159, -147, -777, 1483, -602, 1119, + -1590, 644, -872, 349, 418, 329, -156, -75, 817, 1097, 603, + 610, 1322, -1285, -1465, 384, -1215, -136, 1218, -1335, -874, 220, + -1187, -1659, -1185, -1530, -1278, 794, -1510, -854, -870, 478, -108, + -308, 996, 991, 958, -1460, 1522, 1628, +}; diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index a7ff86bb89..204f378b34 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -40,6 +40,7 @@ add_executable(test_bitcoin common_url_tests.cpp compilerbug_tests.cpp compress_tests.cpp + cpu_features_tests.cpp crypto_tests.cpp cuckoocache_tests.cpp dbwrapper_tests.cpp @@ -66,6 +67,7 @@ add_executable(test_bitcoin miner_tests.cpp miniminer_tests.cpp miniscript_tests.cpp + mlkem_tests.cpp minisketch_tests.cpp multisig_tests.cpp net_peer_connection_tests.cpp @@ -157,6 +159,7 @@ target_json_data_sources(test_bitcoin data/key_io_invalid.json data/key_io_valid.json data/mainnet_launch_difficulty.json + data/mlkem1024_vectors.json data/p2mr_cross_profile_vectors.json data/p2mr_datapqchash_vectors.json data/p2mr_pqc_witness_vectors.json diff --git a/src/test/cpu_features_tests.cpp b/src/test/cpu_features_tests.cpp new file mode 100644 index 0000000000..9d35408f4a --- /dev/null +++ b/src/test/cpu_features_tests.cpp @@ -0,0 +1,243 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#include // IWYU pragma: keep + +#include +#include +#include + +#include + +#include +#include +#include + +using namespace cpu_features; + +namespace { + +constexpr uint32_t LEAF7_EBX_SHA{uint32_t{1} << 29}; + +constexpr X86Features AVX2_USABLE{ + .max_basic_leaf = 7, + .leaf1_ecx = LEAF1_ECX_OSXSAVE | LEAF1_ECX_AVX, + .leaf7_ebx = LEAF7_EBX_AVX2, + .xcr0 = XCR0_XMM_YMM, +}; + +constexpr X86Features MLKEM_X86_NATIVE{ + .max_basic_leaf = 7, + .leaf1_ecx = LEAF1_ECX_OSXSAVE | LEAF1_ECX_AVX | LEAF1_ECX_SSSE3 | LEAF1_ECX_SSE41 | LEAF1_ECX_POPCNT, + .leaf7_ebx = LEAF7_EBX_AVX2 | LEAF7_EBX_BMI2, + .xcr0 = XCR0_XMM_YMM, +}; + +/** A synthetic CPU for DetectX86Features: answers CPUID like hardware and records every query. */ +struct FakeCpu { + X86Features features; + std::vector leaves; + int xgetbv_calls{0}; + bool xgetbv_faulted{false}; +}; +FakeCpu g_cpu; + +void FakeCpuid(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d) +{ + g_cpu.leaves.push_back(leaf); + const bool supported{leaf <= g_cpu.features.max_basic_leaf}; + a = leaf == 0 ? g_cpu.features.max_basic_leaf : 0; + // Real CPUs answer an unsupported leaf with another leaf's data; report + // AVX2 there so that reading it would show up as a wrong result. + b = leaf == 7 ? (supported ? g_cpu.features.leaf7_ebx : LEAF7_EBX_AVX2) : 0; + c = leaf == 1 ? g_cpu.features.leaf1_ecx : 0; + d = 0; + (void)subleaf; +} + +uint64_t FakeXGetBV(uint32_t index) +{ + ++g_cpu.xgetbv_calls; + if ((g_cpu.features.leaf1_ecx & LEAF1_ECX_OSXSAVE) == 0) g_cpu.xgetbv_faulted = true; + return index == 0 ? g_cpu.features.xcr0 : 0; +} + +X86Features DetectFake(const X86Features& features) +{ + g_cpu = FakeCpu{}; + g_cpu.features = features; + return DetectX86Features(FakeCpuid, FakeXGetBV); +} + +} // namespace + +BOOST_AUTO_TEST_SUITE(cpu_features_tests) + +BOOST_AUTO_TEST_CASE(usable_avx2_matrix) +{ + static_assert(HasUsableAVX2(AVX2_USABLE)); + static_assert(!HasUsableAVX2(X86Features{})); + + // Clearing any single requirement makes AVX2 unusable. + X86Features f{AVX2_USABLE}; + f.max_basic_leaf = 6; + BOOST_CHECK(!HasUsableAVX2(f)); + f = AVX2_USABLE; + f.leaf1_ecx &= ~LEAF1_ECX_OSXSAVE; + BOOST_CHECK(!HasUsableAVX2(f)); + f = AVX2_USABLE; + f.leaf1_ecx &= ~LEAF1_ECX_AVX; + BOOST_CHECK(!HasUsableAVX2(f)); + f = AVX2_USABLE; + f.xcr0 = 0x4; // YMM without XMM state + BOOST_CHECK(!HasUsableAVX2(f)); + f = AVX2_USABLE; + f.xcr0 = 0x3; // x87 and XMM, but no YMM state + BOOST_CHECK(!HasUsableAVX2(f)); + f = AVX2_USABLE; + f.leaf7_ebx &= ~LEAF7_EBX_AVX2; + BOOST_CHECK(!HasUsableAVX2(f)); + + // Unrelated bits, SHA-NI included, change nothing. + f = AVX2_USABLE; + f.leaf7_ebx |= LEAF7_EBX_SHA; + f.xcr0 |= 0xe1; + f.max_basic_leaf = 0x24; + BOOST_CHECK(HasUsableAVX2(f)); + + // Detection reads exactly what the CPU reports. + BOOST_CHECK(HasUsableAVX2(DetectFake(AVX2_USABLE))); + BOOST_CHECK_EQUAL(g_cpu.xgetbv_calls, 1); + BOOST_CHECK(!g_cpu.xgetbv_faulted); + BOOST_CHECK(g_cpu.leaves == (std::vector{0, 1, 7})); + + // Without OSXSAVE, XGETBV would fault, so it must not run. + f = AVX2_USABLE; + f.leaf1_ecx &= ~LEAF1_ECX_OSXSAVE; + BOOST_CHECK(!HasUsableAVX2(DetectFake(f))); + BOOST_CHECK_EQUAL(g_cpu.xgetbv_calls, 0); + BOOST_CHECK(!g_cpu.xgetbv_faulted); + + // Without AVX, XCR0 is irrelevant and is not read either. + f = AVX2_USABLE; + f.leaf1_ecx &= ~LEAF1_ECX_AVX; + BOOST_CHECK(!HasUsableAVX2(DetectFake(f))); + BOOST_CHECK_EQUAL(g_cpu.xgetbv_calls, 0); + + // The OS does not save YMM state. + f = AVX2_USABLE; + f.xcr0 = 0x3; + BOOST_CHECK(!HasUsableAVX2(DetectFake(f))); + BOOST_CHECK_EQUAL(g_cpu.xgetbv_calls, 1); + + // Leaf 7 is never queried when the CPU does not support it, even though + // the fake would report AVX2 there. + f = AVX2_USABLE; + f.max_basic_leaf = 6; + BOOST_CHECK(!HasUsableAVX2(DetectFake(f))); + BOOST_CHECK(g_cpu.leaves == (std::vector{0, 1})); + BOOST_CHECK_EQUAL(DetectFake(f).leaf7_ebx, 0U); + + f = AVX2_USABLE; + f.max_basic_leaf = 0; + BOOST_CHECK(!HasUsableAVX2(DetectFake(f))); + BOOST_CHECK(g_cpu.leaves == (std::vector{0})); + BOOST_CHECK_EQUAL(g_cpu.xgetbv_calls, 0); +} + +BOOST_AUTO_TEST_CASE(mlkem_x86_native_matrix) +{ + static_assert(HasMlkemX86Native(MLKEM_X86_NATIVE)); + static_assert(HasUsableAVX2(MLKEM_X86_NATIVE)); + // AVX2 alone is not enough: the assembly also executes SSSE3, SSE4.1, POPCNT and BMI2. + static_assert(!HasMlkemX86Native(AVX2_USABLE)); + + // Clearing any single requirement, the AVX2 ones included, makes the backend unusable. + const auto without_leaf1{[](uint32_t bit) { X86Features f{MLKEM_X86_NATIVE}; f.leaf1_ecx &= ~bit; return f; }}; + const auto without_leaf7{[](uint32_t bit) { X86Features f{MLKEM_X86_NATIVE}; f.leaf7_ebx &= ~bit; return f; }}; + BOOST_CHECK(!HasMlkemX86Native(without_leaf1(LEAF1_ECX_SSSE3))); + BOOST_CHECK(!HasMlkemX86Native(without_leaf1(LEAF1_ECX_SSE41))); + BOOST_CHECK(!HasMlkemX86Native(without_leaf1(LEAF1_ECX_POPCNT))); + BOOST_CHECK(!HasMlkemX86Native(without_leaf1(LEAF1_ECX_OSXSAVE))); + BOOST_CHECK(!HasMlkemX86Native(without_leaf1(LEAF1_ECX_AVX))); + BOOST_CHECK(!HasMlkemX86Native(without_leaf7(LEAF7_EBX_BMI2))); + BOOST_CHECK(!HasMlkemX86Native(without_leaf7(LEAF7_EBX_AVX2))); + X86Features f{MLKEM_X86_NATIVE}; + f.max_basic_leaf = 6; + BOOST_CHECK(!HasMlkemX86Native(f)); + f = MLKEM_X86_NATIVE; + f.xcr0 = 0x3; + BOOST_CHECK(!HasMlkemX86Native(f)); + // Masking BMI2 or POPCNT leaves AVX2 usable, which is exactly why AVX2 alone is not checked. + BOOST_CHECK(HasUsableAVX2(without_leaf7(LEAF7_EBX_BMI2))); + BOOST_CHECK(HasUsableAVX2(without_leaf1(LEAF1_ECX_POPCNT))); + + // Unrelated bits change nothing. + f = MLKEM_X86_NATIVE; + f.leaf7_ebx |= LEAF7_EBX_SHA; + f.leaf1_ecx |= uint32_t{1} << 20; // SSE4.2 + BOOST_CHECK(HasMlkemX86Native(f)); + + // Detection gathers every bit the check reads. + BOOST_CHECK(HasMlkemX86Native(DetectFake(MLKEM_X86_NATIVE))); + BOOST_CHECK(!HasMlkemX86Native(DetectFake(without_leaf7(LEAF7_EBX_BMI2)))); + BOOST_CHECK(!HasMlkemX86Native(DetectFake(without_leaf1(LEAF1_ECX_POPCNT)))); +} + +BOOST_AUTO_TEST_CASE(sha256_x86_code_needs_ssse3) +{ +#if defined(HAVE_GETCPUID) + // SHA256AutoDetect, given synthetic CPUs derived from this one so that the + // code it selects can run its self-test here. Its SSE4, SSE4.1 4-way and + // SHA-NI code all execute SSSE3 instructions (pshufb), so a CPU that + // masks SSSE3 alone, as a virtual machine can, must get the standard code, + // like one without SSE4.1. + const X86Features host{DetectX86Features()}; + const auto detect_masked{[&](uint32_t leaf1_ecx_mask) { + g_cpu = FakeCpu{}; + g_cpu.features = host; + g_cpu.features.leaf1_ecx &= ~leaf1_ecx_mask; + return SHA256AutoDetect(sha256_implementation::USE_ALL, FakeCpuid, FakeXGetBV); + }}; + const std::string host_choice{SHA256AutoDetect()}; + BOOST_TEST_MESSAGE("SHA256 on this CPU: " << host_choice); + // Unmasked, the synthetic CPU gets what this one does. + BOOST_CHECK_EQUAL(detect_masked(0), host_choice); + BOOST_CHECK_EQUAL(detect_masked(LEAF1_ECX_SSSE3), "standard"); + BOOST_CHECK_EQUAL(detect_masked(LEAF1_ECX_SSE41), "standard"); + // Restore this CPU's selection. + BOOST_CHECK_EQUAL(SHA256AutoDetect(), host_choice); +#else + // Other targets ignore the x86 queries. + BOOST_CHECK_EQUAL(SHA256AutoDetect(sha256_implementation::USE_ALL, nullptr, nullptr), SHA256AutoDetect()); +#endif +} + +BOOST_AUTO_TEST_CASE(usable_avx2_host) +{ + const X86Features detected{DetectX86Features()}; + BOOST_CHECK_EQUAL(HasUsableAVX2(), HasUsableAVX2(detected)); + // The cached answer is stable. + BOOST_CHECK_EQUAL(HasUsableAVX2(), HasUsableAVX2()); +#if defined(HAVE_GETCPUID) && (defined(__GNUC__) || defined(__clang__)) + BOOST_TEST_MESSAGE("host: avx2=" << HasUsableAVX2() << " mlkem_x86_native=" << HasMlkemX86Native() + << " sha_ni=" << ((detected.leaf7_ebx & LEAF7_EBX_SHA) != 0)); + BOOST_CHECK_EQUAL(HasMlkemX86Native(), HasMlkemX86Native(detected)); +#ifdef HAVE_BUILTIN_CPU_SUPPORTS + // The compiler's own detection, as an independent check. + __builtin_cpu_init(); + BOOST_CHECK_EQUAL(HasUsableAVX2(), __builtin_cpu_supports("avx2") != 0); + BOOST_CHECK_EQUAL(HasMlkemX86Native(), __builtin_cpu_supports("avx2") && __builtin_cpu_supports("ssse3") && + __builtin_cpu_supports("sse4.1") && __builtin_cpu_supports("popcnt") && + __builtin_cpu_supports("bmi2")); +#endif +#else + // Non-x86 targets and unsupported compilers never report AVX2. + BOOST_CHECK(!HasUsableAVX2()); + BOOST_CHECK(!HasMlkemX86Native()); + BOOST_CHECK_EQUAL(detected.max_basic_leaf, 0U); +#endif +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/data/mlkem1024_vectors.json b/src/test/data/mlkem1024_vectors.json new file mode 100644 index 0000000000..cbe5eceec4 --- /dev/null +++ b/src/test/data/mlkem1024_vectors.json @@ -0,0 +1,144 @@ +{ + "_comment": [ + "ML-KEM-1024 (FIPS 203) vectors for src/test/mlkem_tests.cpp.", + "acvp: test cases copied verbatim from NIST ACVP-Server gen-val/json-files at commit 975de31eb83d87039ec88934fdc47d8c312b892d,", + "ML-KEM-keyGen-FIPS203/internalProjection.json (sha256 d7a62a2c3476957f56dd8d24f9004ea6776ccfe995ffe71a65bb9506dc9c7b1b) and", + "ML-KEM-encapDecap-FIPS203/internalProjection.json (sha256 a556952ce869bb89c3a3196a701dad89647c193a34c86eafb61a9d710d5b810f),", + "parameter set ML-KEM-1024, selected by tcId. Fields not used by the tests are omitted.", + "cctv_modulus: C2SP CCTV ML-KEM/modulus/ML-KEM-1024.txt.gz at commit 6ec11f7d0ef5ebab45ff21ca7e06e643e5182728 lists 1040 invalid", + "encapsulation keys that share every coefficient but one. base_ek is that shared key with every coefficient in range;", + "the tests rebuild all 1040 keys from it and check that the SHA-256 of their hex lines equals sha256_of_txt, the SHA-256 of", + "the decompressed ML-KEM-1024.txt." + ], + "acvp": { + "keyGen": [ + { + "tcId": 51, + "d": "F3A706FAF090C03DB506863AB0B20BD8A1627956318E88C67EB875E8E7266009", + "z": "35D2BC43DD1CC879F765BF2A0C5E297889DDE910E57E2BB0EAE417B90AB7A275", + "ek": "8D0923CA8A2DA2B4146EC25321122B8A5AA8AFE0C03415273008A46EE83031E98AAAA125ABC75D3B30322560C197E75DD0E48A348099F7B2144D7B8A8660A4A97BCF19C0583BD9BB2123033CD7BB5A14B08B817831A673A28170F5F6443C0551913A327CBA18C3A053C4040250403B70AB9588832403AA0FC37665E04980FE1602E7D2715D9CBC00515DF432A4F5B32B3BC92AE3F31700166D498123E94576509B712B18491B1435EE7AB7AEB1AD30D72348C3CC083ABE24A8B12097BF32F792476288EECC3BF630ADCDAC6ACA7950D9839501A448500742BAE37F109203A809B2B960A307E25347A32C3EAB79288173A878789B296E9E8C1C28C5BB3AC472601C9765F7B77225A810C7B85370BEF4A5B079D2015ADA54236B8F33840675F9B2EB427A1B5974CD5C61B24010886C5A7BDA5BBED974AF7217F3338AD719CB308A8BCB1B6D6ED2A1643736C29095E8A8452A3A36C7BB5AE58CBFDC61529466A90F454ED6895B0861083DD1371999B2F559A3A487CF59A074FB49215EA6A6BE656F9AF17B121A2447CB7985590E9738842B899BA57AC311810AE2D9794F37483DD6BCCB64AF6D56588AE94665961C025C3AA2861974C236BCA4BD8FF5509F7AB774593E7C5549E57C2F18D15C0515094AD9A0DFAA0601E524F8231156B627BB25A0DAE04DACD0A66C041CEF400583FC13BAE640291A39A5C5CA8BCA1AD5C683CDD8290891A76940817DD8C9F52678780548E37A05806600801426DBB950C3B2BA34E24CC77864DD91B39F1408C716A69DF63342854E50A245FB50977B9410DED2C93F86B1F9D5A78B87BF81E51CA620A7E8566B19AB700964A40E3266415228D432156E5CBDF52364A90483A55C39B3FB16FC7465A3F8AC801B70B9FB28B583444BA5C1A73722D417A9D6D9B7DEB08BC6B330FF27CF61AB8831E27758C64AF3B12150CB7B33ABC29858106D63686D8762459ABF9413850AE53ED6313F76F83D0FB8AB34374E7DF693E4A1B3E5A8AD0CE820AFE1CF401ACDE650A8101B0946022D52178E19613C42B88B07CC04EAA81DFB28AC9DC076236B67219A30F8F945DD57BD2F335C52D59372308D38993467DB53DA3382B74867B616481BD0091A2232C1116DC88A589DB9107224A681008C67C589186A6929549BEEF92253DB02B0C8AA9F9C875A670266C72BCBDB4F5625043703C1A0457395832E4C335180462ED2220C59E7361903C107D85457F6CD82EB820D0855D97675C2E0151CDB73C2885DDB7849D74541580124E890116A65BC068093B57914E20C937C60A3EB25576F1A976A9583839B672144CD4A45C3477A45C29B4E0BC2BDBD206585C9B7A7741C8B6B5793A92797A15AE7A5B73A74B2971463634BA52AA792AF05530730B6D0A89A346156B733677932BD36593A7496130CC458DCC5CA987C21960604EC8A8C5396056680CBF3F1AAC4F401AA5029FB2150434BB4706C31A2D54E4297939FA7C9C6F85700613CEB65C7F03AC56EB86E2D27CCC6DCB7B9394DCDB942FF222D86958A996C0CB6A8A44F97A70441C95FA71250116EEC20863C0B5A643458788AB001F8869D909922F51EE547A1E889255B3A0599C65842E5AB8D73872F053BC62392EA53896D328102D460BF1609583C22C3B43780EC6DAD0319EB4A5A65B4756C3CB40EAA935183BF8BD46ABE76BA46E199103A5313C3235F49C915E097BCA804DE680781D8365731BEAC6789A9203FB8787C4C070E00A13A6722A66A28236DB179825653D33CCF898B72C6B8450D97AFD3276BB13340519CBEDA708D12A858F54C49F4547195B7788A9150B2649E36AA394121926D568A488B16D3557B2A32AF57D11FC3373F80A28C0723273D362502E7C428AB44D3CBABF9EA585FD1BD0C9846556A1E196B78CF951592984A0A8487A78C2317D7ACA4118E1049750A0788F0D66AD9E48E34731130ABA0B427360A856D96D80B3F028FDD3ABA9035C10106BA1C0934BED36C6D7C7434249654EA89FC22137F4AB903653B75FB25B6F01635E6CC7D39CF1508690562826B49B6FFC59E0DD35022E541F8BA0D304AA5B4E20606907C424395666C54ABC2B8FB009847C86317685000C231215C8C15945860F6A85DDB98A8C3A527F2749D3C027E694E8F0B0F0FA454913AADB635AADD452F7128BF7752569669A8B93290EB92E78F6ADFF23E89F57F3890753B51F12F3F3A8A654E677847", + "dk": "6CBBB491D7B3A3439F53C9027BD727A0E1C78004B4CDB66760E50F1BF97FFF0139F82862F1F3178A077B6AF77735CB802E91316BC207B9F1485FF7A868891E003BB9C6558BE6052C0826AF1CB5836B94C7D5D714EEFC7CFD983E9884B71348499DCA05DEF33B49D1C14BBBB95AB195BD72219B18BD0C704275CC94AB06653BBB9E0EEC74181903A0081AAE068B81EBAE8925335B8134B1480A60F758282C2F011B909641266774513A19090BC672EB3A0708CB577F134151C155C1E560D9B335A35A4D75F223502752CD250C9A7778EFC76A52709813A24679B987A6172F1714641EA6902D364F96357E92B44275EBCCE7F6207BB0A215E98B3AA92DAA46A39AD9132F3874C6239391E716BA600E42BC7084425051F865756A89DAB46A76104E183A12031B99BE86257B583C2E88104FEC3BFCAA50F238BEEC20ABF31219685885F2C3CE2F80C09B874E73A8712D35B8EE819D84B1C59A37947C0B9E25F066EE030ED28172FD5914B562408BA9B9DE9BAB2B95A4F30C22ECE458E133371A693D9458058265A75AD6CA5707617B38A1B6F396FB199CC1C77100741DE37500F1BBB10B9204EEB7C3F5B91A1CC91B6D4072AF410957358ACB0625488B9F6A9915758B7CC9B4AFE7D5AD6A95231A7A711B3A8996E23AFB3695B45B6D919C771D7685D90A7550A9B5B1E7538A365451A5A815009E49326C6541CAF0BA685056BF82EA4AB504161B555568583802800554A9480602437B0B9677FAC638A2508154A658E0623C62666E078D21A5A545A818D66C9547B9749458B735A4AD51812950E8A625A1B2A753ABBBA16415006300AD32D34C8B0B3B2AF4D1CFBDE08B4F06C70DE4BA28FB3763745CCE133DD2E488F89A51E7A586A86AC928867B0541438AF5645DE3211AA94ACA4A1095896EBC363C374A85565C1EBA030B5EA21B9B8A140E131ED07580D5B8590446988DA65939A823A84661B26CA12461AFE373519F43C13987A96A9069CD6B9177B205B25B5197DBCDE66C6457892C1DF96FE50AB7EF3B3DAB0A87436B3451476C44AA22AEF918AE5A1A8AE46920263032E3BC5DEBCFEE43B8AA290CEDF06966939EE6CC7C6AF83CB9D90CC889AB8B5A3D4C224E73916AE057BF5455528CACBED31B554043C6B0EA196854461D2644461470EFC55080E326FED49D62B5A4BB06B14F505EB4486C9A1B033B927836DA656C2475E5B8B823986DCA99B298512D94B77FE0588DBB96BA53A67369B82C2BEAB2CCC0103E549317A99DD97B598FA0A506184EB55C96B17302533C7EC3334E3CEC2BBB22BE0F89442840884FAB4A766A41BC8061D4ABCAC5190C729309AC6151BD434B818BC1E7D0CD435C9CA6DC7182228FF93867134A5CC65C6167C4A8BB830E7FEB863C15BF87B259EB0518FF5248792A625F37C589589799746D36197DC63AABD134A00EF1B49C79A2DF67B28E6B6E989CC9D0577EA5AA1B2FE64B322C7A8B241D503523D875CCDC479D28E3C01BD8383DCB2B684281E6949171F518EA97009237B5CD42AC8FC8A571716EB784B336529DB67728499988B73B80EF5C6D33F3B42ADA7A0A36A7C6A8A0DFB804B3D7BB05803957B913827A16F52ACDD7650EE2971A85DC21F6A94A5B31988E4438A9C1733AB8C48C32653FE800BF81BA66658CD2001ACF3A83E3A0426AB6C3B2600A53621277A43D723AB96758B120F120248C15EDF4294FC38F96EC490BD52094655841D9355038A379D8B666875171F39218E21D37023F5108B6D15389F07C1241E15071C43CD7D160EB5341B719C2055424EF6CBF66815F84943C9481CD1DE685A74C5B9DB37A0713952FD4A67BEC8CA7496F6C9C5FB84A8EAFE68D54654C6BAC9C3AB712C2BC32C0CAB072C18DE3CABBD9224DFAA305DFDB3F85019D7B988577B40F27514822DB42D87489502C53CCE6A095B486D9143D6D160B1C7313BF288913657698117EB145C0C9C55D66C15BF1A2B268F6A5FA259139551ABFE3468F1CC49C01B4232543DEF0BF9AE503B998536310751A8BC302B4B4440B835233797C967E7F9308155B27BA63C65E08BDD1946828196B30C042CDA0B56E4008215AC51F373604F5930BB66297C32BE41A162CA21471CC86657B5CB5307D89091DA9A5137F1C1433AB8AC1D59DE421C7C17A883EAB99BA4714F841998D0923CA8A2DA2B4146EC25321122B8A5AA8AFE0C03415273008A46EE83031E98AAAA125ABC75D3B30322560C197E75DD0E48A348099F7B2144D7B8A8660A4A97BCF19C0583BD9BB2123033CD7BB5A14B08B817831A673A28170F5F6443C0551913A327CBA18C3A053C4040250403B70AB9588832403AA0FC37665E04980FE1602E7D2715D9CBC00515DF432A4F5B32B3BC92AE3F31700166D498123E94576509B712B18491B1435EE7AB7AEB1AD30D72348C3CC083ABE24A8B12097BF32F792476288EECC3BF630ADCDAC6ACA7950D9839501A448500742BAE37F109203A809B2B960A307E25347A32C3EAB79288173A878789B296E9E8C1C28C5BB3AC472601C9765F7B77225A810C7B85370BEF4A5B079D2015ADA54236B8F33840675F9B2EB427A1B5974CD5C61B24010886C5A7BDA5BBED974AF7217F3338AD719CB308A8BCB1B6D6ED2A1643736C29095E8A8452A3A36C7BB5AE58CBFDC61529466A90F454ED6895B0861083DD1371999B2F559A3A487CF59A074FB49215EA6A6BE656F9AF17B121A2447CB7985590E9738842B899BA57AC311810AE2D9794F37483DD6BCCB64AF6D56588AE94665961C025C3AA2861974C236BCA4BD8FF5509F7AB774593E7C5549E57C2F18D15C0515094AD9A0DFAA0601E524F8231156B627BB25A0DAE04DACD0A66C041CEF400583FC13BAE640291A39A5C5CA8BCA1AD5C683CDD8290891A76940817DD8C9F52678780548E37A05806600801426DBB950C3B2BA34E24CC77864DD91B39F1408C716A69DF63342854E50A245FB50977B9410DED2C93F86B1F9D5A78B87BF81E51CA620A7E8566B19AB700964A40E3266415228D432156E5CBDF52364A90483A55C39B3FB16FC7465A3F8AC801B70B9FB28B583444BA5C1A73722D417A9D6D9B7DEB08BC6B330FF27CF61AB8831E27758C64AF3B12150CB7B33ABC29858106D63686D8762459ABF9413850AE53ED6313F76F83D0FB8AB34374E7DF693E4A1B3E5A8AD0CE820AFE1CF401ACDE650A8101B0946022D52178E19613C42B88B07CC04EAA81DFB28AC9DC076236B67219A30F8F945DD57BD2F335C52D59372308D38993467DB53DA3382B74867B616481BD0091A2232C1116DC88A589DB9107224A681008C67C589186A6929549BEEF92253DB02B0C8AA9F9C875A670266C72BCBDB4F5625043703C1A0457395832E4C335180462ED2220C59E7361903C107D85457F6CD82EB820D0855D97675C2E0151CDB73C2885DDB7849D74541580124E890116A65BC068093B57914E20C937C60A3EB25576F1A976A9583839B672144CD4A45C3477A45C29B4E0BC2BDBD206585C9B7A7741C8B6B5793A92797A15AE7A5B73A74B2971463634BA52AA792AF05530730B6D0A89A346156B733677932BD36593A7496130CC458DCC5CA987C21960604EC8A8C5396056680CBF3F1AAC4F401AA5029FB2150434BB4706C31A2D54E4297939FA7C9C6F85700613CEB65C7F03AC56EB86E2D27CCC6DCB7B9394DCDB942FF222D86958A996C0CB6A8A44F97A70441C95FA71250116EEC20863C0B5A643458788AB001F8869D909922F51EE547A1E889255B3A0599C65842E5AB8D73872F053BC62392EA53896D328102D460BF1609583C22C3B43780EC6DAD0319EB4A5A65B4756C3CB40EAA935183BF8BD46ABE76BA46E199103A5313C3235F49C915E097BCA804DE680781D8365731BEAC6789A9203FB8787C4C070E00A13A6722A66A28236DB179825653D33CCF898B72C6B8450D97AFD3276BB13340519CBEDA708D12A858F54C49F4547195B7788A9150B2649E36AA394121926D568A488B16D3557B2A32AF57D11FC3373F80A28C0723273D362502E7C428AB44D3CBABF9EA585FD1BD0C9846556A1E196B78CF951592984A0A8487A78C2317D7ACA4118E1049750A0788F0D66AD9E48E34731130ABA0B427360A856D96D80B3F028FDD3ABA9035C10106BA1C0934BED36C6D7C7434249654EA89FC22137F4AB903653B75FB25B6F01635E6CC7D39CF1508690562826B49B6FFC59E0DD35022E541F8BA0D304AA5B4E20606907C424395666C54ABC2B8FB009847C86317685000C231215C8C15945860F6A85DDB98A8C3A527F2749D3C027E694E8F0B0F0FA454913AADB635AADD452F7128BF7752569669A8B93290EB92E78F6ADFF23E89F57F3890753B51F12F3F3A8A654E6778479370FE5B05DDC92C939F62CBDE4C0FEA36F45CD20C5748CF3AC891A4C260449635D2BC43DD1CC879F765BF2A0C5E297889DDE910E57E2BB0EAE417B90AB7A275" + }, + { + "tcId": 52, + "d": "52F2DF12173088140CA73333D1DB4341949BE452049028EF0DD904BC7E0DDEF7", + "z": "CC96DE9E1F0506BF490A154A6DB46BD2BD1413C3D1DC163190FF171EF0D11E22", + "ek": "88C29C2A91CE80B13EEBE182AA709A1694A456B139DEB1977D531AD2722AC7D2B356AA2684D767C60012B1A2345A69AC52C7A814858E8C741A80C4B5E942CEEE0045B957748E2847EA835E1FE17C70456DBEA13FCEE0384BF431225C8A6B499624F086BF71518C8C269640B436E7783057372825768670B2478C96C4A1A2ADCB1F83A057263362476197C5D878D76B025F83AC802CCC02296A99296CF6B23C5617AA95387C2B0071C23AB7BF614371BC7001E29D7109057DC03053FB38C0084C48890612369C649C0BA3C2603E377C84D15B6DB6BB6E9513ADA9A47210861AA055D14683F0556729192CAEA8AE9A538FF810596D64BC0964B380CBC2F5D706A0A8135C1B87C3480C6D54CFC28A53C339666E24213D33A62EC872D803103987A36256840DD98DA8BC17ED42C34F5C008F6C678C748F3F7404D17B46F29012F079A46A0ABF77C04538254ED4B6BDEA929724AA60014674C427C55B1B0192CB284ED02194F9615417B08A9580DCA78944F43094E48AEBBC8BF0126310D9AA79530457E04DBA9B0D02B045AB28391F923C25CAA69AD038B4586350613D9AE22896F2979D56BBAFDAA294041AA42C21B9AC59E41CBEDBC33AD6996C3451B7DF5B9AE2D003F5092B91847290336A62E804B5F167D7CC4E31F40F6C640C10EB99D7F5519C80147B77843CD083B3B5C03718B1A7EA2DBE25ADBD56762E9150B46B6B4E82B769C384CAB39B3610070C6B6E5981C3B182105981A50567514CA15EB44620C231740FEB6BCBF4980F7B1824A5602A982A68BC8961BBC441B111C93B2374EC88FEF30D4931AB0CE3B5BEA680B1099CFA79B3FFF2B56BB3031A8C07519C7DF4F75A8C887C6A1B7249A700717B2798C02F0EA04E720752D8323376E1635331B07435CDD5BA6D7228281612BF060C694BAB6A00A8700977518BBAC8195853FDD34D44E29FEA7286DCFABDF47C9B2F2347EE7B3133CC275E601AA1874BD6974894C99D9659C1910030AD033138A57F8D9B312208873F733DB9C73007F45C49A007BFAB03AAAC3F107C8C0A032CBE24602A2BCAE4D2658D21899EA89EC5E832C0565701E5BAF0DA89E1B55ECAB7802EC830B8268F6D9CA3883C97AACC20FC4861EB59707AFA18A7B24534956301F238DFFB68BD20A8264ACB6BE4918C85282F63CBACA87A3530515041C5A00CADAC1A0B365B8A6CF626B30A49338461E94B44E20C250A23C7473952A20404FC17055E124792621D417CC7E20381BB24A701D428B1643BC0054B5FF46BA9129C0126889156BC6418A4C3B8C6D41B3D8646A915592305C30886B3995400A2A80771A494795D4B4B9EA6C21E0B21E0933F9F7C5FCB3B4C20AA303F655CB003526F8B04B3C85260C00C8E316DB495417DA22BD3A975FE784434C04DFFA545F91555F01033696BC4B87ABC0665B7A5C972D9A08FC8911147DA3CEF156B5CE85391161F32D09F6F8C0F8A845E268AA044E150944247E2B4354CDC72D3EA01B5970A6061B4F83563390BB33CF1CF9F910858DCCDD6C65E463607DF5289A3684440546816D4C2D59B214A15347EA9580310786A182EC6C2ABB73062FA281AC6806A6A042C0E95312027AD66EA6E245A9F2681A24CF505D5DB43F8A8C39BDC12E07C9F805634C9E5725E18A4229A20A681A0CA5144A8611DC372BFDDF102E5254AC1955450B94CE601AD68B5156996AC8C8A1BF1962A2758B4CBAAAB3A341E9C7C19005AB82BD236CB6294AEF22F1A1BBB32DC6EBD1C15FEB409E6C495EF1013E979981DB92BFF97BBB3B34DC385176C440B29637F68540BD10143AC33CDE4D2331AAB1DA5D9C7A41344A073B41FE536D5890239A96A319852593671F6A22E9480775F265C66D68521A73E9EC83DC6AC2963D3B70C3453861766048B18A135374ED53521A458E1FB024100116F434D1A261CA1C579F4311549243AD629525EBC465F8779C7E1B50344009C487D84A7B2683298EC39C27AF80FB3F28C61D388959A99721857CFD30F98F43A19716CD9E698EC946E69FAC477CBCB8E060EB9C96D3EE64B894C748133A9BDB5758CE45DDF791A68D87D1EF70D9AF626C3A909451A132C0702F0731D5D5959A732684C1759D8D31092669E70ECAB9F804CEF4CA416563AF1F27A219B3FD6BC482901201ADA9836B824AF9BB870DB2C9CCDAC525933E58A16C00B373DD81A3B89C39E9893960D7F6CC246A53DFFB9FE", + "dk": "09766134E63C40A74D70107A121B44651CA43F837316F63F01290286304DA07B45FBCC418092B9FEC00EA7C88A4F20A0ADF9188E23707C88CFFDD6CD86F35A9E9A3D4F842C84A5C8662048395099037ACE5FBA75495A6370B9118D11324788BD4A841C90D219AA6BBC5C829EF2C291AF3AC99CF08672D95225DCC4AF7AAA428B7EE7201EEF8966A10B88067988F62C27FB52470C036535D762DF868BDAD167A383C74246B218CA68EF9B5106979E30BA4B278A731EC370750C2B37135D39B32F811AC667E717BE098B3F278F93815B2270C01723C322F1A3F4D8B1026A39808A9DE51B8453507464E283694BB937778161E604E127CA85F10CEA05CD28458503F100CAA530874BA500A3102CAA3802291DBCF72D457637C557C8F70012EB78B9DF1948F6055974C81AF1FB6DEE601FE40C3D185251441437582B57AB29627BCA1804C3B53EA76325B925982C8CEBF50474A5BD00E43274437229D94494B7536BF473435A1F3C552C45064F4DF31CAE2338394BAFFC8354D1005CDF0CC7D5A8782906135739079DDC727B55A67515C609223FA3C34B2EC24D033397B0D8C6777547E5C635F5E546DF52A207D2CE1AC3A9F85179A8898050078E93093B24122A7F553CEC39BF42B35A07F128288A23C91C9232B835A31288461640CFAB5EAF9A9BAA63AD40044160369C899737FFC8590C83A20CAA997FDA5BE4367DD00B91131C0F4EF61B0D207EEC750A8F4ABD5CACA994B779183051C6F52DCA8696EDC65807758ABABB04203234113052C7F894A478B564EA69D14A1970D2BBB437908F2288D0084EE7D538A3FC628E311D5D558CC2113DD1762DC3FCB336A899B583CEB457A8D55935F6926E1B5A2968352D6B1A35C986915C248E8C193EA3816300C67173930560E8455C560F3817C58848CC9FB3888037B449AA9268876AE4F665B5838329387431408068125F7DA65027917AFD42BD01389FE1F64BEF5B0EC688396CE58D622975D69818888029F6C0005082B9AA14B0EBF9188CE93C3B127237597FEFCC4DCF15B7F70518D8A9B34F1826C5B01CA9F48D1122AD41139C2E625F37DBB431957CD4DB6818292436846745A933014200EFB80FA9B4A4D38824CDD05C1422243A440126855DA2948024E136A23732B98A3319809948B9AE25ABC9B6A5A96AEC8A50167DF81C475EE586DC137DD5ABB7D91A0211258A82982DE2114A808A4DA29B79483A26F3CB73E5985BD68010F233AD7C0464C39CB0EF02938F036E1B099313318E81089CE78C1D008496B69764C08C7BA7DC00328846268B0709104AEBB8C757F22A8A45274CF75E80B1B377A971940852DBE1579B59B7B7901E06ABB7972B60C9357CCC0024C14341EEE51502D79D51D87A6F42897F5955A2B44C73766662F940D47B7DCEE02E22B21E138014D735404A13523EC23ADEF21A3184939AEB0BDA5264798146E1702F42C58185C3B063E1B884455C2B508DE26C5D334C0020C7430D774AAA85B2F9007FA41495A50BAD7315C4CAD429BF2B5B733690A8E422506C36FBC877708BB9F51CB899A48D3885B73933972F97A31DF12C37E2CCFF5608FE500F0B44585CFA0DFFD6932A2096CFE13518FA06B2E93C6F31CFC6E5C1973C2FFEAB73C0D5AA0F7A2C1B922A62065A423187B30B1DD261104933A036A89BF1357961008707D7C4819C6746E137531797E2B554E4C06F57A1811F9B1C0B459B2CBBC4C90293821AC6627B20A1BB8F93752222BA795E0448735840DE552AE26C2B8BBCAFEFF61E0B7898EF606DF88156E9540CCDDB6F4ED180E8E95D2404CD7FC15DACDC4873A2276DCC1892A36B4EB8C973D6858A478FA669052A04CA890C4FEED830553670E5F07FC43C59A0584802E541C26AC4C3C5774F79CF2251274C54BDBFA5B726CA710530C87EE49918A41BC50A3CE5A67C05689646516F2E77A832622EC11C9A69F70B4D014FF35182FD8B6CF2E181710C2EFDA85358712962AA2F04C58360756469AC4BFE861EA019923B68A4B63975064596F39B83FA7083620C20F578B7C96CCCF4FB5D485726DECCBDFBBC6EEEE2185FB46379575B5A875B28653A6945A7409C0402EA40A386A4F0F47D5BB1C972E96216384F9C48489CBB9DEFA92B2B983E11B796F0F176C6D5264290AC88DB13AEB93C022BC8C2266E88C29C2A91CE80B13EEBE182AA709A1694A456B139DEB1977D531AD2722AC7D2B356AA2684D767C60012B1A2345A69AC52C7A814858E8C741A80C4B5E942CEEE0045B957748E2847EA835E1FE17C70456DBEA13FCEE0384BF431225C8A6B499624F086BF71518C8C269640B436E7783057372825768670B2478C96C4A1A2ADCB1F83A057263362476197C5D878D76B025F83AC802CCC02296A99296CF6B23C5617AA95387C2B0071C23AB7BF614371BC7001E29D7109057DC03053FB38C0084C48890612369C649C0BA3C2603E377C84D15B6DB6BB6E9513ADA9A47210861AA055D14683F0556729192CAEA8AE9A538FF810596D64BC0964B380CBC2F5D706A0A8135C1B87C3480C6D54CFC28A53C339666E24213D33A62EC872D803103987A36256840DD98DA8BC17ED42C34F5C008F6C678C748F3F7404D17B46F29012F079A46A0ABF77C04538254ED4B6BDEA929724AA60014674C427C55B1B0192CB284ED02194F9615417B08A9580DCA78944F43094E48AEBBC8BF0126310D9AA79530457E04DBA9B0D02B045AB28391F923C25CAA69AD038B4586350613D9AE22896F2979D56BBAFDAA294041AA42C21B9AC59E41CBEDBC33AD6996C3451B7DF5B9AE2D003F5092B91847290336A62E804B5F167D7CC4E31F40F6C640C10EB99D7F5519C80147B77843CD083B3B5C03718B1A7EA2DBE25ADBD56762E9150B46B6B4E82B769C384CAB39B3610070C6B6E5981C3B182105981A50567514CA15EB44620C231740FEB6BCBF4980F7B1824A5602A982A68BC8961BBC441B111C93B2374EC88FEF30D4931AB0CE3B5BEA680B1099CFA79B3FFF2B56BB3031A8C07519C7DF4F75A8C887C6A1B7249A700717B2798C02F0EA04E720752D8323376E1635331B07435CDD5BA6D7228281612BF060C694BAB6A00A8700977518BBAC8195853FDD34D44E29FEA7286DCFABDF47C9B2F2347EE7B3133CC275E601AA1874BD6974894C99D9659C1910030AD033138A57F8D9B312208873F733DB9C73007F45C49A007BFAB03AAAC3F107C8C0A032CBE24602A2BCAE4D2658D21899EA89EC5E832C0565701E5BAF0DA89E1B55ECAB7802EC830B8268F6D9CA3883C97AACC20FC4861EB59707AFA18A7B24534956301F238DFFB68BD20A8264ACB6BE4918C85282F63CBACA87A3530515041C5A00CADAC1A0B365B8A6CF626B30A49338461E94B44E20C250A23C7473952A20404FC17055E124792621D417CC7E20381BB24A701D428B1643BC0054B5FF46BA9129C0126889156BC6418A4C3B8C6D41B3D8646A915592305C30886B3995400A2A80771A494795D4B4B9EA6C21E0B21E0933F9F7C5FCB3B4C20AA303F655CB003526F8B04B3C85260C00C8E316DB495417DA22BD3A975FE784434C04DFFA545F91555F01033696BC4B87ABC0665B7A5C972D9A08FC8911147DA3CEF156B5CE85391161F32D09F6F8C0F8A845E268AA044E150944247E2B4354CDC72D3EA01B5970A6061B4F83563390BB33CF1CF9F910858DCCDD6C65E463607DF5289A3684440546816D4C2D59B214A15347EA9580310786A182EC6C2ABB73062FA281AC6806A6A042C0E95312027AD66EA6E245A9F2681A24CF505D5DB43F8A8C39BDC12E07C9F805634C9E5725E18A4229A20A681A0CA5144A8611DC372BFDDF102E5254AC1955450B94CE601AD68B5156996AC8C8A1BF1962A2758B4CBAAAB3A341E9C7C19005AB82BD236CB6294AEF22F1A1BBB32DC6EBD1C15FEB409E6C495EF1013E979981DB92BFF97BBB3B34DC385176C440B29637F68540BD10143AC33CDE4D2331AAB1DA5D9C7A41344A073B41FE536D5890239A96A319852593671F6A22E9480775F265C66D68521A73E9EC83DC6AC2963D3B70C3453861766048B18A135374ED53521A458E1FB024100116F434D1A261CA1C579F4311549243AD629525EBC465F8779C7E1B50344009C487D84A7B2683298EC39C27AF80FB3F28C61D388959A99721857CFD30F98F43A19716CD9E698EC946E69FAC477CBCB8E060EB9C96D3EE64B894C748133A9BDB5758CE45DDF791A68D87D1EF70D9AF626C3A909451A132C0702F0731D5D5959A732684C1759D8D31092669E70ECAB9F804CEF4CA416563AF1F27A219B3FD6BC482901201ADA9836B824AF9BB870DB2C9CCDAC525933E58A16C00B373DD81A3B89C39E9893960D7F6CC246A53DFFB9FE7708C928659DFE465AE813012DF3FF6EECB1E081391B0A94172AFF81AA3296C4CC96DE9E1F0506BF490A154A6DB46BD2BD1413C3D1DC163190FF171EF0D11E22" + }, + { + "tcId": 53, + "d": "F2A1E83CAD0055A85EB52F1BC1B9EE24F7F58D9C2FD099B5D38C9F75AD657578", + "z": "26D7035CD963C74AB142936A91EABD3074C583654945F9F5FDC29F754B06EBF2", + "ek": "9331AFCAFB47F0E805E29B98A2A3C2D9D49AFEA806FB1CCD5E24105FCA6C3CCC1551B613927AB4CEE284CB53968DB3767C2154295CC479F06A4560385BB16B29273386320FABC520E5B36FE4D85814079E7E835D7935172DC893FF9277B177B7D8A9ABF9C66DC6A52C25852639540220FBB4E2E576440351F281712910CBF6A02F5A1B786B006DCE399794191D9F6BAFCEF1545E3A960CD204868463FF2B8A20944211408CA82AC182D05AB4C32D6A556FC36381BF163F69377E0028C8E8C70F0AD7A526CA274F9B65BD3ABA9164BF34F74B47C56966E4BF8E1B889E54CA29A037E67C28FE2A2B6710554D3813B55131B2B75DE0A419AB8888151876CD7B476CCA91FD83480D150CB3206F6A40B95E3412F9E7288EF051D3C99ED40B9E29A0A65CA1739F88122C72CB15950EB99302CCF18EDD17446F165930396F85E84E52E221B5668431B55CFA08D048A1386D450938D48CC2A94B38EC1F3F6AB2FA5565BF3B4BF1638AEB6A2440E5CD840C564183331E1AB7CB114A409CAB6D88C84C4545396360B1FB37B36CC405451D7FBA970FA7433B879EA8CA0BFF7C4BB0B98B255412AF695511A9CE168BC4FD50C27E121CA0221DE918778B941019D32020EC2F306C7BDAB38F9518BD51FC374F84B5E4B700F05785AC39CF9CBC0DA4C52150C9485D4A6BB4CC3C54A21DF63C2044B97C8709A5DAA82933197F1EA24A994370A2C9495E0B6660A10AA62BAB0BDC4DA284CC39B7C0E0464798A7365EA106793815B7B2BF02EA70C5B506CA343C38386BEFB69D6E31A1678BBD497BB1E64075B4158205280802234AE612A588CC8B2EC24AF74C5EAC085EEEDB13B827340AE29F348A9E31A20244D20A4B2B41F46A6A4C8293A2B8C5E3D3C49CA1AC0724969360BDD8A0822DF35D16A7CA48F90BB2C7972CD70B8DFB6CF5F1453F69266B04C2C32987DE0565BDA632F4B257CAC7CE7953A70B29681BC4602F3ACC3CFB3BFB63A135A758E461647E344A454AB3A1A72D234268D8870ADB84AAC250127E2AA593E79828848C112C26592A89C4DA358BF20571144C3C715449B8AE46606A51E150F4CA3593BC9ECC8A0382532610328253F488E733965C1742A36626524809C548397A922E79B82FC35B1D0EE41975CBCFF6F38BAAE28F0B452BBF792CD9023EB893767FD36E855774F1600399DA75889B03A95A21F32C0162540CBAB08B82B8346CA43B3E07B44F8C5AE6255CC379B3FE631A434349533B85C164B44D7805DE49442D113D238A9856E1897F0352130369387C3F2BE95435885A117072EC762656BCAB6EC1BE33C80C7835C4670B1D142767E23BC0AE6247529A3C8F348BA37ABE3507AC93C939560A445CE696881248F2A6C0B19B3900DB2E803A4BE0D4231F41A65702502241BBC724AFA441991A0A46DE7C62F6A220E457B2D876C44BB0A5F3116868E2B6D63A44A5C55F53F83E10E534606B76108CA4BF12A1B97A61A28B1BD184A9C2057655D460B65C407CB93272F44509C432DAD7B03A7393CFC160861CAAD4E9C81B9CB1085539D8A2C1855421346ACA46F46C3D12696237B237398B4015467CE7A6030A9666736717E6A740152BC2939998724018525D74724C5BD35B2D1887F9D2C6733A1173129DF3AB625E48C6664207C4D4931AF4130CC1CA0EF9160898890B49674F92A9464837F5597A7CA26BA854A4E0FB756572766DE2B9FAB3820818CBC600AEEF588B59B416F9568E19FC4B03902CC13547DC18197CB1AB23190424EC80633A5D58D861D9BCA5450102E52247A74594936153723C0A561B64F4E75D90E805DD7087FFD3420B4A0CC54CBB32944693A393511CA69E2C2B7A3A5B58784F2E999840BBA8C2C994852151569CA0949616DAF73328B220DBEB7D43794B1373CA3E8AC69178653E889D67E8546D35A40478B9C6410D9376CB28D6080AD41FE834B970403B8687C29011C6C8E24137F22FB7192443190BC2B095594C85C55389D4D49722124F07067FD9F34319565392799C27550934127538524E1655A03F697313130CB0F6799DE565A5DB8B7C23C7A8F53AA7714C7094A86E67894A75A489E6C414B67C52030F40EC5C599969D8E00B7C03492C8CA6DFBA8D641307921712C2840591FA9A847B39794BC22747830FD956C0C17B5C974FDCE96F565355B500A70C3D86EA32C9290514B1373184AAEDB75D90976FE6765F4747BFC946F1", + "dk": "8EC50C8143608F9186988602849A965D48645CA331C1815E506A1C51D73AAF18BF92D39BB18A5E681447EB78883CE48E6C375AA8E36CA82ABA3E188D8D795DDEA50B655B2E9409C35931C934BCB5820C6D2CE975FD955A9BF40695650004B4C62447AF3E1082DEF45A4AEC121DF9C8D36C8264034E55F926BD5053421C809041C1701650EC0959E4EA6D09898B8BCB90F95209C3397525D984F6079063901BB82478F9B1CE08875449355FB16A3921A8CCC9547349C5650BF49D3D9BB523092607F2B4BB437B1A969C35525F321333E9436B66A08699A9CE4017C6D7485145C408FD412E69463A558927F0763ECD8B24C344640535CE10DC41780213A3DB5930DC40CBA012CA5B22B2052C3192B31C9230F4D12FD5767EC1700775C478E263220E40C65EEB36FAA1C18B530B6C5ABF6B11397E694763C8C781CBB3C6C767F24287BCA224370C9D5E9066609078BB4B7BAFA46328F9C203D7AB0419B9177295DA08B802D93A97DB3F1FE964BA7B78F4A5404657603E5A7B95971A1DC2A1B00B54E7FB1C180CCE49917278C961FB61009200070ED90BB752A55661B011E35382A01C57793AFD864AD8D3ABCD752F78168EF12238CC520A6845CE3CA5771D207B3A2A594E687867783E80B057423118753BBCD3926B6640321938A40E793BADA8476D64858296348BD7038DA24E65D4B6FC4C907ED69E0159C76AB07F9111035B6187A41024AB26966F7B64673860EC9AAFEF29C0D7DC8A4DE31344C41D84D07B3A6525A5A884BCA8870E758E4A6C02150505A7587EB6288C80910C0E999C0309A34DC7C47053AA1F01848BC15059E65307207F4C4974DE0ACE30C2A3021B5927B1B0787002AD0A4CABCCC2D30865DB9150F657025658C3A14CC7D088BB1247CB2097A521FC46E75B158C55509D0A6DC8FA6B53016F6C28939C889943075E1B26646E3BAE5B0A52D3553E3D24AD62396AD18CB53C896E8FFC148F57CE11A39677052C134B8D0BA186B6F9BB94A2C96D06627ABC7E29660B9DF9C9B9E1786B39B9148A982B456F4F426368845D0377C043036252A4BA1B61487958A9E05CB272072CBD0927B4667BA7E4837E4B7405B9CD3EE43F7575B8209BCF122622E7E99B414C3AA32C9DEF1B200444B82BCBBA8EBC079FB42E00A830D921C45387BE4F86BC05D64BCE8B948BC61D6FB0CBC9552591F99622C50C498376A106414A6109354184EDAB57D8BC27F6761E18E12A12335BE89944B737C3B3BCC3227148A5A60EA32B83C8C5692B35BD68D72DF7CC264B519C52437B357899B3409239C02251F83460F6AF9B7C73AED1B4EDEB322F794555A0CF5AFC233260A10C6529B5AC55582B153461C25401027748B39260015E00184D553F45372613631E2ED0B4633A0CC52B6F62454DA414A49B17BC88225EC9EAAFBD121494652C925BCE19CB53E3216040B879F409049B780098A189668A7C46751C714B38A6E59EE4D7620AA3B3DA3BB3FF3C4C04E45F3ABB3CEC417CD80908EA701D90A21E22210CB7188D739CB13DFA03E35327C1912453C5709C1B6FEE2CBC5F8714E1A43AEA7422D6E63F247C8065A0CDFB782BDD2CAA781B4B8B76BB7885C7B446BB40A52BB2C8112F3C5DDD077FAC625D54DC26B0A3B52109C8C80137302738B511377376A74E07C1D2C3108180C6ACE97E61F6A343D6C1CFB76D5B18B756179A66D51483B904E5B88FFD700971A37E913829A2F366B8386BC342BB49411890C958040B7E66CCC5FF9C55CBC85209784282F865C7132FD0F82A4B016A9CBB64B03CCDA3F64FE6E760DFA764A7D2A17027B18C8909FDE5C9686608670871CD95B2B1DCCD99C34B22034147F1471BB38D3043ACA4C727BD971C4676BB7DBB987E015084F16E05686994A776704A161C7407007623964359E0F77214B03898C51B4EA6474A7B0711E505F0060C220A80A9C591481643A9796C2AD651ED7783F38633C308B33490867DF28DFD054AE9C49CFF78B508234E17538EF58172E92642F43A5D421B2929210BBEA122853918EFBB91A0446405519D43DA86031A6DBB3A9E24BCBE4E686EE9EAAD71A42A4D27907E89326FB811A5663D18002F7294CED499A29EBA328F5460F80512D42AB48B726E798778A2C2692BCBA2A0052F9A56C1A0631741A6BC846A8C0BC309B7E9709331AFCAFB47F0E805E29B98A2A3C2D9D49AFEA806FB1CCD5E24105FCA6C3CCC1551B613927AB4CEE284CB53968DB3767C2154295CC479F06A4560385BB16B29273386320FABC520E5B36FE4D85814079E7E835D7935172DC893FF9277B177B7D8A9ABF9C66DC6A52C25852639540220FBB4E2E576440351F281712910CBF6A02F5A1B786B006DCE399794191D9F6BAFCEF1545E3A960CD204868463FF2B8A20944211408CA82AC182D05AB4C32D6A556FC36381BF163F69377E0028C8E8C70F0AD7A526CA274F9B65BD3ABA9164BF34F74B47C56966E4BF8E1B889E54CA29A037E67C28FE2A2B6710554D3813B55131B2B75DE0A419AB8888151876CD7B476CCA91FD83480D150CB3206F6A40B95E3412F9E7288EF051D3C99ED40B9E29A0A65CA1739F88122C72CB15950EB99302CCF18EDD17446F165930396F85E84E52E221B5668431B55CFA08D048A1386D450938D48CC2A94B38EC1F3F6AB2FA5565BF3B4BF1638AEB6A2440E5CD840C564183331E1AB7CB114A409CAB6D88C84C4545396360B1FB37B36CC405451D7FBA970FA7433B879EA8CA0BFF7C4BB0B98B255412AF695511A9CE168BC4FD50C27E121CA0221DE918778B941019D32020EC2F306C7BDAB38F9518BD51FC374F84B5E4B700F05785AC39CF9CBC0DA4C52150C9485D4A6BB4CC3C54A21DF63C2044B97C8709A5DAA82933197F1EA24A994370A2C9495E0B6660A10AA62BAB0BDC4DA284CC39B7C0E0464798A7365EA106793815B7B2BF02EA70C5B506CA343C38386BEFB69D6E31A1678BBD497BB1E64075B4158205280802234AE612A588CC8B2EC24AF74C5EAC085EEEDB13B827340AE29F348A9E31A20244D20A4B2B41F46A6A4C8293A2B8C5E3D3C49CA1AC0724969360BDD8A0822DF35D16A7CA48F90BB2C7972CD70B8DFB6CF5F1453F69266B04C2C32987DE0565BDA632F4B257CAC7CE7953A70B29681BC4602F3ACC3CFB3BFB63A135A758E461647E344A454AB3A1A72D234268D8870ADB84AAC250127E2AA593E79828848C112C26592A89C4DA358BF20571144C3C715449B8AE46606A51E150F4CA3593BC9ECC8A0382532610328253F488E733965C1742A36626524809C548397A922E79B82FC35B1D0EE41975CBCFF6F38BAAE28F0B452BBF792CD9023EB893767FD36E855774F1600399DA75889B03A95A21F32C0162540CBAB08B82B8346CA43B3E07B44F8C5AE6255CC379B3FE631A434349533B85C164B44D7805DE49442D113D238A9856E1897F0352130369387C3F2BE95435885A117072EC762656BCAB6EC1BE33C80C7835C4670B1D142767E23BC0AE6247529A3C8F348BA37ABE3507AC93C939560A445CE696881248F2A6C0B19B3900DB2E803A4BE0D4231F41A65702502241BBC724AFA441991A0A46DE7C62F6A220E457B2D876C44BB0A5F3116868E2B6D63A44A5C55F53F83E10E534606B76108CA4BF12A1B97A61A28B1BD184A9C2057655D460B65C407CB93272F44509C432DAD7B03A7393CFC160861CAAD4E9C81B9CB1085539D8A2C1855421346ACA46F46C3D12696237B237398B4015467CE7A6030A9666736717E6A740152BC2939998724018525D74724C5BD35B2D1887F9D2C6733A1173129DF3AB625E48C6664207C4D4931AF4130CC1CA0EF9160898890B49674F92A9464837F5597A7CA26BA854A4E0FB756572766DE2B9FAB3820818CBC600AEEF588B59B416F9568E19FC4B03902CC13547DC18197CB1AB23190424EC80633A5D58D861D9BCA5450102E52247A74594936153723C0A561B64F4E75D90E805DD7087FFD3420B4A0CC54CBB32944693A393511CA69E2C2B7A3A5B58784F2E999840BBA8C2C994852151569CA0949616DAF73328B220DBEB7D43794B1373CA3E8AC69178653E889D67E8546D35A40478B9C6410D9376CB28D6080AD41FE834B970403B8687C29011C6C8E24137F22FB7192443190BC2B095594C85C55389D4D49722124F07067FD9F34319565392799C27550934127538524E1655A03F697313130CB0F6799DE565A5DB8B7C23C7A8F53AA7714C7094A86E67894A75A489E6C414B67C52030F40EC5C599969D8E00B7C03492C8CA6DFBA8D641307921712C2840591FA9A847B39794BC22747830FD956C0C17B5C974FDCE96F565355B500A70C3D86EA32C9290514B1373184AAEDB75D90976FE6765F4747BFC946F1249DBFD72887FAACDB1B900CFCB5B3955EF374D86072D38CE29DD9E906AFD6E326D7035CD963C74AB142936A91EABD3074C583654945F9F5FDC29F754B06EBF2" + } + ], + "encapsulation": [ + { + "tcId": 51, + "ek": "2191ABB6D6BEEE29C5780758A970349879B61A028DEEA5404731292346C81EEB1D17766AFBCAA68C867D91132F34A494E28CAA767241B50902F4825771865FC8D633736248963A253DD52C1A07C7177CB6DF74C43C3C74D7133DA9DC915FE14B5BC30B86153E7B86BE04189F4CE8CEB6E5CB69808D35640AC50335B1633162E28260300646153BE7B26A84C1C565956F8577C95CFA80B68978CD7A79E49C7E808B21CDB460ED8388C4859670333B69980B2F9A0F78D7A5B9D8554893A0B5C52447B15318828989816F43406ABFA4950607A5A57CBA62F5C4340AC3D294191687C7D2B60EF2D83F6BEC15B1618129B78577447BD1E36A87142B9C552C8CD2304F35B81B3BC5196C3840C0346D513EC8862BA14A172C051402F7398707A6133C8E88E5B57F9B9EEC18AED1404EA1C47BE5169E5D558E262A231A75925D25C499650C8AC031BE580CFB185A0E0362CE2194C53713A630CB59600C1942908FF5142B02364B74B3F5474301AB2DA4188DCA640961142F28584EBE810683C046B59CC2CC07025DA63A5E8C5E99E24FBAE183329886279C2D58D0B8CAC1C7B3033F05E765C75414748687FF03C00B36709D41C5968528F178CD03B813B69054D3E20FB6A04A713B1B169AB5F0369810FC5CD967A866B70A2D87514605794930BE9B7B5D49D08F5CB08D84FC5BEECB994DD2A5FC10AEEB4CCE1ED251FD4815A9D06EEA6A77F18C56ACABB1F7C478DD52218B7BBF9770A51C3AC96402548A1BB22DD0C29F422330057179EC2D91E1AEE27BBCB1EB4671324265E28676090896E332535BA2E9169C54113F19A45229C2448078AE08F7213F347244527EF726855CA054AB7A1ED1A40A285A60D04C00CCFA8FD318CF182705DDB888DF652669B3224CF54ED6D00DB5121331B40C7E35B1945925E5AC1503A999899317A2370020D81873330327DB7CC1D430E32C6570FCCFFEEA90A871A4B04CA9A8065AB20A6459F1806221A5482802F14A68893B111607C4E085A5A6410C027C1BF5FA33ADECCEA576211F272CC942ACACC879DE398477780A26E7545ED693EDD66AD24772B0C0C4B033A73ED94E1D02C384D1A288A78F5D8838B5AC0B37855DC59B987EC9B0F5C6868B0319F3372024FC15B247144F925D3EDC453FB8551EC1CA734A37A3972B8CC9B84A217A4D0BC0D08C815359ABA59A07EED998133587E062CF735A8521A18AC2059E3F142F12CCC5AE024CA33B8E4A285BE5A3CED460C2C3675B956294EF2C61C96A88441665D56B2029688EFE4540DC35ABF5957BF9477F58277851719547510F22D01A13D0047C5B88372A654C732AFF395DBF825829A06E6AE6923296218AB9AD33D0B07A0B6A3F3B3DD8428A1F82B72A921853D94C5D400494494F0D168148B5392F386B16C6056D6B8D767B7F4D918A9384C855C8B4DE8450EB6CCC2DF18528CBA0A3552D5704683AA3AAF52751FD5A17801A76A58A75038533C1576E8668B5D290CEB20CB1FF91317D9B418311C11E617D10B4B0E9055AEF5CC6EBF8B91F0244ED9854FDE463E55112A238038750CD016735613B9C461996B90A3B4D4B0D8B8825EC087FB138AB868A04CB3C9249B5B118F81ED06B99E6B521F206A32A4013E876C6C80B36AF39AC197A21B78B5F2713AA45C74335F32676C2AC2902710EC42AF5984687A672E0466831C7B436300B30351795B22B3A939347A7B744831BAFE78C3BF09A8224A958568F75D71F822941DFD64D8969ADA5B73C847272B5914491BBCE5F0C09EF501BABD95190F93201AB413E833F7924B696D5C2B62272245932A6D9C35E1670AA3BC57CD41B8D61A52AAA6D80BB774F922ECB41CD58469C4DCAC698F6730890237602064B148650C17E6EDCB3713382FE755E46F3ABE0C6C45184AE5C5C861E35A0E2BAB2B4E8A4775584351170152512C7E99E96F285B916605B53B91D6C501F51A93129483046CB2DFA16D4A540ED2B6AB18BBDFBA2B201BAA20297AB1FE7133757C07DC041971AA4219157D77628F31AAF520650A2250ADAD1524E58268D21CC62E2469C8425E86359DBCAA066ACB466E06947A1C144E53EE0DC930432C806C493B2BB25757B1E2C775A27A896EC25B58E8A78D4DAC91815A1F3D90635EA25AE13CEFA77877DC0C2F0678ADE307243B438C54AA69DDA539B8A41C1B3C98C77FCEB2C0BC2025A272D927943EA338CDF32A0DEC8B187DF5C", + "m": "2F1E2CA7BD72AF847CAC38CDEFC4D345909D7517543EDF32E2FC491BA05EB5C3", + "c": "D892E0948544D020ECE56F00C6495B9BD1C469AC0F134002C864D10CF61C6C7887890276A7546310EA077741D83428F22B60E2A40ED8BBF5D9227893BB0B7417DF4380323426EBC9744EF35A1BB6DAB1181FF0B677E8C9B6574360994F96EB87C3524E15E468283169D90D8A994BED0DA9CD778CA239CA6C225390221FD408A3EAE541A032D714F3D078DD1EC722CA83FFFC92A416F46FC1E8710C3F8E9CD452C016F483985A5C1D951FE2B03C4AC2C9A0BA71F6FFA4B29EDE76DF69CAD594D10618B94E783FDDAF11CC801ED5036FEB4A70D071079944B7D9F1FFBB98F123DC34A52A40DE57E079A1F8B180DA9E6ECAA47A111F3C054DD9563D7E74B95AC65AB453AFE0BF0DCEC5578FE6FBC2D9EB91932799EA64B2DFA95B2C9C6E931FFB0A5A499DDC42C6D6B563D5A6C1F34DBCB36F57E6C10C69FEADC70A3D01F5F3715F6BEA0D3E2E7AC2B04E01CA80A2E187BB7D9EFA5EE257B9C3CAEA3947D785CA56A76D0B6692A69418935DFC36FD9B26758F5E1043D5C2A6D5CC1556AED592BBCD652F4024BF324910F8CAA2415171F3EC8B1CC494F2B519CBD2B13D317672DE60F0A3D400E6079F85C3CDAD0DA5A17F2EFACCEC03D3C2E85B063AC3CB29F2FC4850705B8D472E35DC12D06A7D021DB302CA883DC7D4A57B7A9E1D023960D4EA5E39C9D5BF328B8E4AC0AFC3A6C990F598F373059AD28EDBEB90412472C637877EA82E7105CAF467A15CE961695F2FF04C750E277E78C1CE9E490A5C45168936BCF79261B95FB20AFCCDCE790E0AEDDE204DA4F6734700FC1CE05239F4EAD0A496FD8F733049EF9B6ECC54D5C635B56D6FCAC7ABA1BA32D707D888B820CE3E794C2562798210D726C6C5DB3D22A214F3F2A1F2477FCB77377AF41D0F3F0AC635E7BF0674CEBB672EA93B9504EFA8C5AAFB7458E41A1D85C28841877FF2F444005CD1C1E4227573E8151184786489C7CC39549F8A29B9C4D68BE38F24081FE239C5D0E14AD5E361720836A72A99CE88E47BF3E6A1A9EB57CDC3724CB4789FB88C135CA9CCD94CCC9BD73EE8A796BFEA36E894ED27B0791E8AD23FCDA5AE7C167D5774AA00468D97BD0FD2B9BD78D01E1E39A66994A369212D95D7886BE00F2B7A1F187C85B873F33CB7D3E4C23993F7BC9FF7ABBC6FD9D5C54F06E5F31B79DA992BC0F2AA79A36BD00483CA0ADD2DCB2BE36C27DC913668F030128E82281E5E66E87C75DCE3FC60431CE6A19C6C45E19A2D84430E945034B7DA6C506EF009493AF22F701088E01C20ACB21972FF28D284D2A57A69D108161938F028011BA985E2B925532B52EED878646579183EB0EF3FB57BE1E05C1BD2B68D0E4A9CE104E0ECDFB30268E6C007812FD88498D97795447E7F88CE8A3BCD161B1E34DCF77413B778F88A3AD66F247F7664D0F6AF31EC1DADC00FEC09D6AC63FC8A802C3752484B5B43555381043FDD30EF5D7DCF4F5B68EB89F08A0FE5ECB65D6D9418CA5283CA398D3129CD3B44EDF5E3568C953CE2B66A28B473CAB0C43918A05C5A4358D1392604495D06B395187CAA6B36050B436B218485466CB6643EDA7AC66AAEEA758715A22A8849066879CC966DA7E0F7E843A1E234920AECF2A4FED2F78C69A6C103A5A535D77976B1E40BB0D75D6E370212017734F2B0BE9F3F87C5583634EA6A998D8FE9CE5E5905F2A6ABB9D635425D06731C1227EB634A3603D081CB1A7C2B0C1685942EFD6F65992DD39AA67CD954FA0310A0F5866AC6121E27E349D5C2ADF37A672C1DFB021A855000F0C0C29489926B997930B20DF641120EF8605CFD9482EEA9344918AEC689F580A94508F318E63EADC3EB7486B9FCD7A92BA16F5E02CF78EF73F528FAE3A43D451C58261A82B3735E09D4F4679CE112803505006CBAE6EAF641A6F66B0C51EE90095735D87AD88C3D19CD7888248FCC3872939605F205976EF6C6AA52F8316E9DEF841594373FCD177F0E24D8975653A29F738FA8F0D457F2D72B7C00A4B7A66FB080F705A8BD599314354E65842AF598B8A2A40F6CB3208762AC6CD467D06A2E987A7AF72E355BEA297DA87BC9250EBE8D8F85CF292200A21D93475BA46BE78C17C1605C10B97659859F08F980114955E68361E180C98015AA46A776070C4F2B328A903A70AD226742F0022279179FD2530390F190E3008F7202C83A6A15866DF848C8A150D12287451DEC8EE7F04C1C3121E4BBA687D7", + "k": "5087E3B0C90BF601DD6501E071270EFF8683621E9F5D67A7A668E50C4F460A75" + }, + { + "tcId": 52, + "ek": "7ED2911897C09E483AFFA62DA7DB18BCFA54D700745D055844778C0804B94BFABC3212B6032C6BA86CCAC5B844A0C1C1130922423A628686AF96EB08AF8BB51482A531CB648B4C535047179742388FA4CBC04118486AA340E50D01F38C4E305833D04D90BACEF4338F59B91AF2D17ECC583516813B8476BC5E4AC5CF4A1E0C621DA498B87FDB6126ECBB79D3CDC047C4EB8B07BB246F2E11BC76D033CA22A92DC041CD646154C0528EA207DAD8066CC3068A026DBA587ACD78215C6BC794C3A61689B6151C7E412148A55A7AD955B85B284C58C56BAEBCA1AA7186B09162A6035B6EA55122C67B823996D2C548C620843D6A939BB2791148CBD18145BAE3AAD82C8A7330716148567AB4AA2C93C739C64B3E63164FCB1A8CD05D9B6C3EFCD3B350F57FC98223C097AF02B77411F9AE3B4285E375923BC63E61B587C396460FC668E9369E5D1341685BC49A090C3B5C52597A7E84C1C9CA2924682808EF45B4E4459A66EC368C80BE3CE2CA5C2007FAD566E7E07C78D009D3CA006F2769476871C57B758967CF39B3166A75625914B90146A1D5A1A3A22164B1D61431E98DE06185436506F349044962A6C5D034C1BA9FAAE2B58E25AA611BCF39383CE0C43DE10B8ECA58347A9A970E56651FE4B6E163CCB9415D7A79B4760193A5CC5C9AD4251EA3AAE9231246D82C7B4ABE1D5932CBF9C439B50C3AC741F9A50FAE32C34B662F7C12937A27BA76D357CF42AECBA7C2075AC4D3050476B43DFAB4380D1854E48C91EAE54B022380156C240F6018F35A5A3933745A887B76BA9625C868C5938E6C618BC2485D20483237065C53F535C5508A8FE82C282A298F2915263059C0B8899EE2B29CB1A45D9A28DCC60179B15F58FA0CDB79338CCA8D8943B2D32B0F4B6408CFB9190EE0697F81AD4285B3E95519083C315CAC73B2920F4ED0B672A95F7FC3873DCA5337E931F79719687078D876A88B6417DA2447827987A04217FC509FCFA99FC2966DDDC05298784BCEE39390B3B48E59555844826A72709FE210350669496C66FF846B60E66EEB957AE834B85C3065340B4CF25574DBCAAF5B732F376256D7267412010B2CEC49653C7222366CA274339090A402E2200E045AE6228848219A08DA8C33403C5779B4DE89475E8466A956C6F378A4A0F79F8F22C17A516BA5CA5C3D21C80866639987A270619B5EABB34B818FC035773A9BBA1AD1179349489A49744A46AA826A7045E1ABA6833043D611679938C29C06EE896665024178DB47CF242798E30278C664C90A39335A33196259A4831F5D875085F64488B054D348AD996A94B37610AE978EFA7926F844983430A797381C0649384E299832F0953C10341E89783B70618A60AB07680D8A677B5237743B37AC4972CA72C5137E8AA4939913DB91B2556AA4A17C09FAD7064289BEF6879BAACBBED5849C93616D3A2BA5E1C0A8F037A7ACA7B89541233586788C9B29F07197B56AA26EB59C727A7502F98F65641129D310A9B036E783552548588A2C5AC789114FC04416602044432DF8469951453D58704279D47700A257944CC1CE1A2A0DF0BF6DE6630EDAB65D4995A023C4AD351B3607578B51B8DDCB43C2549952727EC97066C24230DEAA4BFA55CE0D7394B8F0B84C969F8E443E71C3C38F0B71535B1E68E7A3B08B5BDE9C9D78E655014388AF8CB54AC5B9CA41AB5E2A4BFBB58D0628B4155A102574BB4FF8C172B34BE9BA2408E0C8DF7339A009452D7023F1F912C7C84D5D1A415F1894897B7514EABB771CCAE56C677C506495595012B25AC3409153952699B9A691C09BBA597F128B7B7634004D5BAD0199C213DC892A7A9B7F51A0012C81C000BFB50B2CF7B85A94B2903365BE3B8CB4F92A6392B75866C0908F52CF402A4C8746A2ABFC91161C70B2C21BDDCA2907E29C35FC39EC610B84D28B58D178DE04388764AFAEC4C7E596C953F5A8396CBA69EB99BFC1CF7F0C1F4DC20367F1BA365192F7C35D96D089E0A5934F467EAB3483C8C0B009627150706BE172243CE0CD14684564E586A1E985702CAE6F90B8A6E2771D6059F8C9041F8AAA7F5C06CDA09428DC9904B6AA2B857EE87A3F07AC037AF379DC6AB7ADCB59F0E82E73D474E0C66036B71061EB8F111869EFF94ACA27AB15D02E84F146B9B6B0729A314C96C71A72C583FC446859B01DC664931A3AEDA27E7197BFD78421C3A5E0BC629E393F", + "m": "AE818F586EC2ADADC7AFED63C03344B9DC0C834133D202B43E2372EEED31817B", + "c": "CB327BE1BAEDEAC4CDD83977486EAE157634E3A7623260197B96E4956172FEBACEB6A4C22781095EC6DB8A91204FD3D2FF660CC52026AAC9ED73CEF79B0D39191C601A81923DC309CF5BC526FB76D8200B98438ADA64BFCD0B2105BE5663D14B3BCF0818C53BCD171FA7927E1CFE6B7FC0AFADD15D040949652B779CE05B49C937E6B7E43FAD1283AB3583E8D7A4A3EA0287EAFBFCBF10D07C8DCD6AACF44904027299F24D08C0868CAAFBEDABA6A45F5E00B1F4FFA31D8010B45D5529D3A4D1DD82EC22CC03BB75CF21FA4957EF81FCEFC2D25C53AC5C2559C57AADE716AB26ED3EF8D250C2CACD3F4945194E979C5D5DF64B097694E3052313CE86C7D36CE74FDD9BC4D3CC1FA968E5984AE93E3A4EBB1CB3309DAB6DF63FCB978140624C2240AB80DEDD5EAEA395FD0FA6A5892EA1A162923FB4372CC1BEA7B60295EAE03D200E8B2537B127ECE6EE36EE1D80DA9191029EE666CF29A35C32F9B34D126748C2AADD183BC4BF05D627E1F4B61AEDBF39F0038471A839A141AF16FA371D60B97FE47D450917EC19F5062BAF4C02D2E158ADCA6BF8302F5F18A867F9CE2F010BA079CF39A76AAECE8CD1A7129288B2BFB8562DA666084D7733BDF4C419639BEB1E8C980F86C7C884213A1325CB7BE693543BC056960AE29543D3AA3FC57C994C185642044FA3105BC8FFA80BB020CAD148EB6154A1FD9160C32BF56373084023D470ED7C003FA0A94CAA5C95CF4CDDACA3710926BC7A4F1DA5CD8D66041B1A557547DADAB39712D359228177CDFB5FA779F3446FE8931FFC78BB738A6EBB0F6560F14D6B8BC1E898F35D55D085335D92E8559E7FCB467B04A195EFC179085F5A8B400DC04CEC807E935E6F9F6D38C833DD845D1E4FEF8564BF576D75CF23D1FF74ED1E03DFB96C3B4454794D60FC88C76AD190B21F4940D0BE343434DE890CDCE2DEB58808390B709DF2B84133824A9E5C769713B2D7BEFEF842AA23A92EF9F6FDB6E0871E514AA55A80132C7A8B5ECD39441EC76D862AEA1E93EF98B9AE15CF2458DC7918926102C4C993C14B898EC430F3343647767163FB979943D1185BA420840B876D2F95C709D43C7D29F002DEDDAD4EF03B55825E19E54B1C9FC56B79E8BDCA7591F712DFF92CB9145151B3320D433A45D8B4A47651398E33AC9DA9DE758B6D80373C7DEADF937C293DF202489F99B0248F3A1EF831E6177C907F19F38758BAFCDFD7E56B5232A87E4E86735F055A164335C85DBCDB55D4B99055D00D708EB6A9067ED215C20629D532911480D2AB8600C8B474FC6AC6C3753CA7CEDE2978680032C9FB9AFF2DADF76A81AD7A7FD0639C9170EC2A78F37DB2425FEE690F40CE6D02997D538631F65F01C7F5D7A614EDFDA4117A6242D1575C65AFE5D1808A7F3FB5EDEB429BB53A0AA2584F9958F519B5FC0A4A727C8C5A92583596766DCBFDBA467351C86E3D2231BA22F656C8449CE54D726B069C790BD5C19B6147139F1E2E4B48182E609844D9D858748F7C416A6A1BA35C70A98936C2D9BB1008C390F4B09527ECA0FA719DC42016E0A8622CEBAD83F636E780EF79C2216E7DA50E5DB69BF1660A059AC6CB68AE3969A7210F9E7F8DD172080FA98B60ABCE8831C6CED8B809C85818C263597D3A1A3F012269BB3BBF0F40CAE60BB9F8725A3DAC68CBE49402C90867030F9FA9414A2BB4F5DAEEB292A00F68184150D9E7A6A38AC2094AE61BF06DB01B356C95C52BF6C0CA84DC0D9EE48802FA9EF45C07DE48D4F8E8F715497874886511C9300B950CFB9C110EF9402C70EF54C2132A932E7D5FBBC93522577592DD2F76E2E638AEF9C823A9731BDF2EC047C8B8813F086C7233F2BD8C1BC0B2A2153A9E0B406E19AA57609ABD0DF4F95806CF11EB75E440B9F4D4B2BCEBC880336C97F7104140222A9F6BE97CE3DAD5A8E5980F4C8C81069F850925CDD11799BCC5511769F7BA6434B0E3508163BBC8D9EE8829CF0AB53A7980AA0AAB5127C4A6F3F28960CCDC492BE5E79A50E44906CB4C507DF3C69F363D5AA9CEB18F8DF13667A97519A68F31D8D93B1CD188DB7CCF5BF2B2A3389DA5CD8E755E1E1BEB2649868A13DDCA3050A18BED253568B2FE7D16072D94ADD2D18BA7ACDED627ECC9FEE2A9B90098E3D4323CC807362D90D3B10AE1992D41A4CA5AFA032338BDCBF8ED165D6D7581E6A6C1BEF67C35796265BF2EB37920C2D564E5B9", + "k": "C904A4B2AC6D2D8690799AF342CFC78CA0D61E2B6BDD1D5738F32E87E07905C3" + }, + { + "tcId": 53, + "ek": "811B72ED16206C702861A17100360A1CAA903A2185AB150BC4996B4D8A72FF018545C2A933D87E412111E39CA45FDA3D870383E2829D579C26DE53916977207B851D78E35EDD29B277081AC3B244DC074DC9B7417403030A327252602298B6455E530CCB5C5E702A77492448B7F6C1129427CB28AF7F4031088778B7CB3BF78995D0DA3670057E31492F80F0457BD7A5E9719E229A82AC7AA25FF93A23043AAED482E428CEA4F33BF447B69FC2CDC4E3B142672C08B6B243E65B72335F45A929CD482EB4ACBE81561FE8864FEB5AC9C1D5604C9B19657A3240F26EBD3A3E9DBBA951FACEFC435CDC612F502C4A37DB374B353BDBE53C761BC42CD101F327AB6FC32400E307B739C2D1B37CFA8AAFB675A2EEC8B58144492F1327B0C7C8317CA0AF37882A1A8D0F340E8EBA1C82064873361BDD50BBD1B094DF7787DD2973C53C8364688C7B2CB0536942226A2625822BD88803592B1BA484A36954814EC54ADAB62E38B8013F22C101AA8B11A867B55C2BB673C225F3424263C556FB7988832A57416FFC4268796B7C5ED0A88AA9A0188C9AA51A0402B85C9B84A1A2A2071F2B962889C479533EB2630727F087B714C0223810728013BC725F8F138E2CD20B93ABB1A9561108064FEF2CBE1334831F21C66D8316B1C48B356534CDF7B1941567FC879D7639A7960B5CEC1137619577DD830F444825C89802F4F78EFEA0C4AF40194A7C082F9270A1A12E2E379810DC86E539392505A38DB4B10AF1A2242124BD5B47B3C82B3DD0595C145CE0403B53D003F354B99E872C0CA43E66DCC9F8139B006646DF49BDF225C343201AFF4C3DA8874CAAF501B1C598C97332EA6A99C1B977EDFBB99B17AEB36B79D667997FD33B5EB529175BB2294A2DC926CE0FE672FF9ACE3C5B4A565300C9185A43DB10BB08448134897AAB77606C0CCEE4A7CBE4B92A89693C0217A316CF4C1C14D334034EEC2D64797CA48683B0D8B430011D606749FAE4B8C11722B8732B88F147F3A60743043A19D97B12987BE123A5DFFAA02F8CCAEE75A0DCD49A4ED2AC5287C593C788B37C4F0CD2256406A6589595CDBA1599B6BCAE3CCFC5B7654DFB0F7DF1AF06F15BBBC973FF72BAA8A923D4622637CB70EE044B76211EE31B68EA010FEC0C922E86CCF35B86EF31A84BC560EDF91626FC64E0B24577835542186311011EFED68AF4E851B23067F1E1C236432D05A01F9CE8735095A8A00514778235436A8C383BB57EF815AEB4237C7A144DD0B35B65161A408314612BC2835621CA9AD5A5826C77974DF356B4900C5626856C204CF736B0FE47CED9355087D59D2354CD7388785F831732C54ABD1C82F126C37D87AB8C18A7F575AD2F7C302307469333729D2B4FFA3894BA4B20504CB8A2DA06E67A2102FCB65E3BBACFC7A1ED870D232B33FC0786CA64658812BE487A7C5F8660EAB206C1AAAECC6B3B0594C1CB860CAF9458FE559DA1805E038209BB2932F2D5857C833DD6F71140564749328B60C9044D8C61A993C70DA7A2DA61AACC2A23B91402EDF7448619244AB9CA807C4B527B7FAF139538748CCF9A1C95CB0DBA133334BA1DA987199E455B0BDC9C78F56827BC5439240846571B6D9A3D54A81ADF9B015B631C86C60CB5B6AB623CA63CBA8EC3B64A84F95A81F9A122475D71286C2B07C7AD395E86F44B66B377791C4D5914C0BFA160DC0B6632556E98ABC74B115731424D27E9AB12364C65D11CC059BAE38CC32007BE6052781A42B1C5B2676F619B8F9BB14084A91E06826C66B36339C147455A9284782332A1E5D1CF7E4842380545040284B17257941542DB2B63903069EA45A80EFA3CF2822CEB46CF2880681962056E2A768ABC9485C342D168A65C1A894A33C4D639C96C15A40FDA9CB4DB90AE202B8DC951C76C6038573345D05FAE89195D10BB23B101A773165682811DE4B5B3A496A298B40ECABB5DAA7D8A8A38A0C80880472F37A51419D0283FE6021AA26082291A601C2D77D320A765B07A717F609BB4AD0CB6A78A6EDA5916C7287D842C38577C1831B1CFEFF039DAB02C34FBB5D9A2C282B94E0AD95A17563BF3B78305E3C58960CBCAC7ACD6830674B6A4E74BC13CF7A0FFB8BDCD27CE2227B71DC619D880BB71E78CC528B609375E67229E94960918809B38F4CD9282210FBAC1C2FCFDE66D6B78FCBB332D4953659350B37A07E12C8870EAC6E238CC0B867E30", + "m": "3BB4FF5796B3AEA03E27E75AC0274C077C473CBA776C466F5F3CABC7104E5C25", + "c": "5F083786743B50C1C7329900776F47237978CD0CD1DB311CEB90FCEF3D89FBEF2AAF2E615A3511F41F97AB64EACB6226D95632B0B52ECE80FAF344A8968D48CEFEFBBC94CD43F5158CC2A715187A6F559FA0A8AFA70656ACD8B26B65B480E0458D671E5CFA749BF8A07C10231276AE9C221F124B69B04D5BF6A79A11DD4586798E38457A917132E3F1E1065CB316A87AA3446615D8F7420176DD5DFED430B5D7EE27781782CB6F5404F67C28DC883FC2081A9559F9539B514F78A0C5D020643B7B7E6FA370A2BB78C4457128827B0A37B417E4905D40F81DF77B4E9C661F7B5DDABCDF0BC5D2F5CCB1F1F04B347B97923E780C209E7695945CCD09B2FAF897615A78533DF2D77D360DFF3F6EB87B5C90739DA0143CD4BB8337CBA719F2125CE716AD018EB0564A662D5A08E7E43014137CA29E60F0DBBE5D52FBABE6F1393F6179B0D9A73B82E1C912FD1BA6DDAFB627269B5F7B7F5E08FB30A332F0C8E745EF67B4423A0DB1115CFE4453D98FFE039A60ED3EC093304B210A6A620C28CA0A1988BC06FA31A798649DE00684EA7AB95E19CFF366289AFC46421F69C1DD9C8231C1DAB1466E37E2F3CE0D4675BF1227FAB1271DF7B7F7514CA84CBE81D2C9C1F532D7D7C70D1957B67E4817CB9011B5D0868FE0F71F1DC8E16BDD99F79941B4FE28E6422D8E8F2F4EE1CBED346370AECFE139685E516078238B1F9D07A81CB3C99F8367A05D341A0B372FBE75B6441598D708E38F0C7F1BAC8230CF92390E569DB7E578F86A3F3D0051323537F90D97C7C6D2D7289CB2B7B7749AA095F214459561F19F7B86528A8A92ECA49036C39C01A6C745920362AB0807F7D7B32EB0CC3BE2C1B08B993AFC5F8DBD7FDA67CDB0F4C24A60FC636F129F32178A6F6C8FB2B5BA7D57F90A73174E429C50078EDE7580988484BEAE1639AA0C79357A1362F5325962E26F7217FED96226214D980F5D6704DA43785D5D49FD2C0F395741B61059597DD2E051C56BF9F6FF8AD20C89A38E7402D4B29EAABA32A45AACA0FB180C9B164CCAF3367BA826F515DA0AB639834AACFA8ECFB546E4DA3ABE03E5911DFF9376154D46676F7F15CA356B7CC6147BB9F70E391D96C69547207B973070B93C0171B295BA42656E5DCFA45E71E1AFAB9C018CCBC30F21156F4D7E681C44F422C19F1381FD3476E0CE282C022D8A72D47E45EF2EC9DE55BF4336BA338E8788EEBEAB35770BC16A9B89E35F40599A50BD75D9F5AA0A21C85DE5193DFAFBDBE3FF8765BFDEE4C0C1633C545099053676CFB37931F4B76803C26A1B4A39B2F80158B12556BF132BA797F8DD2DC9DD07A819AD63B5D9B80D0D095149E041AC610192F71B840AA10EE140E39CE45154BF79ADFB970C5E57ECC5C927B907D0CA8A21C3A03E456262728839D6CC756A0AB2D88E9427251A8EDF87FFAFF457D02FC8681BB841EC36AAC0EA42AFBA6F105B3828FCA311989DE28524BADB5F8DE9AB3B05F0285DC4656C99ADC73BE024F9AEDD7C3E7E410BB432D5E37F2B04943D6FB9D5E718AD9E26E4FC0851E7E254A565902D54A8F9E0540B2F72E52CB70F2FCA674A5FF3C87AD7DF3924E79B2EB777099CF37EF4433C1F937933C4E9AD8D99445FE266D0DF317E0E04AAFD82B1DE46AE1F83A2D0824FEB21FF4D1642073E8F43EE12DDBC6B7CDAF0E231A84F0FE55381782F8E5AB72499D7337EF3B0495865CAF1F75CA0A4AF0651CDF8216A3FBFFCABA8509B8F624059FC4DCA843EB5AA7BD99567F548FDE232FD0E68974CE66A1A998236EB8F0E7B9717A26C60ECE9B74BC99175B3EDFB813A193CBF936B0DAC1D9159BCAD65E4B9D7B045A6D126315F1C9902084CCBB5B035F92F5089AD174CF30F5BE5816637E1509E3AB79C7AA12580BF9EB6CDCE6C5867207044608BAE91BF59F3DEB5C338AA338C6C51EBD8199A25C4C7310556B3D1FD826141BED32FD25C289B5E41C78A156432A5B25DA5FC067E7A353C8D185F8180F656820709C010D7AE1DC93CB94B8E29F152EEE733F103EC9F74F35692E350F7FB5366935B83DC0726B90C56CF8EF1A9133066FA3FDB0319284975E4B4896B0649FB9495E2E8D854B37CD56B76A489E4EF052EE59141068264452316F794C0BE7C2DC596B29B83E70DF96E85A33395A6137767B4A4F09B1F063203FF91AD773D7844A60EDC008834E0A0013478A6499DAD7266E17125CB1F4328C815B4", + "k": "D37DE8FAA872CFACF7ABA45349B2EC3A4BBF0EE82BC223C8DEBEF6621F6B3A83" + } + ], + "decapsulation": [ + { + "tcId": 96, + "reason": "modified ciphertext", + "dk": "36BB5D4B256DCFCB14028B1EFD5A74E5606617950DA40B2408CB2C0462A222D4373A7A290FCB2997DAACAE414DD8C3AA7B38773C79C949059CF15B12FA8C522C538FDDE905E798A3E7D5A6F27C93330B8243A12983E5C83880971827B01C2A03B4FABE4B15071AD124A5A00C69D98AFD6BC63F305693B50040595995E99AD375BBD58989CDD2AA92E95532EA88336C5DD9F6C4A1B254FBCC4921B96DE593C3134A2219B7A192E87309BBC6EF95499D594B30A4661F809D03297B6BCB2EA894B724083115B0C6E6C2AF2D239AB69574EA95A746CB23A0DC429EF42A79234CAD8B9E85597BEEC196E85870CA1415CAC40213B64CA34A8247595C89FBC19AD73B715AB3990B7E9691C9E5C767D0780E4E3026DC172494BB4ED87311D20896571608B065224B2C08D3B754837C0B8AD8CDAC7775A8158A0FA93872C972F4048DE1E9C9A8E33D59397E8DE188251A6286BA2CE6F258D1F8A99E78C464F576FCE80A999C6475EC03BC597014DA6084F2C97EB88795F28CDB27437FB11F67977BE1F865722B85C3371D4534CDDD396BD6B12298F4287A7CAA87542AFADCC70A222FC1830182831BD1319D980BA03937B2CEE6C2417A766E841609D8417E9C9D8DFCBC956217A5EC02DEBA6BA6717A0C71009AA663DD4CAFD5FB5142EA14FB52C72FAB33B0070FB38C9C9670323DEC00D9EB9F02F64BF4E65D86C2C216799093040F6FFB80FDF6BDCC649F1F39BCD2C613221B528803A75E16BB5B8B067EBA7765738AF0647DA1026DEA6717CBE4116BB28DFF91C224243C1A72350165C30184AD901A9425538D255B72FD834C8E0090CB0289C2F79EF7997CC2AA5928D453198BC392F6524A88B5AE67B96A1C1782267F044A0AA0D8A011332CB8D34E1C6CC149A4A6A3B99621A308818322FD336EDFB867E4495EC9D92B4BC70F7B170A0D28B3EC365E5DA02C7E50373B3059248A439E0A7A0B3A656863785CA749E7419A9B1595EE28B7DF838F1F2BAF02A065B1242C17DA7974C5332BD7724C77644E88549FB572BA8964979979B5A7901CF05823B4B985E4B3D531C21620689F036646D528DBCCB4645B72EC920FD0F5426614BC469664A48A9EE0B5C7D9A52164E3163C299441E11DD246731180C9BD70219171712C7BB371B213FC976236043F9367569561CC00F033D473809B3C49FA3725F111907015C6214C63347C4B8672980DD856E30B55BDB7B3D22554242393AB54314715CEBB43AB224A911B090DF1CC5E6950504ED063BAB0A0D72A5C0FA8A1824962DE9CB685745659ECADBDC3B335627DDB5959FED1B66108AF4F5548F151ACBB3336A94C59A0A6B28D94B9359AC7C766CD7B9015C3B99EF45280470C37E985CCE280ACABE67AFC54151239125845B94FA26656C4CAD63B0494416464F4C7FE5936BC284E9AE388FA844EF71227D1AA0E872BC6FDE92F894C26E885087523CFB56C7440222150A62BB27C067173025BA2953999BB3746094D546A6AE03BAF8630FB3BC6F57239E7A35C20691FBE1611E6C46EFC25B77029ABAA947D81C286C4816E24F35D883B4D315A529A593421BCC5765B34EF6C4D416C3CD95443F9BA2FA248659EE77AA201987F6A5EBDA78D307559A874963B1927198B3524518ACD6618F2D25AFA4B11C4B396C9111A19398D55079EF62054A812C37FE81DB4565F67038B9EDA7A4B63B640252701844917E8CC7B3B1EDE433D0356A2C0EA7E2DF3C237498231A65E8E87874BD78487EBC1CE800C7D1B56476C220A8BCFBA568BA20B773C993A98302C084B2BA25297049841355A16FF098C420A2220837BA6C89B44356B810576760316254646B0A5A41754A18AF17D998541400CAC0DC830AD9616F4F17840A9CE2D69B3BBE8CF9D60B01B00AC7EC8B1E08B3D886A804B4A4F2FE30CEE898E42109F3A16C91FCA5D1C328B90F00DA0996F2365BE54927D93F83F8F319D8FFC375A5126356A94C6B6C3DEC98CC9F2CF0089B3A6B878C73BBEF04A4442B1B0FBE273A994BFB6E88EF627B6FAE415C37691C9D30E10B45A83E02AC3F16A5B1249E8B15C0304AB6F3482477C44A6FA8B5DB797FCBA271FC1544CA333A6144E72B02E929CAB8FB6011EB66399B8AF76CB188AAC3EE43B992482C4585934A5075428FB9966ABA67B414114E379A05C7F81C351F6E18EEB8C3042E4A8DAD0CED9A28CC492A2573590B15A914CD100BA119939610C6CD77D73FCB5BBBC50609152B3946E2D5969BDA6586A7B26BD4284F41269DEC3C124FC3C73A0A369818D9A491931607AFBC85B54410444F9A426C9491099CB4AD40EDCAB36F6924764403ADE37288FEAC93C2544F16ACB41F0355E2491FD948FB8774A62C0AB9FD3BDF48948C94206E386BDF8D463FAE547B580B1D9531D48E11D1DF6539A7911B5577F7D8B55B17547306CCDF4224950776A2C90ADCE2673983C3FF286260FC6B08138CDC93CA3297873AA9443C7165021D767D3C132B1A66B413B8BD3849E8F6547665601C776A558209FFCFC84EF1415DCD6091F12C790F1CFA2C5CC9C170BB7D0072AE997FF443D853B021B1999E1686AC6580EA6A0A946A2B417C17EEF967A51CBCAB67992A16A8C12623CA327AC7FAC5E4B616844E28F43BA45AD9252C695C4EE6707D60C10085917A3D5CE9D46CC9EA52A7759827FD64571B7B4630554946796B308AB65680FB23091515683D1CB815221B294FA403D79124A9989EB7402F9990712E91D78C89AE45B33647A8FC11C5770AB738A3573E2C32E485A07DD0A009DF27B4456834776ADE5C454E5C70D00CC455115169E8C5D9AC35844E5126E784596B191CACC1C8FF6C2C0E371A81726930808CAF3680CA194D0156BB1662BE1928D3B3A432F102EE9C8BA950C3CD05B394320C568474D78DC7E75862A155CC73CF2A093C87A96653FC1B1288FB130CAA41500541E34725E612C475740079BEA8EF227BF1269B9F0F86E2CA929C3D611CE52552859C7CFCA9BB0F125E2CA1BC1CC67EFF49E27EA6B087700B8723C9DC0175A424D1578458D0BCE007C5858361279B0C76E62A03E1977B880552396320976040D9155DBE7A4AE2AAE1A312B010CC430B310CDA08EBBE690E0A102D24BC508B2A31F61171EFC3C80E0038C20469436776C65656F189A972544529CCCE3D917EEEC4912F779E3435A890000042468F93A3D1F104780D26FFAAB53AA33BEA1A3103C3A51A86C6A7E178BB65C8145F6A41EFB6DCA856FE4847AC574476DD768271031E2E8A70220CAACA92EC136120DE09FFA31C034C0A243EB48762286B1B2218ED2B34C551572B6BF741A9D0D8A31BC1678067A3FD13995CB559325B29C39A78C7798C3806682D3B0C810F5740D438A1861C8B5F11343B9B21A11694BB1049DCC0C872B2575911DE0E8997BD19F2DD7B7D2AC501098CC54444ABCF1C266721122A6ACAC20B17DC5CA3F9B5089136E6EDB6061382E96D08B27D869003B8794BA83A50B1E2E72910699BF02513B35A1630F0BC2E8A666087787DD3B0B10529AF8499F6314039B827786B71CA5DC706DD0610DD220EAF024D577CE14F80649B86EF5F839ECF23F5C595223A294DD9BC0CB7BB59F039082E68B57FBB325911C2D8CAC1C694CDCE01D9E890561B43FEA7BB5E2E00C6D3A650D9838801810BDF5013A8B95E9D76B329C1F2A0A40C4964FC52823E55B2AF6412136314937F98150CA4A2679444773B66CC679519815746B74728A0C9B8346187106AA2185669023A1983A38580505D04E4847B2AA2A4127125013DB519F141B5FE640B06935C2B673819A665D9039848803128204C4B223533C9A3F12C9553C8A4161490FCC5B51CA2B6A81385169C38C78ADA8BA7C42877138A50FBFC5863E4967DE01593597CFC7110A0B9C82506C9FDC7BC4D4D6B245751CCEE2119BE688EA0985C5647294A3B9ABD07296473B70578FBDB0C4C5F16EBE034375818A63E408141B57129093FAD761C54A99133111707656811A2351319C04B99259946FE197B7E89283659623BA26185F887680943C1CB88341420C6C32CE2B5B6304FA99BD35A2D7CAB3E39060056BB4D756A78050835F301AC355BA7A412227027DFD2C681440AF9C81664A9C4DC5955F4BBC2479888A7536A9199199AC6710B495949CA82BB9B1A2B06823BB1A79C6162CFB6A8B8A8893152928A169B9801765DEB2AB890BA2D32C5C4409912C86C12C08972CB3A042BBA1B76675987279067C288FAB89DDD93A75368888A40C2C9A20B60A0FCEE8495DABB859B797C7B763CDB9CEB405213B7BB45F134CB6CB106CD7252EA069FEB250138018FC4A5071473CA81C6A495A46AEF27210D556FABC5D12281CA6D03BEC1E62E762B589499CC3DEB96E2A1119663DB91A6A308E2360A02413CCF01948541C54D5589829053E8513A916F58E7318073CC06A308A53FCBA7D197E3A2DED7516D56C84BA0B51B7FFA0E5D77342681EB856686B73D7942C37D8E2C8", + "c": "4F18ED0D9FF40079EC3A6D67F903ADEE4D5CAF5B4194D969CA222537EECD10ABFF3CE7CDA292B2F632F65A1C73E8CE44B6D7890E123987A8C08FF23F7345313294FB6B3E8E382EAFB352958637A6A01E0839DD1218F1BCA9167894383BE7F651AB0C0E74CA9B19DFA157DB8656E5C4367DED62F7B4432F43F5BA02F394271D79F1AC3DC4B1CAED50723730B337EC50303182ED297AA3908CC28EEA64E80819CB1509FE54302E3E03EBC39BDF0C00A755650AF3610AF7834FFFFEF5E6A2F7B66D68634E5B1B76F2482652B785DC8051494BCE4F41867BAA07EE88C9680E19139EB9BC5128FD5A348E01EE2A7178B3B29BD0C6D6096A7DC3059DA039E2AEB64D5DDE1154BFDFCFAEE6CB6CCDDA5039C64EC3ED3A2CD58FC4F7A4228995CB240B1BA31CBEA59BF6483E100C3816BADAE7414F0E744451FF1FA0912D5C0A2472B1B7C0ABDFB175424DB48AB6F7BE4DB871788AF54960D244035D0E3FFA2FDBCF143C22001D4D7D3CE3BA848562C9CEB024D5DFB8F07689E72B0F2AD31D05890BF96F46002E204EDA434511DAA17385B81E0C513CC36F9D53A6ADB36C1B23C2E37CAC31E5997E1AC3102D4FC2C3A5B3BA3C9772541FCFFF57AE162F5F55FE21DC47A66DA2AB98233DE8EE3058635FE5F3315E7BFBA87E8DD2280930240945E972CC29CCA19060439D317C146DFF758FB183D66542ED00F2841C203095E9020D0F10F416C0B7E641F4B02A9F71F599E42678C85CEACE3EF586EC1B4C4C62D21F2CC75CFA7F58A525CB6A55868EAA31043596E2985D52F32938B304678A89C72D3B91AEB6943A4B34541DA42E1060F6DC03AB7D6AA90B2EC7A4D8B2E1FCE969A8614765F8212A08BCEAFFF50A48C2A522E1A41FC2C2CC89827830D34AD556DBCAB5E69121C22DD035437FC6AD0DD95C0E22916D5284E833CB84286DDFC47DCBE9758B980D78E4DC294A3CE5A1175F7ECBE2D103801089CA744F2BBCEE7212F64B0CCFF10D4E9643027B942134FF2B50DB9084D66857C4DF62B2C6919D9DE84D51170EFC7942B3871FD536D26310FF39AE099E0FA562FB13B0418FFDFD398657259A822A458E3688B42D492117F16E291F913566CFD6CCB097C0C45C2E5416B7D3707E93FEC60037937B3930D1BB6A6CCA5418C1D26083B3286D00EB0AED6C2B7E77028FC70896789AB29F424D91D393F1F63ED8E0D02D3A19F16B80481CAD1809BF92DB925D11651E7DC1B0499E1853865B708D79446166A1C25EDE6FFCD277689B298DA836BF52D4A885C6E8F4320AD031FFBD676378093C0F854236733B002E55636D44A5DB9ECD7358CA24A3EF7638EF03B4B5D8E79DEF61221FDE892987860DFCED702D4CA1536348FCCD99FA58C1031D934CDE4593538474B2EBC8248BD4C4792B2B63B16A152970007574969F34CACB80DC978240C9755903E9F7951ACA608B682FA60A90A8282B3E54862B60CEC8EEB737FBC31B6511052856083CAD928D2E41F226C89A3DE0FC36797714471216EA0FF8A3DC1EEE64E24AFC46A16A818B4FB212410975034F5486A4E84078F59DAD65E5D1EBD37021EEDE2B459041D4250468459D535CA974683CD4584B62B45756EF0D2C0E0F2AF7B946597DCB86129279078C4FEA6563EF1E378BEF179D65B1C8B0432660AF5AFB461478B9A9CBA980925E8035CAFE83C3DBAD397F2BDFAAA08BA05C623115CDEA9A62129D271F237D33CF1266B38F122CD7A365956B9A08D611646CBDA81AD64CF096344F9B905386E7A231526D4C5DE966263B7C8299FC7AC8DAE2CC188AC205F60A14AA108C394260A4630FCCD65AA9F56FB26425B02C6319C305B042EB9E10DBA41221299BD6DD4E02CE5E4085DB6439EF118BB45B6D4C086AF359A3C4B79E9B8BCB93CAB682EB050C745380B65E8775ACDD27F97E934A059B3D663E35BEDC977FBCBBECB3D10C2D8AE2CD8B1940968FF7CE054EA7EEBD0E81E65A372CCD133E098867EDE82DE7C58D99E908B416584645710E2B013A849FC15C209931918CBFB0FCC35D87CA63315719CF8DE8025682F123B6A4288EC654438C6D75522A7E0B80F59BBB44AB8DFCB5372597590AB7644DD1ED74162F5FF389B417981FE46B27396508B28965BC564F8B45D68DDEED1CBB0B3A50EAD14087F3B9E5D8ABD007BC38258D127EC34413DFC4F41EE0B549E106F77C1696DBF04F31A2DCA4691E71D8BDECF4F2721D98E1C8", + "k": "5D1350C0937B6222662F3D0A6194B8F1B60D4AEAF94FB8426C91078B6C12041D" + }, + { + "tcId": 97, + "reason": "valid decapsulation", + "dk": "97250E3DBCCEC512ADC4CAC174BC8F95F32A2A180AD65C0A4005A86B79BF85B47CECAC4F0FC8CDB5D929F422C866AC88EBB91906F26B6A077125CB9D509B24B16AA7C1C39971693F63A83F91AA8303F107D8A54A793BC1E178932314C0EE71A7D1F1A3C53623D66621178067E6085F37B7C2401A50BC50ACEC5814D2F620CB80398918A266293A10032269FAC992205A276C141604639F62C99C557A46338E2570CE54232D7171BDD4CCCC6B0ACCB26742CA100EB9B2A52BC72E7587677C9B0B9C43C16BC56093141D627B76984C6FDE04984604243D286431C8A57C312BAB8CBE468A593613814F6739048CB3AACB9C32A9971938552FDC0BF056B1DF2B0901482534A255F44A79C17B00C485AAEAE57391564FE127C5693695987B67B11A6A4043876D5A77EB4163FFD0A2A538A703DA5E1C1696B1134EDDF43B6EC158895A612DB5A4D993CC09E43957C76F1E24B573478C1B4242FF389D0195B1FFE8A6579828462CB7849C6904326108027F16D8BD5BD872DE80A4D2E77F3CC3001AA69E4D471981E731E6033A3B66092F215146508BFE17372AB424B3974A5F9B0B4C76C9C600823A919502A1414203AD248BAAF2519E6C03647FC9C82A3ABC26796F46416FCAB55D4FF73B64393564C7682BC0703417CA96C3BFCA97950CE29A459C71D9D84A61F02F05127F2F9A549A7650802029752AA126E07F50C2AFAD0159B6CAB32F773B8F9CA6A49A0FAFDA807C62667E4B9EA5075CCCA22A9F791B83A9920B62692BAA3A9B4AC920F5B46EB99B6F869A06769B04C20C4A4C75D35A131EF0CC0EA9B973A7685295CC4A8983AC1A2E4ACC8BDDB86A11EA9596E8A9EE338A6A8570E88B28B35BB0D400BA83C41FAABA7C2636217870B41FC182A6D1A109D54983047A29D372F5EAC59129017203B1BE7566F9B518AB8BA22EA5B8F824BBDE8717F6F559E379981FD68A1FDCB57E2C75BA0C13FA2B598977C9F4343A945A3B130C306A8A8980BC75665504403413D4805A4AA77DD9A43549971BC8D3204786990F7C0470F73CD716BE9E351FD056B28887CD253445D81115BFDA50026779AE7775CB160A4B7379AA5A34BEF4488B15A25D177715748A46DB5C0CA94203C652B2D20737B1ABCD530C61D26F69137C344C5F030228C1646E2E567414063800B268425CB0E008B98C36700D186BAE0763CC030029CCC68CC8AF4C1009F5C4613A3B1066C12E90C9C48E19B31197758AC2A69F2AAB2BC942562B5D78C0831B3B7F44211973EC33978A3CAFEC3318B680BEC68BA5544331EB38427BAAC266B67B663840671E150C9992A5B522517E96A97DB38B241DA868ACD23676327211F460560196F4BC881C2212F1583005C5530C784EAD51A3E9C745579B056E4155CFDA14C6628A9E7B0202563E59BC084FD1BAA662B8A38AACA8E4CF99969D0C243A8895320D80A3948512D9AAC9E0D08D7FDCCADB91A1436276EBD8AC88CC893606A181CAC857435F262AB36B07BEC3A78882B81E533A79E5E54C8FA49D68F033AEE991EB8293E8022D4344097B06C0FD5224A019552510709458A620B98E8A6656D8F75241B1BB68E822E96A3C02339BFF99C4553493C38ABBED5620E2217CF388A97729223D00622ACA95B5CBC1E2C075F1DB5493767130D2AFE4588636851A8676409D1B55B6577650A7BA26DB6C885B04A9169568BC5019824DE600C1E947635F30B5128392A061C739590750A3BB9E975F8159673F199CA8824F8BD35BF1E062CEDA8AC9CCCF3E0A7413F42C2F65965230247EF46C80A08C50584C887715E4BC56EA478EDE317C5EEB0FC4573B62FB29DC795A9069C523B3CE6D16BDFE6B440EC6922818B973833F7484A959E7B3B981CEC0A136D4581429C75C6211C29A29CA36566B0BC6B2023B96DA991A3AB520988914077A8B67B70947BB4C2C3974C458A71272AFE03C459CEB04F889487A8365A6648C759486504156C6334782CC71EBF06534DC3A59738651426DFB527D46D469322777D752A96CE0586D701324F13A781490B475BAFFAA763CE00B6D9ABB08C61FA97A73DDE3800B365E61B5AC1EFBC26586623FBC1ED4502E8274C3C11626F0E01DFC5BCFD4E768CB7C69BDD493FDA01C0AD476804289B3B02185F75EA26A7925949C66F595D90B033FAA4401590FEC389C966B4F9E86A65394569173C309AB4BF7E92D2EEB21E8D12733DA1D5173255F0AC080ABB7C143A0013577DBA999B37022F1D7538613B03B40596E8CCBE92C52CB8961FC8A9431D88A94658BFE906B2ED295C4539BFD191931B89DDBD6063CD3372D6A7E17C74CFD55C9230CC3C7C46811436C92B66F05D88287878D74BC7F644554CA5B913E5391D2F37425F697B604AD9F0CC0E9AA66945386481A1EF177809A597CAC153638643314A9C1AEC06F175918BF8B04AC9005566B4F8BE608A825B3D393436F09D0CA33A5A787B88D7242F9C750C998A15D99CF27C6B0A5D00A8E47A39E09AE6DD8B24FB6B01E35934F493CDB273EC8A6ADF66533C2E777AD639E7167691BACA4C3B4947B465377C35BB10043ACB420D481ADD110A38F0A900A23B387FB78D8AA4E40A32A2E02B8DDC10DE6377194E51EB01B8C9A2A958E85BCD6DC7C21093B296895EE921F7853183EE1BDE3A9A9DFE0CC95DC9536897E69982816F5CEAB300B2AE5C85384B2516C92EC369A4B6B7DB51CB45261AFD2C86A4C3842389B80907532CE76AD20D9A2E850C07983C1753315A1121F10B1C66CC47F7F30701D12CB4E486D24C66754C3B01EC86EB6B056BA77636E4757CCFA11642212754426E6374E95685CB9728D6F357E3EBA3ECF241D118515D9BA4096E55DA801256EC13D182034DD2000DA02305FC282A4B40594EA429BD606261045F3B87D7CF06040C71D14141B4B4150FCD5AC3FEC11EEB9A0EDB3CFF1E80B1CD915B7464101958B8BC3788D3440C7F47F06140E5AA3BE8BA291FE2BBBB91671C9FA077619C6A79968D2E8768BCC052CDB1657E987D1C693BE262E75BA2C088BC8ADF9B7B2525B0B168FF26A5AC77C81ADAC2A2EB2BB53D97B2D78680E830A36152678661816372973E257D70655F1E9C2DFF95449B4C7511886D7D1430677A12CF1843B771B0FD76FE71081B7E165AF37CCE7D0CCFA71124D6A15E100A66E1A8D9B150C91BABEC681C2FBD96A33884032A8BD462305DE0327CBF63A090C2FE4B568C2B54B4AA5B3189C609FDC08EF2803C0DC59254CCB4A956136F19C63768FAED91B343213876ACC0E141AFC30A36865BED609CAC905632872680E59ADBBE76332E13F257097BB4CC8BB4BB278DB331FD8440F0A64C576BB347173FFAB65CC351AD2DABFA822542A477ED1455AEC01467A5453838BB71DF38C261A72887210C4EBBC6D762867C8479CAC682C4652E248A6677132BE56B1F9C2B74E008A878C44BEE92DE8435197583513B409DB87B29DFA32920CA144B437738C70F0C68A58202979D66FE8C50BC53C2817605207CB4FC4784FEEB3275D8AA4A74687FA776C55374E67EAAC488595B5A1445DBC28541CAB5D324DB2D967C6D371AC144A9CEBA82588040BB0C8D4329B81A97EAA42BB649A960A424BABD31DF9379726807EE1245532E0BE252563ACC336CB743F698839EE665F4E294123767031E30224C979786805F369B0C7227732ECB55E90CC6A42B08F652672BB414472AB094B32887803E6DB57DA64B0B9094DFE29154BD0B853E0CD052C2E534902E448561293CA1CF60A24D6122A7B154AF59763A48661F40D11D2C3BB60AB306021F5659E3FD196ED8B59C517B892C457A8F8BBB806AB31B8B7C3F8377B79A214002CE01CAE9975875D2A9CC660AE8F386103B7A276DC883D145EBFE9363D0290E5B63FB6A04CD28B83D8116102C73DD40B21B2E00958924ED0FA33F0546288E6A453914CC7A182876B435026442D050FA19C3BDD451598003F5892A156E718802AA2A1F4882A9146FA78B77BAB9CBB522C3597295DA57940396CEA70B62982B41C07AE5EF33F591A6D1478C19925025D2437DD60B99433097AA463D136C49AECA8162C37F9290C5CDA4F0EDA410C580A203C25BB0C360A99771CD852B73C3FA6FC52B2909D95529D9EB69647A42892390B1CC3BCC6F9C86AA109F8732BE976A85C98401E46C00F5AAC50286DD648B11A3B4523BB126086963C748592A7261ED9BB65E62887C22290F2C9D3E86F289141023A47A1B09B202A0448B444C6321424C5CD8AFBACD04BC17886541AC90C4FD600513034775077B4037B09AA11FF687EE26AC5715B3BF3430FE98C028617CD0040441985AAD41863F819C95A54940F00498F0439F5A04B518506EE96571D9508E5CAA574C6687AE834F8781CAEB17FED8C8721AB3FEC93EBD097A0DC10A493436080A89ED15190D6825C6F3F4DB7B8283E1A2DDFDBFD4C8A3779BC55965A9AC9AE909A994ECEFA6279ADFC260F3686D78AC8C96E95F5C61A739D20DB8C", + "c": "7A0A63634CD704EDAF7FDCB2AB069BE8CAD96BE23DFA96C02BD4EED0BA60E43EC8B79A668B6EC2259D60B748CE5C2BC428A045FF2E6687127BE9D14811213F148D39ED6512509E5160543D9B737FD5C074D5273EEFF6E4E5B09508106C18E604FDE7C927D25DD4CD2F76B7370F58BB6E0798698DB855BD687019FF5C9D2AB6AC398239335D3D4BA465AF994C9146410CAB5376D913D13078C0B101151394B7F5847C90CCA38708C3D4FA257E6144E0100EED9899601802EF337D7B5B1A260273EA5B35A5F61FB48D13FCBE43869E84F4F7CF1D1B6AAE61DBE95D7FF6DA587CAA3D0B4EF865B49BB3149A43A9F89CB76387BF363B0C7FDACAF4CDAA851815E490287CCA86F1F424660294630470A90468D5B39839E1C275A07AF30318FFC50EDC34A24FA531196735E78BEAAD162057E6B5AF92170D1BEC86A415243213FCD7050A2CD34E1B2842C762E13C39666A950964178603627DFCC2DA0A583CEC0D2A51EAFF9E1F5CE4FD58CD239DFD91D3CE683AD5E8D3005FDE4E349FB2C6DE5292561531E868AC91218E2C123747BFE23298481FEC2F127AC884576C6944C79B2ACA24F1DE67DFD178100632FB3AAE44D920DC68E913E82D7D817CD44AF11946856EEA887F321209EA08F394CDB72108211F70B1836A7FD6B6422E987B7F8CDC34E3B2F56243875E441FE050A5E2E963BA804E02ADD36F524F30B6D0FBCA4C8E833A52BE516AD1BF9D7089D80CDF48A7AB09F4175A1906059E7FA6DDFB704243D2E79AFAD7F46C02CCBE5A64DC7F8BE6EE79EDD35DEACB476ADA83113A078A65C96889643EA2BF0F2B297DF76D4EC9E9BBE9FD93DD839FA0827FB2133BA17928FFE64DCE9D74AFE9CFF2B7979AEE75B6FFA85173E97D10A3EC16462BD0AD16550AAA45D716584FA747992D4F3A8E2FB0869A3435A26F21B773C88F4376A8F68757C7678FFFF04C22D566DE7B3146532B9321C1A6408601833ABF2E5046FC4A192E6A0BBA38399E3B7368AA0B41B9886F397162AD1DC321B92C735140A2808250E08DF557B515D3235F35B540FBA3B76747D0377B517D24FF1C5148F62D54F71E54EEBDA2C0AF7B2AC9B420DBCE59647B1F9CC00D16E5D08F6FDECC1F4222C75C19C4023812EBEC86E11DEDD337F15E18131BB5D856AC19C5F7E1FBE411336818812B472D55ADA4A9D202322EB6270E4A2BA7B1BCE1C1352A84518EB456B6F570E428BBAC4B5653AC529327CBF312EE23DF1E72D94143F035A9E4D3C6EC380B0795A418187E5C90FE3872E0D1F82AAC2F206ACE6EA74590BFC42CB05957623D7C02202354149EF5E6323A0D968D715CABAD07B6FE22084F74C8DCA6AA3E0A40DF4C8D85811589D44718B53760359FE2C39F95C98EBEDC70F9832C3A70815F99238A3A9A29C479F31FEC54F91D742914A79C9F9A29889DC2CF159264C5D26BB9BC2B9680A1D2835090855D680F9D48E5CFB82ED8200FFB1FD8626CD7C6BCDEADA75025F55F184F32C2B40EAA11626C873501DC8A3FD1CFF14BE88726E718F1736E93D9C58A67B62E5C952A3926FEEB0FE8080F198E2F00D77AB5788E795CE445994BECDD5D4EF7A282EAF323C81A39276AEB9D4F7F7B11D22904A6B512145F176C2BCA6371178C18EC9B12D233D1B10553FC5262358FF7AED33B0155A79D7D0D637D1BC538D6D5817C83EDD614170864941B3527A1F615D18085258CFE837BDF73520D94CD7ABA32A48A0A80C36680D7C3C8D83B80950C1DD9AEA2406B755F6CDA567B513563514F9E71C1582E9B95FA6E05E3059C2EE2E4317E5D67181EB17A2E65812C6824D2EBFE63E664C06BBD40D162B9F28371D61D8AE363232EADAFE8B571ABE21063FD1E9CD7915D9573D42331710E17EAB4C268BA95532C5A8A7C7CB2EC035847134F8D8260E9A760BD755C32AAA7BCCFBF5A20DA39ACD382566D2BC30176E80B8ACBD129DE56083784DFE7CFA8E36AEE1B911CBCA6101F0786A6791BB6025B0D9C92BD85A426CAC2D38C838062B3208C49AC06F9DBF4D436965FCBCFD3222B093DA213FE641A623ED3DBA61A8FAEC417C14D40CFCC2636A12901633CEB3DC0290EC05C5E7AD3AD828D7F379B1347E11AB25E9F4F2FE8E4CCDD15B3B3ABF961955138849C348D9E9C6A4A5CBC55FE22DC87790BAE23E350FA8F4D27A91454EA4A10589CC95D09066FE7A5E037510C2A6CC1DD0592037619C6CB6294D56C5A", + "k": "8D77E1681DFBEE67FF380AB5C5F66A0C5A6D30E34CCA71523B4ED7DFD51A1A56" + }, + { + "tcId": 98, + "reason": "modified ciphertext", + "dk": "9C004D41EB2EAC59AD6CA481475914C5F07378B05EDF3B7FCA5CA844185449C61678953E72CC0BC5559B062B0F63E29FDB7209A2546DE4B21539E5948A38A42EB0CBFE8325C219288AB7ADAAB772AB46C7FB071B93F2356A22967867901EE7605C689FE8CB13C693B08A3AA302C2B707E4A93243C2B933083A04603CA05A9F996DC205A2CF4A20762BABF970BD32B8481F183D5A8C7170B234AEE2B50BBB88B52AB3696388AE0AAFDA484A5178B015CBA4C2AC29539823C384B45C65AAC95B2496C02132640E21FC04396C696AFA9F77619DE82448B1530A42B5C455072E7462C8F7F35758740C023C8040341824D965ADACB445D52C3FC91607770AB2D8508B96C407B7A104021F6461447A730A38551526F09BFD462D4733918FB588D5CBCEBFABB458111565F809977229696B8FE173B3204556031ACEBB514F49A563711B0A3C637594CBB42EE46FFDE34EE92430F4C41A2EC16F68180D44D21A67525BD81947F8350E5159C9D54C0F295642D0812FC11611CFE3797D7B2B34291B82372E79B6A5B120737297376AA30F6C91CAA7BA4A8FB4173E46AEB7A24EB315443DFACB67D067A5D93AF9C844A32A9048F94F7B2968E35956072CA5C57707A41235C68B8A971C790427CB53B12E32DC5556103F0FC7919CE0A79EF6991A92349D5663E5973C2DCABE876B084958B0A63C38E3778537E86B85237E5367C176E3BC2F2029F7B524B499B84C28B1EDE9B9345BA5B070AC61586D63D79436805EA2EA11B353839736AB5DC384B997025F80C4EF65559F0370AD1B5D385754E806321B906C307A3684BBBC919B01899B5CBEA35D5DB99ECB38A7943C135E49564A3B3C29B18E42F018B50B9C6B7782566209ED93166C121FBA88585A165ECC7AA69219C1DE9C4E019677621B3870997EFA96909A331AB2AA63FC704B86D745F8228E5EF7C6D8E242AA285A06321BB1FC55F36B47BFA22CD8F849689AC3691024537808522816D7F71914B6115E0B71F9F41CE04565926782BFD7C75524AF6B8B63DEBCA86DD97EFBB7C0D73946A0F24316046F23FC4939776D9D9156CEA01071283E24FC02838448C54AA5C03A0AD77B563805022E9131CE77201D746567132B76707FD310CA512517081C2591E03133FAA06E97BB58E625591BA5B5F901F068728008A8188A16E3034CF08C21A379ACAA610B63E5AFEEFA9ACC21191EE618A6777B979980B893B0534C44FBCC770E837B34D60030C47A2A790A09D42C72B4BE53198EB38BA88839368095C375552FBED00841E08F2DB7B991393342D22F3AD6A9F0FC763EF0B5706ACC402B6526CB9DB1EA65AF79A43D184B1588481475BD0175404DE0A266144867924F748A3D0C776062856627D2A21F39CE766CBBF64358FCC86ADA67769583273AEAC70F7277AF501299628EAE607EA4F38473643B3B983943D28CE3411495A88413B531D9D200B2928AB9C6324A5621F2C3352C0549D0A5C146C763530611A292263547CD7B2BA8C54510B9121056A3470D2B5E09CC7CA1377D3FCC18CE74B2C6E5C66B1B3D73AB1D7910679FF821A8525882CBBAE2FA65AA20BEAA78C555461E2D331BE364100A946E5EF90C8ED74E917C79C0E81B3F556AAE5C46FFE15F4C85B6E436BFAE3441AAD9C0215C6BB0D2970E3417565BA104249404E5817871309BB14587A807C402ADC1F45A6B927CB04CA885951ABC6353974214D4804EE7B281E3857BE6BCBD97E47202F5BD28A71E3D179E0475C262B489D63C37C5630CE0A210B4E5CF7DF19B81B1206DA4AB9D367A793194E23122A37823E09B35F718BC9026A26C970F9FE6C0969A1E75F6ACA3255291FACD3397935E349A50778E690AA734E7778B6C4EF0E4915A0106A2D8080C676E00B28BA090101561B204C1791E00A064595F6C7A5799324B4F700AC41B1B1A923234638C8597C5D3D2ABBCC6CA52595248036F9D277706785ED2951FB789A97CA061059245EA249C297CA7E88456CD8A0A0FBB27BDD6A71C0049A6C86F44792C7B470E6A73482AB38A94F247C12A10E546AB184AB0A8E97AE23C1C758A86CFE4BDA5DB2D1D1C8C5A1739A0A4503D82246886CF3CDB3369D2A51616C43D2850459395726BB975A46FBD8C73FDF726008013B17C98D47332ECF0725AB7722974C62021A5FD479E82F442D4DC6C768B171C83025F5B0D676C6A9882B6BED90A6E861247EA0A96FA4183275AB5D484695A1C3994696F38A920225CB2041D29D096DD3645EFA8350337542AE09FC6C5CB49FC3E74BAC9DAF5BD853406EF8CADB206B810E31564B6ADA0C3547E00A90A04793A58A824B4A489E1CF932C1FB12268364B296ACB615CF81229CA33569CAF28929D8FE52527967267606E96E49CE4A11B13E12258D4846B0A2345F053A53C4D4FF14E8970BC3C1B614F074667A624B858C54147B674D7B90330A09D67A31E5961C3B5C162082481B9354695778CF09A1F0A4C33182F969039820033FD071E8E33C87BD2BEE1050A88B86DEC251A976C3D4D3C6E5613A7987B44D37A8CFEB5BF8CE5CB6434284633922FBC8F9A169EB8F0BF20E637ECD28F50C940F2424186722ED6BC9E57547D3DA241F5E517683795B8757275476D511614EA405D29A1CD70EC0FC7EA90DD80C99D4567FCC58B91AB2544417188A6CB55C8CCFB212B57EC1B023668C5A043518B7E17082EAF7A1F60125E938CB7D600A746F10E4A486DFC96A13E50B3B0D166038666C6C02D9DF624754AA6C4824809A6BAE60736F368488AD8949523CA70427B57362049A3958DFBCF27A0A6A1046355C1A4FB2503AD7098A472A22B3AB5E53A91B5C4912431A3D9A18E0EC5075FB720CFC0048830127F78A96FF9837B8716B5D88675D7696793B62F5524827B7A4DA1034F65A8D3D509D95A51CC51BBF49181CB696977B2710C0B9EC9D945E3647D930B207F7CCBD36B6BDA2586F312073A020CC27185CA26411E4750DB354737143CF5540C3190BC225B49AAE549E296C39EB6CC6C28079456174BE7ACFB8A3116D25AE851A116898ED38552439C28F13B83FA78BEA8BB001F8785D0859F9C21697FDB3A7BB54859384E7315C5FD0B5BAD34917F2C3BF1827578354B97984A8A53360EA58CD96753D01502BFF21F1ABB31C9A7B69094761EC957AE08965C4936E9176114684294A631C0E12879592770262B9BD56604451EF6B5548145B316F290A1D2A68D1743B760CB4C373321D977DE48032D20CBF61BC6788C43FB1701E4F65E2821B884E44FCAD016A8865FE0CABF9E682773734B7AB50CCB3697D74441FEC87417965F5B6520591B6B1D550776D1CEFC37A1B5D12CC9E462A23332C79CCBA8C69989F60897696E738656964131C68417C1E89B0F8071C6511D1EA47B6DB37D314A63FC847CD625BDB6E8458C5A0B8B070AE7D1A2E659637D9073AC1B6E99A2C1AE520FE6C6A527B6A64F329672374505D773C0EC11343C24C3E71E422072FBE7121644009BCCC89E92239D7C761D092755800713E821929523BF6A1F52C307857A22BA9B54146B15EA159403DB8CE3C2AD57E665B5190C23E9C3F7C4B90A566292F9C3F1B98AFE683418615727A876DB6078A2B40524E325FD881625C76AD5BC6874A47BA562BEB7D11B46115CB3E10A8CF61D3AC73FBEB8945F4A3128107238268701DBB574C217E618AABA40AA56678219B66376E429975B4BA6F18CEEA451DFF657A6182EF1DB42EB918071F7C0BF1C4F16450DF1823ABA8649B772C273D310AF25BE7F250B6ADB5898D74D2C244BCC05744D074C579825156738EC2B8020DACE305117F173B6FEF86243193F8AC81D782B36F4B6565F22A3525248264C4BD8162E724B6140B1846F9BA0DD8464B2ACBCE11447E6581891C98AA39B5B5C4A299F473E57FB6AA5994E1CF71EDEA78F28C77C1C9B8C00A15F4CE7BCC56BCD3E6075329012A16377A2B2CD29D7CF97C7B34311486589A043C58291DA6AFAD42262A192F70C24E09ABF4D9272907B76BA832711176711271D7E9CAA30BCC5B0228B90A12A5CE182F1F2046830A009E0B6794844A301B6AC1646AF68C2F6BC16B094286D212C3353A26C7A7C6935A8CCC43156888FEDDA0971768F7E558C099AB5AAC79900660500883E8FB078E91120648AAAB0083EA0CB026DF22F82547E99BA0ABFB1741D075DCE1A84A567C4ACB99CF475BF4DFCA20E0650EE278683B6A2C5115024C7AF7EC16C7C65074B8535A13143CD590DD4960B8048498EC1BE5A48BBAC9A5D48002DE4000BDBD8898729C40362A9B04070AFDA881ADBBFE695A01D73B8D722B6C73B0D58A9401B803B1E882C5BFA1CDF4395D7B28EC69614B1C80FC74949AABF5523B52ED2F7DA22260EFD872DCDBDD51C1980ED2CE1524038644E72971011454AE50A899899E840AA421746D93ACC94FB8BDA6619DDAD22F88D9336A144491206B47B3EE659DFABF1E99A66D6DF3543DF0414FD2DF7BC5CC8EA245D0FDB", + "c": "63B513E1172CEC1034C6A626C8018657A5EE6BDDF4C781186EEB42D676CB998E00589EA7DA35756DBA56CA139FC8FD1E19ED21D41DA8A486F4FAB877D38A8FE4EF97BC51F338B086557004D7A53FA4F281DE3FB3FB2383177DC3AF358C6377FD4146E72B5E590C9C1EF2D57B5B75BD84E44D6A8A3CC9E1107C3F642A227E1C35C7954E2499393A8F48CD616890D36BD6A274473C14CA6CE68D84594389FA0D9EC8936F6B1A42AF4601BC909D52B13BC726F5BE1084C7E63D5FA65E16B7D1A36A1C38400EBD8B73C76AF1EF838395ACB2AAAC1761D114432F7FA5243B5EC88BF8126F58A9750F7CEDEB69D732FA12EAC65568D64443CFE4CF60B392C2D1D23A0576165A1AF54B8F21CD145D897084DC863B52E0AFD618CE8666D5F551498179200858607BF1A7FE0B4C0BBBF62E65CE86CFAE3B2E7F2FC8582491E6C8AF96CEEB56CAC2320D2E16DA251F6FDBBE40F86B4DCDB9FBBF80288FAC98686A7E0270AB991877D0094E6762D76EEE5817BEF9718F0976FCD7FA9F56F07DB47A0F0B1B01687F1E7364F99613C9845E697281F29843F2F40F1C1CF0A8BBD810A8EBC35A9E3B7598ECA61751DA922078DEF662A8B80405EF019CAD33DC124EEB339A48BE48FBE703100281E9B4AF4060DF20F45B7E48086059D7A8FF44863EC4BD12F7E6FF22258F7C00E7DA2364F42061E06C4A3C3E19B5A33FDC0E32C663CF8AB2448C3898B0E052B3CC51635EEA77183C7257A6FA2A19E463BD4E29946B79D41EF8E8CE154D7E0CECDAF985A815DC194D1DC436629F0542DFD5A276928AB1A225959BE8BA5E18873C188D1B07A55296F831077505D4774D26DDA37086A67239F134D129291DB08CFEC921B4F4649CDA8F45BDBE49E4456B5C6360B932B628F3F749D0A4059FF98F5ACFB5723770614D916249065534F4FFAAD2F31DDE7D3FF50E538EDED40D68FB0F748FC341CA5B13904D2DCB9FAE332861FD81A1BFE7DBEB4187CD10167D3EF990BF533ABD2D9AEDD8E6E0B5143FC89FFA640FFA1DE761011C8FA8E30B87299053A9AE2BBCC7540B82A78DE99CBAE258CAEE29B0AF4F75FF8D7BDBC7BB67D789B374B71E467EAB0A9BA4ADD8D3F90F6A2C12AF642F3A5FE2E50404D9F0A897A4E04CF1D1434EEB4E3AACA845000A27092C3C97EB2813684C466130C673FCA1A3EBE17D66D8491D986EEBD4152C9B2C21FB70B4DE6A2EF576BAF76C5256E630350D7EB43F164B274108D9DD72B9D3624FB9609FCDBFBE85E253EBF7725156665F9D05D4C0BFE2415FA31B282B71DB83E6B5CDA39F3632B44695AA2F894F7403F367DBF26B117031BAA296655148CBA5588EE35E903047E2960C5F70F65D4C5110CCA521A8AEF81A743F8130DA7786C584BAC7B50C14E6DE49765DF8D2F5B1B4611F95A4B3031E1E7AAE759F3458560DFA9738346BD11400A9E460A74DBB524C27D70C67390193E4B5E515972FFCDC487AC26D347250B9523BF8E4FC4B3F88774B4C1673A597C7E214679BE9E3C2B640C273D163F8654AE1304CBF1A1FAD2EABAA1FEBCE15F0FF808530DEE24EB5257256328BBEA9AD65DB1513EEC31300C3A5EA37A0726DEB74468F158BCF7474759235E74DB6874F1B1DF9EF4D29E091CFEA32B9B81D29B51CD6124358CA2BC47A0113F5DC66BD0A2318702A2133C7A43B8106CD2D8F08971FF205A0F2FF4193038A6AC8902D9A7C980102CD87A6603AC9C3EF3005E886483EA3CF7B649977064D4CC0D33FB8400825FD715D69A20BC76F24E137A63BCC05BA801760F85B7AFC3901D7A11E13F1E5E14A82C0AA5B01DC79EB44844687748C057347EE8B90E3D5B2ACB7B5D81C9F084C8068EC744E3172A043E4C9DA639552BA91CFDF52603D3876892250868E74465CCF735673D71B49E95456A670C0786AD65FD45AED9B51A334DDB2DEDB54A02B2F6C43EA82B2C93375A5F1E165A9ADEB803C91BFB06B85230663EF16DE7FB608F0D85CD8CEC9E7C24A863F18C4E6ABDFD2382FDBFD91812A4B8AFED6B688E76A36B08F38D2F56B4E40D5B8E95AFA86C6EC8CE017863892D07385B2240DC7F10953F68C1440DE5531E089EFA01390382BDD8962B8CCBA235D4BA9A5B46B62AC1D7A3CFB87B6904087DB8EE01E1C455138F5E7E30773813012F5238AD2A35FCD160D685DDFC1CDBE5E6B92F398C5265188CC7916D48810D95290A28A7246791853551659B559D32D", + "k": "AE84FEE1F4075E4CEEF6961D122E621C1380387FF0804B8B40230BD03A44AEE4" + }, + { + "tcId": 99, + "reason": "valid decapsulation", + "dk": "BE5BB6AC09CCFB2235EAF05CB409CBE8325C77803BAE90031C515F011C1F84002BFD5C5B02728A038566AEAC9EA6E46264B7C2E4B0A3FF43359574740F3A1DAA95C75CF38D4A2ABD9716909574A525702C47A060F6B71176014F9D46C9E27AB84F61805F4922D4743900F199079A23AB44B296746B033101F049AAB632A1F66C5E7308597AA18F901C25DB736A8BF929F5A2A62F7854C9A3A3223C87AA497CEEA791B2A30D24F89596AB553BF3B38FAA7D65A129AE411D4C526652D9C7BBB0A1DAF10BD3F392AECB7AE6D7343BB03B0D949737A66C8D4A2273876113DC4F819A3BF03745D8B194BC1AA81E5100D8D968780B3965D733A6E700EC389E895A9B17F7A4B71159F4D119F439829C386E80D7ADB2C7186C476E8D518E634C6F4CA3706F150DDBF30F805611DE98A7CAC261A922BFE9709004EB7FFA58C4EAD47E9394805A1C20A068CA3A379E15C351A6C74D40308A5856421ED726BA2B6A82137CE5916F868A1990087BDA16658EAA00404253AADCA629586E5488218EA7B23A87AEA4AB5E929C66EAB897A917C4124521FD346CF0747FBC70294702541676186B99665B2425F94AABC9B1B350647D24810B63FA4E8A426C34578FCCF82B4C78AA7735C5A9EA35D184C75201A17389B92CF1B81E5BAA77710DA55B7B6ED3928035592696004B17BB8B0759A3718847E30324B46ECB655278E7B9B7EB216D93A180C90A4DB307CE42B070017870E05F594693061818E592C782854E289649BAF48CBE72534D0213811095FE093BFA5A0F2B4A5D1D839E287552A0B4034D03BBA97B487389B9989A883B80B133537EB24659C5078870F575A45B36EC77877F6A29A2C2258C876E5FE664EEEA2D169366AF13B8FE5A33CF9CC5E7B69F0F92264EB11FFED321D737B15D5785F1FB129F8A7F25496BBC6345F66201F5C30196896B5CC324C17C3E9ED290A443C833F73931212CB4A7B6EFA27FBBD30036152EA110A84DD2969B4625D38443E95647A2F96046D80A1A1A2D2530CD0276A96868BC1C89442768BAE7F7C90B8003224B054861508092A029600BEFD93DE55403B5539C9BC78DC6624743F5337E3394127B9DFBEABAB8079F228B55BE1111D1F1739EF3CEB3D18BAC37980307C539C81CA5F8387766C33544C1059016729C9EBDD1CAE1E07D0D38BCCC982F4434C9AC9244071CC17EBB4C99A47F7A3A9E50658DB071524215CAC3D7901619C78E1CAB61D3AE8FCAB03C879A2657A04EF79ECE305923676015592392501453C3B91FC4A7EF3830D9350AA3C3ADF1AB9C25A113F0BC5AFA289EE9DA014F644ADCB76B74235CA06667FC0B96B90C3B3C9417F1F5CF9B57B11FD76A3B82A6DAFA5D3D726159756C081623B798BCD7F8CABDAA927D4307D30884712A4BEED01012166E953645B6E966C125432878606B742586512B1A22A67CA632EF391070AC9BA55582EDE72A2A96BD60EC6A290A6C90D829FE67C75478C4F07107C06335BD460E4F1790012A94A5478CC65C4639C32D191BB9CC86796A7836A79A5C0312B4B48148351B74F2155307F2635D534153599332021CA09ACF38704C85427339078460430A8E7754E8749538759892CA40CBC54491139ABB507DD6E3587DA493DBE949DF1C26486BCC89C297768891032A8879885336B29267997615B3A81C516C8571CFA556128D7B953713102994A585E05FB5764711A47E14D19B5CA22C26B42269279B5B9C677B172DD7417A0A26B8DA380704143783E53FCDF3C0CAA63A225B1C02A1838C1C87CC1790FB938FF8837044FA971C905E1D99552300A3DB613A302951CD1A4DEF352978408CC71C3A22DB01AD796DC813A65AD667FA3B9F8E2C5FEBEBCA8DF29D1FC00963CB204C1A2B5D99CE84E46C21C54A722A69E486B126433C2CA46702ABA4C30308C1B816A0CA2931D8C901FA2BE4CC8D6829B12E6C6264F73D3FC47276AC3241E4A00297B44FA60A63F89FD9F3893EE40285D33F4FB92EE9B03532D9B851E07ACE25061895C9635A76882C6FEB98C6825BBD3B9A977EE7515DEC149AF335E946B6A094280CACC2BF4788941B8605EC87C74C8076236EE98B4965844265697674E72118B909C189762BEC1B5417345177BAA15515EBB0AD15698C9A103928BC3E54B6413AB41AD7F9AE2C485C6B227A71915B6551BBCFB3579163607F60C9F9949FF36568812691C3D78C0C479E8AC18E13B3C59F857FFA66C5AFB1B24B40537C138314D3B1222418F8CC172EA801660518D754982F14386AC80D44742F50027275217D4C0447F62C993EF4B95CAB8865F46387738C0C4223929A70C268BAEE9243C9112C3F3B1857F301E5A80015383AB613004F07110DF16D40E36AB1EA607DB891A6437F1BAC06A81544D576B635D20DE4774832DC88803AB77AE3CB54E118BAC11EBBFA1FC5BB2797557B4F1351FBE9AD41331818F79901AB93BA74BD5142499EA79980042B39C56AF4660D65804BB8438FB3186E536186F2613CC6820F3D60012969BF41CB75F0A422FE945899C7CE81D802BF4742C103B96638184C9014C399B2E13A73256364E48B69BA83651D3077D113481BCC8FFC323F2E8027A738A087A98CED88B59891AEA8FB4980AA6F88E38ACDB61A0671289E2C374DBC929035182BF801D8473E7F6589878A413E4B8A0E39CB874717E2F453CB235495D36FBE534D4E42368FDA4C78B024311990698802BF65581E2BA7C2F9408144069EE82C63F866F6E6C21A45B1C438B5AF9405A7B4CA99C76C6E632F4C41359B574029667FF8985AA3F090BDCA6EF1C35E0F4A970D62551AAA316793825CA936305580B24A9CD16C9072C10CFEAAC1F435246E26B8F14A37AF662014FAA5EE7075A4C04259E18235124D00DB7309D10889BB33F1D69F124C96D8691E0AD01F1D371545C953CFC3CCF7965E1A5B19AE0CB4D3556A5F9B15CAFB5F6093BC9DC7CBDB1859C91964C3CCBEFCBB409663065E524BB208C6B9C7BDA0480862C72EBA89A6C7C8B6E8168A7B8A09969673830C634797961B758C9FC4378C129FA1D311B9B6A36AC87A66E8568949C27CD0851C28942C0BA6A53686AC58359FA76D6E685B3BA03EEBF988877A7C594698FF7861EFC176E265339B792B316B1B79804B7642534E20C8E0394B86089F75D278BBB06A2238C769C20C9F65A4DE827EC14B62FB262C11F63BDA4A8360153157F34ECAB00F66D6B57423762B81549B00AF6FABC53D0A15106AB0ABA083C59B891DBA2D9DD737A24B942525041383395B9B8379D0A1E63707868C2ED3676D508C5B85AB89B9AC48F7789441AC3852662D1F175B51AC1821582C31B83A29C43DD63C40DE62B223A3BD034BBC06945A5AF207F4E3CB6835AB8ABBB01C547A5851A1E847298A092F8442A183A559B9FC62C285ABE871BD9AA11B5FA49FFD158D50076357F2900D78B59623169861052D0C915B4C6FD89097F65A2EA61BA8937A1ECC342C9FF98E6F87954DF3832B31CB6B4ABAE392997A580EE6781A82345B4BCA31F960629B3BAF9B1CC1F49B27FDF27B7B504EB8C475F5EBBD5F9B0F4B3AA982CA73F47505FD0B41A4A343BCD42F48576EA92A72C1F1558334C5EACB1B3D905400B4C47D091A1CE19488911B8559790732420178AA98818B144AC2A1C47E6289738123BC0F8A430741C7136BC2DAE3C556E8CF4AB6BCDDE9AFF4C1B452103F73A1A94EC0B6E4ABB3F0C6C774206DDE35B3A1812C26505D098458B5CC1A0EFBC3DEF77F881C9BD234C0FD81BA1E69BBBC6311EE029931A7666FB17B0BC1A3E98642A289C0854054E51A79F4C94D2DA8B808A7B68A894DC3473BFC05382944CFF7392649C1A8F929023D9015D037A202A88CDA2821392C1BD13ABA1F7647C78909A412BA4306BB2D54C604A6577695810ACB717224A25DECA5448368C112BB084427134A73268267B62C46AD4208CC97C6B314810396CDEAB03C81D538E0F7314619BF61EB7435D21C01DB2B5CC55A06206CDB2114CD71B443A048B9A04BF2FB4FC7901837994E11309005505E8B7115978671DA352792DB359A727B48E027ECABC015575C1021785A31374A362E98A756409168E8B98164BB3611DB385FB5A92C199F51078B0F5C8F711C105816183C7962D8B58CE2CC29B3F7C3446296FC69514F02620D2C567C992621A52113EBC0E44C54D4A113EA5497F91AB31AD3701D603B6250B46D695709CC1D21E2A94FCB5AF6656096C84036085A57AB5331DB7E18F504B4807639816CCD535BE93B9746893C0D833F325741602B5A73308533B8C9E2C596C147CCC59829244701398B03CDFC6DCA65176F3B06DC107BF7B7CBD29B28687424BB22CCBF380C7A7B2D3D7D13B90443E2794443EAF82CCBD7E63CFF57398FFFB5A4E7821D0EEA23811AB1EA0E64FABA847E846AA789F4D09AD7F0DA3AB64C560C7120B8511CF5982191F103CC45042180C7495480D7EE2F431F5ECE88F9A4AB826DCBC0A4A6A4", + "c": "F0BF18E55196F7BEDB00C5F10C013E83040EC31BA323A6F5A62A989E1319993F0325E19718BE8663F644134D292DD6946AEC85433B94093E7CE9FF26B9AB56ED55630846CEFF33B5EEF22892C5425CFA566D9A182EE3FD7463A6B78F56684598DEC8B4F55CABA267A85C6500A0745F55F5EA38D73D484B8A837F6B9F780B4964E495EA41D480F6CB77503C16936B130447C8944537097E30C59BBA795C8CEF67D7070E17196C6176DB38D67F5C70198DC426733906E5E4AD8B3594B12F9FF14241AF8B18A871E851A87DEFCCA5851D84BF7B2B55D980A781F708EBB8B814CD1252A97AB28CBB86A7CC475D74241D4E0E97FC5A8CA3DB84CCE3BF7A1F519F1E1C5304D788B698B84EFBF6B77878B0DAD7ED1958D76E2AEE8E5333BBDD6C824D4975CDC2DEAE906ADD868BD6D7AECD00474B7EF6385F91658DE4E0B89B43F52B79C527DD2045CED383348B85A99B03AF6744A4C0992C5D0BA4D20CEF60468C13F482F45EEB3F7B2333B133FADFB2B3A6CC87E71D05B9778F0FB2FC485CB2567A759015D7F3F373718C8E41169D3C2FEDAC7185401D4A1D638E5479605177148A336760E30059F049B21AA37E9E11F11BED5775B24FDC294FF2B0D114FD57114191E7F537F9F7A2AAAE34B008C167D0825349B223834C23F2A8738AD549504409E5FD2AC80CFA6DDA79A39C5890D46A2B6F701F52ACEE18B5D975CFBBA9A98AC4CE9745ADDDF309DDEF79C450E51D152AB3A7F4EFD172E7D1A19CBEC737917240FF5A18A82863B45CC3DE3EB0EF97A7B77B8DBE7B5768ED8FB91DF1D608348FA06C5939E0DFB7D96DB3F498E67A9952C717E9278692C5291F5E15FA0878ABF863F10EBFFD7DAE5C014EE8F3C4F2CECA74CABED4E3DC58A891F90D57384B9E12722464721B668183AF94B8EFE12EC099B294126C5FBF49118DD2D2FFFA2B374F49BBCE64D60789E22B0B4A496180E0EB798481B4B58393C826258B7632058FB25EA78D9E72DFF823DFF36D738EC87AAF252C7B4AEE5B5CE2F1299A23F2745CE900E25EA6EF5DA30099E9590B61D0F61EE010071A7C443EA01C5C47F0A8C3A2B5BAF01A1BC6ED51F65D6C3ABBA5F500FEE7C613BFA215A35EA25FB9BA6BC9EC3777218DFB5BEC59391CE9BD30442F4B6DBBC4568E2049374E9B94E6FFE27323120DFFE0367F9CAFC67226A958C84AB58049E54A1DC64B015BB36AA184FE1C733B36E1E8682A8AB88956E28B323C4EB7940EA57771827737A5FF62AABCA8F43EAFB05433A838B116AFF9F7B4E16DA1E1A1BEFB4E4B2BE16F5CAD567D6B279B400AC729EBBA513217059B211E582B80864A93773264F6E5CE3B370A39A52C516E75004013690D9EB7B038C29811CCB56923391143B7774144C9CF031CF0337695EACC8406FC52CE3C6240769B2A03EADDBAB53FE2EE753EF02331817F8B9B40ED4AAB07D1B5B331A3A41F78871298DE2575E2F08627F18030B023CC5E15DC5402470B54327D8668EBF7D16F6B445BBC76E5346FE4F635D74563CEC32888F9BBDC3966DFA5E173CC5238CB7A8DC5B7B0C8D5483D719FCAABF35AF31FE3FE9EC7AAF8CDA6B749FAF8512DF81F6C9C14509ADFD1FFE3F4382A8124ADE2E0ACC6CBE96DC0147125084A243A77E490CC115E860AEE32E77C2FEE184A1367C2E821413A6F3FD7B3B50D9D96AFA917FC0FB5F49E0E92C07B8519A1C7817C6BFE5955FC183B087CECFE843937F62C4799FAAEADAE6BAACB7A54D0573CA025BF65004A8B9024519627718DE7D8C9F95587DDEFF430943BD23CDC9580688180C2EA5DE2036E593DFA3EF5010DED150F76FF11AAF9522741E8EB28416ED2495FECD9403D5106AEB42F35B53F7C714EBE4580086DA364E854C589C639DF4DB54D054E016BCCDA848863036E49E106C38ECD8FD89CF07A5960710E80061A0F429EE4DF749D5B54E16D378CB411CF10E1A63E0D807A509DBD898362CC3CDAA257B2559B94C28A471C1B8839C729249CE86AB203FBC2FCFC6168A188F6732C50D2F6357F0CE9463327D7AA4A5392F0279641F84EB3FD8D6B80305ACDD358C4CA5B338EBF072DCC16B6B1724362EAEE0FD3AFB39E5B9A593A5BB119F8E0E0DBB0530B01365DF55AFB81F39A55AD89CE31E5CDFC21097F31AB8B5F8D97F8532887EE9F46B558490A61349BEDFF22ADE08049F86D28EF43AE113E291E4248CD7B91385E43C07C1E99DC215E68", + "k": "3F9581E18A3B18B15E7A2112528B1C28DD00D14B3FF55889E1F6DB6C9CF79274" + }, + { + "tcId": 100, + "reason": "valid decapsulation", + "dk": "B3A0B7A4F289D2D071052C10AFB73E118C34E747312473A80F3610817020A1192EE4B48543AA1B3D9C131ADCA432C6575206C60F6A7E4CEA058B225F57E40B85D25038919B5365935A59253F1A92260A3E4A1448D699342E1CA3ED8756ADE5558375C25151B859BB141D3C1BAC33B788490A7DEB6D8996AFFA619629D1B62A01AADBAB0600F422238A103CD4812420AED239874E2944FE2111DD99C332573B02A661CD6023581B3B028AB05618B039AA62864784D1A093EEC71996E26C312B99546B297F330106F3A4EA054C9874393C675539B659BA4B33387128995C6275310708473EE72001DF18B09F28092AC23A978214FAFBA10DE208A79768836094F6E936CA457B28C64E9F7562FC9C6F9A8B08DC80C04D0909A939C159F920227078F013BCB84A30B73B4921066922C406C2A223DD17CA894479F7ECC2C4A3B21A4045AB712AD0B27D5D472798F35E03598CE9150CB2644ADCD78B98A902BE037D0B3287EF66136776690EA56130A31338176986E34C9571232FE063B53A3EB16C63AB974BB14A8BFA7806E9A0223609A0E36053E8821B743287C6787ACC1B8A03AB433947930CB6CBA2BC0173D1BC9490215E83C7501533B196775E973FB15AB667502F14F55DB7C2A84F8A538C97AB69AC69C19967D16C54D7315279037B3A722C26731AE8C84BDF137FEF328C1EE9433359560B2849DB8B050855BD09908E9039A9DD6ABF0970B9EED9BAF7B6B88509B35741AD7AA405DA1B8F90B57B349250810C73E11C7270911870BBAFD995CB77B55F3F150F49341D48F99A1448BB81DC048013B22445CC88A75C0A27045B18C1DCF44ACC839DA8B0058FFC532B94CF906C97D57B1AAA938E70293131F852EBCB99E5E64CCA12038BD335CA6077B70AA111BCC1BFF616C88C6FDB951A83118B4D3582B57CC00F724711F6139324A73556C150FC091AB532123910F1BA3353833EB5B653763AC1F4F2696866ABF3B117A0636AB76A6735FAACFACC3C98E924E78B52BF3632E2922CAEA22178A51ECB55795BA14854EB2272C259C3DB855A97C33CC5063A2ACD64EA8E57D938950A3CFD962989E39F0BC40C4BF7C36258AD94EC71A4D9A901361EC1711F9E1009F6849F0FF11547290E90046E814639F6E8503FB550EE4B33CE3A905CD7B94E752BE53B298A146585E6C96DA6517FCC221865693CB68B56D2ACCC1497300C8CBAE54E594A674BF489F307A6EE878A6809B1A8EB23A7EB8A9521CD3E583BAC89CB3B627FDB3923300C2963E1A16E352456E55A1C1A66EBA5BCF04473CE25687431C497679D899C230DBC43484B5E554244DF7CAADA8A991FE55A6C113907B8126BC2CAB1FC29B88182C1517789A8CAD1AB5D74A6C3182485DC42130AC14DEE77608A927CA38B9562B61955C51A52F5A0EE27440217C52BF63DE0C3BE81585E9636BA5EA0C507C46F0BF92FBBC3B00A448C724156A5A661C01364BCC2C145A535F5E3C6B738CD936087B525CF5FC101398959D7ECC0CB43CBB9D3993C691C01F12601655C96C787A8A072B53A63567B709AF66275200A32057E95E164B8433CD921AE4038B81E4554C80562DDC25375BC253B9534EF9C466DE52FF7461B55A50ACE9864165BBBF0AB8F323898A306C0809421461304B2087C41C3BDE281A92E69AFD507C8CFB1B3F8C4915EE39505B2C6D100922AEC77DB260FFF91AF741B7B013AC1C7437BC0516FBC1574FB75A108F02B2D489A2BC1C2D8D067AE3C972A5A7FCB50C6C6D1B7E047B216672A19B50691380ADE62409F609979E89372DC5061591548764AC2D58733A060744B31001A00E5EB9BE88031D846CF2456940BD525CAE38C73C16AAF5C4BB949AFB7DB0ACF67CCD3DC839CA9323FFAC08CC58CC92881E3623D8BC2AB0B38CCA3812701DA3950C8C93AEC5FFE9C8A30C7190226B045975193A71F52F8A24EF14E1EB430AE14219AD45F24C5518CAA547CFAAD30C7211FB84CD93057D65413B61B06C596210A662DF08768B6C28C91E2BE1D0CC978ECCBB9B92BF7E2CA4C6779FFCB332404882EC5A32A14CC9086CF5197B617EBB3F628006993985DFB5E79B3CD22A8ACFC147AC083797C994E14D8182347BA4384BF3F448E0026786C3A62A75A28EE3208AFB68C8DF9958FF80DD87B1550F047E7F63DD12C265D9A711B684D849A07911C09C9A8A76DA56B7F68611092BA9448038A76426624B706011E02C997DA250ACF89BAE6875052B8B8F5160936E80DACA41F7F491B7524709B1BCEFFA34091B4CE5158B129B5756BCA8664983311C42BB5A6632594B8A22C43913650CC833541304C7FA36657A9866A5A2C7DC1ACF2372A0D6230EE121C2AECC1FFB7B504A0277CC4CA5B7940608183F896362D0C1FBF361C3F6902DE391521D3BEABD549368B60E3A706508B06DBF215006A04E474AF0A862B6157AB7100B633119118790711A3339C9A77E0A70876761EA0FCB9269AA8388347AD13CBEA4028BD27B0EFAA1029BA3A694786D1648AC13323821A3332D5C5E6BA39C8E33245A50D17EAB9AD3618BD72816AA112114206817A4537BC7BDE984361B85695B6299FFA752985048693BCD6EC83048C3BC2C28224A6CD05CA30CBB13B85006AB1B75CF8D48E5E6C790C94742843A33F74B9EC329202270BB740B5464450AF379E8D9A28F30C29EA769BA77B5221A6C72E373FF9E261B93657BB692D420545397993B5E5CD0152651FE5CCCD9A3CD6C812ADB887CCD03D66D21241A74595D8996B280470FB258611C17EA36A2EB7404C8099992799630843E865A29FDB6FF4F46C2CB63052274F057210C6C3484EA666DFD2A4A2B877F5E21834AA1641F482A4A82742F72C8E4B19E5198F3E7A83A854BD3DC332338008E4336CAE30684A790C509736D89C0B3199C809E62B92257BB3433CE3423CD86B4223EA015FB16F35839F7378301636CD28B20DCE2CB871537690C2974EB5A2B99603D9E31F03A784A7C16CC62095BAB04BD1F38FDF30AFA74164F03AA4EDCC14554550C0D7C10D257A4EA66CF0B5BAEA25135E62A6AF1179478723E278C0D3D3A6BC6B662530012AE701C7971F0D418CEF6526AF4402C3579728825F00F748A8954776D36F7E235A758C5F7BDBBA638B4C44168E0566A0D73387C0183A7B2685DF330E9B346EE96966C77338E009BFC54C5550A3583BE2A4950C065E180E01321C4749A0FA08CDFE23229279394DA513F84758B82ABA850C4C5F60ACD449B520F46561F74809AB6C5D844A4B743C0A341C57DCB49E48616DD533F1460403AB58E516CC1E04ACF2FB26EAA80246016D46B844E7D28616957BDFE43D5B300912B3636770506273CBFEA7627256417C94A67E979208B318A3076F5985C57A6A495CEB7CDE0B1E36080EA4C0939C61C287AAB2317989A4E992D7866D9A937F587166F8A7B93091B897B1626B55C6954BB488CC276AACCC38F9B68E9A206703126EC53C4F493B671B14966489BB04216514BA2C2545AA53206B7B4EA1C9ABDFB2735AFABC5DF04CB7896345FB732F1737A3895BC52AA674304D9AD1ADE4D42E84B03DC5E5AF9DF9C4B8FA2948102B43A32A424C69ED42AA771B73B1FC23A2E5552ECB0B560B18E3C339013BC9E0616C2F3C6669C0A98E3A4312D6857D3802D7146E421BB30446A10BA90F111432488A3E21CA8D28D90B216906F3776D3CA82E6651564456B2DEF9427062594EE3AB4768474B8B47B31448C1E261804C45321008EF08C5B80CACB0C26D41434CBFB6298B7473E319154FF7B5A4413B4FA3080617CA180637F170BFE4D5A8AD4BAAD0F5452EE7A2BF932065364D3F274164B64E574C8423D8AD25135CC0B02EE6E63969307B17DB21FF72AADA431EBE56449CC79555C7150F911F40EA32436B798BB4BE2EF81192EC78E40210C587908DAC153D393C41BA9F541758E6EB2FEA8311F7F53D543A14F96B62CE52254BDC6432F4403A706A64175841D53A6CD08EF517348A5AA3AA1725B9868085F9A99CE9B4D053C50448CA69CB8529AA3384BB9CA089428A447EFC2BBC859C89BB63CDF0C4211148AE08AB3D1DBAC496A823C96B5EC951788955993999643349A3A4E7CDDF499090C3AF876760D6ACB2B5062CAF0B8D68114727FCAA92DB0E41F70F64DB0911FC006D23958A132FE5DBA81AA3637A240E59572C2A77008BBC99A994C6F7B35DFF652E19C64D52648058A2592C57A46E6B3F21771317028B0EF84546121896F734E7EBCB50B352A234BD8622A9E031B977516DFE991A13812F329074355833672AA8516483266793672967737848F684C058794344975C215BB96E14626AF52C882826FFEC4B7B8163B331A894973CECEC54D55CEEB88DD7CF9ADDFA9FDB40D53006A4062DB22311D60E74791ADF3308ADDD65B44D0364754128635B401D8846D91466D6E646B8FBB4E79ED1CB303303ED339B84920B4B6BAE9A506486CE67560757E62E7A10F0DD7238C0B4C7B078", + "c": "572CA32FB92D3840AB3397E12833F96C79B7B3B464DEE37308F07484289CBA75CA07D02D1D7BB3E7164AB6462F22DBD1664AA995BEA910754AAEFF85C0461CEA6A55D4A53CB3ACC6E9C8A72201B8E49C5C321A83D1F535D01D8A1217B9AB0ED6E64927678A266847A88D3F1FDDCC8342AB94BC6F3813634B0789B57B5B6E6D63CD645A85D5995E5D20374FA98483FD70030EBE04976992FD9FC31B9E62FCEE6F6F5D783DC1BF22795F31E96D7E274343615BDB6CF91951C423DBAD58B4667EEFA690FEC8ECF68D31B2B3D8F7E479D4E4ECB0416630EDDA28A31AD00921083D23F8443E35EFD8DF9A581EA03D8837793907336032079663B1DE0E32FCB10304DE467776E8A9B9850C18B2F01A058FBC554E3FC75FCCBE5B2EA30453B2C50FD8EA59453304472F46D955ED003ECE1C908B164D94983CC7A09D3AD09C907235E08624778D169B6728EF4AD2EE8C9E0995EE78C824F9AE4D51CBDBD4B6FD0E3D8ADEFAABD404BBC32F24E2465CFE6608A317D4A1E74B7B479186E17481D52F136CA5808514894499D4CC731C065683BD0D316CBCA851B2B8497EE6FD2C0E2E4D53EEBB4C51E893D71455E9634D4B7D3BE47D53721E83872A05EAC1C0A65E6E95A40E4E4F663C17624E06FD3155C4592CFA4A9B798F2E14793EE54CB9492FDC2515D927BD8D15ECA2D3AA4AD465750B41E4FD27AC120674E24AA0B7812D63745DDAF770387126B81F638F721C0E30C48C6AD3B5F6F43BBDB057202F5C7529BF978EE21F219FC393BB568F8E26C66EA341A9D1B1E6D49B346C5A4861E650AE95C1C954113F9F9F9DE3808D2828D166029EBD47BF6B51F0E7560779F6229EDAB36C3AC780DBD9CFA6212729923F4564868EEDAF361366570C2CB7C8DE2FCE1F860D762E113B00F13C2E848CD2FEAA90749DD4B87F088B03B79B5D2A5E313FFC344D5B5645224F5406608F7B25F96BE988220A128059039BE5E15E14F2033ED10C55AB52A4DF17794C323E9BE97B70BF71DEBC144038B9F98088453EC13350EB179A0CC1733B4E1E6E107CEA6C2C8CFB377D424462B45620B8392CC4B79C2EE8F79A1C24ABBD45520560FF052CAC5BB7607074FA1F6AA76E898DCEC45B210F6757A6ED0652351CADB0A884299DB15674911761277AE9F786A31E8ED198C4A6B212C44AA1DD636E69316038657C23F9E3FFDB7A98DBE6D6F0442F458C36EC49FD9702F12399487A5C7E0EDD4905252BB0937BB02768F648A5F872E6A97DFD4D0E726FC9D2FDBADD843E21660694E51A6251A776CD90B954039A2558A6AFDC31B744B84DFB74C309DFA635491E8A02A4187D4DA900944A60CA88E95053E30144A1FB5BF425DF49B17E74D936BE4CA6B667B2129B7BD389EDEF9F1CB157559F47D02DAB5FCA35A7199C9FEC142E44BE62460FA481BA298728F953DE87BFF32EA554120066DFA20794381104C54C1B73C636F6D29147761F77FC13EF79C08D7AE6858D1188A909BB4C28799391BB9CC4C60670664EED0C9CAA880DBE5420F4813AAB603E95EF25D5F0A765AB803B4947534F681AE4151F3743D738AEA29B6253FFA4F3963749118E198CA638036B4BA8955417A78683181811B29739321A7CCB3E4396B2608EEE157D1680EE57108572D8486029A7B3AD02BEC4ABDC4E3CECE793C20A839DA6E3854F428D24B0567647D7A7843E04D033F1DF7A7517FF30992F6101DF30F7B9861ADDDCD0831D1431844E9B43A3D05366F707889726E2913FA238008E7D6139A880E2F1FE07931CABC16D6EA85171475DBFDF15D200FB9DE2303D9AF2328032629982446494321FAF1415F2694A738A389ECE185F6B0B02E23CC8A77C860C40F1260A2B57C3514EA52BB1B0ECF135F84E26FB91DCA4EE6AE4EEEA59907EFCFA3CA018AEF97D9A95F8BA43D6CAFD096012A6E20EB322060D3426FA1758BA360EA0744F7EB0CF555094DEBFAEB6F0E114A76E1444C225611E17CEE1A60E6AAA3F3A81A690549F504D23F7092F545BD0E8757A4BA652A6353A9D78AE3C01DBB3EE75697E0B79D69625470202AE44D6B3FDD4CAAAEFCA6F05141F06A1EC14675352AE45D3301ECB765FBB80793BD68392016C33B16AE8BFF09885F0C540241C18550C1A9A8EA6AEDFAF4D27567B046AFA2DF697C3AEC2AD36165546334055BB1F8039435187F3FA96CFB5D29BAF07F70C0C09D3821E5177FA42F4E3F0673FD1918D", + "k": "6A8348F32429863588683CF653D2FFC757D19CE3E68213ED6B9B19D243530839" + }, + { + "tcId": 101, + "reason": "modified ciphertext", + "dk": "637578BC4CCAC66A6C58A9C0742A2B205B12BCE7AC72C4CB70AB619F1347310B1FA5D3B42E595C8303404AC4738CE3370B457832C14600162C80146E96459158681C296B750151C0AA883D75F38B53966E40231FE6223465A93E19C06621B65C58F1532F4B3A17BB03AA4294B61885BB1B1238FC7C44AAA96E3AA5FC5A84FFCB51B5995C846ABEDC84056D2A4148063A6DF2305B49CDC929B3A1DC06E301A0838CA44D70CCA092BE18192EE8BC7D0F661C640ABCF26B7F6F35268014786C0660B694A4A75149C345B08C046BB3845BF833C6945BC7249A8B776C65987505C8539CEB01453C04BFD299208E5A8DA70A1F6EB65D1C7C98CA700FE9C53156335E0F2B431FA76AAE962194E3A16F57843D382B2510B7006078CDF7AF67D709F4085D6EEBCF668869B69C168C6A8136D25DC7E1144EBB8FCA72181FBA5965769B54B9A8C2D096C2B48C560BB4989540D71C3602A82FDE4757EA2055E231452032BFDC173D204710B2F4845187BC859501A77793B0822B4AB619EB6840ADBB853B909AE8406F5502CD0DA0702A9353E90273E97A4F14237098627B11452956A470EE036AEA6294323CCD48E0279331ABF396C79268ACE9B3797CE99EB3E7C12497C6753399FE1AA6233360E9F7310E331909E83827719195D610673AB8E470B716169200D7CA199CABA7494D30494AC4B1914C8B127507C26483B52F80822D8854A54ACD9BF46E827BA7103831443428AD900C3B586D5D479342BCA48B379322E9B858671783DA28C8B4626DF9844D5C386E4141594539309099FC0B8E378C22E4F422FBD49375D994FFE70978B80305D94C37743FB0A25F89F637498B0987F21720B27CA863685F9525267308DDF1B17DB81F6BB73D19FB3744966B9CCB3DAE4336EBB34F6C332651262B4B504FEC3291FE4B8C2FC1A6A0EC047BE4152690AB773ABCE18C4134E88F11569C52567CC51738300AA7E8831752C728742A2D95BCBE620963C950BA6C1551B3047E8E606C2368CEFF7462312C5ED9F00E1DB04626A621BEB815DBD36778693542D321D4361DDD155061F0BDDB929401BCC16F086DD146C2E763A19F293D8D624191D59CE0148CA5B21F06347473669B1CF44E92930F4AD8C70AA16ADCE9C857F8771B2BA6EBC1AC024B741E663CD48866E099A456C980E0ABB660DACD8D66024CD247F665A082E90AE2F240014C1DEBA983955486F6A08E219C4D5F842FB00A7E2C0732F3646E3C08856C7C7C4DA2137F00B0AD20191136AB3232B78BBC008929416DB67AE4264313B39ACDB7837312795A464298082941331FF165CB46F76DB638A71B51BB254666715C5A5D83580EF4256D9657DD95266C316F2599521702CA9AF5514A65A742F91FE2A84C655318C632AB5A0653349A2DFE70982C726CFD5B39EEB87BF23C2F800BADB5D2A7C58556AAC0438D6A0EFA4C37991687DDAC918CA6C4A6D174B67CB46A72997A671B89899AF461206E4638046BA416B23635B624040504771AA7EC266B36FA8202B1CCAF11C3F2D25975A5AD3A94783E74C0CFF00796DA051077B35B00A8324BBBF85580E1A96D3E8C2946EA222AFCC75A8C4E17CAB31766006F2225722758403712A0311D53F8C1F51031BA4579D9824A0404ACB45B3C396A7FC95AAA9C6B5970929438FC3E1B89012CC17DE6497C537A83A5BB062B060F833A8D53FC9086EC1FA07C4D32C597A4D1621A142CA9C994DAACAC799866A92A03B51AB3FFEBCB5A522B3308CC44F1714E209E50243485043BF2464CE2340A75F477725784DD35C6A7CB0C7011904B505A0F62573D93AD17940A2D30445ED6A14E95566CE1076CDC4E55832A12349097518A9D5B11AC70147B14143D5092C14559528904688A9DCB0ABCA0D5A655A54820BB7D5D99AE2D966D23448EE75B678D14921B936EE2650670D0AC876C736FA00B1E6269CB4359DCD746C7F20292D1C406691A5978CB8371582F3A598557C7E1144BFC1C0C4E4A21FC83519BFC84B14649D9F89C9DE3C3D26B09FA7C1A7DB509C47320A3E593D37417739BC946E18A0C8591CF6BBF85AB38D7A57D7EB3305AB26F611A7F20092167651830F767448155ED845EF80C996FF6B1B9B95F3E75C7D532B4D6585465456666464E51502714E69503845DD270369A089594D30F6A8CC063707DA3408DFB8511D4677FDF5C875093B4F50BBC2B7AA3F0E3A39D750A41DB3861BA2F68A8735460CACF3130CF1C2040835062551B91124F23B24AF21B165939133815516C65AB24D9245EEA9875267E42B0C990A637D8103DBD6B6C9FF99B17B298CA2A737A44158FD800CFCB417479B7F0094CF2858E8CD21B484C62B78A04DCD8762EC1CAA5105A06ECCEC152A6F68B941487AC4B066665D3217CF629BAE42A3BA1B204030E56AB46A72C67EDE65F16404AB6B724AD049A99A406498A5C0553B286FA4F43103B494685F774A1202654DD953374D152AB51BB8E4171EBB145400CCB065456507B549F18CD63E2BCCD9AA8075C803F8684DCF796E186A65A624369812A3CD195E305A5724336813B9B00E615CC4B6F6575A6178986C62698EA4A92236B70A44222AE3134ECD74D8B693E9F615987C7554AC21522C59B0A939837B299F0B61F569B40D5C79485C46237CC38B7A242317AC1143B8C0955C0C348CD59C110D7247E35073294A693829B1DF3F041A845B9A2E3BAAFB2B1B6B36A7E0A9FD86A82567841E92A270B4606B90984088571B0B39D865B7C2105163E98A23CBB26BFE9902871B6D7691DC1056BEE661ECF874F8D63B2FC18926DE5A161775F807B411A0AC0494530189B6666C7442DA6A0C9479429618BEBF9C2409B81B223A326C3988E4A7179E79542E71DB9B54BD1209806128004F909FBDACAA7013A6C20CD0010409BF9C82AF008C725BE79BBAAB9A7128553A58C30B0A9C231849C92D83C2F7F65632FA4C2788A33F3959C212BB25B6C5B9DD733246C337259094D2C02A46C7437C7AF3EB9BE0F076215CB9687B6BDBF607DB3F0995AEB54B2822352E3A1E5CC304FC6904533142CC6BE5F8B290A1543E80694068547E094C48295C8BEE3AA02E1C17924248E4C018D8A364B0C9061B659B7F14F20B7AC291679958C5CE25A053BEA7815EBC6533ACD1C257A63ECBDE6A911C78B77C6F1307247C700A9C5A9AB830E604C56F8116A43647DA86E9BDB4B0BC6A139062491A51D0A6A405933415D51B445AB783E98CDA8D543B236748015859766403F53B7ED491492062DAB3050E302A3D05081930A3A39DCC6773A6E669A7622EB00B9A3A09BA9A55F2B127000360BF008542414F902C181008A3FC20724C5C7568315611305C10A7A3966472BD8A5D312AC9D57126A8C4B28796AB7719AF040B4C493CD41F6514E81CC3BE182FCE7171C4C9ED62912984ABDC2DCAFE1A172DE4240C446517F94B60A18166B488002803BC6610832D471AEDA1C28324D2C4846350A697F9267D9930518A810407BC32D5C2D45E6BAA2103AEF34B70F1A74EC3CB3F79ABEE7C0A2EFE54E7CB6BC9518485EA47AAFE9BC53F4A7C26A14F2E37FAF8201D680C8C8BA399D26AEC5492E1384C6DAC0BDCBF57EF2E03A9F7620F467868F2711BD87A00FF6C22D4784CCD62934D160FA8ABB06A2707ED0AB6C38A94ACA2DF24C9869D31E0C6C83662CA6A45C67E8FA837D37C97FEC96D0785D12858D8B6AC5C600C1F321C65C200501F21EAE78ABB46608258B12A073598193B787B8AE1B21B1B839B9CE236303A314330BC1600B44F20A06C4299367163378437579691D7D8250D11B9863773E1C960A76E59F5355A8F69796A18206345C4754D53A35C936E1A78CFAE19C3C30448A42BA7567A8239B9579073629B26E99BCC4FB69A4B4D39045D5CEFDA22428096317BC9C0B307820F189F124756FE79E6CC121FBC75DFA94690D5BABAEA200E522913976AF57841233C43C0ADB0F7F343F46F4945BC4032E452DD83499F031243B94B7E5D9ACC5A9B15A3924EC34015E805DCBA85A0DAAB340EC39A715765A14A8832541F171236446CA317A840384BA12E4C63BD33DEFB4CD7EBA8812021A4CD7C6C0E2AE215599E0B938EBF6A51B7C25E739BF33B68F05B85CC1D03617E6C46DC6ADCF09D0E1FC934A152B7FE0742E185D30E5C3BD87AFD5C3C80E113CE017705EA021F13AB9C36322DCB3A01CABBCDA2A6FD81047AAB12C99AA5D4514376C4356B018179D474F6D210F56AC8CFC0573B921C95C0A29240CB24AA07D009098DBA78884050C85857CBD043760FA2334B6A263BC946F96CA17A562E1A02D156558EB65A6599820475A2B18A12AB6E074A4D36C027A0AFF4047629797E6498A2A0A4486BAF743CCD8B1097F73498E6A6B66E3D27A7307FD0FF7686686FC9B513AF511FA3C0FA0CABBAD9C65872DA861A7D3643A21E4A6CF133142466EE68E2D0FD662DFE396971A27281C7BF595FB410928EB6DB1C162D2AF7BF0B5E06E30BB", + "c": "2876FFB597D4BFC600EE1210EF5E0F03FFB231BC973371A36F03628A6B1F4608A3F32BA585A36D4076F2420449A0E3CD5A69F1977491E181B76DCD72AB1541A191C5F81FA16FCE0DD857A217D9C694AE6D63EAF95F589A8CC14F4425912BC7CCEFB06EED3D663A94482379CD34E05DC4D48004939EB558766F06D9B6553A417C96F8BD8CE6BF598546AC1A978854D7A1EE1FB05E03EFC38A8F06B9355E4FF032329D8703815512BA148DF01B6CED36DF0C77029B4B544ECD5328F068A28D39B51EFCB38F392A63BF894FC278FABBE5C1F53543023B9367EC56FEA83583C576412D8BF1E53E4054326CA0F0CA65CCE591792A33EC208AE79BE8CE23BC3B53A32EEFB5550A45307F9698571D08FEA8B15FE8BE8D6B5FA467C1889BB5416ED82B62BDA11391D9E3CBF013482B652015DFCAD6FCEB4AD514B7A098FD8315CA4B4A0B0E93B719D7BEDBA3936A50F54244D3799F93BEAEFCB3ABC5A397F5BE705EC702EED709E20FB8652BD6A3BE6B46ADED9E8696E560D3DC2588026E9B311B9AD0A040DD8B2B6AE34B2FF513271E1B348DFCC563CAC1725040FB4BBCAEEB06AD9289C3FA00647F9F824CABBFAE1AF77D0BCBFE706E870536A953A208B2DF4F5B9373D853913DDD9CD357C68EAA2A5F5B0B3EE66802045975123E196AFAE6A4E97CA81FC0EC6EBF71634E8A3EBE07C0D0DD1B6808970AC17B6A16909A7BA222D4BA83C7E5E92BDDAA107C6A85E0C8F51D4D4C244A4932C8D10A7884FBEC0E830CA62739D48E29AB9DBEE4DB05B9A22CC3595FB5AC490108C71D63D70328FB073BF774B1B90CC75732239ECCED6E999D5D9BE7DEEABC895D0C183F6D4F2C8AD9DDDAFA07FD460C6474D3C90A1960159AA7086933053A16564244E3F2EA767BC77983253C6B6CC3240C9A5971F1C4F28B885035F6A36775E858D3AFBA13A1DAFC6620D8D86F607BB539F978F6028C7D77896EA26B05CA2E5812D3145E33616481FD3BA2C3C325A9C55FB0A5C8843339027E5B13EE7819C0645CEA2BE4912BB6C889B9EC3D12110B8A5783500F2BBC9C10DD1F203C6633CDDCE21A26C797ADADEB05FA7418562FBDDE4D05E34BA62DB0CE78F6AAA355D3B79B3329851D65C833AA0320D0D5846F4FCAA9E8A2AEC5E2151F87A91E25E8F477FABD7A588673CCA54078E0F56058C9B3C13C2F71CC8AE52E225560385D652BEB776F78483F0A5CE05D50D2F27B37E3F7A1FDC00E2C0BD554953903D1FAE2BA1FBD8DD07EBF85D52741561D90605EEBC8C775AFC7870C3C3F4BD5CFBD8C30C0A7292F22B40B2CF68F9E5C000ED731A85ED8D8032041DD92791DBC434CBF41C66038ABA11E855666B4E21C5A3FDF1C4E2FC0AA7F5C088AA43BEBEFFAC40F8F0819847485FC663ADB09A4B65ECF77A4D7485E00DAF97A80CAB31A101C6F8909BED03FE74AADB02BC6231CE4FAFAA9995850723EC737AB6E29A5BE2DDA1DE7B3DEE75E2347F6FE8F15146981F02D6C2DF7F1AB164819EF14A54D17E8E468C119B2C85E0CD782403BAAB1A37C265B938841D5206BB7331B34698FE9B819C76B0668E5F802B0D6D6E19583F437D36A19FC74657F692401168CF93D5CA56CE7999E315E67AE7B69C39005284B6F3E7E89953516678E5AFCFD78F3758FD3D879F7C4F572A98FAF0E805FD09E3CB01C66032B05AA6ABBA9B137CC08D3F23D90BB80CA882BA3E2539B1B39CBA12D858DCEF15B1915A0BA2A12F46A435BBAD82B85595C9BEFC75F697852729FEC43302768AEB22610E7EC9A924181CE5516D2F39EF4E16F1D8B89B131C89CDB0BD6DFC311B0301134EDCECA679E21A21ABCB5170DDB55E833518C025F47DE57A21CCA4606F23468F43E8734348459F9119859942814FC83D9D2FCABD9EE402C4E339DD3609A164B886605584D644773FD6B2D53316C8A12CD6E1B224FC4C4041E5DE371F471001658D4F477A8CCB2BD8511CA9FB0204D8D787B7A8FB22DE749648CC5015B0D987678242677D99936EA70F258F4EF0D079FCB1BDDEF587FD007D6F98A5D612EB121ECA5DD582FD0E57535973B395670255FE46B47062AE919A3923DCAE7EC061D1A28F76FCE3912EC74069F7C0D0291B130A94B7E9D14D88FF9D23934ABC9F589AB1E9EA45A5C1570EBCBD0667280B357E48A14B6ED53B8F2C62AE289FF4F850869C330BC15983110D4349A6A88D72D02511E654722A3AA6775188E3DC765", + "k": "5E163575FA649D21A3CA7DD658F2D98E3C44723EAFB2CC1A2D448DC9A518C008" + } + ], + "encapsulationKeyCheck": [ + { + "tcId": 156, + "testPassed": true, + "reason": "valid encapsulation key", + "ek": "521CA0F404B0DC436D02EC4D4C783A9FE60A027716754CC83A478EF9B292C71B1B4B78038B582944AA857FEC42ECA10F1D42CF97C6790F4B26152C5328279CC5A0AC9C369DFEB82A79E498398B54357C5B6B7AA3935A5BF456865C93669E0721ACC1C1233BC75ADB9A2845BEFAD66045D93A9A1A22024688DA726B38BB84D8C246A7E67F909A5329715212EA2F8D3A4631645887E0A32DB14957A39DBDB550395352944B7D86368BD4B41F11BBABDA9360B5F7160D8A123439B6E327CE6A55BDF08ACDFA1B71C382ABCFE9209B8C960CD95C7EE8B1A9A98CAFA4242FF25D957A82EFAA926E99372203381F801E0FA1898855A1A228C533F295E342122574936CE4925DB87EEEC11ED7306B0340A44B37C1C819ACDC474ADC004FC6CB2C578A2B089CC4894BC53BC4830F3B54D73AA2C10CA7E46C322AF962E3AA6E45287538B920EDFA71356B0BAD9A0D1E0855701CA4C0E23F66AB3C7A4615A5076848D564A9A154A00958CDC9152D61A9915484E65154C4D33BF29285A2632AE16C81040B2C3618718F807FA0821DA33A7B165379EC3757ECC21FB9A08DDAD77A125476E6266D42E20EFBD99488192DD735A196A5B2D310CD10E3CF711B441C642887CBCF5133B056EB8A1E5089540BA2E4404F25B71FD787BACC2BCA73E0B9DE76CEE2664AECF5499A687C7C167F08836A15B832E69B66A1509725720543A123311159063137F25711250016AE983684112A0E5B0AC03AC28468606ACC5CDE2450AF086E386149C76015CA6ABC228948D8029B224B63A0A485518753E9271B74DB35EB080B839A174326A57E332C24F3506EE748D1C56835799EECE13C16E53D3A3966A17548526B0D9132C22788687C8BCF3288784B31539D615344ABCE04258B6F218E48A78A4732B91F75455024ACD237A3B2F143E447533786889693AC3B96492D58765ECA406317085DC659BF893B7C11CE4655B2452A389CD912BB98A63DFA37DFBB71BD58A5CD6A6094114101940CAB2B90EF215A0A356098AB06EF068002E165BC21ABAA68B58891445D434139746CB54402972B12DC75B53C471ED6951498DA464CC4BFA76B2C5E8165D6D1CC76C3506261052E411B1DB34BDB6CBBC4A8CB0EFB980691AF6508CA8AE97518EA0E689747D2CA89FDD44F017C49FDCC6DFB10465C8412B175697F052940909BE126C5D35C0E642B6D251B7D526C11407962DEDC06C8DC138DE5233C5C976265A8DA7B3683C4467497592E8BCC741449AAAC41050BAD867C36872C2119A4C7DB34067A943F0C951168252EA0FB5FC6D769F940BFFDE7C46DAB183E161E02046CC708CE817046EA8063F469148D2336432B36EFC266077AA88C85A26606AAF7EB5AB6F3B64ED95FAC5C3C6DA380D0DC93A30C437A14816C25499B29132D492C3792C1880865B3670857B1758A97B14A29B569844775444C915707E2870B79CA8695759719A34D73062F51560FFF19684F1048FF9730E2669443EA8635F303214A629B71A04E6042B06584E6386F3B396304535BB8356240353EC8EA8158C01565154BAEA1509DE657625C2731A01BF741C5242BA7B909BDD6D47175681B92898AC8872F013852546A982081895729A369F40A70073B7A59BFEE3460982307BDB163D3A05664A91961B12EB4248C190B8D980392BA0ACCD9705B4D3BA54535426F10099FA8198537ADB66656B2D2843BE15F42B93FB4E521280031B27798AFAB68EA084EC0767FB5C76F3A129D15E985AA18C899A9532DF21D3366024D9C214B338DA5891BCAF287C4940E5B3837F66A4AEFD3C4A8406532FB8FB1CB8E45691BC1E960C976501FD532AB164D20A3368B0780C71A0E68C76A59F6CFC7F6663798B102C0A9FFA90C18E33121D127F6467E49471681B186A752B63B593C02F85F18384BB8F6087E59AE3EF879178534DE3361AC4A03448486BAFC394D369F450C234208377833A5B9768E13EB4C855C7893020A8091A8A9709A13693CF44CB842A747274273989C28FE3CC0753573BFFC565BA3BE02D7348FE98CC9BC616AFAC99308805ED0A1480888168790F4FB5417F27BB7F96EC963611D2CB468A98C508140CF5267C90A21E07BC471064EFAA4476D0B95C9A16EBDC34965B3B219DC759AF65392CA3CD486A2B9744AAF5BB33CB727E5BBA176CC4FC28398CD1EF5EEE6429066EDF3FBA01694C6AFF0F246418712A4BE53E1C13A75" + }, + { + "tcId": 159, + "testPassed": false, + "reason": "noisy linear system values too large", + "ek": "020D58896B0BC35C2D70DA8992C424899A8BA9414A1A8319A70659FCC6A3F2B794E84B753B0C8822ABCE063106F4A8706358A0A422AF8FF287EAC05A3D8AA84339996763A8F7A9833D2980F19754D18C37E8C7B46425BF8CC1A75CBB7E1DF5AC70E2056AD37F79F4AB83F87E5E32AA2E36CCA999AA9A4C82E81CC94B4AB471785661349F0FD832297A3CAB38CEFDB45FB020495AD1BC1108186BF4B31D8354422B21AE719809762132E076E5A5B692F40C704ACF1D3904D690AF79F217EA38A8545281F3309CD6EC565A5921FDEC938853A63EBA0A1E8A68187CB5A3D639D35439CE5A80985BBB9C7A9762EC14D3EB5501380D7C298F9FA177EB5A510B04D0CA7A19C763ABE4494A72A5BFC985BC403739DD8A9A13681DD39BCC93E965E4A09E44B97690C36D0D58B4F02126E4A0A07E6AC10E58C24B554ED89AAEB15C49FF53C16433835DD72FB4F69458D1BC99330891F1BE9AB172242412FF819F82377914B269B7120137375BE39B9D32112326EA5CFB5A071B479A73E0532B643B53E58ACF1A987473A3AB017A66DA9AC5946097679BBADAB1A0827783E07F7BC198903215A6777FA3B77DC264CE37636AAA48A5F4208C03636E71C7746083BB7D403035390DD78118FDFC2F7DF282287C96AFE8774A20CAA6DB69104038EDA590C648C69511811E9C28D52C3540721428336EE3D7732A287E51109BA8589227864F5C27C5DD07556B33CB4842340E5C5B027464A74734F088A591EBBC3CAA0899359E75C395A6A57670B04BB38619BBB82A27C85F063CB286160CA46C372604B794D4202EA16F8A757E9DD8A54194C3384850D58213C7533CADA33DD6704F082B4EDFA67FAFA2CAB0C89897FC4C8F2255B839CD653C4F03A728F0D25CF2B99AFDD0419C515DBDBCCA0FC6947B0A0C936AB1F2F290DCE5474183AD4DC85218EA13AFC5B671926842A15D7C367D04EA97890B371297491D21720DE148F9780866E1414EC6844EDA97AE951AFB0AB07FE61B1B856209DB6DD522792C03332645A63C460EC595AC5C49B1DE1980DDE8B4BD5505FFA7CC4749C42F0694BF3ABE4CC6ACF8A7235766513319915B58522487C7F9EB9C2754863A40036E28332265C2A4FC420D95AF87063C244233D5C27CA5B0261669386B624B5AFC992CAB773EC948B8957079B3B3FD27C16CF969005B27B4C438B7269FE317024814AE56862D18362FA47A0035339A485528540BCD27A288489C80AB890BC3992BE39249E507CCC92A250EE631297AB19E6C39AC446EA5D54C675368BD5BC4002463D8CB9A294C3E427A2940BA5F00B97FB2B1714CB6880F5B6A13F868FB936E23FC4B7EB563E2F1C5FB421A449715A1F8CA0D3C6AF726CAA5D93C255BBCB7E60843A54EBA470B32D19E3088C17AE5BEC5408ED9758639C2AE50FCC6C5471B638210F8BB1509691B312082DDA85C91B31517D78D4CF96CD94335494A66B8EC3F9B2478DC264A1B7462DEEA1C44CC0D357C00604ABCC2437404771640D01FA1A9A73E900EB0B7CC8CB3A5EF621EA0118796AC7907612F339654CB519B50A04B6C38A1AA7464E34A8E9B9029A4C1687149B0959070CCF4744A03951B4AB9F6B88D85487E2929338CC6CAC39651EF7C9B802657938B7C368C2B6B0539EC0C152EC95AB702662C21685748A968C68C150A96A9050BF3C597ADE0699930C9780287D8811A68A74AEEC8C335D4666C64AEB257C508A45E32D7A6CF86448B81486296CFEE29C3CA7995FB9B55A8D62C6BD3BE1C267627F4754DFCC1A7818051BB0448219794B87089B8969C72B01FFC168703CD11992383552E14197AB1C99AAF848231579EDDC305D4942B27957700ED5AD44797E0B5953AE06EB25CB0CDC94C6B67A423CC6700D1604E362FB35105A8E6A00ED6587A759B98E37BD358C9FFCC4465F9C5D945A93954550A2B45A6CA90A3593E07163D7A8AADA9848924B135C4911224B52FB5C2823F499F98D0238811B86B55A0D444A2B14B885A0B23F510B5B80280830B5DC705122595578FE076FF859B5FE0878E7B17932A3B99AC58D9A86812AB2006073B8D80BB940931CCFBA269D319D68CC6B1FB5F6EBC288AD208379364EFA73384A8A441C1759BCA9D90016024EB2316D50E8042CB74F81320D089F6D08008007B7031A02BF773BB1C620A73B3A1579FFDBCAEF6652400008B53F560F13D5DCC0B716895116434F2EE28" + }, + { + "tcId": 160, + "testPassed": false, + "reason": "noisy linear system values too large", + "ek": "02BDB4EA986EA8C4815085C9DF937348E86BF9CBA6BB910F2BB760014B2FA93C86280699AF5323D0D2C1DE689F11139BDC58CCC49771D5B1B2F17C5FDD1662EB0C663CC4935DE26E64DCBD855BBDD42CA9F37C9263575C6008AEA8803831124156F71F22743DD257BAD514C49DF4A4C2828E0070130EC1C2026213AD9694CDDA308CD82F0C437557251807392508067D3E3A044AB0665B4A23AFA209128A010C8C91D685BA77C65F119046D14A196C98882D135B14303F7852B1E4DAA8FA873DD2D889186941266B744EECAF47EB84E47A69C86C16C71279DDB8B3DBC7A52B7B2D8D06AAB4F65992E79D82368C6C8A7EDFF47F05E00D2CB35DCB2653FBF51B7A695721C68C602B6B08BBB52AD713480151DD221DBBB129F6D30521ECBECAA98E847470517968FE34CB22EC2245CB0EA0F0B1A4909406EB864990AD7948884E1562F3F4AA823A00553832DBA5B5E15027184133B446BAEA9216CDC3C633481F6AFB0263965F2B66C01EB180564C7187572073C31A00979F591BB6516C0CEE373D117882434A8024E316D1D192EF8AB1DC1013AD07A8E78C226A3884C2A1B8CA372499E784FA10CF9F98A1C22A2F47EB4121B7157AC0C2DA66350FC961367B92F57CAE774443F705115C21793EFB365BCC3DB7D19F18119ADD93620014A69395C31B73A3709685B113B576711C6756132687840F2541BDF6A70A0704E4B4C2DCC33CDC9134586A50AF2010FB640A3D03411A0B8D96E394D27733E7B58C521184BC8AC74592BA596220A1D00AD47240E4593C438B94EF29356D65415EEA24C11644E5C726991A5718A9B21A91CECD83AA172C9495AA1B059845C81550DACB195AE6BB25F32F2CB7928DC1B83FE32700C78C542641573B241C92ACE3766522A84669808DC6F94F98107C3CD15C1830AFE473008594BFD1C779E0017AB9CBBAD49344FCFC6C38B9A5F11956316CC34836768CA99D8C48BDCAE189AB93A556493530CC4E8775ACE118B4095C353C9702AB023928E16981E61F325921605659825034757564A494B8955759DCA7B10593AFDAF396DA7B24FA9570E516BDC2EBBC36D875917224AF4059F6863B511BCC5B186968DB28F122415189B103F4B7AE003E87998244B71326CBB19C34017E892D2676A018268A29A77E29A2697C0655E2455AA544844939974E961F204994DDE56573158C6120A2E61B56D7663926702A1DC77BE18957D789A512632852F60FC6220789CA9FB2868E49F95903C62E0CA13979F582DAB615047B06392BB0DD9B6555D934E9E18BC1987CCA01A0917C82DE6029B593994A195BB501A43EA689FD8B833BDA0F9DB2890C69846F625A84D2C97F47A21E345774DCC8C58AB7AE34BDD838595CF182DFD837EBFC67F4B96F2FB2C2CE046FE62B5FC4291C26B63F258606294101BC5588D13482FC7A88645111B9FC195A8C15FCE93AFD7B1B173C987C282B879A4651939F83D2AF413BBE3F6027299CA09A8838EC79700DA19E92FCC717D69ABB775BED5C19BC9798D6C32AE6279AF473356F5B7994676291E00A2333766C5342FBB2C7B261C3A10B147C8ABF768B4781495C434199A4DA7BC18B2BFFA1268AB745CE972D5265A9735CBB84D67B7B236C3213058680A69C99006CD0850035414B23A40B6670542A98335664EF513700DA6A48EA8290370F5974B868BB0AA7B3A145A77B83A01DEF743404385749B177B5259474F28377AB811D2075F0E49021F184D1B866F3A97BAAEA7E54A292C353AB8C974D29B02ECE23835B163A120CB990D81310085A6A18BC6D69596BAA0C1142B297C18B7B09A1D486A48A812EB5C2875B71AD2D4CBDF8A6901C3C64A8472730B963E7DA999269A849B376CC6947E3218326B96CE175BDB7E79CB1A20FD8439F276C7ACDC47C0E266AC8F270CB349CAFB05D2C02BFCF5321DC3A3320506100E27576D58F0A0202EB2AC800E0801CFC81DC0AC680B3BF25CA9C3B5A40B852A547999637619063828E1C6C0A6FC73B61B011FBA9101944CE179A504BF6A5434948628A0F0A35279480A287E9146690BDDE5A41AAF5148351A3A7C537007100C4764C18BB0C8480C7B34414E833257F1434733896CAA22135DB24C7D2B3ADD53B5A110DE4A5A883B591E029462537657C73A24FB92D6DA735E6035372758299963CE1C91FC62EC34416C0CE6482495C96BD8298D5AD3D88F9738C005A1198ABACE6" + } + ], + "decapsulationKeyCheck": [ + { + "tcId": 146, + "testPassed": false, + "reason": "modified H", + "dk": "B94C8F189B7CEC12B6981C64BE0509E6A2C6F687445732862C984690FACF0AD6817C863F4AA45F9D87C0ED432EB1318354A60BA8EA823E9838FC1B08A9D7C5569976BC624378882CAD589586C95B27C6448819AD8F59A10B60B645BA69C9479F604747D2DA20827B6255AC01CDEB71F5B157729872CCB5A1DDDC715D0768B104AFBF279024366E92954FF45A06DE32C87818033267611B2B9F20A2246DF979FDE2C595929910803A2EB8B4193B029FD51594015A2CB48DEDE90D8913B118F50C9CE4AD1A6B3FDA8B6EECE6673EFB3F7B229CAC7B5B23E86ABCBC2E58E5CACE626FFE040D2EC83A4D457EE014B780727474E51B93A71FEF1545D3C913384A6375E62AC54404809C2BA3245678014D636A435AE50FEB45124AF16518851F2B121FBB982170E5AB454B53AD8A9D3802A400D13A8A724A8A1B9F6F49C30D059695D520BA73A370FAB8F042A3714AC400714F66328E0CD473E15B71AA62773637BDC4B29BBA1A07EE15BC73839B7F08B9EBD4CB36D2427E3CCC92266F19EBA7B6737FCC58848BF0AF174613A4316D7933651FB39625496C26CA9705E48BA105046E8A226DB181543A0C26B455482618460680F522C92F3549F0B87A14915D0FF15C6BA5862C556122311A4DDB07B5248C3ED995F4AC80D015B3E6A71C97226BA61C189A460D6D368F587825EC51C50669269B8B8872B9933FC1BAB5472D90AC8D36E453242826F7A7319FC8C262795947C2810104B98023AC84E854BA59A3319C6EE6905E36D0C3B0B41DFB393CE378046FE292097C528EBB72B91B20C637C531D393BFC43D54692AE4D281E9E637137463A0701D286AA89ED464499A1458913C586A2FAB90506B8097E061C0B710B68EBCACD8353C7CCC0AD37A0EEF32A4CD1232B69AAFCE9AA5620B953BE52EC7594D68D57976EB1B38CC677885840A463C72C3CF17445AE4202AEFFA4A7EE0275134162C4BC3A82BAC03864D39C9406DF72EE60218D83C0E0EDC067A1592D93A8FA32B84CA85BC51D15BFFBA17F7D740143A068879362DE00669A0CF7EF70C604299DC3CB52F023E56F544E3E35CFC9739A64A9392C0AD1D927D62F09D96418F3D926E227C26E02CB1E2C10B89E373A1D515D39C18F259212E5C14A7678A3A761F18E488619C46D2A21590270FE511ACB7C3BB45F9C699E63596DA08827458142791893070D6A130182430665A3152F62328DC4C2294C874F293331604298675A4C2C31F79ADEF67C85329082DDBB222DA4DD0C11B6FD8C9234ACAF4D40679C82BD0479EEA17CCFFD209409C5C2AF148A87C006B65B39436BA5CC6440812ABF7B08364764BEA966982979EB55684D1433E71A471FFF73AD423554514C144CC4377BA07197CA72EA6CC69E215488CB91C7980EF74CBB5E780931A34CDE964DBF21E3BC266556281897530361BB3F8B82CB5415FFFA1CF8E32B12E2124DCE774D6D692E0F803CA357536101CA5944D4E1A063679B0937ABE6BA20A8CCC75FDC03AD4BA2F61522CB8F565B05232E2FC104AE09EA156898E8B2D3D0A7FFBF195E9D1A623E73AF87181B4D932AB259B7D2B3814E721D499950D184E56F109CB1106E9E1062888B6D3185DC30AAA952946C81B6FB83B507773877F7C84EF142C235241E2901D72CC8A6F481A19721A9E877BE36369B4F614E598698083906061074D6055DF372510C543458CCBD9F10881D209176396EDDCB8118B3E5A22A989A56C67912721B290868A75F6D820802A0FCBF00AEE135139D8A1868745FDC22D06F05748D323875187FF298E2D37240AD93EF7A8A3CB462665209DE6F1990828830A65446E2A4C30958BE3AA2D4F192D673A8E99497084544122A56BE4F6BF6BD72E98894E5B696A6A19CCA2C1C79FA976384C3670B8465C2ACBDB46C1B5881F5BFA3480675F83F5535606CB393934A6DBA30469282A3127B5B77BD7709350091FFF453CF8F76370F847ABA2B78131093FE4B699D342BE220E085458E150B081B81FCDD35CDFAC605FBC4F9757C24D7AC542E90EA4E9255B5B29F78479AB36AF1BD691D005AB401269BC9395C9986ED9A2017E3818B1F87DF9846F1EA1A18EB681A0E561F9F379214A6CE44334CDC5C4D489436A5850B272499D32165D10CC8B6577965C56BB55034181BCD7C81488BB9FB1DC8D13C1536870BFD4868F136CBC490B62C79741B273ABA970742D7012AA2165C297A844AB4C27414360B5CDF0E29D31B5CD802CA3B8149B00DA23F7E213A736017C2B9D8846C12E4A5DB12B57E83168D8D6BEF82CBA020116758CB06BF694FA186000458BDA4358C537ADBFAC277FEA5C02E1C8B79998AF6C4F1A6CA47085022CA7A6CAAB0780254FD8C0CB78223D86B8656E56231E9A28F0612747E2075E25B602895FE7C498197A62541420CF839C31B347F478BCED325336BCAEBEE986D52C7AA5B5CA56723F1641CA3480A040D1267A407FE3E92D2C8023DE00846AD79B0AA89CC1F99965E88DBDE06B66415E36D2120E2400F435C3145B57EB28C62C248FEE401525FB3BE6491E02D4156EA38C819AC59D55CAAE14A7D1D3B9E2CB57B7BB58704A8C2D239E622A0D15A589916517768387E21408E0E81A01216A5D4A636686321B919E7019A1697B886E81A376391EDAD6C5E8044F6BAC63A84B02104221F19AA8E3468579A0A156897FEFA66A11A37FDF46088EA586AF8804A9854884597A7327A733315E020693DBC62BF4A33E1DEB74E91437EA30670E7430B7E1596611A10C3595C3AA14F6337B55CA5D0B953068D52EAEB7415BF18E88D7905945CCF3140A75263A25D931F802702EEB40A944BA6AF03FA2509ACFE753EC60CDB0A845CA5131EAD4401E2C8D0A32306FFCC7DD189FA367AB6A4A9A6DD1806BA70AA4440BC80B99D0C91C7931871B436F94B39B1D04448D18085C5AA946E642CE04143670BF7F8514A0797C6FA71BE4BB1416A0B3248075251CB1A5B79E36862AF4947C76D82CE8363B89139808C4CBD14B3C4F1700160B61E5648FB954C25009BACFA265F1A0564673AF000120D93A2D4E211CAACB1D69713D4E183268803163C859A71B92D9117FF807089F47940C57955E3343506762B56272AEE182C719A5478444F29513C75AB44E3C4F0658111473CAB33933C5E2999BD01C5DCB71F1F82F6CA57B9EB6376C388C20064944E3AA803574FD816C119A312140117BA492A3D3204541335C082CAD440F0B22BC55A65E2B852E3EB1042ED046E8A9B755773425C920D2F4260CB29D1DB76A879B36283BCFA8598F799302DA5C14C5AC03230A338EE3C8497899A92CBE645B3D9E91499F441FAA5211D87162E76262BD795409559B294A0453F028A3C737962AA6B9969D8B45A934C7CF22FC2EF51C661541202085554BBB5AA2B207D161A75B7831000B8A532C76C1D01FF3A359212BBF79137B42839A217B009702192A92470F4B3DA4A9397E51CEB6F883C5168DF122076CA783C0C2BBBF572680E5268CFBA41E8024EC58A133A8934C46553302753F39AE7E4B9311A984CDC68360293D94D8A6D857318D5B257291685C5068311839A489485C64AFB7430376108502279FD47042B7216E8F7616EBB408CE4C3876548AFDA6B31596C21EF096B92782D5D745885106C3770E1E04C1DA2A027912137CBA3FFD2632B808AD0BFA8F7D2C9AB4E24245C8131AC5167684190583AD527A25BD8369FD747F29528C61287EEED97D6CCBBF791AC8A90AB9A5E19F57482A89E3A9DE9C00CCBB9884C80F7CC7680B654FE1A3CA5A160431A1C42CA445AF7A4D60CCA7F6A27A7B919C6D70B030025D86341943897296D61B0C8414A696C95F70337C54BDD04C08A3FBC5F9835B9A080496A1308E03A88E76973C85A9DA0C0340B95C29228DE1FB3D4CD96A9204848A92A3C912B0BD8817F6212242D06CFE803E91B656661A51A6A1841DB00F1F8A02518A971F61199F93712C306CB944C52B469B04E76389E71543896A410979009A2A9F09CC58C0868294A72EA8A0FF96A43F543E5E96AA1934545F755382859F962615B8B546D5609B59157DB4CA4D5FB87366E03ED93A2D3E223590A3B9F03C95327A35D65C5B957546D3F0745A31A3AA012719340F13992D8BACB18153A83D59295BC497EFC5C875C4542B430285F54FE245558538B15CFC03B69889E2EA1B1E11CA89D165C9ECC1278C2D5E878C82C13D5AFB7C2DEA9D71437924D97D698180C0B626E683A2AD3BCA5652C9E3021ADF322BBDCB6D9C762D0A12270505A811E3C27FB16A9493047755163DF828E0C8B0CAB4B32EF17C784B0BC1A1922967112122792B20A2917A7FC46016556611416675AB9C5AFF696F80D48B659ACCE23FF815C80547A9762CC0D85C3F2501DD291BA56692474A2E13B3EEB40A73D7BA5E222A09506341F00EAA4E856209AC3525E24569BEA949B03B755A36972911F56028D4EA3F21B1537B186287A191FA400689D618C2C2C20CC091EF65" + }, + { + "tcId": 148, + "testPassed": true, + "reason": "valid decapsulation key", + "dk": "31C48008D52C5365473A1512E7813452784A12410BAA3313EC0C0B4769C6E52CC31776074B2282E035B611AA1AC8DA6DFE63C0A1941B473644162288E53A5EC9C59037999DB3E8C6C7857CA57C42C7501182740E7C041A7404620A9C76F0F611D010703E720CB1004C20E27274B44C0CC3967C8566BB07442A40CD670C0C6869AECD22A57DA32059C19256CABF02F16AF25A3E2C876CB575C12B13871A37B61A00176885B08B158C6B86C02A15539DC264A951C9FAFB2F26DC1C65F3040EAABBA5186A2315944E4B7AE93489E896BCF8F677FB7AA24A180BFACCC19F80678EFC64F8638280B64E1FCBBC4E2672475C5B4F4689E2E6A0B16B262214BE03138364341E774466C5F52C14A5645CDBC5691C9C4D7329C596089EA8B10128BC01E8AB31A56708CA7185FAC4E8E68EF8B8096E6351E19AB2FB2A7E0EB35282198B8BCB3783C05F71B65FE7F389AC166D3E09AB767A14B3575153801A7B38C3F1B085FB28777D326954A66FDE72CCAD43C92E971E2365563F987669E84CDFD43EF9312C5458BEDF9C5EDC757F01365144BA2F4DB954FB2A6168B46F534A0277CC00390B81AB07164A14A830B69ADDB5011D37BE8F7332CBB5AC3E1C042E4373E56B6180459785F08F6311405348990FE907C6F9213321CE23591ECA58180384CEB5B876FC46BE59C559F77AAC21D22BF384901430CB5FF24D4CC2B4AAA0B9A95A4F2EFA60D396040D4315C83911220947A9E0CA7D2C081CC6810370602FE38B1AA9711A87B4331B7F0D4576D1F20A16F0B6EDE5767356419739777C5380FA0411C1052581F412E0C050AB9393D997BEA1E51AE6E32539795E66C46D9C5A04A5B56A8EF808D8AB8CA7D82609E727CC1029FD8002E4E551A7CB05DB27841BDC3B634922B1000D4395AD76039A80745EE72A4A6FA12BEDE61A35D58AE7752B3924CAD762351E598C10354574532583C07B82B4BFACF88C18D12674EB1F3F28CF25DA43BB6541CA6691579651828372C90067E379A982E623BFD062087138D31957F05136ADE1173E0390A62B07EC061918E34089543B9D1C4170F40B1A562C6A11402CA00A3715A41697BF5D8C2E570C5F6AC89D04698F9241CD135C2FA69771A3B9CF1BB793D1E6C945C97C7665658753C31A00778F72C89E30BC2AF746AD23AC21B40E299B57B5D44E55335EDC0A3D1EA42D8C7855166B4CD0229F6CB2A4F9435F412137AA904BCD0ACCEA09B8ED10C9922594BD756E70873B1AC52A575C1210D0A4CD80AB11B7200BC373F6C208D725605207C04D8492F8C9C5E6C1CFF023AE8BA547BE1743582044E5D0C4F1F20534A0A62717AD60D91A6C7A03BCE2721B7503E75581A23BBD8F1848C00AB5509466F144B8FFD5C5EC1370EC8B7C960755A645C4CF23BF4CE9A19E3134D2E5830706910DA7B0CD9AB92D145F2591CA2DC982E6E66716B7754A82BB335709A5249F88BAA71C895BFF403340CBCF70768CAA563F5E9348F4B3C6F29B4C77141FC5D52AA532287876590B15AAFDA8289B9C8FD4EBB0EF8944BB99A0CF83661D78727854B0950A29715A4AA8732B65A08E6BC81F123358CBC3C2E4E93FD2307895597D6D66B16ED139E8B2B260B63E14720BDEA963F8B537A85A1A455BB176D310F9855F6AB019A4D919EB1453C8CA1028260C64513584108DF2D56B6451172B674270744328CACE2E167359FB594106630C887973B35776EC50FD030641312A623908DEC6C81FA573D13C475981ABC5D30F77D4B74999209F4A565C1233E3CB702897379D39B8B68A43C4199D7F64889281BCDFA47ECDB1312D981CB9AB8AAD764AB6EB88216CAE97D93D75E3CF1E654A21209246023B42376344D909C651BA92A4A9CF1CB03A522632B41EC27C8F180CD05A230A2F60A12110C8D832372EF1A1A3BA1769FB49888728BD080DC2E476971756A4E8AAD5C18AA807D0D708714CAB2BF4272C458C20DAE8B089A60945B4B5C5697F31BC4C4DFB1494830323D962C8520A6F7C6C11F8613BE83C22B362A352B47B2B0838392A2DC4745068211E238DC385211BF048ECB2478946C9C2B837CBD0604FB241FDC0445008013DA6480B826EF484AF939C4A399101D380CB55C17E64B081C201D016261407A6C95C570E7BB08C79BA4BBBB92260388342789E7D7784A0E80E602A7EFD97CF79202F38E8A1C0595DD59C97B7B559D344517FD472C8B673FE5012CA935E6DD59E70614ED7C86244132F8771AD2D1A9A884570F40AB82317BBFCC5879844A6C0C757236C94E8D6923906C065B675B54053176B4C6AD2AAB8A5AEC323CCD68A5AF8DB0AE94BB4D431CA1EF198851BBFF64B4EF96C6D0EA045F82C9B7D48337D2A3F1EA7185C2881A4E267DFF8407748C03FCAC286F302A16173128396D76A9ACB217F29353AA87001EA9663D8F48B1E4B0D9AD6CBF81A6B600B763B75493577B85694AF8FA2729323CF5341961F423770217F34466A9115929447AECFD741E39C6796C5A132E831AF3CC2B70A434524C764C9C783EA1D4F269BA091575139C092D91BC46C4402CA0C65B29588B8A34E7C5744FB38189279A507CC9A810588F8166F95833183618AE2470BB25AE12B330AECB4F296B11E8C0EE2E34F12E2ACD0C891CC136D033A17E218A7FFAA7D96807267B083C1CAA8E1D162283AAFD98C5AC1657883CCCDA04A9AC8428B5EA6BFA4B04336D6A6D222B79714CC9696C8CD62B286C04F7AF69F0FDB7ADFC999DA613BC8F6A156161C2E135E7656938EF7C80489B0DFB7AEE086C9E66446EF1173569A8A7E23A0B82B92BC50250CF577236199322B783D6B8C5AB82C9C378716B231C4850ACFCC131F03CA51055A0030CFDCA874CA08BB6E3462FE517DD5F282EC0C5DAEC64414C4534C264D68DB5C2085CA25644059B2AFFCA64BB1D8762A293D42B79E4A9BAAE8D73E1012C114702D2BD19629D646B5300B0500167F3C9B1A7145FCD95BC712BBEC42731D321D7DC8C4C361C271DB1863759B147BA14A524690625823DB8772CA0E2A45AB7C734A1FF0951156A305D70B458AB422A6557AB423641266464A88799987B72572E60A79D0732C84049683310EE475B695556C4A65615651734F56A90D25178D604CD058230E3338FA94003FFC48AE600D2660212D648FA8062B38182006030857882B26E62BF8A23CD5D275B5517F4EA736A298402F155B2141A3AA8A74E5F69678185EB4334FBC380980E850A1399AFA0C4EDB172EFE326BCAFAA3A0DCC07575455C084442E2B5D0105C39B234B409A5BDA7C33C2B5D2994B7CFB90CFA23BA2A65C28E255F0BA3728671AB9E410642D1812D0969C88C721D867E36B8B4808CBD1898B05B563FFC23A306136100975AED0A0DBC1AA8E823C7FC402821F65E31972C3FDCCC96F55B16C3AD48BC4A9CAA05B4C0CB7E863A47296D3C727D7B3B90491364430CCC4756A2203150958770FE354E0F8803D1A64E75A309F7EA01260345AA1C3E5FFB863BAA4B0E5768F20B7D906766F8B86ADD6066CC485FA5C9141D16039E5562E433B511092C1D18C50F7186642152C6986100BA728C136DC28B237E2C8347B28E3FB07C0AE48A42F955476340BD5CBC145860CA6449C597920575AC49C0317F2926BEB2BA7396BD73864B86490E1EBC8A03475A9FF082A7F967F509C8F8BB0BFD4C98174B0B36D06CAC5789919C02B3C0162AD4C4DF639207416BBB5071E5F692668B5D40AC2131988535C68E68C227810C0B68E1B72103B61D98A32A7A05AFE1BB737627955A2276921703A5B0FDE38E415C4E752784FCBBC432A53BC4FC0DB52A46C3D0C3014041C870C00041C183D21D5BF277ADBCAA73D6241CEB018E5550A8E24FA5CAC9310B6735EB5D245239F35B06E803697186680B6AA8BA1730B4CBA23E105565F37B2B16070C3B0E77A965CF577F3C3A82A5192C4C3BB52373C54034B872604ED4A833F9E18E389A4BCCCC6C5991CB49F097771CABD8AA6D53020875080E91D198F93C2EFD276C86586A9E53AC1AA119851C06BE12127AAC3D662AB5325B729F38CAC505A3D0E135CD9A38A8069A7A2A76C5B39B30C7AAACD666EB061093B5C41E720B580A2914552AF509AE2035329D28050C5016AA06AE5EC62E17D84C9024494049816AB32DCF204015A092AEF05857794861175B19A72C9687B6E9EA385963BE3E6A197627359728BF47D4CEDA32779EC51B2F6BC8287241C7B39474805B5AD13DB6C47627AA40F91CCB6CD062DDC58FEA6A22EC98A65F73A0054B3E211ACA6904BC01A9A04B042AC128910B793448A98D19EC1A48DCC451B00F66629AF4DBC2C24101F9480C390226F7B184FF307BBA63B6A1D5B1FA4281AF4C358052CDFDE37B07B2C7627429859EFC72C31BFCA310E4A63E25B10BD4A57CA6A2611E141A8C0CFDE7A700EA3B849017A99F79273DC7FC7702D269197D22112996A9AAAD6803C7CF2AC8132D84A9B5016EFD34D64543E136993D98D12FDA" + } + ] + }, + "cctv_modulus": { + "base_ek": "36c99c6874b42683550244a579ca3bdb4bac1658603bc0c8afd035d0a5415e2c4310d647b6a3984244c3e7456af0e180d453240477461f49cdc6111e47cabf82fbaa36880c869262103a4a20cac30c304ce6990999039e7ef648d08253bc195c980071e61c959f02b409bcb737098af3c6acc98b579c99521d791295153b672a2d17d112d5e87d4878274e01568f9b81ccd50fa8ea34bbf15fc892c82ad46a21a60eef635aa89ca8c9b8cf944a5773ca01488b0ad35aa9ab5babe4c0cd63a90bc929bc3f9a0290851b79626229c54603b883120c0318c7113077aa4d507a366506c6936b56d1883f019918132f87592b5b6343e842323ab3713681b0de78ae607365de49267fa61d5b90ae69d378ab25626791a71dfb93ea06002cd8c32ef28d7e38587924363c1055554b4d7c632b144a958a9139a9ca61523a7e4cb38385f2547aa812c3dbb97ea49e7cd83815a486a6373e179b29a65a005df32b4f72bcb2a5c5371c2bcf8441bea92fe8c2183c6ca97b643480d8cd62d84fe17599f568af4204bd9a1bba594cb5668b3fc44a79397a052faa0569436c6515c5098a9bc25593cea3b694d82ac65516268b58a0f0a5cafb073cf943099b558f6c519a0b96fc4093dbf2362c237872160d7ca7b627606ad1c8169320285b1a3ddeb90d7806183b2bcce6228cb5a166904a6181c789041612af83b2c03326aaf8c468c6a2fa2ac2d8182da2941d7f34265ed54a0ac6b8f819c997a66c26d48c16086a9d77b25d80705da2bf198305ba3115a50ac655253044c4261e1b1d72876aae59b1e63787bf2ab0773c95d0707321ab91b5216d937506b4e3b3e49c18c1f1bbd0e4c1b10a5c2f13cf496769cc463f65e6cd26953692722218005bdcb25f165c9abbcc18b06b8050cb58550c319cf843c560583d0b499750231234c65b243055c68d16834d2a6baf14f96186a65d5de05eaa0b5a5af076d574a1da7334ee45489e337e147b297e963936fc084f3b7a8a324406422d3f677a4a5437965b5115730e949353ac04858d0a56af7ccf340093ee05309736799e8792018433bf55901574c341b2a9df1ccab2955624e58f0413074b1810f957bd202a728855ba1c123a0d3bc88855c59c8a8bafc6329db779b815c5d08297c8e0b328c08fa526aba4aab6c21b8c653c5b0b135b99ec24c11b9edccc4cc3d231efb7a045b61af546c24458ac876c151626cf3a0502f2463d850c9a038a200211abc7c085d29812e0395ebff3c236a34f44c64153e49a2ad002f1e16976b1c18766706fd3b7dde69d29834bee186f4bf4b7abe207175389d1b1b5c9c674e6facdd93158f17628e1ac1197b30902b3277366af0702a0576212cfd9707b47424971bfbb75011cba31b2a58ca062a479586aad4ca5890a8fd805151d5294df479803f4b80d055b19a1b7c483b280c12064213b2ab278eecca661ab635fbcc436fa8412861171f8cc49bc56cf1ba8050c07fc8c670886ae29d05762c26540d1c63888b7d6c9047cd2cbe8f9aa378114722b1d2ca87ce2599dce524ea6c4b83cd32133c36db8162e3c34681fda8ea2c7c9e80c87d34a73e4812aad40af218c75c2551cf86607d43c9f3706a9c30c737a89418ca703d2b1a015728ccb313e9d04117aeb8284f8012fab777640be1ef2592f533468851f8ca70087a83b68bb67cd9947334975e6355a9e50bc32418801458b3aab0d7b68542b581ee5e65990897830d37a72a6b849bb61e8c51cee56832793642f053a29f06bc1a616b912943deb1090a9cb671a904b503f45fa052f140cf0c557ee843c2d8818472c102a91cd7afc5afd9687a61765676508de1050a0fcb7bba82d0740063ac001eb336e81fb5a60c4a7143aaafaf93a62218e760b9dded8c44a6b4de9397d9c677badd9942f346f5b8022ac8c2a54c53856224bdae0146136636acc0b4815bd908b6bc525c0f06732034b8142c16490d419fc85873eeb56f2c002f1126bb3075f5bb16c99642e80f57b4df0479a564e50c78833485936a63c38d1cb8932c9050a0a7ad74a54f2a28788bae082c946da9e8d757afe1880b5d5747717698103538098b177386d36d7bfb96ca1a6b68d6ababc5a4531ea33cb053279b81c84c6d45bb1d14aa12258fc407de6a9a5dd66a9e16b6bd6b9b3bdb45a4a06d763455b4da200013e681c43ea7c943eb186edf29664c5dd42fb0f240f61", + "count": 1040, + "sha256_of_txt": "9c27ef8951a6b6b59b57dcc4e213ae73472c3db956f4298d6c9983c54bb2c263" + } +} diff --git a/src/test/fuzz/CMakeLists.txt b/src/test/fuzz/CMakeLists.txt index eb1c06226e..bbaa22cdfe 100644 --- a/src/test/fuzz/CMakeLists.txt +++ b/src/test/fuzz/CMakeLists.txt @@ -66,6 +66,7 @@ add_executable(fuzz message.cpp miniscript.cpp minisketch.cpp + mlkem.cpp mini_miner.cpp muhash.cpp multiplication_overflow.cpp diff --git a/src/test/fuzz/mlkem.cpp b/src/test/fuzz/mlkem.cpp new file mode 100644 index 0000000000..b622ed25c2 --- /dev/null +++ b/src/test/fuzz/mlkem.cpp @@ -0,0 +1,244 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace mlkem; + +namespace { + +void Initialize() +{ + // MLKEM_FORCE_PORTABLE=1 runs every operation on portable C; unset, empty + // or 0 runs the detected backend. Any other value is refused, so that a + // misspelled setting cannot turn CI's portable replay into a second + // native one, and the portable replay checks that portable C is what runs. + const char* env{std::getenv("MLKEM_FORCE_PORTABLE")}; + const std::string_view setting{env != nullptr ? env : ""}; + if (setting != "" && setting != "0" && setting != "1") { + std::cerr << "MLKEM_FORCE_PORTABLE must be unset, 0 or 1, not '" << setting << "'" << std::endl; + std::abort(); + } + InitializeRuntime(/*force_portable=*/setting == "1"); + const BackendNames active{GetBackendNames()}; + std::cerr << "ML-KEM backend: " << active.arith << "/" << active.keccak + << (setting == "1" ? " (MLKEM_FORCE_PORTABLE=1)" : "") << std::endl; + if (setting == "1" && (active.arith != "portable" || active.keccak != "portable")) { + std::cerr << "MLKEM_FORCE_PORTABLE=1, but the active backend is not portable" << std::endl; + std::abort(); + } +} + +template +std::array ConsumeArray(FuzzedDataProvider& provider) +{ + std::array out{}; + const std::vector bytes{provider.ConsumeBytes(N)}; + std::ranges::copy(bytes, out.begin()); + return out; +} + +/** + * Inputs of both targets, in this order (bytes missing at the end of the input + * are zero): the key generation seed d || z, the encapsulation coins, an + * arbitrary encapsulation key, an arbitrary ciphertext and an arbitrary + * decapsulation key; then, from the back, whether to flip one bit of the + * honest ciphertext, and which. + */ +struct Inputs { + std::array seed; + std::array coins; + PublicKey ek; + Ciphertext ct; + DecapsulationKey dk; + bool flip; + size_t flip_bit; +}; + +Inputs ConsumeInputs(FuzzedDataProvider& provider) +{ + Inputs in{ + .seed = ConsumeArray(provider), + .coins = ConsumeArray(provider), + .ek = ConsumeArray(provider), + .ct = ConsumeArray(provider), + .dk = {}, + .flip = false, + .flip_bit = 0, + }; + std::ranges::copy(ConsumeArray(provider), in.dk.Bytes().begin()); + in.flip = provider.ConsumeBool(); + in.flip_bit = provider.ConsumeIntegralInRange(0, CIPHERTEXT_BYTES * 8 - 1); + return in; +} + +/** FIPS 203 section 7.2 modulus check, written independently of the library. */ +bool PublicKeyInRange(const PublicKey& ek) +{ + for (size_t i{0}; i < 384 * 4; i += 3) { + const unsigned low{ek[i] | ((ek[i + 1] & 0x0fU) << 8)}; + const unsigned high{(ek[i + 1] >> 4) | (unsigned{ek[i + 2]} << 4)}; + if (low >= 3329 || high >= 3329) return false; + } + return true; +} + +/** FIPS 203 section 7.3 hash check, H(ek) stored in dk, using qbit's own SHA3-256. */ +bool SecretKeyHashMatches(const DecapsulationKey& dk) +{ + const std::span bytes{dk.Bytes()}; + std::array hash; + SHA3_256().Write(bytes.subspan(384 * 4, PUBLIC_KEY_BYTES)).Finalize(hash); + return std::ranges::equal(hash, bytes.subspan(384 * 4 + PUBLIC_KEY_BYTES, hash.size())); +} + +bool AllZero(std::span bytes) +{ + return std::ranges::all_of(bytes, [](uint8_t b) { return b == 0; }); +} + +void FlipBit(Ciphertext& ct, size_t bit) +{ + ct[bit / 8] ^= uint8_t(1U << (bit % 8)); +} + +/** Every observable result of one pass over the inputs. */ +std::vector RunAll(const Inputs& in) +{ + std::vector out; + const auto add_error{[&](Error error) { out.push_back(uint8_t(error)); }}; + const auto add_bytes{[&](std::span bytes) { out.insert(out.end(), bytes.begin(), bytes.end()); }}; + + PublicKey ek; + DecapsulationKey dk; + add_error(KeyGen(in.seed, ek, dk)); + add_bytes(ek); + add_bytes(dk.Bytes()); + Ciphertext ct; + SharedSecret ss; + add_error(Encaps(ek, in.coins, ct, ss)); + add_bytes(ct); + add_bytes(ss.Bytes()); + { + SharedSecret received; + add_error(Decaps(dk, ct, received)); + add_bytes(received.Bytes()); + } + if (in.flip) { + Ciphertext corrupted{ct}; + FlipBit(corrupted, in.flip_bit); + SharedSecret received; + add_error(Decaps(dk, corrupted, received)); + add_bytes(received.Bytes()); + } + + add_error(CheckPublicKey(in.ek)); + { + Ciphertext ct2; + SharedSecret ss2; + add_error(Encaps(in.ek, in.coins, ct2, ss2)); + add_bytes(ct2); + add_bytes(ss2.Bytes()); + } + { + SharedSecret received; + add_error(Decaps(dk, in.ct, received)); + add_bytes(received.Bytes()); + } + { + SharedSecret received; + add_error(Decaps(in.dk, in.ct, received)); + add_bytes(received.Bytes()); + } + return out; +} + +} // namespace + +FUZZ_TARGET(mlkem, .init = Initialize) +{ + FuzzedDataProvider provider{buffer.data(), buffer.size()}; + const Inputs in{ConsumeInputs(provider)}; + + // Honest round trips agree. + PublicKey ek; + DecapsulationKey dk; + assert(KeyGen(in.seed, ek, dk) == Error::NONE); + assert(CheckPublicKey(ek) == Error::NONE); + Ciphertext ct; + SharedSecret sent; + assert(Encaps(ek, in.coins, ct, sent) == Error::NONE); + SharedSecret received; + assert(Decaps(dk, ct, received) == Error::NONE); + assert(std::ranges::equal(received.Bytes(), sent.Bytes())); + + // An arbitrary encapsulation key is accepted exactly when it passes the + // modulus check, and a rejected one leaves no output behind. + const Error check{CheckPublicKey(in.ek)}; + assert(check == (PublicKeyInRange(in.ek) ? Error::NONE : Error::INVALID_PUBLIC_KEY)); + { + Ciphertext ct2; + ct2.fill(0xa5); + SharedSecret ss2; + std::ranges::fill(ss2.Bytes(), 0xa5); + assert(Encaps(in.ek, in.coins, ct2, ss2) == check); + if (check != Error::NONE) assert(AllZero(ct2) && AllZero(ss2.Bytes())); + } + + // Decapsulating a ciphertext of the right length always yields a secret + // (implicit rejection). That is never proof the ciphertext was valid: any + // other ciphertext yields a different secret. + { + SharedSecret other; + assert(Decaps(dk, in.ct, other) == Error::NONE); + if (in.ct != ct) assert(!std::ranges::equal(other.Bytes(), sent.Bytes())); + } + if (in.flip) { + Ciphertext corrupted{ct}; + FlipBit(corrupted, in.flip_bit); + SharedSecret rejected; + assert(Decaps(dk, corrupted, rejected) == Error::NONE); + assert(!std::ranges::equal(rejected.Bytes(), sent.Bytes())); + } + + // An arbitrary decapsulation key is rejected exactly when its hash check + // fails, and a rejected one leaves no secret behind. + { + SharedSecret ss; + std::ranges::fill(ss.Bytes(), 0xa5); + const Error error{Decaps(in.dk, in.ct, ss)}; + assert(error == (SecretKeyHashMatches(in.dk) ? Error::NONE : Error::INVALID_SECRET_KEY)); + if (error != Error::NONE) assert(AllZero(ss.Bytes())); + } +} + +FUZZ_TARGET(mlkem_backend_diff, .init = Initialize) +{ + FuzzedDataProvider provider{buffer.data(), buffer.size()}; + const Inputs in{ConsumeInputs(provider)}; + + // The detected backend and portable C agree on every status and output. + const BackendNames detected{GetBackendNames()}; + const std::vector native_results{RunAll(in)}; + { + ForcePortableForTesting portable; + assert(GetBackendNames().arith == "portable" && GetBackendNames().keccak == "portable"); + assert(RunAll(in) == native_results); + } + // The override is restored. + assert(GetBackendNames().arith == detected.arith && GetBackendNames().keccak == detected.keccak); +} diff --git a/src/test/mlkem_tests.cpp b/src/test/mlkem_tests.cpp new file mode 100644 index 0000000000..d04e2ce8dd --- /dev/null +++ b/src/test/mlkem_tests.cpp @@ -0,0 +1,716 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#include // IWYU pragma: keep + +#include + +#include +#include +#include +#include +#include +#include + +#include + +#if defined(__has_feature) +#if __has_feature(memory_sanitizer) +#include +#define MLKEM_TESTS_MSAN 1 +#endif +#endif + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mlkem { +static std::ostream& operator<<(std::ostream& os, Error error) +{ + switch (error) { + case Error::NONE: return os << "NONE"; + case Error::INVALID_PUBLIC_KEY: return os << "INVALID_PUBLIC_KEY"; + case Error::INVALID_SECRET_KEY: return os << "INVALID_SECRET_KEY"; + case Error::INTERNAL: return os << "INTERNAL"; + } + return os << "Error(" << int(error) << ")"; +} +} // namespace mlkem + +using namespace mlkem; + +namespace { + +constexpr int MLKEM_K{4}; +constexpr int MLKEM_N{256}; +constexpr uint16_t MLKEM_Q{3329}; + +UniValue ReadVectors() +{ + UniValue vectors; + BOOST_REQUIRE(vectors.read(json_tests::mlkem1024_vectors)); + return vectors; +} + +template +std::array FromHex(const UniValue& value) +{ + const std::vector bytes{ParseHex(value.get_str())}; + BOOST_REQUIRE_EQUAL(bytes.size(), N); + std::array out; + std::ranges::copy(bytes, out.begin()); + return out; +} + +DecapsulationKey DecapsulationKeyFromHex(const UniValue& value) +{ + DecapsulationKey dk; + std::ranges::copy(FromHex(value), dk.Bytes().begin()); + return dk; +} + +bool AllBytesAre(std::span bytes, uint8_t value) +{ + return std::ranges::all_of(bytes, [&](uint8_t b) { return b == value; }); +} + +/** Whether the capability hooks would let native code run now, on this thread, per its copy of the override. */ +bool HooksAllowNative() +{ + return qbit_mlkem_has_avx2() != 0 || qbit_mlkem_has_neon() != 0; +} + +/** Run `check` with the active backend, then again with portable C forced. */ +void ForEachBackend(const std::function& check) +{ + { + BOOST_TEST_CONTEXT("backend " << GetBackendNames().arith << "/" << GetBackendNames().keccak) { check(); } + } + ForcePortableForTesting portable; + BOOST_REQUIRE_EQUAL(GetBackendNames().arith, "portable"); + BOOST_TEST_CONTEXT("backend portable (forced)") { check(); } +} + +/** + * SHAKE256 truncated to 32 bytes, built on qbit's own Keccak-f[1600]: an + * independent reference for ML-KEM's J, which derives the implicit-rejection + * secret J(z || c). + */ +std::array Shake256To32(std::span input) +{ + constexpr size_t RATE{136}; + uint64_t state[25]{}; + const auto xor_byte{[&](size_t pos, uint8_t byte) { state[pos / 8] ^= uint64_t{byte} << (8 * (pos % 8)); }}; + while (input.size() >= RATE) { + for (size_t i{0}; i < RATE; ++i) xor_byte(i, input[i]); + KeccakF(state); + input = input.subspan(RATE); + } + for (size_t i{0}; i < input.size(); ++i) xor_byte(i, input[i]); + xor_byte(input.size(), 0x1f); + xor_byte(RATE - 1, 0x80); + KeccakF(state); + std::array out; + for (size_t i{0}; i < out.size(); ++i) out[i] = uint8_t(state[i / 8] >> (8 * (i % 8))); + return out; +} + +/** The implicit-rejection secret for ciphertext `ct` under `dk`: J(z || ct), z being dk's last 32 bytes. */ +std::array RejectionSecret(const DecapsulationKey& dk, std::span ct) +{ + std::vector input(dk.Bytes().end() - 32, dk.Bytes().end()); + input.insert(input.end(), ct.begin(), ct.end()); + return Shake256To32(input); +} + +/** Overwrite coefficient `index` of polynomial `poly` in an encapsulation key (FIPS 203 ByteEncode_12). */ +void SetCoefficient(PublicKey& ek, int poly, int index, uint16_t value) +{ + uint8_t* group{ek.data() + 384 * poly + 3 * (index / 2)}; + if (index % 2 == 0) { + group[0] = uint8_t(value); + group[1] = uint8_t((group[1] & 0xf0) | (value >> 8)); + } else { + group[1] = uint8_t((group[1] & 0x0f) | ((value & 0x0f) << 4)); + group[2] = uint8_t(value >> 4); + } +} + +/** + * The 1040 invalid ML-KEM-1024 encapsulation keys of C2SP CCTV + * ML-KEM/modulus/ML-KEM-1024.txt, in file order, rebuilt from the key they + * share, as CCTV's modulus.go generates them. + */ +std::vector CctvModulusKeys(const PublicKey& base) +{ + std::vector keys; + const auto add{[&](int poly, int index, uint16_t value) { + keys.push_back(base); + SetCoefficient(keys.back(), poly, index, value); + }}; + for (int i{0}; i < MLKEM_K; ++i) { + add(i, 0, MLKEM_Q); + add(i, 255, MLKEM_Q); + add(i, 0, (1 << 12) - 1); + add(i, 255, (1 << 12) - 1); + } + int i{0}, j{0}; + uint16_t x{MLKEM_Q}; + bool done_values{false}, done_positions{false}; + while (true) { + add(i, j, x); + if (++x == (1 << 12)) { + x = MLKEM_Q; + done_values = true; + } + if (++j == MLKEM_N) { + j = 0; + ++i; + } + if (i == MLKEM_K) { + i = 0; + done_positions = true; + } + if (done_values && done_positions) break; + } + return keys; +} + +} // namespace + +BOOST_FIXTURE_TEST_SUITE(mlkem_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(acvp_keygen_1024) +{ + const UniValue vectors{ReadVectors()}; + const UniValue& cases{vectors["acvp"]["keyGen"]}; + BOOST_REQUIRE(!cases.empty()); + for (const UniValue& tc : cases.getValues()) { + BOOST_TEST_CONTEXT("ACVP keyGen tcId " << tc["tcId"].getInt()) + { + std::array seed; + const auto d{FromHex<32>(tc["d"])}; + const auto z{FromHex<32>(tc["z"])}; + std::ranges::copy(d, seed.begin()); + std::ranges::copy(z, seed.begin() + 32); + const auto expected_ek{FromHex(tc["ek"])}; + const auto expected_dk{FromHex(tc["dk"])}; + ForEachBackend([&] { + PublicKey ek; + DecapsulationKey dk; + BOOST_REQUIRE_EQUAL(KeyGen(seed, ek, dk), Error::NONE); + BOOST_CHECK(ek == expected_ek); + BOOST_CHECK(std::ranges::equal(dk.Bytes(), expected_dk)); + }); + } + } +} + +BOOST_AUTO_TEST_CASE(acvp_encap_decap_1024) +{ + const UniValue vectors{ReadVectors()}; + const UniValue& acvp{vectors["acvp"]}; + + BOOST_REQUIRE(!acvp["encapsulation"].empty()); + for (const UniValue& tc : acvp["encapsulation"].getValues()) { + BOOST_TEST_CONTEXT("ACVP encapsulation tcId " << tc["tcId"].getInt()) + { + const auto ek{FromHex(tc["ek"])}; + const auto coins{FromHex(tc["m"])}; + const auto expected_ct{FromHex(tc["c"])}; + const auto expected_ss{FromHex(tc["k"])}; + ForEachBackend([&] { + Ciphertext ct; + SharedSecret ss; + BOOST_REQUIRE_EQUAL(Encaps(ek, coins, ct, ss), Error::NONE); + BOOST_CHECK(ct == expected_ct); + BOOST_CHECK(std::ranges::equal(ss.Bytes(), expected_ss)); + }); + } + } + + // Both "valid decapsulation" and "modified ciphertext" cases decapsulate + // successfully; the latter yield the implicit-rejection secret. + BOOST_REQUIRE(!acvp["decapsulation"].empty()); + for (const UniValue& tc : acvp["decapsulation"].getValues()) { + BOOST_TEST_CONTEXT("ACVP decapsulation tcId " << tc["tcId"].getInt() << " (" << tc["reason"].get_str() << ")") + { + const DecapsulationKey dk{DecapsulationKeyFromHex(tc["dk"])}; + const auto ct{FromHex(tc["c"])}; + const auto expected_ss{FromHex(tc["k"])}; + ForEachBackend([&] { + SharedSecret ss; + BOOST_REQUIRE_EQUAL(Decaps(dk, ct, ss), Error::NONE); + BOOST_CHECK(std::ranges::equal(ss.Bytes(), expected_ss)); + }); + } + } + + BOOST_REQUIRE(!acvp["encapsulationKeyCheck"].empty()); + for (const UniValue& tc : acvp["encapsulationKeyCheck"].getValues()) { + BOOST_TEST_CONTEXT("ACVP encapsulationKeyCheck tcId " << tc["tcId"].getInt() << " (" << tc["reason"].get_str() << ")") + { + const auto ek{FromHex(tc["ek"])}; + const Error expected{tc["testPassed"].get_bool() ? Error::NONE : Error::INVALID_PUBLIC_KEY}; + ForEachBackend([&] { + BOOST_CHECK_EQUAL(CheckPublicKey(ek), expected); + const std::array coins{}; + Ciphertext ct; + SharedSecret ss; + BOOST_CHECK_EQUAL(Encaps(ek, coins, ct, ss), expected); + }); + } + } + + BOOST_REQUIRE(!acvp["decapsulationKeyCheck"].empty()); + for (const UniValue& tc : acvp["decapsulationKeyCheck"].getValues()) { + BOOST_TEST_CONTEXT("ACVP decapsulationKeyCheck tcId " << tc["tcId"].getInt() << " (" << tc["reason"].get_str() << ")") + { + const DecapsulationKey dk{DecapsulationKeyFromHex(tc["dk"])}; + const Error expected{tc["testPassed"].get_bool() ? Error::NONE : Error::INVALID_SECRET_KEY}; + ForEachBackend([&] { + const Ciphertext ct{}; + SharedSecret ss; + BOOST_CHECK_EQUAL(Decaps(dk, ct, ss), expected); + }); + } + } +} + +BOOST_AUTO_TEST_CASE(public_key_modulus) +{ + const UniValue vectors{ReadVectors()}; + const UniValue& cctv{vectors["cctv_modulus"]}; + const PublicKey base{FromHex(cctv["base_ek"])}; + const std::array coins{}; + + // The rebuilt keys are exactly CCTV's file. + const std::vector invalid_keys{CctvModulusKeys(base)}; + BOOST_REQUIRE_EQUAL(invalid_keys.size(), size_t(cctv["count"].getInt())); + CSHA256 file_hash; + for (const PublicKey& ek : invalid_keys) { + const std::string line{HexStr(ek) + "\n"}; + file_hash.Write(reinterpret_cast(line.data()), line.size()); + } + std::array digest; + file_hash.Finalize(digest.data()); + BOOST_REQUIRE_EQUAL(HexStr(digest), cctv["sha256_of_txt"].get_str()); + + ForEachBackend([&] { + Ciphertext ct; + SharedSecret ss; + BOOST_CHECK_EQUAL(CheckPublicKey(base), Error::NONE); + BOOST_CHECK_EQUAL(Encaps(base, coins, ct, ss), Error::NONE); + + for (size_t n{0}; n < invalid_keys.size(); ++n) { + BOOST_TEST_CONTEXT("CCTV modulus key " << n) + { + BOOST_CHECK_EQUAL(CheckPublicKey(invalid_keys[n]), Error::INVALID_PUBLIC_KEY); + ct.fill(0xa5); + std::ranges::fill(ss.Bytes(), 0xa5); + BOOST_CHECK_EQUAL(Encaps(invalid_keys[n], coins, ct, ss), Error::INVALID_PUBLIC_KEY); + BOOST_CHECK(AllBytesAre(ct, 0)); + BOOST_CHECK(AllBytesAre(ss.Bytes(), 0)); + } + } + + // The boundary: q - 1 = 3328 is the largest valid coefficient, q = 3329 the smallest invalid one. + for (int poly{0}; poly < MLKEM_K; ++poly) { + for (const int index : {0, 1, 128, 255}) { + BOOST_TEST_CONTEXT("coefficient " << index << " of polynomial " << poly) + { + PublicKey ek{base}; + SetCoefficient(ek, poly, index, MLKEM_Q - 1); + BOOST_CHECK_EQUAL(CheckPublicKey(ek), Error::NONE); + BOOST_CHECK_EQUAL(Encaps(ek, coins, ct, ss), Error::NONE); + SetCoefficient(ek, poly, index, MLKEM_Q); + BOOST_CHECK_EQUAL(CheckPublicKey(ek), Error::INVALID_PUBLIC_KEY); + BOOST_CHECK_EQUAL(Encaps(ek, coins, ct, ss), Error::INVALID_PUBLIC_KEY); + } + } + } + }); +} + +BOOST_AUTO_TEST_CASE(implicit_rejection) +{ + // The reference: ACVP's modified-ciphertext cases equal J(z || c). + const UniValue vectors{ReadVectors()}; + int modified_cases{0}; + for (const UniValue& tc : vectors["acvp"]["decapsulation"].getValues()) { + if (tc["reason"].get_str() != "modified ciphertext") continue; + const DecapsulationKey dk{DecapsulationKeyFromHex(tc["dk"])}; + const auto ct{FromHex(tc["c"])}; + BOOST_CHECK(RejectionSecret(dk, ct) == FromHex(tc["k"])); + ++modified_cases; + } + BOOST_REQUIRE_GT(modified_cases, 0); + + std::array seed; + for (size_t i{0}; i < seed.size(); ++i) seed[i] = uint8_t(i); + std::array coins; + coins.fill(0x42); + + ForEachBackend([&] { + PublicKey ek; + DecapsulationKey dk; + BOOST_REQUIRE_EQUAL(KeyGen(seed, ek, dk), Error::NONE); + Ciphertext ct; + SharedSecret sent; + BOOST_REQUIRE_EQUAL(Encaps(ek, coins, ct, sent), Error::NONE); + SharedSecret received; + BOOST_REQUIRE_EQUAL(Decaps(dk, ct, received), Error::NONE); + BOOST_CHECK(std::ranges::equal(sent.Bytes(), received.Bytes())); + + for (const auto& [byte, bit] : std::vector>{{0, 0}, {784, 3}, {1407, 7}, {CIPHERTEXT_BYTES - 1, 7}}) { + BOOST_TEST_CONTEXT("flipped bit " << bit << " of ciphertext byte " << byte) + { + Ciphertext corrupted{ct}; + corrupted[byte] ^= uint8_t(1 << bit); + SharedSecret rejected; + BOOST_REQUIRE_EQUAL(Decaps(dk, corrupted, rejected), Error::NONE); + BOOST_CHECK(!std::ranges::equal(rejected.Bytes(), sent.Bytes())); + BOOST_CHECK(std::ranges::equal(rejected.Bytes(), RejectionSecret(dk, corrupted))); + } + } + }); +} + +BOOST_AUTO_TEST_CASE(wrapper_lifetimes_and_errors) +{ + const UniValue vectors{ReadVectors()}; + std::array seed; + seed.fill(0x07); + const std::array coins{}; + PublicKey ek; + DecapsulationKey dk; + BOOST_REQUIRE_EQUAL(KeyGen(seed, ek, dk), Error::NONE); + + // An invalid decapsulation key (failed hash check) is its own error, and clears the secret. + for (const UniValue& tc : vectors["acvp"]["decapsulationKeyCheck"].getValues()) { + if (tc["testPassed"].get_bool()) continue; + const DecapsulationKey bad_dk{DecapsulationKeyFromHex(tc["dk"])}; + SharedSecret ss; + std::ranges::fill(ss.Bytes(), 0x5a); + BOOST_CHECK_EQUAL(Decaps(bad_dk, Ciphertext{}, ss), Error::INVALID_SECRET_KEY); + BOOST_CHECK(AllBytesAre(ss.Bytes(), 0)); + } + + // Injected library results: anything but the documented invalid-key codes + // is INTERNAL, and every failure clears the outputs the library wrote. + { + InjectResultForTesting inject{Operation::KEYGEN, upstream::ERR_FAIL}; + PublicKey ek2; + DecapsulationKey dk2; + BOOST_CHECK_EQUAL(KeyGen(seed, ek2, dk2), Error::INTERNAL); + BOOST_CHECK(AllBytesAre(ek2, 0)); + BOOST_CHECK(AllBytesAre(dk2.Bytes(), 0)); + } + { + InjectResultForTesting inject{Operation::KEYGEN, upstream::ERR_INVALID_PK}; + PublicKey ek2; + DecapsulationKey dk2; + BOOST_CHECK_EQUAL(KeyGen(seed, ek2, dk2), Error::INTERNAL); + BOOST_CHECK(AllBytesAre(dk2.Bytes(), 0)); + } + for (const auto& [result, expected] : std::vector>{ + {upstream::ERR_INVALID_PK, Error::INVALID_PUBLIC_KEY}, + {upstream::ERR_INVALID_SK, Error::INTERNAL}, + {upstream::ERR_OUT_OF_MEMORY, Error::INTERNAL}}) { + BOOST_TEST_CONTEXT("upstream result " << result) + { + { + InjectResultForTesting inject{Operation::CHECK_PUBLIC_KEY, result}; + BOOST_CHECK_EQUAL(CheckPublicKey(ek), expected); + } + InjectResultForTesting inject{Operation::ENCAPS, result}; + Ciphertext ct; + SharedSecret ss; + BOOST_CHECK_EQUAL(Encaps(ek, coins, ct, ss), expected); + BOOST_CHECK(AllBytesAre(ct, 0)); + BOOST_CHECK(AllBytesAre(ss.Bytes(), 0)); + } + } + for (const auto& [result, expected] : std::vector>{ + {upstream::ERR_INVALID_SK, Error::INVALID_SECRET_KEY}, + {upstream::ERR_INVALID_PK, Error::INTERNAL}, + {upstream::ERR_RNG_FAIL, Error::INTERNAL}, + {upstream::ERR_PCT_FAIL, Error::INTERNAL}}) { + BOOST_TEST_CONTEXT("upstream result " << result) + { + InjectResultForTesting inject{Operation::DECAPS, result}; + SharedSecret ss; + BOOST_CHECK_EQUAL(Decaps(dk, Ciphertext{}, ss), expected); + BOOST_CHECK(AllBytesAre(ss.Bytes(), 0)); + } + } + { + // Guards nest and restore what they replaced. + InjectResultForTesting outer{Operation::ENCAPS, upstream::ERR_FAIL}; + { + InjectResultForTesting inner{Operation::ENCAPS, upstream::ERR_INVALID_PK}; + Ciphertext ct; + SharedSecret ss; + BOOST_CHECK_EQUAL(Encaps(ek, coins, ct, ss), Error::INVALID_PUBLIC_KEY); + } + Ciphertext ct; + SharedSecret ss; + BOOST_CHECK_EQUAL(Encaps(ek, coins, ct, ss), Error::INTERNAL); + } + // Without injection every operation succeeds again. + Ciphertext ct; + SharedSecret ss; + BOOST_CHECK_EQUAL(CheckPublicKey(ek), Error::NONE); + BOOST_REQUIRE_EQUAL(Encaps(ek, coins, ct, ss), Error::NONE); + + // Moves transfer the secret and clear the source; Clear() wipes. + DecapsulationKey moved_dk{std::move(dk)}; + BOOST_CHECK(AllBytesAre(dk.Bytes(), 0)); // NOLINT(bugprone-use-after-move) + SharedSecret decapsulated; + BOOST_REQUIRE_EQUAL(Decaps(moved_dk, ct, decapsulated), Error::NONE); + BOOST_CHECK(std::ranges::equal(decapsulated.Bytes(), ss.Bytes())); + + SharedSecret assigned; + assigned = std::move(decapsulated); + BOOST_CHECK(std::ranges::equal(assigned.Bytes(), ss.Bytes())); + BOOST_CHECK(AllBytesAre(decapsulated.Bytes(), 0)); // NOLINT(bugprone-use-after-move) + SharedSecret& alias{assigned}; + assigned = std::move(alias); + BOOST_CHECK(std::ranges::equal(assigned.Bytes(), ss.Bytes())); + assigned.Clear(); + BOOST_CHECK(AllBytesAre(assigned.Bytes(), 0)); + + // Destruction wipes the bytes. + alignas(SharedSecret) unsigned char storage[sizeof(SharedSecret)]; + SharedSecret* secret{new (storage) SharedSecret}; + std::ranges::fill(secret->Bytes(), 0xc3); + secret->~SharedSecret(); +#ifdef MLKEM_TESTS_MSAN + // MemorySanitizer marks a destroyed object's storage uninitialized; read + // the bytes the destructor left behind anyway. + __msan_unpoison(storage, sizeof(storage)); +#endif + BOOST_CHECK(AllBytesAre(storage, 0)); +} + +BOOST_AUTO_TEST_CASE(backend_selection) +{ + const std::string_view compiled_arith{qbit_mlkem_compiled_arith_backend()}; + const std::string_view compiled_keccak{qbit_mlkem_compiled_keccak_backend()}; + const BackendNames active{GetBackendNames()}; + BOOST_TEST_MESSAGE("compiled " << compiled_arith << "/" << compiled_keccak << ", active " << active.arith << "/" << active.keccak); + + // The configure step's decision reaches the compiled library. +#if ENABLE_MLKEM_NATIVE && (defined(__x86_64__) || defined(_M_X64)) + BOOST_CHECK_MESSAGE(compiled_arith == "x86_64-avx2" && compiled_keccak == "x86_64-avx2", + "FAIL: configure chose native x86_64, but the library was compiled with " << compiled_arith << "/" << compiled_keccak + << ". Cause: cmake/mlkem-native.cmake and src/crypto/mlkem_config.h disagree. Fix: compare the probes in mlkem_native_detect_arch with the header's conditions."); +#ifdef HAVE_BUILTIN_CPU_SUPPORTS + // On a host with every extension the x86_64 assembly executes, native code + // must be what runs; without any one of them (BMI2 or POPCNT masked in a + // virtual machine, say), portable C must. + __builtin_cpu_init(); + const bool x86_native{__builtin_cpu_supports("avx2") && __builtin_cpu_supports("ssse3") && __builtin_cpu_supports("sse4.1") && + __builtin_cpu_supports("popcnt") && __builtin_cpu_supports("bmi2")}; + const std::string_view expected{x86_native ? "x86_64-avx2" : "portable"}; + BOOST_CHECK_MESSAGE(active.arith == expected && active.keccak == expected, + "FAIL: active backend " << active.arith << "/" << active.keccak << ", expected " << expected + << ". Cause: cached detection (cpu_features::HasMlkemX86Native) or the capability hook disagrees with the CPU. Fix: check cpu_features_tests and qbit_mlkem_has_avx2."); +#endif +#elif defined(__aarch64__) && defined(__APPLE__) && ENABLE_MLKEM_NATIVE + BOOST_CHECK_EQUAL(compiled_arith, "aarch64-neon"); + BOOST_CHECK_EQUAL(compiled_keccak, "aarch64"); + BOOST_CHECK_EQUAL(active.arith, "aarch64-neon"); + BOOST_CHECK_EQUAL(active.keccak, "aarch64"); +#else + // WITH_MLKEM_NATIVE=OFF, MemorySanitizer, or a target without a native + // backend, AArch64 ELF included: its assembly lacks BTI landing pads. + BOOST_CHECK_MESSAGE(!ENABLE_MLKEM_NATIVE, "native build for a target without a qbit backend"); + BOOST_CHECK_EQUAL(compiled_arith, "portable"); + BOOST_CHECK_EQUAL(compiled_keccak, "portable"); + BOOST_CHECK_EQUAL(active.arith, "portable"); + BOOST_CHECK_EQUAL(active.keccak, "portable"); +#endif + + // The portable guard forces portable C, nests, and restores what it replaced. + { + ForcePortableForTesting outer; + BOOST_CHECK_EQUAL(GetBackendNames().arith, "portable"); + BOOST_CHECK_EQUAL(GetBackendNames().keccak, "portable"); + { + ForcePortableForTesting inner; + BOOST_CHECK_EQUAL(GetBackendNames().arith, "portable"); + } + BOOST_CHECK_EQUAL(GetBackendNames().arith, "portable"); + BOOST_CHECK_EQUAL(GetBackendNames().keccak, "portable"); + } + BOOST_CHECK_EQUAL(GetBackendNames().arith, active.arith); + BOOST_CHECK_EQUAL(GetBackendNames().keccak, active.keccak); + + // The startup override. + InitializeRuntime(/*force_portable=*/true); + BOOST_CHECK_EQUAL(GetBackendNames().arith, "portable"); + BOOST_CHECK_EQUAL(GetBackendNames().keccak, "portable"); + InitializeRuntime(/*force_portable=*/false); + BOOST_CHECK_EQUAL(GetBackendNames().arith, active.arith); + BOOST_CHECK_EQUAL(GetBackendNames().keccak, active.keccak); +} + +BOOST_AUTO_TEST_CASE(every_entry_point_takes_the_override) +{ + // Each entry point fixes the backend for its operation at entry, from its + // own thread's copy of the override. Run each alone on a new thread, whose + // copy starts unset, and then ask the hooks what that copy says. The + // hooks can say native only where this CPU and build allow it. + (void)GetBackendNames(); + const bool native_possible{HooksAllowNative()}; + BOOST_TEST_MESSAGE("hooks can allow native code here: " << native_possible); + + std::array seed; + seed.fill(0x11); + const std::array coins{}; + PublicKey ek; + DecapsulationKey dk; + Ciphertext ct; + SharedSecret ss; + BOOST_REQUIRE_EQUAL(KeyGen(seed, ek, dk), Error::NONE); + BOOST_REQUIRE_EQUAL(Encaps(ek, coins, ct, ss), Error::NONE); + const std::vector>> entry_points{ + {"KeyGen", [&] { PublicKey e; DecapsulationKey d; (void)KeyGen(seed, e, d); }}, + {"CheckPublicKey", [&] { (void)CheckPublicKey(ek); }}, + {"Encaps", [&] { Ciphertext c; SharedSecret s; (void)Encaps(ek, coins, c, s); }}, + {"Decaps", [&] { SharedSecret s; (void)Decaps(dk, ct, s); }}, + {"GetBackendNames", [] { (void)GetBackendNames(); }}, + }; + for (const auto& entry : entry_points) { + const std::string_view name{entry.first}; + const std::function& entry_point{entry.second}; + bool native_while_forced{true}; + bool native_after{!native_possible}; + std::thread{[&] { + { + ForcePortableForTesting portable; + entry_point(); + native_while_forced = HooksAllowNative(); + } + entry_point(); + native_after = HooksAllowNative(); + }}.join(); + BOOST_CHECK_MESSAGE(!native_while_forced, "FAIL: after mlkem::" << name << " on a new thread with portable C forced, the hooks allow native code. " + << "Cause: it entered the library without taking the override, so native code can run. " + << "Fix: call BeginOperation() first in every entry point of src/crypto/mlkem.cpp."); + BOOST_CHECK_MESSAGE(native_after == native_possible, "FAIL: after mlkem::" << name << " with the override cleared, the hooks allow native code: " + << native_after << ", expected " << native_possible << "."); + } +} + +BOOST_AUTO_TEST_CASE(override_copy_is_per_thread) +{ + // Thread A takes the override unset; thread B then takes it set. A's copy + // must not change: a copy shared between threads would switch A's running + // operation to the other backend. + (void)GetBackendNames(); + const bool native_possible{HooksAllowNative()}; + std::promise a_took, b_took; + std::future a_took_future{a_took.get_future()}; + std::future b_took_future{b_took.get_future()}; + bool a_before{!native_possible}; + bool a_after{!native_possible}; + std::thread a{[&] { + (void)GetBackendNames(); + a_before = HooksAllowNative(); + a_took.set_value(); + b_took_future.wait(); + a_after = HooksAllowNative(); + }}; + a_took_future.wait(); + bool b_native{true}; + { + ForcePortableForTesting portable; + std::thread{[&] { + (void)GetBackendNames(); + b_native = HooksAllowNative(); + }}.join(); + } + b_took.set_value(); + a.join(); + BOOST_CHECK(!b_native); + BOOST_CHECK_EQUAL(a_before, native_possible); + BOOST_CHECK_MESSAGE(a_after == a_before, "FAIL: thread A's override changed when thread B took it. " + << "Cause: the copy the hooks read is shared between threads. Fix: keep t_force_portable thread_local in src/crypto/mlkem.cpp."); +} + +BOOST_AUTO_TEST_CASE(override_changed_by_another_thread) +{ + const BackendNames before{GetBackendNames()}; + // The outputs with nothing changing the override; every backend gives them. + std::array seed; + for (size_t i{0}; i < seed.size(); ++i) seed[i] = uint8_t(i * 7 + 1); + std::array coins; + for (size_t i{0}; i < coins.size(); ++i) coins[i] = uint8_t(i * 3 + 2); + PublicKey expected_ek; + DecapsulationKey expected_dk; + Ciphertext expected_ct; + SharedSecret expected_ss; + BOOST_REQUIRE_EQUAL(KeyGen(seed, expected_ek, expected_dk), Error::NONE); + BOOST_REQUIRE_EQUAL(Encaps(expected_ek, coins, expected_ct, expected_ss), Error::NONE); + + // A third thread flips the override as fast as it can while two threads + // run operations at once. Each operation must finish on the backend it + // started with: on x86_64, one that mixed backends would give wrong + // outputs, and under ThreadSanitizer a shared copy of the override races. + std::atomic_bool stop{false}; + std::atomic flips{0}; + std::thread flipper{[&] { + while (!stop.load(std::memory_order_relaxed)) { + ForcePortableForTesting portable; + flips.fetch_add(1, std::memory_order_relaxed); + } + }}; + while (flips.load(std::memory_order_relaxed) == 0) std::this_thread::yield(); + std::atomic wrong{0}; + std::atomic portable_seen{0}; + constexpr int ROUNDS{50}; + const auto run_rounds{[&] { + for (int round{0}; round < ROUNDS; ++round) { + PublicKey ek; + DecapsulationKey dk; + Ciphertext ct; + SharedSecret ss; + SharedSecret decapsulated; + const bool ok{KeyGen(seed, ek, dk) == Error::NONE && Encaps(ek, coins, ct, ss) == Error::NONE && + Decaps(dk, ct, decapsulated) == Error::NONE}; + if (!ok || ek != expected_ek || !std::ranges::equal(dk.Bytes(), expected_dk.Bytes()) || ct != expected_ct || + !std::ranges::equal(ss.Bytes(), expected_ss.Bytes()) || !std::ranges::equal(decapsulated.Bytes(), expected_ss.Bytes())) { + ++wrong; + } + if (GetBackendNames().arith == "portable") ++portable_seen; + } + }}; + std::thread second{run_rounds}; + run_rounds(); + second.join(); + stop.store(true, std::memory_order_relaxed); + flipper.join(); + BOOST_TEST_MESSAGE(flips.load() << " flips; portable active at " << portable_seen.load() << " of " << 2 * ROUNDS << " checks"); + BOOST_CHECK_MESSAGE(wrong.load() == 0, "FAIL: " << wrong.load() << " of " << 2 * ROUNDS << " rounds gave wrong outputs while another thread changed the override. " + << "Cause: an operation mixed backends. Fix: the capability hooks in src/crypto/mlkem.cpp must read the thread_local copy taken at entry."); + BOOST_CHECK_EQUAL(GetBackendNames().arith, before.arith); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index fdf0081c26..6952990600 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -29,6 +29,7 @@ add_library(bitcoin_util STATIC EXCLUDE_FROM_ALL threadnames.cpp time.cpp tokenpipe.cpp + ../compat/cpu_features.cpp ../logging.cpp ../random.cpp ../randomenv.cpp diff --git a/test/fuzz/qbit_corpora/MANIFEST.json b/test/fuzz/qbit_corpora/MANIFEST.json index 887c3f231f..5265d09ab8 100644 --- a/test/fuzz/qbit_corpora/MANIFEST.json +++ b/test/fuzz/qbit_corpora/MANIFEST.json @@ -170,6 +170,136 @@ "semantics": "Payload validation, header validation and CheckBlock in one input." } ], + "mlkem": [ + { + "file": "honest-round-trip", + "format": "fdp", + "size": 96, + "sha256": "60563d6bf3a00873f259f212eb2916e929f9a2912b6738402345c16a2fc6cb51", + "semantics": "Key generation, encapsulation and decapsulation from fixed seed and coins agree; the all-zero encapsulation key is in range, and the all-zero decapsulation key fails its hash check." + }, + { + "file": "cctv-base-key", + "format": "fdp", + "size": 1664, + "sha256": "fc04cec04516e98b8c4c8c855bc74c30e5d672a04d7be3a44415274c05c1ff4c", + "semantics": "The key shared by the C2SP CCTV modulus vectors, every coefficient in range: accepted." + }, + { + "file": "coefficient-3328", + "format": "fdp", + "size": 1664, + "sha256": "6c274898e6747f79cb4fec2dde374e46f37e578d1e4ee605e0a8ce9c377212b0", + "semantics": "First coefficient q - 1 = 3328, the largest valid value: accepted." + }, + { + "file": "coefficient-3329", + "format": "fdp", + "size": 1664, + "sha256": "9dea64bc7eef0413f8c5bdfea2f74b4cf1acb148ed1bfa846935db0de425eb2e", + "semantics": "First coefficient q = 3329, the smallest invalid value: rejected, outputs cleared." + }, + { + "file": "last-coefficient-4095", + "format": "fdp", + "size": 1664, + "sha256": "465c31c294e5e8a5e0effcedf2e346b2de1643635a66309d7f9856c3fb9a0a42", + "semantics": "Last coefficient of the last polynomial 4095: rejected, outputs cleared." + }, + { + "file": "acvp-valid-decapsulation", + "format": "fdp", + "size": 6400, + "sha256": "22599a5c645de5def70e8e074118f35683d5a324fbc17cd817cc5623c19d5746", + "semantics": "NIST ACVP decapsulation tcId 97 (valid): its key, ciphertext and decapsulation key." + }, + { + "file": "acvp-modified-ciphertext", + "format": "fdp", + "size": 6400, + "sha256": "fea33fe3acf805c569b6d4804a257b82275dcd6dba1269fa7ff330091563e563", + "semantics": "NIST ACVP decapsulation tcId 96 (modified ciphertext): implicit rejection." + }, + { + "file": "acvp-modified-key-hash", + "format": "fdp", + "size": 6400, + "sha256": "b937bf76921e91b22e043794ca5b8780b35f60e8384d90faa558ef0e15da9497", + "semantics": "NIST ACVP decapsulation key check tcId 146 (modified H): rejected, secret cleared." + }, + { + "file": "honest-ciphertext-bit-flip", + "format": "fdp", + "size": 6403, + "sha256": "c555ad9f071c42fb7772c778d34a769259a926cae98b853ad5ce4b72f1bbc1a2", + "semantics": "Flip bit 4321 of the honest ciphertext: decapsulation still yields a secret, a different one." + } + ], + "mlkem_backend_diff": [ + { + "file": "honest-round-trip", + "format": "fdp", + "size": 96, + "sha256": "60563d6bf3a00873f259f212eb2916e929f9a2912b6738402345c16a2fc6cb51", + "semantics": "Key generation, encapsulation and decapsulation from fixed seed and coins agree; the all-zero encapsulation key is in range, and the all-zero decapsulation key fails its hash check." + }, + { + "file": "cctv-base-key", + "format": "fdp", + "size": 1664, + "sha256": "fc04cec04516e98b8c4c8c855bc74c30e5d672a04d7be3a44415274c05c1ff4c", + "semantics": "The key shared by the C2SP CCTV modulus vectors, every coefficient in range: accepted." + }, + { + "file": "coefficient-3328", + "format": "fdp", + "size": 1664, + "sha256": "6c274898e6747f79cb4fec2dde374e46f37e578d1e4ee605e0a8ce9c377212b0", + "semantics": "First coefficient q - 1 = 3328, the largest valid value: accepted." + }, + { + "file": "coefficient-3329", + "format": "fdp", + "size": 1664, + "sha256": "9dea64bc7eef0413f8c5bdfea2f74b4cf1acb148ed1bfa846935db0de425eb2e", + "semantics": "First coefficient q = 3329, the smallest invalid value: rejected, outputs cleared." + }, + { + "file": "last-coefficient-4095", + "format": "fdp", + "size": 1664, + "sha256": "465c31c294e5e8a5e0effcedf2e346b2de1643635a66309d7f9856c3fb9a0a42", + "semantics": "Last coefficient of the last polynomial 4095: rejected, outputs cleared." + }, + { + "file": "acvp-valid-decapsulation", + "format": "fdp", + "size": 6400, + "sha256": "22599a5c645de5def70e8e074118f35683d5a324fbc17cd817cc5623c19d5746", + "semantics": "NIST ACVP decapsulation tcId 97 (valid): its key, ciphertext and decapsulation key." + }, + { + "file": "acvp-modified-ciphertext", + "format": "fdp", + "size": 6400, + "sha256": "fea33fe3acf805c569b6d4804a257b82275dcd6dba1269fa7ff330091563e563", + "semantics": "NIST ACVP decapsulation tcId 96 (modified ciphertext): implicit rejection." + }, + { + "file": "acvp-modified-key-hash", + "format": "fdp", + "size": 6400, + "sha256": "b937bf76921e91b22e043794ca5b8780b35f60e8384d90faa558ef0e15da9497", + "semantics": "NIST ACVP decapsulation key check tcId 146 (modified H): rejected, secret cleared." + }, + { + "file": "honest-ciphertext-bit-flip", + "format": "fdp", + "size": 6403, + "sha256": "c555ad9f071c42fb7772c778d34a769259a926cae98b853ad5ce4b72f1bbc1a2", + "semantics": "Flip bit 4321 of the honest ciphertext: decapsulation still yields a secret, a different one." + } + ], "p2mr_script": [ { "file": "legacy-checksigpqc-default-valid", diff --git a/test/fuzz/qbit_corpora/README.md b/test/fuzz/qbit_corpora/README.md index 4bb66bda48..01a509aea3 100644 --- a/test/fuzz/qbit_corpora/README.md +++ b/test/fuzz/qbit_corpora/README.md @@ -9,6 +9,8 @@ upstream `qa-assets` corpus: | `asert_edge_cases` | [`asert_edge_cases/`](asert_edge_cases) | | `asert_math` | [`asert_math/`](asert_math) | | `auxpow` | [`auxpow/`](auxpow) | +| `mlkem` | [`mlkem/`](mlkem) | +| `mlkem_backend_diff` | [`mlkem_backend_diff/`](mlkem_backend_diff) | | `p2mr_script` | [`p2mr_script/`](p2mr_script) | | `pqc` | [`pqc/`](pqc) | @@ -46,6 +48,16 @@ arrays and strings from the front of the input, integers from the back. real keys and signatures that are built once per process, on first use. Tagged inputs with another version byte are ignored. +`mlkem` and `mlkem_backend_diff` share one untagged layout: the key generation +seed `d || z` (64 bytes), encapsulation coins (32), an arbitrary encapsulation +key (1568), ciphertext (1568) and decapsulation key (3168), then, from the back, +whether to flip one bit of the honest ciphertext and which. Fields cut off by +the end of the input are zero. The seeds use the C2SP CCTV modulus key, the +3328/3329 coefficient boundary and NIST ACVP decapsulation vectors from +`src/test/data/mlkem1024_vectors.json`. Both targets run on native code where +the CPU has it, or on portable C with `MLKEM_FORCE_PORTABLE=1`; +`mlkem_backend_diff` also compares every result with forced portable C. + The fixture bodies check that an unmodified fixture verifies. `pqc` also checks that any modified signature, public key or message is rejected. Legacy and generation seeds keep key generation and signing covered in every replay. @@ -63,6 +75,6 @@ from a reused corpus by hand if they are no longer wanted. A target path that is not a directory, or a symlink, directory or other non-regular file at a managed name, is an error; all target paths and managed names are checked before any file is written. -`test/fuzz/test_runner.py --require_qbit_corpus` then fails unless all six +`test/fuzz/test_runner.py --require_qbit_corpus` then fails unless all eight targets are selected, each has at least one regular input file, and the fuzz binary reports replaying them. diff --git a/test/fuzz/qbit_corpora/generate_seeds.py b/test/fuzz/qbit_corpora/generate_seeds.py index 62883464b0..db6942e068 100755 --- a/test/fuzz/qbit_corpora/generate_seeds.py +++ b/test/fuzz/qbit_corpora/generate_seeds.py @@ -557,6 +557,99 @@ def auxpow_cases() -> list[dict]: # --- manifest ---------------------------------------------------------------- +# --- mlkem, mlkem_backend_diff ----------------------------------------------- + +MLKEM_SEED_BYTES = 64 +MLKEM_COINS_BYTES = 32 +MLKEM_EK_BYTES = 1568 +MLKEM_CT_BYTES = 1568 +MLKEM_DK_BYTES = 3168 +MLKEM_Q = 3329 +MLKEM_VECTORS = CORPORA_DIR.parents[2] / "src" / "test" / "data" / "mlkem1024_vectors.json" + + +def mlkem_vectors() -> dict: + return json.loads(MLKEM_VECTORS.read_text(encoding="utf-8")) + + +def mlkem_set_coefficient(ek: bytes, poly: int, index: int, value: int) -> bytes: + """Overwrite one 12-bit coefficient of an encapsulation key (FIPS 203 ByteEncode_12).""" + out = bytearray(ek) + pos = 384 * poly + 3 * (index // 2) + if index % 2 == 0: + out[pos] = value & 0xFF + out[pos + 1] = (out[pos + 1] & 0xF0) | (value >> 8) + else: + out[pos + 1] = (out[pos + 1] & 0x0F) | ((value & 0x0F) << 4) + out[pos + 2] = value >> 4 + return bytes(out) + + +def mlkem_input(*, ek: bytes | None = None, ct: bytes | None = None, dk: bytes | None = None, + flip_bit: int | None = None) -> bytes: + """The shared layout of both ML-KEM targets. + + Front: key generation seed d || z (64 bytes), encapsulation coins (32), an + arbitrary encapsulation key (1568), ciphertext (1568) and decapsulation key + (3168). Back: whether to flip a bit of the honest ciphertext, and which. + The harness zero-fills fields cut off by the end of the input, so a seed + may stop after any field. A bit flip needs every front field present. + """ + enc = FdpEncoder() + enc.raw(pattern(MLKEM_SEED_BYTES, 0x61)) + enc.raw(pattern(MLKEM_COINS_BYTES, 0x71)) + fields = [(ek, MLKEM_EK_BYTES), (ct, MLKEM_CT_BYTES), (dk, MLKEM_DK_BYTES)] + if flip_bit is not None: + fields = [(value if value is not None else bytes(size), size) for value, size in fields] + while fields and fields[-1][0] is None: + fields.pop() + for value, size in fields: + value = value if value is not None else bytes(size) + if len(value) != size: + raise ValueError(f"ML-KEM field of {len(value)} bytes, expected {size}") + enc.raw(value) + if flip_bit is not None: + enc.boolean(True) + enc.integral(flip_bit, SIZE_T, 0, MLKEM_CT_BYTES * 8 - 1) + return enc.build() + + +def mlkem_cases() -> list[dict]: + vectors = mlkem_vectors() + base_ek = bytes.fromhex(vectors["cctv_modulus"]["base_ek"]) + acvp = vectors["acvp"] + decaps = {tc["tcId"]: tc for tc in acvp["decapsulation"]} + valid = next(tc for tc in decaps.values() if tc["reason"] == "valid decapsulation") + modified = next(tc for tc in decaps.values() if tc["reason"] == "modified ciphertext") + bad_dk = next(tc for tc in acvp["decapsulationKeyCheck"] if not tc["testPassed"]) + return [ + case("honest-round-trip", "fdp", mlkem_input(), + "Key generation, encapsulation and decapsulation from fixed seed and coins agree; the all-zero " + "encapsulation key is in range, and the all-zero decapsulation key fails its hash check."), + case("cctv-base-key", "fdp", mlkem_input(ek=base_ek), + "The key shared by the C2SP CCTV modulus vectors, every coefficient in range: accepted."), + case("coefficient-3328", "fdp", mlkem_input(ek=mlkem_set_coefficient(base_ek, 0, 0, MLKEM_Q - 1)), + "First coefficient q - 1 = 3328, the largest valid value: accepted."), + case("coefficient-3329", "fdp", mlkem_input(ek=mlkem_set_coefficient(base_ek, 0, 0, MLKEM_Q)), + "First coefficient q = 3329, the smallest invalid value: rejected, outputs cleared."), + case("last-coefficient-4095", "fdp", mlkem_input(ek=mlkem_set_coefficient(base_ek, 3, 255, (1 << 12) - 1)), + "Last coefficient of the last polynomial 4095: rejected, outputs cleared."), + case("acvp-valid-decapsulation", "fdp", + mlkem_input(ek=bytes.fromhex(valid["dk"])[1536:1536 + MLKEM_EK_BYTES], ct=bytes.fromhex(valid["c"]), + dk=bytes.fromhex(valid["dk"])), + f"NIST ACVP decapsulation tcId {valid['tcId']} (valid): its key, ciphertext and decapsulation key."), + case("acvp-modified-ciphertext", "fdp", + mlkem_input(ek=bytes.fromhex(modified["dk"])[1536:1536 + MLKEM_EK_BYTES], ct=bytes.fromhex(modified["c"]), + dk=bytes.fromhex(modified["dk"])), + f"NIST ACVP decapsulation tcId {modified['tcId']} (modified ciphertext): implicit rejection."), + case("acvp-modified-key-hash", "fdp", + mlkem_input(ct=bytes.fromhex(modified["c"]), dk=bytes.fromhex(bad_dk["dk"])), + f"NIST ACVP decapsulation key check tcId {bad_dk['tcId']} (modified H): rejected, secret cleared."), + case("honest-ciphertext-bit-flip", "fdp", mlkem_input(flip_bit=4321), + "Flip bit 4321 of the honest ciphertext: decapsulation still yields a secret, a different one."), + ] + + def case(name: str, fmt: str, data: bytes, semantics: str) -> dict: return {"name": name, "format": fmt, "data": data, "semantics": semantics} @@ -566,6 +659,8 @@ def case(name: str, fmt: str, data: bytes, semantics: str) -> dict: "asert_edge_cases": asert_edge_cases_cases, "asert_math": asert_math_cases, "auxpow": auxpow_cases, + "mlkem": mlkem_cases, + "mlkem_backend_diff": mlkem_cases, "p2mr_script": p2mr_cases, "pqc": pqc_cases, } diff --git a/test/fuzz/qbit_corpora/mlkem/acvp-modified-ciphertext b/test/fuzz/qbit_corpora/mlkem/acvp-modified-ciphertext new file mode 100644 index 0000000000000000000000000000000000000000..1846c14ca16955f2c6d493906518e26e316ecd1a GIT binary patch literal 6400 zcmeI$cOw*z!vJuz&gSgB_sC^)#o1?%GtSC7n>c%R_Bh)aXYb6y*`wmjvk4VR%1nw- zq53^9;CTt(H}U!7<5IcFW|hB44NbulJ&5YN00p!j3cFTqv0E1`(HNwT6F-dZzr6pr z?MTF}`aOpYW|`J7ZGuESX5dOLu>DxnJ?PmNF+}YLE_l%@y>Z4A86#L91UovGiKR3NwT6kNVIi6o@qmbKamuIw0G{}HeaiR9xhz-8&85%Zo&a=Q_ z0;J*IBaOB`(IO+7Nf$uxZTzUvEjYNk&O1J<6v_^Aj=Gt)he61+f7gr5K-+RJHEW%N$J)7E*(k>{Yb*CcN~FBevo~ zQzV!jroiXb1=CSQ2=&oUv7Am>LOnAPj7J{nz10W^w#F z!-9pC7<_7xP}XrKyi4F5o%8&5Q7L-t3kWCbM}jL}vD#@GGe_uTQ7sU~*xD)rBRi%7 zT&o|lhxenb1dHxdgptwYS#2KFKa+UI0g@!99+$<@&sS#;?dR^)kgOO^0J`dUU08W; z!KfY5Og>$VVz?gy@tCc&SXkSleD2d**3%Ox@Mq*FExU9gA~GSb->MII>Ge=6kFQ%T zTjXBU)X{@g&71su;#kqW4)Hoy^*lHJgyhGclcFX<^!zuxMCn1hhfhiP2yM+Wql!S9 zovLflEBRNoT3__T1X4P?__J4VMpn$o-j^X96*L*3VU~zHQR>Sog|k)#!d>MmPcy?y z@F#@87;K~OIZHhKWUQ{Ssm(OZ=Mf>yaPk(MQdr}S|(Yz6u+xzMce9e)>E3twRRbqCrX zaQRE+Pz%qgu0>iB1IRwo?K@qk`X<;Y~n5#b#qDk#{e?ijTjqDmbqENd*zVVP?0zg=dG zD7ISzvmiZi6PFLvPWRb4l77wlde{*t#?`o_cEaO&;M$W8Eyr&U1V}KjdpsE6@R7|H zbgmX+oHJ$nN>g<~6nUcR@A;dw{nXguxjh8))|((8taLh0-}(=HFkhSBb#?#Bjw|?Z zLA!S&offE)+Vm>tTs>F^bRTo$`!HKs<~h!yFh!}cng^ir9>{h!+{_YzQC2&+GH<1( z4Iz}Vf2gt9OtIb@SB(AY`mp4R0c$q$SucjQ^ygYiznS2LuNEKcdfOcs>zRXnOL zH*+le*!>W|KFdt;7bYw7kn{__>;pB?Q!51J4;RXxiKhSlUhdca?Dh9@GqL9kJJ%Pn z=u6&0nL`a1Od%=JEq71TX1r+G87l{d&BgxM$!8OvN(4pVy2_R&*C~8r@#GcND!b2m zjUBfcU@t!`KHH!5n}27fGHHVUsw%aXGybz)AiZ!_h>Ek0>qSG^mHtC|YGszecLxwt z+7NB?f4tRs(hihD$j-i3tsgo=HI2J7dVjt!?hlH{xP&^dXp=h9s$bvRTx0?Z5OEvF zfHy;ejrIFvC2)JPWAh-E|CnM$vZ6_hIci=$^AUk=3xBe3d_cJ zoFAv)*r%8_&>l9915Yn)I)-n+4T6i2Fy6&~HqTh@J+k`CzopmcU)x$D_~7tV*4FLO z-Vcyd^i5M-)`}Q~tPq9H5pqs^t|r%6s{$0mw(y9AhK~M% zFE{-31TW^U5D zm(BPG*RILDii$&Hb5`2hNKITS@0#L@8~s0(XB3rm4X!tg9gv(qe^gRzgi8}7bg~uG zx}~A~tY-n@ym<;9i|cb8#jnO4WMC|PtATL^e&vtd`t^9Eq@-J?53_h6QD{g}l{{eh`uevRsQ}EB?=$!7*5^;WRccaS`G0;5QjLLUHDG>f9 z;y2q>o=ARZsYX{jxPqAHk4s&F6{5U?v2u$*4_ppVYNHNO=vxX{XK?o$Y2sKn`d%b& zT9#{>a!?Dh@HcWi@(@vrNx;SouJVlm-EHr01n~I*uN&{HEJM#05PB27kW+_P zqI#WDL)+eq{*Eg!ATN91@lfH#R`HYJp%dIPwRFbrJepmjDCg$_xyjSh5!(2DJ0+=k z#)lMXo?j284<@p(M}Te?TV0(XT1jHnq^eN|J}f9N^JT{^-Z7Qn2C`va%sM2oPB;_n ztryFlglT2^hgwL)QK2r&hO}F=ZhMr3i{X@g?#9bsL;lZ%AL-q!CA^<~MO6@hu5`q( zwJBO>J!|_b*ASJe+3~VQ>W_iFH?2!|hn0o1fN^+j#t-8cJUczmk9U5?uoP{6HwD#h z)qC>*#oaLEu<`S-XD=4l++!pHJ-;5$WbL|)rcqVO)H~{ z*+e_IBuKljUl=mqMu=myE9|ltlO@1qcR=^Eq>pgf;6{OV0sxwH? zgFB?(jFxVw1=-=SRB=UlGka5>OA)jtvLaioLYN_wH^B@mfFc_fKnFs=Pztcx2TP2h z%6K#179QHfoI%&DjGRGUkY#;5nrZs<0Im|3^)XGD!!>14YX97J`B!sg#JKxq*zF-D zw%k4Rgmf4l$n?%R>3k-xKeg~Yb5m67NhhyoarDPu2&7~(UWr>re*V3R?|P>|$%R-{ zOiL3=$Ox==R>|`Pz-S<}uTa#nio4xsJPZxXkjR!21tQh~9pwBDpfW*mq1B8)J#g zEg++Ne1rOPH8SU3#j7&BGRY{wX^}?y)QM%t9(^ZnFdPw_@!dVPhQ$BalXdnpgKt;X zzq~OaAutD0ff#`&PHNBPzzADr3ELz8BpPKC<<63%kNvFtiH#X+fs4JiWJlxkuPhAG zdY*X>_;9f%h&&-PpsapV%^)Y$ziPteHwvw9tQ#oeuLHy<3jC1^_|Wh98R|6iR?={a zE}E5+MhthN=;T-}6=$ZTCJPtJ(5j@2qEbb8xkWfUg&sgk%a{vK#ri%Z=I}~BC#gns z3Q4iPiwv2NliWZWg}a-i!K!+YgX!+Y*vOuzxm@3E1-l25KjYRwV=R1LRiqxeYhs&o z-R}0tlYF|ROHB`^VcjD4`i#n*o~gGD{O+Gvz~UIlNA4BfU@tKE~ty~ zw%ApXOufL+rCleP`~-?Fx6r0aPX0N%mu@6- zm<$LNTEE{cucEGX~jNl)~lthbO}9<-hqEL;S#RVOqZj@73xf zDi5M+iA$dNIE_5bk`0A7g%qi=i?X8?gv>MMJ=o4y?5+(N#&Jb63>@q#Syp!|ulQh1 zK>U+K?r!Txo}}o?cTt9Jy&A%z#7R)r!?|cRo^7oMq}H{sj^m_%j444AK#fk>si}q= zNvU5(7f{r(oYs1N0+i^*L2Gm^RX7-{S|Te2FqEvdC?+d`DA=+5qsY{x6yQsYZk#UD zR`87{4+|q@5!Ti1XsBm})nuH-Vv`{n)NM4=vh77I7iSR~%?r}rxS_+v3g-?^qPDo{ z&K>lFG#`|qf$6__>A2Fr&;=6&iW@pFEZg4Z^W$$73m1ytvXJ&j@-pnd3eYaiW zjs*V9#~uhh>~tU{Yke%ApckXvc#U@KE502R% V!J1w?{nlQ-!5bJUr8m3N{{#Do>a+j= literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem/acvp-modified-key-hash b/test/fuzz/qbit_corpora/mlkem/acvp-modified-key-hash new file mode 100644 index 0000000000000000000000000000000000000000..863f876e39f5d730718019864cdfc7d1c258f268 GIT binary patch literal 6400 zcmeIyg<2Dg!v$~{-8GQzW=uwdGz_GqdmuF$X=LQ6!RSuO(V=v!bPSLbDFs1VN+}iP z`(MC!3Gcat-*e7!DBWN*%bFwCMvdSohIU^;UNj!^IhJiQnP<;a=p>BdDuwl2i~BSk z@jI1&X12&#pwvwq11N|0-bna19}779y*?3yS8lOd!tIe$Er(qzGJ+KqEjGBxGw#B!f$ z87~Lw8A5VopB&s6HIh>XVvCu!Dl9fo=6j>k5vPu~9qNzOuloQ$a|GBBmraVJt=LmT ze;KiMneT>TkeE}UP)tV3aN4o+Au-c5J;^^)F|ZQe$w=`QIZmw^ob;yy>20jR!@ukO z%G)k)H>YPdEWtmX@*8)@Zz7Gfbw3!WqQV-8&Jw5GD4FO=dk2gJ|5{6?;hqWmg|xR7 z&yTN=ctng4l$6WuyzbJs{X%2h?Z`e1B7pxd)5a43D3i{o~ry zs`0LDEZ|W*JQK5YdcOA)Vi)$OE;4mdkVK4^1bXB@BQ#TyVXs~S2?u&V1f{peuE5n! zaele-V85d*tNErq z%us0!5PYe;odR3qpF*7$_J0*a7tn?gDYdliqKM^5i7;328e*gyl3fir*F8(s3bONwWC z>VU8uF3ATG!;Or%6*r5~EyuR;%#xBreWOQ|_XxH21dfl=i%!%Ja<55Bsj40S(6y+h z|NSe6vfwL<5r(EoCw7Qxb3M9zA;gg>_p3Im%mbT9cvX8#+E0Gm^Z| z+Q_n{fwM;pwP=?znxJfaQxO^tE)~OeUwrNtU%C|wrPfKmH(subA3wYjC!L2pymfU7 zAyJ@;SCbBSf&(^l^46%{D4h+k#PVX!gJsY0IyzSmO3od)IliEU5>OQPr@o?k6(&wk z?`I2tTMpr|W$S%`W}CzXww+A!CNGa7L`0lgJ?Mpeae@N#$|SMV$j$s1m8l>ryrbJ( zW~_a(+v6J2j5JR?OBE{`{X~avQ7A?au z)g5M&{mgks=U%hIBRbijjaHQ;2+zP2RduKma$V94-&;6Y^5r`FA_C1&UFO4ggO-@U z>I2;rmMwK1J1or4DL|A|!neJeRuR)E+AIW7ior45>6~3A`TS7{ymhigo!|bA^&d@O zNN@dgJu=GMD zD>TktZ|j&>^e*xw4T>{NPzRL|V{bj%BNzTB;W35xp_A)7GmJ^Y1UIC>2I>?!HTxV2mSU%nM-x3m7jY2g1p;U~J| zQ31y{PXReNaT^*wU~Z5MT}l0N%GysRZ+N_*lJK*4Z?$orXh4=Pl{RujgKlf|iha9N z`zzsXI78mXg#n~;v&x;_Z}si>s~`Ss1z#t7_0k!O!@tvZ6|g$LBOuwY9KT7>un5p<4_p7Eb8=gatfdR1P9`=Ky&p&($@T9yx zG;$l*jr0!;++)eENo}M3y*2wBVUPNc+Ep_^_@J_t@o%-ptL%daK?BJ z5n7J^0xsXZMyReyb5e;cpbBR}KFMDviKWa(dxU}@FZUy7UjWAUch7|y#!qJy$Irk; zJdJ!g;H!;agX2qSnYW{D#eK{~XY>OBvBgouy(i1l(u-=53Nu;!K1?UIT>()umT}1! zZyssVRYANF5^#?8JU(qNer!I$x8w=}{1-{w-mTH5^5py-W>5JUH3^Z|+*6OGv@d!> zftj{cmoU$26v)hh8^Wx=LDb2cqO_KK)r6dIJk-~zC7RKcD*erbSCId<7E(4j?1-?^8ji-r??v)7#T}aVSz#crUhHSs zNzae8bg=!AQ`il`_u0uJ2{%sb_wQxl9G4k`EM>nRRWx-M{v@b&!=v3~@Cg@RD zy6mi71hmXlu=IG`m|^IN%-?nx?56HY8)#aUQV5?RT?(Z<7q6P774Ir~^P#BPku3Mg z9(ck^-|ZtJ_=H7v#x*29j#N{@e`M~FhV?g2;q%`*k=p`Bk_;f-kvfqkJe0oNl#2H) zcpt#Hsz6D(Nf=6>wI-jA5sAab4;U@jJoG)f{iFhvCrm_0i*6ITm(NAslH02lZYp#> z1}K?dYwYgA?(^kpbh5Jso7u7=94|`pRMQo+UT}x=Z-ZKQhRD#n0TmzVmtHacu;K#4 zfVHkk^1&R8-x3{OX)adMX9!U3(KL0B^h1A*7w${ttrN!tS^@J*O-_G+0BF)1>y zReJnr=3$XU1rBMKy%N%!Ja2mh8}Xo|U92GeLg` z<(K;&I~ap{uHh}l;yd@$I@_fc=IFF!QAN_IKbN^j3pG3k@^^mG9c$u^e8mwWvcLnb z<-%MF#%Jc2B-1p6M_&mA6T6leZAVBOa)mS}S$uk)7`_ZhhzgH82ZDHvRd+Jj{AZGU zo-#A}GnXcW?K-)4+yM6q+>x;v_6|h)_GVi$eAHv=(3I)5OXBE7l1EN?r<*-A$x7DT z_2_CZ*GvJfwhUBK(9J3*QkZJ`g7l2eNN%+Pl>q%aEJ}D`@pOqN!6@NDC{0v`mvl|( zwhl9`$q(l&*}p6#7K|l@YrGKF6U$BBZ4fnJ5_OeNE7bOlGgalT_WXA9-t!Z<0R7Cy z-NS=vrgCVatUg#)PujOf6DB;pp*hr<$YJwa66yLp_S(dpa9U1IvStH=?=C0|5$Nm= z`{bK#NW$>|Rk*u#^1ah)tn7}4kSKraDqK=~$4yCWYWdZ$CAx=Wb_urCWdD4?Am3tG zKXgp(h_dR4&&EdRHV)A!{f1>FbDp44h1Pu_J9nz!#abn9q%33WUBs_AANGohw65s# zJ%?XE5nMW6-_#{%$3~aY>K>McZT{LN;taObGs*Cmc`);SGd$Jkflrgb#Qw6@%~ZM} z6F1-Y%cyiZuK{f`$BBH^wt&WZ-#}4c+6FH8aDJUeqn?1Ox_#&D#ZC#Na~280^Z<)- z)_HDs=xU{L!$nFGHHW}Dwyh!-hY+{bm3McbJ^(I5P)dvEP0nvddmL!;y1M03z$M3O@W+qv9BQQT2~luhSV?5L$^w6=O4RO= zCY!L7$nmlYJ1CV@T+HWw;xl6z#h4l$l<-wfc|cR$5v zjN)!ch|{E1ElbuKP1GF`ahX;Qxkav>gu0=tW2dA4>#s2rOx0R(8b5OBCagQ2DHsr1 z(f|Xn?Q#-hJB7vbthUpBmv@i7s>4dVWV~ckI0>~ib{mjVfq$dDXo?_QDl}nLS(qhG zsHy~v7}TsmMaDf~R%0g;4_0c$2hVFB)_0q|l;oS@T)g5ZZ!2Nz_DCp}7JM>SXO)h~ z2CTz?V@-HGq!MYnliDGL^`LjoHj1U0+GQFX4aPL9evSu@j&BjBBxc1zI`}3Rf|v=J zJmH$&ypm*Sz2w9k_dWQU)rDS_lQZ5-x8J(5jcCE4)+PjkLzZL$3Lcv0_=&ON5_ylH z3sZ|N;tIpb2QUd|X;JAaR@KWz8hz-xiydP~z2qkHs*N2S?tgE=kKE%=`BTM?lKLk_z9Y5L^$%2k{y3&T^N9% zr3>#<{>v}kL)=XEJ3Rta)Ul%moup$PV+f98B8Y7^4&{3r>;5+|O2|0HK``=cEy8R1 zJ%V|%p0v06YnjDUVf1eOmsdD5{RMH8RN)hDWN?G87_2n|KBQt4R0Vw#s&7A2ds7)2 zmifxtw?zzKi;`qgNBifku#m?AYx1U^`$~kHzMa=2#{3!k6>C#OjGtE_mWEfC7M!mT zU%yU{r_u`(;!gegM^SIhBMUz+DGM`2+wwN?(G3^3+Pv34q*7*=xeK#VO+_4A3l&Ea zz$hu7!-1(R+Ycn>)vGZkqFIc8;DkP)S4VrOk|1!3gv&@~EC*3BIkn)r@`t=CWm6@) zyapyo)8|1ZG4W-2{PaDYnjh_pt>_~w?0$jG0{#R)wZmA4R?}b9Hc&OA0%qBNT zbLk?`8iHxS6=0I2qAL_lZfcD-sZ+EPvKh@kADavsH5E|@#9tfiK+WQ1TVeNjT?Oc! zV-|KAEP4Sx^?gVnMbs`KC1yy``cJs%afzQgf_E*#JsLUK#lIh0+57}+ZalRMVBQiJ zc{}a>Dp-`1R)Bz@j%If_s?{SW8~>#lgW{dw_GAmTqaA!19->P&RFNg*O2fsC5aF!M z42>GKXE1Z2QFRGwd}aOM?t@%Th8NB3DBsaf5@^*2g9cY&%6zz=g;=Uxvb1!-bnRyy zqyeA$?6`$V44qUjC>x%|JCZa>@$Ep}-_>ikJPERv&KIJ&wijGJR_1Co31f6jsL1@S RLWo{r93CDfAIv<5{~ub%>|Ou> literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem/acvp-valid-decapsulation b/test/fuzz/qbit_corpora/mlkem/acvp-valid-decapsulation new file mode 100644 index 0000000000000000000000000000000000000000..6fe5560bae189a5f8255ea549ea143776a9c6b64 GIT binary patch literal 6400 zcmeI$1wR~&;{b3@$8?wS+=+8cH68!pW{=4#XZeXfauQc1qVEpkgE`}e zmG3iK=dMr~BuxQSBm3_qeVfk(T>M%t1YIjXvfAdYQ9VP=;HyUs{*d->y#%>c9kAKu zujhFhmnGJk=UQb2Opr7gd_NNrz2G`s5s=(rGY#N#Bpr=u>K)T8$CVD*Z7A&Y<-XoD zPo!;ARI~L?oj(&cn@@KBjmnkSLgl)q{LS^fGcTl5`DYEu%h+w!o1Td`u>Rm8WsaF>yU#=UR@Y56SYJWKA8iFU?=$ z=Zp!qpu(YWNnpCIW|=fyx2lY!<>gj~aZek&bdK>;rCkA)+1loNSIO~{kY-q!pSuSO zrh1|)H$uleWa4={jwYsyb3L|sg{mrpG?SF46ZtE2tIlf+aG&!EbEv1g}sd> zwFrz(PZ-+Ri{v6v1i{f)*&LzfK$XL{pBoxJ9xR~u&@#zU9tDC7cZ*HpWZ*Be=C3oerabj~opXwZN#amsA-&)yER=VS~S|03*WB4e8|V9}02 z^X&+cP&h?AsSG`Ts2h+$78LNwdXv!d`fU8;Pm8BrGp4}A%@s|;kP6}J*NP!5l$$;$ zRI$B>uC1~QC(8>Del*Vz^oKaL-ppym^yFCs!_%?jzqS#QNotV#w>mroAMga`?-fYM z#g4k+f7s(ilrgNCz%1|6Zhk{nxFl%y_G!GltM0gp|t3)0N1J0tbn z7GT||6>YmqT%B+W^Z1lrwwqLbX8$CAs>zF?m%)PWv$`c1xUi1dC%syCKgh6E`_J)S zBODYmRcrW6Nba#c8!;qi_Ym)=yS;w~f^qL=rPQT=$o-YZX ztZjB3QXJb5)u=J(Qj|xkVbHY3_ER>WKPKvs*Aj9t%*O}AOvua!s(K{JU%5FNfz)|| zeI-5vc(GxjaDp3;wi)ga$YuA7%msu-TZSvY|Dd{-e-nuTBqnT|TTj(fRfX_3Ff+QPp_% z6x%7rzU)cs`X7URgiR8Cy%RrVBf=`_aD)$7TMC1a4&ah$rQYV$0^nTj~1r=|(NmIwi5 zMTz3RHhlq(s!eu<)d?^LTv>6O1kTcV2OWrMNML$skl^#Nk|u71W6oD>E#+{FLd8ok%iL|A+}x9#~!`5|vqX#Hw=!0AY3wztSF#e1Bu)Tvi*(Ozg$ zuw465)D1}!akipI&8140CjZry#-z;D$wIj}Ao~4i;Clg;f9{XIw4sTqynx5}e@?Ih zs}Ae$#DaTeDQyP5i3&W@`YhkVeVHEZOTj%D`8at=2W5k5*VaPAEFE?7CiCEJ+7BA^#{HvXRQM&imwHQPC&pvlCL73GxGxi&Yw%y+;eoL(f%IR zOe}kS`+bRx1<~TXd*%>*zhe{Gs7z+`!?mlplded!;YUd?3^^jBNFKd2cl)j}0Z4z8 zDHu7Iczw(32FQis`!?Xi-**ex43q_0-(#6^mi=?D0eYko9W-~N{>+g}) zGwvCW-we6myM2g|&@mUv;To$|OT>*=7!ftm`K`|+iXa=1lP^;YfH#t<_b2(t`$3!8 zCb;^A15DK_W8zE>((bIg>DPE1rMuOBRrR{-F3U!kzT$mkHPyQpvG~qzarL%xv^4^Z z?c&^Bv~`K#C>edi@MSr}=feVr2KS6T(bdhh5*)-BSeawYas=jYXs&oZ zvionVYkre%I9i*&lp`ZTR|A1j_L+a>{0~-sfH7j6h(+%@?aYx$;Wk@qsB$ELlgEOt z?Cuz%@FXwI&%UXdsOK*?v2oz+UTvcasoF=ww1imtQ@4 z!V12zj)goZg(A>o*bT>}#ZR_C5M5o#n<7wB>G%?Tw#(G2XaA5;UQESmyh&z)1Dvu{ z@5@r3=oTmCFODs!*XH!lhvqsHzB@;tF=V`pqLCWvg_OqKL#eJTT^pPt`A>k?0p2;= zQz-WyQF}C4p8xL!%LzJt;C%DD3zo>^vj0mMbAadZUr zDhR)}n}*Jj(MxC!=Es-{yey_!m0W2jF#l}^mCpFyvZjb@Sy%e%3G5Zil-$Pq*vR;= z8+44sF1aky0u}f4D=`Utsej`mJ_eXQ$$b|-0I5ci#!R@yY);H>DaN45*|u9M)jWe7 zy>_f3Or>F_#$|qEHloGo2XBJU1SO)MjwGTS+=1-v;i9gw!f@Wzi@&gAmr>8RYAewG zf8+f&*ifHgJfh95r`>){pQvDe*h@CE9MA#lJQ98sjOShB@By^BupKHgvf~oZMwiw& zEzaqz&gpHCJ*2U2XlcZbb4MPt7H9QRXaYC*lngb$0{UP@Ec^Q7$x{7i#1%^x>rgEf z2T@{e7)uY}`ID0Dw+Yq!bK`5N@iEX&&eW$E!k@dh>oR$c^Gj>oLL|zSw0~Wgp(<;; zwaghUlZ<1})Rq?bWhy|pFr^FSQ4qzb{fqFm=D4(I*=JZ6KJ5bIM=7LDZx6B5c-_d9LSoIgT2P~ z39|A2Pk!^z-1lB~`Fs@0LyJ>1&69IufkPSS8!#A1x?xYMMo{k=Hj>a4d5JNmoPF25 z4Z7u?b;BfG^fM9u>xm%pQ%EEFJpU4qWo3LQFy$eGE{Je%XGW-LB465tT&Tkymn&=J z>hN}6urlPe(;DsHDteFEB-=TQbXRZZERY<@ThI8g^L7X5rm|>gJj#{gqT3%&Ypn|e zb%c9gmw4bqoQVO^3|nuXZ~1?Ssa`#aR+_+Jt}T0}%M^Peag3*+=td&OWvXM6@|RH- z+{N3OukTn)W*Cf}#)(tNLIWz)HSn5wQyTv@E%U;2iN?-?6^8i3e00=(=DHn>zbm@P zOS{a#g+_g)80YdPM&|O-LC^qs$n5zs`762??X_Zqw`K1t|U>21mDvX!OrJx z>o^1PlN^l$k^C}Tp}t)lV{x@B6(sMU#^Z;B*8K0zr?Nn}{&?-fL2f_0flVouqBVU# zcn3cq8Sn3GKZg}?Y@j&yx}$`H2FPh2glv;Tu(whQxps=Qc2dK)dMWgk4vZo6fhe6_d-4ViylJzJRU+B;rg!G#&Z|Biw2@MVj(7mUF&A%9 zt3WzJIh+o-hJD)saeu0e?-QVB3lVGq4+pd`ojTiUNJVlt>8zkgZHyO zLXFqH0XGR@HLQdjLPz6S6r;7cm;x>(^HEBzmy*DqnBnP^8VgN{3ptI5#&Nfv2sb%3 zcXoR6!t9!EZqul|hF6^_ylnMw%&JT<#Gm@s$qF~`b|{IJo95a^oM6hFJk+S!wCZd| z+q#&*TitSLhi)2*DvJOBtW<1=h>ceUYN35qC`|p)Qej>xIj=$z8uBEjDmQ^XPF6*v%EfzI-vvs4j(nlWaN5 zx`@j;1&C;Bkwg%V{xRb%XR_p@^3AoWsiZ#6IA!hm z^#OEo2Zdc&QXvhby%iR7%`+3{jk@)UDNi-AF-!`fKN_++*G>N*5&7bRa?)B6`U|ud zW|!$P#nZX?lHu)NPc4e+Y(ZduK)h<;>jvAC&i>fN(F&P$8#>U8J(PO92s9^Sw ziw23OIU2$P_CLpv$-Hpx#<2eycTI$<4zaJBh{gp@j4|+kJ7D|$d};jzJ|F4vc0K1$ zIV$JU8J=pR7OpMjrCiB>fArrU{r5-z{n3Aa^xq%- ze}7b+qv;e=U$$bJxe;gYqx!73Pf&$fsL*Kf0UFl)o1ao9 TaT7I*^+w-KGY6D%?xy|^oqFqc literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem/cctv-base-key b/test/fuzz/qbit_corpora/mlkem/cctv-base-key new file mode 100644 index 0000000000000000000000000000000000000000..f4eb8e847187176031bccc800c98f954df4e8137 GIT binary patch literal 1664 zcmV-`27mcsp4%8zlFtlDhQk0odbjj3ZmZ}eVxQa@R+G>TN{7S&K6|+JGHL+AzuIngeqTd`@meUYTipK;&e!KWJaG$~)UjtQJ^cJHW`V&^6GdL0&9F5Y|VwqnJWO!{c=zU0dCr$xYkDGza)eorZG`sO%$dbq^ z)M_E74)0@HsGO+DxX+YIS98h%NQ(;7TB)mBtK`7VW2p-!MW{W?>N_jbY1uv=vX+vyf6~zgPo5EF- z&ZD-J*eb?V7AA{Wpzx*2`v*MvLkXKzk8Dwz3zqyqliTt(EF*Yw77cu-wkKd}(a08) zAShcJJ>Izucm^0dE6nC1jJ2U=kV;{J$B6_M60d`@z%wSQ_{3<&qWUVr*cdIMlpTLG zCSKJ_3dXqj8OfKXY$nu<76@vccd}i8a9yIm8G{A7F%_i>##JRSM8qZ@8y#|oYOYza z<~N7GDzJAvmC$f=A*+$KA#Ia&2DIa||=hlT)k&g^dbUuYAum0F&+oFqbxYo`;eFgfqWYkQH>pL9(ge9LlnlRwU(*1QQ2K z7!dhay&x)bh*i2A5;_e#$cRU8;oT< zTMH9gnd~IN8=l!ZxE%M8-i=SW^ixQx83HQDT7Py7;j7Tx2xg@7gLGR zv9-y@bmsca*)dr0b|~Sj5tp+G0<$M`X0HbVpjToN&)IN$M?y()zq@q-9J(>GrHr6r zqSXR?i!#1q=uLjAsajt|`!0V!~xW(Z)E4x7Nu7eA3J4`KmX86mlyaEU0|q zS)I;OPNu}TJkudF!)>@0E<7}7AKH$h$I0jnhto=PkyEs%V!#pOHe;W`UNi(4DiKQ?u0xoh!{sK5Gs+)di+}bmWQSn zWoKmw-Vjir{I|QPEeAjbI=}(zGj4(VT42Pd6gsN<`8r}Dj&=*3-q^%SYfb4neVk`| zt=W_>G;dpgBCL!mRK++}B1_uf6k#@FYRn5r6}^y)YsDqN@MkguOMybcWRTPu{Dp@; z>sIo>0`U@Svj<;Wv22-SE`arWP4GvWR!&gIh%-o8Hl{o{(aVW4$ps1ude=%+@}h@` zy5NGzM%tc@b$b37fVI_hcNb}a15<#Qv3EFaHrKzoY@w#MjcU5QT17GHGs^`sdAJ;e z#?)J}(Mq8rSo}bJ=BcIKW~t$8Yu355y|h|N2G?UnTTP+>0X}FPL+X5#KCy=F@|I-9 K-9q~hBoARom;=uM literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem/coefficient-3328 b/test/fuzz/qbit_corpora/mlkem/coefficient-3328 new file mode 100644 index 0000000000000000000000000000000000000000..21a0a6fa803542c309e985ea41a5bc211ff77070 GIT binary patch literal 1664 zcmV-`27mcsp4%8zlFtlDhQk0odbjj3ZmZ}eVxQa@R+G>TN{7S&K6|+JGHL+AzuIngeqTd`@meUYTipK;&e!KWJaG$~)UjtQJ^cJHW`V&^6GdL0&9F5Y|VwqnJWO!{c=zU0dCr$xYkDGza)eorZG`sO%$dbq^ z)M_E74)0@HsGO+DxX+YIS98h%NQ(;7TB)mBtK`7VW2p-!MW{W?>N_jbY1uv=vX+vyf6~zgPo5EF- z&ZD-J*eb?V7AA{Wpzx*2`v*MvLkXKzk8Dwz3zqyqliTt(EF*Yw77cu-wkKd}(a08) zAShcJJ>Izucm^0dE6nC1jJ2U=kV;{J$B6_M60d`@z%wSQ_{3<&qWUVr*cdIMlpTLG zCSKJ_3dXqj8OfKXY$nu<76@vccd}i8a9yIm8G{A7F%_i>##JRSM8qZ@8y#|oYOYza z<~N7GDzJAvmC$f=A*+$KA#Ia&2DIa||=hlT)k&g^dbUuYAum0F&+oFqbxYo`;eFgfqWYkQH>pL9(ge9LlnlRwU(*1QQ2K z7!dhay&x)bh*i2A5;_e#$cRU8;oT< zTMH9gnd~IN8=l!ZxE%M8-i=SW^ixQx83HQDT7Py7;j7Tx2xg@7gLGR zv9-y@bmsca*)dr0b|~Sj5tp+G0<$M`X0HbVpjToN&)IN$M?y()zq@q-9J(>GrHr6r zqSXR?i!#1q=uLjAsajt|`!0V!~xW(Z)E4x7Nu7eA3J4`KmX86mlyaEU0|q zS)I;OPNu}TJkudF!)>@0E<7}7AKH$h$I0jnhto=PkyEs%V!#pOHe;W`UNi(4DiKQ?u0xoh!{sK5Gs+)di+}bmWQSn zWoKmw-Vjir{I|QPEeAjbI=}(zGj4(VT42Pd6gsN<`8r}Dj&=*3-q^%SYfb4neVk`| zt=W_>G;dpgBCL!mRK++}B1_uf6k#@FYRn5r6}^y)YsDqN@MkguOMybcWRTPu{Dp@; z>sIo>0`U@Svj<;Wv22-SE`arWP4GvWR!&gIh%-o8Hl{o{(aVW4$ps1ude=%+@}h@` zy5NGzM%tc@b$b37fVI_hcNb}a15<#Qv3EFaHrKzoY@w#MjcU5QT17GHGs^`sdAJ;e z#?)J}(Mq8rSo}bJ=BcIKW~t$8Yu355y|h|N2G?UnTTP+>0X}FPL+X5#KCy=F@|I-9 K-9q~hBoAR0MFX7x literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem/coefficient-3329 b/test/fuzz/qbit_corpora/mlkem/coefficient-3329 new file mode 100644 index 0000000000000000000000000000000000000000..037364c402af2300a1fc628a5b5bf506490e915b GIT binary patch literal 1664 zcmV-`27mcsp4%8zlFtlDhQk0odbjj3ZmZ}eVxQa@R+G>TN{7S&K6|+JGHL+AzuIngeqTd`@meUYTipK;&e!KWJaG$~)UjtQJ^cJHW`V&^6GdL0&9F5Y|VwqnJWO!{c=zU0dCr$xYkDGza)eorZG`sO%$dbq^ z)M_E74)0@HsGO+DxX+YIS98h%NQ(;7TB)mBtK`7VW2p-!MW{W?>N_jbY1uv=vX+vyf6~zgPo5EF- z&ZD-J*eb?V7AA{Wpzx*2`v*MvLkXKzk8Dwz3zqyqliTt(EF*Yw77cu-wkKd}(a08) zAShcJJ>Izucm^0dE6nC1jJ2U=kV;{J$B6_M60d`@z%wSQ_{3<&qWUVr*cdIMlpTLG zCSKJ_3dXqj8OfKXY$nu<76@vccd}i8a9yIm8G{A7F%_i>##JRSM8qZ@8y#|oYOYza z<~N7GDzJAvmC$f=A*+$KA#Ia&2DIa||=hlT)k&g^dbUuYAum0F&+oFqbxYo`;eFgfqWYkQH>pL9(ge9LlnlRwU(*1QQ2K z7!dhay&x)bh*i2A5;_e#$cRU8;oT< zTMH9gnd~IN8=l!ZxE%M8-i=SW^ixQx83HQDT7Py7;j7Tx2xg@7gLGR zv9-y@bmsca*)dr0b|~Sj5tp+G0<$M`X0HbVpjToN&)IN$M?y()zq@q-9J(>GrHr6r zqSXR?i!#1q=uLjAsajt|`!0V!~xW(Z)E4x7Nu7eA3J4`KmX86mlyaEU0|q zS)I;OPNu}TJkudF!)>@0E<7}7AKH$h$I0jnhto=PkyEs%V!#pOHe;W`UNi(4DiKQ?u0xoh!{sK5Gs+)di+}bmWQSn zWoKmw-Vjir{I|QPEeAjbI=}(zGj4(VT42Pd6gsN<`8r}Dj&=*3-q^%SYfb4neVk`| zt=W_>G;dpgBCL!mRK++}B1_uf6k#@FYRn5r6}^y)YsDqN@MkguOMybcWRTPu{Dp@; z>sIo>0`U@Svj<;Wv22-SE`arWP4GvWR!&gIh%-o8Hl{o{(aVW4$ps1ude=%+@}h@` zy5NGzM%tc@b$b37fVI_hcNb}a15<#Qv3EFaHrKzoY@w#MjcU5QT17GHGs^`sdAJ;e z#?)J}(Mq8rSo}bJ=BcIKW~t$8Yu355y|h|N2G?UnTTP+>0X}FPL+X5#KCy=F@|I-9 K-9q~hBoAR6Wdoi7 literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem/honest-ciphertext-bit-flip b/test/fuzz/qbit_corpora/mlkem/honest-ciphertext-bit-flip new file mode 100644 index 0000000000000000000000000000000000000000..e90fa0ac39fb8f88ac32e96c5e5c7600b649e4b6 GIT binary patch literal 6403 zcmYeCcUvNK(s>@ww!;jzRolN9=B<9Anl%58WZ2{jyk6}`811Ted^O5n^HME&!Ck5F zDHr*?JB~8h*X;aeT(I_)dP?EC*BYsd?#o0@yDZ?_b)3bqZubwTN{7S&K6|+JGHL+AzuIngeqTd`@meUYTipK;&e!KWJaTN{7S&K6|+JGHL+AzuIngeqTd`@meUYTipK;&e!KWJaG$~)UjtQJ^cJHW`V&^6GdL0&9F5Y|VwqnJWO!{c=zU0dCr$xYkDGza)eorZG`sO%$dbq^ z)M_E74)0@HsGO+DxX+YIS98h%NQ(;7TB)mBtK`7VW2p-!MW{W?>N_jbY1uv=vX+vyf6~zgPo5EF- z&ZD-J*eb?V7AA{Wpzx*2`v*MvLkXKzk8Dwz3zqyqliTt(EF*Yw77cu-wkKd}(a08) zAShcJJ>Izucm^0dE6nC1jJ2U=kV;{J$B6_M60d`@z%wSQ_{3<&qWUVr*cdIMlpTLG zCSKJ_3dXqj8OfKXY$nu<76@vccd}i8a9yIm8G{A7F%_i>##JRSM8qZ@8y#|oYOYza z<~N7GDzJAvmC$f=A*+$KA#Ia&2DIa||=hlT)k&g^dbUuYAum0F&+oFqbxYo`;eFgfqWYkQH>pL9(ge9LlnlRwU(*1QQ2K z7!dhay&x)bh*i2A5;_e#$cRU8;oT< zTMH9gnd~IN8=l!ZxE%M8-i=SW^ixQx83HQDT7Py7;j7Tx2xg@7gLGR zv9-y@bmsca*)dr0b|~Sj5tp+G0<$M`X0HbVpjToN&)IN$M?y()zq@q-9J(>GrHr6r zqSXR?i!#1q=uLjAsajt|`!0V!~xW(Z)E4x7Nu7eA3J4`KmX86mlyaEU0|q zS)I;OPNu}TJkudF!)>@0E<7}7AKH$h$I0jnhto=PkyEs%V!#pOHe;W`UNi(4DiKQ?u0xoh!{sK5Gs+)di+}bmWQSn zWoKmw-Vjir{I|QPEeAjbI=}(zGj4(VT42Pd6gsN<`8r}Dj&=*3-q^%SYfb4neVk`| zt=W_>G;dpgBCL!mRK++}B1_uf6k#@FYRn5r6}^y)YsDqN@MkguOMybcWRTPu{Dp@; z>sIo>0`U@Svj<;Wv22-SE`arWP4GvWR!&gIh%-o8Hl{o{(aVW4$ps1ude=%+@}h@` zy5NGzM%tc@b$b37fVI_hcNb}a15<#Qv3EFaHrKzoY@w#MjcU5QT17GHGs^`sdAJ;e z#?)J}(Mq8rSo}bJ=BcIKW~t$8Yu355z4ZS|2G?UnTTP+>0X}FPL+X5#KCy=F@|I-9 K-9q~hBoAR`T?DiM literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem_backend_diff/acvp-modified-ciphertext b/test/fuzz/qbit_corpora/mlkem_backend_diff/acvp-modified-ciphertext new file mode 100644 index 0000000000000000000000000000000000000000..1846c14ca16955f2c6d493906518e26e316ecd1a GIT binary patch literal 6400 zcmeI$cOw*z!vJuz&gSgB_sC^)#o1?%GtSC7n>c%R_Bh)aXYb6y*`wmjvk4VR%1nw- zq53^9;CTt(H}U!7<5IcFW|hB44NbulJ&5YN00p!j3cFTqv0E1`(HNwT6F-dZzr6pr z?MTF}`aOpYW|`J7ZGuESX5dOLu>DxnJ?PmNF+}YLE_l%@y>Z4A86#L91UovGiKR3NwT6kNVIi6o@qmbKamuIw0G{}HeaiR9xhz-8&85%Zo&a=Q_ z0;J*IBaOB`(IO+7Nf$uxZTzUvEjYNk&O1J<6v_^Aj=Gt)he61+f7gr5K-+RJHEW%N$J)7E*(k>{Yb*CcN~FBevo~ zQzV!jroiXb1=CSQ2=&oUv7Am>LOnAPj7J{nz10W^w#F z!-9pC7<_7xP}XrKyi4F5o%8&5Q7L-t3kWCbM}jL}vD#@GGe_uTQ7sU~*xD)rBRi%7 zT&o|lhxenb1dHxdgptwYS#2KFKa+UI0g@!99+$<@&sS#;?dR^)kgOO^0J`dUU08W; z!KfY5Og>$VVz?gy@tCc&SXkSleD2d**3%Ox@Mq*FExU9gA~GSb->MII>Ge=6kFQ%T zTjXBU)X{@g&71su;#kqW4)Hoy^*lHJgyhGclcFX<^!zuxMCn1hhfhiP2yM+Wql!S9 zovLflEBRNoT3__T1X4P?__J4VMpn$o-j^X96*L*3VU~zHQR>Sog|k)#!d>MmPcy?y z@F#@87;K~OIZHhKWUQ{Ssm(OZ=Mf>yaPk(MQdr}S|(Yz6u+xzMce9e)>E3twRRbqCrX zaQRE+Pz%qgu0>iB1IRwo?K@qk`X<;Y~n5#b#qDk#{e?ijTjqDmbqENd*zVVP?0zg=dG zD7ISzvmiZi6PFLvPWRb4l77wlde{*t#?`o_cEaO&;M$W8Eyr&U1V}KjdpsE6@R7|H zbgmX+oHJ$nN>g<~6nUcR@A;dw{nXguxjh8))|((8taLh0-}(=HFkhSBb#?#Bjw|?Z zLA!S&offE)+Vm>tTs>F^bRTo$`!HKs<~h!yFh!}cng^ir9>{h!+{_YzQC2&+GH<1( z4Iz}Vf2gt9OtIb@SB(AY`mp4R0c$q$SucjQ^ygYiznS2LuNEKcdfOcs>zRXnOL zH*+le*!>W|KFdt;7bYw7kn{__>;pB?Q!51J4;RXxiKhSlUhdca?Dh9@GqL9kJJ%Pn z=u6&0nL`a1Od%=JEq71TX1r+G87l{d&BgxM$!8OvN(4pVy2_R&*C~8r@#GcND!b2m zjUBfcU@t!`KHH!5n}27fGHHVUsw%aXGybz)AiZ!_h>Ek0>qSG^mHtC|YGszecLxwt z+7NB?f4tRs(hihD$j-i3tsgo=HI2J7dVjt!?hlH{xP&^dXp=h9s$bvRTx0?Z5OEvF zfHy;ejrIFvC2)JPWAh-E|CnM$vZ6_hIci=$^AUk=3xBe3d_cJ zoFAv)*r%8_&>l9915Yn)I)-n+4T6i2Fy6&~HqTh@J+k`CzopmcU)x$D_~7tV*4FLO z-Vcyd^i5M-)`}Q~tPq9H5pqs^t|r%6s{$0mw(y9AhK~M% zFE{-31TW^U5D zm(BPG*RILDii$&Hb5`2hNKITS@0#L@8~s0(XB3rm4X!tg9gv(qe^gRzgi8}7bg~uG zx}~A~tY-n@ym<;9i|cb8#jnO4WMC|PtATL^e&vtd`t^9Eq@-J?53_h6QD{g}l{{eh`uevRsQ}EB?=$!7*5^;WRccaS`G0;5QjLLUHDG>f9 z;y2q>o=ARZsYX{jxPqAHk4s&F6{5U?v2u$*4_ppVYNHNO=vxX{XK?o$Y2sKn`d%b& zT9#{>a!?Dh@HcWi@(@vrNx;SouJVlm-EHr01n~I*uN&{HEJM#05PB27kW+_P zqI#WDL)+eq{*Eg!ATN91@lfH#R`HYJp%dIPwRFbrJepmjDCg$_xyjSh5!(2DJ0+=k z#)lMXo?j284<@p(M}Te?TV0(XT1jHnq^eN|J}f9N^JT{^-Z7Qn2C`va%sM2oPB;_n ztryFlglT2^hgwL)QK2r&hO}F=ZhMr3i{X@g?#9bsL;lZ%AL-q!CA^<~MO6@hu5`q( zwJBO>J!|_b*ASJe+3~VQ>W_iFH?2!|hn0o1fN^+j#t-8cJUczmk9U5?uoP{6HwD#h z)qC>*#oaLEu<`S-XD=4l++!pHJ-;5$WbL|)rcqVO)H~{ z*+e_IBuKljUl=mqMu=myE9|ltlO@1qcR=^Eq>pgf;6{OV0sxwH? zgFB?(jFxVw1=-=SRB=UlGka5>OA)jtvLaioLYN_wH^B@mfFc_fKnFs=Pztcx2TP2h z%6K#179QHfoI%&DjGRGUkY#;5nrZs<0Im|3^)XGD!!>14YX97J`B!sg#JKxq*zF-D zw%k4Rgmf4l$n?%R>3k-xKeg~Yb5m67NhhyoarDPu2&7~(UWr>re*V3R?|P>|$%R-{ zOiL3=$Ox==R>|`Pz-S<}uTa#nio4xsJPZxXkjR!21tQh~9pwBDpfW*mq1B8)J#g zEg++Ne1rOPH8SU3#j7&BGRY{wX^}?y)QM%t9(^ZnFdPw_@!dVPhQ$BalXdnpgKt;X zzq~OaAutD0ff#`&PHNBPzzADr3ELz8BpPKC<<63%kNvFtiH#X+fs4JiWJlxkuPhAG zdY*X>_;9f%h&&-PpsapV%^)Y$ziPteHwvw9tQ#oeuLHy<3jC1^_|Wh98R|6iR?={a zE}E5+MhthN=;T-}6=$ZTCJPtJ(5j@2qEbb8xkWfUg&sgk%a{vK#ri%Z=I}~BC#gns z3Q4iPiwv2NliWZWg}a-i!K!+YgX!+Y*vOuzxm@3E1-l25KjYRwV=R1LRiqxeYhs&o z-R}0tlYF|ROHB`^VcjD4`i#n*o~gGD{O+Gvz~UIlNA4BfU@tKE~ty~ zw%ApXOufL+rCleP`~-?Fx6r0aPX0N%mu@6- zm<$LNTEE{cucEGX~jNl)~lthbO}9<-hqEL;S#RVOqZj@73xf zDi5M+iA$dNIE_5bk`0A7g%qi=i?X8?gv>MMJ=o4y?5+(N#&Jb63>@q#Syp!|ulQh1 zK>U+K?r!Txo}}o?cTt9Jy&A%z#7R)r!?|cRo^7oMq}H{sj^m_%j444AK#fk>si}q= zNvU5(7f{r(oYs1N0+i^*L2Gm^RX7-{S|Te2FqEvdC?+d`DA=+5qsY{x6yQsYZk#UD zR`87{4+|q@5!Ti1XsBm})nuH-Vv`{n)NM4=vh77I7iSR~%?r}rxS_+v3g-?^qPDo{ z&K>lFG#`|qf$6__>A2Fr&;=6&iW@pFEZg4Z^W$$73m1ytvXJ&j@-pnd3eYaiW zjs*V9#~uhh>~tU{Yke%ApckXvc#U@KE502R% V!J1w?{nlQ-!5bJUr8m3N{{#Do>a+j= literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem_backend_diff/acvp-modified-key-hash b/test/fuzz/qbit_corpora/mlkem_backend_diff/acvp-modified-key-hash new file mode 100644 index 0000000000000000000000000000000000000000..863f876e39f5d730718019864cdfc7d1c258f268 GIT binary patch literal 6400 zcmeIyg<2Dg!v$~{-8GQzW=uwdGz_GqdmuF$X=LQ6!RSuO(V=v!bPSLbDFs1VN+}iP z`(MC!3Gcat-*e7!DBWN*%bFwCMvdSohIU^;UNj!^IhJiQnP<;a=p>BdDuwl2i~BSk z@jI1&X12&#pwvwq11N|0-bna19}779y*?3yS8lOd!tIe$Er(qzGJ+KqEjGBxGw#B!f$ z87~Lw8A5VopB&s6HIh>XVvCu!Dl9fo=6j>k5vPu~9qNzOuloQ$a|GBBmraVJt=LmT ze;KiMneT>TkeE}UP)tV3aN4o+Au-c5J;^^)F|ZQe$w=`QIZmw^ob;yy>20jR!@ukO z%G)k)H>YPdEWtmX@*8)@Zz7Gfbw3!WqQV-8&Jw5GD4FO=dk2gJ|5{6?;hqWmg|xR7 z&yTN=ctng4l$6WuyzbJs{X%2h?Z`e1B7pxd)5a43D3i{o~ry zs`0LDEZ|W*JQK5YdcOA)Vi)$OE;4mdkVK4^1bXB@BQ#TyVXs~S2?u&V1f{peuE5n! zaele-V85d*tNErq z%us0!5PYe;odR3qpF*7$_J0*a7tn?gDYdliqKM^5i7;328e*gyl3fir*F8(s3bONwWC z>VU8uF3ATG!;Or%6*r5~EyuR;%#xBreWOQ|_XxH21dfl=i%!%Ja<55Bsj40S(6y+h z|NSe6vfwL<5r(EoCw7Qxb3M9zA;gg>_p3Im%mbT9cvX8#+E0Gm^Z| z+Q_n{fwM;pwP=?znxJfaQxO^tE)~OeUwrNtU%C|wrPfKmH(subA3wYjC!L2pymfU7 zAyJ@;SCbBSf&(^l^46%{D4h+k#PVX!gJsY0IyzSmO3od)IliEU5>OQPr@o?k6(&wk z?`I2tTMpr|W$S%`W}CzXww+A!CNGa7L`0lgJ?Mpeae@N#$|SMV$j$s1m8l>ryrbJ( zW~_a(+v6J2j5JR?OBE{`{X~avQ7A?au z)g5M&{mgks=U%hIBRbijjaHQ;2+zP2RduKma$V94-&;6Y^5r`FA_C1&UFO4ggO-@U z>I2;rmMwK1J1or4DL|A|!neJeRuR)E+AIW7ior45>6~3A`TS7{ymhigo!|bA^&d@O zNN@dgJu=GMD zD>TktZ|j&>^e*xw4T>{NPzRL|V{bj%BNzTB;W35xp_A)7GmJ^Y1UIC>2I>?!HTxV2mSU%nM-x3m7jY2g1p;U~J| zQ31y{PXReNaT^*wU~Z5MT}l0N%GysRZ+N_*lJK*4Z?$orXh4=Pl{RujgKlf|iha9N z`zzsXI78mXg#n~;v&x;_Z}si>s~`Ss1z#t7_0k!O!@tvZ6|g$LBOuwY9KT7>un5p<4_p7Eb8=gatfdR1P9`=Ky&p&($@T9yx zG;$l*jr0!;++)eENo}M3y*2wBVUPNc+Ep_^_@J_t@o%-ptL%daK?BJ z5n7J^0xsXZMyReyb5e;cpbBR}KFMDviKWa(dxU}@FZUy7UjWAUch7|y#!qJy$Irk; zJdJ!g;H!;agX2qSnYW{D#eK{~XY>OBvBgouy(i1l(u-=53Nu;!K1?UIT>()umT}1! zZyssVRYANF5^#?8JU(qNer!I$x8w=}{1-{w-mTH5^5py-W>5JUH3^Z|+*6OGv@d!> zftj{cmoU$26v)hh8^Wx=LDb2cqO_KK)r6dIJk-~zC7RKcD*erbSCId<7E(4j?1-?^8ji-r??v)7#T}aVSz#crUhHSs zNzae8bg=!AQ`il`_u0uJ2{%sb_wQxl9G4k`EM>nRRWx-M{v@b&!=v3~@Cg@RD zy6mi71hmXlu=IG`m|^IN%-?nx?56HY8)#aUQV5?RT?(Z<7q6P774Ir~^P#BPku3Mg z9(ck^-|ZtJ_=H7v#x*29j#N{@e`M~FhV?g2;q%`*k=p`Bk_;f-kvfqkJe0oNl#2H) zcpt#Hsz6D(Nf=6>wI-jA5sAab4;U@jJoG)f{iFhvCrm_0i*6ITm(NAslH02lZYp#> z1}K?dYwYgA?(^kpbh5Jso7u7=94|`pRMQo+UT}x=Z-ZKQhRD#n0TmzVmtHacu;K#4 zfVHkk^1&R8-x3{OX)adMX9!U3(KL0B^h1A*7w${ttrN!tS^@J*O-_G+0BF)1>y zReJnr=3$XU1rBMKy%N%!Ja2mh8}Xo|U92GeLg` z<(K;&I~ap{uHh}l;yd@$I@_fc=IFF!QAN_IKbN^j3pG3k@^^mG9c$u^e8mwWvcLnb z<-%MF#%Jc2B-1p6M_&mA6T6leZAVBOa)mS}S$uk)7`_ZhhzgH82ZDHvRd+Jj{AZGU zo-#A}GnXcW?K-)4+yM6q+>x;v_6|h)_GVi$eAHv=(3I)5OXBE7l1EN?r<*-A$x7DT z_2_CZ*GvJfwhUBK(9J3*QkZJ`g7l2eNN%+Pl>q%aEJ}D`@pOqN!6@NDC{0v`mvl|( zwhl9`$q(l&*}p6#7K|l@YrGKF6U$BBZ4fnJ5_OeNE7bOlGgalT_WXA9-t!Z<0R7Cy z-NS=vrgCVatUg#)PujOf6DB;pp*hr<$YJwa66yLp_S(dpa9U1IvStH=?=C0|5$Nm= z`{bK#NW$>|Rk*u#^1ah)tn7}4kSKraDqK=~$4yCWYWdZ$CAx=Wb_urCWdD4?Am3tG zKXgp(h_dR4&&EdRHV)A!{f1>FbDp44h1Pu_J9nz!#abn9q%33WUBs_AANGohw65s# zJ%?XE5nMW6-_#{%$3~aY>K>McZT{LN;taObGs*Cmc`);SGd$Jkflrgb#Qw6@%~ZM} z6F1-Y%cyiZuK{f`$BBH^wt&WZ-#}4c+6FH8aDJUeqn?1Ox_#&D#ZC#Na~280^Z<)- z)_HDs=xU{L!$nFGHHW}Dwyh!-hY+{bm3McbJ^(I5P)dvEP0nvddmL!;y1M03z$M3O@W+qv9BQQT2~luhSV?5L$^w6=O4RO= zCY!L7$nmlYJ1CV@T+HWw;xl6z#h4l$l<-wfc|cR$5v zjN)!ch|{E1ElbuKP1GF`ahX;Qxkav>gu0=tW2dA4>#s2rOx0R(8b5OBCagQ2DHsr1 z(f|Xn?Q#-hJB7vbthUpBmv@i7s>4dVWV~ckI0>~ib{mjVfq$dDXo?_QDl}nLS(qhG zsHy~v7}TsmMaDf~R%0g;4_0c$2hVFB)_0q|l;oS@T)g5ZZ!2Nz_DCp}7JM>SXO)h~ z2CTz?V@-HGq!MYnliDGL^`LjoHj1U0+GQFX4aPL9evSu@j&BjBBxc1zI`}3Rf|v=J zJmH$&ypm*Sz2w9k_dWQU)rDS_lQZ5-x8J(5jcCE4)+PjkLzZL$3Lcv0_=&ON5_ylH z3sZ|N;tIpb2QUd|X;JAaR@KWz8hz-xiydP~z2qkHs*N2S?tgE=kKE%=`BTM?lKLk_z9Y5L^$%2k{y3&T^N9% zr3>#<{>v}kL)=XEJ3Rta)Ul%moup$PV+f98B8Y7^4&{3r>;5+|O2|0HK``=cEy8R1 zJ%V|%p0v06YnjDUVf1eOmsdD5{RMH8RN)hDWN?G87_2n|KBQt4R0Vw#s&7A2ds7)2 zmifxtw?zzKi;`qgNBifku#m?AYx1U^`$~kHzMa=2#{3!k6>C#OjGtE_mWEfC7M!mT zU%yU{r_u`(;!gegM^SIhBMUz+DGM`2+wwN?(G3^3+Pv34q*7*=xeK#VO+_4A3l&Ea zz$hu7!-1(R+Ycn>)vGZkqFIc8;DkP)S4VrOk|1!3gv&@~EC*3BIkn)r@`t=CWm6@) zyapyo)8|1ZG4W-2{PaDYnjh_pt>_~w?0$jG0{#R)wZmA4R?}b9Hc&OA0%qBNT zbLk?`8iHxS6=0I2qAL_lZfcD-sZ+EPvKh@kADavsH5E|@#9tfiK+WQ1TVeNjT?Oc! zV-|KAEP4Sx^?gVnMbs`KC1yy``cJs%afzQgf_E*#JsLUK#lIh0+57}+ZalRMVBQiJ zc{}a>Dp-`1R)Bz@j%If_s?{SW8~>#lgW{dw_GAmTqaA!19->P&RFNg*O2fsC5aF!M z42>GKXE1Z2QFRGwd}aOM?t@%Th8NB3DBsaf5@^*2g9cY&%6zz=g;=Uxvb1!-bnRyy zqyeA$?6`$V44qUjC>x%|JCZa>@$Ep}-_>ikJPERv&KIJ&wijGJR_1Co31f6jsL1@S RLWo{r93CDfAIv<5{~ub%>|Ou> literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem_backend_diff/acvp-valid-decapsulation b/test/fuzz/qbit_corpora/mlkem_backend_diff/acvp-valid-decapsulation new file mode 100644 index 0000000000000000000000000000000000000000..6fe5560bae189a5f8255ea549ea143776a9c6b64 GIT binary patch literal 6400 zcmeI$1wR~&;{b3@$8?wS+=+8cH68!pW{=4#XZeXfauQc1qVEpkgE`}e zmG3iK=dMr~BuxQSBm3_qeVfk(T>M%t1YIjXvfAdYQ9VP=;HyUs{*d->y#%>c9kAKu zujhFhmnGJk=UQb2Opr7gd_NNrz2G`s5s=(rGY#N#Bpr=u>K)T8$CVD*Z7A&Y<-XoD zPo!;ARI~L?oj(&cn@@KBjmnkSLgl)q{LS^fGcTl5`DYEu%h+w!o1Td`u>Rm8WsaF>yU#=UR@Y56SYJWKA8iFU?=$ z=Zp!qpu(YWNnpCIW|=fyx2lY!<>gj~aZek&bdK>;rCkA)+1loNSIO~{kY-q!pSuSO zrh1|)H$uleWa4={jwYsyb3L|sg{mrpG?SF46ZtE2tIlf+aG&!EbEv1g}sd> zwFrz(PZ-+Ri{v6v1i{f)*&LzfK$XL{pBoxJ9xR~u&@#zU9tDC7cZ*HpWZ*Be=C3oerabj~opXwZN#amsA-&)yER=VS~S|03*WB4e8|V9}02 z^X&+cP&h?AsSG`Ts2h+$78LNwdXv!d`fU8;Pm8BrGp4}A%@s|;kP6}J*NP!5l$$;$ zRI$B>uC1~QC(8>Del*Vz^oKaL-ppym^yFCs!_%?jzqS#QNotV#w>mroAMga`?-fYM z#g4k+f7s(ilrgNCz%1|6Zhk{nxFl%y_G!GltM0gp|t3)0N1J0tbn z7GT||6>YmqT%B+W^Z1lrwwqLbX8$CAs>zF?m%)PWv$`c1xUi1dC%syCKgh6E`_J)S zBODYmRcrW6Nba#c8!;qi_Ym)=yS;w~f^qL=rPQT=$o-YZX ztZjB3QXJb5)u=J(Qj|xkVbHY3_ER>WKPKvs*Aj9t%*O}AOvua!s(K{JU%5FNfz)|| zeI-5vc(GxjaDp3;wi)ga$YuA7%msu-TZSvY|Dd{-e-nuTBqnT|TTj(fRfX_3Ff+QPp_% z6x%7rzU)cs`X7URgiR8Cy%RrVBf=`_aD)$7TMC1a4&ah$rQYV$0^nTj~1r=|(NmIwi5 zMTz3RHhlq(s!eu<)d?^LTv>6O1kTcV2OWrMNML$skl^#Nk|u71W6oD>E#+{FLd8ok%iL|A+}x9#~!`5|vqX#Hw=!0AY3wztSF#e1Bu)Tvi*(Ozg$ zuw465)D1}!akipI&8140CjZry#-z;D$wIj}Ao~4i;Clg;f9{XIw4sTqynx5}e@?Ih zs}Ae$#DaTeDQyP5i3&W@`YhkVeVHEZOTj%D`8at=2W5k5*VaPAEFE?7CiCEJ+7BA^#{HvXRQM&imwHQPC&pvlCL73GxGxi&Yw%y+;eoL(f%IR zOe}kS`+bRx1<~TXd*%>*zhe{Gs7z+`!?mlplded!;YUd?3^^jBNFKd2cl)j}0Z4z8 zDHu7Iczw(32FQis`!?Xi-**ex43q_0-(#6^mi=?D0eYko9W-~N{>+g}) zGwvCW-we6myM2g|&@mUv;To$|OT>*=7!ftm`K`|+iXa=1lP^;YfH#t<_b2(t`$3!8 zCb;^A15DK_W8zE>((bIg>DPE1rMuOBRrR{-F3U!kzT$mkHPyQpvG~qzarL%xv^4^Z z?c&^Bv~`K#C>edi@MSr}=feVr2KS6T(bdhh5*)-BSeawYas=jYXs&oZ zvionVYkre%I9i*&lp`ZTR|A1j_L+a>{0~-sfH7j6h(+%@?aYx$;Wk@qsB$ELlgEOt z?Cuz%@FXwI&%UXdsOK*?v2oz+UTvcasoF=ww1imtQ@4 z!V12zj)goZg(A>o*bT>}#ZR_C5M5o#n<7wB>G%?Tw#(G2XaA5;UQESmyh&z)1Dvu{ z@5@r3=oTmCFODs!*XH!lhvqsHzB@;tF=V`pqLCWvg_OqKL#eJTT^pPt`A>k?0p2;= zQz-WyQF}C4p8xL!%LzJt;C%DD3zo>^vj0mMbAadZUr zDhR)}n}*Jj(MxC!=Es-{yey_!m0W2jF#l}^mCpFyvZjb@Sy%e%3G5Zil-$Pq*vR;= z8+44sF1aky0u}f4D=`Utsej`mJ_eXQ$$b|-0I5ci#!R@yY);H>DaN45*|u9M)jWe7 zy>_f3Or>F_#$|qEHloGo2XBJU1SO)MjwGTS+=1-v;i9gw!f@Wzi@&gAmr>8RYAewG zf8+f&*ifHgJfh95r`>){pQvDe*h@CE9MA#lJQ98sjOShB@By^BupKHgvf~oZMwiw& zEzaqz&gpHCJ*2U2XlcZbb4MPt7H9QRXaYC*lngb$0{UP@Ec^Q7$x{7i#1%^x>rgEf z2T@{e7)uY}`ID0Dw+Yq!bK`5N@iEX&&eW$E!k@dh>oR$c^Gj>oLL|zSw0~Wgp(<;; zwaghUlZ<1})Rq?bWhy|pFr^FSQ4qzb{fqFm=D4(I*=JZ6KJ5bIM=7LDZx6B5c-_d9LSoIgT2P~ z39|A2Pk!^z-1lB~`Fs@0LyJ>1&69IufkPSS8!#A1x?xYMMo{k=Hj>a4d5JNmoPF25 z4Z7u?b;BfG^fM9u>xm%pQ%EEFJpU4qWo3LQFy$eGE{Je%XGW-LB465tT&Tkymn&=J z>hN}6urlPe(;DsHDteFEB-=TQbXRZZERY<@ThI8g^L7X5rm|>gJj#{gqT3%&Ypn|e zb%c9gmw4bqoQVO^3|nuXZ~1?Ssa`#aR+_+Jt}T0}%M^Peag3*+=td&OWvXM6@|RH- z+{N3OukTn)W*Cf}#)(tNLIWz)HSn5wQyTv@E%U;2iN?-?6^8i3e00=(=DHn>zbm@P zOS{a#g+_g)80YdPM&|O-LC^qs$n5zs`762??X_Zqw`K1t|U>21mDvX!OrJx z>o^1PlN^l$k^C}Tp}t)lV{x@B6(sMU#^Z;B*8K0zr?Nn}{&?-fL2f_0flVouqBVU# zcn3cq8Sn3GKZg}?Y@j&yx}$`H2FPh2glv;Tu(whQxps=Qc2dK)dMWgk4vZo6fhe6_d-4ViylJzJRU+B;rg!G#&Z|Biw2@MVj(7mUF&A%9 zt3WzJIh+o-hJD)saeu0e?-QVB3lVGq4+pd`ojTiUNJVlt>8zkgZHyO zLXFqH0XGR@HLQdjLPz6S6r;7cm;x>(^HEBzmy*DqnBnP^8VgN{3ptI5#&Nfv2sb%3 zcXoR6!t9!EZqul|hF6^_ylnMw%&JT<#Gm@s$qF~`b|{IJo95a^oM6hFJk+S!wCZd| z+q#&*TitSLhi)2*DvJOBtW<1=h>ceUYN35qC`|p)Qej>xIj=$z8uBEjDmQ^XPF6*v%EfzI-vvs4j(nlWaN5 zx`@j;1&C;Bkwg%V{xRb%XR_p@^3AoWsiZ#6IA!hm z^#OEo2Zdc&QXvhby%iR7%`+3{jk@)UDNi-AF-!`fKN_++*G>N*5&7bRa?)B6`U|ud zW|!$P#nZX?lHu)NPc4e+Y(ZduK)h<;>jvAC&i>fN(F&P$8#>U8J(PO92s9^Sw ziw23OIU2$P_CLpv$-Hpx#<2eycTI$<4zaJBh{gp@j4|+kJ7D|$d};jzJ|F4vc0K1$ zIV$JU8J=pR7OpMjrCiB>fArrU{r5-z{n3Aa^xq%- ze}7b+qv;e=U$$bJxe;gYqx!73Pf&$fsL*Kf0UFl)o1ao9 TaT7I*^+w-KGY6D%?xy|^oqFqc literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem_backend_diff/cctv-base-key b/test/fuzz/qbit_corpora/mlkem_backend_diff/cctv-base-key new file mode 100644 index 0000000000000000000000000000000000000000..f4eb8e847187176031bccc800c98f954df4e8137 GIT binary patch literal 1664 zcmV-`27mcsp4%8zlFtlDhQk0odbjj3ZmZ}eVxQa@R+G>TN{7S&K6|+JGHL+AzuIngeqTd`@meUYTipK;&e!KWJaG$~)UjtQJ^cJHW`V&^6GdL0&9F5Y|VwqnJWO!{c=zU0dCr$xYkDGza)eorZG`sO%$dbq^ z)M_E74)0@HsGO+DxX+YIS98h%NQ(;7TB)mBtK`7VW2p-!MW{W?>N_jbY1uv=vX+vyf6~zgPo5EF- z&ZD-J*eb?V7AA{Wpzx*2`v*MvLkXKzk8Dwz3zqyqliTt(EF*Yw77cu-wkKd}(a08) zAShcJJ>Izucm^0dE6nC1jJ2U=kV;{J$B6_M60d`@z%wSQ_{3<&qWUVr*cdIMlpTLG zCSKJ_3dXqj8OfKXY$nu<76@vccd}i8a9yIm8G{A7F%_i>##JRSM8qZ@8y#|oYOYza z<~N7GDzJAvmC$f=A*+$KA#Ia&2DIa||=hlT)k&g^dbUuYAum0F&+oFqbxYo`;eFgfqWYkQH>pL9(ge9LlnlRwU(*1QQ2K z7!dhay&x)bh*i2A5;_e#$cRU8;oT< zTMH9gnd~IN8=l!ZxE%M8-i=SW^ixQx83HQDT7Py7;j7Tx2xg@7gLGR zv9-y@bmsca*)dr0b|~Sj5tp+G0<$M`X0HbVpjToN&)IN$M?y()zq@q-9J(>GrHr6r zqSXR?i!#1q=uLjAsajt|`!0V!~xW(Z)E4x7Nu7eA3J4`KmX86mlyaEU0|q zS)I;OPNu}TJkudF!)>@0E<7}7AKH$h$I0jnhto=PkyEs%V!#pOHe;W`UNi(4DiKQ?u0xoh!{sK5Gs+)di+}bmWQSn zWoKmw-Vjir{I|QPEeAjbI=}(zGj4(VT42Pd6gsN<`8r}Dj&=*3-q^%SYfb4neVk`| zt=W_>G;dpgBCL!mRK++}B1_uf6k#@FYRn5r6}^y)YsDqN@MkguOMybcWRTPu{Dp@; z>sIo>0`U@Svj<;Wv22-SE`arWP4GvWR!&gIh%-o8Hl{o{(aVW4$ps1ude=%+@}h@` zy5NGzM%tc@b$b37fVI_hcNb}a15<#Qv3EFaHrKzoY@w#MjcU5QT17GHGs^`sdAJ;e z#?)J}(Mq8rSo}bJ=BcIKW~t$8Yu355y|h|N2G?UnTTP+>0X}FPL+X5#KCy=F@|I-9 K-9q~hBoARom;=uM literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem_backend_diff/coefficient-3328 b/test/fuzz/qbit_corpora/mlkem_backend_diff/coefficient-3328 new file mode 100644 index 0000000000000000000000000000000000000000..21a0a6fa803542c309e985ea41a5bc211ff77070 GIT binary patch literal 1664 zcmV-`27mcsp4%8zlFtlDhQk0odbjj3ZmZ}eVxQa@R+G>TN{7S&K6|+JGHL+AzuIngeqTd`@meUYTipK;&e!KWJaG$~)UjtQJ^cJHW`V&^6GdL0&9F5Y|VwqnJWO!{c=zU0dCr$xYkDGza)eorZG`sO%$dbq^ z)M_E74)0@HsGO+DxX+YIS98h%NQ(;7TB)mBtK`7VW2p-!MW{W?>N_jbY1uv=vX+vyf6~zgPo5EF- z&ZD-J*eb?V7AA{Wpzx*2`v*MvLkXKzk8Dwz3zqyqliTt(EF*Yw77cu-wkKd}(a08) zAShcJJ>Izucm^0dE6nC1jJ2U=kV;{J$B6_M60d`@z%wSQ_{3<&qWUVr*cdIMlpTLG zCSKJ_3dXqj8OfKXY$nu<76@vccd}i8a9yIm8G{A7F%_i>##JRSM8qZ@8y#|oYOYza z<~N7GDzJAvmC$f=A*+$KA#Ia&2DIa||=hlT)k&g^dbUuYAum0F&+oFqbxYo`;eFgfqWYkQH>pL9(ge9LlnlRwU(*1QQ2K z7!dhay&x)bh*i2A5;_e#$cRU8;oT< zTMH9gnd~IN8=l!ZxE%M8-i=SW^ixQx83HQDT7Py7;j7Tx2xg@7gLGR zv9-y@bmsca*)dr0b|~Sj5tp+G0<$M`X0HbVpjToN&)IN$M?y()zq@q-9J(>GrHr6r zqSXR?i!#1q=uLjAsajt|`!0V!~xW(Z)E4x7Nu7eA3J4`KmX86mlyaEU0|q zS)I;OPNu}TJkudF!)>@0E<7}7AKH$h$I0jnhto=PkyEs%V!#pOHe;W`UNi(4DiKQ?u0xoh!{sK5Gs+)di+}bmWQSn zWoKmw-Vjir{I|QPEeAjbI=}(zGj4(VT42Pd6gsN<`8r}Dj&=*3-q^%SYfb4neVk`| zt=W_>G;dpgBCL!mRK++}B1_uf6k#@FYRn5r6}^y)YsDqN@MkguOMybcWRTPu{Dp@; z>sIo>0`U@Svj<;Wv22-SE`arWP4GvWR!&gIh%-o8Hl{o{(aVW4$ps1ude=%+@}h@` zy5NGzM%tc@b$b37fVI_hcNb}a15<#Qv3EFaHrKzoY@w#MjcU5QT17GHGs^`sdAJ;e z#?)J}(Mq8rSo}bJ=BcIKW~t$8Yu355y|h|N2G?UnTTP+>0X}FPL+X5#KCy=F@|I-9 K-9q~hBoAR0MFX7x literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem_backend_diff/coefficient-3329 b/test/fuzz/qbit_corpora/mlkem_backend_diff/coefficient-3329 new file mode 100644 index 0000000000000000000000000000000000000000..037364c402af2300a1fc628a5b5bf506490e915b GIT binary patch literal 1664 zcmV-`27mcsp4%8zlFtlDhQk0odbjj3ZmZ}eVxQa@R+G>TN{7S&K6|+JGHL+AzuIngeqTd`@meUYTipK;&e!KWJaG$~)UjtQJ^cJHW`V&^6GdL0&9F5Y|VwqnJWO!{c=zU0dCr$xYkDGza)eorZG`sO%$dbq^ z)M_E74)0@HsGO+DxX+YIS98h%NQ(;7TB)mBtK`7VW2p-!MW{W?>N_jbY1uv=vX+vyf6~zgPo5EF- z&ZD-J*eb?V7AA{Wpzx*2`v*MvLkXKzk8Dwz3zqyqliTt(EF*Yw77cu-wkKd}(a08) zAShcJJ>Izucm^0dE6nC1jJ2U=kV;{J$B6_M60d`@z%wSQ_{3<&qWUVr*cdIMlpTLG zCSKJ_3dXqj8OfKXY$nu<76@vccd}i8a9yIm8G{A7F%_i>##JRSM8qZ@8y#|oYOYza z<~N7GDzJAvmC$f=A*+$KA#Ia&2DIa||=hlT)k&g^dbUuYAum0F&+oFqbxYo`;eFgfqWYkQH>pL9(ge9LlnlRwU(*1QQ2K z7!dhay&x)bh*i2A5;_e#$cRU8;oT< zTMH9gnd~IN8=l!ZxE%M8-i=SW^ixQx83HQDT7Py7;j7Tx2xg@7gLGR zv9-y@bmsca*)dr0b|~Sj5tp+G0<$M`X0HbVpjToN&)IN$M?y()zq@q-9J(>GrHr6r zqSXR?i!#1q=uLjAsajt|`!0V!~xW(Z)E4x7Nu7eA3J4`KmX86mlyaEU0|q zS)I;OPNu}TJkudF!)>@0E<7}7AKH$h$I0jnhto=PkyEs%V!#pOHe;W`UNi(4DiKQ?u0xoh!{sK5Gs+)di+}bmWQSn zWoKmw-Vjir{I|QPEeAjbI=}(zGj4(VT42Pd6gsN<`8r}Dj&=*3-q^%SYfb4neVk`| zt=W_>G;dpgBCL!mRK++}B1_uf6k#@FYRn5r6}^y)YsDqN@MkguOMybcWRTPu{Dp@; z>sIo>0`U@Svj<;Wv22-SE`arWP4GvWR!&gIh%-o8Hl{o{(aVW4$ps1ude=%+@}h@` zy5NGzM%tc@b$b37fVI_hcNb}a15<#Qv3EFaHrKzoY@w#MjcU5QT17GHGs^`sdAJ;e z#?)J}(Mq8rSo}bJ=BcIKW~t$8Yu355y|h|N2G?UnTTP+>0X}FPL+X5#KCy=F@|I-9 K-9q~hBoAR6Wdoi7 literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem_backend_diff/honest-ciphertext-bit-flip b/test/fuzz/qbit_corpora/mlkem_backend_diff/honest-ciphertext-bit-flip new file mode 100644 index 0000000000000000000000000000000000000000..e90fa0ac39fb8f88ac32e96c5e5c7600b649e4b6 GIT binary patch literal 6403 zcmYeCcUvNK(s>@ww!;jzRolN9=B<9Anl%58WZ2{jyk6}`811Ted^O5n^HME&!Ck5F zDHr*?JB~8h*X;aeT(I_)dP?EC*BYsd?#o0@yDZ?_b)3bqZubwTN{7S&K6|+JGHL+AzuIngeqTd`@meUYTipK;&e!KWJaTN{7S&K6|+JGHL+AzuIngeqTd`@meUYTipK;&e!KWJaG$~)UjtQJ^cJHW`V&^6GdL0&9F5Y|VwqnJWO!{c=zU0dCr$xYkDGza)eorZG`sO%$dbq^ z)M_E74)0@HsGO+DxX+YIS98h%NQ(;7TB)mBtK`7VW2p-!MW{W?>N_jbY1uv=vX+vyf6~zgPo5EF- z&ZD-J*eb?V7AA{Wpzx*2`v*MvLkXKzk8Dwz3zqyqliTt(EF*Yw77cu-wkKd}(a08) zAShcJJ>Izucm^0dE6nC1jJ2U=kV;{J$B6_M60d`@z%wSQ_{3<&qWUVr*cdIMlpTLG zCSKJ_3dXqj8OfKXY$nu<76@vccd}i8a9yIm8G{A7F%_i>##JRSM8qZ@8y#|oYOYza z<~N7GDzJAvmC$f=A*+$KA#Ia&2DIa||=hlT)k&g^dbUuYAum0F&+oFqbxYo`;eFgfqWYkQH>pL9(ge9LlnlRwU(*1QQ2K z7!dhay&x)bh*i2A5;_e#$cRU8;oT< zTMH9gnd~IN8=l!ZxE%M8-i=SW^ixQx83HQDT7Py7;j7Tx2xg@7gLGR zv9-y@bmsca*)dr0b|~Sj5tp+G0<$M`X0HbVpjToN&)IN$M?y()zq@q-9J(>GrHr6r zqSXR?i!#1q=uLjAsajt|`!0V!~xW(Z)E4x7Nu7eA3J4`KmX86mlyaEU0|q zS)I;OPNu}TJkudF!)>@0E<7}7AKH$h$I0jnhto=PkyEs%V!#pOHe;W`UNi(4DiKQ?u0xoh!{sK5Gs+)di+}bmWQSn zWoKmw-Vjir{I|QPEeAjbI=}(zGj4(VT42Pd6gsN<`8r}Dj&=*3-q^%SYfb4neVk`| zt=W_>G;dpgBCL!mRK++}B1_uf6k#@FYRn5r6}^y)YsDqN@MkguOMybcWRTPu{Dp@; z>sIo>0`U@Svj<;Wv22-SE`arWP4GvWR!&gIh%-o8Hl{o{(aVW4$ps1ude=%+@}h@` zy5NGzM%tc@b$b37fVI_hcNb}a15<#Qv3EFaHrKzoY@w#MjcU5QT17GHGs^`sdAJ;e z#?)J}(Mq8rSo}bJ=BcIKW~t$8Yu355z4ZS|2G?UnTTP+>0X}FPL+X5#KCy=F@|I-9 K-9q~hBoAR`T?DiM literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/overlay.py b/test/fuzz/qbit_corpora/overlay.py index ce00d7d3df..0533e6507f 100755 --- a/test/fuzz/qbit_corpora/overlay.py +++ b/test/fuzz/qbit_corpora/overlay.py @@ -29,6 +29,8 @@ "asert_edge_cases", "asert_math", "auxpow", + "mlkem", + "mlkem_backend_diff", "p2mr_script", "pqc", ) diff --git a/test/fuzz/test_qbit_fuzz_runner.py b/test/fuzz/test_qbit_fuzz_runner.py index d4066f98ea..23a3d8a0da 100755 --- a/test/fuzz/test_qbit_fuzz_runner.py +++ b/test/fuzz/test_qbit_fuzz_runner.py @@ -31,7 +31,7 @@ SRC_DIR = Path(__file__).resolve().parents[2] RUNNER = SRC_DIR / "test" / "fuzz" / "test_runner.py" CORPORA_DIR = SRC_DIR / "test" / "fuzz" / "qbit_corpora" -REQUIRED = ("asert_chain_transition", "asert_edge_cases", "asert_math", "auxpow", "p2mr_script", "pqc") +REQUIRED = ("asert_chain_transition", "asert_edge_cases", "asert_math", "auxpow", "mlkem", "mlkem_backend_diff", "p2mr_script", "pqc") INTEGRATION = {} STUB_SOURCE = r''' diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index 12fb595ad4..d04b655c9d 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -24,6 +24,8 @@ "asert_edge_cases", "asert_math", "auxpow", + "mlkem", + "mlkem_backend_diff", "p2mr_script", "pqc", ) diff --git a/test/lint/README.md b/test/lint/README.md index df1dfc3931..16493b0508 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -130,6 +130,22 @@ The check accepts the exact documented downstream lockfile patch for `RUSTSEC-2026-0204-crossbeam-epoch-0.9.20`. Any other subtree drift remains a failure. +mlkem-native-check.sh +===================== +Run this script from the root of the repository to verify offline that the +vendored `src/mlkem-native` tree is byte-identical to the mlkem-native release +pinned in `contrib/devtools/mlkem-native.pin`: + +``` +test/lint/mlkem-native-check.sh +``` + +It compares the git tree id of `src/mlkem-native/mlkem` and the blob id of +`src/mlkem-native/LICENSE` in `HEAD` with the pinned ids, and fails on any other +entry or uncommitted change under `src/mlkem-native`. The lint runner's +`subtree` lint runs it. For the update workflow, see +[`doc/subtrees/mlkem-native.md`](../../doc/subtrees/mlkem-native.md). + lint-libbitcoinpqc-vectors.py ============================= Run this script from the root of the repository to verify bounded30 SPHINCS+ diff --git a/test/lint/lint-locale-dependence.py b/test/lint/lint-locale-dependence.py index 7e90b5cd80..9bb55ded2b 100755 --- a/test/lint/lint-locale-dependence.py +++ b/test/lint/lint-locale-dependence.py @@ -56,6 +56,7 @@ "src/leveldb/", "src/secp256k1/", "src/minisketch/", + "src/mlkem-native/", "src/tinyformat.h", ] diff --git a/test/lint/lint_ignore_dirs.py b/test/lint/lint_ignore_dirs.py index 34dc945b5d..c5144dc424 100644 --- a/test/lint/lint_ignore_dirs.py +++ b/test/lint/lint_ignore_dirs.py @@ -10,5 +10,6 @@ "src/minisketch/", "src/ipc/libmultiprocess/", "src/libbitcoinpqc/", + "src/mlkem-native/", "contrib/photon/src/vendor/", ] diff --git a/test/lint/mlkem-native-check.sh b/test/lint/mlkem-native-check.sh new file mode 100755 index 0000000000..15ae3cdb04 --- /dev/null +++ b/test/lint/mlkem-native-check.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2026-present The qbit core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit. + +export LC_ALL=C +set -euo pipefail + +readonly PREFIX="src/mlkem-native" +readonly PIN_FILE="contrib/devtools/mlkem-native.pin" + +usage() { + cat <&2 + echo " Cause: ${cause}" >&2 + echo " Fix: ${fix}" >&2 + exit 1 +} + +pin_value() { + local key="$1" + local value + value="$(awk -F= -v key="${key}" '$1 == key { print substr($0, length(key) + 2); exit }' "${PIN_FILE}")" + if [[ -z "${value}" ]]; then + fail "${PIN_FILE} has no ${key}= entry" \ + "the pin file was edited by hand or truncated" \ + "restore ${PIN_FILE} from git, or regenerate it with contrib/devtools/update-mlkem-native.sh" + fi + printf '%s\n' "${value}" +} + +require_object_id() { + local key="$1" + local value="$2" + if [[ ! "${value}" =~ ^[0-9a-f]{40}$ ]]; then + fail "${PIN_FILE} ${key}=${value} is not a 40-digit object id" \ + "the pin file was edited by hand" \ + "restore ${PIN_FILE} from git, or regenerate it with contrib/devtools/update-mlkem-native.sh" + fi +} + +if [[ "${1-}" == "--help" || "${1-}" == "-h" ]]; then + usage + exit 0 +fi + +if [[ $# -ne 0 ]]; then + usage >&2 + exit 2 +fi + +if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then + fail "not inside a git work tree" \ + "the check compares git object ids" \ + "run it from the root of a qbit checkout" +fi +cd "$(git rev-parse --show-toplevel)" + +if [[ ! -f "${PIN_FILE}" ]]; then + fail "${PIN_FILE} not found" \ + "the pin file was deleted or the check runs outside the repository root" \ + "restore ${PIN_FILE} from git" +fi + +pinned_tag="$(pin_value tag)" +pinned_commit="$(pin_value commit)" +pinned_tree="$(pin_value mlkem_tree)" +pinned_license="$(pin_value license_blob)" +require_object_id commit "${pinned_commit}" +require_object_id mlkem_tree "${pinned_tree}" +require_object_id license_blob "${pinned_license}" + +echo "${PIN_FILE} pins mlkem-native ${pinned_tag} commit ${pinned_commit}" + +top_entries="$(git ls-tree --name-only "HEAD:${PREFIX}" 2>/dev/null || true)" +if [[ -z "${top_entries}" ]]; then + fail "${PREFIX} not found in HEAD" \ + "the vendored tree was removed or never committed" \ + "run contrib/devtools/update-mlkem-native.sh and commit the result" +fi +if [[ "${top_entries}" != $'LICENSE\nmlkem' ]]; then + fail "${PREFIX} in HEAD holds entries other than LICENSE and mlkem: $(printf '%s' "${top_entries}" | tr '\n' ' ')" \ + "a file was added next to the vendored tree" \ + "move qbit-owned files out of ${PREFIX} (ML-KEM glue lives in src/crypto/mlkem*)" +fi + +current_tree="$(git rev-parse "HEAD:${PREFIX}/mlkem")" +echo "${PREFIX}/mlkem in HEAD is tree ${current_tree}" +if [[ "${current_tree}" != "${pinned_tree}" ]]; then + git diff --stat "${pinned_tree}" "${current_tree}" >&2 2>/dev/null || true + fail "${PREFIX}/mlkem tree ${current_tree} differs from pinned tree ${pinned_tree}" \ + "a vendored file was edited, added, removed or changed mode, or the pin was changed without re-vendoring" \ + "never edit ${PREFIX}; restore it with contrib/devtools/update-mlkem-native.sh" +fi + +current_license="$(git rev-parse "HEAD:${PREFIX}/LICENSE")" +echo "${PREFIX}/LICENSE in HEAD is blob ${current_license}" +if [[ "${current_license}" != "${pinned_license}" ]]; then + fail "${PREFIX}/LICENSE blob ${current_license} differs from pinned blob ${pinned_license}" \ + "the vendored LICENSE was edited, or the pin was changed without re-vendoring" \ + "never edit ${PREFIX}; restore it with contrib/devtools/update-mlkem-native.sh" +fi + +dirty="$(git status --porcelain --untracked-files=all -- "${PREFIX}")" +if [[ -n "${dirty}" ]]; then + printf '%s\n' "${dirty}" >&2 + fail "uncommitted changes under ${PREFIX}" \ + "a vendored file was edited, added or removed in the work tree or index" \ + "discard them with: git checkout HEAD -- ${PREFIX} && git clean -fdx -- ${PREFIX}" +fi + +echo "GOOD" diff --git a/test/lint/test_runner/src/main.rs b/test/lint/test_runner/src/main.rs index 3ea377f623..9c8083aa61 100644 --- a/test/lint/test_runner/src/main.rs +++ b/test/lint/test_runner/src/main.rs @@ -224,6 +224,8 @@ fn get_pathspecs_default_excludes() -> Vec { // qbit-specific vendored source. Public release snapshot branches may not // carry git-subtree metadata until the public upstream repository exists. "src/libbitcoinpqc", + // Vendored byte for byte and pinned by tree id; never reformatted. + "src/mlkem-native", "doc/release-notes/release-notes-*", // archived notes "contrib/photon/src/vendor/", // externally sourced vendored code ]) @@ -246,6 +248,10 @@ fn lint_subtree() -> LintResult { .status() .expect("command_error") .success(); + good &= Command::new("test/lint/mlkem-native-check.sh") + .status() + .expect("command_error") + .success(); if good { Ok(()) } else { @@ -697,6 +703,8 @@ fn lint_markdown() -> LintResult { let bin_name = "mlc"; let mut md_ignore_paths = get_subtrees(); md_ignore_paths.push("./doc/development/README_doxygen.md"); + // Upstream's READMEs link to files outside the vendored mlkem/ directory. + md_ignore_paths.push("src/mlkem-native"); let md_ignore_path_str = md_ignore_paths.join(","); let mut cmd = Command::new(bin_name); diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 77aed1aa8a..84facc2482 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -12,6 +12,7 @@ unsigned-integer-overflow:*/include/c++/ unsigned-integer-overflow:FuzzedDataProvider::ConsumeIntegralInRange unsigned-integer-overflow:leveldb/ unsigned-integer-overflow:minisketch/ +unsigned-integer-overflow:mlkem-native/ unsigned-integer-overflow:secp256k1/ unsigned-integer-overflow:test/fuzz/crypto_diff_fuzz_chacha20.cpp implicit-integer-sign-change:*/include/boost/ @@ -20,18 +21,22 @@ implicit-integer-sign-change:*/new_allocator.h implicit-integer-sign-change:*/qarraydata.h implicit-integer-sign-change:crc32c/ implicit-integer-sign-change:minisketch/ +implicit-integer-sign-change:mlkem-native/ implicit-integer-sign-change:secp256k1/ implicit-signed-integer-truncation:*/include/c++/ implicit-signed-integer-truncation:leveldb/ +implicit-signed-integer-truncation:mlkem-native/ implicit-signed-integer-truncation:secp256k1/ implicit-signed-integer-truncation,implicit-integer-sign-change:secp256k1_modinv64_posdivsteps_62_var implicit-unsigned-integer-truncation:*/include/c++/ implicit-unsigned-integer-truncation:leveldb/ +implicit-unsigned-integer-truncation:mlkem-native/ implicit-unsigned-integer-truncation:secp256k1/ implicit-unsigned-integer-truncation:test/fuzz/crypto_diff_fuzz_chacha20.cpp shift-base:*/include/c++/ shift-base:leveldb/ shift-base:minisketch/ +shift-base:mlkem-native/ shift-base:secp256k1/ shift-base:test/fuzz/crypto_diff_fuzz_chacha20.cpp From a9373780e4e1fecce4c493336490162f887844ad Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Wed, 21 Oct 2026 22:59:08 -0400 Subject: [PATCH 09/24] net: hybrid post-quantum v2 handshake with key confirmation (#184 part 4) (#230) * bip324: derive hybrid keys from an ordered transport transcript Add the hybrid ML-KEM-1024 key recipe of #184 to BIP324Cipher. Nothing calls it from the transport yet; E3 does. - Initialize(..., retain_for_hybrid) keeps the raw BIP324 ECDH secret, starts a running SHA256 transcript with ellswift_I || ellswift_R, and snapshots the network magic and cipher direction. The default (false) keeps today's secret lifetime and the existing vectors. - AddVersionContents hashes CompactSize(len) || contents, responder first, through a small stream adapter without copying the contents. - SwitchToHybrid derives HKDF-SHA256 with salt "qbit_v2_hybrid_mlkem1024" || magic over ss_ECDH || ss_MLKEM || TH, re-emplaces all four packet ciphers with counters at 0 and replaces the session id; garbage terminators stay. Misuse (not initialized, not retained, wrong contents count, a second switch) returns false and changes nothing. - The retained secret, IKM, HKDF state and derived keys are wiped on switch, in DiscardHybridSecret and in the destructor. Copying is deleted. The same recipe goes into test_framework/crypto/bip324_cipher.py, and contrib/devtools/generate-pq-transport-vectors.py uses it, the Python ML-KEM-1024 and the BIP324 ECDH helpers (imported by path, never the C++ code) to write src/test/data/pq_transport_vectors.json: inputs, every intermediate, both confirmations and first packets, and negative cases. Its --check mode runs in lint through test/lint/lint-pq-transport-vectors.py. bip324_tests gains hybrid_vectors (both roles and self-decrypt mode, with every intermediate recomputed from qbit's own primitives), hybrid_transcript_binding, hybrid_api_order and hybrid_rekey_boundaries. The bip324_cipher_roundtrip fuzz target gains an optional switch, transcript and secret mutations and discards. The existing BIP324 vectors are unchanged. Part of #184 (#199, E1). Co-Authored-By: Claude Opus 5.5 * doc: specify hybrid v2 transport (doc/design/pq-transport.md) Add the specification of the hybrid ML-KEM-1024 v2 transport from #184's design, written for interoperating implementations and outside review: the record grammar and header registry (0x00 reserved, 0xF0 for the ML-KEM-1024 offer and accept, qbit's values from 0xF0 upward), byte layouts, parse rules (unparseable contents are ignored, unlike the Transport Feature Negotiation draft), the key recipe byte by byte, the responder and initiator message rules and why progress still holds, the switch boundary, key confirmation MUSTs, failure handling, security notes, an interop walk-through of the vectors file, the convergence and retirement policy, and a contributor section. The spec quotes vector 0 in a marked block, and generate-pq-transport-vectors.py now fails (in --check mode too, so in lint) when a quoted value differs from the vectors. Part of #184 (#199, H2). Co-Authored-By: Claude Opus 5.5 * net: implement PQ feature parsing and per-connection KEM state Add src/bip324_pq.{h,cpp} with PQHandshake, the version-contents parser and one connection's ML-KEM-1024 state for the hybrid v2 transport of #184. Nothing calls it from V2Transport yet; E3 does, so behavior is unchanged. - ParseContents is a single noexcept pass over a SpanReader. The whole contents must parse as records CompactSize(len) || header || payload (len >= 1, canonical, within the contents) before anything else is looked at; any framing error means no features. Each length is checked against what is left before SpanReader::ignore(). The first 0xF0 record wins and is returned as a span into the contents. - CheckRecordLength applies the role's length rule to that record (EK_LENGTH or CT_LENGTH); AcceptOffer runs the modulus check (EK_MODULUS) and encapsulates; DecapsulateAccept decapsulates and wipes the single-use key. SerializeRecord writes fd 21 06 f0 || payload. - Key generation draws two independent 32-byte values and encapsulation one, from GetStrongRandBytes in production or the test constructor's PQRandomSource; the ML-KEM operations come through PQKemOps. Seeds and coins are wiped. - Internal ML-KEM errors return mlkem::Error::INTERNAL and record which operation failed; nothing asserts. The first recorded failure wins, and any failure, ClearSecrets() and the destructor wipe the decapsulation key. net_tests gains pq_records_grammar, pq_records_first_wins, pq_record_semantics, pq_records_max_contents and pq_handshake_negotiation (vector 0's ek, ct and shared secret from deterministic entropy, plus injected keygen, check, encaps and decaps failures). The new pq_records fuzz target checks the parser against an independent reading of the grammar; it joins the required qbit seed corpora with 15 generated seeds, including the nightly mutation list in ci/test/03_test_script.sh. Part of #184 (#199, E2). Co-Authored-By: Claude Opus 5.5 * net: negotiate hybrid v2 keys and require key confirmation Wire PQHandshake into V2Transport per #184 and #199 (E3). Production construction stays PQ-off (V2PQOptions{} in MakeTransport); part 6 adds the option, and with the switch off the wire bytes are today's. - New states: SendState::AWAITING_VERSION and RecvState::CONFIRM, with the transition checks and diagrams updated. - With PQMode::NEGOTIATE the ciphers retain the ECDH secret and start the transcript. A responder generates its key only once the initiator's whole public key has arrived, queues its terminator and the offer, and then sends nothing until the initiator's version packet is processed. An initiator queues only its terminator and holds its version packet. While holding, SetMessageToSend refuses and GetBytesToSend reports more=false. - ProcessReceivedPacketBytes takes the send lock around ProcessVersionContents, so the switch of all four ciphers and the session id is atomic under both locks; application packets take no extra lock. A legacy peer (no own record, or contents that do not parse) keeps the ECDH keys. A valid offer is answered with the accept under the ECDH keys, then both sides switch and append a 20-byte key confirmation. Packets are only ever appended to the send buffer. - In CONFIRM the peer's first hybrid packet must decrypt to length 0, checked as soon as its 3 length bytes arrive, then verify its tag with empty AAD and carry the ignore bit. It never becomes a message. - Off and fallback neither parse nor validate the peer's contents; a responder that could not offer never decapsulates. - Internal ML-KEM failures never assert. Key generation, key check and encapsulation failures continue as plain v2 with an empty version packet; a decapsulation failure after the initiator committed closes. The decapsulation key and the retained ECDH secret are wiped on switch, legacy outcome, every failure and in destructors. - Transport::Info gains transport_pq and transport_pq_status; until the confirmation the status is pending, the classification stays detecting and no session id is published. V2Transport::GetPQSnapshot exposes the negotiation's progress and first failure for part 5. - V2PQOptions::corrupt_shared_secret inverts byte 0 of the local ML-KEM secret before derivation, as the vectors' negative case models. net_tests gains v2_pq_pair, v2_pq_legacy_matrix (including two real transports in every mode pair), v2_pq_v1_detection, v2_pq_send_hold, v2_pq_switch_boundary, v2_pq_confirmation_length, _tag and _not_decoy, v2_pq_corrupt_ct, v2_pq_internal_errors and v2_pq_rekey, and the existing random-contents scenarios also run with the negotiation on against a legacy peer. The bidirectional v2 and v1v2 transport fuzz targets take a PQ mode and an ML-KEM entropy seed per side from the input and check that the keys are hybrid exactly when both sides negotiate; both become required qbit corpus targets with generated seeds, including in the nightly mutation list. Part of #184 (#199, E3). Co-Authored-By: Claude Opus 5.5 * fuzz: add a stateful malicious-peer target for PQ negotiation Add the p2p_v2_pq_malicious_peer target (#199, G4): a peer that completes the ECDH part of BIP324 honestly, so every packet it sends authenticates, and drives the hybrid negotiation of a V2Transport from the attacker's side. The tested transport runs in either role and in each mode (a responder never falls back), built with E3's deterministic test constructor and counting ML-KEM operations. The fuzzer scripts the peer step by step: its key, terminator, decoys (also carrying own records), version packets (honest, duplicate and unknown records, trailing corruption, malformed lengths, a key that fails the modulus check, blind, damaged and unsolicited ciphertexts, an accept without the peer's own switch), confirmation variants, application packets, unauthenticated bytes, partial delivery and collection, messages for the transport to send, and a disconnect at any point. The peer reads everything the transport sends and switches to hybrid keys itself when it played its part honestly. The variant generator follows the E3 verifier's malpeer harness. Checked after every step: no assertion or hang; GetBytesToSend stays append-only and its 'more' predictions hold; refused messages stay untouched and a holding initiator takes none; nothing is delivered between the switch and the peer's confirmation; the transport sends no decoy but its confirmation, which decrypts to length 0 under the peer's keys when both switched; only the same keys confirm, with equal session ids, and an honest exchange delivered in full confirms; off and fallback make no ML-KEM call, and a responder decapsulates only an accept to its own offer; once the peer's version packet is processed, or the transport failed, no hybrid secret remains; a close under the hybrid keys before confirmation is a confirmation failure. The target joins the required qbit seed corpora with 16 generated seeds, each a script whose outcome is documented in the manifest: honest exchanges in both roles (with maximum garbage, extra records and fragmented delivery), every malformed record and confirmation failure, a damaged ciphertext, a legacy peer, a fallback initiator, an unsolicited accept and a disconnect while the offer is pending. Part of #184 (#199, G4). Co-Authored-By: Claude Opus 5.5 * fuzz: drop a stray libFuzzer artifact and ignore new ones The G4 commit added crash-ef5ec86784262f7cd94f46933e5f6760bee654b5 at the repository root by accident. It is an input for p2p_v2_pq_malicious_peer left over from one of the harness false positives fixed during G4's development, not a transport defect: it replays cleanly on that target and on pq_records, bip324_cipher_roundtrip and both bidirectional transport targets. libFuzzer writes its crash, leak, out-of-memory, slow-unit and timeout artifacts to the working directory, so ignore those prefixes at the root. Co-Authored-By: Claude Opus 5.5 * test: check hybrid secret wipes, close paths and pending status Close the test gaps the #230 deep review's mutants exposed; each mutant below survived bip324_tests and net_tests before and is now caught. - The switch wipes the retained ECDH secret and the transcript (hybrid_api_order, v2_pq_pair, v2_pq_legacy_matrix), and a legacy outcome wipes them and the decapsulation key (v2_pq_legacy_matrix, in every arm and for every pair of modes). - v2_pq_close_wipes: a missing garbage terminator, an oversized version packet length and a damaged version packet tag close with both secrets wiped at once, in both roles. These closes come before the peer's version packet authenticated: no PQ failure, pending, and a responder's offer counts as sent. - The status is pending from the start of the handshake until the peer's version packet is processed, not legacy_peer (v2_pq_pair, v2_pq_legacy_matrix, v2_pq_close_wipes). - A decapsulation failure closes on the initiator's version packet itself, with no confirmation queued, instead of one packet later. Co-Authored-By: Claude Opus 5.5 * net: wipe hybrid secrets whenever the peer's version is processed ProcessVersionContents returned early, without wiping, when this side was not awaiting the peer's version packet: the negotiation off, in fallback, or a responder that could not offer. Every such path in this PR wipes (or never retains) before it sends its version packet, so this is correct today. But a later path that declines the negotiation after Initialize(..., retain_for_hybrid=true) would keep the ECDH secret for the whole connection; part 5's load shedding first had exactly that bug. Wipe in that branch too, a no-op unless such a path kept a secret, so that no hybrid secret outlives the peer's version packet by construction. A test-only option, V2PQOptions::retain_without_negotiation, retains the secret with the negotiation off to stand in for such a path; production passes default options. v2_pq_version_wipes checks the wipe in both roles and that plain v2 continues. v2_pq_internal_errors now also checks that the keygen failure wipes at once, which the new wipe would otherwise mask until the initiator's version packet arrives. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .gitignore | 7 + ci/test/03_test_script.sh | 3 +- .../devtools/generate-pq-transport-vectors.py | 381 ++++ doc/design/pq-transport.md | 507 ++++++ doc/development/fuzzing.md | 9 +- src/CMakeLists.txt | 1 + src/bip324.cpp | 93 +- src/bip324.h | 44 +- src/bip324_pq.cpp | 200 ++ src/bip324_pq.h | 200 ++ src/net.cpp | 344 +++- src/net.h | 129 +- src/test/CMakeLists.txt | 1 + src/test/bip324_tests.cpp | 463 +++++ src/test/data/pq_transport_vectors.json | 239 +++ src/test/fuzz/CMakeLists.txt | 2 + src/test/fuzz/bip324.cpp | 86 +- src/test/fuzz/p2p_transport_serialization.cpp | 53 +- src/test/fuzz/p2p_v2_pq_malicious_peer.cpp | 639 +++++++ src/test/fuzz/pq_records.cpp | 96 + src/test/net_tests.cpp | 1621 ++++++++++++++++- .../test_framework/crypto/bip324_cipher.py | 148 +- test/fuzz/qbit_corpora/MANIFEST.json | 354 +++- test/fuzz/qbit_corpora/README.md | 38 +- test/fuzz/qbit_corpora/generate_seeds.py | 267 +++ test/fuzz/qbit_corpora/overlay.py | 4 + .../v1-silent-negotiating-responder | Bin 0 -> 114 bytes .../v1-version-to-negotiating-responder | Bin 0 -> 119 bytes ...rsion-to-negotiating-responder-max-garbage | Bin 0 -> 119 bytes .../v1-version-to-off-responder | Bin 0 -> 183 bytes .../p2p_transport_bidirectional_v2/both-off | Bin 0 -> 214 bytes .../both-off-interleaved | Bin 0 -> 390 bytes .../initiator-fallback-responder-off | Bin 0 -> 219 bytes .../initiator-fallback-responder-offers | Bin 0 -> 214 bytes ...ator-fallback-responder-offers-interleaved | Bin 0 -> 343 bytes .../initiator-negotiates-responder-off | Bin 0 -> 214 bytes ...iator-negotiates-responder-off-interleaved | Bin 0 -> 331 bytes .../initiator-off-responder-offers | Bin 0 -> 214 bytes ...initiator-off-responder-offers-interleaved | Bin 0 -> 359 bytes .../negotiate-both | Bin 0 -> 214 bytes .../negotiate-both-interleaved | Bin 0 -> 343 bytes .../negotiate-both-interleaved-long-garbage | Bin 0 -> 326 bytes .../negotiate-both-max-garbage | Bin 0 -> 214 bytes .../initiator-confirmation-length | Bin 0 -> 86 bytes .../initiator-confirmation-tag | Bin 0 -> 86 bytes .../initiator-ek-length | Bin 0 -> 83 bytes .../initiator-ek-modulus | Bin 0 -> 83 bytes .../initiator-fallback-ignores-offer | Bin 0 -> 88 bytes .../p2p_v2_pq_malicious_peer/initiator-hybrid | Bin 0 -> 90 bytes .../initiator-hybrid-extras-fragmented | Bin 0 -> 309 bytes .../initiator-noncompliant-responder | Bin 0 -> 85 bytes .../responder-confirmation-not-decoy | Bin 0 -> 86 bytes .../responder-corrupt-ct | Bin 0 -> 90 bytes .../responder-ct-length | Bin 0 -> 83 bytes .../responder-disconnect-after-offer | Bin 0 -> 81 bytes .../p2p_v2_pq_malicious_peer/responder-hybrid | Bin 0 -> 90 bytes .../responder-hybrid-extras | Bin 0 -> 97 bytes .../responder-legacy-peer-burst | Bin 0 -> 89 bytes .../responder-off-unsolicited-accept | Bin 0 -> 87 bytes test/fuzz/qbit_corpora/pq_records/accept | Bin 0 -> 1572 bytes .../qbit_corpora/pq_records/empty-own-record | 1 + test/fuzz/qbit_corpora/pq_records/long-offer | Bin 0 -> 1573 bytes .../pq_records/malformed-first-then-offer | Bin 0 -> 1576 bytes .../pq_records/minimal-unknown-records | 1 + .../pq_records/non-canonical-length | Bin 0 -> 5 bytes test/fuzz/qbit_corpora/pq_records/offer | Bin 0 -> 1572 bytes .../pq_records/offer-duplicate-unknown | Bin 0 -> 3148 bytes .../pq_records/offer-trailing-byte | Bin 0 -> 1573 bytes .../fuzz/qbit_corpora/pq_records/one-past-end | 1 + .../qbit_corpora/pq_records/oversized-length | Bin 0 -> 7 bytes .../pq_records/reserved-and-unknown-headers | Bin 0 -> 7 bytes test/fuzz/qbit_corpora/pq_records/short-offer | Bin 0 -> 1571 bytes .../pq_records/truncated-wide-length | Bin 0 -> 6 bytes .../pq_records/unknown-then-accept | Bin 0 -> 1578 bytes test/fuzz/qbit_corpora/pq_records/zero-length | Bin 0 -> 7 bytes test/fuzz/test_qbit_fuzz_runner.py | 4 +- test/fuzz/test_runner.py | 4 + test/lint/lint-pq-transport-vectors.py | 19 + 78 files changed, 5874 insertions(+), 85 deletions(-) create mode 100755 contrib/devtools/generate-pq-transport-vectors.py create mode 100644 doc/design/pq-transport.md create mode 100644 src/bip324_pq.cpp create mode 100644 src/bip324_pq.h create mode 100644 src/test/data/pq_transport_vectors.json create mode 100644 src/test/fuzz/p2p_v2_pq_malicious_peer.cpp create mode 100644 src/test/fuzz/pq_records.cpp create mode 100644 test/fuzz/qbit_corpora/p2p_transport_bidirectional_v1v2/v1-silent-negotiating-responder create mode 100644 test/fuzz/qbit_corpora/p2p_transport_bidirectional_v1v2/v1-version-to-negotiating-responder create mode 100644 test/fuzz/qbit_corpora/p2p_transport_bidirectional_v1v2/v1-version-to-negotiating-responder-max-garbage create mode 100644 test/fuzz/qbit_corpora/p2p_transport_bidirectional_v1v2/v1-version-to-off-responder create mode 100644 test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/both-off create mode 100644 test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/both-off-interleaved create mode 100644 test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/initiator-fallback-responder-off create mode 100644 test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/initiator-fallback-responder-offers create mode 100644 test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/initiator-fallback-responder-offers-interleaved create mode 100644 test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/initiator-negotiates-responder-off create mode 100644 test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/initiator-negotiates-responder-off-interleaved create mode 100644 test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/initiator-off-responder-offers create mode 100644 test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/initiator-off-responder-offers-interleaved create mode 100644 test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/negotiate-both create mode 100644 test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/negotiate-both-interleaved create mode 100644 test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/negotiate-both-interleaved-long-garbage create mode 100644 test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/negotiate-both-max-garbage create mode 100644 test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/initiator-confirmation-length create mode 100644 test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/initiator-confirmation-tag create mode 100644 test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/initiator-ek-length create mode 100644 test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/initiator-ek-modulus create mode 100644 test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/initiator-fallback-ignores-offer create mode 100644 test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/initiator-hybrid create mode 100644 test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/initiator-hybrid-extras-fragmented create mode 100644 test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/initiator-noncompliant-responder create mode 100644 test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/responder-confirmation-not-decoy create mode 100644 test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/responder-corrupt-ct create mode 100644 test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/responder-ct-length create mode 100644 test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/responder-disconnect-after-offer create mode 100644 test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/responder-hybrid create mode 100644 test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/responder-hybrid-extras create mode 100644 test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/responder-legacy-peer-burst create mode 100644 test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/responder-off-unsolicited-accept create mode 100644 test/fuzz/qbit_corpora/pq_records/accept create mode 100644 test/fuzz/qbit_corpora/pq_records/empty-own-record create mode 100644 test/fuzz/qbit_corpora/pq_records/long-offer create mode 100644 test/fuzz/qbit_corpora/pq_records/malformed-first-then-offer create mode 100644 test/fuzz/qbit_corpora/pq_records/minimal-unknown-records create mode 100644 test/fuzz/qbit_corpora/pq_records/non-canonical-length create mode 100644 test/fuzz/qbit_corpora/pq_records/offer create mode 100644 test/fuzz/qbit_corpora/pq_records/offer-duplicate-unknown create mode 100644 test/fuzz/qbit_corpora/pq_records/offer-trailing-byte create mode 100644 test/fuzz/qbit_corpora/pq_records/one-past-end create mode 100644 test/fuzz/qbit_corpora/pq_records/oversized-length create mode 100644 test/fuzz/qbit_corpora/pq_records/reserved-and-unknown-headers create mode 100644 test/fuzz/qbit_corpora/pq_records/short-offer create mode 100644 test/fuzz/qbit_corpora/pq_records/truncated-wide-length create mode 100644 test/fuzz/qbit_corpora/pq_records/unknown-then-accept create mode 100644 test/fuzz/qbit_corpora/pq_records/zero-length create mode 100755 test/lint/lint-pq-transport-vectors.py diff --git a/.gitignore b/.gitignore index 65201c99d0..1a053bda74 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,11 @@ target/ /guix-build-* +# libFuzzer artifacts, written to the working directory +/crash-* +/leak-* +/oom-* +/slow-unit-* +/timeout-* + /ci/scratch/ diff --git a/ci/test/03_test_script.sh b/ci/test/03_test_script.sh index 6399230d18..d4f2fbe36f 100755 --- a/ci/test/03_test_script.sh +++ b/ci/test/03_test_script.sh @@ -371,6 +371,7 @@ if [ "$RUN_FUZZ_TESTS" = "true" ]; then --require_qbit_corpus \ --mutate_min_time="${QBIT_FUZZ_MUTATE_MIN_TIME}" \ "${DIR_FUZZ_IN}" \ - asert_chain_transition asert_edge_cases asert_math auxpow mlkem mlkem_backend_diff p2mr_script pqc + asert_chain_transition asert_edge_cases asert_math auxpow mlkem mlkem_backend_diff p2mr_script \ + p2p_transport_bidirectional_v1v2 p2p_transport_bidirectional_v2 p2p_v2_pq_malicious_peer pq_records pqc fi fi diff --git a/contrib/devtools/generate-pq-transport-vectors.py b/contrib/devtools/generate-pq-transport-vectors.py new file mode 100755 index 0000000000..1796fbe1ca --- /dev/null +++ b/contrib/devtools/generate-pq-transport-vectors.py @@ -0,0 +1,381 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026-present The qbit core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://www.opensource.org/licenses/mit-license.php. +"""Generate the hybrid post-quantum v2 transport vectors (src/test/data/pq_transport_vectors.json). + +The values come from the Python reference in test/functional/test_framework/crypto, imported by +path: bip324_cipher.py for the key recipe and packet encryption, mlkem.py for ML-KEM-1024, and +ellswift.py, secp256k1.py and hkdf.py for the BIP324 ECDH part. This script never calls the C++ +code under test. Every input is derived from a fixed label, so the output is reproducible byte for +byte. The recipe is specified in doc/design/pq-transport.md. + +Run without arguments to rewrite the file, or with --check (as the lint does) to fail when the +committed file differs from what the reference produces. Both modes also fail when a value the +specification quotes in a "pq-transport-vectors" block differs from the vectors. +""" + +import argparse +import hashlib +import importlib +import json +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +FUNCTIONAL_DIR = ROOT / "test" / "functional" +OUTPUT = ROOT / "src" / "test" / "data" / "pq_transport_vectors.json" +SPEC = ROOT / "doc" / "design" / "pq-transport.md" +# A block of "name = hex" lines quoting one vector, introduced by an HTML comment. +SPEC_BLOCK = re.compile(r"\n```text\n(.*?)\n```", re.DOTALL) + +PQ_MLKEM1024 = 0xF0 +# Network magic in wire order (src/kernel/chainparams.cpp); bip324_tests checks it against the chain. +MAGIC = {"main": bytes.fromhex("444f24a8"), "regtest": bytes.fromhex("a66b1fda")} +# First application messages: a v2 short-id ping (18) and the matching pong (19). +PING = bytes([18]) + bytes.fromhex("0123456789abcdef") +PONG = bytes([19]) + bytes.fromhex("0123456789abcdef") + + +def load_reference(): + """Import the test framework's crypto modules from the source tree.""" + sys.path.insert(0, str(FUNCTIONAL_DIR)) + names = ("bip324_cipher", "ellswift", "hkdf", "mlkem", "secp256k1") + return {name: importlib.import_module(f"test_framework.crypto.{name}") for name in names} + + +def seed(label, size=32): + """Deterministic input bytes for a label.""" + out = b"" + counter = 0 + while len(out) < size: + out += hashlib.sha256(f"qbit pq transport vectors/{label}/{counter}".encode()).digest() + counter += 1 + return out[:size] + + +def tagged_hash(tag, data): + tag_hash = hashlib.sha256(tag.encode()).digest() + return hashlib.sha256(tag_hash + tag_hash + data).digest() + + +def make_key(ref, label): + """Deterministic private key and ElligatorSwift encoding of its public key.""" + secp, ellswift = ref["secp256k1"], ref["ellswift"] + priv = int.from_bytes(seed(f"{label}/privkey"), "big") % (secp.GE.ORDER - 1) + 1 + x = (priv * secp.G).x + counter = 0 + while True: + u_seed = seed(f"{label}/ellswift/{counter}", 33) + u = secp.FE(int.from_bytes(u_seed[:32], "big")) + t = ellswift.xswiftec_inv(x, u, u_seed[32] & 7) if u != 0 else None + if t is not None: + assert ellswift.xswiftec(u, t) == x + return priv.to_bytes(32, "big"), u.to_bytes() + t.to_bytes() + counter += 1 + + +def bip324_ecdh(ref, priv, ellswift_theirs, ellswift_ours, initiating): + """The BIP324 ECDH secret (what CKey::ComputeBIP324ECDHSecret returns).""" + x = ref["ellswift"].ellswift_ecdh_xonly(ellswift_theirs, priv) + pubkeys = ellswift_ours + ellswift_theirs if initiating else ellswift_theirs + ellswift_ours + return tagged_hash("bip324_ellswift_xonly_ecdh", pubkeys + x) + + +def record(ref, header, payload): + """One version-contents record: CompactSize(1 + len(payload)) || header || payload.""" + return ref["bip324_cipher"].ser_compact_size(1 + len(payload)) + bytes([header]) + payload + + +class Exchange: + """One side's view of a hybrid handshake, from the version contents onwards.""" + + def __init__(self, ref, ss_ecdh, ellswift_i, ellswift_r, contents_first, contents_second, ss_mlkem, magic): + cipher = ref["bip324_cipher"] + # The transcript always lists the responder's contents first; a misbehaving side may not. + self.transcript_input = cipher.hybrid_transcript_input(ellswift_i, ellswift_r, contents_first, contents_second) + self.transcript_hash = hashlib.sha256(self.transcript_input).digest() + self.keys = cipher.hybrid_derive_keys(ss_ecdh, ss_mlkem, self.transcript_hash, magic) + + def ciphers(self, ref, initiator): + return ref["bip324_cipher"].hybrid_ciphers(self.keys, initiator) + + +def first_packets(ref, exchange, initiator, contents): + """The key confirmation and the first application packet one side sends.""" + cipher = ref["bip324_cipher"] + send_l, send_p, _, _ = exchange.ciphers(ref, initiator) + confirmation = cipher.encrypt_packet(send_l, send_p, b"", ignore=True) + assert len(confirmation) == 20 + return confirmation, cipher.encrypt_packet(send_l, send_p, contents) + + +def receive(ref, exchange, initiator, packets): + """Decrypt packets in order as one side; return each result of decrypt_packet.""" + _, _, recv_l, recv_p = exchange.ciphers(ref, initiator) + return [ref["bip324_cipher"].decrypt_packet(recv_l, recv_p, packet) for packet in packets] + + +def make_vector(ref, index, chain, comment, shape): + mlkem, hkdf = ref["mlkem"], ref["hkdf"] + magic = MAGIC[chain] + priv_i, ellswift_i = make_key(ref, f"{index}/initiator") + priv_r, ellswift_r = make_key(ref, f"{index}/responder") + ss_ecdh = bip324_ecdh(ref, priv_i, ellswift_r, ellswift_i, initiating=True) + assert ss_ecdh == bip324_ecdh(ref, priv_r, ellswift_i, ellswift_r, initiating=False) + + # Today's BIP324 values, which the switch must leave alone (garbage terminators) or replace + # (session id). + salt_ecdh = b"bitcoin_v2_shared_secret" + magic + garbage_terminators = hkdf.hkdf_sha256(length=32, ikm=ss_ecdh, salt=salt_ecdh, info=b"garbage_terminators") + ecdh_session_id = hkdf.hkdf_sha256(length=32, ikm=ss_ecdh, salt=salt_ecdh, info=b"session_id") + + keygen_seed = seed(f"{index}/mlkem_keygen", mlkem.SEED_SIZE) + encaps_m = seed(f"{index}/mlkem_encaps", mlkem.MSG_SIZE) + ek, dk = mlkem.mlkem1024_keygen(keygen_seed) + assert mlkem.mlkem1024_check_ek(ek) + ss_mlkem, ct = mlkem.mlkem1024_encaps(ek, encaps_m) + assert mlkem.mlkem1024_decaps(dk, ct) == ss_mlkem + + offer = record(ref, PQ_MLKEM1024, ek) + accept = record(ref, PQ_MLKEM1024, ct) + assert offer[:4] == accept[:4] == bytes.fromhex("fd2106f0") and len(offer) == len(accept) == 1572 + if shape == "single": + contents_r, contents_i = offer, accept + elif shape == "extra_records": + # The first 0xF0 record wins, wherever it sits: the initiator encapsulates to the offer's ek + # and ignores the second 0xF0 record's (different, valid) ek. Unknown headers are ignored + # too, but the transcript covers every record of both contents. + other_ek, _ = mlkem.mlkem1024_keygen(seed(f"{index}/mlkem_keygen_duplicate", mlkem.SEED_SIZE)) + contents_r = offer + record(ref, PQ_MLKEM1024, other_ek) + record(ref, 0xF1, bytes.fromhex("0001")) + contents_i = record(ref, 0x01, b"qbit") + accept + else: + # A 65,535-byte unknown record: its record length (65,536) and the responder's contents + # length in the transcript (67,113) both take the 0xfe CompactSize form. + assert shape == "large_contents" + contents_r = offer + record(ref, 0xF1, bytes(65535)) + contents_i = accept + assert contents_r[1572] == 0xFE and ref["bip324_cipher"].ser_compact_size(len(contents_r))[0] == 0xFE + + exchange = Exchange(ref, ss_ecdh, ellswift_i, ellswift_r, contents_r, contents_i, ss_mlkem, magic) + if shape == "single": + assert len(exchange.transcript_input) == 3278 + init_confirmation, init_first = first_packets(ref, exchange, True, PING) + resp_confirmation, resp_first = first_packets(ref, exchange, False, PONG) + assert receive(ref, exchange, False, [init_confirmation, init_first]) == [(0, (True, b"")), (len(PING), (False, PING))] + assert receive(ref, exchange, True, [resp_confirmation, resp_first]) == [(0, (True, b"")), (len(PONG), (False, PONG))] + + vector = { + "comment": comment, + "chain": chain, + "magic": magic.hex(), + "initiator_privkey": priv_i.hex(), + "initiator_ellswift": ellswift_i.hex(), + "responder_privkey": priv_r.hex(), + "responder_ellswift": ellswift_r.hex(), + "ss_ecdh": ss_ecdh.hex(), + "ecdh_session_id": ecdh_session_id.hex(), + "initiator_garbage_terminator": garbage_terminators[:16].hex(), + "responder_garbage_terminator": garbage_terminators[16:].hex(), + "mlkem_keygen_seed": keygen_seed.hex(), + "mlkem_encaps_m": encaps_m.hex(), + "ek": ek.hex(), + "ct": ct.hex(), + "ss_mlkem": ss_mlkem.hex(), + "contents_responder": contents_r.hex(), + "contents_initiator": contents_i.hex(), + "transcript_input": exchange.transcript_input.hex(), + "transcript_hash": exchange.transcript_hash.hex(), + "salt": exchange.keys["salt"].hex(), + "ikm": exchange.keys["ikm"].hex(), + "prk": exchange.keys["prk"].hex(), + } + for label in ref["bip324_cipher"].HYBRID_KEY_LABELS: + vector[label] = exchange.keys[label].hex() + vector.update({ + "initiator_confirmation": init_confirmation.hex(), + "responder_confirmation": resp_confirmation.hex(), + "initiator_first_contents": PING.hex(), + "initiator_first_packet": init_first.hex(), + "responder_first_contents": PONG.hex(), + "responder_first_packet": resp_first.hex(), + }) + state = {"ss_ecdh": ss_ecdh, "ellswift_i": ellswift_i, "ellswift_r": ellswift_r, "dk": dk, "ct": ct, + "ss_mlkem": ss_mlkem, "contents_r": contents_r, "contents_i": contents_i, "magic": magic, + "exchange": exchange, "confirmations": {True: init_confirmation, False: resp_confirmation}} + return vector, state + + +def make_negative(ref, index, state, comment, initiator, contents_first, contents_second, ss_mlkem): + """One side derives from a different transcript or secret than its honest peer. + + Records what that side derives and sends, and what each side decodes from the other's key + confirmation: a nonzero length fails at once (reason length), a zero length on the tag.""" + bad = Exchange(ref, state["ss_ecdh"], state["ellswift_i"], state["ellswift_r"], contents_first, contents_second, + ss_mlkem, state["magic"]) + honest = state["exchange"] + assert bad.keys["session_id"] != honest.keys["session_id"] + confirmation, _ = first_packets(ref, bad, initiator, b"") + [(length, result)] = receive(ref, bad, initiator, [state["confirmations"][not initiator]]) + [(peer_length, peer_result)] = receive(ref, honest, not initiator, [confirmation]) + assert result is None and peer_result is None + negative = { + "comment": comment, + "vector": index, + "role": "initiator" if initiator else "responder", + "transcript_contents": [contents_first.hex(), contents_second.hex()], + "ss_mlkem": ss_mlkem.hex(), + "session_id": bad.keys["session_id"].hex(), + "confirmation": confirmation.hex(), + "decrypted_length": length, + "failure": "length" if length else "tag", + "peer_decrypted_length": peer_length, + "peer_failure": "length" if peer_length else "tag", + } + return negative + + +def make_confirmation_failure(ref, index, state, comment, receiver_is_initiator, packet): + """A first hybrid packet that fails the receiver's key confirmation check under the honest keys.""" + [(length, result)] = receive(ref, state["exchange"], receiver_is_initiator, [packet]) + if length: + failure = "length" + elif result is None: + failure = "tag" + else: + assert result == (False, b"") + failure = "not_decoy" + return { + "comment": comment, + "vector": index, + "receiver": "initiator" if receiver_is_initiator else "responder", + "packet": packet.hex(), + "decrypted_length": length, + "failure": failure, + } + + +def generate(): + ref = load_reference() + cipher, mlkem = ref["bip324_cipher"], ref["mlkem"] + vectors = [] + states = [] + for index, (chain, comment, shape) in enumerate(( + ("main", "Mainnet, one offer record and one accept record (the ordinary exchange).", "single"), + ("regtest", "Regtest, one offer record and one accept record.", "single"), + ("main", "Mainnet, extra records. The responder's contents are its offer, a second 0xf0 record with a " + "different valid ek, and an unknown 0xf1 record; the initiator encapsulates to the ek of the " + "first 0xf0 record and ignores the rest. The initiator's contents are an unknown 0x01 record " + "followed by its accept, which is still the first 0xf0 record there. The transcript covers every " + "record of both contents.", "extra_records"), + ("main", "Mainnet; the responder's contents are its offer followed by an unknown 0xf1 record with a " + "65,535-byte zero payload, so that record's length (65,536) and the responder's contents length " + "in the transcript (67,113) both use the 0xfe CompactSize form.", "large_contents"), + )): + vector, state = make_vector(ref, index, chain, comment, shape) + vectors.append(vector) + states.append(state) + + state = states[0] + bad_ct = bytes([state["ct"][0] ^ 0x01]) + state["ct"][1:] + bad_ss = mlkem.mlkem1024_decaps(state["dk"], bad_ct) + assert bad_ss != state["ss_mlkem"] + flipped_ss = bytes([state["ss_mlkem"][0] ^ 0xFF]) + state["ss_mlkem"][1:] + negatives = [ + make_negative(ref, 0, state, "Bit 0 of the ciphertext flipped in transit: the responder hashes the " + "damaged accept and decapsulation returns the implicit-rejection secret.", + False, state["contents_r"], record(ref, PQ_MLKEM1024, bad_ct), bad_ss), + make_negative(ref, 0, state, "The initiator's ML-KEM shared secret has byte 0 inverted after " + "encapsulation (a local fault); EK and CT are untouched.", + True, state["contents_r"], state["contents_i"], flipped_ss), + make_negative(ref, 0, state, "The responder hashes the two contents in the wrong order.", + False, state["contents_i"], state["contents_r"], state["ss_mlkem"]), + ] + + # Damaged first packets under the honest keys of vector 0, one for each confirmation failure reason. + init_confirmation = state["confirmations"][True] + resp_send_l, resp_send_p, _, _ = state["exchange"].ciphers(ref, False) + confirmation_failures = [ + make_confirmation_failure(ref, 0, state, "The initiator's confirmation with bit 0 of its first byte flipped: " + "the length decrypts to 1.", + False, bytes([init_confirmation[0] ^ 0x01]) + init_confirmation[1:]), + make_confirmation_failure(ref, 0, state, "The initiator's confirmation with bit 0 of its last tag byte flipped: " + "the length decrypts to 0 and the tag fails.", + False, init_confirmation[:-1] + bytes([init_confirmation[-1] ^ 0x01])), + make_confirmation_failure(ref, 0, state, "The responder's first hybrid packet is an authenticated empty packet " + "without the ignore bit.", + True, cipher.encrypt_packet(resp_send_l, resp_send_p, b"", ignore=False)), + ] + assert [failure["failure"] for failure in confirmation_failures] == ["length", "tag", "not_decoy"] + + document = { + "comment": [ + "Hybrid post-quantum BIP324 v2 transport: the ML-KEM-1024 key recipe under version-contents " + "header 0xf0, specified in doc/design/pq-transport.md.", + "Generated by contrib/devtools/generate-pq-transport-vectors.py from the Python reference in " + "test/functional/test_framework/crypto. Do not edit by hand.", + "Byte strings are lowercase hex. Packets are BIP324 packets: encrypted length (3) || encrypted " + "header and contents || tag (16). Each side's key confirmation is its hybrid packet 0, and its " + "first application packet is hybrid packet 1.", + "confirmation_failures are first hybrid packets that the named receiver must reject under the honest " + "keys of the vector; negative_vectors are sides that derived from different inputs than their peer.", + ], + "header": f"{PQ_MLKEM1024:02x}", + "salt_label": cipher.HYBRID_SALT_LABEL.decode(), + "labels": list(cipher.HYBRID_KEY_LABELS), + "vectors": vectors, + "confirmation_failures": confirmation_failures, + "negative_vectors": negatives, + } + return document + + +def check_spec(document): + """Return every value the specification quotes that differs from the vectors.""" + spec = SPEC.relative_to(ROOT) + blocks = SPEC_BLOCK.findall(SPEC.read_text(encoding="utf8")) + if not blocks: + return [f"{spec}: no quoted vector block found"] + errors = [] + for index, body in blocks: + if int(index) >= len(document["vectors"]): + errors.append(f"{spec}: quotes vector {index}, which does not exist") + continue + vector = document["vectors"][int(index)] + for line in body.splitlines(): + name, _, value = line.partition(" = ") + if vector.get(name) != value: + errors.append(f"{spec}: vector {index} quotes {name} = {value}, which does not match the vectors") + return errors + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--check", action="store_true", help=f"fail if {OUTPUT.relative_to(ROOT)} is stale") + args = parser.parse_args() + + document = generate() + text = json.dumps(document, indent=2) + "\n" + spec_errors = check_spec(document) + for error in spec_errors: + print(error, file=sys.stderr) + if args.check: + current = OUTPUT.read_text(encoding="utf8") if OUTPUT.exists() else None + if current != text: + print(f"{OUTPUT.relative_to(ROOT)} does not match the Python reference. If the recipe change is " + "intended, it needs a new version-contents header (doc/design/pq-transport.md); then run " + f"{Path(__file__).relative_to(ROOT)} to regenerate it.", file=sys.stderr) + sys.exit(1) + if spec_errors: + sys.exit(1) + print(f"{OUTPUT.relative_to(ROOT)} and the values {SPEC.relative_to(ROOT)} quotes match the Python reference.") + return + OUTPUT.write_text(text, encoding="utf8") + print(f"Wrote {OUTPUT.relative_to(ROOT)}") + if spec_errors: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/doc/design/pq-transport.md b/doc/design/pq-transport.md new file mode 100644 index 0000000000..c78d76ae60 --- /dev/null +++ b/doc/design/pq-transport.md @@ -0,0 +1,507 @@ +# Hybrid post-quantum v2 transport + +This document specifies how two qbit nodes add an ML-KEM-1024 key exchange to +the [BIP324](https://github.com/bitcoin/bips/blob/master/bip-0324.mediawiki) v2 +P2P transport, so that their session keys stay confidential unless both +secp256k1 ECDH and ML-KEM-1024 are broken. It is written so that another +implementation can interoperate, and it is the starting point for outside +review. The tracking issue is +[Qbit-Org/qbit#184](https://github.com/Qbit-Org/qbit/issues/184). + +The key words MUST, MUST NOT, SHOULD and MAY are to be interpreted as described +in [RFC 2119](https://www.rfc-editor.org/rfc/rfc2119) and +[RFC 8174](https://www.rfc-editor.org/rfc/rfc8174) when, and only when, they +appear in all capitals. + +## Overview + +- **Threat.** A passive observer can record v2 traffic today and, with a + cryptographically relevant quantum computer, later recover the ECDH secret + from the two 64-byte ElligatorSwift keys at the start of each connection. + Hybrid keys protect such recordings. The symmetric layer (ChaCha20-Poly1305 + with FSChaCha20 rekeying every 224 packets) is unchanged. +- **Hybrid, and ECDH stays.** Session keys come from HKDF-SHA256 over the BIP324 + ECDH secret, an ML-KEM-1024 shared secret + ([FIPS 203](https://csrc.nist.gov/pubs/fips/203/final), NIST category 5) and + a hash of the negotiation transcript. +- **No extra round trip.** The responder puts its ML-KEM encapsulation key in + its BIP324 version packet. The initiator waits for that packet, which arrives + in the same flight as today, and puts the ciphertext in its own version + packet. Both sides switch keys right after the version packets, and each + side's first packet under the new keys is a 20-byte key confirmation. +- **Compatible.** Version packet contents are ignored by BIP324 receivers, so + legacy v2 peers keep today's ECDH-only keys with no disconnect, and v1 peers + are unaffected. No consensus rule changes. +- **Cost.** 3,184 extra bytes per hybrid connection (two 1,572-byte records and + two 20-byte confirmations), and one ML-KEM key generation, encapsulation and + decapsulation. + +```text +Initiator Responder +ellswift_I || garbage_I ----------> + <---------- ellswift_R || garbage_R + <---------- terminator_R || version_R{F0 || ek} then holds +terminator_I ----------> +holds its version until version_R arrives +version_I{F0 || ct}, ECDH keys ----------> +switch to hybrid keys decapsulate; switch to hybrid keys +confirmation, hybrid keys ----------> + <---------- confirmation, hybrid keys +application packets <----------> application packets +``` + +Neither side waits for the other's confirmation before sending. A receiver +delivers application messages only after the peer's confirmation verifies. + +## Terms + +- **Contents**: the plaintext contents of a BIP324 version packet, which + BIP324 reserves for future versions. +- **Record**: one entry in the contents, as defined below. +- **Own record**: a record with header `0xF0`. +- **Offer**: the responder's own record, carrying an ML-KEM-1024 encapsulation + key `ek` (1,568 bytes). +- **Accept**: the initiator's own record, carrying an ML-KEM-1024 ciphertext + `ct` (1,568 bytes). +- **ECDH keys**: the four BIP324 packet ciphers derived from the ECDH secret + alone. **Hybrid keys**: the four packet ciphers derived by the recipe below. +- **Switch on**: a node configured to offer (as responder) and accept (as + initiator). With the switch off, a node behaves exactly as BIP324 does today. + +## Wire format + +### Records + +Version packet contents are zero or more records: + +```text +contents = record* +record = CompactSize(len) || header (1 byte) || payload (len - 1 bytes) +``` + +`len` counts the header plus the payload and MUST be at least 1. `CompactSize` +is Bitcoin's variable-length integer, and a receiver MUST reject non-canonical +encodings as a parse failure. The shape follows the bitcoin-dev +[Transport Feature Negotiation](https://gnusha.org/pi/bitcoindev/29174459-1523-4cf8-a20f-5ce1abc302c3n@googlegroups.com/) +draft. + +### Header registry + +| Header | Meaning | +|---|---| +| `0x00` | Reserved. | +| `0xF0` | `PQ_MLKEM1024`: the ML-KEM-1024 offer (sent by the responder) and accept (sent by the initiator). The sender's role says which one a record is. | + +qbit assigns its own header values from `0xF0` upward, far from the low values +a Transport Feature Negotiation table would assign first. Every other value is +unassigned, and a receiver ignores records with headers it doesn't know. + +### Offer and accept + +- The responder sends an offer only when its switch is on. The offer's payload + is the 1,568-byte encapsulation key `ek`. +- The initiator sends an accept only after it received a valid offer and its + switch is on. The accept's payload is the 1,568-byte ciphertext `ct`. + Otherwise the initiator sends empty contents. +- qbit sends the offer and the accept as the entire contents, but a receiver + MUST handle other records alongside them as the parse rules below say. + +| Object | Layout | Bytes | +|---|---|---:| +| Offer contents | `fd 21 06 f0 ‖ ek` | 1,572 | +| Accept contents | `fd 21 06 f0 ‖ ct` | 1,572 | +| Record length | CompactSize 1,569 (header plus payload) | `fd 21 06` | +| Encrypted version packet | encrypted length 3 ‖ encrypted payload (header `00` ‖ contents) 1,573 ‖ tag 16 | 1,592 | +| Encrypted confirmation | encrypted length 3 (plaintext `00 00 00`) ‖ encrypted header 1 (plaintext `80`) ‖ tag 16 | 20 | + +**AAD.** Both version packets travel under the ECDH keys, with BIP324's +associated data rule: a sender's first packet in its direction authenticates +its garbage, so if a decoy already consumed the garbage, the version packet +uses empty AAD. Confirmations always use empty AAD. + +### Parse rules + +A receiver parses the contents in one pass: + +1. **The whole contents must parse as records.** Empty contents, or a parse + failure anywhere (truncation, a zero length, a non-canonical or oversized + length, trailing bytes), mean "no features": the connection continues with + ECDH keys and MUST NOT be dropped for it. This deliberately differs from the + Transport Feature Negotiation draft, which disconnects on unparseable + contents: BIP324 receivers ignore version contents today, and qbit v1.0.0 + relies on that. +2. **The first own record wins.** Later `0xF0` records and records with unknown + headers are ignored, but the transcript still covers them. +3. **Only after the whole grammar validates** is the first own record checked. + An offer's payload MUST be exactly 1,568 bytes and MUST pass the FIPS 203 + section 7.2 encapsulation key check (every encoded coefficient below q). An + accept's payload MUST be exactly 1,568 bytes. A malformed first own record + disconnects the connection. +4. **A corrupted `ct` of the right length can't be detected**, because ML-KEM + decapsulation rejects implicitly. The two sides derive different keys, and + the key confirmation fails at once. + +A responder that sent no offer MUST ignore an own record in the initiator's +contents and never decapsulate. A node whose switch is off MUST NOT parse or +validate the peer's contents. qbit's receive limit for any packet, 4,000,013 +bytes of contents, bounds the parser's work. + +## Message rules + +### Responder + +- It MUST send its version packet after its garbage terminator without waiting + to receive anything more from the initiator, such as the initiator's garbage + terminator or version packet. It MAY send decoy packets before the version + packet, as BIP324 allows. BIP324 implementations already behave this way. +- After sending an offer, it MUST send nothing more, decoys included, until it + has processed the initiator's version packet. +- On the initiator's version packet: with no own record, or contents that fail + to parse, it keeps the ECDH keys and discards its ML-KEM state. With a + malformed accept, it disconnects. With a valid-length accept, it decapsulates + and switches. + +### Initiator + +- With its switch on, it MAY hold its version packet until the responder's + version packet arrives, and then sends the ciphertext in it. With its switch + off, it sends its version packet at once, as today. +- On the responder's version packet: with no own record, or contents that fail + to parse, it sends empty contents and keeps the ECDH keys. With a malformed + offer, it disconnects. With a valid offer, it encapsulates, sends the accept + under the ECDH keys and switches. +- It MAY send decoy packets before its version packet, under the ECDH keys. + +### Why progress still holds + +BIP324's version negotiation phase lists the initiator's steps as receiving a +packet and then sending its own version packet, and notes that version packets +*can* be sent right after the garbage terminator. Deployed initiators send at +once; a hybrid qbit initiator forgoes that optional immediate send and uses the +listed order. All of the progress conditions BIP324 sets for handshake +upgrades still hold: in particular, after sending its garbage terminator each +party moves to version negotiation without waiting for more bytes. The +initiator's wait always ends, because every responder, legacy or hybrid, MUST +send its version packet without waiting to receive anything more from the +initiator. Responders never hold before their own version packet. + +BIP324 names post-quantum cryptography upgrades to the handshake among the +features its version negotiation phase is meant to carry. + +## Key schedule + +### Retained secret + +Each side keeps the raw 32-byte BIP324 ECDH secret `ss_ECDH` (the tagged hash +`bip324_ellswift_xonly_ecdh` that BIP324's HKDF consumes) past its normal +lifetime, until it either switches or decides to stay on the ECDH keys. An +implementation SHOULD wipe it on every path. + +### Transcript + +```text +TH = SHA256( ellswift_I (64) || ellswift_R (64) + || CompactSize(len(contents_R)) || contents_R + || CompactSize(len(contents_I)) || contents_I ) +``` + +- `ellswift_I` and `ellswift_R` are the 64-byte ElligatorSwift keys the + initiator and the responder sent, in that order, whatever the local role. +- `contents_R` and `contents_I` are the full version packet contents each side + sent, responder first, including unknown and duplicate records. +- Packet framing, garbage, terminators, tags, decoys and confirmations are not + hashed. +- For the ordinary one-record exchange, the TH input is 3,278 bytes. + +### Derivation + +HKDF-SHA256 ([RFC 5869](https://www.rfc-editor.org/rfc/rfc5869)), with one +32-byte output block per label: + +```text +salt = ASCII("qbit_v2_hybrid_mlkem1024") || network_magic 24 + 4 = 28 bytes, no NUL +IKM = ss_ECDH (32) || ss_MLKEM (32) || TH (32) 96 bytes +PRK = HMAC-SHA256(key = salt, msg = IKM) +initiator_L, initiator_P, responder_L, responder_P, session_id + = HMAC-SHA256(PRK, ASCII(label) || 0x01) 32 bytes each +``` + +The network magic is the chain's 4-byte message start in wire order, the same +bytes BIP324's salt uses: mainnet `44 4f 24 a8`, regtest `a6 6b 1f da`. +`ss_MLKEM` is the 32-byte ML-KEM-1024 shared secret: the output of +encapsulation at the initiator and of decapsulation at the responder. + +### Switch + +- Switching replaces all four packet ciphers at once: the initiator sends with + `initiator_L` and `initiator_P` and receives with `responder_L` and + `responder_P`, and the responder the reverse. Each new cipher starts with its + packet counter at 0, and BIP324's rekeying every 224 packets then derives + from the hybrid keys. +- The session id becomes the hybrid `session_id`. +- The garbage terminators are not re-derived. +- The initiator switches right after queuing its version packet, and the + responder right after processing the initiator's version packet. + +### Recipe changes + +The recipe, its labels, salt and transcript layout are a wire contract between +releases. Any change takes a new header with its own salt label. A label-only +change under the same header would let two adjacent releases both accept the +offer, derive different keys and disconnect at the first hybrid packet. + +## The switch boundary + +| Direction | Packets | Keys | +|---|---|---| +| Initiator to responder | ElligatorSwift key, garbage, garbage terminator | none (plaintext) | +| Initiator to responder | decoys before the version packet; the version packet (accept or empty) | ECDH | +| Initiator to responder | after a switch: confirmation (hybrid packet 0), then decoys and application packets | hybrid | +| Responder to initiator | ElligatorSwift key, garbage, garbage terminator | none (plaintext) | +| Responder to initiator | decoys before the version packet; the version packet (offer or empty) | ECDH | +| Responder to initiator | after a switch: confirmation (hybrid packet 0), then decoys and application packets | hybrid | + +Without a switch, every packet after the garbage terminators uses the ECDH +keys, exactly as in BIP324. + +## Key confirmation + +**Sending.** Right after switching, a node MUST send a key confirmation as its +first packet under the hybrid keys: an empty decoy packet with header `0x80` +and empty AAD, 20 bytes on the wire. It consumes hybrid packet 0 in that +direction. A node MUST NOT wait for the peer's confirmation before sending +application packets. + +**Receiving.** The first packet a node receives under the hybrid keys MUST meet +all of the following, and otherwise the node MUST disconnect at once: + +- its decrypted length is 0, checked as soon as the 3 length bytes arrive, + before waiting for or allocating a body; +- its tag verifies; +- its ignore bit is set. Other header bits are ignored, as BIP324 specifies. + +A node MUST NOT deliver application messages, or report the connection as +hybrid, until the peer's confirmation verifies. Without this check, a key +mismatch decrypts to a random length, and about 24% of the time the receiver +would wait until its handshake timeout for bytes that never arrive. + +## Failure handling + +- Internal ML-KEM errors not caused by the peer never crash the node. A + responder that can't generate a key sends no offer. An initiator that can't + check the key or encapsulate continues as if its switch were off for that + connection. A responder whose decapsulation fails internally closes the + connection, because the initiator has already switched. +- Key generation draws two independent 32-byte strong random values for the + 64-byte seed, and encapsulation draws one 32-byte value. +- The decapsulation key, the retained ECDH secret, ML-KEM temporaries and + derived keys are wiped on switch, on staying with ECDH keys, at disconnect + and timeout, and in destructors. + +How qbit counts, logs and reports these outcomes, its fallback after repeated +failures and its load shedding are local policy, described in +[#184](https://github.com/Qbit-Org/qbit/issues/184); they don't change the wire +behavior above. + +qbit reports each v2 connection's negotiation status as one of these values: + +- `pending` from the start of the handshake until one of the outcomes below; +- `hybrid` once the peer's confirmation verifies; +- `legacy_peer` when the peer's version packet carries no usable offer or + accept, so the ECDH keys stay; +- `off` when the switch is off, or after a local key generation, key check or + encapsulation fault that left the connection on plain v2; +- `fallback` for an outbound connection that runs plain v2 because of earlier + failures; inbound connections never fall back. + +There is no failed status. A connection that closes on any failure above or +in key confirmation, local faults included, reports `pending` until it is +removed. + +## Security notes + +- **Version contents have ECDH-only protection.** The EK, the CT and any future + record travel under the ECDH keys. A future attacker who breaks ECDH learns + them, which is fine for ML-KEM's public values but matters for any future + record that carries secrets. +- **The guarantee holds for confirmed hybrid connections.** A switch that is + off on either side, a local fault, load shedding and fallback all permit + ECDH-only sessions. `getpeerinfo` reports `transport_pq` only once the peer's + confirmation verified. +- **This is not authentication.** BIP324 stays unauthenticated, and an active + man-in-the-middle can still relay two separate sessions. The hybrid session + id can be compared out of band the same way BIP324's can. +- **Downgrade.** Stripping an offer or an accept means forging packets under + the ECDH keys, so it takes an active attacker who breaks ECDH in real time + or sits in the middle, and an unauthenticated session can't stop the latter + anyway. A passive recorder can't downgrade a connection. +- **Transcript binding.** Both keys, both contents and the network magic feed + the derivation, so two sides that saw different negotiation bytes derive + different keys and fail the confirmation instead of continuing. +- **Distinguishability.** Hybrid sessions are distinguishable from legacy v2 + by packet sizes. Users who need to hide that should use Tor or I2P. +- **Forward secrecy.** Each connection uses a fresh ML-KEM key pair, and the + decapsulation key is wiped after the switch. + +## Interop recipe + +The vectors file +[`src/test/data/pq_transport_vectors.json`](../../src/test/data/pq_transport_vectors.json) +holds every input and intermediate. All byte strings are lowercase hex. To +check an implementation against vector 0: + +1. Select the chain in `chain` and check that its message start is `magic`. +2. Take `initiator_privkey` with `initiator_ellswift` and `responder_privkey` + with `responder_ellswift`. Each side's BIP324 ECDH gives `ss_ecdh`; BIP324's + own HKDF gives `ecdh_session_id` and the two garbage terminators, which the + switch leaves alone. +3. Run ML-KEM-1024 KeyGen on `mlkem_keygen_seed` (`d || z`) to get `ek`, and + Encaps on `ek` with `mlkem_encaps_m` to get `ct` and `ss_mlkem`. + Decapsulating `ct` with the decapsulation key gives `ss_mlkem` too. +4. Build `contents_responder` (`fd 21 06 f0 || ek`) and `contents_initiator` + (`fd 21 06 f0 || ct`). In vector 2 the responder's offer is followed by a + second `0xF0` record with a different valid `ek` and an unknown `0xF1` + record, and the initiator's accept is preceded by an unknown `0x01` record: + the KEM uses the first `0xF0` record of each contents, and the transcript + covers every record. Vector 3 appends a 65,535-byte unknown record to the + offer, so that record's length and the responder's contents length in the + transcript both take the `0xfe` CompactSize form. +5. Concatenate the transcript input and compare it with `transcript_input`, and + its SHA256 with `transcript_hash`. +6. Compare `salt`, `ikm`, `prk` and the five derived values. +7. Encrypt the initiator's confirmation (empty contents, header `0x80`, empty + AAD) with fresh ciphers keyed by `initiator_L` and `initiator_P`; it must + equal `initiator_confirmation`. Hybrid packet 1 with + `initiator_first_contents` (a short-id `ping`) must equal + `initiator_first_packet`. Do the same for the responder. +8. For each entry in `negative_vectors`, the side named by `role` derives from + `transcript_contents` and `ss_mlkem` instead of the honest values of vector + `vector`. It must + get `session_id` and send `confirmation`, and decrypting the honest peer's + confirmation must fail with `decrypted_length` (reason `length` when nonzero, + else `tag`); the honest peer fails on its confirmation with + `peer_decrypted_length`. +9. Each entry in `confirmation_failures` is a first hybrid packet that the side + named by `receiver` must reject under the honest keys of vector `vector`: + the length decrypts to `decrypted_length`, and the check fails with + `failure`, one of `length`, `tag` (a zero length with a damaged tag) and + `not_decoy`. + +### Vector 0 + + +```text +magic = 444f24a8 +initiator_ellswift = 6b5fb320cab84c788ef6393262172b222facedee5e55f4e2fa44bae00b1f36102dd8739200e025a0443a6c6d3d52f003e4b94d296ed3a68e9dacc567f0c7bd09 +responder_ellswift = 3d3449542df2dc02aedca2a14e5c6b22f2b797b3191488cdbdc1ff6ad3ef40a77e62023395a490bbacc9de3da7fd37f70592ec361e34c01c761ad4ea2029712c +ss_ecdh = 7e8dd0eda7d613290fcfc7919af8675a2405e6585378777b9d2f02e592f1fb1d +ecdh_session_id = ab8e9ae6fe9d33e2e10f847855298fbe295ed45625608b425ca118d09dbfd0a4 +ss_mlkem = 22c8bf5c4858ac2e5f94b6e945ca142ccc28b98e7b954a78899630a7bfa3135f +transcript_hash = 66f0655505928bdeec6ce4cf2b1de08ff16c2cb1cf04b781011e30ca6687c240 +salt = 716269745f76325f6879627269645f6d6c6b656d31303234444f24a8 +prk = cacbbffc106ea035465bc498d22562e83989c039390ddc1ea36e0fe7ba0ac6b4 +initiator_L = d723af332d033d229fe91db011f04cd005657fcc35d59b93b6771e77dc9bd572 +initiator_P = 72462d2ed71f2b6b9c7435745166155bbd61ab04e2e21798945d1282f690e7d2 +responder_L = 167756b477acb286099da2a866ad9e2d94ec719236dc61836f2956451148ec12 +responder_P = 95eb7e61e0e575bd36db2a0b589553984942db0da9f43f8d8bc48012f2323d21 +session_id = c749d7dadf82b12615fa4ccdeef8cbce4ffb935ad22a7eac1fbc52d8d1394ea4 +initiator_confirmation = 69abb2b1c9601dad7baf8038bc3601eaa42527de +responder_confirmation = a634faec713fb192a52b9e810216b8a36d39516b +initiator_first_packet = 4bbe69dca55d32377b3cee47741171e83115bae7d56d7a9a81dd94394a +responder_first_packet = be28dfc2f7f52321d961b644a0ff9fb473d4f9055c6e9523c696751721 +``` + +The generator's `--check` mode fails if any value above differs from the file. + +## Convergence policy and recipe retirement + +- qbit tracks bitcoin-dev, bitcoin/bips and upstream transport changes such as + [bitcoin/bitcoin#36376](https://github.com/bitcoin/bitcoin/issues/36376) + (rare BIP324 decoys). The rule against decoys between an offer and the + initiator's version packet binds only offering qbit responders, so upstream + decoys from legacy peers stay compatible. +- If a Bitcoin standard for post-quantum transport ships, qbit implements it as + a new header. At most two post-quantum recipes are supported at once, and + while two exist, `getpeerinfo` reports which one each hybrid peer uses. +- The `0xF0` recipe stays for at least two major releases after that standard + ships. The release before its removal announces it in its release notes and + warns at startup while `0xF0` connections are still in use. + +## Discussion + +This specification, its Python reference and the vectors are to be posted to +the bitcoin-dev threads +[Transport Feature Negotiation](https://gnusha.org/pi/bitcoindev/29174459-1523-4cf8-a20f-5ce1abc302c3n@googlegroups.com/) +and +[A Post-Quantum Path for BIP 324](https://groups.google.com/g/bitcoindev/c/n_5WuKVYqwI/), +asking for a reserved private-use header range and discussing ML-KEM-768 +against ML-KEM-1024. Links to those posts will be added here. + +## For contributors + +### Build and test + +```sh +cmake -B build +cmake --build build -j 8 --target test_bitcoin # builds build/bin/test_qbit +ctest --test-dir build -R bip324_tests +build/bin/test_qbit --run_test=bip324_tests +``` + +`bip324_tests` replays the vectors file (`hybrid_vectors`) and checks +transcript binding, API misuse and rekey boundaries. The Python reference has +its own unit tests: + +```sh +build/test/functional/test_runner.py feature_framework_unit_tests.py +``` + +The `bip324_cipher_roundtrip` fuzz target covers the switch, mismatched +transcripts, damaged packets and rekeys: + +```sh +cmake -B build_fuzz -DBUILD_FOR_FUZZING=ON -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \ + -DSANITIZERS=fuzzer,address,undefined +cmake --build build_fuzz --target fuzz +FUZZ=bip324_cipher_roundtrip build_fuzz/bin/fuzz +``` + +### ML-KEM backends + +qbit uses native ML-KEM code on x86_64 with AVX2, detected at run time, and on +macOS arm64, and portable C everywhere else. That includes AArch64 Linux and +every other AArch64 ELF build: upstream's AArch64 assembly has no BTI landing +pads or GNU property note, so linking it would drop BTI/PAC/GCS marking from the +whole binary, and hardening wins. The CMake option +`WITH_MLKEM_NATIVE=AUTO|ON|OFF` controls the build: `OFF` builds portable C +only, and `ON` makes missing native support a configure error, as on AArch64 +ELF. At runtime, the hidden `-mlkemportable` option, and +`-test=mlkem_portable` on regtest, force portable C; they arrive with the +operator options ([#209](https://github.com/Qbit-Org/qbit/issues/209)). + +### Regenerating the vectors + +```sh +contrib/devtools/generate-pq-transport-vectors.py # rewrite the vectors file +contrib/devtools/generate-pq-transport-vectors.py --check # what the lint runs +``` + +The generator imports the Python reference from +`test/functional/test_framework/crypto` (`bip324_cipher.py` for the recipe, +`mlkem.py` for ML-KEM-1024) and never calls the C++ code. Every input comes +from a fixed label, so the output is byte-for-byte reproducible. Changing the +recipe changes the vectors, and both `--check` and `bip324_tests` fail; a +deliberate recipe change needs a new header. + +## References + +- [BIP324](https://github.com/bitcoin/bips/blob/master/bip-0324.mediawiki), + the v2 P2P transport, and its version negotiation phase. +- [FIPS 203](https://csrc.nist.gov/pubs/fips/203/final), ML-KEM. +- [RFC 5869](https://www.rfc-editor.org/rfc/rfc5869), HKDF. +- [mlkem-native v2.0.0](https://github.com/pq-code-package/mlkem-native/tree/v2.0.0), + the C library qbit uses. +- bitcoin-dev, + [Transport Feature Negotiation](https://gnusha.org/pi/bitcoindev/29174459-1523-4cf8-a20f-5ce1abc302c3n@googlegroups.com/) + (2026-09-17), and + [A Post-Quantum Path for BIP 324](https://groups.google.com/g/bitcoindev/c/n_5WuKVYqwI/). diff --git a/doc/development/fuzzing.md b/doc/development/fuzzing.md index 4d43732d84..bd99a5eff0 100644 --- a/doc/development/fuzzing.md +++ b/doc/development/fuzzing.md @@ -41,7 +41,10 @@ $ ctest --test-dir build_fuzz -R '^required_fuzz_targets$' --output-on-failure ## qbit seed corpora `qa-assets` has no inputs for `asert_chain_transition`, `asert_edge_cases`, -`asert_math`, `auxpow`, `mlkem`, `mlkem_backend_diff`, `p2mr_script` or `pqc`. +`asert_math`, `auxpow`, `mlkem`, `mlkem_backend_diff`, `p2mr_script`, +`p2p_v2_pq_malicious_peer`, `pq_records` or `pqc`, and its inputs for +`p2p_transport_bidirectional_v2` and `p2p_transport_bidirectional_v1v2` predate +the hybrid v2 negotiation. Generated seeds for these targets, their manifest and the input format are described in [`test/fuzz/qbit_corpora/README.md`](../../test/fuzz/qbit_corpora/README.md). @@ -52,7 +55,7 @@ $ test/fuzz/qbit_corpora/overlay.py qa-assets/fuzz_corpora $ build_fuzz/test/fuzz/test_runner.py --require_qbit_corpus qa-assets/fuzz_corpora ``` -`--require_qbit_corpus` fails if any of the eight targets is not compiled or not +`--require_qbit_corpus` fails if any of the twelve targets is not compiled or not selected (including through `--exclude`), has no regular input files, or is not reported as replayed by the fuzz executable. The runner prints the number of input files present and replayed for each target. @@ -68,7 +71,7 @@ replay with a mutation phase of that many seconds per target. The phase starts from the corpus, writes new inputs to a temporary directory and leaves the corpus unchanged. It fails on non-libFuzzer builds. The nightly native fuzz job sets `QBIT_FUZZ_MUTATE_MIN_TIME`, which makes `ci/test/03_test_script.sh` run -this phase for the eight targets after the normal replay. +this phase for the twelve targets after the normal replay. The budget is libFuzzer's `-max_total_time`, which counts time spent loading the seed corpus (libFuzzer's `INITED` line) as well as mutation. A run can end diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0e05808cdc..7d7ce68d34 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -191,6 +191,7 @@ add_library(bitcoin_node STATIC EXCLUDE_FROM_ALL auxpow_validation.cpp banman.cpp bip324.cpp + bip324_pq.cpp blockencodings.cpp blockfilter.cpp consensus/tx_verify.cpp diff --git a/src/bip324.cpp b/src/bip324.cpp index 9c5a1572bb..3ba2c8aeaa 100644 --- a/src/bip324.cpp +++ b/src/bip324.cpp @@ -8,9 +8,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -22,6 +24,20 @@ #include #include +namespace { + +/** Serialization stream that feeds every written byte into a running SHA256, without copying. */ +class TranscriptWriter +{ + CSHA256& m_hasher; + +public: + explicit TranscriptWriter(CSHA256& hasher) noexcept : m_hasher{hasher} {} + void write(std::span src) noexcept { m_hasher.Write(UCharCast(src.data()), src.size()); } +}; + +} // namespace + BIP324Cipher::BIP324Cipher(const CKey& key, std::span ent32) noexcept : m_key(key) { @@ -31,7 +47,12 @@ BIP324Cipher::BIP324Cipher(const CKey& key, std::span ent32) no BIP324Cipher::BIP324Cipher(const CKey& key, const EllSwiftPubKey& pubkey) noexcept : m_key(key), m_our_pubkey(pubkey) {} -void BIP324Cipher::Initialize(const EllSwiftPubKey& their_pubkey, bool initiator, bool self_decrypt) noexcept +BIP324Cipher::~BIP324Cipher() +{ + DiscardHybridSecret(); +} + +void BIP324Cipher::Initialize(const EllSwiftPubKey& their_pubkey, bool initiator, bool self_decrypt, bool retain_for_hybrid) noexcept { // Determine salt (fixed string + network magic bytes) const auto& message_header = Params().MessageStart(); @@ -63,6 +84,19 @@ void BIP324Cipher::Initialize(const EllSwiftPubKey& their_pubkey, bool initiator // Derive session id from shared secret. hkdf.Expand32("session_id", UCharCast(m_session_id.data())); + // Keep what SwitchToHybrid() needs: the ECDH secret itself, the transcript started with the + // actual-role public keys (initiator first), the network magic and the cipher direction. + if (retain_for_hybrid) { + m_retained_ecdh = ecdh_secret; + m_hybrid_transcript.emplace(); + const EllSwiftPubKey& ellswift_initiator{initiator ? m_our_pubkey : their_pubkey}; + const EllSwiftPubKey& ellswift_responder{initiator ? their_pubkey : m_our_pubkey}; + m_hybrid_transcript->Write(UCharCast(ellswift_initiator.data()), ellswift_initiator.size()); + m_hybrid_transcript->Write(UCharCast(ellswift_responder.data()), ellswift_responder.size()); + std::copy(std::begin(message_header), std::end(message_header), m_hybrid_magic.begin()); + m_hybrid_send_is_initiator = side; + } + // Wipe all variables that contain information which could be used to re-derive encryption keys. memory_cleanse(ecdh_secret.data(), ecdh_secret.size()); memory_cleanse(hkdf_32_okm.data(), sizeof(hkdf_32_okm)); @@ -70,6 +104,63 @@ void BIP324Cipher::Initialize(const EllSwiftPubKey& their_pubkey, bool initiator m_key = CKey(); } +bool BIP324Cipher::AddVersionContents(std::span contents) noexcept +{ + if (!m_hybrid_transcript || m_version_contents_count >= 2) return false; + TranscriptWriter writer{*m_hybrid_transcript}; + WriteCompactSize(writer, contents.size()); + writer.write(contents); + ++m_version_contents_count; + return true; +} + +bool BIP324Cipher::SwitchToHybrid(std::span ss_mlkem) noexcept +{ + if (!*this || !m_retained_ecdh || !m_hybrid_transcript || m_version_contents_count != 2 || m_hybrid_switched) return false; + + // Determine salt (fixed string + network magic bytes, as snapshotted by Initialize). + std::string salt = std::string{"qbit_v2_hybrid_mlkem1024"} + std::string(std::begin(m_hybrid_magic), std::end(m_hybrid_magic)); + + // IKM = ss_ECDH (32) || ss_MLKEM (32) || TH (32), where finishing the transcript gives TH. + std::array ikm; + std::copy(m_retained_ecdh->begin(), m_retained_ecdh->end(), ikm.begin()); + std::copy(ss_mlkem.begin(), ss_mlkem.end(), ikm.begin() + ECDH_SECRET_SIZE); + m_hybrid_transcript->Finalize(UCharCast(ikm.data() + ECDH_SECRET_SIZE + 32)); + + // Derive the hybrid keys, and re-emplace all four ciphers, so their counters start at 0. + bool side = m_hybrid_send_is_initiator; + CHKDF_HMAC_SHA256_L32 hkdf(UCharCast(ikm.data()), ikm.size(), salt); + std::array hkdf_32_okm; + hkdf.Expand32("initiator_L", UCharCast(hkdf_32_okm.data())); + (side ? m_send_l_cipher : m_recv_l_cipher).emplace(hkdf_32_okm, REKEY_INTERVAL); + hkdf.Expand32("initiator_P", UCharCast(hkdf_32_okm.data())); + (side ? m_send_p_cipher : m_recv_p_cipher).emplace(hkdf_32_okm, REKEY_INTERVAL); + hkdf.Expand32("responder_L", UCharCast(hkdf_32_okm.data())); + (side ? m_recv_l_cipher : m_send_l_cipher).emplace(hkdf_32_okm, REKEY_INTERVAL); + hkdf.Expand32("responder_P", UCharCast(hkdf_32_okm.data())); + (side ? m_recv_p_cipher : m_send_p_cipher).emplace(hkdf_32_okm, REKEY_INTERVAL); + + // Replace the session id. The garbage terminators are not re-derived. + hkdf.Expand32("session_id", UCharCast(m_session_id.data())); + + // Wipe everything that could re-derive the keys, including the retained secret. + memory_cleanse(ikm.data(), ikm.size()); + memory_cleanse(hkdf_32_okm.data(), hkdf_32_okm.size()); + memory_cleanse(&hkdf, sizeof(hkdf)); + DiscardHybridSecret(); + m_hybrid_switched = true; + return true; +} + +void BIP324Cipher::DiscardHybridSecret() noexcept +{ + if (m_retained_ecdh) { + memory_cleanse(m_retained_ecdh->data(), m_retained_ecdh->size()); + m_retained_ecdh.reset(); + } + m_hybrid_transcript.reset(); +} + void BIP324Cipher::Encrypt(std::span contents, std::span aad, bool ignore, std::span output) noexcept { assert(output.size() == contents.size() + EXPANSION); diff --git a/src/bip324.h b/src/bip324.h index c6b82cb969..4d80becfc5 100644 --- a/src/bip324.h +++ b/src/bip324.h @@ -7,10 +7,12 @@ #include #include +#include #include #include #include +#include #include #include #include @@ -40,10 +42,23 @@ class BIP324Cipher std::array m_send_garbage_terminator; std::array m_recv_garbage_terminator; + /** Hybrid key schedule state, only kept when Initialize() was called with retain_for_hybrid. */ + std::optional m_retained_ecdh; + std::optional m_hybrid_transcript; + std::array m_hybrid_magic{}; + uint8_t m_version_contents_count{0}; + bool m_hybrid_send_is_initiator{false}; + bool m_hybrid_switched{false}; + public: /** No default constructor; keys must be provided to create a BIP324Cipher. */ BIP324Cipher() = delete; + /** Wipes any retained hybrid secret. */ + ~BIP324Cipher(); + BIP324Cipher(const BIP324Cipher&) = delete; + BIP324Cipher& operator=(const BIP324Cipher&) = delete; + /** Initialize a BIP324 cipher with specified key and encoding entropy (testing only). */ BIP324Cipher(const CKey& key, std::span ent32) noexcept; @@ -58,8 +73,35 @@ class BIP324Cipher * initiator is set to true if we are the initiator establishing the v2 P2P connection. * self_decrypt is only for testing, and swaps encryption/decryption keys, so that encryption * and decryption can be tested without knowing the other side's private key. + * retain_for_hybrid keeps the ECDH secret and starts the hybrid transcript, so that + * SwitchToHybrid() can later replace the keys. Without it, the ECDH secret is wiped here. + */ + void Initialize(const EllSwiftPubKey& their_pubkey, bool initiator, bool self_decrypt = false, + bool retain_for_hybrid = false) noexcept; + + /** Add the full version packet contents of one side to the hybrid transcript, the + * responder's first and the initiator's second. + * + * Returns false, changing nothing, unless Initialize() retained the hybrid state, no + * DiscardHybridSecret() or SwitchToHybrid() happened since, and fewer than two contents + * were added. */ - void Initialize(const EllSwiftPubKey& their_pubkey, bool initiator, bool self_decrypt = false) noexcept; + [[nodiscard]] bool AddVersionContents(std::span contents) noexcept; + + /** Replace all four packet ciphers (counters at 0) and the session id with keys derived from + * the retained ECDH secret, the ML-KEM shared secret and the transcript. The garbage + * terminators are unchanged, and the retained secret is wiped. + * + * Returns false, changing nothing, unless the cipher is initialized, the hybrid state is + * retained, both version contents were added and no earlier switch happened. + */ + [[nodiscard]] bool SwitchToHybrid(std::span ss_mlkem) noexcept; + + /** Wipe the retained ECDH secret and reset the transcript; the current ciphers stay in use. */ + void DiscardHybridSecret() noexcept; + + /** Whether the retained ECDH secret or the transcript is still held. */ + bool HoldsHybridSecret() const noexcept { return m_retained_ecdh.has_value() || m_hybrid_transcript.has_value(); } /** Determine whether this cipher is fully initialized. */ explicit operator bool() const noexcept { return m_send_l_cipher.has_value(); } diff --git a/src/bip324_pq.cpp b/src/bip324_pq.cpp new file mode 100644 index 0000000000..285da14ebe --- /dev/null +++ b/src/bip324_pq.cpp @@ -0,0 +1,200 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#include + +#include +#include +#include +#include +#include + +#include +#include + +namespace { + +void StrongFill32(void*, std::span out) noexcept +{ + GetStrongRandBytes(out); +} + +const PQKemOps DEFAULT_KEM_OPS{ + .keygen = &mlkem::KeyGen, + .check_public_key = &mlkem::CheckPublicKey, + .encaps = &mlkem::Encaps, + .decaps = &mlkem::Decaps, +}; + +} // namespace + +PQRandomSource DefaultPQRandomSource() noexcept +{ + return {.fill32 = &StrongFill32, .context = nullptr}; +} + +const PQKemOps& DefaultPQKemOps() noexcept +{ + return DEFAULT_KEM_OPS; +} + +PQHandshake::ParsedContents PQHandshake::ParseContents(std::span contents) noexcept +{ + try { + SpanReader reader{contents}; + std::optional> first; + while (!reader.empty()) { + // ReadCompactSize rejects non-canonical encodings and truncation by throwing. + const uint64_t len{ReadCompactSize(reader)}; + // Check the length against what is left before reading or skipping anything, as + // SpanReader::ignore() does not check bounds. + if (len == 0 || len > reader.size()) return {.kind = ParseKind::INVALID_GRAMMAR}; + uint8_t header; + reader >> header; + const size_t payload_len{size_t(len - 1)}; + if (header == PQ_MLKEM1024 && !first) { + first = contents.subspan(contents.size() - reader.size(), payload_len); + } + reader.ignore(payload_len); + } + if (first) return {.kind = ParseKind::OWN_RECORD, .payload = *first}; + return {.kind = ParseKind::NO_RECORD}; + } catch (const std::ios_base::failure&) { + return {.kind = ParseKind::INVALID_GRAMMAR}; + } +} + +PQHandshake::Record PQHandshake::SerializeRecord(std::span payload) noexcept +{ + // CompactSize(1 + 1568) = CompactSize(0x0621) = fd 21 06. + static_assert(1 + mlkem::PUBLIC_KEY_BYTES == 0x0621); + Record record; + record[0] = std::byte{0xfd}; + record[1] = std::byte{0x21}; + record[2] = std::byte{0x06}; + record[3] = std::byte{PQ_MLKEM1024}; + std::ranges::copy(std::as_bytes(payload), record.begin() + 4); + return record; +} + +PQHandshake::PQHandshake(bool initiating, PQMode mode) noexcept + : PQHandshake{initiating, mode, DefaultPQRandomSource(), DefaultPQKemOps()} {} + +PQHandshake::PQHandshake(bool initiating, PQMode mode, PQRandomSource random, const PQKemOps& ops) noexcept + : m_initiating{initiating}, m_mode{mode}, m_random{random}, m_ops{ops} {} + +PQHandshake::~PQHandshake() +{ + ClearSecrets(); +} + +std::optional PQHandshake::CheckRecordLength(std::span payload) noexcept +{ + // Only a negotiating initiator before it accepts, or a responder holding its offer's key, + // has a record to check: anything else must ignore the peer's records. + if (!Assume(m_mode == PQMode::NEGOTIATE && (m_initiating ? m_snapshot.offer == PQOfferState::NONE : m_dk != nullptr))) { + SetFailure(PQFailure::CIPHER_STATE_INTERNAL); + return std::nullopt; + } + if (payload.size() != mlkem::PUBLIC_KEY_BYTES) { + SetFailure(m_initiating ? PQFailure::EK_LENGTH : PQFailure::CT_LENGTH); + return std::nullopt; + } + return Payload{UCharCast(payload.data()), mlkem::PUBLIC_KEY_BYTES}; +} + +mlkem::Error PQHandshake::MakeOffer(Record& offer) noexcept +{ + // Only a negotiating responder offers, and only once. + if (!Assume(!m_initiating && m_mode == PQMode::NEGOTIATE && m_snapshot.offer == PQOfferState::NONE && !m_dk)) { + SetFailure(PQFailure::KEYGEN_INTERNAL); + return mlkem::Error::INTERNAL; + } + // ProcRand allows at most 32 bytes per call, so the 64-byte seed d || z takes two draws. + mlkem::Secret seed; + m_random.fill32(m_random.context, seed.Bytes().first<32>()); + m_random.fill32(m_random.context, seed.Bytes().last<32>()); + auto dk{std::make_unique()}; + mlkem::PublicKey ek; + if (m_ops.keygen(seed.Bytes(), ek, *dk) != mlkem::Error::NONE) { + SetFailure(PQFailure::KEYGEN_INTERNAL); + return mlkem::Error::INTERNAL; + } + offer = SerializeRecord(ek); + m_dk = std::move(dk); + return mlkem::Error::NONE; +} + +void PQHandshake::SetOfferSent() noexcept +{ + if (Assume(!m_initiating && m_dk != nullptr && m_snapshot.offer == PQOfferState::NONE)) { + m_snapshot.offer = PQOfferState::SENT; + } +} + +mlkem::Error PQHandshake::AcceptOffer(Payload ek, Record& accept, mlkem::SharedSecret& ss) noexcept +{ + // Only a negotiating initiator accepts, and only once. + if (!Assume(m_initiating && m_mode == PQMode::NEGOTIATE && m_snapshot.offer == PQOfferState::NONE)) { + SetFailure(PQFailure::CHECK_EK_INTERNAL); + ss.Clear(); + return mlkem::Error::INTERNAL; + } + m_snapshot.offer = PQOfferState::RECEIVED; + switch (m_ops.check_public_key(ek)) { + case mlkem::Error::NONE: + break; + case mlkem::Error::INVALID_PUBLIC_KEY: + SetFailure(PQFailure::EK_MODULUS); + ss.Clear(); + return mlkem::Error::INVALID_PUBLIC_KEY; + default: + SetFailure(PQFailure::CHECK_EK_INTERNAL); + ss.Clear(); + return mlkem::Error::INTERNAL; + } + mlkem::Secret coins; + m_random.fill32(m_random.context, coins.Bytes()); + mlkem::Ciphertext ct; + // Encaps repeats the modulus check; failing it now is a local fault, not the peer's. + if (m_ops.encaps(ek, coins.Bytes(), ct, ss) != mlkem::Error::NONE) { + SetFailure(PQFailure::ENCAPS_INTERNAL); + ss.Clear(); + return mlkem::Error::INTERNAL; + } + accept = SerializeRecord(ct); + return mlkem::Error::NONE; +} + +mlkem::Error PQHandshake::DecapsulateAccept(Payload ct, mlkem::SharedSecret& ss) noexcept +{ + if (!Assume(m_dk != nullptr)) { + SetFailure(PQFailure::DECAPS_INTERNAL); + ss.Clear(); + return mlkem::Error::INTERNAL; + } + const mlkem::Error result{m_ops.decaps(*m_dk, ct, ss)}; + // The key is single use. + ClearSecrets(); + if (result != mlkem::Error::NONE) { + SetFailure(PQFailure::DECAPS_INTERNAL); + ss.Clear(); + return mlkem::Error::INTERNAL; + } + return mlkem::Error::NONE; +} + +void PQHandshake::SetFailure(PQFailure failure) noexcept +{ + if (m_snapshot.failure == PQFailure::NONE) m_snapshot.failure = failure; + ClearSecrets(); +} + +void PQHandshake::ClearSecrets() noexcept +{ + if (m_dk) { + m_dk->Clear(); + m_dk.reset(); + } +} diff --git a/src/bip324_pq.h b/src/bip324_pq.h new file mode 100644 index 0000000000..d8f0f8cf8a --- /dev/null +++ b/src/bip324_pq.h @@ -0,0 +1,200 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#ifndef QBIT_BIP324_PQ_H +#define QBIT_BIP324_PQ_H + +#include + +#include +#include +#include +#include +#include +#include + +/** + * Hybrid post-quantum negotiation for the BIP324 v2 transport (doc/design/pq-transport.md). + * + * Version-packet contents are zero or more records CompactSize(len) || header || payload, where + * len >= 1 counts the header and the payload. A responder offers its ML-KEM-1024 encapsulation key + * in a PQ_MLKEM1024 record, and an initiator accepts with the ciphertext in a record with the same + * header; the sender's role says which one a record is. + */ + +/** Record header of the ML-KEM-1024 offer (responder) and accept (initiator). 0x00 is reserved. */ +inline constexpr uint8_t PQ_MLKEM1024 = 0xF0; +/** An offer or accept record: CompactSize(1569) (3 bytes), the header and the 1,568-byte payload. */ +inline constexpr size_t PQ_RECORD_BYTES = 1572; +/** A key confirmation on the wire: encrypted length (3), encrypted header (1) and tag (16). */ +inline constexpr size_t PQ_CONFIRMATION_BYTES = 20; + +static_assert(mlkem::PUBLIC_KEY_BYTES == mlkem::CIPHERTEXT_BYTES); +static_assert(PQ_RECORD_BYTES == 3 + 1 + mlkem::PUBLIC_KEY_BYTES); + +/** What one side of a v2 connection does about the hybrid key exchange. */ +enum class PQMode : uint8_t { + //! Today's v2: empty version contents, the peer's contents ignored. + OFF = 0, + //! Responders offer an encapsulation key, initiators accept a valid offer. + NEGOTIATE = 1, + //! Plain v2 toward an endpoint in the fallback set (outbound only); behaves like OFF. + FALLBACK = 2, +}; + +/** What getpeerinfo reports as transport_pq_status. */ +enum class PQStatus : uint8_t { V1 = 0, OFF = 1, PENDING = 2, HYBRID = 3, LEGACY_PEER = 4, FALLBACK = 5 }; + +/** Offer progress. SENT (responder): the offer was queued for sending (SetOfferSent()), not + * necessarily received. RECEIVED (initiator): an offer of the right length was handed to + * AcceptOffer(). */ +enum class PQOfferState : uint8_t { NONE = 0, SENT = 1, RECEIVED = 2 }; + +/** The first failure of a connection's negotiation; later failures don't replace it. */ +enum class PQFailure : uint8_t { + NONE = 0, + // malformed_record: the first own record of the peer's contents. + EK_LENGTH = 1, + EK_MODULUS = 2, + CT_LENGTH = 3, + // first_packet_failed: the peer's key confirmation. + CONFIRM_LENGTH = 4, + CONFIRM_TAG = 5, + CONFIRM_NOT_DECOY = 6, + // internal_error: local faults, never caused by peer input. + KEYGEN_INTERNAL = 7, + CHECK_EK_INTERNAL = 8, + ENCAPS_INTERNAL = 9, + DECAPS_INTERNAL = 10, + CIPHER_STATE_INTERNAL = 11, +}; + +/** Entropy for key generation and encapsulation, 32 bytes per call. */ +struct PQRandomSource { + using Fill32 = void (*)(void*, std::span) noexcept; + Fill32 fill32; + void* context{nullptr}; +}; + +/** The ML-KEM operations a handshake uses. Must have static lifetime. */ +struct PQKemOps { + decltype(&mlkem::KeyGen) keygen; + decltype(&mlkem::CheckPublicKey) check_public_key; + decltype(&mlkem::Encaps) encaps; + decltype(&mlkem::Decaps) decaps; +}; + +/** Production entropy: GetStrongRandBytes. */ +PQRandomSource DefaultPQRandomSource() noexcept; +/** Production operations: the mlkem:: wrapper. */ +const PQKemOps& DefaultPQKemOps() noexcept; + +/** + * The version-contents parser and one connection's KEM state. + * + * There is no mutex: the owning transport's receive lock protects the object. The operations + * record their own failures (EK_MODULUS and the internal ones); callers record the others with + * SetFailure(). Internal ML-KEM errors never assert: they return mlkem::Error::INTERNAL, and the + * caller decides what happens to the connection. + */ +class PQHandshake +{ +public: + enum class ParseKind : uint8_t { NO_RECORD = 0, INVALID_GRAMMAR = 1, OWN_RECORD = 2 }; + struct ParsedContents { + ParseKind kind{ParseKind::NO_RECORD}; + //! OWN_RECORD only: the payload of the first PQ_MLKEM1024 record, pointing into the contents. + std::span payload{}; + }; + using Record = std::array; + using Payload = std::span; + struct Snapshot { + PQOfferState offer{PQOfferState::NONE}; + //! The peer's version packet authenticated. + bool version_received{false}; + //! The hybrid keys were installed. + bool switched{false}; + //! The peer's key confirmation verified. + bool confirmed{false}; + PQFailure failure{PQFailure::NONE}; + }; + + /** + * Parse version-packet contents in one pass, without copying or allocating. + * + * The whole contents must parse as records. Empty contents, or a framing error anywhere + * (truncation, a zero, non-canonical or oversized length, trailing bytes), give NO_RECORD or + * INVALID_GRAMMAR: no features, never a reason to disconnect. Otherwise the first + * PQ_MLKEM1024 record wins; later ones and unknown headers are ignored. Nothing about the + * payload is checked here. + */ + static ParsedContents ParseContents(std::span contents) noexcept; + + /** An offer or accept record carrying payload. */ + static Record SerializeRecord(std::span payload) noexcept; + + /** Production entropy and operations. */ + PQHandshake(bool initiating, PQMode mode) noexcept; + /** Test entropy and operations; ops must have static lifetime. */ + PQHandshake(bool initiating, PQMode mode, PQRandomSource random, const PQKemOps& ops) noexcept; + PQHandshake(bool initiating, PQMode mode, PQRandomSource random, const PQKemOps&& ops) = delete; + ~PQHandshake(); + PQHandshake(const PQHandshake&) = delete; + PQHandshake& operator=(const PQHandshake&) = delete; + + /** + * Check the length of the first own record of the peer's contents: the offer (for an + * initiator, before AcceptOffer()) or the accept (for a responder holding the decapsulation + * key of its offer) carries exactly 1,568 bytes. A wrong length records EK_LENGTH or + * CT_LENGTH, wipes the secrets and returns nullopt. A call in any other state is a local + * bug: it records CIPHER_STATE_INTERNAL and returns nullopt. + */ + std::optional CheckRecordLength(std::span payload) noexcept; + + /** + * Responder: generate a key pair from two independent 32-byte draws. Success writes the + * offer and keeps the decapsulation key; the caller queues the offer and then calls + * SetOfferSent(). Failure (KEYGEN_INTERNAL) writes no offer. + */ + [[nodiscard]] mlkem::Error MakeOffer(Record& offer) noexcept; + + /** Responder: the offer from MakeOffer() was queued for sending. */ + void SetOfferSent() noexcept; + + /** + * Initiator: check the offered key and encapsulate to it with one 32-byte draw. Returns + * INVALID_PUBLIC_KEY (EK_MODULUS) for a key that fails the modulus check, which is the + * peer's fault, and INTERNAL (CHECK_EK_INTERNAL, ENCAPS_INTERNAL) for local faults. Only + * success writes the accept record and the shared secret. + */ + [[nodiscard]] mlkem::Error AcceptOffer(Payload ek, Record& accept, mlkem::SharedSecret& ss) noexcept; + + /** + * Responder: decapsulate the accepted ciphertext. A corrupted ciphertext of the right + * length is not an error (implicit rejection). The decapsulation key is wiped either way; + * failure (DECAPS_INTERNAL) clears ss. + */ + [[nodiscard]] mlkem::Error DecapsulateAccept(Payload ct, mlkem::SharedSecret& ss) noexcept; + + void SetVersionReceived() noexcept { m_snapshot.version_received = true; } + void SetSwitched() noexcept { m_snapshot.switched = true; } + void SetConfirmed() noexcept { m_snapshot.confirmed = true; } + /** Record a failure unless one is recorded already, and wipe the secrets. */ + void SetFailure(PQFailure failure) noexcept; + Snapshot GetSnapshot() const noexcept { return m_snapshot; } + /** Whether a decapsulation key is held (between MakeOffer() and its wipe). */ + bool HasDecapsulationKey() const noexcept { return m_dk != nullptr; } + /** Wipe and release the decapsulation key. */ + void ClearSecrets() noexcept; + +private: + const bool m_initiating; + const PQMode m_mode; + PQRandomSource m_random; + const PQKemOps& m_ops; + std::unique_ptr m_dk; + Snapshot m_snapshot; +}; + +#endif // QBIT_BIP324_PQ_H diff --git a/src/net.cpp b/src/net.cpp index 9c0391fb99..2c330ce0dc 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -729,7 +729,7 @@ V1Transport::V1Transport(const NodeId node_id) noexcept Transport::Info V1Transport::GetInfo() const noexcept { - return {.transport_type = TransportProtocolType::V1, .session_id = {}}; + return {.transport_type = TransportProtocolType::V1, .session_id = {}, .transport_pq = false, .transport_pq_status = PQStatus::V1}; } int V1Transport::readHeader(std::span msg_bytes) @@ -1000,10 +1000,13 @@ void V2Transport::StartSendingHandshake() noexcept // We cannot wipe m_send_garbage as it will still be used as AAD later in the handshake. } -V2Transport::V2Transport(NodeId nodeid, bool initiating, const CKey& key, std::span ent32, std::vector garbage) noexcept +V2Transport::V2Transport(NodeId nodeid, bool initiating, const CKey& key, std::span ent32, + std::vector garbage, V2PQOptions pq, PQRandomSource random, const PQKemOps& ops) noexcept : m_cipher{key, ent32}, m_initiating{initiating}, m_nodeid{nodeid}, m_v1_fallback{nodeid}, + m_pq_options{EffectivePQOptions(initiating, pq)}, m_recv_state{initiating ? RecvState::KEY : RecvState::KEY_MAYBE_V1}, + m_pq{initiating, m_pq_options.mode, random, ops}, m_send_garbage{std::move(garbage)}, m_send_state{initiating ? SendState::AWAITING_KEY : SendState::MAYBE_V1} { @@ -1015,9 +1018,20 @@ V2Transport::V2Transport(NodeId nodeid, bool initiating, const CKey& key, std::s } } -V2Transport::V2Transport(NodeId nodeid, bool initiating) noexcept +V2PQOptions V2Transport::EffectivePQOptions(bool initiating, V2PQOptions pq) noexcept +{ + // Inbound connections never fall back (#184): a responder in fallback mode is a caller bug. + if (!Assume(initiating || pq.mode != PQMode::FALLBACK)) pq.mode = PQMode::OFF; + return pq; +} + +V2Transport::V2Transport(NodeId nodeid, bool initiating, const CKey& key, std::span ent32, std::vector garbage) noexcept + : V2Transport{nodeid, initiating, key, ent32, std::move(garbage), V2PQOptions{}, DefaultPQRandomSource(), DefaultPQKemOps()} {} + +V2Transport::V2Transport(NodeId nodeid, bool initiating, V2PQOptions pq) noexcept : V2Transport{nodeid, initiating, GenerateRandomKey(), - MakeByteSpan(GetRandHash()), GenerateRandomGarbage()} {} + MakeByteSpan(GetRandHash()), GenerateRandomGarbage(), + pq, DefaultPQRandomSource(), DefaultPQKemOps()} {} void V2Transport::SetReceiveState(RecvState recv_state) noexcept { @@ -1034,6 +1048,9 @@ void V2Transport::SetReceiveState(RecvState recv_state) noexcept Assume(recv_state == RecvState::VERSION); break; case RecvState::VERSION: + Assume(recv_state == RecvState::APP || recv_state == RecvState::CONFIRM); + break; + case RecvState::CONFIRM: Assume(recv_state == RecvState::APP); break; case RecvState::APP: @@ -1059,6 +1076,9 @@ void V2Transport::SetSendState(SendState send_state) noexcept Assume(send_state == SendState::V1 || send_state == SendState::AWAITING_KEY); break; case SendState::AWAITING_KEY: + Assume(send_state == SendState::READY || send_state == SendState::AWAITING_VERSION); + break; + case SendState::AWAITING_VERSION: Assume(send_state == SendState::READY); break; case SendState::READY: @@ -1144,33 +1164,50 @@ bool V2Transport::ProcessReceivedKeyBytes() noexcept // Other side's key has been fully received, and can now be Diffie-Hellman combined with // our key to initialize the encryption ciphers. - // Initialize the ciphers. + // Initialize the ciphers. The hybrid negotiation needs the ECDH secret and the transcript + // until the switch decision. EllSwiftPubKey ellswift(MakeByteSpan(m_recv_buffer)); + const bool negotiate{m_pq_options.mode == PQMode::NEGOTIATE}; LOCK(m_send_mutex); - m_cipher.Initialize(ellswift, m_initiating); + m_cipher.Initialize(ellswift, m_initiating, /*self_decrypt=*/false, + /*retain_for_hybrid=*/negotiate || m_pq_options.retain_without_negotiation); // Switch receiver state to GARB_GARBTERM. SetReceiveState(RecvState::GARB_GARBTERM); m_recv_buffer.clear(); - // Switch sender state to READY. - SetSendState(SendState::READY); - // Append the garbage terminator to the send buffer. m_send_buffer.resize(m_send_buffer.size() + BIP324Cipher::GARBAGE_TERMINATOR_LEN); std::copy(m_cipher.GetSendGarbageTerminator().begin(), m_cipher.GetSendGarbageTerminator().end(), MakeWritableByteSpan(m_send_buffer).last(BIP324Cipher::GARBAGE_TERMINATOR_LEN).begin()); - // Construct version packet in the send buffer, with the sent garbage data as AAD. - m_send_buffer.resize(m_send_buffer.size() + BIP324Cipher::EXPANSION + VERSION_CONTENTS.size()); - m_cipher.Encrypt( - /*contents=*/VERSION_CONTENTS, - /*aad=*/MakeByteSpan(m_send_garbage), - /*ignore=*/false, - /*output=*/MakeWritableByteSpan(m_send_buffer).last(BIP324Cipher::EXPANSION + VERSION_CONTENTS.size())); - // We no longer need the garbage. - ClearShrink(m_send_garbage); + if (!negotiate) { + // Switch sender state to READY, and send an empty version packet. + SetSendState(SendState::READY); + AppendVersionPacket(VERSION_CONTENTS); + } else if (m_initiating) { + // Hold the version packet until the responder's arrives: it answers their offer. + SetSendState(SendState::AWAITING_VERSION); + } else { + // Offer an encapsulation key in the version packet, and send nothing else until the + // initiator's version packet is processed. The key is generated only now that the + // initiator's whole public key has arrived. + PQHandshake::Record offer; + if (m_pq.MakeOffer(offer) == mlkem::Error::NONE) { + // Unreachable failure: the transcript was just started. + if (!Assume(m_cipher.AddVersionContents(offer))) return FailCipherState(); + SetSendState(SendState::AWAITING_VERSION); + AppendVersionPacket(offer); + m_pq.SetOfferSent(); + } else { + // A local fault: continue without the hybrid negotiation, as plain v2. + LogDebug(BCLog::NET, "V2 transport: no hybrid offer after an internal error (keygen), peer=%d\n", m_nodeid); + ClearHybridSecrets(); + SetSendState(SendState::READY); + AppendVersionPacket(VERSION_CONTENTS); + } + } } else { // We still have to receive more key bytes. } @@ -1207,7 +1244,8 @@ bool V2Transport::ProcessReceivedGarbageBytes() noexcept bool V2Transport::ProcessReceivedPacketBytes() noexcept { AssertLockHeld(m_recv_mutex); - Assume(m_recv_state == RecvState::VERSION || m_recv_state == RecvState::APP); + AssertLockNotHeld(m_send_mutex); + Assume(m_recv_state == RecvState::VERSION || m_recv_state == RecvState::CONFIRM || m_recv_state == RecvState::APP); // The maximum permitted contents length for a packet, consisting of: // - 0x00 byte: indicating long message type encoding @@ -1220,11 +1258,20 @@ bool V2Transport::ProcessReceivedPacketBytes() noexcept if (m_recv_buffer.size() == BIP324Cipher::LENGTH_LEN) { // Length descriptor received. m_recv_len = m_cipher.DecryptLength(MakeByteSpan(m_recv_buffer)); + if (m_recv_state == RecvState::CONFIRM && m_recv_len != 0) { + // The key confirmation is empty. With mismatched keys the length decrypts to a random + // value: fail now instead of waiting for bytes that may never come. + m_pq.SetFailure(PQFailure::CONFIRM_LENGTH); + LogDebug(BCLog::NET, "V2 transport error: key confirmation failed (length), peer=%d\n", m_nodeid); + return false; + } if (m_recv_len > MAX_CONTENTS_LEN) { LogDebug(BCLog::NET, "V2 transport error: packet too large (%u bytes), peer=%d\n", m_recv_len, m_nodeid); return false; } } else if (m_recv_buffer.size() > BIP324Cipher::LENGTH_LEN && m_recv_buffer.size() == m_recv_len + BIP324Cipher::EXPANSION) { + // The key confirmation is handled before, and never as, an ordinary decoy. + if (m_recv_state == RecvState::CONFIRM) return ProcessConfirmation(); // Ciphertext received, decrypt it into m_recv_decode_buffer. // Note that it is impossible to reach this branch without hitting the branch above first, // as GetMaxBytesToProcess only allows up to LENGTH_LEN into the buffer before that point. @@ -1248,11 +1295,14 @@ bool V2Transport::ProcessReceivedPacketBytes() noexcept // decoy, which we simply ignore, use the current state to decide what to do with it. if (!ignore) { switch (m_recv_state) { - case RecvState::VERSION: - // Version message received; transition to application phase. The contents is - // ignored, but can be used for future extensions. - SetReceiveState(RecvState::APP); + case RecvState::VERSION: { + // Version message received; act on its contents (or ignore them), and transition + // to the key confirmation or the application phase. The send lock makes a switch + // of all ciphers atomic. + LOCK(m_send_mutex); + if (!ProcessVersionContents(MakeByteSpan(m_recv_decode_buffer))) return false; break; + } case RecvState::APP: // Application message decrypted correctly. It can be extracted using GetMessage(). SetReceiveState(RecvState::APP_READY); @@ -1273,6 +1323,167 @@ bool V2Transport::ProcessReceivedPacketBytes() noexcept return true; } +bool V2Transport::ProcessConfirmation() noexcept +{ + AssertLockHeld(m_recv_mutex); + Assume(m_recv_state == RecvState::CONFIRM); + Assume(m_recv_len == 0 && m_recv_buffer.size() == PQ_CONFIRMATION_BYTES); + // The peer's first packet under the hybrid keys: an empty decoy, with empty AAD. + bool ignore{false}; + if (!m_cipher.Decrypt( + /*input=*/MakeByteSpan(m_recv_buffer).subspan(BIP324Cipher::LENGTH_LEN), + /*aad=*/{}, + /*ignore=*/ignore, + /*contents=*/{})) { + m_pq.SetFailure(PQFailure::CONFIRM_TAG); + LogDebug(BCLog::NET, "V2 transport error: key confirmation failed (tag), peer=%d\n", m_nodeid); + return false; + } + if (!ignore) { + m_pq.SetFailure(PQFailure::CONFIRM_NOT_DECOY); + LogDebug(BCLog::NET, "V2 transport error: key confirmation failed (not_decoy), peer=%d\n", m_nodeid); + return false; + } + // Feed the last 4 bytes of the Poly1305 authentication tag (and its timing) into our RNG. + RandAddEvent(ReadLE32(m_recv_buffer.data() + m_recv_buffer.size() - 4)); + m_pq.SetConfirmed(); + ClearShrink(m_recv_buffer); + SetReceiveState(RecvState::APP); + return true; +} + +bool V2Transport::ProcessVersionContents(std::span contents) noexcept +{ + AssertLockHeld(m_recv_mutex); + AssertLockHeld(m_send_mutex); + Assume(m_recv_state == RecvState::VERSION); + m_pq.SetVersionReceived(); + + // Without the negotiation (switch off, fallback, or a responder that could not offer), the + // contents are neither parsed nor validated. Nothing is held for a switch past this point, + // whichever path declined it (a no-op unless that path kept a secret). + if (m_send_state != SendState::AWAITING_VERSION) { + ClearHybridSecrets(); + SetReceiveState(RecvState::APP); + return true; + } + Assume(m_pq_options.mode == PQMode::NEGOTIATE); + + // A legacy peer: empty contents, no own record, or contents that do not parse. Keep the ECDH + // keys; an initiator now sends its empty version packet. + const auto parsed{PQHandshake::ParseContents(contents)}; + if (parsed.kind != PQHandshake::ParseKind::OWN_RECORD) { + ClearHybridSecrets(); + SetSendState(SendState::READY); + if (m_initiating) AppendVersionPacket(VERSION_CONTENTS); + SetReceiveState(RecvState::APP); + return true; + } + + // The first own record is the responder's offer or the initiator's accept, and has one length. + const auto payload{m_pq.CheckRecordLength(parsed.payload)}; + if (!payload) { + if (m_pq.GetSnapshot().failure == PQFailure::CIPHER_STATE_INTERNAL) return FailCipherState(); + LogDebug(BCLog::NET, "V2 transport error: malformed hybrid record (%s), peer=%d\n", + m_initiating ? "ek_length" : "ct_length", m_nodeid); + ClearHybridSecrets(); + return false; + } + + // Both contents go into the transcript, the responder's first. The unreachable cipher state + // failures close, whatever was committed. + if (!Assume(m_cipher.AddVersionContents(contents))) return FailCipherState(); + mlkem::SharedSecret ss; + if (m_initiating) { + PQHandshake::Record accept; + const mlkem::Error result{m_pq.AcceptOffer(*payload, accept, ss)}; + if (result == mlkem::Error::INVALID_PUBLIC_KEY) { + LogDebug(BCLog::NET, "V2 transport error: malformed hybrid record (ek_modulus), peer=%d\n", m_nodeid); + ClearHybridSecrets(); + return false; + } + if (result != mlkem::Error::NONE) { + // A local fault before we committed to anything (AcceptOffer recorded which): + // continue as if the switch were off. + LogDebug(BCLog::NET, "V2 transport: no hybrid accept after an internal error, peer=%d\n", m_nodeid); + ClearHybridSecrets(); + SetSendState(SendState::READY); + AppendVersionPacket(VERSION_CONTENTS); + SetReceiveState(RecvState::APP); + return true; + } + if (!Assume(m_cipher.AddVersionContents(accept))) return FailCipherState(); + if (m_pq_options.corrupt_shared_secret) ss.Bytes()[0] ^= 0xff; + // The accept travels under the ECDH keys, then both directions switch. + SetSendState(SendState::READY); + AppendVersionPacket(accept); + } else { + // The initiator has committed to the hybrid keys, so every failure from here closes. + if (m_pq.DecapsulateAccept(*payload, ss) != mlkem::Error::NONE) { + LogDebug(BCLog::NET, "V2 transport error: hybrid key exchange failed (decaps), peer=%d\n", m_nodeid); + ClearHybridSecrets(); + return false; + } + if (m_pq_options.corrupt_shared_secret) ss.Bytes()[0] ^= 0xff; + SetSendState(SendState::READY); + } + + // Unreachable failure: both contents are in the transcript. + if (!Assume(m_cipher.SwitchToHybrid(std::as_bytes(ss.Bytes())))) return FailCipherState(); + m_pq.SetSwitched(); + // Our first packet under the hybrid keys is the key confirmation, and theirs must be too. + AppendConfirmationPacket(); + SetReceiveState(RecvState::CONFIRM); + return true; +} + +void V2Transport::AppendVersionPacket(std::span contents) noexcept +{ + AssertLockHeld(m_recv_mutex); + AssertLockHeld(m_send_mutex); + // Construct version packet in the send buffer, after any unsent bytes, with the sent garbage + // data as AAD. + m_send_buffer.resize(m_send_buffer.size() + BIP324Cipher::EXPANSION + contents.size()); + m_cipher.Encrypt( + /*contents=*/contents, + /*aad=*/MakeByteSpan(m_send_garbage), + /*ignore=*/false, + /*output=*/MakeWritableByteSpan(m_send_buffer).last(BIP324Cipher::EXPANSION + contents.size())); + // We no longer need the garbage. + ClearShrink(m_send_garbage); +} + +void V2Transport::AppendConfirmationPacket() noexcept +{ + AssertLockHeld(m_recv_mutex); + AssertLockHeld(m_send_mutex); + static_assert(BIP324Cipher::EXPANSION == PQ_CONFIRMATION_BYTES); + m_send_buffer.resize(m_send_buffer.size() + PQ_CONFIRMATION_BYTES); + m_cipher.Encrypt( + /*contents=*/{}, + /*aad=*/{}, + /*ignore=*/true, + /*output=*/MakeWritableByteSpan(m_send_buffer).last(PQ_CONFIRMATION_BYTES)); +} + +void V2Transport::ClearHybridSecrets() noexcept +{ + AssertLockHeld(m_recv_mutex); + AssertLockHeld(m_send_mutex); + m_pq.ClearSecrets(); + m_cipher.DiscardHybridSecret(); +} + +bool V2Transport::FailCipherState() noexcept +{ + AssertLockHeld(m_recv_mutex); + AssertLockHeld(m_send_mutex); + m_pq.SetFailure(PQFailure::CIPHER_STATE_INTERNAL); + LogDebug(BCLog::NET, "V2 transport error: hybrid key exchange failed (cipher_state), peer=%d\n", m_nodeid); + ClearHybridSecrets(); + return false; +} + size_t V2Transport::GetMaxBytesToProcess() noexcept { AssertLockHeld(m_recv_mutex); @@ -1297,8 +1508,9 @@ size_t V2Transport::GetMaxBytesToProcess() noexcept // Process garbage bytes one by one (because terminator may appear anywhere). return 1; case RecvState::VERSION: + case RecvState::CONFIRM: case RecvState::APP: - // These three states all involve decoding a packet. Process the length descriptor first, + // These states all involve decoding a packet. Process the length descriptor first, // so that we know where the current packet ends (and we don't process bytes from the next // packet or decoy yet). Then, process the ciphertext bytes of the current packet. if (m_recv_buffer.size() < BIP324Cipher::LENGTH_LEN) { @@ -1350,6 +1562,7 @@ bool V2Transport::ReceivedBytes(std::span& msg_bytes) noexcept m_recv_buffer.reserve(MAX_GARBAGE_LEN + BIP324Cipher::GARBAGE_TERMINATOR_LEN); break; case RecvState::VERSION: + case RecvState::CONFIRM: case RecvState::APP: { // During states where a packet is being received, as much as is expected but never // more than MAX_RESERVE_AHEAD bytes in addition to what is received so far. @@ -1389,12 +1602,25 @@ bool V2Transport::ReceivedBytes(std::span& msg_bytes) noexcept break; case RecvState::GARB_GARBTERM: - if (!ProcessReceivedGarbageBytes()) return false; + if (!ProcessReceivedGarbageBytes()) { + if (m_pq_options.mode == PQMode::NEGOTIATE) { + LOCK(m_send_mutex); + ClearHybridSecrets(); + } + return false; + } break; case RecvState::VERSION: + case RecvState::CONFIRM: case RecvState::APP: - if (!ProcessReceivedPacketBytes()) return false; + if (!ProcessReceivedPacketBytes()) { + if (m_pq_options.mode == PQMode::NEGOTIATE) { + LOCK(m_send_mutex); + ClearHybridSecrets(); + } + return false; + } break; case RecvState::APP_READY: @@ -1587,18 +1813,80 @@ Transport::Info V2Transport::GetInfo() const noexcept Transport::Info info; // Do not report v2 and session ID until the version packet has been received - // and verified (confirming that the other side very likely has the same keys as us). + // and verified (confirming that the other side very likely has the same keys as us), and after + // a switch to hybrid keys, until the peer's key confirmation has verified. if (m_recv_state != RecvState::KEY_MAYBE_V1 && m_recv_state != RecvState::KEY && - m_recv_state != RecvState::GARB_GARBTERM && m_recv_state != RecvState::VERSION) { + m_recv_state != RecvState::GARB_GARBTERM && m_recv_state != RecvState::VERSION && + m_recv_state != RecvState::CONFIRM) { info.transport_type = TransportProtocolType::V2; info.session_id = uint256(MakeUCharSpan(m_cipher.GetSessionID())); } else { info.transport_type = TransportProtocolType::DETECTING; } + info.transport_pq = m_pq.GetSnapshot().confirmed; + info.transport_pq_status = GetPQStatus(); return info; } +PQStatus V2Transport::GetPQStatus() const noexcept +{ + AssertLockHeld(m_recv_mutex); + switch (m_pq_options.mode) { + case PQMode::OFF: return PQStatus::OFF; + case PQMode::FALLBACK: return PQStatus::FALLBACK; + case PQMode::NEGOTIATE: break; + } + const PQHandshake::Snapshot snapshot{m_pq.GetSnapshot()}; + if (snapshot.confirmed) return PQStatus::HYBRID; + switch (snapshot.failure) { + case PQFailure::NONE: + // The peer's version packet had no usable offer or accept: the ECDH keys stay. + if (snapshot.version_received && !snapshot.switched) return PQStatus::LEGACY_PEER; + return PQStatus::PENDING; + case PQFailure::KEYGEN_INTERNAL: + case PQFailure::CHECK_EK_INTERNAL: + case PQFailure::ENCAPS_INTERNAL: + // A local fault before anything was committed: the connection continues as plain v2. + return PQStatus::OFF; + case PQFailure::EK_LENGTH: + case PQFailure::EK_MODULUS: + case PQFailure::CT_LENGTH: + case PQFailure::CONFIRM_LENGTH: + case PQFailure::CONFIRM_TAG: + case PQFailure::CONFIRM_NOT_DECOY: + case PQFailure::DECAPS_INTERNAL: + case PQFailure::CIPHER_STATE_INTERNAL: + // The connection closes. There is no failed status, so it reports pending until it is + // removed, local faults included. + return PQStatus::PENDING; + } + Assume(false); + return PQStatus::PENDING; +} + +PQHandshake::Snapshot V2Transport::GetPQSnapshot() const noexcept +{ + AssertLockNotHeld(m_recv_mutex); + LOCK(m_recv_mutex); + return m_pq.GetSnapshot(); +} + +bool V2Transport::HoldsHybridSecretsForTesting() const noexcept +{ + AssertLockNotHeld(m_recv_mutex); + LOCK(m_recv_mutex); + return m_pq.HasDecapsulationKey() || m_cipher.HoldsHybridSecret(); +} + +void V2Transport::DiscardHybridSecretForTesting() noexcept +{ + AssertLockNotHeld(m_recv_mutex); + AssertLockNotHeld(m_send_mutex); + LOCK2(m_recv_mutex, m_send_mutex); + m_cipher.DiscardHybridSecret(); +} + std::pair CConnman::SocketSendData(CNode& node) const { auto it = node.vSendMsg.begin(); diff --git a/src/net.h b/src/net.h index d41fc325ef..8f061ed795 100644 --- a/src/net.h +++ b/src/net.h @@ -7,6 +7,7 @@ #define QBIT_NET_H #include +#include #include #include #include @@ -264,6 +265,9 @@ class Transport { { TransportProtocolType transport_type; std::optional session_id; + /** Whether the session keys are hybrid: true only once the peer's key confirmation verified. */ + bool transport_pq{false}; + PQStatus transport_pq_status{PQStatus::OFF}; }; /** Retrieve information about this transport. */ @@ -451,12 +455,24 @@ class V1Transport final : public Transport bool ShouldReconnectV1() const noexcept override { return false; } }; +/** The hybrid post-quantum negotiation of one v2 connection (doc/design/pq-transport.md). */ +struct V2PQOptions { + PQMode mode{PQMode::OFF}; + /** Test only: retain the ECDH secret and the transcript at the key exchange even without the + * negotiation, as a path that declines it afterwards without wiping them would. */ + bool retain_without_negotiation{false}; + /** Test only: invert byte 0 of the local ML-KEM shared secret before deriving keys, so the + * peer's key confirmation check fails. EK and CT are unchanged. */ + bool corrupt_shared_secret{false}; +}; + class V2Transport final : public Transport { private: - /** Contents of the version packet to send. BIP324 stipulates that senders should leave this - * empty, and receivers should ignore it. Future extensions can change what is sent as long as - * an empty version packet contents is interpreted as no extensions supported. */ + /** Contents of the version packet to send without an offer or accept. BIP324 stipulates that + * senders should leave this empty, and receivers should ignore it. Extensions (such as the + * hybrid negotiation, see PQHandshake) change what is sent, and an empty version packet + * contents is interpreted as no extensions supported. */ static constexpr std::array VERSION_CONTENTS = {}; /** The length of the V1 prefix to match bytes initially received by responders with to @@ -475,12 +491,12 @@ class V2Transport final : public Transport * * start(responder) * | - * | start(initiator) /---------\ - * | | | | - * v v v | - * KEY_MAYBE_V1 -> KEY -> GARB_GARBTERM -> VERSION -> APP -> APP_READY - * | - * \-------> V1 + * | start(initiator) /---------\ + * | | | | + * v v v | + * KEY_MAYBE_V1 -> KEY -> GARB_GARBTERM -> VERSION ------------> APP -> APP_READY + * | | ^ + * \-------> V1 \----> CONFIRM ----/ */ enum class RecvState : uint8_t { /** (Responder only) either v2 public key or v1 header. @@ -511,10 +527,20 @@ class V2Transport final : public Transport * first received packet in this state (whether it's a decoy or not) is expected to * authenticate the garbage received during the GARB_GARBTERM state as associated * authenticated data (AAD). The first non-decoy packet in this state is interpreted as - * version negotiation (currently, that means ignoring the contents, but it can be used for - * negotiating future extensions), and afterwards the state becomes APP. */ + * version negotiation. Without the hybrid negotiation its contents are ignored, and the + * state becomes APP. With it, the contents are an offer or an accept (or a legacy peer's), + * see ProcessVersionContents(): the state becomes CONFIRM if the keys switched to hybrid, + * and APP otherwise. */ VERSION, + /** Key confirmation (hybrid keys only). + * + * The first packet under the hybrid keys must be an empty decoy. Its length is checked as + * soon as its 3 bytes arrive, then its tag and ignore bit. If that fails, the connection + * aborts; otherwise the peer is confirmed, nothing is delivered and the state becomes + * APP. */ + CONFIRM, + /** Application packet. * * A packet is received, and decrypted/verified. If that succeeds, the state becomes @@ -543,9 +569,9 @@ class V2Transport final : public Transport * | start(initiator) * | | * v v - * MAYBE_V1 -> AWAITING_KEY -> READY - * | - * \-----> V1 + * MAYBE_V1 -> AWAITING_KEY ---------------------> READY + * | | ^ + * \-----> V1 \-----> AWAITING_VERSION ------/ */ enum class SendState : uint8_t { /** (Responder only) Not sending until v1 or v2 is detected. @@ -560,15 +586,26 @@ class V2Transport final : public Transport * * This is the initial state for initiators. The public key and garbage is sent out. When * the receiver receives the other side's public key and transitions to GARB_GARBTERM, the - * sender state becomes READY. */ + * sender state becomes READY, or AWAITING_VERSION with the hybrid negotiation. */ AWAITING_KEY, + /** (Hybrid negotiation only) Waiting for the other side's version packet. + * + * The ciphers are initialized and the garbage terminator is queued. A responder has also + * queued its version packet with the offer; an initiator holds its version packet, which + * answers the offer. Nothing else is sent, so no message can be provided, and bytes still + * in the send buffer go out. When the receiver processes the other side's version packet, + * the sender state becomes READY, and the version packet (initiator) and the key + * confirmation (after a switch to hybrid keys) are appended to the send buffer. */ + AWAITING_VERSION, + /** Normal sending state. * * In this state, the ciphers are initialized, so packets can be sent. When this state is - * entered, the garbage terminator and version packet are appended to the send buffer (in - * addition to the key and garbage which may still be there). In this state a message can be - * provided if the send buffer is empty. */ + * entered, the garbage terminator and version packet (and, after a switch to hybrid keys, + * the key confirmation) have been appended to the send buffer (in addition to the key and + * garbage which may still be there). In this state a message can be provided if the send + * buffer is empty. */ READY, /** This transport is using v1 fallback. @@ -585,6 +622,8 @@ class V2Transport final : public Transport const NodeId m_nodeid; /** Encapsulate a V1Transport to fall back to. */ V1Transport m_v1_fallback; + /** The hybrid negotiation options of this connection. */ + const V2PQOptions m_pq_options; /** Lock for receiver-side fields. */ mutable Mutex m_recv_mutex ACQUIRED_BEFORE(m_send_mutex); @@ -599,6 +638,8 @@ class V2Transport final : public Transport std::vector m_recv_decode_buffer GUARDED_BY(m_recv_mutex); /** Current receiver state. */ RecvState m_recv_state GUARDED_BY(m_recv_mutex); + /** The hybrid negotiation state: KEM secrets and progress. */ + PQHandshake m_pq GUARDED_BY(m_recv_mutex); /** Lock for sending-side fields. If both sending and receiving fields are accessed, * m_recv_mutex must be acquired before m_send_mutex. */ @@ -632,8 +673,32 @@ class V2Transport final : public Transport bool ProcessReceivedKeyBytes() noexcept EXCLUSIVE_LOCKS_REQUIRED(m_recv_mutex, !m_send_mutex); /** Process bytes in m_recv_buffer, while in GARB_GARBTERM state. */ bool ProcessReceivedGarbageBytes() noexcept EXCLUSIVE_LOCKS_REQUIRED(m_recv_mutex); - /** Process bytes in m_recv_buffer, while in VERSION/APP state. */ - bool ProcessReceivedPacketBytes() noexcept EXCLUSIVE_LOCKS_REQUIRED(m_recv_mutex); + /** Process bytes in m_recv_buffer, while in VERSION/CONFIRM/APP state. */ + bool ProcessReceivedPacketBytes() noexcept EXCLUSIVE_LOCKS_REQUIRED(m_recv_mutex, !m_send_mutex); + /** Process the peer's key confirmation in m_recv_buffer, while in CONFIRM state. */ + bool ProcessConfirmation() noexcept EXCLUSIVE_LOCKS_REQUIRED(m_recv_mutex); + /** Act on the contents of the peer's version packet, and leave the VERSION state. Returns + * false if the connection must close. */ + bool ProcessVersionContents(std::span contents) noexcept EXCLUSIVE_LOCKS_REQUIRED(m_recv_mutex, m_send_mutex); + /** Append our version packet to the send buffer, under the current (ECDH) keys, with our + * garbage as AAD, and wipe the garbage. */ + void AppendVersionPacket(std::span contents) noexcept EXCLUSIVE_LOCKS_REQUIRED(m_recv_mutex, m_send_mutex); + /** Append our key confirmation (an empty decoy, empty AAD) under the hybrid keys. */ + void AppendConfirmationPacket() noexcept EXCLUSIVE_LOCKS_REQUIRED(m_recv_mutex, m_send_mutex); + /** Wipe the decapsulation key and the retained ECDH secret, keeping the current keys. */ + void ClearHybridSecrets() noexcept EXCLUSIVE_LOCKS_REQUIRED(m_recv_mutex, m_send_mutex); + /** Close after an inconsistent hybrid cipher state, a local bug: record and log + * CIPHER_STATE_INTERNAL, wipe, and return false. */ + bool FailCipherState() noexcept EXCLUSIVE_LOCKS_REQUIRED(m_recv_mutex, m_send_mutex); + /** The options this side uses: a responder never falls back, so its FALLBACK becomes OFF. */ + static V2PQOptions EffectivePQOptions(bool initiating, V2PQOptions pq) noexcept; + /** The status getpeerinfo reports for the hybrid negotiation. OFF and FALLBACK follow the + * mode. With the negotiation on: HYBRID once the peer's confirmation verified, LEGACY_PEER + * when the peer's version packet carried no usable offer or accept, OFF after a key + * generation, key check or encapsulation fault that continued as plain v2, and PENDING + * otherwise. There is no failed status: a connection closing on any other failure, local + * faults included, reports PENDING until it is removed. */ + PQStatus GetPQStatus() const noexcept EXCLUSIVE_LOCKS_REQUIRED(m_recv_mutex); public: static constexpr uint32_t MAX_GARBAGE_LEN = 4095; @@ -642,12 +707,24 @@ class V2Transport final : public Transport * * @param[in] nodeid the node's NodeId (only for debug log output). * @param[in] initiating whether we are the initiator side. + * @param[in] pq the hybrid negotiation options (off by default). Only an initiator + * falls back: PQMode::FALLBACK on a responder is a caller bug + * (Assume), and it runs and reports as PQMode::OFF. */ - V2Transport(NodeId nodeid, bool initiating) noexcept; + V2Transport(NodeId nodeid, bool initiating, V2PQOptions pq = {}) noexcept; - /** Construct a V2 transport with specified keys and garbage (test use only). */ + /** Construct a V2 transport with specified keys and garbage, without the hybrid negotiation + * (test use only). */ V2Transport(NodeId nodeid, bool initiating, const CKey& key, std::span ent32, std::vector garbage) noexcept; + /** Construct a V2 transport with specified keys, garbage, hybrid negotiation options, ML-KEM + * entropy and operations (test use only). ops must have static lifetime. A responder's + * PQMode::FALLBACK is handled as in the constructor above. */ + V2Transport(NodeId nodeid, bool initiating, const CKey& key, std::span ent32, + std::vector garbage, V2PQOptions pq, PQRandomSource random, const PQKemOps& ops) noexcept; + V2Transport(NodeId nodeid, bool initiating, const CKey& key, std::span ent32, + std::vector garbage, V2PQOptions pq, PQRandomSource random, const PQKemOps&& ops) = delete; + // Receive side functions. bool ReceivedMessageComplete() const noexcept override EXCLUSIVE_LOCKS_REQUIRED(!m_recv_mutex); bool ReceivedBytes(std::span& msg_bytes) noexcept override EXCLUSIVE_LOCKS_REQUIRED(!m_recv_mutex, !m_send_mutex); @@ -662,6 +739,14 @@ class V2Transport final : public Transport // Miscellaneous functions. bool ShouldReconnectV1() const noexcept override EXCLUSIVE_LOCKS_REQUIRED(!m_recv_mutex, !m_send_mutex); Info GetInfo() const noexcept override EXCLUSIVE_LOCKS_REQUIRED(!m_recv_mutex); + /** The hybrid negotiation's progress and first failure. */ + PQHandshake::Snapshot GetPQSnapshot() const noexcept EXCLUSIVE_LOCKS_REQUIRED(!m_recv_mutex); + + /** Test only: whether the decapsulation key or the retained ECDH secret is still held. */ + bool HoldsHybridSecretsForTesting() const noexcept EXCLUSIVE_LOCKS_REQUIRED(!m_recv_mutex); + /** Test only: discard the retained ECDH secret and transcript, as an inconsistent cipher state + * would. Every later use of them fails closed. */ + void DiscardHybridSecretForTesting() noexcept EXCLUSIVE_LOCKS_REQUIRED(!m_recv_mutex, !m_send_mutex); }; struct CNodeOptions diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 204f378b34..673e436286 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -166,6 +166,7 @@ target_json_data_sources(test_bitcoin data/p2mr_script_boundary_vectors.json data/p2mr_v1_manifest.json data/p2mr_vectors.json + data/pq_transport_vectors.json data/script_tests.json data/sighash.json data/testnet4_launch_difficulty.json diff --git a/src/test/bip324_tests.cpp b/src/test/bip324_tests.cpp index 2e71bf22a5..3ed4464e49 100644 --- a/src/test/bip324_tests.cpp +++ b/src/test/bip324_tests.cpp @@ -2,11 +2,19 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include + #include #include +#include +#include +#include +#include #include #include +#include #include +#include #include #include #include @@ -15,12 +23,88 @@ #include #include #include +#include +#include +#include +#include +#include +#include #include #include +#include namespace { +using Bytes = std::vector; + +Bytes ToBytes(std::span bytes) { return {bytes.begin(), bytes.end()}; } + +Bytes Concat(std::initializer_list> parts) +{ + Bytes out; + for (const auto& part : parts) out.insert(out.end(), part.begin(), part.end()); + return out; +} + +Bytes CompactSize(uint64_t size) +{ + DataStream stream; + WriteCompactSize(stream, size); + return {stream.begin(), stream.end()}; +} + +/** A version-contents record: CompactSize(1 + payload size) || header || payload. */ +Bytes Record(uint8_t header, std::span payload) +{ + return Concat({CompactSize(1 + payload.size()), std::as_bytes(std::span{&header, 1}), payload}); +} + +Bytes EncryptPacket(BIP324Cipher& sender, std::span contents, bool ignore) +{ + Bytes packet(contents.size() + BIP324Cipher::EXPANSION); + sender.Encrypt(contents, {}, ignore, packet); + return packet; +} + +/** Decrypt one complete packet: nullopt if its decrypted length doesn't fit the packet or its tag fails. */ +std::optional> DecryptPacket(BIP324Cipher& receiver, std::span packet, uint32_t& length) +{ + length = receiver.DecryptLength(packet.first(BIP324Cipher::LENGTH_LEN)); + if (packet.size() != length + BIP324Cipher::EXPANSION) return std::nullopt; + Bytes contents(length); + bool ignore{false}; + if (!receiver.Decrypt(packet.subspan(BIP324Cipher::LENGTH_LEN), {}, ignore, contents)) return std::nullopt; + return std::make_pair(ignore, std::move(contents)); +} + +bool Receives(BIP324Cipher& receiver, std::span packet, std::span contents, bool ignore) +{ + uint32_t length; + const auto result = DecryptPacket(receiver, packet, length); + return result && result->first == ignore && std::ranges::equal(result->second, contents); +} + +/** Add both version contents (responder's first) and switch to the hybrid keys. */ +bool SwitchWith(BIP324Cipher& cipher, std::span contents_first, std::span contents_second, + std::span ss_mlkem) +{ + return cipher.AddVersionContents(contents_first) && cipher.AddVersionContents(contents_second) && cipher.SwitchToHybrid(ss_mlkem); +} + +void InitializePair(BIP324Cipher& initiator, BIP324Cipher& responder, bool retain_for_hybrid) +{ + initiator.Initialize(responder.GetOurPubKey(), /*initiator=*/true, /*self_decrypt=*/false, retain_for_hybrid); + responder.Initialize(initiator.GetOurPubKey(), /*initiator=*/false, /*self_decrypt=*/false, retain_for_hybrid); +} + +/** Check one packet each way between an initiator and a responder. */ +void CheckPair(BIP324Cipher& initiator, BIP324Cipher& responder, std::span contents) +{ + BOOST_CHECK(Receives(responder, EncryptPacket(initiator, contents, false), contents, false)); + BOOST_CHECK(Receives(initiator, EncryptPacket(responder, contents, false), contents, false)); +} + struct BIP324Test : BasicTestingSetup { void TestBIP324PacketVector( uint32_t in_idx, @@ -300,4 +384,383 @@ BOOST_AUTO_TEST_CASE(packet_test_vectors) { "27cf3c977dec7a089ae3660b0ec0bf5e15d9704df4e3d9e54831552dddbb07afabc03b42c9f912a5225c855850a30fc74db1d04ee00b19085fbfe6786084f559cf8531bffbd706db7a6bec021b2c946d1dd5ecbc84299205434bbde2910fb3543f75a08153e1471e02649be9c4f2e1767b7415243fa8eeedc55cb77a36651966"); } +BOOST_AUTO_TEST_CASE(hybrid_vectors) +{ + // The vectors come from an independent Python reference (contrib/devtools/generate-pq-transport-vectors.py). + UniValue doc; + BOOST_REQUIRE(doc.read(json_tests::pq_transport_vectors)); + BOOST_CHECK_EQUAL(doc["header"].get_str(), "f0"); + BOOST_CHECK_EQUAL(doc["salt_label"].get_str(), "qbit_v2_hybrid_mlkem1024"); + const std::vector labels{"initiator_L", "initiator_P", "responder_L", "responder_P", "session_id"}; + BOOST_REQUIRE_EQUAL(doc["labels"].size(), labels.size()); + for (size_t i = 0; i < labels.size(); ++i) BOOST_CHECK_EQUAL(doc["labels"][i].get_str(), labels[i]); + + const UniValue& vectors = doc["vectors"]; + BOOST_REQUIRE(vectors.size() >= 4); + for (size_t index = 0; index < vectors.size(); ++index) { + const UniValue& vec = vectors[index]; + BOOST_TEST_MESSAGE("hybrid vector " << index << ": " << vec["comment"].get_str()); + const auto hex = [&](const std::string& key) { return ParseHex(vec[key].get_str()); }; + const std::string chain = vec["chain"].get_str(); + BOOST_REQUIRE(chain == "main" || chain == "regtest"); + SelectParams(chain == "main" ? ChainType::MAIN : ChainType::REGTEST); + BOOST_CHECK(std::ranges::equal(hex("magic"), MakeByteSpan(Params().MessageStart()))); + + CKey key_i, key_r; + const auto priv_i = ParseHex(vec["initiator_privkey"].get_str()), priv_r = ParseHex(vec["responder_privkey"].get_str()); + key_i.Set(priv_i.begin(), priv_i.end(), true); + key_r.Set(priv_r.begin(), priv_r.end(), true); + BOOST_REQUIRE(key_i.IsValid() && key_r.IsValid()); + const Bytes ellswift_i = hex("initiator_ellswift"), ellswift_r = hex("responder_ellswift"); + const EllSwiftPubKey pub_i{ellswift_i}, pub_r{ellswift_r}; + const Bytes ss_ecdh = hex("ss_ecdh"), contents_r = hex("contents_responder"), contents_i = hex("contents_initiator"); + const Bytes ss_mlkem_bytes = hex("ss_mlkem"); + BOOST_REQUIRE_EQUAL(ss_mlkem_bytes.size(), 32U); + const std::span ss_mlkem{ss_mlkem_bytes}; + + // The inputs and every intermediate agree with qbit's own primitives. + BOOST_CHECK(std::ranges::equal(key_i.ComputeBIP324ECDHSecret(pub_r, pub_i, /*initiating=*/true), ss_ecdh)); + BOOST_CHECK(std::ranges::equal(key_r.ComputeBIP324ECDHSecret(pub_i, pub_r, /*initiating=*/false), ss_ecdh)); + BOOST_CHECK(!std::ranges::search(contents_r, Record(0xF0, hex("ek"))).empty()); + BOOST_CHECK(!std::ranges::search(contents_i, Record(0xF0, hex("ct"))).empty()); + const Bytes transcript_input = Concat({ellswift_i, ellswift_r, CompactSize(contents_r.size()), contents_r, + CompactSize(contents_i.size()), contents_i}); + BOOST_CHECK(transcript_input == hex("transcript_input")); + Bytes transcript_hash(CSHA256::OUTPUT_SIZE); + CSHA256().Write(UCharCast(transcript_input.data()), transcript_input.size()).Finalize(UCharCast(transcript_hash.data())); + BOOST_CHECK(transcript_hash == hex("transcript_hash")); + const std::string salt_label{"qbit_v2_hybrid_mlkem1024"}; + const Bytes salt = Concat({std::as_bytes(std::span{salt_label}), MakeByteSpan(Params().MessageStart())}); + BOOST_CHECK(salt == hex("salt")); + const Bytes ikm = Concat({ss_ecdh, ss_mlkem, transcript_hash}); + BOOST_CHECK(ikm == hex("ikm")); + Bytes prk(CHMAC_SHA256::OUTPUT_SIZE); + CHMAC_SHA256(UCharCast(salt.data()), salt.size()).Write(UCharCast(ikm.data()), ikm.size()).Finalize(UCharCast(prk.data())); + BOOST_CHECK(prk == hex("prk")); + for (const auto& label : labels) { + Bytes okm(CHMAC_SHA256::OUTPUT_SIZE); + const uint8_t one{1}; + CHMAC_SHA256(UCharCast(prk.data()), prk.size()).Write(UCharCast(label.data()), label.size()).Write(&one, 1).Finalize(UCharCast(okm.data())); + BOOST_CHECK_MESSAGE(okm == hex(label), label); + } + + // The derived length and packet keys produce each side's key confirmation. + for (const std::string role : {"initiator", "responder"}) { + FSChaCha20 l_cipher{hex(role + "_L"), BIP324Cipher::REKEY_INTERVAL}; + FSChaCha20Poly1305 p_cipher{hex(role + "_P"), BIP324Cipher::REKEY_INTERVAL}; + Bytes confirmation(BIP324Cipher::EXPANSION); + const std::array zero_length{}; + const std::array header{BIP324Cipher::IGNORE_BIT}; + l_cipher.Crypt(zero_length, std::span{confirmation}.first(BIP324Cipher::LENGTH_LEN)); + p_cipher.Encrypt(header, {}, {}, std::span{confirmation}.subspan(BIP324Cipher::LENGTH_LEN)); + BOOST_CHECK(confirmation == hex(role + "_confirmation")); + } + + // BIP324Cipher in both roles, and in self-decrypt mode, which decrypts its own role's packets. + for (const bool self_decrypt : {false, true}) { + for (const bool initiator : {true, false}) { + const std::string ours{initiator ? "initiator" : "responder"}, theirs{initiator ? "responder" : "initiator"}; + BIP324Cipher cipher(initiator ? key_i : key_r, initiator ? pub_i : pub_r); + cipher.Initialize(initiator ? pub_r : pub_i, initiator, self_decrypt, /*retain_for_hybrid=*/true); + BOOST_CHECK(std::ranges::equal(cipher.GetSessionID(), hex("ecdh_session_id"))); + const Bytes send_terminator = ToBytes(cipher.GetSendGarbageTerminator()); + const Bytes recv_terminator = ToBytes(cipher.GetReceiveGarbageTerminator()); + BOOST_CHECK(send_terminator == hex(ours + "_garbage_terminator")); + BOOST_CHECK(recv_terminator == hex(theirs + "_garbage_terminator")); + + BOOST_REQUIRE(SwitchWith(cipher, contents_r, contents_i, ss_mlkem)); + BOOST_CHECK(std::ranges::equal(cipher.GetSessionID(), hex("session_id"))); + BOOST_CHECK(std::ranges::equal(cipher.GetSendGarbageTerminator(), send_terminator)); + BOOST_CHECK(std::ranges::equal(cipher.GetReceiveGarbageTerminator(), recv_terminator)); + + const std::string sender{self_decrypt ? ours : theirs}; + BOOST_CHECK(Receives(cipher, hex(sender + "_confirmation"), {}, /*ignore=*/true)); + BOOST_CHECK(Receives(cipher, hex(sender + "_first_packet"), hex(sender + "_first_contents"), /*ignore=*/false)); + if (!self_decrypt) { + BOOST_CHECK(EncryptPacket(cipher, {}, /*ignore=*/true) == hex(ours + "_confirmation")); + BOOST_CHECK(EncryptPacket(cipher, hex(ours + "_first_contents"), /*ignore=*/false) == hex(ours + "_first_packet")); + } + } + } + } + + const auto hex = [](const UniValue& obj, const std::string& key) { return ParseHex(obj[key].get_str()); }; + // A cipher in one role of a vector, initialized with the hybrid state retained. + const auto make_cipher = [&](const UniValue& vec, bool as_initiator) { + SelectParams(vec["chain"].get_str() == "main" ? ChainType::MAIN : ChainType::REGTEST); + CKey key; + const auto priv = ParseHex(vec[as_initiator ? "initiator_privkey" : "responder_privkey"].get_str()); + key.Set(priv.begin(), priv.end(), true); + const EllSwiftPubKey pub_i{hex(vec, "initiator_ellswift")}, pub_r{hex(vec, "responder_ellswift")}; + auto cipher = std::make_unique(key, as_initiator ? pub_i : pub_r); + cipher->Initialize(as_initiator ? pub_r : pub_i, as_initiator, /*self_decrypt=*/false, /*retain_for_hybrid=*/true); + return cipher; + }; + // Why a receiver's key confirmation check rejects a first hybrid packet ("none" if it passes). + const auto confirmation_failure = [](BIP324Cipher& receiver, std::span packet, uint32_t& length) -> std::string { + const auto result = DecryptPacket(receiver, packet, length); + if (length != 0) return "length"; + if (!result) return "tag"; + return result->first ? "none" : "not_decoy"; + }; + + // First hybrid packets that fail the receiver's check under the honest keys, one per reason. + const UniValue& failures = doc["confirmation_failures"]; + BOOST_REQUIRE_EQUAL(failures.size(), 3U); + std::set reasons; + for (size_t index = 0; index < failures.size(); ++index) { + const UniValue& fail = failures[index]; + BOOST_TEST_MESSAGE("confirmation failure " << index << ": " << fail["comment"].get_str()); + const UniValue& vec = vectors[fail["vector"].getInt()]; + const bool initiator{fail["receiver"].get_str() == "initiator"}; + BOOST_REQUIRE(initiator || fail["receiver"].get_str() == "responder"); + const Bytes ss_mlkem = hex(vec, "ss_mlkem"), packet = hex(fail, "packet"); + BOOST_REQUIRE_EQUAL(ss_mlkem.size(), 32U); + BOOST_REQUIRE_EQUAL(packet.size(), BIP324Cipher::EXPANSION); + auto receiver = make_cipher(vec, initiator); + BOOST_REQUIRE(SwitchWith(*receiver, hex(vec, "contents_responder"), hex(vec, "contents_initiator"), + std::span{ss_mlkem})); + uint32_t length; + BOOST_CHECK_EQUAL(confirmation_failure(*receiver, packet, length), fail["failure"].get_str()); + BOOST_CHECK_EQUAL(length, fail["decrypted_length"].getInt()); + reasons.insert(fail["failure"].get_str()); + } + BOOST_CHECK((reasons == std::set{"length", "tag", "not_decoy"})); + + // One side derives from a different transcript or ML-KEM secret than its honest peer, so each + // side's key confirmation fails at the other: at once on a nonzero length, else on the tag. + const UniValue& negatives = doc["negative_vectors"]; + BOOST_REQUIRE(negatives.size() >= 3); + for (size_t index = 0; index < negatives.size(); ++index) { + const UniValue& neg = negatives[index]; + BOOST_TEST_MESSAGE("negative hybrid vector " << index << ": " << neg["comment"].get_str()); + const UniValue& vec = vectors[neg["vector"].getInt()]; + const bool initiator{neg["role"].get_str() == "initiator"}; + BOOST_REQUIRE(initiator || neg["role"].get_str() == "responder"); + + const Bytes ss_bad = hex(neg, "ss_mlkem"), ss_good = hex(vec, "ss_mlkem"); + BOOST_REQUIRE(ss_bad.size() == 32 && ss_good.size() == 32); + auto mismatched = make_cipher(vec, initiator); + const UniValue& transcript_contents = neg["transcript_contents"]; + BOOST_REQUIRE_EQUAL(transcript_contents.size(), 2U); + BOOST_REQUIRE(SwitchWith(*mismatched, ParseHex(transcript_contents[0].get_str()), + ParseHex(transcript_contents[1].get_str()), std::span{ss_bad})); + BOOST_CHECK(std::ranges::equal(mismatched->GetSessionID(), hex(neg, "session_id"))); + BOOST_CHECK(!std::ranges::equal(mismatched->GetSessionID(), hex(vec, "session_id"))); + BOOST_CHECK(EncryptPacket(*mismatched, {}, /*ignore=*/true) == hex(neg, "confirmation")); + uint32_t length; + BOOST_CHECK_EQUAL(confirmation_failure(*mismatched, hex(vec, initiator ? "responder_confirmation" : "initiator_confirmation"), length), + neg["failure"].get_str()); + BOOST_CHECK_EQUAL(length, neg["decrypted_length"].getInt()); + + auto honest = make_cipher(vec, !initiator); + BOOST_REQUIRE(SwitchWith(*honest, hex(vec, "contents_responder"), hex(vec, "contents_initiator"), + std::span{ss_good})); + BOOST_CHECK_EQUAL(confirmation_failure(*honest, hex(neg, "confirmation"), length), neg["peer_failure"].get_str()); + BOOST_CHECK_EQUAL(length, neg["peer_decrypted_length"].getInt()); + } +} + +BOOST_AUTO_TEST_CASE(hybrid_transcript_binding) +{ + const CKey key_i = GenerateRandomKey(), key_r = GenerateRandomKey(); + const Bytes ent_i = m_rng.randbytes(32), ent_r = m_rng.randbytes(32); + const Bytes other_ent_i = m_rng.randbytes(32), other_ent_r = m_rng.randbytes(32); + const Bytes offer = Record(0xF0, m_rng.randbytes(1568)), accept = Record(0xF0, m_rng.randbytes(1568)); + BOOST_REQUIRE_EQUAL(offer.size(), 1572U); + const Bytes unknown = Record(0xF1, std::vector{std::byte{0x00}, std::byte{0x01}}); + const Bytes other_unknown = Record(0xF1, std::vector{std::byte{0x00}, std::byte{0x02}}); + std::array ss; + m_rng.fillrand(ss); + std::array other_ss{ss}; + other_ss[31] ^= std::byte{0x80}; + const auto flip = [](Bytes bytes, size_t pos) { bytes.at(pos) ^= std::byte{0x01}; return bytes; }; + + struct Case { + std::string name; + bool ecdh_changes; + ChainType chain; + Bytes ent_i, ent_r, contents_r, contents_i; + std::array ss; + }; + const std::vector cases{ + {"base", false, ChainType::MAIN, ent_i, ent_r, offer, accept, ss}, + {"initiator ellswift", true, ChainType::MAIN, other_ent_i, ent_r, offer, accept, ss}, + {"responder ellswift", true, ChainType::MAIN, ent_i, other_ent_r, offer, accept, ss}, + {"ek", false, ChainType::MAIN, ent_i, ent_r, flip(offer, 1000), accept, ss}, + {"ct", false, ChainType::MAIN, ent_i, ent_r, offer, flip(accept, 1000), ss}, + {"unknown record", false, ChainType::MAIN, ent_i, ent_r, Concat({offer, unknown}), accept, ss}, + {"unknown record payload", false, ChainType::MAIN, ent_i, ent_r, Concat({offer, other_unknown}), accept, ss}, + {"duplicate record", false, ChainType::MAIN, ent_i, ent_r, Concat({offer, offer}), accept, ss}, + {"record order", false, ChainType::MAIN, ent_i, ent_r, Concat({unknown, offer}), accept, ss}, + {"initiator records", false, ChainType::MAIN, ent_i, ent_r, offer, Concat({accept, unknown}), ss}, + {"length prefix", false, ChainType::MAIN, ent_i, ent_r, Concat({offer, std::span{accept}.first(1)}), Bytes(accept.begin() + 1, accept.end()), ss}, + {"swapped contents", false, ChainType::MAIN, ent_i, ent_r, accept, offer, ss}, + {"ml-kem secret", false, ChainType::MAIN, ent_i, ent_r, offer, accept, other_ss}, + {"magic", true, ChainType::REGTEST, ent_i, ent_r, offer, accept, ss}, + }; + + std::vector> derived; + Bytes base_ecdh_session_id; + for (const auto& c : cases) { + BOOST_TEST_MESSAGE("binding case: " << c.name); + SelectParams(c.chain); + BIP324Cipher initiator(key_i, c.ent_i), responder(key_r, c.ent_r); + InitializePair(initiator, responder, /*retain_for_hybrid=*/true); + const Bytes ecdh_session_id = ToBytes(initiator.GetSessionID()); + if (base_ecdh_session_id.empty()) base_ecdh_session_id = ecdh_session_id; + // Only the hybrid layer sees the version contents and the ML-KEM secret. + BOOST_CHECK((ecdh_session_id != base_ecdh_session_id) == c.ecdh_changes); + + // Two honest sides still agree. + BOOST_REQUIRE(SwitchWith(initiator, c.contents_r, c.contents_i, c.ss)); + BOOST_REQUIRE(SwitchWith(responder, c.contents_r, c.contents_i, c.ss)); + BOOST_CHECK(std::ranges::equal(initiator.GetSessionID(), responder.GetSessionID())); + const Bytes confirmation_i = EncryptPacket(initiator, {}, /*ignore=*/true); + const Bytes confirmation_r = EncryptPacket(responder, {}, /*ignore=*/true); + BOOST_CHECK(Receives(responder, confirmation_i, {}, /*ignore=*/true)); + BOOST_CHECK(Receives(initiator, confirmation_r, {}, /*ignore=*/true)); + CheckPair(initiator, responder, unknown); + derived.push_back({ToBytes(initiator.GetSessionID()), confirmation_i, confirmation_r}); + } + + // Every change gives a different session id and different keys in both directions. + for (size_t a = 0; a < cases.size(); ++a) { + for (size_t b = a + 1; b < cases.size(); ++b) { + for (size_t value = 0; value < 3; ++value) { + BOOST_CHECK_MESSAGE(derived[a][value] != derived[b][value], cases[a].name << " vs " << cases[b].name << " value " << value); + } + } + } +} + +BOOST_AUTO_TEST_CASE(hybrid_api_order) +{ + SelectParams(ChainType::MAIN); + const CKey key_i = GenerateRandomKey(), key_r = GenerateRandomKey(); + const Bytes ent_i = m_rng.randbytes(32), ent_r = m_rng.randbytes(32); + const Bytes contents_r = Record(0xF0, m_rng.randbytes(1568)), contents_i = Record(0xF0, m_rng.randbytes(1568)); + std::array ss; + m_rng.fillrand(ss); + const Bytes message = m_rng.randbytes(40); + + // Before Initialize, nothing is accepted. + { + BIP324Cipher cipher(key_i, ent_i); + BOOST_CHECK(!cipher.SwitchToHybrid(ss)); + BOOST_CHECK(!cipher.AddVersionContents(contents_r)); + BOOST_CHECK(!cipher.SwitchToHybrid(ss)); + cipher.DiscardHybridSecret(); + BOOST_CHECK(!cipher); + } + + // Without retain_for_hybrid (the default), today's secret lifetime: nothing to switch with. + { + BIP324Cipher initiator(key_i, ent_i), responder(key_r, ent_r); + InitializePair(initiator, responder, /*retain_for_hybrid=*/false); + const Bytes session_id = ToBytes(initiator.GetSessionID()); + BOOST_CHECK(!initiator.HoldsHybridSecret()); + BOOST_CHECK(!initiator.AddVersionContents(contents_r)); + BOOST_CHECK(!initiator.SwitchToHybrid(ss)); + BOOST_CHECK(std::ranges::equal(initiator.GetSessionID(), session_id)); + CheckPair(initiator, responder, message); + } + + // With retention: switching early or twice, and a third contents call, are rejected and change nothing. + { + BIP324Cipher initiator(key_i, ent_i), responder(key_r, ent_r); + InitializePair(initiator, responder, /*retain_for_hybrid=*/true); + const Bytes ecdh_session_id = ToBytes(initiator.GetSessionID()); + BOOST_CHECK(!initiator.SwitchToHybrid(ss)); + BOOST_CHECK(initiator.AddVersionContents(contents_r)); + BOOST_CHECK(!initiator.SwitchToHybrid(ss)); + BOOST_CHECK(std::ranges::equal(initiator.GetSessionID(), ecdh_session_id)); + // The ECDH keys keep working while the switch is pending. + CheckPair(initiator, responder, message); + BOOST_CHECK(initiator.AddVersionContents(contents_i)); + BOOST_CHECK(!initiator.AddVersionContents(contents_i)); + BOOST_CHECK(responder.AddVersionContents(contents_r)); + BOOST_CHECK(responder.AddVersionContents(contents_i)); + BOOST_CHECK(!responder.AddVersionContents({})); + + // The switch wipes the retained secret and the transcript. + BOOST_CHECK(initiator.HoldsHybridSecret() && responder.HoldsHybridSecret()); + BOOST_CHECK(initiator.SwitchToHybrid(ss)); + BOOST_CHECK(responder.SwitchToHybrid(ss)); + BOOST_CHECK(!initiator.HoldsHybridSecret() && !responder.HoldsHybridSecret()); + const Bytes hybrid_session_id = ToBytes(initiator.GetSessionID()); + BOOST_CHECK(hybrid_session_id != ecdh_session_id); + BOOST_CHECK(std::ranges::equal(responder.GetSessionID(), hybrid_session_id)); + BOOST_CHECK(!initiator.SwitchToHybrid(ss)); + BOOST_CHECK(!initiator.AddVersionContents(contents_i)); + BOOST_CHECK(std::ranges::equal(initiator.GetSessionID(), hybrid_session_id)); + CheckPair(initiator, responder, message); + initiator.DiscardHybridSecret(); + CheckPair(initiator, responder, message); + } + + // DiscardHybridSecret (a legacy outcome): the ECDH keys stay in use and nothing can switch later. + { + BIP324Cipher initiator(key_i, ent_i), responder(key_r, ent_r); + InitializePair(initiator, responder, /*retain_for_hybrid=*/true); + const Bytes ecdh_session_id = ToBytes(initiator.GetSessionID()); + BOOST_CHECK(initiator.AddVersionContents(contents_r)); + BOOST_CHECK(initiator.AddVersionContents(contents_i)); + BOOST_CHECK(initiator.HoldsHybridSecret()); + initiator.DiscardHybridSecret(); + BOOST_CHECK(!initiator.HoldsHybridSecret()); + initiator.DiscardHybridSecret(); + responder.DiscardHybridSecret(); + BOOST_CHECK(!responder.HoldsHybridSecret()); + BOOST_CHECK(!initiator.SwitchToHybrid(ss)); + BOOST_CHECK(!initiator.AddVersionContents(contents_i)); + BOOST_CHECK(!responder.AddVersionContents(contents_r)); + BOOST_CHECK(!responder.SwitchToHybrid(ss)); + BOOST_CHECK(std::ranges::equal(initiator.GetSessionID(), ecdh_session_id)); + CheckPair(initiator, responder, message); + } +} + +BOOST_AUTO_TEST_CASE(hybrid_rekey_boundaries) +{ + SelectParams(ChainType::MAIN); + const CKey key_i = GenerateRandomKey(), key_r = GenerateRandomKey(); + const Bytes ent_i = m_rng.randbytes(32), ent_r = m_rng.randbytes(32); + const Bytes contents_r = Record(0xF0, m_rng.randbytes(1568)), contents_i = Record(0xF0, m_rng.randbytes(1568)); + std::array ss; + m_rng.fillrand(ss); + // Hybrid packets 0..449 in each direction cross the rekeys after packets 223 and 447. + constexpr unsigned HYBRID_PACKETS{2 * BIP324Cipher::REKEY_INTERVAL + 2}; + + std::optional> fresh_stream; + for (const unsigned ecdh_packets : {0U, 1U, 223U, 224U, 447U}) { + BOOST_TEST_MESSAGE("switch after " << ecdh_packets << " ECDH packets each way"); + BIP324Cipher initiator(key_i, ent_i), responder(key_r, ent_r); + InitializePair(initiator, responder, /*retain_for_hybrid=*/true); + for (unsigned i = 0; i < ecdh_packets; ++i) { + BOOST_REQUIRE(Receives(responder, EncryptPacket(initiator, {}, /*ignore=*/true), {}, /*ignore=*/true)); + BOOST_REQUIRE(Receives(initiator, EncryptPacket(responder, {}, /*ignore=*/true), {}, /*ignore=*/true)); + } + BOOST_REQUIRE(SwitchWith(initiator, contents_r, contents_i, ss)); + BOOST_REQUIRE(SwitchWith(responder, contents_r, contents_i, ss)); + + std::vector stream; + for (unsigned i = 0; i < HYBRID_PACKETS; ++i) { + // Packet 0 is the key confirmation, an empty decoy. + const bool ignore{i == 0}; + const Bytes contents(ignore ? 0 : 1 + i % 7, std::byte(i & 0xff)); + stream.push_back(EncryptPacket(initiator, contents, ignore)); + BOOST_REQUIRE_MESSAGE(Receives(responder, stream.back(), contents, ignore), "initiator packet " << i); + stream.push_back(EncryptPacket(responder, contents, ignore)); + BOOST_REQUIRE_MESSAGE(Receives(initiator, stream.back(), contents, ignore), "responder packet " << i); + } + // All four hybrid ciphers start at 0, whatever the ECDH ciphers carried before the switch. + if (!fresh_stream) { + fresh_stream = std::move(stream); + } else { + BOOST_CHECK(stream == *fresh_stream); + } + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/data/pq_transport_vectors.json b/src/test/data/pq_transport_vectors.json new file mode 100644 index 0000000000..04943c09e5 --- /dev/null +++ b/src/test/data/pq_transport_vectors.json @@ -0,0 +1,239 @@ +{ + "comment": [ + "Hybrid post-quantum BIP324 v2 transport: the ML-KEM-1024 key recipe under version-contents header 0xf0, specified in doc/design/pq-transport.md.", + "Generated by contrib/devtools/generate-pq-transport-vectors.py from the Python reference in test/functional/test_framework/crypto. Do not edit by hand.", + "Byte strings are lowercase hex. Packets are BIP324 packets: encrypted length (3) || encrypted header and contents || tag (16). Each side's key confirmation is its hybrid packet 0, and its first application packet is hybrid packet 1.", + "confirmation_failures are first hybrid packets that the named receiver must reject under the honest keys of the vector; negative_vectors are sides that derived from different inputs than their peer." + ], + "header": "f0", + "salt_label": "qbit_v2_hybrid_mlkem1024", + "labels": [ + "initiator_L", + "initiator_P", + "responder_L", + "responder_P", + "session_id" + ], + "vectors": [ + { + "comment": "Mainnet, one offer record and one accept record (the ordinary exchange).", + "chain": "main", + "magic": "444f24a8", + "initiator_privkey": "b90007e04658c9f83710e61170984a6261ee2091d3f237a1f065666e741e96eb", + "initiator_ellswift": "6b5fb320cab84c788ef6393262172b222facedee5e55f4e2fa44bae00b1f36102dd8739200e025a0443a6c6d3d52f003e4b94d296ed3a68e9dacc567f0c7bd09", + "responder_privkey": "5047c9486dd8881232243d049d47d5afacf4438dffde0579cdbe28f807231f18", + "responder_ellswift": "3d3449542df2dc02aedca2a14e5c6b22f2b797b3191488cdbdc1ff6ad3ef40a77e62023395a490bbacc9de3da7fd37f70592ec361e34c01c761ad4ea2029712c", + "ss_ecdh": "7e8dd0eda7d613290fcfc7919af8675a2405e6585378777b9d2f02e592f1fb1d", + "ecdh_session_id": "ab8e9ae6fe9d33e2e10f847855298fbe295ed45625608b425ca118d09dbfd0a4", + "initiator_garbage_terminator": "4f698c0b17c882f46d09697e67cbcd75", + "responder_garbage_terminator": "2c85474143c749d2ee0489bb61920613", + "mlkem_keygen_seed": "1b5f52efd3c68106de6e780c8e7e0d57e78445b65a778779d2cd78cec5ed6289f973c11bd432e4642d99bb2d272e4da7032f4645430ac8d5ac98b9d782835ccd", + "mlkem_encaps_m": "3ddd894ca239fdae4e54bf00f7608fb9a1348fce50f7ba09f163b00e82f1474c", + "ek": "4ef53cd8d75470aa5492cc10d1b519a02ac168e5394226756c89ace3a2174cb08fe962b490f3c2cf88a60391513083a330718b73257c554c7cf0ca1865fbbf682435605c5a7f0636b19ab1c7e2c9cb40cf62c47940b17855f7c2e4d1869409ad7b3b047052bfd7366195d84890f03ac23032a63c545ed99f07e55bc81771836917b5139954d413162b9b5745556865710a5401bd9b6e3382adef4108b80833ed2337aaf2392a9704e239cb4a35332bf8257c288200389d054ba9acf772c61ab9f74875353b1bb933bff0e12f72b8c1537c4b47a74750d5617eb4c99d72af4e401961f588ff9a617b775ef7280a8a2843f1b408e6cbcf7385493a3b9e86230ba3c5676c5cb8885c317855a103e915aeac2cb27c36392494bb2aac04f481d38ba33cf7376711cce831380c580bdd887b3183a8fb2642857b3283c51a82c788e5293ddbd01bc46609f6017f40294f4b43842a709db0a7203565ceebda5721c1201569217ad817cd6316fc8c7f3036a4d5a679095bb4376c9e75360a64051f17fb8445f4556d37180c83cbee261629620ecec139bb8c609da90f9da2a8e1297793546ef90c0bffb3b0c836215bb9108191af7803cd5f26163c577404002aaed44de693892cb04f87e060dc5b9db9d00a4ea6b2734505c04857c31ccaa68bc833733d1e2837e0b5aec46354867933824399f132805893330c73c11c988d1adb6aac9b9bbcbc41263158e2b960c30665571484900ac7621a3e7fd1252ffb271f763ddc98293a5b929790468b9b4ed848c9ba531adaa31fec60367bc48eae612777cb79d7b56a17f8a6afc1c1febb86ec286ef9260cb27079c0d5a74910b3ef26b09ec96de697735e91841b9224d33abf2e1b14b15460d9055093268c75c17aa1ba15b5463e68d32c99a10d40c603b45c1fb81b5965c17379b898cba3cd1471bb07c8586f232d5c29ade9831f06163a3d60c672535e8fb0a01fd4435853480b073ec0ac32bc8a876cc3cd43bbb7bf8b3dc77874be68bff1d0cdf6a2900d8bca43f29b87c4395c7a794c39a45e89c3d9e3bf7046b5bc204a75a30a707242525c81cf4ac93c2a8b266c59eaaa0d997348e9fc7bd0294231c559a5cc1465a22317e714cc1431c092bed0665f242098c556b1c091a18519c1fa91354d6403f71972e78c23a28a276798826b3b711e76aa17f27c24214d6c1c6e8a53736cd986d9e446068532f1a6214cc98eb1625cd064456b0481abe2a38aeaa26845b77b1926b791a4a3b56739997edeb097284a9a6217438fd92b65399d0ecb2f23ccb1c8397496413615c26304d5272d60bc34f3ba6b5c652f693bc702be90349f27fa019173508a94bdf5f1153b341d20d040eb506ea0d40b561c0178c662a7c5b6bd130535355a8a5438ae1c0e2ab872f92800ba319b34d587e52860eb183b0bdb24ec024a111b369d3871c1c7ce9ec97207a208b4ec32ea142c61d23bdbb40325ca0756b36c5de19bed851d0ce6a30c175e467957d72b65f5552007dab6cd474d095ab556353111b9664a012d7fd767b3d549138580a087103d72bb602520f9393317e72e3fbc5d7206a24ccaae03fa844619a4acaa8c246053ed122c9765bc9164aa1c7257ac5893e0619bb965c48f24b5a5747d02f38705a09c905b82918856487382f40c28b73916b8059c29b39853189997978e49b05df1495c66a777e5f84ef99575f51b33fb157d41e54a7611c082387c6da242c3042fce7711525104f46233cad9882e45796a09c59b8471287b19c4f6205ea0b3543c341f8c169c11010cd52abda1b934024b9181c047ecad01844709d7834aa056648097f2109e4fac640d32ad80cc5581ac60296a5b51d615241c8cc0bcccfe777b97e6a5475011603a8ad7eaafe741572018840b989d83f986163633bc68919d571211d317c0a3c0347529b49101c2d451844b5083404da25b4d37931743139953fcc8374369260b88193284bc96250d02c05ecc6809a44a38e24261083dc3953d4e8a093bda800b4b2614d76a4355c6a900251e3cb9de069ed2ebaa34b34aae3661362218b5350b4f68c9bc6b11d0f5753e792cf300acdb94c52eab50d406967f498a8327420d6397886380a94019b307b8188a4496b223ebe27075b87c5d8680855ac7c5b892d357247aecb55e6089a2dba345f65d259b8d40a253a450a546b46abc3d1804f4c6ecc2a992fdff4e7d", + "ct": "79578b92edfe5618d5dd9284a2004e45ebe3a37cc4c0eef53eeaf8d322303b91edfb0aa7c453d4c0d06bc29c2e904f20db8e6e0567d6a1303ad190d7c087a76da634a6e0ac11f4f78496ca11862c87acb0fc273061479a89758886d0b0d78375b1edf317ddfa889507df4e74853fe34892580a127e104f0bc4df583d8415b54367efafea0995c7e7ed7a1dedb8112a22e8278f90361dbf0a1f7af1246b609c9981f95a919312540d8bbd207eb75cf9feb4f9d2689b071f1f5b0d790573ca40db398374d3e50e3c2d46bb966567cd9c9b1589ee62206ead95a80483d4fa567ff697d40d804acdde8ec654fd5ba413133fabb99db19d949ecde78d55cb563766e714cf2138f50457809757b618b004ad48c1cb0b0f1c95e0fbe1f43813d5f035c4b3ad1b6a5c1c5fa0a1d27a0cb6d3de7b3a054e6fcbc24e286e6a77e45ab56bc5ecb070a00c22974d05832832bd40ab96010039be5da0464731993e2bccfcc468533cbfe3089a6f579d5a06cbe4bf08ddb85fe9996ed95fe9442130e49c61d1bf0881968672eae12c568189f3285671358e68ae0fbb5075ed137127788aa1f96033c89f41ade84cc61aaeae9b2c86862b1e956c74d39bd6e8f00907fd586cd099ab4224883c2e0c07ac4bb3f74a5a59975ef0e844583370cf077671fbfc2a333182dce2ac9cc8f3ba0d73cd27ae60aabc1d7689f7bd2d7ecd5d97164293cc88fe7adbadb3a24798f7b2be578c7780a681cc902b69b7fe4aa8f535e4a9ed83d002c1ea0f28e711be6231cabde2ea19655688c8d70b413420651852db3bcad5c6b66267d516088b3955a6884ca44a83c9f179c5901ff4edefea074ee0cd425a7dd8a855ea49affbd7a8604c490213dfa7bf48ac12b738fae7cd1697e757f26a59826e64b7799a47703e037c9ca7cf18828574092a9b0af255a5180eef96f96e5d2f51cc621d7a03c7bb05eb80d386415f243f5d59451d5f95c93f4616cca643cd6be65109d440461e752d2999745facd5eacaea338d6ce940b54c8dba44f059c94fa65a74456b747c065f2aba2b8580931e1e5db4cd6505d4cc50ce68e15a1f9e0069edd1a8d5bbfa9180f0a88fef1d773a5b417b67ddd8729c3a781bfcdeb527126579f5d11c5550525869a3236e1f21940542c7ce69e3a265c24d9aefeab6c0852636708a74718729f35a401fd73b46b204ec92b40d9ec0e5f924c967a8a4e219910d40af2d19e76869bfb7c95868e4572334bc35f7071d6254cb3cdf82ee94d3b685fe48a49ad95503ca19b39e316e57a32a1eaa9af4573a32bfe94dd77c744484781c2efe891e2b4b65b38e12dbab69b436d4b0ac6a9b512ee898b95906163f32839fa2985fadead35d79fa03a284debc60c51daa8a017c1d4dc4dc7e884b1374d131fde2ca6e591672b259754b4f76bbf6ed3eeeee94e4e7c496344a3b38c8b4dea12684bbc0713ec6b7684479e883a0a515d681581beab290d952fa20f6f9ae63dc00a55d1ef3f592d9a86d97a40ae74717623dc1ffdf2588f241d6600e6f34163db805273266f58e75e074d7d079d40755012da5ba3203d2bfdc9100e361148456a7ac423ba9cf3bd135f3e64c2f262fcdb728e0682e8297cc2af4e0467603990459d923cf6974cc27f9c02f535d6962cd67bbfd1287ba4148647153fd33ef9e482e2555ff6ee243ce6ab277af1494944b2913ecf29819507571427d7c1852d684a6359a72698b65ef040b8a03c3fffca6f4a9e13359db9a1e3d2e4fac6d469357d9d9ca6663f6a3f80076c6ec937aa16d23b12fb0ee483c260ad6e751a2814720e36dfd26202b14c1cd18e3f9477e924292d84d905c056bd73ceecede186f3b6987abe328fb3e1e48b2bed3885b9d9e9af6bdebc0c7f9cc6f7954b38905e3d8ee41f780ca446792326a94837c367104983ea9b906a7f7a8b4bbf44700694c27e7670e4a7b3e5df7218d8d67fd68bda52592e650de25b1a81ed304d81cadbad366833d94d7e69a8433c9d7edfb1043b99a08265de814121119d3f7eaa74c8082bd9b4a91bb2478605a2f2e077679665d24391ea37c748c5c6e35f00fd202a13c032270b0d30f0af8bb1d7c2570055980748fd7761d6a801043ecc04051f7d60a5b7361436549762a9f3ba7eb7344658b154b8b680ba8ac2eeec07e4cc3cac211255c3f901e713db12984a00450011790a0f2d6af22d5d35b83c5483dd4e8", + "ss_mlkem": "22c8bf5c4858ac2e5f94b6e945ca142ccc28b98e7b954a78899630a7bfa3135f", + "contents_responder": "fd2106f04ef53cd8d75470aa5492cc10d1b519a02ac168e5394226756c89ace3a2174cb08fe962b490f3c2cf88a60391513083a330718b73257c554c7cf0ca1865fbbf682435605c5a7f0636b19ab1c7e2c9cb40cf62c47940b17855f7c2e4d1869409ad7b3b047052bfd7366195d84890f03ac23032a63c545ed99f07e55bc81771836917b5139954d413162b9b5745556865710a5401bd9b6e3382adef4108b80833ed2337aaf2392a9704e239cb4a35332bf8257c288200389d054ba9acf772c61ab9f74875353b1bb933bff0e12f72b8c1537c4b47a74750d5617eb4c99d72af4e401961f588ff9a617b775ef7280a8a2843f1b408e6cbcf7385493a3b9e86230ba3c5676c5cb8885c317855a103e915aeac2cb27c36392494bb2aac04f481d38ba33cf7376711cce831380c580bdd887b3183a8fb2642857b3283c51a82c788e5293ddbd01bc46609f6017f40294f4b43842a709db0a7203565ceebda5721c1201569217ad817cd6316fc8c7f3036a4d5a679095bb4376c9e75360a64051f17fb8445f4556d37180c83cbee261629620ecec139bb8c609da90f9da2a8e1297793546ef90c0bffb3b0c836215bb9108191af7803cd5f26163c577404002aaed44de693892cb04f87e060dc5b9db9d00a4ea6b2734505c04857c31ccaa68bc833733d1e2837e0b5aec46354867933824399f132805893330c73c11c988d1adb6aac9b9bbcbc41263158e2b960c30665571484900ac7621a3e7fd1252ffb271f763ddc98293a5b929790468b9b4ed848c9ba531adaa31fec60367bc48eae612777cb79d7b56a17f8a6afc1c1febb86ec286ef9260cb27079c0d5a74910b3ef26b09ec96de697735e91841b9224d33abf2e1b14b15460d9055093268c75c17aa1ba15b5463e68d32c99a10d40c603b45c1fb81b5965c17379b898cba3cd1471bb07c8586f232d5c29ade9831f06163a3d60c672535e8fb0a01fd4435853480b073ec0ac32bc8a876cc3cd43bbb7bf8b3dc77874be68bff1d0cdf6a2900d8bca43f29b87c4395c7a794c39a45e89c3d9e3bf7046b5bc204a75a30a707242525c81cf4ac93c2a8b266c59eaaa0d997348e9fc7bd0294231c559a5cc1465a22317e714cc1431c092bed0665f242098c556b1c091a18519c1fa91354d6403f71972e78c23a28a276798826b3b711e76aa17f27c24214d6c1c6e8a53736cd986d9e446068532f1a6214cc98eb1625cd064456b0481abe2a38aeaa26845b77b1926b791a4a3b56739997edeb097284a9a6217438fd92b65399d0ecb2f23ccb1c8397496413615c26304d5272d60bc34f3ba6b5c652f693bc702be90349f27fa019173508a94bdf5f1153b341d20d040eb506ea0d40b561c0178c662a7c5b6bd130535355a8a5438ae1c0e2ab872f92800ba319b34d587e52860eb183b0bdb24ec024a111b369d3871c1c7ce9ec97207a208b4ec32ea142c61d23bdbb40325ca0756b36c5de19bed851d0ce6a30c175e467957d72b65f5552007dab6cd474d095ab556353111b9664a012d7fd767b3d549138580a087103d72bb602520f9393317e72e3fbc5d7206a24ccaae03fa844619a4acaa8c246053ed122c9765bc9164aa1c7257ac5893e0619bb965c48f24b5a5747d02f38705a09c905b82918856487382f40c28b73916b8059c29b39853189997978e49b05df1495c66a777e5f84ef99575f51b33fb157d41e54a7611c082387c6da242c3042fce7711525104f46233cad9882e45796a09c59b8471287b19c4f6205ea0b3543c341f8c169c11010cd52abda1b934024b9181c047ecad01844709d7834aa056648097f2109e4fac640d32ad80cc5581ac60296a5b51d615241c8cc0bcccfe777b97e6a5475011603a8ad7eaafe741572018840b989d83f986163633bc68919d571211d317c0a3c0347529b49101c2d451844b5083404da25b4d37931743139953fcc8374369260b88193284bc96250d02c05ecc6809a44a38e24261083dc3953d4e8a093bda800b4b2614d76a4355c6a900251e3cb9de069ed2ebaa34b34aae3661362218b5350b4f68c9bc6b11d0f5753e792cf300acdb94c52eab50d406967f498a8327420d6397886380a94019b307b8188a4496b223ebe27075b87c5d8680855ac7c5b892d357247aecb55e6089a2dba345f65d259b8d40a253a450a546b46abc3d1804f4c6ecc2a992fdff4e7d", + "contents_initiator": "fd2106f079578b92edfe5618d5dd9284a2004e45ebe3a37cc4c0eef53eeaf8d322303b91edfb0aa7c453d4c0d06bc29c2e904f20db8e6e0567d6a1303ad190d7c087a76da634a6e0ac11f4f78496ca11862c87acb0fc273061479a89758886d0b0d78375b1edf317ddfa889507df4e74853fe34892580a127e104f0bc4df583d8415b54367efafea0995c7e7ed7a1dedb8112a22e8278f90361dbf0a1f7af1246b609c9981f95a919312540d8bbd207eb75cf9feb4f9d2689b071f1f5b0d790573ca40db398374d3e50e3c2d46bb966567cd9c9b1589ee62206ead95a80483d4fa567ff697d40d804acdde8ec654fd5ba413133fabb99db19d949ecde78d55cb563766e714cf2138f50457809757b618b004ad48c1cb0b0f1c95e0fbe1f43813d5f035c4b3ad1b6a5c1c5fa0a1d27a0cb6d3de7b3a054e6fcbc24e286e6a77e45ab56bc5ecb070a00c22974d05832832bd40ab96010039be5da0464731993e2bccfcc468533cbfe3089a6f579d5a06cbe4bf08ddb85fe9996ed95fe9442130e49c61d1bf0881968672eae12c568189f3285671358e68ae0fbb5075ed137127788aa1f96033c89f41ade84cc61aaeae9b2c86862b1e956c74d39bd6e8f00907fd586cd099ab4224883c2e0c07ac4bb3f74a5a59975ef0e844583370cf077671fbfc2a333182dce2ac9cc8f3ba0d73cd27ae60aabc1d7689f7bd2d7ecd5d97164293cc88fe7adbadb3a24798f7b2be578c7780a681cc902b69b7fe4aa8f535e4a9ed83d002c1ea0f28e711be6231cabde2ea19655688c8d70b413420651852db3bcad5c6b66267d516088b3955a6884ca44a83c9f179c5901ff4edefea074ee0cd425a7dd8a855ea49affbd7a8604c490213dfa7bf48ac12b738fae7cd1697e757f26a59826e64b7799a47703e037c9ca7cf18828574092a9b0af255a5180eef96f96e5d2f51cc621d7a03c7bb05eb80d386415f243f5d59451d5f95c93f4616cca643cd6be65109d440461e752d2999745facd5eacaea338d6ce940b54c8dba44f059c94fa65a74456b747c065f2aba2b8580931e1e5db4cd6505d4cc50ce68e15a1f9e0069edd1a8d5bbfa9180f0a88fef1d773a5b417b67ddd8729c3a781bfcdeb527126579f5d11c5550525869a3236e1f21940542c7ce69e3a265c24d9aefeab6c0852636708a74718729f35a401fd73b46b204ec92b40d9ec0e5f924c967a8a4e219910d40af2d19e76869bfb7c95868e4572334bc35f7071d6254cb3cdf82ee94d3b685fe48a49ad95503ca19b39e316e57a32a1eaa9af4573a32bfe94dd77c744484781c2efe891e2b4b65b38e12dbab69b436d4b0ac6a9b512ee898b95906163f32839fa2985fadead35d79fa03a284debc60c51daa8a017c1d4dc4dc7e884b1374d131fde2ca6e591672b259754b4f76bbf6ed3eeeee94e4e7c496344a3b38c8b4dea12684bbc0713ec6b7684479e883a0a515d681581beab290d952fa20f6f9ae63dc00a55d1ef3f592d9a86d97a40ae74717623dc1ffdf2588f241d6600e6f34163db805273266f58e75e074d7d079d40755012da5ba3203d2bfdc9100e361148456a7ac423ba9cf3bd135f3e64c2f262fcdb728e0682e8297cc2af4e0467603990459d923cf6974cc27f9c02f535d6962cd67bbfd1287ba4148647153fd33ef9e482e2555ff6ee243ce6ab277af1494944b2913ecf29819507571427d7c1852d684a6359a72698b65ef040b8a03c3fffca6f4a9e13359db9a1e3d2e4fac6d469357d9d9ca6663f6a3f80076c6ec937aa16d23b12fb0ee483c260ad6e751a2814720e36dfd26202b14c1cd18e3f9477e924292d84d905c056bd73ceecede186f3b6987abe328fb3e1e48b2bed3885b9d9e9af6bdebc0c7f9cc6f7954b38905e3d8ee41f780ca446792326a94837c367104983ea9b906a7f7a8b4bbf44700694c27e7670e4a7b3e5df7218d8d67fd68bda52592e650de25b1a81ed304d81cadbad366833d94d7e69a8433c9d7edfb1043b99a08265de814121119d3f7eaa74c8082bd9b4a91bb2478605a2f2e077679665d24391ea37c748c5c6e35f00fd202a13c032270b0d30f0af8bb1d7c2570055980748fd7761d6a801043ecc04051f7d60a5b7361436549762a9f3ba7eb7344658b154b8b680ba8ac2eeec07e4cc3cac211255c3f901e713db12984a00450011790a0f2d6af22d5d35b83c5483dd4e8", + "transcript_input": "6b5fb320cab84c788ef6393262172b222facedee5e55f4e2fa44bae00b1f36102dd8739200e025a0443a6c6d3d52f003e4b94d296ed3a68e9dacc567f0c7bd093d3449542df2dc02aedca2a14e5c6b22f2b797b3191488cdbdc1ff6ad3ef40a77e62023395a490bbacc9de3da7fd37f70592ec361e34c01c761ad4ea2029712cfd2406fd2106f04ef53cd8d75470aa5492cc10d1b519a02ac168e5394226756c89ace3a2174cb08fe962b490f3c2cf88a60391513083a330718b73257c554c7cf0ca1865fbbf682435605c5a7f0636b19ab1c7e2c9cb40cf62c47940b17855f7c2e4d1869409ad7b3b047052bfd7366195d84890f03ac23032a63c545ed99f07e55bc81771836917b5139954d413162b9b5745556865710a5401bd9b6e3382adef4108b80833ed2337aaf2392a9704e239cb4a35332bf8257c288200389d054ba9acf772c61ab9f74875353b1bb933bff0e12f72b8c1537c4b47a74750d5617eb4c99d72af4e401961f588ff9a617b775ef7280a8a2843f1b408e6cbcf7385493a3b9e86230ba3c5676c5cb8885c317855a103e915aeac2cb27c36392494bb2aac04f481d38ba33cf7376711cce831380c580bdd887b3183a8fb2642857b3283c51a82c788e5293ddbd01bc46609f6017f40294f4b43842a709db0a7203565ceebda5721c1201569217ad817cd6316fc8c7f3036a4d5a679095bb4376c9e75360a64051f17fb8445f4556d37180c83cbee261629620ecec139bb8c609da90f9da2a8e1297793546ef90c0bffb3b0c836215bb9108191af7803cd5f26163c577404002aaed44de693892cb04f87e060dc5b9db9d00a4ea6b2734505c04857c31ccaa68bc833733d1e2837e0b5aec46354867933824399f132805893330c73c11c988d1adb6aac9b9bbcbc41263158e2b960c30665571484900ac7621a3e7fd1252ffb271f763ddc98293a5b929790468b9b4ed848c9ba531adaa31fec60367bc48eae612777cb79d7b56a17f8a6afc1c1febb86ec286ef9260cb27079c0d5a74910b3ef26b09ec96de697735e91841b9224d33abf2e1b14b15460d9055093268c75c17aa1ba15b5463e68d32c99a10d40c603b45c1fb81b5965c17379b898cba3cd1471bb07c8586f232d5c29ade9831f06163a3d60c672535e8fb0a01fd4435853480b073ec0ac32bc8a876cc3cd43bbb7bf8b3dc77874be68bff1d0cdf6a2900d8bca43f29b87c4395c7a794c39a45e89c3d9e3bf7046b5bc204a75a30a707242525c81cf4ac93c2a8b266c59eaaa0d997348e9fc7bd0294231c559a5cc1465a22317e714cc1431c092bed0665f242098c556b1c091a18519c1fa91354d6403f71972e78c23a28a276798826b3b711e76aa17f27c24214d6c1c6e8a53736cd986d9e446068532f1a6214cc98eb1625cd064456b0481abe2a38aeaa26845b77b1926b791a4a3b56739997edeb097284a9a6217438fd92b65399d0ecb2f23ccb1c8397496413615c26304d5272d60bc34f3ba6b5c652f693bc702be90349f27fa019173508a94bdf5f1153b341d20d040eb506ea0d40b561c0178c662a7c5b6bd130535355a8a5438ae1c0e2ab872f92800ba319b34d587e52860eb183b0bdb24ec024a111b369d3871c1c7ce9ec97207a208b4ec32ea142c61d23bdbb40325ca0756b36c5de19bed851d0ce6a30c175e467957d72b65f5552007dab6cd474d095ab556353111b9664a012d7fd767b3d549138580a087103d72bb602520f9393317e72e3fbc5d7206a24ccaae03fa844619a4acaa8c246053ed122c9765bc9164aa1c7257ac5893e0619bb965c48f24b5a5747d02f38705a09c905b82918856487382f40c28b73916b8059c29b39853189997978e49b05df1495c66a777e5f84ef99575f51b33fb157d41e54a7611c082387c6da242c3042fce7711525104f46233cad9882e45796a09c59b8471287b19c4f6205ea0b3543c341f8c169c11010cd52abda1b934024b9181c047ecad01844709d7834aa056648097f2109e4fac640d32ad80cc5581ac60296a5b51d615241c8cc0bcccfe777b97e6a5475011603a8ad7eaafe741572018840b989d83f986163633bc68919d571211d317c0a3c0347529b49101c2d451844b5083404da25b4d37931743139953fcc8374369260b88193284bc96250d02c05ecc6809a44a38e24261083dc3953d4e8a093bda800b4b2614d76a4355c6a900251e3cb9de069ed2ebaa34b34aae3661362218b5350b4f68c9bc6b11d0f5753e792cf300acdb94c52eab50d406967f498a8327420d6397886380a94019b307b8188a4496b223ebe27075b87c5d8680855ac7c5b892d357247aecb55e6089a2dba345f65d259b8d40a253a450a546b46abc3d1804f4c6ecc2a992fdff4e7dfd2406fd2106f079578b92edfe5618d5dd9284a2004e45ebe3a37cc4c0eef53eeaf8d322303b91edfb0aa7c453d4c0d06bc29c2e904f20db8e6e0567d6a1303ad190d7c087a76da634a6e0ac11f4f78496ca11862c87acb0fc273061479a89758886d0b0d78375b1edf317ddfa889507df4e74853fe34892580a127e104f0bc4df583d8415b54367efafea0995c7e7ed7a1dedb8112a22e8278f90361dbf0a1f7af1246b609c9981f95a919312540d8bbd207eb75cf9feb4f9d2689b071f1f5b0d790573ca40db398374d3e50e3c2d46bb966567cd9c9b1589ee62206ead95a80483d4fa567ff697d40d804acdde8ec654fd5ba413133fabb99db19d949ecde78d55cb563766e714cf2138f50457809757b618b004ad48c1cb0b0f1c95e0fbe1f43813d5f035c4b3ad1b6a5c1c5fa0a1d27a0cb6d3de7b3a054e6fcbc24e286e6a77e45ab56bc5ecb070a00c22974d05832832bd40ab96010039be5da0464731993e2bccfcc468533cbfe3089a6f579d5a06cbe4bf08ddb85fe9996ed95fe9442130e49c61d1bf0881968672eae12c568189f3285671358e68ae0fbb5075ed137127788aa1f96033c89f41ade84cc61aaeae9b2c86862b1e956c74d39bd6e8f00907fd586cd099ab4224883c2e0c07ac4bb3f74a5a59975ef0e844583370cf077671fbfc2a333182dce2ac9cc8f3ba0d73cd27ae60aabc1d7689f7bd2d7ecd5d97164293cc88fe7adbadb3a24798f7b2be578c7780a681cc902b69b7fe4aa8f535e4a9ed83d002c1ea0f28e711be6231cabde2ea19655688c8d70b413420651852db3bcad5c6b66267d516088b3955a6884ca44a83c9f179c5901ff4edefea074ee0cd425a7dd8a855ea49affbd7a8604c490213dfa7bf48ac12b738fae7cd1697e757f26a59826e64b7799a47703e037c9ca7cf18828574092a9b0af255a5180eef96f96e5d2f51cc621d7a03c7bb05eb80d386415f243f5d59451d5f95c93f4616cca643cd6be65109d440461e752d2999745facd5eacaea338d6ce940b54c8dba44f059c94fa65a74456b747c065f2aba2b8580931e1e5db4cd6505d4cc50ce68e15a1f9e0069edd1a8d5bbfa9180f0a88fef1d773a5b417b67ddd8729c3a781bfcdeb527126579f5d11c5550525869a3236e1f21940542c7ce69e3a265c24d9aefeab6c0852636708a74718729f35a401fd73b46b204ec92b40d9ec0e5f924c967a8a4e219910d40af2d19e76869bfb7c95868e4572334bc35f7071d6254cb3cdf82ee94d3b685fe48a49ad95503ca19b39e316e57a32a1eaa9af4573a32bfe94dd77c744484781c2efe891e2b4b65b38e12dbab69b436d4b0ac6a9b512ee898b95906163f32839fa2985fadead35d79fa03a284debc60c51daa8a017c1d4dc4dc7e884b1374d131fde2ca6e591672b259754b4f76bbf6ed3eeeee94e4e7c496344a3b38c8b4dea12684bbc0713ec6b7684479e883a0a515d681581beab290d952fa20f6f9ae63dc00a55d1ef3f592d9a86d97a40ae74717623dc1ffdf2588f241d6600e6f34163db805273266f58e75e074d7d079d40755012da5ba3203d2bfdc9100e361148456a7ac423ba9cf3bd135f3e64c2f262fcdb728e0682e8297cc2af4e0467603990459d923cf6974cc27f9c02f535d6962cd67bbfd1287ba4148647153fd33ef9e482e2555ff6ee243ce6ab277af1494944b2913ecf29819507571427d7c1852d684a6359a72698b65ef040b8a03c3fffca6f4a9e13359db9a1e3d2e4fac6d469357d9d9ca6663f6a3f80076c6ec937aa16d23b12fb0ee483c260ad6e751a2814720e36dfd26202b14c1cd18e3f9477e924292d84d905c056bd73ceecede186f3b6987abe328fb3e1e48b2bed3885b9d9e9af6bdebc0c7f9cc6f7954b38905e3d8ee41f780ca446792326a94837c367104983ea9b906a7f7a8b4bbf44700694c27e7670e4a7b3e5df7218d8d67fd68bda52592e650de25b1a81ed304d81cadbad366833d94d7e69a8433c9d7edfb1043b99a08265de814121119d3f7eaa74c8082bd9b4a91bb2478605a2f2e077679665d24391ea37c748c5c6e35f00fd202a13c032270b0d30f0af8bb1d7c2570055980748fd7761d6a801043ecc04051f7d60a5b7361436549762a9f3ba7eb7344658b154b8b680ba8ac2eeec07e4cc3cac211255c3f901e713db12984a00450011790a0f2d6af22d5d35b83c5483dd4e8", + "transcript_hash": "66f0655505928bdeec6ce4cf2b1de08ff16c2cb1cf04b781011e30ca6687c240", + "salt": "716269745f76325f6879627269645f6d6c6b656d31303234444f24a8", + "ikm": "7e8dd0eda7d613290fcfc7919af8675a2405e6585378777b9d2f02e592f1fb1d22c8bf5c4858ac2e5f94b6e945ca142ccc28b98e7b954a78899630a7bfa3135f66f0655505928bdeec6ce4cf2b1de08ff16c2cb1cf04b781011e30ca6687c240", + "prk": "cacbbffc106ea035465bc498d22562e83989c039390ddc1ea36e0fe7ba0ac6b4", + "initiator_L": "d723af332d033d229fe91db011f04cd005657fcc35d59b93b6771e77dc9bd572", + "initiator_P": "72462d2ed71f2b6b9c7435745166155bbd61ab04e2e21798945d1282f690e7d2", + "responder_L": "167756b477acb286099da2a866ad9e2d94ec719236dc61836f2956451148ec12", + "responder_P": "95eb7e61e0e575bd36db2a0b589553984942db0da9f43f8d8bc48012f2323d21", + "session_id": "c749d7dadf82b12615fa4ccdeef8cbce4ffb935ad22a7eac1fbc52d8d1394ea4", + "initiator_confirmation": "69abb2b1c9601dad7baf8038bc3601eaa42527de", + "responder_confirmation": "a634faec713fb192a52b9e810216b8a36d39516b", + "initiator_first_contents": "120123456789abcdef", + "initiator_first_packet": "4bbe69dca55d32377b3cee47741171e83115bae7d56d7a9a81dd94394a", + "responder_first_contents": "130123456789abcdef", + "responder_first_packet": "be28dfc2f7f52321d961b644a0ff9fb473d4f9055c6e9523c696751721" + }, + { + "comment": "Regtest, one offer record and one accept record.", + "chain": "regtest", + "magic": "a66b1fda", + "initiator_privkey": "0df7f6c17b741f30d8bd9f5746f019cc1ea364e4e9a75817e181b549d7b0b5b3", + "initiator_ellswift": "fd575a36275251cbf6a287fcafac3419d93b941834964ef2c71235474a86325e075cdcd6a60c478c638ad23cf2c3436395e56b1bb02da9d45c785ba20b6ef484", + "responder_privkey": "eb4a8dbc4505b9df7274e18bc3b620950db233416242a1f0fe3aefcff30deca2", + "responder_ellswift": "cdcdc96b9b01f8dffb480f2cb943e33fcfaa1dd3b20c6be6e25375bdc0f2358fb70615a2103fedde8396327d92966af4a5022bd5537c3816fd21bc036d7dcca3", + "ss_ecdh": "7cc5f298684b388f81e651af731d3d6231f0d0c4c01ce8aae3695f7c24d0d8d3", + "ecdh_session_id": "45094627e9ead1daaa94b2544a58e5086e5e8d0e8d880b971b686e8ec330bd86", + "initiator_garbage_terminator": "3f75a8e72d0759da9311620c7a73dfb9", + "responder_garbage_terminator": "0cebb7d6501e09172b2962ddd16203f9", + "mlkem_keygen_seed": "2147741c766fc3c925963d84d939aa8d54ecf928052d948102f5585d2d100287d9ba0dbbdb1518dec4061dfb7cc2248c03c90ec2f49e361ca59d007f61f88703", + "mlkem_encaps_m": "16c0ebdd39df0e8a59e598c9e300a6a7b22e4e51f6b38a54c77a0c07ef44bb53", + "ek": "36a9c8146c2f789c479f2acd47c7c64f7594550b059d2a4cb7f3add2970ed600888ad782df574a72608283085a433c0ceb9300f5440141848e8352874b83839659629de75e953a0b25a74e8d449c7eb215de1042ef95b140bac0d35cb21c31204c8c22e09861d0a63c6af3029e489e50b023b6dab3295a6d1a7533f571741a5602d25286cbab504b5724371712aa65a84c47782ffc69f1414c81f47479368e6dab1cb14c1fabe24a9531c2e5e97e54c2625b56204104cd11fb77ee999682d745a832c10120ae13b63a016ab99a46a57f549ea5fa6668d2491a4b8689c96b1f8414b6815a22ba70dd2a459cf64365e72677f3b7a74c33a3a62f2fa345ca3b60b41c7abe1a2e1e6b3f43d68be86cce15eba2470204ecc458cb082196d814d9c22aa58675d3337e64b36198d92c1b17470222cab7696f0d8ca4488966205360dfca5d85d1a90ba80b58688adadc02299716bf086dcf5cbc06414cb1fc58091c5a68281f42940fc6da795bb320fefb88cd60727db44a8fda0110c92529f64f726b1834470b1224a1a5c59845b945a1e1a2316039f4b64df9934b40f89df766c279e75a91450ad0e68276f91a3efac90536b836364d55080056d0622533800126b818685a9ad8b052ec72aed13c80085ac9b07a6186553740c7c8a27f1a593a19719eae0519de338408dc7441724248f2b5803682cbf36debd97b265ab60362b59ab2585cf518252565ab1143ed8a53bf40a03d97022b0100afe597226b9e1d3793c98bb9b0b5b167031cb0bc775c16699e785cd60aabeee250a55b5c0881af22265962855a13654dd7022603078b038745c1aa515188642280b2ce68437b5bc458b463cdab65fac07755c862a8d1cd09ca11fb625c61136e4f4c022d6b5f94f5276098bf709b56a5a57a736422a93898041a42075926de23b90f105f488c1d1cb95e7ba5049ee7b23d4766ef6823d45a1d623cbc9d2a9ea1b43dd0b4b30026b2e0d84533142bf9c83a98575ee8d92427d22ee4a03e66e0a11f8a23674b9c0cc248a42b2e1aa4bc422a5f3715523b3769d330628fb61e0a5215c94a4c31408e968c700fc859e2f2230e3c6db387a703fa90b7e05e2c4136848c0372c9c73a424db9a2a497c7b22e565b0317ac085a848122b219dabbb706bb13e35762d26f9b59a475309a50787e0cac18982931f7759b7ca20cdf0246350640f4a9afa2857425519a0c86508db0a87b526c281c0818dc00c7e7c346821ec449759cf602e0bbace8e1a5f493196c1c265c338ca346430ff6796fb58f50e6670db7929ec38c4433bc2d0189dcb873223a56e2f3276065244067c394dc2a433a44e3b329bc28b4e366c9158861cb191ca7343b679c4782c2162831854a170a39f07995aa2502a9a6abbbbab28bc7aed1995136048d81099497aaf3a051659065db5194193064328a56e3f90b39cccd5e511c70d191cbf6b690635db56674fb52b5849b6c655cc789326c900282b1c87a916039af087daba8171af6a2bfe15182192a3e2741b49c9a97427211bc45ec890cbb6221d0637eec0a3e47598a490c8dd8a11102f1ae2425029c072b878615703064b63b7b4e614a47a36080e88dcbdc15408c26e21464766a5740dc96d042ab81a470c85133ae1b9bede688872a742e94aef3cb03081653a40379962b5f5ec34617535ecc471eadac4b205719831158658b00ab8911d8352157e0866625944b690bc5b71101ec244bf4493fb049b5a1c91e3aa200268bdf68831fb347f2614205ab640c8cc59ef7bd3fa7bf45d315db6bc1b1c80d3e692f249788520766033c827674709e8460a3a685333726779a0a76308356367a00228b50f2a6854c6947da1b70d489b6d46fae76352132b024263874c60b147269f78711a9e22a92e05b4b05a6f3507ef723a0716611530459c96a895e1269f9d545edc4a9a665b47cda99d149100c642723601ffd449ac2574a20f71e9b59293f543ab95832955c852f1246758738ddcb76d5db5d8af237345302e4c5884ff10e14c974d7fbc986ec16863ca4c7c0309c92ae6ff9a1d7d55021fc83d1777b500535e74aad34523299068d8d6bba212c2d6ad9a29024a1f192ae96d49cb48835df87367110be3b79710aabccb0055c39cc30fbe3b8b6a68e96c4173616134318ad8bfa64f779c9cd673889c315195180783bb84b15d3877961f7b077e9dc5eda19438ba0", + "ct": "b061c303ccf518a36279d1972c4a5343e63eff500b479471fadc34ce13e6500c6e22a57d56a75a8c3de538cbe8d3c925807af223d9516893cee61cf67cbd010674da54545221c56455174c4f4c9ab094728b179051edb581a4611574e29a8c73b203b8a54c3eb46151dc394f73b71086411a2ac6fa461cc4e7e9d41b84229edd8deb4209c79899b7c668707ebc7ca3b157ea93fc0815a75f17fe167d05b898af756c7ae5268c881d2b38fe7d0b77eb6860a20effe16807194e0e1c190fabf32b97f37e2984b4ef600ee215c6a19fd179d29ddaaa0d65bf03bea367c819fee7ce333c689ee60c9b46dde659c55d5114b70faf031f66a62c2c2ea2302674b324d032efb54980f14459aece5bba882b1a303c0168183e07ee9da0bfe56537fb83ccca73f263f4e2a98fcae96964183e93231e0a00c5293d0e007c57e8cd019cc35625d2d3cac71026d86d11f78bba42e32f301748be147654ad88f261dfd4a48a2e93d7a2d78e37d1fa00557de8d4c5afb8fb2c5e77e641b0f77a931a167ee15189b02dde09842f8b5b1938c7bf0bddddf3d46103f9a0d2bcb04b18786b4ce751b56977c41d9401dc9be25a81c948d6d6da4065478beaa38ebfe8dac127be0d88919dd924f5ece5afd2baae1c2d38002d8916375ccb85fb0bc9c6de9c56d01c95305c6bbfdf5c9f58436b06c499f3968805924342e0a9c7ba1e1f564d633c1aa1a67e6b3727c48d502ab2e1ee8245197f7390a219d5e99cf70d0813a200acbb39ed68680991d7d8c88303ba70d045112dcc68b3f0bea17a390ade438c780ce98418734c59904ce7a5e048d77d1493ef345290a467034cf9c40cfbbc546bca323637d579d4a36ee43ed24387309907b6ec277f0aaf9fc97534328b9f442efeffe398768b82b3d0e579527f7dbbdd2a07b2ef898191020253f007f67aaa54ffb77fffb47246a5308b291d83ff347f41b740c023fffe9b2fcdf3dc2b7009eda3370ededde10613409594540c7458bcab9154fa3edc1aabed7e3f20580749b0f2a680b705023ccde800622561307ea1c3ce925bba2bd4c9397996c33a09daa8a4b9df3079e4d3ddc6abec1f8d7d9f2a0e668c6af584c6153d8b6abb44cf099a0ae51cc4b7f324c8e063b5cfa152d6eb2996ba79473b790427b86c071dc504df51963f42be903f4c97519634b24b0a3c36a39237fe7a30e39cf95902553397aee1708d8f1a5fbe7ade6dc8e45dd3bcd040b427167936b941870eba6206655c57f5bdaa84118d7fb5625e8e5f1ee9396f479d64c6d1bcfd5d0e839784f170ac9e1347c0ca4c893ba419680cf9cf668fd64b817813200136e99a464cda883e736959be84ebde9b8499982eeced11784ec8b2cee9a3942485b93e82ca34f49c97a4cb0cbedef734cd6866b1b9143bb75693f59941604c0bac544830a4edbcaf9f7153e5d330d97231d9021ead5600274a3809a25ebf2ca01b68a83e55bda94872f27000b08fb5ea0e9baf1dde890ac843576bd98cec38aeaf527bef947dc3712e02c8ecbd5315e7c5a0394d908d152997d69d0468b8b10a6b876233143f8fe415297d8dc3e61bff5817af0873e7f56831c084eece18e909801158fab83f35c3fae602ec4d85be2370694b4304c943fb077df9fcea1fd7943572909fb40e452d5c1eb4a24b499a51d976a991987d39975b0a23e33fd8e07c54d88f34dc004a76bdb20243967722c79a32c65ddc25b7a677a3cde595c98710f575025dba871dee9c087a89feee0aa82a05bad08998774525b5ba783286092faf676f59bd3929e44697f16248f306093d246eb558c228a4e909f8640752f804c7270946958a3bcaa2cf8640bbec909dae7eb10e0cce1a80156f200ebfbb0fce05bcc03849f2919ab9fe0f61c24a1b831af0f36c44dac399f9512c00cc9cc28d6f28eeef5a0ab53ec58e7d80a63461ae232e05772f6c06d92176e109fdea0009fab76afb22b648bc54b61012bc020fe104b17297bb25b790592346c507d9298e4c4d90d7304efc4bc42ec679a3a92a72fa4b24f36c1b070f043f35f755285a5c4daff87c853863c912b523fff1a23cd4d0b93d672b82873061f02901063ae79637ab3d1724a860de0e4932cbf2033c52fcf671cc9fc2dc9774e97162d69502166bf2f454a9ba20d0d128624d7bc90cfe40476af4970e28be4dfab73b5ad05d34509ac1e28ecc1c4ad3c0ecb3e21d", + "ss_mlkem": "88d05ce5eedd4bf68930cd8c235e2098a78f77496b623e40af0a45121c1732d9", + "contents_responder": "fd2106f036a9c8146c2f789c479f2acd47c7c64f7594550b059d2a4cb7f3add2970ed600888ad782df574a72608283085a433c0ceb9300f5440141848e8352874b83839659629de75e953a0b25a74e8d449c7eb215de1042ef95b140bac0d35cb21c31204c8c22e09861d0a63c6af3029e489e50b023b6dab3295a6d1a7533f571741a5602d25286cbab504b5724371712aa65a84c47782ffc69f1414c81f47479368e6dab1cb14c1fabe24a9531c2e5e97e54c2625b56204104cd11fb77ee999682d745a832c10120ae13b63a016ab99a46a57f549ea5fa6668d2491a4b8689c96b1f8414b6815a22ba70dd2a459cf64365e72677f3b7a74c33a3a62f2fa345ca3b60b41c7abe1a2e1e6b3f43d68be86cce15eba2470204ecc458cb082196d814d9c22aa58675d3337e64b36198d92c1b17470222cab7696f0d8ca4488966205360dfca5d85d1a90ba80b58688adadc02299716bf086dcf5cbc06414cb1fc58091c5a68281f42940fc6da795bb320fefb88cd60727db44a8fda0110c92529f64f726b1834470b1224a1a5c59845b945a1e1a2316039f4b64df9934b40f89df766c279e75a91450ad0e68276f91a3efac90536b836364d55080056d0622533800126b818685a9ad8b052ec72aed13c80085ac9b07a6186553740c7c8a27f1a593a19719eae0519de338408dc7441724248f2b5803682cbf36debd97b265ab60362b59ab2585cf518252565ab1143ed8a53bf40a03d97022b0100afe597226b9e1d3793c98bb9b0b5b167031cb0bc775c16699e785cd60aabeee250a55b5c0881af22265962855a13654dd7022603078b038745c1aa515188642280b2ce68437b5bc458b463cdab65fac07755c862a8d1cd09ca11fb625c61136e4f4c022d6b5f94f5276098bf709b56a5a57a736422a93898041a42075926de23b90f105f488c1d1cb95e7ba5049ee7b23d4766ef6823d45a1d623cbc9d2a9ea1b43dd0b4b30026b2e0d84533142bf9c83a98575ee8d92427d22ee4a03e66e0a11f8a23674b9c0cc248a42b2e1aa4bc422a5f3715523b3769d330628fb61e0a5215c94a4c31408e968c700fc859e2f2230e3c6db387a703fa90b7e05e2c4136848c0372c9c73a424db9a2a497c7b22e565b0317ac085a848122b219dabbb706bb13e35762d26f9b59a475309a50787e0cac18982931f7759b7ca20cdf0246350640f4a9afa2857425519a0c86508db0a87b526c281c0818dc00c7e7c346821ec449759cf602e0bbace8e1a5f493196c1c265c338ca346430ff6796fb58f50e6670db7929ec38c4433bc2d0189dcb873223a56e2f3276065244067c394dc2a433a44e3b329bc28b4e366c9158861cb191ca7343b679c4782c2162831854a170a39f07995aa2502a9a6abbbbab28bc7aed1995136048d81099497aaf3a051659065db5194193064328a56e3f90b39cccd5e511c70d191cbf6b690635db56674fb52b5849b6c655cc789326c900282b1c87a916039af087daba8171af6a2bfe15182192a3e2741b49c9a97427211bc45ec890cbb6221d0637eec0a3e47598a490c8dd8a11102f1ae2425029c072b878615703064b63b7b4e614a47a36080e88dcbdc15408c26e21464766a5740dc96d042ab81a470c85133ae1b9bede688872a742e94aef3cb03081653a40379962b5f5ec34617535ecc471eadac4b205719831158658b00ab8911d8352157e0866625944b690bc5b71101ec244bf4493fb049b5a1c91e3aa200268bdf68831fb347f2614205ab640c8cc59ef7bd3fa7bf45d315db6bc1b1c80d3e692f249788520766033c827674709e8460a3a685333726779a0a76308356367a00228b50f2a6854c6947da1b70d489b6d46fae76352132b024263874c60b147269f78711a9e22a92e05b4b05a6f3507ef723a0716611530459c96a895e1269f9d545edc4a9a665b47cda99d149100c642723601ffd449ac2574a20f71e9b59293f543ab95832955c852f1246758738ddcb76d5db5d8af237345302e4c5884ff10e14c974d7fbc986ec16863ca4c7c0309c92ae6ff9a1d7d55021fc83d1777b500535e74aad34523299068d8d6bba212c2d6ad9a29024a1f192ae96d49cb48835df87367110be3b79710aabccb0055c39cc30fbe3b8b6a68e96c4173616134318ad8bfa64f779c9cd673889c315195180783bb84b15d3877961f7b077e9dc5eda19438ba0", + "contents_initiator": "fd2106f0b061c303ccf518a36279d1972c4a5343e63eff500b479471fadc34ce13e6500c6e22a57d56a75a8c3de538cbe8d3c925807af223d9516893cee61cf67cbd010674da54545221c56455174c4f4c9ab094728b179051edb581a4611574e29a8c73b203b8a54c3eb46151dc394f73b71086411a2ac6fa461cc4e7e9d41b84229edd8deb4209c79899b7c668707ebc7ca3b157ea93fc0815a75f17fe167d05b898af756c7ae5268c881d2b38fe7d0b77eb6860a20effe16807194e0e1c190fabf32b97f37e2984b4ef600ee215c6a19fd179d29ddaaa0d65bf03bea367c819fee7ce333c689ee60c9b46dde659c55d5114b70faf031f66a62c2c2ea2302674b324d032efb54980f14459aece5bba882b1a303c0168183e07ee9da0bfe56537fb83ccca73f263f4e2a98fcae96964183e93231e0a00c5293d0e007c57e8cd019cc35625d2d3cac71026d86d11f78bba42e32f301748be147654ad88f261dfd4a48a2e93d7a2d78e37d1fa00557de8d4c5afb8fb2c5e77e641b0f77a931a167ee15189b02dde09842f8b5b1938c7bf0bddddf3d46103f9a0d2bcb04b18786b4ce751b56977c41d9401dc9be25a81c948d6d6da4065478beaa38ebfe8dac127be0d88919dd924f5ece5afd2baae1c2d38002d8916375ccb85fb0bc9c6de9c56d01c95305c6bbfdf5c9f58436b06c499f3968805924342e0a9c7ba1e1f564d633c1aa1a67e6b3727c48d502ab2e1ee8245197f7390a219d5e99cf70d0813a200acbb39ed68680991d7d8c88303ba70d045112dcc68b3f0bea17a390ade438c780ce98418734c59904ce7a5e048d77d1493ef345290a467034cf9c40cfbbc546bca323637d579d4a36ee43ed24387309907b6ec277f0aaf9fc97534328b9f442efeffe398768b82b3d0e579527f7dbbdd2a07b2ef898191020253f007f67aaa54ffb77fffb47246a5308b291d83ff347f41b740c023fffe9b2fcdf3dc2b7009eda3370ededde10613409594540c7458bcab9154fa3edc1aabed7e3f20580749b0f2a680b705023ccde800622561307ea1c3ce925bba2bd4c9397996c33a09daa8a4b9df3079e4d3ddc6abec1f8d7d9f2a0e668c6af584c6153d8b6abb44cf099a0ae51cc4b7f324c8e063b5cfa152d6eb2996ba79473b790427b86c071dc504df51963f42be903f4c97519634b24b0a3c36a39237fe7a30e39cf95902553397aee1708d8f1a5fbe7ade6dc8e45dd3bcd040b427167936b941870eba6206655c57f5bdaa84118d7fb5625e8e5f1ee9396f479d64c6d1bcfd5d0e839784f170ac9e1347c0ca4c893ba419680cf9cf668fd64b817813200136e99a464cda883e736959be84ebde9b8499982eeced11784ec8b2cee9a3942485b93e82ca34f49c97a4cb0cbedef734cd6866b1b9143bb75693f59941604c0bac544830a4edbcaf9f7153e5d330d97231d9021ead5600274a3809a25ebf2ca01b68a83e55bda94872f27000b08fb5ea0e9baf1dde890ac843576bd98cec38aeaf527bef947dc3712e02c8ecbd5315e7c5a0394d908d152997d69d0468b8b10a6b876233143f8fe415297d8dc3e61bff5817af0873e7f56831c084eece18e909801158fab83f35c3fae602ec4d85be2370694b4304c943fb077df9fcea1fd7943572909fb40e452d5c1eb4a24b499a51d976a991987d39975b0a23e33fd8e07c54d88f34dc004a76bdb20243967722c79a32c65ddc25b7a677a3cde595c98710f575025dba871dee9c087a89feee0aa82a05bad08998774525b5ba783286092faf676f59bd3929e44697f16248f306093d246eb558c228a4e909f8640752f804c7270946958a3bcaa2cf8640bbec909dae7eb10e0cce1a80156f200ebfbb0fce05bcc03849f2919ab9fe0f61c24a1b831af0f36c44dac399f9512c00cc9cc28d6f28eeef5a0ab53ec58e7d80a63461ae232e05772f6c06d92176e109fdea0009fab76afb22b648bc54b61012bc020fe104b17297bb25b790592346c507d9298e4c4d90d7304efc4bc42ec679a3a92a72fa4b24f36c1b070f043f35f755285a5c4daff87c853863c912b523fff1a23cd4d0b93d672b82873061f02901063ae79637ab3d1724a860de0e4932cbf2033c52fcf671cc9fc2dc9774e97162d69502166bf2f454a9ba20d0d128624d7bc90cfe40476af4970e28be4dfab73b5ad05d34509ac1e28ecc1c4ad3c0ecb3e21d", + "transcript_input": "fd575a36275251cbf6a287fcafac3419d93b941834964ef2c71235474a86325e075cdcd6a60c478c638ad23cf2c3436395e56b1bb02da9d45c785ba20b6ef484cdcdc96b9b01f8dffb480f2cb943e33fcfaa1dd3b20c6be6e25375bdc0f2358fb70615a2103fedde8396327d92966af4a5022bd5537c3816fd21bc036d7dcca3fd2406fd2106f036a9c8146c2f789c479f2acd47c7c64f7594550b059d2a4cb7f3add2970ed600888ad782df574a72608283085a433c0ceb9300f5440141848e8352874b83839659629de75e953a0b25a74e8d449c7eb215de1042ef95b140bac0d35cb21c31204c8c22e09861d0a63c6af3029e489e50b023b6dab3295a6d1a7533f571741a5602d25286cbab504b5724371712aa65a84c47782ffc69f1414c81f47479368e6dab1cb14c1fabe24a9531c2e5e97e54c2625b56204104cd11fb77ee999682d745a832c10120ae13b63a016ab99a46a57f549ea5fa6668d2491a4b8689c96b1f8414b6815a22ba70dd2a459cf64365e72677f3b7a74c33a3a62f2fa345ca3b60b41c7abe1a2e1e6b3f43d68be86cce15eba2470204ecc458cb082196d814d9c22aa58675d3337e64b36198d92c1b17470222cab7696f0d8ca4488966205360dfca5d85d1a90ba80b58688adadc02299716bf086dcf5cbc06414cb1fc58091c5a68281f42940fc6da795bb320fefb88cd60727db44a8fda0110c92529f64f726b1834470b1224a1a5c59845b945a1e1a2316039f4b64df9934b40f89df766c279e75a91450ad0e68276f91a3efac90536b836364d55080056d0622533800126b818685a9ad8b052ec72aed13c80085ac9b07a6186553740c7c8a27f1a593a19719eae0519de338408dc7441724248f2b5803682cbf36debd97b265ab60362b59ab2585cf518252565ab1143ed8a53bf40a03d97022b0100afe597226b9e1d3793c98bb9b0b5b167031cb0bc775c16699e785cd60aabeee250a55b5c0881af22265962855a13654dd7022603078b038745c1aa515188642280b2ce68437b5bc458b463cdab65fac07755c862a8d1cd09ca11fb625c61136e4f4c022d6b5f94f5276098bf709b56a5a57a736422a93898041a42075926de23b90f105f488c1d1cb95e7ba5049ee7b23d4766ef6823d45a1d623cbc9d2a9ea1b43dd0b4b30026b2e0d84533142bf9c83a98575ee8d92427d22ee4a03e66e0a11f8a23674b9c0cc248a42b2e1aa4bc422a5f3715523b3769d330628fb61e0a5215c94a4c31408e968c700fc859e2f2230e3c6db387a703fa90b7e05e2c4136848c0372c9c73a424db9a2a497c7b22e565b0317ac085a848122b219dabbb706bb13e35762d26f9b59a475309a50787e0cac18982931f7759b7ca20cdf0246350640f4a9afa2857425519a0c86508db0a87b526c281c0818dc00c7e7c346821ec449759cf602e0bbace8e1a5f493196c1c265c338ca346430ff6796fb58f50e6670db7929ec38c4433bc2d0189dcb873223a56e2f3276065244067c394dc2a433a44e3b329bc28b4e366c9158861cb191ca7343b679c4782c2162831854a170a39f07995aa2502a9a6abbbbab28bc7aed1995136048d81099497aaf3a051659065db5194193064328a56e3f90b39cccd5e511c70d191cbf6b690635db56674fb52b5849b6c655cc789326c900282b1c87a916039af087daba8171af6a2bfe15182192a3e2741b49c9a97427211bc45ec890cbb6221d0637eec0a3e47598a490c8dd8a11102f1ae2425029c072b878615703064b63b7b4e614a47a36080e88dcbdc15408c26e21464766a5740dc96d042ab81a470c85133ae1b9bede688872a742e94aef3cb03081653a40379962b5f5ec34617535ecc471eadac4b205719831158658b00ab8911d8352157e0866625944b690bc5b71101ec244bf4493fb049b5a1c91e3aa200268bdf68831fb347f2614205ab640c8cc59ef7bd3fa7bf45d315db6bc1b1c80d3e692f249788520766033c827674709e8460a3a685333726779a0a76308356367a00228b50f2a6854c6947da1b70d489b6d46fae76352132b024263874c60b147269f78711a9e22a92e05b4b05a6f3507ef723a0716611530459c96a895e1269f9d545edc4a9a665b47cda99d149100c642723601ffd449ac2574a20f71e9b59293f543ab95832955c852f1246758738ddcb76d5db5d8af237345302e4c5884ff10e14c974d7fbc986ec16863ca4c7c0309c92ae6ff9a1d7d55021fc83d1777b500535e74aad34523299068d8d6bba212c2d6ad9a29024a1f192ae96d49cb48835df87367110be3b79710aabccb0055c39cc30fbe3b8b6a68e96c4173616134318ad8bfa64f779c9cd673889c315195180783bb84b15d3877961f7b077e9dc5eda19438ba0fd2406fd2106f0b061c303ccf518a36279d1972c4a5343e63eff500b479471fadc34ce13e6500c6e22a57d56a75a8c3de538cbe8d3c925807af223d9516893cee61cf67cbd010674da54545221c56455174c4f4c9ab094728b179051edb581a4611574e29a8c73b203b8a54c3eb46151dc394f73b71086411a2ac6fa461cc4e7e9d41b84229edd8deb4209c79899b7c668707ebc7ca3b157ea93fc0815a75f17fe167d05b898af756c7ae5268c881d2b38fe7d0b77eb6860a20effe16807194e0e1c190fabf32b97f37e2984b4ef600ee215c6a19fd179d29ddaaa0d65bf03bea367c819fee7ce333c689ee60c9b46dde659c55d5114b70faf031f66a62c2c2ea2302674b324d032efb54980f14459aece5bba882b1a303c0168183e07ee9da0bfe56537fb83ccca73f263f4e2a98fcae96964183e93231e0a00c5293d0e007c57e8cd019cc35625d2d3cac71026d86d11f78bba42e32f301748be147654ad88f261dfd4a48a2e93d7a2d78e37d1fa00557de8d4c5afb8fb2c5e77e641b0f77a931a167ee15189b02dde09842f8b5b1938c7bf0bddddf3d46103f9a0d2bcb04b18786b4ce751b56977c41d9401dc9be25a81c948d6d6da4065478beaa38ebfe8dac127be0d88919dd924f5ece5afd2baae1c2d38002d8916375ccb85fb0bc9c6de9c56d01c95305c6bbfdf5c9f58436b06c499f3968805924342e0a9c7ba1e1f564d633c1aa1a67e6b3727c48d502ab2e1ee8245197f7390a219d5e99cf70d0813a200acbb39ed68680991d7d8c88303ba70d045112dcc68b3f0bea17a390ade438c780ce98418734c59904ce7a5e048d77d1493ef345290a467034cf9c40cfbbc546bca323637d579d4a36ee43ed24387309907b6ec277f0aaf9fc97534328b9f442efeffe398768b82b3d0e579527f7dbbdd2a07b2ef898191020253f007f67aaa54ffb77fffb47246a5308b291d83ff347f41b740c023fffe9b2fcdf3dc2b7009eda3370ededde10613409594540c7458bcab9154fa3edc1aabed7e3f20580749b0f2a680b705023ccde800622561307ea1c3ce925bba2bd4c9397996c33a09daa8a4b9df3079e4d3ddc6abec1f8d7d9f2a0e668c6af584c6153d8b6abb44cf099a0ae51cc4b7f324c8e063b5cfa152d6eb2996ba79473b790427b86c071dc504df51963f42be903f4c97519634b24b0a3c36a39237fe7a30e39cf95902553397aee1708d8f1a5fbe7ade6dc8e45dd3bcd040b427167936b941870eba6206655c57f5bdaa84118d7fb5625e8e5f1ee9396f479d64c6d1bcfd5d0e839784f170ac9e1347c0ca4c893ba419680cf9cf668fd64b817813200136e99a464cda883e736959be84ebde9b8499982eeced11784ec8b2cee9a3942485b93e82ca34f49c97a4cb0cbedef734cd6866b1b9143bb75693f59941604c0bac544830a4edbcaf9f7153e5d330d97231d9021ead5600274a3809a25ebf2ca01b68a83e55bda94872f27000b08fb5ea0e9baf1dde890ac843576bd98cec38aeaf527bef947dc3712e02c8ecbd5315e7c5a0394d908d152997d69d0468b8b10a6b876233143f8fe415297d8dc3e61bff5817af0873e7f56831c084eece18e909801158fab83f35c3fae602ec4d85be2370694b4304c943fb077df9fcea1fd7943572909fb40e452d5c1eb4a24b499a51d976a991987d39975b0a23e33fd8e07c54d88f34dc004a76bdb20243967722c79a32c65ddc25b7a677a3cde595c98710f575025dba871dee9c087a89feee0aa82a05bad08998774525b5ba783286092faf676f59bd3929e44697f16248f306093d246eb558c228a4e909f8640752f804c7270946958a3bcaa2cf8640bbec909dae7eb10e0cce1a80156f200ebfbb0fce05bcc03849f2919ab9fe0f61c24a1b831af0f36c44dac399f9512c00cc9cc28d6f28eeef5a0ab53ec58e7d80a63461ae232e05772f6c06d92176e109fdea0009fab76afb22b648bc54b61012bc020fe104b17297bb25b790592346c507d9298e4c4d90d7304efc4bc42ec679a3a92a72fa4b24f36c1b070f043f35f755285a5c4daff87c853863c912b523fff1a23cd4d0b93d672b82873061f02901063ae79637ab3d1724a860de0e4932cbf2033c52fcf671cc9fc2dc9774e97162d69502166bf2f454a9ba20d0d128624d7bc90cfe40476af4970e28be4dfab73b5ad05d34509ac1e28ecc1c4ad3c0ecb3e21d", + "transcript_hash": "b5daca7d9f565561eb65188fe847b42035584b7d5a10a37b03780093d4759002", + "salt": "716269745f76325f6879627269645f6d6c6b656d31303234a66b1fda", + "ikm": "7cc5f298684b388f81e651af731d3d6231f0d0c4c01ce8aae3695f7c24d0d8d388d05ce5eedd4bf68930cd8c235e2098a78f77496b623e40af0a45121c1732d9b5daca7d9f565561eb65188fe847b42035584b7d5a10a37b03780093d4759002", + "prk": "4f41efbac786e859440a53e08165ef9c65c8994578090ed105a5a21b732d8e02", + "initiator_L": "b52abaf0643e95f6604bb5b96cd74bb2082396f60e77db1129ebbf087a6acf25", + "initiator_P": "ca616100d18947e39791dbd10291ee65633ca83d2c18313553d0def027a2fd15", + "responder_L": "08276142743e4d7060120d65f4b191385dc9557862c984dc9e5d26fd45aca1d3", + "responder_P": "fa2b20ce6b6184abba4452a743b4bcd1c131a7d772306c0cc55241a09e37eb7d", + "session_id": "a4d00d3bbf7b2827f5174b6ae15392d50cf7c10301d9176b8a112dbc8ea3030c", + "initiator_confirmation": "18842cc0feb7d46095f2b5248c365780ed704865", + "responder_confirmation": "be555e4bc6f9008845aab5051714e5b762f7f343", + "initiator_first_contents": "120123456789abcdef", + "initiator_first_packet": "849e3283bc0b5f18b87f6a0d43aa9a84e3f67f67f1e514ba6f8829bc4b", + "responder_first_contents": "130123456789abcdef", + "responder_first_packet": "dafaef5b2d7f8349646d1b0c75da82a2b1850b1802184e814a9c4c8193" + }, + { + "comment": "Mainnet, extra records. The responder's contents are its offer, a second 0xf0 record with a different valid ek, and an unknown 0xf1 record; the initiator encapsulates to the ek of the first 0xf0 record and ignores the rest. The initiator's contents are an unknown 0x01 record followed by its accept, which is still the first 0xf0 record there. The transcript covers every record of both contents.", + "chain": "main", + "magic": "444f24a8", + "initiator_privkey": "7a8fce38025c8d5e293a3fa1700aeb0377cd4f24deddfb9c942b8dcf43d482e9", + "initiator_ellswift": "0c80fda2ca53954371093ebf6c8a05459a8a7b191144bb9b2b45cfcf9325f65e12d1fc1ac3eb53bb888e25deaa0d76ca2ee327264a1b6c0c33f214bde668ff6d", + "responder_privkey": "72f7f006e2be303b761305ebee497574abad3ea7b1542ea5b6d7e7ebb390dd5f", + "responder_ellswift": "e9455f0cabdb2176c839614c5b5b438eb0233819ff63197572b15b5657fc3664fae78cf87f8a1012873d9b7c30e4d11fedef245c7fe3a50cdcfc1a52a0fb5ecc", + "ss_ecdh": "5c673cb7a5b0f773e77a240a65a089b914d26e08092a2eb87d5f6dc82039d79a", + "ecdh_session_id": "b5de8be0eee3939ad30ad9ad402515b289297d9eef37b58327842f0bd94796b8", + "initiator_garbage_terminator": "e548e658f884c0717202aca407756c5b", + "responder_garbage_terminator": "9b050c8ac06961ec6cdb826dadcb4f70", + "mlkem_keygen_seed": "b04358d9549f8211c95e3e95d6f8d967d3f6319039028314fd7c43e68177f234166d646ef8bf960a873ffd68585ff6439fe8d9fd996ae9025fc9bce73beafcf5", + "mlkem_encaps_m": "94b194e2b14bdb50be8c305fc471b35865217c8d97aecf8848088398b8afa05d", + "ek": "391892c89cc4fbd7cf2b9b8eb563a681446d51ac93006d179dc79417544f12432319ebc23d77385b047582e922ecdc6dd1359166fc6353f1cd13404723f8a35f202d570a71baf435f6c14bb0d748ae8809e290107f222eb3a12ac180999ba1afaaf642dbc9294a60ac6ecc02fc8a825d541cd5e723bdc9ce84fa97847009317c0c5ab348acd921eafb8730729329714b710c091b383249465a124534b3db4d451a77ede1b60cc308cd9578a2d22050324e0aa724efc14c24555c8f6a4258a998e6ba0230c22bdc267d5ca9780d603f91712533d96ea2899b9989688ba97597245c72412f73c83a471883061c627283c19e355a7ad80d29565401955987a2af908b9f1fa265b4a12f1ca34aa86027d79876a8c3b0766a5064637684a40497e962d8e5acee309b37215a458227ae09654023c5317a91955147e6880d8187053b40872ed8c8aac0a7d690794751940a379a0617a9c61c4d2ba5c4b36a7c9ad8cc1e5acca6db3b7f5b4a394ba87a03cbe036787a37a39dfbbb86d183d050054e83a61a3b9e5a17ba11c16862e35c6b5705f16143bd856996ec159a039f0600071434928151c2b7a4ced4b6a54237b17a32cf15116e2d03aaff7b1e07b0bcfa2c844f0932d15085739a7040937a8a62783518c6e63007c0ba10b280944db447296369aa495a84e3471fbaad4f50473240102509ce45726e26f95d0256035ce21a39c1430ca1884b5519bcf6833bfa65c3aa89677b5d18164a6230a34a26c204ba6bb3d162f42135e567c490b58d1dd1b5f808bcd8e029e734bcab762a49f8b015cc63dbe9b4dfb8446b31452a1a428fa0a5c7e499a62a620ac915f3d85ebc39ac944b2977c07898987b7c2771f9136e6d2b0aa42a6c391c8456e17aa6d6b1e5036ed3fb5f00475ac4808de28ba686c027b38862db4149c84a86395c05824947fa3876a867be2f10666a5c683b8b190ed841b473a22bb35949f74a9580ceb370284127c445c759d7ccc5e66a4bb950b4a54071f496c570f5a641991c705a9b3bf2131ab6254a120c3a5259687c256c7a0ba23ac234eab05f34423907b9d16cb76371c3233c74327729540a1a8bfaa786f324c4c446c2b5307534937e98804da8598c95a434e7b99ca2144bf753d3a2564685cd3fdb1574b472276b933b579a1f88a197c7ad6f1c02b3b7c6c8250855e15585f253b9174cf9700e5a762cc129cce48b4cb39588c74854e4c7735d085295735533a9674fb889c2e5c225f7206f383fd429c3b0a050c95950579b806f3c7324c022d253b254054236eb054200888ca93e5b2aa5df5aafd6840fbe780dd926624b0a22bebb443b9a033f33189fd5532d0750adf3c1a346164355210eab12876c1304da9898c2c8f5a475f766af2a092af3e31bd8e7407dc8a9faf096fda24ccb230337caa5e4a8aa0d497a50f59a08ab12437223506b92a0e9bd84f80865e81b3ffb671074cdf0d03e1537bc50a87b9433af26b10f01349c56373112dc0ccdc24b8d15b33237993732b9ff5b8a089b88e54102b7336f4b8ba6c5c6c6bca19a691592f65b57b4f47c8e2889e8d8c72bd29856d83a8687a46d6ccef7e9be6af75aa9367c97fbaac7251cd0cc444d81115824beddfb26c3a430788781c1dca72d96c4bf77a8b02812c792c3c472272fd25048d3230b494b6058326ec66d3a90a23747b30a7328cb8b8c4e95b6183920d40653681293d3f7c094e76d8cb98b481585c610ae34e5226f9abd37938875737400d0178bbc743c5c6727778d227010ab88bcac680d11c581fd215b33d2c2ddd3631de5a1848b76c9514780698727e7b53108c4d122aefd9a55174047beac7e06297fbd383218424e95978b7833389843c321476cd555c4e10561f7540769bc1c7bb444cb7b8c500813eef5a7f4d89ac0f315fcc7bd3b55c350c74685e81e30919ab32c5f33a8cea2518850768b99d311f310a19cf954b80b4c897928155705f4200d70a5039443cd2f484ebe936329e4b5c7540b47f4663f31c6dc535aeb0a360eb02a6d0890db24b31732acf59b7303405d798aa5527c274dc774349124b310194c16cac635a803523df22b35e4f66101807c61ec5aac6bc60e955568119b08c4163d3c4218750d4ca0c0a677b864e3b4beea934a745df18bb8dab289b561414b9494b25d116502a23d2bce210cd4b9cb2ffaf261d18ca277589c727be84555102b19", + "ct": "f9002f5e1e408d9aa90199fca791fc86e7eedd51fa56fb897954f9a30610855b4f5c737fbebb93618c3e6664fd400ed471e3243a63c8d287ef7f435719e540f24c789f4790340678ee90468f3e42586e98c5a42ff9f3fd8c255af6441a16ce2581ee8fd5970acf91e7a7497ba144990c2c8a84f37b5e65a238ffae62740f9e8c8fc309d43156a27dadebfdbe04a23553b3ea9779e86eac2b1cd1b1281bd486ce3e597d0aef96d3aae22660a3951a303afb7335978718e38387d68cc6f7eef0f8fb0b9b7a37df6a46a7eb27b1bf7dec788fc4e88ab252a573fde9dec0b693fbb00e6647db52d994252473861b218479179c1ab32e82e81ad71eaec92a2dafc2efb84ff11ce686fec244581d17cc0b91a9db6b6fe6c49e5919337d324981d023227e028600285ef795a449c69895e7dd96ef925365623a862d03ebd86ffa3fff2acb6b4165a73e030259ef0b94d0abc36a7a87ba2f12565c850af548e170d4927bce6de8b2e888ac7ee47266ba230629da59d6573ccb0651c0dcd5bb9926262a70ef9633a914bd57c8aaf4219d0cbe8713748070e93e33f5ac5aa937d9553e58451cc3a8699c940295bb3867e654a51887b0e82fc35eec05958503a256a19e97520e4a6d04a194c6c4d2e6d5abdad5c91d6aed8e9f5f9ef963ff92e334ef04b86b77b6e75a135173755508978831a831df77298cbb8ec4c99c00eb2696b473777532f3ea911ba03dd99e086cbb7a64c5a1c38154300fbe11e311599a2b6e1a14df63ec7cf2462b835c64b89cf6249aab2835865d78b2c16859acd0bbbd9481315d1a21d22b3cbb4f57cc32bbbb08bf6a7a7032899927ff1903c7aef6658b703b0663f4780c4c2927fbd279d32c05514e9c00d37754ceb4c98501de4334c6a35fd19159bfc571f42d1ca02343aa690da3a8f4a98874dfb9d02fb474b5ea399c820bec6334e186448236c7f809616c2df4ba1863f34f36770fa88b1f2d0dcb267f675ad9e990c5f6cd4e95a4f1fa5085a414ec200d218a34c399ca5b900a3b36084270019f1edddd97037951d60bf3c1524e5e17db214659f70188132e33f19c697fd708aa04d2262eb09c59040c0192795c53bae668442566e237bc23f0c8eec5b569cdee9522a698a312e8e5ddc788a39eead2bb8719b298f25c6045cf0c2cce2928c729c54c8d8925c8e043c566fd6b2dc9e725fd47b743d571e4299434849d4c7789d7f56ff01b3aef1dced7f601cfc427df3ebcf704aa722164cf2d3ce6491456ce0bc5a4279bec79e48e8222808ac0d51fa0e51e25de4a69942fc215b5a87aca441de5b4395863cddb03dc9c3ac6bc833ad08eae1e2438be5b5c4cce66bd7c1fcc59d7b927494ed4569fe6f5db9569edf8b65a5fd4bae073c699ee6c50200d5b5824542a40873a18d5b881ca86ccdb1c9ba5387095e7ffb03da4b3ddb5d2a0e5395a21ac4288105a2385fe1477eceac41e7f5d520e854f9511d62a13839395f6c37390ed8ca46b92ef285222b1addfa8a3a119b0e37366bcbc782693fb6e079ba4ded9a75a29016ff6af04ac1fc04bf6173e58cc922f7c7ac6060dea4b4169a420c7e80d8c034d118de811cf600102b235e1ef3fbb33dd9d8fe9d2a83c55192ae8c3a3476cd94bd6beae4e3f281b8d3477dc68d508db659b9472c8564f1877bf1a319e88cc304c8f0c0708a15787f75082ba35c5aa9b9bb01b1fe8f55a259429c0484f233b8c445112ebfbb3c77ad8b977a97497675a28e810df5e4cb905c7e840fcfa126a2da3d246626e73df022ba75563b696335ea8da1035d22634c1c029e00d808a5b1e4e663475a907c5f18e4c3ffc711ccca38030cf4f2dab4bcfb0a69d6ab835ab7f514f40e623508a4e4b9b98cc177000d002304b84978dff9db4ac1fdea8376839e339ce5daeb2a8918a06c540a6c43082aa3cb99473eeb6067ca8e7dbe31d60e7c37569da75e690fb79a55dea00ce872cba268eca78be5a9b367855a280097c1d9b345ad69040cfbec6fcfdc137c276d23aab7e8c81ccc658677454f394b1913f22d27c1f7f0be0c4bbd2a355a719de059e17e1efe3ff2b304d27779fe3780c30e8dc4c8352586db8de9a0d83fb69b0ab90241733ecedc94c3ddcb3c65012e9971b547d35bc7f47101049ddf77c278e373b184fcf0befbba38b628f3da097fab9ac37819a9c0a019093f8e398e7df873fc9c497f971f2763d9f", + "ss_mlkem": "c1608e04a8b9a552945aefe36cef26784e1b20d96516f9c5db47b8a9847062d3", + "contents_responder": "fd2106f0391892c89cc4fbd7cf2b9b8eb563a681446d51ac93006d179dc79417544f12432319ebc23d77385b047582e922ecdc6dd1359166fc6353f1cd13404723f8a35f202d570a71baf435f6c14bb0d748ae8809e290107f222eb3a12ac180999ba1afaaf642dbc9294a60ac6ecc02fc8a825d541cd5e723bdc9ce84fa97847009317c0c5ab348acd921eafb8730729329714b710c091b383249465a124534b3db4d451a77ede1b60cc308cd9578a2d22050324e0aa724efc14c24555c8f6a4258a998e6ba0230c22bdc267d5ca9780d603f91712533d96ea2899b9989688ba97597245c72412f73c83a471883061c627283c19e355a7ad80d29565401955987a2af908b9f1fa265b4a12f1ca34aa86027d79876a8c3b0766a5064637684a40497e962d8e5acee309b37215a458227ae09654023c5317a91955147e6880d8187053b40872ed8c8aac0a7d690794751940a379a0617a9c61c4d2ba5c4b36a7c9ad8cc1e5acca6db3b7f5b4a394ba87a03cbe036787a37a39dfbbb86d183d050054e83a61a3b9e5a17ba11c16862e35c6b5705f16143bd856996ec159a039f0600071434928151c2b7a4ced4b6a54237b17a32cf15116e2d03aaff7b1e07b0bcfa2c844f0932d15085739a7040937a8a62783518c6e63007c0ba10b280944db447296369aa495a84e3471fbaad4f50473240102509ce45726e26f95d0256035ce21a39c1430ca1884b5519bcf6833bfa65c3aa89677b5d18164a6230a34a26c204ba6bb3d162f42135e567c490b58d1dd1b5f808bcd8e029e734bcab762a49f8b015cc63dbe9b4dfb8446b31452a1a428fa0a5c7e499a62a620ac915f3d85ebc39ac944b2977c07898987b7c2771f9136e6d2b0aa42a6c391c8456e17aa6d6b1e5036ed3fb5f00475ac4808de28ba686c027b38862db4149c84a86395c05824947fa3876a867be2f10666a5c683b8b190ed841b473a22bb35949f74a9580ceb370284127c445c759d7ccc5e66a4bb950b4a54071f496c570f5a641991c705a9b3bf2131ab6254a120c3a5259687c256c7a0ba23ac234eab05f34423907b9d16cb76371c3233c74327729540a1a8bfaa786f324c4c446c2b5307534937e98804da8598c95a434e7b99ca2144bf753d3a2564685cd3fdb1574b472276b933b579a1f88a197c7ad6f1c02b3b7c6c8250855e15585f253b9174cf9700e5a762cc129cce48b4cb39588c74854e4c7735d085295735533a9674fb889c2e5c225f7206f383fd429c3b0a050c95950579b806f3c7324c022d253b254054236eb054200888ca93e5b2aa5df5aafd6840fbe780dd926624b0a22bebb443b9a033f33189fd5532d0750adf3c1a346164355210eab12876c1304da9898c2c8f5a475f766af2a092af3e31bd8e7407dc8a9faf096fda24ccb230337caa5e4a8aa0d497a50f59a08ab12437223506b92a0e9bd84f80865e81b3ffb671074cdf0d03e1537bc50a87b9433af26b10f01349c56373112dc0ccdc24b8d15b33237993732b9ff5b8a089b88e54102b7336f4b8ba6c5c6c6bca19a691592f65b57b4f47c8e2889e8d8c72bd29856d83a8687a46d6ccef7e9be6af75aa9367c97fbaac7251cd0cc444d81115824beddfb26c3a430788781c1dca72d96c4bf77a8b02812c792c3c472272fd25048d3230b494b6058326ec66d3a90a23747b30a7328cb8b8c4e95b6183920d40653681293d3f7c094e76d8cb98b481585c610ae34e5226f9abd37938875737400d0178bbc743c5c6727778d227010ab88bcac680d11c581fd215b33d2c2ddd3631de5a1848b76c9514780698727e7b53108c4d122aefd9a55174047beac7e06297fbd383218424e95978b7833389843c321476cd555c4e10561f7540769bc1c7bb444cb7b8c500813eef5a7f4d89ac0f315fcc7bd3b55c350c74685e81e30919ab32c5f33a8cea2518850768b99d311f310a19cf954b80b4c897928155705f4200d70a5039443cd2f484ebe936329e4b5c7540b47f4663f31c6dc535aeb0a360eb02a6d0890db24b31732acf59b7303405d798aa5527c274dc774349124b310194c16cac635a803523df22b35e4f66101807c61ec5aac6bc60e955568119b08c4163d3c4218750d4ca0c0a677b864e3b4beea934a745df18bb8dab289b561414b9494b25d116502a23d2bce210cd4b9cb2ffaf261d18ca277589c727be84555102b19fd2106f02e0151b6e29076483f5e416de08aafc9b5bbdf05c906c94a468bc219536b1d81c3ff6201220bb3efd564d77c5a1577c46762a0ea8c9feb0501f37628f078577bb9bcef2ac773d079bfa34c365c275bd936b5cc62ac7b2b5faba3a4250c1cbbc1d83443b790af42ba620a205877da585a55bf27f83853c8cf6f23a203039f0a519f1df2754fa0363f82cb10b39a71487ba10925460b8709239af8c060c492c1c374aa5ee06b5dd484e13a7f677c4758d639e4f5b94038acb8837ca4cb33f1a52a2d69821f676f6640532cc8aad7d2891416303609c1b2a3c35f532502211249791f9802902a729c87bc9bc3ba4741c8adadaab8353b92d839b65aea9aa37c5e04ec7842d92f32e627b6b259baa7192b840769e28dcf3780c6356fe5f35e1b0b7c85dc93e6ea0109f205fbd0b35955049217987200be27fb6347db7e15a4b0decbc2fb521357344061c65d0684650ca788501c12d2a644f6db7f82a1ce15f50b5c1cb3e7573b804aa2792113b1c6093b3992dac72bc470040e0c6427092fbd761881899e23547b4d9717b235c94b5827c5da3499fc49b20bbed1c755f535489d776e15e596c7d207dde4cef26261b5001673da163013b1d31c3e72419568010ed67b6d3f737dff3161e0687286ab1cf9c9ccd08a80a0708e87816391a90a17983c503045893b70f728b1b267502c1a5ef1c42309561efae324c43658197417f33196f25c1a52e4482e5605cf6426f2056b983b634a300126097589e89026f37306e0ab8b01c633296ed9db4e75dc330b631be7531f31e891dd3cadf2661d26ab13bb8581ceec3002e2a339d432473358cdc8752d2863d8fb976aa2a215e57dbe1670cde00c915572c098681051cea8c43c969c1f358c5fa0ea437e1751c9c5cf83d3a665b736802abc73d1a73c610f18e3baecf697fec172d23b28ee941869e2a3da0a98b0c4ce15f1a9b1331d4ab4045bf56fb9519eee602b5f12175915b0731cc152c6110b7771758715176993924aa13114b7a6d921f82c668f91784a9c5761530522c06c6eb873a8795c33c860332304d8ac16520790c30c4a7080516de898ba2b19764a0eb304198ef864670103b55b63b3b364cca943c4a38e43c18e0b373f7d9c8dc6ba4ea3c6489461af6644c2cf0c9d1d4095eff5c62c1a4b0217c66eda16eb15bb556160d6311640311520b3651e92678d75b0d3e596554ba6c10037e95c8ab2b457bd0bb4b484c33606c3a1980638483a5fd0a3ff9c9d3dfb742ed08fa2a704deb092e9c79d84378b60735b07a309584b5289d49cfc242543761c9f586a4ad4a99ab2968c4262899a5487aaa9056061e33c1325f29f8c395f12608e466c0d99c58321cc3e110953afbacf81240b9c5917dbe5794654839b1777b8eca888111b84fb7578172043830774316cdbba8159997931a82ad3fa23a81997b0c461c817920d44afd6aa7841599cbedc01aee34166649fdf4c4d28045147409585a07564167c685b26aee168bb59ad29e82e00f1abe0589440341242a20af77884af53a38ecb028f728c9bea9ac7849527079e0530305edbc02131aab7666f31b5a0df1647678b0b79b36d3b5c53ad0c8d8ca90e5da5a1b6726712b9be65fb185c34baff42544d0175e5e2cd8fc249343737c346c330a660b243197cd9c8e1448192164bf8f6b8018b871d0373b2b3518ec6652c596c182804c3053fa4d75afbe052e8285ab5e6c0aa742311b7201976a390e7004cc015cdb34c91229536a6c18150cccb90a02f32048647541eca49b0e498f9f1324a2832bcf41c9fe08a09754e7552c21df20ce9a610dcd0ca48a6ce949456a01b11a718229f32b699291defcc42b3c7100e226c637b2ad2a855dde8831d89b7c3139a3454386a813b1e14aa31088a436735b029b75d127ae1e917c8ca5f4b36c8f001ae897475d3e71b7e447571969fec2668877c7402ea8f4ee11d9a47bddcf04dedea15eb735125b74e79235e95f047cc9379eeb6ae0503458807783fa1254b949449ea2f8df39b3a5c1651f06a5cb528fec135e3cbc695a6cb32306e4026bf3ac326d117c1b7aa1e8f99ab40519f0c368513668f89104df510367b54415d1c257a621efb257d4d807a21937a4474867c2615f1e17fb6f87dbde557bcfa33dc751d6c39611d9329dcd593b444cd8f431e052b23e8428c9f393129eb403ee2025d01245c97d673a7c51a8703f10001", + "contents_initiator": "050171626974fd2106f0f9002f5e1e408d9aa90199fca791fc86e7eedd51fa56fb897954f9a30610855b4f5c737fbebb93618c3e6664fd400ed471e3243a63c8d287ef7f435719e540f24c789f4790340678ee90468f3e42586e98c5a42ff9f3fd8c255af6441a16ce2581ee8fd5970acf91e7a7497ba144990c2c8a84f37b5e65a238ffae62740f9e8c8fc309d43156a27dadebfdbe04a23553b3ea9779e86eac2b1cd1b1281bd486ce3e597d0aef96d3aae22660a3951a303afb7335978718e38387d68cc6f7eef0f8fb0b9b7a37df6a46a7eb27b1bf7dec788fc4e88ab252a573fde9dec0b693fbb00e6647db52d994252473861b218479179c1ab32e82e81ad71eaec92a2dafc2efb84ff11ce686fec244581d17cc0b91a9db6b6fe6c49e5919337d324981d023227e028600285ef795a449c69895e7dd96ef925365623a862d03ebd86ffa3fff2acb6b4165a73e030259ef0b94d0abc36a7a87ba2f12565c850af548e170d4927bce6de8b2e888ac7ee47266ba230629da59d6573ccb0651c0dcd5bb9926262a70ef9633a914bd57c8aaf4219d0cbe8713748070e93e33f5ac5aa937d9553e58451cc3a8699c940295bb3867e654a51887b0e82fc35eec05958503a256a19e97520e4a6d04a194c6c4d2e6d5abdad5c91d6aed8e9f5f9ef963ff92e334ef04b86b77b6e75a135173755508978831a831df77298cbb8ec4c99c00eb2696b473777532f3ea911ba03dd99e086cbb7a64c5a1c38154300fbe11e311599a2b6e1a14df63ec7cf2462b835c64b89cf6249aab2835865d78b2c16859acd0bbbd9481315d1a21d22b3cbb4f57cc32bbbb08bf6a7a7032899927ff1903c7aef6658b703b0663f4780c4c2927fbd279d32c05514e9c00d37754ceb4c98501de4334c6a35fd19159bfc571f42d1ca02343aa690da3a8f4a98874dfb9d02fb474b5ea399c820bec6334e186448236c7f809616c2df4ba1863f34f36770fa88b1f2d0dcb267f675ad9e990c5f6cd4e95a4f1fa5085a414ec200d218a34c399ca5b900a3b36084270019f1edddd97037951d60bf3c1524e5e17db214659f70188132e33f19c697fd708aa04d2262eb09c59040c0192795c53bae668442566e237bc23f0c8eec5b569cdee9522a698a312e8e5ddc788a39eead2bb8719b298f25c6045cf0c2cce2928c729c54c8d8925c8e043c566fd6b2dc9e725fd47b743d571e4299434849d4c7789d7f56ff01b3aef1dced7f601cfc427df3ebcf704aa722164cf2d3ce6491456ce0bc5a4279bec79e48e8222808ac0d51fa0e51e25de4a69942fc215b5a87aca441de5b4395863cddb03dc9c3ac6bc833ad08eae1e2438be5b5c4cce66bd7c1fcc59d7b927494ed4569fe6f5db9569edf8b65a5fd4bae073c699ee6c50200d5b5824542a40873a18d5b881ca86ccdb1c9ba5387095e7ffb03da4b3ddb5d2a0e5395a21ac4288105a2385fe1477eceac41e7f5d520e854f9511d62a13839395f6c37390ed8ca46b92ef285222b1addfa8a3a119b0e37366bcbc782693fb6e079ba4ded9a75a29016ff6af04ac1fc04bf6173e58cc922f7c7ac6060dea4b4169a420c7e80d8c034d118de811cf600102b235e1ef3fbb33dd9d8fe9d2a83c55192ae8c3a3476cd94bd6beae4e3f281b8d3477dc68d508db659b9472c8564f1877bf1a319e88cc304c8f0c0708a15787f75082ba35c5aa9b9bb01b1fe8f55a259429c0484f233b8c445112ebfbb3c77ad8b977a97497675a28e810df5e4cb905c7e840fcfa126a2da3d246626e73df022ba75563b696335ea8da1035d22634c1c029e00d808a5b1e4e663475a907c5f18e4c3ffc711ccca38030cf4f2dab4bcfb0a69d6ab835ab7f514f40e623508a4e4b9b98cc177000d002304b84978dff9db4ac1fdea8376839e339ce5daeb2a8918a06c540a6c43082aa3cb99473eeb6067ca8e7dbe31d60e7c37569da75e690fb79a55dea00ce872cba268eca78be5a9b367855a280097c1d9b345ad69040cfbec6fcfdc137c276d23aab7e8c81ccc658677454f394b1913f22d27c1f7f0be0c4bbd2a355a719de059e17e1efe3ff2b304d27779fe3780c30e8dc4c8352586db8de9a0d83fb69b0ab90241733ecedc94c3ddcb3c65012e9971b547d35bc7f47101049ddf77c278e373b184fcf0befbba38b628f3da097fab9ac37819a9c0a019093f8e398e7df873fc9c497f971f2763d9f", + "transcript_input": "0c80fda2ca53954371093ebf6c8a05459a8a7b191144bb9b2b45cfcf9325f65e12d1fc1ac3eb53bb888e25deaa0d76ca2ee327264a1b6c0c33f214bde668ff6de9455f0cabdb2176c839614c5b5b438eb0233819ff63197572b15b5657fc3664fae78cf87f8a1012873d9b7c30e4d11fedef245c7fe3a50cdcfc1a52a0fb5eccfd4c0cfd2106f0391892c89cc4fbd7cf2b9b8eb563a681446d51ac93006d179dc79417544f12432319ebc23d77385b047582e922ecdc6dd1359166fc6353f1cd13404723f8a35f202d570a71baf435f6c14bb0d748ae8809e290107f222eb3a12ac180999ba1afaaf642dbc9294a60ac6ecc02fc8a825d541cd5e723bdc9ce84fa97847009317c0c5ab348acd921eafb8730729329714b710c091b383249465a124534b3db4d451a77ede1b60cc308cd9578a2d22050324e0aa724efc14c24555c8f6a4258a998e6ba0230c22bdc267d5ca9780d603f91712533d96ea2899b9989688ba97597245c72412f73c83a471883061c627283c19e355a7ad80d29565401955987a2af908b9f1fa265b4a12f1ca34aa86027d79876a8c3b0766a5064637684a40497e962d8e5acee309b37215a458227ae09654023c5317a91955147e6880d8187053b40872ed8c8aac0a7d690794751940a379a0617a9c61c4d2ba5c4b36a7c9ad8cc1e5acca6db3b7f5b4a394ba87a03cbe036787a37a39dfbbb86d183d050054e83a61a3b9e5a17ba11c16862e35c6b5705f16143bd856996ec159a039f0600071434928151c2b7a4ced4b6a54237b17a32cf15116e2d03aaff7b1e07b0bcfa2c844f0932d15085739a7040937a8a62783518c6e63007c0ba10b280944db447296369aa495a84e3471fbaad4f50473240102509ce45726e26f95d0256035ce21a39c1430ca1884b5519bcf6833bfa65c3aa89677b5d18164a6230a34a26c204ba6bb3d162f42135e567c490b58d1dd1b5f808bcd8e029e734bcab762a49f8b015cc63dbe9b4dfb8446b31452a1a428fa0a5c7e499a62a620ac915f3d85ebc39ac944b2977c07898987b7c2771f9136e6d2b0aa42a6c391c8456e17aa6d6b1e5036ed3fb5f00475ac4808de28ba686c027b38862db4149c84a86395c05824947fa3876a867be2f10666a5c683b8b190ed841b473a22bb35949f74a9580ceb370284127c445c759d7ccc5e66a4bb950b4a54071f496c570f5a641991c705a9b3bf2131ab6254a120c3a5259687c256c7a0ba23ac234eab05f34423907b9d16cb76371c3233c74327729540a1a8bfaa786f324c4c446c2b5307534937e98804da8598c95a434e7b99ca2144bf753d3a2564685cd3fdb1574b472276b933b579a1f88a197c7ad6f1c02b3b7c6c8250855e15585f253b9174cf9700e5a762cc129cce48b4cb39588c74854e4c7735d085295735533a9674fb889c2e5c225f7206f383fd429c3b0a050c95950579b806f3c7324c022d253b254054236eb054200888ca93e5b2aa5df5aafd6840fbe780dd926624b0a22bebb443b9a033f33189fd5532d0750adf3c1a346164355210eab12876c1304da9898c2c8f5a475f766af2a092af3e31bd8e7407dc8a9faf096fda24ccb230337caa5e4a8aa0d497a50f59a08ab12437223506b92a0e9bd84f80865e81b3ffb671074cdf0d03e1537bc50a87b9433af26b10f01349c56373112dc0ccdc24b8d15b33237993732b9ff5b8a089b88e54102b7336f4b8ba6c5c6c6bca19a691592f65b57b4f47c8e2889e8d8c72bd29856d83a8687a46d6ccef7e9be6af75aa9367c97fbaac7251cd0cc444d81115824beddfb26c3a430788781c1dca72d96c4bf77a8b02812c792c3c472272fd25048d3230b494b6058326ec66d3a90a23747b30a7328cb8b8c4e95b6183920d40653681293d3f7c094e76d8cb98b481585c610ae34e5226f9abd37938875737400d0178bbc743c5c6727778d227010ab88bcac680d11c581fd215b33d2c2ddd3631de5a1848b76c9514780698727e7b53108c4d122aefd9a55174047beac7e06297fbd383218424e95978b7833389843c321476cd555c4e10561f7540769bc1c7bb444cb7b8c500813eef5a7f4d89ac0f315fcc7bd3b55c350c74685e81e30919ab32c5f33a8cea2518850768b99d311f310a19cf954b80b4c897928155705f4200d70a5039443cd2f484ebe936329e4b5c7540b47f4663f31c6dc535aeb0a360eb02a6d0890db24b31732acf59b7303405d798aa5527c274dc774349124b310194c16cac635a803523df22b35e4f66101807c61ec5aac6bc60e955568119b08c4163d3c4218750d4ca0c0a677b864e3b4beea934a745df18bb8dab289b561414b9494b25d116502a23d2bce210cd4b9cb2ffaf261d18ca277589c727be84555102b19fd2106f02e0151b6e29076483f5e416de08aafc9b5bbdf05c906c94a468bc219536b1d81c3ff6201220bb3efd564d77c5a1577c46762a0ea8c9feb0501f37628f078577bb9bcef2ac773d079bfa34c365c275bd936b5cc62ac7b2b5faba3a4250c1cbbc1d83443b790af42ba620a205877da585a55bf27f83853c8cf6f23a203039f0a519f1df2754fa0363f82cb10b39a71487ba10925460b8709239af8c060c492c1c374aa5ee06b5dd484e13a7f677c4758d639e4f5b94038acb8837ca4cb33f1a52a2d69821f676f6640532cc8aad7d2891416303609c1b2a3c35f532502211249791f9802902a729c87bc9bc3ba4741c8adadaab8353b92d839b65aea9aa37c5e04ec7842d92f32e627b6b259baa7192b840769e28dcf3780c6356fe5f35e1b0b7c85dc93e6ea0109f205fbd0b35955049217987200be27fb6347db7e15a4b0decbc2fb521357344061c65d0684650ca788501c12d2a644f6db7f82a1ce15f50b5c1cb3e7573b804aa2792113b1c6093b3992dac72bc470040e0c6427092fbd761881899e23547b4d9717b235c94b5827c5da3499fc49b20bbed1c755f535489d776e15e596c7d207dde4cef26261b5001673da163013b1d31c3e72419568010ed67b6d3f737dff3161e0687286ab1cf9c9ccd08a80a0708e87816391a90a17983c503045893b70f728b1b267502c1a5ef1c42309561efae324c43658197417f33196f25c1a52e4482e5605cf6426f2056b983b634a300126097589e89026f37306e0ab8b01c633296ed9db4e75dc330b631be7531f31e891dd3cadf2661d26ab13bb8581ceec3002e2a339d432473358cdc8752d2863d8fb976aa2a215e57dbe1670cde00c915572c098681051cea8c43c969c1f358c5fa0ea437e1751c9c5cf83d3a665b736802abc73d1a73c610f18e3baecf697fec172d23b28ee941869e2a3da0a98b0c4ce15f1a9b1331d4ab4045bf56fb9519eee602b5f12175915b0731cc152c6110b7771758715176993924aa13114b7a6d921f82c668f91784a9c5761530522c06c6eb873a8795c33c860332304d8ac16520790c30c4a7080516de898ba2b19764a0eb304198ef864670103b55b63b3b364cca943c4a38e43c18e0b373f7d9c8dc6ba4ea3c6489461af6644c2cf0c9d1d4095eff5c62c1a4b0217c66eda16eb15bb556160d6311640311520b3651e92678d75b0d3e596554ba6c10037e95c8ab2b457bd0bb4b484c33606c3a1980638483a5fd0a3ff9c9d3dfb742ed08fa2a704deb092e9c79d84378b60735b07a309584b5289d49cfc242543761c9f586a4ad4a99ab2968c4262899a5487aaa9056061e33c1325f29f8c395f12608e466c0d99c58321cc3e110953afbacf81240b9c5917dbe5794654839b1777b8eca888111b84fb7578172043830774316cdbba8159997931a82ad3fa23a81997b0c461c817920d44afd6aa7841599cbedc01aee34166649fdf4c4d28045147409585a07564167c685b26aee168bb59ad29e82e00f1abe0589440341242a20af77884af53a38ecb028f728c9bea9ac7849527079e0530305edbc02131aab7666f31b5a0df1647678b0b79b36d3b5c53ad0c8d8ca90e5da5a1b6726712b9be65fb185c34baff42544d0175e5e2cd8fc249343737c346c330a660b243197cd9c8e1448192164bf8f6b8018b871d0373b2b3518ec6652c596c182804c3053fa4d75afbe052e8285ab5e6c0aa742311b7201976a390e7004cc015cdb34c91229536a6c18150cccb90a02f32048647541eca49b0e498f9f1324a2832bcf41c9fe08a09754e7552c21df20ce9a610dcd0ca48a6ce949456a01b11a718229f32b699291defcc42b3c7100e226c637b2ad2a855dde8831d89b7c3139a3454386a813b1e14aa31088a436735b029b75d127ae1e917c8ca5f4b36c8f001ae897475d3e71b7e447571969fec2668877c7402ea8f4ee11d9a47bddcf04dedea15eb735125b74e79235e95f047cc9379eeb6ae0503458807783fa1254b949449ea2f8df39b3a5c1651f06a5cb528fec135e3cbc695a6cb32306e4026bf3ac326d117c1b7aa1e8f99ab40519f0c368513668f89104df510367b54415d1c257a621efb257d4d807a21937a4474867c2615f1e17fb6f87dbde557bcfa33dc751d6c39611d9329dcd593b444cd8f431e052b23e8428c9f393129eb403ee2025d01245c97d673a7c51a8703f10001fd2a06050171626974fd2106f0f9002f5e1e408d9aa90199fca791fc86e7eedd51fa56fb897954f9a30610855b4f5c737fbebb93618c3e6664fd400ed471e3243a63c8d287ef7f435719e540f24c789f4790340678ee90468f3e42586e98c5a42ff9f3fd8c255af6441a16ce2581ee8fd5970acf91e7a7497ba144990c2c8a84f37b5e65a238ffae62740f9e8c8fc309d43156a27dadebfdbe04a23553b3ea9779e86eac2b1cd1b1281bd486ce3e597d0aef96d3aae22660a3951a303afb7335978718e38387d68cc6f7eef0f8fb0b9b7a37df6a46a7eb27b1bf7dec788fc4e88ab252a573fde9dec0b693fbb00e6647db52d994252473861b218479179c1ab32e82e81ad71eaec92a2dafc2efb84ff11ce686fec244581d17cc0b91a9db6b6fe6c49e5919337d324981d023227e028600285ef795a449c69895e7dd96ef925365623a862d03ebd86ffa3fff2acb6b4165a73e030259ef0b94d0abc36a7a87ba2f12565c850af548e170d4927bce6de8b2e888ac7ee47266ba230629da59d6573ccb0651c0dcd5bb9926262a70ef9633a914bd57c8aaf4219d0cbe8713748070e93e33f5ac5aa937d9553e58451cc3a8699c940295bb3867e654a51887b0e82fc35eec05958503a256a19e97520e4a6d04a194c6c4d2e6d5abdad5c91d6aed8e9f5f9ef963ff92e334ef04b86b77b6e75a135173755508978831a831df77298cbb8ec4c99c00eb2696b473777532f3ea911ba03dd99e086cbb7a64c5a1c38154300fbe11e311599a2b6e1a14df63ec7cf2462b835c64b89cf6249aab2835865d78b2c16859acd0bbbd9481315d1a21d22b3cbb4f57cc32bbbb08bf6a7a7032899927ff1903c7aef6658b703b0663f4780c4c2927fbd279d32c05514e9c00d37754ceb4c98501de4334c6a35fd19159bfc571f42d1ca02343aa690da3a8f4a98874dfb9d02fb474b5ea399c820bec6334e186448236c7f809616c2df4ba1863f34f36770fa88b1f2d0dcb267f675ad9e990c5f6cd4e95a4f1fa5085a414ec200d218a34c399ca5b900a3b36084270019f1edddd97037951d60bf3c1524e5e17db214659f70188132e33f19c697fd708aa04d2262eb09c59040c0192795c53bae668442566e237bc23f0c8eec5b569cdee9522a698a312e8e5ddc788a39eead2bb8719b298f25c6045cf0c2cce2928c729c54c8d8925c8e043c566fd6b2dc9e725fd47b743d571e4299434849d4c7789d7f56ff01b3aef1dced7f601cfc427df3ebcf704aa722164cf2d3ce6491456ce0bc5a4279bec79e48e8222808ac0d51fa0e51e25de4a69942fc215b5a87aca441de5b4395863cddb03dc9c3ac6bc833ad08eae1e2438be5b5c4cce66bd7c1fcc59d7b927494ed4569fe6f5db9569edf8b65a5fd4bae073c699ee6c50200d5b5824542a40873a18d5b881ca86ccdb1c9ba5387095e7ffb03da4b3ddb5d2a0e5395a21ac4288105a2385fe1477eceac41e7f5d520e854f9511d62a13839395f6c37390ed8ca46b92ef285222b1addfa8a3a119b0e37366bcbc782693fb6e079ba4ded9a75a29016ff6af04ac1fc04bf6173e58cc922f7c7ac6060dea4b4169a420c7e80d8c034d118de811cf600102b235e1ef3fbb33dd9d8fe9d2a83c55192ae8c3a3476cd94bd6beae4e3f281b8d3477dc68d508db659b9472c8564f1877bf1a319e88cc304c8f0c0708a15787f75082ba35c5aa9b9bb01b1fe8f55a259429c0484f233b8c445112ebfbb3c77ad8b977a97497675a28e810df5e4cb905c7e840fcfa126a2da3d246626e73df022ba75563b696335ea8da1035d22634c1c029e00d808a5b1e4e663475a907c5f18e4c3ffc711ccca38030cf4f2dab4bcfb0a69d6ab835ab7f514f40e623508a4e4b9b98cc177000d002304b84978dff9db4ac1fdea8376839e339ce5daeb2a8918a06c540a6c43082aa3cb99473eeb6067ca8e7dbe31d60e7c37569da75e690fb79a55dea00ce872cba268eca78be5a9b367855a280097c1d9b345ad69040cfbec6fcfdc137c276d23aab7e8c81ccc658677454f394b1913f22d27c1f7f0be0c4bbd2a355a719de059e17e1efe3ff2b304d27779fe3780c30e8dc4c8352586db8de9a0d83fb69b0ab90241733ecedc94c3ddcb3c65012e9971b547d35bc7f47101049ddf77c278e373b184fcf0befbba38b628f3da097fab9ac37819a9c0a019093f8e398e7df873fc9c497f971f2763d9f", + "transcript_hash": "5d1a7fb607f5d333c32e7c392994ec07f7c9dad1452331c65baaf3411fff1c96", + "salt": "716269745f76325f6879627269645f6d6c6b656d31303234444f24a8", + "ikm": "5c673cb7a5b0f773e77a240a65a089b914d26e08092a2eb87d5f6dc82039d79ac1608e04a8b9a552945aefe36cef26784e1b20d96516f9c5db47b8a9847062d35d1a7fb607f5d333c32e7c392994ec07f7c9dad1452331c65baaf3411fff1c96", + "prk": "fa8b0e64aba8e1924e3c5d05a7e1a711f9515b54c35468d0ccfd5b75177eb605", + "initiator_L": "b0ad68d1bf92b8bb7b157917f4e649e53b397c5c26292dbe2ba8bf32f09e5d47", + "initiator_P": "9b73c730d2eb2bb958b3034466419e0d9cafc875cafe2b05d99608ff790b967f", + "responder_L": "635da4c1f2ec229cda0e468a26220643a4cd863a0260537390fd59bad00b725f", + "responder_P": "edefe23ec9ccf6293555169e7454dbca231adc295f503197e7e483a158d35394", + "session_id": "7bad230ffe2b9a7e25ba58653ab6367ecc0b66854ea28e17874653ffce5a9bbc", + "initiator_confirmation": "a27114b9c378240c62f7c79420f9631e7858bb77", + "responder_confirmation": "7851573884e1f1f5bfd00def86bd4fb3b866d0a9", + "initiator_first_contents": "120123456789abcdef", + "initiator_first_packet": "c00884e7caa1e7a4a7247257bc392f10e0bf8d3aa4033acc50eec55125", + "responder_first_contents": "130123456789abcdef", + "responder_first_packet": "94af7850890ac6b2c548297f7254d529851380ff39f54e8f6fb6300e6f" + }, + { + "comment": "Mainnet; the responder's contents are its offer followed by an unknown 0xf1 record with a 65,535-byte zero payload, so that record's length (65,536) and the responder's contents length in the transcript (67,113) both use the 0xfe CompactSize form.", + "chain": "main", + "magic": "444f24a8", + "initiator_privkey": "f58415cb1b0141b0be1f891c12fd7184900ad751e4fcbb8b6436ff2a984a858e", + "initiator_ellswift": "00867888931969393e24c82320dce065753ed7cb7d4d429564beda218d92bb1054439ce4f81947b99c7c4ba8b561973fb630f5e90b17c64f1010014e0ee05274", + "responder_privkey": "9b97db6e8dcf81f31fa28d52106e27e6da08e96191137acc3db09222e0a88156", + "responder_ellswift": "22cc1b75b5e7aa62a229cb2ce96c8f9db05015f88ae40ca56161c6d05a7a482ed9ef2868a50f66e055705409098a11b58b933e7c65347239df5958c64576a4de", + "ss_ecdh": "bf3f6c4be548a2afbf33f9b33c2d0b52b3f133e87ca0dc636281ccc833ceab8f", + "ecdh_session_id": "b53d9d41bd2c8036d429d3ba1c046b6efc1ac1543904a246e4e410ea3dec7eb3", + "initiator_garbage_terminator": "8b2db9f6d682634c80d6ad8ece5f175e", + "responder_garbage_terminator": "b5ab29e9ac027058fabe3dbbc9f48b77", + "mlkem_keygen_seed": "cc4b6414cef160474eec6b225c5dc019f240b2ed6753f7f04d80cc5b587ecaf3666c614a9f7d6c8d56a5b7ca074ce15306f170164f0208888731e623824b7ec3", + "mlkem_encaps_m": "8b28bd873ef412198de56f5fdac093e39f9b01dbe2a8392811993df620a3a8cb", + "ek": "d9f1814882bf6a3513b9c69b993535a6935a7903132fb67b85da8ad3c26db6f46c84fb078b348d52e75cc6421bdef50220c10b81b132c5c92944376f5132382bf0781c18a28b132985241a6a358536e6a45df18709ecacc7121116fabb287c638e815c9e7bb9f46b0fdc972be0aca13f0974b517202aa47b23c64c8a719ecc901df9f4bc309a3615e092a82b21477a7f900b5c0857694fdb25b44b0fde954a9f75658665904183bdab66c455b951f46a57dfdac732e6cf03048cefb813b4f71795fbba857013b7d6ac61f9875b801aebf298c8b0633db092ea4b3829a93870c167d5081bb7aaa745350ded309a46e1325b87afc346009781b81626547eab3b98c6664c19b2438bb8862485ab57836f1908782bc755ea94e5375a4b80139270c4bb332a6f5c2eaec03b890008504b2af26505b104b72ae742f430132440cca2abc65963bb9709770659623d8239710a5beac68f498a0a2a091ef0f78e19cc0e42222674d8b4fba57dde3301be234f527aa31c53be2be18649b17a047caff4e57a0c93ba8a3823d441c3f1901808c19b1871afec811a9d583d6f1516640c09dfd16e25ab35efb82a419bcbfdd225d3b57760f12b8d56a2fdac0f52b05526f86ce9c73bd1dcadeaaa6056b36fc70495c66a9cecda24f6365521246bd534a77f3c898c382397200fb8530546ec240814504e0971b9b990768434932781c7035942237f4af92aaf881d04fc3d90960ed0a2c278e385155a01712c33590582a6250953e931a9f03f0493272a09716923c8bb8b41ddd762306c85e3a12d66ccbe5f271f943659bc6288f67b5599a2ac4e7cc621460cd8aa5d57013c916854589a0bcb076646896d09f677a855818ae89a3655b327b4b9a63ac0429290d5d4762ef95c5d0969276a10b19930a3f69e849abaf0d97f78893ff40a598253300e54c6b6576e36079c278b5d697401c77049a31a44eee245440b397e2398b9208a19283b8e817d679c2ee4295411f7be4c6bc2b7329420d939c30400645c7baf8cc757fa84a815c49ec7bb9ea19d9dcc0ef9d3562db79f0f810ddd14690d51276df631388a51e54116d0d247e88b140b658e7f7203523c96a05053ced944ed0890fdfa3d23d46c9d41c5b5d34bbc1b862d2b19c6138e4a46866e313244fb6fc3706ba5ac692e6610fc800b2ae57eba9499f0295b83eaae1bf03e5ba75626688f5f7b6b25c430cec60c89f445653042e6746ac3f173d18a30b2a022555001767051c948cc9392b95a5234d5c1c33629648be9238de63d1a15a2182149c89b84727961ab046ad6d7639e9032bca94e480169b5f22b438418a3c926bd944de38750b0f1a802b9a92c6c2119553a88751bb48b65f451710bd7167a31b13ad81cd36631df339b292309bef1a719080cc72abe9917465757bcb5018a84f6ad3b601c018a3b5cab962cc7b49321bb0517a5bc3a2889272b52b10d629b1316bc17dd2b323a7468f8e52c2a43b8b736bbe116327522a2975aa5bfb8b5da976c12f3beafb36210b273a967024f6b030b98139ca49b484ac2937a50374670eafc1fe4078d9d36874fa5b462aa5e6b32b4d66a948b86bcbeb706563042034227fe521149a4b45fcc86959c09f8a1a333712347102dcfcb3158280e24d0af4251c6a5925583a03cb4445b3b595932bacf43579ae6d797d56c5261567739fa8c9ce3178b3501be2121f9b249ab35bdc26008321b36d5365f6e9530156267493b8fd60c04ed386c96ec58345990b540c59179035593a454d63d43a97888d2ade3c4b4704b5b14940585b23ae8b022a8e17ae2f886bf834bf3190936790f1b856d92588476f140c9b32fa906c0cfa21daba5498197beecd17e768475ec296b675c7bbcb9b5920900d94381305552b5e58bbe203fc7b849322310eb5c477d66097e913a532c5dd79c123ba21508fccb5ab8a81c4459bb74b3d9c46143d0616ac0c196609d22a8903c5639d17aa04cd0228a7680d3101d00a044e66b4bfbc148fca3a773d92eafc9489d6051ff591f931cca910585e2299bd5f6af8b72a813f98642a28f9324c90b3403001329304ca0e15068574c617b69006e1b5d21d8cad3168669331e7d3445fd5a8cd232a4de426c87a090304c0dc16680353ac7bab44cbfa1c916884f5e917e18993fe4c778f5193cec47e5af8d3ae1b706f4b31bdf47234de201b4583d57532cec513dc3ed", + "ct": "14d9b16a1a42a50c6faf6efc3aaab564575de5e587d0c06dd158b93ed1a795a32ee291c56a4ba254d805f7546691514206735a80ea73591bc3a20040f5979a689f083ba1cb1f2ba204b25a94ce83bf48854e05203de1137582b4e482165b4b462a63ec53d5dc2291dc61da93de28691edf28fe72ffbda9361c4b73c555b1165dae61557500f32c5247ab886e0482a2fffe0d1d48abb3f732089df546ecf7b5a53e4803fc39aea2d2b5ebe31bd1c4e90db7de6d239a025f359dd7ea8533ecb131084ed0d35461b2a14622af9eb437a327f36a842e3178c1f32ad13b95e314f2758e0cb1059ab74ba366b6f3a87d79645c7aee5822c9b75ea47ce6654ac61637a608913571093c282aa7e77ce2612a073e4cf40ceba3958b592d9e32236a0110362256230368f9560596be51003596013ed48891f57989eebb7b009a0eac1dd2556af27c3305d571ed6d252157dc8a17924b2e6a80febe0ff3a29c24be8b2ca8a56505f7c55901b8a602488375b1b34d63b1ba167199dfb51c56bc025b2be55cdc9b136702d06eb71946cd4e710c51cd907c59243ababe45861a4bd7a6d097e6e8ecf5a8d60ee5bf5570bda3e1ecb873bbc119695268f2203d08e6f786a7d7af68207a9833a37bd4c65c079bf21ebcf22ea028c01547378a11a258ebdb1279014fd8d527923b5cf93c374d5c990d18bfd5eba7c5b14bfa445ad69667d1c38ca3be167657d42aa621908e9cba43a316e3c000f5c2d8802433210e3bf3c6cb7df77097e88ae0c8d23cf8a73582fa351d60b77037b9fbadf683a12fc30473493f20f8262ca2383f000a97b7adc14174167678ed0b3b8559fd528aefc320964156591cade0c9dac72b36c32f9a47147e6285a52f491a4b6c6150c9e16b63b77eae68ccf7add5502593075fa20aaa16f88d498e55d56c69ab61b83b7aa900c09a9f6d497e5711553ad9962abf0cb0f4f99830cdd6a9a9ae6f54d9e7dd8bbd533acc4a6233bbd49d6a2a2159dcd59c9613122ce4684ed49357125141a8c473aa85d31d3624c8bd0d44989f61ef9f0101490cca8be88ef630ab2f52906fca2b2981c9cfdf4a82db342d5534d33313311044f7a8034f604418d9694760a3fb8ab98b12fa0e83af9c2ce43f9dfe9b63dff9cf9c209f97a1c526660e7a39b4082966a53825b67ddc22afbb4e69bb90138bfc834c2b2c6380f5666db64d153519e33296ebbe577f6b98a13c3d54688754f9ace99168cd7bee489055fb6e50b4ac9104896d8867749d47964bb597fa3b29a4457a431879b96f633f59a23e663dc8820ad97ac85e6cc722d3bb93c780c4a7b909b1a2c9a5745cefd55bde2383d1e9ed1104923814b6b5b71bb3be9d1215c49f0120fd7e04db1d8b0921f209ccbd404f0d9bb69d754717536e402fd19e261ed162f0d98bd1f0f95e3ee720b21e9554bab58445c15d8bc99f6afcaa63ed40641feca1ea820a8de86b5e7f3da3fcadfa7d111242695f10180db0c72607a342b63c5917ca4d7aea3e8ccdb511ff487eaa01bad8c0ce0f6ac3060946b109095edc01094fa943cf5271097c7589efbad0c8ab661b8f21c0880567f160f6ddada20c109dbd2ff0fd9564638a0593bc3d03149c5de51053f73fb1717789f51ff2cadfa525536667c725a649b6bfaeb6b8859257792dbf0400cdb16bfd55c5f15fa6a33bd2fe5667b61c703706ead079dfba5b9db7069de1ecba6ee577273af84bbba8af7d494a2645e0999d46fa305c534a21b3b8493e13ec2a33f608e7748d87214d6a06a1f7a286287a3b0d9b726e2ad20a44062726ffdfa15ab839770ac3466237d234149fa8007ec336d77fdc73d357e0c696f94deae9d83e8836e5cff457ad98e11e173ec929e5daee3beea54e71f927dbd77b115a136be17a5596d1c2ff43a35aa3e941654264a59ff4b26aae2bd790f6d66b4d0570648ca6b00a93c6918a9d01af9e115bd1d2de88035538f2477b1a723da647c77ea9b36ee7795737d779bfa9caeaf2dbaff42b7e22d3d3a04677888074db0fea004eccb48ccb5df257248d10b940683c132bd540c107cbcfcc9c4a28c080cf4b213938ee30078f6b048709a3dbd85eec4e317be3b43cedbf1af4e259f39d4373b6c6608f23866c7462d80f70a305332b6aefb6e07dd688edf4248e3b7a95d36c0be7025397c8229b3e33c361cebf215470154dba18acbc7d72d4e1c651409b01e9", + "ss_mlkem": "50124a2dd20a655c9d431594979a253093dc976e424ed1076e09cced7ef06941", + "contents_responder": "fd2106f0d9f1814882bf6a3513b9c69b993535a6935a7903132fb67b85da8ad3c26db6f46c84fb078b348d52e75cc6421bdef50220c10b81b132c5c92944376f5132382bf0781c18a28b132985241a6a358536e6a45df18709ecacc7121116fabb287c638e815c9e7bb9f46b0fdc972be0aca13f0974b517202aa47b23c64c8a719ecc901df9f4bc309a3615e092a82b21477a7f900b5c0857694fdb25b44b0fde954a9f75658665904183bdab66c455b951f46a57dfdac732e6cf03048cefb813b4f71795fbba857013b7d6ac61f9875b801aebf298c8b0633db092ea4b3829a93870c167d5081bb7aaa745350ded309a46e1325b87afc346009781b81626547eab3b98c6664c19b2438bb8862485ab57836f1908782bc755ea94e5375a4b80139270c4bb332a6f5c2eaec03b890008504b2af26505b104b72ae742f430132440cca2abc65963bb9709770659623d8239710a5beac68f498a0a2a091ef0f78e19cc0e42222674d8b4fba57dde3301be234f527aa31c53be2be18649b17a047caff4e57a0c93ba8a3823d441c3f1901808c19b1871afec811a9d583d6f1516640c09dfd16e25ab35efb82a419bcbfdd225d3b57760f12b8d56a2fdac0f52b05526f86ce9c73bd1dcadeaaa6056b36fc70495c66a9cecda24f6365521246bd534a77f3c898c382397200fb8530546ec240814504e0971b9b990768434932781c7035942237f4af92aaf881d04fc3d90960ed0a2c278e385155a01712c33590582a6250953e931a9f03f0493272a09716923c8bb8b41ddd762306c85e3a12d66ccbe5f271f943659bc6288f67b5599a2ac4e7cc621460cd8aa5d57013c916854589a0bcb076646896d09f677a855818ae89a3655b327b4b9a63ac0429290d5d4762ef95c5d0969276a10b19930a3f69e849abaf0d97f78893ff40a598253300e54c6b6576e36079c278b5d697401c77049a31a44eee245440b397e2398b9208a19283b8e817d679c2ee4295411f7be4c6bc2b7329420d939c30400645c7baf8cc757fa84a815c49ec7bb9ea19d9dcc0ef9d3562db79f0f810ddd14690d51276df631388a51e54116d0d247e88b140b658e7f7203523c96a05053ced944ed0890fdfa3d23d46c9d41c5b5d34bbc1b862d2b19c6138e4a46866e313244fb6fc3706ba5ac692e6610fc800b2ae57eba9499f0295b83eaae1bf03e5ba75626688f5f7b6b25c430cec60c89f445653042e6746ac3f173d18a30b2a022555001767051c948cc9392b95a5234d5c1c33629648be9238de63d1a15a2182149c89b84727961ab046ad6d7639e9032bca94e480169b5f22b438418a3c926bd944de38750b0f1a802b9a92c6c2119553a88751bb48b65f451710bd7167a31b13ad81cd36631df339b292309bef1a719080cc72abe9917465757bcb5018a84f6ad3b601c018a3b5cab962cc7b49321bb0517a5bc3a2889272b52b10d629b1316bc17dd2b323a7468f8e52c2a43b8b736bbe116327522a2975aa5bfb8b5da976c12f3beafb36210b273a967024f6b030b98139ca49b484ac2937a50374670eafc1fe4078d9d36874fa5b462aa5e6b32b4d66a948b86bcbeb706563042034227fe521149a4b45fcc86959c09f8a1a333712347102dcfcb3158280e24d0af4251c6a5925583a03cb4445b3b595932bacf43579ae6d797d56c5261567739fa8c9ce3178b3501be2121f9b249ab35bdc26008321b36d5365f6e9530156267493b8fd60c04ed386c96ec58345990b540c59179035593a454d63d43a97888d2ade3c4b4704b5b14940585b23ae8b022a8e17ae2f886bf834bf3190936790f1b856d92588476f140c9b32fa906c0cfa21daba5498197beecd17e768475ec296b675c7bbcb9b5920900d94381305552b5e58bbe203fc7b849322310eb5c477d66097e913a532c5dd79c123ba21508fccb5ab8a81c4459bb74b3d9c46143d0616ac0c196609d22a8903c5639d17aa04cd0228a7680d3101d00a044e66b4bfbc148fca3a773d92eafc9489d6051ff591f931cca910585e2299bd5f6af8b72a813f98642a28f9324c90b3403001329304ca0e15068574c617b69006e1b5d21d8cad3168669331e7d3445fd5a8cd232a4de426c87a090304c0dc16680353ac7bab44cbfa1c916884f5e917e18993fe4c778f5193cec47e5af8d3ae1b706f4b31bdf47234de201b4583d57532cec513dc3edfe00000100f1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "contents_initiator": "fd2106f014d9b16a1a42a50c6faf6efc3aaab564575de5e587d0c06dd158b93ed1a795a32ee291c56a4ba254d805f7546691514206735a80ea73591bc3a20040f5979a689f083ba1cb1f2ba204b25a94ce83bf48854e05203de1137582b4e482165b4b462a63ec53d5dc2291dc61da93de28691edf28fe72ffbda9361c4b73c555b1165dae61557500f32c5247ab886e0482a2fffe0d1d48abb3f732089df546ecf7b5a53e4803fc39aea2d2b5ebe31bd1c4e90db7de6d239a025f359dd7ea8533ecb131084ed0d35461b2a14622af9eb437a327f36a842e3178c1f32ad13b95e314f2758e0cb1059ab74ba366b6f3a87d79645c7aee5822c9b75ea47ce6654ac61637a608913571093c282aa7e77ce2612a073e4cf40ceba3958b592d9e32236a0110362256230368f9560596be51003596013ed48891f57989eebb7b009a0eac1dd2556af27c3305d571ed6d252157dc8a17924b2e6a80febe0ff3a29c24be8b2ca8a56505f7c55901b8a602488375b1b34d63b1ba167199dfb51c56bc025b2be55cdc9b136702d06eb71946cd4e710c51cd907c59243ababe45861a4bd7a6d097e6e8ecf5a8d60ee5bf5570bda3e1ecb873bbc119695268f2203d08e6f786a7d7af68207a9833a37bd4c65c079bf21ebcf22ea028c01547378a11a258ebdb1279014fd8d527923b5cf93c374d5c990d18bfd5eba7c5b14bfa445ad69667d1c38ca3be167657d42aa621908e9cba43a316e3c000f5c2d8802433210e3bf3c6cb7df77097e88ae0c8d23cf8a73582fa351d60b77037b9fbadf683a12fc30473493f20f8262ca2383f000a97b7adc14174167678ed0b3b8559fd528aefc320964156591cade0c9dac72b36c32f9a47147e6285a52f491a4b6c6150c9e16b63b77eae68ccf7add5502593075fa20aaa16f88d498e55d56c69ab61b83b7aa900c09a9f6d497e5711553ad9962abf0cb0f4f99830cdd6a9a9ae6f54d9e7dd8bbd533acc4a6233bbd49d6a2a2159dcd59c9613122ce4684ed49357125141a8c473aa85d31d3624c8bd0d44989f61ef9f0101490cca8be88ef630ab2f52906fca2b2981c9cfdf4a82db342d5534d33313311044f7a8034f604418d9694760a3fb8ab98b12fa0e83af9c2ce43f9dfe9b63dff9cf9c209f97a1c526660e7a39b4082966a53825b67ddc22afbb4e69bb90138bfc834c2b2c6380f5666db64d153519e33296ebbe577f6b98a13c3d54688754f9ace99168cd7bee489055fb6e50b4ac9104896d8867749d47964bb597fa3b29a4457a431879b96f633f59a23e663dc8820ad97ac85e6cc722d3bb93c780c4a7b909b1a2c9a5745cefd55bde2383d1e9ed1104923814b6b5b71bb3be9d1215c49f0120fd7e04db1d8b0921f209ccbd404f0d9bb69d754717536e402fd19e261ed162f0d98bd1f0f95e3ee720b21e9554bab58445c15d8bc99f6afcaa63ed40641feca1ea820a8de86b5e7f3da3fcadfa7d111242695f10180db0c72607a342b63c5917ca4d7aea3e8ccdb511ff487eaa01bad8c0ce0f6ac3060946b109095edc01094fa943cf5271097c7589efbad0c8ab661b8f21c0880567f160f6ddada20c109dbd2ff0fd9564638a0593bc3d03149c5de51053f73fb1717789f51ff2cadfa525536667c725a649b6bfaeb6b8859257792dbf0400cdb16bfd55c5f15fa6a33bd2fe5667b61c703706ead079dfba5b9db7069de1ecba6ee577273af84bbba8af7d494a2645e0999d46fa305c534a21b3b8493e13ec2a33f608e7748d87214d6a06a1f7a286287a3b0d9b726e2ad20a44062726ffdfa15ab839770ac3466237d234149fa8007ec336d77fdc73d357e0c696f94deae9d83e8836e5cff457ad98e11e173ec929e5daee3beea54e71f927dbd77b115a136be17a5596d1c2ff43a35aa3e941654264a59ff4b26aae2bd790f6d66b4d0570648ca6b00a93c6918a9d01af9e115bd1d2de88035538f2477b1a723da647c77ea9b36ee7795737d779bfa9caeaf2dbaff42b7e22d3d3a04677888074db0fea004eccb48ccb5df257248d10b940683c132bd540c107cbcfcc9c4a28c080cf4b213938ee30078f6b048709a3dbd85eec4e317be3b43cedbf1af4e259f39d4373b6c6608f23866c7462d80f70a305332b6aefb6e07dd688edf4248e3b7a95d36c0be7025397c8229b3e33c361cebf215470154dba18acbc7d72d4e1c651409b01e9", + "transcript_input": "00867888931969393e24c82320dce065753ed7cb7d4d429564beda218d92bb1054439ce4f81947b99c7c4ba8b561973fb630f5e90b17c64f1010014e0ee0527422cc1b75b5e7aa62a229cb2ce96c8f9db05015f88ae40ca56161c6d05a7a482ed9ef2868a50f66e055705409098a11b58b933e7c65347239df5958c64576a4defe29060100fd2106f0d9f1814882bf6a3513b9c69b993535a6935a7903132fb67b85da8ad3c26db6f46c84fb078b348d52e75cc6421bdef50220c10b81b132c5c92944376f5132382bf0781c18a28b132985241a6a358536e6a45df18709ecacc7121116fabb287c638e815c9e7bb9f46b0fdc972be0aca13f0974b517202aa47b23c64c8a719ecc901df9f4bc309a3615e092a82b21477a7f900b5c0857694fdb25b44b0fde954a9f75658665904183bdab66c455b951f46a57dfdac732e6cf03048cefb813b4f71795fbba857013b7d6ac61f9875b801aebf298c8b0633db092ea4b3829a93870c167d5081bb7aaa745350ded309a46e1325b87afc346009781b81626547eab3b98c6664c19b2438bb8862485ab57836f1908782bc755ea94e5375a4b80139270c4bb332a6f5c2eaec03b890008504b2af26505b104b72ae742f430132440cca2abc65963bb9709770659623d8239710a5beac68f498a0a2a091ef0f78e19cc0e42222674d8b4fba57dde3301be234f527aa31c53be2be18649b17a047caff4e57a0c93ba8a3823d441c3f1901808c19b1871afec811a9d583d6f1516640c09dfd16e25ab35efb82a419bcbfdd225d3b57760f12b8d56a2fdac0f52b05526f86ce9c73bd1dcadeaaa6056b36fc70495c66a9cecda24f6365521246bd534a77f3c898c382397200fb8530546ec240814504e0971b9b990768434932781c7035942237f4af92aaf881d04fc3d90960ed0a2c278e385155a01712c33590582a6250953e931a9f03f0493272a09716923c8bb8b41ddd762306c85e3a12d66ccbe5f271f943659bc6288f67b5599a2ac4e7cc621460cd8aa5d57013c916854589a0bcb076646896d09f677a855818ae89a3655b327b4b9a63ac0429290d5d4762ef95c5d0969276a10b19930a3f69e849abaf0d97f78893ff40a598253300e54c6b6576e36079c278b5d697401c77049a31a44eee245440b397e2398b9208a19283b8e817d679c2ee4295411f7be4c6bc2b7329420d939c30400645c7baf8cc757fa84a815c49ec7bb9ea19d9dcc0ef9d3562db79f0f810ddd14690d51276df631388a51e54116d0d247e88b140b658e7f7203523c96a05053ced944ed0890fdfa3d23d46c9d41c5b5d34bbc1b862d2b19c6138e4a46866e313244fb6fc3706ba5ac692e6610fc800b2ae57eba9499f0295b83eaae1bf03e5ba75626688f5f7b6b25c430cec60c89f445653042e6746ac3f173d18a30b2a022555001767051c948cc9392b95a5234d5c1c33629648be9238de63d1a15a2182149c89b84727961ab046ad6d7639e9032bca94e480169b5f22b438418a3c926bd944de38750b0f1a802b9a92c6c2119553a88751bb48b65f451710bd7167a31b13ad81cd36631df339b292309bef1a719080cc72abe9917465757bcb5018a84f6ad3b601c018a3b5cab962cc7b49321bb0517a5bc3a2889272b52b10d629b1316bc17dd2b323a7468f8e52c2a43b8b736bbe116327522a2975aa5bfb8b5da976c12f3beafb36210b273a967024f6b030b98139ca49b484ac2937a50374670eafc1fe4078d9d36874fa5b462aa5e6b32b4d66a948b86bcbeb706563042034227fe521149a4b45fcc86959c09f8a1a333712347102dcfcb3158280e24d0af4251c6a5925583a03cb4445b3b595932bacf43579ae6d797d56c5261567739fa8c9ce3178b3501be2121f9b249ab35bdc26008321b36d5365f6e9530156267493b8fd60c04ed386c96ec58345990b540c59179035593a454d63d43a97888d2ade3c4b4704b5b14940585b23ae8b022a8e17ae2f886bf834bf3190936790f1b856d92588476f140c9b32fa906c0cfa21daba5498197beecd17e768475ec296b675c7bbcb9b5920900d94381305552b5e58bbe203fc7b849322310eb5c477d66097e913a532c5dd79c123ba21508fccb5ab8a81c4459bb74b3d9c46143d0616ac0c196609d22a8903c5639d17aa04cd0228a7680d3101d00a044e66b4bfbc148fca3a773d92eafc9489d6051ff591f931cca910585e2299bd5f6af8b72a813f98642a28f9324c90b3403001329304ca0e15068574c617b69006e1b5d21d8cad3168669331e7d3445fd5a8cd232a4de426c87a090304c0dc16680353ac7bab44cbfa1c916884f5e917e18993fe4c778f5193cec47e5af8d3ae1b706f4b31bdf47234de201b4583d57532cec513dc3edfe00000100f1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000fd2406fd2106f014d9b16a1a42a50c6faf6efc3aaab564575de5e587d0c06dd158b93ed1a795a32ee291c56a4ba254d805f7546691514206735a80ea73591bc3a20040f5979a689f083ba1cb1f2ba204b25a94ce83bf48854e05203de1137582b4e482165b4b462a63ec53d5dc2291dc61da93de28691edf28fe72ffbda9361c4b73c555b1165dae61557500f32c5247ab886e0482a2fffe0d1d48abb3f732089df546ecf7b5a53e4803fc39aea2d2b5ebe31bd1c4e90db7de6d239a025f359dd7ea8533ecb131084ed0d35461b2a14622af9eb437a327f36a842e3178c1f32ad13b95e314f2758e0cb1059ab74ba366b6f3a87d79645c7aee5822c9b75ea47ce6654ac61637a608913571093c282aa7e77ce2612a073e4cf40ceba3958b592d9e32236a0110362256230368f9560596be51003596013ed48891f57989eebb7b009a0eac1dd2556af27c3305d571ed6d252157dc8a17924b2e6a80febe0ff3a29c24be8b2ca8a56505f7c55901b8a602488375b1b34d63b1ba167199dfb51c56bc025b2be55cdc9b136702d06eb71946cd4e710c51cd907c59243ababe45861a4bd7a6d097e6e8ecf5a8d60ee5bf5570bda3e1ecb873bbc119695268f2203d08e6f786a7d7af68207a9833a37bd4c65c079bf21ebcf22ea028c01547378a11a258ebdb1279014fd8d527923b5cf93c374d5c990d18bfd5eba7c5b14bfa445ad69667d1c38ca3be167657d42aa621908e9cba43a316e3c000f5c2d8802433210e3bf3c6cb7df77097e88ae0c8d23cf8a73582fa351d60b77037b9fbadf683a12fc30473493f20f8262ca2383f000a97b7adc14174167678ed0b3b8559fd528aefc320964156591cade0c9dac72b36c32f9a47147e6285a52f491a4b6c6150c9e16b63b77eae68ccf7add5502593075fa20aaa16f88d498e55d56c69ab61b83b7aa900c09a9f6d497e5711553ad9962abf0cb0f4f99830cdd6a9a9ae6f54d9e7dd8bbd533acc4a6233bbd49d6a2a2159dcd59c9613122ce4684ed49357125141a8c473aa85d31d3624c8bd0d44989f61ef9f0101490cca8be88ef630ab2f52906fca2b2981c9cfdf4a82db342d5534d33313311044f7a8034f604418d9694760a3fb8ab98b12fa0e83af9c2ce43f9dfe9b63dff9cf9c209f97a1c526660e7a39b4082966a53825b67ddc22afbb4e69bb90138bfc834c2b2c6380f5666db64d153519e33296ebbe577f6b98a13c3d54688754f9ace99168cd7bee489055fb6e50b4ac9104896d8867749d47964bb597fa3b29a4457a431879b96f633f59a23e663dc8820ad97ac85e6cc722d3bb93c780c4a7b909b1a2c9a5745cefd55bde2383d1e9ed1104923814b6b5b71bb3be9d1215c49f0120fd7e04db1d8b0921f209ccbd404f0d9bb69d754717536e402fd19e261ed162f0d98bd1f0f95e3ee720b21e9554bab58445c15d8bc99f6afcaa63ed40641feca1ea820a8de86b5e7f3da3fcadfa7d111242695f10180db0c72607a342b63c5917ca4d7aea3e8ccdb511ff487eaa01bad8c0ce0f6ac3060946b109095edc01094fa943cf5271097c7589efbad0c8ab661b8f21c0880567f160f6ddada20c109dbd2ff0fd9564638a0593bc3d03149c5de51053f73fb1717789f51ff2cadfa525536667c725a649b6bfaeb6b8859257792dbf0400cdb16bfd55c5f15fa6a33bd2fe5667b61c703706ead079dfba5b9db7069de1ecba6ee577273af84bbba8af7d494a2645e0999d46fa305c534a21b3b8493e13ec2a33f608e7748d87214d6a06a1f7a286287a3b0d9b726e2ad20a44062726ffdfa15ab839770ac3466237d234149fa8007ec336d77fdc73d357e0c696f94deae9d83e8836e5cff457ad98e11e173ec929e5daee3beea54e71f927dbd77b115a136be17a5596d1c2ff43a35aa3e941654264a59ff4b26aae2bd790f6d66b4d0570648ca6b00a93c6918a9d01af9e115bd1d2de88035538f2477b1a723da647c77ea9b36ee7795737d779bfa9caeaf2dbaff42b7e22d3d3a04677888074db0fea004eccb48ccb5df257248d10b940683c132bd540c107cbcfcc9c4a28c080cf4b213938ee30078f6b048709a3dbd85eec4e317be3b43cedbf1af4e259f39d4373b6c6608f23866c7462d80f70a305332b6aefb6e07dd688edf4248e3b7a95d36c0be7025397c8229b3e33c361cebf215470154dba18acbc7d72d4e1c651409b01e9", + "transcript_hash": "a4538eb235f3f1a1128dbb1a2c213fc655ccd42fafa41c442c850244c903dbf9", + "salt": "716269745f76325f6879627269645f6d6c6b656d31303234444f24a8", + "ikm": "bf3f6c4be548a2afbf33f9b33c2d0b52b3f133e87ca0dc636281ccc833ceab8f50124a2dd20a655c9d431594979a253093dc976e424ed1076e09cced7ef06941a4538eb235f3f1a1128dbb1a2c213fc655ccd42fafa41c442c850244c903dbf9", + "prk": "a54dca5a686a40867aeb980ece78e3cc23c54cffa3a617662fcbd56b6c3788d1", + "initiator_L": "37aa1cb24cd70feceb65e1ffa7b909d39119aecb0bd6c2bb68f14b7a1b2552d1", + "initiator_P": "fc86c07860059c9b0885a1faa48daae92ba13d49eba4e0dbfad705ef165eacc4", + "responder_L": "eaf470586dea69230ed91bb8f454b25301198d3dc5aade5f3d854f61137fe942", + "responder_P": "08017b2336a5e6d8a5b379ea84d5d1fe6469952cf6c19254b3955cf0f50b12c4", + "session_id": "94376e50745dd11d5a962abe2057bb7f14dcc9e94131f49f42c3e032f004311c", + "initiator_confirmation": "b727dfa3a9b214c5e7b40559d6aebc4136fa1513", + "responder_confirmation": "dde08146e6422b6322a931bd70dff2d2a4a73124", + "initiator_first_contents": "120123456789abcdef", + "initiator_first_packet": "a13e6541e85df6f59b20c77fe3a74e78adac4b710dd99d5c59aecb326a", + "responder_first_contents": "130123456789abcdef", + "responder_first_packet": "d187660dd5bb303079116fceb19188396845a7c2e8b6530d499917880d" + } + ], + "confirmation_failures": [ + { + "comment": "The initiator's confirmation with bit 0 of its first byte flipped: the length decrypts to 1.", + "vector": 0, + "receiver": "responder", + "packet": "68abb2b1c9601dad7baf8038bc3601eaa42527de", + "decrypted_length": 1, + "failure": "length" + }, + { + "comment": "The initiator's confirmation with bit 0 of its last tag byte flipped: the length decrypts to 0 and the tag fails.", + "vector": 0, + "receiver": "responder", + "packet": "69abb2b1c9601dad7baf8038bc3601eaa42527df", + "decrypted_length": 0, + "failure": "tag" + }, + { + "comment": "The responder's first hybrid packet is an authenticated empty packet without the ignore bit.", + "vector": 0, + "receiver": "initiator", + "packet": "a634fa6c1db68e36c90668fe20a632b91a491ece", + "decrypted_length": 0, + "failure": "not_decoy" + } + ], + "negative_vectors": [ + { + "comment": "Bit 0 of the ciphertext flipped in transit: the responder hashes the damaged accept and decapsulation returns the implicit-rejection secret.", + "vector": 0, + "role": "responder", + "transcript_contents": [ + "fd2106f04ef53cd8d75470aa5492cc10d1b519a02ac168e5394226756c89ace3a2174cb08fe962b490f3c2cf88a60391513083a330718b73257c554c7cf0ca1865fbbf682435605c5a7f0636b19ab1c7e2c9cb40cf62c47940b17855f7c2e4d1869409ad7b3b047052bfd7366195d84890f03ac23032a63c545ed99f07e55bc81771836917b5139954d413162b9b5745556865710a5401bd9b6e3382adef4108b80833ed2337aaf2392a9704e239cb4a35332bf8257c288200389d054ba9acf772c61ab9f74875353b1bb933bff0e12f72b8c1537c4b47a74750d5617eb4c99d72af4e401961f588ff9a617b775ef7280a8a2843f1b408e6cbcf7385493a3b9e86230ba3c5676c5cb8885c317855a103e915aeac2cb27c36392494bb2aac04f481d38ba33cf7376711cce831380c580bdd887b3183a8fb2642857b3283c51a82c788e5293ddbd01bc46609f6017f40294f4b43842a709db0a7203565ceebda5721c1201569217ad817cd6316fc8c7f3036a4d5a679095bb4376c9e75360a64051f17fb8445f4556d37180c83cbee261629620ecec139bb8c609da90f9da2a8e1297793546ef90c0bffb3b0c836215bb9108191af7803cd5f26163c577404002aaed44de693892cb04f87e060dc5b9db9d00a4ea6b2734505c04857c31ccaa68bc833733d1e2837e0b5aec46354867933824399f132805893330c73c11c988d1adb6aac9b9bbcbc41263158e2b960c30665571484900ac7621a3e7fd1252ffb271f763ddc98293a5b929790468b9b4ed848c9ba531adaa31fec60367bc48eae612777cb79d7b56a17f8a6afc1c1febb86ec286ef9260cb27079c0d5a74910b3ef26b09ec96de697735e91841b9224d33abf2e1b14b15460d9055093268c75c17aa1ba15b5463e68d32c99a10d40c603b45c1fb81b5965c17379b898cba3cd1471bb07c8586f232d5c29ade9831f06163a3d60c672535e8fb0a01fd4435853480b073ec0ac32bc8a876cc3cd43bbb7bf8b3dc77874be68bff1d0cdf6a2900d8bca43f29b87c4395c7a794c39a45e89c3d9e3bf7046b5bc204a75a30a707242525c81cf4ac93c2a8b266c59eaaa0d997348e9fc7bd0294231c559a5cc1465a22317e714cc1431c092bed0665f242098c556b1c091a18519c1fa91354d6403f71972e78c23a28a276798826b3b711e76aa17f27c24214d6c1c6e8a53736cd986d9e446068532f1a6214cc98eb1625cd064456b0481abe2a38aeaa26845b77b1926b791a4a3b56739997edeb097284a9a6217438fd92b65399d0ecb2f23ccb1c8397496413615c26304d5272d60bc34f3ba6b5c652f693bc702be90349f27fa019173508a94bdf5f1153b341d20d040eb506ea0d40b561c0178c662a7c5b6bd130535355a8a5438ae1c0e2ab872f92800ba319b34d587e52860eb183b0bdb24ec024a111b369d3871c1c7ce9ec97207a208b4ec32ea142c61d23bdbb40325ca0756b36c5de19bed851d0ce6a30c175e467957d72b65f5552007dab6cd474d095ab556353111b9664a012d7fd767b3d549138580a087103d72bb602520f9393317e72e3fbc5d7206a24ccaae03fa844619a4acaa8c246053ed122c9765bc9164aa1c7257ac5893e0619bb965c48f24b5a5747d02f38705a09c905b82918856487382f40c28b73916b8059c29b39853189997978e49b05df1495c66a777e5f84ef99575f51b33fb157d41e54a7611c082387c6da242c3042fce7711525104f46233cad9882e45796a09c59b8471287b19c4f6205ea0b3543c341f8c169c11010cd52abda1b934024b9181c047ecad01844709d7834aa056648097f2109e4fac640d32ad80cc5581ac60296a5b51d615241c8cc0bcccfe777b97e6a5475011603a8ad7eaafe741572018840b989d83f986163633bc68919d571211d317c0a3c0347529b49101c2d451844b5083404da25b4d37931743139953fcc8374369260b88193284bc96250d02c05ecc6809a44a38e24261083dc3953d4e8a093bda800b4b2614d76a4355c6a900251e3cb9de069ed2ebaa34b34aae3661362218b5350b4f68c9bc6b11d0f5753e792cf300acdb94c52eab50d406967f498a8327420d6397886380a94019b307b8188a4496b223ebe27075b87c5d8680855ac7c5b892d357247aecb55e6089a2dba345f65d259b8d40a253a450a546b46abc3d1804f4c6ecc2a992fdff4e7d", + "fd2106f078578b92edfe5618d5dd9284a2004e45ebe3a37cc4c0eef53eeaf8d322303b91edfb0aa7c453d4c0d06bc29c2e904f20db8e6e0567d6a1303ad190d7c087a76da634a6e0ac11f4f78496ca11862c87acb0fc273061479a89758886d0b0d78375b1edf317ddfa889507df4e74853fe34892580a127e104f0bc4df583d8415b54367efafea0995c7e7ed7a1dedb8112a22e8278f90361dbf0a1f7af1246b609c9981f95a919312540d8bbd207eb75cf9feb4f9d2689b071f1f5b0d790573ca40db398374d3e50e3c2d46bb966567cd9c9b1589ee62206ead95a80483d4fa567ff697d40d804acdde8ec654fd5ba413133fabb99db19d949ecde78d55cb563766e714cf2138f50457809757b618b004ad48c1cb0b0f1c95e0fbe1f43813d5f035c4b3ad1b6a5c1c5fa0a1d27a0cb6d3de7b3a054e6fcbc24e286e6a77e45ab56bc5ecb070a00c22974d05832832bd40ab96010039be5da0464731993e2bccfcc468533cbfe3089a6f579d5a06cbe4bf08ddb85fe9996ed95fe9442130e49c61d1bf0881968672eae12c568189f3285671358e68ae0fbb5075ed137127788aa1f96033c89f41ade84cc61aaeae9b2c86862b1e956c74d39bd6e8f00907fd586cd099ab4224883c2e0c07ac4bb3f74a5a59975ef0e844583370cf077671fbfc2a333182dce2ac9cc8f3ba0d73cd27ae60aabc1d7689f7bd2d7ecd5d97164293cc88fe7adbadb3a24798f7b2be578c7780a681cc902b69b7fe4aa8f535e4a9ed83d002c1ea0f28e711be6231cabde2ea19655688c8d70b413420651852db3bcad5c6b66267d516088b3955a6884ca44a83c9f179c5901ff4edefea074ee0cd425a7dd8a855ea49affbd7a8604c490213dfa7bf48ac12b738fae7cd1697e757f26a59826e64b7799a47703e037c9ca7cf18828574092a9b0af255a5180eef96f96e5d2f51cc621d7a03c7bb05eb80d386415f243f5d59451d5f95c93f4616cca643cd6be65109d440461e752d2999745facd5eacaea338d6ce940b54c8dba44f059c94fa65a74456b747c065f2aba2b8580931e1e5db4cd6505d4cc50ce68e15a1f9e0069edd1a8d5bbfa9180f0a88fef1d773a5b417b67ddd8729c3a781bfcdeb527126579f5d11c5550525869a3236e1f21940542c7ce69e3a265c24d9aefeab6c0852636708a74718729f35a401fd73b46b204ec92b40d9ec0e5f924c967a8a4e219910d40af2d19e76869bfb7c95868e4572334bc35f7071d6254cb3cdf82ee94d3b685fe48a49ad95503ca19b39e316e57a32a1eaa9af4573a32bfe94dd77c744484781c2efe891e2b4b65b38e12dbab69b436d4b0ac6a9b512ee898b95906163f32839fa2985fadead35d79fa03a284debc60c51daa8a017c1d4dc4dc7e884b1374d131fde2ca6e591672b259754b4f76bbf6ed3eeeee94e4e7c496344a3b38c8b4dea12684bbc0713ec6b7684479e883a0a515d681581beab290d952fa20f6f9ae63dc00a55d1ef3f592d9a86d97a40ae74717623dc1ffdf2588f241d6600e6f34163db805273266f58e75e074d7d079d40755012da5ba3203d2bfdc9100e361148456a7ac423ba9cf3bd135f3e64c2f262fcdb728e0682e8297cc2af4e0467603990459d923cf6974cc27f9c02f535d6962cd67bbfd1287ba4148647153fd33ef9e482e2555ff6ee243ce6ab277af1494944b2913ecf29819507571427d7c1852d684a6359a72698b65ef040b8a03c3fffca6f4a9e13359db9a1e3d2e4fac6d469357d9d9ca6663f6a3f80076c6ec937aa16d23b12fb0ee483c260ad6e751a2814720e36dfd26202b14c1cd18e3f9477e924292d84d905c056bd73ceecede186f3b6987abe328fb3e1e48b2bed3885b9d9e9af6bdebc0c7f9cc6f7954b38905e3d8ee41f780ca446792326a94837c367104983ea9b906a7f7a8b4bbf44700694c27e7670e4a7b3e5df7218d8d67fd68bda52592e650de25b1a81ed304d81cadbad366833d94d7e69a8433c9d7edfb1043b99a08265de814121119d3f7eaa74c8082bd9b4a91bb2478605a2f2e077679665d24391ea37c748c5c6e35f00fd202a13c032270b0d30f0af8bb1d7c2570055980748fd7761d6a801043ecc04051f7d60a5b7361436549762a9f3ba7eb7344658b154b8b680ba8ac2eeec07e4cc3cac211255c3f901e713db12984a00450011790a0f2d6af22d5d35b83c5483dd4e8" + ], + "ss_mlkem": "2add583d143a71963d6fe31fe39ed47923325d513c27a8c40f4ed5d74ec7f30c", + "session_id": "44c4e64069b50d8cd3e333a38dcd6fa2c9543b4d52a7db8fc2114ad7e7136304", + "confirmation": "db84ca2e545823c5d3e0fb8c6b45995ab779a106", + "decrypted_length": 8641960, + "failure": "length", + "peer_decrypted_length": 3190909, + "peer_failure": "length" + }, + { + "comment": "The initiator's ML-KEM shared secret has byte 0 inverted after encapsulation (a local fault); EK and CT are untouched.", + "vector": 0, + "role": "initiator", + "transcript_contents": [ + "fd2106f04ef53cd8d75470aa5492cc10d1b519a02ac168e5394226756c89ace3a2174cb08fe962b490f3c2cf88a60391513083a330718b73257c554c7cf0ca1865fbbf682435605c5a7f0636b19ab1c7e2c9cb40cf62c47940b17855f7c2e4d1869409ad7b3b047052bfd7366195d84890f03ac23032a63c545ed99f07e55bc81771836917b5139954d413162b9b5745556865710a5401bd9b6e3382adef4108b80833ed2337aaf2392a9704e239cb4a35332bf8257c288200389d054ba9acf772c61ab9f74875353b1bb933bff0e12f72b8c1537c4b47a74750d5617eb4c99d72af4e401961f588ff9a617b775ef7280a8a2843f1b408e6cbcf7385493a3b9e86230ba3c5676c5cb8885c317855a103e915aeac2cb27c36392494bb2aac04f481d38ba33cf7376711cce831380c580bdd887b3183a8fb2642857b3283c51a82c788e5293ddbd01bc46609f6017f40294f4b43842a709db0a7203565ceebda5721c1201569217ad817cd6316fc8c7f3036a4d5a679095bb4376c9e75360a64051f17fb8445f4556d37180c83cbee261629620ecec139bb8c609da90f9da2a8e1297793546ef90c0bffb3b0c836215bb9108191af7803cd5f26163c577404002aaed44de693892cb04f87e060dc5b9db9d00a4ea6b2734505c04857c31ccaa68bc833733d1e2837e0b5aec46354867933824399f132805893330c73c11c988d1adb6aac9b9bbcbc41263158e2b960c30665571484900ac7621a3e7fd1252ffb271f763ddc98293a5b929790468b9b4ed848c9ba531adaa31fec60367bc48eae612777cb79d7b56a17f8a6afc1c1febb86ec286ef9260cb27079c0d5a74910b3ef26b09ec96de697735e91841b9224d33abf2e1b14b15460d9055093268c75c17aa1ba15b5463e68d32c99a10d40c603b45c1fb81b5965c17379b898cba3cd1471bb07c8586f232d5c29ade9831f06163a3d60c672535e8fb0a01fd4435853480b073ec0ac32bc8a876cc3cd43bbb7bf8b3dc77874be68bff1d0cdf6a2900d8bca43f29b87c4395c7a794c39a45e89c3d9e3bf7046b5bc204a75a30a707242525c81cf4ac93c2a8b266c59eaaa0d997348e9fc7bd0294231c559a5cc1465a22317e714cc1431c092bed0665f242098c556b1c091a18519c1fa91354d6403f71972e78c23a28a276798826b3b711e76aa17f27c24214d6c1c6e8a53736cd986d9e446068532f1a6214cc98eb1625cd064456b0481abe2a38aeaa26845b77b1926b791a4a3b56739997edeb097284a9a6217438fd92b65399d0ecb2f23ccb1c8397496413615c26304d5272d60bc34f3ba6b5c652f693bc702be90349f27fa019173508a94bdf5f1153b341d20d040eb506ea0d40b561c0178c662a7c5b6bd130535355a8a5438ae1c0e2ab872f92800ba319b34d587e52860eb183b0bdb24ec024a111b369d3871c1c7ce9ec97207a208b4ec32ea142c61d23bdbb40325ca0756b36c5de19bed851d0ce6a30c175e467957d72b65f5552007dab6cd474d095ab556353111b9664a012d7fd767b3d549138580a087103d72bb602520f9393317e72e3fbc5d7206a24ccaae03fa844619a4acaa8c246053ed122c9765bc9164aa1c7257ac5893e0619bb965c48f24b5a5747d02f38705a09c905b82918856487382f40c28b73916b8059c29b39853189997978e49b05df1495c66a777e5f84ef99575f51b33fb157d41e54a7611c082387c6da242c3042fce7711525104f46233cad9882e45796a09c59b8471287b19c4f6205ea0b3543c341f8c169c11010cd52abda1b934024b9181c047ecad01844709d7834aa056648097f2109e4fac640d32ad80cc5581ac60296a5b51d615241c8cc0bcccfe777b97e6a5475011603a8ad7eaafe741572018840b989d83f986163633bc68919d571211d317c0a3c0347529b49101c2d451844b5083404da25b4d37931743139953fcc8374369260b88193284bc96250d02c05ecc6809a44a38e24261083dc3953d4e8a093bda800b4b2614d76a4355c6a900251e3cb9de069ed2ebaa34b34aae3661362218b5350b4f68c9bc6b11d0f5753e792cf300acdb94c52eab50d406967f498a8327420d6397886380a94019b307b8188a4496b223ebe27075b87c5d8680855ac7c5b892d357247aecb55e6089a2dba345f65d259b8d40a253a450a546b46abc3d1804f4c6ecc2a992fdff4e7d", + "fd2106f079578b92edfe5618d5dd9284a2004e45ebe3a37cc4c0eef53eeaf8d322303b91edfb0aa7c453d4c0d06bc29c2e904f20db8e6e0567d6a1303ad190d7c087a76da634a6e0ac11f4f78496ca11862c87acb0fc273061479a89758886d0b0d78375b1edf317ddfa889507df4e74853fe34892580a127e104f0bc4df583d8415b54367efafea0995c7e7ed7a1dedb8112a22e8278f90361dbf0a1f7af1246b609c9981f95a919312540d8bbd207eb75cf9feb4f9d2689b071f1f5b0d790573ca40db398374d3e50e3c2d46bb966567cd9c9b1589ee62206ead95a80483d4fa567ff697d40d804acdde8ec654fd5ba413133fabb99db19d949ecde78d55cb563766e714cf2138f50457809757b618b004ad48c1cb0b0f1c95e0fbe1f43813d5f035c4b3ad1b6a5c1c5fa0a1d27a0cb6d3de7b3a054e6fcbc24e286e6a77e45ab56bc5ecb070a00c22974d05832832bd40ab96010039be5da0464731993e2bccfcc468533cbfe3089a6f579d5a06cbe4bf08ddb85fe9996ed95fe9442130e49c61d1bf0881968672eae12c568189f3285671358e68ae0fbb5075ed137127788aa1f96033c89f41ade84cc61aaeae9b2c86862b1e956c74d39bd6e8f00907fd586cd099ab4224883c2e0c07ac4bb3f74a5a59975ef0e844583370cf077671fbfc2a333182dce2ac9cc8f3ba0d73cd27ae60aabc1d7689f7bd2d7ecd5d97164293cc88fe7adbadb3a24798f7b2be578c7780a681cc902b69b7fe4aa8f535e4a9ed83d002c1ea0f28e711be6231cabde2ea19655688c8d70b413420651852db3bcad5c6b66267d516088b3955a6884ca44a83c9f179c5901ff4edefea074ee0cd425a7dd8a855ea49affbd7a8604c490213dfa7bf48ac12b738fae7cd1697e757f26a59826e64b7799a47703e037c9ca7cf18828574092a9b0af255a5180eef96f96e5d2f51cc621d7a03c7bb05eb80d386415f243f5d59451d5f95c93f4616cca643cd6be65109d440461e752d2999745facd5eacaea338d6ce940b54c8dba44f059c94fa65a74456b747c065f2aba2b8580931e1e5db4cd6505d4cc50ce68e15a1f9e0069edd1a8d5bbfa9180f0a88fef1d773a5b417b67ddd8729c3a781bfcdeb527126579f5d11c5550525869a3236e1f21940542c7ce69e3a265c24d9aefeab6c0852636708a74718729f35a401fd73b46b204ec92b40d9ec0e5f924c967a8a4e219910d40af2d19e76869bfb7c95868e4572334bc35f7071d6254cb3cdf82ee94d3b685fe48a49ad95503ca19b39e316e57a32a1eaa9af4573a32bfe94dd77c744484781c2efe891e2b4b65b38e12dbab69b436d4b0ac6a9b512ee898b95906163f32839fa2985fadead35d79fa03a284debc60c51daa8a017c1d4dc4dc7e884b1374d131fde2ca6e591672b259754b4f76bbf6ed3eeeee94e4e7c496344a3b38c8b4dea12684bbc0713ec6b7684479e883a0a515d681581beab290d952fa20f6f9ae63dc00a55d1ef3f592d9a86d97a40ae74717623dc1ffdf2588f241d6600e6f34163db805273266f58e75e074d7d079d40755012da5ba3203d2bfdc9100e361148456a7ac423ba9cf3bd135f3e64c2f262fcdb728e0682e8297cc2af4e0467603990459d923cf6974cc27f9c02f535d6962cd67bbfd1287ba4148647153fd33ef9e482e2555ff6ee243ce6ab277af1494944b2913ecf29819507571427d7c1852d684a6359a72698b65ef040b8a03c3fffca6f4a9e13359db9a1e3d2e4fac6d469357d9d9ca6663f6a3f80076c6ec937aa16d23b12fb0ee483c260ad6e751a2814720e36dfd26202b14c1cd18e3f9477e924292d84d905c056bd73ceecede186f3b6987abe328fb3e1e48b2bed3885b9d9e9af6bdebc0c7f9cc6f7954b38905e3d8ee41f780ca446792326a94837c367104983ea9b906a7f7a8b4bbf44700694c27e7670e4a7b3e5df7218d8d67fd68bda52592e650de25b1a81ed304d81cadbad366833d94d7e69a8433c9d7edfb1043b99a08265de814121119d3f7eaa74c8082bd9b4a91bb2478605a2f2e077679665d24391ea37c748c5c6e35f00fd202a13c032270b0d30f0af8bb1d7c2570055980748fd7761d6a801043ecc04051f7d60a5b7361436549762a9f3ba7eb7344658b154b8b680ba8ac2eeec07e4cc3cac211255c3f901e713db12984a00450011790a0f2d6af22d5d35b83c5483dd4e8" + ], + "ss_mlkem": "ddc8bf5c4858ac2e5f94b6e945ca142ccc28b98e7b954a78899630a7bfa3135f", + "session_id": "caa12cb88128baf00d2ca768fea2a7b0d0740af35c5fa5524dc7783b525f3082", + "confirmation": "8b4724ce6bcb46d34283fb52710bc1b12c7b25c4", + "decrypted_length": 1490822, + "failure": "length", + "peer_decrypted_length": 9891042, + "peer_failure": "length" + }, + { + "comment": "The responder hashes the two contents in the wrong order.", + "vector": 0, + "role": "responder", + "transcript_contents": [ + "fd2106f079578b92edfe5618d5dd9284a2004e45ebe3a37cc4c0eef53eeaf8d322303b91edfb0aa7c453d4c0d06bc29c2e904f20db8e6e0567d6a1303ad190d7c087a76da634a6e0ac11f4f78496ca11862c87acb0fc273061479a89758886d0b0d78375b1edf317ddfa889507df4e74853fe34892580a127e104f0bc4df583d8415b54367efafea0995c7e7ed7a1dedb8112a22e8278f90361dbf0a1f7af1246b609c9981f95a919312540d8bbd207eb75cf9feb4f9d2689b071f1f5b0d790573ca40db398374d3e50e3c2d46bb966567cd9c9b1589ee62206ead95a80483d4fa567ff697d40d804acdde8ec654fd5ba413133fabb99db19d949ecde78d55cb563766e714cf2138f50457809757b618b004ad48c1cb0b0f1c95e0fbe1f43813d5f035c4b3ad1b6a5c1c5fa0a1d27a0cb6d3de7b3a054e6fcbc24e286e6a77e45ab56bc5ecb070a00c22974d05832832bd40ab96010039be5da0464731993e2bccfcc468533cbfe3089a6f579d5a06cbe4bf08ddb85fe9996ed95fe9442130e49c61d1bf0881968672eae12c568189f3285671358e68ae0fbb5075ed137127788aa1f96033c89f41ade84cc61aaeae9b2c86862b1e956c74d39bd6e8f00907fd586cd099ab4224883c2e0c07ac4bb3f74a5a59975ef0e844583370cf077671fbfc2a333182dce2ac9cc8f3ba0d73cd27ae60aabc1d7689f7bd2d7ecd5d97164293cc88fe7adbadb3a24798f7b2be578c7780a681cc902b69b7fe4aa8f535e4a9ed83d002c1ea0f28e711be6231cabde2ea19655688c8d70b413420651852db3bcad5c6b66267d516088b3955a6884ca44a83c9f179c5901ff4edefea074ee0cd425a7dd8a855ea49affbd7a8604c490213dfa7bf48ac12b738fae7cd1697e757f26a59826e64b7799a47703e037c9ca7cf18828574092a9b0af255a5180eef96f96e5d2f51cc621d7a03c7bb05eb80d386415f243f5d59451d5f95c93f4616cca643cd6be65109d440461e752d2999745facd5eacaea338d6ce940b54c8dba44f059c94fa65a74456b747c065f2aba2b8580931e1e5db4cd6505d4cc50ce68e15a1f9e0069edd1a8d5bbfa9180f0a88fef1d773a5b417b67ddd8729c3a781bfcdeb527126579f5d11c5550525869a3236e1f21940542c7ce69e3a265c24d9aefeab6c0852636708a74718729f35a401fd73b46b204ec92b40d9ec0e5f924c967a8a4e219910d40af2d19e76869bfb7c95868e4572334bc35f7071d6254cb3cdf82ee94d3b685fe48a49ad95503ca19b39e316e57a32a1eaa9af4573a32bfe94dd77c744484781c2efe891e2b4b65b38e12dbab69b436d4b0ac6a9b512ee898b95906163f32839fa2985fadead35d79fa03a284debc60c51daa8a017c1d4dc4dc7e884b1374d131fde2ca6e591672b259754b4f76bbf6ed3eeeee94e4e7c496344a3b38c8b4dea12684bbc0713ec6b7684479e883a0a515d681581beab290d952fa20f6f9ae63dc00a55d1ef3f592d9a86d97a40ae74717623dc1ffdf2588f241d6600e6f34163db805273266f58e75e074d7d079d40755012da5ba3203d2bfdc9100e361148456a7ac423ba9cf3bd135f3e64c2f262fcdb728e0682e8297cc2af4e0467603990459d923cf6974cc27f9c02f535d6962cd67bbfd1287ba4148647153fd33ef9e482e2555ff6ee243ce6ab277af1494944b2913ecf29819507571427d7c1852d684a6359a72698b65ef040b8a03c3fffca6f4a9e13359db9a1e3d2e4fac6d469357d9d9ca6663f6a3f80076c6ec937aa16d23b12fb0ee483c260ad6e751a2814720e36dfd26202b14c1cd18e3f9477e924292d84d905c056bd73ceecede186f3b6987abe328fb3e1e48b2bed3885b9d9e9af6bdebc0c7f9cc6f7954b38905e3d8ee41f780ca446792326a94837c367104983ea9b906a7f7a8b4bbf44700694c27e7670e4a7b3e5df7218d8d67fd68bda52592e650de25b1a81ed304d81cadbad366833d94d7e69a8433c9d7edfb1043b99a08265de814121119d3f7eaa74c8082bd9b4a91bb2478605a2f2e077679665d24391ea37c748c5c6e35f00fd202a13c032270b0d30f0af8bb1d7c2570055980748fd7761d6a801043ecc04051f7d60a5b7361436549762a9f3ba7eb7344658b154b8b680ba8ac2eeec07e4cc3cac211255c3f901e713db12984a00450011790a0f2d6af22d5d35b83c5483dd4e8", + "fd2106f04ef53cd8d75470aa5492cc10d1b519a02ac168e5394226756c89ace3a2174cb08fe962b490f3c2cf88a60391513083a330718b73257c554c7cf0ca1865fbbf682435605c5a7f0636b19ab1c7e2c9cb40cf62c47940b17855f7c2e4d1869409ad7b3b047052bfd7366195d84890f03ac23032a63c545ed99f07e55bc81771836917b5139954d413162b9b5745556865710a5401bd9b6e3382adef4108b80833ed2337aaf2392a9704e239cb4a35332bf8257c288200389d054ba9acf772c61ab9f74875353b1bb933bff0e12f72b8c1537c4b47a74750d5617eb4c99d72af4e401961f588ff9a617b775ef7280a8a2843f1b408e6cbcf7385493a3b9e86230ba3c5676c5cb8885c317855a103e915aeac2cb27c36392494bb2aac04f481d38ba33cf7376711cce831380c580bdd887b3183a8fb2642857b3283c51a82c788e5293ddbd01bc46609f6017f40294f4b43842a709db0a7203565ceebda5721c1201569217ad817cd6316fc8c7f3036a4d5a679095bb4376c9e75360a64051f17fb8445f4556d37180c83cbee261629620ecec139bb8c609da90f9da2a8e1297793546ef90c0bffb3b0c836215bb9108191af7803cd5f26163c577404002aaed44de693892cb04f87e060dc5b9db9d00a4ea6b2734505c04857c31ccaa68bc833733d1e2837e0b5aec46354867933824399f132805893330c73c11c988d1adb6aac9b9bbcbc41263158e2b960c30665571484900ac7621a3e7fd1252ffb271f763ddc98293a5b929790468b9b4ed848c9ba531adaa31fec60367bc48eae612777cb79d7b56a17f8a6afc1c1febb86ec286ef9260cb27079c0d5a74910b3ef26b09ec96de697735e91841b9224d33abf2e1b14b15460d9055093268c75c17aa1ba15b5463e68d32c99a10d40c603b45c1fb81b5965c17379b898cba3cd1471bb07c8586f232d5c29ade9831f06163a3d60c672535e8fb0a01fd4435853480b073ec0ac32bc8a876cc3cd43bbb7bf8b3dc77874be68bff1d0cdf6a2900d8bca43f29b87c4395c7a794c39a45e89c3d9e3bf7046b5bc204a75a30a707242525c81cf4ac93c2a8b266c59eaaa0d997348e9fc7bd0294231c559a5cc1465a22317e714cc1431c092bed0665f242098c556b1c091a18519c1fa91354d6403f71972e78c23a28a276798826b3b711e76aa17f27c24214d6c1c6e8a53736cd986d9e446068532f1a6214cc98eb1625cd064456b0481abe2a38aeaa26845b77b1926b791a4a3b56739997edeb097284a9a6217438fd92b65399d0ecb2f23ccb1c8397496413615c26304d5272d60bc34f3ba6b5c652f693bc702be90349f27fa019173508a94bdf5f1153b341d20d040eb506ea0d40b561c0178c662a7c5b6bd130535355a8a5438ae1c0e2ab872f92800ba319b34d587e52860eb183b0bdb24ec024a111b369d3871c1c7ce9ec97207a208b4ec32ea142c61d23bdbb40325ca0756b36c5de19bed851d0ce6a30c175e467957d72b65f5552007dab6cd474d095ab556353111b9664a012d7fd767b3d549138580a087103d72bb602520f9393317e72e3fbc5d7206a24ccaae03fa844619a4acaa8c246053ed122c9765bc9164aa1c7257ac5893e0619bb965c48f24b5a5747d02f38705a09c905b82918856487382f40c28b73916b8059c29b39853189997978e49b05df1495c66a777e5f84ef99575f51b33fb157d41e54a7611c082387c6da242c3042fce7711525104f46233cad9882e45796a09c59b8471287b19c4f6205ea0b3543c341f8c169c11010cd52abda1b934024b9181c047ecad01844709d7834aa056648097f2109e4fac640d32ad80cc5581ac60296a5b51d615241c8cc0bcccfe777b97e6a5475011603a8ad7eaafe741572018840b989d83f986163633bc68919d571211d317c0a3c0347529b49101c2d451844b5083404da25b4d37931743139953fcc8374369260b88193284bc96250d02c05ecc6809a44a38e24261083dc3953d4e8a093bda800b4b2614d76a4355c6a900251e3cb9de069ed2ebaa34b34aae3661362218b5350b4f68c9bc6b11d0f5753e792cf300acdb94c52eab50d406967f498a8327420d6397886380a94019b307b8188a4496b223ebe27075b87c5d8680855ac7c5b892d357247aecb55e6089a2dba345f65d259b8d40a253a450a546b46abc3d1804f4c6ecc2a992fdff4e7d" + ], + "ss_mlkem": "22c8bf5c4858ac2e5f94b6e945ca142ccc28b98e7b954a78899630a7bfa3135f", + "session_id": "4eb19b11efa8f32faa26babc856315c4d15ce8aed78d7b211045ca2a92bb4db2", + "confirmation": "02a3575120e8cd720e0673fe02bae7bc2b03bd39", + "decrypted_length": 7974615, + "failure": "length", + "peer_decrypted_length": 11376548, + "peer_failure": "length" + } + ] +} diff --git a/src/test/fuzz/CMakeLists.txt b/src/test/fuzz/CMakeLists.txt index bbaa22cdfe..26e6304502 100644 --- a/src/test/fuzz/CMakeLists.txt +++ b/src/test/fuzz/CMakeLists.txt @@ -79,6 +79,7 @@ add_executable(fuzz p2p_handshake.cpp p2p_headers_presync.cpp p2p_transport_serialization.cpp + p2p_v2_pq_malicious_peer.cpp p2mr_script.cpp pcp.cpp package_eval.cpp @@ -92,6 +93,7 @@ add_executable(fuzz policy_estimator_io.cpp poolresource.cpp pow.cpp + pq_records.cpp pqc.cpp prevector.cpp primitives_transaction.cpp diff --git a/src/test/fuzz/bip324.cpp b/src/test/fuzz/bip324.cpp index b787bd1668..63f3970340 100644 --- a/src/test/fuzz/bip324.cpp +++ b/src/test/fuzz/bip324.cpp @@ -11,7 +11,10 @@ #include #include +#include #include +#include +#include #include namespace { @@ -43,15 +46,20 @@ FUZZ_TARGET(bip324_cipher_roundtrip, .init=Initialize) auto resp_ent = provider.ConsumeBytes(32); resp_ent.resize(32); - // Initialize ciphers by exchanging public keys. + // Initialize ciphers by exchanging public keys, optionally retaining the hybrid key state. + const bool hybrid = provider.ConsumeBool(); BIP324Cipher initiator(init_key, init_ent); assert(!initiator); BIP324Cipher responder(resp_key, resp_ent); assert(!responder); - initiator.Initialize(responder.GetOurPubKey(), true); + std::array ss_init{}; + assert(!initiator.SwitchToHybrid(ss_init)); + initiator.Initialize(responder.GetOurPubKey(), true, /*self_decrypt=*/false, hybrid); assert(initiator); - responder.Initialize(initiator.GetOurPubKey(), false); + responder.Initialize(initiator.GetOurPubKey(), false, /*self_decrypt=*/false, hybrid); assert(responder); + // Switching needs both version contents first. + assert(!initiator.SwitchToHybrid(ss_init)); // Initialize RNG deterministically, to generate contents and AAD. We assume that there are no // (potentially buggy) edge cases triggered by specific values of contents/AAD, so we can avoid @@ -63,8 +71,72 @@ FUZZ_TARGET(bip324_cipher_roundtrip, .init=Initialize) assert(std::ranges::equal(initiator.GetSessionID(), responder.GetSessionID())); assert(std::ranges::equal(initiator.GetSendGarbageTerminator(), responder.GetReceiveGarbageTerminator())); assert(std::ranges::equal(initiator.GetReceiveGarbageTerminator(), responder.GetSendGarbageTerminator())); + const std::vector garbage_terminators{initiator.GetSendGarbageTerminator().begin(), initiator.GetSendGarbageTerminator().end()}; + + // With the hybrid state retained, both sides switch keys (or discard the state, as on a legacy + // outcome) before a chosen packet. The version contents and the ML-KEM secret come from the + // RNG; the responder may hash a mutated transcript or use a mutated secret, after which every + // packet must fail to decrypt. + std::optional switch_before; + bool discard{false}; + std::vector contents_r, contents_i, resp_first, resp_second; + std::array ss_resp{}; + if (hybrid) { + switch_before = provider.ConsumeIntegralInRange(0, 1000); + contents_r = rng.randbytes(provider.ConsumeIntegralInRange(0, 4096)); + contents_i = rng.randbytes(provider.ConsumeIntegralInRange(0, 4096)); + rng.fillrand(ss_init); + resp_first = contents_r; + resp_second = contents_i; + ss_resp = ss_init; + const auto mutate = [&](std::vector& bytes) { + if (bytes.empty()) { + bytes.push_back(std::byte{0}); + } else { + bytes[provider.ConsumeIntegralInRange(0, bytes.size() - 1)] ^= std::byte(1U << provider.ConsumeIntegralInRange(0, 7)); + } + }; + switch (provider.ConsumeIntegralInRange(0, 5)) { + case 0: break; + case 1: mutate(resp_first); break; + case 2: mutate(resp_second); break; + case 3: std::swap(resp_first, resp_second); break; + case 4: ss_resp[provider.ConsumeIntegralInRange(0, ss_resp.size() - 1)] ^= std::byte(1U << provider.ConsumeIntegralInRange(0, 7)); break; + case 5: discard = true; break; + } + } else { + assert(!initiator.AddVersionContents({})); + } + const bool views_differ{resp_first != contents_r || resp_second != contents_i || ss_resp != ss_init}; + bool keys_differ{false}; + unsigned packet{0}; LIMITED_WHILE(provider.remaining_bytes(), 1000) { + if (switch_before == packet++) { + if (discard) { + initiator.DiscardHybridSecret(); + responder.DiscardHybridSecret(); + assert(!initiator.AddVersionContents(contents_r)); + assert(!responder.SwitchToHybrid(ss_resp)); + } else { + const std::vector ecdh_session_id{initiator.GetSessionID().begin(), initiator.GetSessionID().end()}; + assert(initiator.AddVersionContents(contents_r)); + assert(initiator.AddVersionContents(contents_i)); + assert(!initiator.AddVersionContents(contents_i)); + assert(initiator.SwitchToHybrid(ss_init)); + assert(responder.AddVersionContents(resp_first)); + assert(responder.AddVersionContents(resp_second)); + assert(responder.SwitchToHybrid(ss_resp)); + assert(!initiator.SwitchToHybrid(ss_init)); + assert(!responder.AddVersionContents({})); + keys_differ = views_differ; + assert(std::ranges::equal(initiator.GetSessionID(), responder.GetSessionID()) == !keys_differ); + assert(!std::ranges::equal(initiator.GetSessionID(), ecdh_session_id)); + assert(std::ranges::equal(initiator.GetSendGarbageTerminator(), garbage_terminators)); + assert(std::ranges::equal(responder.GetReceiveGarbageTerminator(), garbage_terminators)); + } + } + // Mode: // - Bit 0: whether the ignore bit is set in message // - Bit 1: whether the responder (0) or initiator (1) sends @@ -107,7 +179,8 @@ FUZZ_TARGET(bip324_cipher_roundtrip, .init=Initialize) // Decrypt length uint32_t dec_length = receiver.DecryptLength(std::span{ciphertext}.first(initiator.LENGTH_LEN)); - if (!damage) { + const bool expect_failure{damage || keys_differ}; + if (!expect_failure) { assert(dec_length == length); } else { // For performance reasons, don't try to decode if length got increased too much. @@ -120,8 +193,9 @@ FUZZ_TARGET(bip324_cipher_roundtrip, .init=Initialize) std::vector decrypt(dec_length); bool dec_ignore{false}; bool ok = receiver.Decrypt(std::span{ciphertext}.subspan(initiator.LENGTH_LEN), aad, dec_ignore, decrypt); - // Decryption *must* fail if the packet was damaged, and succeed if it wasn't. - assert(!ok == damage); + // Decryption *must* fail if the packet was damaged or the sides derived different hybrid + // keys, and succeed otherwise. + assert(!ok == expect_failure); if (!ok) break; assert(ignore == dec_ignore); assert(decrypt == contents); diff --git a/src/test/fuzz/p2p_transport_serialization.cpp b/src/test/fuzz/p2p_transport_serialization.cpp index 88432ec100..9c6bfc272f 100644 --- a/src/test/fuzz/p2p_transport_serialization.cpp +++ b/src/test/fuzz/p2p_transport_serialization.cpp @@ -2,7 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include #include +#include +#include #include #include #include @@ -13,10 +16,12 @@ #include #include +#include #include #include #include #include +#include #include namespace { @@ -336,8 +341,23 @@ std::unique_ptr MakeV1Transport(NodeId nodeid) noexcept return std::make_unique(nodeid); } +/** One side's ML-KEM entropy, SHA256(seed || counter), with the seed from the fuzzer input. */ +struct FuzzPQEntropy { + std::array seed{}; + uint32_t counter{0}; + + static void Fill(void* context, std::span out) noexcept + { + auto& self{*static_cast(context)}; + uint8_t counter_le[4]; + WriteLE32(counter_le, self.counter++); + CSHA256().Write(self.seed.data(), self.seed.size()).Write(counter_le, sizeof(counter_le)).Finalize(out.data()); + } +}; + template -std::unique_ptr MakeV2Transport(NodeId nodeid, bool initiator, RNG& rng, FuzzedDataProvider& provider) +std::unique_ptr MakeV2Transport(NodeId nodeid, bool initiator, RNG& rng, FuzzedDataProvider& provider, + FuzzPQEntropy& entropy, PQMode& mode) { // Retrieve key auto key = ConsumePrivateKey(provider); @@ -366,8 +386,15 @@ std::unique_ptr MakeV2Transport(NodeId nodeid, bool initiator, RNG& r CSHA256().Write(UCharCast(ent.data()), ent.size()) .Write(garb.data(), garb.size()) .Finalize(UCharCast(ent.data())); - - return std::make_unique(nodeid, initiator, key, ent, std::move(garb)); + // Retrieve the hybrid negotiation mode (only an initiator falls back), and the seed of this + // side's ML-KEM entropy. + mode = initiator ? provider.PickValueInArray({PQMode::OFF, PQMode::NEGOTIATE, PQMode::FALLBACK}) : + provider.PickValueInArray({PQMode::OFF, PQMode::NEGOTIATE}); + const auto pq_seed{provider.ConsumeBytes(entropy.seed.size())}; + std::ranges::copy(pq_seed, entropy.seed.begin()); + + return std::make_unique(nodeid, initiator, key, ent, std::move(garb), V2PQOptions{.mode = mode}, + PQRandomSource{.fill32 = &FuzzPQEntropy::Fill, .context = &entropy}, DefaultPQKemOps()); } } // namespace @@ -385,22 +412,32 @@ FUZZ_TARGET(p2p_transport_bidirectional, .init = initialize_p2p_transport_serial FUZZ_TARGET(p2p_transport_bidirectional_v2, .init = initialize_p2p_transport_serialization) { - // Test with two V2 transports talking to each other. + // Test with two V2 transports talking to each other, each with or without the hybrid + // negotiation. FuzzedDataProvider provider{buffer.data(), buffer.size()}; InsecureRandomContext rng(provider.ConsumeIntegral()); - auto t1 = MakeV2Transport(NodeId{0}, true, rng, provider); - auto t2 = MakeV2Transport(NodeId{1}, false, rng, provider); + FuzzPQEntropy entropy1, entropy2; + PQMode mode1, mode2; + auto t1 = MakeV2Transport(NodeId{0}, true, rng, provider, entropy1, mode1); + auto t2 = MakeV2Transport(NodeId{1}, false, rng, provider, entropy2, mode2); if (!t1 || !t2) return; SimulationTest(*t1, *t2, rng, provider); + // The keys are hybrid, and confirmed on both sides, exactly when both sides negotiate. + const bool hybrid{mode1 == PQMode::NEGOTIATE && mode2 == PQMode::NEGOTIATE}; + assert(t1->GetInfo().transport_pq == hybrid); + assert(t2->GetInfo().transport_pq == hybrid); } FUZZ_TARGET(p2p_transport_bidirectional_v1v2, .init = initialize_p2p_transport_serialization) { - // Test with a V1 initiator talking to a V2 responder. + // Test with a V1 initiator talking to a V2 responder, with or without the hybrid negotiation. FuzzedDataProvider provider{buffer.data(), buffer.size()}; InsecureRandomContext rng(provider.ConsumeIntegral()); + FuzzPQEntropy entropy; + PQMode mode; auto t1 = MakeV1Transport(NodeId{0}); - auto t2 = MakeV2Transport(NodeId{1}, false, rng, provider); + auto t2 = MakeV2Transport(NodeId{1}, false, rng, provider, entropy, mode); if (!t1 || !t2) return; SimulationTest(*t1, *t2, rng, provider); + assert(!t2->GetInfo().transport_pq); } diff --git a/src/test/fuzz/p2p_v2_pq_malicious_peer.cpp b/src/test/fuzz/p2p_v2_pq_malicious_peer.cpp new file mode 100644 index 0000000000..781ef4f292 --- /dev/null +++ b/src/test/fuzz/p2p_v2_pq_malicious_peer.cpp @@ -0,0 +1,639 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +/** How often each ML-KEM operation ran for the current input. */ +struct KemCalls { + int keygen{0}, check{0}, encaps{0}, decaps{0}; +}; +KemCalls g_kem_calls; + +const PQKemOps COUNTING_KEM_OPS{ + .keygen = [](std::span seed, mlkem::PublicKey& ek, mlkem::DecapsulationKey& dk) noexcept { + ++g_kem_calls.keygen; + return mlkem::KeyGen(seed, ek, dk); + }, + .check_public_key = [](std::span ek) noexcept { + ++g_kem_calls.check; + return mlkem::CheckPublicKey(ek); + }, + .encaps = [](std::span ek, std::span coins, + mlkem::Ciphertext& ct, mlkem::SharedSecret& ss) noexcept { + ++g_kem_calls.encaps; + return mlkem::Encaps(ek, coins, ct, ss); + }, + .decaps = [](const mlkem::DecapsulationKey& dk, std::span ct, mlkem::SharedSecret& ss) noexcept { + ++g_kem_calls.decaps; + return mlkem::Decaps(dk, ct, ss); + }, +}; + +/** ML-KEM entropy from a generator seeded by the fuzzer input. */ +struct Entropy { + InsecureRandomContext rng; + + explicit Entropy(uint64_t seed) : rng{seed} {} + + static void Fill(void* context, std::span out) noexcept + { + static_cast(context)->rng.fillrand(std::as_writable_bytes(out)); + } + + PQRandomSource Source() { return {.fill32 = &Fill, .context = this}; } +}; + +std::vector Record(uint8_t header, std::span payload) +{ + std::vector out; + VectorWriter writer{out, 0}; + WriteCompactSize(writer, 1 + payload.size()); + out.push_back(header); + out.insert(out.end(), payload.begin(), payload.end()); + return out; +} + +void Append(std::vector& out, std::span bytes) +{ + out.insert(out.end(), bytes.begin(), bytes.end()); +} + +/** What the peer's first non-decoy packet, the one the transport takes as its version packet, was. */ +enum class PeerVersion { NONE, HONEST_OFFER, HONEST_ACCEPT, OTHER }; + +/** + * A malicious peer of a V2Transport. It completes the ECDH part honestly, so every packet it sends + * authenticates, and chooses the negotiation plaintext, packet boundaries, delivery and timing. + * The peer reads everything the transport sends, and switches to hybrid keys itself when it played + * its part of the negotiation honestly. + */ +class MaliciousPeer +{ + FuzzedDataProvider& m_provider; + InsecureRandomContext m_rng; + const bool m_test_initiator; + const PQMode m_mode; + std::vector m_transport_garbage, m_peer_garbage; + Entropy m_transport_entropy, m_peer_entropy; + V2Transport m_transport; + BIP324Cipher m_cipher; + PQHandshake m_peer_pq; + + // The peer side. + std::vector m_to_transport; //!< scheduled, not yet delivered + bool m_key_sent{false}, m_terminator_sent{false}, m_sent_aad{false}; + PeerVersion m_peer_version{PeerVersion::NONE}; + std::optional m_offer; + bool m_accepted{false}; + bool m_switched{false}; + bool m_raw_sent{false}; //!< we sent unauthenticated bytes + bool m_sent_between{false}; //!< we sent a packet after our version packet and before our switch + std::optional m_confirmation_honest; //!< whether our first hybrid packet is a valid confirmation + + // What the transport sent. + std::vector m_from_transport; //!< received, not yet processed + bool m_have_key{false}, m_garbage_read{false}, m_recv_aad_used{false}, m_lost_sync{false}; + std::optional m_recv_len; + std::optional> m_transport_version; + bool m_confirmation_read{false}; + + // The transport's state as observed. + bool m_closed{false}; + std::vector m_unsent; //!< bytes GetBytesToSend reported and were not marked sent + std::optional m_expect_more, m_expect_more_next, m_last_more, m_last_more_next; + +public: + MaliciousPeer(FuzzedDataProvider& provider, bool test_initiator, PQMode mode, const CKey& transport_key, + const CKey& peer_key, size_t transport_garbage_len, size_t peer_garbage_len, uint64_t seed) + : m_provider{provider}, + m_rng{seed}, + m_test_initiator{test_initiator}, + m_mode{mode}, + m_transport_garbage{m_rng.randbytes(transport_garbage_len)}, + m_peer_garbage{m_rng.randbytes(peer_garbage_len)}, + m_transport_entropy{m_rng.rand64()}, + m_peer_entropy{m_rng.rand64()}, + m_transport{0, test_initiator, transport_key, MakeByteSpan(m_rng.rand256()), m_transport_garbage, + V2PQOptions{.mode = mode}, m_transport_entropy.Source(), COUNTING_KEM_OPS}, + m_cipher{peer_key, MakeByteSpan(m_rng.rand256())}, + m_peer_pq{!test_initiator, PQMode::NEGOTIATE, m_peer_entropy.Source(), DefaultPQKemOps()} {} + + /** Check GetBytesToSend: stable between calls, append-only, and as predicted by 'more'. */ + Transport::BytesToSend CheckBytesToSend() + { + const auto& [bytes, more, type] = m_transport.GetBytesToSend(/*have_next_message=*/false); + const auto& [bytes_next, more_next, type_next] = m_transport.GetBytesToSend(/*have_next_message=*/true); + assert(std::ranges::equal(bytes, bytes_next)); + assert(type == type_next); + if (more) assert(more_next); + assert(m_unsent.size() <= bytes.size()); + assert(std::ranges::equal(m_unsent, bytes.first(m_unsent.size()))); + if (m_expect_more) assert(!bytes.empty() == *m_expect_more); + m_unsent.assign(bytes.begin(), bytes.end()); + m_last_more = more; + m_last_more_next = more_next; + return {bytes, more, type}; + } + + /** Take up to max bytes the transport has to send, and process them as the peer. */ + void Collect(size_t max) + { + const auto& [bytes, more, type] = CheckBytesToSend(); + const size_t n{std::min(max, bytes.size())}; + if (n == 0) return; + m_from_transport.insert(m_from_transport.end(), bytes.begin(), bytes.begin() + n); + m_transport.MarkBytesSent(n); + m_unsent.erase(m_unsent.begin(), m_unsent.begin() + n); + if (m_unsent.empty()) { + m_expect_more = m_last_more; + m_expect_more_next = m_last_more_next; + } + CheckBytesToSend(); + ReadTransportOutput(); + } + + /** Deliver up to max scheduled bytes to the transport, taking every complete message. */ + void Deliver(size_t max) + { + size_t n{std::min(max, m_to_transport.size())}; + while (n > 0 && !m_closed) { + std::span bytes{std::span{m_to_transport}.first(n)}; + const bool ok{m_transport.ReceivedBytes(bytes)}; + const size_t used{n - bytes.size()}; + m_to_transport.erase(m_to_transport.begin(), m_to_transport.begin() + used); + n -= used; + // Received bytes can make bytes to send appear. + if (m_expect_more == false) m_expect_more = std::nullopt; + if (m_expect_more_next == false) m_expect_more_next = std::nullopt; + if (!ok) { + OnClose(); + return; + } + const bool message{TakeMessages()}; + if (used == 0 && !message) break; + } + } + + bool TakeMessages() + { + bool any{false}; + while (m_transport.ReceivedMessageComplete()) { + // Nothing is delivered between the switch and the peer's confirmation. + const auto snapshot{m_transport.GetPQSnapshot()}; + assert(!snapshot.switched || snapshot.confirmed); + bool reject{false}; + (void)m_transport.GetReceivedMessage({}, reject); + any = true; + } + return any; + } + + void Flush() + { + while (true) { + const size_t before{m_to_transport.size() + m_unsent.size()}; + Deliver(m_to_transport.size()); + Collect(std::numeric_limits::max()); + if (m_to_transport.size() + m_unsent.size() == before || (m_closed && m_unsent.empty())) break; + } + } + + void OnClose() + { + m_closed = true; + // A failed transport keeps no hybrid secrets, never reports hybrid keys, and a failure + // under the hybrid keys before the peer's confirmation is a confirmation failure. + assert(!m_transport.HoldsHybridSecretsForTesting()); + const auto snapshot{m_transport.GetPQSnapshot()}; + assert(!m_transport.GetInfo().transport_pq || snapshot.confirmed); + if (snapshot.switched && !snapshot.confirmed) { + assert(snapshot.failure == PQFailure::CONFIRM_LENGTH || snapshot.failure == PQFailure::CONFIRM_TAG || + snapshot.failure == PQFailure::CONFIRM_NOT_DECOY); + } + } + + /** Packets need the ciphers, and must follow our key and garbage, or the transport would take + * them as our key. */ + bool CanSendPackets() const { return m_key_sent && m_have_key; } + + /** Schedule a packet to the transport; the first one authenticates our garbage. */ + void SendPacket(std::span contents, bool ignore, PeerVersion kind = PeerVersion::OTHER) + { + std::vector packet(contents.size() + BIP324Cipher::EXPANSION); + m_cipher.Encrypt(MakeByteSpan(contents), m_sent_aad ? std::span{} : MakeByteSpan(m_peer_garbage), ignore, + MakeWritableByteSpan(packet)); + m_sent_aad = true; + if (m_peer_version != PeerVersion::NONE && !m_switched) m_sent_between = true; + if (m_switched && !m_confirmation_honest) m_confirmation_honest = contents.empty() && ignore; + if (!ignore && m_peer_version == PeerVersion::NONE) m_peer_version = kind; + Append(m_to_transport, packet); + } + + void SendKey() + { + if (m_key_sent) return; + Append(m_to_transport, MakeUCharSpan(m_cipher.GetOurPubKey())); + Append(m_to_transport, m_peer_garbage); + m_key_sent = true; + } + + void SendTerminator() + { + if (!m_key_sent || !m_have_key || m_terminator_sent) return; + Append(m_to_transport, MakeUCharSpan(m_cipher.GetSendGarbageTerminator())); + m_terminator_sent = true; + } + + void SendDecoy(uint8_t kind) + { + if (!CanSendPackets()) return; + std::vector contents; + switch (kind) { + case 0: contents = m_rng.randbytes(m_rng.randrange(64)); break; + case 1: contents = Record(PQ_MLKEM1024, m_rng.randbytes(mlkem::CIPHERTEXT_BYTES)); break; + case 2: break; + default: contents = m_rng.randbytes(m_rng.randrange(2000)); break; + } + SendPacket(contents, /*ignore=*/true); + } + + /** The peer's version packet: an offer (the transport initiates) or an accept, honest or not. */ + void SendVersion(uint8_t variant) + { + if (!CanSendPackets()) return; + const bool first{m_peer_version == PeerVersion::NONE}; + std::vector contents; + PeerVersion kind{PeerVersion::OTHER}; + mlkem::SharedSecret ss; + bool switch_after{false}; + const auto random_record = [&](size_t size) { return Record(PQ_MLKEM1024, m_rng.randbytes(size)); }; + if (m_test_initiator) { + // We are the responder: offer variants. + const auto offer = [&]() -> std::span { + if (!m_offer) { + PQHandshake::Record record; + assert(m_peer_pq.MakeOffer(record) == mlkem::Error::NONE); + m_offer = record; + } + return *m_offer; + }; + switch (variant) { + case 0: break; + case 1: contents = m_rng.randbytes(m_rng.randrange(2000)); break; + case 2: contents = random_record(m_rng.randrange(1600)); break; + case 3: contents = Record(PQ_MLKEM1024, std::vector(mlkem::PUBLIC_KEY_BYTES, 0xff)); break; + case 4: Append(contents, MakeUCharSpan(offer())); Append(contents, std::vector{0xfd, 0x00}); break; + case 5: Append(contents, MakeUCharSpan(offer())); kind = PeerVersion::HONEST_OFFER; break; + case 6: + Append(contents, Record(0x01, m_rng.randbytes(m_rng.randrange(20)))); + Append(contents, MakeUCharSpan(offer())); + Append(contents, random_record(m_rng.randrange(20))); + kind = PeerVersion::HONEST_OFFER; + break; + case 7: Append(contents, Record(0xf1, m_rng.randbytes(m_rng.randrange(20)))); break; + case 8: contents = random_record(mlkem::PUBLIC_KEY_BYTES - 1); break; + default: + Append(contents, MakeUCharSpan(offer())); + Append(contents, random_record(mlkem::PUBLIC_KEY_BYTES)); + kind = PeerVersion::HONEST_OFFER; + break; + } + // A transport that sent its version packet before ours does not negotiate, so we + // have already dropped the transcript. + if (first && !m_transport_version) assert(m_cipher.AddVersionContents(MakeByteSpan(contents))); + } else { + // We are the initiator: accept variants, answering the transport's offer if it made one. + PQHandshake::Record accept; + bool have_accept{false}; + if (!m_accepted && m_transport_version && first) { + const auto parsed{PQHandshake::ParseContents(MakeByteSpan(*m_transport_version))}; + if (parsed.kind == PQHandshake::ParseKind::OWN_RECORD) { + const auto ek{m_peer_pq.CheckRecordLength(parsed.payload)}; + assert(ek); + assert(m_peer_pq.AcceptOffer(*ek, accept, ss) == mlkem::Error::NONE); + m_accepted = have_accept = true; + } + } + switch (variant) { + case 0: break; + case 1: contents = m_rng.randbytes(m_rng.randrange(2000)); break; + case 2: contents = random_record(m_rng.randrange(1600)); break; + case 3: contents = random_record(mlkem::CIPHERTEXT_BYTES); break; // blind + case 4: + if (have_accept) { + Append(contents, MakeUCharSpan(accept)); + contents[4 + m_rng.randrange(mlkem::CIPHERTEXT_BYTES)] ^= 1 << m_rng.randrange(8); + } + break; + case 5: + case 6: + if (have_accept) { + if (variant == 6) Append(contents, Record(0x02, m_rng.randbytes(m_rng.randrange(20)))); + Append(contents, MakeUCharSpan(accept)); + if (variant == 6) Append(contents, random_record(m_rng.randrange(20))); + kind = PeerVersion::HONEST_ACCEPT; + switch_after = true; + } + break; + case 7: + // An honest accept, but we never switch: a non-compliant peer. + if (have_accept) Append(contents, MakeUCharSpan(accept)); + break; + case 8: contents = random_record(mlkem::CIPHERTEXT_BYTES + 1); break; + default: + if (have_accept) { + Append(contents, MakeUCharSpan(accept)); + Append(contents, std::vector{0xfe, 0x00}); + } + break; + } + if (switch_after) { + assert(m_cipher.AddVersionContents(MakeByteSpan(*m_transport_version))); + assert(m_cipher.AddVersionContents(MakeByteSpan(contents))); + } + } + SendPacket(contents, /*ignore=*/false, kind); + if (switch_after && first) { + assert(m_cipher.SwitchToHybrid(std::as_bytes(ss.Bytes()))); + m_switched = true; + } + } + + void SendConfirmation(uint8_t variant) + { + if (!CanSendPackets()) return; + switch (variant) { + case 0: SendPacket({}, /*ignore=*/true); break; + case 1: SendPacket(m_rng.randbytes(1 + m_rng.randrange(30)), /*ignore=*/true); break; + case 2: SendPacket({}, /*ignore=*/false); break; + default: { + const bool first_hybrid{m_switched && !m_confirmation_honest}; + SendPacket({}, /*ignore=*/true); + if (first_hybrid) m_confirmation_honest = false; + // Damage the length (variant 3) or the header and tag (otherwise). + const size_t pos{m_to_transport.size() - BIP324Cipher::EXPANSION + + (variant == 3 ? m_rng.randrange(BIP324Cipher::LENGTH_LEN) : BIP324Cipher::LENGTH_LEN + m_rng.randrange(17))}; + m_to_transport[pos] ^= 1 << m_rng.randrange(8); + break; + } + } + } + + void SendApplication(uint8_t variant) + { + if (!CanSendPackets()) return; + switch (variant) { + case 0: SendPacket(std::vector{18, 1, 2, 3, 4, 5, 6, 7, 8}, /*ignore=*/false); break; // ping + case 1: { + std::vector contents(1 + CMessageHeader::MESSAGE_TYPE_SIZE); + std::ranges::copy(std::string_view{"foobar"}, contents.begin() + 1); + SendPacket(contents, /*ignore=*/false); + break; + } + case 2: SendPacket({}, /*ignore=*/false); break; + default: SendPacket(m_rng.randbytes(m_rng.randrange(100)), /*ignore=*/false); break; + } + } + + /** Unauthenticated bytes, after our key (bytes in place of the key would only make the ECDH + * part dishonest). */ + void SendRaw(uint8_t size) + { + if (!m_key_sent) return; + Append(m_to_transport, m_rng.randbytes(size)); + m_raw_sent = true; + } + + /** Give the transport a message to send; refused messages stay untouched. */ + void QueueMessage(uint8_t variant) + { + static const std::array TYPES{"ping", "pong", "version", "barfoo"}; + CSerializedNetMsg msg; + msg.m_type = TYPES[variant % TYPES.size()]; + msg.data = m_rng.randbytes(m_rng.randrange(200)); + const CSerializedNetMsg copy{msg.Copy()}; + const auto snapshot{m_transport.GetPQSnapshot()}; + const bool holding{m_test_initiator && m_mode == PQMode::NEGOTIATE && !snapshot.version_received}; + const bool queued{m_transport.SetMessageToSend(msg)}; + if (!queued) assert(msg.m_type == copy.m_type && msg.data == copy.data); + // An initiator holding its version packet takes no message. + if (holding) assert(!queued); + m_expect_more = m_expect_more_next; + m_expect_more_next = std::nullopt; + CheckBytesToSend(); + } + + /** Process what the transport sent: its key, garbage and terminator, then its packets. */ + void ReadTransportOutput() + { + if (!m_have_key) { + if (m_from_transport.size() < EllSwiftPubKey::size()) return; + m_cipher.Initialize(EllSwiftPubKey{MakeByteSpan(m_from_transport).first(EllSwiftPubKey::size())}, !m_test_initiator, + /*self_decrypt=*/false, /*retain_for_hybrid=*/true); + m_from_transport.erase(m_from_transport.begin(), m_from_transport.begin() + EllSwiftPubKey::size()); + m_have_key = true; + } + if (!m_garbage_read) { + const size_t len{m_transport_garbage.size() + BIP324Cipher::GARBAGE_TERMINATOR_LEN}; + if (m_from_transport.size() < len) return; + assert(std::ranges::equal(std::span{m_from_transport}.first(m_transport_garbage.size()), m_transport_garbage)); + assert(std::ranges::equal(MakeByteSpan(m_from_transport).subspan(m_transport_garbage.size(), BIP324Cipher::GARBAGE_TERMINATOR_LEN), + m_cipher.GetReceiveGarbageTerminator())); + m_from_transport.erase(m_from_transport.begin(), m_from_transport.begin() + len); + m_garbage_read = true; + } + while (!m_lost_sync) { + if (!m_recv_len) { + if (m_from_transport.size() < BIP324Cipher::LENGTH_LEN) return; + m_recv_len = m_cipher.DecryptLength(MakeByteSpan(m_from_transport).first(BIP324Cipher::LENGTH_LEN)); + // Key agreement: when both sides switched, the transport's first hybrid packet is its + // confirmation, so its length decrypts to 0 under our keys. + if (m_switched && !m_confirmation_read && m_transport.GetPQSnapshot().switched) assert(*m_recv_len == 0); + } + const size_t size{*m_recv_len + BIP324Cipher::EXPANSION}; + if (m_from_transport.size() < size) return; + std::vector contents(*m_recv_len); + bool ignore{false}; + const bool ok{m_cipher.Decrypt(MakeByteSpan(m_from_transport).subspan(BIP324Cipher::LENGTH_LEN, size - BIP324Cipher::LENGTH_LEN), + m_recv_aad_used ? std::span{} : MakeByteSpan(m_transport_garbage), ignore, + MakeWritableByteSpan(contents))}; + m_recv_aad_used = true; + m_recv_len.reset(); + m_from_transport.erase(m_from_transport.begin(), m_from_transport.begin() + size); + if (!ok) { + // Only keys that differ can fail: one side switched and the other did not. + assert(m_switched != m_transport.GetPQSnapshot().switched); + m_lost_sync = true; + return; + } + if (!m_transport_version) { + // The transport sends no decoy before its version packet. + assert(!ignore); + m_transport_version = contents; + const auto parsed{PQHandshake::ParseContents(MakeByteSpan(contents))}; + const bool own_record{parsed.kind == PQHandshake::ParseKind::OWN_RECORD}; + if (!m_test_initiator) { + // A responder offers at once exactly when it negotiates. + assert(own_record == (m_mode == PQMode::NEGOTIATE)); + } else { + // An initiator accepts exactly an honest offer, with the negotiation on. + assert(own_record == (m_mode == PQMode::NEGOTIATE && m_peer_version == PeerVersion::HONEST_OFFER)); + if (own_record) { + assert(m_cipher.AddVersionContents(MakeByteSpan(contents))); + const auto ct{m_peer_pq.CheckRecordLength(parsed.payload)}; + assert(ct); + mlkem::SharedSecret ss; + assert(m_peer_pq.DecapsulateAccept(*ct, ss) == mlkem::Error::NONE); + assert(m_cipher.SwitchToHybrid(std::as_bytes(ss.Bytes()))); + m_switched = true; + } else { + m_cipher.DiscardHybridSecret(); + } + } + } else if (m_switched && !m_confirmation_read) { + // The transport's first packet under the hybrid keys is its key confirmation. + assert(ignore && contents.empty()); + m_confirmation_read = true; + } else { + // The transport sends no other decoys. + assert(!ignore); + } + } + } + + /** Invariants that hold between any two steps. */ + void CheckState() + { + const auto info{m_transport.GetInfo()}; + if (info.transport_pq_status == PQStatus::V1) return; + const auto snapshot{m_transport.GetPQSnapshot()}; + assert(info.transport_pq == snapshot.confirmed); + if (snapshot.confirmed) { + // Only a peer with the same keys can have produced a valid confirmation. + assert(m_switched && info.transport_pq_status == PQStatus::HYBRID); + assert(info.session_id && std::ranges::equal(MakeUCharSpan(*info.session_id), MakeUCharSpan(m_cipher.GetSessionID()))); + } + if (snapshot.switched && !snapshot.confirmed) { + assert(info.transport_type == TransportProtocolType::DETECTING && !info.session_id); + assert(info.transport_pq_status == PQStatus::PENDING); + } + switch (m_mode) { + case PQMode::OFF: + case PQMode::FALLBACK: + // Neither generates, parses nor validates anything. + assert(!snapshot.switched && snapshot.offer == PQOfferState::NONE); + assert(g_kem_calls.keygen + g_kem_calls.check + g_kem_calls.encaps + g_kem_calls.decaps == 0); + assert(info.transport_pq_status == (m_mode == PQMode::OFF ? PQStatus::OFF : PQStatus::FALLBACK)); + break; + case PQMode::NEGOTIATE: + assert(g_kem_calls.keygen <= (m_test_initiator ? 0 : 1)); + assert(g_kem_calls.check + g_kem_calls.encaps <= (m_test_initiator ? 2 : 0)); + // A responder decapsulates only an accept to the offer it sent. + assert(g_kem_calls.decaps <= ((!m_test_initiator && snapshot.offer == PQOfferState::SENT) ? 1 : 0)); + break; + } + // Once the peer's version packet is processed, or the transport failed, no hybrid secret + // remains. + if (snapshot.version_received || m_closed) assert(!m_transport.HoldsHybridSecretsForTesting()); + } + + /** Run the fuzzer's script of steps, then deliver and take everything left. */ + void Run() + { + LIMITED_WHILE(m_provider.remaining_bytes(), 300) + { + bool stop{false}; + CallOneOf( + m_provider, + [&] { SendKey(); }, + [&] { SendTerminator(); }, + [&] { SendDecoy(m_provider.ConsumeIntegralInRange(0, 3)); }, + [&] { SendVersion(m_provider.ConsumeIntegralInRange(0, 9)); }, + [&] { SendConfirmation(m_provider.ConsumeIntegralInRange(0, 4)); }, + [&] { SendApplication(m_provider.ConsumeIntegralInRange(0, 3)); }, + [&] { Deliver(m_provider.ConsumeIntegralInRange(1, 1 << 16)); }, + [&] { Collect(m_provider.ConsumeIntegralInRange(1, 1 << 16)); }, + [&] { Flush(); }, + [&] { QueueMessage(m_provider.ConsumeIntegralInRange(0, 3)); }, + [&] { SendRaw(m_provider.ConsumeIntegralInRange(1, 64)); }, + // The peer disconnects. + [&] { stop = true; }); + CheckState(); + if (stop) return; + } + Flush(); + CheckState(); + // An honest exchange with a valid confirmation, delivered in full, confirms (later damaged + // packets may still close the connection). + const auto snapshot{m_transport.GetPQSnapshot()}; + if (m_switched && snapshot.switched && !m_raw_sent && !m_sent_between && m_confirmation_honest == true) { + assert(snapshot.confirmed); + } + } +}; + +CKey KeyFrom(FuzzedDataProvider& provider) +{ + auto bytes{provider.ConsumeBytes(32)}; + bytes.resize(32); + CKey key; + key.Set(bytes.begin(), bytes.end(), /*fCompressedIn=*/true); + return key; +} + +void initialize_p2p_v2_pq_malicious_peer() +{ + static ECC_Context ecc_context{}; + SelectParams(ChainType::REGTEST); +} + +} // namespace + +FUZZ_TARGET(p2p_v2_pq_malicious_peer, .init = initialize_p2p_v2_pq_malicious_peer) +{ + FuzzedDataProvider provider{buffer.data(), buffer.size()}; + g_kem_calls = {}; + // The tested transport's role and mode (a responder never falls back), both keys, both + // garbage lengths and the seed of everything else that is random. + const uint8_t flags{provider.ConsumeIntegral()}; + const bool test_initiator{(flags & 1) != 0}; + const PQMode mode{test_initiator ? std::array{PQMode::OFF, PQMode::NEGOTIATE, PQMode::FALLBACK}[(flags >> 1) % 3] : + std::array{PQMode::OFF, PQMode::NEGOTIATE}[(flags >> 1) % 2]}; + const CKey transport_key{KeyFrom(provider)}; + const CKey peer_key{KeyFrom(provider)}; + if (!transport_key.IsValid() || !peer_key.IsValid()) return; + const size_t transport_garbage_len{provider.ConsumeIntegralInRange(0, V2Transport::MAX_GARBAGE_LEN)}; + const size_t peer_garbage_len{provider.ConsumeIntegralInRange(0, V2Transport::MAX_GARBAGE_LEN)}; + const uint64_t seed{provider.ConsumeIntegral()}; + MaliciousPeer peer{provider, test_initiator, mode, transport_key, peer_key, transport_garbage_len, peer_garbage_len, seed}; + peer.Run(); +} diff --git a/src/test/fuzz/pq_records.cpp b/src/test/fuzz/pq_records.cpp new file mode 100644 index 0000000000..6d4ae13ffa --- /dev/null +++ b/src/test/fuzz/pq_records.cpp @@ -0,0 +1,96 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +/** What the version-contents grammar says about an input, computed without the serialization code. */ +struct Expected { + bool valid{false}; + //! Offset and length of the first PQ_MLKEM1024 record's payload. + std::optional> first; +}; + +/** + * An independent reading of the record grammar: CompactSize(len) || header || payload, len >= 1, + * with the minimal CompactSize encoding and no bytes left over. Lengths above MAX_SIZE need more + * input than any fuzz input has, so "longer than what is left" covers them. + */ +Expected Oracle(std::span in) +{ + Expected expected; + size_t pos{0}; + while (pos < in.size()) { + const uint8_t tag{in[pos++]}; + uint64_t len{tag}; + if (tag >= 0xfd) { + const size_t width{tag == 0xfd ? 2U : tag == 0xfe ? 4U : 8U}; + const uint64_t minimum{tag == 0xfd ? 0xfdU : tag == 0xfe ? 0x10000U : 0x100000000U}; + if (in.size() - pos < width) return {}; + len = 0; + for (size_t i{0}; i < width; ++i) len |= uint64_t{in[pos + i]} << (8 * i); + pos += width; + if (len < minimum) return {}; + } + if (len == 0 || len > in.size() - pos) return {}; + if (in[pos] == PQ_MLKEM1024 && !expected.first) expected.first = {pos + 1, size_t(len - 1)}; + pos += size_t(len); + } + expected.valid = true; + return expected; +} + +void CheckParse(std::span in) +{ + const auto contents{std::as_bytes(in)}; + const auto parsed{PQHandshake::ParseContents(contents)}; + const Expected expected{Oracle(in)}; + if (!expected.valid) { + assert(parsed.kind == PQHandshake::ParseKind::INVALID_GRAMMAR); + assert(parsed.payload.empty()); + } else if (!expected.first) { + assert(parsed.kind == PQHandshake::ParseKind::NO_RECORD); + assert(parsed.payload.empty()); + } else { + assert(parsed.kind == PQHandshake::ParseKind::OWN_RECORD); + // The payload points into the contents, at the first own record. + assert(parsed.payload.data() == contents.data() + expected.first->first); + assert(parsed.payload.size() == expected.first->second); + } +} + +} // namespace + +FUZZ_TARGET(pq_records) +{ + CheckParse(buffer); + + // A record built from the first payload bytes parses back to them, and as the first record of + // longer contents it wins if, and only if, the rest is valid grammar. + if (buffer.size() < mlkem::PUBLIC_KEY_BYTES) return; + const auto record{PQHandshake::SerializeRecord(buffer.first())}; + const auto alone{PQHandshake::ParseContents(record)}; + assert(alone.kind == PQHandshake::ParseKind::OWN_RECORD); + assert(alone.payload.data() == record.data() + 4); + assert(std::ranges::equal(alone.payload, std::as_bytes(buffer.first()))); + + std::vector combined(record.size() + buffer.size()); + std::ranges::copy(MakeUCharSpan(record), combined.begin()); + std::ranges::copy(buffer, combined.begin() + record.size()); + CheckParse(combined); + const auto parsed{PQHandshake::ParseContents(std::as_bytes(std::span{combined}))}; + assert((parsed.kind == PQHandshake::ParseKind::OWN_RECORD) == Oracle(buffer).valid); +} diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 48197b2235..5f0c37ea2a 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -2,10 +2,16 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include + +#include #include #include #include #include +#include +#include +#include #include #include #include @@ -24,12 +30,16 @@ #include #include +#include #include +#include +#include #include #include #include #include +#include using namespace std::literals; using namespace util::hex_literals; @@ -1054,6 +1064,7 @@ class V2TransportTester V2Transport m_transport; //!< V2Transport being tested BIP324Cipher m_cipher; //!< Cipher to help with the other side bool m_test_initiator; //!< Whether m_transport is the initiator (true) or responder (false) + PQHandshake m_peer_pq; //!< The other side's hybrid negotiation state std::vector m_sent_garbage; //!< The garbage we've sent to m_transport. std::vector m_recv_garbage; //!< The garbage we've received from m_transport. @@ -1064,11 +1075,21 @@ class V2TransportTester public: /** Construct a tester object. test_initiator: whether the tested transport is initiator. */ - explicit V2TransportTester(FastRandomContext& rng, bool test_initiator) + explicit V2TransportTester(FastRandomContext& rng, bool test_initiator, V2PQOptions pq = {}) : m_rng{rng}, - m_transport{0, test_initiator}, + m_transport{0, test_initiator, pq}, m_cipher{GenerateRandomTestKey(m_rng), MakeByteSpan(m_rng.rand256())}, - m_test_initiator(test_initiator) {} + m_test_initiator(test_initiator), + m_peer_pq{!test_initiator, PQMode::NEGOTIATE} {} + + /** Construct a tester object whose transport uses the given ML-KEM entropy and operations. */ + V2TransportTester(FastRandomContext& rng, bool test_initiator, V2PQOptions pq, PQRandomSource random, const PQKemOps& ops) + : m_rng{rng}, + m_transport{0, test_initiator, GenerateRandomTestKey(rng), MakeByteSpan(rng.rand256()), + rng.randbytes(rng.randrange(V2Transport::MAX_GARBAGE_LEN + 1)), pq, random, ops}, + m_cipher{GenerateRandomTestKey(m_rng), MakeByteSpan(m_rng.rand256())}, + m_test_initiator(test_initiator), + m_peer_pq{!test_initiator, PQMode::NEGOTIATE} {} /** Data type returned by Interact: * @@ -1135,6 +1156,43 @@ class V2TransportTester /** Expose the cipher. */ BIP324Cipher& GetCipher() { return m_cipher; } + /** Expose the transport being tested. */ + V2Transport& GetTransport() { return m_transport; } + + /** The bytes scheduled to be sent to the transport. */ + std::vector& ToSend() { return m_to_send; } + + /** The bytes received from the transport and not processed yet. */ + std::vector& Received() { return m_received; } + + /** Deliver scheduled bytes to the transport, all in one call or one byte per call. Returns false + * (leaving the rest scheduled) at the first transport error. */ + bool Deliver(bool byte_by_byte = false) + { + while (!m_to_send.empty()) { + std::span to_send{m_to_send}; + if (byte_by_byte) to_send = to_send.first(1); + const size_t old_len{to_send.size()}; + const bool ret{m_transport.ReceivedBytes(to_send)}; + m_to_send.erase(m_to_send.begin(), m_to_send.begin() + (old_len - to_send.size())); + if (!ret) return false; + BOOST_REQUIRE(to_send.empty() || m_transport.ReceivedMessageComplete()); + if (!to_send.empty()) return true; + } + return true; + } + + /** Take every byte the transport has to send. */ + void Collect() + { + while (true) { + const auto& [bytes, _more, _msg_type] = m_transport.GetBytesToSend(false); + if (bytes.empty()) break; + m_received.insert(m_received.end(), bytes.begin(), bytes.end()); + m_transport.MarkBytesSent(bytes.size()); + } + } + /** Schedule bytes to be sent to the transport. */ void Send(std::span data) { @@ -1192,12 +1250,13 @@ class V2TransportTester * Many other V2TransportTester functions cannot be called until after ReceiveKey() has been * called, as no encryption keys are set up before that point. */ - void ReceiveKey() + void ReceiveKey(bool retain_for_hybrid = false) { // When processing a key, enough bytes need to have been received already. BOOST_REQUIRE(m_received.size() >= EllSwiftPubKey::size()); // Initialize the cipher using it (acting as the opposite side of the tested transport). - m_cipher.Initialize(MakeByteSpan(m_received).first(EllSwiftPubKey::size()), !m_test_initiator); + m_cipher.Initialize(MakeByteSpan(m_received).first(EllSwiftPubKey::size()), !m_test_initiator, + /*self_decrypt=*/false, retain_for_hybrid); // Strip the processed bytes off the front of the receive buffer. m_received.erase(m_received.begin(), m_received.begin() + EllSwiftPubKey::size()); } @@ -1296,6 +1355,78 @@ class V2TransportTester BOOST_CHECK(contents.empty()); } + /** Expect a version packet to have been received, process it and return its contents (only + * after ReceiveKey). */ + std::vector ReceiveVersionContents() + { + return ReceivePacket(/*aad=*/MakeByteSpan(m_recv_garbage)); + } + + /** Expect a key confirmation: exactly one empty decoy packet (only after a switch). */ + void ReceiveConfirmation() + { + BOOST_REQUIRE(m_received.size() >= PQ_CONFIRMATION_BYTES); + const auto packet{MakeByteSpan(std::span{m_received}.first(PQ_CONFIRMATION_BYTES))}; + BOOST_CHECK_EQUAL(m_cipher.DecryptLength(packet.first(BIP324Cipher::LENGTH_LEN)), 0U); + bool ignore{false}; + BOOST_CHECK(m_cipher.Decrypt(packet.subspan(BIP324Cipher::LENGTH_LEN), /*aad=*/{}, ignore, /*contents=*/{})); + BOOST_CHECK(ignore); + m_received.erase(m_received.begin(), m_received.begin() + PQ_CONFIRMATION_BYTES); + } + + /** Schedule our key confirmation, an empty decoy (only after a switch). */ + void SendConfirmation() { SendPacket(/*content=*/{}, /*aad=*/{}, /*ignore=*/true); } + + /** As a hybrid responder (after ReceiveKey(true) and SendGarbageTerm): schedule our version + * packet with an offer, followed by extra contents. */ + void SendOffer(std::span extra = {}) + { + PQHandshake::Record offer; + BOOST_REQUIRE(m_peer_pq.MakeOffer(offer) == mlkem::Error::NONE); + std::vector contents{offer.begin(), offer.end()}; + contents.insert(contents.end(), extra.begin(), extra.end()); + BOOST_REQUIRE(m_cipher.AddVersionContents(contents)); + SendVersion(MakeUCharSpan(contents)); + } + + /** As a hybrid responder: process the transport's version packet, which must carry a valid + * accept, and switch keys; invert_ss inverts byte 0 of our ML-KEM secret first. */ + void ReceiveAcceptAndSwitch(bool invert_ss = false) + { + const auto contents{ReceiveVersionContents()}; + BOOST_REQUIRE(m_cipher.AddVersionContents(MakeByteSpan(contents))); + const auto parsed{PQHandshake::ParseContents(MakeByteSpan(contents))}; + BOOST_REQUIRE(parsed.kind == PQHandshake::ParseKind::OWN_RECORD); + const auto ct{m_peer_pq.CheckRecordLength(parsed.payload)}; + BOOST_REQUIRE(ct); + mlkem::SharedSecret ss; + BOOST_REQUIRE(m_peer_pq.DecapsulateAccept(*ct, ss) == mlkem::Error::NONE); + if (invert_ss) ss.Bytes()[0] ^= 0xff; + BOOST_REQUIRE(m_cipher.SwitchToHybrid(std::as_bytes(ss.Bytes()))); + } + + /** As a hybrid initiator (after ReceiveKey(true) and SendGarbageTerm): process the transport's + * version packet, which must carry a valid offer, schedule our version packet with the accept + * under the ECDH keys, and switch keys. damage_ct flips a ciphertext bit after hashing it; + * invert_ss inverts byte 0 of our ML-KEM secret before switching. */ + void AcceptOfferAndSwitch(bool damage_ct = false, bool invert_ss = false) + { + const auto contents{ReceiveVersionContents()}; + BOOST_REQUIRE(m_cipher.AddVersionContents(MakeByteSpan(contents))); + const auto parsed{PQHandshake::ParseContents(MakeByteSpan(contents))}; + BOOST_REQUIRE(parsed.kind == PQHandshake::ParseKind::OWN_RECORD); + const auto ek{m_peer_pq.CheckRecordLength(parsed.payload)}; + BOOST_REQUIRE(ek); + PQHandshake::Record accept; + mlkem::SharedSecret ss; + BOOST_REQUIRE(m_peer_pq.AcceptOffer(*ek, accept, ss) == mlkem::Error::NONE); + BOOST_REQUIRE(m_cipher.AddVersionContents(accept)); + if (damage_ct) accept[4 + m_rng.randrange(mlkem::CIPHERTEXT_BYTES)] ^= std::byte(1 << m_rng.randrange(8)); + SendVersion(MakeUCharSpan(accept)); + if (invert_ss) ss.Bytes()[0] ^= 0xff; + BOOST_REQUIRE(m_cipher.SwitchToHybrid(std::as_bytes(ss.Bytes()))); + } + /** Expect application packet to have been received, with specified short id and payload. * (only after ReceiveKey). */ void ReceiveMessage(uint8_t short_id, std::span payload) @@ -1438,8 +1569,11 @@ BOOST_AUTO_TEST_CASE(v2transport_test) BOOST_CHECK(!ret); } - // Various valid but unusual scenarios. - for (int i = 0; i < 50; ++i) { + // Various valid but unusual scenarios. The last 50 have the hybrid negotiation on, with a legacy + // peer. + for (int i = 0; i < 100; ++i) { + /** Whether the tested transport negotiates hybrid keys. */ + const bool pq{i >= 50}; /** Whether an initiator or responder is being tested. */ bool initiator = m_rng.randbool(); /** Use either 0 bytes or the maximum possible (4095 bytes) garbage length. */ @@ -1448,11 +1582,15 @@ BOOST_AUTO_TEST_CASE(v2transport_test) unsigned num_ignore_version = m_rng.randrange(10); /** What data to send in the version packet (ignored by BIP324 peers, but reserved for future extensions). */ auto ver_data = m_rng.randbytes(m_rng.randbool() ? 0 : m_rng.randrange(1000)); + // A legacy peer's random contents never carry an own record. + while (pq && PQHandshake::ParseContents(MakeByteSpan(ver_data)).kind == PQHandshake::ParseKind::OWN_RECORD) { + ver_data = m_rng.randbytes(m_rng.randrange(1000)); + } /** Whether to immediately send key and garbage out (required for responders, optional otherwise). */ bool send_immediately = !initiator || m_rng.randbool(); /** How many decoy packets to send before the first and second real message. */ unsigned num_decoys_1 = m_rng.randrange(1000), num_decoys_2 = m_rng.randrange(1000); - V2TransportTester tester(m_rng, initiator); + V2TransportTester tester(m_rng, initiator, {.mode = pq ? PQMode::NEGOTIATE : PQMode::OFF}); if (send_immediately) { tester.SendKey(); tester.SendGarbage(garb_len); @@ -1474,7 +1612,13 @@ BOOST_AUTO_TEST_CASE(v2transport_test) ret = tester.Interact(); BOOST_REQUIRE(ret && ret->empty()); tester.ReceiveGarbage(); - tester.ReceiveVersion(); + if (pq && !initiator) { + // The tested responder offered; the legacy peer ignores the offer. + const auto contents{tester.ReceiveVersionContents()}; + BOOST_CHECK(PQHandshake::ParseContents(MakeByteSpan(contents)).kind == PQHandshake::ParseKind::OWN_RECORD); + } else { + tester.ReceiveVersion(); + } tester.CompareSessionIDs(); for (unsigned d = 0; d < num_decoys_1; ++d) { auto decoy_data = m_rng.randbytes(m_rng.randrange(1000)); @@ -1499,6 +1643,9 @@ BOOST_AUTO_TEST_CASE(v2transport_test) BOOST_CHECK(!(*ret)[2]); BOOST_CHECK((*ret)[3] && (*ret)[3]->m_type == "foobar" && (*ret)[3]->m_recv.empty()); tester.ReceiveMessage("barfoo", {}); + const auto info{tester.GetTransport().GetInfo()}; + BOOST_CHECK(!info.transport_pq); + BOOST_CHECK(info.transport_pq_status == (pq ? PQStatus::LEGACY_PEER : PQStatus::OFF)); } // Too long garbage (initiator). @@ -1583,4 +1730,1460 @@ BOOST_AUTO_TEST_CASE(v2transport_test) } } +namespace { + +/** Deterministic PQ entropy: the queued 32-byte values in order, then SHA256(seed || counter). */ +struct TestPQEntropy { + std::deque> queued; + uint256 seed; + uint64_t counter{0}; + + explicit TestPQEntropy(const uint256& seed_in) : seed{seed_in} {} + + static void Fill(void* context, std::span out) noexcept + { + auto& self{*static_cast(context)}; + if (!self.queued.empty()) { + std::ranges::copy(self.queued.front(), out.begin()); + self.queued.pop_front(); + return; + } + uint8_t counter_le[8]; + WriteLE64(counter_le, self.counter++); + CSHA256().Write(self.seed.begin(), self.seed.size()).Write(counter_le, sizeof(counter_le)).Finalize(out.data()); + } + + PQRandomSource Source() { return {.fill32 = &Fill, .context = this}; } + + void Queue(std::span bytes) + { + BOOST_REQUIRE(bytes.size() % 32 == 0); + for (size_t i{0}; i < bytes.size(); i += 32) { + std::array value; + std::ranges::copy(bytes.subspan(i, 32), value.begin()); + queued.push_back(value); + } + } +}; + +/** How often each ML-KEM operation ran through COUNTING_KEM_OPS. */ +struct KemCallCounts { + int keygen{0}, check{0}, encaps{0}, decaps{0}; + int total() const { return keygen + check + encaps + decaps; } +}; +KemCallCounts g_kem_calls; + +const PQKemOps COUNTING_KEM_OPS{ + .keygen = [](std::span seed, mlkem::PublicKey& ek, mlkem::DecapsulationKey& dk) noexcept { + ++g_kem_calls.keygen; + return mlkem::KeyGen(seed, ek, dk); + }, + .check_public_key = [](std::span ek) noexcept { + ++g_kem_calls.check; + return mlkem::CheckPublicKey(ek); + }, + .encaps = [](std::span ek, std::span coins, + mlkem::Ciphertext& ct, mlkem::SharedSecret& ss) noexcept { + ++g_kem_calls.encaps; + return mlkem::Encaps(ek, coins, ct, ss); + }, + .decaps = [](const mlkem::DecapsulationKey& dk, std::span ct, mlkem::SharedSecret& ss) noexcept { + ++g_kem_calls.decaps; + return mlkem::Decaps(dk, ct, ss); + }, +}; + +std::vector HexBytes(std::string_view hex) +{ + auto bytes{TryParseHex(hex)}; + BOOST_REQUIRE(bytes); + return std::move(*bytes); +} + +std::vector Concat(std::initializer_list> parts) +{ + std::vector out; + for (const auto& part : parts) out.insert(out.end(), part.begin(), part.end()); + return out; +} + +/** A record CompactSize(1 + payload size) || header || payload. */ +std::vector MakeRecord(uint8_t header, std::span payload) +{ + DataStream stream; + WriteCompactSize(stream, 1 + payload.size()); + stream << header; + stream.write(payload); + return {stream.begin(), stream.end()}; +} + +/** A valid encapsulation key, from a fixed seed. */ +mlkem::PublicKey TestEncapsulationKey() +{ + std::array seed{}; + seed.fill(0x5a); + mlkem::PublicKey ek; + mlkem::DecapsulationKey dk; + BOOST_REQUIRE(mlkem::KeyGen(seed, ek, dk) == mlkem::Error::NONE); + return ek; +} + +/** ek with its first coefficient set to q = 3329, which fails the modulus check. */ +mlkem::PublicKey BadModulusKey(mlkem::PublicKey ek) +{ + ek[0] = 0x01; + ek[1] = (ek[1] & 0xF0) | 0x0D; + return ek; +} + +void CheckParse(std::span contents, PQHandshake::ParseKind kind, std::optional> payload = std::nullopt) +{ + const auto parsed{PQHandshake::ParseContents(contents)}; + BOOST_CHECK(parsed.kind == kind); + if (payload) { + BOOST_CHECK(parsed.payload.data() == contents.data() + payload->first); + BOOST_CHECK_EQUAL(parsed.payload.size(), payload->second); + } else { + BOOST_CHECK(parsed.payload.empty()); + } +} + +} // namespace + +BOOST_AUTO_TEST_CASE(pq_records_grammar) +{ + using enum PQHandshake::ParseKind; + // Empty contents, as every BIP324 implementation sends today: no features. + CheckParse({}, NO_RECORD); + // Unknown and reserved headers, with and without payload, are ignored. + CheckParse(HexBytes("0100"), NO_RECORD); + CheckParse(HexBytes("0301aabb"), NO_RECORD); + CheckParse(HexBytes("0100" "02f1aa" "01ff"), NO_RECORD); + // An own record with an empty payload parses; its length is checked later, by the role. + CheckParse(HexBytes("01f0"), OWN_RECORD, std::pair{2, 0}); + CheckParse(HexBytes("0201aa" "03f0bbcc"), OWN_RECORD, std::pair{5, 2}); + // len = 0, alone or after valid records. + CheckParse(HexBytes("00"), INVALID_GRAMMAR); + CheckParse(HexBytes("0201aa" "00"), INVALID_GRAMMAR); + CheckParse(HexBytes("00" "01f0"), INVALID_GRAMMAR); + // Truncated records and truncated CompactSize encodings. + CheckParse(HexBytes("05f0aabb"), INVALID_GRAMMAR); + CheckParse(HexBytes("02f0"), INVALID_GRAMMAR); + CheckParse(HexBytes("fd"), INVALID_GRAMMAR); + CheckParse(HexBytes("fd21"), INVALID_GRAMMAR); + CheckParse(HexBytes("fe010000"), INVALID_GRAMMAR); + CheckParse(HexBytes("ff01000000000000"), INVALID_GRAMMAR); + // Non-canonical CompactSize encodings of lengths that would otherwise fit. + CheckParse(HexBytes("fd0200" "f0aa"), INVALID_GRAMMAR); + CheckParse(HexBytes("fe02000000" "f0aa"), INVALID_GRAMMAR); + CheckParse(HexBytes("ff0200000000000000" "f0aa"), INVALID_GRAMMAR); + // Oversized lengths: beyond the contents (one byte, and far), exactly MAX_SIZE (which + // ReadCompactSize accepts), beyond MAX_SIZE (which it rejects), and beyond size_t on 32-bit + // platforms. None is used to skip bytes before it is checked. + static_assert(MAX_SIZE == 0x02000000); + CheckParse(HexBytes("03f0aa"), INVALID_GRAMMAR); + CheckParse(HexBytes("fdfd00" "f0aa"), INVALID_GRAMMAR); + CheckParse(HexBytes("fe00000002" "f0aa"), INVALID_GRAMMAR); + CheckParse(HexBytes("fe01000002" "f0aa"), INVALID_GRAMMAR); + CheckParse(HexBytes("ffffffffffffffffff" "f0aa"), INVALID_GRAMMAR); + CheckParse(HexBytes("ff0000000001000000" "f0aa"), INVALID_GRAMMAR); + // Trailing corruption after valid records, including after a valid own record. + CheckParse(HexBytes("0201aa" "07"), INVALID_GRAMMAR); + CheckParse(HexBytes("02f0aa" "fd"), INVALID_GRAMMAR); + CheckParse(HexBytes("02f0aa" "0301aa"), INVALID_GRAMMAR); + // Canonical 3-byte CompactSize lengths parse. + std::vector long_unknown{HexBytes("fdfd00" "01")}; + long_unknown.resize(long_unknown.size() + 0xfc); + CheckParse(long_unknown, NO_RECORD); +} + +BOOST_AUTO_TEST_CASE(pq_records_first_wins) +{ + using enum PQHandshake::ParseKind; + const mlkem::PublicKey ek{TestEncapsulationKey()}; + const auto offer{PQHandshake::SerializeRecord(ek)}; + BOOST_CHECK_EQUAL(HexStr(std::span{offer}.first(4)), "fd2106f0"); + BOOST_CHECK(std::ranges::equal(std::span{offer}.subspan(4), std::as_bytes(std::span{ek}))); + const auto malformed{MakeRecord(PQ_MLKEM1024, HexBytes("aabb"))}; + const auto unknown{MakeRecord(0xf1, HexBytes("cc"))}; + + // A valid first own record wins over a malformed duplicate, wherever unknown records are. + CheckParse(offer, OWN_RECORD, std::pair{4, mlkem::PUBLIC_KEY_BYTES}); + CheckParse(Concat({offer, malformed}), OWN_RECORD, std::pair{4, mlkem::PUBLIC_KEY_BYTES}); + CheckParse(Concat({unknown, offer, unknown, malformed}), OWN_RECORD, std::pair{unknown.size() + 4, mlkem::PUBLIC_KEY_BYTES}); + // A malformed first own record wins over a valid second one; the role's length check then fails. + CheckParse(Concat({malformed, offer}), OWN_RECORD, std::pair{2, 2}); + CheckParse(Concat({unknown, malformed, offer}), OWN_RECORD, std::pair{unknown.size() + 2, 2}); + // Any later framing corruption means no features, even after a valid own record. + CheckParse(Concat({offer, HexBytes("00")}), INVALID_GRAMMAR); + CheckParse(Concat({offer, unknown, HexBytes("05f0")}), INVALID_GRAMMAR); + CheckParse(Concat({offer, HexBytes("fd0300f0aabb")}), INVALID_GRAMMAR); + auto truncated{Concat({unknown, offer})}; + truncated.pop_back(); + CheckParse(truncated, INVALID_GRAMMAR); +} + +BOOST_AUTO_TEST_CASE(pq_record_semantics) +{ + const mlkem::PublicKey ek{TestEncapsulationKey()}; + for (const size_t size : {size_t{0}, mlkem::PUBLIC_KEY_BYTES - 1, mlkem::PUBLIC_KEY_BYTES + 1}) { + std::vector payload(size, std::byte{0x42}); + // The responder's offer, at the initiator. + PQHandshake initiator{/*initiating=*/true, PQMode::NEGOTIATE}; + const auto contents{MakeRecord(PQ_MLKEM1024, payload)}; + const auto parsed{PQHandshake::ParseContents(contents)}; + BOOST_REQUIRE(parsed.kind == PQHandshake::ParseKind::OWN_RECORD); + BOOST_CHECK(!initiator.CheckRecordLength(parsed.payload)); + BOOST_CHECK(initiator.GetSnapshot().failure == PQFailure::EK_LENGTH); + BOOST_CHECK(initiator.GetSnapshot().offer == PQOfferState::NONE); + // The initiator's accept, at the responder, which wipes its key. + PQHandshake responder{/*initiating=*/false, PQMode::NEGOTIATE}; + PQHandshake::Record offer; + BOOST_REQUIRE(responder.MakeOffer(offer) == mlkem::Error::NONE); + BOOST_CHECK(!responder.CheckRecordLength(parsed.payload)); + BOOST_CHECK(responder.GetSnapshot().failure == PQFailure::CT_LENGTH); + BOOST_CHECK(!responder.HasDecapsulationKey()); + } + + // A key of the right length that fails the modulus check. + g_kem_calls = {}; + TestPQEntropy entropy{uint256::ONE}; + PQHandshake initiator{/*initiating=*/true, PQMode::NEGOTIATE, entropy.Source(), COUNTING_KEM_OPS}; + const auto contents{PQHandshake::SerializeRecord(BadModulusKey(ek))}; + const auto payload{initiator.CheckRecordLength(PQHandshake::ParseContents(contents).payload)}; + BOOST_REQUIRE(payload); + PQHandshake::Record accept; + accept.fill(std::byte{0x33}); + mlkem::SharedSecret ss; + BOOST_CHECK(initiator.AcceptOffer(*payload, accept, ss) == mlkem::Error::INVALID_PUBLIC_KEY); + BOOST_CHECK(initiator.GetSnapshot().failure == PQFailure::EK_MODULUS); + BOOST_CHECK(initiator.GetSnapshot().offer == PQOfferState::RECEIVED); + BOOST_CHECK(std::ranges::all_of(accept, [](std::byte b) { return b == std::byte{0x33}; })); + BOOST_CHECK_EQUAL(g_kem_calls.check, 1); + BOOST_CHECK_EQUAL(g_kem_calls.total(), 1); + // No entropy was drawn for a rejected key. + BOOST_CHECK_EQUAL(entropy.counter, 0U); +} + +BOOST_AUTO_TEST_CASE(pq_records_max_contents) +{ + using enum PQHandshake::ParseKind; + // The largest contents a v2 packet can carry (1 + 12 + 4,000,000 bytes), as minimal unknown + // records. One pass, no copies, no exception. + constexpr size_t MAX_CONTENTS{4'000'013}; + std::vector contents; + contents.reserve(MAX_CONTENTS); + while (contents.size() + 2 < MAX_CONTENTS) { + contents.push_back(std::byte{0x01}); + contents.push_back(std::byte{0x01}); + } + // Ending in a malformed record: no features. + std::vector malformed{contents}; + malformed.push_back(std::byte{0x05}); + BOOST_REQUIRE_EQUAL(malformed.size(), MAX_CONTENTS); + CheckParse(malformed, INVALID_GRAMMAR); + // Ending in a valid unknown record: still no features. + std::vector valid{contents.begin(), contents.end() - 2}; + for (const auto b : HexBytes("0201aa")) valid.push_back(b); + BOOST_REQUIRE_EQUAL(valid.size(), MAX_CONTENTS); + CheckParse(valid, NO_RECORD); + // Ending in an own record: found at the very end. + const auto offer{PQHandshake::SerializeRecord(TestEncapsulationKey())}; + std::vector own{HexBytes("0201aa")}; + while (own.size() + 2 + offer.size() <= MAX_CONTENTS) { + own.push_back(std::byte{0x01}); + own.push_back(std::byte{0x01}); + } + own.insert(own.end(), offer.begin(), offer.end()); + BOOST_REQUIRE_EQUAL(own.size(), MAX_CONTENTS); + CheckParse(own, OWN_RECORD, std::pair{MAX_CONTENTS - mlkem::PUBLIC_KEY_BYTES, mlkem::PUBLIC_KEY_BYTES}); +} + +BOOST_AUTO_TEST_CASE(pq_handshake_negotiation) +{ + UniValue doc; + BOOST_REQUIRE(doc.read(json_tests::pq_transport_vectors)); + const UniValue& vector{doc["vectors"][0]}; + const auto keygen_seed{ParseHex(vector["mlkem_keygen_seed"].get_str())}; + const auto encaps_m{ParseHex(vector["mlkem_encaps_m"].get_str())}; + + // With the vector's entropy, MakeOffer draws d then z and AcceptOffer draws m, so the offer, + // the accept and both secrets are the vector's. + { + TestPQEntropy responder_entropy{uint256::ZERO}, initiator_entropy{uint256::ONE}; + responder_entropy.Queue(keygen_seed); + initiator_entropy.Queue(encaps_m); + PQHandshake responder{/*initiating=*/false, PQMode::NEGOTIATE, responder_entropy.Source(), DefaultPQKemOps()}; + PQHandshake initiator{/*initiating=*/true, PQMode::NEGOTIATE, initiator_entropy.Source(), DefaultPQKemOps()}; + + PQHandshake::Record offer; + BOOST_REQUIRE(responder.MakeOffer(offer) == mlkem::Error::NONE); + BOOST_CHECK_EQUAL(HexStr(offer), "fd2106f0" + vector["ek"].get_str()); + BOOST_CHECK(responder.HasDecapsulationKey()); + // SENT means queued, which only the caller knows. + BOOST_CHECK(responder.GetSnapshot().offer == PQOfferState::NONE); + responder.SetOfferSent(); + BOOST_CHECK(responder.GetSnapshot().offer == PQOfferState::SENT); + BOOST_CHECK(responder_entropy.queued.empty()); + + const auto ek{initiator.CheckRecordLength(PQHandshake::ParseContents(offer).payload)}; + BOOST_REQUIRE(ek); + PQHandshake::Record accept; + mlkem::SharedSecret ss_initiator; + BOOST_REQUIRE(initiator.AcceptOffer(*ek, accept, ss_initiator) == mlkem::Error::NONE); + BOOST_CHECK_EQUAL(HexStr(accept), "fd2106f0" + vector["ct"].get_str()); + BOOST_CHECK_EQUAL(HexStr(ss_initiator.Bytes()), vector["ss_mlkem"].get_str()); + BOOST_CHECK(initiator.GetSnapshot().offer == PQOfferState::RECEIVED); + BOOST_CHECK(initiator.GetSnapshot().failure == PQFailure::NONE); + + const auto ct{responder.CheckRecordLength(PQHandshake::ParseContents(accept).payload)}; + BOOST_REQUIRE(ct); + mlkem::SharedSecret ss_responder; + BOOST_REQUIRE(responder.DecapsulateAccept(*ct, ss_responder) == mlkem::Error::NONE); + BOOST_CHECK_EQUAL(HexStr(ss_responder.Bytes()), vector["ss_mlkem"].get_str()); + // The decapsulation key is single use. + BOOST_CHECK(!responder.HasDecapsulationKey()); + BOOST_CHECK(responder.GetSnapshot().failure == PQFailure::NONE); + } + + // Random production entropy: both sides agree, and every handshake has fresh keys. + std::optional> previous; + for (int i = 0; i < 2; ++i) { + PQHandshake responder{/*initiating=*/false, PQMode::NEGOTIATE}; + PQHandshake initiator{/*initiating=*/true, PQMode::NEGOTIATE}; + PQHandshake::Record offer, accept; + mlkem::SharedSecret ss_initiator, ss_responder; + BOOST_REQUIRE(responder.MakeOffer(offer) == mlkem::Error::NONE); + BOOST_REQUIRE(initiator.AcceptOffer(*initiator.CheckRecordLength(PQHandshake::ParseContents(offer).payload), accept, ss_initiator) == mlkem::Error::NONE); + BOOST_REQUIRE(responder.DecapsulateAccept(*responder.CheckRecordLength(PQHandshake::ParseContents(accept).payload), ss_responder) == mlkem::Error::NONE); + BOOST_CHECK(std::ranges::equal(ss_initiator.Bytes(), ss_responder.Bytes())); + std::vector current(offer.size()); + std::ranges::copy(MakeUCharSpan(offer), current.begin()); + if (previous) BOOST_CHECK(*previous != current); + previous = current; + } + + // A corrupted ciphertext of the right length is not an error: it yields a different secret. + { + PQHandshake responder{/*initiating=*/false, PQMode::NEGOTIATE}; + PQHandshake initiator{/*initiating=*/true, PQMode::NEGOTIATE}; + PQHandshake::Record offer, accept; + mlkem::SharedSecret ss_initiator, ss_responder; + BOOST_REQUIRE(responder.MakeOffer(offer) == mlkem::Error::NONE); + BOOST_REQUIRE(initiator.AcceptOffer(*initiator.CheckRecordLength(PQHandshake::ParseContents(offer).payload), accept, ss_initiator) == mlkem::Error::NONE); + accept[100] ^= std::byte{0x01}; + BOOST_REQUIRE(responder.DecapsulateAccept(*responder.CheckRecordLength(PQHandshake::ParseContents(accept).payload), ss_responder) == mlkem::Error::NONE); + BOOST_CHECK(!std::ranges::equal(ss_initiator.Bytes(), ss_responder.Bytes())); + } + + using mlkem::InjectResultForTesting; + using mlkem::Operation; + namespace upstream = mlkem::upstream; + const auto all_bytes_are = [](std::span bytes, uint8_t value) { + return std::ranges::all_of(bytes, [&](uint8_t b) { return b == value; }); + }; + + // Key generation fails: no offer, no key, a recorded internal error. + { + PQHandshake responder{/*initiating=*/false, PQMode::NEGOTIATE}; + PQHandshake::Record offer; + offer.fill(std::byte{0x33}); + { + InjectResultForTesting inject{Operation::KEYGEN, upstream::ERR_FAIL}; + BOOST_CHECK(responder.MakeOffer(offer) == mlkem::Error::INTERNAL); + } + BOOST_CHECK(std::ranges::all_of(offer, [](std::byte b) { return b == std::byte{0x33}; })); + BOOST_CHECK(!responder.HasDecapsulationKey()); + BOOST_CHECK(responder.GetSnapshot().offer == PQOfferState::NONE); + BOOST_CHECK(responder.GetSnapshot().failure == PQFailure::KEYGEN_INTERNAL); + } + + // The key check or encapsulation fails internally: no accept, the secret cleared. + for (const auto& [op, failure] : {std::pair{Operation::CHECK_PUBLIC_KEY, PQFailure::CHECK_EK_INTERNAL}, + std::pair{Operation::ENCAPS, PQFailure::ENCAPS_INTERNAL}}) { + for (const int result : {upstream::ERR_FAIL, upstream::ERR_OUT_OF_MEMORY}) { + PQHandshake initiator{/*initiating=*/true, PQMode::NEGOTIATE}; + const mlkem::PublicKey ek{TestEncapsulationKey()}; + PQHandshake::Record accept; + accept.fill(std::byte{0x33}); + mlkem::SharedSecret ss; + std::ranges::fill(ss.Bytes(), 0x44); + { + InjectResultForTesting inject{op, result}; + BOOST_CHECK(initiator.AcceptOffer(ek, accept, ss) == mlkem::Error::INTERNAL); + } + BOOST_CHECK(std::ranges::all_of(accept, [](std::byte b) { return b == std::byte{0x33}; })); + BOOST_CHECK(all_bytes_are(ss.Bytes(), 0)); + BOOST_CHECK(initiator.GetSnapshot().failure == failure); + } + } + // An encapsulation that rejects the key after the check passed is a local fault too. + { + PQHandshake initiator{/*initiating=*/true, PQMode::NEGOTIATE}; + PQHandshake::Record accept; + mlkem::SharedSecret ss; + InjectResultForTesting inject{Operation::ENCAPS, upstream::ERR_INVALID_PK}; + BOOST_CHECK(initiator.AcceptOffer(TestEncapsulationKey(), accept, ss) == mlkem::Error::INTERNAL); + BOOST_CHECK(initiator.GetSnapshot().failure == PQFailure::ENCAPS_INTERNAL); + } + + // Decapsulation fails internally (including a failed key hash check): the secret cleared, the + // key wiped. + for (const int result : {upstream::ERR_FAIL, upstream::ERR_INVALID_SK}) { + PQHandshake responder{/*initiating=*/false, PQMode::NEGOTIATE}; + PQHandshake initiator{/*initiating=*/true, PQMode::NEGOTIATE}; + PQHandshake::Record offer, accept; + mlkem::SharedSecret ss_initiator, ss_responder; + BOOST_REQUIRE(responder.MakeOffer(offer) == mlkem::Error::NONE); + BOOST_REQUIRE(initiator.AcceptOffer(*initiator.CheckRecordLength(PQHandshake::ParseContents(offer).payload), accept, ss_initiator) == mlkem::Error::NONE); + std::ranges::fill(ss_responder.Bytes(), 0x44); + { + InjectResultForTesting inject{Operation::DECAPS, result}; + BOOST_CHECK(responder.DecapsulateAccept(*responder.CheckRecordLength(PQHandshake::ParseContents(accept).payload), ss_responder) == mlkem::Error::INTERNAL); + } + BOOST_CHECK(all_bytes_are(ss_responder.Bytes(), 0)); + BOOST_CHECK(!responder.HasDecapsulationKey()); + BOOST_CHECK(responder.GetSnapshot().failure == PQFailure::DECAPS_INTERNAL); + } + + // ClearSecrets() and any recorded failure wipe the key; the first failure is kept. + { + PQHandshake responder{/*initiating=*/false, PQMode::NEGOTIATE}; + PQHandshake::Record offer; + BOOST_REQUIRE(responder.MakeOffer(offer) == mlkem::Error::NONE); + responder.ClearSecrets(); + BOOST_CHECK(!responder.HasDecapsulationKey()); + BOOST_CHECK(responder.GetSnapshot().failure == PQFailure::NONE); + } + { + PQHandshake responder{/*initiating=*/false, PQMode::NEGOTIATE}; + PQHandshake::Record offer; + BOOST_REQUIRE(responder.MakeOffer(offer) == mlkem::Error::NONE); + responder.SetFailure(PQFailure::CONFIRM_TAG); + BOOST_CHECK(!responder.HasDecapsulationKey()); + responder.SetFailure(PQFailure::CONFIRM_LENGTH); + BOOST_CHECK(responder.GetSnapshot().failure == PQFailure::CONFIRM_TAG); + } + + // The snapshot records progress. + { + PQHandshake initiator{/*initiating=*/true, PQMode::NEGOTIATE}; + BOOST_CHECK(!initiator.GetSnapshot().version_received); + initiator.SetVersionReceived(); + initiator.SetSwitched(); + initiator.SetConfirmed(); + const auto snapshot{initiator.GetSnapshot()}; + BOOST_CHECK(snapshot.version_received && snapshot.switched && snapshot.confirmed); + BOOST_CHECK(snapshot.failure == PQFailure::NONE); + } +} + +namespace { + +constexpr V2PQOptions PQ_ON{.mode = PQMode::NEGOTIATE}; + +/** Drive a tester, as the hybrid peer, through the handshake with its transport: both sides + * switch, and the tester checks the transport's key confirmation. The tester's own confirmation + * is scheduled (not delivered) if send_confirmation. Returns the ECDH session id. */ +uint256 HybridHandshake(V2TransportTester& tester, bool test_initiator, bool send_confirmation = true) +{ + uint256 ecdh_session_id; + if (test_initiator) { + // The transport initiates and holds its version packet; we (the responder) offer at once. + auto ret{tester.Interact()}; + BOOST_REQUIRE(ret && ret->empty()); + tester.SendKey(); + tester.SendGarbage(); + tester.ReceiveKey(/*retain_for_hybrid=*/true); + ecdh_session_id = uint256(MakeUCharSpan(tester.GetCipher().GetSessionID())); + tester.SendGarbageTerm(); + tester.SendOffer(); + ret = tester.Interact(); + BOOST_REQUIRE(ret && ret->empty()); + tester.ReceiveGarbage(); + tester.ReceiveAcceptAndSwitch(); + } else { + // The transport responds with an offer; we (the initiator) accept it. + tester.SendKey(); + tester.SendGarbage(); + auto ret{tester.Interact()}; + BOOST_REQUIRE(ret && ret->empty()); + tester.ReceiveKey(/*retain_for_hybrid=*/true); + ecdh_session_id = uint256(MakeUCharSpan(tester.GetCipher().GetSessionID())); + tester.SendGarbageTerm(); + tester.ReceiveGarbage(); + tester.AcceptOfferAndSwitch(); + ret = tester.Interact(); + BOOST_REQUIRE(ret && ret->empty()); + } + tester.ReceiveConfirmation(); + if (send_confirmation) tester.SendConfirmation(); + return ecdh_session_id; +} + +void CheckPQ(V2Transport& transport, PQStatus status, PQFailure failure = PQFailure::NONE) +{ + const auto info{transport.GetInfo()}; + BOOST_CHECK(info.transport_pq_status == status); + BOOST_CHECK_EQUAL(info.transport_pq, status == PQStatus::HYBRID); + BOOST_CHECK(transport.GetPQSnapshot().failure == failure); +} + +/** Exchange count messages each way between two connected transports, fragmenting the bytes at + * random, and check that each arrives intact and in order. */ +void ExchangeMessages(FastRandomContext& rng, V2Transport& initiator, V2Transport& responder, int count) +{ + const std::array transports{&initiator, &responder}; + std::array, 2> to_send; + std::array, 2> expected; + std::array, 2> in_flight; + for (int side = 0; side < 2; ++side) { + for (int i = 0; i < count; ++i) { + CSerializedNetMsg msg; + msg.m_type = i == 0 ? "version" : "ping"; + msg.data = rng.randbytes(rng.randrange(1000)); + expected[side].push_back(msg.Copy()); + to_send[side].push_back(std::move(msg)); + } + } + bool progress{true}; + while (progress) { + progress = false; + for (int side = 0; side < 2; ++side) { + if (!to_send[side].empty() && transports[side]->SetMessageToSend(to_send[side].front())) { + to_send[side].pop_front(); + progress = true; + } + const auto& [bytes, _more, _type] = transports[side]->GetBytesToSend(!to_send[side].empty()); + if (!bytes.empty()) { + const size_t n{1 + rng.randrange(bytes.size())}; + in_flight[side].insert(in_flight[side].end(), bytes.begin(), bytes.begin() + n); + transports[side]->MarkBytesSent(n); + progress = true; + } + if (!in_flight[side].empty()) { + std::span received{std::span{in_flight[side]}.first(1 + rng.randrange(in_flight[side].size()))}; + const size_t old_size{received.size()}; + BOOST_REQUIRE(transports[!side]->ReceivedBytes(received)); + in_flight[side].erase(in_flight[side].begin(), in_flight[side].begin() + (old_size - received.size())); + if (received.size() != old_size) progress = true; + } + while (transports[!side]->ReceivedMessageComplete()) { + bool reject{false}; + const CNetMessage msg{transports[!side]->GetReceivedMessage({}, reject)}; + BOOST_REQUIRE(!reject && !expected[side].empty()); + BOOST_CHECK_EQUAL(msg.m_type, expected[side].front().m_type); + BOOST_CHECK(std::ranges::equal(MakeUCharSpan(msg.m_recv), expected[side].front().data)); + expected[side].pop_front(); + progress = true; + } + } + } + BOOST_CHECK(expected[0].empty() && expected[1].empty()); +} + +} // namespace + +BOOST_AUTO_TEST_CASE(v2_pq_pair) +{ + for (const bool test_initiator : {true, false}) { + BOOST_TEST_CONTEXT("test_initiator=" << test_initiator) + { + V2TransportTester tester(m_rng, test_initiator, PQ_ON); + auto& transport{tester.GetTransport()}; + // Pending from the start of the handshake. + CheckPQ(transport, PQStatus::PENDING); + BOOST_CHECK(!transport.GetPQSnapshot().version_received); + const uint256 ecdh_session_id{HybridHandshake(tester, test_initiator, /*send_confirmation=*/false)}; + + // Switched, not confirmed: pending, still detecting, and no session id. The switch + // wiped the decapsulation key and the retained ECDH secret. + auto info{transport.GetInfo()}; + BOOST_CHECK(info.transport_type == TransportProtocolType::DETECTING); + BOOST_CHECK(!info.session_id); + CheckPQ(transport, PQStatus::PENDING); + auto snapshot{transport.GetPQSnapshot()}; + BOOST_CHECK(snapshot.version_received && snapshot.switched && !snapshot.confirmed); + BOOST_CHECK(snapshot.offer == (test_initiator ? PQOfferState::RECEIVED : PQOfferState::SENT)); + BOOST_CHECK(!transport.HoldsHybridSecretsForTesting()); + + // The transport sends application messages without waiting for our confirmation. + const auto msg_data_1{m_rng.randbytes(m_rng.randrange(100000))}; + tester.AddMessage("barfoo", msg_data_1); + auto ret{tester.Interact()}; + BOOST_REQUIRE(ret && ret->empty()); + tester.ReceiveMessage("barfoo", msg_data_1); + + // It delivers ours only after our confirmation. + const auto msg_data_2{m_rng.randbytes(m_rng.randrange(100000))}; + tester.SendConfirmation(); + tester.SendMessage(uint8_t(4), msg_data_2); // cmpctblock short id + tester.SendMessage("tx", msg_data_1); + ret = tester.Interact(); + BOOST_REQUIRE(ret && ret->size() == 2); + BOOST_CHECK((*ret)[0] && (*ret)[0]->m_type == "cmpctblock" && std::ranges::equal((*ret)[0]->m_recv, MakeByteSpan(msg_data_2))); + BOOST_CHECK((*ret)[1] && (*ret)[1]->m_type == "tx" && std::ranges::equal((*ret)[1]->m_recv, MakeByteSpan(msg_data_1))); + + // Hybrid: v2 with the hybrid session id, which both sides share. + info = transport.GetInfo(); + BOOST_CHECK(info.transport_type == TransportProtocolType::V2); + CheckPQ(transport, PQStatus::HYBRID); + tester.CompareSessionIDs(); + BOOST_CHECK(info.session_id && *info.session_id != ecdh_session_id); + snapshot = transport.GetPQSnapshot(); + BOOST_CHECK(snapshot.switched && snapshot.confirmed); + } + } +} + +BOOST_AUTO_TEST_CASE(v2_pq_legacy_matrix) +{ + // The tested transport (on, off or in fallback) against a scripted peer (hybrid or legacy), in + // both roles. A legacy peer sends its version packet and an application burst as soon as it + // can, without waiting for ours. + for (const bool test_initiator : {true, false}) { + for (const PQMode local : {PQMode::OFF, PQMode::NEGOTIATE, PQMode::FALLBACK}) { + // Inbound connections never fall back. + if (local == PQMode::FALLBACK && !test_initiator) continue; + for (const bool peer_hybrid : {false, true}) { + BOOST_TEST_CONTEXT("test_initiator=" << test_initiator << " local=" << int(local) << " peer_hybrid=" << peer_hybrid) + { + g_kem_calls = {}; + TestPQEntropy entropy{m_rng.rand256()}; + V2TransportTester tester(m_rng, test_initiator, {.mode = local}, entropy.Source(), COUNTING_KEM_OPS); + auto& transport{tester.GetTransport()}; + const bool hybrid{local == PQMode::NEGOTIATE && peer_hybrid}; + const bool offered{local == PQMode::NEGOTIATE && !test_initiator}; + const auto burst_1{m_rng.randbytes(m_rng.randrange(10000))}; + const auto burst_2{m_rng.randbytes(m_rng.randrange(1000))}; + const auto send_burst = [&] { + tester.SendMessage(uint8_t(14), burst_1); // inv short id + tester.SendMessage("foobar", burst_2); + }; + // Until the peer's version packet is processed, the negotiation is pending. + const PQStatus initial_status{local == PQMode::OFF ? PQStatus::OFF : + local == PQMode::FALLBACK ? PQStatus::FALLBACK : PQStatus::PENDING}; + CheckPQ(transport, initial_status); + + if (test_initiator) { + auto ret{tester.Interact()}; + BOOST_REQUIRE(ret && ret->empty()); + } + tester.SendKey(); + tester.SendGarbage(); + if (!test_initiator) { + auto ret{tester.Interact()}; + BOOST_REQUIRE(ret && ret->empty()); + } + tester.ReceiveKey(/*retain_for_hybrid=*/peer_hybrid); + tester.SendGarbageTerm(); + // After the key exchange, before the peer's version packet: still pending, and + // only the negotiation holds secrets for the switch. + auto ret{tester.Interact()}; + BOOST_REQUIRE(ret && ret->empty()); + CheckPQ(transport, initial_status); + BOOST_CHECK(!transport.GetPQSnapshot().version_received); + BOOST_CHECK_EQUAL(transport.HoldsHybridSecretsForTesting(), local == PQMode::NEGOTIATE); + if (!peer_hybrid) { + tester.SendVersion(); + send_burst(); + } else if (test_initiator) { + // A hybrid responder offers at once. + tester.SendOffer(); + } + ret = tester.Interact(); + BOOST_REQUIRE(ret); + BOOST_CHECK_EQUAL(ret->size(), peer_hybrid ? 0U : 2U); + tester.ReceiveGarbage(); + if (!peer_hybrid) { + // A legacy peer ignores an offer. + const auto contents{tester.ReceiveVersionContents()}; + BOOST_CHECK(offered ? PQHandshake::ParseContents(MakeByteSpan(contents)).kind == PQHandshake::ParseKind::OWN_RECORD : contents.empty()); + } else if (test_initiator) { + if (hybrid) { + tester.ReceiveAcceptAndSwitch(); + tester.ReceiveConfirmation(); + tester.SendConfirmation(); + } else { + BOOST_CHECK(tester.ReceiveVersionContents().empty()); + tester.GetCipher().DiscardHybridSecret(); + } + send_burst(); + } else { + // A hybrid initiator answers the responder's version packet. + if (hybrid) { + tester.AcceptOfferAndSwitch(); + tester.SendConfirmation(); + } else { + BOOST_CHECK(tester.ReceiveVersionContents().empty()); + tester.GetCipher().DiscardHybridSecret(); + tester.SendVersion(); + } + send_burst(); + } + if (peer_hybrid) { + ret = tester.Interact(); + BOOST_REQUIRE(ret && ret->size() == 2); + if (hybrid && !test_initiator) tester.ReceiveConfirmation(); + } + BOOST_CHECK((*ret)[0] && (*ret)[0]->m_type == "inv" && std::ranges::equal((*ret)[0]->m_recv, MakeByteSpan(burst_1))); + BOOST_CHECK((*ret)[1] && (*ret)[1]->m_type == "foobar" && std::ranges::equal((*ret)[1]->m_recv, MakeByteSpan(burst_2))); + + // The transport's messages arrive under the keys the peer expects. + tester.AddMessage("barfoo", burst_2); + ret = tester.Interact(); + BOOST_REQUIRE(ret && ret->empty()); + tester.ReceiveMessage("barfoo", burst_2); + BOOST_CHECK(tester.Received().empty()); + tester.CompareSessionIDs(); + const PQStatus status{local == PQMode::OFF ? PQStatus::OFF : + local == PQMode::FALLBACK ? PQStatus::FALLBACK : + hybrid ? PQStatus::HYBRID : PQStatus::LEGACY_PEER}; + CheckPQ(transport, status); + // The switch and a legacy outcome both wiped what the switch needed. + BOOST_CHECK(!transport.HoldsHybridSecretsForTesting()); + + // Off and fallback neither generate, parse nor validate anything; a responder + // never decapsulates without an accept. + BOOST_CHECK_EQUAL(g_kem_calls.keygen, offered ? 1 : 0); + BOOST_CHECK_EQUAL(g_kem_calls.check, hybrid && test_initiator ? 1 : 0); + BOOST_CHECK_EQUAL(g_kem_calls.encaps, hybrid && test_initiator ? 1 : 0); + BOOST_CHECK_EQUAL(g_kem_calls.decaps, hybrid && !test_initiator ? 1 : 0); + } + } + } + } + + // Two transports in every combination of modes. + for (const PQMode initiator_mode : {PQMode::OFF, PQMode::NEGOTIATE, PQMode::FALLBACK}) { + for (const PQMode responder_mode : {PQMode::OFF, PQMode::NEGOTIATE}) { + BOOST_TEST_CONTEXT("initiator=" << int(initiator_mode) << " responder=" << int(responder_mode)) + { + V2Transport initiator{0, /*initiating=*/true, {.mode = initiator_mode}}; + V2Transport responder{1, /*initiating=*/false, {.mode = responder_mode}}; + ExchangeMessages(m_rng, initiator, responder, 5); + const auto initiator_info{initiator.GetInfo()}, responder_info{responder.GetInfo()}; + BOOST_CHECK(initiator_info.transport_type == TransportProtocolType::V2); + BOOST_CHECK(initiator_info.session_id && initiator_info.session_id == responder_info.session_id); + const bool hybrid{initiator_mode == PQMode::NEGOTIATE && responder_mode == PQMode::NEGOTIATE}; + const auto expected = [&](PQMode mode) { + if (mode == PQMode::OFF) return PQStatus::OFF; + if (mode == PQMode::FALLBACK) return PQStatus::FALLBACK; + return hybrid ? PQStatus::HYBRID : PQStatus::LEGACY_PEER; + }; + CheckPQ(initiator, expected(initiator_mode)); + CheckPQ(responder, expected(responder_mode)); + BOOST_CHECK(!initiator.HoldsHybridSecretsForTesting()); + BOOST_CHECK(!responder.HoldsHybridSecretsForTesting()); + } + } + } + + // A responder never falls back: its FALLBACK is a caller bug (Assume), and it runs and reports + // as off. Debug builds abort on the Assume instead. + if constexpr (!G_FUZZING_BUILD && !G_ABORT_ON_FAILED_ASSUME) { + V2Transport initiator{0, /*initiating=*/true, PQ_ON}; + V2Transport responder{1, /*initiating=*/false, {.mode = PQMode::FALLBACK}}; + ExchangeMessages(m_rng, initiator, responder, 2); + CheckPQ(initiator, PQStatus::LEGACY_PEER); + CheckPQ(responder, PQStatus::OFF); + } +} + +BOOST_AUTO_TEST_CASE(v2_pq_v1_detection) +{ + // A responder with the negotiation on detects v1 peers and peers from another network as + // before, without generating a key. + g_kem_calls = {}; + { + TestPQEntropy entropy{m_rng.rand256()}; + V2TransportTester tester(m_rng, false, PQ_ON, entropy.Source(), COUNTING_KEM_OPS); + tester.SendV1Version(Params().MessageStart()); + BOOST_CHECK(tester.Interact()); + const auto info{tester.GetTransport().GetInfo()}; + BOOST_CHECK(info.transport_type == TransportProtocolType::V1); + BOOST_CHECK(info.transport_pq_status == PQStatus::V1); + BOOST_CHECK(!info.transport_pq); + } + { + TestPQEntropy entropy{m_rng.rand256()}; + V2TransportTester tester(m_rng, false, PQ_ON, entropy.Source(), COUNTING_KEM_OPS); + tester.SendV1Version(CChainParams::Main()->MessageStart()); + BOOST_CHECK(!tester.Interact()); + } + BOOST_CHECK_EQUAL(g_kem_calls.total(), 0); + + // An initiator asks for the v1 retry exactly as before: after sending at least 24 bytes and + // before receiving anything. + for (const PQMode mode : {PQMode::OFF, PQMode::NEGOTIATE, PQMode::FALLBACK}) { + V2TransportTester tester(m_rng, true, {.mode = mode}); + auto& transport{tester.GetTransport()}; + BOOST_CHECK(!transport.ShouldReconnectV1()); + BOOST_REQUIRE(tester.Interact()); + BOOST_CHECK(transport.ShouldReconnectV1()); + tester.SendKey(); + tester.ToSend().resize(1); + BOOST_REQUIRE(tester.Deliver()); + BOOST_CHECK(!transport.ShouldReconnectV1()); + } +} + +BOOST_AUTO_TEST_CASE(v2_pq_send_hold) +{ + for (const bool test_initiator : {true, false}) { + BOOST_TEST_CONTEXT("test_initiator=" << test_initiator) + { + V2TransportTester tester(m_rng, test_initiator, PQ_ON); + auto& transport{tester.GetTransport()}; + if (test_initiator) tester.Collect(); + tester.SendKey(); + tester.SendGarbage(); + BOOST_REQUIRE(tester.Deliver()); + + // A responder's offer counts as sent once it is queued. + BOOST_CHECK(transport.GetPQSnapshot().offer == (test_initiator ? PQOfferState::NONE : PQOfferState::SENT)); + + // Holding: the terminator (initiator) or the key, garbage, terminator and offer + // (responder) is queued. Send part of it, and look at all of it. + auto [bytes, more, _type] = transport.GetBytesToSend(/*have_next_message=*/true); + BOOST_CHECK(!more); + BOOST_REQUIRE(bytes.size() > (test_initiator ? 0U : PQ_RECORD_BYTES)); + const std::vector queued{bytes.begin(), bytes.end()}; + tester.Received().insert(tester.Received().end(), queued.begin(), queued.end()); + const size_t sent{test_initiator ? 1 + m_rng.randrange(queued.size() - 1) : queued.size() - 1 - m_rng.randrange(PQ_RECORD_BYTES)}; + transport.MarkBytesSent(sent); + + // No message is taken, and nothing more will be sendable. + CSerializedNetMsg msg; + msg.m_type = "ping"; + msg.data = m_rng.randbytes(8); + const auto data{msg.data}; + BOOST_CHECK(!transport.SetMessageToSend(msg)); + BOOST_CHECK(msg.m_type == "ping" && msg.data == data); + for (const bool have_next : {false, true}) { + const auto& [rest, rest_more, _rest_type] = transport.GetBytesToSend(have_next); + BOOST_CHECK(!rest_more); + BOOST_CHECK(std::ranges::equal(rest, std::span{queued}.subspan(sent))); + } + + // The peer's version packet: the transport appends after the unsent bytes. + tester.ReceiveKey(/*retain_for_hybrid=*/true); + tester.SendGarbageTerm(); + if (test_initiator) { + tester.SendOffer(); + } else { + tester.ReceiveGarbage(); + tester.AcceptOfferAndSwitch(); + } + BOOST_REQUIRE(tester.Deliver()); + const auto& [after, after_more, _after_type] = transport.GetBytesToSend(/*have_next_message=*/true); + BOOST_CHECK(after_more); + const size_t appended{(test_initiator ? BIP324Cipher::EXPANSION + PQ_RECORD_BYTES : 0) + PQ_CONFIRMATION_BYTES}; + BOOST_REQUIRE_EQUAL(after.size(), queued.size() - sent + appended); + BOOST_CHECK(std::ranges::equal(after.first(queued.size() - sent), std::span{queued}.subspan(sent))); + tester.Received().insert(tester.Received().end(), after.end() - appended, after.end()); + transport.MarkBytesSent(after.size()); + if (test_initiator) { + tester.ReceiveGarbage(); + tester.ReceiveAcceptAndSwitch(); + } + tester.ReceiveConfirmation(); + + // Now the held message goes out under the hybrid keys. + BOOST_CHECK(transport.SetMessageToSend(msg)); + tester.Collect(); + tester.ReceiveMessage(uint8_t(18), data); // ping short id + } + } +} + +BOOST_AUTO_TEST_CASE(v2_pq_switch_boundary) +{ + for (const bool test_initiator : {true, false}) { + for (const bool byte_by_byte : {false, true}) { + BOOST_TEST_CONTEXT("test_initiator=" << test_initiator << " byte_by_byte=" << byte_by_byte) + { + V2TransportTester tester(m_rng, test_initiator, PQ_ON); + auto& transport{tester.GetTransport()}; + const auto payload{m_rng.randbytes(m_rng.randrange(1000))}; + if (test_initiator) { + // The offer (ECDH keys), then confirmation and message (hybrid keys). + tester.Collect(); + tester.SendKey(); + tester.SendGarbage(); + tester.ReceiveKey(/*retain_for_hybrid=*/true); + tester.SendGarbageTerm(); + tester.SendOffer(); + BOOST_REQUIRE(tester.Deliver(byte_by_byte)); + tester.Collect(); + tester.ReceiveGarbage(); + tester.ReceiveAcceptAndSwitch(); + tester.ReceiveConfirmation(); + } else { + // The accept (ECDH keys), confirmation and message (hybrid keys), coalesced. + tester.SendKey(); + tester.SendGarbage(); + BOOST_REQUIRE(tester.Deliver(byte_by_byte)); + tester.Collect(); + tester.ReceiveKey(/*retain_for_hybrid=*/true); + tester.SendGarbageTerm(); + tester.ReceiveGarbage(); + tester.AcceptOfferAndSwitch(); + } + tester.SendConfirmation(); + const size_t before_message{tester.ToSend().size()}; + tester.SendMessage(uint8_t(14), payload); // inv short id + std::vector message{tester.ToSend().begin() + before_message, tester.ToSend().end()}; + tester.ToSend().resize(before_message); + + // Up to the end of the confirmation: confirmed, and no message. + BOOST_REQUIRE(tester.Deliver(byte_by_byte)); + BOOST_CHECK(!transport.ReceivedMessageComplete()); + CheckPQ(transport, PQStatus::HYBRID); + if (!test_initiator) { + tester.Collect(); + tester.ReceiveConfirmation(); + } + + // Then the message, under the hybrid keys. + tester.Send(message); + BOOST_REQUIRE(tester.Deliver(byte_by_byte)); + BOOST_REQUIRE(transport.ReceivedMessageComplete()); + bool reject{false}; + const CNetMessage msg{transport.GetReceivedMessage({}, reject)}; + BOOST_CHECK(!reject && msg.m_type == "inv" && std::ranges::equal(msg.m_recv, MakeByteSpan(payload))); + BOOST_CHECK(!transport.ReceivedMessageComplete()); + tester.CompareSessionIDs(); + } + } + } + + // All at once: the accept, the confirmation and a message coalesced in one delivery. + { + V2TransportTester tester(m_rng, false, PQ_ON); + auto& transport{tester.GetTransport()}; + const auto payload{m_rng.randbytes(100)}; + tester.SendKey(); + tester.SendGarbage(); + BOOST_REQUIRE(tester.Deliver()); + tester.Collect(); + tester.ReceiveKey(/*retain_for_hybrid=*/true); + tester.SendGarbageTerm(); + tester.ReceiveGarbage(); + tester.AcceptOfferAndSwitch(); + tester.SendConfirmation(); + tester.SendMessage(uint8_t(14), payload); + BOOST_REQUIRE(tester.Deliver()); + BOOST_REQUIRE(transport.ReceivedMessageComplete()); + bool reject{false}; + const CNetMessage msg{transport.GetReceivedMessage({}, reject)}; + BOOST_CHECK(!reject && msg.m_type == "inv" && std::ranges::equal(msg.m_recv, MakeByteSpan(payload))); + CheckPQ(transport, PQStatus::HYBRID); + } +} + +BOOST_AUTO_TEST_CASE(v2_pq_confirmation_length) +{ + for (const bool test_initiator : {true, false}) { + for (const size_t length : {size_t{1}, size_t{17}, size_t{4'000'014}}) { + BOOST_TEST_CONTEXT("test_initiator=" << test_initiator << " length=" << length) + { + V2TransportTester tester(m_rng, test_initiator, PQ_ON); + auto& transport{tester.GetTransport()}; + HybridHandshake(tester, test_initiator, /*send_confirmation=*/false); + // An authenticated decoy with a nonzero length, of which only the length arrives. + tester.SendPacket(std::vector(length), /*aad=*/{}, /*ignore=*/true); + tester.ToSend().resize(BIP324Cipher::LENGTH_LEN); + BOOST_CHECK(!tester.Deliver(/*byte_by_byte=*/true)); + BOOST_CHECK(tester.ToSend().empty()); + CheckPQ(transport, PQStatus::PENDING, PQFailure::CONFIRM_LENGTH); + BOOST_CHECK(!transport.ShouldReconnectV1()); + } + } + } +} + +BOOST_AUTO_TEST_CASE(v2_pq_confirmation_tag) +{ + for (const bool test_initiator : {true, false}) { + for (const bool damage_header : {false, true}) { + BOOST_TEST_CONTEXT("test_initiator=" << test_initiator << " damage_header=" << damage_header) + { + V2TransportTester tester(m_rng, test_initiator, PQ_ON); + auto& transport{tester.GetTransport()}; + HybridHandshake(tester, test_initiator, /*send_confirmation=*/false); + tester.SendConfirmation(); + BOOST_REQUIRE_EQUAL(tester.ToSend().size(), PQ_CONFIRMATION_BYTES); + tester.ToSend()[damage_header ? BIP324Cipher::LENGTH_LEN : PQ_CONFIRMATION_BYTES - 1] ^= 1 << m_rng.randrange(8); + // Nothing fails before byte 20. + std::vector last{tester.ToSend().back()}; + tester.ToSend().pop_back(); + BOOST_REQUIRE(tester.Deliver(/*byte_by_byte=*/true)); + CheckPQ(transport, PQStatus::PENDING); + tester.Send(last); + BOOST_CHECK(!tester.Deliver()); + CheckPQ(transport, PQStatus::PENDING, PQFailure::CONFIRM_TAG); + } + } + } +} + +BOOST_AUTO_TEST_CASE(v2_pq_confirmation_not_decoy) +{ + for (const bool test_initiator : {true, false}) { + BOOST_TEST_CONTEXT("test_initiator=" << test_initiator) + { + V2TransportTester tester(m_rng, test_initiator, PQ_ON); + auto& transport{tester.GetTransport()}; + HybridHandshake(tester, test_initiator, /*send_confirmation=*/false); + // An authenticated empty packet without the ignore bit (an empty application message). + tester.SendPacket(/*content=*/{}, /*aad=*/{}, /*ignore=*/false); + BOOST_CHECK(!tester.Deliver(/*byte_by_byte=*/true)); + BOOST_CHECK(tester.ToSend().empty()); + BOOST_CHECK(!transport.ReceivedMessageComplete()); + CheckPQ(transport, PQStatus::PENDING, PQFailure::CONFIRM_NOT_DECOY); + } + } +} + +BOOST_AUTO_TEST_CASE(v2_pq_corrupt_ct) +{ + // A ciphertext damaged in transit, of the right length: the responder switches to different + // keys, and the first packet under them fails. No continuation with ECDH keys. + { + V2TransportTester tester(m_rng, false, PQ_ON); + auto& transport{tester.GetTransport()}; + tester.SendKey(); + tester.SendGarbage(); + BOOST_REQUIRE(tester.Interact()); + tester.ReceiveKey(/*retain_for_hybrid=*/true); + tester.SendGarbageTerm(); + tester.ReceiveGarbage(); + tester.AcceptOfferAndSwitch(/*damage_ct=*/true); + tester.SendConfirmation(); + BOOST_CHECK(!tester.Interact()); + const auto snapshot{transport.GetPQSnapshot()}; + BOOST_CHECK(snapshot.switched && !snapshot.confirmed); + BOOST_CHECK(snapshot.failure == PQFailure::CONFIRM_LENGTH || snapshot.failure == PQFailure::CONFIRM_TAG); + BOOST_CHECK(transport.GetInfo().transport_pq_status == PQStatus::PENDING); + } + + // The local fault injection: the transport inverts byte 0 of its ML-KEM secret. Each side + // rejects the other's confirmation, and an initiator does not retry with v1. + for (const bool test_initiator : {true, false}) { + BOOST_TEST_CONTEXT("test_initiator=" << test_initiator) + { + V2TransportTester tester(m_rng, test_initiator, {.mode = PQMode::NEGOTIATE, .corrupt_shared_secret = true}); + auto& transport{tester.GetTransport()}; + if (test_initiator) { + BOOST_REQUIRE(tester.Interact()); + tester.SendKey(); + tester.SendGarbage(); + tester.ReceiveKey(/*retain_for_hybrid=*/true); + tester.SendGarbageTerm(); + tester.SendOffer(); + BOOST_REQUIRE(tester.Interact()); + tester.ReceiveGarbage(); + tester.ReceiveAcceptAndSwitch(); + } else { + tester.SendKey(); + tester.SendGarbage(); + BOOST_REQUIRE(tester.Interact()); + tester.ReceiveKey(/*retain_for_hybrid=*/true); + tester.SendGarbageTerm(); + tester.ReceiveGarbage(); + tester.AcceptOfferAndSwitch(); + BOOST_REQUIRE(tester.Interact()); + } + // The transport's confirmation does not verify under our honest keys. + BOOST_REQUIRE_EQUAL(tester.Received().size(), PQ_CONFIRMATION_BYTES); + const auto packet{MakeByteSpan(tester.Received())}; + bool ignore{false}; + BOOST_CHECK(tester.GetCipher().DecryptLength(packet.first(BIP324Cipher::LENGTH_LEN)) != 0 || + !tester.GetCipher().Decrypt(packet.subspan(BIP324Cipher::LENGTH_LEN), {}, ignore, {})); + tester.SendConfirmation(); + BOOST_CHECK(!tester.Interact()); + const auto snapshot{transport.GetPQSnapshot()}; + BOOST_CHECK(snapshot.switched && !snapshot.confirmed); + BOOST_CHECK(snapshot.failure == PQFailure::CONFIRM_LENGTH || snapshot.failure == PQFailure::CONFIRM_TAG); + BOOST_CHECK(!transport.ShouldReconnectV1()); + } + } + + // The injected fault is exactly byte 0 of the ML-KEM secret inverted, EK and CT untouched: a + // peer that inverts the same byte derives the transport's keys. pq_transport_vectors.json + // models the same change in a negative vector. + for (const bool test_initiator : {true, false}) { + BOOST_TEST_CONTEXT("test_initiator=" << test_initiator) + { + V2TransportTester tester(m_rng, test_initiator, {.mode = PQMode::NEGOTIATE, .corrupt_shared_secret = true}); + auto& transport{tester.GetTransport()}; + if (test_initiator) { + BOOST_REQUIRE(tester.Interact()); + tester.SendKey(); + tester.SendGarbage(); + tester.ReceiveKey(/*retain_for_hybrid=*/true); + tester.SendGarbageTerm(); + tester.SendOffer(); + BOOST_REQUIRE(tester.Interact()); + tester.ReceiveGarbage(); + tester.ReceiveAcceptAndSwitch(/*invert_ss=*/true); + } else { + tester.SendKey(); + tester.SendGarbage(); + BOOST_REQUIRE(tester.Interact()); + tester.ReceiveKey(/*retain_for_hybrid=*/true); + tester.SendGarbageTerm(); + tester.ReceiveGarbage(); + tester.AcceptOfferAndSwitch(/*damage_ct=*/false, /*invert_ss=*/true); + BOOST_REQUIRE(tester.Interact()); + } + tester.ReceiveConfirmation(); + tester.SendConfirmation(); + BOOST_REQUIRE(tester.Interact()); + CheckPQ(transport, PQStatus::HYBRID); + tester.CompareSessionIDs(); + } + } + UniValue doc; + BOOST_REQUIRE(doc.read(json_tests::pq_transport_vectors)); + int modeled{0}; + for (const UniValue& neg : doc["negative_vectors"].getValues()) { + const UniValue& vec{doc["vectors"][neg["vector"].getInt()]}; + auto ss{ParseHex(vec["ss_mlkem"].get_str())}; + ss[0] ^= 0xff; + const UniValue& contents{neg["transcript_contents"]}; + if (HexStr(ss) == neg["ss_mlkem"].get_str() && contents[0].get_str() == vec["contents_responder"].get_str() && + contents[1].get_str() == vec["contents_initiator"].get_str()) { + ++modeled; + } + } + BOOST_CHECK_EQUAL(modeled, 1); +} + +BOOST_AUTO_TEST_CASE(v2_pq_internal_errors) +{ + using mlkem::InjectResultForTesting; + using mlkem::Operation; + namespace upstream = mlkem::upstream; + + // Failures before anything is committed: an empty version packet, and plain v2 continues. + for (const auto& [op, failure] : {std::pair{Operation::KEYGEN, PQFailure::KEYGEN_INTERNAL}, + std::pair{Operation::CHECK_PUBLIC_KEY, PQFailure::CHECK_EK_INTERNAL}, + std::pair{Operation::ENCAPS, PQFailure::ENCAPS_INTERNAL}}) { + const bool test_initiator{op != Operation::KEYGEN}; + BOOST_TEST_CONTEXT("failure=" << int(failure)) + { + g_kem_calls = {}; + TestPQEntropy entropy{m_rng.rand256()}; + V2TransportTester tester(m_rng, test_initiator, PQ_ON, entropy.Source(), COUNTING_KEM_OPS); + auto& transport{tester.GetTransport()}; + if (test_initiator) { + BOOST_REQUIRE(tester.Interact()); + tester.SendKey(); + tester.SendGarbage(); + tester.ReceiveKey(/*retain_for_hybrid=*/true); + tester.SendGarbageTerm(); + tester.SendOffer(); + InjectResultForTesting inject{op, upstream::ERR_FAIL}; + BOOST_REQUIRE(tester.Interact()); + } else { + tester.SendKey(); + tester.SendGarbage(); + { + InjectResultForTesting inject{op, upstream::ERR_FAIL}; + BOOST_REQUIRE(tester.Interact()); + } + // No offer was queued, and the retained secret is wiped at once, not only when the + // initiator's version packet arrives. + BOOST_CHECK(transport.GetPQSnapshot().offer == PQOfferState::NONE); + BOOST_CHECK(!transport.HoldsHybridSecretsForTesting()); + tester.ReceiveKey(/*retain_for_hybrid=*/true); + tester.SendGarbageTerm(); + } + tester.ReceiveGarbage(); + BOOST_CHECK(tester.ReceiveVersionContents().empty()); + tester.GetCipher().DiscardHybridSecret(); + if (!test_initiator) { + // An unsolicited own record (an accept to an offer never made) is ignored and + // never decapsulated. + tester.SendVersion(MakeUCharSpan(MakeRecord(PQ_MLKEM1024, MakeByteSpan(m_rng.randbytes(mlkem::CIPHERTEXT_BYTES))))); + } + const auto payload{m_rng.randbytes(100)}; + tester.SendMessage(uint8_t(14), payload); + tester.AddMessage("barfoo", payload); + const auto ret{tester.Interact()}; + BOOST_REQUIRE(ret && ret->size() == 1); + BOOST_CHECK((*ret)[0] && (*ret)[0]->m_type == "inv" && std::ranges::equal((*ret)[0]->m_recv, MakeByteSpan(payload))); + tester.ReceiveMessage("barfoo", payload); + tester.CompareSessionIDs(); + CheckPQ(transport, PQStatus::OFF, failure); + BOOST_CHECK(!transport.GetPQSnapshot().switched); + BOOST_CHECK(!transport.HoldsHybridSecretsForTesting()); + BOOST_CHECK_EQUAL(g_kem_calls.decaps, 0); + BOOST_CHECK_EQUAL(g_kem_calls.keygen, test_initiator ? 0 : 1); + } + } + + // Decapsulation fails after the initiator committed to its accept: close at once, on the + // version packet, without queuing a confirmation. There is no failed status, so the closing + // connection reports pending. + { + V2TransportTester tester(m_rng, false, PQ_ON); + auto& transport{tester.GetTransport()}; + tester.SendKey(); + tester.SendGarbage(); + BOOST_REQUIRE(tester.Interact()); + tester.ReceiveKey(/*retain_for_hybrid=*/true); + tester.SendGarbageTerm(); + tester.ReceiveGarbage(); + tester.AcceptOfferAndSwitch(); + InjectResultForTesting inject{Operation::DECAPS, upstream::ERR_FAIL}; + BOOST_CHECK(!tester.Deliver()); + BOOST_CHECK(tester.ToSend().empty()); + BOOST_CHECK(std::get<0>(transport.GetBytesToSend(false)).empty()); + const auto snapshot{transport.GetPQSnapshot()}; + BOOST_CHECK(snapshot.version_received && !snapshot.switched); + CheckPQ(transport, PQStatus::PENDING, PQFailure::DECAPS_INTERNAL); + BOOST_CHECK(!transport.HoldsHybridSecretsForTesting()); + } + + // An inconsistent cipher state (a local bug, behind Assume) closes in both roles, also before + // the initiator committed to anything. Debug builds abort on the Assume instead. + if constexpr (!G_FUZZING_BUILD && !G_ABORT_ON_FAILED_ASSUME) { + for (const bool test_initiator : {true, false}) { + BOOST_TEST_CONTEXT("cipher_state test_initiator=" << test_initiator) + { + g_kem_calls = {}; + TestPQEntropy entropy{m_rng.rand256()}; + V2TransportTester tester(m_rng, test_initiator, PQ_ON, entropy.Source(), COUNTING_KEM_OPS); + auto& transport{tester.GetTransport()}; + if (test_initiator) tester.Collect(); + tester.SendKey(); + tester.SendGarbage(); + BOOST_REQUIRE(tester.Deliver()); + tester.Collect(); + tester.ReceiveKey(/*retain_for_hybrid=*/true); + tester.SendGarbageTerm(); + transport.DiscardHybridSecretForTesting(); + if (test_initiator) { + tester.SendOffer(); + } else { + tester.ReceiveGarbage(); + tester.AcceptOfferAndSwitch(); + } + BOOST_CHECK(!tester.Deliver()); + // Nothing more is queued: no version packet from the initiator, no confirmation. + BOOST_CHECK(std::get<0>(transport.GetBytesToSend(false)).empty()); + BOOST_CHECK(!transport.GetPQSnapshot().switched); + CheckPQ(transport, PQStatus::PENDING, PQFailure::CIPHER_STATE_INTERNAL); + BOOST_CHECK(!transport.HoldsHybridSecretsForTesting()); + BOOST_CHECK_EQUAL(g_kem_calls.check + g_kem_calls.encaps + g_kem_calls.decaps, 0); + } + } + } +} + +BOOST_AUTO_TEST_CASE(v2_pq_malformed_record) +{ + // The first own record of the peer's version packet has the wrong length, or (offer only) a + // key that fails the modulus check: close with that failure, wipe both secrets, and queue + // nothing, without any encapsulation or decapsulation. + const auto valid_offer{PQHandshake::SerializeRecord(TestEncapsulationKey())}; + const auto bad_payload = [&](size_t size) { return MakeRecord(PQ_MLKEM1024, MakeByteSpan(m_rng.randbytes(size))); }; + struct Case { + std::string name; + bool test_initiator; + std::vector contents; + PQFailure failure; + }; + std::vector cases{ + {"ek 1567", true, bad_payload(mlkem::PUBLIC_KEY_BYTES - 1), PQFailure::EK_LENGTH}, + {"ek 1569", true, bad_payload(mlkem::PUBLIC_KEY_BYTES + 1), PQFailure::EK_LENGTH}, + {"ek modulus", true, MakeRecord(PQ_MLKEM1024, std::vector(mlkem::PUBLIC_KEY_BYTES, std::byte{0xff})), PQFailure::EK_MODULUS}, + {"malformed first, valid second", true, Concat({bad_payload(2), valid_offer}), PQFailure::EK_LENGTH}, + }; + for (const size_t size : {size_t{0}, mlkem::CIPHERTEXT_BYTES - 1, mlkem::CIPHERTEXT_BYTES + 1, size_t{3000}}) { + cases.push_back({strprintf("ct %u", size), false, bad_payload(size), PQFailure::CT_LENGTH}); + } + for (const auto& c : cases) { + BOOST_TEST_CONTEXT(c.name) + { + g_kem_calls = {}; + TestPQEntropy entropy{m_rng.rand256()}; + V2TransportTester tester(m_rng, c.test_initiator, PQ_ON, entropy.Source(), COUNTING_KEM_OPS); + auto& transport{tester.GetTransport()}; + if (c.test_initiator) tester.Collect(); + tester.SendKey(); + tester.SendGarbage(); + BOOST_REQUIRE(tester.Deliver()); + // Everything queued so far: the key, garbage and terminator, and a responder's offer. + tester.Collect(); + BOOST_CHECK(transport.HoldsHybridSecretsForTesting()); + tester.ReceiveKey(); + tester.SendGarbageTerm(); + if (!c.test_initiator) { + tester.ReceiveGarbage(); + BOOST_CHECK(PQHandshake::ParseContents(MakeByteSpan(tester.ReceiveVersionContents())).kind == PQHandshake::ParseKind::OWN_RECORD); + } + tester.SendVersion(MakeUCharSpan(c.contents)); + BOOST_CHECK(!tester.Deliver()); + BOOST_CHECK(std::get<0>(transport.GetBytesToSend(false)).empty()); + const auto snapshot{transport.GetPQSnapshot()}; + BOOST_CHECK(snapshot.version_received && !snapshot.switched); + CheckPQ(transport, PQStatus::PENDING, c.failure); + BOOST_CHECK(!transport.HoldsHybridSecretsForTesting()); + BOOST_CHECK_EQUAL(g_kem_calls.check, c.failure == PQFailure::EK_MODULUS ? 1 : 0); + BOOST_CHECK_EQUAL(g_kem_calls.encaps + g_kem_calls.decaps, 0); + BOOST_CHECK_EQUAL(g_kem_calls.keygen, c.test_initiator ? 0 : 1); + } + } +} + +BOOST_AUTO_TEST_CASE(v2_pq_close_wipes) +{ + // A close the transport detects before the peer's version packet authenticated (a missing + // garbage terminator, or a version packet that is too long or fails its tag) wipes the + // retained ECDH secret and a responder's decapsulation key at once. + enum class Close { GARBAGE_TERMINATOR, VERSION_LENGTH, VERSION_TAG }; + for (const bool test_initiator : {true, false}) { + for (const Close close : {Close::GARBAGE_TERMINATOR, Close::VERSION_LENGTH, Close::VERSION_TAG}) { + BOOST_TEST_CONTEXT("test_initiator=" << test_initiator << " close=" << int(close)) + { + V2TransportTester tester(m_rng, test_initiator, PQ_ON); + auto& transport{tester.GetTransport()}; + if (test_initiator) tester.Collect(); + tester.SendKey(); + BOOST_REQUIRE(tester.Deliver()); + tester.Collect(); + tester.ReceiveKey(); + if (close == Close::GARBAGE_TERMINATOR) { + tester.SendGarbage(V2Transport::MAX_GARBAGE_LEN + BIP324Cipher::GARBAGE_TERMINATOR_LEN); + } else { + tester.SendGarbage(); + tester.SendGarbageTerm(); + const size_t version_start{tester.ToSend().size()}; + if (close == Close::VERSION_LENGTH) { + // Only the length of a version packet one byte over the maximum arrives. + tester.SendVersion(std::vector(4'000'014)); + tester.ToSend().resize(version_start + BIP324Cipher::LENGTH_LEN); + } else { + tester.SendVersion(); + tester.ToSend().back() ^= 1 << m_rng.randrange(8); + } + } + // Nothing fails before the last byte, and the secrets for the switch are held. + const std::vector last{tester.ToSend().back()}; + tester.ToSend().pop_back(); + BOOST_REQUIRE(tester.Deliver()); + BOOST_CHECK(transport.HoldsHybridSecretsForTesting()); + tester.Send(last); + BOOST_CHECK(!tester.Deliver()); + BOOST_CHECK(!transport.HoldsHybridSecretsForTesting()); + // No PQ failure: the close is the transport's, before the peer's version packet. + const auto snapshot{transport.GetPQSnapshot()}; + BOOST_CHECK(!snapshot.version_received && !snapshot.switched); + BOOST_CHECK(snapshot.offer == (test_initiator ? PQOfferState::NONE : PQOfferState::SENT)); + CheckPQ(transport, PQStatus::PENDING); + } + } + } +} + +BOOST_AUTO_TEST_CASE(v2_pq_version_wipes) +{ + // However a transport came to send its version packet without the negotiation, processing the + // peer's version packet wipes anything still held for a switch. A test option retains the ECDH + // secret and the transcript with the negotiation off, as a path that declined it after the key + // exchange without wiping them would. + for (const bool test_initiator : {true, false}) { + BOOST_TEST_CONTEXT("test_initiator=" << test_initiator) + { + V2TransportTester tester(m_rng, test_initiator, {.mode = PQMode::OFF, .retain_without_negotiation = true}); + auto& transport{tester.GetTransport()}; + if (test_initiator) tester.Collect(); + tester.SendKey(); + tester.SendGarbage(); + BOOST_REQUIRE(tester.Deliver()); + tester.Collect(); + tester.ReceiveKey(); + tester.SendGarbageTerm(); + BOOST_REQUIRE(tester.Deliver()); + // The transport sent its empty version packet, and still holds the retained secret. + BOOST_CHECK(transport.HoldsHybridSecretsForTesting()); + tester.ReceiveGarbage(); + tester.ReceiveVersion(); + tester.SendVersion(); + BOOST_REQUIRE(tester.Deliver()); + BOOST_CHECK(!transport.HoldsHybridSecretsForTesting()); + + // The connection continues as plain v2. + const auto payload{m_rng.randbytes(100)}; + tester.SendMessage("foobar", payload); + tester.AddMessage("barfoo", payload); + const auto ret{tester.Interact()}; + BOOST_REQUIRE(ret && ret->size() == 1); + BOOST_CHECK((*ret)[0] && (*ret)[0]->m_type == "foobar" && std::ranges::equal((*ret)[0]->m_recv, MakeByteSpan(payload))); + tester.ReceiveMessage("barfoo", payload); + tester.CompareSessionIDs(); + CheckPQ(transport, PQStatus::OFF); + } + } +} + +BOOST_AUTO_TEST_CASE(v2_pq_rekey) +{ + for (const bool test_initiator : {true, false}) { + BOOST_TEST_CONTEXT("test_initiator=" << test_initiator) + { + V2TransportTester tester(m_rng, test_initiator, PQ_ON); + auto& transport{tester.GetTransport()}; + // 222 decoys before our version packet: the transport's ECDH receive counter is at 223, + // one packet before a rekey, when it switches. + constexpr int DECOYS{BIP324Cipher::REKEY_INTERVAL - 2}; + if (test_initiator) { + BOOST_REQUIRE(tester.Interact()); + tester.SendKey(); + tester.SendGarbage(); + tester.ReceiveKey(/*retain_for_hybrid=*/true); + tester.SendGarbageTerm(); + for (int i = 0; i < DECOYS; ++i) tester.SendVersion(m_rng.randbytes(m_rng.randrange(10)), /*vers_ignore=*/true); + tester.SendOffer(); + BOOST_REQUIRE(tester.Interact()); + tester.ReceiveGarbage(); + tester.ReceiveAcceptAndSwitch(); + } else { + tester.SendKey(); + tester.SendGarbage(); + BOOST_REQUIRE(tester.Interact()); + tester.ReceiveKey(/*retain_for_hybrid=*/true); + tester.SendGarbageTerm(); + tester.ReceiveGarbage(); + for (int i = 0; i < DECOYS; ++i) tester.SendVersion(m_rng.randbytes(m_rng.randrange(10)), /*vers_ignore=*/true); + tester.AcceptOfferAndSwitch(); + BOOST_REQUIRE(tester.Interact()); + } + tester.ReceiveConfirmation(); + tester.SendConfirmation(); + + // More than two rekey intervals of hybrid packets each way. + constexpr int MESSAGES{2 * BIP324Cipher::REKEY_INTERVAL + 2}; + std::vector> payloads; + for (int i = 0; i < MESSAGES; ++i) { + payloads.push_back(m_rng.randbytes(m_rng.randrange(100))); + tester.SendMessage(uint8_t(14), payloads.back()); + tester.AddMessage("barfoo", payloads.back()); + } + const auto ret{tester.Interact()}; + BOOST_REQUIRE(ret && ret->size() == MESSAGES); + for (int i = 0; i < MESSAGES; ++i) { + BOOST_CHECK((*ret)[i] && (*ret)[i]->m_type == "inv" && std::ranges::equal((*ret)[i]->m_recv, MakeByteSpan(payloads[i]))); + tester.ReceiveMessage("barfoo", payloads[i]); + } + CheckPQ(transport, PQStatus::HYBRID); + tester.CompareSessionIDs(); + } + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/test/functional/test_framework/crypto/bip324_cipher.py b/test/functional/test_framework/crypto/bip324_cipher.py index c9f0fa0151..8b6c3c112f 100644 --- a/test/functional/test_framework/crypto/bip324_cipher.py +++ b/test/functional/test_framework/crypto/bip324_cipher.py @@ -3,7 +3,8 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test-only implementation of ChaCha20 Poly1305 AEAD Construction in RFC 8439 and FSChaCha20Poly1305 for BIP 324 +"""Test-only implementation of ChaCha20 Poly1305 AEAD Construction in RFC 8439 and FSChaCha20Poly1305 for BIP 324, +and of qbit's hybrid post-quantum v2 key schedule (doc/design/pq-transport.md). It is designed for ease of understanding, not performance. @@ -11,9 +12,11 @@ anything but tests. """ +import hashlib import unittest -from .chacha20 import chacha20_block, REKEY_INTERVAL +from .chacha20 import chacha20_block, FSChaCha20, REKEY_INTERVAL +from .hkdf import hkdf_sha256, hmac_sha256 from .poly1305 import Poly1305 @@ -88,6 +91,81 @@ def encrypt(self, aad, plaintext): return self._crypt(aad, plaintext, False) +LENGTH_LEN = 3 +IGNORE_BIT = 0x80 + + +def encrypt_packet(send_l, send_p, contents, aad=b"", ignore=False): + """Encrypt one BIP324 packet with the sender's FSChaCha20 (length) and FSChaCha20Poly1305.""" + header = bytes([IGNORE_BIT if ignore else 0]) + return send_l.crypt(len(contents).to_bytes(LENGTH_LEN, 'little')) + send_p.encrypt(aad, header + contents) + + +def decrypt_packet(recv_l, recv_p, packet, aad=b""): + """Decrypt one complete BIP324 packet. + + Returns (length, None) if the decrypted length does not match the packet size or the tag + fails, and (length, (ignore, contents)) otherwise.""" + length = int.from_bytes(recv_l.crypt(packet[:LENGTH_LEN]), 'little') + if len(packet) != LENGTH_LEN + 1 + length + 16: + return length, None + plaintext = recv_p.decrypt(aad, packet[LENGTH_LEN:]) + if plaintext is None: + return length, None + return length, (bool(plaintext[0] & IGNORE_BIT), plaintext[1:]) + + +# qbit hybrid v2 key schedule: the ML-KEM-1024 recipe under version-contents header 0xF0. Any change +# to this recipe needs a new header with its own salt label (doc/design/pq-transport.md). +HYBRID_SALT_LABEL = b"qbit_v2_hybrid_mlkem1024" +HYBRID_KEY_LABELS = ("initiator_L", "initiator_P", "responder_L", "responder_P", "session_id") + + +def ser_compact_size(n): + """Serialize n as a Bitcoin CompactSize.""" + if n < 253: + return n.to_bytes(1, 'little') + if n <= 0xffff: + return b"\xfd" + n.to_bytes(2, 'little') + if n <= 0xffffffff: + return b"\xfe" + n.to_bytes(4, 'little') + return b"\xff" + n.to_bytes(8, 'little') + + +def hybrid_transcript_input(ellswift_initiator, ellswift_responder, contents_responder, contents_initiator): + """Return the bytes the transcript hash TH covers. + + Both EllSwift keys in role order, then the full version packet contents each side sent, the + responder's first, each preceded by its CompactSize length.""" + assert len(ellswift_initiator) == 64 and len(ellswift_responder) == 64 + return (ellswift_initiator + ellswift_responder + + ser_compact_size(len(contents_responder)) + contents_responder + + ser_compact_size(len(contents_initiator)) + contents_initiator) + + +def hybrid_derive_keys(ss_ecdh, ss_mlkem, transcript_hash, magic): + """Derive the hybrid v2 keys with HKDF-SHA256 (RFC 5869). + + ss_ecdh is the 32-byte BIP324 ECDH secret, ss_mlkem the 32-byte ML-KEM-1024 shared secret, + transcript_hash = SHA256(hybrid_transcript_input(...)) and magic the 4 network magic bytes + in wire order. Returns a dict with the salt, IKM and PRK, and the five derived 32-byte + values keyed by label.""" + assert len(ss_ecdh) == 32 and len(ss_mlkem) == 32 and len(transcript_hash) == 32 and len(magic) == 4 + salt = HYBRID_SALT_LABEL + magic + ikm = ss_ecdh + ss_mlkem + transcript_hash + keys = {"salt": salt, "ikm": ikm, "prk": hmac_sha256(salt, ikm)} + for label in HYBRID_KEY_LABELS: + keys[label] = hkdf_sha256(length=32, ikm=ikm, salt=salt, info=label.encode('ascii')) + return keys + + +def hybrid_ciphers(keys, initiator): + """Return fresh (send_L, send_P, recv_L, recv_P) ciphers for one side from hybrid_derive_keys().""" + ours, theirs = ("initiator", "responder") if initiator else ("responder", "initiator") + return (FSChaCha20(keys[ours + "_L"]), FSChaCha20Poly1305(keys[ours + "_P"]), + FSChaCha20(keys[theirs + "_L"]), FSChaCha20Poly1305(keys[theirs + "_P"])) + + # Test vectors from RFC8439 consisting of plaintext, aad, 32 byte key, 12 byte nonce and ciphertext AEAD_TESTS = [ # RFC 8439 Example from section 2.8.2 @@ -201,3 +279,69 @@ def test_fschacha20poly1305aead(self): dec_aead.decrypt(b"", None) plaintext = dec_aead.decrypt(aad, ciphertext) self.assertEqual(plain, plaintext) + + +class TestFrameworkHybridKeys(unittest.TestCase): + ELLSWIFT_I = bytes(range(64)) + ELLSWIFT_R = bytes(range(64, 128)) + OFFER = b"\xfd\x21\x06\xf0" + bytes([0x11]) * 1568 + ACCEPT = b"\xfd\x21\x06\xf0" + bytes([0x22]) * 1568 + SS_ECDH = bytes([0x33]) * 32 + SS_MLKEM = bytes([0x44]) * 32 + MAINNET = bytes.fromhex("444f24a8") + + def derive(self, ellswift_i=ELLSWIFT_I, ellswift_r=ELLSWIFT_R, contents_r=OFFER, contents_i=ACCEPT, + ss_ecdh=SS_ECDH, ss_mlkem=SS_MLKEM, magic=MAINNET): + th_input = hybrid_transcript_input(ellswift_i, ellswift_r, contents_r, contents_i) + return hybrid_derive_keys(ss_ecdh, ss_mlkem, hashlib.sha256(th_input).digest(), magic) + + def test_layout(self): + """Transcript and HKDF input sizes match the specification.""" + self.assertEqual(len(hybrid_transcript_input(self.ELLSWIFT_I, self.ELLSWIFT_R, self.OFFER, self.ACCEPT)), 3278) + self.assertEqual(hybrid_transcript_input(self.ELLSWIFT_I, self.ELLSWIFT_R, b"", b""), + self.ELLSWIFT_I + self.ELLSWIFT_R + b"\x00\x00") + keys = self.derive() + self.assertEqual(keys["salt"], b"qbit_v2_hybrid_mlkem1024\x44\x4f\x24\xa8") + self.assertEqual(len(keys["ikm"]), 96) + for label in HYBRID_KEY_LABELS: + self.assertEqual(keys[label], hmac_sha256(keys["prk"], label.encode('ascii') + b"\x01")) + for n, encoding in ((252, "fc"), (253, "fdfd00"), (0xffff, "fdffff"), (0x10000, "fe00000100"), + (0x100000000, "ff0000000001000000")): + self.assertEqual(ser_compact_size(n).hex(), encoding) + + def test_binding(self): + """Every input changes all five derived values.""" + base = self.derive() + variants = [ + self.derive(ellswift_i=b"\x01" + self.ELLSWIFT_I[1:]), + self.derive(ellswift_r=self.ELLSWIFT_R[:-1] + b"\x00"), + self.derive(contents_r=self.OFFER[:-1] + b"\x00"), + self.derive(contents_i=self.ACCEPT[:-1] + b"\x00"), + self.derive(contents_r=self.ACCEPT, contents_i=self.OFFER), + self.derive(contents_r=self.OFFER + self.ACCEPT[:1], contents_i=self.ACCEPT[1:]), + self.derive(contents_i=self.ACCEPT + b"\x01\xf1"), + self.derive(ss_ecdh=self.SS_MLKEM, ss_mlkem=self.SS_ECDH), + self.derive(magic=bytes.fromhex("a66b1fda")), + ] + seen = {base["session_id"]} + for keys in variants: + for label in HYBRID_KEY_LABELS: + self.assertNotEqual(keys[label], base[label]) + seen.add(keys["session_id"]) + self.assertEqual(len(seen), len(variants) + 1) + + def test_packets(self): + """Both sides' hybrid ciphers agree, across a rekey.""" + keys = self.derive() + i_send_l, i_send_p, i_recv_l, i_recv_p = hybrid_ciphers(keys, initiator=True) + r_send_l, r_send_p, r_recv_l, r_recv_p = hybrid_ciphers(keys, initiator=False) + confirmation = encrypt_packet(i_send_l, i_send_p, b"", ignore=True) + self.assertEqual(len(confirmation), 20) + self.assertEqual(decrypt_packet(r_recv_l, r_recv_p, confirmation), (0, (True, b""))) + for i in range(REKEY_INTERVAL + 1): + contents = i.to_bytes(2, 'little') + self.assertEqual(decrypt_packet(i_recv_l, i_recv_p, encrypt_packet(r_send_l, r_send_p, contents)), + (2, (False, contents))) + damaged = bytearray(encrypt_packet(r_send_l, r_send_p, b"x")) + damaged[-1] ^= 1 + self.assertEqual(decrypt_packet(i_recv_l, i_recv_p, bytes(damaged)), (1, None)) diff --git a/test/fuzz/qbit_corpora/MANIFEST.json b/test/fuzz/qbit_corpora/MANIFEST.json index 5265d09ab8..f2bc3b70d3 100644 --- a/test/fuzz/qbit_corpora/MANIFEST.json +++ b/test/fuzz/qbit_corpora/MANIFEST.json @@ -12,7 +12,8 @@ "fdp": "Untagged FuzzedDataProvider layout for targets without a tagged format.", "legacy": "Untagged pre-existing FuzzedDataProvider layout of pqc/p2mr_script, unchanged.", "qbfx-v1-fixture": "b'QBFX' + target tag ('P' pqc, 'M' p2mr_script) + 0x01 + selector whose low nibble is not 0xF, then the FuzzedDataProvider layout of the cached-fixture body.", - "qbfx-v1-generation": "b'QBFX' + target tag + 0x01 + selector whose low nibble is 0xF, then the legacy FuzzedDataProvider layout." + "qbfx-v1-generation": "b'QBFX' + target tag + 0x01 + selector whose low nibble is 0xF, then the legacy FuzzedDataProvider layout.", + "raw": "The whole input is the data the target parses, without a FuzzedDataProvider layout." }, "targets": { "asert_chain_transition": [ @@ -414,6 +415,357 @@ "semantics": "Tagged generation selector 0x0F followed by the legacy-checksigpqc-default-valid layout." } ], + "p2p_transport_bidirectional_v1v2": [ + { + "file": "v1-version-to-negotiating-responder", + "format": "fdp", + "size": 119, + "sha256": "35e6f63d5ac205b1e4d48389eeea77e0d1e55d798fb7a156233754548fca1a19", + "semantics": "A v1 initiator's version message reaches a responder with the negotiation on: v1 detected, no key generated." + }, + { + "file": "v1-version-to-off-responder", + "format": "fdp", + "size": 183, + "sha256": "19e798b8d20825d67726f8067d7283dddb834821e7ae0e7fd4940f790ac43ad2", + "semantics": "A v1 initiator's version message reaches a responder with the negotiation off: v1 detected." + }, + { + "file": "v1-version-to-negotiating-responder-max-garbage", + "format": "fdp", + "size": 119, + "sha256": "69c2c84bdbe6b60ec78bab5775ce5e65ff6eab9529aee686530b93625438bbc7", + "semantics": "A v1 initiator's version message reaches a negotiating responder configured with 4,095 bytes of garbage, which it never sends: v1 detected." + }, + { + "file": "v1-silent-negotiating-responder", + "format": "fdp", + "size": 114, + "sha256": "28ff6ed50c2b9700aaee84e470ec094fc48d2698e55cc4282cda42dd9d12530c", + "semantics": "A v1 initiator that sends nothing: the negotiating responder stays undecided and sends nothing." + } + ], + "p2p_transport_bidirectional_v2": [ + { + "file": "negotiate-both", + "format": "fdp", + "size": 214, + "sha256": "513f1a34d1c2e612ef36e2d5c34f8c48776acff59ad10169392b114925bf0581", + "semantics": "Both sides negotiate, no garbage; the final flush completes the handshake: hybrid keys, confirmed both ways." + }, + { + "file": "negotiate-both-max-garbage", + "format": "fdp", + "size": 214, + "sha256": "d49337112b58a48f382feb53e798c218a9bd7d6459d0f83054c0af71924a3b79", + "semantics": "Both sides negotiate with 4,095 bytes of garbage each, so the held version packet's AAD is the longest; the final flush completes the handshake." + }, + { + "file": "negotiate-both-interleaved", + "format": "fdp", + "size": 343, + "sha256": "8b793b1259c0a61cf02a467c4302840314635f52e593f5f920976992d8dfc9ef", + "semantics": "Both sides negotiate, 64 and 17 bytes of garbage; a fixed 48-byte pattern drives the interleaving loop, then the final flush completes the handshake: hybrid." + }, + { + "file": "negotiate-both-interleaved-long-garbage", + "format": "fdp", + "size": 326, + "sha256": "978e1df57d71dc44cca090e6fe25415e2e1f4e5050cacfc16610bce1d484875a", + "semantics": "Both sides negotiate, 1,000 and 64 bytes of garbage; a fixed 48-byte pattern drives the interleaving loop, then the final flush completes the handshake: hybrid." + }, + { + "file": "initiator-negotiates-responder-off", + "format": "fdp", + "size": 214, + "sha256": "9306059152bd46f04435b2b6dbe8bafe816102d4810cb2927a8d8af6f249fa83", + "semantics": "The initiator holds its version packet for an empty one; the final flush completes the handshake: legacy_peer, ECDH keys." + }, + { + "file": "initiator-negotiates-responder-off-interleaved", + "format": "fdp", + "size": 331, + "sha256": "bd1fbf465f64843526c618a09ed968336afb0ef65d95add1b2bf088fc0a75f2a", + "semantics": "The initiator negotiates, the responder is off; a fixed 48-byte pattern drives the interleaving loop, then the final flush completes the handshake: ECDH keys." + }, + { + "file": "initiator-off-responder-offers", + "format": "fdp", + "size": 214, + "sha256": "0e8c0e8cc82faf65135f7a440a25f65fcfffbfdc11b01ab35573ec3d35695f55", + "semantics": "The responder offers to an initiator that is off and ignores it; the final flush completes the handshake: ECDH keys." + }, + { + "file": "initiator-off-responder-offers-interleaved", + "format": "fdp", + "size": 359, + "sha256": "3b68416cfaec3415673d47174ec0512b788cbe3062109708336b8ade9929e445", + "semantics": "The responder offers to an initiator that is off; a fixed 48-byte pattern drives the interleaving loop, then the final flush completes the handshake: ECDH keys." + }, + { + "file": "initiator-fallback-responder-offers", + "format": "fdp", + "size": 214, + "sha256": "e1cd4020339b10060518069f2434f25dea38db0fe8cfa0e80885a946d32d7461", + "semantics": "An initiator in fallback sends plain v2 and never parses the offer; the final flush completes the handshake: ECDH keys." + }, + { + "file": "both-off", + "format": "fdp", + "size": 214, + "sha256": "3ad084a8e2f8494a595d9c44ebd9b72db47efda77c76d531c197ca6d686334f0", + "semantics": "Today's v2 on both sides; the final flush completes the handshake." + }, + { + "file": "both-off-interleaved", + "format": "fdp", + "size": 390, + "sha256": "9378972edb1ddf9b65d7ae33ec515901e11aa937ec1ebd355c4834d96dd6192a", + "semantics": "Today's v2 on both sides; a fixed 48-byte pattern drives the interleaving loop, then the final flush completes the handshake." + }, + { + "file": "initiator-fallback-responder-off", + "format": "fdp", + "size": 219, + "sha256": "f315c712be3bad0f7bf8eb9484ffa876675bc4bd90932dac84b13ac051e746ad", + "semantics": "Plain v2 on both sides; the final flush completes the handshake." + }, + { + "file": "initiator-fallback-responder-offers-interleaved", + "format": "fdp", + "size": 343, + "sha256": "cec2460db80a9d17fcd35bb12215b7517034cf9d732b0e328f6a6e2026fe7aff", + "semantics": "An initiator in fallback never parses the offer; a fixed 48-byte pattern drives the interleaving loop, then the final flush completes the handshake: ECDH keys." + } + ], + "p2p_v2_pq_malicious_peer": [ + { + "file": "initiator-hybrid", + "format": "fdp", + "size": 90, + "sha256": "96925cdab372557e025a9cac0dc3b5e3ca71d1b4b9499077b0e7f1d5a6e9c211", + "semantics": "The transport initiates; we offer honestly and switch on its accept: hybrid, confirmed, messages both ways." + }, + { + "file": "responder-hybrid", + "format": "fdp", + "size": 90, + "sha256": "796086e85f8379e80d9f7cdb75f60544d260051be35af905e966c33773bde94c", + "semantics": "The transport offers; we accept honestly and switch: hybrid, confirmed, messages both ways." + }, + { + "file": "initiator-hybrid-extras-fragmented", + "format": "fdp", + "size": 309, + "sha256": "f45f8b17efcd2bfc33d6ade02c70da02d30dbdd2bfdf1d19a1fe69a65ef16eec", + "semantics": "Maximum garbage both ways, own records inside decoys, an offer between unknown and duplicate records, delivered in 97-byte and then 1-byte pieces: hybrid." + }, + { + "file": "responder-hybrid-extras", + "format": "fdp", + "size": 97, + "sha256": "bb33f24d2a9e2696ee9428633897934427610f8681bc2e65e941ec010a88afcc", + "semantics": "Decoys, then an accept between unknown and duplicate records: hybrid." + }, + { + "file": "initiator-ek-length", + "format": "fdp", + "size": 83, + "sha256": "1727485faecda04ec7f676fe38b5dd837d06554f4bc76cc0e0b91a3665860467", + "semantics": "A 1,567-byte offer: the transport closes with ek_length and never sends its version packet." + }, + { + "file": "initiator-ek-modulus", + "format": "fdp", + "size": 83, + "sha256": "032bd86ffdea759d3b842e38891187a844f2d27713e5d5704fff6086d06bca4f", + "semantics": "An offer whose coefficients are all above q: the transport closes with ek_modulus." + }, + { + "file": "responder-ct-length", + "format": "fdp", + "size": 83, + "sha256": "42661de0c7b810020c0c76d463661a203c262faab397d6f3e9c09dd40fc7281a", + "semantics": "A 1,569-byte accept: the transport closes with ct_length." + }, + { + "file": "responder-corrupt-ct", + "format": "fdp", + "size": 90, + "sha256": "f7ba2b5a427de4fb48e9fe038386188f5a28d1ff8d3d1a270878135bcbf46c0d", + "semantics": "An accept with one ciphertext bit flipped: the transport switches to other keys and our confirmation fails." + }, + { + "file": "responder-disconnect-after-offer", + "format": "fdp", + "size": 81, + "sha256": "612b7d3ab9f066d1a40a408047aac95b3d78ddd9408a716ac0ecf52ea5b34737", + "semantics": "The transport offers and we disconnect before sending our version packet." + }, + { + "file": "initiator-confirmation-length", + "format": "fdp", + "size": 86, + "sha256": "2ca916c42ec30c4e55391852d1e45b46666d312e15ebd821ff4d1da9aa3db3ec", + "semantics": "An honest exchange, then a nonzero-length first hybrid packet: the transport fails it at byte 3." + }, + { + "file": "initiator-confirmation-tag", + "format": "fdp", + "size": 86, + "sha256": "23721132e8dba8e0d357b1f99ca1a51fd44e6b8a2e8b79309f629d8fb1bc23b4", + "semantics": "An honest offer followed by a duplicate record, then a confirmation with a damaged header or tag." + }, + { + "file": "responder-confirmation-not-decoy", + "format": "fdp", + "size": 86, + "sha256": "2b7d387a3b986e3dfea29386069868d90441e69bf02e869adf920b4fcc79e185", + "semantics": "An honest exchange, then an empty first hybrid packet without the ignore bit." + }, + { + "file": "responder-legacy-peer-burst", + "format": "fdp", + "size": 89, + "sha256": "079588a0165c1ca65c75ac701e625169401a08909479b05ca704b1f18a6a3102", + "semantics": "We ignore the offer like a legacy peer and send application messages at once: plain v2, legacy_peer." + }, + { + "file": "initiator-fallback-ignores-offer", + "format": "fdp", + "size": 88, + "sha256": "549406bb7e9c49617e70b98e2bb336f2a8424d6e1fb153a8009a527eca77bc91", + "semantics": "An initiator in fallback sends plain v2 at once and never parses our offer: no ML-KEM operation." + }, + { + "file": "initiator-noncompliant-responder", + "format": "fdp", + "size": 85, + "sha256": "e2156d55b4e66ccf22a17b4c4d7bb0d66d6b8000f2db1322111c7f6dd122bc9c", + "semantics": "We offer and then send a decoy under the ECDH keys where our confirmation belongs: the transport fails it." + }, + { + "file": "responder-off-unsolicited-accept", + "format": "fdp", + "size": 87, + "sha256": "600263bb14d01a4deefa4ec87a235eae35154b976a7866f594bd4635bb7578b4", + "semantics": "A responder that is off gets an accept-shaped record it never asked for: ignored, plain v2." + } + ], + "pq_records": [ + { + "file": "offer", + "format": "raw", + "size": 1572, + "sha256": "4db1dcc35718f86fbfacd7a3169154ce9b90ac4899d8e1ba77e607e807829335", + "semantics": "The ordinary offer of vector 0 of src/test/data/pq_transport_vectors.json: fd 21 06 f0 || ek." + }, + { + "file": "accept", + "format": "raw", + "size": 1572, + "sha256": "f55f95816bc7d6c6e5f6879842f244866bb908c313cb7065fd610f16d38638cd", + "semantics": "The ordinary accept of vector 0: fd 21 06 f0 || ct." + }, + { + "file": "offer-duplicate-unknown", + "format": "raw", + "size": 3148, + "sha256": "ec7fa0f262d470033d09f505582b1025f84a482d55b66fc3331a4e235c43cd10", + "semantics": "Vector 2's responder contents: its offer, a second valid offer and an unknown 0xf1 record. The first wins." + }, + { + "file": "unknown-then-accept", + "format": "raw", + "size": 1578, + "sha256": "144903f355c909a634fc4807c3a0ec127296f12892493ffb2fae80e771078f9e", + "semantics": "Vector 2's initiator contents: an unknown 0x01 record, then the accept." + }, + { + "file": "reserved-and-unknown-headers", + "format": "raw", + "size": 7, + "sha256": "6cf59e77bd2bad687e0f9dd4576f15cfa41ee211c670b123613ca60183984c2e", + "semantics": "Valid grammar without an own record: no features." + }, + { + "file": "empty-own-record", + "format": "raw", + "size": 2, + "sha256": "6cc0ab95752bf25ec58c91b1d603c5eb41b8fbd7ee1694d5a85005056efb1ab3", + "semantics": "An own record with an empty payload: parses, and fails the role's length check later." + }, + { + "file": "short-offer", + "format": "raw", + "size": 1571, + "sha256": "ce05ab17bb9a676487edda33fdbfd01306de9e7aa41ef39ef2a0094c2f82be2a", + "semantics": "A 1,567-byte own payload (ek_length or ct_length)." + }, + { + "file": "long-offer", + "format": "raw", + "size": 1573, + "sha256": "ebc5c90456da104eae9bd81785e97a6d0031aede95adaae5b5f880e6dd4fb0f5", + "semantics": "A 1,569-byte own payload (ek_length or ct_length)." + }, + { + "file": "malformed-first-then-offer", + "format": "raw", + "size": 1576, + "sha256": "7933842445af30e309967eb36ccec32f3ebd76f062aadbba6cb2f9e7b369b765", + "semantics": "A malformed first own record before a valid one: the first wins." + }, + { + "file": "offer-trailing-byte", + "format": "raw", + "size": 1573, + "sha256": "01e1fbb6a72ae6bd158ef5902e60d37de318f2d831271ec7eed4179f473d92fe", + "semantics": "A valid offer followed by a truncated CompactSize: invalid grammar, no features." + }, + { + "file": "zero-length", + "format": "raw", + "size": 7, + "sha256": "e034a290523f42c301c35ac085df2ed5293c495c58689bbff1b939f071f22507", + "semantics": "A record with len = 0 after a valid one: invalid grammar." + }, + { + "file": "one-past-end", + "format": "raw", + "size": 3, + "sha256": "19b27e599c63eef9fedd1c64bbb785530914c07f4f03ab723eae7474994d3c51", + "semantics": "A length one byte past the end of the contents: invalid grammar." + }, + { + "file": "truncated-wide-length", + "format": "raw", + "size": 6, + "sha256": "0d8bf907c7f5c26d45de142db505a5389e1b79c301614ba42178e88275eb2d13", + "semantics": "A valid record, then a 0xfe CompactSize cut off after two of its four bytes: invalid grammar." + }, + { + "file": "non-canonical-length", + "format": "raw", + "size": 5, + "sha256": "0682a6be8935622a35773009dfb0e26c0857c4e72424029c526122b1dec714cd", + "semantics": "Length 2 in a 3-byte CompactSize: invalid grammar." + }, + { + "file": "oversized-length", + "format": "raw", + "size": 7, + "sha256": "a70a4eb5f4febe8115e5b1feefcf4591c2589f5e5da8b1889ab858f7aa51eca9", + "semantics": "Length MAX_SIZE + 1 (0x02000001), which ReadCompactSize rejects: invalid grammar." + }, + { + "file": "minimal-unknown-records", + "format": "raw", + "size": 2001, + "sha256": "38a04844262d1d0d0a74e3ad0989c2a3cb206b859f8f084ff774ed7d5fca85cc", + "semantics": "1,000 one-byte unknown records ending in a truncated one: invalid grammar after a long valid prefix." + } + ], "pqc": [ { "file": "legacy-generated-valid-sign-verify", diff --git a/test/fuzz/qbit_corpora/README.md b/test/fuzz/qbit_corpora/README.md index 01a509aea3..426a8218c6 100644 --- a/test/fuzz/qbit_corpora/README.md +++ b/test/fuzz/qbit_corpora/README.md @@ -1,7 +1,8 @@ # qbit fuzz seed corpora Flat binary seed inputs for the qbit-specific fuzz targets that have no -upstream `qa-assets` corpus: +upstream `qa-assets` corpus, and for the v2 transport targets, whose +`qa-assets` inputs predate the hybrid negotiation: | Target | Directory | | --- | --- | @@ -12,6 +13,10 @@ upstream `qa-assets` corpus: | `mlkem` | [`mlkem/`](mlkem) | | `mlkem_backend_diff` | [`mlkem_backend_diff/`](mlkem_backend_diff) | | `p2mr_script` | [`p2mr_script/`](p2mr_script) | +| `p2p_transport_bidirectional_v1v2` | [`p2p_transport_bidirectional_v1v2/`](p2p_transport_bidirectional_v1v2) | +| `p2p_transport_bidirectional_v2` | [`p2p_transport_bidirectional_v2/`](p2p_transport_bidirectional_v2) | +| `p2p_v2_pq_malicious_peer` | [`p2p_v2_pq_malicious_peer/`](p2p_v2_pq_malicious_peer) | +| `pq_records` | [`pq_records/`](pq_records) | | `pqc` | [`pqc/`](pqc) | ## Provenance @@ -62,6 +67,35 @@ The fixture bodies check that an unmodified fixture verifies. `pqc` also checks that any modified signature, public key or message is rejected. Legacy and generation seeds keep key generation and signing covered in every replay. +`pq_records` takes the whole input, untagged (`raw`), as BIP324 version-packet +contents: records `CompactSize(len) || header || payload`. The seeds are the +offer and accept records of `src/test/data/pq_transport_vectors.json`, records +of the wrong length, duplicate and unknown records, and framing errors that make +the whole contents invalid. The harness checks the parser against an +independent reading of the grammar. + +`p2p_transport_bidirectional_v2` and `p2p_transport_bidirectional_v1v2` use +their harness's untagged layout: each v2 side's key, garbage and ellswift +entropy, then its hybrid negotiation mode (off, negotiate or fallback for an +initiator; off or negotiate for a responder, which never falls back) and the +32-byte seed of its ML-KEM entropy, then the simulation steps. The seeds cover +every pair of modes and the v1 detection; some end before the simulation loop, +so the final flush completes the handshake, and some add a fixed pattern that +drives the loop first. The v2 harness checks that the keys are hybrid on both +sides exactly when both negotiate. + +`p2p_v2_pq_malicious_peer` drives a v2 transport from a malicious peer that +completes the ECDH part honestly, so every packet it sends authenticates, and +chooses the negotiation plaintext. Its layout: the tested transport's role and +mode, both keys, both garbage lengths and a seed (everything else random comes +from it), then a script of steps, each a choice byte and a fixed-width +parameter: send our key, terminator, a decoy, a version packet variant, a +confirmation variant or an application packet; deliver or take some bytes; +flush; give the transport a message; send unauthenticated bytes; disconnect. +The seeds script honest hybrid exchanges in both roles, each malformed record +and confirmation failure, a damaged ciphertext, a legacy peer, a fallback +initiator, an unsolicited record and a disconnect while the offer is pending. + ## Use in CI [`overlay.py`](overlay.py) copies each seed to @@ -75,6 +109,6 @@ from a reused corpus by hand if they are no longer wanted. A target path that is not a directory, or a symlink, directory or other non-regular file at a managed name, is an error; all target paths and managed names are checked before any file is written. -`test/fuzz/test_runner.py --require_qbit_corpus` then fails unless all eight +`test/fuzz/test_runner.py --require_qbit_corpus` then fails unless all twelve targets are selected, each has at least one regular input file, and the fuzz binary reports replaying them. diff --git a/test/fuzz/qbit_corpora/generate_seeds.py b/test/fuzz/qbit_corpora/generate_seeds.py index db6942e068..a0e741cb8a 100755 --- a/test/fuzz/qbit_corpora/generate_seeds.py +++ b/test/fuzz/qbit_corpora/generate_seeds.py @@ -650,6 +650,268 @@ def mlkem_cases() -> list[dict]: ] +# --- pq_records --------------------------------------------------------------- + +PQ_VECTORS = CORPORA_DIR.parents[2] / "src" / "test" / "data" / "pq_transport_vectors.json" +PQ_MLKEM1024 = 0xF0 + + +def compact_size(n: int) -> bytes: + if n < 0xFD: + return bytes([n]) + if n <= 0xFFFF: + return b"\xfd" + n.to_bytes(2, "little") + if n <= 0xFFFFFFFF: + return b"\xfe" + n.to_bytes(4, "little") + return b"\xff" + n.to_bytes(8, "little") + + +def pq_record(header: int, payload: bytes) -> bytes: + """A version-contents record: CompactSize(1 + len(payload)) || header || payload.""" + return compact_size(1 + len(payload)) + bytes([header]) + payload + + +def pq_records_cases() -> list[dict]: + vectors = json.loads(PQ_VECTORS.read_text(encoding="utf-8"))["vectors"] + ordinary, extra = vectors[0], vectors[2] + offer = pq_record(PQ_MLKEM1024, bytes.fromhex(ordinary["ek"])) + accept = pq_record(PQ_MLKEM1024, bytes.fromhex(ordinary["ct"])) + unknown = pq_record(0xF1, b"qbit") + minimal_unknown = pq_record(0x01, b"") * 1000 + return [ + case("offer", "raw", offer, + "The ordinary offer of vector 0 of src/test/data/pq_transport_vectors.json: fd 21 06 f0 || ek."), + case("accept", "raw", accept, "The ordinary accept of vector 0: fd 21 06 f0 || ct."), + case("offer-duplicate-unknown", "raw", bytes.fromhex(extra["contents_responder"]), + "Vector 2's responder contents: its offer, a second valid offer and an unknown 0xf1 record. The first wins."), + case("unknown-then-accept", "raw", bytes.fromhex(extra["contents_initiator"]), + "Vector 2's initiator contents: an unknown 0x01 record, then the accept."), + case("reserved-and-unknown-headers", "raw", pq_record(0x00, b"") + pq_record(0xF1, b"\xaa") + pq_record(0xFF, b""), + "Valid grammar without an own record: no features."), + case("empty-own-record", "raw", pq_record(PQ_MLKEM1024, b""), + "An own record with an empty payload: parses, and fails the role's length check later."), + case("short-offer", "raw", pq_record(PQ_MLKEM1024, bytes.fromhex(ordinary["ek"])[:-1]), + "A 1,567-byte own payload (ek_length or ct_length)."), + case("long-offer", "raw", pq_record(PQ_MLKEM1024, bytes.fromhex(ordinary["ek"]) + b"\x00"), + "A 1,569-byte own payload (ek_length or ct_length)."), + case("malformed-first-then-offer", "raw", pq_record(PQ_MLKEM1024, b"\xaa\xbb") + offer, + "A malformed first own record before a valid one: the first wins."), + case("offer-trailing-byte", "raw", offer + b"\xfd", + "A valid offer followed by a truncated CompactSize: invalid grammar, no features."), + case("zero-length", "raw", unknown + b"\x00", "A record with len = 0 after a valid one: invalid grammar."), + case("one-past-end", "raw", b"\x03\xf0\xaa", + "A length one byte past the end of the contents: invalid grammar."), + case("truncated-wide-length", "raw", b"\x02\x01\xaa\xfe\x00\x00", + "A valid record, then a 0xfe CompactSize cut off after two of its four bytes: invalid grammar."), + case("non-canonical-length", "raw", b"\xfd\x02\x00\xf0\xaa", + "Length 2 in a 3-byte CompactSize: invalid grammar."), + case("oversized-length", "raw", b"\xfe\x01\x00\x00\x02\xf0\xaa", + "Length MAX_SIZE + 1 (0x02000001), which ReadCompactSize rejects: invalid grammar."), + case("minimal-unknown-records", "raw", minimal_unknown + b"\x05", + "1,000 one-byte unknown records ending in a truncated one: invalid grammar after a long valid prefix."), + ] + + +# --- p2p_transport_bidirectional_v2, p2p_transport_bidirectional_v1v2 --------- + +SECP256K1_ORDER = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141 +V2_MAX_GARBAGE_LEN = 4095 +PQ_MODES = {"off": 0, "negotiate": 1, "fallback": 2} +# A responder never falls back, so the harness picks its mode from the first two only. +PQ_RESPONDER_MODES = 2 +SIMULATION_MAX_MESSAGE = 75000 +SIMULATION_ACTIONS = 6 + + +def transport_key(label: str) -> bytes: + key = hashlib.sha256(f"qbit fuzz seed key/{label}".encode()).digest() + assert 0 < int.from_bytes(key, "big") < SECP256K1_ORDER + return key + + +def v2_transport_side(enc: FdpEncoder, label: str, garbage_len: int, mode: str, initiator: bool) -> None: + """MakeV2Transport: key, compression flag, garbage, ellswift entropy, PQ mode and ML-KEM seed.""" + enc.raw(transport_key(label)) + enc.boolean(True) + enc.integral(garbage_len, SIZE_T, 0, V2_MAX_GARBAGE_LEN) + if garbage_len <= 64: + enc.raw(pattern(garbage_len, 0x31)) + enc.raw(hashlib.sha256(f"qbit fuzz seed ent/{label}".encode()).digest()) + if not initiator and PQ_MODES[mode] >= PQ_RESPONDER_MODES: + raise ValueError(f"{label}: a responder cannot be in {mode} mode") + enc.pick(PQ_MODES[mode], len(PQ_MODES) if initiator else PQ_RESPONDER_MODES) + enc.raw(hashlib.sha256(f"qbit fuzz seed mlkem/{label}".encode()).digest()) + + +def transport_v2_input(name: str, initiator: tuple[int, str], responder: tuple[int, str], tail: int) -> bytes: + enc = FdpEncoder() + enc.integral(int.from_bytes(hashlib.sha256(name.encode()).digest()[:8], "big"), UINT64) + v2_transport_side(enc, f"{name}/initiator", *initiator, initiator=True) + v2_transport_side(enc, f"{name}/responder", *responder, initiator=False) + # Both sides' first message ("version") sizes. + enc.integral(1000, UINT32, 0, SIMULATION_MAX_MESSAGE) + enc.integral(200, UINT32, 0, SIMULATION_MAX_MESSAGE) + # Further bytes drive the interleaving loop from the back; the final flush completes the rest. + enc.raw(pattern(tail, 0x47)) + return enc.build() + + +def transport_v2_cases() -> list[dict]: + flush = "the final flush completes the handshake" + loop = "a fixed 48-byte pattern drives the interleaving loop, then the final flush completes the handshake" + cases = [ + ("negotiate-both", (0, "negotiate"), (0, "negotiate"), 0, + f"Both sides negotiate, no garbage; {flush}: hybrid keys, confirmed both ways."), + ("negotiate-both-max-garbage", (V2_MAX_GARBAGE_LEN, "negotiate"), (V2_MAX_GARBAGE_LEN, "negotiate"), 0, + f"Both sides negotiate with 4,095 bytes of garbage each, so the held version packet's AAD is the longest; {flush}."), + ("negotiate-both-interleaved", (64, "negotiate"), (17, "negotiate"), 48, + f"Both sides negotiate, 64 and 17 bytes of garbage; {loop}: hybrid."), + ("negotiate-both-interleaved-long-garbage", (1000, "negotiate"), (64, "negotiate"), 48, + f"Both sides negotiate, 1,000 and 64 bytes of garbage; {loop}: hybrid."), + ("initiator-negotiates-responder-off", (0, "negotiate"), (0, "off"), 0, + f"The initiator holds its version packet for an empty one; {flush}: legacy_peer, ECDH keys."), + ("initiator-negotiates-responder-off-interleaved", (64, "negotiate"), (5, "off"), 48, + f"The initiator negotiates, the responder is off; {loop}: ECDH keys."), + ("initiator-off-responder-offers", (0, "off"), (0, "negotiate"), 0, + f"The responder offers to an initiator that is off and ignores it; {flush}: ECDH keys."), + ("initiator-off-responder-offers-interleaved", (33, "off"), (64, "negotiate"), 48, + f"The responder offers to an initiator that is off; {loop}: ECDH keys."), + ("initiator-fallback-responder-offers", (0, "fallback"), (0, "negotiate"), 0, + f"An initiator in fallback sends plain v2 and never parses the offer; {flush}: ECDH keys."), + ("both-off", (0, "off"), (0, "off"), 0, f"Today's v2 on both sides; {flush}."), + ("both-off-interleaved", (64, "off"), (64, "off"), 48, f"Today's v2 on both sides; {loop}."), + ("initiator-fallback-responder-off", (2, "fallback"), (3, "off"), 0, f"Plain v2 on both sides; {flush}."), + ("initiator-fallback-responder-offers-interleaved", (64, "fallback"), (17, "negotiate"), 48, + f"An initiator in fallback never parses the offer; {loop}: ECDH keys."), + ] + return [case(name, "fdp", transport_v2_input(name, initiator, responder, tail), semantics) + for name, initiator, responder, tail, semantics in cases] + + +def transport_v1v2_input(name: str, responder: tuple[int, str], v1_message: bool) -> bytes: + enc = FdpEncoder() + enc.integral(int.from_bytes(hashlib.sha256(name.encode()).digest()[:8], "big"), UINT64) + v2_transport_side(enc, f"{name}/responder", *responder, initiator=False) + enc.integral(500, UINT32, 0, SIMULATION_MAX_MESSAGE) + enc.integral(300, UINT32, 0, SIMULATION_MAX_MESSAGE) + if v1_message: + # One loop step: the v1 initiator takes its version message (the next message's type index + # and size are drawn when it is queued); the final flush delivers it. + enc.pick(0, SIMULATION_ACTIONS) + enc.integral(0, UINT8) + enc.integral(100, UINT32, 0, SIMULATION_MAX_MESSAGE) + return enc.build() + + +def transport_v1v2_cases() -> list[dict]: + return [ + case("v1-version-to-negotiating-responder", "fdp", transport_v1v2_input("v1-negotiate", (0, "negotiate"), True), + "A v1 initiator's version message reaches a responder with the negotiation on: v1 detected, no key " + "generated."), + case("v1-version-to-off-responder", "fdp", transport_v1v2_input("v1-off", (64, "off"), True), + "A v1 initiator's version message reaches a responder with the negotiation off: v1 detected."), + case("v1-version-to-negotiating-responder-max-garbage", "fdp", + transport_v1v2_input("v1-negotiate-max-garbage", (V2_MAX_GARBAGE_LEN, "negotiate"), True), + "A v1 initiator's version message reaches a negotiating responder configured with 4,095 bytes of garbage, " + "which it never sends: v1 detected."), + case("v1-silent-negotiating-responder", "fdp", transport_v1v2_input("v1-silent", (0, "negotiate"), False), + "A v1 initiator that sends nothing: the negotiating responder stays undecided and sends nothing."), + ] + + +# --- p2p_v2_pq_malicious_peer --------------------------------------------------- + +# The harness's steps, in its CallOneOf order, and the range of each step's parameter. +MALPEER_STEPS = { + "key": None, "terminator": None, "decoy": (UINT8, 0, 3), "version": (UINT8, 0, 9), + "confirmation": (UINT8, 0, 4), "application": (UINT8, 0, 3), "deliver": (UINT32, 1, 1 << 16), + "collect": (UINT32, 1, 1 << 16), "flush": None, "queue": (UINT8, 0, 3), "raw": (UINT8, 1, 64), + "disconnect": None, +} +MALPEER_INITIATOR_MODES = {"off": 0, "negotiate": 1, "fallback": 2} +MALPEER_RESPONDER_MODES = {"off": 0, "negotiate": 1} + + +def malpeer_input(name: str, test_initiator: bool, mode: str, steps: list, garbage: tuple[int, int] = (0, 0)) -> bytes: + """The tested transport's role and mode, both keys, both garbage lengths, a seed, then the steps.""" + enc = FdpEncoder() + modes = MALPEER_INITIATOR_MODES if test_initiator else MALPEER_RESPONDER_MODES + enc.integral((modes[mode] << 1) | int(test_initiator), UINT8) + enc.raw(transport_key(f"{name}/transport")) + enc.raw(transport_key(f"{name}/peer")) + enc.integral(garbage[0], UINT16, 0, V2_MAX_GARBAGE_LEN) + enc.integral(garbage[1], UINT16, 0, V2_MAX_GARBAGE_LEN) + enc.integral(int.from_bytes(hashlib.sha256(name.encode()).digest()[:8], "big"), UINT64) + names = list(MALPEER_STEPS) + for step in steps: + step_name, param = (step, None) if isinstance(step, str) else step + enc.pick(names.index(step_name), len(names)) + spec = MALPEER_STEPS[step_name] + if spec is not None: + type_name, lo, hi = spec + enc.integral(param, type_name, lo, hi) + return enc.build() + + +def malpeer_cases() -> list[dict]: + # Version variants (see SendVersion): as responder 5, 6 and 9 are honest offers, 3 a key that + # fails the modulus check, 8 a 1,567-byte key; as initiator 5 and 6 are honest accepts after + # which we switch, 3 a blind ciphertext, 4 a damaged one, 7 an honest accept without our + # switch, 8 a 1,569-byte ciphertext. Confirmation variants: 0 honest, 1 nonzero length, 2 not + # a decoy, 4 damaged tag. + as_responder = ["flush", "key", "terminator"] # the transport initiates + as_initiator = ["key", "flush", "terminator"] # the transport responds and offers + finish = [("confirmation", 0), ("application", 0), ("queue", 0), "flush"] + cases = [ + ("initiator-hybrid", True, "negotiate", as_responder + [("version", 5), "flush"] + finish, (0, 0), + "The transport initiates; we offer honestly and switch on its accept: hybrid, confirmed, messages both ways."), + ("responder-hybrid", False, "negotiate", as_initiator + [("version", 5), "flush"] + finish, (0, 0), + "The transport offers; we accept honestly and switch: hybrid, confirmed, messages both ways."), + ("initiator-hybrid-extras-fragmented", True, "negotiate", + as_responder + [("decoy", 1), ("decoy", 2), ("version", 6)] + [("deliver", 97)] * 40 + + ["flush", ("confirmation", 0), ("decoy", 0), ("application", 1)] + [("deliver", 1)] * 30 + + [("collect", 5), ("queue", 1), "flush"], (V2_MAX_GARBAGE_LEN, V2_MAX_GARBAGE_LEN), + "Maximum garbage both ways, own records inside decoys, an offer between unknown and duplicate records, " + "delivered in 97-byte and then 1-byte pieces: hybrid."), + ("responder-hybrid-extras", False, "negotiate", + as_initiator + [("decoy", 0), ("decoy", 1), ("version", 6), ("collect", 3), "flush"] + finish, + (V2_MAX_GARBAGE_LEN, 17), "Decoys, then an accept between unknown and duplicate records: hybrid."), + ("initiator-ek-length", True, "negotiate", as_responder + [("version", 8), "flush"], (0, 0), + "A 1,567-byte offer: the transport closes with ek_length and never sends its version packet."), + ("initiator-ek-modulus", True, "negotiate", as_responder + [("version", 3), "flush"], (0, 0), + "An offer whose coefficients are all above q: the transport closes with ek_modulus."), + ("responder-ct-length", False, "negotiate", as_initiator + [("version", 8), "flush"], (0, 0), + "A 1,569-byte accept: the transport closes with ct_length."), + ("responder-corrupt-ct", False, "negotiate", as_initiator + [("version", 4), "flush"] + finish, (0, 0), + "An accept with one ciphertext bit flipped: the transport switches to other keys and our confirmation fails."), + ("responder-disconnect-after-offer", False, "negotiate", as_initiator + ["disconnect"], (0, 0), + "The transport offers and we disconnect before sending our version packet."), + ("initiator-confirmation-length", True, "negotiate", + as_responder + [("version", 5), "flush", ("confirmation", 1), "flush"], (0, 0), + "An honest exchange, then a nonzero-length first hybrid packet: the transport fails it at byte 3."), + ("initiator-confirmation-tag", True, "negotiate", + as_responder + [("version", 9), "flush", ("confirmation", 4), "flush"], (0, 0), + "An honest offer followed by a duplicate record, then a confirmation with a damaged header or tag."), + ("responder-confirmation-not-decoy", False, "negotiate", + as_initiator + [("version", 5), "flush", ("confirmation", 2), "flush"], (0, 0), + "An honest exchange, then an empty first hybrid packet without the ignore bit."), + ("responder-legacy-peer-burst", False, "negotiate", + as_initiator + [("version", 0), ("application", 0), ("application", 1), ("queue", 0), "flush"], (0, 64), + "We ignore the offer like a legacy peer and send application messages at once: plain v2, legacy_peer."), + ("initiator-fallback-ignores-offer", True, "fallback", + as_responder + ["flush", ("version", 5), ("application", 0), ("queue", 0), "flush"], (5, 5), + "An initiator in fallback sends plain v2 at once and never parses our offer: no ML-KEM operation."), + ("initiator-noncompliant-responder", True, "negotiate", + as_responder + [("version", 5), ("decoy", 2), "flush"], (0, 0), + "We offer and then send a decoy under the ECDH keys where our confirmation belongs: the transport fails it."), + ("responder-off-unsolicited-accept", False, "off", + as_initiator + [("version", 3), ("application", 0), ("queue", 2), "flush"], (0, 0), + "A responder that is off gets an accept-shaped record it never asked for: ignored, plain v2."), + ] + return [case(name, "fdp", malpeer_input(name, ti, mode, steps, garbage), semantics) + for name, ti, mode, steps, garbage, semantics in cases] + + def case(name: str, fmt: str, data: bytes, semantics: str) -> dict: return {"name": name, "format": fmt, "data": data, "semantics": semantics} @@ -662,6 +924,10 @@ def case(name: str, fmt: str, data: bytes, semantics: str) -> dict: "mlkem": mlkem_cases, "mlkem_backend_diff": mlkem_cases, "p2mr_script": p2mr_cases, + "p2p_transport_bidirectional_v1v2": transport_v1v2_cases, + "p2p_transport_bidirectional_v2": transport_v2_cases, + "p2p_v2_pq_malicious_peer": malpeer_cases, + "pq_records": pq_records_cases, "pqc": pqc_cases, } @@ -681,6 +947,7 @@ def build_all() -> tuple[dict, dict[str, list[dict]]]: "is not 0xF, then the FuzzedDataProvider layout of the cached-fixture body.", "qbfx-v1-generation": "b'QBFX' + target tag + 0x01 + selector whose low nibble is 0xF, then the legacy " "FuzzedDataProvider layout.", + "raw": "The whole input is the data the target parses, without a FuzzedDataProvider layout.", }, "targets": { target: [ diff --git a/test/fuzz/qbit_corpora/overlay.py b/test/fuzz/qbit_corpora/overlay.py index 0533e6507f..72d8b52a2a 100755 --- a/test/fuzz/qbit_corpora/overlay.py +++ b/test/fuzz/qbit_corpora/overlay.py @@ -32,6 +32,10 @@ "mlkem", "mlkem_backend_diff", "p2mr_script", + "p2p_transport_bidirectional_v1v2", + "p2p_transport_bidirectional_v2", + "p2p_v2_pq_malicious_peer", + "pq_records", "pqc", ) MANIFEST_NAME = "MANIFEST.json" diff --git a/test/fuzz/qbit_corpora/p2p_transport_bidirectional_v1v2/v1-silent-negotiating-responder b/test/fuzz/qbit_corpora/p2p_transport_bidirectional_v1v2/v1-silent-negotiating-responder new file mode 100644 index 0000000000000000000000000000000000000000..afd6e5ba6947ec93e359d34c0f760c1316786294 GIT binary patch literal 114 zcmV-&0FD3UqJiOdOSwKn-=CWEFLMhAeh=tP<@G{U@5as)Igl1WR-i{Ng+Wo|;?bi` z#lz#*;`Aw5+rDYC>(Cpj6DHk|@-|8&>yqjz1UCat&ZqNJpcz?ejvFCh?^|wa)M&yi U0RZ#?009610qcUC>EC3c5-5c?N&o-= literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/p2p_transport_bidirectional_v1v2/v1-version-to-negotiating-responder b/test/fuzz/qbit_corpora/p2p_transport_bidirectional_v1v2/v1-version-to-negotiating-responder new file mode 100644 index 0000000000000000000000000000000000000000..3312cdb7dfb53c26a8e7b8e0c463b6eede2e7f8f GIT binary patch literal 119 zcmV--0EqvT2rq__Ml<0H5*Z6nI=@5*U@bf_Xr4tPtnK$M8Aqf4&D1}t;<_ z?kem|2}_|gjjL~nP#$>C1YFW;&TD}MP{53ostMrd<3 zO$+O5lPooA!l<{Ri0~(R$oR~z3~c=POcj1t)y?bMsYfc^kA|rQZi2WL>9OjP{5q~= Z00000ECB%Y0RRF24*{yiY^uuIri~BxJoEqn literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/p2p_transport_bidirectional_v1v2/v1-version-to-off-responder b/test/fuzz/qbit_corpora/p2p_transport_bidirectional_v1v2/v1-version-to-off-responder new file mode 100644 index 0000000000000000000000000000000000000000..e54445aacb7901c7aba426803b6fb2a38f6c155a GIT binary patch literal 183 zcmV;o07(DHSk&LZMo}Eeb~2k5zf!wR+-8D6z*q4r;M&InI3><8ZmZ}eVxQa@R+G>T zN{7S&K6|+JGHL+BO-Wynz(hp3D#sffoy7x3h z%99LOLYR|yoQi7^)8DZmE-&We*;ha==J>gxiOerH^)%kmJ67;6lmMqM?4j*?C78Vm lyb=_lmP;}~C&WEu00000ECB%Y0RR9%00HTQE+G_1AW<4-QiK2i literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/both-off b/test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/both-off new file mode 100644 index 0000000000000000000000000000000000000000..41fb37c2e40159ddd880391114aecb328bf2d421 GIT binary patch literal 214 zcmV;{04e{UXi1S4p>2Q0UD#&Od4A^u6~hi&&%{)4ccgG7Q=# z+70eLkX`ZOss{iZ?~T-Th_I9Z)&kU;K8~Ovlu8b6e%ztH`x0JZ2D$IH3DLb3LQem3 zjFYSRxcdWNiE(h1k%cCxhBw_E_{mwznywi>ni_8Y!Ll>FiNJX_fQ||b{6lmm=H6|WoSPO}}ze5>Th008I%00000 Q0RR910pC#O)$_fT%=3F{rvLx| literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/both-off-interleaved b/test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/both-off-interleaved new file mode 100644 index 0000000000000000000000000000000000000000..2ba2427ab569368ca6195b90c5944a76ea394fb7 GIT binary patch literal 390 zcmX?h!F1d6JcGx7TV}>>I?=_^`DnSU{Wawm_1~8!Y}cJRiHR#%8p=3$6e&dN4+m?~L$mLD|AYLn3YZtup{cgRNoe=3X&Xf$~c=TM|nh(H>L z8bQM5*gJF10=d-wHMCz?XUx<>j;-ZwC78WnwczbLWif86k_uQFa&?XgUP4uQcSRW} zWy&cW3*ls;Ja0OfsbBPho)_!fi#2?9&*+zx-E(9Ua2)Fy1UEOiL~EsXllC7@B_uKVY7IwQmSSl@Q)S=4K#sp8^r*9==f)d?I0IdfNLBTSsBw1 zY1D4@>NS5%v3sb-+s>uJ5RaO)ld z%1EW-;;Bp6n8%TYSS5E@>6%{7au=nu>_ZOcUg4XR!L31SNbMILu>V5H008I%00961 Q0RjL30Tcs1XjT!DQ39=G=l}o! literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/initiator-fallback-responder-offers-interleaved b/test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/initiator-fallback-responder-offers-interleaved new file mode 100644 index 0000000000000000000000000000000000000000..8e9fec4edeb924c27d96a264acccdffb41b0d44e GIT binary patch literal 343 zcmV-d0jU1l5$E5mak~(4$Jr@fa4(*7^3oZMFvT zN{7S&K6|+JGHL+BO-Wynz(hp3D#sffoy7x3^ z(dM~p$gW9H`_%G`NB#1=t3fO?7)bF`V1=~r9N0ZY3|9C}OLhp)qVZudZmZ}eVxQa@ zR+G>TN{7S&!30@Fj@LS-kGCm<@hnR%K%86XeQb^WcTatE-}kgyAfqcs`a0AZMm8|* zlvIZ)?n4akjQsbV0U`qvWoS-IM})!tJ9xG7FKnsiBVe7{7gUkX3rK~+|2%oN^Du3y p=OkgC+Za`n&kRY1!vH;c$N&K70{{UL009C(00E^}(HD-m62f2MqwW9z literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/initiator-negotiates-responder-off b/test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/initiator-negotiates-responder-off new file mode 100644 index 0000000000000000000000000000000000000000..2f95347f744b26ba77bf824b875bfb9f86a07658 GIT binary patch literal 214 zcmV;{04e_&W3b`417ob4Cf`zb#=OL&u`mg1M3@#9vY*-UWAf24u5;^Oxj;T9!pbk|hia|T@p^o|%%e47jz=ZQahhxxX|%BAsZ57Zx(aRd<~stkUKFx+#r zooz9}yY4|o?zl`49#Xc4q@c0^Q3y*gugz6Uv9v;od(55=nK9t(nFuqbpJKkB=Jqot zBM!!)%8ZuzM@9ymazLgJ4(#>98@F=Myb2K|ME2(dOC7CN62%D_QiCDL008I%00000 Q0RaF20g4`20nh|Q+hbr~!T$?uJT*mn`X-ApB4Qa+c9CLJE`w6ZmZ}eVxQa@R+G>T zN{7S&K6|+JGHL+BO-Wynz(hp3D#sffoy7x5I z%W@i^xV~FR&wnexnKeo_8t;g{O`=L>{>xJuga*R$Ok5~?Jf4kN`xRVX(IC0d*4e3s z3aE;O*&3st35k`^^V6R;j?)!rjPFZk*YirDCrg!(ZuN9MuMmL;ufPv6ZmZ}e?}fnx zzSs51XJ85(^ed8!FYv^=o<(jZ{1Op+fJHv9S8F|>v`*y zq1HXcBm0JqJ6Wo3u!0)#bV_`4Jn>RPUNTke3b(|)l{RiEP>E?KWMWHR*S_;lV?ZVX zMe*SYB`m;7%~Lt(_zA1^bS_o>Q5OBG`#9U`)=Y(+KxfVAuA5Gr=TLv9D0JwUAf>g| zWhEdPNZIoStvFFPOW2mkIyWiR&{njp|;=_m-U)GX8Dbo8E3P*>2746}!3KRK0t6N%?Nm!^VgyHh*|s zyKY@Kjgw8O{N>}J^RLE{pab~4JB~8h*X;aeT(I_)ddk9k(h*ZH@%wZhV|J+B_1z?I z-+lJLipld5r`7Gb%3bh$y_f9j$F+AST`u3Z%VxsF_g9on3x5O@IDcElIdwL_PVV0? zS!=cKS!zGab1U_H?fb~R<={W-imjjYb5=f6PMCX3JY?cIZjaVO|7|L_eKyEl^;{)! o-ffA{N#}Vy+YU3>R-IsAc)`rT=)l0ppvb`ZoKJdkarLQX0QDBEivR!s literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/negotiate-both b/test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/negotiate-both new file mode 100644 index 0000000000000000000000000000000000000000..3ea9a529f02f169c4f083ca83a1232e1ad2fb893 GIT binary patch literal 214 zcmV;{04e_t%+#1^B1s!t0_x*J()3t%3(W$?3iRE!lPq05ctsaUwU*^wvn}zyqvYBcP5ix7F2GM=7Pt|NT(`7;gfZUe{2008I%00961 Q0RaF20p0Adbj%J`XXt`v+W-In literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/negotiate-both-interleaved b/test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/negotiate-both-interleaved new file mode 100644 index 0000000000000000000000000000000000000000..d8e8e79fc2a237173037482913fd9708250ef466 GIT binary patch literal 343 zcmaE`@HK06YSOX)dm5gocUTLmUvZbId@sVP{z^eS$G3{xFmLq>)uj1%B*P|O;Pq-h z!f02$kgX@k^YL0SpWZwLSeVI;m_sxrg7I)l;^KpMyBIe~jrQxfT!2cx@ zA}J!D)<>Rn*g3&b>Jp3e^!zC=zQ(UqQaJgmxBE9I$9>6z@0Fk2GnHu3mCuwD=H3zynRt%dqxH~#o62pU4RTjKS4o_A nTOxGQc^=QU!wj}nCm0xBFf%X;GB7eaFfiKrwh5n$sPP8?Sl2rT;jc|2VM>woa{x&&NEyfwK6|@s}$Ex))ZmZ}eVxQa@R+G>TN{7S&K6|+J zGHL+BO-Wynz(hp3D#sffoy7x5dfLTXUYq-zT zYL?(zIkIuDO3TIPa>dz#2c%$klu;0z%4eQd5kl~dDi7hJ zNA4^~gu(thc(w8`Y^mlWV4d0*RFTdLNQJ`xJbAYBFm0;mBw?Q07*&$b3`vH=06luh Y008I%00BS%0RiX(0VXPT2APJdx0lD6rvLx| literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/negotiate-both-max-garbage b/test/fuzz/qbit_corpora/p2p_transport_bidirectional_v2/negotiate-both-max-garbage new file mode 100644 index 0000000000000000000000000000000000000000..6f09e28e944ad0bee00f88f50657d6b2e65f0ac4 GIT binary patch literal 214 zcmV;{04e_~2;1n`v@qA=lxZNl!2}pGe(|_8jT&Hx*dzPa;BWh2uo33CeT zeJNgX?9o>;cqXBXaX&6qK6Z_wSJV}{pz7LW21e}QR(BDyI889G_oZO=Vmf}%nHr(5 z5XG+;quwHrYFTFFHSScgguo4fMRia@6H(r(de~5c^xBlw=Aj1=ayN!$Qh2k9&E2xd z@c86APib&Ilooc18fVy=t;fO$%C)z*P1#o%F>e;%a*@NOQNHC@f^|a3008I%00I9G Q0RjIH0U`V}lN|2-n?i4_;N+@D|2l sc^}5=yvH8Kkqd(Oy_6O)9-Tymvd)uZF8t4qa@WVl)gR6Ol0 sA37V^e*h{QF~$0wh4emKCBIn+1Ox~P0|5XC-IJeAnGlV30000011s7j0ssI2 literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/initiator-ek-length b/test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/initiator-ek-length new file mode 100644 index 0000000000000000000000000000000000000000..bc3d7f074b742db5be97c5453797d279d8f7ba33 GIT binary patch literal 83 zcmV-Z0IdHzF~J4$V^yk-Te_3#P97(%7!$vwHk#Uj>i{2~65`DEP(w&}jt_U*W%Rsb poy5sM8;5tr_}!+wXs_>GrS8KB2m=8C2#bRv5!ahg>Hq)$00Wu@CHVjV literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/initiator-ek-modulus b/test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/initiator-ek-modulus new file mode 100644 index 0000000000000000000000000000000000000000..bc37e00bae2ac6bc8e086d6061da34bdfa9461c8 GIT binary patch literal 83 zcmV-Z0IdI!b_5c6?00SfxBtHNE literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/initiator-fallback-ignores-offer b/test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/initiator-fallback-ignores-offer new file mode 100644 index 0000000000000000000000000000000000000000..5bb104af2988e95962fd313b8bec0a4f563cc7f4 GIT binary patch literal 88 zcmV-e0H^+qK5_%5*XM91hFwQF)=bRGH|51 Q&lB_%wr}{)|DT^30CXNV761SM literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/initiator-noncompliant-responder b/test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/initiator-noncompliant-responder new file mode 100644 index 0000000000000000000000000000000000000000..6d959d383c44c7c3587ffaef1880e59230157834 GIT binary patch literal 85 zcmV-b0IL6)+2nmkc{lTLP0;rO^-UTsS`pn4cF(k5RsFur64D*Ng_zILrKpusWqreB r_*Hc&)Wsu<7$EPs`HpF#PW*=m0s;jC0RRY^a*p2S)>d->00003S_~(q literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/responder-confirmation-not-decoy b/test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/responder-confirmation-not-decoy new file mode 100644 index 0000000000000000000000000000000000000000..9deee9b3677d5c19981253084c5538298468e1f9 GIT binary patch literal 86 zcmV-c0IB~w>$C8_arq-DCa45f-t*uESSD{{5nm~x8r2MZIogp$%h^3>8eQ}Sm4rdQ sbjo$x4-xxS=*365K_uBmCc^Xx0t5&J0|5vC!-i>FHHTsk000000<26TMF0Q* literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/responder-corrupt-ct b/test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/responder-corrupt-ct new file mode 100644 index 0000000000000000000000000000000000000000..1e18ff61a12bd4e76e491b796b1c3df380056a11 GIT binary patch literal 90 zcmV-g0Hy!Y{JEcKgW*28jW{M}R+^&Hi-|q3aONSdRb-6FSx`f?7?#9JIA4EyO-JW$ wq*T8ys9kdJ>YAVF4?LGAtl8!W00{sE00amG0|5vC|J+Fyn;*ktf%Qsi_ItK?cBL~Cmt^XJDs-C~X00c|`+Akl` literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/responder-disconnect-after-offer b/test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/responder-disconnect-after-offer new file mode 100644 index 0000000000000000000000000000000000000000..afec4d1436fed27196268b6b7861942b5f6b7d8c GIT binary patch literal 81 zcmV-X0IvV!)Zq|G<+C{^*IURjKzq^>CD`RNm3Y;w-7h?u> nMCV%w&&-8Y6e4!yq8Q%-Ww7-N0SEx1QRcR`ZqsD|00002-*h4D literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/responder-hybrid b/test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/responder-hybrid new file mode 100644 index 0000000000000000000000000000000000000000..9f72a775ea06cfcd9b5daf411bbf0102b6c6f156 GIT binary patch literal 90 zcmV-g0HyzEq_n4aoYp<{X5~|KspedsN)Kg|(RE7b>)UoI6jr$RQJga0)hXB~eip=w wh=_?NweXR2_sQehqxqe6tbOPR00{sE00amH0|5vC(!0pgUxjUi000000(i zNk{+wd-F|%rW3N3Aoor+nD_Gt00{sE00amE00#yG0RjL50SExT;x_KcomvhN0RImH D3wtQ8 literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/responder-legacy-peer-burst b/test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/responder-legacy-peer-burst new file mode 100644 index 0000000000000000000000000000000000000000..b13b5a4c9655091a3fe99036a8bdfdcb38ffba99 GIT binary patch literal 89 zcmV-f0H*)76gtO4OUR;18&9#WqVE3ao}AO1%{Y8A4Gps*8`(WrS!JhsAq7!=S literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/responder-off-unsolicited-accept b/test/fuzz/qbit_corpora/p2p_v2_pq_malicious_peer/responder-off-unsolicited-accept new file mode 100644 index 0000000000000000000000000000000000000000..d38420d8cb95651639bb7c9404ff3c9ce3e4c0d9 GIT binary patch literal 87 zcmY#>quUmFd5TTP*_0RG57s%`E_H31-toZv$;-u?#RBxtrq7p-m~k_5YtFe|mqYqD rm|LEN^Q`Fj{@`c*fo|80{B9gfoD8hY%#0ii)mo`a4jt__3?Kjidzd83 literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/pq_records/accept b/test/fuzz/qbit_corpora/pq_records/accept new file mode 100644 index 0000000000000000000000000000000000000000..25edad2ef35baa579501cb70e04ad41b4716becd GIT binary patch literal 1572 zcmV+<2HW}lAqMbySBsME{#F>(-I9c&08T~giE+lFgua$`wFMTQ`ErF zYr>o^kWV1nj&22K)}b&u(U8}`ho^0(G^XIJ5%l+jmdX)^EQhSH{3kGBN1BOsh=$Ox z*MoJj?eiDi`iPYW-%fOeKjTP}SPBw;5Kjxl-&j3_6}3ZW@2~0!mB;7pdL8Y!5h^0+ zCy$Ue9lr`6dhsM{V4RtO`C5^a5>yR~y&!(KT>1XA`O;{c2Ol3>4S5A~%0SyWgLKp7 z4m>SJyOw2V&77MRiSA+`ZmpH51cTK2R)6-F)D3`2&EAg2RQ+3|6B9qHxt+0{l%CD! zjaAE5H)iJ)&mlPV1XqBUSGE|i1g%KH%L@-2mEil~^f(jM@HNDftO_fr;}dR&h0sXs!>t zP<8DSaVL0+q4{7l$e%&2=uE~MuCALbhK4I1m27m=o7U*?2?zaHY|xpjLL`VhE(`~( zOSAV%T3MG~@aRNXGjPubc5(asDl;*H+~Ta9$n&}lbIm8NV5+JKRA5x!zE%Dv+18D&<8 z$kz)&G$3UdQrkPq)yB4BXVn%6i#b)Mh)kqPgURuE#gHHL?eFRbPT1JwtP;04`sd9Sm*-dVYFUDAWVd;mM{qs^e4MAx7=nd#2`ZZk@>Qi6 z4)2!vZe1@?%wipS1IN1s>wwdSL0=?4U0FpPUzN!}Mi$JbL(OaEQ3=#QMjmx7DVcO% ztkvqu>NAaO=|Htijk-keS;FUAD#ed z?a`>!yZVuU@TiaP9d|lgL3?N2*m9gYcpLoQwI>o~dG*m8RZvn`X`>@Y6*Djuqu^jA7Ezv)fa zd~`&FcpNVNi5@FUWwVYF+pB4`Hq@}JYMW6m=$N@#1{OaugP)?9U#;rXU3vNgqJ-YO zV8tD(iUE8bO~l-Oh)WZ6(J}qv%5GT}anFLwcBhP7c%qRK4FH>DF)`X@tnsRB2W$y$FiUY&{{HFA& z;WJs=njSqaPpoZ5lULc<%4TEsqxb-J#_W@Np=~3vFR<=NJSGa(=TV}8M%hnP%x`&A zvxx=c*zQ61fXYN?k}_(PgM2n|1eiXlxdx~AsIis zr6rQ)P~Ii6s2>ZeSAg1wtc zk-H>#U|KIO2X=X8T_iakqkMFXTy8b+57GjmJOd(du+t9;_`4l^C2$2vKe9ixo?YXbY&UF7Cj7OvB2;5hYwdkREY8u_=V0 W1W*ANkf8F`uOijcTZ6?&J=Ext3Ke1i literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/pq_records/empty-own-record b/test/fuzz/qbit_corpora/pq_records/empty-own-record new file mode 100644 index 0000000000..2e32a41313 --- /dev/null +++ b/test/fuzz/qbit_corpora/pq_records/empty-own-record @@ -0,0 +1 @@ +ð \ No newline at end of file diff --git a/test/fuzz/qbit_corpora/pq_records/long-offer b/test/fuzz/qbit_corpora/pq_records/long-offer new file mode 100644 index 0000000000000000000000000000000000000000..9821de5c1c592ff423631fac4b8fc4759b68827e GIT binary patch literal 1573 zcmV+=2HN@kA_nkI^*q?uRB)0-2y^TN-F zrUQ{tFoUBoaf@>$d{s<*@X8ov`@d);HDFv?e+D+Onz6^?$;&{`V#Iksv3OPY!sOA0 zlnJeSI|Oi2zt=WlmDos-@H)aUGNwFKUfG`q&bEDwhP}Im=2lGb{Kdd?)KZ#!5|fB zA$r&s&0`k)jDIjTq}8T*30t%`Y@T&C3SBhu~n` zTb;Sk3Qnf7b43NfNLRxg%BG9RGjly2C^z7>uEb+hhIuoBLz(e1fLN0=40FL8n2j3S zYOI@^yu3jsF<9ccV8aGwR}_Sh3ddp^K7Y|AFZ(ASc0JsfDLPw{mykw_n@-qB$+}Y- z+M^%rU^aWij;>)RcguO#wQ3jmrmw-l{=0_kC~o;C46<-}z}2To5VP+lu%5|n=9hC` zk%SwPB-1*-E*liFRAAW!P?IK%b-{X}x)rrXK4{Y{nV}6p#sjomAGjM?Wx;cKxR}eM z%@lFF2gq1&BQ0Dht?7dw1{OL!V8(J&UXQS#AJjuwQ%DO3KESLpyo!fx!_7mxx4(-$ z$9QzUXut8$&Gw>@4U5V{@|%amIb3>qOgW@piNo3Bzi>viydX+-qY7|xLQ-6T&q~QW zDvKs;S?a0{nR7_#{Cm(TLNUczrOXs%q9YgQ6wDMcz>>buW?v*An8jAHz>%Sa8NvFI zHBDp#_Zf2Mj3c6oCuf*~YddiscB&Whd?X=FY#eTiQ*&(DhS}ss28A;5rXftpj0g-c1ij=+e@fABX9U#y^>rif>)C*P| z0eHq@r^U9t69qLjT8dOSt{e_3xN`X@0J<@oG}VXYC}8UtI}6(+>;g&=8#bLdalyyV zp2>0tq6oC?GU^m8VbVL>v;!r|2UfFeUE!PUg&hp$qYM{bMtN7)D`oXnAP3sE%|}fM zTD4X+F%h|DN&zi@*JrcUNfU*Dpob7Wa=Tz9Ao)2n7w0ZNyj^kzqD;!J1NwwU8KkVL zj3i)F?Gh}PWxSDOsvL4xtXPxaVVk*S#E&GkrF4A)^M?hXoRC|Bk%(4EbAt2?D7QHl zxCNXkvzSvDnU|N2Nw8h9cjfp_`IU9`8#DVAeL>|)b`ijWIDBoQLc;_v&UX<~ zQ3Uj2Gs@YBE=7523B{X)aVUEk#P%RwptDpwG#`u>oDl&G)hfNAxikVxk%7QR?5zQW zM+w)1N}yI`fS2+Ro=>b~4Kl5O%vFJ`U@2-_QPveC9E`xc%>H+Km*%BMP!V7{ir4C| z=RsE>7=#O$orC#?7B(}yXpx;)5)soEz@xx4bt$xw0m9T#giBC^Kuw}sO*fMlLlc=( z{Kz*$X(kJZ88U>tmL&}Wz+TK~38YFm;zD5vJ;RkfPKpUT+JFm7CKT6dLsiD903{wg zx!wkz((9@;vr4WuVKyQdwKWS*Xvw^55zzH@K6xzj0Ib`T#V)H*)CQJ+Ns5CfLJeb= zh+}}MKpC?KxEP8=ma-%3;&64id|ifsg<8kOxRTRXBzo+%USNr$+oMJHT_u~1K%!Hm XP^Ct+YP>xd1oX!2!l{z||4w}X(s$J< literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/pq_records/malformed-first-then-offer b/test/fuzz/qbit_corpora/pq_records/malformed-first-then-offer new file mode 100644 index 0000000000000000000000000000000000000000..fc1998efe30949677feff4d7ddbdb16665a27572 GIT binary patch literal 1576 zcmV+@2G{um@T$B0AqMbH^*q?uRB)0-2y z^TN-FrUQ{tFoUBoaf@>$d{s<*@X8ov`@d);HDFv?e+D+Onz6^?$;&{`V#Iksv3OPY z!sOA0lnJeSI|Oi2zt=WlmDos-@H)aUGNwFKUfG`q&bEDwhP}Im=2lGb{Kdd?)KZ# z!5|fBA$r&s&0`k)jDIjTq}8T*30t%`Y@T&C3SB zhu~n`Tb;Sk3Qnf7b43NfNLRxg%BG9RGjly2C^z7>uEb+hhIuoBLz(e1fLN0=40FL8 zn2j3SYOI@^yu3jsF<9ccV8aGwR}_Sh3ddp^K7Y|AFZ(ASc0JsfDLPw{mykw_n@-qB z$+}Y-+M^%rU^aWij;>)RcguO#wQ3jmrmw-l{=0_kC~o;C46<-}z}2To5VP+lu%5|n z=9hC`k%SwPB-1*-E*liFRAAW!P?IK%b-{X}x)rrXK4{Y{nV}6p#sjomAGjM?Wx;cK zxR}eM%@lFF2gq1&BQ0Dht?7dw1{OL!V8(J&UXQS#AJjuwQ%DO3KESLpyo!fx!_7mx zx4(-$$9QzUXut8$&Gw>@4U5V{@|%amIb3>qOgW@piNo3Bzi>viydX+-qY7|xLQ-6T z&q~QWDvKs;S?a0{nR7_#{Cm(TLNUczrOXs%q9YgQ6wDMcz>>buW?v*An8jAHz>%Sa z8NvFIHBDp#_Zf2Mj3c6oCuf*~YddiscB&Whd?X=FY#eTiQ*&(DhS}ss28A;5rXftp zj0g-c1ij=+e@fABX9U#y^>rif> z)C*P|0eHq@r^U9t69qLjT8dOSt{e_3xN`X@0J<@oG}VXYC}8UtI}6(+>;g&=8#bLd zalyyVp2>0tq6oC?GU^m8VbVL>v;!r|2UfFeUE!PUg&hp$qYM{bMtN7)D`oXnAP3sE z%|}fMTD4X+F%h|DN&zi@*JrcUNfU*Dpob7Wa=Tz9Ao)2n7w0ZNyj^kzqD;!J1NwwU z8KkVLj3i)F?Gh}PWxSDOsvL4xtXPxaVVk*S#E&GkrF4A)^M?hXoRC|Bk%(4EbAt2? zD7QHlxCNXkvzSvDnU|N2Nw8h9cjfp_`IU9`8#DVAeL>|)b`ijWIDBoQLc;_v z&UX<~Q3Uj2Gs@YBE=7523B{X)aVUEk#P%RwptDpwG#`u>oDl&G)hfNAxikVxk%7QR z?5zQWM+w)1N}yI`fS2+Ro=>b~4Kl5O%vFJ`U@2-_QPveC9E`xc%>H+Km*%BMP!V7{ zir4C|=RsE>7=#O$orC#?7B(}yXpx;)5)soEz@xx4bt$xw0m9T#giBC^Kuw}sO*fMl zLlc=({Kz*$X(kJZ88U>tmL&}Wz+TK~38YFm;zD5vJ;RkfPKpUT+JFm7CKT6dLsiD9 z03{wgx!wkz((9@;vr4WuVKyQdwKWS*Xvw^55zzH@K6xzj0Ib`T#V)H*)CQJ+Ns5Cf zLJeb=h+}}MKpC?KxEP8=ma-%3;&64id|ifsg<8kOxRTRXBzo+%USNr$+oMJHT_u~1 aK%!HmP^Ct+YP>xd1oX!2!l{z||4w~jOxJ(_ literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/pq_records/minimal-unknown-records b/test/fuzz/qbit_corpora/pq_records/minimal-unknown-records new file mode 100644 index 0000000000..fd5fa385d5 --- /dev/null +++ b/test/fuzz/qbit_corpora/pq_records/minimal-unknown-records @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/fuzz/qbit_corpora/pq_records/non-canonical-length b/test/fuzz/qbit_corpora/pq_records/non-canonical-length new file mode 100644 index 0000000000000000000000000000000000000000..935388e0c4db09ab560a17736a7a8156a2d6c864 GIT binary patch literal 5 Mcmey%#PDGi00)Qyng9R* literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/pq_records/offer b/test/fuzz/qbit_corpora/pq_records/offer new file mode 100644 index 0000000000000000000000000000000000000000..090c62bf1ead53b7fa4a8f0edfcfbd7f8a46cc22 GIT binary patch literal 1572 zcmV+<2HW}lAqMbH^*q?uRB)0-2y^TN-F zrUQ{tFoUBoaf@>$d{s<*@X8ov`@d);HDFv?e+D+Onz6^?$;&{`V#Iksv3OPY!sOA0 zlnJeSI|Oi2zt=WlmDos-@H)aUGNwFKUfG`q&bEDwhP}Im=2lGb{Kdd?)KZ#!5|fB zA$r&s&0`k)jDIjTq}8T*30t%`Y@T&C3SBhu~n` zTb;Sk3Qnf7b43NfNLRxg%BG9RGjly2C^z7>uEb+hhIuoBLz(e1fLN0=40FL8n2j3S zYOI@^yu3jsF<9ccV8aGwR}_Sh3ddp^K7Y|AFZ(ASc0JsfDLPw{mykw_n@-qB$+}Y- z+M^%rU^aWij;>)RcguO#wQ3jmrmw-l{=0_kC~o;C46<-}z}2To5VP+lu%5|n=9hC` zk%SwPB-1*-E*liFRAAW!P?IK%b-{X}x)rrXK4{Y{nV}6p#sjomAGjM?Wx;cKxR}eM z%@lFF2gq1&BQ0Dht?7dw1{OL!V8(J&UXQS#AJjuwQ%DO3KESLpyo!fx!_7mxx4(-$ z$9QzUXut8$&Gw>@4U5V{@|%amIb3>qOgW@piNo3Bzi>viydX+-qY7|xLQ-6T&q~QW zDvKs;S?a0{nR7_#{Cm(TLNUczrOXs%q9YgQ6wDMcz>>buW?v*An8jAHz>%Sa8NvFI zHBDp#_Zf2Mj3c6oCuf*~YddiscB&Whd?X=FY#eTiQ*&(DhS}ss28A;5rXftpj0g-c1ij=+e@fABX9U#y^>rif>)C*P| z0eHq@r^U9t69qLjT8dOSt{e_3xN`X@0J<@oG}VXYC}8UtI}6(+>;g&=8#bLdalyyV zp2>0tq6oC?GU^m8VbVL>v;!r|2UfFeUE!PUg&hp$qYM{bMtN7)D`oXnAP3sE%|}fM zTD4X+F%h|DN&zi@*JrcUNfU*Dpob7Wa=Tz9Ao)2n7w0ZNyj^kzqD;!J1NwwU8KkVL zj3i)F?Gh}PWxSDOsvL4xtXPxaVVk*S#E&GkrF4A)^M?hXoRC|Bk%(4EbAt2?D7QHl zxCNXkvzSvDnU|N2Nw8h9cjfp_`IU9`8#DVAeL>|)b`ijWIDBoQLc;_v&UX<~ zQ3Uj2Gs@YBE=7523B{X)aVUEk#P%RwptDpwG#`u>oDl&G)hfNAxikVxk%7QR?5zQW zM+w)1N}yI`fS2+Ro=>b~4Kl5O%vFJ`U@2-_QPveC9E`xc%>H+Km*%BMP!V7{ir4C| z=RsE>7=#O$orC#?7B(}yXpx;)5)soEz@xx4bt$xw0m9T#giBC^Kuw}sO*fMlLlc=( z{Kz*$X(kJZ88U>tmL&}Wz+TK~38YFm;zD5vJ;RkfPKpUT+JFm7CKT6dLsiD903{wg zx!wkz((9@;vr4WuVKyQdwKWS*Xvw^55zzH@K6xzj0Ib`T#V)H*)CQJ+Ns5CfLJeb= zh+}}MKpC?KxEP8=ma-%3;&64id|ifsg<8kOxRTRXBzo+%USNr$+oMJHT_u~1K%!Hm WP^Ct+YP>xd1oX!2!l{z||4x1P9MvcQ literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/pq_records/offer-duplicate-unknown b/test/fuzz/qbit_corpora/pq_records/offer-duplicate-unknown new file mode 100644 index 0000000000000000000000000000000000000000..090ca6b92276b3952db8d07fdd1a01ba20791312 GIT binary patch literal 3148 zcmV-S472n7AqMa{7?Q}G#QWFJE1QnBW2S*bZBeX~0BskY$CMXTPZC2T8SBD5cQ{)F zb%N<4?A&e9HIZigV^i_X6F^5J_@iGSEmsP0y7V>n!Ar2$NUn$p;*b!3A}+I`D#3u6 zo1w3&_CnjqDN10hZp;Gwih^BK9M$I|y~)mm`j>=o2{C*OTC+&3*&*uthcI%JDRE12 z3<(=JGD${S5=At#+f79pckSV}48sV`m3X4kAW$++3a2FR!AvAoT#sr(SgDxix&ko5 zE8Hf1T&Z{sU_X&@B{SJ>qKTWCiD-+db(bVuazQV1$T~+Dg9aR8a)ZI1HClSu4JlSs z0hL*YqOXvPpC6)Sw4pB?qe`e?C)b#EsKc;!YEWcjc7&t^m+4~I<*e>7n>Qg^MS>@; z31vVd#W8x3l~G6Lhz)^<1v@~8F4)Mbz^B%bc}G!{3OAYt7pcY^O)I6uvub>r*vuYU z%%x)H%>V&hzER|WB5L%oG* zmh2Up1D^%}2NX1tflQEQC)9GSN_lbDD5K zlX{9`cr_Tt<}e4qx)8E}lufipDPw7>Nm_*CM<2SaPf$lPKoBJf&P8%=Ciz_gRs&q( z8acs3455fiRT;eYgFE_V!>WmAdtDe7N@6ghN+!Ytx@)u1V)P+3JDoOaT70hGX>9pUtL~AicDjGtMpryy;nWidY3dt4o*j~IjtdvVBcffd< zn0tIDarqN&Z7T|-Dr`9%gjV5trq;3L18&p%UjRp1#DI8LaC$orR+d)alN`^UH z1%gRO`Z#u|XTC2GW@=n$JBt|(*g>>&qARmmN%u;XfX=gUC_yL0MaNm!%*EzvOSw?A zr9g4?mc?-Ora_q;a9W!?@)H`iB}x(uI#O9^d?jpp3!*y0H0rQlG(tHCxzTL5V{yYH zJajU5DO3s?i~6UA^CZN?M#8l)bu^QHn1D^FS&WsWH0QaTq7+N_Q`4eWMup8k+ZA-Q zawlt(J6DoK`n665I^U!b^=6vobfCH!`{Z zTZ#yqh~+^7w=-``i>AfK#=N1LX%&+8TUWI7e2yrI=-9_A(wJ7*I);a&ZEVi>>Aq_B zTB$aCm;0*6B^=PqL`{JaSR}sP`zFJrFnEW7!Q7`Umc+kzsIVv!$CAUuawjj+P)O4w z3rS00STb(LZ90&mH%GGyb12J;j82ud7&#!+22*GflhgOWl;>@Xxr;~@g~kxBH02_1 zn!PuZh;?&x0MHkUymUNVXD4@!B5)9^h`g+54H3nG{UKX3(!$--V;$w8go}2`QAdDj zhbQN?F$l!bBCh?KRTn@i<#3A^AMq& z`Bb{_gA#txNL zXc3zT#1=g~LKt-oOrXG~cerHZw7%+-N_1WEi@4gdiM3%tOO%wdT@hshqCG3lAq>>H z%P;!!VbP4DcUYWqd+0?~5Gxt|AqMa+0a3Q%kakEvUO{c(im%DFyWa)L2FXfBi^3UG zYaM~Z|6&0m3$yRlWY>IJ6?epEVxa1bpX&ty^L8lkcvpM5yzeT_?&0SWR2`_QvlRRoe3m~sHVC;MYZ+kO?Ku-?nU`%)8EG(cg-T?T|@45x@t91_x| zME2W%f}ze8^$T1av*%YkfJ&lyArrC22|GEG+Q%!za0CtvWG4wPy>=LZiJl`=drg-Y zvNg#|SSQ8WG@1NKvJ1Y^$5r(;NS$|X73G%4(g)q-&hlbmwEz}#+7>VqvC|wrazT}7 z0S?xCZ9j8;|1n|UXmW`r{P0VWA`iRh3f^K%B^tBV1~ zGbwJ_+fH@dGYexI=Tjds=#kw#t@36aCaV*>g@MlOFaqMEIn**oGg!^YbuB1k*!!1i zqM{Y$eZCfO&EO1?RdT?XXb@4(sKh*$oF6reU!dwkeiu>6#m|G&re(J_fGWIm(Wg9N z4;bUR?Dm)b!E(|&DDIRPY2u^W3Yf6O&K2>gu`?Y?v;*k zgCWd55eZYTy3c_m3!GUO+vRyiRD+urcew1Rh!Gov`*nC1AVY%(bTMq(x`A1lc`>Le z)A}Q*8JDocVaOMf4MeZjs(3+JoW9%vuH!*wWS`$mO(+CWM?jT@pmk&xd}v!HuHk6A zS*JEG!jCh3io(~uT!Ir%L0#bjGOA3$Apz92c88mFkaihAu+1AW^XaI zpx+ipXNwDYvu!(EQ>_e*jHwP?rJ=TRXA-%-W&0RhG`jynR80YO<>Jkc!bvnYH^WB5 zFs5L#Lm7P8$l*kRk`_z&_P7CyhaCfRva?Z+#$_y7Y#1m6!v#O2*IN7FQs^jJwdTO8 zbR!YBAQ^U}kmmqQz!lB2Opzj$Hm1RWP|VAapf556hDTH$%1N-~nECNCN+>eC^cvK^hw@!H@UX}1i%#(TUwyp&OMTiG@KcOW{l$1&8 zFOBn?I$Rb}@M>JODE`4URQ>7s0ox9*>!;KvACzHiZ*rkBJaX z^$<3DR6$)FC3<2W`z3u%fO;X5dPH=Fd?pp~;eWRHeZA#Zy!tcTbscOuVI7kx+|`q` mM9q&w9tA5S=t7L2IWZ~gKtAFET>&Iqm)3Kq#TthL@c;n{XT8_} literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/pq_records/offer-trailing-byte b/test/fuzz/qbit_corpora/pq_records/offer-trailing-byte new file mode 100644 index 0000000000000000000000000000000000000000..3fdbbddeaaf6b5184013fdf6caa4af45f7011302 GIT binary patch literal 1573 zcmV+=2HN@kAqMbH^*q?uRB)0-2y^TN-F zrUQ{tFoUBoaf@>$d{s<*@X8ov`@d);HDFv?e+D+Onz6^?$;&{`V#Iksv3OPY!sOA0 zlnJeSI|Oi2zt=WlmDos-@H)aUGNwFKUfG`q&bEDwhP}Im=2lGb{Kdd?)KZ#!5|fB zA$r&s&0`k)jDIjTq}8T*30t%`Y@T&C3SBhu~n` zTb;Sk3Qnf7b43NfNLRxg%BG9RGjly2C^z7>uEb+hhIuoBLz(e1fLN0=40FL8n2j3S zYOI@^yu3jsF<9ccV8aGwR}_Sh3ddp^K7Y|AFZ(ASc0JsfDLPw{mykw_n@-qB$+}Y- z+M^%rU^aWij;>)RcguO#wQ3jmrmw-l{=0_kC~o;C46<-}z}2To5VP+lu%5|n=9hC` zk%SwPB-1*-E*liFRAAW!P?IK%b-{X}x)rrXK4{Y{nV}6p#sjomAGjM?Wx;cKxR}eM z%@lFF2gq1&BQ0Dht?7dw1{OL!V8(J&UXQS#AJjuwQ%DO3KESLpyo!fx!_7mxx4(-$ z$9QzUXut8$&Gw>@4U5V{@|%amIb3>qOgW@piNo3Bzi>viydX+-qY7|xLQ-6T&q~QW zDvKs;S?a0{nR7_#{Cm(TLNUczrOXs%q9YgQ6wDMcz>>buW?v*An8jAHz>%Sa8NvFI zHBDp#_Zf2Mj3c6oCuf*~YddiscB&Whd?X=FY#eTiQ*&(DhS}ss28A;5rXftpj0g-c1ij=+e@fABX9U#y^>rif>)C*P| z0eHq@r^U9t69qLjT8dOSt{e_3xN`X@0J<@oG}VXYC}8UtI}6(+>;g&=8#bLdalyyV zp2>0tq6oC?GU^m8VbVL>v;!r|2UfFeUE!PUg&hp$qYM{bMtN7)D`oXnAP3sE%|}fM zTD4X+F%h|DN&zi@*JrcUNfU*Dpob7Wa=Tz9Ao)2n7w0ZNyj^kzqD;!J1NwwU8KkVL zj3i)F?Gh}PWxSDOsvL4xtXPxaVVk*S#E&GkrF4A)^M?hXoRC|Bk%(4EbAt2?D7QHl zxCNXkvzSvDnU|N2Nw8h9cjfp_`IU9`8#DVAeL>|)b`ijWIDBoQLc;_v&UX<~ zQ3Uj2Gs@YBE=7523B{X)aVUEk#P%RwptDpwG#`u>oDl&G)hfNAxikVxk%7QR?5zQW zM+w)1N}yI`fS2+Ro=>b~4Kl5O%vFJ`U@2-_QPveC9E`xc%>H+Km*%BMP!V7{ir4C| z=RsE>7=#O$orC#?7B(}yXpx;)5)soEz@xx4bt$xw0m9T#giBC^Kuw}sO*fMlLlc=( z{Kz*$X(kJZ88U>tmL&}Wz+TK~38YFm;zD5vJ;RkfPKpUT+JFm7CKT6dLsiD903{wg zx!wkz((9@;vr4WuVKyQdwKWS*Xvw^55zzH@K6xzj0Ib`T#V)H*)CQJ+Ns5CfLJeb= zh+}}MKpC?KxEP8=ma-%3;&64id|ifsg<8kOxRTRXBzo+%USNr$+oMJHT_u~1K%!Hm XP^Ct+YP>xd1oX!2!l{z||4x1V%}~}Q literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/pq_records/one-past-end b/test/fuzz/qbit_corpora/pq_records/one-past-end new file mode 100644 index 0000000000..f04166c321 --- /dev/null +++ b/test/fuzz/qbit_corpora/pq_records/one-past-end @@ -0,0 +1 @@ +ðª \ No newline at end of file diff --git a/test/fuzz/qbit_corpora/pq_records/oversized-length b/test/fuzz/qbit_corpora/pq_records/oversized-length new file mode 100644 index 0000000000000000000000000000000000000000..dc20e4e3ead2014273a73bcbcc90631d889c2da1 GIT binary patch literal 7 Ocmeyz$iTq#VHE%gj{=+k literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/pq_records/reserved-and-unknown-headers b/test/fuzz/qbit_corpora/pq_records/reserved-and-unknown-headers new file mode 100644 index 0000000000000000000000000000000000000000..b46a2bc79dd8453c09e36211bb0ddc638c530c12 GIT binary patch literal 7 OcmZQ%VEVX<@jn0t+X9~e literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/pq_records/short-offer b/test/fuzz/qbit_corpora/pq_records/short-offer new file mode 100644 index 0000000000000000000000000000000000000000..df732639161642d50a9cb69394509d5f7146e910 GIT binary patch literal 1571 zcmV+;2Hg4mAO`SG^*q?uRB)0-2y^TN-F zrUQ{tFoUBoaf@>$d{s<*@X8ov`@d);HDFv?e+D+Onz6^?$;&{`V#Iksv3OPY!sOA0 zlnJeSI|Oi2zt=WlmDos-@H)aUGNwFKUfG`q&bEDwhP}Im=2lGb{Kdd?)KZ#!5|fB zA$r&s&0`k)jDIjTq}8T*30t%`Y@T&C3SBhu~n` zTb;Sk3Qnf7b43NfNLRxg%BG9RGjly2C^z7>uEb+hhIuoBLz(e1fLN0=40FL8n2j3S zYOI@^yu3jsF<9ccV8aGwR}_Sh3ddp^K7Y|AFZ(ASc0JsfDLPw{mykw_n@-qB$+}Y- z+M^%rU^aWij;>)RcguO#wQ3jmrmw-l{=0_kC~o;C46<-}z}2To5VP+lu%5|n=9hC` zk%SwPB-1*-E*liFRAAW!P?IK%b-{X}x)rrXK4{Y{nV}6p#sjomAGjM?Wx;cKxR}eM z%@lFF2gq1&BQ0Dht?7dw1{OL!V8(J&UXQS#AJjuwQ%DO3KESLpyo!fx!_7mxx4(-$ z$9QzUXut8$&Gw>@4U5V{@|%amIb3>qOgW@piNo3Bzi>viydX+-qY7|xLQ-6T&q~QW zDvKs;S?a0{nR7_#{Cm(TLNUczrOXs%q9YgQ6wDMcz>>buW?v*An8jAHz>%Sa8NvFI zHBDp#_Zf2Mj3c6oCuf*~YddiscB&Whd?X=FY#eTiQ*&(DhS}ss28A;5rXftpj0g-c1ij=+e@fABX9U#y^>rif>)C*P| z0eHq@r^U9t69qLjT8dOSt{e_3xN`X@0J<@oG}VXYC}8UtI}6(+>;g&=8#bLdalyyV zp2>0tq6oC?GU^m8VbVL>v;!r|2UfFeUE!PUg&hp$qYM{bMtN7)D`oXnAP3sE%|}fM zTD4X+F%h|DN&zi@*JrcUNfU*Dpob7Wa=Tz9Ao)2n7w0ZNyj^kzqD;!J1NwwU8KkVL zj3i)F?Gh}PWxSDOsvL4xtXPxaVVk*S#E&GkrF4A)^M?hXoRC|Bk%(4EbAt2?D7QHl zxCNXkvzSvDnU|N2Nw8h9cjfp_`IU9`8#DVAeL>|)b`ijWIDBoQLc;_v&UX<~ zQ3Uj2Gs@YBE=7523B{X)aVUEk#P%RwptDpwG#`u>oDl&G)hfNAxikVxk%7QR?5zQW zM+w)1N}yI`fS2+Ro=>b~4Kl5O%vFJ`U@2-_QPveC9E`xc%>H+Km*%BMP!V7{ir4C| z=RsE>7=#O$orC#?7B(}yXpx;)5)soEz@xx4bt$xw0m9T#giBC^Kuw}sO*fMlLlc=( z{Kz*$X(kJZ88U>tmL&}Wz+TK~38YFm;zD5vJ;RkfPKpUT+JFm7CKT6dLsiD903{wg zx!wkz((9@;vr4WuVKyQdwKWS*Xvw^55zzH@K6xzj0Ib`T#V)H*)CQJ+Ns5CfLJeb= zh+}}MKpC?KxEP8=ma-%3;&64id|ifsg<8kOxRTRXBzo+%USNr$+oMJHT_u~1K%!Hm VP^Ct+YP>xd1oX!2!l{z||4tjy)T#gg literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/pq_records/truncated-wide-length b/test/fuzz/qbit_corpora/pq_records/truncated-wide-length new file mode 100644 index 0000000000000000000000000000000000000000..d74217a42a986d49d6a65628cc66276800d37d54 GIT binary patch literal 6 NcmZQ#T=kEE0RRQL0jvN3 literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/pq_records/unknown-then-accept b/test/fuzz/qbit_corpora/pq_records/unknown-then-accept new file mode 100644 index 0000000000000000000000000000000000000000..50770534019a44fc4f6a115087c9e0de471d9b04 GIT binary patch literal 1578 zcmV+_2G#im0dZn!bp0U)@c95QULHV=nyCSq{HKxphUf0xQTkTd;(C4ugb)t3s-k>{sLd!a;`3@nO-^Lt)pqB#GqVssCljE}zv<&OTXv3h$QFs^TVKqm>#kI{R}qmxmbRgNN3P#`o^< z`1=c+dNw7n~Zi zE`sP9*B-9PDlMF#9Oj1p!bDgd7t9NhsoQIB=ER;^88dw{NrBKKB7Opf04QGf zm840=n3d<vgnAce&ljyx+4ZD+F90DJj(`Az}(flnIAc9ct~4pI@H&WB-!lH17nsYj?KiS`$%obyWzLh%u-!-*+jDyN<-k zoB-=4mb7zsbu#nnksF{r*`5e&yLx2Bp~HbxFb}>F;}Kb!D{dMT-(&22@N%W(3+!Vw;f6$lHplo0VQek*x)@{gPd0ZCsEZ#h4a+8fXIk0mkj3`R zPL-tb`cQ?W6zm`kA&NA^naW#`3OhCkLT~|}9^KuS19?%_3-iHJPF@$=Ax2sE0f-YW zGx3~hf7b}A1kxrhu$);03;~jPTvNK{XhbDu;y1h_@W}4PwQ0@nl_I8?qY~)l-N%Td zp6b%OhZ(Y%@?2m=&kQWiDJaJ&#Y~NfCCK1I#b*6$Ey?F4{YSS$)p6u0lr)5$Om~Ub z^>6SSI`19M*Y*L=#3$cAy!QmEav@~TEj;E)6jsg)#iS>j?0Mvlf+B#5z||k1j;_nN*SRN2PSjSP z=Jnf^Y3=y7T3^(<;B&^A?rcyX4O>_wR4PD+IvCZsfy#!=+Z>yvIB=Ec|FAuzv)#4Q zpyfGQA*@1(5LzRJ{uFoY>ck#@T~ZE(Pn8kYDiec~mG;APknN15Ym)CMQX;Xf->9RZ z8L;DXX1u(3CX@Sa2b-kc*{5150dMxN1gsyxOZFE&Sj>_ye0s(P4eColX`~>>=naem zO%aXg5zk-&0@)!d8J+I0M3Ujx+adwc)nVjHh5K{fC+pZn>1S1kU-DA#{B)kH^O$(I;(z+fy~BO zXLMBal(CUNBGP;xe+%HmyV9dory1S_o)_Wo0X#P!j2v8&rKYynja!5J}zld?$`KI~Y&T3-7z5i(-#GpqKi& ctT%z0oC*PullbG9=ii4v$;6lWaq@OOpMc67@&Et; literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/pq_records/zero-length b/test/fuzz/qbit_corpora/pq_records/zero-length new file mode 100644 index 0000000000000000000000000000000000000000..0548523376e5cb6fe2cb88ecc9f9c3adc77041ac GIT binary patch literal 7 OcmZSmSeTSq!T Date: Wed, 21 Oct 2026 22:59:08 -0400 Subject: [PATCH 10/24] net: hybrid handshake outcomes, 3-strike fallback and load shedding (#184 part 5) (#235) * net: record a first-cause-wins close cause Add NodeCloseCause and CNode::RequestDisconnect(cause). It records the cause with a compare-exchange from NONE before setting fDisconnect, so the first cause wins and later or concurrent calls, including cleanup, cannot relabel it. CloseSocketDisconnect(cause) goes through it. The socket sites record PEER_EOF, PEER_RESET and SEND_ERROR, InactivityCheck() and the ping timeout record TIMEOUT, and every other writer in net.cpp and net_processing.cpp records LOCAL by calling RequestDisconnect() instead of setting fDisconnect directly. A node that reaches DisconnectNodes() with no cause is finalized there as LOCAL. fDisconnect stays for its readers, and no protocol behavior changes. The close_cause_first_wins test drives each site through CConnman and PeerManager: EOF, reset, send error, inactivity, receive-parse failure, operator disconnect, network off, eviction, shutdown and the ping timeout. It also checks that repeated and racing calls keep the first cause and that no node is deleted without one. Work item F1 of #206 (#184 part 5). Co-Authored-By: Claude Opus 5.5 * net: attribute PQ outcomes per direction with bounded failure rings Count every hybrid negotiation outcome exactly once, per direction, and keep the recent non-success outcomes as evidence. This is the data source for the fallback (F3), load shedding (F4) and the RPC (F6). CConnman::ObservePQ() snapshots the transport under its own lock, releases it, and only then takes m_pq_mutex; nothing calls into the transport, socket, peer manager or node enumeration while holding it. It runs after ReceiveMsgBytes() and once more, finalizing, in DisconnectNodes() before the disconnected pool keeps the node. Each CNode records the events already accounted, so an event seen after receive, at disconnect and in cleanup counts once. Finalization derives closed_after_switch (outbound, switched but unconfirmed, closed by EOF, reset or timeout) and abandoned (an offer queued, no authenticated version) from the F1 close cause, marks the node final and wipes the transport's pending secrets. With the network off it wipes but records nothing, since every close is then our own. - Transport gains GetPQSnapshot() and ClearPQSecrets(), with no-op defaults for V1Transport. The snapshot gains the legacy reason (no_features or parse_error) and the transport's own reason for an abandoned offer (garbage_terminator, version_length, version_tag), set at V2Transport's existing failure sites. - CNode keeps a typed PQEndpointKey captured at creation through CNodeOptions::pq_endpoint: the resolved CService with its port, the normalized hostname and port of a name-proxy destination (never the proxy's address), or an inbound peer's remote address and source port. CNodeOptions::pq carries the negotiation mode to the transport; CConnman's mode stays off until part 6 adds the option. - Counters count events: switched at key installation, even if the confirmation fails; each failure once; abandoned, closed_after_switch and the shed placeholder per direction. - Two rings of at most 256 entries, inbound and outbound (manual included), each with its own sequence and dropped count, so inbound noise can't evict outbound evidence. Reasons come from the fixed vocabulary only. - instance_id and since identify the process's counting period, and GetPQTransportStats() returns an owning copy. - One "v2 pq:" line per outcome, with fixed fields. Outbound failures log at info level, each from its own call site; inbound outcomes, switched, legacy_peer and abandoned are net debug, from separate call sites so inbound noise can't rate-limit outbound lines. net_pq_tests covers once-only observation (including racing observers), endpoint identity, the ring bounds, switched-then-failed, and every abandoned and closed_after_switch cause; a confirmed handshake that the peer later closes, or that times out, counts nothing. It checks the exact text and level of each outcome's log line in both directions (info for outbound failures, net debug otherwise), and the status each failure reports (off for a local fault before commit, pending while a failed connection closes). pq_cipher_state_closes checks that an inconsistent cipher state closes in both roles, the initiator before it commits included, and counts once as internal_error/cipher_state. pq_finalization_wipes checks, for every close cause (EOF, reset, timeout, local, send error, network off, transport failure) and both roles, that the pending secrets survive the socket close and are gone once the connection is finalized, before the node is deleted. pq_name_proxy_endpoint drives ConnectNode() through a name proxy (a mocked SOCKS5 server) and checks that the endpoint key is the normalized destination and port, never the proxy. ProxySettingsRestorerForTesting, a test-only netbase guard, saves and restores the proxy settings so that no later test sees them. Production construction stays PQ-off. Work item F2 of #206 (#184 part 5). Co-Authored-By: Claude Opus 5.5 * net: apply bounded per-endpoint fallback after repeated PQ failures Keep upgraded nodes connected when the hybrid negotiation with one endpoint keeps failing, without letting an outsider downgrade connections cheaply: a single failure is telemetry only. - Counted failures are outbound only, per endpoint key: malformed_record, first_packet_failed, and closed_after_switch (eof, reset or timeout). Our own closes, send errors, legacy outcomes and inbound failures are neutral: they neither advance nor reset a streak. - The third consecutive counted failure puts the endpoint in the fallback set for 1 h, then 4 h, then 24 h for each later entry (PQ_FALLBACK_WINDOWS). At expiry the streak restarts at 0 and the escalation stays. A failure from a connection already running during a window neither extends it nor enters again. - A confirmed hybrid handshake moves the endpoint into a separate success set, never evicted until restart, and drops its history. The check and the insert share m_pq_mutex with the counters and rings, so a late failure can't reinsert a protected endpoint. - The history holds at most 1,000 endpoints, evicting the oldest insertion first; insertion orders are unique, so equal times can't tie. It is memory only and uses the mockable NodeClock. - ConnectNode() asks IsPQFallback(endpoint, now): a fallback connection runs PQMode::FALLBACK, plain v2 that neither parses nor validates the offer. No retry loop is added: the next attempt the connection logic makes asks again, existing connections never renegotiate, and inbound connections never fall back. - Entering counts the outbound fallback counter once, adds a ring entry and logs one warning, "v2 pq: fallback cause= until=" with the usual fields. - When 8 distinct endpoints have counted failures and none has succeeded since startup, it warns once that this node's own ML-KEM code may be at fault, naming the active backends. It names no remedy: this tree has no option that forces portable code yet. - GetPQTransportStats() gains the fallback set and the failure streaks for part 6. net_pq_tests covers the threshold for each counted variant with neutral closes in between, the escalating windows under mock time (including new connections getting FALLBACK and existing ones not), success protection against a late failure and 1,001 endpoints of churn, oldest-insertion eviction with tied times and insertion order opposite to key order, the local-fault warning (once, naming the backends, native or portable; never after a success), and a fallback connection completing as plain v2 against a malformed offer. pq_closed_after_switch now uses an endpoint per connection, so that its three counted failures don't enter fallback, and pq_cipher_state_closes checks that a local fault never counts toward the fallback. Work item F3 of #206 (#184 part 5). Co-Authored-By: Claude Opus 5.5 * net: shed inbound PQ offers under handshake floods Bound the extra CPU an inbound connection flood can cost. Each offer costs the responder one ML-KEM-1024 key generation; above a rate threshold, responders stop offering and cost exactly what plain v2 costs today. - V2PQOptions gains an offer gate (a function pointer and context, like PQRandomSource). A negotiating responder calls it at its key generation point, once the initiator's 64-byte key has arrived; when it declines, the responder wipes the retained secret and sends empty version contents, and its status is off. - CConnman's gate counts the attempted offers per second. More than threshold_per_s in the current second starts shedding; ten consecutive complete seconds with fewer than half the threshold's attempts stop it. The quiet seconds before a flood don't count toward its end, so shedding can't flap within a second. The seconds come from the steady clock (MockableSteadyClock), so a wall clock step can't merge seconds or stretch a period; only the reported start time is wall-clock time. The socket handler also checks the clock, so shedding stops on time when the flood stops entirely. - Each shed offer increments the inbound shed counter and adds no ring entry, so a flood can't wash out the evidence. Starting and stopping each log one info line: "v2 pq: load_shedding started threshold_per_s=" and "v2 pq: load_shedding stopped". - The threshold defaults to DEFAULT_PQ_SHED_THRESHOLD_PER_S (1,000) and CConnman::Options::pq_shed_threshold_per_s overrides it for unit tests and the flood lab; there is no new user-facing option. - GetPQTransportStats() gains load_shedding {active, threshold_per_s, since} for part 6. - Outbound connections are never affected. The gate's mutex is taken under the transport's locks, and nothing holding it calls into a transport or takes m_pq_mutex. net_pq_tests drives inbound connections through the accept path under mock time: offers up to the threshold, the first one above it shed (empty contents, so the initiator sees a legacy responder; status off; shed +1), outbound connections still hybrid, a second at half the threshold restarting the quiet run, offers resuming only after ten quiet seconds, and each transition logged once at info level. A wall clock step back by an hour under a trickle of offers sheds nothing, and a step forward doesn't end a flood early. The real socket handler thread, run as Start() runs it, ends a flood that stopped. With the switch off the gate is never asked. Work item F4 of #206 (#184 part 5). Co-Authored-By: Claude Opus 5.5 * test: pin the shed wipe, inbound-success and outbound-abandoned guards Three guards had no test, and each survived a mutation in the deep review of PR #235: - pq_shed_wipes_secrets: a shed responder wipes the retained ECDH secret and transcript at once (ClearHybridSecrets() in the shed branch), not only at finalization, since it runs plain v2 for the whole connection. - pq_inbound_success_does_not_protect: only outbound successes enter the success set, which is never evicted. An inbound peer's key is its address and source port, so an inbound success must neither protect an endpoint nor silence the local-fault warning. - pq_outbound_never_abandoned: only responders offer, so an outbound connection closed before the peer's version, by either side, counts no abandoned and adds no outbound ring entry. Follow-up to work items F2-F4 of #206 (#184 part 5). Co-Authored-By: Claude Opus 5.5 * test: restore mocked globals on every path in net_pq_tests Three tests changed process-wide state and restored it only at their end: pq_endpoint_identity, pq_name_proxy_endpoint and pq_fallback_windows replaced CreateSock, pq_name_proxy_endpoint also cleared the SOCKS5 interrupt, and pq_log_lines set fLogIPs. A failed BOOST_REQUIRE in between throws past the restore. Every later suite in the same test_qbit process would then connect through a mock socket or log differently, and fail far from the cause. A GlobalOverride guard, a MockCreateSock() helper built on it, and a Socks5InterruptCleared guard now restore them on every path, as ProxySettingsRestorerForTesting already does for the proxy settings. Co-Authored-By: Claude Opus 5.5 * doc: name load shedding in the off status of pq-transport.md A responder that sheds its offer under an inbound flood runs plain v2 and reports off (#206 F4), but the status list in doc/design/pq-transport.md named only the switch and local faults. Co-Authored-By: Claude Opus 5.5 * test: check the shed wipe before the initiator's version arrives Part 4 now wipes the hybrid secrets whenever the peer's version is processed (#230, L5). pq_shed_wipes_secrets already checks right after the initiator's key, before any version, so a shed branch that doesn't wipe is still caught. With the merged part 4, a variant of the test that checks after the version lets that mutant survive. Make the precondition explicit, so the check can't drift past the version. Follow-up to work item F4 of #206 (#184 part 5). Co-Authored-By: Claude Opus 5.5 * test: pin the local-fault set's cap The distinct failing endpoints that the local-fault warning counts are never evicted, so the cap at PQ_LOCAL_FAULT_THRESHOLD is their only bound. No test saw it: the warning fires once either way, and an uncapped set survived mutation in the deep review of PR #235 (finding 5). ConnmanTestMsg::PQLocalFaultEndpoints() reads the set's size, test only. pq_local_fault_warning checks that the set holds exactly the threshold once the warning fires, and still does after 12 distinct endpoints have failed. Follow-up to work item F3 of #206 (#184 part 5). Co-Authored-By: Claude Opus 5.5 * test: build PQ test peers without a conditional of prvalues MSVC requires an accessible copy constructor for `cond ? a : b` with prvalue operands, and V2Transport has none, so test_bitcoin failed to build on the Windows CI job (C2280). Return the transport from a lambda instead. Co-Authored-By: Claude Opus 5.5 * net: finalize PQ state of the connections deleted at shutdown Finalization ran only in DisconnectNodes(). StopNodes() swaps the nodes out of m_nodes and deletes them itself, so a node still held at shutdown was never marked finalized, and its pending decapsulation key and retained ECDH secret stayed until the transport's destructor (PR #235 review). StopNodes() now runs the same finalization, ObservePQ(finalizing), before deleting each node: it marks the node final and wipes the pending secrets. A shutdown close is our own decision, so like the network-off path it records nothing: no abandoned offer, ring entry or log line. ObservePQ() gains a record_outcomes flag for that; the nodes in the disconnected pool were already finalized by DisconnectNodes(). pq_shutdown_finalizes stops a connection manager that holds an inbound connection with its offer pending, an outbound one awaiting the version, and a hybrid one. At deletion no node is unfinalized or holds a hybrid secret, and no counter or ring changes. Follow-up to work item F2 of #206 (#184 part 5). Co-Authored-By: Claude Opus 5.5 * test: report would-block from DynSock where Windows reads it net_pq_tests failed on the Windows CI job: no handshake completed, offers were abandoned with reason reset instead of eof, and the rings stayed empty. The log shows the cause: "socket recv error, disconnecting peer=0: The operation completed successfully. (0)". DynSock::Pipe::GetBytes() signals an empty pipe like a non-blocking recv(2) with no data: it returns -1 and sets errno to EAGAIN. The socket handler reads the error with WSAGetLastError(). That is errno on POSIX, but on Windows it is Winsock's own last error, which the mock never set. So Windows saw error 0, treated every idle read as a permanent receive error, and closed the connection as PEER_RESET before the peer's next bytes arrived. Production is right: real sockets set the Winsock error, an orderly close still reads 0 (EOF), and a peer that closes with unread data resets the connection on Windows and Linux alike. GetBytes() now also calls WSASetLastError(WSAEWOULDBLOCK) on Windows. mock_socket_would_block pins it: an empty pipe reads -1 with the would-block error from WSAGetLastError(), EOF reads 0, and a socket handler pass over a connection with nothing to read leaves it open. Making the mock report error 0 on Linux reproduces the Windows failures exactly, and the new test catches it. Four checks dereferenced Streak() or Fallback() without requiring a value; on Windows one read an indeterminate streak. They now BOOST_REQUIRE it first. Co-Authored-By: Claude Opus 5.5 * net: keep active fallback windows when the endpoint history is full When the history of outbound endpoints with counted failures reached PQ_MAX_ENDPOINT_HISTORY, a failure from a new endpoint evicted the oldest insertion, even one inside an active fallback window. Entries whose window ended stay in the map to remember escalation, so a later one-strike failure could drop an in-window fallback: IsPQFallback then returned false, the next outbound connection tried hybrid again, and the endpoint's window and escalation started over. Evict the oldest insertion outside a fallback window instead, and only when every endpoint is in one, the window that ends first (the oldest insertion of those that end together). Found by Cursor Bugbot on #235. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- doc/design/pq-transport.md | 5 +- src/bip324_pq.h | 19 + src/net.cpp | 565 ++++++++++- src/net.h | 401 +++++++- src/net_processing.cpp | 88 +- src/netbase.cpp | 16 + src/netbase.h | 18 + src/test/CMakeLists.txt | 1 + src/test/net_pq_tests.cpp | 1827 ++++++++++++++++++++++++++++++++++++ src/test/net_tests.cpp | 221 +++++ src/test/util/net.cpp | 7 +- src/test/util/net.h | 57 +- 12 files changed, 3144 insertions(+), 81 deletions(-) create mode 100644 src/test/net_pq_tests.cpp diff --git a/doc/design/pq-transport.md b/doc/design/pq-transport.md index c78d76ae60..6953aad18b 100644 --- a/doc/design/pq-transport.md +++ b/doc/design/pq-transport.md @@ -309,8 +309,9 @@ qbit reports each v2 connection's negotiation status as one of these values: - `hybrid` once the peer's confirmation verifies; - `legacy_peer` when the peer's version packet carries no usable offer or accept, so the ECDH keys stay; -- `off` when the switch is off, or after a local key generation, key check or - encapsulation fault that left the connection on plain v2; +- `off` when the switch is off, when a responder sheds its offer under load, + or after a local key generation, key check or encapsulation fault that left + the connection on plain v2; - `fallback` for an outbound connection that runs plain v2 because of earlier failures; inbound connections never fall back. diff --git a/src/bip324_pq.h b/src/bip324_pq.h index d8f0f8cf8a..73462f0d4d 100644 --- a/src/bip324_pq.h +++ b/src/bip324_pq.h @@ -70,6 +70,13 @@ enum class PQFailure : uint8_t { CIPHER_STATE_INTERNAL = 11, }; +/** Why the peer counts as a legacy peer: its contents carried no own record, or did not parse. */ +enum class PQLegacyReason : uint8_t { NONE = 0, NO_FEATURES = 1, PARSE_ERROR = 2 }; + +/** Why the transport closed before the peer's version packet authenticated (the transport's own + * reasons for an abandoned offer). The first one recorded is kept. */ +enum class PQAbort : uint8_t { NONE = 0, GARBAGE_TERMINATOR = 1, VERSION_LENGTH = 2, VERSION_TAG = 3 }; + /** Entropy for key generation and encapsulation, 32 bytes per call. */ struct PQRandomSource { using Fill32 = void (*)(void*, std::span) noexcept; @@ -118,6 +125,9 @@ class PQHandshake //! The peer's key confirmation verified. bool confirmed{false}; PQFailure failure{PQFailure::NONE}; + //! Set when the negotiation ended with a legacy peer. + PQLegacyReason legacy{PQLegacyReason::NONE}; + PQAbort abort{PQAbort::NONE}; }; /** @@ -180,6 +190,15 @@ class PQHandshake void SetVersionReceived() noexcept { m_snapshot.version_received = true; } void SetSwitched() noexcept { m_snapshot.switched = true; } void SetConfirmed() noexcept { m_snapshot.confirmed = true; } + /** The peer's contents parsed as kind, without an own record: a legacy peer. */ + void SetLegacy(ParseKind kind) noexcept + { + m_snapshot.legacy = kind == ParseKind::INVALID_GRAMMAR ? PQLegacyReason::PARSE_ERROR : PQLegacyReason::NO_FEATURES; + } + void SetAbort(PQAbort abort) noexcept + { + if (m_snapshot.abort == PQAbort::NONE) m_snapshot.abort = abort; + } /** Record a failure unless one is recorded already, and wipe the secrets. */ void SetFailure(PQFailure failure) noexcept; Snapshot GetSnapshot() const noexcept { return m_snapshot; } diff --git a/src/net.cpp b/src/net.cpp index 2c330ce0dc..256b99dc2e 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -44,6 +45,7 @@ #include #include #include +#include #include TRACEPOINT_SEMAPHORE(net, closed_connection); @@ -462,6 +464,8 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo std::unique_ptr i2p_transient_session; for (auto& target_addr: connect_to) { + // What the hybrid negotiation is accounted to: the destination, never a proxy. + PQEndpointKey pq_endpoint{CService{target_addr}}; if (target_addr.IsValid()) { const bool use_proxy{GetProxy(target_addr.GetNetwork(), proxy)}; bool proxyConnectionFailed = false; @@ -512,6 +516,7 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo std::string host; uint16_t port{default_port}; SplitHostPort(std::string(pszDest), port, host); + pq_endpoint = MakePQNameEndpoint(host, port); bool proxyConnectionFailed; sock = ConnectThroughProxy(proxy, host, port, proxyConnectionFailed); } @@ -520,6 +525,12 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo continue; } + // Repeated hybrid failures to this endpoint put it in the fallback set: plain v2, without + // parsing or validating the peer's offer. The next attempt, whenever the connection logic + // makes one, asks again. + PQMode pq_mode{m_pq_mode}; + if (pq_mode == PQMode::NEGOTIATE && use_v2transport && IsPQFallback(pq_endpoint, Now())) pq_mode = PQMode::FALLBACK; + NetPermissionFlags permission_flags = NetPermissionFlags::None; std::vector whitelist_permissions = conn_type == ConnectionType::MANUAL ? vWhitelistedRangeOutgoing : std::vector{}; AddWhitelistPermissionFlags(permission_flags, target_addr, whitelist_permissions); @@ -545,6 +556,8 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo .recv_flood_size = nReceiveFloodSize, .use_v2transport = use_v2transport, .is_archive_connection = is_archive_connection, + .pq = {.mode = pq_mode}, + .pq_endpoint = std::move(pq_endpoint), }); pnode->AddRef(); @@ -557,9 +570,19 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo return nullptr; } -void CNode::CloseSocketDisconnect() +void CNode::RequestDisconnect(NodeCloseCause cause) noexcept { + Assume(cause != NodeCloseCause::NONE); + // Compare-exchange from NONE so the first cause wins, and record it before + // setting fDisconnect so a thread that sees fDisconnect also sees the cause. + NodeCloseCause expected{NodeCloseCause::NONE}; + m_close_cause.compare_exchange_strong(expected, cause); fDisconnect = true; +} + +void CNode::CloseSocketDisconnect(NodeCloseCause cause) +{ + RequestDisconnect(cause); LOCK(m_sock_mutex); if (m_sock) { LogDebug(BCLog::NET, "Resetting socket for peer=%d%s", GetId(), LogIP(fLogIPs)); @@ -1189,6 +1212,12 @@ bool V2Transport::ProcessReceivedKeyBytes() noexcept } else if (m_initiating) { // Hold the version packet until the responder's arrives: it answers their offer. SetSendState(SendState::AWAITING_VERSION); + } else if (const PQOfferGate& gate{m_pq_options.offer_gate}; gate.allow && !gate.allow(gate.context)) { + // Load shedding: no offer, which is exactly plain v2. + m_pq_shed = true; + ClearHybridSecrets(); + SetSendState(SendState::READY); + AppendVersionPacket(VERSION_CONTENTS); } else { // Offer an encapsulation key in the version packet, and send nothing else until the // initiator's version packet is processed. The key is generated only now that the @@ -1230,6 +1259,7 @@ bool V2Transport::ProcessReceivedGarbageBytes() noexcept // We've reached the maximum length for garbage + garbage terminator, and the // terminator still does not match. Abort. LogDebug(BCLog::NET, "V2 transport error: missing garbage terminator, peer=%d\n", m_nodeid); + m_pq.SetAbort(PQAbort::GARBAGE_TERMINATOR); return false; } else { // We still need to receive more garbage and/or garbage terminator bytes. @@ -1267,6 +1297,7 @@ bool V2Transport::ProcessReceivedPacketBytes() noexcept } if (m_recv_len > MAX_CONTENTS_LEN) { LogDebug(BCLog::NET, "V2 transport error: packet too large (%u bytes), peer=%d\n", m_recv_len, m_nodeid); + if (m_recv_state == RecvState::VERSION) m_pq.SetAbort(PQAbort::VERSION_LENGTH); return false; } } else if (m_recv_buffer.size() > BIP324Cipher::LENGTH_LEN && m_recv_buffer.size() == m_recv_len + BIP324Cipher::EXPANSION) { @@ -1284,6 +1315,7 @@ bool V2Transport::ProcessReceivedPacketBytes() noexcept /*contents=*/MakeWritableByteSpan(m_recv_decode_buffer)); if (!ret) { LogDebug(BCLog::NET, "V2 transport error: packet decryption failure (%u bytes), peer=%d\n", m_recv_len, m_nodeid); + if (m_recv_state == RecvState::VERSION) m_pq.SetAbort(PQAbort::VERSION_TAG); return false; } // We have decrypted a valid packet with the AAD we expected, so clear the expected AAD. @@ -1373,6 +1405,7 @@ bool V2Transport::ProcessVersionContents(std::span contents) no // keys; an initiator now sends its empty version packet. const auto parsed{PQHandshake::ParseContents(contents)}; if (parsed.kind != PQHandshake::ParseKind::OWN_RECORD) { + m_pq.SetLegacy(parsed.kind); ClearHybridSecrets(); SetSendState(SendState::READY); if (m_initiating) AppendVersionPacket(VERSION_CONTENTS); @@ -1837,6 +1870,7 @@ PQStatus V2Transport::GetPQStatus() const noexcept case PQMode::FALLBACK: return PQStatus::FALLBACK; case PQMode::NEGOTIATE: break; } + if (m_pq_shed) return PQStatus::OFF; const PQHandshake::Snapshot snapshot{m_pq.GetSnapshot()}; if (snapshot.confirmed) return PQStatus::HYBRID; switch (snapshot.failure) { @@ -1887,6 +1921,14 @@ void V2Transport::DiscardHybridSecretForTesting() noexcept m_cipher.DiscardHybridSecret(); } +void V2Transport::ClearPQSecrets() noexcept +{ + AssertLockNotHeld(m_recv_mutex); + AssertLockNotHeld(m_send_mutex); + LOCK2(m_recv_mutex, m_send_mutex); + ClearHybridSecrets(); +} + std::pair CConnman::SocketSendData(CNode& node) const { auto it = node.vSendMsg.begin(); @@ -1950,7 +1992,7 @@ std::pair CConnman::SocketSendData(CNode& node) const int nErr = WSAGetLastError(); if (nErr != WSAEWOULDBLOCK && nErr != WSAEMSGSIZE && nErr != WSAEINTR && nErr != WSAEINPROGRESS) { LogDebug(BCLog::NET, "socket send error, %s: %s\n", node.DisconnectMsg(fLogIPs), NetworkErrorString(nErr)); - node.CloseSocketDisconnect(); + node.CloseSocketDisconnect(NodeCloseCause::SEND_ERROR); } } break; @@ -2016,7 +2058,7 @@ bool CConnman::AttemptToEvictConnection() pnode->ConnectionTypeAsString().c_str(), pnode->ConnectedThroughNetwork(), Ticks(pnode->m_connected)); - pnode->fDisconnect = true; + pnode->RequestDisconnect(); return true; } } @@ -2136,6 +2178,9 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr&& sock, .prefer_evict = discouraged, .recv_flood_size = nReceiveFloodSize, .use_v2transport = use_v2transport, + .pq = {.mode = m_pq_mode, .offer_gate = {.allow = &CConnman::AllowPQOffer, .context = this}}, + // The actual remote address and source port. + .pq_endpoint = addr, }); pnode->AddRef(); m_msgproc->InitializeNode(*pnode, local_services); @@ -2192,6 +2237,488 @@ bool CConnman::AddConnection(const std::string& address, ConnectionType conn_typ return true; } +PQNameEndpoint MakePQNameEndpoint(std::string_view hostname, uint16_t port) +{ + // DNS names are case-insensitive, and "example.com." is the fully qualified "example.com". + if (hostname.ends_with('.')) hostname.remove_suffix(1); + return {.hostname = ToLower(hostname), .port = port}; +} + +std::string_view PQOutcomeString(PQOutcome outcome) noexcept +{ + switch (outcome) { + case PQOutcome::SWITCHED: return "switched"; + case PQOutcome::CONFIRMED: return "confirmed"; + case PQOutcome::LEGACY_PEER: return "legacy_peer"; + case PQOutcome::MALFORMED_RECORD: return "malformed_record"; + case PQOutcome::FIRST_PACKET_FAILED: return "first_packet_failed"; + case PQOutcome::ABANDONED: return "abandoned"; + case PQOutcome::CLOSED_AFTER_SWITCH: return "closed_after_switch"; + case PQOutcome::FALLBACK: return "fallback"; + case PQOutcome::INTERNAL_ERROR: return "internal_error"; + } // no default case, so the compiler can warn about missing cases + assert(false); +} + +namespace { +/** The index of a direction in the hybrid negotiation counters and failure rings. */ +size_t PQDirection(bool inbound) { return inbound ? 0 : 1; } + +/** The outcome a negotiation failure counts as. */ +PQOutcome PQFailureOutcome(PQFailure failure) noexcept +{ + switch (failure) { + case PQFailure::EK_LENGTH: + case PQFailure::EK_MODULUS: + case PQFailure::CT_LENGTH: + return PQOutcome::MALFORMED_RECORD; + case PQFailure::CONFIRM_LENGTH: + case PQFailure::CONFIRM_TAG: + case PQFailure::CONFIRM_NOT_DECOY: + return PQOutcome::FIRST_PACKET_FAILED; + case PQFailure::KEYGEN_INTERNAL: + case PQFailure::CHECK_EK_INTERNAL: + case PQFailure::ENCAPS_INTERNAL: + case PQFailure::DECAPS_INTERNAL: + case PQFailure::CIPHER_STATE_INTERNAL: + return PQOutcome::INTERNAL_ERROR; + case PQFailure::NONE: + break; + } // no default case, so the compiler can warn about missing cases + assert(false); +} + +/** The reason word of a negotiation failure. */ +std::string_view PQFailureReason(PQFailure failure) noexcept +{ + switch (failure) { + case PQFailure::EK_LENGTH: return "ek_length"; + case PQFailure::EK_MODULUS: return "ek_modulus"; + case PQFailure::CT_LENGTH: return "ct_length"; + case PQFailure::CONFIRM_LENGTH: return "length"; + case PQFailure::CONFIRM_TAG: return "tag"; + case PQFailure::CONFIRM_NOT_DECOY: return "not_decoy"; + case PQFailure::KEYGEN_INTERNAL: return "keygen"; + case PQFailure::CHECK_EK_INTERNAL: return "check_public_key"; + case PQFailure::ENCAPS_INTERNAL: return "encaps"; + case PQFailure::DECAPS_INTERNAL: return "decaps"; + case PQFailure::CIPHER_STATE_INTERNAL: return "cipher_state"; + case PQFailure::NONE: break; + } // no default case, so the compiler can warn about missing cases + assert(false); +} + +/** The reason word of a close cause. A node without one is finalized as LOCAL. */ +std::string_view PQCloseCauseReason(NodeCloseCause cause) noexcept +{ + switch (cause) { + case NodeCloseCause::PEER_EOF: return "eof"; + case NodeCloseCause::PEER_RESET: return "reset"; + case NodeCloseCause::TIMEOUT: return "timeout"; + case NodeCloseCause::SEND_ERROR: return "send_error"; + case NodeCloseCause::NONE: + case NodeCloseCause::LOCAL: return "local"; + } // no default case, so the compiler can warn about missing cases + assert(false); +} + +/** The reason word of the transport closing before the peer's version arrived. */ +std::string_view PQAbortReason(PQAbort abort) noexcept +{ + switch (abort) { + case PQAbort::GARBAGE_TERMINATOR: return "garbage_terminator"; + case PQAbort::VERSION_LENGTH: return "version_length"; + case PQAbort::VERSION_TAG: return "version_tag"; + case PQAbort::NONE: break; + } // no default case, so the compiler can warn about missing cases + assert(false); +} + +/** The counter of an outcome, or nullptr for a success, which isn't counted. */ +std::atomic* PQCounter(PQCounters& counters, PQOutcome outcome) noexcept +{ + switch (outcome) { + case PQOutcome::SWITCHED: return &counters.switched; + case PQOutcome::CONFIRMED: return nullptr; + case PQOutcome::LEGACY_PEER: return &counters.legacy_peer; + case PQOutcome::MALFORMED_RECORD: return &counters.malformed_record; + case PQOutcome::FIRST_PACKET_FAILED: return &counters.first_packet_failed; + case PQOutcome::ABANDONED: return &counters.abandoned; + case PQOutcome::CLOSED_AFTER_SWITCH: return &counters.closed_after_switch; + case PQOutcome::FALLBACK: return &counters.fallback; + case PQOutcome::INTERNAL_ERROR: return &counters.internal_error; + } // no default case, so the compiler can warn about missing cases + assert(false); +} + +/** The window an endpoint enters next: escalating per completed window, then the last one. */ +std::chrono::seconds PQFallbackWindow(const PQEndpointHistory& history) noexcept +{ + return PQ_FALLBACK_WINDOWS[std::min(history.completed_windows, PQ_FALLBACK_WINDOWS.size() - 1)]; +} + +/** End an expired fallback window: the streak restarts at 0, and the escalation level stays. */ +void ExpirePQFallback(PQEndpointHistory& history, NodeSeconds now) noexcept +{ + if (!history.expires || now < *history.expires) return; + history.entered.reset(); + history.expires.reset(); + history.streak = 0; + history.completed_windows = static_cast(std::min(history.completed_windows + 1, PQ_FALLBACK_WINDOWS.size())); +} + +PQCounts LoadPQCounts(const PQCounters& counters) noexcept +{ + return { + .switched = counters.switched.load(), + .legacy_peer = counters.legacy_peer.load(), + .malformed_record = counters.malformed_record.load(), + .first_packet_failed = counters.first_packet_failed.load(), + .abandoned = counters.abandoned.load(), + .internal_error = counters.internal_error.load(), + .closed_after_switch = counters.closed_after_switch.load(), + .fallback = counters.fallback.load(), + .shed = counters.shed.load(), + }; +} +} // namespace + +void CConnman::ObservePQ(CNode& node, bool finalizing, bool record_outcomes) +{ + AssertLockNotHeld(m_pq_mutex); + // The transport takes and releases its own lock. + const PQHandshake::Snapshot snapshot{node.m_transport->GetPQSnapshot()}; + // Nothing to account yet, as on every connection without the negotiation. + if (!finalizing && !snapshot.switched && snapshot.legacy == PQLegacyReason::NONE && snapshot.failure == PQFailure::NONE) return; + const NodeCloseCause cause{node.GetCloseCause()}; + const bool record{record_outcomes && fNetworkActive}; + PQLogLines lines; + { + LOCK(m_pq_mutex); + if (node.m_pq_finalized) return; + if (record) lines = AccountPQ(node, snapshot, cause, finalizing, Now()); + if (finalizing) node.m_pq_finalized = true; + } + if (finalizing) node.m_transport->ClearPQSecrets(); + for (const PQLogLine& line : lines.outcomes) LogPQOutcome(node, line); + if (lines.local_fault) LogPQLocalFault(); +} + +CConnman::PQLogLines CConnman::AccountPQ(CNode& node, const PQHandshake::Snapshot& snapshot, NodeCloseCause cause, + bool finalizing, NodeSeconds now) +{ + AssertLockHeld(m_pq_mutex); + const bool inbound{node.IsInboundConn()}; + PQLogLines lines; + // Each event counts once per connection, however often it is observed. + const auto account{[&](PQOutcome outcome, std::string_view reason) EXCLUSIVE_LOCKS_REQUIRED(m_pq_mutex) { + const uint16_t bit{static_cast(1U << static_cast(outcome))}; + if (node.m_pq_accounted_events & bit) return; + node.m_pq_accounted_events |= bit; + if (std::atomic* counter{PQCounter(m_pq_counters[PQDirection(inbound)], outcome)}) ++*counter; + if (outcome == PQOutcome::CONFIRMED) { + // A confirmed hybrid handshake protects an outbound endpoint until restart: no + // failure, however late, can count against it again. + if (!inbound) { + m_pq_successful_endpoints.insert(node.m_pq_endpoint); + m_pq_history.erase(node.m_pq_endpoint); + } + return; + } + if (outcome != PQOutcome::SWITCHED) { + AddPQFailure({.sequence = 0, + .time = now, + .endpoint = node.m_pq_endpoint, + .inbound = inbound, + .connection_type = node.m_conn_type, + .peer_id = node.GetId(), + .outcome = outcome, + .reason = std::string{reason}}); + } + lines.outcomes.push_back({outcome, reason}); + // Only failures the peer side caused count toward the fallback, and only outbound. + if (!inbound && (outcome == PQOutcome::MALFORMED_RECORD || outcome == PQOutcome::FIRST_PACKET_FAILED || + outcome == PQOutcome::CLOSED_AFTER_SWITCH)) { + if (const auto until{CountPQFailure(node, outcome, reason, now, lines.local_fault)}) { + lines.outcomes.push_back({PQOutcome::FALLBACK, PQOutcomeString(outcome), until}); + } + } + }}; + + // switched counts the key installation, even if the confirmation fails later. + if (snapshot.switched) account(PQOutcome::SWITCHED, {}); + if (snapshot.confirmed) account(PQOutcome::CONFIRMED, {}); + if (snapshot.legacy != PQLegacyReason::NONE) { + account(PQOutcome::LEGACY_PEER, snapshot.legacy == PQLegacyReason::PARSE_ERROR ? "parse_error" : "no_features"); + } + if (snapshot.failure != PQFailure::NONE) account(PQFailureOutcome(snapshot.failure), PQFailureReason(snapshot.failure)); + if (finalizing && snapshot.failure == PQFailure::NONE) { + // We switched, and the peer closed or stopped responding before its confirmation + // verified. Our own closes and send errors are not the peer's doing. + if (!inbound && snapshot.switched && !snapshot.confirmed && + (cause == NodeCloseCause::PEER_EOF || cause == NodeCloseCause::PEER_RESET || cause == NodeCloseCause::TIMEOUT)) { + account(PQOutcome::CLOSED_AFTER_SWITCH, PQCloseCauseReason(cause)); + } + // Our offer was queued, but no authenticated version arrived before the close. An offer + // that failed internally is internal_error only. + if (snapshot.offer == PQOfferState::SENT && !snapshot.version_received) { + account(PQOutcome::ABANDONED, snapshot.abort != PQAbort::NONE ? PQAbortReason(snapshot.abort) : PQCloseCauseReason(cause)); + } + } + return lines; +} + +std::optional CConnman::CountPQFailure(const CNode& node, PQOutcome outcome, std::string_view reason, NodeSeconds now, + bool& local_fault) +{ + AssertLockHeld(m_pq_mutex); + const PQEndpointKey& endpoint{node.m_pq_endpoint}; + if (m_pq_successful_endpoints.contains(endpoint)) return std::nullopt; + + // Many endpoints failing, and none succeeding, points at this node's own ML-KEM code. + if (m_pq_failed_endpoints_for_warning.size() < PQ_LOCAL_FAULT_THRESHOLD) m_pq_failed_endpoints_for_warning.insert(endpoint); + if (!m_pq_local_fault_warned && m_pq_failed_endpoints_for_warning.size() >= PQ_LOCAL_FAULT_THRESHOLD && + m_pq_successful_endpoints.empty()) { + m_pq_local_fault_warned = true; + local_fault = true; + } + + auto it{m_pq_history.find(endpoint)}; + if (it == m_pq_history.end()) { + if (m_pq_history.size() >= PQ_MAX_ENDPOINT_HISTORY) { + // Spare active fallback windows: evict the oldest insertion outside one, and only when + // every endpoint is in one, the window that ends first. + m_pq_history.erase(std::ranges::min_element(m_pq_history, {}, [now](const auto& entry) { + const PQEndpointHistory& history{entry.second}; + const bool active{history.expires && now < *history.expires}; + return std::tuple{active, active ? *history.expires : NodeSeconds{}, history.insertion_order}; + })); + } + it = m_pq_history.emplace(endpoint, PQEndpointHistory{.insertion_order = m_pq_history_insertions++}).first; + } + PQEndpointHistory& history{it->second}; + ExpirePQFallback(history, now); + // A connection that was already running when the window began can't extend it. + if (history.expires) return std::nullopt; + ++history.streak; + history.last_failure = now; + history.cause = outcome; + history.reason = reason; + if (history.streak < PQ_FAILURE_THRESHOLD) return std::nullopt; + + history.entered = now; + history.expires = now + PQFallbackWindow(history); + ++m_pq_counters[PQDirection(/*inbound=*/false)].fallback; + AddPQFailure({.sequence = 0, + .time = now, + .endpoint = endpoint, + .inbound = false, + .connection_type = node.m_conn_type, + .peer_id = node.GetId(), + .outcome = PQOutcome::FALLBACK, + .reason = std::string{PQOutcomeString(outcome)}}); + return history.expires; +} + +bool CConnman::IsPQFallback(const PQEndpointKey& endpoint, NodeSeconds now) +{ + AssertLockNotHeld(m_pq_mutex); + LOCK(m_pq_mutex); + const auto it{m_pq_history.find(endpoint)}; + if (it == m_pq_history.end()) return false; + ExpirePQFallback(it->second, now); + return it->second.expires.has_value(); +} + +void CConnman::AddPQFailure(PQFailureEntry entry) +{ + AssertLockHeld(m_pq_mutex); + PQFailureRing& ring{m_pq_failures[PQDirection(entry.inbound)]}; + entry.sequence = ++ring.last_sequence; + ring.entries.push_back(std::move(entry)); + if (ring.entries.size() > PQ_FAILURE_RING_SIZE) { + ring.entries.pop_front(); + ++ring.dropped; + } +} + +void CConnman::LogPQOutcome(const CNode& node, const PQLogLine& line) const +{ + const auto backends{mlkem::GetBackendNames()}; + const std::string_view word{PQOutcomeString(line.outcome)}; + const std::string conn_type{node.ConnectionTypeAsString()}; + const std::string peeraddr{node.LogIP(fLogIPs)}; + if (node.IsInboundConn()) { + // Outsiders can cause every inbound outcome at will, so these are debug lines. + switch (line.outcome) { + case PQOutcome::SWITCHED: + LogDebug(BCLog::NET, "v2 pq: switched role=responder conn_type=%s peer=%d%s", conn_type, node.GetId(), peeraddr); + return; + case PQOutcome::LEGACY_PEER: + case PQOutcome::ABANDONED: + LogDebug(BCLog::NET, "v2 pq: %s reason=%s role=responder conn_type=%s peer=%d%s", word, line.reason, conn_type, node.GetId(), peeraddr); + return; + case PQOutcome::MALFORMED_RECORD: + case PQOutcome::FIRST_PACKET_FAILED: + case PQOutcome::INTERNAL_ERROR: + LogDebug(BCLog::NET, "v2 pq: %s reason=%s role=responder conn_type=%s peer=%d%s arith=%s keccak=%s", + word, line.reason, conn_type, node.GetId(), peeraddr, backends.arith, backends.keccak); + return; + case PQOutcome::CONFIRMED: + case PQOutcome::CLOSED_AFTER_SWITCH: + case PQOutcome::FALLBACK: + return; + } // no default case, so the compiler can warn about missing cases + return; + } + // Outbound failures are logged at info level, each from its own call site. + switch (line.outcome) { + case PQOutcome::SWITCHED: + LogDebug(BCLog::NET, "v2 pq: switched role=initiator conn_type=%s peer=%d%s", conn_type, node.GetId(), peeraddr); + return; + case PQOutcome::LEGACY_PEER: + case PQOutcome::ABANDONED: + LogDebug(BCLog::NET, "v2 pq: %s reason=%s role=initiator conn_type=%s peer=%d%s", word, line.reason, conn_type, node.GetId(), peeraddr); + return; + case PQOutcome::MALFORMED_RECORD: + LogInfo("v2 pq: malformed_record reason=%s role=initiator conn_type=%s peer=%d%s arith=%s keccak=%s", + line.reason, conn_type, node.GetId(), peeraddr, backends.arith, backends.keccak); + return; + case PQOutcome::FIRST_PACKET_FAILED: + LogInfo("v2 pq: first_packet_failed reason=%s role=initiator conn_type=%s peer=%d%s arith=%s keccak=%s", + line.reason, conn_type, node.GetId(), peeraddr, backends.arith, backends.keccak); + return; + case PQOutcome::CLOSED_AFTER_SWITCH: + LogInfo("v2 pq: closed_after_switch cause=%s role=initiator conn_type=%s peer=%d%s arith=%s keccak=%s", + line.reason, conn_type, node.GetId(), peeraddr, backends.arith, backends.keccak); + return; + case PQOutcome::INTERNAL_ERROR: + LogInfo("v2 pq: internal_error reason=%s role=initiator conn_type=%s peer=%d%s arith=%s keccak=%s", + line.reason, conn_type, node.GetId(), peeraddr, backends.arith, backends.keccak); + return; + case PQOutcome::FALLBACK: + LogWarning("v2 pq: fallback cause=%s until=%s role=initiator conn_type=%s peer=%d%s arith=%s keccak=%s", + line.reason, FormatISO8601DateTime(TicksSinceEpoch(line.until.value_or(NodeSeconds{}))), + conn_type, node.GetId(), peeraddr, backends.arith, backends.keccak); + return; + case PQOutcome::CONFIRMED: + return; + } // no default case, so the compiler can warn about missing cases +} + +void CConnman::LogPQLocalFault() const +{ + const auto backends{mlkem::GetBackendNames()}; + LogWarning("v2 pq: local_fault arith=%s keccak=%s: hybrid handshakes failed with %u distinct endpoints and none succeeded " + "since startup, so this node's own ML-KEM code may be at fault.", + backends.arith, backends.keccak, PQ_LOCAL_FAULT_THRESHOLD); +} + +PQTransportStats CConnman::GetPQTransportStats() const +{ + AssertLockNotHeld(m_pq_mutex); + AssertLockNotHeld(m_pq_shed_mutex); + // The two mutexes never nest. + PQLoadSheddingStats load_shedding; + { + LOCK(m_pq_shed_mutex); + load_shedding = {.active = m_pq_shedding_since.has_value(), + .threshold_per_s = m_pq_shed_threshold, + .since = m_pq_shedding_since.value_or(NodeSeconds{})}; + } + LOCK(m_pq_mutex); + PQTransportStats stats{ + .instance_id = m_pq_instance_id, + .since = m_pq_since, + .inbound = LoadPQCounts(m_pq_counters[PQDirection(/*inbound=*/true)]), + .outbound = LoadPQCounts(m_pq_counters[PQDirection(/*inbound=*/false)]), + .inbound_failures = m_pq_failures[PQDirection(/*inbound=*/true)], + .outbound_failures = m_pq_failures[PQDirection(/*inbound=*/false)], + .fallback_set = {}, + .failure_streaks = {}, + .load_shedding = load_shedding, + }; + const NodeSeconds now{Now()}; + for (const auto& [endpoint, history] : m_pq_history) { + if (history.expires) { + // A window that expired ended the streak; nothing to report until the next failure. + if (now < *history.expires) { + stats.fallback_set.push_back({endpoint, history.cause, history.reason, history.streak, *history.entered, *history.expires}); + } + } else if (history.streak > 0) { + stats.failure_streaks.push_back({endpoint, history.cause, history.reason, history.streak, history.last_failure, PQFallbackWindow(history)}); + } + } + return stats; +} + +bool CConnman::AllowPQOffer(void* connman) noexcept +{ + return static_cast(connman)->AllowPQOffer(); +} + +bool CConnman::AllowPQOffer() noexcept +{ + AssertLockNotHeld(m_pq_shed_mutex); + const PQShedSeconds now{PQShedNow()}; + bool stopped; + std::optional started; + bool offer; + { + LOCK(m_pq_shed_mutex); + stopped = AdvancePQShedding(now); + ++m_pq_shed_attempts; + if (!m_pq_shedding_since && m_pq_shed_attempts > m_pq_shed_threshold) { + m_pq_shedding_since = Now(); + // The quiet seconds before the flood don't count toward its end. + m_pq_quiet_since.reset(); + started = m_pq_shed_threshold; + } + offer = !m_pq_shedding_since; + } + if (!offer) ++m_pq_counters[PQDirection(/*inbound=*/true)].shed; + if (stopped) LogPQShedding(std::nullopt); + if (started) LogPQShedding(started); + return offer; +} + +bool CConnman::AdvancePQShedding(PQShedSeconds now) +{ + AssertLockHeld(m_pq_shed_mutex); + if (now > m_pq_shed_second) { + // Close the current second, and the seconds without attempts since: a second is quiet + // with fewer than half the threshold's attempts. + if (2 * m_pq_shed_attempts >= m_pq_shed_threshold) { + m_pq_quiet_since.reset(); + if (now > m_pq_shed_second + 1s) m_pq_quiet_since = m_pq_shed_second + 1s; + } else if (!m_pq_quiet_since) { + m_pq_quiet_since = m_pq_shed_second; + } + m_pq_shed_second = now; + m_pq_shed_attempts = 0; + } + if (m_pq_shedding_since && m_pq_quiet_since && now - *m_pq_quiet_since >= PQ_SHED_QUIET_PERIOD) { + m_pq_shedding_since.reset(); + return true; + } + return false; +} + +void CConnman::UpdatePQShedding(PQShedSeconds now) +{ + AssertLockNotHeld(m_pq_shed_mutex); + if (WITH_LOCK(m_pq_shed_mutex, return AdvancePQShedding(now))) LogPQShedding(std::nullopt); +} + +void CConnman::LogPQShedding(std::optional started_threshold) +{ + if (started_threshold) { + LogInfo("v2 pq: load_shedding started threshold_per_s=%u", *started_threshold); + } else { + LogInfo("v2 pq: load_shedding stopped"); + } +} + void CConnman::DisconnectNodes() { AssertLockNotHeld(m_nodes_mutex); @@ -2210,7 +2737,7 @@ void CConnman::DisconnectNodes() for (CNode* pnode : m_nodes) { if (!pnode->fDisconnect) { LogDebug(BCLog::NET, "Network not active, %s\n", pnode->DisconnectMsg(fLogIPs)); - pnode->fDisconnect = true; + pnode->RequestDisconnect(); } } } @@ -2241,9 +2768,13 @@ void CConnman::DisconnectNodes() // release outbound grant (if any) pnode->grantOutbound.Release(); - // close socket and cleanup + // close socket and cleanup; a node with no recorded close cause counts as LOCAL pnode->CloseSocketDisconnect(); + // account the hybrid negotiation's close outcome and wipe its pending secrets + // before the disconnected pool keeps the node + ObservePQ(*pnode, /*finalizing=*/true); + // update connection count by network if (pnode->IsManualOrFullOutboundConn()) --m_network_conn_counts[pnode->addr.GetNetwork()]; @@ -2472,6 +3003,7 @@ void CConnman::SocketHandlerConnected(const std::vector& nodes, ); pnode->CloseSocketDisconnect(); } + ObservePQ(*pnode, /*finalizing=*/false); RecordBytesRecv(nBytes); if (notify) { pnode->MarkReceivedMsgsForProcessing(); @@ -2484,7 +3016,7 @@ void CConnman::SocketHandlerConnected(const std::vector& nodes, if (!pnode->fDisconnect) { LogDebug(BCLog::NET, "socket closed, %s\n", pnode->DisconnectMsg(fLogIPs)); } - pnode->CloseSocketDisconnect(); + pnode->CloseSocketDisconnect(NodeCloseCause::PEER_EOF); } else if (nBytes < 0) { @@ -2495,12 +3027,12 @@ void CConnman::SocketHandlerConnected(const std::vector& nodes, if (!pnode->fDisconnect) { LogDebug(BCLog::NET, "socket recv error, %s: %s\n", pnode->DisconnectMsg(fLogIPs), NetworkErrorString(nErr)); } - pnode->CloseSocketDisconnect(); + pnode->CloseSocketDisconnect(NodeCloseCause::PEER_RESET); } } } - if (InactivityCheck(*pnode)) pnode->fDisconnect = true; + if (InactivityCheck(*pnode)) pnode->RequestDisconnect(NodeCloseCause::TIMEOUT); } } @@ -2525,6 +3057,7 @@ void CConnman::ThreadSocketHandler() { DisconnectNodes(); NotifyNumConnectionsChanged(); + if (m_pq_mode == PQMode::NEGOTIATE) UpdatePQShedding(PQShedNow()); SocketHandler(); } } @@ -3781,6 +4314,9 @@ void CConnman::StopNodes() for (CNode* pnode : nodes) { LogDebug(BCLog::NET, "Stopping node, %s", pnode->DisconnectMsg(fLogIPs)); pnode->CloseSocketDisconnect(); + // Wipe the hybrid negotiation's pending secrets before the node goes, as DisconnectNodes() + // does. A shutdown close is our own decision, so it records no outcome. + ObservePQ(*pnode, /*finalizing=*/true, /*record_outcomes=*/false); DeleteNode(pnode); } @@ -3944,7 +4480,7 @@ bool CConnman::DisconnectNode(const std::string& strNode) LOCK(m_nodes_mutex); if (CNode* pnode = FindNode(strNode)) { LogDebug(BCLog::NET, "disconnect by address%s match, %s", (fLogIPs ? strprintf("=%s", strNode) : ""), pnode->DisconnectMsg(fLogIPs)); - pnode->fDisconnect = true; + pnode->RequestDisconnect(); return true; } return false; @@ -3957,7 +4493,7 @@ bool CConnman::DisconnectNode(const CSubNet& subnet) for (CNode* pnode : m_nodes) { if (subnet.Match(pnode->addr)) { LogDebug(BCLog::NET, "disconnect by subnet%s match, %s", (fLogIPs ? strprintf("=%s", subnet.ToString()) : ""), pnode->DisconnectMsg(fLogIPs)); - pnode->fDisconnect = true; + pnode->RequestDisconnect(); disconnected = true; } } @@ -3975,7 +4511,7 @@ bool CConnman::DisconnectNode(NodeId id) for(CNode* pnode : m_nodes) { if (id == pnode->GetId()) { LogDebug(BCLog::NET, "disconnect by id, %s", pnode->DisconnectMsg(fLogIPs)); - pnode->fDisconnect = true; + pnode->RequestDisconnect(); return true; } } @@ -4087,10 +4623,10 @@ ServiceFlags CConnman::GetLocalServices() const return m_local_services; } -static std::unique_ptr MakeTransport(NodeId id, bool use_v2transport, bool inbound) noexcept +static std::unique_ptr MakeTransport(NodeId id, bool use_v2transport, bool inbound, const V2PQOptions& pq) noexcept { if (use_v2transport) { - return std::make_unique(id, /*initiating=*/!inbound); + return std::make_unique(id, /*initiating=*/!inbound, pq); } else { return std::make_unique(id); } @@ -4106,7 +4642,7 @@ CNode::CNode(NodeId idIn, ConnectionType conn_type_in, bool inbound_onion, CNodeOptions&& node_opts) - : m_transport{MakeTransport(idIn, node_opts.use_v2transport, conn_type_in == ConnectionType::INBOUND)}, + : m_transport{MakeTransport(idIn, node_opts.use_v2transport, conn_type_in == ConnectionType::INBOUND, node_opts.pq)}, m_permission_flags{node_opts.permission_flags}, m_sock{sock}, m_connected{GetTime()}, @@ -4119,6 +4655,7 @@ CNode::CNode(NodeId idIn, nKeyedNetGroup{nKeyedNetGroupIn}, m_conn_type{conn_type_in}, m_is_archive_connection{node_opts.is_archive_connection}, + m_pq_endpoint{node_opts.pq_endpoint ? std::move(*node_opts.pq_endpoint) : PQEndpointKey{CService{addrIn}}}, id{idIn}, nLocalHostNonce{nLocalHostNonceIn}, m_recv_flood_size{node_opts.recv_flood_size}, diff --git a/src/net.h b/src/net.h index 8f061ed795..92ec68797b 100644 --- a/src/net.h +++ b/src/net.h @@ -33,8 +33,11 @@ #include #include #include +#include +#include #include +#include #include #include #include @@ -44,8 +47,12 @@ #include #include #include +#include +#include +#include #include #include +#include #include class AddrMan; @@ -371,6 +378,12 @@ class Transport { /** Whether upon disconnections, a reconnect with V1 is warranted. */ virtual bool ShouldReconnectV1() const noexcept = 0; + + /** The hybrid negotiation's progress and first failure (nothing, without a negotiation). */ + virtual PQHandshake::Snapshot GetPQSnapshot() const noexcept { return {}; } + + /** Wipe the hybrid negotiation's pending secrets; the current keys stay in use. */ + virtual void ClearPQSecrets() noexcept {} }; class V1Transport final : public Transport @@ -455,6 +468,14 @@ class V1Transport final : public Transport bool ShouldReconnectV1() const noexcept override { return false; } }; +/** Asked by a negotiating responder, at its key generation point, whether to offer: load + * shedding (#184 part 5). Each call counts one attempted offer. */ +struct PQOfferGate { + using Allow = bool (*)(void* context) noexcept; + Allow allow{nullptr}; + void* context{nullptr}; +}; + /** The hybrid post-quantum negotiation of one v2 connection (doc/design/pq-transport.md). */ struct V2PQOptions { PQMode mode{PQMode::OFF}; @@ -464,6 +485,8 @@ struct V2PQOptions { /** Test only: invert byte 0 of the local ML-KEM shared secret before deriving keys, so the * peer's key confirmation check fails. EK and CT are unchanged. */ bool corrupt_shared_secret{false}; + /** Responders only; without one, a negotiating responder always offers. */ + PQOfferGate offer_gate{}; }; class V2Transport final : public Transport @@ -640,6 +663,8 @@ class V2Transport final : public Transport RecvState m_recv_state GUARDED_BY(m_recv_mutex); /** The hybrid negotiation state: KEM secrets and progress. */ PQHandshake m_pq GUARDED_BY(m_recv_mutex); + /** Whether the offer gate declined to offer (load shedding): plain v2, status off. */ + bool m_pq_shed GUARDED_BY(m_recv_mutex){false}; /** Lock for sending-side fields. If both sending and receiving fields are accessed, * m_recv_mutex must be acquired before m_send_mutex. */ @@ -739,8 +764,8 @@ class V2Transport final : public Transport // Miscellaneous functions. bool ShouldReconnectV1() const noexcept override EXCLUSIVE_LOCKS_REQUIRED(!m_recv_mutex, !m_send_mutex); Info GetInfo() const noexcept override EXCLUSIVE_LOCKS_REQUIRED(!m_recv_mutex); - /** The hybrid negotiation's progress and first failure. */ - PQHandshake::Snapshot GetPQSnapshot() const noexcept EXCLUSIVE_LOCKS_REQUIRED(!m_recv_mutex); + PQHandshake::Snapshot GetPQSnapshot() const noexcept override EXCLUSIVE_LOCKS_REQUIRED(!m_recv_mutex); + void ClearPQSecrets() noexcept override EXCLUSIVE_LOCKS_REQUIRED(!m_recv_mutex, !m_send_mutex); /** Test only: whether the decapsulation key or the retained ECDH secret is still held. */ bool HoldsHybridSecretsForTesting() const noexcept EXCLUSIVE_LOCKS_REQUIRED(!m_recv_mutex); @@ -749,6 +774,192 @@ class V2Transport final : public Transport void DiscardHybridSecretForTesting() noexcept EXCLUSIVE_LOCKS_REQUIRED(!m_recv_mutex, !m_send_mutex); }; +/** + * Why a connection closed. The first recorded cause wins (see + * CNode::RequestDisconnect()), so cleanup that runs after a peer closed the + * connection never relabels it. + */ +enum class NodeCloseCause : uint8_t { + //! No cause recorded. A node that reaches DisconnectNodes() without one + //! (fDisconnect set directly) is finalized there as LOCAL. + NONE = 0, + //! Our own decision: operator, eviction, feeler, protocol policy, network off or shutdown. + LOCAL = 1, + //! recv() returned 0: the peer closed the connection. + PEER_EOF = 2, + //! recv() failed with a non-transient error, such as a connection reset. + PEER_RESET = 3, + //! send() failed with a non-transient error. + SEND_ERROR = 4, + //! The peer stopped responding: InactivityCheck() or the ping timeout. + TIMEOUT = 5, +}; + +/** A name-proxy destination: the SOCKS destination hostname and the effective port. */ +struct PQNameEndpoint { + std::string hostname; + uint16_t port{0}; + friend bool operator==(const PQNameEndpoint&, const PQNameEndpoint&) = default; + friend auto operator<=>(const PQNameEndpoint&, const PQNameEndpoint&) = default; +}; + +/** + * The endpoint a connection's hybrid negotiation outcomes are accounted to. A resolved + * destination, and an inbound peer, is its CService: the address, of any network, and the port + * (for an inbound peer, its source port). A name-proxy destination, never resolved locally, is + * its normalized hostname and port; never the proxy's own address. + */ +using PQEndpointKey = std::variant; + +/** The endpoint of a name-proxy destination: hostname in lowercase ASCII, one trailing root dot removed. */ +PQNameEndpoint MakePQNameEndpoint(std::string_view hostname, uint16_t port); + +/** A hybrid negotiation event. Each one's word (PQOutcomeString) is its counter key and log word. */ +enum class PQOutcome : uint8_t { + SWITCHED = 0, + //! The peer's key confirmation verified: a success, not counted (switched counted it). + CONFIRMED = 1, + LEGACY_PEER = 2, + MALFORMED_RECORD = 3, + FIRST_PACKET_FAILED = 4, + //! A responder's offer was queued, but no authenticated version arrived before the close. + ABANDONED = 5, + //! Outbound: the peer closed (or timed out) after we switched, before its confirmation. + CLOSED_AFTER_SWITCH = 6, + //! Outbound: an endpoint entered the fallback set. + FALLBACK = 7, + INTERNAL_ERROR = 8, +}; + +std::string_view PQOutcomeString(PQOutcome outcome) noexcept; + +/** Hybrid negotiation event counts of one direction. They count events, not exclusive final + * states: a switch whose confirmation then fails counts switched and first_packet_failed. */ +template +struct PQCountsBase { + T switched{0}; + T legacy_peer{0}; + T malformed_record{0}; + T first_packet_failed{0}; + //! Inbound only: only responders offer. + T abandoned{0}; + T internal_error{0}; + //! Outbound only. + T closed_after_switch{0}; + T fallback{0}; + //! Inbound only: offers not made because of load shedding. + T shed{0}; +}; +using PQCounters = PQCountsBase>; +using PQCounts = PQCountsBase; + +/** A non-success hybrid negotiation outcome, as kept in a failure ring. */ +struct PQFailureEntry { + uint64_t sequence; + NodeSeconds time; + PQEndpointKey endpoint; + bool inbound; + ConnectionType connection_type; + NodeId peer_id; + PQOutcome outcome; + //! Assigned only from the fixed reason vocabulary, never from peer bytes. + std::string reason; +}; + +/** The most recent failures of one direction, oldest first, with their own sequence. */ +struct PQFailureRing { + //! The sequence of the newest entry ever added (the first is 1); 0 if none. + uint64_t last_sequence{0}; + //! How many entries were evicted to keep the bound. + uint64_t dropped{0}; + std::deque entries; +}; + +/** At most this many entries in each failure ring. */ +inline constexpr size_t PQ_FAILURE_RING_SIZE{256}; + +/** Consecutive counted failures (outbound malformed_record, first_packet_failed and + * closed_after_switch) that put an endpoint in the fallback set. */ +inline constexpr uint32_t PQ_FAILURE_THRESHOLD{3}; +/** An endpoint's fallback windows: its first entry, its second, then every later one. */ +inline constexpr std::array PQ_FALLBACK_WINDOWS{ + std::chrono::seconds{3600}, std::chrono::seconds{14400}, std::chrono::seconds{86400}}; +/** At most this many endpoints in the fallback, streak and escalation history. Eviction spares + * endpoints in an active fallback window while any other is left. */ +inline constexpr size_t PQ_MAX_ENDPOINT_HISTORY{1000}; +/** Distinct endpoints with counted failures, and no success since startup, that suggest a local fault. */ +inline constexpr size_t PQ_LOCAL_FAULT_THRESHOLD{8}; + +/** Inbound offers attempted within one second above which responders stop offering. R3's flood + * measurement (#222) sets the final value; CConnman::Options can override it. */ +inline constexpr uint64_t DEFAULT_PQ_SHED_THRESHOLD_PER_S{1000}; +/** Consecutive seconds with fewer than half the threshold's attempts that end load shedding. */ +inline constexpr std::chrono::seconds PQ_SHED_QUIET_PERIOD{10}; + +/** Inbound offer load shedding. */ +struct PQLoadSheddingStats { + bool active; + uint64_t threshold_per_s; + //! When the current shedding period started; the epoch (0) when inactive. + NodeSeconds since; +}; + +/** An endpoint's counted failures and fallback windows. */ +struct PQEndpointHistory { + //! Consecutive counted failures since the last fallback window ended. + uint32_t streak{0}; + //! Fallback windows that ended; saturates at PQ_FALLBACK_WINDOWS.size(). + uint8_t completed_windows{0}; + //! The active fallback window, if any. + std::optional entered{}; + std::optional expires{}; + //! The latest counted failure. + NodeSeconds last_failure{}; + PQOutcome cause{PQOutcome::MALFORMED_RECORD}; + //! Assigned only from the fixed reason vocabulary. + std::string reason{}; + //! Outside fallback windows, the oldest insertion is evicted first; unique, so equal times can't tie. + uint64_t insertion_order{0}; +}; + +/** An endpoint in the fallback set. */ +struct PQFallbackStats { + PQEndpointKey endpoint; + //! The failure that completed the streak, and its reason. + PQOutcome cause; + std::string reason; + uint32_t streak; + NodeSeconds entered; + NodeSeconds expires; +}; + +/** An endpoint with counted failures that is not in the fallback set. */ +struct PQStreakStats { + PQEndpointKey endpoint; + //! The latest counted failure, and its reason. + PQOutcome cause; + std::string reason; + uint32_t streak; + NodeSeconds last_failure; + //! The window the endpoint would enter at PQ_FAILURE_THRESHOLD. + std::chrono::seconds next_window; +}; + +/** An owning copy of the hybrid transport statistics, without atomics. */ +struct PQTransportStats { + //! Random per process start; counters, rings and history count from since. + uint256 instance_id; + NodeSeconds since; + PQCounts inbound; + PQCounts outbound; + PQFailureRing inbound_failures; + //! Outbound connections, manual ones included. + PQFailureRing outbound_failures; + std::vector fallback_set; + std::vector failure_streaks; + PQLoadSheddingStats load_shedding; +}; + struct CNodeOptions { NetPermissionFlags permission_flags = NetPermissionFlags::None; @@ -757,6 +968,10 @@ struct CNodeOptions size_t recv_flood_size{DEFAULT_MAXRECEIVEBUFFER * 1000}; bool use_v2transport = false; bool is_archive_connection = false; + //! The hybrid negotiation of a v2 transport; off unless set. + V2PQOptions pq{}; + //! The endpoint key of the connection; the peer's address and port if unset. + std::optional pq_endpoint{}; }; /** Information about a peer */ @@ -818,7 +1033,7 @@ class CNode /** fSuccessfullyConnected is set to true on receiving VERACK from the peer. */ std::atomic_bool fSuccessfullyConnected{false}; // Setting fDisconnect to true will cause the node to be disconnected the - // next time DisconnectNodes() runs + // next time DisconnectNodes() runs. Set it with RequestDisconnect(). std::atomic_bool fDisconnect{false}; CountingSemaphoreGrant<> grantOutbound; std::atomic nRefCount{0}; @@ -829,6 +1044,13 @@ class CNode const ConnectionType m_conn_type; const bool m_is_archive_connection{false}; + //! The endpoint this connection's hybrid negotiation outcomes are accounted to. + const PQEndpointKey m_pq_endpoint; + + //! The PQOutcome events already accounted for this connection (a bit per outcome), and + //! whether its accounting is final. Accessed only by CConnman, under its m_pq_mutex. + uint16_t m_pq_accounted_events{0}; + bool m_pq_finalized{false}; /** Move all messages from the received queue to the processing queue. */ void MarkReceivedMsgsForProcessing() @@ -1033,7 +1255,19 @@ class CNode nRefCount--; } - void CloseSocketDisconnect() EXCLUSIVE_LOCKS_REQUIRED(!m_sock_mutex); + /** + * Mark the node for disconnection and record why. Only the first cause is + * kept: later or concurrent calls, including cleanup, leave it unchanged. + * The cause is recorded before fDisconnect is set, so a thread that sees + * fDisconnect also sees the cause. + */ + void RequestDisconnect(NodeCloseCause cause = NodeCloseCause::LOCAL) noexcept; + + /** RequestDisconnect(cause), then close the socket. */ + void CloseSocketDisconnect(NodeCloseCause cause = NodeCloseCause::LOCAL) EXCLUSIVE_LOCKS_REQUIRED(!m_sock_mutex); + + /** The first recorded close cause, or NONE if none has been recorded. */ + NodeCloseCause GetCloseCause() const noexcept { return m_close_cause.load(); } void CopyStats(CNodeStats& stats) EXCLUSIVE_LOCKS_REQUIRED(!m_subver_mutex, !m_addr_local_mutex, !cs_vSend, !cs_vRecv); @@ -1066,6 +1300,9 @@ class CNode const uint64_t nLocalHostNonce; std::atomic m_greatest_common_version{INIT_PROTO_VERSION}; + /** Written only by RequestDisconnect(), which keeps the first cause. */ + std::atomic m_close_cause{NodeCloseCause::NONE}; + const size_t m_recv_flood_size; std::list vRecvMsg; // Used only by SocketHandler thread @@ -1173,9 +1410,11 @@ class CConnman bool m_i2p_accept_incoming; bool whitelist_forcerelay = DEFAULT_WHITELISTFORCERELAY; bool whitelist_relay = DEFAULT_WHITELISTRELAY; + //! Inbound offer load shedding threshold (unit tests and the flood lab). + uint64_t pq_shed_threshold_per_s{DEFAULT_PQ_SHED_THRESHOLD_PER_S}; }; - void Init(const Options& connOptions) EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex, !m_total_bytes_sent_mutex) + void Init(const Options& connOptions) EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex, !m_total_bytes_sent_mutex, !m_pq_shed_mutex) { AssertLockNotHeld(m_total_bytes_sent_mutex); @@ -1207,6 +1446,11 @@ class CConnman m_onion_binds = connOptions.onion_binds; whitelist_forcerelay = connOptions.whitelist_forcerelay; whitelist_relay = connOptions.whitelist_relay; + { + LOCK(m_pq_shed_mutex); + // At least 1: a threshold of 0 would shed every offer and never stop. + m_pq_shed_threshold = std::max(connOptions.pq_shed_threshold_per_s, 1); + } } CConnman(uint64_t seed0, uint64_t seed1, AddrMan& addrman, const NetGroupManager& netgroupman, @@ -1214,11 +1458,11 @@ class CConnman ~CConnman(); - bool Start(CScheduler& scheduler, const Options& options) EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex, !m_added_nodes_mutex, !m_addr_fetches_mutex, !mutexMsgProc); + bool Start(CScheduler& scheduler, const Options& options) EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex, !m_added_nodes_mutex, !m_addr_fetches_mutex, !mutexMsgProc, !m_pq_shed_mutex); void StopThreads(); - void StopNodes(); - void Stop() + void StopNodes() EXCLUSIVE_LOCKS_REQUIRED(!m_pq_mutex); + void Stop() EXCLUSIVE_LOCKS_REQUIRED(!m_pq_mutex) { StopThreads(); StopNodes(); @@ -1228,7 +1472,7 @@ class CConnman bool GetNetworkActive() const { return fNetworkActive; }; bool GetUseAddrmanOutgoing() const { return m_use_addrman_outgoing; }; void SetNetworkActive(bool active); - void OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CountingSemaphoreGrant<>&& grant_outbound, const char* strDest, ConnectionType conn_type, bool use_v2transport, bool is_archive_connection = false) EXCLUSIVE_LOCKS_REQUIRED(!m_unused_i2p_sessions_mutex); + void OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CountingSemaphoreGrant<>&& grant_outbound, const char* strDest, ConnectionType conn_type, bool use_v2transport, bool is_archive_connection = false) EXCLUSIVE_LOCKS_REQUIRED(!m_unused_i2p_sessions_mutex, !m_pq_mutex); bool CheckIncomingNonce(uint64_t nonce); void ASMapHealthCheck(); @@ -1324,7 +1568,7 @@ class CConnman * - Max total outbound connection capacity filled * - Max connection capacity for type is filled */ - bool AddConnection(const std::string& address, ConnectionType conn_type, bool use_v2transport) EXCLUSIVE_LOCKS_REQUIRED(!m_unused_i2p_sessions_mutex); + bool AddConnection(const std::string& address, ConnectionType conn_type, bool use_v2transport) EXCLUSIVE_LOCKS_REQUIRED(!m_unused_i2p_sessions_mutex, !m_pq_mutex); size_t GetNodeCount(ConnectionDirection) const; std::map getNetLocalAddresses() const; @@ -1373,6 +1617,13 @@ class CConnman uint64_t GetTotalBytesRecv() const; uint64_t GetTotalBytesSent() const EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); + /** An owning copy of the hybrid transport counters, failure rings and endpoint history. */ + PQTransportStats GetPQTransportStats() const EXCLUSIVE_LOCKS_REQUIRED(!m_pq_mutex, !m_pq_shed_mutex); + + /** Whether new outbound connections to endpoint run plain v2, because repeated hybrid + * failures put it in the fallback set. Ends a window that expired. */ + bool IsPQFallback(const PQEndpointKey& endpoint, NodeSeconds now) EXCLUSIVE_LOCKS_REQUIRED(!m_pq_mutex); + /** Get a unique deterministic randomizer. */ CSipHasher GetDeterministicRandomizer(uint64_t id) const; @@ -1405,10 +1656,10 @@ class CConnman bool Bind(const CService& addr, unsigned int flags, NetPermissionFlags permissions); bool InitBinds(const Options& options); - void ThreadOpenAddedConnections() EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex, !m_unused_i2p_sessions_mutex, !m_reconnections_mutex); + void ThreadOpenAddedConnections() EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex, !m_unused_i2p_sessions_mutex, !m_reconnections_mutex, !m_pq_mutex); void AddAddrFetch(const std::string& strDest) EXCLUSIVE_LOCKS_REQUIRED(!m_addr_fetches_mutex); - void ProcessAddrFetch() EXCLUSIVE_LOCKS_REQUIRED(!m_addr_fetches_mutex, !m_unused_i2p_sessions_mutex); - void ThreadOpenConnections(std::vector connect, std::span seed_nodes) EXCLUSIVE_LOCKS_REQUIRED(!m_addr_fetches_mutex, !m_added_nodes_mutex, !m_nodes_mutex, !m_unused_i2p_sessions_mutex, !m_reconnections_mutex); + void ProcessAddrFetch() EXCLUSIVE_LOCKS_REQUIRED(!m_addr_fetches_mutex, !m_unused_i2p_sessions_mutex, !m_pq_mutex); + void ThreadOpenConnections(std::vector connect, std::span seed_nodes) EXCLUSIVE_LOCKS_REQUIRED(!m_addr_fetches_mutex, !m_added_nodes_mutex, !m_nodes_mutex, !m_unused_i2p_sessions_mutex, !m_reconnections_mutex, !m_pq_mutex); void ThreadMessageHandler() EXCLUSIVE_LOCKS_REQUIRED(!mutexMsgProc); void ThreadI2PAcceptIncoming(); void AcceptConnection(const ListenSocket& hListenSocket); @@ -1426,7 +1677,7 @@ class CConnman const CService& addr_bind, const CService& addr); - void DisconnectNodes() EXCLUSIVE_LOCKS_REQUIRED(!m_reconnections_mutex, !m_nodes_mutex); + void DisconnectNodes() EXCLUSIVE_LOCKS_REQUIRED(!m_reconnections_mutex, !m_nodes_mutex, !m_pq_mutex); void NotifyNumConnectionsChanged(); /** Return true if the peer is inactive and should be disconnected. */ bool InactivityCheck(const CNode& node) const; @@ -1441,7 +1692,7 @@ class CConnman /** * Check connected and listening sockets for IO readiness and process them accordingly. */ - void SocketHandler() EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex, !mutexMsgProc); + void SocketHandler() EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex, !mutexMsgProc, !m_pq_mutex); /** * Do the read/write for connected sockets that are ready for IO. @@ -1450,7 +1701,7 @@ class CConnman */ void SocketHandlerConnected(const std::vector& nodes, const Sock::EventsPerSock& events_per_sock) - EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex, !mutexMsgProc); + EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex, !mutexMsgProc, !m_pq_mutex); /** * Accept incoming connections, one from each read-ready listening socket. @@ -1458,7 +1709,7 @@ class CConnman */ void SocketHandlerListening(const Sock::EventsPerSock& events_per_sock); - void ThreadSocketHandler() EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex, !mutexMsgProc, !m_nodes_mutex, !m_reconnections_mutex); + void ThreadSocketHandler() EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex, !mutexMsgProc, !m_nodes_mutex, !m_reconnections_mutex, !m_pq_mutex, !m_pq_shed_mutex); void ThreadDNSAddressSeed() EXCLUSIVE_LOCKS_REQUIRED(!m_addr_fetches_mutex, !m_nodes_mutex); uint64_t CalculateKeyedNetGroup(const CNetAddr& ad) const; @@ -1474,7 +1725,7 @@ class CConnman bool AlreadyConnectedToAddress(const CAddress& addr); bool AttemptToEvictConnection(); - CNode* ConnectNode(CAddress addrConnect, const char *pszDest, bool fCountFailure, ConnectionType conn_type, bool use_v2transport, bool is_archive_connection = false) EXCLUSIVE_LOCKS_REQUIRED(!m_unused_i2p_sessions_mutex); + CNode* ConnectNode(CAddress addrConnect, const char *pszDest, bool fCountFailure, ConnectionType conn_type, bool use_v2transport, bool is_archive_connection = false) EXCLUSIVE_LOCKS_REQUIRED(!m_unused_i2p_sessions_mutex, !m_pq_mutex); void AddWhitelistPermissionFlags(NetPermissionFlags& flags, std::optional addr, const std::vector& ranges) const; void DeleteNode(CNode* pnode); @@ -1739,7 +1990,7 @@ class CConnman std::list m_reconnections GUARDED_BY(m_reconnections_mutex); /** Attempt reconnections, if m_reconnections non-empty. */ - void PerformReconnections() EXCLUSIVE_LOCKS_REQUIRED(!m_reconnections_mutex, !m_unused_i2p_sessions_mutex); + void PerformReconnections() EXCLUSIVE_LOCKS_REQUIRED(!m_reconnections_mutex, !m_unused_i2p_sessions_mutex, !m_pq_mutex); /** * Cap on the size of `m_unused_i2p_sessions`, to ensure it does not @@ -1784,6 +2035,118 @@ class CConnman std::vector m_nodes_copy; }; + /** One hybrid negotiation event of a connection, to log once m_pq_mutex is released. */ + struct PQLogLine { + PQOutcome outcome; + std::string_view reason; + //! FALLBACK: the end of the window. + std::optional until{}; + }; + + /** What AccountPQ() leaves to log. */ + struct PQLogLines { + std::vector outcomes; + //! Whether the one-time local-fault warning is due. + bool local_fault{false}; + }; + + /** + * Account the hybrid negotiation events of node that are new since its last observation. It + * runs after the node received bytes, and once more, finalizing, before DisconnectNodes() + * hands the node to the disconnected pool: that derives closed_after_switch and abandoned + * from the close cause, marks the node finalized so no later observation counts anything, + * and wipes the transport's pending secrets. With the network off, every close is our own + * decision, so nothing is recorded, but finalization still wipes. StopNodes() finalizes the + * connections it deletes at shutdown the same way, with record_outcomes false: shutdown + * closes are our own too. + * + * The transport is snapshotted under its own lock, which is released before m_pq_mutex is + * taken; nothing calls into the transport, socket, peer manager or node enumeration while + * holding m_pq_mutex. + */ + void ObservePQ(CNode& node, bool finalizing, bool record_outcomes = true) EXCLUSIVE_LOCKS_REQUIRED(!m_pq_mutex); + + /** Count each event of snapshot not yet accounted for node, once, add a ring entry for each + * non-success outcome, and update the endpoint history. Returns what to log. */ + PQLogLines AccountPQ(CNode& node, const PQHandshake::Snapshot& snapshot, NodeCloseCause cause, + bool finalizing, NodeSeconds now) EXCLUSIVE_LOCKS_REQUIRED(m_pq_mutex); + + /** + * Count an outbound connection's failure against its endpoint. The third consecutive one + * enters the fallback set, unless the endpoint completed a hybrid handshake since startup. + * A failure during an active window neither extends it nor enters again. Returns the end of + * a window it started, and sets local_fault when the local-fault warning is due. + */ + std::optional CountPQFailure(const CNode& node, PQOutcome outcome, std::string_view reason, NodeSeconds now, + bool& local_fault) EXCLUSIVE_LOCKS_REQUIRED(m_pq_mutex); + + /** Add a non-success outcome to the ring of its direction. */ + void AddPQFailure(PQFailureEntry entry) EXCLUSIVE_LOCKS_REQUIRED(m_pq_mutex); + + /** Log one event of node: inbound and outbound lines come from separate call sites, because + * the logger rate-limits per call site. */ + void LogPQOutcome(const CNode& node, const PQLogLine& line) const; + + /** Warn that this node's own ML-KEM code may be at fault. */ + void LogPQLocalFault() const; + + /** The hybrid negotiation mode of new v2 connections. Off until the option sets it (part 6); + * set before the connection threads start. */ + PQMode m_pq_mode{PQMode::OFF}; + + mutable Mutex m_pq_mutex; + //! Event counters, inbound [0] and outbound [1]. Written under m_pq_mutex, except shed, + //! which the offer gate counts under m_pq_shed_mutex. + std::array m_pq_counters; + //! Failure rings, inbound [0] and outbound [1]: inbound noise can't evict outbound evidence. + std::array m_pq_failures GUARDED_BY(m_pq_mutex); + const uint256 m_pq_instance_id{GetRandHash()}; + const NodeSeconds m_pq_since{Now()}; + //! Outbound endpoints with counted failures: streaks, fallback windows and escalation. + std::map m_pq_history GUARDED_BY(m_pq_mutex); + uint64_t m_pq_history_insertions GUARDED_BY(m_pq_mutex){0}; + //! Outbound endpoints that completed a hybrid handshake since startup. Never evicted; its + //! size is bounded by the distinct successful outbound endpoints. + std::set m_pq_successful_endpoints GUARDED_BY(m_pq_mutex); + //! Distinct endpoints with counted failures, up to PQ_LOCAL_FAULT_THRESHOLD. + std::set m_pq_failed_endpoints_for_warning GUARDED_BY(m_pq_mutex); + bool m_pq_local_fault_warned GUARDED_BY(m_pq_mutex){false}; + + /** + * Inbound offer load shedding, with one-second counters. The seconds come from the steady + * clock, so a wall clock step can't merge or stretch them; only the reported start time is + * wall-clock time. Its own mutex is taken under the transport's locks (the offer gate runs + * inside V2Transport), so nothing holding it calls into a transport, and it never nests with + * m_pq_mutex. + */ + using PQShedSeconds = std::chrono::time_point; + static PQShedSeconds PQShedNow() noexcept { return std::chrono::time_point_cast(MockableSteadyClock::now()); } + mutable Mutex m_pq_shed_mutex; + uint64_t m_pq_shed_threshold GUARDED_BY(m_pq_shed_mutex){DEFAULT_PQ_SHED_THRESHOLD_PER_S}; + //! The current second, and the offers attempted in it. + PQShedSeconds m_pq_shed_second GUARDED_BY(m_pq_shed_mutex){}; + uint64_t m_pq_shed_attempts GUARDED_BY(m_pq_shed_mutex){0}; + //! The first of the run of complete seconds with fewer than half the threshold's attempts. + std::optional m_pq_quiet_since GUARDED_BY(m_pq_shed_mutex); + //! When the current shedding period started (wall clock, for the stats); unset when offering. + std::optional m_pq_shedding_since GUARDED_BY(m_pq_shed_mutex); + + /** The offer gate of inbound transports: count one attempted offer, and decline it while + * shedding, counting it as shed. */ + bool AllowPQOffer() noexcept EXCLUSIVE_LOCKS_REQUIRED(!m_pq_shed_mutex); + //! The PQOfferGate function; the transport calls it under its own locks, never under m_pq_shed_mutex. + static bool AllowPQOffer(void* connman) noexcept NO_THREAD_SAFETY_ANALYSIS; + + /** Close the seconds before now, and stop shedding after PQ_SHED_QUIET_PERIOD quiet ones. + * Returns whether shedding stopped. */ + bool AdvancePQShedding(PQShedSeconds now) EXCLUSIVE_LOCKS_REQUIRED(m_pq_shed_mutex); + + /** Stop shedding when the flood is over, even if no new offer is attempted. */ + void UpdatePQShedding(PQShedSeconds now) EXCLUSIVE_LOCKS_REQUIRED(!m_pq_shed_mutex); + + /** Log a load shedding transition: started (with the threshold) or stopped. */ + static void LogPQShedding(std::optional started_threshold); + const CChainParams& m_params; friend struct ConnmanTestMsg; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 047d470352..dfdfa2e0d4 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2397,7 +2397,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& !pfrom.HasPermission(NetPermissionFlags::Download) // nodes with the download permission may exceed target ) { LogDebug(BCLog::NET, "historical block serving limit reached, %s\n", pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); return; } tip = m_chainman.ActiveChain().Tip(); @@ -2407,7 +2407,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& )) { LogDebug(BCLog::NET, "Ignore block request below NODE_NETWORK_LIMITED threshold, %s\n", pfrom.DisconnectMsg(fLogIPs)); //disconnect node and prevent it from stalling (would otherwise wait for the missing block) - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); return; } // Pruned nodes may have deleted the block, so check whether @@ -2439,7 +2439,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& } else { LogError("Cannot load block from disk, %s\n", pfrom.DisconnectMsg(fLogIPs)); } - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); return; } MakeAndPushMessage(pfrom, NetMsgType::BLOCK, std::span{block_data}); @@ -2453,7 +2453,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& } else { LogError("Cannot load block from disk, %s\n", pfrom.DisconnectMsg(fLogIPs)); } - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); return; } pblock = pblockRead; @@ -2974,7 +2974,7 @@ void PeerManagerImpl::UpdatePeerStateForReceivedHeaders(CNode& pfrom, Peer& peer // as an anti-DoS measure. if (pfrom.IsOutboundOrBlockRelayConn()) { LogInfo("outbound peer headers chain has insufficient work, %s\n", pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); } } } @@ -3305,7 +3305,7 @@ bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& node, Peer& peer, if (!supported_filter_type) { LogDebug(BCLog::NET, "peer requested unsupported block filter type: %d, %s\n", static_cast(filter_type), node.DisconnectMsg(fLogIPs)); - node.fDisconnect = true; + node.RequestDisconnect(); return false; } @@ -3317,7 +3317,7 @@ bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& node, Peer& peer, if (!stop_index || !BlockRequestAllowed(stop_index)) { LogDebug(BCLog::NET, "peer requested invalid block hash: %s, %s\n", stop_hash.ToString(), node.DisconnectMsg(fLogIPs)); - node.fDisconnect = true; + node.RequestDisconnect(); return false; } } @@ -3327,13 +3327,13 @@ bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& node, Peer& peer, LogDebug(BCLog::NET, "peer sent invalid getcfilters/getcfheaders with " "start height %d and stop height %d, %s\n", start_height, stop_height, node.DisconnectMsg(fLogIPs)); - node.fDisconnect = true; + node.RequestDisconnect(); return false; } if (stop_height - start_height >= max_height_diff) { LogDebug(BCLog::NET, "peer requested too many cfilters/cfheaders: %d / %d, %s\n", stop_height - start_height + 1, max_height_diff, node.DisconnectMsg(fLogIPs)); - node.fDisconnect = true; + node.RequestDisconnect(); return false; } @@ -3630,7 +3630,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, nServices, GetDesirableServiceFlags(nServices), pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); return; } if ((pfrom.m_conn_type == ConnectionType::OUTBOUND_FULL_RELAY || @@ -3640,13 +3640,13 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, LogDebug(BCLog::NET, "peer advertises NODE_WITNESS_PRUNED in full-validation mode (%08x), %s\n", nServices, pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); return; } if (pfrom.m_is_archive_connection && (nServices & NODE_WITNESS_PRUNED)) { LogPrintf("Archive peer advertises NODE_WITNESS_PRUNED, %s\n", pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); return; } if (pfrom.m_is_archive_connection && !HasArchiveServiceFlags(nServices)) { @@ -3654,14 +3654,14 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, nServices, ServiceFlags(NODE_NETWORK | NODE_WITNESS | NODE_ARCHIVE), pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); return; } if (nVersion < MIN_PEER_PROTO_VERSION) { // disconnect from peers older than this proto version LogDebug(BCLog::NET, "peer using obsolete version %i, %s\n", nVersion, pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); return; } @@ -3687,7 +3687,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (pfrom.IsInboundConn() && !m_connman.CheckIncomingNonce(nNonce)) { LogPrintf("connected to self at %s, disconnecting\n", pfrom.addr.ToStringAddrPort()); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); return; } @@ -3839,7 +3839,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Feeler connections exist only to verify if address is online. if (pfrom.IsFeelerConn()) { LogDebug(BCLog::NET, "feeler connection completed, %s\n", pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); } return; } @@ -3943,7 +3943,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (pfrom.fSuccessfullyConnected) { // Disconnect peers that send a wtxidrelay message after VERACK. LogDebug(BCLog::NET, "wtxidrelay received after verack, %s\n", pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); return; } if (pfrom.GetCommonVersion() >= WTXID_RELAY_VERSION) { @@ -3965,7 +3965,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (pfrom.fSuccessfullyConnected) { // Disconnect peers that send a SENDADDRV2 message after VERACK. LogDebug(BCLog::NET, "sendaddrv2 received after verack, %s\n", pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); return; } peer->m_wants_addrv2 = true; @@ -3983,14 +3983,14 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (pfrom.fSuccessfullyConnected) { LogDebug(BCLog::NET, "sendtxrcncl received after verack, %s\n", pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); return; } // Peer must not offer us reconciliations if we specified no tx relay support in VERSION. if (RejectIncomingTxs(pfrom)) { LogDebug(BCLog::NET, "sendtxrcncl received to which we indicated no tx relay, %s\n", pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); return; } @@ -4000,7 +4000,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, const auto* tx_relay = peer->GetTxRelay(); if (!tx_relay || !WITH_LOCK(tx_relay->m_bloom_filter_mutex, return tx_relay->m_relay_txs)) { LogDebug(BCLog::NET, "sendtxrcncl received which indicated no tx relay to us, %s\n", pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); return; } @@ -4018,11 +4018,11 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, break; case ReconciliationRegisterResult::ALREADY_REGISTERED: LogDebug(BCLog::NET, "txreconciliation protocol violation (sendtxrcncl received from already registered peer), %s\n", pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); return; case ReconciliationRegisterResult::PROTOCOL_VIOLATION: LogDebug(BCLog::NET, "txreconciliation protocol violation, %s\n", pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); return; } return; @@ -4125,7 +4125,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // AddrFetch: Require multiple addresses to avoid disconnecting on self-announcements if (pfrom.IsAddrFetchConn() && vAddr.size() > 1) { LogDebug(BCLog::NET, "addrfetch connection completed, %s\n", pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); } return; } @@ -4175,7 +4175,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } else if (inv.IsGenTxMsg()) { if (reject_tx_invs) { LogDebug(BCLog::NET, "transaction (%s) inv sent in violation of protocol, %s\n", inv.hash.ToString(), pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); return; } const GenTxid gtxid = ToGenTxid(inv); @@ -4252,7 +4252,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (locator.vHave.size() > MAX_LOCATOR_SZ) { LogDebug(BCLog::NET, "getblocks locator size %lld > %d, %s\n", locator.vHave.size(), MAX_LOCATOR_SZ, pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); return; } @@ -4374,7 +4374,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (locator.vHave.size() > MAX_LOCATOR_SZ) { LogDebug(BCLog::NET, "getheaders locator size %lld > %d, %s\n", locator.vHave.size(), MAX_LOCATOR_SZ, pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); return; } @@ -4450,7 +4450,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (msg_type == NetMsgType::TX) { if (RejectIncomingTxs(pfrom)) { LogDebug(BCLog::NET, "transaction sent in violation of protocol, %s", pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); return; } @@ -4937,7 +4937,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (!pfrom.HasPermission(NetPermissionFlags::NoBan)) { LogDebug(BCLog::NET, "mempool request with bloom filters disabled, %s\n", pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); } return; } @@ -4947,7 +4947,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (!pfrom.HasPermission(NetPermissionFlags::NoBan)) { LogDebug(BCLog::NET, "mempool request with bandwidth limit reached, %s\n", pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); } return; } @@ -5037,7 +5037,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (msg_type == NetMsgType::FILTERLOAD) { if (!(peer->m_our_services & NODE_BLOOM)) { LogDebug(BCLog::NET, "filterload received despite not offering bloom services, %s\n", pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); return; } CBloomFilter filter; @@ -5062,7 +5062,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (msg_type == NetMsgType::FILTERADD) { if (!(peer->m_our_services & NODE_BLOOM)) { LogDebug(BCLog::NET, "filteradd received despite not offering bloom services, %s\n", pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); return; } std::vector vData; @@ -5090,7 +5090,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (msg_type == NetMsgType::FILTERCLEAR) { if (!(peer->m_our_services & NODE_BLOOM)) { LogDebug(BCLog::NET, "filterclear received despite not offering bloom services, %s\n", pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); return; } auto tx_relay = peer->GetTxRelay(); @@ -5183,7 +5183,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (!(old_services & NODE_WITNESS_PRUNED) && pfrom.m_is_archive_connection) { LogPrintf("Archive peer implies NODE_WITNESS_PRUNED via NOTFOUND, %s\n", pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); } RemoveBlockRequest(block_hash, pfrom.GetId()); continue; @@ -5191,7 +5191,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, LogInfo("Peer sent NOTFOUND for in-flight block %s, %s\n", block_hash.ToString(), pfrom.DisconnectMsg(fLogIPs)); - pfrom.fDisconnect = true; + pfrom.RequestDisconnect(); } m_txdownloadman.ReceivedNotFound(pfrom.GetId(), tx_invs); return; @@ -5230,7 +5230,7 @@ bool PeerManagerImpl::MaybeDiscourageAndDisconnect(CNode& pnode, Peer& peer) // all peers on the same local address) LogDebug(BCLog::NET, "Warning: disconnecting but not discouraging %s peer %d!\n", pnode.m_inbound_onion ? "inbound onion" : "local", peer.m_id); - pnode.fDisconnect = true; + pnode.RequestDisconnect(); return true; } @@ -5359,7 +5359,7 @@ void PeerManagerImpl::ConsiderEviction(CNode& pto, Peer& peer, std::chrono::seco if (state.m_chain_sync.m_sent_getheaders) { // They've run out of time to catch up! LogInfo("Outbound peer has old chain, best known block = %s, %s\n", state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : "", pto.DisconnectMsg(fLogIPs)); - pto.fDisconnect = true; + pto.RequestDisconnect(); } else { assert(state.m_chain_sync.m_work_header); // Here, we assume that the getheaders message goes out, @@ -5417,7 +5417,7 @@ void PeerManagerImpl::EvictExtraOutboundPeers(std::chrono::seconds now) CNodeState *node_state = State(pnode->GetId()); if (node_state == nullptr || (now - pnode->m_connected >= MINIMUM_CONNECT_TIME && node_state->vBlocksInFlight.empty())) { - pnode->fDisconnect = true; + pnode->RequestDisconnect(); LogDebug(BCLog::NET, "disconnecting extra block-relay-only peer=%d (last block received at time %d)\n", pnode->GetId(), count_seconds(pnode->m_last_block_time)); return true; @@ -5470,7 +5470,7 @@ void PeerManagerImpl::EvictExtraOutboundPeers(std::chrono::seconds now) CNodeState &state = *State(pnode->GetId()); if (now - pnode->m_connected > MINIMUM_CONNECT_TIME && state.vBlocksInFlight.empty()) { LogDebug(BCLog::NET, "disconnecting extra outbound peer=%d (last block announcement received at time %d)\n", pnode->GetId(), oldest_block_announcement); - pnode->fDisconnect = true; + pnode->RequestDisconnect(); return true; } else { LogDebug(BCLog::NET, "keeping outbound peer=%d chosen for eviction (connect time: %d, blocks_in_flight: %d)\n", @@ -5526,7 +5526,7 @@ void PeerManagerImpl::MaybeSendPing(CNode& node_to, Peer& peer, std::chrono::mic // The ping timeout is using mocktime. To disable the check during // testing, increase -peertimeout. LogDebug(BCLog::NET, "ping timeout: %fs, %s", 0.000001 * count_microseconds(now - peer.m_ping_start.load()), node_to.DisconnectMsg(fLogIPs)); - node_to.fDisconnect = true; + node_to.RequestDisconnect(NodeCloseCause::TIMEOUT); return; } @@ -5755,7 +5755,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) if (pto->IsAddrFetchConn() && current_time - pto->m_connected > 10 * AVG_ADDRESS_BROADCAST_INTERVAL) { LogDebug(BCLog::NET, "addrfetch connection timeout, %s\n", pto->DisconnectMsg(fLogIPs)); - pto->fDisconnect = true; + pto->RequestDisconnect(); return true; } @@ -6103,7 +6103,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // the download window should be much larger than the to-be-downloaded set of blocks, so disconnection // should only happen during initial block download. LogInfo("Peer is stalling block download, %s\n", pto->DisconnectMsg(fLogIPs)); - pto->fDisconnect = true; + pto->RequestDisconnect(); // Increase timeout for the next peer so that we don't disconnect multiple peers if our own // bandwidth is insufficient. const auto new_timeout = std::min(2 * stalling_timeout, BLOCK_STALLING_TIMEOUT_MAX); @@ -6123,7 +6123,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) const auto block_download_timeout_interval = std::max(std::chrono::seconds{consensusParams.nPowTargetSpacing}, BLOCK_DOWNLOAD_TIMEOUT_INTERVAL_MIN); if (current_time > state.m_downloading_since + block_download_timeout_interval * (BLOCK_DOWNLOAD_TIMEOUT_BASE + BLOCK_DOWNLOAD_TIMEOUT_PER_PEER * nOtherPeersWithValidatedDownloads)) { LogInfo("Timeout downloading block %s, %s\n", queuedBlock.pindex->GetBlockHash().ToString(), pto->DisconnectMsg(fLogIPs)); - pto->fDisconnect = true; + pto->RequestDisconnect(); return true; } } @@ -6139,7 +6139,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // problems if we can't get any outbound peers. if (!pto->HasPermission(NetPermissionFlags::NoBan)) { LogInfo("Timeout downloading headers, %s\n", pto->DisconnectMsg(fLogIPs)); - pto->fDisconnect = true; + pto->RequestDisconnect(); return true; } else { LogInfo("Timeout downloading headers from noban peer, not %s\n", pto->DisconnectMsg(fLogIPs)); diff --git a/src/netbase.cpp b/src/netbase.cpp index d5f21b4f4d..84b8c8a355 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -16,10 +16,12 @@ #include #include +#include #include #include #include #include +#include #include #include @@ -790,6 +792,20 @@ bool GetNameProxy(Proxy &nameProxyOut) { return true; } +ProxySettingsRestorerForTesting::ProxySettingsRestorerForTesting() +{ + LOCK(g_proxyinfo_mutex); + std::copy(std::begin(proxyInfo), std::end(proxyInfo), m_proxies.begin()); + m_name_proxy = nameProxy; +} + +ProxySettingsRestorerForTesting::~ProxySettingsRestorerForTesting() +{ + LOCK(g_proxyinfo_mutex); + std::copy(m_proxies.begin(), m_proxies.end(), std::begin(proxyInfo)); + nameProxy = m_name_proxy; +} + bool HaveNameProxy() { LOCK(g_proxyinfo_mutex); return nameProxy.IsValid(); diff --git a/src/netbase.h b/src/netbase.h index 18f6522790..9396ed7801 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -219,6 +220,23 @@ bool SetNameProxy(const Proxy &addrProxy); bool HaveNameProxy(); bool GetNameProxy(Proxy &nameProxyOut); +/** + * Test only: saves every proxy setting (each network's proxy and the name proxy) when created, + * and restores them when destroyed. The settings can't be unset otherwise, so a test that sets a + * proxy uses this to keep it from leaking into later tests. + */ +class ProxySettingsRestorerForTesting +{ + std::array m_proxies; + Proxy m_name_proxy; + +public: + ProxySettingsRestorerForTesting(); + ~ProxySettingsRestorerForTesting(); + ProxySettingsRestorerForTesting(const ProxySettingsRestorerForTesting&) = delete; + ProxySettingsRestorerForTesting& operator=(const ProxySettingsRestorerForTesting&) = delete; +}; + using DNSLookupFn = std::function(const std::string&, bool)>; extern DNSLookupFn g_dns_lookup; diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 673e436286..bf2fcebf52 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -72,6 +72,7 @@ add_executable(test_bitcoin multisig_tests.cpp net_peer_connection_tests.cpp net_peer_eviction_tests.cpp + net_pq_tests.cpp net_tests.cpp netbase_tests.cpp node_init_tests.cpp diff --git a/src/test/net_pq_tests.cpp b/src/test/net_pq_tests.cpp new file mode 100644 index 0000000000..b63e6aeae9 --- /dev/null +++ b/src/test/net_pq_tests.cpp @@ -0,0 +1,1827 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std::literals; + +namespace { + +//! Messages are never processed: the tests drive the connection manager directly. +class NoMessages final : public NetEventsInterface +{ +public: + //! Nodes deleted, and of those, the ones deleted unfinalized or still holding hybrid secrets. + int m_deleted{0}; + int m_deleted_unfinalized{0}; + int m_deleted_with_secrets{0}; + + void InitializeNode(const CNode&, ServiceFlags) override {} + //! CConnman calls this right before it deletes the node. + void FinalizeNode(const CNode& node) override + { + ++m_deleted; + if (!node.m_pq_finalized) ++m_deleted_unfinalized; + const auto* transport{dynamic_cast(node.m_transport.get())}; + if (transport && transport->HoldsHybridSecretsForTesting()) ++m_deleted_with_secrets; + } + bool HasAllDesirableServiceFlags(ServiceFlags) const override { return true; } + bool HasUndesirableServiceFlags(ServiceFlags) const override { return false; } + bool ProcessMessages(CNode*, std::atomic&) override EXCLUSIVE_LOCKS_REQUIRED(g_msgproc_mutex) { return false; } + bool SendMessages(CNode*) override EXCLUSIVE_LOCKS_REQUIRED(g_msgproc_mutex) { return false; } +}; + +//! A DynSock whose Recv() or Send() can fail as if the peer reset the connection. +class FailingSock : public DynSock +{ +public: + using DynSock::DynSock; + + std::atomic m_fail_recv{false}; + std::atomic m_fail_send{false}; + + ssize_t Recv(void* buf, size_t len, int flags) const override + { + return m_fail_recv ? Fail() : DynSock::Recv(buf, len, flags); + } + ssize_t Send(const void* buf, size_t len, int flags) const override + { + return m_fail_send ? Fail() : DynSock::Send(buf, len, flags); + } + +private: + FailingSock& operator=(Sock&&) override + { + assert(false && "Move of Sock into FailingSock not allowed."); + return *this; + } + + static ssize_t Fail() + { +#ifdef WIN32 + WSASetLastError(WSAECONNRESET); +#else + errno = ECONNRESET; +#endif + return -1; + } +}; + +//! One connection under test: a CNode on a FailingSock, whose pipes are the peer's side. +struct Link { + std::shared_ptr pipes; + std::shared_ptr sock; + CNode* node; +}; + +//! The far side of a v2 connection, built by hand: it sends no garbage and any version contents. +class RawPeer +{ + BIP324Cipher m_cipher; + const bool m_initiator; + +public: + explicit RawPeer(bool initiator) : m_cipher{GenerateRandomKey(), MakeByteSpan(GetRandHash())}, m_initiator{initiator} {} + + std::vector Key() const + { + const EllSwiftPubKey& key{m_cipher.GetOurPubKey()}; + const auto bytes{MakeUCharSpan(std::span{key.data(), key.size()})}; + return {bytes.begin(), bytes.end()}; + } + + void Initialize(std::span their_key) + { + BOOST_REQUIRE_EQUAL(their_key.size(), EllSwiftPubKey::size()); + m_cipher.Initialize(EllSwiftPubKey{MakeByteSpan(their_key)}, m_initiator); + } + + //! Our garbage terminator, then a version packet with contents (empty garbage as AAD). + std::vector TerminatorAndVersion(std::span contents, bool damage_tag = false) + { + std::vector out(BIP324Cipher::GARBAGE_TERMINATOR_LEN + BIP324Cipher::EXPANSION + contents.size()); + std::ranges::copy(MakeUCharSpan(m_cipher.GetSendGarbageTerminator()), out.begin()); + m_cipher.Encrypt(contents, /*aad=*/{}, /*ignore=*/false, MakeWritableByteSpan(out).subspan(BIP324Cipher::GARBAGE_TERMINATOR_LEN)); + if (damage_tag) out.back() ^= 0x01; + return out; + } +}; + +/** A record CompactSize(1 + payload size) || header || payload. */ +std::vector MakeRecord(uint8_t header, size_t payload_size) +{ + DataStream stream; + WriteCompactSize(stream, 1 + payload_size); + stream << header; + stream.write(std::vector(payload_size, std::byte{0x42})); + return {stream.begin(), stream.end()}; +} + +/** The level a line is logged at, as #184's logging table sets it. */ +enum class LineLevel { NET_DEBUG, INFO, WARNING }; + +/** + * Matches a log line that ends with one of texts, logged at level. The logger prints a level + * prefix right before the message: "[net] " for a net debug line, "[warning] " for a warning, + * nothing for an info line. The test setup turns on -logsourcelocations, so without a prefix the + * bracket right before the message is the function signature. + */ +std::function AtLevel(std::vector texts, LineLevel level) +{ + BOOST_REQUIRE(LogInstance().m_log_sourcelocations); + return [texts, level](const std::string* line) { + if (line == nullptr) return true; + return std::ranges::any_of(texts, [&](const std::string& text) { + if (!line->ends_with(text + "\n")) return false; + const std::string_view head{std::string_view{*line}.substr(0, line->size() - text.size() - 1)}; + const size_t open{head.rfind('[')}; + if (open == std::string_view::npos || !head.ends_with("] ")) return false; + const std::string_view bracket{head.substr(open)}; + switch (level) { + case LineLevel::NET_DEBUG: return bracket == "[net] "; + case LineLevel::WARNING: return bracket == "[warning] "; + case LineLevel::INFO: return bracket.find('(') != std::string_view::npos; + } + return false; + }); + }; +} + +/** Expect a log line that is exactly text, logged at level. */ +DebugLogHelper ExpectLine(std::string text, LineLevel level) { return DebugLogHelper{text, AtLevel({text}, level)}; } + +/** The backends every failure line names. */ +std::string Backends() +{ + const auto backends{mlkem::GetBackendNames()}; + return strprintf(" arith=%s keccak=%s", backends.arith, backends.keccak); +} + +/** Counts the log lines that contain text while alive. */ +class LogLineCounter +{ + const std::string m_text; + std::list>::iterator m_callback; + +public: + std::atomic m_count{0}; + + explicit LogLineCounter(std::string text) : m_text{std::move(text)} + { + m_callback = LogInstance().PushBackCallback([this](const std::string& line) { + if (line.find(m_text) != std::string::npos) ++m_count; + }); + } + ~LogLineCounter() { LogInstance().DeleteCallback(m_callback); } +}; + +/** Sets a global while alive, and puts the old value back on every path: a failed BOOST_REQUIRE + * throws past the end of the test, and later suites in the same process must not see it. */ +template +class GlobalOverride +{ + T& m_global; + T m_previous; + +public: + GlobalOverride(T& global, T value) : m_global{global}, m_previous{std::exchange(global, std::move(value))} {} + ~GlobalOverride() { m_global = std::move(m_previous); } + GlobalOverride(const GlobalOverride&) = delete; + GlobalOverride& operator=(const GlobalOverride&) = delete; +}; + +/** A CreateSock that hands out sockets with contents, or ZeroSocks without. */ +GlobalOverride MockCreateSock(std::string contents = {}) +{ + if (contents.empty()) return {CreateSock, [](int, int, int) -> std::unique_ptr { return std::make_unique(); }}; + return {CreateSock, [contents](int, int, int) -> std::unique_ptr { return std::make_unique(contents); }}; +} + +/** Clears the SOCKS5 interrupt while alive, and sets it again on every path if it was set. Every + * CConnman::Interrupt() sets it, such as an earlier test's at teardown. */ +class Socks5InterruptCleared +{ + const bool m_was_set{g_socks5_interrupt}; + +public: + Socks5InterruptCleared() { g_socks5_interrupt.reset(); } + ~Socks5InterruptCleared() + { + if (m_was_set) g_socks5_interrupt(); + } + Socks5InterruptCleared(const Socks5InterruptCleared&) = delete; + Socks5InterruptCleared& operator=(const Socks5InterruptCleared&) = delete; +}; + +/** Our outbound node's peer: a responder. */ +V2Transport Responder(V2PQOptions pq = {.mode = PQMode::NEGOTIATE}) { return {/*nodeid=*/1000, /*initiating=*/false, pq}; } +/** Our inbound node's peer: an initiator. */ +V2Transport Initiator(V2PQOptions pq = {.mode = PQMode::NEGOTIATE}) { return {/*nodeid=*/1000, /*initiating=*/true, pq}; } + +/** 64 bytes an initiator could send as its key: any 64 bytes are a valid ElligatorSwift key. */ +std::vector InitiatorKey() { return FastRandomContext{}.randbytes(EllSwiftPubKey::size()); } + +struct PQNetSetup : public RegTestingSetup { + NoMessages m_events; + ConnmanTestMsg m_connman{0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman, Params()}; + NodeId m_next_id{0}; + + PQNetSetup() + { + static_assert(PQ_FAILURE_THRESHOLD == 3); + InitConnman(); + m_connman.SetPeerConnectTimeout(60s); + m_connman.SetPQMode(PQMode::NEGOTIATE); + SetMockTime(GetTime()); + MockableSteadyClock::SetMockTime(STEADY_START); + } + + ~PQNetSetup() + { + m_connman.Stop(); + SetMockTime(0s); + MockableSteadyClock::ClearMockTime(); + } + + //! Where the mocked steady clock, which drives load shedding, starts. + static constexpr std::chrono::seconds STEADY_START{3600}; + + /** Init() with room for inbound peers, v2 accepted, and the given load shedding threshold. */ + void InitConnman(uint64_t shed_threshold = DEFAULT_PQ_SHED_THRESHOLD_PER_S) + { + CConnman::Options options; + options.m_max_automatic_connections = DEFAULT_MAX_PEER_CONNECTIONS; + options.m_local_services = NODE_P2P_V2; + options.pq_shed_threshold_per_s = shed_threshold; + m_connman.Init(options); + m_connman.SetMsgProc(&m_events); + } + + /** An inbound connection through the accept path, with the transport options it gives. */ + Link Accept(const CService& addr = LookupNumeric("10.9.0.1", 50000)) + { + auto pipes{std::make_shared()}; + m_connman.CreateNodeFromAcceptedSocketPublic(std::make_unique(pipes, std::make_shared()), + LookupNumeric("127.0.0.1", 18444), addr); + CNode* node{m_connman.TestNodes().back()}; + auto sock{std::static_pointer_cast(WITH_LOCK(node->m_sock_mutex, return node->m_sock))}; + BOOST_REQUIRE(node->IsInboundConn()); + return {pipes, sock, node}; + } + + /** A connection the connection manager holds, as ConnectNode() or the accept path would add it. */ + Link Add(ConnectionType conn_type, PQMode mode = PQMode::NEGOTIATE, std::optional endpoint = std::nullopt, + const CService& addr = LookupNumeric("1.2.3.4", 8333)) + { + auto pipes{std::make_shared()}; + auto sock{std::make_shared(pipes, std::make_shared())}; + CNode* node{new CNode{m_next_id++, sock, CAddress{addr, NODE_NONE}, /*nKeyedNetGroupIn=*/0, /*nLocalHostNonceIn=*/0, + CAddress{}, /*addrNameIn=*/"", conn_type, /*inbound_onion=*/false, + CNodeOptions{.use_v2transport = true, .pq = {.mode = mode}, .pq_endpoint = std::move(endpoint)}}}; + node->AddRef(); // the reference m_nodes holds + m_connman.AddTestNode(*node); + return {pipes, sock, node}; + } + + /** Two socket handler passes over links: each node sends what it has queued, receives once, + * then sends what that queued. */ + void Pass(const std::vector& links) + { + Sock::EventsPerSock events; + std::vector nodes; + for (const Link& link : links) { + events.emplace(link.sock, Sock::Events{Sock::RECV | Sock::SEND}).first->second.occurred = Sock::RECV | Sock::SEND; + nodes.push_back(link.node); + } + m_connman.SocketHandlerConnectedPublic(nodes, events); + m_connman.SocketHandlerConnectedPublic(nodes, events); + } + void Pass(const Link& link) { Pass(std::vector{link}); } + + /** Everything the node sent so far. */ + static std::vector Sent(const Link& link) + { + std::vector out; + uint8_t buf[4096]; + ssize_t read; + while ((read = link.pipes->send.GetBytes(buf, sizeof(buf))) > 0) out.insert(out.end(), buf, buf + read); + return out; + } + + /** The node's key, at the start of what it sent; the rest is dropped. */ + static std::vector NodeKey(const Link& link) + { + std::vector sent{Sent(link)}; + BOOST_REQUIRE(sent.size() >= EllSwiftPubKey::size()); + sent.resize(EllSwiftPubKey::size()); + return sent; + } + + static void Receive(const Link& link, std::span bytes) { link.pipes->recv.PushBytes(bytes.data(), bytes.size()); } + + /** What peer has to send. */ + static std::vector PeerBytes(Transport& peer) + { + std::vector out; + while (true) { + const auto [bytes, more, msg_type] = peer.GetBytesToSend(/*have_next_message=*/false); + if (bytes.empty()) return out; + out.insert(out.end(), bytes.begin(), bytes.end()); + peer.MarkBytesSent(bytes.size()); + } + } + + /** Queue what peer has to send for the node. */ + static void PeerToNode(const Link& link, Transport& peer) { Receive(link, PeerBytes(peer)); } + + /** Feed peer what the node sent; false if peer failed. */ + static bool NodeToPeer(const Link& link, Transport& peer) + { + const std::vector sent{Sent(link)}; + std::span bytes{sent}; + while (!bytes.empty()) { + if (!peer.ReceivedBytes(bytes)) return false; + if (peer.ReceivedMessageComplete()) { + bool reject{false}; + peer.GetReceivedMessage({}, reject); + } + } + return true; + } + + /** Run socket handler passes, moving bytes both ways. A peer that fails gets nothing more, but + * what it queued before still reaches the node. */ + void Exchange(const Link& link, Transport& peer) + { + bool peer_failed{false}; + for (int round{0}; round < 10; ++round) { + PeerToNode(link, peer); + Pass(link); + if (peer_failed) { + Sent(link); + } else { + peer_failed = !NodeToPeer(link, peer); + } + } + } + + /** Remove the disconnected nodes; those with a reference left stay in the disconnected pool. */ + void Disconnect() { m_connman.DisconnectNodesPublic(); } + + PQTransportStats Stats() const { return m_connman.GetPQTransportStats(); } + + /** How a test connection ends. */ + enum class End { MALFORMED, CONFIRMATION, PEER_EOF, PEER_RESET, TIMEOUT, LOCAL, SEND_ERROR, LEGACY, SUCCESS }; + + /** One outbound connection to endpoint that ends as end; the connection manager finalizes it. */ + void Connect(const PQEndpointKey& endpoint, End end) + { + Link link{Add(ConnectionType::OUTBOUND_FULL_RELAY, PQMode::NEGOTIATE, endpoint)}; + switch (end) { + case End::MALFORMED: { // an offer of the wrong length + Pass(link); + RawPeer peer{/*initiator=*/false}; + peer.Initialize(NodeKey(link)); + Receive(link, peer.Key()); + Receive(link, peer.TerminatorAndVersion(MakeRecord(PQ_MLKEM1024, 100))); + Pass(link); + break; + } + case End::CONFIRMATION: { + V2Transport peer{Responder({.mode = PQMode::NEGOTIATE, .corrupt_shared_secret = true})}; + Exchange(link, peer); + break; + } + case End::LEGACY: + case End::SUCCESS: { + // Not `cond ? a : b`: MSVC requires a copy constructor for a conditional of + // prvalues, and V2Transport has none. + V2Transport peer{[&] { if (end == End::LEGACY) return Responder({}); return Responder(); }()}; + Exchange(link, peer); + BOOST_REQUIRE(link.node->m_transport->GetPQSnapshot().confirmed == (end == End::SUCCESS)); + link.node->RequestDisconnect(); + break; + } + case End::PEER_EOF: + case End::PEER_RESET: + case End::TIMEOUT: + case End::LOCAL: + case End::SEND_ERROR: { // we switch, and the connection closes before the peer's confirmation + V2Transport peer{Responder()}; + Pass(link); + BOOST_REQUIRE(NodeToPeer(link, peer)); + PeerToNode(link, peer); + if (end == End::SEND_ERROR) link.sock->m_fail_send = true; + Pass(link); + BOOST_REQUIRE(link.node->m_transport->GetPQSnapshot().switched); + if (end == End::PEER_EOF) link.pipes->recv.Eof(); + if (end == End::PEER_RESET) link.sock->m_fail_recv = true; + if (end == End::TIMEOUT) SetMockTime(GetTime() + 61s); + if (end == End::LOCAL) link.node->RequestDisconnect(); + Pass(link); + break; + } + } + BOOST_REQUIRE(link.node->fDisconnect); + Disconnect(); + } + + std::optional Streak(const PQEndpointKey& endpoint) const + { + for (const PQStreakStats& streak : Stats().failure_streaks) { + if (streak.endpoint == endpoint) return streak; + } + return std::nullopt; + } + + std::optional Fallback(const PQEndpointKey& endpoint) const + { + for (const PQFallbackStats& fallback : Stats().fallback_set) { + if (fallback.endpoint == endpoint) return fallback; + } + return std::nullopt; + } +}; + +/** A distinct outbound endpoint per index. */ +PQEndpointKey TestEndpoint(uint8_t subnet, int index) +{ + return LookupNumeric(strprintf("10.%d.%d.%d", subnet, index / 250, 1 + index % 250), 8333); +} + +} // namespace + +BOOST_FIXTURE_TEST_SUITE(net_pq_tests, PQNetSetup) + +BOOST_AUTO_TEST_CASE(mock_socket_would_block) +{ + // These tests drive the socket handler through DynSock. An empty pipe must read like a + // non-blocking socket without data: -1, with the would-block error where the socket handler + // reads it, WSAGetLastError(). That is errno except on Windows, which keeps Winsock's error + // apart; with errno alone, Windows saw error 0 and closed every idle connection as a reset. + auto pipes{std::make_shared()}; + const DynSock sock{pipes, std::make_shared()}; + uint8_t buf[1]; + errno = 0; +#ifdef WIN32 + WSASetLastError(0); +#endif + BOOST_CHECK_EQUAL(sock.Recv(buf, sizeof(buf), 0), -1); + const int err{WSAGetLastError()}; + BOOST_CHECK(err == WSAEWOULDBLOCK || err == WSAEAGAIN); + pipes->recv.Eof(); + BOOST_CHECK_EQUAL(sock.Recv(buf, sizeof(buf), 0), 0); + + // So a socket handler pass over a connection with nothing to read keeps it open. + Link link{Add(ConnectionType::OUTBOUND_FULL_RELAY)}; + Pass(link); + BOOST_CHECK(!link.node->fDisconnect); + BOOST_CHECK(link.node->GetCloseCause() == NodeCloseCause::NONE); +} + +BOOST_AUTO_TEST_CASE(pq_endpoint_identity) +{ + // Name-proxy destinations: lowercase ASCII, one trailing root dot removed, the port kept. + BOOST_CHECK(MakePQNameEndpoint("Example.COM.", 8444) == (PQNameEndpoint{"example.com", 8444})); + BOOST_CHECK(MakePQNameEndpoint("example.com", 8444) == MakePQNameEndpoint("EXAMPLE.com.", 8444)); + BOOST_CHECK(MakePQNameEndpoint("example.com..", 8444) == (PQNameEndpoint{"example.com.", 8444})); + BOOST_CHECK(MakePQNameEndpoint("example.com", 8444) != MakePQNameEndpoint("example.com", 8445)); + BOOST_CHECK(MakePQNameEndpoint("example.com", 8444) != MakePQNameEndpoint("example.org", 8444)); + BOOST_CHECK(MakePQNameEndpoint("\xC3\x84" "B.example", 1) == (PQNameEndpoint{"\xC3\x84" "b.example", 1})); // ASCII only + + // Resolved destinations keep the address and the port. + const CService ipv4{LookupNumeric("1.2.3.4", 8333)}; + const PQEndpointKey a{ipv4}; + BOOST_CHECK(a != PQEndpointKey{LookupNumeric("1.2.3.4", 8334)}); + BOOST_CHECK(a != PQEndpointKey{LookupNumeric("2001:db8::1", 8333)}); + BOOST_CHECK(a == PQEndpointKey{LookupNumeric("::ffff:1.2.3.4", 8333)}); // the same IPv4 address + // Private addresses share a network class, NET_UNROUTABLE, but stay distinct endpoints. + BOOST_CHECK(PQEndpointKey{LookupNumeric("10.0.0.1", 8333)} != PQEndpointKey{LookupNumeric("10.0.0.2", 8333)}); + // A name never aliases an address, even one that spells the same. + BOOST_CHECK(a != PQEndpointKey{MakePQNameEndpoint("1.2.3.4", 8333)}); + + // A map keyed by endpoint orders them strictly. + std::map keys; + for (const PQEndpointKey& key : {a, PQEndpointKey{LookupNumeric("1.2.3.4", 8334)}, PQEndpointKey{MakePQNameEndpoint("A.example", 1)}, + PQEndpointKey{MakePQNameEndpoint("a.example.", 1)}, PQEndpointKey{MakePQNameEndpoint("b.example", 1)}}) { + ++keys[key]; + } + BOOST_CHECK_EQUAL(keys.size(), 4U); + + // A node without a captured endpoint uses its address and port. + Link plain{Add(ConnectionType::OUTBOUND_FULL_RELAY)}; + BOOST_CHECK(plain.node->m_pq_endpoint == a); + Link named{Add(ConnectionType::MANUAL, PQMode::NEGOTIATE, MakePQNameEndpoint("Seed.Example.", 18444), CService{})}; + BOOST_CHECK(named.node->m_pq_endpoint == PQEndpointKey{(PQNameEndpoint{"seed.example", 18444})}); + + // Outbound construction captures the resolved destination and port. + { + const auto create_sock{MockCreateSock()}; + const std::unique_ptr outbound{m_connman.ConnectNodeOnly("5.6.7.8:18555", ConnectionType::MANUAL, /*use_v2transport=*/true)}; + BOOST_REQUIRE(outbound); + BOOST_CHECK(outbound->m_pq_endpoint == PQEndpointKey{LookupNumeric("5.6.7.8", 18555)}); + const std::unique_ptr ipv6{m_connman.ConnectNodeOnly("[2a01:4f8::2]:18555", ConnectionType::MANUAL, /*use_v2transport=*/true)}; + BOOST_REQUIRE(ipv6); + BOOST_CHECK(ipv6->m_pq_endpoint == PQEndpointKey{LookupNumeric("2a01:4f8::2", 18555)}); + } + + // Inbound uses the actual remote address and source port. + m_connman.CreateNodeFromAcceptedSocketPublic(std::make_unique(), LookupNumeric("127.0.0.1", 18444), LookupNumeric("9.8.7.6", 51234)); + const std::vector nodes{m_connman.TestNodes()}; + BOOST_REQUIRE(!nodes.empty()); + BOOST_CHECK(nodes.back()->IsInboundConn()); + BOOST_CHECK(nodes.back()->m_pq_endpoint == PQEndpointKey{LookupNumeric("9.8.7.6", 51234)}); +} + +BOOST_AUTO_TEST_CASE(proxy_settings_restorer) +{ + // The guard puts back exactly what was set before it, nested or not. + BOOST_REQUIRE(!HaveNameProxy()); + Proxy proxy; + BOOST_REQUIRE(!GetProxy(NET_IPV4, proxy)); + { + ProxySettingsRestorerForTesting outer; + BOOST_REQUIRE(SetNameProxy(Proxy{LookupNumeric("127.0.0.1", 9050)})); + { + ProxySettingsRestorerForTesting inner; + BOOST_REQUIRE(SetNameProxy(Proxy{LookupNumeric("127.0.0.2", 9050)})); + BOOST_REQUIRE(SetProxy(NET_IPV4, Proxy{LookupNumeric("127.0.0.3", 9050)})); + } + BOOST_REQUIRE(GetNameProxy(proxy)); + BOOST_CHECK(proxy.proxy == LookupNumeric("127.0.0.1", 9050)); + BOOST_CHECK(!GetProxy(NET_IPV4, proxy)); + } + BOOST_CHECK(!HaveNameProxy()); + BOOST_CHECK(!GetProxy(NET_IPV4, proxy)); +} + +BOOST_AUTO_TEST_CASE(pq_name_proxy_endpoint) +{ + // Through a name proxy, a destination by name is never resolved locally: ConnectNode() asks + // the proxy for it, and the endpoint key is the normalized SOCKS destination and the + // effective port, never the proxy's own address. + const CService proxy{LookupNumeric("127.0.0.9", 9050)}; + { + const Socks5InterruptCleared socks5_interrupt; + ProxySettingsRestorerForTesting restore_proxies; + BOOST_REQUIRE(SetNameProxy(Proxy{proxy})); + // The proxy accepts without authentication, then reports the connection made. + const auto create_sock{MockCreateSock(std::string{"\x05\x00" "\x05\x00\x00\x01" "\x00\x00\x00\x00" "\x00\x00", 12})}; + for (const auto& [dest, key] : {std::pair{"Seed.Example.COM.:18555", PQNameEndpoint{"seed.example.com", 18555}}, + std::pair{"seed.example.com", PQNameEndpoint{"seed.example.com", Params().GetDefaultPort()}}, + std::pair{"other.example:18555", PQNameEndpoint{"other.example", 18555}}}) { + const std::unique_ptr node{m_connman.ConnectNodeOnly(dest, ConnectionType::MANUAL, /*use_v2transport=*/true)}; + BOOST_REQUIRE(node); + BOOST_CHECK(node->m_pq_endpoint == PQEndpointKey{key}); + BOOST_CHECK(node->m_pq_endpoint != PQEndpointKey{proxy}); + } + } + // No later test sees the name proxy. + BOOST_CHECK(!HaveNameProxy()); +} + +BOOST_AUTO_TEST_CASE(pq_observation_once) +{ + // A hybrid handshake: switched once, however often it is observed. + Link link{Add(ConnectionType::OUTBOUND_FULL_RELAY)}; + link.node->AddRef(); // keep the node through DisconnectNodes() + V2Transport peer{Responder()}; + Exchange(link, peer); + BOOST_REQUIRE(link.node->m_transport->GetPQSnapshot().confirmed); + BOOST_CHECK(link.node->m_transport->GetInfo().transport_pq_status == PQStatus::HYBRID); + for (int i{0}; i < 3; ++i) m_connman.ObservePQPublic(*link.node, /*finalizing=*/false); + BOOST_CHECK_EQUAL(Stats().outbound.switched, 1U); + + // A switch whose confirmation fails, seen after receive, at disconnect and in cleanup. + Link failing{Add(ConnectionType::OUTBOUND_FULL_RELAY)}; + failing.node->AddRef(); + V2Transport bad_peer{Responder({.mode = PQMode::NEGOTIATE, .corrupt_shared_secret = true})}; + Exchange(failing, bad_peer); + BOOST_REQUIRE(failing.node->fDisconnect); + m_connman.ObservePQPublic(*failing.node, /*finalizing=*/false); + PQTransportStats stats{Stats()}; + BOOST_CHECK_EQUAL(stats.outbound.switched, 2U); + BOOST_CHECK_EQUAL(stats.outbound.first_packet_failed, 1U); + BOOST_CHECK_EQUAL(stats.outbound_failures.entries.size(), 1U); + + // Racing observers, as from the socket handler and cleanup, still count each event once. + std::vector threads; + for (int i{0}; i < 4; ++i) { + threads.emplace_back([&, i] { m_connman.ObservePQPublic(*failing.node, /*finalizing=*/i % 2 == 0); }); + } + for (auto& thread : threads) thread.join(); + Disconnect(); + for (const Link& l : {link, failing}) { + m_connman.ObservePQPublic(*l.node, /*finalizing=*/false); + m_connman.ObservePQPublic(*l.node, /*finalizing=*/true); + BOOST_CHECK(l.node->m_pq_finalized); + } + stats = Stats(); + BOOST_CHECK_EQUAL(stats.outbound.switched, 2U); + BOOST_CHECK_EQUAL(stats.outbound.first_packet_failed, 1U); + BOOST_CHECK_EQUAL(stats.outbound.closed_after_switch, 0U); // our own close after the failure + BOOST_CHECK_EQUAL(stats.outbound_failures.entries.size(), 1U); + BOOST_CHECK_EQUAL(stats.outbound_failures.last_sequence, 1U); + BOOST_CHECK_EQUAL(stats.inbound.switched + stats.inbound.first_packet_failed, 0U); + BOOST_CHECK(stats.inbound_failures.entries.empty()); + for (const Link& l : {link, failing}) l.node->Release(); + Disconnect(); + + // Instance identity: random per connection manager, counted from its creation. + ConnmanTestMsg other{0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman, Params()}; + BOOST_CHECK(other.GetPQTransportStats().instance_id != stats.instance_id); + BOOST_CHECK(stats.since <= Now()); + BOOST_CHECK_EQUAL(other.GetPQTransportStats().outbound.switched, 0U); +} + +BOOST_AUTO_TEST_CASE(pq_ring_bounds) +{ + // One outbound failure: the responder offers no features. + Link outbound{Add(ConnectionType::MANUAL)}; + V2Transport legacy{Responder({})}; + Exchange(outbound, legacy); + BOOST_CHECK_EQUAL(Stats().outbound.legacy_peer, 1U); + + // 300 inbound failures: each offer is abandoned when the peer closes. + std::vector inbound; + for (int i{0}; i < 300; ++i) { + inbound.push_back(Add(ConnectionType::INBOUND, PQMode::NEGOTIATE, std::nullopt, LookupNumeric("10.0.0.1", 40000 + i))); + Receive(inbound.back(), InitiatorKey()); + } + Pass(inbound); + for (const Link& link : inbound) { + BOOST_REQUIRE(link.node->m_transport->GetPQSnapshot().offer == PQOfferState::SENT); + link.pipes->recv.Eof(); + } + Pass(inbound); + Disconnect(); + + const PQTransportStats stats{Stats()}; + BOOST_CHECK_EQUAL(stats.inbound.abandoned, 300U); + BOOST_CHECK_EQUAL(stats.inbound_failures.entries.size(), PQ_FAILURE_RING_SIZE); + BOOST_CHECK_EQUAL(stats.inbound_failures.dropped, 44U); + BOOST_CHECK_EQUAL(stats.inbound_failures.last_sequence, 300U); + for (size_t i{0}; i < stats.inbound_failures.entries.size(); ++i) { + const PQFailureEntry& entry{stats.inbound_failures.entries[i]}; + BOOST_CHECK_EQUAL(entry.sequence, 45 + i); + BOOST_CHECK(entry.inbound); + BOOST_CHECK(entry.outcome == PQOutcome::ABANDONED); + BOOST_CHECK_EQUAL(entry.reason, "eof"); + } + BOOST_CHECK(stats.inbound_failures.entries.back().endpoint == PQEndpointKey{LookupNumeric("10.0.0.1", 40299)}); + + // The outbound entry survives, with its own sequence. + BOOST_REQUIRE_EQUAL(stats.outbound_failures.entries.size(), 1U); + const PQFailureEntry& entry{stats.outbound_failures.entries.front()}; + BOOST_CHECK_EQUAL(entry.sequence, 1U); + BOOST_CHECK_EQUAL(stats.outbound_failures.last_sequence, 1U); + BOOST_CHECK_EQUAL(stats.outbound_failures.dropped, 0U); + BOOST_CHECK(entry.outcome == PQOutcome::LEGACY_PEER); + BOOST_CHECK_EQUAL(entry.reason, "no_features"); + BOOST_CHECK(!entry.inbound); + BOOST_CHECK(entry.connection_type == ConnectionType::MANUAL); + BOOST_CHECK_EQUAL(entry.peer_id, outbound.node->GetId()); + BOOST_CHECK(entry.endpoint == PQEndpointKey{LookupNumeric("1.2.3.4", 8333)}); + BOOST_CHECK(entry.time == Now()); +} + +BOOST_AUTO_TEST_CASE(pq_switched_then_failed) +{ + for (const bool inbound : {false, true}) { + Link link{Add(inbound ? ConnectionType::INBOUND : ConnectionType::OUTBOUND_FULL_RELAY)}; + // Not `cond ? a : b`: MSVC requires a copy constructor for a conditional of + // prvalues, and V2Transport has none. + V2Transport peer{[&] { if (inbound) return Initiator({.mode = PQMode::NEGOTIATE, .corrupt_shared_secret = true}); return Responder({.mode = PQMode::NEGOTIATE, .corrupt_shared_secret = true}); }()}; + Exchange(link, peer); + const PQHandshake::Snapshot snapshot{link.node->m_transport->GetPQSnapshot()}; + BOOST_CHECK(snapshot.switched); + BOOST_CHECK(snapshot.failure == PQFailure::CONFIRM_LENGTH || snapshot.failure == PQFailure::CONFIRM_TAG); + BOOST_CHECK(link.node->fDisconnect); + Disconnect(); + const PQTransportStats stats{Stats()}; + const PQCounts& counts{inbound ? stats.inbound : stats.outbound}; + BOOST_CHECK_EQUAL(counts.switched, 1U); + BOOST_CHECK_EQUAL(counts.first_packet_failed, 1U); + BOOST_CHECK_EQUAL(counts.closed_after_switch + counts.abandoned + counts.malformed_record, 0U); + const PQFailureRing& ring{inbound ? stats.inbound_failures : stats.outbound_failures}; + BOOST_REQUIRE_EQUAL(ring.entries.size(), 1U); + BOOST_CHECK(ring.entries.front().outcome == PQOutcome::FIRST_PACKET_FAILED); + BOOST_CHECK(ring.entries.front().reason == "length" || ring.entries.front().reason == "tag"); + } +} + +BOOST_AUTO_TEST_CASE(pq_abandoned) +{ + // An offer is queued, and the connection closes before an authenticated version arrives. + struct Case { + std::string reason; + std::function close; + }; + const std::vector cases{ + {"eof", [](PQNetSetup&, const Link& link, RawPeer&) { link.pipes->recv.Eof(); }}, + {"reset", [](PQNetSetup&, const Link& link, RawPeer&) { link.sock->m_fail_recv = true; }}, + {"timeout", [](PQNetSetup&, const Link&, RawPeer&) { SetMockTime(GetTime() + 61s); }}, + {"local", [](PQNetSetup& setup, const Link& link, RawPeer&) { BOOST_CHECK(setup.m_connman.DisconnectNode(link.node->GetId())); }}, + // The peer's version packet doesn't authenticate. + {"version_tag", [](PQNetSetup&, const Link& link, RawPeer& peer) { + Receive(link, peer.TerminatorAndVersion({}, /*damage_tag=*/true)); + }}, + // The peer's version packet is longer than any packet may be. + {"version_length", [](PQNetSetup&, const Link& link, RawPeer& peer) { + const auto packet{peer.TerminatorAndVersion(std::vector(MAX_PROTOCOL_MESSAGE_LENGTH + 14))}; + Receive(link, std::span{packet}.first(BIP324Cipher::GARBAGE_TERMINATOR_LEN + BIP324Cipher::LENGTH_LEN)); + }}, + // No garbage terminator in the longest garbage allowed. + {"garbage_terminator", [](PQNetSetup&, const Link& link, RawPeer&) { + Receive(link, std::vector(V2Transport::MAX_GARBAGE_LEN + BIP324Cipher::GARBAGE_TERMINATOR_LEN, 0)); + }}, + }; + for (const Case& test : cases) { + BOOST_TEST_MESSAGE("abandoned: " + test.reason); + // The peer sends its key, so the node offers, and then not the version it is asked for. + Link link{Add(ConnectionType::INBOUND, PQMode::NEGOTIATE, std::nullopt, LookupNumeric("10.0.0.2", 50000))}; + RawPeer peer{/*initiator=*/true}; + Receive(link, peer.Key()); + Pass(link); + BOOST_REQUIRE(link.node->m_transport->GetPQSnapshot().offer == PQOfferState::SENT); + peer.Initialize(NodeKey(link)); + test.close(*this, link, peer); + for (int i{0}; i < 4; ++i) Pass(link); + BOOST_CHECK(link.node->fDisconnect); + const auto line{ExpectLine(strprintf("v2 pq: abandoned reason=%s role=responder conn_type=inbound peer=%d", test.reason, link.node->GetId()), + LineLevel::NET_DEBUG)}; + Disconnect(); + } + // A failed send of the queued offer. + { + Link link{Add(ConnectionType::INBOUND)}; + Receive(link, InitiatorKey()); + link.sock->m_fail_send = true; + Pass(link); // receives the key and queues the offer + Pass(link); // the send fails + BOOST_CHECK(link.node->GetCloseCause() == NodeCloseCause::SEND_ERROR); + Disconnect(); + } + + const PQTransportStats stats{Stats()}; + BOOST_CHECK_EQUAL(stats.inbound.abandoned, cases.size() + 1); + BOOST_CHECK_EQUAL(stats.inbound.switched, 0U); + BOOST_REQUIRE_EQUAL(stats.inbound_failures.entries.size(), cases.size() + 1); + for (size_t i{0}; i < cases.size(); ++i) BOOST_CHECK_EQUAL(stats.inbound_failures.entries[i].reason, cases[i].reason); + BOOST_CHECK_EQUAL(stats.inbound_failures.entries.back().reason, "send_error"); + BOOST_CHECK(stats.inbound_failures.entries.front().endpoint == PQEndpointKey{LookupNumeric("10.0.0.2", 50000)}); + BOOST_CHECK(stats.outbound_failures.entries.empty()); + + // An offer the peer answered is not abandoned, however the connection ends. + Link answered{Add(ConnectionType::INBOUND)}; + V2Transport peer{Initiator({})}; + Exchange(answered, peer); + BOOST_CHECK(answered.node->m_transport->GetPQSnapshot().legacy == PQLegacyReason::NO_FEATURES); + answered.pipes->recv.Eof(); + Pass(answered); + Disconnect(); + BOOST_CHECK_EQUAL(Stats().inbound.abandoned, cases.size() + 1); + BOOST_CHECK_EQUAL(Stats().inbound.legacy_peer, 1U); +} + +BOOST_AUTO_TEST_CASE(pq_outbound_never_abandoned) +{ + // Only responders offer, so an outbound connection that closes before the peer's version is + // never abandoned, whoever closes it: the outbound count stays 0 and its ring stays empty. + for (const bool peer_closes : {true, false}) { + Link link{Add(ConnectionType::OUTBOUND_FULL_RELAY, PQMode::NEGOTIATE, TestEndpoint(10, peer_closes ? 1 : 2))}; + Pass(link); + Receive(link, InitiatorKey()); // any 64 bytes are a valid key, here the responder's + Pass(link); + BOOST_REQUIRE(!link.node->m_transport->GetPQSnapshot().version_received); + if (peer_closes) { + link.pipes->recv.Eof(); + } else { + link.node->RequestDisconnect(); + } + Pass(link); + BOOST_REQUIRE(link.node->fDisconnect); + Disconnect(); + } + const PQTransportStats stats{Stats()}; + BOOST_CHECK_EQUAL(stats.outbound.abandoned, 0U); + BOOST_CHECK(stats.outbound_failures.entries.empty()); +} + +BOOST_AUTO_TEST_CASE(pq_closed_after_switch) +{ + // We switched, and the peer closed or stopped responding before its confirmation arrived. + // Each connection goes to its own endpoint, so none enters the fallback set. + uint16_t port{8000}; + for (const std::string cause : {"eof", "reset", "timeout", "local", "send_error"}) { + Link link{Add(ConnectionType::OUTBOUND_FULL_RELAY, PQMode::NEGOTIATE, LookupNumeric("1.2.3.4", ++port))}; + V2Transport peer{Responder()}; + Pass(link); // our key and garbage + BOOST_REQUIRE(NodeToPeer(link, peer)); + PeerToNode(link, peer); // the peer's key, garbage, terminator and offer + if (cause == "send_error") link.sock->m_fail_send = true; + Pass(link); + BOOST_REQUIRE(link.node->m_transport->GetPQSnapshot().switched); + if (cause == "eof") link.pipes->recv.Eof(); + if (cause == "reset") link.sock->m_fail_recv = true; + if (cause == "timeout") SetMockTime(GetTime() + 61s); + if (cause == "local") link.node->RequestDisconnect(); + Pass(link); + Pass(link); + BOOST_REQUIRE(link.node->fDisconnect); + const bool counted{cause == "eof" || cause == "reset" || cause == "timeout"}; + // Count the lines rather than require one, so that a missing line fails the check instead + // of throwing from a destructor. + const std::string text{strprintf("v2 pq: closed_after_switch cause=%s role=initiator conn_type=outbound-full-relay peer=%d%s", + cause, link.node->GetId(), Backends())}; + const auto expected{AtLevel({text}, LineLevel::INFO)}; + int lines{0}, expected_lines{0}; + const uint64_t before{Stats().outbound.closed_after_switch}; + { + DebugLogHelper watch{"v2 pq: closed_after_switch", [&](const std::string* line) { + if (line) { + ++lines; + if (expected(line)) ++expected_lines; + } + return false; + }}; + Disconnect(); + } + BOOST_CHECK_EQUAL(lines, counted ? 1 : 0); + BOOST_CHECK_EQUAL(expected_lines, counted ? 1 : 0); + BOOST_CHECK_EQUAL(Stats().outbound.closed_after_switch, before + (counted ? 1 : 0)); + } + const PQTransportStats stats{Stats()}; + BOOST_CHECK_EQUAL(stats.outbound.switched, 5U); + BOOST_CHECK_EQUAL(stats.outbound.closed_after_switch, 3U); + BOOST_REQUIRE_EQUAL(stats.outbound_failures.entries.size(), 3U); + BOOST_CHECK_EQUAL(stats.outbound_failures.entries[2].reason, "timeout"); + + // An inbound switch closed by the peer is not counted: closed_after_switch is outbound only. + Link inbound{Add(ConnectionType::INBOUND)}; + V2Transport peer{Initiator()}; + PeerToNode(inbound, peer); + Pass(inbound); + BOOST_REQUIRE(NodeToPeer(inbound, peer)); + const std::vector accept{PeerBytes(peer)}; // and the confirmation, held back + Receive(inbound, std::span{accept}.first(accept.size() - PQ_CONFIRMATION_BYTES)); + Pass(inbound); + BOOST_REQUIRE(inbound.node->m_transport->GetPQSnapshot().switched); + BOOST_REQUIRE(!inbound.node->m_transport->GetPQSnapshot().confirmed); + inbound.pipes->recv.Eof(); + Pass(inbound); + Disconnect(); + BOOST_CHECK_EQUAL(Stats().inbound.switched, 1U); + BOOST_CHECK_EQUAL(Stats().inbound.closed_after_switch + Stats().inbound.abandoned, 0U); + + // After a confirmed handshake, a peer close or a timeout is an ordinary close: nothing is + // counted, the ring gets no entry, and nothing is logged. + int lines{0}; + { + DebugLogHelper watch{"v2 pq: closed_after_switch", [&](const std::string* line) { + if (line) ++lines; + return false; + }}; + for (const NodeCloseCause cause : {NodeCloseCause::PEER_EOF, NodeCloseCause::PEER_RESET, NodeCloseCause::TIMEOUT}) { + Link link{Add(ConnectionType::OUTBOUND_FULL_RELAY, PQMode::NEGOTIATE, LookupNumeric("1.2.3.6", 8333))}; + V2Transport responder{Responder()}; + Exchange(link, responder); + BOOST_REQUIRE(link.node->m_transport->GetPQSnapshot().confirmed); + if (cause == NodeCloseCause::PEER_EOF) link.pipes->recv.Eof(); + if (cause == NodeCloseCause::PEER_RESET) link.sock->m_fail_recv = true; + if (cause == NodeCloseCause::TIMEOUT) SetMockTime(GetTime() + 61s); + Pass(link); + BOOST_REQUIRE(link.node->GetCloseCause() == cause); + Disconnect(); + } + } + BOOST_CHECK_EQUAL(lines, 0); + BOOST_CHECK_EQUAL(Stats().outbound.switched, stats.outbound.switched + 3); + BOOST_CHECK_EQUAL(Stats().outbound.closed_after_switch, stats.outbound.closed_after_switch); + BOOST_CHECK_EQUAL(Stats().outbound_failures.last_sequence, stats.outbound_failures.last_sequence); +} + +BOOST_AUTO_TEST_CASE(pq_log_lines) +{ + // Outbound failures are info lines; inbound lines and outbound successes are net debug lines. + const auto line{[](std::string_view text, const Link& link, bool failure) { + return ExpectLine(strprintf("v2 pq: %s conn_type=%s peer=%d%s", text, link.node->ConnectionTypeAsString(), link.node->GetId(), + failure ? Backends() : ""), + failure && !link.node->IsInboundConn() ? LineLevel::INFO : LineLevel::NET_DEBUG); + }}; + + // Outbound: a hybrid handshake, a legacy peer, malformed records, a parse error. + { + Link link{Add(ConnectionType::OUTBOUND_FULL_RELAY)}; + V2Transport peer{Responder()}; + const auto expect{line("switched role=initiator", link, false)}; + Exchange(link, peer); + } + { + Link link{Add(ConnectionType::BLOCK_RELAY)}; + V2Transport peer{Responder({})}; + const auto expect{line("legacy_peer reason=no_features role=initiator", link, false)}; + Exchange(link, peer); + } + const auto raw_responder{[&](const Link& link, std::span contents) { + Pass(link); + RawPeer peer{/*initiator=*/false}; + peer.Initialize(NodeKey(link)); + Receive(link, peer.Key()); + Receive(link, peer.TerminatorAndVersion(contents)); + Pass(link); + }}; + { + Link link{Add(ConnectionType::MANUAL)}; + const auto expect{line("legacy_peer reason=parse_error role=initiator", link, false)}; + raw_responder(link, std::vector{std::byte{0x05}, std::byte{0x01}}); + } + { + Link link{Add(ConnectionType::MANUAL)}; + const auto expect{line("malformed_record reason=ek_length role=initiator", link, true)}; + raw_responder(link, MakeRecord(PQ_MLKEM1024, 100)); + BOOST_CHECK(link.node->fDisconnect); + // A closing connection reports pending until it is removed. + BOOST_CHECK(link.node->m_transport->GetInfo().transport_pq_status == PQStatus::PENDING); + } + // With mismatched keys, the confirmation's length decrypts to a random value, rarely to 0. + const auto confirm_failed{[](const Link& link, std::string_view role) { + std::vector texts; + for (const std::string_view reason : {"length", "tag"}) { + texts.push_back(strprintf("v2 pq: first_packet_failed reason=%s role=%s conn_type=%s peer=%d%s", reason, role, + link.node->ConnectionTypeAsString(), link.node->GetId(), Backends())); + } + return DebugLogHelper{"v2 pq: first_packet_failed reason=", AtLevel(texts, role == "initiator" ? LineLevel::INFO : LineLevel::NET_DEBUG)}; + }}; + { + Link link{Add(ConnectionType::FEELER)}; + const auto expect{confirm_failed(link, "initiator")}; + V2Transport peer{Responder({.mode = PQMode::NEGOTIATE, .corrupt_shared_secret = true})}; + Exchange(link, peer); + } + { + // A local fault before we committed: the connection continues as plain v2, status off. + Link link{Add(ConnectionType::ADDR_FETCH)}; + const auto expect{line("internal_error reason=encaps role=initiator", link, true)}; + V2Transport peer{Responder()}; + mlkem::InjectResultForTesting inject{mlkem::Operation::ENCAPS, mlkem::upstream::ERR_FAIL}; + Exchange(link, peer); + BOOST_CHECK(!link.node->fDisconnect); + BOOST_CHECK(link.node->m_transport->GetInfo().transport_pq_status == PQStatus::OFF); + } + + // Inbound: the same outcomes, at debug level from their own call sites. + { + Link link{Add(ConnectionType::INBOUND)}; + V2Transport peer{Initiator()}; + const auto expect{line("switched role=responder", link, false)}; + Exchange(link, peer); + BOOST_CHECK(link.node->m_transport->GetPQSnapshot().confirmed); + } + { + Link link{Add(ConnectionType::INBOUND)}; + V2Transport peer{Initiator({})}; + const auto expect{line("legacy_peer reason=no_features role=responder", link, false)}; + Exchange(link, peer); + } + const auto raw_initiator{[&](const Link& link, std::span contents) { + RawPeer peer{/*initiator=*/true}; + Receive(link, peer.Key()); + Pass(link); + peer.Initialize(NodeKey(link)); + Receive(link, peer.TerminatorAndVersion(contents)); + Pass(link); + }}; + { + Link link{Add(ConnectionType::INBOUND)}; + const auto expect{line("legacy_peer reason=parse_error role=responder", link, false)}; + raw_initiator(link, std::vector{std::byte{0x05}, std::byte{0x01}}); + } + { + Link link{Add(ConnectionType::INBOUND)}; + const auto expect{line("malformed_record reason=ct_length role=responder", link, true)}; + raw_initiator(link, MakeRecord(PQ_MLKEM1024, 100)); + } + { + Link link{Add(ConnectionType::INBOUND)}; + const auto expect{confirm_failed(link, "responder")}; + V2Transport peer{Initiator({.mode = PQMode::NEGOTIATE, .corrupt_shared_secret = true})}; + Exchange(link, peer); + } + { + Link link{Add(ConnectionType::INBOUND)}; + const auto expect{line("internal_error reason=keygen role=responder", link, true)}; + mlkem::InjectResultForTesting inject{mlkem::Operation::KEYGEN, mlkem::upstream::ERR_FAIL}; + Receive(link, InitiatorKey()); + Pass(link); + // No offer, so the connection continues as plain v2, status off. + BOOST_CHECK(!link.node->fDisconnect); + BOOST_CHECK(link.node->m_transport->GetInfo().transport_pq_status == PQStatus::OFF); + } + { + Link link{Add(ConnectionType::INBOUND)}; + const auto expect{line("internal_error reason=decaps role=responder", link, true)}; + V2Transport peer{Initiator()}; + mlkem::InjectResultForTesting inject{mlkem::Operation::DECAPS, mlkem::upstream::ERR_FAIL}; + Exchange(link, peer); + // The initiator has committed, so this local fault closes, and reports pending until the + // connection is removed. Counting reads the failure, never the status. + BOOST_CHECK(link.node->fDisconnect); + BOOST_CHECK(link.node->m_transport->GetInfo().transport_pq_status == PQStatus::PENDING); + } + + // -logips adds the peer's address. + { + Link link{Add(ConnectionType::MANUAL)}; + V2Transport peer{Responder()}; + const GlobalOverride log_ips{fLogIPs, true}; + const auto expect{ExpectLine(strprintf("v2 pq: switched role=initiator conn_type=manual peer=%d peeraddr=1.2.3.4:8333", link.node->GetId()), + LineLevel::NET_DEBUG)}; + Exchange(link, peer); + } + Disconnect(); + + const PQTransportStats stats{Stats()}; + for (const PQCounts& counts : {stats.inbound, stats.outbound}) { + BOOST_CHECK_EQUAL(counts.legacy_peer, 2U); + BOOST_CHECK_EQUAL(counts.malformed_record, 1U); + BOOST_CHECK_EQUAL(counts.first_packet_failed, 1U); + } + BOOST_CHECK_EQUAL(stats.outbound.switched, 3U); // hybrid, corrupted, -logips + BOOST_CHECK_EQUAL(stats.outbound.internal_error, 1U); + BOOST_CHECK_EQUAL(stats.inbound.switched, 2U); // hybrid, corrupted + BOOST_CHECK_EQUAL(stats.inbound.internal_error, 2U); +} + +BOOST_AUTO_TEST_CASE(pq_cipher_state_closes) +{ + // An inconsistent cipher state is a local fault that closes the connection in either role, + // an initiator that hasn't committed yet included. It counts once, as internal_error with + // reason cipher_state: an inbound offer that was queued is not also abandoned. + // DiscardHybridSecretForTesting() induces it; the failing check is an Assume, so only + // release builds can exercise it. + if constexpr (!G_FUZZING_BUILD && !G_ABORT_ON_FAILED_ASSUME) { + const PQEndpointKey endpoint{LookupNumeric("1.2.3.5", 8333)}; + for (int i{0}; i < 3; ++i) { + // Outbound: the keys are initialized, and the responder's offer arrives after the discard. + Link link{Add(ConnectionType::OUTBOUND_FULL_RELAY, PQMode::NEGOTIATE, endpoint)}; + auto& transport{dynamic_cast(*link.node->m_transport)}; + V2Transport peer{Responder()}; + Pass(link); + BOOST_REQUIRE(NodeToPeer(link, peer)); + const std::vector bytes{PeerBytes(peer)}; + Receive(link, std::span{bytes}.first(EllSwiftPubKey::size())); + Pass(link); + BOOST_REQUIRE(transport.HoldsHybridSecretsForTesting()); + transport.DiscardHybridSecretForTesting(); + const std::string text{strprintf("v2 pq: internal_error reason=cipher_state role=initiator conn_type=outbound-full-relay peer=%d%s", + link.node->GetId(), Backends())}; + DebugLogHelper expect{text, AtLevel({text}, LineLevel::INFO)}; + Receive(link, std::span{bytes}.subspan(EllSwiftPubKey::size())); + Pass(link); + BOOST_CHECK(link.node->fDisconnect); + BOOST_CHECK(transport.GetPQSnapshot().failure == PQFailure::CIPHER_STATE_INTERNAL); + BOOST_CHECK(transport.GetInfo().transport_pq_status == PQStatus::PENDING); + Disconnect(); + } + { + // Inbound: the offer is queued, and the initiator's accept arrives after the discard. + Link link{Add(ConnectionType::INBOUND)}; + auto& transport{dynamic_cast(*link.node->m_transport)}; + V2Transport peer{Initiator()}; + PeerToNode(link, peer); + Pass(link); + BOOST_REQUIRE(transport.GetPQSnapshot().offer == PQOfferState::SENT); + BOOST_REQUIRE(NodeToPeer(link, peer)); + transport.DiscardHybridSecretForTesting(); + const std::string text{strprintf("v2 pq: internal_error reason=cipher_state role=responder conn_type=inbound peer=%d%s", + link.node->GetId(), Backends())}; + DebugLogHelper expect{text, AtLevel({text}, LineLevel::NET_DEBUG)}; + PeerToNode(link, peer); + Pass(link); + BOOST_CHECK(link.node->fDisconnect); + BOOST_CHECK(transport.GetPQSnapshot().failure == PQFailure::CIPHER_STATE_INTERNAL); + Disconnect(); + } + const PQTransportStats stats{Stats()}; + BOOST_CHECK_EQUAL(stats.outbound.internal_error, 3U); + BOOST_CHECK_EQUAL(stats.inbound.internal_error, 1U); + BOOST_CHECK_EQUAL(stats.inbound.abandoned + stats.outbound.closed_after_switch + stats.outbound.switched, 0U); + BOOST_REQUIRE_EQUAL(stats.inbound_failures.entries.size(), 1U); + BOOST_CHECK_EQUAL(stats.inbound_failures.entries.front().reason, "cipher_state"); + BOOST_REQUIRE_EQUAL(stats.outbound_failures.entries.size(), 3U); + for (const PQFailureEntry& entry : stats.outbound_failures.entries) { + BOOST_CHECK(entry.outcome == PQOutcome::INTERNAL_ERROR); + BOOST_CHECK_EQUAL(entry.reason, "cipher_state"); + } + // A local fault never counts toward the fallback. + BOOST_CHECK(!Streak(endpoint)); + BOOST_CHECK(!m_connman.IsPQFallback(endpoint, Now())); + BOOST_CHECK_EQUAL(stats.outbound.fallback, 0U); + } +} + +BOOST_AUTO_TEST_CASE(pq_finalization_wipes) +{ + // Every way a connection can close while hybrid secrets are pending: an inbound offer's + // decapsulation key and either role's retained ECDH secret. Finalization wipes them before + // the disconnected pool releases the node, not only when the node is deleted. + enum class Close { PEER_EOF, PEER_RESET, TIMEOUT, LOCAL, SEND_ERROR, NETWORK_OFF, TRANSPORT }; + for (const bool inbound : {true, false}) { + for (const Close close : {Close::PEER_EOF, Close::PEER_RESET, Close::TIMEOUT, Close::LOCAL, Close::SEND_ERROR, + Close::NETWORK_OFF, Close::TRANSPORT}) { + Link link{Add(inbound ? ConnectionType::INBOUND : ConnectionType::OUTBOUND_FULL_RELAY)}; + link.node->AddRef(); // keep the node in the disconnected pool + const auto& transport{dynamic_cast(*link.node->m_transport)}; + if (!inbound) Pass(link); // our key and garbage + // The peer's key: an inbound node queues its offer, an outbound one its terminator. + Receive(link, InitiatorKey()); + if (close == Close::SEND_ERROR) link.sock->m_fail_send = true; + Pass(link); + BOOST_CHECK_EQUAL(transport.GetPQSnapshot().offer == PQOfferState::SENT, inbound); + if (close != Close::SEND_ERROR) BOOST_REQUIRE(transport.HoldsHybridSecretsForTesting()); + switch (close) { + case Close::PEER_EOF: link.pipes->recv.Eof(); break; + case Close::PEER_RESET: link.sock->m_fail_recv = true; break; + case Close::TIMEOUT: SetMockTime(GetTime() + 61s); break; + case Close::LOCAL: link.node->RequestDisconnect(); break; + case Close::SEND_ERROR: break; + case Close::NETWORK_OFF: m_connman.SetNetworkActive(false); break; + case Close::TRANSPORT: // garbage without a terminator; the transport wipes as it fails + Receive(link, std::vector(V2Transport::MAX_GARBAGE_LEN + BIP324Cipher::GARBAGE_TERMINATOR_LEN, 0)); + break; + } + Pass(link); + // A peer or socket close leaves the secrets to finalization. + if (close != Close::TRANSPORT && close != Close::NETWORK_OFF) { + BOOST_REQUIRE(link.node->fDisconnect); + BOOST_CHECK(transport.HoldsHybridSecretsForTesting()); + } + const uint64_t abandoned{Stats().inbound.abandoned}; + Disconnect(); + m_connman.SetNetworkActive(true); + BOOST_CHECK(link.node->m_pq_finalized); + BOOST_CHECK(!transport.HoldsHybridSecretsForTesting()); + // With the network off, every close is ours: nothing is recorded. + BOOST_CHECK_EQUAL(Stats().inbound.abandoned, abandoned + (inbound && close != Close::NETWORK_OFF ? 1 : 0)); + link.node->Release(); + Disconnect(); + } + } + + // The wipe is effective: a peer's accept that arrives later can no longer switch the keys. + // The transport then fails an Assume, so only release builds can exercise it. + if constexpr (!G_FUZZING_BUILD && !G_ABORT_ON_FAILED_ASSUME) { + Link link{Add(ConnectionType::INBOUND)}; + link.node->AddRef(); + V2Transport peer{Initiator()}; + PeerToNode(link, peer); + Pass(link); + BOOST_REQUIRE(NodeToPeer(link, peer)); // the peer now holds its accept for us + link.node->RequestDisconnect(); + Disconnect(); + const std::vector bytes{PeerBytes(peer)}; + std::span span{bytes}; + BOOST_CHECK(!link.node->m_transport->ReceivedBytes(span)); + const PQHandshake::Snapshot snapshot{link.node->m_transport->GetPQSnapshot()}; + BOOST_CHECK(snapshot.version_received); + BOOST_CHECK(!snapshot.switched); + BOOST_CHECK(snapshot.failure == PQFailure::CIPHER_STATE_INTERNAL); + // Nothing is counted after finalization. + m_connman.ObservePQPublic(*link.node, /*finalizing=*/false); + BOOST_CHECK_EQUAL(Stats().inbound.internal_error, 0U); + link.node->Release(); + Disconnect(); + } +} + +BOOST_AUTO_TEST_CASE(pq_shutdown_finalizes) +{ + // Shutdown deletes the connections it still holds itself, without DisconnectNodes(). It + // finalizes them first: no node is deleted holding hybrid secrets, but a shutdown close is + // our own decision, so nothing is recorded, as with the network off. + Link inbound{Add(ConnectionType::INBOUND)}; // its offer is pending: decapsulation key and ECDH secret + Receive(inbound, InitiatorKey()); + Pass(inbound); + BOOST_REQUIRE(inbound.node->m_transport->GetPQSnapshot().offer == PQOfferState::SENT); + Link outbound{Add(ConnectionType::OUTBOUND_FULL_RELAY)}; // awaiting the version: the ECDH secret + Pass(outbound); + Receive(outbound, InitiatorKey()); + Pass(outbound); + Link hybrid{Add(ConnectionType::MANUAL)}; + V2Transport peer{Responder()}; + Exchange(hybrid, peer); + BOOST_REQUIRE(hybrid.node->m_transport->GetPQSnapshot().confirmed); + for (const Link& link : {inbound, outbound}) { + BOOST_REQUIRE(dynamic_cast(*link.node->m_transport).HoldsHybridSecretsForTesting()); + } + const PQTransportStats before{Stats()}; + const int deleted{m_events.m_deleted}; + m_connman.Stop(); + BOOST_CHECK_EQUAL(m_events.m_deleted - deleted, 3); + BOOST_CHECK_EQUAL(m_events.m_deleted_with_secrets, 0); + BOOST_CHECK_EQUAL(m_events.m_deleted_unfinalized, 0); + // No outcome: the pending offer isn't abandoned, and no ring entry is added. + const PQTransportStats after{Stats()}; + BOOST_CHECK_EQUAL(after.inbound.abandoned, before.inbound.abandoned); + BOOST_CHECK_EQUAL(after.inbound.switched, before.inbound.switched); + BOOST_CHECK_EQUAL(after.outbound.switched, before.outbound.switched); + BOOST_CHECK_EQUAL(after.inbound_failures.last_sequence, before.inbound_failures.last_sequence); + BOOST_CHECK_EQUAL(after.outbound_failures.last_sequence, before.outbound_failures.last_sequence); +} + +BOOST_AUTO_TEST_CASE(pq_fallback_threshold) +{ + using enum End; + struct Case { + End end; + std::string cause; + std::vector reasons; + }; + int index{0}; + for (const Case& test : {Case{MALFORMED, "malformed_record", {"ek_length"}}, Case{CONFIRMATION, "first_packet_failed", {"length", "tag"}}, + Case{PEER_EOF, "closed_after_switch", {"eof"}}, Case{PEER_RESET, "closed_after_switch", {"reset"}}, + Case{TIMEOUT, "closed_after_switch", {"timeout"}}}) { + BOOST_TEST_MESSAGE("counted failure: " + test.cause + " " + test.reasons.front()); + const PQEndpointKey endpoint{TestEndpoint(3, index++)}; + Connect(endpoint, test.end); + Connect(endpoint, test.end); + // Our own closes, send errors and legacy outcomes are neutral: they neither advance nor + // reset the streak. + Connect(endpoint, LOCAL); + Connect(endpoint, SEND_ERROR); + Connect(endpoint, LEGACY); + BOOST_CHECK(!m_connman.IsPQFallback(endpoint, Now())); + const auto streak{Streak(endpoint)}; + BOOST_REQUIRE(streak); + BOOST_CHECK_EQUAL(streak->streak, 2U); + BOOST_CHECK_EQUAL(PQOutcomeString(streak->cause), test.cause); + BOOST_CHECK(std::ranges::count(test.reasons, streak->reason) == 1); + BOOST_CHECK(streak->next_window == 3600s); + + // The third enters the fallback set once, with one warning. + const uint64_t fallbacks{Stats().outbound.fallback}; + const NodeSeconds failure_time{Now() + (test.end == TIMEOUT ? 61s : 0s)}; + { + const std::string until{FormatISO8601DateTime(TicksSinceEpoch(failure_time + 3600s))}; + const std::string text{strprintf("v2 pq: fallback cause=%s until=%s role=initiator conn_type=outbound-full-relay peer=%d%s", + test.cause, until, m_next_id, Backends())}; + DebugLogHelper expect{text, AtLevel({text}, LineLevel::WARNING)}; + Connect(endpoint, test.end); + } + BOOST_CHECK(m_connman.IsPQFallback(endpoint, Now())); + PQTransportStats stats{Stats()}; + BOOST_CHECK_EQUAL(stats.outbound.fallback, fallbacks + 1); + const auto fallback{Fallback(endpoint)}; + BOOST_REQUIRE(fallback); + BOOST_CHECK_EQUAL(PQOutcomeString(fallback->cause), test.cause); + BOOST_CHECK_EQUAL(fallback->streak, 3U); + BOOST_CHECK(fallback->entered == failure_time); + BOOST_CHECK(fallback->expires == failure_time + 3600s); + BOOST_CHECK(!Streak(endpoint)); + const PQFailureEntry& entry{stats.outbound_failures.entries.back()}; + BOOST_CHECK(entry.outcome == PQOutcome::FALLBACK); + BOOST_CHECK_EQUAL(entry.reason, test.cause); + BOOST_CHECK(entry.endpoint == endpoint); + BOOST_CHECK(!entry.inbound); + + // A failure from a connection that was already running doesn't enter again. + Connect(endpoint, test.end); + stats = Stats(); + BOOST_CHECK_EQUAL(stats.outbound.fallback, fallbacks + 1); + BOOST_REQUIRE(Fallback(endpoint)); + BOOST_CHECK(Fallback(endpoint)->expires == failure_time + 3600s); + } + + // Inbound failures never count: inbound connections never fall back. + const CService inbound_addr{LookupNumeric("10.3.1.1", 50000)}; + for (int i{0}; i < 3; ++i) { + Link link{Add(ConnectionType::INBOUND, PQMode::NEGOTIATE, std::nullopt, inbound_addr)}; + RawPeer peer{/*initiator=*/true}; + Receive(link, peer.Key()); + Pass(link); + peer.Initialize(NodeKey(link)); + Receive(link, peer.TerminatorAndVersion(MakeRecord(PQ_MLKEM1024, 100))); + Pass(link); + BOOST_REQUIRE(link.node->fDisconnect); + Disconnect(); + } + BOOST_CHECK_EQUAL(Stats().inbound.malformed_record, 3U); + BOOST_CHECK(!Streak(inbound_addr)); + BOOST_CHECK(!m_connman.IsPQFallback(inbound_addr, Now())); +} + +BOOST_AUTO_TEST_CASE(pq_fallback_windows) +{ + const PQEndpointKey endpoint{LookupNumeric("5.6.7.8", 18555)}; + const auto create_sock{MockCreateSock()}; + // A new outbound connection to the endpoint, as the connection logic opens one. + const auto new_status{[&] { + const std::unique_ptr node{m_connman.ConnectNodeOnly("5.6.7.8:18555", ConnectionType::OUTBOUND_FULL_RELAY, /*use_v2transport=*/true)}; + BOOST_REQUIRE(node); + return node->m_transport->GetInfo().transport_pq_status; + }}; + BOOST_CHECK(new_status() == PQStatus::PENDING); + // A session that is running when the endpoint enters the fallback set. + Link running{Add(ConnectionType::OUTBOUND_FULL_RELAY, PQMode::NEGOTIATE, endpoint)}; + + for (const std::chrono::seconds window : {3600s, 14400s, 86400s, 86400s}) { + Connect(endpoint, End::MALFORMED); + Connect(endpoint, End::MALFORMED); + BOOST_REQUIRE(Streak(endpoint)); + BOOST_CHECK(Streak(endpoint)->next_window == window); + const NodeSeconds start{Now()}; + const uint64_t fallbacks{Stats().outbound.fallback}; + Connect(endpoint, End::MALFORMED); + BOOST_CHECK_EQUAL(Stats().outbound.fallback, fallbacks + 1); + BOOST_REQUIRE(Fallback(endpoint)); + BOOST_CHECK(Fallback(endpoint)->expires == start + window); + BOOST_CHECK(new_status() == PQStatus::FALLBACK); + + // A failure during the window neither extends it nor enters again. + SetMockTime(TicksSinceEpoch(start + window - 1s)); + Connect(endpoint, End::MALFORMED); + BOOST_CHECK(m_connman.IsPQFallback(endpoint, Now())); + BOOST_REQUIRE(Fallback(endpoint)); + BOOST_CHECK(Fallback(endpoint)->expires == start + window); + BOOST_CHECK_EQUAL(Stats().outbound.fallback, fallbacks + 1); + + // At expiry the window ends, and the streak restarts at 0. + SetMockTime(TicksSinceEpoch(start + window)); + BOOST_CHECK(!Fallback(endpoint)); + BOOST_CHECK(!m_connman.IsPQFallback(endpoint, Now())); + BOOST_CHECK(!Streak(endpoint)); + BOOST_CHECK(new_status() == PQStatus::PENDING); + } + // Existing sessions never renegotiate. + BOOST_CHECK(running.node->m_transport->GetInfo().transport_pq_status == PQStatus::PENDING); +} + +BOOST_AUTO_TEST_CASE(pq_success_protection) +{ + const PQEndpointKey endpoint{TestEndpoint(4, 0)}; + // A connection that will fail late: we switch now, and the peer's bad confirmation waits. + Link late{Add(ConnectionType::OUTBOUND_FULL_RELAY, PQMode::NEGOTIATE, endpoint)}; + V2Transport bad_peer{Responder({.mode = PQMode::NEGOTIATE, .corrupt_shared_secret = true})}; + Pass(late); + BOOST_REQUIRE(NodeToPeer(late, bad_peer)); + PeerToNode(late, bad_peer); + Pass(late); + BOOST_REQUIRE(late.node->m_transport->GetPQSnapshot().switched); + BOOST_CHECK(!NodeToPeer(late, bad_peer)); // the peer switched to other keys, and fails ours + const std::vector late_confirmation{PeerBytes(bad_peer)}; + + // An earlier failure, then a success: the history goes, and the endpoint is protected. + Connect(endpoint, End::MALFORMED); + BOOST_CHECK(Streak(endpoint)); + Connect(endpoint, End::SUCCESS); + BOOST_CHECK(!Streak(endpoint)); + + // The late failure arrives after the success, and can't undo it. + Receive(late, late_confirmation); + Pass(late); + BOOST_REQUIRE(late.node->m_transport->GetPQSnapshot().failure != PQFailure::NONE); + Disconnect(); + BOOST_CHECK(!Streak(endpoint)); + + // More than PQ_MAX_ENDPOINT_HISTORY other endpoints churn the history. + for (int i{1}; i <= int{PQ_MAX_ENDPOINT_HISTORY} + 1; ++i) Connect(TestEndpoint(4, i), End::MALFORMED); + BOOST_CHECK_EQUAL(Stats().failure_streaks.size(), PQ_MAX_ENDPOINT_HISTORY); + + // The protected endpoint never falls back; its failures are still counted. + for (const End end : {End::MALFORMED, End::CONFIRMATION, End::PEER_EOF, End::MALFORMED}) Connect(endpoint, end); + BOOST_CHECK(!m_connman.IsPQFallback(endpoint, Now())); + BOOST_CHECK(!Streak(endpoint)); + BOOST_CHECK(!Fallback(endpoint)); + const PQTransportStats stats{Stats()}; + BOOST_CHECK_EQUAL(stats.outbound.fallback, 0U); + BOOST_CHECK_EQUAL(stats.outbound.first_packet_failed, 2U); + BOOST_CHECK_EQUAL(stats.outbound.closed_after_switch, 1U); +} + +BOOST_AUTO_TEST_CASE(pq_history_eviction) +{ + // 1,001 endpoints fail once each, all at the same time, inserted in descending key order so + // that insertion order and map order disagree. + const int newest{int{PQ_MAX_ENDPOINT_HISTORY}}; + for (int i{newest}; i >= 0; --i) Connect(TestEndpoint(6, i), End::MALFORMED); + PQTransportStats stats{Stats()}; + BOOST_CHECK_EQUAL(stats.failure_streaks.size(), PQ_MAX_ENDPOINT_HISTORY); + // The first insertion, which has the largest key, is the one evicted. + BOOST_CHECK(!Streak(TestEndpoint(6, newest))); + BOOST_CHECK(Streak(TestEndpoint(6, 0))); + BOOST_CHECK(Streak(TestEndpoint(6, 1))); + BOOST_CHECK(Streak(TestEndpoint(6, newest - 1))); + // The rings keep their own bound and counts. + BOOST_CHECK_EQUAL(stats.outbound.malformed_record, PQ_MAX_ENDPOINT_HISTORY + 1); + BOOST_CHECK_EQUAL(stats.outbound_failures.last_sequence, PQ_MAX_ENDPOINT_HISTORY + 1); + BOOST_CHECK_EQUAL(stats.outbound_failures.entries.size(), PQ_FAILURE_RING_SIZE); + BOOST_CHECK_EQUAL(stats.outbound_failures.dropped, PQ_MAX_ENDPOINT_HISTORY + 1 - PQ_FAILURE_RING_SIZE); + + // Eviction follows insertion, not the latest failure or the key: the oldest remaining + // insertion fails again, later than every other, and is still the next to go. + SetMockTime(GetTime() + 1s); + Connect(TestEndpoint(6, newest - 1), End::MALFORMED); + BOOST_REQUIRE(Streak(TestEndpoint(6, newest - 1))); + BOOST_CHECK_EQUAL(Streak(TestEndpoint(6, newest - 1))->streak, 2U); + Connect(TestEndpoint(6, newest + 1), End::MALFORMED); + BOOST_CHECK(!Streak(TestEndpoint(6, newest - 1))); + BOOST_CHECK(Streak(TestEndpoint(6, newest - 2))); + BOOST_CHECK(Streak(TestEndpoint(6, 0))); + BOOST_CHECK(Streak(TestEndpoint(6, newest + 1))); + BOOST_CHECK_EQUAL(Stats().failure_streaks.size(), PQ_MAX_ENDPOINT_HISTORY); +} + +BOOST_AUTO_TEST_CASE(pq_history_eviction_spares_fallback) +{ + // The oldest insertion is in a fallback window; every later endpoint has one failure. + const PQEndpointKey in_window{TestEndpoint(8, 0)}; + for (uint32_t i{0}; i < PQ_FAILURE_THRESHOLD; ++i) Connect(in_window, End::MALFORMED); + BOOST_REQUIRE(Fallback(in_window)); + for (int i{1}; i < int{PQ_MAX_ENDPOINT_HISTORY}; ++i) Connect(TestEndpoint(8, i), End::MALFORMED); + // A new endpoint evicts the oldest insertion outside a window, and the window stays. + Connect(TestEndpoint(8, int{PQ_MAX_ENDPOINT_HISTORY}), End::MALFORMED); + BOOST_CHECK(m_connman.IsPQFallback(in_window, Now())); + BOOST_CHECK(Fallback(in_window)); + BOOST_CHECK(!Streak(TestEndpoint(8, 1))); + BOOST_CHECK(Streak(TestEndpoint(8, 2))); + BOOST_CHECK(Streak(TestEndpoint(8, int{PQ_MAX_ENDPOINT_HISTORY}))); +} + +BOOST_AUTO_TEST_CASE(pq_history_eviction_all_in_fallback) +{ + // The oldest insertion fails once now and enters its window last, so it ends last. + const PQEndpointKey oldest{TestEndpoint(9, 0)}; + Connect(oldest, End::MALFORMED); + for (int i{1}; i < int{PQ_MAX_ENDPOINT_HISTORY}; ++i) { + for (uint32_t n{0}; n < PQ_FAILURE_THRESHOLD; ++n) Connect(TestEndpoint(9, i), End::MALFORMED); + } + SetMockTime(GetTime() + 600s); + for (uint32_t n{1}; n < PQ_FAILURE_THRESHOLD; ++n) Connect(oldest, End::MALFORMED); + BOOST_REQUIRE(Fallback(oldest)); + BOOST_REQUIRE_EQUAL(Stats().fallback_set.size(), PQ_MAX_ENDPOINT_HISTORY); + // With every endpoint in a window, the window that ends first goes; of those that end + // together, the oldest insertion. + Connect(TestEndpoint(9, int{PQ_MAX_ENDPOINT_HISTORY}), End::MALFORMED); + BOOST_CHECK(Fallback(oldest)); + BOOST_CHECK(!Fallback(TestEndpoint(9, 1))); + BOOST_CHECK(!m_connman.IsPQFallback(TestEndpoint(9, 1), Now())); + BOOST_CHECK(Fallback(TestEndpoint(9, 2))); + BOOST_CHECK(Streak(TestEndpoint(9, int{PQ_MAX_ENDPOINT_HISTORY}))); +} + +namespace { +/** The warning names the backends that ran, and no remedy: no option forces portable code yet. */ +std::string LocalFaultLine() +{ + const auto backends{mlkem::GetBackendNames()}; + return strprintf("v2 pq: local_fault arith=%s keccak=%s: hybrid handshakes failed with 8 distinct endpoints and none succeeded " + "since startup, so this node's own ML-KEM code may be at fault.", + backends.arith, backends.keccak); +} +} // namespace + +BOOST_AUTO_TEST_CASE(pq_local_fault_warning) +{ + // Whatever backends this machine runs. + LogLineCounter warnings{"v2 pq: local_fault"}; + for (int i{0}; i < 7; ++i) Connect(TestEndpoint(7, i), End::MALFORMED); + BOOST_CHECK_EQUAL(warnings.m_count, 0); + { + const std::string text{LocalFaultLine()}; + DebugLogHelper expect{text, AtLevel({text}, LineLevel::WARNING)}; + Connect(TestEndpoint(7, 7), End::MALFORMED); + } + BOOST_CHECK_EQUAL(warnings.m_count, 1); + BOOST_CHECK_EQUAL(m_connman.PQLocalFaultEndpoints(), PQ_LOCAL_FAULT_THRESHOLD); + // It warns once, and behavior doesn't change. + for (int i{0}; i < 12; ++i) Connect(TestEndpoint(7, i), End::MALFORMED); + BOOST_CHECK_EQUAL(warnings.m_count, 1); + BOOST_CHECK(m_connman.IsPQFallback(TestEndpoint(7, 0), Now()) == false); + BOOST_REQUIRE(Streak(TestEndpoint(7, 0))); + BOOST_CHECK_EQUAL(Streak(TestEndpoint(7, 0))->streak, 2U); + // The set of failing endpoints stops growing at the threshold: it is never evicted, so the + // cap is its only bound. + BOOST_CHECK_EQUAL(m_connman.PQLocalFaultEndpoints(), PQ_LOCAL_FAULT_THRESHOLD); +} + +BOOST_AUTO_TEST_CASE(pq_local_fault_warning_portable) +{ + mlkem::ForcePortableForTesting portable; + BOOST_CHECK_EQUAL(mlkem::GetBackendNames().arith, "portable"); + const std::string text{LocalFaultLine()}; + BOOST_CHECK(text.find("arith=portable keccak=portable:") != std::string::npos); + LogLineCounter warnings{"v2 pq: local_fault"}; + { + DebugLogHelper expect{text, AtLevel({text}, LineLevel::WARNING)}; + for (int i{0}; i < 8; ++i) Connect(TestEndpoint(7, i), End::MALFORMED); + } + BOOST_CHECK_EQUAL(warnings.m_count, 1); +} + +BOOST_AUTO_TEST_CASE(pq_local_fault_after_success) +{ + // One success since startup rules out a local fault. + LogLineCounter warnings{"v2 pq: local_fault"}; + Connect(TestEndpoint(8, 0), End::SUCCESS); + for (int i{1}; i <= 12; ++i) Connect(TestEndpoint(8, i), End::MALFORMED); + BOOST_CHECK_EQUAL(warnings.m_count, 0); +} + +BOOST_AUTO_TEST_CASE(pq_inbound_success_does_not_protect) +{ + // Only outbound successes enter the success set, which is never evicted. An inbound peer's + // key is its address and source port, so letting inbound successes in would let a peer that + // reconnects grow the set without bound, and silence the local-fault warning. + const CService addr{LookupNumeric("10.7.0.1", 8333)}; + Link inbound{Add(ConnectionType::INBOUND, PQMode::NEGOTIATE, std::nullopt, addr)}; + V2Transport peer{Initiator()}; + Exchange(inbound, peer); + BOOST_REQUIRE(inbound.node->m_transport->GetPQSnapshot().confirmed); + inbound.node->RequestDisconnect(); + Disconnect(); + LogLineCounter warnings{"v2 pq: local_fault"}; + // An outbound failure to the same address and port still starts a streak... + Connect(addr, End::MALFORMED); + BOOST_CHECK(Streak(addr)); + // ...and with no outbound success, eight failing endpoints still warn. + for (int i{1}; i < 8; ++i) Connect(TestEndpoint(9, i), End::MALFORMED); + BOOST_CHECK_EQUAL(warnings.m_count, 1); +} + +BOOST_AUTO_TEST_CASE(pq_fallback_connection) +{ + // A fallback connection to a peer that sends a malformed offer: plain v2, with the offer + // neither parsed nor validated. + Link link{Add(ConnectionType::OUTBOUND_FULL_RELAY, PQMode::FALLBACK)}; + Pass(link); + RawPeer raw{/*initiator=*/false}; + raw.Initialize(NodeKey(link)); + Receive(link, raw.Key()); + Receive(link, raw.TerminatorAndVersion(MakeRecord(PQ_MLKEM1024, 100))); + Pass(link); + BOOST_CHECK(!link.node->fDisconnect); + Transport::Info info{link.node->m_transport->GetInfo()}; + BOOST_CHECK(info.transport_type == TransportProtocolType::V2); + BOOST_CHECK(info.transport_pq_status == PQStatus::FALLBACK); + BOOST_CHECK(!info.transport_pq); + + // Toward an upgraded responder, which offers: our version stays empty, and the peer sees a + // legacy initiator. + Link upgraded{Add(ConnectionType::OUTBOUND_FULL_RELAY, PQMode::FALLBACK)}; + V2Transport peer{Responder()}; + Exchange(upgraded, peer); + BOOST_CHECK(!upgraded.node->fDisconnect); + info = upgraded.node->m_transport->GetInfo(); + BOOST_CHECK(info.transport_type == TransportProtocolType::V2); + BOOST_CHECK(info.transport_pq_status == PQStatus::FALLBACK); + BOOST_CHECK(peer.GetInfo().transport_pq_status == PQStatus::LEGACY_PEER); + + // Nothing is counted for either. + link.node->RequestDisconnect(); + upgraded.node->RequestDisconnect(); + Disconnect(); + const PQTransportStats stats{Stats()}; + BOOST_CHECK_EQUAL(stats.outbound.switched + stats.outbound.legacy_peer + stats.outbound.malformed_record, 0U); + BOOST_CHECK(stats.outbound_failures.entries.empty()); +} + +BOOST_AUTO_TEST_CASE(pq_load_shedding) +{ + BOOST_CHECK_EQUAL(Stats().load_shedding.threshold_per_s, DEFAULT_PQ_SHED_THRESHOLD_PER_S); + // A low threshold, as unit tests and the flood lab set it. + InitConnman(/*shed_threshold=*/10); + BOOST_CHECK_EQUAL(Stats().load_shedding.threshold_per_s, 10U); + LogLineCounter started{"v2 pq: load_shedding started"}; + LogLineCounter stopped{"v2 pq: load_shedding stopped"}; + + // One attempted offer: an inbound connection whose 64-byte key arrived. + const auto attempt{[&] { + Link link{Accept()}; + Receive(link, InitiatorKey()); + Pass(link); + return link.node->m_transport->GetPQSnapshot().offer == PQOfferState::SENT; + }}; + // The steady clock drives the seconds; the wall clock only dates the start. + const NodeSeconds start{Now()}; + const auto at{[&](std::chrono::seconds offset) { + SetMockTime(TicksSinceEpoch(start + offset)); + MockableSteadyClock::SetMockTime(STEADY_START + offset); + }}; + + // Up to the threshold in one second, responders offer. + for (int i{0}; i < 10; ++i) BOOST_CHECK(attempt()); + BOOST_CHECK(!Stats().load_shedding.active); + BOOST_CHECK_EQUAL(started.m_count, 0); + + // More than the threshold within the second starts shedding, with one line. + { + const std::string text{"v2 pq: load_shedding started threshold_per_s=10"}; + DebugLogHelper expect{text, AtLevel({text}, LineLevel::INFO)}; + BOOST_CHECK(!attempt()); + } + // A shed responder sends empty contents, which is exactly plain v2: the peer sees a legacy + // responder, and the connection's status is off. + Link shed{Accept()}; + V2Transport peer{Initiator()}; + Exchange(shed, peer); + BOOST_CHECK(!shed.node->fDisconnect); + BOOST_CHECK(shed.node->m_transport->GetInfo().transport_type == TransportProtocolType::V2); + BOOST_CHECK(shed.node->m_transport->GetInfo().transport_pq_status == PQStatus::OFF); + BOOST_CHECK(peer.GetInfo().transport_pq_status == PQStatus::LEGACY_PEER); + BOOST_CHECK(shed.node->m_transport->GetPQSnapshot().legacy == PQLegacyReason::NONE); + PQTransportStats stats{Stats()}; + BOOST_CHECK(stats.load_shedding.active); + BOOST_CHECK(stats.load_shedding.since == start); + BOOST_CHECK_EQUAL(stats.inbound.shed, 2U); + BOOST_CHECK_EQUAL(started.m_count, 1); + + // Outbound connections are never affected. + Link outbound{Add(ConnectionType::OUTBOUND_FULL_RELAY)}; + V2Transport responder{Responder()}; + Exchange(outbound, responder); + BOOST_CHECK(outbound.node->m_transport->GetInfo().transport_pq_status == PQStatus::HYBRID); + + // Nine quiet seconds (fewer than half the threshold), then a second at half: not quiet, so + // shedding goes on and the run starts over. + for (int second{1}; second <= 10; ++second) { + at(std::chrono::seconds{second}); + for (int i{0}; i < (second == 10 ? 5 : 4); ++i) BOOST_CHECK(!attempt()); + } + // Ten quiet seconds: offers resume in the eleventh, with one line. + for (int second{11}; second <= 20; ++second) { + at(std::chrono::seconds{second}); + for (int i{0}; i < 4; ++i) BOOST_CHECK(!attempt()); + } + BOOST_CHECK(Stats().load_shedding.active); + at(21s); + { + const std::string text{"v2 pq: load_shedding stopped"}; + DebugLogHelper expect{text, AtLevel({text}, LineLevel::INFO)}; + BOOST_CHECK(attempt()); + } + stats = Stats(); + BOOST_CHECK(!stats.load_shedding.active); + BOOST_CHECK(stats.load_shedding.since == NodeSeconds{}); + BOOST_CHECK_EQUAL(stats.inbound.shed, 2U + 9 * 4 + 5 + 10 * 4); + // Shed offers add no ring entries: a flood can't wash out the evidence. + BOOST_CHECK(stats.inbound_failures.entries.empty()); + + // A second flood, which then stops entirely: the socket handler ends shedding on time even + // though no offer is attempted. + for (int i{0}; i < 11; ++i) attempt(); + BOOST_CHECK(Stats().load_shedding.active); + BOOST_CHECK(Stats().load_shedding.since == start + 21s); + at(31s); + m_connman.UpdatePQSheddingPublic(); + BOOST_CHECK(Stats().load_shedding.active); + at(32s); + m_connman.UpdatePQSheddingPublic(); + m_connman.UpdatePQSheddingPublic(); + BOOST_CHECK(!Stats().load_shedding.active); + BOOST_CHECK_EQUAL(started.m_count, 2); + BOOST_CHECK_EQUAL(stopped.m_count, 2); +} + +BOOST_AUTO_TEST_CASE(pq_load_shedding_off) +{ + // With the switch off, inbound transports never ask the gate: nothing is counted or shed. + InitConnman(/*shed_threshold=*/1); + m_connman.SetPQMode(PQMode::OFF); + for (int i{0}; i < 5; ++i) { + Link link{Accept()}; + Receive(link, InitiatorKey()); + Pass(link); + BOOST_CHECK(link.node->m_transport->GetInfo().transport_pq_status == PQStatus::OFF); + } + BOOST_CHECK(!Stats().load_shedding.active); + BOOST_CHECK_EQUAL(Stats().inbound.shed, 0U); +} + +BOOST_AUTO_TEST_CASE(pq_shed_wipes_secrets) +{ + // A shed responder runs plain v2 for the rest of the connection, so it keeps no hybrid + // secret: the retained ECDH secret and transcript go at once, not at finalization. + InitConnman(/*shed_threshold=*/1); + Link offered{Accept()}; + Receive(offered, InitiatorKey()); + Pass(offered); + BOOST_REQUIRE(offered.node->m_transport->GetPQSnapshot().offer == PQOfferState::SENT); + BOOST_CHECK(dynamic_cast(*offered.node->m_transport).HoldsHybridSecretsForTesting()); + Link shed{Accept(LookupNumeric("10.9.0.2", 50001))}; + Receive(shed, InitiatorKey()); + Pass(shed); + BOOST_REQUIRE(Stats().load_shedding.active); + BOOST_REQUIRE(shed.node->m_transport->GetPQSnapshot().offer == PQOfferState::NONE); + BOOST_REQUIRE(!shed.node->fDisconnect); + // Check before the initiator's version arrives: processing it wipes whatever is still held, + // which would hide a shed branch that didn't wipe. + BOOST_REQUIRE(!shed.node->m_transport->GetPQSnapshot().version_received); + BOOST_CHECK(!dynamic_cast(*shed.node->m_transport).HoldsHybridSecretsForTesting()); +} + +BOOST_AUTO_TEST_CASE(pq_load_shedding_wall_clock_step) +{ + // Offers trickle in, one per second, far below the threshold, while the wall clock steps back + // an hour (an NTP correction or a VM restore). The steady clock still tells the seconds apart, + // so nothing is shed. + InitConnman(/*shed_threshold=*/5); + const auto attempt{[&] { + Link link{Accept()}; + Receive(link, InitiatorKey()); + Pass(link); + const bool offered{link.node->m_transport->GetPQSnapshot().offer == PQOfferState::SENT}; + link.node->RequestDisconnect(); + Disconnect(); + return offered; + }}; + const NodeSeconds wall{Now()}; + BOOST_CHECK(attempt()); + for (int i{1}; i <= 120; ++i) { + SetMockTime(TicksSinceEpoch(wall - 3600s + std::chrono::seconds{i})); + MockableSteadyClock::SetMockTime(STEADY_START + std::chrono::seconds{i}); + m_connman.UpdatePQSheddingPublic(); + BOOST_CHECK(attempt()); + } + BOOST_CHECK(!Stats().load_shedding.active); + BOOST_CHECK_EQUAL(Stats().inbound.shed, 0U); + + // A flood within one steady second starts shedding, and a wall clock step forward doesn't + // end it early: only ten quiet steady seconds do. + MockableSteadyClock::SetMockTime(STEADY_START + 200s); + for (int i{0}; i < 6; ++i) attempt(); + BOOST_CHECK(Stats().load_shedding.active); + SetMockTime(TicksSinceEpoch(wall + 3600s)); + m_connman.UpdatePQSheddingPublic(); + BOOST_CHECK(!attempt()); + BOOST_CHECK(Stats().load_shedding.active); + MockableSteadyClock::SetMockTime(STEADY_START + 211s); + m_connman.UpdatePQSheddingPublic(); + BOOST_CHECK(!Stats().load_shedding.active); + BOOST_CHECK(attempt()); +} + +BOOST_AUTO_TEST_CASE(pq_load_shedding_socket_thread) +{ + // The socket handler thread ends shedding on time when the flood stops entirely, without a + // further offer: the real thread function runs, as Start() runs it. + InitConnman(/*shed_threshold=*/2); + LogLineCounter stopped{"v2 pq: load_shedding stopped"}; + for (int i{0}; i < 3; ++i) { + Link link{Accept()}; + Receive(link, InitiatorKey()); + Pass(link); + link.node->RequestDisconnect(); + } + Disconnect(); // so the thread has no sockets to serve + BOOST_REQUIRE(Stats().load_shedding.active); + MockableSteadyClock::SetMockTime(STEADY_START + PQ_SHED_QUIET_PERIOD + 1s); + m_connman.StartSocketHandlerThread(); + const auto deadline{SteadyClock::now() + 10s}; + while (Stats().load_shedding.active && SteadyClock::now() < deadline) { + std::this_thread::sleep_for(10ms); + } + m_connman.StopSocketHandlerThread(); + BOOST_CHECK(!Stats().load_shedding.active); + BOOST_CHECK_EQUAL(stopped.m_count, 1); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 5f0c37ea2a..ebcb9adfb0 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -34,11 +35,16 @@ #include #include +#include +#include #include #include +#include +#include #include #include #include +#include #include using namespace std::literals; @@ -179,6 +185,221 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) BOOST_CHECK_EQUAL(pnode4->ConnectedThroughNetwork(), Network::NET_ONION); } +namespace { +//! A socket whose Send() and Recv() fail as if the peer reset the connection. +class ResetSock : public ZeroSock +{ +public: + ssize_t Send(const void*, size_t, int) const override { return Fail(); } + ssize_t Recv(void*, size_t, int) const override { return Fail(); } + +private: + static ssize_t Fail() + { +#ifdef WIN32 + WSASetLastError(WSAECONNRESET); +#else + errno = ECONNRESET; +#endif + return -1; + } + + ResetSock& operator=(Sock&&) override + { + assert(false && "Move of Sock into ResetSock not allowed."); + return *this; + } +}; + +//! Message processing that only records each node's close cause when CConnman deletes it. +class CloseCauseRecorder final : public NetEventsInterface +{ +public: + std::map m_finalized; + + void InitializeNode(const CNode&, ServiceFlags) override {} + void FinalizeNode(const CNode& node) override { m_finalized.emplace(node.GetId(), node.GetCloseCause()); } + bool HasAllDesirableServiceFlags(ServiceFlags) const override { return true; } + bool HasUndesirableServiceFlags(ServiceFlags) const override { return false; } + bool ProcessMessages(CNode*, std::atomic&) override EXCLUSIVE_LOCKS_REQUIRED(g_msgproc_mutex) { return false; } + bool SendMessages(CNode*) override EXCLUSIVE_LOCKS_REQUIRED(g_msgproc_mutex) { return false; } +}; +} // namespace + +BOOST_AUTO_TEST_CASE(close_cause_first_wins) +{ + constexpr std::array causes{NodeCloseCause::PEER_EOF, NodeCloseCause::PEER_RESET, NodeCloseCause::SEND_ERROR, + NodeCloseCause::TIMEOUT, NodeCloseCause::LOCAL}; + NodeId id{0}; + const auto new_node{[&](std::shared_ptr sock, ConnectionType conn_type = ConnectionType::OUTBOUND_FULL_RELAY) { + return new CNode{id++, + std::move(sock), + CAddress{LookupNumeric("1.2.3.4", 8333), NODE_NONE}, + /*nKeyedNetGroupIn=*/0, + /*nLocalHostNonceIn=*/0, + CAddress{}, + /*addrNameIn=*/"", + conn_type, + /*inbound_onion=*/false}; + }}; + + // Each cause is recorded, and repeated cleanup keeps it. + for (const auto cause : causes) { + const std::unique_ptr node{new_node(nullptr)}; + BOOST_CHECK(node->GetCloseCause() == NodeCloseCause::NONE); + BOOST_CHECK(!node->fDisconnect); + node->RequestDisconnect(cause); + BOOST_CHECK(node->GetCloseCause() == cause); + BOOST_CHECK(node->fDisconnect); + for (const auto later : causes) { + node->RequestDisconnect(later); + node->CloseSocketDisconnect(later); + } + node->CloseSocketDisconnect(); + BOOST_CHECK(node->GetCloseCause() == cause); + } + + // Racing writers, like the socket, message handler and RPC threads, all + // see the same cause once their own call returns. + for (int round{0}; round < 50; ++round) { + const std::unique_ptr node{new_node(nullptr)}; + std::atomic start{false}; + std::array seen{}; + std::vector threads; + for (size_t i{0}; i < causes.size(); ++i) { + threads.emplace_back([&, i] { + while (!start) std::this_thread::yield(); + node->CloseSocketDisconnect(causes[i]); + node->RequestDisconnect(); + seen[i] = node->GetCloseCause(); + }); + } + start = true; + for (auto& thread : threads) thread.join(); + const NodeCloseCause winner{node->GetCloseCause()}; + BOOST_CHECK(winner != NodeCloseCause::NONE); + for (const auto cause : seen) BOOST_CHECK(cause == winner); + } + + // Each disconnect site records its cause. + CloseCauseRecorder events; + ConnmanTestMsg connman{0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman, Params()}; + connman.SetMsgProc(&events); + connman.SetPeerConnectTimeout(60s); + const auto now{GetTime()}; + SetMockTime(now); + + const std::shared_ptr eof_sock{std::make_shared("")}; + const std::shared_ptr reset_sock{std::make_shared()}; + CNode* eof_node{new_node(eof_sock)}; + CNode* reset_node{new_node(reset_sock)}; + CNode* send_node{new_node(std::make_shared())}; + CNode* timeout_node{new_node(std::make_shared())}; + CNode* local_node{new_node(std::make_shared())}; + CNode* unexplained_node{new_node(std::make_shared())}; + // Zeros never start with the network magic, so the transport rejects them. + const std::shared_ptr garbage_sock{std::make_shared()}; + CNode* garbage_node{new_node(garbage_sock)}; + CNode* network_off_node{new_node(std::make_shared())}; + const std::vector nodes{eof_node, reset_node, send_node, timeout_node, local_node, unexplained_node, garbage_node, network_off_node}; + for (CNode* node : nodes) { + node->AddRef(); // the reference m_nodes holds, as in ConnectNode() + node->AddRef(); // keeps the node alive through DisconnectNodes() below + connman.AddTestNode(*node); + } + + // Our own decision, as from the RPC thread. + BOOST_CHECK(connman.DisconnectNode(local_node->GetId())); + BOOST_CHECK(local_node->GetCloseCause() == NodeCloseCause::LOCAL); + // A failed optimistic send, as from the message handler thread. + connman.PushMessage(send_node, NetMsg::Make(NetMsgType::VERACK)); + BOOST_CHECK(send_node->GetCloseCause() == NodeCloseCause::SEND_ERROR); + // Setting fDisconnect directly leaves the cause unknown. + unexplained_node->fDisconnect = true; + + // One socket handler pass after the inactivity timeout, so the timeout + // check runs for every node with an open socket after it was read. + SetMockTime(now + 61s); + Sock::EventsPerSock events_per_sock; + for (const auto& sock : {eof_sock, reset_sock, garbage_sock}) { + events_per_sock.emplace(sock, Sock::Events{Sock::RECV}).first->second.occurred = Sock::RECV; + } + connman.SocketHandlerConnectedPublic({eof_node, reset_node, send_node, timeout_node, local_node, garbage_node}, events_per_sock); + BOOST_CHECK(eof_node->GetCloseCause() == NodeCloseCause::PEER_EOF); + BOOST_CHECK(reset_node->GetCloseCause() == NodeCloseCause::PEER_RESET); + BOOST_CHECK(send_node->GetCloseCause() == NodeCloseCause::SEND_ERROR); + BOOST_CHECK(timeout_node->GetCloseCause() == NodeCloseCause::TIMEOUT); + BOOST_CHECK(local_node->GetCloseCause() == NodeCloseCause::LOCAL); + BOOST_CHECK(garbage_node->GetCloseCause() == NodeCloseCause::LOCAL); + BOOST_CHECK(unexplained_node->GetCloseCause() == NodeCloseCause::NONE); + BOOST_CHECK(network_off_node->GetCloseCause() == NodeCloseCause::NONE); + + // With the network off, DisconnectNodes() disconnects the remaining node + // as LOCAL. Finalization counts the unexplained disconnect as LOCAL and + // keeps every recorded cause. + connman.SetNetworkActive(false); + connman.DisconnectNodesPublic(); + connman.SetNetworkActive(true); + BOOST_CHECK(connman.TestNodes().empty()); + BOOST_CHECK(eof_node->GetCloseCause() == NodeCloseCause::PEER_EOF); + BOOST_CHECK(reset_node->GetCloseCause() == NodeCloseCause::PEER_RESET); + BOOST_CHECK(send_node->GetCloseCause() == NodeCloseCause::SEND_ERROR); + BOOST_CHECK(timeout_node->GetCloseCause() == NodeCloseCause::TIMEOUT); + BOOST_CHECK(local_node->GetCloseCause() == NodeCloseCause::LOCAL); + BOOST_CHECK(garbage_node->GetCloseCause() == NodeCloseCause::LOCAL); + BOOST_CHECK(unexplained_node->GetCloseCause() == NodeCloseCause::LOCAL); + BOOST_CHECK(network_off_node->GetCloseCause() == NodeCloseCause::LOCAL); + + for (CNode* node : nodes) node->Release(); + connman.DisconnectNodesPublic(); // deletes the released nodes + + // Eviction records LOCAL. Enough inbound candidates that some survive the + // protections in SelectNodeToEvict(); outbound connections are never evicted. + std::vector inbound_nodes; + for (int i{0}; i < 40; ++i) { + inbound_nodes.push_back(new_node(nullptr, ConnectionType::INBOUND)); + } + CNode* shutdown_node{new_node(std::make_shared())}; + for (CNode* node : inbound_nodes) { + node->AddRef(); + connman.AddTestNode(*node); + } + shutdown_node->AddRef(); + connman.AddTestNode(*shutdown_node); + BOOST_CHECK(connman.AttemptToEvictConnectionPublic()); + std::vector evicted; + std::copy_if(inbound_nodes.begin(), inbound_nodes.end(), std::back_inserter(evicted), [](const CNode* node) { return node->fDisconnect.load(); }); + BOOST_CHECK_EQUAL(evicted.size(), 1U); + for (const CNode* node : evicted) BOOST_CHECK(node->GetCloseCause() == NodeCloseCause::LOCAL); + + // Shutdown deletes the remaining nodes and records LOCAL for any without a cause. + const NodeId shutdown_id{shutdown_node->GetId()}; + connman.Stop(); + BOOST_CHECK(connman.TestNodes().empty()); + BOOST_CHECK(events.m_finalized.at(shutdown_id) == NodeCloseCause::LOCAL); + for (const auto& [finalized_id, cause] : events.m_finalized) BOOST_CHECK(cause != NodeCloseCause::NONE); + + // The ping timeout records TIMEOUT, from the message handler thread. + { + LOCK(NetEventsInterface::g_msgproc_mutex); + auto& fixture_connman{static_cast(*m_node.connman)}; + const std::unique_ptr ping_node{new_node(nullptr, ConnectionType::INBOUND)}; + // The handshake ends by sending the first ping. + fixture_connman.Handshake(*ping_node, + /*successfully_connected=*/true, + /*remote_services=*/ServiceFlags(NODE_NETWORK | NODE_WITNESS), + /*local_services=*/ServiceFlags(NODE_NETWORK | NODE_WITNESS), + /*version=*/PROTOCOL_VERSION, + /*relay_txs=*/true); + BOOST_CHECK(!ping_node->fDisconnect); + SetMockTime(GetTime() + TIMEOUT_INTERVAL + 1s); + m_node.peerman->SendMessages(ping_node.get()); + BOOST_CHECK(ping_node->GetCloseCause() == NodeCloseCause::TIMEOUT); + m_node.peerman->FinalizeNode(*ping_node); + } + SetMockTime(0s); +} + BOOST_AUTO_TEST_CASE(cnetaddr_basic) { CNetAddr addr; diff --git a/src/test/util/net.cpp b/src/test/util/net.cpp index 73ae996b7a..589e0ac02b 100644 --- a/src/test/util/net.cpp +++ b/src/test/util/net.cpp @@ -261,7 +261,12 @@ ssize_t DynSock::Pipe::GetBytes(void* buf, size_t len, int flags) if (m_eof) { return 0; } - errno = EAGAIN; // Same as recv(2) on a non-blocking socket. + // Same as recv(2) on a non-blocking socket, reported where callers read it: + // WSAGetLastError() is errno except on Windows, where it is Winsock's own error. + errno = EAGAIN; +#ifdef WIN32 + WSASetLastError(WSAEWOULDBLOCK); +#endif return -1; } diff --git a/src/test/util/net.h b/src/test/util/net.h index 3af8dfba13..7081638a9b 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -92,7 +93,61 @@ struct ConnmanTestMsg : public CConnman { bool AlreadyConnectedPublic(const CAddress& addr) { return AlreadyConnectedToAddress(addr); }; CNode* ConnectNodePublic(PeerManager& peerman, const char* pszDest, ConnectionType conn_type) - EXCLUSIVE_LOCKS_REQUIRED(!m_unused_i2p_sessions_mutex); + EXCLUSIVE_LOCKS_REQUIRED(!m_unused_i2p_sessions_mutex, !m_pq_mutex); + + void SocketHandlerConnectedPublic(const std::vector& nodes, const Sock::EventsPerSock& events_per_sock) + EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex, !mutexMsgProc, !m_pq_mutex) + { + SocketHandlerConnected(nodes, events_per_sock); + } + + void DisconnectNodesPublic() EXCLUSIVE_LOCKS_REQUIRED(!m_nodes_mutex, !m_reconnections_mutex, !m_pq_mutex) + { + DisconnectNodes(); + } + + void SetPQMode(PQMode mode) { m_pq_mode = mode; } + + /** How many distinct failing endpoints the local-fault warning has counted. */ + size_t PQLocalFaultEndpoints() const EXCLUSIVE_LOCKS_REQUIRED(!m_pq_mutex) + { + return WITH_LOCK(m_pq_mutex, return m_pq_failed_endpoints_for_warning.size()); + } + + void UpdatePQSheddingPublic() EXCLUSIVE_LOCKS_REQUIRED(!m_pq_shed_mutex) { UpdatePQShedding(PQShedNow()); } + + /** Run the socket handler thread, as Start() does. */ + void StartSocketHandlerThread() + { + threadSocketHandler = std::thread(&util::TraceThread, "net", [this] { ThreadSocketHandler(); }); + } + + /** Stop the socket handler thread, and let the connection manager run again. */ + void StopSocketHandlerThread() + { + interruptNet(); + if (threadSocketHandler.joinable()) threadSocketHandler.join(); + interruptNet.reset(); + } + + void ObservePQPublic(CNode& node, bool finalizing) EXCLUSIVE_LOCKS_REQUIRED(!m_pq_mutex) + { + ObservePQ(node, finalizing); + } + + /** ConnectNode() without adding the node to the connection manager; the caller owns it. */ + CNode* ConnectNodeOnly(const char* dest, ConnectionType conn_type, bool use_v2transport) + EXCLUSIVE_LOCKS_REQUIRED(!m_unused_i2p_sessions_mutex, !m_pq_mutex) + { + return ConnectNode(CAddress{}, dest, /*fCountFailure=*/false, conn_type, use_v2transport); + } + + void CreateNodeFromAcceptedSocketPublic(std::unique_ptr&& sock, const CService& addr_bind, const CService& addr) + { + CreateNodeFromAcceptedSocket(std::move(sock), NetPermissionFlags::None, addr_bind, addr); + } + + bool AttemptToEvictConnectionPublic() { return AttemptToEvictConnection(); } }; constexpr ServiceFlags ALL_SERVICE_FLAGS[]{ From 9e2e884dc2da3c9da4b4a4c6679cc61ea9cb9f13 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Thu, 22 Oct 2026 12:52:15 -0400 Subject: [PATCH 11/24] rpc, init: -v2pqtransport, getpqtransportinfo and hybrid peer fields (#184 part 6) (#236) * init: add -v2pqtransport and hybrid transport test options Make the hybrid post-quantum v2 transport configurable, off by default until the default-on change in part 9, through one configuration that the startup warnings, the startup line, every new connection and RPC share. - PQTransportConfig {v2_enabled, pq_requested, fail_first_packet} is computed once in AppInitParameterInteraction() from the effective -v2transport, -v2pqtransport and -test settings. NODE_P2P_V2 now comes from the same value. CConnman::Options::m_pq carries it, and CConnman::Init() derives m_pq_mode from it: NEGOTIATE only with both switches on. This is the only way the mode is set; the unit-test hook ConnmanTestMsg::SetPQMode is removed and net_pq_tests passes a configuration instead. - -v2pqtransport (DEFAULT_V2_PQ_TRANSPORT = false) has no effect with -v2transport=0. An explicit -v2pqtransport=1 then warns: "-v2pqtransport=1 has no effect because -v2transport=0. Set -v2transport=1 to use hybrid post-quantum v2 transport." An explicit 0 or the default never warns. - Booleans are read as integers, so -v2pqtransport=true is 0. A value that doesn't parse completely as an int, under InterpretBool's whitespace and sign rules, warns and names the value it is read as: true, 1abc and an overflowing number. An empty value and a negation are valid. InterpretBool itself is unchanged. - -test=pq_fail_first_packet (regtest) sets corrupt_shared_secret on inbound and outbound transports, so every hybrid handshake fails its key confirmation. -test=mlkem_portable (regtest) and the hidden -mlkemportable (debug category, all chains) force portable ML-KEM code through mlkem::InitializeRuntime(), called before the connection manager exists. - The startup line is always logged and names the active backends: "v2 pq: enabled arith= keccak=", "v2 pq: disabled (-v2pqtransport=0) ..." or "v2 pq: disabled (-v2transport=0) ..." when v2 is off, whatever the PQ switch says. - The local-fault warning names -mlkemportable as the remedy again while native code runs: " Restart with -mlkemportable to use the portable implementation." - MakeTransport() takes the CNodeOptions. rpc_pqtransport.py checks the switch from the command line, qbit.conf, -nov2pqtransport and across restarts on a live two-node connection ("v2 pq: switched"), the exact warnings and their absence, -help and -help-debug, all three startup lines, portable code from both options, the test options' regtest-only check, and that the test options print nothing to stderr. net_pq_tests' pq_config checks the mode of inbound and outbound connections for each configuration and the effect of fail_first_packet; the local-fault tests check the remedy appears only while native code runs. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 19268db84f0a34f1296ddb27600bbf9d13ad8a29) * rpc: expose hybrid transport state and diagnostics Give operators and monitoring scripts everything they need to see the hybrid post-quantum v2 transport, using the RPC contract in #184. - getpqtransportinfo renders the configuration and part 5's PQTransportStats field for field: enabled; status (enabled, disabled_v2pqtransport or disabled_v2transport); instance_id (64 lowercase hex); the active arith and keccak backends; since; per direction handshake counters, inbound shed and outbound closed_after_switch and fallback included; load_shedding (since is 0 when inactive); fallback_set with window_seconds; failure_streaks with next_window_seconds; and both failure rings, oldest first. Every field is always present and empty collections are []. An endpoint is {kind, network, address, port}: an address's own network (127.0.0.1 is ipv4), or name_proxy and the normalized hostname. The help text lists the fixed reason vocabulary and the counter semantics. - getpeerinfo gains transport_pq and transport_pq_status (pending, hybrid, legacy_peer, fallback, off or v1), carried by CNodeStats from Transport::Info. The help documents part 4's rule: there is no failed status, so a connection closing on any hybrid failure reports pending until it is removed. transport_protocol_type keeps its values, and session_id is the hybrid session id once transport_pq is true. - getnetworkinfo gains connections_pq. getpeerinfo's visibility filter (a peer without node state is skipped) moves into GetListedPeers(), and connections_pq counts transport_pq over exactly those peers, before cs_main is taken. - getpqtransportinfo joins the rpc fuzz target's safe list, and doc/rpc/annotations/v30.2-delta.yml describes the three methods. - qbit-cli -netinfo shows 2p in the v column for a hybrid peer, and its legend explains it. A server without transport_pq, such as v1.0.0, counts as not hybrid. - The GUI's peer details read "v2 (hybrid post-quantum)" for a hybrid peer, and the transport type tooltip explains it. rpc_pqtransport.py gains the RPC cases: every getpqtransportinfo field with the switch off, with v2 off, on, and with an endpoint in the fallback set, and a new instance_id per restart; each transport_pq_status on both sides of a connection; exact per-direction counts for hybrid, legacy and failing connections, with -test=pq_fail_first_packet failing handshakes in both directions; an abandoned offer after eof and after a handshake timeout, with no switch; 300 inbound failures leaving 256 entries with contiguous sequences, 44 dropped, and the outbound ring's entry kept; and connections_pq equal to getpeerinfo's transport_pq count, 0 with the switch off. rpc_net.py's exact getpeerinfo dict gains both fields, interface_bitcoin_cli.py checks 2p and 2 with one hybrid and one legacy peer and the legend, and p2p_v2_previous_release.py runs this release's qbit-cli -netinfo against v1.0.0. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit fc3eb7b6af2e0e0cf11c36b2a2aa9939848c59de) * init: warn when a PQ option's setting is read differently than written ArgsManager's boolean parsing stays as it is (it matches Bitcoin Core), but -v2pqtransport and -mlkemportable now warn whenever the text given doesn't say what the setting is read as, naming what it was read as and the effect. - A non-numeric value is read by InterpretBool's integer rules: true, yes and on as 0, 1abc or an overflowing number as 1. It warns with the value given: "-v2pqtransport=true is not a number and was read as -v2pqtransport=0: hybrid transport is OFF. Use -v2pqtransport=1 or -v2pqtransport=0." Surrounding whitespace and a sign are allowed, as before. - A double negative, -nov2pqtransport given 0 or any non-numeric value (on the command line or in qbit.conf), switches the option on. ArgsManager keeps only the result of a negation, a bool, so the text is gone; the warning says what it can: "-nov2pqtransport was given 0 or a non-numeric value, so it was read as -v2pqtransport=1: hybrid transport is ON. Use -v2pqtransport=0 to switch it off." -nov2pqtransport=1abc is read as off, like -nov2pqtransport, and can't be told apart, so it doesn't warn. - The effect is the real one: hybrid transport is ON only with -v2transport on too, and the contradiction warning still follows. - -mlkemportable gets the same two warnings, so -mlkemportable=true, which keeps native code, no longer passes silently. Its effect names the code that runs, -test=mlkem_portable included. The setting is now read once, in AppInitParameterInteraction(), and AppInitMain() passes the stored result to mlkem::InitializeRuntime(). - The -v2pqtransport and -mlkemportable help, and getpqtransportinfo's help, say that such values warn at startup. rpc_pqtransport.py checks the exact warnings and startup states for -v2pqtransport=true, yes, on, 1abc and an overflow, none for " 1", +1 and -1, both warnings with -v2transport=0; -nov2pqtransport=0, true and on (and nov2pqtransport=true in qbit.conf) warning and enabled; no warning for -nov2pqtransport, =1 and =1abc; and, for -mlkemportable, =true warning while the default code stays, =1abc, -nomlkemportable=0 and =true warning with portable code, no warning for -nomlkemportable, =1 and -mlkemportable=0, and the test option's effect. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 8ae2a47c6363643fe02c76851e1449ee74f18c40) * test: cover getpqtransportinfo's rendering paths The review of #236 found nine plausible mutants of the RPC rendering and startup code that every test passed. Each now fails a test. net_pq_tests calls the RPCs (getpqtransportinfo with the fixture's own connection manager): - pq_rpc_status: status names v2 first, so v2 off reads disabled_v2transport whatever the hybrid switch says; enabled follows. - pq_rpc_load_shedding: with a threshold of 1, the second offer in one second is shed: load_shedding is {active: true, threshold_per_s: 1, since: now}, and inbound shed counts it. - pq_rpc_endpoints: IPv4, IPv4-mapped IPv6, IPv6, ::1, onion, I2P and name-proxy endpoints, each with its kind, its own network, its canonical address and its port, in the outbound ring and the failure streaks. - pq_rpc_windows: one endpoint's windows escalate, 3600, 14400, then 86400, in window_seconds (equal to expires - entered) and next_window_seconds, and an expired window is listed nowhere. - pq_rpc_listed_peers: a confirmed hybrid connection the peer manager doesn't know is in getnetworkinfo's connections but not in connections_pq or getpeerinfo; once known, both show it. rpc_pqtransport.py: - a default node runs native ML-KEM-1024 code where the build and CPU have it. The test config gains ENABLE_MLKEM_NATIVE, and the test reads the CPU's flags on Linux x86_64 (the features cpu_features::HasMlkemX86Native() requires), assumes native code on macOS arm64, and expects portable code without native support; it doesn't check where it can't tell; - getpqtransportinfo with -v2transport=0 alone: disabled_v2transport; - a name-proxy destination through a SOCKS5 proxy that resolves it to a failing node, end to end: the ring entry and the streak name {name_proxy, name_proxy, pqtest.example, 8333} for "PQTest.Example.:8333"; - with local IPv6, an IPv6 connection: {address, ipv6, ::1, port} on the outbound side, and ipv6 ::1 on the inbound side. The whitespace case (-v2pqtransport= 1 doesn't warn) is in the previous commit. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 7e0f826ddca8c400f1ded4936bb8aa2ec13147eb) * ci: require the PQ jobs for test/CMakeLists.txt changes test/CMakeLists.txt now passes ENABLE_MLKEM_NATIVE to the functional test config, so the contract test that every file mentioning ML-KEM is classified as PQ-relevant failed. Classify it with the other build wiring. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 9e04cfbf5178738e65107551dd4973e0159fbe73) * rpc: construct getpqtransportinfo's help entries in place clang-tidy's modernize-use-emplace failed the tidy CI job on the three direction-specific help entries. Build them with emplace_back. Co-Authored-By: Claude Opus 5.5 * test: expect the node's line ending between two startup warnings On Windows the node ends each stderr line with CRLF, so the two rpc_pqtransport.py cases that expect two warnings failed on both Windows jobs: they joined the warnings with "\n". Join them with os.linesep, as feature_config_args.py does. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- ci/checks/classify_merge_profile.py | 1 + ci/checks/test_classify_merge_profile.py | 1 + doc/rpc/annotations/v30.2-delta.yml | 25 + src/bitcoin-cli.cpp | 8 +- src/common/args.cpp | 2 + src/init.cpp | 96 ++- src/net.cpp | 36 +- src/net.h | 39 +- src/qt/rpcconsole.cpp | 9 +- src/rpc/net.cpp | 348 +++++++++- src/test/fuzz/rpc.cpp | 1 + src/test/net_pq_tests.cpp | 250 +++++++- src/test/util/net.h | 2 - test/CMakeLists.txt | 1 + test/config.ini.in | 1 + test/functional/interface_bitcoin_cli.py | 40 +- test/functional/p2p_v2_previous_release.py | 22 +- test/functional/rpc_net.py | 5 + test/functional/rpc_pqtransport.py | 708 +++++++++++++++++++++ test/functional/test_runner.py | 1 + 20 files changed, 1549 insertions(+), 47 deletions(-) create mode 100755 test/functional/rpc_pqtransport.py diff --git a/ci/checks/classify_merge_profile.py b/ci/checks/classify_merge_profile.py index f00e503cbc..af6d234960 100755 --- a/ci/checks/classify_merge_profile.py +++ b/ci/checks/classify_merge_profile.py @@ -73,6 +73,7 @@ "src/test/CMakeLists.txt", "src/test/fuzz/CMakeLists.txt", "src/util/CMakeLists.txt", + "test/CMakeLists.txt", # The jobs and the gate: this classifier, the workflows that run it and # the PQ jobs, the scripts and evidence check of those jobs, and the # tests of that wiring. diff --git a/ci/checks/test_classify_merge_profile.py b/ci/checks/test_classify_merge_profile.py index 917059be4f..4fdb90d35c 100755 --- a/ci/checks/test_classify_merge_profile.py +++ b/ci/checks/test_classify_merge_profile.py @@ -273,6 +273,7 @@ def test_pq_unit_jobs_required_for_mlkem_and_transport_paths(self) -> None: "src/test/CMakeLists.txt", "src/test/fuzz/CMakeLists.txt", "src/bench/CMakeLists.txt", + "test/CMakeLists.txt", "cmake/bitcoin-build-config.h.in", # Gate and CI wiring, and its tests. ".github/workflows/ci.yml", diff --git a/doc/rpc/annotations/v30.2-delta.yml b/doc/rpc/annotations/v30.2-delta.yml index 1bc02b5f77..2a93e0963b 100644 --- a/doc/rpc/annotations/v30.2-delta.yml +++ b/doc/rpc/annotations/v30.2-delta.yml @@ -25,6 +25,31 @@ "Lane-specific estimates filter which blocks contribute work while using the same active-chain elapsed-time window as the all-lane estimate; they do not use a lane-local timestamp window." ] }, + "getpqtransportinfo": { + "status_override": "new_since_v30_2", + "delta_summary": "Adds the hybrid post-quantum v2 transport (ML-KEM-1024 and ECDH) diagnostics RPC.", + "delta_notes": [ + "This method does not exist in Bitcoin Core v30.2.", + "Every field is always present, and empty collections are [].", + "Counters count events since startup, not exclusive outcomes; they start over when instance_id changes." + ] + }, + "getpeerinfo": { + "status_override": "semantic_changed_since_v30_2", + "delta_summary": "Adds transport_pq and transport_pq_status, and session_id is the hybrid session id for a hybrid post-quantum peer.", + "delta_notes": [ + "transport_pq is true once the peer's key confirmation verified.", + "transport_pq_status is pending, hybrid, legacy_peer, fallback, off or v1. A connection that closes on a hybrid failure reports pending until it is removed.", + "transport_protocol_type keeps its values: detecting until the handshake completes, then v2." + ] + }, + "getnetworkinfo": { + "status_override": "surface_changed_since_v30_2", + "delta_summary": "Adds connections_pq, the number of hybrid post-quantum connections.", + "delta_notes": [ + "connections_pq counts transport_pq over the same peers getpeerinfo lists." + ] + }, "getmininginfo": { "status_override": "semantic_changed_since_v30_2", "delta_summary": "networkhashps is the default total qbit chainwork hashrate estimate, and next is the permissionless/native candidate.", diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index a03bd5663c..d7020a75a1 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -407,6 +407,7 @@ class NetinfoRequestHandler : public BaseRequestHandler std::string age; std::string services; std::string transport_protocol_type; + bool is_transport_pq; double min_ping; double ping; int64_t addr_processed; @@ -555,10 +556,12 @@ class NetinfoRequestHandler : public BaseRequestHandler const std::string services{FormatServices(services_names)}; const std::string sub_version{peer["subver"].isNull() ? "" : peer["subver"].get_str()}; const std::string transport{peer["transport_protocol_type"].isNull() ? "v1" : peer["transport_protocol_type"].get_str()}; + // A server without hybrid post-quantum transport, such as v1.0.0, has no transport_pq. + const bool is_transport_pq{peer["transport_pq"].isNull() ? false : peer["transport_pq"].get_bool()}; const bool is_addr_relay_enabled{peer["addr_relay_enabled"].isNull() ? false : peer["addr_relay_enabled"].get_bool()}; const bool is_bip152_hb_from{peer["bip152_hb_from"].isNull() ? false : peer["bip152_hb_from"].get_bool()}; const bool is_bip152_hb_to{peer["bip152_hb_to"].isNull() ? false : peer["bip152_hb_to"].get_bool()}; - m_peers.push_back({addr, sub_version, conn_type, NETWORK_SHORT_NAMES[network_id], age, services, transport, min_ping, ping, addr_processed, addr_rate_limited, last_blck, last_recv, last_send, last_trxn, peer_id, mapped_as, version, is_addr_relay_enabled, is_bip152_hb_from, is_bip152_hb_to, is_outbound, is_tx_relay}); + m_peers.push_back({addr, sub_version, conn_type, NETWORK_SHORT_NAMES[network_id], age, services, transport, is_transport_pq, min_ping, ping, addr_processed, addr_rate_limited, last_blck, last_recv, last_send, last_trxn, peer_id, mapped_as, version, is_addr_relay_enabled, is_bip152_hb_from, is_bip152_hb_to, is_outbound, is_tx_relay}); m_max_addr_length = std::max(addr.length() + 1, m_max_addr_length); m_max_addr_processed_length = std::max(ToString(addr_processed).length(), m_max_addr_processed_length); m_max_addr_rate_limited_length = std::max(ToString(addr_rate_limited).length(), m_max_addr_rate_limited_length); @@ -593,7 +596,7 @@ class NetinfoRequestHandler : public BaseRequestHandler peer.network, m_max_services_length, // variable spacing peer.services, - (peer.transport_protocol_type.size() == 2 && peer.transport_protocol_type[0] == 'v') ? peer.transport_protocol_type[1] : ' ', + peer.is_transport_pq ? "2p" : (peer.transport_protocol_type.size() == 2 && peer.transport_protocol_type[0] == 'v') ? peer.transport_protocol_type.substr(1) : "", PingTimeToString(peer.min_ping), PingTimeToString(peer.ping), peer.last_send ? ToString(time_now - peer.last_send) : "", @@ -720,6 +723,7 @@ class NetinfoRequestHandler : public BaseRequestHandler " \"2\" - P2P_V2: peer supports version 2 P2P transport protocol, as defined in BIP 324\n" " \"u\" - UNKNOWN: unrecognized bit flag\n" " v Version of transport protocol used for the connection\n" + " \"2p\" - v2 with hybrid post-quantum session keys (getpeerinfo \"transport_pq\" is true)\n" " mping Minimum observed ping time, in milliseconds (ms)\n" " ping Last observed ping time, in milliseconds (ms)\n" " send Time since last message sent to the peer, in seconds\n" diff --git a/src/common/args.cpp b/src/common/args.cpp index ce0b414cca..dbd5604ec2 100644 --- a/src/common/args.cpp +++ b/src/common/args.cpp @@ -745,6 +745,8 @@ const std::vector TEST_OPTIONS_DOC{ "reindex_after_failure_noninteractive_yes (When asked for a reindex after failure interactively, simulate as-if answered with 'yes')", "bip94 (enforce BIP94 consensus rules)", "disable_witness_pruning (disable witness pruning in regtest)", + "pq_fail_first_packet (corrupt this node's ML-KEM shared secret on every hybrid post-quantum v2 handshake, so the peer's key confirmation fails)", + "mlkem_portable (use the portable ML-KEM-1024 code instead of native code)", }; bool HasTestOption(const ArgsManager& args, const std::string& test_option) diff --git a/src/init.cpp b/src/init.cpp index 297a4f2824..4dba9c2790 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -565,6 +566,7 @@ void SetupServerArgs(ArgsManager& argsman, bool can_listen_ipc) argsman.AddArg("-i2pacceptincoming", strprintf("Whether to accept inbound I2P connections (default: %i). Ignored if -i2psam is not set. Listening for inbound I2P connections is done through the SAM proxy, not by binding to a local address and port.", DEFAULT_I2P_ACCEPT_INCOMING), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-onlynet=", "Make automatic outbound connections only to network (" + Join(GetNetworkNames(), ", ") + "). Inbound and manual connections are not affected by this option. It can be specified multiple times to allow multiple networks.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-v2transport", strprintf("Support v2 transport (default: %u)", DEFAULT_V2_TRANSPORT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-v2pqtransport", strprintf("Support hybrid post-quantum key exchange (ML-KEM-1024) in v2 transport. Has no effect with -v2transport=0. A non-numeric value, or a double negative such as -nov2pqtransport=0, warns at startup and states how it was read (default: %u)", DEFAULT_V2_PQ_TRANSPORT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peerbloomfilters", strprintf("Support filtering of blocks and transaction with bloom filters (default: %u)", DEFAULT_PEERBLOOMFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peerblockfilters", strprintf("Serve compact block filters to peers per BIP 157 (default: %u)", DEFAULT_PEERBLOCKFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-txreconciliation", strprintf("Enable transaction reconciliations per BIP 330 (default: %d)", DEFAULT_TXRECONCILIATION_ENABLE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); @@ -649,6 +651,7 @@ void SetupServerArgs(ArgsManager& argsman, bool can_listen_ipc) argsman.AddArg("-limitdescendantsize=", strprintf("Do not accept transactions if any ancestor would have more than kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT_KVB), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-test=