Hybrid post-quantum v2 transport (#184) — integration

.github/workflows/ci.yml

@@ -68,6 +68,8 @@ jobs:
       profile: ${{ steps.classify.outputs.profile }}
       release_policy_only: ${{ steps.classify.outputs.release_policy_only }}
       source_validation_required: ${{ steps.classify.outputs.source_validation_required }}
+      pq_unit_required: ${{ steps.classify.outputs.pq_unit_required }}
+      pq_functional_required: ${{ steps.classify.outputs.pq_functional_required }}
     steps:
       - name: Checkout
         uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
@@ -93,23 +95,23 @@ jobs:
               echo "::error::Pull request base SHA is unavailable."
               exit 1
             fi
-            git diff --name-only "${BASE_SHA}...HEAD" > "${changed_files}"
+            git diff --name-only --no-renames "${BASE_SHA}...HEAD" > "${changed_files}"
           elif [[ "${EVENT_NAME}" == "merge_group" ]]; then
             base_branch="${BASE_REF#refs/heads/}"
             if [[ -z "${base_branch}" ]]; then
               echo "::error::Merge-group base branch is unavailable."
               exit 1
             fi
             git fetch --no-tags --prune origin "+refs/heads/${base_branch}:refs/remotes/origin/${base_branch}"
-            git diff --name-only "origin/${base_branch}...HEAD" > "${changed_files}"
+            git diff --name-only --no-renames "origin/${base_branch}...HEAD" > "${changed_files}"
           elif [[ "${EVENT_NAME}" == "push" ]]; then
             if [[ -n "${BEFORE_SHA}" && ! "${BEFORE_SHA}" =~ ^0+$ ]]; then
-              git diff --name-only "${BEFORE_SHA}..HEAD" > "${changed_files}"
+              git diff --name-only --no-renames "${BEFORE_SHA}..HEAD" > "${changed_files}"
             else
               git fetch --no-tags --prune origin "${DEFAULT_BRANCH}"
               merge_base="$(git merge-base "origin/${DEFAULT_BRANCH}" HEAD 2>/dev/null || true)"
               if [[ -n "${merge_base}" ]]; then
-                git diff --name-only "${merge_base}...HEAD" > "${changed_files}"
+                git diff --name-only --no-renames "${merge_base}...HEAD" > "${changed_files}"
               else
                 git diff-tree --no-commit-id --name-only -r HEAD > "${changed_files}"
               fi
@@ -973,6 +975,7 @@ jobs:
       - name: Run public docs lint
         run: |
           test/lint/lint-qbit-public-docs.py
+          test/lint/lint-pq-doc-tokens.py
           python3 -m unittest \
             ci.release.test_verify_p2mr_v1_conformance.CheckedInP2MRIntegrationMatrixTest
 
@@ -1068,6 +1071,36 @@ jobs:
           path: |
             build/doc/rpc/**
 
+  # Changes to the ML-KEM library, the v2 transport, or their build and CI
+  # wiring (ci/checks/classify_merge_profile.py) must pass these jobs of the PQ
+  # workflow on the change itself (for a pull request, its merge commit), on
+  # GitHub-hosted runners: AArch64 Linux, and macOS on Apple Silicon and on
+  # Intel, the only jobs that build and run ML-KEM's native code on macOS.
+  pq-unit:
+    name: 'PQ unit tests'
+    needs: classify-changes
+    if: ${{ needs.classify-changes.outputs.pq_unit_required == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.draft != true) }}
+    uses: ./.github/workflows/ci-pq.yml
+    with:
+      source_ref: ${{ github.sha }}
+      jobs: aarch64-unit,macos-arm64-unit,macos-x86_64-unit
+
+  # Changes that can break hybrid post-quantum v2 transport (every change that
+  # pq-unit requires, and the hybrid transport paths of
+  # ci/checks/classify_merge_profile.py that its jobs do not cover) must pass
+  # the whole functional suite with it, the pq-functional job of the PQ
+  # workflow, on the change itself, on a GitHub-hosted runner. No other job
+  # runs the suite hybrid: the ci-matrix jobs run v1 between nodes, and its
+  # i686 job plain v2.
+  pq-functional:
+    name: 'PQ functional tests'
+    needs: classify-changes
+    if: ${{ needs.classify-changes.outputs.pq_functional_required == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.draft != true) }}
+    uses: ./.github/workflows/ci-pq.yml
+    with:
+      source_ref: ${{ github.sha }}
+      jobs: pq-functional
+
   full-validation-gate:
     name: 'Full Validation Gate'
     runs-on: ubuntu-24.04
@@ -1081,6 +1114,8 @@ jobs:
       - lint
       - public-docs-lint
       - rpc-docs
+      - pq-unit
+      - pq-functional
     if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.draft != true) }}
     steps:
       - name: Require trusted label for fork PRs
@@ -1103,6 +1138,10 @@ jobs:
           LINT_RESULT: ${{ needs.lint.result }}
           PUBLIC_DOCS_LINT_RESULT: ${{ needs.public-docs-lint.result }}
           RPC_DOCS_RESULT: ${{ needs.rpc-docs.result }}
+          PQ_UNIT_REQUIRED: ${{ needs.classify-changes.outputs.pq_unit_required }}
+          PQ_UNIT_RESULT: ${{ needs.pq-unit.result }}
+          PQ_FUNCTIONAL_REQUIRED: ${{ needs.classify-changes.outputs.pq_functional_required }}
+          PQ_FUNCTIONAL_RESULT: ${{ needs.pq-functional.result }}
         run: |
           set -euo pipefail
 
@@ -1118,9 +1157,41 @@ jobs:
             fi
           }
 
+          # A job that only some changes need may be skipped only when the
+          # classifier says this change does not need it; a missing or other
+          # classifier output fails.
+          check_required_result() {
+            local name="$1"
+            local output="$2"
+            local required="$3"
+            local result="$4"
+
+            case "${required}" in
+              true)
+                check_result "${name}" "${result}"
+                ;;
+              false)
+                echo "${name}: ${result} (not required for this change)"
+                if [[ "${result}" != "skipped" && "${result}" != "success" ]]; then
+                  echo "::error title=Full Validation failed::${name} result was ${result}."
+                  failed=1
+                fi
+                ;;
+              *)
+                echo "::error title=Full Validation failed::The classifier did not report whether ${name} are required (${output}='${required}')."
+                failed=1
+                ;;
+            esac
+          }
+
           check_result "classify validation profile" "${CLASSIFY_CHANGES_RESULT}"
           echo "CI profile: ${CI_PROFILE:-unavailable}"
 
+          # The PQ unit jobs: AArch64 Linux, macOS arm64 and x86_64.
+          check_required_result "PQ unit tests" pq_unit_required "${PQ_UNIT_REQUIRED}" "${PQ_UNIT_RESULT}"
+          # The whole functional suite with hybrid transport.
+          check_required_result "PQ functional tests" pq_functional_required "${PQ_FUNCTIONAL_REQUIRED}" "${PQ_FUNCTIONAL_RESULT}"
+
           if [[ "${SOURCE_VALIDATION_REQUIRED}" != "true" ]]; then
             echo "Validation profile ${VALIDATION_PROFILE} does not require full source validation."
             exit "${failed}"

.github/workflows/core-checks.yml

@@ -6,6 +6,15 @@ name: Core Checks
 
 on:
   pull_request:
+    # The same activity types as the Required Merge Gate, which waits for
+    # its own event's Core Checks Gate instead of an earlier event's failure.
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - ready_for_review
+      - labeled
+      - unlabeled
     branches:
       - main
       - 1.0.0
@@ -56,6 +65,10 @@ jobs:
         with:
           fetch-depth: 0
 
+      # The same git commands as in ci.yml and required-merge-gate.yml, so that
+      # all three classify a change alike (ci/checks/test_pq_merge_gate_contract.py).
+      # --no-renames lists both paths of a rename: the old one alone may need
+      # the source checks.
       - name: Write changed file list
         id: changed-files
         env:
@@ -75,23 +88,23 @@ jobs:
               echo "::error::Pull request base SHA is unavailable."
               exit 1
             fi
-            git diff --name-only "${BASE_SHA}...HEAD" > "${changed_files}"
+            git diff --name-only --no-renames "${BASE_SHA}...HEAD" > "${changed_files}"
           elif [[ "${EVENT_NAME}" == "merge_group" ]]; then
             base_branch="${BASE_REF#refs/heads/}"
             if [[ -z "${base_branch}" ]]; then
               echo "::error::Merge-group base branch is unavailable."
               exit 1
             fi
             git fetch --no-tags --prune origin "+refs/heads/${base_branch}:refs/remotes/origin/${base_branch}"
-            git diff --name-only "origin/${base_branch}...HEAD" > "${changed_files}"
+            git diff --name-only --no-renames "origin/${base_branch}...HEAD" > "${changed_files}"
           elif [[ "${EVENT_NAME}" == "push" ]]; then
             if [[ -n "${BEFORE_SHA}" && ! "${BEFORE_SHA}" =~ ^0+$ ]]; then
-              git diff --name-only "${BEFORE_SHA}..HEAD" > "${changed_files}"
+              git diff --name-only --no-renames "${BEFORE_SHA}..HEAD" > "${changed_files}"
             else
               git fetch --no-tags --prune origin "${DEFAULT_BRANCH}"
               merge_base="$(git merge-base "origin/${DEFAULT_BRANCH}" HEAD 2>/dev/null || true)"
               if [[ -n "${merge_base}" ]]; then
-                git diff --name-only "${merge_base}...HEAD" > "${changed_files}"
+                git diff --name-only --no-renames "${merge_base}...HEAD" > "${changed_files}"
               else
                 git diff-tree --no-commit-id --name-only -r HEAD > "${changed_files}"
               fi
@@ -131,12 +144,45 @@ jobs:
     needs: classify-changes
     if: ${{ needs.classify-changes.outputs.source_validation_required == 'true' }}
     runs-on: ubuntu-24.04
-    timeout-minutes: 45
+    # Room for a run that misses the package cache below to download its
+    # packages from a slower mirror.
+    timeout-minutes: 90
 
     steps:
       - name: Checkout
         uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
 
+      # Hosted runners fetch packages from an Azure mirror in their own region
+      # first, and apt falls back to the next mirror only on an error, not when
+      # a mirror is slow. One region's mirror has served packages at 30 kB/s,
+      # so this job tries the main archive first and keeps the regional
+      # mirror as the last fallback.
+      - name: Prefer the main Ubuntu archive
+        run: |
+          if [ -f /etc/apt/apt-mirrors.txt ]; then
+            sudo sed -i -E 's#^(http://azure\.archive\.ubuntu\.com/ubuntu/[[:space:]]+priority:)[0-9]+$#\19#' /etc/apt/apt-mirrors.txt
+            cat /etc/apt/apt-mirrors.txt
+          fi
+
+      # The packages the steps below install, so that a slow mirror delays
+      # only a run that misses the cache. apt reuses a cached package of the
+      # expected size without checking its hash, so only push runs save the
+      # cache, as with the ccache caches: a pull request cannot change what
+      # another run installs.
+      - name: Restore apt package cache
+        id: apt-cache
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        with:
+          path: ${{ runner.temp }}/apt-archives
+          key: build-smoke-apt-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/workflows/core-checks.yml') }}
+          restore-keys: build-smoke-apt-${{ runner.os }}-${{ runner.arch }}-
+
+      - name: Keep downloaded packages in the cache directory
+        run: |
+          mkdir -p "${RUNNER_TEMP}/apt-archives/partial"
+          printf 'Dir::Cache::Archives "%s/apt-archives/";\nAPT::Keep-Downloaded-Packages "true";\n' "${RUNNER_TEMP}" \
+            | sudo tee /etc/apt/apt.conf.d/90build-smoke-cache > /dev/null
+
       - name: Install build dependencies
         run: |
           sudo apt-get update
@@ -153,6 +199,38 @@ jobs:
       - name: Test integrated libbitcoinpqc build policy
         run: python3 ci/checks/test_libbitcoinpqc_build_policy.py
 
+      # Every compiler and tool the build-policy test uses, each listed as
+      # required so that none of its cases can skip: the AArch64 cross compiler
+      # proves that AArch64 ELF builds refuse native ML-KEM (no BTI landing
+      # pads), MinGW and Clang cover the Windows and Clang builds, and cc -m32
+      # an x86 target without a backend. The AArch64 C++ cross compiler, its nm
+      # and qemu-aarch64 run the AArch64 backend's portable-override check: a
+      # static test binary under emulation. gcc-aarch64-linux-gnu only
+      # recommends its libc headers, so they are installed explicitly. For -m32,
+      # libc6-dev-i386 provides the 32-bit headers: gcc-multilib conflicts with
+      # the AArch64 cross compiler on Ubuntu 24.04. clang-21 comes from
+      # apt.llvm.org, as in ci/test/01_base_install.sh.
+      - name: Test integrated mlkem-native build policy
+        env:
+          MLKEM_POLICY_REQUIRED_COMPILERS: >-
+            cc cc-m32 clang-21 aarch64-linux-gnu-gcc aarch64-linux-gnu-g++ aarch64-linux-gnu-nm qemu-aarch64
+            x86_64-w64-mingw32-gcc-posix
+          APT_LLVM_V: 21
+        run: |
+          sudo install -d -m 0755 /etc/apt/keyrings
+          curl --fail --silent --show-error --location --retry 5 https://apt.llvm.org/llvm-snapshot.gpg.key \
+            | sudo tee /etc/apt/keyrings/apt.llvm.org.asc > /dev/null
+          codename="$(. /etc/os-release && echo "${VERSION_CODENAME}")"
+          echo "deb [signed-by=/etc/apt/keyrings/apt.llvm.org.asc] http://apt.llvm.org/${codename}/ llvm-toolchain-${codename}-${APT_LLVM_V} main" \
+            | sudo tee "/etc/apt/sources.list.d/llvm-toolchain-${codename}-${APT_LLVM_V}.list" > /dev/null
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross qemu-user \
+            gcc-mingw-w64-x86-64-posix \
+            libc6-dev-i386 \
+            "clang-${APT_LLVM_V}" "llvm-${APT_LLVM_V}"
+          python3 ci/checks/test_mlkem_build_policy.py -v
+
       - name: Test IBD timeout wiring
         run: python3 ci/checks/test_ibd_timeout_wiring.py
 
@@ -189,6 +267,18 @@ jobs:
           build/bin/qbitd -version
           build/bin/qbit-cli -version
 
+      # apt leaves its lock and partial directory readable only by root.
+      - name: Make the apt package cache readable
+        if: ${{ !cancelled() && github.event_name != 'pull_request' && steps.apt-cache.outputs.cache-hit != 'true' }}
+        run: sudo chown -R "$(id -u):$(id -g)" "${RUNNER_TEMP}/apt-archives"
+
+      - name: Save apt package cache
+        if: ${{ !cancelled() && github.event_name != 'pull_request' && steps.apt-cache.outputs.cache-hit != 'true' }}
+        uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        with:
+          path: ${{ runner.temp }}/apt-archives
+          key: ${{ steps.apt-cache.outputs.cache-primary-key }}
+
   focused-unit-suites:
     name: focused unit suites
     needs: classify-changes
@@ -402,6 +492,7 @@ jobs:
         with:
           fetch-depth: 0
 
+      # --no-renames, so that moving a release file away still runs the tests.
       - name: Detect release file changes
         id: release_changes
         env:
@@ -417,11 +508,11 @@ jobs:
               git fetch --no-tags --prune origin main
               merge_base="$(git merge-base origin/main HEAD 2>/dev/null || true)"
               if [[ -n "${merge_base}" ]]; then
-                git diff --name-only "${merge_base}" HEAD > "${changed_files}"
+                git diff --name-only --no-renames "${merge_base}" HEAD > "${changed_files}"
               else
                 git diff-tree --no-commit-id --name-only -r HEAD > "${changed_files}"
               fi
-            elif ! git diff --name-only "${before}" HEAD > "${changed_files}"; then
+            elif ! git diff --name-only --no-renames "${before}" HEAD > "${changed_files}"; then
               echo "Could not diff push range; running release validator tests."
               echo "changed=true" >> "$GITHUB_OUTPUT"
               exit 0
@@ -436,7 +527,7 @@ jobs:
             fi
 
             git fetch --no-tags --prune origin "+refs/heads/${base_branch}:refs/remotes/origin/${base_branch}"
-            if ! git diff --name-only "origin/${base_branch}...HEAD" > "${changed_files}"; then
+            if ! git diff --name-only --no-renames "origin/${base_branch}...HEAD" > "${changed_files}"; then
               echo "Could not diff against origin/${base_branch}; running release validator tests."
               echo "changed=true" >> "$GITHUB_OUTPUT"
               exit 0