ci: decouple builds into actions and test every package in Build and Test

.github/actionlint.yaml

@@ -0,0 +1,6 @@
+# Labels of the self-hosted NPU runners, for actionlint's runs-on check.
+self-hosted-runner:
+  labels:
+    - flm
+    - npu
+    - sweep

.github/actions/detect-changes/action.yml

@@ -0,0 +1,90 @@
+name: Detect changes
+description: >
+  Decides what a Build and Test run does: whether anything beyond docs changed,
+  which model families the change touches, whether the ASR code changed, and
+  which test tier applies. Needs a checkout with enough history to diff
+  (fetch-depth: 0).
+
+inputs:
+  tier:
+    description: Tier requested by the caller or dispatch; empty picks one from the event.
+    required: false
+    default: ""
+
+outputs:
+  run:
+    description: '"true" unless only documentation changed.'
+    value: ${{ steps.changes.outputs.run_tests }}
+  families:
+    description: Comma-separated model families the change touches.
+    value: ${{ steps.changes.outputs.families }}
+  asr-full:
+    description: '"true" when the ASR code changed, so the whole ASR set runs.'
+    value: ${{ steps.changes.outputs.asr_full }}
+  suites:
+    description: '"all", or the comma-separated npu_test suites the change can affect.'
+    value: ${{ steps.tier.outputs.suites }}
+  hrx-suites:
+    description: >
+      The suites for the HRX packages: the same, less api and serving, which
+      test server logic the backend does not touch. Empty when nothing is left.
+    value: ${{ steps.tier.outputs.hrx-suites }}
+  source:
+    description: '"true" when the models-from-source builds need testing (the chat suite runs).'
+    value: ${{ steps.tier.outputs.source }}
+  tier:
+    description: install, pr or full.
+    value: ${{ steps.tier.outputs.tier }}
+  short-sha:
+    description: First 7 characters of the commit, as the .deb artifact names use it.
+    value: ${{ steps.tier.outputs.short-sha }}
+
+runs:
+  using: composite
+  steps:
+    # Pull requests diff their own head against their base, pushes the new tip
+    # against the previous one. A pull request's github.sha is the merge with
+    # the current main, which would also count whatever landed on main since
+    # the base. Everything else (nightly, dispatch, a tag) has no base and
+    # tests everything; so does a push that creates a branch, whose "before"
+    # is zeros.
+    - name: Classify changed files
+      id: changes
+      shell: bash
+      working-directory: ${{ github.workspace }}
+      env:
+        BASE: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || (github.event_name == 'push' && github.ref_type == 'branch' && github.event.before) || '' }}
+        HEAD: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      run: python3 .github/script/ci/changes.py --base "${BASE}" --head "${HEAD}" --github-output
+
+    - name: Pick tier
+      id: tier
+      shell: bash
+      env:
+        REQUESTED: ${{ inputs.tier }}
+        EVENT: ${{ github.event_name }}
+        SHA: ${{ github.sha }}
+        SUITES: ${{ steps.changes.outputs.suites }}
+      run: |
+        tier="${REQUESTED}"
+        if [ -z "${tier}" ]; then
+          if [ "${EVENT}" = "schedule" ]; then tier=full; else tier=pr; fi
+        fi
+        # The nightly and any requested tier test everything, whatever changed.
+        suites="${SUITES:-all}"
+        [ "${EVENT}" = "pull_request" ] || [ "${EVENT}" = "push" ] || suites=all
+        if [ "${suites}" = "all" ]; then
+          hrx="chat,prefix_cache,embedding,decision,asr"
+        else
+          hrx=$(echo "${suites}" | tr ',' '\n' | { grep -vxE 'api|serving' || true; } | paste -sd, -)
+        fi
+        source=false
+        { [ "${suites}" = "all" ] || echo ",${suites}," | grep -q ",chat,"; } && source=true
+        {
+          echo "tier=${tier}"
+          echo "short-sha=${SHA:0:7}"
+          echo "suites=${suites}"
+          echo "hrx-suites=${hrx}"
+          echo "source=${source}"
+        } >> "$GITHUB_OUTPUT"
+        echo "Tier: ${tier}; suites: ${suites}; HRX suites: ${hrx:-none}; source tests: ${source}"

.github/actions/install-package/action.yml

@@ -0,0 +1,122 @@
+name: Install package
+description: >
+  Makes a downloaded FastFlowLM package runnable on an NPU runner and reports
+  the flm binary to test. A portable package is unpacked. A .deb or MSI is
+  installed when the runner account may install software (passwordless sudo,
+  or an elevated Windows session) and nothing is installed already; otherwise
+  it is unpacked and run in place, with a warning that the install itself went
+  untested. Pair with remove-package (if: always()) to put the runner back.
+
+inputs:
+  kind:
+    description: portable, deb or msi.
+    required: true
+  path:
+    description: Directory the package artifact was downloaded to, relative to the workspace.
+    required: true
+
+outputs:
+  flm-bin:
+    description: Absolute path of the flm binary to test.
+    value: ${{ steps.linux.outputs.flm-bin || steps.windows.outputs.flm-bin }}
+  mode:
+    description: '"installed" or "unpacked"; remove-package undoes an install only.'
+    value: ${{ steps.linux.outputs.mode || steps.windows.outputs.mode }}
+
+runs:
+  using: composite
+  steps:
+    - name: Prepare package (Linux)
+      id: linux
+      if: runner.os == 'Linux'
+      shell: bash
+      working-directory: ${{ github.workspace }}
+      env:
+        KIND: ${{ inputs.kind }}
+        PKG_DIR: ${{ inputs.path }}
+      run: |
+        set -euo pipefail
+        unpack="${GITHUB_WORKSPACE}/flm-unpacked"
+        rm -rf "${unpack}" && mkdir -p "${unpack}"
+        case "${KIND}" in
+          portable)
+            tar xzf "$(ls "${PKG_DIR}"/*_linux.tar.gz | head -1)" -C "${unpack}"
+            echo "flm-bin=${unpack}/flm" >> "$GITHUB_OUTPUT"
+            echo "mode=unpacked" >> "$GITHUB_OUTPUT"
+            ;;
+          deb)
+            deb=$(ls "${PKG_DIR}"/*.deb | head -1)
+            # Installing over a FastFlowLM someone put on this runner would
+            # leave it removed afterwards, so only install onto a clean one.
+            if sudo -n true 2>/dev/null && ! dpkg -s fastflowlm >/dev/null 2>&1; then
+              sudo -n apt-get install -y --no-install-recommends "$(realpath "${deb}")"
+              echo "flm-bin=/opt/fastflowlm/bin/flm" >> "$GITHUB_OUTPUT"
+              echo "mode=installed" >> "$GITHUB_OUTPUT"
+            else
+              reason="no passwordless sudo"
+              dpkg -s fastflowlm >/dev/null 2>&1 && reason="fastflowlm is already installed on this runner"
+              echo "::warning::Not installing $(basename "${deb}") (${reason}); testing its unpacked payload instead."
+              # flm finds its model lists and xclbins in ../share/flm next to
+              # the binary before its built-in /opt prefix, so the unpacked
+              # tree runs as it is.
+              dpkg-deb -x "${deb}" "${unpack}"
+              echo "flm-bin=${unpack}/opt/fastflowlm/bin/flm" >> "$GITHUB_OUTPUT"
+              echo "mode=unpacked" >> "$GITHUB_OUTPUT"
+            fi
+            ;;
+          *)
+            echo "::error::Unknown Linux package kind '${KIND}'."
+            exit 1
+            ;;
+        esac
+
+    - name: Prepare package (Windows)
+      id: windows
+      if: runner.os == 'Windows'
+      shell: powershell
+      working-directory: ${{ github.workspace }}
+      env:
+        KIND: ${{ inputs.kind }}
+        PKG_DIR: ${{ inputs.path }}
+      run: |
+        $ErrorActionPreference = "Stop"
+        function Out-Step($line) {
+          # No BOM: Windows PowerShell 5.1's Out-File -Encoding utf8 writes one,
+          # which corrupts the output name.
+          [System.IO.File]::AppendAllText($env:GITHUB_OUTPUT, "$line`n")
+        }
+        switch ($env:KIND) {
+          'portable' {
+            $flm = Join-Path (Resolve-Path $env:PKG_DIR) 'flm.exe'
+            if (-not (Test-Path $flm)) { throw "flm.exe not found in $env:PKG_DIR" }
+            Out-Step "flm-bin=$flm"
+            Out-Step "mode=unpacked"
+          }
+          'msi' {
+            $msi = (Get-ChildItem $env:PKG_DIR -Filter *.msi | Select-Object -First 1).FullName
+            $installed = Join-Path $env:ProgramFiles 'flm\flm.exe'
+            $admin = ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole(
+              [Security.Principal.WindowsBuiltInRole]::Administrator)
+            if ($admin -and -not (Test-Path $installed)) {
+              $log = Join-Path $env:GITHUB_WORKSPACE 'msi-install.log'
+              $p = Start-Process msiexec.exe -Wait -PassThru -ArgumentList @('/i', "`"$msi`"", '/qn', '/l*v', "`"$log`"")
+              if ($p.ExitCode -ne 0) { throw "msiexec /i failed with exit code $($p.ExitCode); see msi-install.log" }
+              Out-Step "flm-bin=$installed"
+              Out-Step "mode=installed"
+            } else {
+              $reason = if (-not $admin) { 'the runner session is not elevated' } else { 'FastFlowLM is already installed on this runner' }
+              Write-Host "::warning::Not installing $(Split-Path $msi -Leaf) ($reason); testing its unpacked payload instead."
+              # An administrative install only extracts the files; it needs no
+              # elevation and registers nothing.
+              $target = Join-Path $env:GITHUB_WORKSPACE 'flm-unpacked'
+              if (Test-Path $target) { Remove-Item $target -Recurse -Force }
+              $p = Start-Process msiexec.exe -Wait -PassThru -ArgumentList @('/a', "`"$msi`"", '/qn', "TARGETDIR=`"$target`"")
+              if ($p.ExitCode -ne 0) { throw "msiexec /a failed with exit code $($p.ExitCode)" }
+              $flm = Get-ChildItem $target -Recurse -Filter flm.exe | Select-Object -First 1
+              if (-not $flm) { throw "flm.exe not found in the extracted MSI" }
+              Out-Step "flm-bin=$($flm.FullName)"
+              Out-Step "mode=unpacked"
+            }
+          }
+          default { throw "Unknown Windows package kind '$env:KIND'." }
+        }

.github/actions/lin-build/action.yml

@@ -0,0 +1,177 @@
+name: Linux build
+description: >
+  Builds one Linux variant inside the fastflowlm build-environment container.
+  Expects the repository to be checked out with submodules, history and tags
+  (the version comes from git describe). A deb variant comes out as a finished
+  .deb, since dpkg-buildpackage builds and packages in one go; a portable
+  variant comes out as a CMake build tree for the caller to install and pack.
+
+inputs:
+  variant:
+    description: >
+      XRT-deb, HRX-deb, XRT-portable, HRX-portable, or Source-portable (the
+      model engines built from source instead of the prebuilt libraries in
+      lib/xrt).
+    required: true
+  distro:
+    description: The container's distribution, "ubuntu" or "debian". Names the .deb.
+    required: false
+    default: ""
+  distro-version:
+    description: The container's distribution version, e.g. "24.04". Names the .deb.
+    required: false
+    default: ""
+  codename:
+    description: The container's release codename, e.g. "noble". Goes in the .deb changelog.
+    required: false
+    default: ""
+
+outputs:
+  version:
+    description: git describe without the leading "v".
+    value: ${{ steps.version.outputs.version }}
+  package:
+    description: The built .deb, relative to the workspace (deb variants only).
+    value: ${{ steps.deb.outputs.package }}
+  build-dir:
+    description: The CMake build tree, relative to the workspace (portable variants only).
+    value: ${{ steps.portable.outputs.build-dir }}
+
+runs:
+  using: composite
+  steps:
+    - name: Check variant
+      shell: bash
+      env:
+        VARIANT: ${{ inputs.variant }}
+        DISTRO: ${{ inputs.distro }}
+        DISTRO_VERSION: ${{ inputs.distro-version }}
+        CODENAME: ${{ inputs.codename }}
+      run: |
+        case "${VARIANT}" in
+          XRT-deb|HRX-deb)
+            if [ -z "${DISTRO}" ] || [ -z "${DISTRO_VERSION}" ] || [ -z "${CODENAME}" ]; then
+              echo "::error::${VARIANT} needs distro, distro-version and codename."
+              exit 1
+            fi
+            ;;
+          XRT-portable|HRX-portable|Source-portable) ;;
+          *)
+            echo "::error::Unknown Linux build variant '${VARIANT}'."
+            exit 1
+            ;;
+        esac
+
+    - name: Configure git safe directory
+      shell: bash
+      run: git config --global --add safe.directory "${GITHUB_WORKSPACE}"
+
+    # Stripped for Debian version compatibility, and so the tarball names
+    # match the release tag without its "v".
+    - name: Get version from git describe
+      id: version
+      shell: bash
+      working-directory: ${{ github.workspace }}
+      run: echo "version=$(git describe --tags --always | sed 's/^v//')" >> "$GITHUB_OUTPUT"
+
+    # The build-environment image is rebuilt nightly from main's Dockerfile, so
+    # a Build-Depends change reaches it only after merge. Install the ones the
+    # image may still lack, plus what the portable and HRX builds need (patchelf
+    # rewrites the HRX engine RUNPATHs, zstd unpacks the HRX release).
+    - name: Install build tools and upgrade Rust toolchain
+      shell: bash
+      run: |
+        apt-get update
+        apt-get install -y --no-install-recommends \
+          ca-certificates curl libedit-dev nasm patchelf uuid-dev zstd
+        curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
+          sh -s -- -y --profile minimal --default-toolchain stable
+        echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
+
+    - name: Fetch pinned HRX release
+      id: hrx
+      if: startsWith(inputs.variant, 'HRX-')
+      shell: bash
+      working-directory: ${{ github.workspace }}
+      run: ./hrx-integration/fetch-hrx-release.sh
+
+    # debian/rules reads FLM_USE_HRX and FLM_CMAKE_ARGS from the environment.
+    - name: Build Debian package
+      id: deb
+      if: endsWith(inputs.variant, '-deb')
+      shell: bash
+      working-directory: ${{ github.workspace }}
+      env:
+        VARIANT: ${{ inputs.variant }}
+        DISTRO: ${{ inputs.distro }}${{ inputs.distro-version }}
+        CODENAME: ${{ inputs.codename }}
+        DEB_VERSION: ${{ steps.version.outputs.version }}
+        HRX_PREFIX: ${{ steps.hrx.outputs.prefix }}
+        DEBEMAIL: noreply@example.com
+        DEBFULLNAME: fastflowlm
+      run: |
+        set -euo pipefail
+        rustc --version
+
+        sed -e "s|@@DEB_VERSION@@|${DEB_VERSION}|g" \
+            -e "s|@@DEB_CODENAME@@|${CODENAME}|g" \
+            -e "s|@@DEB_DATE@@|$(date -R)|g" \
+            debian/changelog.in > debian/changelog
+        chmod +x debian/rules
+
+        if [ "${VARIANT}" = "HRX-deb" ]; then
+          if [ -z "${HRX_PREFIX}" ]; then
+            echo "::error::fetch-hrx-release.sh did not report the HRX package prefix."
+            exit 1
+          fi
+          export FLM_USE_HRX=ON
+          export FLM_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${HRX_PREFIX}"
+          package="HRX_fastflowlm_${DEB_VERSION}_${DISTRO}_amd64.deb"
+        else
+          package="fastflowlm_${DEB_VERSION}_${DISTRO}_amd64.deb"
+        fi
+
+        dpkg-buildpackage -b -us -uc
+        mv "../fastflowlm_${DEB_VERSION}_amd64.deb" "./${package}"
+        ls -lah "./${package}"
+        .github/script/ci/check_no_copyleft_libs.sh "./${package}"
+        echo "package=${package}" >> "$GITHUB_OUTPUT"
+
+    # The engines take more memory per compile than the rest of the tree, so
+    # the source build runs fewer jobs at once.
+    - name: Build portable distribution
+      id: portable
+      if: endsWith(inputs.variant, '-portable')
+      shell: bash
+      working-directory: ${{ github.workspace }}/src
+      env:
+        VARIANT: ${{ inputs.variant }}
+        HRX_PREFIX: ${{ steps.hrx.outputs.prefix }}
+      run: |
+        set -euo pipefail
+        case "${VARIANT}" in
+          XRT-portable)
+            cmake --preset linux-portable
+            cmake --build build -j"$(nproc)"
+            build_dir=src/build
+            ;;
+          HRX-portable)
+            if [ -z "${HRX_PREFIX}" ]; then
+              echo "::error::fetch-hrx-release.sh did not report the HRX package prefix."
+              exit 1
+            fi
+            cmake --preset linux-portable \
+              -DFLM_USE_HRX=ON \
+              -DCMAKE_PREFIX_PATH="${HRX_PREFIX}"
+            cmake --build build -j"$(nproc)"
+            build_dir=src/build
+            ;;
+          Source-portable)
+            cmake --preset linux-portable \
+              -B build-models-from-source \
+              -DFLM_BUILD_GEMMA4E=ON
+            cmake --build build-models-from-source --parallel 2
+            build_dir=src/build-models-from-source
+            ;;
+        esac
+        echo "build-dir=${build_dir}" >> "$GITHUB_OUTPUT"