diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 000000000..2e5cf063b --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,6 @@ +# Labels of the self-hosted NPU runners, for actionlint's runs-on check. +self-hosted-runner: + labels: + - flm + - npu + - sweep diff --git a/.github/actions/detect-changes/action.yml b/.github/actions/detect-changes/action.yml new file mode 100644 index 000000000..74b17f3db --- /dev/null +++ b/.github/actions/detect-changes/action.yml @@ -0,0 +1,90 @@ +name: Detect changes +description: > + Decides what a Build and Test run does: whether anything beyond docs changed, + which model families the change touches, whether the ASR code changed, and + which test tier applies. Needs a checkout with enough history to diff + (fetch-depth: 0). + +inputs: + tier: + description: Tier requested by the caller or dispatch; empty picks one from the event. + required: false + default: "" + +outputs: + run: + description: '"true" unless only documentation changed.' + value: ${{ steps.changes.outputs.run_tests }} + families: + description: Comma-separated model families the change touches. + value: ${{ steps.changes.outputs.families }} + asr-full: + description: '"true" when the ASR code changed, so the whole ASR set runs.' + value: ${{ steps.changes.outputs.asr_full }} + suites: + description: '"all", or the comma-separated npu_test suites the change can affect.' + value: ${{ steps.tier.outputs.suites }} + hrx-suites: + description: > + The suites for the HRX packages: the same, less api and serving, which + test server logic the backend does not touch. Empty when nothing is left. + value: ${{ steps.tier.outputs.hrx-suites }} + source: + description: '"true" when the models-from-source builds need testing (the chat suite runs).' + value: ${{ steps.tier.outputs.source }} + tier: + description: install, pr or full. + value: ${{ steps.tier.outputs.tier }} + short-sha: + description: First 7 characters of the commit, as the .deb artifact names use it. + value: ${{ steps.tier.outputs.short-sha }} + +runs: + using: composite + steps: + # Pull requests diff their own head against their base, pushes the new tip + # against the previous one. A pull request's github.sha is the merge with + # the current main, which would also count whatever landed on main since + # the base. Everything else (nightly, dispatch, a tag) has no base and + # tests everything; so does a push that creates a branch, whose "before" + # is zeros. + - name: Classify changed files + id: changes + shell: bash + working-directory: ${{ github.workspace }} + env: + BASE: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || (github.event_name == 'push' && github.ref_type == 'branch' && github.event.before) || '' }} + HEAD: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + run: python3 .github/script/ci/changes.py --base "${BASE}" --head "${HEAD}" --github-output + + - name: Pick tier + id: tier + shell: bash + env: + REQUESTED: ${{ inputs.tier }} + EVENT: ${{ github.event_name }} + SHA: ${{ github.sha }} + SUITES: ${{ steps.changes.outputs.suites }} + run: | + tier="${REQUESTED}" + if [ -z "${tier}" ]; then + if [ "${EVENT}" = "schedule" ]; then tier=full; else tier=pr; fi + fi + # The nightly and any requested tier test everything, whatever changed. + suites="${SUITES:-all}" + [ "${EVENT}" = "pull_request" ] || [ "${EVENT}" = "push" ] || suites=all + if [ "${suites}" = "all" ]; then + hrx="chat,prefix_cache,embedding,decision,asr" + else + hrx=$(echo "${suites}" | tr ',' '\n' | { grep -vxE 'api|serving' || true; } | paste -sd, -) + fi + source=false + { [ "${suites}" = "all" ] || echo ",${suites}," | grep -q ",chat,"; } && source=true + { + echo "tier=${tier}" + echo "short-sha=${SHA:0:7}" + echo "suites=${suites}" + echo "hrx-suites=${hrx}" + echo "source=${source}" + } >> "$GITHUB_OUTPUT" + echo "Tier: ${tier}; suites: ${suites}; HRX suites: ${hrx:-none}; source tests: ${source}" diff --git a/.github/actions/install-package/action.yml b/.github/actions/install-package/action.yml new file mode 100644 index 000000000..d22e89490 --- /dev/null +++ b/.github/actions/install-package/action.yml @@ -0,0 +1,122 @@ +name: Install package +description: > + Makes a downloaded FastFlowLM package runnable on an NPU runner and reports + the flm binary to test. A portable package is unpacked. A .deb or MSI is + installed when the runner account may install software (passwordless sudo, + or an elevated Windows session) and nothing is installed already; otherwise + it is unpacked and run in place, with a warning that the install itself went + untested. Pair with remove-package (if: always()) to put the runner back. + +inputs: + kind: + description: portable, deb or msi. + required: true + path: + description: Directory the package artifact was downloaded to, relative to the workspace. + required: true + +outputs: + flm-bin: + description: Absolute path of the flm binary to test. + value: ${{ steps.linux.outputs.flm-bin || steps.windows.outputs.flm-bin }} + mode: + description: '"installed" or "unpacked"; remove-package undoes an install only.' + value: ${{ steps.linux.outputs.mode || steps.windows.outputs.mode }} + +runs: + using: composite + steps: + - name: Prepare package (Linux) + id: linux + if: runner.os == 'Linux' + shell: bash + working-directory: ${{ github.workspace }} + env: + KIND: ${{ inputs.kind }} + PKG_DIR: ${{ inputs.path }} + run: | + set -euo pipefail + unpack="${GITHUB_WORKSPACE}/flm-unpacked" + rm -rf "${unpack}" && mkdir -p "${unpack}" + case "${KIND}" in + portable) + tar xzf "$(ls "${PKG_DIR}"/*_linux.tar.gz | head -1)" -C "${unpack}" + echo "flm-bin=${unpack}/flm" >> "$GITHUB_OUTPUT" + echo "mode=unpacked" >> "$GITHUB_OUTPUT" + ;; + deb) + deb=$(ls "${PKG_DIR}"/*.deb | head -1) + # Installing over a FastFlowLM someone put on this runner would + # leave it removed afterwards, so only install onto a clean one. + if sudo -n true 2>/dev/null && ! dpkg -s fastflowlm >/dev/null 2>&1; then + sudo -n apt-get install -y --no-install-recommends "$(realpath "${deb}")" + echo "flm-bin=/opt/fastflowlm/bin/flm" >> "$GITHUB_OUTPUT" + echo "mode=installed" >> "$GITHUB_OUTPUT" + else + reason="no passwordless sudo" + dpkg -s fastflowlm >/dev/null 2>&1 && reason="fastflowlm is already installed on this runner" + echo "::warning::Not installing $(basename "${deb}") (${reason}); testing its unpacked payload instead." + # flm finds its model lists and xclbins in ../share/flm next to + # the binary before its built-in /opt prefix, so the unpacked + # tree runs as it is. + dpkg-deb -x "${deb}" "${unpack}" + echo "flm-bin=${unpack}/opt/fastflowlm/bin/flm" >> "$GITHUB_OUTPUT" + echo "mode=unpacked" >> "$GITHUB_OUTPUT" + fi + ;; + *) + echo "::error::Unknown Linux package kind '${KIND}'." + exit 1 + ;; + esac + + - name: Prepare package (Windows) + id: windows + if: runner.os == 'Windows' + shell: powershell + working-directory: ${{ github.workspace }} + env: + KIND: ${{ inputs.kind }} + PKG_DIR: ${{ inputs.path }} + run: | + $ErrorActionPreference = "Stop" + function Out-Step($line) { + # No BOM: Windows PowerShell 5.1's Out-File -Encoding utf8 writes one, + # which corrupts the output name. + [System.IO.File]::AppendAllText($env:GITHUB_OUTPUT, "$line`n") + } + switch ($env:KIND) { + 'portable' { + $flm = Join-Path (Resolve-Path $env:PKG_DIR) 'flm.exe' + if (-not (Test-Path $flm)) { throw "flm.exe not found in $env:PKG_DIR" } + Out-Step "flm-bin=$flm" + Out-Step "mode=unpacked" + } + 'msi' { + $msi = (Get-ChildItem $env:PKG_DIR -Filter *.msi | Select-Object -First 1).FullName + $installed = Join-Path $env:ProgramFiles 'flm\flm.exe' + $admin = ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole( + [Security.Principal.WindowsBuiltInRole]::Administrator) + if ($admin -and -not (Test-Path $installed)) { + $log = Join-Path $env:GITHUB_WORKSPACE 'msi-install.log' + $p = Start-Process msiexec.exe -Wait -PassThru -ArgumentList @('/i', "`"$msi`"", '/qn', '/l*v', "`"$log`"") + if ($p.ExitCode -ne 0) { throw "msiexec /i failed with exit code $($p.ExitCode); see msi-install.log" } + Out-Step "flm-bin=$installed" + Out-Step "mode=installed" + } else { + $reason = if (-not $admin) { 'the runner session is not elevated' } else { 'FastFlowLM is already installed on this runner' } + Write-Host "::warning::Not installing $(Split-Path $msi -Leaf) ($reason); testing its unpacked payload instead." + # An administrative install only extracts the files; it needs no + # elevation and registers nothing. + $target = Join-Path $env:GITHUB_WORKSPACE 'flm-unpacked' + if (Test-Path $target) { Remove-Item $target -Recurse -Force } + $p = Start-Process msiexec.exe -Wait -PassThru -ArgumentList @('/a', "`"$msi`"", '/qn', "TARGETDIR=`"$target`"") + if ($p.ExitCode -ne 0) { throw "msiexec /a failed with exit code $($p.ExitCode)" } + $flm = Get-ChildItem $target -Recurse -Filter flm.exe | Select-Object -First 1 + if (-not $flm) { throw "flm.exe not found in the extracted MSI" } + Out-Step "flm-bin=$($flm.FullName)" + Out-Step "mode=unpacked" + } + } + default { throw "Unknown Windows package kind '$env:KIND'." } + } diff --git a/.github/actions/lin-build/action.yml b/.github/actions/lin-build/action.yml new file mode 100644 index 000000000..eace5dc3d --- /dev/null +++ b/.github/actions/lin-build/action.yml @@ -0,0 +1,177 @@ +name: Linux build +description: > + Builds one Linux variant inside the fastflowlm build-environment container. + Expects the repository to be checked out with submodules, history and tags + (the version comes from git describe). A deb variant comes out as a finished + .deb, since dpkg-buildpackage builds and packages in one go; a portable + variant comes out as a CMake build tree for the caller to install and pack. + +inputs: + variant: + description: > + XRT-deb, HRX-deb, XRT-portable, HRX-portable, or Source-portable (the + model engines built from source instead of the prebuilt libraries in + lib/xrt). + required: true + distro: + description: The container's distribution, "ubuntu" or "debian". Names the .deb. + required: false + default: "" + distro-version: + description: The container's distribution version, e.g. "24.04". Names the .deb. + required: false + default: "" + codename: + description: The container's release codename, e.g. "noble". Goes in the .deb changelog. + required: false + default: "" + +outputs: + version: + description: git describe without the leading "v". + value: ${{ steps.version.outputs.version }} + package: + description: The built .deb, relative to the workspace (deb variants only). + value: ${{ steps.deb.outputs.package }} + build-dir: + description: The CMake build tree, relative to the workspace (portable variants only). + value: ${{ steps.portable.outputs.build-dir }} + +runs: + using: composite + steps: + - name: Check variant + shell: bash + env: + VARIANT: ${{ inputs.variant }} + DISTRO: ${{ inputs.distro }} + DISTRO_VERSION: ${{ inputs.distro-version }} + CODENAME: ${{ inputs.codename }} + run: | + case "${VARIANT}" in + XRT-deb|HRX-deb) + if [ -z "${DISTRO}" ] || [ -z "${DISTRO_VERSION}" ] || [ -z "${CODENAME}" ]; then + echo "::error::${VARIANT} needs distro, distro-version and codename." + exit 1 + fi + ;; + XRT-portable|HRX-portable|Source-portable) ;; + *) + echo "::error::Unknown Linux build variant '${VARIANT}'." + exit 1 + ;; + esac + + - name: Configure git safe directory + shell: bash + run: git config --global --add safe.directory "${GITHUB_WORKSPACE}" + + # Stripped for Debian version compatibility, and so the tarball names + # match the release tag without its "v". + - name: Get version from git describe + id: version + shell: bash + working-directory: ${{ github.workspace }} + run: echo "version=$(git describe --tags --always | sed 's/^v//')" >> "$GITHUB_OUTPUT" + + # The build-environment image is rebuilt nightly from main's Dockerfile, so + # a Build-Depends change reaches it only after merge. Install the ones the + # image may still lack, plus what the portable and HRX builds need (patchelf + # rewrites the HRX engine RUNPATHs, zstd unpacks the HRX release). + - name: Install build tools and upgrade Rust toolchain + shell: bash + run: | + apt-get update + apt-get install -y --no-install-recommends \ + ca-certificates curl libedit-dev nasm patchelf uuid-dev zstd + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ + sh -s -- -y --profile minimal --default-toolchain stable + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + + - name: Fetch pinned HRX release + id: hrx + if: startsWith(inputs.variant, 'HRX-') + shell: bash + working-directory: ${{ github.workspace }} + run: ./hrx-integration/fetch-hrx-release.sh + + # debian/rules reads FLM_USE_HRX and FLM_CMAKE_ARGS from the environment. + - name: Build Debian package + id: deb + if: endsWith(inputs.variant, '-deb') + shell: bash + working-directory: ${{ github.workspace }} + env: + VARIANT: ${{ inputs.variant }} + DISTRO: ${{ inputs.distro }}${{ inputs.distro-version }} + CODENAME: ${{ inputs.codename }} + DEB_VERSION: ${{ steps.version.outputs.version }} + HRX_PREFIX: ${{ steps.hrx.outputs.prefix }} + DEBEMAIL: noreply@example.com + DEBFULLNAME: fastflowlm + run: | + set -euo pipefail + rustc --version + + sed -e "s|@@DEB_VERSION@@|${DEB_VERSION}|g" \ + -e "s|@@DEB_CODENAME@@|${CODENAME}|g" \ + -e "s|@@DEB_DATE@@|$(date -R)|g" \ + debian/changelog.in > debian/changelog + chmod +x debian/rules + + if [ "${VARIANT}" = "HRX-deb" ]; then + if [ -z "${HRX_PREFIX}" ]; then + echo "::error::fetch-hrx-release.sh did not report the HRX package prefix." + exit 1 + fi + export FLM_USE_HRX=ON + export FLM_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${HRX_PREFIX}" + package="HRX_fastflowlm_${DEB_VERSION}_${DISTRO}_amd64.deb" + else + package="fastflowlm_${DEB_VERSION}_${DISTRO}_amd64.deb" + fi + + dpkg-buildpackage -b -us -uc + mv "../fastflowlm_${DEB_VERSION}_amd64.deb" "./${package}" + ls -lah "./${package}" + .github/script/ci/check_no_copyleft_libs.sh "./${package}" + echo "package=${package}" >> "$GITHUB_OUTPUT" + + # The engines take more memory per compile than the rest of the tree, so + # the source build runs fewer jobs at once. + - name: Build portable distribution + id: portable + if: endsWith(inputs.variant, '-portable') + shell: bash + working-directory: ${{ github.workspace }}/src + env: + VARIANT: ${{ inputs.variant }} + HRX_PREFIX: ${{ steps.hrx.outputs.prefix }} + run: | + set -euo pipefail + case "${VARIANT}" in + XRT-portable) + cmake --preset linux-portable + cmake --build build -j"$(nproc)" + build_dir=src/build + ;; + HRX-portable) + if [ -z "${HRX_PREFIX}" ]; then + echo "::error::fetch-hrx-release.sh did not report the HRX package prefix." + exit 1 + fi + cmake --preset linux-portable \ + -DFLM_USE_HRX=ON \ + -DCMAKE_PREFIX_PATH="${HRX_PREFIX}" + cmake --build build -j"$(nproc)" + build_dir=src/build + ;; + Source-portable) + cmake --preset linux-portable \ + -B build-models-from-source \ + -DFLM_BUILD_GEMMA4E=ON + cmake --build build-models-from-source --parallel 2 + build_dir=src/build-models-from-source + ;; + esac + echo "build-dir=${build_dir}" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/npu-test/action.yml b/.github/actions/npu-test/action.yml new file mode 100644 index 000000000..4e5a8fec2 --- /dev/null +++ b/.github/actions/npu-test/action.yml @@ -0,0 +1,189 @@ +name: NPU test +description: > + Runs the npu_test suites (.github/script/npu_test) against one flm binary in + server mode, adds the summary to the job page and uploads the full report. + Needs .github/script, src/test/openai_api_test, src/test/openai_usage and + src/test/d3_video/fixtures in the checkout, and Python 3.9+ on the runner + (standard library only). + +inputs: + flm-bin: + description: Absolute path of the flm binary to test. + required: true + label: + description: Names this package in the report, e.g. linux-xrt-portable. + required: true + tier: + description: install, pr or full. + required: true + suites: + description: Comma-separated suites, or "all". + required: false + default: all + families: + description: Comma-separated model families the change touches. + required: false + default: "" + models: + description: Comma-separated model ids that restrict every suite. + required: false + default: "" + asr-full: + description: '"true" to run the whole ASR evaluation set on the pr tier.' + required: false + default: "false" + asr-expected-key: + description: Row of asr_eval/expected.json to check WER against (linux, linux-hrx, windows, windows-hrx). + required: false + default: "" + +runs: + using: composite + steps: + # A runner takes one job at a time, so an flm left running from an earlier + # job's workspace is stale and holds the NPU: a new server then hangs at + # start. Only processes under the runner's work directory are touched, + # never one a person started. One that survives SIGKILL is stuck in the + # NPU driver, and only a reboot frees the runner. + - name: Clear stale flm processes (Linux) + if: runner.os == 'Linux' + shell: bash + run: | + work="$(dirname "$(dirname "${GITHUB_WORKSPACE}")")" + stale=$(pgrep -u "$(id -u)" -f "^${work}/.*flm(-real)? serve" || true) + [ -z "${stale}" ] && exit 0 + echo "::warning::Killing flm left over from an earlier job: $(ps -o pid=,stat=,args= -p ${stale} | tr '\n' ';')" + kill -9 ${stale} 2>/dev/null || true + sleep 5 + left=$(ps -o pid=,stat=,args= -p ${stale} 2>/dev/null || true) + if [ -n "${left}" ]; then + echo "::error::flm from an earlier job survived SIGKILL and holds the NPU (state D means stuck in the driver); reboot this runner (${RUNNER_NAME}): ${left}" + exit 1 + fi + + - name: Clear stale flm processes (Windows) + if: runner.os == 'Windows' + shell: powershell + run: | + $work = Split-Path (Split-Path $env:GITHUB_WORKSPACE) + $stale = Get-Process -Name flm -ErrorAction SilentlyContinue | + Where-Object { $_.Path -and $_.Path.StartsWith($work, [StringComparison]::OrdinalIgnoreCase) } + if (-not $stale) { exit 0 } + Write-Host "::warning::Killing flm left over from an earlier job: $($stale | ForEach-Object { "$($_.Id) $($_.Path)" })" + $stale | Stop-Process -Force -ErrorAction SilentlyContinue + Start-Sleep -Seconds 5 + $left = $stale | Where-Object { -not $_.HasExited } + if ($left) { + Write-Host "::error::flm from an earlier job survived Stop-Process and holds the NPU; reboot this runner ($env:RUNNER_NAME): $($left.Id)" + exit 1 + } + + # Models persist on the runner between jobs, so only the first job on a + # runner pays for a download. + - name: Run tests (Linux) + id: linux + if: runner.os == 'Linux' + shell: bash + working-directory: ${{ github.workspace }}/.github/script + env: + FLM_BIN: ${{ inputs.flm-bin }} + LABEL: ${{ inputs.label }} + TIER: ${{ inputs.tier }} + SUITES: ${{ inputs.suites }} + FAMILIES: ${{ inputs.families }} + MODELS: ${{ inputs.models }} + ASR_FULL: ${{ inputs.asr-full }} + ASR_KEY: ${{ inputs.asr-expected-key }} + run: | + export FLM_MODEL_PATH="${FLM_MODEL_PATH:-/scratch}" + args=(--flm-bin "${FLM_BIN}" --label "${LABEL}" --tier "${TIER}" --suites "${SUITES}" + --families "${FAMILIES}" --models "${MODELS}" --asr-expected-key "${ASR_KEY}" + --output-dir "${GITHUB_WORKSPACE}/npu-test-results/${LABEL}") + [ "${ASR_FULL}" = "true" ] && args+=(--asr-full) + code=0 + python3 -m npu_test "${args[@]}" || code=$? + echo "code=${code}" >> "$GITHUB_OUTPUT" + + - name: Run tests (Windows) + id: windows + if: runner.os == 'Windows' + shell: powershell + working-directory: ${{ github.workspace }}/.github/script + env: + FLM_BIN: ${{ inputs.flm-bin }} + LABEL: ${{ inputs.label }} + TIER: ${{ inputs.tier }} + SUITES: ${{ inputs.suites }} + FAMILIES: ${{ inputs.families }} + MODELS: ${{ inputs.models }} + ASR_FULL: ${{ inputs.asr-full }} + ASR_KEY: ${{ inputs.asr-expected-key }} + run: | + if (-not $env:FLM_MODEL_PATH) { + $env:FLM_MODEL_PATH = Join-Path $env:USERPROFILE ".flm" + New-Item -ItemType Directory -Force $env:FLM_MODEL_PATH | Out-Null + } + # --opt=value throughout: Windows PowerShell 5.1 drops an empty argument + # when calling a native program, which would leave --families with no + # value. Model output and transcripts are not all representable in the + # console code page, so Python (and the test scripts it starts) write UTF-8. + $env:PYTHONUTF8 = '1' + $outDir = Join-Path $env:GITHUB_WORKSPACE "npu-test-results\$env:LABEL" + $cmdArgs = @('-m', 'npu_test', "--flm-bin=$env:FLM_BIN", "--label=$env:LABEL", + "--tier=$env:TIER", "--suites=$env:SUITES", "--families=$env:FAMILIES", + "--models=$env:MODELS", "--asr-expected-key=$env:ASR_KEY", + "--output-dir=$outDir") + if ($env:ASR_FULL -eq 'true') { $cmdArgs += '--asr-full' } + & python $cmdArgs + [System.IO.File]::AppendAllText($env:GITHUB_OUTPUT, "code=$LASTEXITCODE`n") + exit 0 + + - name: Add summary (Linux) + if: always() && runner.os == 'Linux' + shell: bash + working-directory: ${{ github.workspace }} + env: + LABEL: ${{ inputs.label }} + run: | + summary="npu-test-results/${LABEL}/summary.md" + if [ -f "${summary}" ]; then + cat "${summary}" >> "$GITHUB_STEP_SUMMARY" + else + echo "### ❌ ${LABEL}: no report was written" >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Add summary (Windows) + if: always() && runner.os == 'Windows' + shell: powershell + working-directory: ${{ github.workspace }} + env: + LABEL: ${{ inputs.label }} + run: | + $summary = "npu-test-results\$env:LABEL\summary.md" + $text = if (Test-Path $summary) { Get-Content $summary -Raw -Encoding UTF8 } else { "### ❌ ${env:LABEL}: no report was written`n" } + [System.IO.File]::AppendAllText($env:GITHUB_STEP_SUMMARY, $text, [System.Text.UTF8Encoding]::new($false)) + + # Server logs included, so a failure can be diagnosed without the runner. + - name: Upload report + if: always() + uses: actions/upload-artifact@v4 + with: + name: npu-test-${{ inputs.label }} + path: npu-test-results/${{ inputs.label }}/ + if-no-files-found: warn + retention-days: 30 + + # Last, so a failed run still gets its summary and report uploaded. + - name: Fail on test failures (Linux) + if: runner.os == 'Linux' && steps.linux.outputs.code != '0' + shell: bash + run: | + echo "::error::NPU tests failed; see the summary and the npu-test-${{ inputs.label }} artifact." + exit 1 + + - name: Fail on test failures (Windows) + if: runner.os == 'Windows' && steps.windows.outputs.code != '0' + shell: powershell + run: | + Write-Host "::error::NPU tests failed; see the summary and the npu-test-${{ inputs.label }} artifact." + exit 1 diff --git a/.github/actions/remove-package/action.yml b/.github/actions/remove-package/action.yml new file mode 100644 index 000000000..cc8dda535 --- /dev/null +++ b/.github/actions/remove-package/action.yml @@ -0,0 +1,37 @@ +name: Remove package +description: > + Undoes what install-package installed, so the next job on this runner starts + clean. Unpacked packages live in the workspace and need nothing. Run it with + if: always(). + +inputs: + kind: + description: portable, deb or msi. + required: true + mode: + description: The mode install-package reported. + required: true + path: + description: Directory the package artifact was downloaded to (for the MSI). + required: false + default: "" + +runs: + using: composite + steps: + - name: Remove .deb + if: runner.os == 'Linux' && inputs.kind == 'deb' && inputs.mode == 'installed' + shell: bash + run: sudo -n apt-get remove -y fastflowlm + + - name: Remove MSI + if: runner.os == 'Windows' && inputs.kind == 'msi' && inputs.mode == 'installed' + shell: powershell + working-directory: ${{ github.workspace }} + env: + PKG_DIR: ${{ inputs.path }} + run: | + $ErrorActionPreference = "Stop" + $msi = (Get-ChildItem $env:PKG_DIR -Filter *.msi | Select-Object -First 1).FullName + $p = Start-Process msiexec.exe -Wait -PassThru -ArgumentList @('/x', "`"$msi`"", '/qn') + if ($p.ExitCode -ne 0) { throw "msiexec /x failed with exit code $($p.ExitCode)" } diff --git a/.github/actions/win-build/action.yml b/.github/actions/win-build/action.yml new file mode 100644 index 000000000..fdbbc78e9 --- /dev/null +++ b/.github/actions/win-build/action.yml @@ -0,0 +1,130 @@ +name: Windows build +description: > + Builds flm.exe for one Windows variant. Expects the repository to be checked + out with history and tags (the version comes from git describe); packaging + the result is left to the caller. + +inputs: + variant: + description: > + XRT (the default build), HRX (the HRX runtime), or Source (the model + engines built from source instead of the prebuilt DLLs in lib\xrt). + required: true + +outputs: + build-dir: + description: The CMake build tree, relative to the workspace. + value: ${{ steps.build.outputs.build-dir }} + hrx-runtime: + description: Absolute path to the fetched hrx.dll (HRX only). + value: ${{ steps.hrx-runtime.outputs.path }} + +runs: + using: composite + steps: + - name: Check variant + shell: powershell + env: + VARIANT: ${{ inputs.variant }} + run: | + if ($env:VARIANT -notin @('XRT', 'HRX', 'Source')) { + Write-Host "::error::Unknown Windows build variant '$env:VARIANT'." + exit 1 + } + + # A full-history checkout also applies to submodules, which pulls the full + # msgpack-c history (~95 MB) and keeps getting reset mid-transfer on the + # runner network. The pinned commits alone are under 1 MB. + - name: Checkout submodules + shell: powershell + working-directory: ${{ github.workspace }} + run: | + for ($i = 1; $i -le 3; $i++) { + git submodule update --init --recursive --depth 1 --jobs 4 + if ($LASTEXITCODE -eq 0) { exit 0 } + Write-Host "Submodule fetch failed (attempt $i), retrying..." + Start-Sleep -Seconds 15 + } + exit 1 + + # setup_windows_deps.ps1 lays out the C:\dev dependencies the self-hosted + # runner has. HRX needs the same Boost and curl as the XRT build; the XRT + # parts of the tree go unused there. + - name: Restore build dependencies + uses: actions/cache@v4 + with: + path: C:\dev + key: windows-deps-${{ hashFiles('.github/script/ci/setup_windows_deps.ps1', '.github/windows-deps/**') }} + + - name: Set up build dependencies + shell: powershell + working-directory: ${{ github.workspace }} + run: .github\script\ci\setup_windows_deps.ps1 + + - name: Fetch pinned HRX release + id: hrx + if: inputs.variant == 'HRX' + shell: powershell + working-directory: ${{ github.workspace }} + run: .\hrx-integration\fetch-hrx-release.ps1 + + - name: Configure CMake + shell: powershell + working-directory: ${{ github.workspace }}/src + env: + VARIANT: ${{ inputs.variant }} + HRX_PREFIX: ${{ steps.hrx.outputs.prefix }} + run: | + $ErrorActionPreference = "Stop" + switch ($env:VARIANT) { + 'XRT' { + cmake --preset windows-vs18 + } + 'HRX' { + if (-not $env:HRX_PREFIX) { + throw 'fetch-hrx-release.ps1 did not report the HRX package prefix' + } + cmake --preset windows-vs18 -DFLM_USE_HRX=ON "-DCMAKE_PREFIX_PATH=$env:HRX_PREFIX" + } + 'Source' { + cmake --preset windows-vs18 ` + -B build-models-from-source ` + -DFLM_BUILD_GEMMA4E=ON + } + } + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + # The engines take more memory per compile than the rest of the tree, so + # the source build runs fewer jobs at once. + - name: Build + id: build + shell: powershell + working-directory: ${{ github.workspace }}/src + env: + VARIANT: ${{ inputs.variant }} + run: | + $ErrorActionPreference = "Stop" + if ($env:VARIANT -eq 'Source') { + cmake --build build-models-from-source --config Release --parallel 2 + $buildDir = 'src/build-models-from-source' + } else { + cmake --build --preset windows-default --config Release --parallel + $buildDir = 'src/build' + } + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + # No BOM: Windows PowerShell 5.1's Out-File -Encoding utf8 writes one, + # which corrupts the output name. + [System.IO.File]::AppendAllText($env:GITHUB_OUTPUT, "build-dir=$buildDir`n") + + - name: Locate HRX runtime + id: hrx-runtime + if: inputs.variant == 'HRX' + shell: powershell + working-directory: ${{ github.workspace }} + run: | + $ErrorActionPreference = "Stop" + $hrxDll = Get-ChildItem -Path .\hrx-integration\.hrx-release -Recurse -Filter 'hrx.dll' -ErrorAction SilentlyContinue | Select-Object -First 1 + if (-not $hrxDll) { + throw 'hrx.dll not found under hrx-integration\.hrx-release; HRX fetch step may have failed' + } + [System.IO.File]::AppendAllText($env:GITHUB_OUTPUT, "path=$($hrxDll.FullName)`n") diff --git a/.github/actions/win-msi/action.yml b/.github/actions/win-msi/action.yml new file mode 100644 index 000000000..cc92fdf85 --- /dev/null +++ b/.github/actions/win-msi/action.yml @@ -0,0 +1,65 @@ +name: Windows MSI +description: > + Builds the MSI installer around an already-built flm.exe, so the MSI and the + portable zip ship the same binary. Everything else the installer stages + (DLLs, xclbins, model lists) comes from the checkout, so no submodules or + history are needed. + +inputs: + flm-exe: + description: Path to the flm.exe to package, relative to the workspace. + required: true + +outputs: + msi: + description: The built installer, relative to the workspace. + value: src/wix/output/flm-setup.msi + +runs: + using: composite + steps: + # get_files.bat (inno and wix) stage flm.exe from build\, where a local + # build puts it. + - name: Place flm.exe + shell: powershell + working-directory: ${{ github.workspace }} + env: + FLM_EXE: ${{ inputs.flm-exe }} + run: | + $ErrorActionPreference = "Stop" + New-Item -ItemType Directory -Force src\build | Out-Null + Copy-Item -Path $env:FLM_EXE -Destination src\build\flm.exe -Force + + - name: Stage installer package files + shell: powershell + working-directory: ${{ github.workspace }}/src/inno + run: .\get_files.bat + + # The image only has WiX 3; flm.wxs is WiX 5. Same version as the + # self-hosted runner had. + - name: Install WiX + shell: powershell + run: | + $ErrorActionPreference = "Stop" + dotnet tool install --global wix --version 5.0.2 + if ($LASTEXITCODE -ne 0) { throw "dotnet tool install wix failed" } + $wixToolsPath = Join-Path $env:USERPROFILE ".dotnet\tools" + & "$wixToolsPath\wix.exe" extension add --global WixToolset.UI.wixext/5.0.2 + if ($LASTEXITCODE -ne 0) { throw "wix extension add failed" } + + $wixToolsPath | Out-File ` + -FilePath $env:GITHUB_PATH ` + -Encoding utf8 ` + -Append + + - name: Verify WiX + shell: powershell + run: | + $wix = Get-Command wix -ErrorAction Stop + Write-Host "Using WiX: $($wix.Source)" + wix --version + + - name: Build MSI installer + shell: powershell + working-directory: ${{ github.workspace }}/src + run: .\wix\build.ps1 diff --git a/.github/script/asr_gate.sh b/.github/script/asr_gate.sh deleted file mode 100755 index 5dd41aab3..000000000 --- a/.github/script/asr_gate.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash -# Decides whether a change needs the ASR test (.github/workflows/asr-test.yml). -# -# asr_gate.sh e.g. asr_gate.sh origin/main...HEAD or asr_gate.sh .. -# -# The workflow's `paths:` filter is deliberately wide: src/server/rest_handler.cpp and src/server/server.cpp host -# /v1/audio/transcriptions, but they also hold every other endpoint, so every server change triggers the workflow. This -# script narrows that down on a hosted runner, before an NPU machine is reserved: -# -# * a changed file that only serves ASR (the Whisper engine, its kernels and libraries, the audio decoders and front -# ends, the multipart parser, the ASR test and its data, this workflow) always runs the test; -# * a change to the shared server files runs the test only if a changed line mentions audio or speech handling -# (audio, transcription, whisper, multipart, speech, asr, wav, mp3, flac, ogg, opus, m4a, pcm, resample); -# * anything else (e.g. a model-family change that adds a tag to the REST handler) skips it. -# -# When in doubt it runs the test: an unreadable range, or no files at all, means "run". Prints "run=true|false" on -# stdout (the workflow appends it to $GITHUB_OUTPUT) and the reason on stderr. Keep asr_only below in step with the -# `paths:` lists in asr-test.yml: everything there except the two shared server files. -set -euo pipefail - -range=${1:-} -shared='^src/server/(rest_handler|server)\.cpp$' -asr_only='^(src/common/whisper/|src/include/whisper/|src/xclbins/Whisper-V3-Turbo-NPU2/|src/lib/(xrt|hrx)/[^/]*whisper_npu|src/common/audio_process_utils/|src/include/audio_process_utils/|src/common/audio/|src/include/audio/|src/common/AutoModel/[^/]*_audio\.cpp$|src/detail/gemma4e_npu/gemma4e_audio|src/include/pocketfft/|src/include/dr_libs/|src/cmake/FlmAudioCodecs\.cmake$|src/server/multipart\.(cpp|hpp)$|src/test/whisper_npu/|\.github/script/(asr_test\.py|asr_summary\.py|asr_gate\.sh|sweep_common\.py)$|\.github/script/assets/asr_eval/|\.github/workflows/asr-test\.yml$)' -asr_words='audio|transcri|whisper|multipart|speech|\basr\b|\bwav\b|mp3|flac|\bogg\b|opus|m4a|\bpcm\b|resampl' - -say() { echo "run=$1"; echo "asr_gate: $2" >&2; exit 0; } - -[ -n "$range" ] || say true "no diff range given: running" -files=$(git diff --name-only "$range" 2>/dev/null) || say true "cannot diff $range (shallow clone or force-push): running" -[ -n "$files" ] || say true "no changed files found in $range: running" - -hit=$(printf '%s\n' "$files" | grep -E "$asr_only" || true) -[ -z "$hit" ] || say true "ASR-only files changed: $(printf '%s' "$hit" | head -3 | tr '\n' ' ')" - -touched=$(printf '%s\n' "$files" | grep -E "$shared" || true) -[ -n "$touched" ] || say false "no ASR-related files changed" - -# Only the shared server files changed: look at the changed lines themselves. -touched_list=$(echo $touched) -# shellcheck disable=SC2086 -lines=$(git diff -U0 "$range" -- $touched_list | grep -E '^[+-]' | grep -v -E '^(\+\+\+|---)' || true) -if printf '%s\n' "$lines" | grep -q -i -E "$asr_words"; then - say true "changed lines in $touched_list mention audio or speech: $(printf '%s\n' "$lines" | grep -i -E "$asr_words" | head -1 | cut -c1-100)" -fi -say false "only $touched_list changed, and no changed line is about audio or speech" diff --git a/.github/script/asr_summary.py b/.github/script/asr_summary.py deleted file mode 100644 index 9b55203c3..000000000 --- a/.github/script/asr_summary.py +++ /dev/null @@ -1,298 +0,0 @@ -#!/usr/bin/env python3 -"""Render ASR test results as one Markdown report, a section per model. - -Reads every *_results.json that asr_test.py wrote under the given directory -(only those in an asr-results folder when the artifacts also hold baseline -runs) and prints GitHub-flavoured Markdown to stdout, for $GITHUB_STEP_SUMMARY. -Each model gets its own section with the platforms side by side, so a reader -can see at once which model failed where and why. - -asr_test.py also calls render() for the report of a single run. -""" - -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path - -PLATFORM_ORDER = ["linux", "linux-hrx", "windows", "windows-hrx", - "linux-models-from-source", "windows-models-from-source"] -PLATFORM_LABELS = { - "linux": "Linux", "linux-hrx": "Linux HRX", "windows": "Windows", - "windows-hrx": "Windows HRX", "linux-models-from-source": "Linux (models from source)", - "windows-models-from-source": "Windows (models from source)", -} -WHISPER_PREFIX = "whisper-v3" - - -def pct(value) -> str: - return "-" if value is None else f"{value * 100:.2f}%" - - -def points(current, base) -> str: - if current is None or base is None: - return "" - return f" ({(current - base) * 100:+.2f})" - - -def label(platform: str) -> str: - return PLATFORM_LABELS.get(platform, platform) - - -def cell(text: str) -> str: - return " ".join(str(text).split()).replace("|", "\\|") - - -def reference_name(result: dict) -> str: - base = result.get("baseline") or {} - if base.get("kind") == "build": - return "main" - if base.get("kind") == "expected": - return "expected" - return "" - - -def model_checks(result: dict, model: str) -> list[dict]: - return [c for c in result.get("checks", []) if c["model"] == model] - - -def verdict(result: dict | None, model: str) -> str: - if result is None: - return "no results" - checks = model_checks(result, model) - if not checks: - return "not run" - return "**FAIL**" if any(c["status"] == "FAIL" for c in checks) else "**PASS**" - - -def whisper_models(results: list[dict]) -> list[str]: - models = [] - for result in results: - for check in result.get("checks", []): - if check["model"].startswith(WHISPER_PREFIX) and check["model"] not in models: - models.append(check["model"]) - return models or [f"{WHISPER_PREFIX}:turbo"] - - -def chat_models(results: list[dict]) -> list[str]: - models = [] - for result in results: - for model in result.get("chat", {}): - if model not in models: - models.append(model) - return models - - -def problems(lines: list[str], results: list[dict], model: str) -> None: - """Failed checks and warnings for one model, grouped by platform.""" - lines += ["**Failed checks and warnings**", ""] - any_listed = False - for result in results: - rows = [c for c in model_checks(result, model) if c["status"] != "ok"] - if not rows: - continue - any_listed = True - lines.append(f"*{label(result['platform'])}*") - lines.append("") - for row in rows: - status = "FAIL" if row["status"] == "FAIL" else "warning" - group = "" if row["group"] == "core" else f" [{row['group']}]" - detail = f": {cell(row['detail'])}" if row["detail"] else "" - lines.append(f"- **{status}**{group} {cell(row['area'])}: " - f"{cell(row['check'])}{detail}") - lines.append("") - if not any_listed: - lines += ["_None._", ""] - - -def whisper_section(lines: list[str], results: list[dict], model: str) -> None: - columns = [label(r["platform"]) for r in results] - ref = next((reference_name(r) for r in results if reference_name(r)), "") - lines += [f"## {model}: speech-to-text (`/v1/audio/transcriptions`)", "", - "**Verdict:** " + " · ".join(f"{label(r['platform'])} {verdict(r, model)}" - for r in results), ""] - - lines += ["**Accuracy**", "", - "Error rate on each FLEURS set: WER, or CER for ja and zh. " - + (f"In brackets, the change in points against {ref}. " if ref else "") - + "Lower is better.", "", - "| Set | Metric | " + " | ".join(columns) + " |", - "|---|---|" + "---|" * len(columns)] - set_names: list[str] = [] - for result in results: - set_names += [s for s in result.get("sets", {}) if s not in set_names] - for set_name in set_names: - metric = next((r["sets"][set_name]["metric"].upper() for r in results - if set_name in r.get("sets", {})), "") - cells = [] - for result in results: - current = result.get("sets", {}).get(set_name) - base = ((result.get("baseline") or {}).get("sets", {}).get(set_name) - or {}).get("error_rate") - cells.append("-" if current is None else - pct(current["error_rate"]) + points(current["error_rate"], base)) - lines.append(f"| {set_name} | {metric} | " + " | ".join(cells) + " |") - cells = [] - for result in results: - current = (result.get("overall") or {}).get("error_rate") - base = ((result.get("baseline") or {}).get("overall") or {}).get("error_rate") - cells.append(f"**{pct(current)}**{points(current, base)}") - lines.append("| **pooled** | all | " + " | ".join(cells) + " |") - lines.append("") - - lines += ["**Behaviour**", "", - "| Measure | " + " | ".join(columns) + " |", - "|---|" + "---|" * len(columns)] - - def row(name: str, render) -> None: - lines.append(f"| {name} | " + " | ".join(render(r) for r in results) + " |") - - def runaways(result: dict) -> str: - current = (result.get("overall") or {}).get("runaways") - base = ((result.get("baseline") or {}).get("overall") or {}).get("runaways") - if current is None: - return "-" - return f"{current}" + (f" ({ref} {base})" if base is not None else "") - - def changed(result: dict) -> str: - sets = result.get("sets", {}).values() - if not any("changed" in s for s in sets): - return "-" - return f"{sum(s.get('changed', 0) for s in sets)}/{sum(s['files'] for s in sets)}" - - def speed(result: dict) -> str: - sets = result.get("sets", {}) - audio = sum(s.get("audio_s", 0) for s in sets.values()) - wall = sum(s.get("wall_s", 0) for s in sets.values()) - base_sets = (result.get("baseline") or {}).get("sets", {}) - base_audio = sum(s.get("audio_s", 0) for s in base_sets.values()) - base_wall = sum(s.get("wall_s", 0) for s in base_sets.values()) - text = f"{wall / audio:.3f}" if audio else "-" - if base_audio and ref == "main": - text += f" (main {base_wall / base_audio:.3f})" - return text - - row("Runaway transcripts (repetition loops)", runaways) - row("Files whose transcript changed vs main", changed) - row("Processing seconds per audio second", speed) - lines.append("") - - groups: list[str] = [] - for result in results: - groups += [g for g in result.get("features", {}) if g not in groups] - lines += ["**Checks by group** (passed/total; required groups fail the run)", "", - "| Group | Required | " + " | ".join(columns) + " |", - "|---|---|" + "---|" * len(columns)] - for group in groups: - required = "yes" if group == "core" or group in results[0].get("required", []) else "no" - cells = [] - for result in results: - checks = [c for c in model_checks(result, model) if c["group"] == group] - if not checks: - cells.append("-") - continue - passed = sum(c["status"] == "ok" for c in checks) - cells.append(f"{passed}/{len(checks)}") - lines.append(f"| {group} | {required} | " + " | ".join(cells) + " |") - lines.append("") - problems(lines, results, model) - - -def chat_section(lines: list[str], results: list[dict], model: str) -> None: - columns = [label(r["platform"]) for r in results] - lines += [f"## {model}: chat transcription (audio sent to the chat model)", "", - "**Verdict:** " + " · ".join(f"{label(r['platform'])} {verdict(r, model)}" - for r in results), "", - "| Measure | " + " | ".join(columns) + " |", - "|---|" + "---|" * len(columns)] - cells, clips = [], [] - for result in results: - current = result.get("chat", {}).get(model) - base = ((result.get("baseline") or {}).get("chat") or {}).get(model) - if current is None: - cells.append("-") - clips.append("-") - continue - text = pct(current["error_rate"]) - if base: - text += f" (main {pct(base['error_rate'])})" - cells.append(text) - clips.append(str(current["files"])) - lines.append("| WER on English clips | " + " | ".join(cells) + " |") - lines.append("| Clips | " + " | ".join(clips) + " |") - lines.append("") - problems(lines, results, model) - - -def render(results: list[dict], title: str = "ASR Test") -> str: - results = sorted(results, key=lambda r: (PLATFORM_ORDER.index(r["platform"]) - if r["platform"] in PLATFORM_ORDER else 99, - r["platform"])) - lines = [f"# {title}", ""] - if not results: - return "\n".join(lines + ["_No results were produced._", ""]) - - whisper = whisper_models(results) - chats = chat_models(results) - refs = {reference_name(r) for r in results} - {""} - required = results[0].get("required", []) - lines += [ - "Builds under test: flm " + ", ".join(sorted({str(r.get('version')) for r in results})) - + ". Compared against: " + (" and ".join( - "main's latest build, run first on the same machine" if r == "main" - else "the expected rates in asr_eval/expected.json" for r in sorted(refs)) - or "nothing") + ".", - "", - "Required feature groups: " + (", ".join(required) or "none") - + ". Other groups are measured and reported as warnings.", - "", - "| Model | " + " | ".join(label(r["platform"]) for r in results) + " |", - "|---|" + "---|" * len(results), - ] - for model in whisper + chats: - lines.append(f"| {model} | " + " | ".join(verdict(r, model) for r in results) + " |") - lines.append("") - - for model in whisper: - whisper_section(lines, results, model) - for model in chats: - chat_section(lines, results, model) - - other = [c for r in results for c in r.get("checks", []) - if c["status"] == "FAIL" and c["model"] not in whisper + chats] - if other: - lines += ["## Other failures", ""] - lines += [f"- {cell(c['model'])}: {cell(c['check'])} {cell(c['detail'])}" for c in other] - lines.append("") - return "\n".join(lines) - - -def load_results(root: Path) -> list[dict]: - paths = sorted(root.rglob("*_results.json")) - # Artifacts also carry the baseline's own run; report the builds under test. - if any("asr-results" in p.parts for p in paths): - paths = [p for p in paths if "asr-results" in p.parts] - results = [] - for path in paths: - try: - result = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: - print(f"") - continue - if "platform" in result and "checks" in result: - results.append(result) - return results - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument("directory", help="Directory holding the downloaded artifacts.") - args = parser.parse_args(argv) - print(render(load_results(Path(args.directory)))) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/script/asr_test.py b/.github/script/asr_test.py deleted file mode 100644 index c6e5d8dc0..000000000 --- a/.github/script/asr_test.py +++ /dev/null @@ -1,725 +0,0 @@ -#!/usr/bin/env python3 -"""Measure and check ASR through /v1/audio/transcriptions and chat audio. - -sweep_audio.py only asks for a non-empty transcript. This runs the FLEURS -evaluation set in assets/asr_eval/ (see build_asr_eval.py) and measures: - - accuracy WER per set, CER for ja and zh, against human transcripts, - each file's errors capped at its reference length so one - runaway transcript cannot swamp a set; and a pooled rate - over every set - runaways transcripts over 1.5 times the reference length, which is - what a decoder stuck in a repetition loop produces - speed seconds of processing per second of audio - formats json, no response_format and verbose_json agree - chat audio WER of chat models that transcribe audio: Gemma 4 E2B, whose - audio front end E4B and the flash variants share, and 12B, - which has its own - -and checks these feature groups: - - verbose_json segments that are ordered, inside the clip, non-empty and - join into the text; duration equal to the clip length - timestamp_rules segments never overlap; the first one starts within 1 s - window_tail no text repeated across 30 s window boundaries - language the language field is honoured; unknown codes fail - prompt a prompt is accepted and helps with the words it names - quiet_log transcripts stay out of the server log - -A failed check in a group fails the run only if the group is listed in ---require; otherwise it is reported as "warn". Core checks always count. The -groups exist so a feature can be required once main has it, and measured -before then. - -With --baseline, the results of another build (normally main's, run on the -same machine just before) become the reference. The run fails if the pooled -error rate rises by more than --tolerance points, the rate of any set of ten -or more files by more than --set-tolerance, or a file turns into a runaway, or -repeats more phrases, while also getting more errors than in the baseline. -The long-form sets are left to the pooled rate and those last two checks: a -single file there moves by many points whenever one decode flips, which HRX -builds do between identical runs. The report also shows how -many transcripts changed. Even a numerically equivalent change, such as a -different FFT, flips a few borderline decodes, so a few changed files and -small per-set moves are expected; --max-drift is for builds that should -reproduce the baseline exactly. Without a baseline, --expected gives fixed -error rates to stay under, with the same --tolerance on the pooled rate and ---set-tolerance per set; the workflow passes --tolerance 2 there, since those -rates come from one machine per platform. - -Every response body, and a results JSON that --baseline can read back, are -written to --output-dir. -""" - -from __future__ import annotations - -import base64 -import json -import re -import socket -import sys -import time -import unicodedata -from collections import Counter -from pathlib import Path - -from asr_summary import label as platform_label, render -from sweep_common import ASSETS_DIR, SweepTask, run_task - -FEATURE_GROUPS = [ - "verbose_json", "timestamp_rules", "window_tail", "language", "prompt", "quiet_log", -] -# verbose_json's language, as OpenAI reports it: openai-whisper's lower-case names. -LANGUAGE_NAMES = { - "en": "english", "de": "german", "es": "spanish", "fr": "french", - "ja": "japanese", "zh": "chinese", "ar": "arabic", -} -CHAT_PROMPT = ( - "Transcribe the speech in this audio exactly, in its original language. " - "Reply with the transcript only." -) -# MP3 encoder padding and the server's own rounding. -DURATION_TOLERANCE_S = 0.1 -# The encoder is not bit-exact between passes, so two requests for the same -# clip can differ in a word or two (see the PR #794 notes). -SAME_TEXT_MAX_WER = 0.02 -REPEAT_NGRAM = 6 -RUNAWAY_RATIO = 1.5 -# Smaller sets are judged through the pooled rate only; see the docstring. -SET_CHECK_MIN_FILES = 10 -ARABIC_DIACRITICS = re.compile("[\u064b-\u065f\u0670\u06d6-\u06ed]") - - -# -- scoring ------------------------------------------------------------------ - -def normalize(text: str, language: str) -> str: - """Lowercase, no punctuation or symbols, single spaces, NFKC.""" - text = unicodedata.normalize("NFKC", text).lower() - if language == "ar": - text = ARABIC_DIACRITICS.sub("", text) - text = "".join(" " if unicodedata.category(c)[0] in "PS" else c for c in text) - return " ".join(text.split()) - - -def tokens(text: str, language: str, metric: str) -> list[str]: - text = normalize(text, language) - return list(text.replace(" ", "")) if metric == "cer" else text.split() - - -def edit_distance(ref: list[str], hyp: list[str]) -> int: - previous = list(range(len(hyp) + 1)) - for i, ref_token in enumerate(ref, 1): - current = [i] - for j, hyp_token in enumerate(hyp, 1): - current.append(min(previous[j] + 1, current[j - 1] + 1, - previous[j - 1] + (ref_token != hyp_token))) - previous = current - return previous[-1] - - -def error_rate(ref: str, hyp: str, language: str, metric: str) -> tuple[int, int]: - """(edits, reference length), so rates can be pooled over a set.""" - ref_tokens = tokens(ref, language, metric) - return edit_distance(ref_tokens, tokens(hyp, language, metric)), len(ref_tokens) - - -def pooled(pairs) -> float: - """Error rate over many files, each file's edits capped at its length.""" - edits = sum(min(e, n) for e, n in pairs) - length = sum(n for _, n in pairs) - return edits / length if length else 0.0 - - -def repeats(text: str, n: int = REPEAT_NGRAM) -> int: - """n-word sequences that occur more than once.""" - words = normalize(text, "en").split() - counts = Counter(tuple(words[i:i + n]) for i in range(len(words) - n + 1)) - return sum(c - 1 for c in counts.values() if c > 1) - - -def pct(value: float | None) -> str: - return "-" if value is None else f"{value * 100:.2f}%" - - -# -- test --------------------------------------------------------------------- - -class AsrTest(SweepTask): - name = "asr" - needs_asr = True - csv_header = ["Model", "Area", "Group", "Check", "Status", "Detail"] - - @staticmethod - def add_arguments(parser) -> None: - parser.add_argument("--eval-dir", default=str(ASSETS_DIR / "asr_eval"), - help="Directory holding manifest.json and the clips.") - parser.add_argument("--sets", nargs="+", default=None, metavar="SET", - help="Run only these sets (default: all).") - parser.add_argument("--require", nargs="*", default=[], metavar="GROUP", - choices=FEATURE_GROUPS, - help="Feature groups whose failures fail the run.") - parser.add_argument("--baseline", default=None, metavar="RESULTS_JSON", - help="Results of a reference build to compare against.") - parser.add_argument("--expected", default=None, metavar="JSON", - help="Error rates per platform and set to stay under when there " - "is no baseline. Builds decode differently per platform, " - "so only --platform's entry is used.") - parser.add_argument("--tolerance", type=float, default=1.0, - help="Allowed rise in the pooled error rate, in points (default: 1.0).") - parser.add_argument("--set-tolerance", type=float, default=5.0, - help="Allowed rise in the error rate of any set of " - f"{SET_CHECK_MIN_FILES} or more files, in points " - "(default: 5.0). One flipped decode moves a ten-file " - "set by a point or more.") - parser.add_argument("--max-drift", type=float, default=None, - help="Fail a set whose transcripts differ from the baseline's " - "by more than this fraction, for builds meant to change " - "nothing. Unset: drift is only reported.") - parser.add_argument("--chat-models", nargs="*", - default=["gemma4-it:e2b", "gemma4-it:12b"], metavar="TAG", - help="Chat models to transcribe with (default: gemma4-it:e2b " - "gemma4-it:12b). Pass none to skip.") - parser.add_argument("--chat-clips", type=int, default=5, - help="en-short clips sent to each chat model (default: 5).") - parser.add_argument("--chat-tolerance", type=float, default=3.0, - help="Allowed rise in chat WER against the baseline, in points.") - - def __init__(self, args): - super().__init__(args) - self.rows: list[list[str]] = [] - self.check_failures: set[str] = set() - self.results: dict = {"sets": {}, "items": {}, "chat": {}, "features": {}, - "repeats": {}, "language": {}} - self.baseline = (json.loads(Path(args.baseline).read_text(encoding="utf-8")) - if args.baseline else None) - self.expected = None - if args.expected: - platforms = json.loads(Path(args.expected).read_text(encoding="utf-8"))["platforms"] - self.expected = platforms.get(args.platform) - if self.expected is None: - print(f"::warning::{args.expected} has no expected rates for " - f"{args.platform}; accuracy is reported but not checked.") - self.responses_dir = self.output_dir / "responses" - self.responses_dir.mkdir(exist_ok=True) - - def start_server(self, append_log: bool = False) -> None: - """Refuses to start when another server already holds the port. - - flm exits at once when it cannot bind, but the other server answers - /v1/version, so the run would carry on against a server without ASR, - or one that is running someone else's job. - """ - if not self.args.no_server: - try: - socket.create_connection((self.args.host, self.args.port), timeout=2).close() - except OSError: - pass - else: - raise RuntimeError(f"Port {self.args.port} on {self.args.host} is already in " - "use by another server; pass --port to use a free one.") - super().start_server(append_log) - - def select_models(self, catalog: list[dict]) -> list[str]: - models = [e["id"] for e in catalog if e["id"].startswith("whisper-v3")] - if not models: - print("[asr] No Whisper models found. The /v1/models fallback hides " - "Whisper, so pass --models explicitly if 'flm list' failed.") - return models - - # -- recording --------------------------------------------------------- - - def check(self, writer, model: str, area: str, name: str, ok: bool, - detail: str = "", group: str = "core") -> None: - required = group == "core" or group in self.args.require - status = "ok" if ok else ("FAIL" if required else "warn") - row = [model, area, group, name, status, detail] - self.rows.append(row) - error = None if ok or not required else f"{area}: {name} ({detail})" - if error: - self.check_failures.add(f"{model}: {error}") - self.record(writer, row, model=model, error=error) - feature = self.results["features"].setdefault(group, {"passed": 0, "total": 0}) - feature["total"] += 1 - feature["passed"] += ok - if not ok or group == "core": - print(f" {status:4} [{group}] {area}: {name}" + (f" ({detail})" if detail else "")) - - def save_body(self, model: str, label: str, text: str) -> None: - safe = re.sub(r"[^\w.-]", "_", f"{model}_{label}") - (self.responses_dir / f"{safe}.json").write_text(text, encoding="utf-8") - - # -- requests ---------------------------------------------------------- - - def transcribe(self, model: str, path: Path, label: str, **fields) -> tuple[int, dict | None, float]: - """(HTTP status, parsed body or None, seconds). The body is saved first.""" - kwargs = {key: value for key, value in fields.items() if value is not None} - started = time.monotonic() - with open(path, "rb") as handle: - raw = self.client.audio.transcriptions.with_raw_response.create( - model=model, file=handle, **kwargs) - seconds = time.monotonic() - started - response = raw.http_response - self.save_body(model, label, response.text) - try: - body = json.loads(response.text) - except json.JSONDecodeError: - body = None - return response.status_code, body, seconds - - def request(self, writer, model: str, path: Path, label: str, area: str, - **fields) -> tuple[dict | None, float]: - """A request that is expected to succeed; failures become core checks.""" - try: - status, body, seconds = self.transcribe(model, path, label, **fields) - except Exception as exc: - self.check(writer, model, area, f"{label}: request succeeds", False, str(exc)) - return None, 0.0 - if status >= 400 or body is None or "error" in body: - # The server answers with an empty body when ASR is not loaded. - self.check(writer, model, area, f"{label}: request succeeds", False, - f"HTTP {status}: {json.dumps(body)[:120] if body else 'not JSON'}") - return None, seconds - return body, seconds - - # -- formats ----------------------------------------------------------- - - def same_text(self, a: str, b: str) -> tuple[bool, str]: - edits, length = error_rate(a, b, "en", "wer") - rate = edits / length if length else float(bool(b.strip())) - return rate <= SAME_TEXT_MAX_WER, f"{rate:.1%} apart" if rate else "identical" - - def check_segments(self, writer, model: str, area: str, body: dict, duration: float, - failures: Counter) -> None: - """verbose_json and timestamp_rules checks, tallied into `failures`.""" - segments = body.get("segments") or [] - text = body.get("text") or "" - problems = { - "duration is the clip length": - not isinstance(body.get("duration"), (int, float)) - or abs(body["duration"] - duration) > DURATION_TOLERANCE_S, - "segment ids count up from 0": - [s.get("id") for s in segments] != list(range(len(segments))), - "segments lie inside the clip": - any(not 0 <= s["start"] <= s["end"] <= duration + DURATION_TOLERANCE_S - for s in segments), - "segments are in order": - any(b["start"] < a["start"] for a, b in zip(segments, segments[1:])), - "no empty or zero-length segments": - any(s["end"] - s["start"] < 0.01 or not s["text"].strip() for s in segments), - "segments join into the text": - "".join(s["text"] for s in segments).strip() != text.strip(), - } - for name, bad in problems.items(): - failures[("verbose_json", name)] += bad - rules = { - "segments never overlap": - any(b["start"] < a["end"] - 0.005 for a, b in zip(segments, segments[1:])), - "first segment starts within 1 s": - bool(segments) and segments[0]["start"] > 1.0, - } - for name, bad in rules.items(): - failures[("timestamp_rules", name)] += bad - - def run_formats(self, writer, model: str, item: dict, path: Path) -> None: - """json, default and verbose_json on one clip must agree.""" - area = f"formats ({item['file']})" - plain, _ = self.request(writer, model, path, "formats_json", area, response_format="json") - default, _ = self.request(writer, model, path, "formats_default", area) - verbose, _ = self.request(writer, model, path, "formats_verbose_json", area, - response_format="verbose_json") - if plain is None: - return - text = plain.get("text") or "" - self.check(writer, model, area, "json text is non-empty", bool(text.strip())) - if default is not None: - self.check(writer, model, area, "default returns only model and text", - set(default) == {"model", "text"}, ", ".join(sorted(default))) - ok, detail = self.same_text(text, default.get("text") or "") - self.check(writer, model, area, "default text matches json", ok, detail) - if verbose is not None: - has_segments = "segments" in verbose - self.check(writer, model, area, "verbose_json returns segments", has_segments, - "" if has_segments else "server ignores verbose_json", - group="verbose_json") - if has_segments: - ok, detail = self.same_text(text, verbose.get("text") or "") - self.check(writer, model, area, "verbose_json text matches json", ok, detail, - group="verbose_json") - - # -- evaluation -------------------------------------------------------- - - def run_eval(self, writer, model: str, items: list[dict], eval_dir: Path) -> None: - """One verbose_json request per clip: accuracy, speed and segments.""" - segment_failures: Counter = Counter() - verbose_items = 0 - by_set: dict[str, list[dict]] = {} - for item in items: - by_set.setdefault(item["set"], []).append(item) - - for set_name, set_items in by_set.items(): - print(f" set {set_name}: {len(set_items)} files") - pairs, audio_s, wall_s, detected, runaways = [], 0.0, 0.0, 0, 0 - for item in set_items: - body, seconds = self.request(writer, model, eval_dir / item["file"], - f"eval_{item['file']}", f"eval {set_name}", - response_format="verbose_json") - if body is None: - continue - text = body.get("text") or "" - edits, length = error_rate(item["reference"], text, item["language"], - item["metric"]) - pairs.append((edits, length)) - runaway = len(tokens(text, item["language"], item["metric"])) > RUNAWAY_RATIO * length - runaways += runaway - audio_s += item["duration"] - wall_s += seconds - self.results["items"][item["file"]] = { - "set": set_name, "language": item["language"], "metric": item["metric"], - "text": text, "seconds": seconds, "edits": edits, "length": length, - "runaway": runaway} - if body.get("language") == LANGUAGE_NAMES[item["language"]]: - detected += 1 - if "segments" in body: - verbose_items += 1 - self.check_segments(writer, model, f"eval {set_name}", body, - item["duration"], segment_failures) - if item["set"] in ("en-long", "en-longform"): - self.results["repeats"][item["file"]] = { - "reference": repeats(item["reference"]), "text": repeats(text)} - - rate = pooled(pairs) - self.results["sets"][set_name] = { - "metric": set_items[0]["metric"], "error_rate": rate, "files": len(pairs), - "runaways": runaways, - "audio_s": audio_s, "wall_s": wall_s, - "rtf": wall_s / audio_s if audio_s else None, - "language_detected": detected if verbose_items else None, - } - print(f" {set_items[0]['metric'].upper()} {pct(rate)}, {runaways} runaways, " - f"{audio_s:.0f} s of audio in {wall_s:.1f} s") - self.check(writer, model, f"eval {set_name}", "every file transcribed", - len(pairs) == len(set_items), f"{len(pairs)}/{len(set_items)}") - - all_items = self.results["items"].values() - self.results["overall"] = { - "error_rate": pooled([(r["edits"], r["length"]) for r in all_items]), - "runaways": sum(r["runaway"] for r in all_items), - "files": len(self.results["items"]), - } - print(f" pooled error rate {pct(self.results['overall']['error_rate'])}, " - f"{self.results['overall']['runaways']} runaways") - - # Without segments, run_formats has already reported verbose_json. - for (group, name), bad in sorted(segment_failures.items()): - self.check(writer, model, "eval", name, bad == 0, - f"{bad} of {verbose_items} files" if bad else f"{verbose_items} files", - group=group) - - for name, counts in self.results["repeats"].items(): - self.check(writer, model, f"eval {name}", - f"no {REPEAT_NGRAM}-word phrase repeated beyond the reference", - counts["text"] <= counts["reference"], - f"{counts['text']} repeats, reference {counts['reference']}", - group="window_tail") - - # -- feature probes ---------------------------------------------------- - - def run_language(self, writer, model: str, items: list[dict], eval_dir: Path) -> None: - """Forcing the language: same or better accuracy, and actually honoured.""" - group, area = "language", "language" - for language in sorted({i["language"] for i in items} - {"en"}): - probe = [i for i in items if i["language"] == language][:3] - pairs, reported = [], [] - for item in probe: - body, _ = self.request(writer, model, eval_dir / item["file"], - f"language_{item['file']}", area, - response_format="verbose_json", language=language) - if body is not None: - pairs.append(error_rate(item["reference"], body.get("text") or "", - language, item["metric"])) - if "segments" in body: - reported.append(body.get("language")) - auto = pooled([(self.results["items"][i["file"]]["edits"], - self.results["items"][i["file"]]["length"]) - for i in probe if i["file"] in self.results["items"]]) - forced = pooled(pairs) - self.results["language"][language] = {"auto": auto, "forced": forced} - self.check(writer, model, area, f"{language}: forcing it does not hurt", - len(pairs) == len(probe) - and forced <= auto + self.args.set_tolerance / 100, - f"{pct(forced)} forced, {pct(auto)} detected", group=group) - # Only verbose_json reports the language. Forcing a wrong language - # is no test: Whisper often keeps transcribing what it hears. - if reported: - self.check(writer, model, area, f"{language}: response reports it", - all(r == LANGUAGE_NAMES[language] for r in reported), - ", ".join(sorted({str(r) for r in reported})), group=group) - - english = next((i for i in items if i["set"] == "en-short"), None) - if english is None: - return - path = eval_dir / english["file"] - try: - status, bad_body, _ = self.transcribe(model, path, "language_invalid", language="xx") - rejected = status >= 400 or bad_body is None or "error" in bad_body - detail = f"HTTP {status}" - except Exception as exc: - rejected, detail = True, type(exc).__name__ - self.check(writer, model, area, "an unknown language code is rejected", rejected, - detail, group=group) - - def run_prompt(self, writer, model: str, items: list[dict], eval_dir: Path) -> None: - """A prompt naming the clip's proper nouns should not hurt, and should help them.""" - group, area = "prompt", "prompt" - probe = [i for i in items if i["set"] == "en-short"][:5] - hits_without = hits_with = names_total = 0 - pairs_without, pairs_with = [], [] - for item in probe: - names = sorted({w.strip(".,;:!?\"'()") for w in item["raw_reference"].split()[1:] - if w[:1].isupper()} - {""}) - if not names: - continue - body, _ = self.request(writer, model, eval_dir / item["file"], - f"prompt_{item['file']}", area, prompt=" ".join(names)) - before = self.results["items"].get(item["file"]) - if body is None or before is None: - continue - after = body.get("text") or "" - pairs_with.append(error_rate(item["reference"], after, "en", "wer")) - pairs_without.append((before["edits"], before["length"])) - names_total += len(names) - hits_without += sum(n in before["text"] for n in names) - hits_with += sum(n in after for n in names) - if not names_total: - return - with_rate, without_rate = pooled(pairs_with), pooled(pairs_without) - self.results["prompt"] = {"names": names_total, "hits_with": hits_with, - "hits_without": hits_without, - "wer_with": with_rate, "wer_without": without_rate} - self.check(writer, model, area, "a prompt does not hurt accuracy", - with_rate <= without_rate + self.args.set_tolerance / 100, - f"WER {pct(with_rate)} with, {pct(without_rate)} without", group=group) - self.check(writer, model, area, "names in the prompt are spelled as given", - hits_with >= hits_without, - f"{hits_with}/{names_total} with, {hits_without}/{names_total} without", - group=group) - - def run_quiet_log(self, writer, model: str) -> None: - # Streamed output interleaves timestamp markers with the text. - log = re.sub(r"<\|[\d.]+\|>", "", self.server_log_path.read_text( - encoding="utf-8", errors="replace")) - log = "".join(log.split()) - leaked = [f for f, r in self.results["items"].items() - if len("".join(r["text"].split())) >= 20 - and "".join(r["text"].split())[:20] in log] - self.check(writer, model, "server log", "transcripts are not logged", not leaked, - f"{len(leaked)} of {len(self.results['items'])} transcripts found" - if leaked else "", group="quiet_log") - - # -- chat audio -------------------------------------------------------- - - def run_chat(self, writer, items: list[dict], eval_dir: Path) -> None: - probe = [i for i in items if i["set"] == "en-short"][:self.args.chat_clips] - for model in self.args.chat_models: - print(f"\n--- Chat transcription: {model} ---") - pairs, failed = [], 0 - for item in probe: - encoded = base64.b64encode((eval_dir / item["file"]).read_bytes()).decode() - try: - response = self.client.chat.completions.create( - model=model, temperature=0.0, stream=False, - messages=[{"role": "user", "content": [ - {"type": "text", "text": CHAT_PROMPT}, - {"type": "input_audio", - "input_audio": {"data": encoded, "format": "mp3"}}, - ]}], - **self.completion_kwargs()) - text = response.choices[0].message.content or "" - except Exception as exc: - failed += 1 - text = "" - print(f" ERROR {item['file']}: {exc}") - self.save_body(model, f"chat_{item['file']}", json.dumps({"text": text})) - pairs.append(error_rate(item["reference"], text, "en", "wer")) - rate = pooled(pairs) - self.results["chat"][model] = {"error_rate": rate, "files": len(probe)} - print(f" WER {pct(rate)} over {len(probe)} clips") - self.check(writer, model, "chat", "every clip transcribed", failed == 0, - f"{len(probe) - failed}/{len(probe)}") - self.check(writer, model, "chat", "WER is under 50%", rate < 0.5, pct(rate)) - base = (self.baseline or {}).get("chat", {}).get(model) - if base: - self.check(writer, model, "chat", "WER no worse than the baseline", - rate - base["error_rate"] <= self.args.chat_tolerance / 100, - f"{pct(rate)} vs {pct(base['error_rate'])}") - - # -- comparison -------------------------------------------------------- - - def baseline_overall(self) -> dict | None: - """The baseline's pooled rate over the same files, so --sets compares like with like.""" - base_items = (self.baseline or {}).get("items", {}) - shared = [base_items[f] for f in self.results["items"] if f in base_items] - if not shared: - return None - return {"error_rate": pooled([(r["edits"], r["length"]) for r in shared]), - "runaways": sum(r.get("runaway", False) for r in shared)} - - def compare(self, writer, model: str) -> None: - """Error rates against the baseline build, or the expected values.""" - tolerance = self.args.tolerance / 100 - set_tolerance = self.args.set_tolerance / 100 - overall = self.results["overall"] - if self.baseline: - base_items = self.baseline.get("items", {}) - base_overall = self.baseline_overall() - if base_overall: - self.check(writer, model, "eval", "pooled error rate no worse than the baseline", - overall["error_rate"] - base_overall["error_rate"] <= tolerance, - f"{pct(overall['error_rate'])} vs {pct(base_overall['error_rate'])}") - # A file that crosses the runaway length while making no more - # errors, for instance by also transcribing what the baseline - # dropped, is not a regression. - new_runaways = sorted( - f for f, r in self.results["items"].items() - if r.get("runaway") and f in base_items - and not base_items[f].get("runaway") - and r["edits"] > base_items[f]["edits"]) - self.check(writer, model, "eval", "no new runaway transcripts", - not new_runaways, - ", ".join(new_runaways) if new_runaways else - f"{overall['runaways']} runaways, baseline {base_overall['runaways']}") - for set_name, current in self.results["sets"].items(): - base = self.baseline["sets"].get(set_name) - if not base: - continue - delta = current["error_rate"] - base["error_rate"] - if current["files"] >= SET_CHECK_MIN_FILES: - self.check(writer, model, f"eval {set_name}", - "error rate no worse than the baseline", delta <= set_tolerance, - f"{pct(current['error_rate'])} vs {pct(base['error_rate'])}") - compared = [(f, r) for f, r in self.results["items"].items() - if r["set"] == set_name and f in base_items] - drift = pooled([error_rate(base_items[f]["text"], r["text"], r["language"], - r["metric"]) for f, r in compared]) - current["drift"] = drift - # Normalized, so whitespace and punctuation alone do not count. - current["changed"] = sum( - normalize(base_items[f]["text"], r["language"]) - != normalize(r["text"], r["language"]) for f, r in compared) - if self.args.max_drift is not None: - self.check(writer, model, f"eval {set_name}", - "transcripts match the baseline's", drift <= self.args.max_drift, - f"{pct(drift)} of words differ") - if base.get("rtf") and current.get("rtf") and current["rtf"] > base["rtf"] * 1.2: - print(f"::warning::{set_name}: {current['rtf']:.3f} s per audio second, " - f"baseline {base['rtf']:.3f}") - for name, counts in self.results["repeats"].items(): - base = self.baseline.get("repeats", {}).get(name) - if base and name in base_items: - more_errors = self.results["items"][name]["edits"] > base_items[name]["edits"] - self.check(writer, model, f"eval {name}", - "no more repeated phrases than the baseline", - counts["text"] <= base["text"] or not more_errors, - f"{counts['text']} vs {base['text']}") - elif self.expected: - limit = self.expected.get("overall") - if limit is not None: - self.check(writer, model, "eval", "pooled error rate within the expected value", - overall["error_rate"] <= limit + tolerance, - f"{pct(overall['error_rate'])} vs expected {pct(limit)}") - for set_name, current in self.results["sets"].items(): - limit = self.expected.get("sets", {}).get(set_name) - if limit is not None and current["files"] >= SET_CHECK_MIN_FILES: - self.check(writer, model, f"eval {set_name}", - "error rate within the expected value", - current["error_rate"] <= limit + set_tolerance, - f"{pct(current['error_rate'])} vs expected {pct(limit)}") - - # -- report ------------------------------------------------------------ - - @property - def report_path(self) -> Path: - return self.output_dir / f"{self.name}_{self.args.platform}_v{self.version}.md" - - @property - def results_path(self) -> Path: - return self.output_dir / f"{self.name}_{self.args.platform}_v{self.version}_results.json" - - def baseline_snapshot(self) -> dict | None: - """What this run was compared against, in one shape for asr_summary.py.""" - if self.baseline: - return { - "kind": "build", - "version": self.baseline.get("version"), - "overall": self.baseline_overall(), - "sets": {name: self.baseline["sets"][name] for name in self.results["sets"] - if name in self.baseline.get("sets", {})}, - "chat": self.baseline.get("chat", {}), - } - if self.expected: - return { - "kind": "expected", - "overall": {"error_rate": self.expected.get("overall")}, - "sets": {name: {"error_rate": rate} - for name, rate in self.expected.get("sets", {}).items()}, - } - return None - - def finish_results(self) -> None: - checks = [dict(zip(("model", "area", "group", "check", "status", "detail"), row)) - for row in self.rows] - # Failures that are not a check, such as the server crashing, still - # belong in the report. - checks += [{"model": "server", "area": "run", "group": "core", "check": failure, - "status": "FAIL", "detail": ""} - for failure in self.failures if failure not in self.check_failures] - self.results.update({ - "platform": self.args.platform, - "version": self.version, - "required": list(self.args.require), - "checks": checks, - "baseline": self.baseline_snapshot(), - }) - - def write_report(self) -> None: - self.report_path.write_text( - render([self.results], title=f"ASR test: {platform_label(self.args.platform)}") - + "\n", encoding="utf-8") - - # -- entry point ------------------------------------------------------- - - def run(self) -> None: - eval_dir = Path(self.args.eval_dir) - manifest = json.loads((eval_dir / "manifest.json").read_text(encoding="utf-8")) - items = [i for i in manifest["items"] - if not self.args.sets or i["set"] in self.args.sets] - print(f"[asr] {len(items)} files from {eval_dir}, " - f"required groups: {', '.join(self.args.require) or 'none'}") - self.results["version"] = self.version - - models = self.resolve_models() - if not models: - self.failures.append("no Whisper model to test") - handle, writer = self.open_csv() - try: - for model in self.iter_models(models): - print(f"\n--- ASR: {model} ---") - formats_item = next((i for i in items if i["set"] == "en-long"), items[0]) - self.run_formats(writer, model, formats_item, eval_dir / formats_item["file"]) - self.run_eval(writer, model, items, eval_dir) - self.run_language(writer, model, items, eval_dir) - self.run_prompt(writer, model, items, eval_dir) - self.run_quiet_log(writer, model) - self.compare(writer, model) - self.run_chat(writer, items, eval_dir) - finally: - handle.close() - self.finish_results() - self.results_path.write_text(json.dumps(self.results, indent=1, ensure_ascii=False), - encoding="utf-8") - self.write_report() - print(f"\nASR test complete. Report: {self.report_path}") - self.write_summary(models) - - -if __name__ == "__main__": - sys.exit(run_task(AsrTest)) diff --git a/.github/script/ci/changes.py b/.github/script/ci/changes.py new file mode 100644 index 000000000..a965e3408 --- /dev/null +++ b/.github/script/ci/changes.py @@ -0,0 +1,510 @@ +#!/usr/bin/env python3 +"""Decides, from the files a pull request or push changes, what CI must test. + + changes.py --base --head [--github-output] + changes.py --files src/server/server.cpp docs/index.md + +Outputs (printed, and appended to $GITHUB_OUTPUT with --github-output): + run_tests false only when every changed file is documentation or otherwise + never built or packaged, so no NPU runner is reserved for it + families comma-separated model families (details.family in + src/model_list.json) whose engine or catalog entry changed; the + pr tier tests every model in them instead of one per engine + asr_full true when speech-recognition code changed, so the pr tier runs + the whole ASR evaluation set + suites "all", or the comma-separated npu_test suites the change can + affect when every changed file is scoped to some (an ASR-only + change runs asr, a d3-only change runs decision) + summary one line for the job log + +When in doubt it tests more: an all-zero or missing base (a new branch), or a +range git cannot diff (shallow clone, force-push), counts as "everything +changed". It still names no families then: the pr tier already runs one model +of every engine, and testing every model of every family is the nightly job. + +Standard library only; --files skips git for the file list, for tests. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Iterable, Optional + +REPO_ROOT = Path(__file__).resolve().parents[3] +MODEL_LIST = "src/model_list.json" +MODEL_INFO = "src/model_info.json" + +# Files that are never built, packaged or read by a test. A change made only +# of these skips the NPU tests. LICENSE_*.txt and THIRD_PARTY_NOTICES.txt are +# not here: the packages ship them. +NON_CODE = re.compile( + r"^(docs/" + r"|assets/" # README images and the like, at the repo root + r"|src/tb_files/" + r"|src/iceCream\.jpg$" + r"|\.github/ISSUE_TEMPLATE/" + r"|\.github/(PULL_REQUEST_TEMPLATE[^/]*|CODEOWNERS|FUNDING\.yml)$" + r"|(.*/)?\.gitignore$|\.mailmap$" + r"|.*\.md$" + r"|.*~$)", # editor backups such as src/home_install.sh~ + re.IGNORECASE) + +# Engine -> the catalog families it serves, from the registry in +# src/common/AutoModel/builtin_backends.cpp (RegisterFlm(family)) and +# the frontends in src/include/AutoModel/modeling_*.hpp. An engine's name +# appears in several spellings across the tree; every spelling below is the +# engine's name with "_npu" and a "lib" prefix stripped: +# src/lib/{xrt,hrx}/[lib]_npu.{so,dll,lib} prebuilt engine libraries +# src/include/models//flm/** their headers +# src/include/AutoModel/modeling_.hpp frontends (and the .cpp +# src/common/AutoModel/modeling_[_image|_audio].cpp beside them) +# src/test/[_npu]/** engine unit tests +ENGINES = { + # names families + ("llama", "llama3"): ("llama3", "deepseek-r1"), + ("qwen3",): ("qwen3", "qwen3-it", "qwen3-tk", "deepseek-r1-0528"), + ("qwen2",): ("qwen2",), + ("qwen2vl",): ("qwen2vl",), + # Qwen3VL_Flash derives from Qwen3VL and its engine includes qwen3vl_npu. + ("qwen3vl",): ("qwen3vl", "qwen3vl-flash"), + ("qwen3vl_flash",): ("qwen3vl-flash",), + ("qwen3_5vl",): ("qwen3.5",), + ("minicpm_v_4_7", "minicpm_v"): ("minicpm-v-4.7",), + ("qwen3_6_moe",): ("qwen3.6-moe",), + ("qwen3_8mtp", "qwen3_8_mtp"): ("qwen3.8-mtp",), + ("gemma", "gemma3"): ("gemma3",), + ("gemma_text", "gemma3_text"): ("gemma3-text",), + # Gemma4e_Flash derives from Gemma4e and its engine includes gemma4e_npu. + ("gemma4e",): ("gemma4e", "gemma4e-flash"), + ("gemma4e_flash",): ("gemma4e-flash",), + ("gemma4_12b",): ("gemma4-12b",), + ("hunyuan",): ("hunyuan",), + ("gpt_oss",): ("gpt-oss",), + ("lfm2",): ("lfm2", "lfm2.5-tk"), + ("lfm3",): ("lfm3",), + ("nanbeige",): ("nanbeige",), + ("phi4",): ("phi4",), + ("gemma_embedding",): ("embed-gemma",), + ("d3",): ("d3",), + ("whisper",): ("whisper-v3",), + # Built but registered for no catalog family. + ("qwen3_5_omni",): (), +} +ENGINE_FAMILIES = {name: fams for names, fams in ENGINES.items() for name in names} + +# The corelib backends (src/common/models//core, src/include/models/ +# /core, src/test/_core) serve only the family they register for +# in builtin_backends.cpp, not every family of the flm engine of that name. +CORE_FAMILIES = { + "gemma4": ("gemma4e",), + "phi4": ("phi4",), + "qwen3": ("qwen3",), + "qwen35": ("qwen3.5",), + "qwen36": ("qwen3.6-moe",), +} + +# Subsystems outside the per-engine layout, each serving one family. +SUBSYSTEMS = ( + ("whisper engine", r"src/(common|include)/whisper/", ("whisper-v3",)), + ("embedding engine", r"src/(common|include)/AutoEmbeddingModel/|src/include/embedding_model\.hpp$", + ("embed-gemma",)), + ("decision engine", r"src/(common|include)/(decision|AutoDecisionModel)/" + r"|src/include/decision_model\.hpp$|src/test/d3_\w+/", ("d3",)), + # src/detail builds the gemma4e engine from source; gemma4e_flash + # includes it. + ("gemma4e engine source", r"src/detail/", ("gemma4e", "gemma4e-flash")), +) + +# Files that only serve speech recognition: any change runs the whole ASR +# set. Ported from the paths: filter of the old asr-test.yml and asr_gate.sh, +# with the test scripts moved to npu_test. +ASR_ONLY = re.compile( + r"^(src/(common|include)/whisper/" + r"|src/xclbins/Whisper-V3-Turbo-NPU2/" + r"|src/lib/(xrt|hrx)/[^/]*whisper_npu" + r"|src/(common|include)/audio_process_utils/" + r"|src/(common|include)/audio/" + r"|src/common/AutoModel/[^/]*_audio\.cpp$" + r"|src/detail/gemma4e_npu/gemma4e_audio" + r"|src/include/(pocketfft|dr_libs)/" + r"|src/cmake/FlmAudioCodecs\.cmake$" + r"|src/server/multipart\.(cpp|hpp)$" + r"|src/test/whisper_npu/" + r"|\.github/script/npu_test/suites/(asr|whisper)[^/]*\.py$" + r"|\.github/script/npu_test/tools/build_asr_eval\.py$" + r"|\.github/script/npu_test/assets/asr_eval/)") +# These host /v1/audio/transcriptions but also every other endpoint, so they +# count only when a changed line is about audio or speech. +ASR_SHARED = re.compile(r"^src/server/(rest_handler|server)\.cpp$") +ASR_WORDS = re.compile(r"audio|transcri|whisper|multipart|speech|\basr\b|\bwav\b|mp3|flac" + r"|\bogg\b|opus|m4a|\bpcm\b|resampl", re.IGNORECASE) + + +# The npu_test suites, in run order; "all" when a change is not scoped. +SUITES = ("chat", "serving", "prefix_cache", "api", "embedding", "decision", "asr") + +# Test-side files that matter to one suite only. Anything else in npu_test +# (the client, server, harness, plan) runs every suite. +TEST_SCOPES = ( + (r"\.github/script/npu_test/suites/(\w+)\.py$", None), # the suite itself + (r"\.github/script/npu_test/assets/d3_goldens[^/]*\.jsonl$", {"decision"}), + (r"\.github/script/npu_test/assets/vision/|\.github/script/npu_test/tools/make_vision_assets\.py$", + {"chat"}), + (r"src/test/(openai_api_test|openai_usage)/", {"api"}), +) + + +def family_suites(families: Iterable[str], catalog: dict[str, dict]) -> set: + """The suites that exercise models of these families.""" + out = set() + for fam in families: + if fam == "d3": + out.add("decision") + elif fam == "embed-gemma": + out.add("embedding") + elif fam == "whisper-v3": + out.add("asr") + else: + out |= {"chat", "prefix_cache"} + # Chat models with an audio encoder are tested by the asr suite. + if any(family_of(e) == fam and e.get("asr") for e in catalog.values()): + out.add("asr") + return out + + +def is_non_code(path: str) -> bool: + return bool(NON_CODE.match(path)) + + +class Git: + """The diff range; None from any method means git could not answer.""" + + def __init__(self, repo: Path, base: str, head: str): + self.repo, self.base, self.head = repo, base, head + # Diff from the merge base, as base...head does, so a pull request + # is judged by its own commits rather than by what main gained since. + self.merge_base = self._git("merge-base", base, head) + if self.merge_base: + self.merge_base = self.merge_base.strip() + + def _git(self, *args: str) -> Optional[str]: + try: + res = subprocess.run(["git", "-C", str(self.repo), *args], + capture_output=True, text=True, encoding="utf-8", errors="replace") + except OSError: + return None + return res.stdout if res.returncode == 0 else None + + def files(self) -> Optional[list[str]]: + # --no-renames lists both sides of a rename, so a file moved out of + # an engine directory still counts against that engine. + out = self._git("diff", "--name-only", "--no-renames", f"{self.base}...{self.head}") + return None if out is None else [l for l in out.splitlines() if l.strip()] + + def changed_lines(self, paths: list[str]) -> Optional[str]: + out = self._git("diff", "-U0", f"{self.base}...{self.head}", "--", *paths) + if out is None: + return None + return "\n".join(l for l in out.splitlines() + if l[:1] in ("+", "-") and not l.startswith(("+++", "---"))) + + def show(self, rev_side: str, path: str) -> Optional[str]: + rev = self.merge_base if rev_side == "base" else self.head + return self._git("show", f"{rev}:{path}") if rev else None + + +@dataclass +class Decision: + run_tests: bool = True + families: set = field(default_factory=set) + asr_full: bool = False + # None: every suite. Otherwise the suites every changed file is scoped to. + suites: Optional[set] = field(default_factory=set) + reasons: list = field(default_factory=list) + + def all_suites(self, path: str) -> None: + if self.suites is not None: + self.reasons.append(f"{path} is not scoped to particular suites: running them all") + self.suites = None + + def add_suites(self, names: set) -> None: + if self.suites is not None: + self.suites |= names + + def summary(self) -> str: + if not self.run_tests: + return "documentation only: NPU tests skipped" + parts = ["NPU tests run"] + parts.append("families in full: " + (", ".join(sorted(self.families)) or "none")) + parts.append("ASR: whole set" if self.asr_full else "ASR: pr subset") + parts.append("suites: " + self.suites_value().replace(",", ", ")) + return "; ".join(parts) + + def suites_value(self) -> str: + if self.suites is None or not self.suites: + return "all" + return ",".join(s for s in SUITES if s in self.suites) + + def outputs(self) -> dict[str, str]: + return {"run_tests": str(self.run_tests).lower(), + "families": ",".join(sorted(self.families)), + "asr_full": str(self.asr_full).lower(), + "suites": self.suites_value(), + "summary": self.summary()} + + +def load_catalog(text: Optional[str]) -> dict[str, dict]: + """model id ("name:tag") -> entry, from model_list.json text.""" + if not text: + return {} + try: + models = json.loads(text).get("models", {}) + except (ValueError, AttributeError): + return {} + return {f"{name}:{tag}": entry for name, tags in models.items() + if isinstance(tags, dict) for tag, entry in tags.items() if isinstance(entry, dict)} + + +def family_of(entry: dict) -> str: + details = entry.get("details") + return details.get("family", "") if isinstance(details, dict) else "" + + +def catalog_diff(base: dict[str, dict], head: dict[str, dict]) -> set: + """Families with an entry added, removed or edited between two catalogs.""" + fams = set() + for mid in set(base) | set(head): + if base.get(mid) != head.get(mid): + fams.update(family_of(e) for e in (base.get(mid), head.get(mid)) if e) + return fams - {""} + + +def model_info_diff(base_text: Optional[str], head_text: Optional[str], + catalogs: Iterable[dict[str, dict]]) -> Optional[set]: + """Families whose model_info.json entry changed. Keys are model ids or a + catalog entry's model_info_key.""" + try: + base = json.loads(base_text) if base_text else {} + head = json.loads(head_text) if head_text else {} + except ValueError: + return None + if not isinstance(base, dict) or not isinstance(head, dict): + return None + keys = {k for k in set(base) | set(head) if base.get(k) != head.get(k)} + fams = set() + for catalog in catalogs: + for mid, entry in catalog.items(): + if mid in keys or entry.get("model_info_key") in keys: + fams.add(family_of(entry)) + return fams - {""} + + +def engine_families(path: str, xclbin_families: dict[str, set]) -> Optional[tuple[str, tuple]]: + """(why, families) for a path inside one engine's code, else None.""" + parts = path.split("/") + if len(parts) < 3 or parts[0] != "src": + return None + top, sub = parts[1], parts[2] + if top == "lib" and sub in ("xrt", "hrx") and len(parts) == 4: + stem = parts[3].rsplit(".", 1)[0] + if parts[3].endswith(".so") and stem.startswith("lib"): + stem = stem[3:] + name = stem[:-4] if stem.endswith("_npu") else stem + if name in ENGINE_FAMILIES: + return f"engine library {name}", ENGINE_FAMILIES[name] + return None + if top == "include" and sub == "models" and len(parts) >= 5: + name, kind = parts[3], parts[4] + if kind == "core" and name in CORE_FAMILIES: + return f"{name} core backend", CORE_FAMILIES[name] + if name in ENGINE_FAMILIES: + return f"engine headers {name}", ENGINE_FAMILIES[name] + return None + if top == "common" and sub == "models" and len(parts) >= 5 and parts[3] in CORE_FAMILIES: + return f"{parts[3]} core backend", CORE_FAMILIES[parts[3]] + if top in ("include", "common") and sub == "AutoModel" and len(parts) == 4: + m = re.match(r"modeling_(\w+?)(_image|_audio)?\.(hpp|cpp)$", parts[3]) + if m and m.group(1) in ENGINE_FAMILIES: + return f"frontend modeling_{m.group(1)}", ENGINE_FAMILIES[m.group(1)] + return None + if top == "test" and len(parts) >= 4: + name = sub[:-4] if sub.endswith("_npu") else sub + if name.endswith("_core") and name[:-5] in CORE_FAMILIES: + return f"{name} tests", CORE_FAMILIES[name[:-5]] + if name in ENGINE_FAMILIES: + return f"{sub} tests", ENGINE_FAMILIES[name] + return None + if top == "xclbins" and len(parts) >= 4: + return f"xclbins {sub}", tuple(sorted(xclbin_families.get(sub, ()))) + return None + + +def decide(files: Optional[list[str]], git: Optional[Git] = None, + head_catalog_text: Optional[str] = None) -> Decision: + """The decision for a list of changed files (None: git could not tell). + + git supplies the changed lines and the base and head versions of the + catalogs; without it those checks fall back as described inline. + """ + d = Decision(suites=None) + if files is None: + d.asr_full = True + d.reasons.append("changed files unknown (new branch, shallow clone or force-push): " + "running everything, ASR set in full") + return d + if not files: + d.reasons.append("no changed files: running the tests anyway") + return d + + code = [f for f in files if not is_non_code(f)] + d.suites = set() # narrowed per file below; None once any file is unscoped + if not code: + d.run_tests = False + d.reasons.append(f"all {len(files)} changed file(s) are documentation or non-code") + return d + d.reasons.append(f"{len(code)} of {len(files)} changed file(s) are code, e.g. {code[0]}") + + head_catalog = load_catalog(head_catalog_text) + xclbin_families: dict[str, set] = {} + for entry in head_catalog.values(): + if entry.get("name"): + xclbin_families.setdefault(entry["name"], set()).add(family_of(entry)) + + for path in code: + hit = engine_families(path, xclbin_families) + if hit is None: + hit = next(((why, fams) for why, pattern, fams in SUBSYSTEMS + if re.match(pattern, path)), None) + if hit and hit[1]: + new = set(hit[1]) - d.families + if new: + d.families |= new + d.reasons.append(f"{path}: {hit[0]} -> {', '.join(sorted(new))}") + + # Which suites this file can affect. + scope = None + for pattern, names in TEST_SCOPES: + m = re.match(pattern, path) + if m: + scope = names if names is not None else ( + {m.group(1)} if m.group(1) in SUITES else None) + break + if scope is None and ASR_ONLY.match(path): + scope = {"asr"} + if scope is None and hit and hit[1]: + scope = family_suites(hit[1], head_catalog) + if scope is None and path not in (MODEL_LIST, MODEL_INFO): + d.all_suites(path) + elif scope is not None: + d.add_suites(scope) + + if MODEL_LIST in code: + base = load_catalog(git.show("base", MODEL_LIST)) if git else {} + if git is None or not base or not head_catalog: + d.reasons.append(f"{MODEL_LIST} changed but cannot be compared; no family added for it") + d.all_suites(MODEL_LIST) + else: + fams = catalog_diff(base, head_catalog) + d.families |= fams + if fams: + d.add_suites(family_suites(fams, head_catalog)) + else: + d.all_suites(MODEL_LIST) + d.reasons.append(f"{MODEL_LIST}: entries changed in " + + (", ".join(sorted(fams)) or "no family (top-level fields only)")) + if MODEL_INFO in code: + fams = None + if git: + fams = model_info_diff(git.show("base", MODEL_INFO), git.show("head", MODEL_INFO), + [load_catalog(git.show("base", MODEL_LIST)), head_catalog]) + if fams is None: + d.reasons.append(f"{MODEL_INFO} changed but cannot be compared; no family added for it") + d.all_suites(MODEL_INFO) + else: + d.families |= fams + if fams: + d.add_suites(family_suites(fams, head_catalog)) + else: + d.all_suites(MODEL_INFO) + d.reasons.append(f"{MODEL_INFO}: entries changed in " + (", ".join(sorted(fams)) or "no family")) + + # A family the head catalog lacks (removed, or never listed) has no model + # the plan could run. + if head_catalog: + known = {family_of(e) for e in head_catalog.values()} + gone = d.families - known + if gone: + d.families -= gone + d.reasons.append(f"not in the head catalog, so not tested: {', '.join(sorted(gone))}") + + asr = [f for f in code if ASR_ONLY.match(f)] + shared = [f for f in code if ASR_SHARED.match(f)] + if asr: + d.asr_full = True + d.reasons.append(f"ASR-only file(s) changed, e.g. {asr[0]}: whole ASR set") + elif shared: + lines = git.changed_lines(shared) if git else None + if lines is None: + d.asr_full = True + d.reasons.append(f"{', '.join(shared)} changed and the lines cannot be read: whole ASR set") + else: + words = [l for l in lines.splitlines() if ASR_WORDS.search(l)] + if words: + d.asr_full = True + d.reasons.append(f"changed lines in {', '.join(shared)} mention audio or speech: " + f"{words[0][:100]}") + else: + d.reasons.append(f"{', '.join(shared)} changed, but no changed line is about audio") + return d + + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--base", default="", help="Base commit; empty or all-zero means unknown.") + parser.add_argument("--head", default="HEAD") + parser.add_argument("--files", nargs="*", + help="Changed files, instead of diffing base...head. With --base the " + "catalog diffs and changed lines still come from git.") + parser.add_argument("--repo", type=Path, default=REPO_ROOT) + parser.add_argument("--github-output", action="store_true", + help="Also append the outputs to $GITHUB_OUTPUT.") + args = parser.parse_args(argv) + + known_base = bool(args.base.strip("0 ")) + git = Git(args.repo, args.base, args.head) if known_base else None + if args.files is not None: + files: Optional[list[str]] = [f.replace("\\", "/") for f in args.files] + else: + files = git.files() if git else None + head_catalog = git.show("head", MODEL_LIST) if git else None + if head_catalog is None: + try: + head_catalog = (args.repo / MODEL_LIST).read_text(encoding="utf-8") + except OSError: + head_catalog = None + + decision = decide(files, git, head_catalog) + for reason in decision.reasons: + print(f"changes: {reason}") + outputs = decision.outputs() + for key, value in outputs.items(): + print(f"{key}={value}") + if args.github_output: + target = os.environ.get("GITHUB_OUTPUT") + if not target: + print("--github-output given but $GITHUB_OUTPUT is not set", file=sys.stderr) + return 2 + with open(target, "a", encoding="utf-8") as fh: + fh.writelines(f"{k}={v}\n" for k, v in outputs.items()) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/script/check_no_copyleft_libs.sh b/.github/script/ci/check_no_copyleft_libs.sh similarity index 100% rename from .github/script/check_no_copyleft_libs.sh rename to .github/script/ci/check_no_copyleft_libs.sh diff --git a/.github/script/ci/merge_reports.py b/.github/script/ci/merge_reports.py new file mode 100644 index 000000000..31a788753 --- /dev/null +++ b/.github/script/ci/merge_reports.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Merges the npu_test reports of one Build and Test run into one summary. + + merge_reports.py [--output summary.md] + +Each package's job already puts its own report on its job page; this puts +them side by side on the run page, so one look shows which packages and +runners failed which suites. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("reports", type=Path) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + + reports = [] + for path in sorted(args.reports.glob("**/results.json")): + try: + reports.append(json.loads(path.read_text(encoding="utf-8"))) + except (OSError, json.JSONDecodeError) as exc: + print(f"skipping {path}: {exc}", file=sys.stderr) + + lines = ["## NPU tests", ""] + if not reports: + lines.append("No test reports were uploaded (tests skipped, or every test job failed " + "before writing one).") + else: + suites = sorted({s for r in reports for s in r["suites"]}) + lines += ["| Package | Tier | flm | " + " | ".join(suites) + " | Time |", + "|---|---|---|" + "---|" * len(suites) + "---|"] + for report in reports: + cells = [] + for suite in suites: + results = [r for r in report["results"] if r["suite"] == suite] + if suite not in report["suites"]: + cells.append("") + elif any(r["status"] == "fail" for r in results): + failed = sum(1 for r in results if r["status"] == "fail") + cells.append(f"❌ {failed}") + elif any(r["status"] == "warn" for r in results): + cells.append("⚠️") + elif results: + cells.append("✅") + else: + cells.append("–") + lines.append(f"| {report['label']} | {report['tier']} | {report['flm_version']} | " + + " | ".join(cells) + f" | {report['seconds'] / 60:.0f} min |") + lines += ["", "❌ n: n failed cases, ⚠️ warnings only, – nothing to run. " + "Each test job's page lists the individual cases.", ""] + + failures = [(rep["label"], r) for rep in reports for r in rep["results"] + if r["status"] == "fail"] + if failures: + lines += ["
Failed cases", "", + "| Package | Suite | Case | Model | Detail |", "|---|---|---|---|---|"] + for label, r in failures[:200]: + detail = r["detail"].replace("|", "\\|").replace("\n", " ")[:200] + lines.append(f"| {label} | {r['suite']} | {r['case']} | {r['model']} | {detail} |") + if len(failures) > 200: + lines.append(f"| | | | | {len(failures) - 200} more not shown |") + lines += ["", "
", ""] + + text = "\n".join(lines) + "\n" + if args.output: + args.output.write_text(text, encoding="utf-8") + print(text) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/script/setup_windows_deps.ps1 b/.github/script/ci/setup_windows_deps.ps1 similarity index 99% rename from .github/script/setup_windows_deps.ps1 rename to .github/script/ci/setup_windows_deps.ps1 index 7ed9fd9d0..9a27a5729 100644 --- a/.github/script/setup_windows_deps.ps1 +++ b/.github/script/ci/setup_windows_deps.ps1 @@ -31,7 +31,7 @@ param( $ErrorActionPreference = "Stop" $ProgressPreference = "SilentlyContinue" -$DepsDir = Join-Path $PSScriptRoot "..\windows-deps" | Resolve-Path +$DepsDir = Join-Path $PSScriptRoot "..\..\windows-deps" | Resolve-Path # Boost: GitHub release archive, pinned by the SHA-256 Boost publishes beside it. $BoostUrl = "https://github.com/boostorg/boost/releases/download/boost-1.88.0/boost-1.88.0-b2-nodocs.7z" diff --git a/.github/script/ci/test_changes.py b/.github/script/ci/test_changes.py new file mode 100644 index 000000000..143eb0467 --- /dev/null +++ b/.github/script/ci/test_changes.py @@ -0,0 +1,240 @@ +"""Tests for changes.py: python3 .github/script/ci/test_changes.py""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import changes # noqa: E402 + +SCRIPT = HERE / "changes.py" + + +def cli(*args: str, env: dict | None = None) -> dict[str, str]: + """Runs changes.py and returns its key=value lines.""" + res = subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True, text=True, + env=env, check=True) + return dict(l.split("=", 1) for l in res.stdout.splitlines() if "=" in l and ":" not in l.split("=")[0]) + + +def families(out: dict[str, str]) -> set[str]: + return set(filter(None, out["families"].split(","))) + + +class FakeGit: + """Stands in for changes.Git: fixed file contents and changed lines.""" + + def __init__(self, base=None, head=None, lines=""): + self.contents = {"base": base or {}, "head": head or {}} + self.lines = lines + + def show(self, side, path): + return self.contents[side].get(path) + + def changed_lines(self, paths): + return self.lines + + +def catalog(**entries) -> str: + """model_list.json text from id=(name, family) pairs, ids spelled name__tag.""" + models: dict = {} + for mid, (name, family) in entries.items(): + base, tag = mid.split("__") + models.setdefault(base, {})[tag] = {"name": name, "details": {"family": family}} + return json.dumps({"model_path": "models", "models": models}) + + +class Cli(unittest.TestCase): + def test_docs_only(self): + out = cli("--files", "docs/docs/models/d3.md", "README.md", "src/README.md", + "assets/logo.png", "src/tb_files/panda.png", "src/iceCream.jpg", + "src/home_install.sh~", ".github/ISSUE_TEMPLATE/bug.yml", "src/wix/.gitignore") + self.assertEqual(out["run_tests"], "false") + self.assertEqual(out["families"], "") + self.assertEqual(out["asr_full"], "false") + + def test_packaged_text_runs(self): + for f in ("LICENSE_BINARY.txt", "THIRD_PARTY_NOTICES.txt", "Dockerfile", "debian/control", + ".github/workflows/ci.yml", "src/CMakeLists.txt", "hrx-integration/x.patch"): + self.assertEqual(cli("--files", f)["run_tests"], "true", f) + + def test_mixed(self): + out = cli("--files", "docs/index.md", "src/runner/runner.cpp") + self.assertEqual(out["run_tests"], "true") + self.assertEqual(families(out), set()) + self.assertEqual(out["asr_full"], "false") + + def test_engine_library_maps_to_every_family_it_serves(self): + out = cli("--files", "src/lib/hrx/libqwen3_npu.so") + self.assertEqual(families(out), {"qwen3", "qwen3-it", "qwen3-tk", "deepseek-r1-0528"}) + self.assertEqual(families(cli("--files", "src/lib/xrt/llama_npu.dll")), {"llama3", "deepseek-r1"}) + + def test_engine_paths(self): + cases = { + "src/include/AutoModel/modeling_lfm2.hpp": {"lfm2", "lfm2.5-tk"}, + "src/common/AutoModel/modeling_qwen3vl_image.cpp": {"qwen3vl", "qwen3vl-flash"}, + "src/include/models/gemma_text/flm/aie2p/gemma_text_gemm.hpp": {"gemma3-text"}, + "src/include/models/qwen3/core/x.hpp": {"qwen3"}, + "src/common/models/qwen35/core/dev04/a.cpp": {"qwen3.5"}, + "src/test/gemma4_core/t.cpp": {"gemma4e"}, + "src/test/qwen3_8_mtp_npu/t.cpp": {"qwen3.8-mtp"}, + "src/detail/gemm/gemm.cpp": {"gemma4e", "gemma4e-flash"}, + "src/xclbins/Gemma3-4B-NPU2/a.xclbin": {"gemma3"}, + "src/common/AutoEmbeddingModel/modeling_gemma_embedding.cpp": {"embed-gemma"}, + "src/common/decision/py_json.cpp": {"d3"}, + "src/test/d3_contract/t.cpp": {"d3"}, + "src/include/whisper/whisper_npu.hpp": {"whisper-v3"}, + # Shared by every engine: the pr tier's one model per engine covers it. + "src/lib/xrt/libgemm.so": set(), + "src/common/AutoModel/automodel.cpp": set(), + # A built engine no catalog family uses. + "src/test/qwen3_5_omni_npu/t.cpp": set(), + } + for path, expected in cases.items(): + self.assertEqual(families(cli("--files", path)), expected, path) + + def test_asr_paths(self): + for f in ("src/common/whisper/modeling_whisper.cpp", "src/xclbins/Whisper-V3-Turbo-NPU2/x", + "src/lib/hrx/libwhisper_npu.so", "src/common/AutoModel/modeling_gemma4e_audio.cpp", + "src/detail/gemma4e_npu/gemma4e_audio.cpp", "src/server/multipart.cpp", + ".github/script/npu_test/assets/asr_eval/expected.json"): + self.assertEqual(cli("--files", f)["asr_full"], "true", f) + self.assertEqual(cli("--files", "src/detail/gemma4e_npu/gemma4e_image.cpp")["asr_full"], "false") + # No git to read the changed lines from: assume they matter. + self.assertEqual(cli("--files", "src/server/server.cpp")["asr_full"], "true") + + def test_unknown_base_runs_everything(self): + out = cli("--base", "0" * 40, "--head", "HEAD") + self.assertEqual((out["run_tests"], out["asr_full"], out["families"]), ("true", "true", "")) + + def test_github_output(self): + with tempfile.TemporaryDirectory() as tmp: + target = Path(tmp) / "out" + cli("--files", "docs/a.md", "--github-output", env={**os.environ, "GITHUB_OUTPUT": str(target)}) + lines = target.read_text(encoding="utf-8").splitlines() + self.assertIn("run_tests=false", lines) + self.assertEqual([l.split("=")[0] for l in lines], ["run_tests", "families", "asr_full", "suites", "summary"]) + + def test_suites_scoped_to_the_change(self): + cases = { + ("src/common/whisper/whisper.cpp",): "asr", + (".github/script/npu_test/assets/asr_eval/manifest.json",): "asr", + ("src/common/AutoDecisionModel/auto_decision_model.cpp",): "decision", + (".github/script/npu_test/assets/d3_goldens.jsonl",): "decision", + (".github/script/npu_test/suites/serving.py",): "serving", + ("src/test/openai_api_test/test_errors.py",): "api", + ("src/lib/xrt/libqwen3_npu.so",): "chat,prefix_cache", + # An engine with an audio encoder is also tested by asr. + ("src/lib/xrt/libgemma4e_npu.so",): "chat,prefix_cache,asr", + ("src/test/openai_api_test/test_errors.py", "src/lib/xrt/libd3_npu.so"): "api,decision", + # Shared code, or the harness itself, runs everything. + ("src/server/server.cpp",): "all", + (".github/script/npu_test/harness.py",): "all", + ("src/common/whisper/whisper.cpp", "src/CMakeLists.txt"): "all", + } + for files, expected in cases.items(): + with self.subTest(files=files): + self.assertEqual(cli("--files", *files)["suites"], expected) + + +class Decide(unittest.TestCase): + def test_model_list_change_names_changed_families(self): + base = catalog(qwen3__0_6b=("Qwen3-0.6B-NPU2", "qwen3"), lfm2__1b=("LFM2", "lfm2")) + head = json.loads(base) + head["models"]["lfm2"]["1b"]["default_context_length"] = 4096 + head["models"]["d3"] = {"lite": {"name": "D3", "details": {"family": "d3"}}} + head = json.dumps(head) + git = FakeGit(base={changes.MODEL_LIST: base}, head={changes.MODEL_LIST: head}) + d = changes.decide([changes.MODEL_LIST], git, head) + self.assertEqual(d.families, {"lfm2", "d3"}) + + def test_model_list_without_git_adds_nothing(self): + d = changes.decide([changes.MODEL_LIST], None, catalog(a__b=("A", "qwen3"))) + self.assertTrue(d.run_tests) + self.assertEqual(d.families, set()) + + def test_model_info_change_maps_through_catalog(self): + cat = catalog(**{"qwen3-core__4b": ("x", "qwen3"), "lfm2__1b": ("y", "lfm2")}) + git = FakeGit(base={changes.MODEL_LIST: cat, changes.MODEL_INFO: '{"lfm2:1b": 1, "q": 2}'}, + head={changes.MODEL_LIST: cat, changes.MODEL_INFO: '{"lfm2:1b": 3, "q": 2}'}) + self.assertEqual(changes.decide([changes.MODEL_INFO], git, cat).families, {"lfm2"}) + + def test_shared_server_files_need_audio_lines(self): + files = ["src/server/rest_handler.cpp"] + self.assertFalse(changes.decide(files, FakeGit(lines="+ tag = \"qwen3\";")).asr_full) + self.assertTrue(changes.decide(files, FakeGit(lines="- // handle audio upload")).asr_full) + + def test_families_missing_from_catalog_are_dropped(self): + d = changes.decide(["src/lib/hrx/libqwen3_npu.so"], None, catalog(a__b=("A", "qwen3"))) + self.assertEqual(d.families, {"qwen3"}) + + def test_table_matches_catalog(self): + """Every family the tables name exists in the real catalog, and every + catalog family is reachable from some engine path.""" + text = (changes.REPO_ROOT / changes.MODEL_LIST).read_text(encoding="utf-8") + known = {changes.family_of(e) for e in changes.load_catalog(text).values()} + named = {f for fams in changes.ENGINES.values() for f in fams} + named |= {f for fams in changes.CORE_FAMILIES.values() for f in fams} + named |= {f for _, _, fams in changes.SUBSYSTEMS for f in fams} + self.assertEqual(named - known, set()) + self.assertEqual(known - named, set()) + + +@unittest.skipUnless(shutil.which("git"), "git not installed") +class RealGit(unittest.TestCase): + """base...head through a throwaway repository.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.repo = Path(self.tmp.name) + self.git("init", "-q") + self.write(changes.MODEL_LIST, catalog(qwen3__0_6b=("Qwen3-0.6B-NPU2", "qwen3"), + gemma3__1b=("Gemma3-1B-NPU2", "gemma3-text"))) + self.write("src/server/server.cpp", "int main() {}\n") + self.base = self.commit() + + def tearDown(self): + self.tmp.cleanup() + + def git(self, *args): + return subprocess.run(["git", "-C", str(self.repo), "-c", "user.name=t", "-c", "user.email=t@t", + *args], capture_output=True, text=True, check=True).stdout.strip() + + def write(self, rel, text): + path = self.repo / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + def commit(self): + self.git("add", "-A") + self.git("commit", "-q", "-m", "c") + return self.git("rev-parse", "HEAD") + + def run_cli(self): + return cli("--repo", str(self.repo), "--base", self.base, "--head", self.commit()) + + def test_model_list_and_server_lines(self): + cat = json.loads((self.repo / changes.MODEL_LIST).read_text(encoding="utf-8")) + cat["models"]["gemma3"]["1b"]["flm_min_version"] = "0.9.40" + self.write(changes.MODEL_LIST, json.dumps(cat)) + self.write("src/server/server.cpp", "int main() { /* new tag */ }\n") + out = self.run_cli() + self.assertEqual((out["run_tests"], out["families"], out["asr_full"]), ("true", "gemma3-text", "false")) + + def test_docs_only_and_audio_line(self): + self.write("docs/index.md", "hi\n") + self.assertEqual(self.run_cli()["run_tests"], "false") + self.write("src/server/server.cpp", "int main() { transcribe(); }\n") + self.assertEqual(self.run_cli()["asr_full"], "true") + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/script/npu_test/README.md b/.github/script/npu_test/README.md new file mode 100644 index 000000000..9e6425a72 --- /dev/null +++ b/.github/script/npu_test/README.md @@ -0,0 +1,80 @@ +# NPU tests + +Tests one FastFlowLM package on an NPU machine, in server mode: it starts +`flm serve` with the flags each suite needs, drives it over HTTP, and writes a +report. Python 3.9+ standard library only, so a runner needs nothing installed. + +CI runs it from `.github/workflows/build-and-test.yml` through the +`.github/actions/npu-test` action, once per package (XRT and HRX portable, the +Ubuntu 24.04 .deb, the MSI, the models-from-source builds). + +## Running it + +From `.github/script`: + +```bash +python3 -m npu_test --flm-bin /path/to/flm --label my-machine --tier pr +python3 -m npu_test --flm-bin flm --label dev --suites chat --models qwen3:0.6b,llama3.2:1b +``` + +| Option | Meaning | +|---|---| +| `--tier install\|pr\|full` | How much to run; see below | +| `--suites a,b` | Only these suites (default: all) | +| `--models a,b` | Restrict every suite to these model ids | +| `--families a,b` | Families a change touches; the pr tier tests all their models | +| `--asr-full` | The whole ASR evaluation set on the pr tier | +| `--asr-expected-key KEY` | Check WER against this row of `assets/asr_eval/expected.json` | +| `--output-dir DIR` | Where `results.json`, `summary.md` and the server logs go | + +Models download on first use into `$FLM_MODEL_PATH`. Exit code 1 means a case +failed; warnings and skips do not fail a run. + +## Tiers + +| Tier | Runs on | Covers | +|---|---|---| +| `install` | an installed .deb or MSI | one chat model, Whisper on two clips, embeddings, `d3:lite` | +| `pr` | pull requests, pushes, release tags | one model per engine family, every model of a touched family, the feature suites, a subset of the ASR set | +| `full` | nightly | every catalog model, the whole ASR set, every d3 size | + +Which models each suite runs at each tier is in `plan.py`, in one place. + +On a pull request, `.github/script/ci/changes.py` also picks the suites: only +those the changed files can affect (an ASR-only change runs `asr`, a d3-only +change `decision`, an engine change `chat` and `prefix_cache`), or all of them +when any changed file is shared code. The HRX packages never run `api` or +`serving`, which test server logic the backend does not touch. + +## Suites + +| Suite | Checks | +|---|---| +| `chat` | Per engine family: a known answer, a streamed two-turn conversation, a non-streamed reply and its usage; for vision models, the colour of a generated test image | +| `serving` | Queue full (503), concurrent requests, client disconnect releasing the NPU, cancel, model switching, unknown model, `/v1/models` | +| `prefix_cache` | Multi-slot KV cache (`--prefix-slots 2`): a hit when a conversation returns, LRU eviction, cached and cold replies agree | +| `api` | `src/test/openai_api_test` (tools, logprobs, response_format, reasoning, sampling, continue_final_message, errors) and `src/test/openai_usage` | +| `embedding` | Single and batch vectors, similarity ordering, determinism | +| `decision` | d3 goldens against the upstream model, answer types, errors, the turbo/ultra prefix features | +| `asr` | Whisper WER/CER on the FLEURS set (`assets/asr_eval`), response formats, segments, language, prompt; chat-model transcription (Gemma 4, Nemotron 3.5) | + +## Layout + +``` +run.py entry point and arguments +plan.py catalog and per-tier model selection +harness.py context, cases (pass/fail/warn/skip), report +server.py one flm serve process +client.py HTTP: JSON, SSE streams, multipart +suites/ one module per suite: NAME, DESCRIPTION, run(ctx) +assets/ images, audio, the ASR evaluation set, d3 goldens +tools/ scripts that regenerate assets +``` + +## Adding a suite + +Write `suites/.py` with `NAME`, `DESCRIPTION` and `run(ctx)`, record each +check with `with ctx.case(NAME, "", model, server):`, and add it to +`suites/__init__.py`. Use `expect()` for what a correct server always does, +`warn()` where the ground truth is uncertain, and `skip()` where a check does +not apply to a model. diff --git a/.github/script/npu_test/__init__.py b/.github/script/npu_test/__init__.py new file mode 100644 index 000000000..8e179a466 --- /dev/null +++ b/.github/script/npu_test/__init__.py @@ -0,0 +1 @@ +"""NPU tests for FastFlowLM in server mode. See README.md.""" diff --git a/.github/script/npu_test/__main__.py b/.github/script/npu_test/__main__.py new file mode 100644 index 000000000..1f9f29ecd --- /dev/null +++ b/.github/script/npu_test/__main__.py @@ -0,0 +1,5 @@ +import sys + +from .run import main + +sys.exit(main()) diff --git a/.github/script/assets/asr_eval/README.md b/.github/script/npu_test/assets/asr_eval/README.md similarity index 94% rename from .github/script/assets/asr_eval/README.md rename to .github/script/npu_test/assets/asr_eval/README.md index 5edc4629b..cb2376ddf 100644 --- a/.github/script/assets/asr_eval/README.md +++ b/.github/script/npu_test/assets/asr_eval/README.md @@ -36,6 +36,6 @@ because their builds decode differently: Windows and Linux differ by up to ## Rebuilding -`.github/script/build_asr_eval.py` regenerates this directory from the FLEURS +`.github/script/npu_test/tools/build_asr_eval.py` regenerates this directory from the FLEURS Parquet files; its docstring says where to get them. The selection is deterministic, so the same Parquet revision gives the same set. diff --git a/.github/script/assets/asr_eval/ar-fleurs-00.mp3 b/.github/script/npu_test/assets/asr_eval/ar-fleurs-00.mp3 similarity index 100% rename from .github/script/assets/asr_eval/ar-fleurs-00.mp3 rename to .github/script/npu_test/assets/asr_eval/ar-fleurs-00.mp3 diff --git a/.github/script/assets/asr_eval/ar-fleurs-01.mp3 b/.github/script/npu_test/assets/asr_eval/ar-fleurs-01.mp3 similarity index 100% rename from .github/script/assets/asr_eval/ar-fleurs-01.mp3 rename to .github/script/npu_test/assets/asr_eval/ar-fleurs-01.mp3 diff --git a/.github/script/assets/asr_eval/ar-fleurs-02.mp3 b/.github/script/npu_test/assets/asr_eval/ar-fleurs-02.mp3 similarity index 100% rename from .github/script/assets/asr_eval/ar-fleurs-02.mp3 rename to .github/script/npu_test/assets/asr_eval/ar-fleurs-02.mp3 diff --git a/.github/script/assets/asr_eval/ar-fleurs-03.mp3 b/.github/script/npu_test/assets/asr_eval/ar-fleurs-03.mp3 similarity index 100% rename from .github/script/assets/asr_eval/ar-fleurs-03.mp3 rename to .github/script/npu_test/assets/asr_eval/ar-fleurs-03.mp3 diff --git a/.github/script/assets/asr_eval/ar-fleurs-04.mp3 b/.github/script/npu_test/assets/asr_eval/ar-fleurs-04.mp3 similarity index 100% rename from .github/script/assets/asr_eval/ar-fleurs-04.mp3 rename to .github/script/npu_test/assets/asr_eval/ar-fleurs-04.mp3 diff --git a/.github/script/assets/asr_eval/ar-fleurs-05.mp3 b/.github/script/npu_test/assets/asr_eval/ar-fleurs-05.mp3 similarity index 100% rename from .github/script/assets/asr_eval/ar-fleurs-05.mp3 rename to .github/script/npu_test/assets/asr_eval/ar-fleurs-05.mp3 diff --git a/.github/script/assets/asr_eval/ar-fleurs-06.mp3 b/.github/script/npu_test/assets/asr_eval/ar-fleurs-06.mp3 similarity index 100% rename from .github/script/assets/asr_eval/ar-fleurs-06.mp3 rename to .github/script/npu_test/assets/asr_eval/ar-fleurs-06.mp3 diff --git a/.github/script/assets/asr_eval/ar-fleurs-07.mp3 b/.github/script/npu_test/assets/asr_eval/ar-fleurs-07.mp3 similarity index 100% rename from .github/script/assets/asr_eval/ar-fleurs-07.mp3 rename to .github/script/npu_test/assets/asr_eval/ar-fleurs-07.mp3 diff --git a/.github/script/assets/asr_eval/ar-fleurs-08.mp3 b/.github/script/npu_test/assets/asr_eval/ar-fleurs-08.mp3 similarity index 100% rename from .github/script/assets/asr_eval/ar-fleurs-08.mp3 rename to .github/script/npu_test/assets/asr_eval/ar-fleurs-08.mp3 diff --git a/.github/script/assets/asr_eval/ar-fleurs-09.mp3 b/.github/script/npu_test/assets/asr_eval/ar-fleurs-09.mp3 similarity index 100% rename from .github/script/assets/asr_eval/ar-fleurs-09.mp3 rename to .github/script/npu_test/assets/asr_eval/ar-fleurs-09.mp3 diff --git a/.github/script/assets/asr_eval/de-fleurs-00.mp3 b/.github/script/npu_test/assets/asr_eval/de-fleurs-00.mp3 similarity index 100% rename from .github/script/assets/asr_eval/de-fleurs-00.mp3 rename to .github/script/npu_test/assets/asr_eval/de-fleurs-00.mp3 diff --git a/.github/script/assets/asr_eval/de-fleurs-01.mp3 b/.github/script/npu_test/assets/asr_eval/de-fleurs-01.mp3 similarity index 100% rename from .github/script/assets/asr_eval/de-fleurs-01.mp3 rename to .github/script/npu_test/assets/asr_eval/de-fleurs-01.mp3 diff --git a/.github/script/assets/asr_eval/de-fleurs-02.mp3 b/.github/script/npu_test/assets/asr_eval/de-fleurs-02.mp3 similarity index 100% rename from .github/script/assets/asr_eval/de-fleurs-02.mp3 rename to .github/script/npu_test/assets/asr_eval/de-fleurs-02.mp3 diff --git a/.github/script/assets/asr_eval/de-fleurs-03.mp3 b/.github/script/npu_test/assets/asr_eval/de-fleurs-03.mp3 similarity index 100% rename from .github/script/assets/asr_eval/de-fleurs-03.mp3 rename to .github/script/npu_test/assets/asr_eval/de-fleurs-03.mp3 diff --git a/.github/script/assets/asr_eval/de-fleurs-04.mp3 b/.github/script/npu_test/assets/asr_eval/de-fleurs-04.mp3 similarity index 100% rename from .github/script/assets/asr_eval/de-fleurs-04.mp3 rename to .github/script/npu_test/assets/asr_eval/de-fleurs-04.mp3 diff --git a/.github/script/assets/asr_eval/de-fleurs-05.mp3 b/.github/script/npu_test/assets/asr_eval/de-fleurs-05.mp3 similarity index 100% rename from .github/script/assets/asr_eval/de-fleurs-05.mp3 rename to .github/script/npu_test/assets/asr_eval/de-fleurs-05.mp3 diff --git a/.github/script/assets/asr_eval/de-fleurs-06.mp3 b/.github/script/npu_test/assets/asr_eval/de-fleurs-06.mp3 similarity index 100% rename from .github/script/assets/asr_eval/de-fleurs-06.mp3 rename to .github/script/npu_test/assets/asr_eval/de-fleurs-06.mp3 diff --git a/.github/script/assets/asr_eval/de-fleurs-07.mp3 b/.github/script/npu_test/assets/asr_eval/de-fleurs-07.mp3 similarity index 100% rename from .github/script/assets/asr_eval/de-fleurs-07.mp3 rename to .github/script/npu_test/assets/asr_eval/de-fleurs-07.mp3 diff --git a/.github/script/assets/asr_eval/de-fleurs-08.mp3 b/.github/script/npu_test/assets/asr_eval/de-fleurs-08.mp3 similarity index 100% rename from .github/script/assets/asr_eval/de-fleurs-08.mp3 rename to .github/script/npu_test/assets/asr_eval/de-fleurs-08.mp3 diff --git a/.github/script/assets/asr_eval/de-fleurs-09.mp3 b/.github/script/npu_test/assets/asr_eval/de-fleurs-09.mp3 similarity index 100% rename from .github/script/assets/asr_eval/de-fleurs-09.mp3 rename to .github/script/npu_test/assets/asr_eval/de-fleurs-09.mp3 diff --git a/.github/script/assets/asr_eval/en-long-0.mp3 b/.github/script/npu_test/assets/asr_eval/en-long-0.mp3 similarity index 100% rename from .github/script/assets/asr_eval/en-long-0.mp3 rename to .github/script/npu_test/assets/asr_eval/en-long-0.mp3 diff --git a/.github/script/assets/asr_eval/en-long-1.mp3 b/.github/script/npu_test/assets/asr_eval/en-long-1.mp3 similarity index 100% rename from .github/script/assets/asr_eval/en-long-1.mp3 rename to .github/script/npu_test/assets/asr_eval/en-long-1.mp3 diff --git a/.github/script/assets/asr_eval/en-long-2.mp3 b/.github/script/npu_test/assets/asr_eval/en-long-2.mp3 similarity index 100% rename from .github/script/assets/asr_eval/en-long-2.mp3 rename to .github/script/npu_test/assets/asr_eval/en-long-2.mp3 diff --git a/.github/script/assets/asr_eval/en-longform-0.mp3 b/.github/script/npu_test/assets/asr_eval/en-longform-0.mp3 similarity index 100% rename from .github/script/assets/asr_eval/en-longform-0.mp3 rename to .github/script/npu_test/assets/asr_eval/en-longform-0.mp3 diff --git a/.github/script/assets/asr_eval/en-short-00.mp3 b/.github/script/npu_test/assets/asr_eval/en-short-00.mp3 similarity index 100% rename from .github/script/assets/asr_eval/en-short-00.mp3 rename to .github/script/npu_test/assets/asr_eval/en-short-00.mp3 diff --git a/.github/script/assets/asr_eval/en-short-01.mp3 b/.github/script/npu_test/assets/asr_eval/en-short-01.mp3 similarity index 100% rename from .github/script/assets/asr_eval/en-short-01.mp3 rename to .github/script/npu_test/assets/asr_eval/en-short-01.mp3 diff --git a/.github/script/assets/asr_eval/en-short-02.mp3 b/.github/script/npu_test/assets/asr_eval/en-short-02.mp3 similarity index 100% rename from .github/script/assets/asr_eval/en-short-02.mp3 rename to .github/script/npu_test/assets/asr_eval/en-short-02.mp3 diff --git a/.github/script/assets/asr_eval/en-short-03.mp3 b/.github/script/npu_test/assets/asr_eval/en-short-03.mp3 similarity index 100% rename from .github/script/assets/asr_eval/en-short-03.mp3 rename to .github/script/npu_test/assets/asr_eval/en-short-03.mp3 diff --git a/.github/script/assets/asr_eval/en-short-04.mp3 b/.github/script/npu_test/assets/asr_eval/en-short-04.mp3 similarity index 100% rename from .github/script/assets/asr_eval/en-short-04.mp3 rename to .github/script/npu_test/assets/asr_eval/en-short-04.mp3 diff --git a/.github/script/assets/asr_eval/en-short-05.mp3 b/.github/script/npu_test/assets/asr_eval/en-short-05.mp3 similarity index 100% rename from .github/script/assets/asr_eval/en-short-05.mp3 rename to .github/script/npu_test/assets/asr_eval/en-short-05.mp3 diff --git a/.github/script/assets/asr_eval/en-short-06.mp3 b/.github/script/npu_test/assets/asr_eval/en-short-06.mp3 similarity index 100% rename from .github/script/assets/asr_eval/en-short-06.mp3 rename to .github/script/npu_test/assets/asr_eval/en-short-06.mp3 diff --git a/.github/script/assets/asr_eval/en-short-07.mp3 b/.github/script/npu_test/assets/asr_eval/en-short-07.mp3 similarity index 100% rename from .github/script/assets/asr_eval/en-short-07.mp3 rename to .github/script/npu_test/assets/asr_eval/en-short-07.mp3 diff --git a/.github/script/assets/asr_eval/en-short-08.mp3 b/.github/script/npu_test/assets/asr_eval/en-short-08.mp3 similarity index 100% rename from .github/script/assets/asr_eval/en-short-08.mp3 rename to .github/script/npu_test/assets/asr_eval/en-short-08.mp3 diff --git a/.github/script/assets/asr_eval/en-short-09.mp3 b/.github/script/npu_test/assets/asr_eval/en-short-09.mp3 similarity index 100% rename from .github/script/assets/asr_eval/en-short-09.mp3 rename to .github/script/npu_test/assets/asr_eval/en-short-09.mp3 diff --git a/.github/script/assets/asr_eval/en-short-10.mp3 b/.github/script/npu_test/assets/asr_eval/en-short-10.mp3 similarity index 100% rename from .github/script/assets/asr_eval/en-short-10.mp3 rename to .github/script/npu_test/assets/asr_eval/en-short-10.mp3 diff --git a/.github/script/assets/asr_eval/en-short-11.mp3 b/.github/script/npu_test/assets/asr_eval/en-short-11.mp3 similarity index 100% rename from .github/script/assets/asr_eval/en-short-11.mp3 rename to .github/script/npu_test/assets/asr_eval/en-short-11.mp3 diff --git a/.github/script/assets/asr_eval/en-short-12.mp3 b/.github/script/npu_test/assets/asr_eval/en-short-12.mp3 similarity index 100% rename from .github/script/assets/asr_eval/en-short-12.mp3 rename to .github/script/npu_test/assets/asr_eval/en-short-12.mp3 diff --git a/.github/script/assets/asr_eval/en-short-13.mp3 b/.github/script/npu_test/assets/asr_eval/en-short-13.mp3 similarity index 100% rename from .github/script/assets/asr_eval/en-short-13.mp3 rename to .github/script/npu_test/assets/asr_eval/en-short-13.mp3 diff --git a/.github/script/assets/asr_eval/en-short-14.mp3 b/.github/script/npu_test/assets/asr_eval/en-short-14.mp3 similarity index 100% rename from .github/script/assets/asr_eval/en-short-14.mp3 rename to .github/script/npu_test/assets/asr_eval/en-short-14.mp3 diff --git a/.github/script/assets/asr_eval/en-short-15.mp3 b/.github/script/npu_test/assets/asr_eval/en-short-15.mp3 similarity index 100% rename from .github/script/assets/asr_eval/en-short-15.mp3 rename to .github/script/npu_test/assets/asr_eval/en-short-15.mp3 diff --git a/.github/script/assets/asr_eval/en-short-16.mp3 b/.github/script/npu_test/assets/asr_eval/en-short-16.mp3 similarity index 100% rename from .github/script/assets/asr_eval/en-short-16.mp3 rename to .github/script/npu_test/assets/asr_eval/en-short-16.mp3 diff --git a/.github/script/assets/asr_eval/en-short-17.mp3 b/.github/script/npu_test/assets/asr_eval/en-short-17.mp3 similarity index 100% rename from .github/script/assets/asr_eval/en-short-17.mp3 rename to .github/script/npu_test/assets/asr_eval/en-short-17.mp3 diff --git a/.github/script/assets/asr_eval/en-short-18.mp3 b/.github/script/npu_test/assets/asr_eval/en-short-18.mp3 similarity index 100% rename from .github/script/assets/asr_eval/en-short-18.mp3 rename to .github/script/npu_test/assets/asr_eval/en-short-18.mp3 diff --git a/.github/script/assets/asr_eval/en-short-19.mp3 b/.github/script/npu_test/assets/asr_eval/en-short-19.mp3 similarity index 100% rename from .github/script/assets/asr_eval/en-short-19.mp3 rename to .github/script/npu_test/assets/asr_eval/en-short-19.mp3 diff --git a/.github/script/assets/asr_eval/es-fleurs-00.mp3 b/.github/script/npu_test/assets/asr_eval/es-fleurs-00.mp3 similarity index 100% rename from .github/script/assets/asr_eval/es-fleurs-00.mp3 rename to .github/script/npu_test/assets/asr_eval/es-fleurs-00.mp3 diff --git a/.github/script/assets/asr_eval/es-fleurs-01.mp3 b/.github/script/npu_test/assets/asr_eval/es-fleurs-01.mp3 similarity index 100% rename from .github/script/assets/asr_eval/es-fleurs-01.mp3 rename to .github/script/npu_test/assets/asr_eval/es-fleurs-01.mp3 diff --git a/.github/script/assets/asr_eval/es-fleurs-02.mp3 b/.github/script/npu_test/assets/asr_eval/es-fleurs-02.mp3 similarity index 100% rename from .github/script/assets/asr_eval/es-fleurs-02.mp3 rename to .github/script/npu_test/assets/asr_eval/es-fleurs-02.mp3 diff --git a/.github/script/assets/asr_eval/es-fleurs-03.mp3 b/.github/script/npu_test/assets/asr_eval/es-fleurs-03.mp3 similarity index 100% rename from .github/script/assets/asr_eval/es-fleurs-03.mp3 rename to .github/script/npu_test/assets/asr_eval/es-fleurs-03.mp3 diff --git a/.github/script/assets/asr_eval/es-fleurs-04.mp3 b/.github/script/npu_test/assets/asr_eval/es-fleurs-04.mp3 similarity index 100% rename from .github/script/assets/asr_eval/es-fleurs-04.mp3 rename to .github/script/npu_test/assets/asr_eval/es-fleurs-04.mp3 diff --git a/.github/script/assets/asr_eval/es-fleurs-05.mp3 b/.github/script/npu_test/assets/asr_eval/es-fleurs-05.mp3 similarity index 100% rename from .github/script/assets/asr_eval/es-fleurs-05.mp3 rename to .github/script/npu_test/assets/asr_eval/es-fleurs-05.mp3 diff --git a/.github/script/assets/asr_eval/es-fleurs-06.mp3 b/.github/script/npu_test/assets/asr_eval/es-fleurs-06.mp3 similarity index 100% rename from .github/script/assets/asr_eval/es-fleurs-06.mp3 rename to .github/script/npu_test/assets/asr_eval/es-fleurs-06.mp3 diff --git a/.github/script/assets/asr_eval/es-fleurs-07.mp3 b/.github/script/npu_test/assets/asr_eval/es-fleurs-07.mp3 similarity index 100% rename from .github/script/assets/asr_eval/es-fleurs-07.mp3 rename to .github/script/npu_test/assets/asr_eval/es-fleurs-07.mp3 diff --git a/.github/script/assets/asr_eval/es-fleurs-08.mp3 b/.github/script/npu_test/assets/asr_eval/es-fleurs-08.mp3 similarity index 100% rename from .github/script/assets/asr_eval/es-fleurs-08.mp3 rename to .github/script/npu_test/assets/asr_eval/es-fleurs-08.mp3 diff --git a/.github/script/assets/asr_eval/es-fleurs-09.mp3 b/.github/script/npu_test/assets/asr_eval/es-fleurs-09.mp3 similarity index 100% rename from .github/script/assets/asr_eval/es-fleurs-09.mp3 rename to .github/script/npu_test/assets/asr_eval/es-fleurs-09.mp3 diff --git a/.github/script/assets/asr_eval/expected.json b/.github/script/npu_test/assets/asr_eval/expected.json similarity index 100% rename from .github/script/assets/asr_eval/expected.json rename to .github/script/npu_test/assets/asr_eval/expected.json diff --git a/.github/script/assets/asr_eval/fr-fleurs-00.mp3 b/.github/script/npu_test/assets/asr_eval/fr-fleurs-00.mp3 similarity index 100% rename from .github/script/assets/asr_eval/fr-fleurs-00.mp3 rename to .github/script/npu_test/assets/asr_eval/fr-fleurs-00.mp3 diff --git a/.github/script/assets/asr_eval/fr-fleurs-01.mp3 b/.github/script/npu_test/assets/asr_eval/fr-fleurs-01.mp3 similarity index 100% rename from .github/script/assets/asr_eval/fr-fleurs-01.mp3 rename to .github/script/npu_test/assets/asr_eval/fr-fleurs-01.mp3 diff --git a/.github/script/assets/asr_eval/fr-fleurs-02.mp3 b/.github/script/npu_test/assets/asr_eval/fr-fleurs-02.mp3 similarity index 100% rename from .github/script/assets/asr_eval/fr-fleurs-02.mp3 rename to .github/script/npu_test/assets/asr_eval/fr-fleurs-02.mp3 diff --git a/.github/script/assets/asr_eval/fr-fleurs-03.mp3 b/.github/script/npu_test/assets/asr_eval/fr-fleurs-03.mp3 similarity index 100% rename from .github/script/assets/asr_eval/fr-fleurs-03.mp3 rename to .github/script/npu_test/assets/asr_eval/fr-fleurs-03.mp3 diff --git a/.github/script/assets/asr_eval/fr-fleurs-04.mp3 b/.github/script/npu_test/assets/asr_eval/fr-fleurs-04.mp3 similarity index 100% rename from .github/script/assets/asr_eval/fr-fleurs-04.mp3 rename to .github/script/npu_test/assets/asr_eval/fr-fleurs-04.mp3 diff --git a/.github/script/assets/asr_eval/fr-fleurs-05.mp3 b/.github/script/npu_test/assets/asr_eval/fr-fleurs-05.mp3 similarity index 100% rename from .github/script/assets/asr_eval/fr-fleurs-05.mp3 rename to .github/script/npu_test/assets/asr_eval/fr-fleurs-05.mp3 diff --git a/.github/script/assets/asr_eval/fr-fleurs-06.mp3 b/.github/script/npu_test/assets/asr_eval/fr-fleurs-06.mp3 similarity index 100% rename from .github/script/assets/asr_eval/fr-fleurs-06.mp3 rename to .github/script/npu_test/assets/asr_eval/fr-fleurs-06.mp3 diff --git a/.github/script/assets/asr_eval/fr-fleurs-07.mp3 b/.github/script/npu_test/assets/asr_eval/fr-fleurs-07.mp3 similarity index 100% rename from .github/script/assets/asr_eval/fr-fleurs-07.mp3 rename to .github/script/npu_test/assets/asr_eval/fr-fleurs-07.mp3 diff --git a/.github/script/assets/asr_eval/fr-fleurs-08.mp3 b/.github/script/npu_test/assets/asr_eval/fr-fleurs-08.mp3 similarity index 100% rename from .github/script/assets/asr_eval/fr-fleurs-08.mp3 rename to .github/script/npu_test/assets/asr_eval/fr-fleurs-08.mp3 diff --git a/.github/script/assets/asr_eval/fr-fleurs-09.mp3 b/.github/script/npu_test/assets/asr_eval/fr-fleurs-09.mp3 similarity index 100% rename from .github/script/assets/asr_eval/fr-fleurs-09.mp3 rename to .github/script/npu_test/assets/asr_eval/fr-fleurs-09.mp3 diff --git a/.github/script/assets/asr_eval/ja-fleurs-00.mp3 b/.github/script/npu_test/assets/asr_eval/ja-fleurs-00.mp3 similarity index 100% rename from .github/script/assets/asr_eval/ja-fleurs-00.mp3 rename to .github/script/npu_test/assets/asr_eval/ja-fleurs-00.mp3 diff --git a/.github/script/assets/asr_eval/ja-fleurs-01.mp3 b/.github/script/npu_test/assets/asr_eval/ja-fleurs-01.mp3 similarity index 100% rename from .github/script/assets/asr_eval/ja-fleurs-01.mp3 rename to .github/script/npu_test/assets/asr_eval/ja-fleurs-01.mp3 diff --git a/.github/script/assets/asr_eval/ja-fleurs-02.mp3 b/.github/script/npu_test/assets/asr_eval/ja-fleurs-02.mp3 similarity index 100% rename from .github/script/assets/asr_eval/ja-fleurs-02.mp3 rename to .github/script/npu_test/assets/asr_eval/ja-fleurs-02.mp3 diff --git a/.github/script/assets/asr_eval/ja-fleurs-03.mp3 b/.github/script/npu_test/assets/asr_eval/ja-fleurs-03.mp3 similarity index 100% rename from .github/script/assets/asr_eval/ja-fleurs-03.mp3 rename to .github/script/npu_test/assets/asr_eval/ja-fleurs-03.mp3 diff --git a/.github/script/assets/asr_eval/ja-fleurs-04.mp3 b/.github/script/npu_test/assets/asr_eval/ja-fleurs-04.mp3 similarity index 100% rename from .github/script/assets/asr_eval/ja-fleurs-04.mp3 rename to .github/script/npu_test/assets/asr_eval/ja-fleurs-04.mp3 diff --git a/.github/script/assets/asr_eval/ja-fleurs-05.mp3 b/.github/script/npu_test/assets/asr_eval/ja-fleurs-05.mp3 similarity index 100% rename from .github/script/assets/asr_eval/ja-fleurs-05.mp3 rename to .github/script/npu_test/assets/asr_eval/ja-fleurs-05.mp3 diff --git a/.github/script/assets/asr_eval/ja-fleurs-06.mp3 b/.github/script/npu_test/assets/asr_eval/ja-fleurs-06.mp3 similarity index 100% rename from .github/script/assets/asr_eval/ja-fleurs-06.mp3 rename to .github/script/npu_test/assets/asr_eval/ja-fleurs-06.mp3 diff --git a/.github/script/assets/asr_eval/ja-fleurs-07.mp3 b/.github/script/npu_test/assets/asr_eval/ja-fleurs-07.mp3 similarity index 100% rename from .github/script/assets/asr_eval/ja-fleurs-07.mp3 rename to .github/script/npu_test/assets/asr_eval/ja-fleurs-07.mp3 diff --git a/.github/script/assets/asr_eval/ja-fleurs-08.mp3 b/.github/script/npu_test/assets/asr_eval/ja-fleurs-08.mp3 similarity index 100% rename from .github/script/assets/asr_eval/ja-fleurs-08.mp3 rename to .github/script/npu_test/assets/asr_eval/ja-fleurs-08.mp3 diff --git a/.github/script/assets/asr_eval/ja-fleurs-09.mp3 b/.github/script/npu_test/assets/asr_eval/ja-fleurs-09.mp3 similarity index 100% rename from .github/script/assets/asr_eval/ja-fleurs-09.mp3 rename to .github/script/npu_test/assets/asr_eval/ja-fleurs-09.mp3 diff --git a/.github/script/assets/asr_eval/manifest.json b/.github/script/npu_test/assets/asr_eval/manifest.json similarity index 100% rename from .github/script/assets/asr_eval/manifest.json rename to .github/script/npu_test/assets/asr_eval/manifest.json diff --git a/.github/script/assets/asr_eval/zh-fleurs-00.mp3 b/.github/script/npu_test/assets/asr_eval/zh-fleurs-00.mp3 similarity index 100% rename from .github/script/assets/asr_eval/zh-fleurs-00.mp3 rename to .github/script/npu_test/assets/asr_eval/zh-fleurs-00.mp3 diff --git a/.github/script/assets/asr_eval/zh-fleurs-01.mp3 b/.github/script/npu_test/assets/asr_eval/zh-fleurs-01.mp3 similarity index 100% rename from .github/script/assets/asr_eval/zh-fleurs-01.mp3 rename to .github/script/npu_test/assets/asr_eval/zh-fleurs-01.mp3 diff --git a/.github/script/assets/asr_eval/zh-fleurs-02.mp3 b/.github/script/npu_test/assets/asr_eval/zh-fleurs-02.mp3 similarity index 100% rename from .github/script/assets/asr_eval/zh-fleurs-02.mp3 rename to .github/script/npu_test/assets/asr_eval/zh-fleurs-02.mp3 diff --git a/.github/script/assets/asr_eval/zh-fleurs-03.mp3 b/.github/script/npu_test/assets/asr_eval/zh-fleurs-03.mp3 similarity index 100% rename from .github/script/assets/asr_eval/zh-fleurs-03.mp3 rename to .github/script/npu_test/assets/asr_eval/zh-fleurs-03.mp3 diff --git a/.github/script/assets/asr_eval/zh-fleurs-04.mp3 b/.github/script/npu_test/assets/asr_eval/zh-fleurs-04.mp3 similarity index 100% rename from .github/script/assets/asr_eval/zh-fleurs-04.mp3 rename to .github/script/npu_test/assets/asr_eval/zh-fleurs-04.mp3 diff --git a/.github/script/assets/asr_eval/zh-fleurs-05.mp3 b/.github/script/npu_test/assets/asr_eval/zh-fleurs-05.mp3 similarity index 100% rename from .github/script/assets/asr_eval/zh-fleurs-05.mp3 rename to .github/script/npu_test/assets/asr_eval/zh-fleurs-05.mp3 diff --git a/.github/script/assets/asr_eval/zh-fleurs-06.mp3 b/.github/script/npu_test/assets/asr_eval/zh-fleurs-06.mp3 similarity index 100% rename from .github/script/assets/asr_eval/zh-fleurs-06.mp3 rename to .github/script/npu_test/assets/asr_eval/zh-fleurs-06.mp3 diff --git a/.github/script/assets/asr_eval/zh-fleurs-07.mp3 b/.github/script/npu_test/assets/asr_eval/zh-fleurs-07.mp3 similarity index 100% rename from .github/script/assets/asr_eval/zh-fleurs-07.mp3 rename to .github/script/npu_test/assets/asr_eval/zh-fleurs-07.mp3 diff --git a/.github/script/assets/asr_eval/zh-fleurs-08.mp3 b/.github/script/npu_test/assets/asr_eval/zh-fleurs-08.mp3 similarity index 100% rename from .github/script/assets/asr_eval/zh-fleurs-08.mp3 rename to .github/script/npu_test/assets/asr_eval/zh-fleurs-08.mp3 diff --git a/.github/script/assets/asr_eval/zh-fleurs-09.mp3 b/.github/script/npu_test/assets/asr_eval/zh-fleurs-09.mp3 similarity index 100% rename from .github/script/assets/asr_eval/zh-fleurs-09.mp3 rename to .github/script/npu_test/assets/asr_eval/zh-fleurs-09.mp3 diff --git a/.github/script/assets/d3_goldens.jsonl b/.github/script/npu_test/assets/d3_goldens.jsonl similarity index 100% rename from .github/script/assets/d3_goldens.jsonl rename to .github/script/npu_test/assets/d3_goldens.jsonl diff --git a/.github/script/assets/d3_goldens_mini.jsonl b/.github/script/npu_test/assets/d3_goldens_mini.jsonl similarity index 100% rename from .github/script/assets/d3_goldens_mini.jsonl rename to .github/script/npu_test/assets/d3_goldens_mini.jsonl diff --git a/.github/script/assets/d3_goldens_nano.jsonl b/.github/script/npu_test/assets/d3_goldens_nano.jsonl similarity index 100% rename from .github/script/assets/d3_goldens_nano.jsonl rename to .github/script/npu_test/assets/d3_goldens_nano.jsonl diff --git a/.github/script/assets/d3_goldens_turbo.jsonl b/.github/script/npu_test/assets/d3_goldens_turbo.jsonl similarity index 100% rename from .github/script/assets/d3_goldens_turbo.jsonl rename to .github/script/npu_test/assets/d3_goldens_turbo.jsonl diff --git a/.github/script/assets/d3_goldens_ultra.jsonl b/.github/script/npu_test/assets/d3_goldens_ultra.jsonl similarity index 100% rename from .github/script/assets/d3_goldens_ultra.jsonl rename to .github/script/npu_test/assets/d3_goldens_ultra.jsonl diff --git a/.github/script/assets/test_audio.mp3 b/.github/script/npu_test/assets/test_audio.mp3 similarity index 100% rename from .github/script/assets/test_audio.mp3 rename to .github/script/npu_test/assets/test_audio.mp3 diff --git a/.github/script/assets/test_image1.jpg b/.github/script/npu_test/assets/test_image1.jpg similarity index 100% rename from .github/script/assets/test_image1.jpg rename to .github/script/npu_test/assets/test_image1.jpg diff --git a/.github/script/assets/test_image2.jpg b/.github/script/npu_test/assets/test_image2.jpg similarity index 100% rename from .github/script/assets/test_image2.jpg rename to .github/script/npu_test/assets/test_image2.jpg diff --git a/.github/script/npu_test/assets/vision/blue_square.png b/.github/script/npu_test/assets/vision/blue_square.png new file mode 100644 index 000000000..9c1cbedba Binary files /dev/null and b/.github/script/npu_test/assets/vision/blue_square.png differ diff --git a/.github/script/npu_test/assets/vision/red_square.png b/.github/script/npu_test/assets/vision/red_square.png new file mode 100644 index 000000000..95914e748 Binary files /dev/null and b/.github/script/npu_test/assets/vision/red_square.png differ diff --git a/.github/script/npu_test/client.py b/.github/script/npu_test/client.py new file mode 100644 index 000000000..d935de555 --- /dev/null +++ b/.github/script/npu_test/client.py @@ -0,0 +1,208 @@ +"""A small HTTP client for the FLM server, standard library only. + +The NPU runners sit behind a proxy that intermittently blocks downloads, so the +harness avoids anything that needs `pip install`. This covers what the suites +need: JSON requests, Server-Sent Events streams, and multipart uploads. +""" + +from __future__ import annotations + +import json +import time +import urllib.error +import urllib.request +import uuid +from dataclasses import dataclass, field + + +class HttpError(Exception): + """A response with a non-2xx status. Carries the status and the body.""" + + def __init__(self, status: int, body: str, url: str): + self.status = status + self.body = body + self.url = url + super().__init__(f"HTTP {status} from {url}: {body[:500]}") + + def json(self): + try: + return json.loads(self.body) + except json.JSONDecodeError: + return None + + +@dataclass +class ChatResult: + """One chat completion, streamed or not, folded into a single shape.""" + + content: str = "" + reasoning: str = "" + finish_reason: str | None = None + usage: dict = field(default_factory=dict) + tool_calls: list = field(default_factory=list) + chunks: int = 0 + seconds: float = 0.0 + raw: dict | None = None # the full body of a non-streamed response + + @property + def cached_tokens(self) -> int | None: + details = self.usage.get("prompt_tokens_details") or {} + value = details.get("cached_tokens") + return value if isinstance(value, int) else None + + def usage_line(self) -> str: + """The server's prefill/decode accounting as one line, for the log.""" + u = self.usage + parts = [] + if isinstance(u.get("prompt_tokens"), int): + part = f"prefill {u['prompt_tokens']} tok" + if self.cached_tokens: + part += f" ({self.cached_tokens} cached)" + parts.append(part) + if isinstance(u.get("completion_tokens"), int): + part = f"decode {u['completion_tokens']} tok" + if isinstance(u.get("decoding_speed_tps"), (int, float)): + part += f" @ {u['decoding_speed_tps']:.1f} tps" + parts.append(part) + if u.get("load_duration"): + parts.append(f"load {u['load_duration']:.2f}s") + return " | ".join(parts) + + +class Client: + def __init__(self, base_url: str, timeout: float = 600.0): + # base_url is the server root, e.g. http://127.0.0.1:52625 + self.base_url = base_url.rstrip("/") + self.timeout = timeout + + # -- plumbing ------------------------------------------------------------ + + def _open(self, method: str, path: str, body: bytes | None, headers: dict, + timeout: float | None): + url = f"{self.base_url}{path}" + request = urllib.request.Request(url, data=body, method=method, headers=headers) + try: + return urllib.request.urlopen(request, timeout=timeout or self.timeout) + except urllib.error.HTTPError as exc: + text = exc.read().decode("utf-8", "replace") + raise HttpError(exc.code, text, url) from None + + def get_json(self, path: str, timeout: float | None = None): + with self._open("GET", path, None, {}, timeout) as resp: + return json.loads(resp.read().decode("utf-8")) + + def post_json(self, path: str, body: dict, timeout: float | None = None): + data = json.dumps(body).encode("utf-8") + with self._open("POST", path, data, {"Content-Type": "application/json"}, + timeout) as resp: + return json.loads(resp.read().decode("utf-8")) + + def post_raw(self, path: str, data: bytes, content_type: str, + timeout: float | None = None) -> tuple[int, bytes]: + """POSTs arbitrary bytes and returns (status, body); non-2xx raises.""" + with self._open("POST", path, data, {"Content-Type": content_type}, + timeout) as resp: + return resp.status, resp.read() + + def post_multipart(self, path: str, fields: dict, files: dict, + timeout: float | None = None) -> tuple[int, bytes]: + """POSTs multipart/form-data. files maps name -> (filename, bytes, mime).""" + boundary = f"----flm{uuid.uuid4().hex}" + out = bytearray() + for name, value in fields.items(): + if value is None: + continue + out += f"--{boundary}\r\n".encode() + out += f'Content-Disposition: form-data; name="{name}"\r\n\r\n'.encode() + out += str(value).encode("utf-8") + b"\r\n" + for name, (filename, payload, mime) in files.items(): + out += f"--{boundary}\r\n".encode() + out += (f'Content-Disposition: form-data; name="{name}"; ' + f'filename="{filename}"\r\n').encode() + out += f"Content-Type: {mime}\r\n\r\n".encode() + out += payload + b"\r\n" + out += f"--{boundary}--\r\n".encode() + return self.post_raw(path, bytes(out), f"multipart/form-data; boundary={boundary}", + timeout) + + def stream_events(self, path: str, body: dict, timeout: float | None = None, + deadline: float | None = None): + """POSTs body and yields each SSE `data:` payload, parsed, until [DONE]. + + The socket timeout bounds the wait for the next line only; deadline + (seconds) bounds the whole stream, so a model that never stops + producing tokens still ends the request. + """ + data = json.dumps(body).encode("utf-8") + resp = self._open("POST", path, data, {"Content-Type": "application/json", + "Accept": "text/event-stream"}, timeout) + end = time.monotonic() + (deadline or timeout or self.timeout) + try: + for raw_line in resp: + if time.monotonic() > end: + raise TimeoutError(f"stream exceeded {deadline or timeout or self.timeout:.0f}s") + line = raw_line.decode("utf-8", "replace").strip() + if not line.startswith("data:"): + continue + payload = line[len("data:"):].strip() + if payload == "[DONE]": + return + yield json.loads(payload) + finally: + resp.close() + + # -- endpoints ----------------------------------------------------------- + + def version(self) -> str: + try: + return str(self.get_json("/v1/version", timeout=10).get("version", "unknown")) + except Exception: + return "unknown" + + def chat(self, model: str, messages: list, *, stream: bool = True, + timeout: float | None = None, **extra) -> ChatResult: + """One /v1/chat/completions request. extra is merged into the body. + + Streamed requests ask for usage, as an OpenAI client must, so both + modes come back with the server's usage block. + """ + body = {"model": model, "messages": messages, "stream": stream, **extra} + if stream: + body.setdefault("stream_options", {"include_usage": True}) + started = time.monotonic() + result = ChatResult() + if stream: + calls: dict[int, dict] = {} + for chunk in self.stream_events("/v1/chat/completions", body, timeout=timeout): + result.chunks += 1 + if chunk.get("usage"): + result.usage = chunk["usage"] + if "error" in chunk: + raise HttpError(500, json.dumps(chunk), "/v1/chat/completions (stream)") + for choice in chunk.get("choices") or []: + delta = choice.get("delta") or {} + result.content += delta.get("content") or "" + result.reasoning += delta.get("reasoning_content") or "" + for call in delta.get("tool_calls") or []: + slot = calls.setdefault(call.get("index", 0), + {"function": {"name": "", "arguments": ""}}) + fn = call.get("function") or {} + slot["function"]["name"] += fn.get("name") or "" + slot["function"]["arguments"] += fn.get("arguments") or "" + if call.get("id"): + slot["id"] = call["id"] + if choice.get("finish_reason"): + result.finish_reason = choice["finish_reason"] + result.tool_calls = [calls[i] for i in sorted(calls)] + else: + data = self.post_json("/v1/chat/completions", body, timeout=timeout) + result.raw = data + choice = (data.get("choices") or [{}])[0] + message = choice.get("message") or {} + result.content = message.get("content") or "" + result.reasoning = message.get("reasoning_content") or "" + result.tool_calls = message.get("tool_calls") or [] + result.finish_reason = choice.get("finish_reason") + result.usage = data.get("usage") or {} + result.seconds = time.monotonic() - started + return result diff --git a/.github/script/npu_test/harness.py b/.github/script/npu_test/harness.py new file mode 100644 index 000000000..cdfe237cd --- /dev/null +++ b/.github/script/npu_test/harness.py @@ -0,0 +1,268 @@ +"""What every suite shares: the run context, cases, and the report. + +A suite is a module in suites/ with NAME, a one-line DESCRIPTION, and +run(ctx). It records each check as a case: + + with ctx.case("chat", "stream-two-turns", model) as case: + reply = server.client.chat(model, messages) + expect(reply.content, "empty content") + case.data["tokens"] = reply.usage.get("completion_tokens") + +An AssertionError (or expect()) in the block records FAIL, skip() records +SKIP, warn() records WARN, and any other exception records FAIL with its type. +The block never raises, so one broken model cannot stop the suite. When the +server dies under a case, it is restarted for the next one; once restarts run +out, ServerGone ends the suite and its remaining models are reported as not +attempted. +""" + +from __future__ import annotations + +import json +import sys +import time +import traceback +from dataclasses import asdict, dataclass, field +from pathlib import Path + +from .plan import Plan +from .server import FlmServer, ServerError + +PASS, FAIL, SKIP, WARN = "pass", "fail", "skip", "warn" + +SCRIPT_DIR = Path(__file__).resolve().parent +ASSETS = SCRIPT_DIR / "assets" +REPO_ROOT = SCRIPT_DIR.parents[2] + + +class Skip(Exception): + """Raised inside a case: the check does not apply here.""" + + +class Warn(Exception): + """Raised inside a case: worth a look, but not a failure.""" + + +class ServerGone(Exception): + """The suite's server died and could not be restarted.""" + + +def expect(condition, message: str) -> None: + if not condition: + raise AssertionError(message) + + +def skip(message: str): + raise Skip(message) + + +def warn(message: str): + raise Warn(message) + + +@dataclass +class Result: + suite: str + case: str + status: str + model: str = "" + detail: str = "" + seconds: float = 0.0 + data: dict = field(default_factory=dict) + + +class Case: + def __init__(self, ctx: "Context", suite: str, name: str, model: str, + server: FlmServer | None): + self.ctx = ctx + self.suite = suite + self.name = name + self.model = model + self.server = server + self.data: dict = {} + self.status = PASS + self._started = 0.0 + + def __enter__(self) -> "Case": + self._started = time.monotonic() + return self + + def __exit__(self, exc_type, exc, tb) -> bool: + seconds = time.monotonic() - self._started + detail = "" + if exc_type is None: + self.status = PASS + elif issubclass(exc_type, Skip): + self.status, detail = SKIP, str(exc) + elif issubclass(exc_type, Warn): + self.status, detail = WARN, str(exc) + elif issubclass(exc_type, ServerGone): + return False # let the suite end + elif issubclass(exc_type, AssertionError): + self.status, detail = FAIL, str(exc) or "assertion failed" + elif issubclass(exc_type, KeyboardInterrupt): + return False + else: + self.status = FAIL + detail = f"{exc_type.__name__}: {exc}" + if self.ctx.verbose: + traceback.print_exception(exc_type, exc, tb) + self.ctx.record(Result(self.suite, self.name, self.status, self.model, detail, + round(seconds, 2), self.data)) + if self.status == FAIL and self.server is not None and self.server.died(): + self.ctx.record(Result(self.suite, "server-crash", FAIL, self.model, + f"flm serve exited (code {self.server.proc.returncode}) " + f"during {self.name}", + data={"log_tail": self.server.log_tail()})) + if not self.server.restart(): + raise ServerGone(f"server for {self.suite} could not be restarted") + return True + + @property + def ok(self) -> bool: + return self.status in (PASS, WARN) + + +class Context: + def __init__(self, *, flm_bin: str, label: str, tier: str, plan: Plan, + out_dir: Path, request_timeout: float, port: int | None = None, + verbose: bool = False): + self.flm_bin = flm_bin + self.label = label + self.tier = tier + self.plan = plan + self.out_dir = Path(out_dir) + self.out_dir.mkdir(parents=True, exist_ok=True) + self.request_timeout = request_timeout + self.port = port + self.verbose = verbose + self.results: list[Result] = [] + self.servers: list[FlmServer] = [] + self.version = "unknown" + self.assets = ASSETS + self.repo_root = REPO_ROOT + + # -- servers and cases ----------------------------------------------------- + + def server(self, name: str, args: list[str] | None = None, **kwargs) -> FlmServer: + """A server for one suite, not yet started; use it as a context manager.""" + kwargs.setdefault("port", self.port) + server = FlmServer(self.flm_bin, name=name, log_dir=self.out_dir, args=args, + request_timeout=self.request_timeout, **kwargs) + self.servers.append(server) + return server + + def case(self, suite: str, name: str, model: str = "", + server: FlmServer | None = None) -> Case: + return Case(self, suite, name, model, server) + + def record(self, result: Result) -> None: + self.results.append(result) + mark = {PASS: "PASS", FAIL: "FAIL", SKIP: "SKIP", WARN: "WARN"}[result.status] + where = f" [{result.model}]" if result.model else "" + line = f" {mark} {result.suite}/{result.case}{where} ({result.seconds:.1f}s)" + if result.detail: + line += f": {result.detail}" + print(line, flush=True) + + def not_attempted(self, suite: str, models: list[str], reason: str) -> None: + for model in models: + self.record(Result(suite, "not-attempted", FAIL, model, reason)) + + def run_suite(self, module) -> None: + """Runs one suite module, turning a fatal error into a recorded failure.""" + print(f"\n== {module.NAME}: {module.DESCRIPTION}", flush=True) + started = time.monotonic() + try: + module.run(self) + except ServerGone as exc: + self.record(Result(module.NAME, "server-gone", FAIL, detail=str(exc))) + except ServerError as exc: + self.record(Result(module.NAME, "server-start", FAIL, detail=str(exc))) + except KeyboardInterrupt: + raise + except Exception as exc: + traceback.print_exc() + self.record(Result(module.NAME, "suite-error", FAIL, + detail=f"{type(exc).__name__}: {exc}")) + for server in self.servers: + if server.unkillable: + server.unkillable = False # report it once + self.record(Result(module.NAME, "server-stop", FAIL, + detail=f"flm serve (pid {server.proc.pid}) survived SIGKILL, " + f"likely stuck in the NPU driver; later suites may " + f"find the NPU busy", + data={"log_tail": server.log_tail()})) + print(f"== {module.NAME} done in {time.monotonic() - started:.0f}s", flush=True) + + def asset(self, relative: str) -> Path: + path = self.assets / relative + if not path.is_file(): + raise FileNotFoundError(f"test asset missing: {path}") + return path + + # -- report ---------------------------------------------------------------- + + @property + def failed(self) -> bool: + return any(r.status == FAIL for r in self.results) + + def write_report(self, suites: list[str], seconds: float) -> Path: + counts = {s: sum(1 for r in self.results if r.status == s) + for s in (PASS, FAIL, WARN, SKIP)} + report = { + "label": self.label, + "tier": self.tier, + "flm_version": self.version, + "suites": suites, + "seconds": round(seconds, 1), + "counts": counts, + "results": [asdict(r) for r in self.results], + } + path = self.out_dir / "results.json" + path.write_text(json.dumps(report, indent=2), encoding="utf-8") + (self.out_dir / "summary.md").write_text(render_markdown(report), encoding="utf-8") + return path + + +def render_markdown(report: dict) -> str: + """One package's results as Markdown: counts, then every non-pass case.""" + c = report["counts"] + icon = "❌" if c["fail"] else ("⚠️" if c["warn"] else "✅") + lines = [ + f"### {icon} {report['label']} ({report['tier']} tier, flm {report['flm_version']})", + "", + f"{c['pass']} passed, {c['fail']} failed, {c['warn']} warnings, {c['skip']} skipped " + f"in {report['seconds'] / 60:.1f} min. Suites: {', '.join(report['suites'])}.", + "", + ] + notable = [r for r in report["results"] if r["status"] in (FAIL, WARN)] + if notable: + lines += ["| | Suite | Case | Model | Detail |", "|---|---|---|---|---|"] + for r in notable: + mark = "❌" if r["status"] == FAIL else "⚠️" + detail = r["detail"].replace("|", "\\|").replace("\n", " ")[:300] + lines.append(f"| {mark} | {r['suite']} | {r['case']} | {r['model']} | {detail} |") + lines.append("") + per_suite = {} + for r in report["results"]: + s = per_suite.setdefault(r["suite"], {PASS: 0, FAIL: 0, WARN: 0, SKIP: 0, "s": 0.0}) + s[r["status"]] += 1 + s["s"] += r["seconds"] + if per_suite: + lines += ["| Suite | Pass | Fail | Warn | Skip | Time |", "|---|---|---|---|---|---|"] + for name, s in per_suite.items(): + lines.append(f"| {name} | {s[PASS]} | {s[FAIL]} | {s[WARN]} | {s[SKIP]} " + f"| {s['s'] / 60:.1f} min |") + lines.append("") + return "\n".join(lines) + + +def configure_stdout() -> None: + # The live log is the only progress a CI watcher sees; block buffering would + # hold it until the job ends. Model output can be any script, which the + # Windows console code page cannot always encode, and a print must never + # be what ends a suite. + for stream in (sys.stdout, sys.stderr): + if hasattr(stream, "reconfigure"): + stream.reconfigure(line_buffering=True, encoding="utf-8", errors="backslashreplace") diff --git a/.github/script/npu_test/plan.py b/.github/script/npu_test/plan.py new file mode 100644 index 000000000..63dd78fc6 --- /dev/null +++ b/.github/script/npu_test/plan.py @@ -0,0 +1,236 @@ +"""The model catalog, and which models each suite runs at each tier. + +Tiers: + install a handful of requests, to show an installed .deb or MSI works + pr one model per engine family, plus every model of a family the + change touches (--families) + full every model in the catalog (nightly) + +The chat suite covers language models family by family, so there is no +separate sweep of every model on a pull request: one model per engine family +exercises the engine, and the families a change touches run in full. +""" + +from __future__ import annotations + +import json +import subprocess +from dataclasses import dataclass, field + +TIERS = ("install", "pr", "full") + +# One model per engine family for the pr tier, the smallest that still covers +# the engine. None leaves a family out of the pr tier unless the change +# touches it: those are the large models whose load alone costs minutes, on +# engines no other model shares. A family missing from this table (one added +# to the catalog later) gets its smallest model, so it is never silently +# untested. +PR_CHAT = { + "hunyuan": "hy-mt2-flash:1.8b", + "nanbeige": "nanbeige4.1:3b", + "qwen3vl": "qwen3vl-it:4b", + "qwen3vl-flash": "qwen3vl-flash:4b", + "gemma4e": "gemma4-it:e2b", + "gemma4-12b": "gemma4-it:12b", + "gemma4e-flash": "gemma4e-flash:e2b", + "qwen3.5": "qwen3.5:0.8b", + "minicpm-v-4.7": "minicpm-v-4.7:1b", + "qwen3.6-moe": None, # 35 GB + "qwen3.8-mtp": None, # 27 GB, ~2 tok/s + "lfm2": "lfm2:1.2b", + "lfm3": "lfm3:1.7b", + "lfm2.5-tk": "lfm2.5-tk:1.2b", + "phi4": "phi4-mini-it:4b", + "gemma3-text": "gemma3:1b", + "gemma3": "gemma3:4b", + "llama3": "llama3.2:1b", + "deepseek-r1": None, # the llama3 engine, already covered + "deepseek-r1-0528": None, # the qwen3 engine, already covered + "qwen3": "qwen3:0.6b", + "qwen3-tk": None, # the qwen3 engine, already covered + "qwen3-it": None, # the qwen3 engine, already covered + "gpt-oss": None, # 20 GB + "qwen2": "qwen2.5-it:3b", + "qwen2vl": "qwen2.5vl-it:3b", +} + +# Families that are not chat models; each has its own suite. +NON_CHAT_FAMILIES = {"embed-gemma", "d3", "whisper-v3"} + +# Models that reason even when asked not to. Their answer comes after the +# reasoning, so they need a larger token budget before any content appears. +ALWAYS_THINKS_FAMILIES = {"qwen3-tk", "lfm2.5-tk", "deepseek-r1", "deepseek-r1-0528", + "nanbeige", "gpt-oss"} + +# Per-suite model lists for the tiers that do not derive from PR_CHAT. +SUITE_MODELS = { + "api": { + "install": [], + "pr": ["qwen3:0.6b", "gemma4-it:e2b"], + "full": ["qwen3:0.6b", "qwen3:4b", "qwen3-it:4b", "gemma4-it:e2b", "qwen3.5:2b", + "llama3.2:3b", "gpt-oss:20b", "lfm2.5-tk:1.2b"], + }, + "prefix_cache": { + "install": [], + "pr": ["qwen3:0.6b", "gemma4-it:e2b"], + "full": ["qwen3:0.6b", "gemma4-it:e2b", "qwen3.5:0.8b", "llama3.2:1b", + "lfm2:1.2b", "phi4-mini-it:4b", "gemma3:1b"], + }, + "serving": { + "install": [], + "pr": ["qwen3:0.6b", "llama3.2:1b"], + "full": ["qwen3:0.6b", "llama3.2:1b"], + }, + # Chat models fed audio through /v1/chat/completions. Whisper itself is + # always tested; these are the chat models with an audio encoder. + "asr_chat": { + "install": [], + # 12b has its own audio front end, separate from the e2b/e4b one. + "pr": ["gemma4-it:e2b", "gemma4-it:e4b", "nemotron3.5-bolt:e4b", "gemma4-it:12b"], + "full": None, # every catalog model with asr=true, Whisper aside + }, + "embedding": { + "install": ["embed-gemma:300m"], + "pr": ["embed-gemma:300m"], + "full": None, # every embed-gemma model + }, + "decision": { + "install": ["d3:lite"], + "pr": ["d3:lite"], + "full": None, # every d3 model + }, + "whisper": { + "install": ["whisper-v3:turbo"], + "pr": ["whisper-v3:turbo"], + "full": None, # every whisper-v3 model + }, +} + +INSTALL_CHAT = ["qwen3:0.6b"] + + +@dataclass +class Model: + id: str + family: str = "" + vlm: bool = False + asr: bool = False + think: bool = False + labels: list = field(default_factory=list) + size: float = 0.0 + + @property + def single_turn(self) -> bool: + return "single-turn" in self.labels + + @property + def always_thinks(self) -> bool: + return self.family in ALWAYS_THINKS_FAMILIES + + @property + def translation(self) -> bool: + # Not lfm2-trans: that is LFM2-2.6B-Transcript, a meeting summarizer. + return "translation" in self.labels or self.id.startswith("translategemma") + + +def load_catalog(flm_bin: str) -> dict[str, Model]: + """The catalog as `flm list --json --filter all` reports it on this machine. + + That is the only listing that carries the vlm/asr flags and families, and it + already leaves out models this NPU cannot run. + """ + proc = subprocess.run([flm_bin, "list", "--json", "--filter", "all"], + capture_output=True, text=True, timeout=300, check=True) + start = proc.stdout.find("{") # tolerate banner lines before the JSON + if start < 0: + raise RuntimeError("no JSON in `flm list --json` output") + models = {} + for entry in json.loads(proc.stdout[start:]).get("models", []): + model_id = entry.get("model") or entry.get("name") + if not model_id: + continue + details = entry.get("details") or {} + models[model_id] = Model( + id=model_id, + family=details.get("family", ""), + vlm=bool(entry.get("vlm")), + asr=bool(entry.get("asr")), + think=bool(details.get("think")), + labels=list(entry.get("label") or []), + size=float(entry.get("size") or 0), + ) + return models + + +class Plan: + def __init__(self, catalog: dict[str, Model], tier: str, families: list[str], + only_models: list[str] | None = None): + if tier not in TIERS: + raise ValueError(f"unknown tier {tier!r}") + self.catalog = catalog + self.tier = tier + self.families = set(families) + self.only_models = only_models + + def _known(self, ids) -> list[str]: + """Keeps catalog models, in order, dropping ones this machine lacks.""" + out = [] + for model_id in ids: + if model_id in self.catalog and model_id not in out: + if not self.only_models or model_id in self.only_models: + out.append(model_id) + return out + + def _family(self, family: str) -> list[str]: + return [m.id for m in self.catalog.values() if m.family == family] + + def _touched(self, predicate) -> list[str]: + return [m.id for m in self.catalog.values() + if m.family in self.families and predicate(m)] + + def chat(self) -> list[str]: + # --models names exactly what to test, whatever the tier would pick. + if self.only_models: + return [m for m in self.only_models if m in self.catalog + and self.catalog[m].family not in NON_CHAT_FAMILIES] + if self.tier == "install": + return self._known(INSTALL_CHAT) + chat_models = [m for m in self.catalog.values() if m.family not in NON_CHAT_FAMILIES] + if self.tier == "full": + return self._known(m.id for m in chat_models) + picked = [] + for family in sorted({m.family for m in chat_models}): + if family in PR_CHAT: + if PR_CHAT[family]: + picked.append(PR_CHAT[family]) + else: + members = sorted((m for m in chat_models if m.family == family), + key=lambda m: (m.size, m.id)) + picked.append(members[0].id) + picked += self._touched(lambda m: m.family not in NON_CHAT_FAMILIES) + return self._known(picked) + + def suite(self, name: str) -> list[str]: + """Models for a suite with a fixed list in SUITE_MODELS.""" + table = SUITE_MODELS[name] + listed = table[self.tier] + if listed is None: # full tier: derive from the catalog + listed = { + "asr_chat": [m.id for m in self.catalog.values() + if m.asr and m.family != "whisper-v3"], + "embedding": self._family("embed-gemma"), + "decision": self._family("d3"), + "whisper": self._family("whisper-v3"), + }[name] + extra = [] + if self.tier == "pr": + extra = { + "asr_chat": self._touched(lambda m: m.asr and m.family != "whisper-v3"), + "decision": self._touched(lambda m: m.family == "d3"), + "embedding": self._touched(lambda m: m.family == "embed-gemma"), + "whisper": self._touched(lambda m: m.family == "whisper-v3"), + }.get(name, []) + return self._known(list(listed) + extra) + + def model(self, model_id: str) -> Model: + return self.catalog.get(model_id) or Model(id=model_id) diff --git a/.github/script/npu_test/run.py b/.github/script/npu_test/run.py new file mode 100644 index 000000000..bc65196ad --- /dev/null +++ b/.github/script/npu_test/run.py @@ -0,0 +1,111 @@ +"""Runs the NPU test suites against one flm binary and writes a report. + + python -m npu_test --flm-bin path/to/flm --label linux-xrt-portable \ + --tier pr --output-dir results + +Run from .github/script (or with it on PYTHONPATH). Exits 1 if any case +failed; warnings and skips do not fail the run. +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +import time + +from .harness import FAIL, Context, Result, configure_stdout +from .plan import TIERS, Plan, load_catalog +from .suites import SUITES, ORDER + + +def parse_args(argv=None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--flm-bin", default=os.environ.get("FLM_BIN", "flm"), + help="The flm binary under test (default: $FLM_BIN, else flm).") + parser.add_argument("--label", required=True, + help="Names the package under test in the report, " + "e.g. linux-xrt-portable.") + parser.add_argument("--tier", choices=TIERS, default="pr") + parser.add_argument("--suites", default="all", + help=f"Comma-separated subset of: {', '.join(ORDER)} (default: all).") + parser.add_argument("--families", default="", + help="Comma-separated model families the change touches; " + "the pr tier tests every model in them.") + parser.add_argument("--asr-full", action="store_true", + help="Run the whole ASR evaluation set even on the pr tier.") + parser.add_argument("--models", default="", + help="Comma-separated model ids; restricts every suite to these.") + parser.add_argument("--asr-expected-key", default="", + help="Row of asr_eval/expected.json to check WER against, " + "e.g. linux-hrx (default: none, core checks only).") + parser.add_argument("--output-dir", default="npu-test-results") + parser.add_argument("--request-timeout", type=float, default=600.0) + # Not $FLM_SERVE_PORT: a runner may set it for an flm of its own, which then + # holds that port (it did on a Windows runner, failing every server start). + parser.add_argument("--port", type=int, default=None, + help="Port for the servers this run starts (default: a free port " + "for each server).") + parser.add_argument("-v", "--verbose", action="store_true") + return parser.parse_args(argv) + + +def main(argv=None) -> int: + configure_stdout() + args = parse_args(argv) + started = time.monotonic() + + names = ORDER if args.suites == "all" else [s.strip() for s in args.suites.split(",") if s.strip()] + unknown = [n for n in names if n not in SUITES] + if unknown: + print(f"unknown suite(s): {', '.join(unknown)}; known: {', '.join(ORDER)}") + return 2 + + version = subprocess.run([args.flm_bin, "version"], capture_output=True, text=True) + print(version.stdout.strip() or version.stderr.strip()) + if version.returncode != 0: + print(f"`{args.flm_bin} version` failed with code {version.returncode}") + return 1 + + # Without the catalog nothing can run, but the failure still belongs in the + # report, so a broken package shows up as a failed case, not a traceback. + catalog_error = "" + try: + catalog = load_catalog(args.flm_bin) + except Exception as exc: + catalog, catalog_error = {}, f"`flm list --json` failed: {type(exc).__name__}: {exc}" + families = [f.strip() for f in args.families.split(",") if f.strip()] + only = [m.strip() for m in args.models.split(",") if m.strip()] or None + plan = Plan(catalog, args.tier, families, only_models=only) + ctx = Context(flm_bin=args.flm_bin, label=args.label, tier=args.tier, plan=plan, + out_dir=args.output_dir, request_timeout=args.request_timeout, + port=args.port, verbose=args.verbose) + ctx.asr_full = args.asr_full or args.tier == "full" + ctx.asr_expected_key = args.asr_expected_key + + print(f"{args.label}: {args.tier} tier, {len(catalog)} catalog models, " + f"suites: {', '.join(names)}" + (f", touched families: {', '.join(families)}" + if families else "")) + try: + if catalog_error: + print(catalog_error) + ctx.record(Result("setup", "catalog", FAIL, detail=catalog_error)) + else: + for name in names: + ctx.run_suite(SUITES[name]) + finally: + # The version the servers reported, which is what the report should name. + ctx.version = (version.stdout.strip().split()[-1] if version.stdout.strip() + else "unknown") + path = ctx.write_report(names, time.monotonic() - started) + print(f"\nReport: {path}") + + counts = {s: sum(1 for r in ctx.results if r.status == s) for s in ("pass", "fail", "warn", "skip")} + print(f"{counts['pass']} passed, {counts['fail']} failed, {counts['warn']} warnings, " + f"{counts['skip']} skipped") + return 1 if ctx.failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/script/npu_test/server.py b/.github/script/npu_test/server.py new file mode 100644 index 000000000..739999f73 --- /dev/null +++ b/.github/script/npu_test/server.py @@ -0,0 +1,189 @@ +"""Starts, watches and stops one `flm serve` process. + +Each suite gets its own server with the flags it needs (`-a 1` for Whisper, +`--prefix-slots 2`, ...), so a crash in one suite cannot leak into the next, +and its log is kept as a separate file in the report. +""" + +from __future__ import annotations + +import os +import signal +import socket +import subprocess +import time +import urllib.error +import urllib.request +from pathlib import Path + +from .client import Client + +IS_WINDOWS = os.name == "nt" +# Matches utils::get_server_port() in src/common/utils.cpp. +DEFAULT_PORT = 52625 + + +class ServerError(RuntimeError): + pass + + +class FlmServer: + def __init__(self, flm_bin: str, *, name: str, log_dir: Path, + args: list[str] | None = None, host: str = "127.0.0.1", + port: int | None = None, ready_timeout: float = 300.0, + request_timeout: float = 600.0, max_restarts: int = 3): + self.flm_bin = flm_bin + self.name = name + self.args = list(args or []) + self.host = host + # None: a free port per start. A server stuck in the NPU driver cannot + # be killed and keeps its port, which would otherwise stop every later + # suite from starting. + self.fixed_port = port + self.port = port or DEFAULT_PORT + self.unkillable = False + self.ready_timeout = ready_timeout + self.max_restarts = max_restarts + self.restarts = 0 + self.log_path = Path(log_dir) / f"server-{name}.log" + self.proc: subprocess.Popen | None = None + self._log = None + self.client = Client(f"http://{host}:{self.port}", timeout=request_timeout) + self.version = "unknown" + + @property + def base_url(self) -> str: + return self.client.base_url + + @property + def command(self) -> list[str]: + # -a / -e default to 0 so a suite only gets the sidecars it asks for. + cmd = [self.flm_bin, "serve", "--host", self.host, "--port", str(self.port)] + if "-a" not in self.args and "--asr" not in self.args: + cmd += ["-a", "0"] + if "-e" not in self.args and "--embed" not in self.args: + cmd += ["-e", "0"] + return cmd + self.args + + # -- lifecycle ------------------------------------------------------------- + + def __enter__(self) -> "FlmServer": + self.start() + return self + + def __exit__(self, *exc) -> bool: + self.stop() + return False + + def _port_answers(self) -> bool: + try: + with urllib.request.urlopen(f"{self.base_url}/v1/version", timeout=3): + return True + except (urllib.error.URLError, OSError): + return False + + def _free_port(self) -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind((self.host, 0)) + return sock.getsockname()[1] + + def start(self, append: bool = False) -> None: + if self.fixed_port is None: + self.port = self._free_port() + # Suites hold on to self.client, so move it rather than replace it. + self.client.base_url = f"http://{self.host}:{self.port}" + # A server left behind by an earlier job (or anything else on the port) + # would answer every request in place of the one under test. + if self._port_answers(): + raise ServerError(f"port {self.port} already answers /v1/version; another " + f"server is running, so the package under test cannot start") + self.log_path.parent.mkdir(parents=True, exist_ok=True) + self._log = open(self.log_path, "ab" if append else "wb") + if append: + self._log.write(f"\n=== restart {self.restarts} ===\n".encode()) + self._log.flush() + print(f"[server:{self.name}] {' '.join(self.command)}", flush=True) + kwargs: dict = {"stdout": self._log, "stderr": subprocess.STDOUT} + if IS_WINDOWS: + # So Ctrl-Break can stop the server without also stopping us. + kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP + self.proc = subprocess.Popen(self.command, **kwargs) + self._wait_ready() + self.version = self.client.version() + + def _wait_ready(self) -> None: + deadline = time.monotonic() + self.ready_timeout + while time.monotonic() < deadline: + if self.proc is not None and self.proc.poll() is not None: + raise ServerError(f"flm serve exited with code {self.proc.returncode} " + f"before it was ready; see {self.log_path.name}") + try: + with urllib.request.urlopen(f"{self.base_url}/v1/version", timeout=5): + pass + except (urllib.error.URLError, OSError): + time.sleep(2) + continue + # Something answered; make sure it is our process and not one that + # took the port between the check in start() and now. + time.sleep(1) + if self.proc is not None and self.proc.poll() is not None: + raise ServerError(f"flm serve exited with code {self.proc.returncode}; " + f"port {self.port} is answered by another process") + return + raise ServerError(f"flm serve did not answer within {self.ready_timeout:.0f}s; " + f"see {self.log_path.name}") + + def stop(self) -> None: + if self.proc is not None and self.proc.poll() is None: + try: + self.proc.send_signal(signal.CTRL_BREAK_EVENT if IS_WINDOWS else signal.SIGINT) + self.proc.wait(timeout=30) + except (subprocess.TimeoutExpired, OSError, ValueError): + self.proc.kill() + try: + self.proc.wait(timeout=15) + except subprocess.TimeoutExpired: + # Still there after SIGKILL: stuck in the kernel, which for + # flm means the NPU driver. Nothing here can free it. + self.unkillable = True + print(f"[server:{self.name}] pid {self.proc.pid} survived SIGKILL; " + f"it is likely stuck in the NPU driver", flush=True) + if self._log is not None: + self._log.close() + self._log = None + + def alive(self) -> bool: + return self.proc is not None and self.proc.poll() is None + + def died(self, grace: float = 5.0) -> bool: + """True once the process has exited, waiting briefly for one on its way out. + + A request that kills the server often fails before the process is gone, + and calling it alive there would blame the next model for the crash. + """ + deadline = time.monotonic() + grace + while time.monotonic() < deadline: + if not self.alive(): + return True + time.sleep(0.2) + return not self.alive() + + def restart(self) -> bool: + """Brings the server back after a crash. False once the budget is spent.""" + if self.restarts >= self.max_restarts: + return False + self.restarts += 1 + self.stop() + try: + self.start(append=True) + except ServerError as exc: + print(f"[server:{self.name}] restart failed: {exc}", flush=True) + return False + return True + + def log_tail(self, lines: int = 40) -> str: + try: + text = self.log_path.read_text(encoding="utf-8", errors="replace") + except OSError: + return "" + return "\n".join(text.splitlines()[-lines:]) diff --git a/.github/script/npu_test/suites/__init__.py b/.github/script/npu_test/suites/__init__.py new file mode 100644 index 000000000..5cf1fe6cf --- /dev/null +++ b/.github/script/npu_test/suites/__init__.py @@ -0,0 +1,10 @@ +"""The test suites, in the order a run executes them. + +Each module defines NAME, DESCRIPTION and run(ctx); see harness.py. The fast, +broad suites come first so a broken build shows up within minutes. +""" + +from . import api, asr, chat, decision, embedding, prefix_cache, serving + +ORDER = ["chat", "serving", "prefix_cache", "api", "embedding", "decision", "asr"] +SUITES = {m.NAME: m for m in (chat, serving, prefix_cache, api, embedding, decision, asr)} diff --git a/.github/script/npu_test/suites/api.py b/.github/script/npu_test/suites/api.py new file mode 100644 index 000000000..cf39e9070 --- /dev/null +++ b/.github/script/npu_test/suites/api.py @@ -0,0 +1,285 @@ +"""The OpenAI API contract: runs the live tests in src/test against our server. + +src/test/openai_api_test (output parsing, streaming, tools, response_format, +logprobs, sampling, continue_final_message, errors) and +src/test/openai_usage (usage and cache token counts) are standalone scripts +that developers run by hand. Running them here, as subprocesses, keeps one +copy of each check. Every PASS/FAIL/SKIP line they print becomes its own case, +so a failure shows by name in the report rather than as "run_all.py exited 1". +""" + +from __future__ import annotations + +import os +import re +import subprocess +import sys +import threading +import time +from pathlib import Path + +from ..harness import FAIL, PASS, SKIP, Result, ServerGone, expect + +NAME = "api" +DESCRIPTION = "OpenAI API and usage contract tests from src/test, per model" + +# A run_all.py pass is about a minute on qwen3:0.6b; the budget is for large +# models on the full tier, whose answers come after up to 1024 reasoning tokens. +RUN_ALL_TIMEOUT = 30 * 60 +USAGE_TIMEOUT = 10 * 60 + +# flm_api.run_cases(): "PASS name (1.2s)", "FAIL name (1.2s): why", +# "FAIL name: cannot reach the server: why", "SKIP name: why". +CASE_LINE = re.compile(r"^(PASS|FAIL|SKIP) (\S+)(?: \(([\d.]+)s\))?(?:: ?(.*))?$") +MODULE_LINE = re.compile(r"^== (\S+)$") +SUMMARY_LINE = re.compile(r"^(\d+) passed, (\d+) failed, (\d+) skipped$") +STATUS = {"PASS": PASS, "FAIL": FAIL, "SKIP": SKIP} + +# test_usage_contract.py: "Test 1: ...", "Test 1 PASSED", "FAIL: why". +USAGE_TESTS = {"1": "usage.non_stream", "2": "usage.stream"} +USAGE_START = re.compile(r"^Test (\d+): ") +USAGE_PASSED = re.compile(r"^Test (\d+) PASSED$") +USAGE_FAIL = re.compile(r"^FAIL: (.*)$") + + +def run(ctx) -> None: + models = ctx.plan.suite(NAME) + if not models: + return + tests = ctx.repo_root / "src" / "test" + run_all = tests / "openai_api_test" / "run_all.py" + usage = tests / "openai_usage" / "test_usage_contract.py" + with ctx.case(NAME, "scripts-present") as case: + missing = [str(p) for p in (run_all, usage) if not p.is_file()] + # The workflow sparse-checks-out these directories; a path change there + # would otherwise look like every model passing nothing. + expect(not missing, f"missing: {', '.join(missing)}") + if case.status != PASS: + ctx.not_attempted(NAME, models, "API test scripts missing") + return + + with ctx.server(NAME) as server: + for i, model in enumerate(models): + try: + test_model(ctx, server, model, run_all, usage) + except ServerGone: + ctx.not_attempted(NAME, models[i + 1:], "server could not be restarted") + raise + + +def test_model(ctx, server, model: str, run_all: Path, usage: Path) -> None: + with ctx.case(NAME, "load", model, server) as case: + # Loading (and downloading) gets its own case and timeout, so a model + # that cannot load is one failure rather than every case timing out. + server.client.chat(model, [{"role": "user", "content": "Hi"}], stream=False, + max_tokens=1, temperature=0, top_k=1) + if case.status != PASS: + return + + common = [model, "--endpoint", server.base_url, "--timeout", str(ctx.request_timeout)] + always_thinks = ["--always-thinks"] if ctx.plan.model(model).always_thinks else [] + with ctx.case(NAME, "openai_api_test", model, server) as case: + parser = RunAllParser(ctx, model, server) + out = run_script(ctx, run_all, common + always_thinks, model, RUN_ALL_TIMEOUT, + on_line=parser.feed) + case.data.update(out.data()) + parser.flush() + # Before the output checks: after a crash, they would only blame run_all.py. + expect(server.alive(), f"flm serve exited during run_all.py, in " + f"{parser.module} after {parser.last_case or 'no case'}") + parser.check(out) + with ctx.case(NAME, "openai_usage", model, server) as case: + out = run_script(ctx, usage, common, model, USAGE_TIMEOUT) + case.data.update(out.data()) + check_usage(ctx, model, out) + expect(server.alive(), f"flm serve exited during test_usage_contract.py " + f"({out.status_text()})") + + +class ScriptOutput: + def __init__(self, script: Path, log: Path): + self.script = script + self.log = log + self.lines: list[str] = [] + self.returncode: int | None = None + self.killed = "" # why run_script() stopped it, if it did + self.seconds = 0.0 + + def tail(self, count: int = 40) -> str: + return "\n".join(self.lines[-count:]) + + def last_line(self) -> str: + return next((l.strip() for l in reversed(self.lines) if l.strip()), "(no output)") + + def status_text(self) -> str: + if self.killed: + return f"killed: {self.killed}" + return f"exit code {self.returncode}" + + def data(self) -> dict: + data = {"log": self.log.name, "exit_code": self.returncode, + "seconds": round(self.seconds, 1)} + if self.killed or self.returncode != 0: + data["output_tail"] = self.tail() + return data + + +def run_script(ctx, script: Path, args: list[str], model: str, timeout: float, + on_line=None) -> ScriptOutput: + """Runs one test script to completion or timeout, keeping its output. + + on_line sees each line as it comes and may return a reason to stop the + script early. The output also goes to a log file next to the server logs, since the + report keeps only a tail of it. + """ + safe = re.sub(r"[^A-Za-z0-9._-]+", "_", model) + out = ScriptOutput(script, ctx.out_dir / f"api-{script.stem}-{safe}.log") + env = dict(os.environ, PYTHONUNBUFFERED="1", PYTHONIOENCODING="utf-8", + PYTHONDONTWRITEBYTECODE="1") + cmd = [sys.executable, "-u", str(script)] + args + print(f" [{NAME}] {model}: {script.name} (log: {out.log.name})", flush=True) + started = time.monotonic() + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + cwd=str(script.parent), env=env) + + def kill(reason: str) -> None: + if not out.killed: + out.killed = reason + proc.kill() + + # A timer rather than communicate(timeout=) so lines are seen as they come. + timer = threading.Timer(timeout, kill, args=(f"timed out after {timeout:.0f}s",)) + timer.start() + try: + with open(out.log, "w", encoding="utf-8") as log: + log.write(" ".join(cmd) + "\n\n") + for raw in proc.stdout: + line = raw.decode("utf-8", errors="replace").rstrip("\r\n") + out.lines.append(line) + log.write(line + "\n") + log.flush() + stop = on_line(line) if on_line is not None else None + if stop: + kill(stop) + out.returncode = proc.wait() + finally: + timer.cancel() + if proc.poll() is None: + proc.kill() + proc.wait() + out.seconds = time.monotonic() - started + return out + + +class RunAllParser: + """Turns run_all.py output into one case per result line, as it arrives. + + A FAIL's message can span lines, so each result is recorded when the + next marker line (or the end of output) closes it. + """ + + def __init__(self, ctx, model: str, server): + self.ctx = ctx + self.model = model + self.server = server + self.module = "" + self.summary = None + self.counts = {PASS: 0, FAIL: 0, SKIP: 0} + self.last_case = "" + self._pending: Result | None = None + + def feed(self, line: str) -> str | None: + m = CASE_LINE.match(line) + if m: + self.flush() + mark, name, seconds, detail = m.groups() + self._pending = Result(NAME, name, STATUS[mark], self.model, + (detail or "").strip(), float(seconds or 0)) + if "cannot reach the server" in self._pending.detail and self.server.died(grace=3): + # Every later case would fail the same way; the crash is the + # one failure worth reporting, and the server needs a restart. + return "flm serve died" + return None + m = MODULE_LINE.match(line) or SUMMARY_LINE.match(line) + if m: + self.flush() + if len(m.groups()) == 1: + self.module = m.group(1) + else: + self.summary = tuple(int(g) for g in m.groups()) + return + if line.startswith("Traceback (most recent call last)"): + # run_cases() lets anything but an assertion or a connection error + # escape, which ends the run; check() reports that, not the case + # that happened to print last. + self.flush() + elif self._pending is not None and self._pending.status == FAIL and line.strip(): + self._pending.detail += "\n" + line + + def flush(self) -> None: + if self._pending is not None: + self.ctx.record(self._pending) + self.counts[self._pending.status] += 1 + self.last_case = self._pending.case + self._pending = None + + def check(self, out: ScriptOutput) -> None: + """Checks what the per-case lines cannot show: a timeout, a crash + between cases, or output this parser did not understand.""" + total = sum(self.counts.values()) + expect(not out.killed, f"run_all.py {out.status_text()}, in {self.module or 'startup'} " + f"after {self.last_case or 'no case'}") + expect(total, f"run_all.py printed no result lines ({out.status_text()}): " + f"{out.last_line()}") + expect(self.summary is not None, + f"run_all.py stopped in {self.module} after {self.last_case} " + f"({out.status_text()}): {out.last_line()}") + parsed = (self.counts[PASS], self.counts[FAIL], self.counts[SKIP]) + expect(parsed == self.summary, f"parsed {parsed} pass/fail/skip but run_all.py " + f"reported {self.summary}") + expect((out.returncode != 0) == (self.counts[FAIL] > 0), + f"run_all.py exit code {out.returncode} with {self.counts[FAIL]} failed cases") + + +def check_usage(ctx, model: str, out: ScriptOutput) -> None: + """Records the usage script's two tests as cases. + + It stops at the first failure, printing "FAIL: why" for a contract + violation or a traceback for anything else, so a test that started and + did not pass failed with that message, and a later one never ran. + """ + started, passed = [], set() + reason = "" + for line in out.lines: + m = USAGE_START.match(line) + if m: + started.append(m.group(1)) + m = USAGE_PASSED.match(line) + if m: + passed.add(m.group(1)) + m = USAGE_FAIL.match(line) + if m: + reason = m.group(1) + expect(started, f"test_usage_contract.py printed no tests ({out.status_text()}): " + f"{out.last_line()}") + blocked = "" + for number, name in USAGE_TESTS.items(): + if number in passed: + ctx.record(Result(NAME, name, PASS, model)) + elif number in started: + why = reason or out.last_line() + if out.killed: + why = f"{out.status_text()}; last line: {out.last_line()}" + ctx.record(Result(NAME, name, FAIL, model, why, + data={"output_tail": out.tail(15)})) + blocked = blocked or name + elif blocked: + ctx.record(Result(NAME, name, SKIP, model, f"not run: {blocked} failed first")) + else: + ctx.record(Result(NAME, name, FAIL, model, + f"never started ({out.status_text()}): {out.last_line()}")) + expect(not out.killed, f"test_usage_contract.py {out.status_text()}") + all_passed = all(n in passed for n in USAGE_TESTS) + expect((out.returncode == 0) == all_passed, + f"test_usage_contract.py exit code {out.returncode} but " + f"{len(passed)} of {len(USAGE_TESTS)} tests passed") diff --git a/.github/script/npu_test/suites/asr.py b/.github/script/npu_test/suites/asr.py new file mode 100644 index 000000000..00dc99a81 --- /dev/null +++ b/.github/script/npu_test/suites/asr.py @@ -0,0 +1,617 @@ +"""Whisper through /v1/audio/transcriptions, and chat models fed audio. + +Runs the FLEURS evaluation set in assets/asr_eval/ and scores it: WER per set +(CER for ja and zh) against human transcripts, NFKC-normalised, each file's +edits capped at its reference length so one runaway transcript cannot swamp a +set, plus a pooled rate over every set. Around that it checks feature groups: + + verbose_json segments ordered, inside the clip, non-empty, joining into + the text; duration equal to the clip length + timestamp_rules segments never overlap; the first starts within 1 s + window_tail no phrase repeated across 30 s window boundaries + language forcing the language is honoured; unknown codes rejected + prompt a prompt naming proper nouns does not hurt and helps them + quiet_log transcripts stay out of the server log + +Core checks, verbose_json, timestamp_rules and prompt fail the run. The others +only warn: window_tail because main's HRX build loops on long-form audio, +language and quiet_log because main never required them. + +The eval set is chosen by tier: install sends 2 en-short clips and skips the +groups that cost extra requests; pr sends every en-short clip and the first 3 +of every other set; full (or --asr-full) sends all 84. +""" + +from __future__ import annotations + +import base64 +import json +import re +import time +import unicodedata +from collections import Counter +from pathlib import Path + +from ..client import HttpError +from ..harness import ServerGone, expect, skip, warn + +NAME = "asr" +DESCRIPTION = "Whisper transcription accuracy and features, and chat models fed audio" + +# verbose_json's language, as OpenAI reports it: openai-whisper's lower-case names. +LANGUAGE_NAMES = { + "en": "english", "de": "german", "es": "spanish", "fr": "french", + "ja": "japanese", "zh": "chinese", "ar": "arabic", +} +CHAT_PROMPT = ("Transcribe the speech in this audio exactly, in its original language. " + "Reply with the transcript only.") +CHAT_CLIPS = 5 +# MP3 encoder padding and the server's own rounding. +DURATION_TOLERANCE_S = 0.1 +# The encoder is not bit-exact between passes, so two requests for the same +# clip can differ in a word or two (see the PR #794 notes). +SAME_TEXT_MAX_WER = 0.02 +REPEAT_NGRAM = 6 +RUNAWAY_RATIO = 1.5 +LONG_SETS = ("en-long", "en-longform") +# expected.json holds main's full-set rates from one machine per platform. +# The pooled rate gets 2 points of slack. A set gets 5, and only with 10 or +# more files: one flipped decode moves a ten-file set by a point or more, and +# a single long-form file by many, which HRX builds do between identical runs. +POOLED_TOLERANCE = 0.02 +SET_TOLERANCE = 0.05 +SET_CHECK_MIN_FILES = 10 +PR_CLIPS_PER_SET = 3 +INSTALL_CLIPS = 2 +# A sanity bound that holds on every platform (main is at 8-10% on en-short), +# so a broken decoder fails even when no expected.json row is given. +EN_SHORT_MAX_WER = 0.5 +ARABIC_DIACRITICS = re.compile("[\u064b-\u065f\u0670\u06d6-\u06ed]") + + +# -- scoring ------------------------------------------------------------------ + +def normalize(text: str, language: str) -> str: + """Lowercase, no punctuation or symbols, single spaces, NFKC.""" + text = unicodedata.normalize("NFKC", text).lower() + if language == "ar": + text = ARABIC_DIACRITICS.sub("", text) + text = "".join(" " if unicodedata.category(c)[0] in "PS" else c for c in text) + return " ".join(text.split()) + + +def tokens(text: str, language: str, metric: str) -> list[str]: + text = normalize(text, language) + return list(text.replace(" ", "")) if metric == "cer" else text.split() + + +def edit_distance(ref: list[str], hyp: list[str]) -> int: + previous = list(range(len(hyp) + 1)) + for i, ref_token in enumerate(ref, 1): + current = [i] + for j, hyp_token in enumerate(hyp, 1): + current.append(min(previous[j] + 1, current[j - 1] + 1, + previous[j - 1] + (ref_token != hyp_token))) + previous = current + return previous[-1] + + +def error_rate(ref: str, hyp: str, language: str, metric: str) -> tuple[int, int]: + """(edits, reference length), so rates can be pooled over a set.""" + ref_tokens = tokens(ref, language, metric) + return edit_distance(ref_tokens, tokens(hyp, language, metric)), len(ref_tokens) + + +def pooled(pairs) -> float: + """Error rate over many files, each file's edits capped at its length.""" + pairs = list(pairs) + edits = sum(min(e, n) for e, n in pairs) + length = sum(n for _, n in pairs) + return edits / length if length else 0.0 + + +def repeats(text: str, n: int = REPEAT_NGRAM) -> int: + """n-word sequences that occur more than once.""" + words = normalize(text, "en").split() + counts = Counter(tuple(words[i:i + n]) for i in range(len(words) - n + 1)) + return sum(c - 1 for c in counts.values() if c > 1) + + +def same_text(a: str, b: str) -> tuple[bool, str]: + edits, length = error_rate(a, b, "en", "wer") + rate = edits / length if length else float(bool(b.strip())) + return rate <= SAME_TEXT_MAX_WER, f"{rate:.1%} apart" if rate else "identical" + + +def pct(value: float | None) -> str: + return "-" if value is None else f"{value * 100:.2f}%" + + +def settle(problems: list[str], required: bool) -> None: + """Ends a case that collected several checks: fail, warn, or pass.""" + if problems: + message = "; ".join(problems) + if required: + raise AssertionError(message) + warn(message) + + +# -- selection ---------------------------------------------------------------- + +def select_items(manifest_items: list[dict], tier: str, full: bool) -> list[dict]: + """The clips this tier sends, in manifest order (so the subset is stable).""" + if full: + return list(manifest_items) + taken: Counter = Counter() + out = [] + for item in manifest_items: + name = item["set"] + if tier == "install": + keep = name == "en-short" and taken[name] < INSTALL_CLIPS + else: + keep = name == "en-short" or taken[name] < PR_CLIPS_PER_SET + if keep: + taken[name] += 1 + out.append(item) + return out + + +# -- requests ----------------------------------------------------------------- + +def transcribe(client, model: str, path: Path, **fields) -> tuple[dict, float]: + """(parsed body, seconds). Anything but a JSON object without "error" raises.""" + form = {"model": model, **{k: v for k, v in fields.items() if v is not None}} + started = time.monotonic() + _, raw = client.post_multipart("/v1/audio/transcriptions", form, + {"file": (path.name, path.read_bytes(), "audio/mpeg")}) + seconds = time.monotonic() - started + try: + body = json.loads(raw.decode("utf-8", "replace")) + except json.JSONDecodeError: + body = None + # The server answers with an empty body when no ASR model is loaded. + expect(isinstance(body, dict), f"not a JSON object: {raw[:120]!r}") + expect("error" not in body, f"error body: {json.dumps(body)[:200]}") + return body, seconds + + +class WhisperRun: + """One Whisper model's run: its eval results, shared by the later cases.""" + + def __init__(self, ctx, server, model: str, items: list[dict], all_items: list[dict]): + self.ctx = ctx + self.server = server + self.client = server.client + self.model = model + self.items = items + self.all_items = all_items + self.eval_dir = ctx.assets / "asr_eval" + self.results: dict = {"items": {}, "sets": {}, "overall": {}, "language": {}, + "prompt": {}} + self.segment_failures: Counter = Counter() + self.verbose_items = 0 + + def case(self, name: str): + return self.ctx.case(NAME, name, self.model, self.server) + + def path(self, item: dict) -> Path: + return self.ctx.asset(f"asr_eval/{item['file']}") + + # -- formats ------------------------------------------------------------- + + def formats(self) -> None: + """json, no response_format and verbose_json on one clip must agree.""" + if self.ctx.tier == "install": + item = self.items[0] + else: + item = next((i for i in self.items if i["set"] == "en-long"), self.items[0]) + path = self.path(item) + plain = verbose = None + with self.case("formats") as case: + case.data["file"] = item["file"] + plain, _ = transcribe(self.client, self.model, path, response_format="json") + text = plain.get("text") or "" + expect(text.strip(), "json text is empty") + default, _ = transcribe(self.client, self.model, path) + expect(set(default) == {"model", "text"}, + f"default format returns {', '.join(sorted(default))}, not model and text") + ok, detail = same_text(text, default.get("text") or "") + expect(ok, f"default text differs from json: {detail}") + with self.case("verbose-json") as case: + case.data["file"] = item["file"] + if plain is None: + skip("the json request failed") + verbose, _ = transcribe(self.client, self.model, path, + response_format="verbose_json") + expect("segments" in verbose, "no segments: the server ignores verbose_json") + ok, detail = same_text(plain.get("text") or "", verbose.get("text") or "") + expect(ok, f"verbose_json text differs from json: {detail}") + + # -- evaluation ---------------------------------------------------------- + + def check_segments(self, body: dict, duration: float) -> None: + """Tallies the verbose_json and timestamp_rules problems of one response.""" + segments = body.get("segments") or [] + text = body.get("text") or "" + problems = { + ("verbose_json", "duration is the clip length"): + not isinstance(body.get("duration"), (int, float)) + or abs(body["duration"] - duration) > DURATION_TOLERANCE_S, + ("verbose_json", "segment ids count up from 0"): + [s.get("id") for s in segments] != list(range(len(segments))), + ("verbose_json", "segments lie inside the clip"): + any(not 0 <= s["start"] <= s["end"] <= duration + DURATION_TOLERANCE_S + for s in segments), + ("verbose_json", "segments are in order"): + any(b["start"] < a["start"] for a, b in zip(segments, segments[1:])), + ("verbose_json", "no empty or zero-length segments"): + any(s["end"] - s["start"] < 0.01 or not s["text"].strip() for s in segments), + ("verbose_json", "segments join into the text"): + "".join(s["text"] for s in segments).strip() != text.strip(), + ("timestamp_rules", "segments never overlap"): + any(b["start"] < a["end"] - 0.005 for a, b in zip(segments, segments[1:])), + ("timestamp_rules", "first segment starts within 1 s"): + bool(segments) and segments[0]["start"] > 1.0, + } + for key, bad in problems.items(): + self.segment_failures[key] += bool(bad) + + def eval_set(self, set_name: str, set_items: list[dict]) -> None: + """One verbose_json request per clip: accuracy, speed, segments.""" + with self.case(f"eval-{set_name}") as case: + pairs, audio_s, wall_s, detected, runaways = [], 0.0, 0.0, 0, [] + errors = [] + for item in set_items: + try: + body, seconds = transcribe(self.client, self.model, self.path(item), + response_format="verbose_json") + except Exception as exc: # one bad file must not hide the rest + errors.append(f"{item['file']}: {type(exc).__name__}: {exc}"[:200]) + if not self.server.alive(): + break # the case fails and the server is restarted + continue + text = body.get("text") or "" + edits, length = error_rate(item["reference"], text, item["language"], + item["metric"]) + runaway = (len(tokens(text, item["language"], item["metric"])) + > RUNAWAY_RATIO * length) + pairs.append((edits, length)) + audio_s += item["duration"] + wall_s += seconds + if runaway: + runaways.append(item["file"]) + self.results["items"][item["file"]] = { + "set": set_name, "language": item["language"], "metric": item["metric"], + "text": text, "seconds": round(seconds, 2), "edits": edits, + "length": length, "runaway": runaway} + if body.get("language") == LANGUAGE_NAMES[item["language"]]: + detected += 1 + if "segments" in body: + self.verbose_items += 1 + self.check_segments(body, item["duration"]) + rate = pooled(pairs) + stats = { + "metric": set_items[0]["metric"], "error_rate": round(rate, 4), + "files": len(pairs), "of": len(set_items), "runaways": len(runaways), + "audio_s": round(audio_s, 1), "wall_s": round(wall_s, 1), + "rtf": round(wall_s / audio_s, 4) if audio_s else None, + "language_detected": detected, + } + self.results["sets"][set_name] = stats + case.data.update(stats) + if errors: + raise AssertionError(f"{len(pairs)}/{len(set_items)} transcribed; {errors[0]}") + if set_name == "en-short" and pairs: + expect(rate < EN_SHORT_MAX_WER, f"WER {pct(rate)} on en-short, " + f"over the {pct(EN_SHORT_MAX_WER)} sanity bound") + # Long-form runaways are window_tail's to report. Elsewhere one is + # worth a look, but a single odd decode is not a regression. + if runaways and set_name not in LONG_SETS: + warn(f"runaway transcripts: {', '.join(runaways)}") + + def evaluate(self) -> None: + by_set: dict[str, list[dict]] = {} + for item in self.items: + by_set.setdefault(item["set"], []).append(item) + for set_name, set_items in by_set.items(): + self.eval_set(set_name, set_items) + scored = self.results["items"].values() + self.results["overall"] = { + "error_rate": round(pooled((r["edits"], r["length"]) for r in scored), 4), + "runaways": sum(r["runaway"] for r in scored), + "files": len(self.results["items"]), + } + + def segment_cases(self) -> None: + for group, name in (("verbose_json", "segments"), + ("timestamp_rules", "timestamp-rules")): + with self.case(name) as case: + case.data["files"] = self.verbose_items + if not self.verbose_items: + skip("no verbose_json response with segments to check") + bad = [f"{check}: {count} of {self.verbose_items} files" + for (g, check), count in sorted(self.segment_failures.items()) + if g == group and count] + expect(not bad, "; ".join(bad)) + + def window_tail(self) -> None: + long_items = [f for f, r in self.results["items"].items() if r["set"] in LONG_SETS] + if not long_items: + return + reference = {i["file"]: i["reference"] for i in self.items} + with self.case("window-tail") as case: + problems = [] + for name in long_items: + text = self.results["items"][name]["text"] + counts = {"reference": repeats(reference[name]), "text": repeats(text)} + case.data[name] = counts + if counts["text"] > counts["reference"]: + problems.append(f"{name}: {counts['text']} repeated " + f"{REPEAT_NGRAM}-word phrases, reference {counts['reference']}") + settle(problems, required=False) + + # -- feature probes ------------------------------------------------------ + + def language(self) -> None: + """Forcing the language: no worse than detecting it, and reported back.""" + for language in sorted({i["language"] for i in self.items} - {"en"}): + probe = [i for i in self.items if i["language"] == language][:PR_CLIPS_PER_SET] + with self.case(f"language-{language}") as case: + pairs, reported, problems, failed = [], [], [], [] + for item in probe: + try: + body, _ = transcribe(self.client, self.model, self.path(item), + response_format="verbose_json", + language=language) + except (HttpError, AssertionError) as exc: + # A valid language must be accepted and transcribed, as + # the old asr_test.py required; only how well it does + # below is a warning. + failed.append(f"{item['file']}: {type(exc).__name__}: {exc}"[:200]) + continue + pairs.append(error_rate(item["reference"], body.get("text") or "", + language, item["metric"])) + if "segments" in body: + reported.append(body.get("language")) + auto = pooled((self.results["items"][i["file"]]["edits"], + self.results["items"][i["file"]]["length"]) + for i in probe if i["file"] in self.results["items"]) + forced = pooled(pairs) + case.data.update({"forced": round(forced, 4), "detected": round(auto, 4)}) + self.results["language"][language] = case.data.copy() + if pairs and forced > auto + SET_TOLERANCE: + problems.append(f"forcing it hurts: {pct(forced)} forced, " + f"{pct(auto)} detected") + # Only verbose_json reports the language. Forcing a wrong + # language is no test: Whisper often keeps transcribing what + # it hears. + if any(r != LANGUAGE_NAMES[language] for r in reported): + problems.append("response reports " + + ", ".join(sorted({str(r) for r in reported}))) + expect(not failed, "forced-language requests failed: " + "; ".join(failed)) + settle(problems, required=False) + + english = next((i for i in self.items if i["set"] == "en-short"), None) + if english is None: + return + with self.case("language-invalid") as case: + try: + body, _ = transcribe(self.client, self.model, self.path(english), + language="xx") + except HttpError as exc: + case.data["status"] = exc.status + if exc.status >= 500: + warn(f"language 'xx' got HTTP {exc.status}, not a 4xx") + return + except AssertionError as exc: # an error body is a rejection too + case.data["status"] = str(exc)[:120] + return + warn(f"language 'xx' was accepted: {json.dumps(body)[:120]}") + + def prompt(self) -> None: + """A prompt naming the clip's proper nouns should not hurt, and should help them.""" + probe = [i for i in self.items if i["set"] == "en-short"][:5] + with self.case("prompt") as case: + hits_without = hits_with = names_total = 0 + pairs_without, pairs_with = [], [] + for item in probe: + names = sorted({w.strip(".,;:!?\"'()") for w in item["raw_reference"].split()[1:] + if w[:1].isupper()} - {""}) + before = self.results["items"].get(item["file"]) + if not names or before is None: + continue + body, _ = transcribe(self.client, self.model, self.path(item), + prompt=" ".join(names)) + after = body.get("text") or "" + pairs_with.append(error_rate(item["reference"], after, "en", "wer")) + pairs_without.append((before["edits"], before["length"])) + names_total += len(names) + hits_without += sum(n in before["text"] for n in names) + hits_with += sum(n in after for n in names) + if not names_total: + skip("no en-short clip with proper nouns was transcribed") + with_rate, without_rate = pooled(pairs_with), pooled(pairs_without) + case.data.update({"names": names_total, "hits_with": hits_with, + "hits_without": hits_without, "wer_with": round(with_rate, 4), + "wer_without": round(without_rate, 4)}) + self.results["prompt"] = dict(case.data) + problems = [] + if with_rate > without_rate + SET_TOLERANCE: + problems.append(f"a prompt hurts accuracy: WER {pct(with_rate)} with, " + f"{pct(without_rate)} without") + if hits_with < hits_without: + problems.append(f"names spelled as given {hits_with}/{names_total} with the " + f"prompt, {hits_without}/{names_total} without") + settle(problems, required=True) + + def quiet_log(self) -> None: + with self.case("quiet-log") as case: + try: + log = self.server.log_path.read_text(encoding="utf-8", errors="replace") + except OSError as exc: + skip(f"cannot read the server log: {exc}") + # Streamed output interleaves timestamp markers with the text. + log = "".join(re.sub(r"<\|[\d.]+\|>", "", log).split()) + leaked = [] + for name, r in self.results["items"].items(): + squashed = "".join(r["text"].split()) + if len(squashed) >= 20 and squashed[:20] in log: + leaked.append(name) + case.data["leaked"] = len(leaked) + if leaked: + warn(f"{len(leaked)} of {len(self.results['items'])} transcripts found in " + f"the server log, e.g. {leaked[0]}") + + # -- expected rates ------------------------------------------------------ + + def compare_expected(self) -> None: + """Rates against expected.json's row for this platform. + + Those rates are main's over the full set. A set that ran in full is + compared as it is. The pooled rate of a subset is not comparable to + the full pooled rate, so it is compared to the expected set rates + weighted by this run's reference lengths per set, and only warns: + three clips of a set are far noisier than ten. + """ + key = self.ctx.asr_expected_key + if not key or not self.results["items"]: + return + with self.case("wer-expected") as case: + platforms = json.loads(self.ctx.asset("asr_eval/expected.json") + .read_text(encoding="utf-8"))["platforms"] + expected = platforms.get(key) + if expected is None: + warn(f"expected.json has no row for {key!r}; accuracy is not checked") + set_limits = expected.get("sets", {}) + # Judged by what was sent, not what came back: a failed file is + # already a failure of its eval case. + selected = Counter(i["set"] for i in self.items) + complete = {name for name, n in Counter(i["set"] for i in self.all_items).items() + if selected[name] == n} + subset = len(self.items) < len(self.all_items) + overall = self.results["overall"]["error_rate"] + failures, warnings = [], [] + + if subset: + lengths: Counter = Counter() + for r in self.results["items"].values(): + lengths[r["set"]] += r["length"] + total = sum(lengths[s] for s in lengths if s in set_limits) + limit = (sum(set_limits[s] * n for s, n in lengths.items() if s in set_limits) + / total if total else None) + tolerance, bucket = SET_TOLERANCE, warnings + else: + limit, tolerance, bucket = expected.get("overall"), POOLED_TOLERANCE, failures + case.data.update({"key": key, "subset": subset, "pooled": overall, + "pooled_expected": None if limit is None else round(limit, 4), + "sets": {}}) + if limit is not None and overall > limit + tolerance: + bucket.append(f"pooled {pct(overall)} vs expected {pct(limit)}") + + for name, current in self.results["sets"].items(): + limit = set_limits.get(name) + if limit is None: + continue + case.data["sets"][name] = {"error_rate": current["error_rate"], + "expected": limit, "rtf": current["rtf"]} + if (name in complete and current["files"] >= SET_CHECK_MIN_FILES + and current["error_rate"] > limit + SET_TOLERANCE): + failures.append(f"{name} {pct(current['error_rate'])} vs expected " + f"{pct(limit)}") + expect(not failures, "; ".join(failures + warnings)) + settle(warnings, required=False) + + def save(self) -> None: + """Every transcript and rate, for a reader chasing a WER change.""" + safe = re.sub(r"[^\w.-]", "_", self.model) + path = self.ctx.out_dir / f"asr-{safe}.json" + path.write_text(json.dumps({"model": self.model, **self.results}, indent=1, + ensure_ascii=False), encoding="utf-8") + + def run(self) -> None: + try: + self.formats() + self.evaluate() + self.segment_cases() + self.window_tail() + if self.ctx.tier != "install": + self.language() + self.prompt() + self.quiet_log() + self.compare_expected() + finally: + self.save() + + +# -- chat audio --------------------------------------------------------------- + +def run_chat(ctx, models: list[str], clips: list[dict]) -> None: + """Chat models with an audio encoder transcribe a few en-short clips.""" + audio = [(item, base64.b64encode(ctx.asset(f"asr_eval/{item['file']}").read_bytes()) + .decode("ascii")) for item in clips] + with ctx.server("asr-chat") as server: + for index, model_id in enumerate(models): + model = ctx.plan.model(model_id) + # Sampling and think state persist on the loaded model, so pin both. + extra = {"temperature": 0, "top_k": 1, + "max_tokens": 1024 if model.always_thinks else 256} + if model.think: + extra["think"] = False + try: + with ctx.case(NAME, "chat-audio", model_id, server) as case: + pairs, errors, seconds = [], [], 0.0 + for item, encoded in audio: + try: + reply = server.client.chat(model_id, [{"role": "user", "content": [ + {"type": "text", "text": CHAT_PROMPT}, + {"type": "input_audio", + "input_audio": {"data": encoded, "format": "mp3"}}, + ]}], stream=False, **extra) + text = reply.content + seconds += reply.seconds + except Exception as exc: + errors.append(f"{item['file']}: {type(exc).__name__}: {exc}"[:200]) + if not server.alive(): + break # the case fails and the server is restarted + text = "" + # An empty transcript counts as every word missed. + pairs.append(error_rate(item["reference"], text, "en", "wer")) + rate = pooled(pairs) + case.data.update({"wer": round(rate, 4), "clips": len(audio), + "transcribed": len(audio) - len(errors), + "seconds": round(seconds, 1)}) + if errors: + raise AssertionError(f"{len(audio) - len(errors)}/{len(audio)} " + f"transcribed; {errors[0]}") + expect(rate < 0.5, f"WER {pct(rate)}, not under 50%") + except ServerGone as exc: + ctx.not_attempted(NAME, models[index + 1:], str(exc)) + return + + +# -- entry point -------------------------------------------------------------- + +def run(ctx) -> None: + whisper = ctx.plan.suite("whisper") + chat = ctx.plan.suite("asr_chat") + if not whisper and not chat: + return + manifest = json.loads(ctx.asset("asr_eval/manifest.json").read_text(encoding="utf-8")) + all_items = manifest["items"] + items = select_items(all_items, ctx.tier, bool(getattr(ctx, "asr_full", False))) + print(f"[asr] {len(items)} of {len(all_items)} clips, expected rates: " + f"{getattr(ctx, 'asr_expected_key', '') or 'none'}", flush=True) + + if whisper: + # -a 1 loads the ASR sidecar, which serves every Whisper tag. + with ctx.server("asr", ["-a", "1"]) as server: + for index, model in enumerate(whisper): + try: + WhisperRun(ctx, server, model, items, all_items).run() + except ServerGone as exc: + # This model's remaining cases are lost too. + ctx.not_attempted(NAME, whisper[index:], str(exc)) + break + if chat: + clips = [i for i in all_items if i["set"] == "en-short"][:CHAT_CLIPS] + run_chat(ctx, chat, clips) diff --git a/.github/script/npu_test/suites/chat.py b/.github/script/npu_test/suites/chat.py new file mode 100644 index 000000000..ad8f789d9 --- /dev/null +++ b/.github/script/npu_test/suites/chat.py @@ -0,0 +1,206 @@ +"""Every chat model answers, streams, holds a second turn and, if it has a +vision encoder, sees. + +Replaces the old per-model sweeps, which only checked that a reply arrived: +each check here has a known answer, so a model that loads but emits garbage +fails. One plain server serves the whole suite; the model switches with each +request, which also exercises unload/load between engines. +""" + +from __future__ import annotations + +import base64 +from pathlib import Path + +from ..harness import ServerGone, expect, skip, warn +from ..server import ServerError + +NAME = "chat" +DESCRIPTION = "per-model answer, streaming, two turns, non-streamed usage, vision" + +# Greedy, so the content checks see the model's single most likely answer. +# FLM keeps sampling and think settings on the loaded model between requests, +# so every request sends them rather than relying on what came before. +GREEDY = {"temperature": 0, "top_k": 1, "top_p": 1.0} + +ANSWER_PROMPT = "What is the capital of France? Answer with one word." +ANSWER_WORD = "paris" + +TURNS = ("Name one primary color. Answer in one short sentence.", + "Now name a different primary color. Answer in one short sentence.") +NON_STREAM_PROMPT = "Name the largest planet in the solar system. Answer in one short sentence." + +# Translators answer every prompt with a translation, so they get one to do. +TRANSLATE_TEXT = "Thank you very much." +TRANSLATE_WORD = "merci" +TRANSLATE_TURNS = ("Thank you very much.", "Good night.") + +COLOR_PROMPT = "What color is the square? Answer with one word." +COLOR_IMAGES = {"red": "vision/red_square.png", "blue": "vision/blue_square.png"} +PHOTO = "test_image1.jpg" +PHOTO_PROMPT = "Describe this image in one sentence." + +# Token budgets. A model that always reasons spends hundreds of tokens before +# any content, so it gets a larger one; everything else stays short to keep +# the NPU runners free. +ANSWER_TOKENS = 64 +TURN_TOKENS = 128 +THINKING_TOKENS = 1024 + +# lfm2-trans is LFM2-2.6B-Transcript, a meeting-transcript summarizer with +# its own input format, not a translator; plan.Model.translation matches it +# by id prefix anyway. It gets the plain question, and a wrong answer is only +# a warning, since the question is outside what it was tuned for. +OFF_FORMAT_PREFIXES = ("lfm2-trans",) + +MIME = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg"} + + +def run(ctx) -> None: + models = ctx.plan.chat() + if not models: + return + try: + with ctx.server("chat") as server: + for index, model_id in enumerate(models): + try: + _model_cases(ctx, server, ctx.plan.model(model_id)) + except ServerGone as exc: + ctx.not_attempted(NAME, models[index + 1:], str(exc)) + raise + except ServerError as exc: + ctx.not_attempted(NAME, models, f"server did not start: {exc}") + raise + + +def _model_cases(ctx, server, model) -> None: + client = server.client + # A model that crashed the server once is likely to again; its remaining + # cases are dropped to leave the restart budget for the other models. + restarts = server.restarts + + def crashed(): + return server.restarts > restarts + + thinks = model.always_thinks + translator = model.translation and not model.id.startswith(OFF_FORMAT_PREFIXES) + + def chat(messages, max_tokens, stream=True): + return client.chat(model.id, messages, stream=stream, think=False, + max_tokens=THINKING_TOKENS if thinks else max_tokens, **GREEDY) + + def check_reply(reply, what): + expect(reply.finish_reason, f"{what}: no finish_reason") + if thinks: + expect(reply.content.strip() or reply.reasoning.strip(), + f"{what}: no content or reasoning") + else: + expect(reply.content.strip(), f"{what}: empty content") + + with ctx.case(NAME, "answer", model.id, server) as case: + if translator: + prompt, word = _translate_prompt(model.id, TRANSLATE_TEXT), TRANSLATE_WORD + else: + prompt, word = ANSWER_PROMPT, ANSWER_WORD + reply = chat([{"role": "user", "content": prompt}], ANSWER_TOKENS) + case.data.update(_timing(reply), content=reply.content[:200]) + if not reply.content.strip() and thinks and reply.finish_reason == "length": + # Whether the reasoning fits the budget is the model's doing, not + # the server's; a reasoning stream that ends cleanly with no + # answer still fails below. + warn(f"reasoning used all {THINKING_TOKENS} tokens before an answer") + expect(reply.content.strip(), "empty content") + if word not in reply.content.lower(): + message = f"expected {word!r} in {reply.content[:120]!r}" + if model.id.startswith(OFF_FORMAT_PREFIXES): + warn(message) + raise AssertionError(message) + + if crashed(): + return + with ctx.case(NAME, "stream-two-turns", model.id, server) as case: + turns = ([_translate_prompt(model.id, t) for t in TRANSLATE_TURNS] if translator + else list(TURNS)) + messages = [{"role": "user", "content": turns[0]}] + first = chat(messages, TURN_TOKENS) + case.data["turn1"] = _timing(first) + check_reply(first, "turn 1") + if model.single_turn: + skip("turn 1 passed; turn 2 not sent to a single-turn model") + messages += [{"role": "assistant", "content": first.content or first.reasoning[:500]}, + {"role": "user", "content": turns[1]}] + second = chat(messages, TURN_TOKENS) + case.data["turn2"] = _timing(second) + check_reply(second, "turn 2") + case.data["cached_tokens"] = second.cached_tokens + if not second.cached_tokens: + # Reusing the first turn's KV cache is an optimisation not every + # engine implements, so its absence is worth seeing, not failing. + warn(f"turn 2 reported cached_tokens={second.cached_tokens}") + + if crashed(): + return + with ctx.case(NAME, "non-stream", model.id, server) as case: + prompt = (_translate_prompt(model.id, TRANSLATE_TEXT) if translator + else NON_STREAM_PROMPT) + reply = chat([{"role": "user", "content": prompt}], TURN_TOKENS, stream=False) + case.data.update(_timing(reply)) + check_reply(reply, "non-stream") + usage = reply.usage + prompt_tokens, completion_tokens = usage.get("prompt_tokens"), usage.get("completion_tokens") + expect(isinstance(prompt_tokens, int) and prompt_tokens > 0, + f"usage.prompt_tokens is {prompt_tokens!r}") + expect(isinstance(completion_tokens, int) and completion_tokens > 0, + f"usage.completion_tokens is {completion_tokens!r}") + expect(usage.get("total_tokens") == prompt_tokens + completion_tokens, + f"usage.total_tokens {usage.get('total_tokens')!r} != " + f"{prompt_tokens} + {completion_tokens}") + + if not model.vlm or crashed(): + return + with ctx.case(NAME, "vision", model.id, server) as case: + if model.translation: + # A translator answers the question with its translation, so the + # colour word is not a fair check; the photo case still runs. + skip("translation model: does not answer questions about images") + for color, relative in COLOR_IMAGES.items(): + reply = chat([_image_message(ctx.asset(relative), COLOR_PROMPT)], ANSWER_TOKENS) + case.data[color] = {**_timing(reply), "content": reply.content[:120]} + expect(reply.content.strip(), f"{color} square: empty content") + expect(color in reply.content.lower(), + f"{color} square: answer was {reply.content[:120]!r}") + + if crashed(): + return + with ctx.case(NAME, "vision-photo", model.id, server) as case: + reply = chat([_image_message(ctx.asset(PHOTO), PHOTO_PROMPT)], TURN_TOKENS) + case.data.update(_timing(reply), content=reply.content[:200]) + check_reply(reply, "photo") + + +def _translate_prompt(model_id: str, text: str) -> str: + """The prompt format each translator's model card documents.""" + if model_id.startswith("translategemma"): + return ("You are a professional English (en) to French (fr) translator. Your goal " + "is to accurately convey the meaning and nuances of the original English " + "text while adhering to French grammar, vocabulary, and cultural " + "sensitivities.\nProduce only the French translation, without any " + "additional explanations or commentary. Please translate the following " + f"English text into French:\n\n\n{text}") + return f"Translate the following segment into French, without additional explanation.\n\n{text}" + + +def _image_message(path: Path, prompt: str) -> dict: + encoded = base64.b64encode(path.read_bytes()).decode("ascii") + url = f"data:{MIME.get(path.suffix.lower(), 'image/jpeg')};base64,{encoded}" + return {"role": "user", "content": [{"type": "text", "text": prompt}, + {"type": "image_url", "image_url": {"url": url}}]} + + +def _timing(reply) -> dict: + """The server's load/prefill/decode accounting for one request.""" + keys = ("prompt_tokens", "completion_tokens", "load_duration", "prefill_duration_ttft", + "decoding_duration", "decoding_speed_tps") + out = {k: reply.usage[k] for k in keys if isinstance(reply.usage.get(k), (int, float))} + out["seconds"] = round(reply.seconds, 2) + return out diff --git a/.github/script/npu_test/suites/decision.py b/.github/script/npu_test/suites/decision.py new file mode 100644 index 000000000..323190feb --- /dev/null +++ b/.github/script/npu_test/suites/decision.py @@ -0,0 +1,344 @@ +"""The decision (d3) models through POST /v1/systemone. + +Each model gets its own server, since --decision takes one tag. Per model: + goldens stored requests answered by the upstream BF16 model: every answer has the + upstream shape, top-1 agrees on >= 85% of them, mean |dp| <= 0.05 + types one choice, one noul, one score, and all three in one request + two-letter a 27-option choice (the 27th option's code is "AA") + invalid a malformed question -> 422 naming it, nothing run; a body that is not JSON -> 422 + too-long a prompt over the input limit -> max_length_exceeded inside a 200 reply + image a PNG answered, a broken one -> 422 "images[0]: ..." + d3:turbo and d3:ultra also get the prefix-bidirectional checks (box, state counted once, + assets, track, long_context), each gated on what /v1/systemone/models lists. +The install tier runs goldens and types only. +""" + +from __future__ import annotations + +import base64 +import json +import time + +from ..client import HttpError +from ..harness import FAIL, Result, ServerGone, expect, skip, warn +from ..server import ServerError + +NAME = "decision" +DESCRIPTION = "/v1/systemone on each d3 model: BF16 goldens, question types, errors, images" + +MIN_AGREEMENT = 0.85 +MAX_MEAN_DP = 0.05 +PREFIX_MODELS = ("d3:turbo", "d3:ultra") + +STATE = ("The order arrived damaged yesterday. The customer has a receipt and asks for " + "a replacement today.") + +# 96 x 64 PNG: a yellow box on a blue background +IMAGE = ("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGAAAABACAIAAABqVuVZAAAAiElEQVR42u3bMRXAIAxAQcKrjk5V1QkZlYAM" + "5FQEYmoApi4Z7u8ZuJeMxHm/RfsqAkCAAAECBAgQIK06/gzPp+V81dWHDXJigAABAiRAgAABAgQIECBAAgQIECBAgAABEiBAgAABAgQIkAAB" + "AgQIECBAgABpXfiSaYMAAQIECBAgQAIECBCgbH3ZCwYBFiTfPwAAAABJRU5ErkJggg==") + +BACKGROUND_QUESTION = {"type": "choice", "instructions": "What is the main colour of the background?", + "criteria": {"blue": "Blue", "red": "Red", "green": "Green"}} + +# The 4 s, 48 x 32 H.264 clip of the d3 video tests. +TRACK_CLIP = "src/test/d3_video/fixtures/h264.mp4" + + +def answer_shape_ok(answer: dict) -> bool: + """The upstream reply shape, key order included: clients index these positionally.""" + if not isinstance(answer, dict): + return False + if "error" in answer: + return set(answer) == {"type", "error", "message"} + return list(answer) == { + "choice": ["type", "choice", "probabilities", "confidence"], + "noul": ["type", "noul"], + "score": ["type", "score", "probabilities", "confidence", "legend"], + }.get(answer.get("type"), None) + + +def post(server, body) -> tuple[int, dict]: + """POST /v1/systemone, returning (status, JSON body) for 2xx and errors alike. + + A body that is not JSON (a proxy's or crash page's 5xx) comes back as + {"_raw": its start}, so the case fails on it rather than raising. + """ + data = body if isinstance(body, bytes) else json.dumps(body).encode("utf-8") + try: + status, raw = server.client.post_raw("/v1/systemone", data, "application/json") + text = raw.decode("utf-8", "replace") + except HttpError as exc: + status, text = exc.status, exc.body + try: + parsed = json.loads(text or "{}") + except ValueError: + return status, {"_raw": text[:200]} + return status, parsed if isinstance(parsed, dict) else {"_raw": text[:200]} + + +def brief(body: dict) -> str: + return json.dumps(body)[:200] + + +def server_name(model: str) -> str: + # The log file is named after the server; ':' is not allowed in Windows file names. + return f"{NAME}-{model.replace(':', '-')}" + + +def run(ctx) -> None: + for model in ctx.plan.suite(NAME): + # One server per model: a model whose server will not start or stay up + # must not cost the other models their run. + try: + with ctx.server(server_name(model), ["--decision", model]) as server: + check_model(ctx, server, model) + except ServerError as exc: + ctx.record(Result(NAME, "server-start", FAIL, model, str(exc))) + except ServerGone as exc: + ctx.record(Result(NAME, "server-gone", FAIL, model, str(exc))) + + +def check_model(ctx, server, model: str) -> None: + goldens(ctx, server, model) + types(ctx, server, model) + if ctx.tier == "install": + return + two_letter(ctx, server, model) + invalid(ctx, server, model) + too_long(ctx, server, model) + if model in PREFIX_MODELS: + prefix_bidirectional(ctx, server, model) + image(ctx, server, model) + + +# -- cases ------------------------------------------------------------------- + +def goldens(ctx, server, model: str) -> None: + with ctx.case(NAME, "goldens", model, server) as case: + # Each d3 size has its own BF16 answers: d3_goldens.jsonl is d3:lite's, + # d3_goldens_.jsonl the others'. + size = model.split(":", 1)[-1] + name = "d3_goldens.jsonl" if size == "lite" else f"d3_goldens_{size}.jsonl" + path = ctx.assets / name + if not path.is_file(): + # A model added to the catalog before its goldens: nothing to + # compare against yet, and the other cases still cover it. + warn(f"no BF16 goldens for {model} yet ({name}); add them to compare its answers") + agree = total = 0 + dps: list[float] = [] + bad: list[str] = [] + started = time.monotonic() + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + rec = json.loads(line) + status, body = post(server, rec["request"]) + if status != 200: + bad.append(f"{rec['id']}: status {status}") + continue + wanted = rec["response"]["answers"] + got_all = body.get("answers") or {} + if list(got_all) != list(wanted): + bad.append(f"{rec['id']}: answers {list(got_all)}") + continue + for key, want in wanted.items(): + got = got_all[key] + if not answer_shape_ok(got) or "error" in got: + bad.append(f"{rec['id']}/{key}") + continue + if "probabilities" in want: + p = list(want["probabilities"].values()) + q = list(got["probabilities"].values()) + if len(p) != len(q): + bad.append(f"{rec['id']}/{key}: {len(q)} probabilities, want {len(p)}") + continue + dps.append(sum(abs(a - b) for a, b in zip(p, q)) / len(p)) + agree += max(range(len(p)), key=p.__getitem__) == max(range(len(q)), key=q.__getitem__) + total += 1 + elif "noul" in want: + dps.append(abs(want["noul"] - got["noul"])) + agree += (want["noul"] >= 0.5) == (got["noul"] >= 0.5) + total += 1 + share = agree / max(total, 1) + mean_dp = sum(dps) / max(len(dps), 1) + case.data.update(agree=agree, total=total, mean_dp=round(mean_dp, 4), + seconds_per_request=round((time.monotonic() - started) / max(total, 1), 3)) + expect(not bad, f"bad replies: {'; '.join(bad[:5])}") + expect(total > 0, f"{name} has no answer to compare") + expect(share >= MIN_AGREEMENT and mean_dp <= MAX_MEAN_DP, + f"top-1 {agree}/{total} (min {MIN_AGREEMENT}), mean |dp| {mean_dp:.4f} " + f"(max {MAX_MEAN_DP})") + + +def types(ctx, server, model: str) -> None: + with ctx.case(NAME, "types", model, server): + questions = { + "route": {"type": "choice", "instructions": "Which team should handle this?", + "criteria": {"returns": "Refunds and replacements", "billing": "Payments", + "general": None}}, + "urgent": {"type": "noul", "instructions": "Does this need a reply today?"}, + "anger": {"type": "score", "instructions": "How upset is the customer?", + "criteria": ["calm", "annoyed", "angry"]}, + } + status, body = post(server, {"state": STATE, "questions": questions}) + expect(status == 200, f"status {status}: {brief(body)}") + answers = body.get("answers") or {} + expect(list(answers) == list(questions), f"answers {list(answers)}, want {list(questions)}") + wrong = [k for k, a in answers.items() if not answer_shape_ok(a) or "error" in a] + expect(not wrong, f"bad answers {wrong}: {brief(body)}") + usage = body.get("usage") or {} + # A decision is read off the prefill; nothing is generated. + expect(usage.get("input_tokens", 0) > 0 and usage.get("output_tokens") == 0, + f"usage {usage}") + + +def two_letter(ctx, server, model: str) -> None: + with ctx.case(NAME, "two-letter", model, server): + criteria = {f"option{i:02d}": f"Option number {i}" for i in range(27)} + status, body = post(server, {"state": STATE, + "questions": {"q": {"type": "choice", "criteria": criteria}}}) + expect(status == 200, f"status {status}: {brief(body)}") + answer = (body.get("answers") or {}).get("q", {}) + expect(answer_shape_ok(answer) and "error" not in answer, f"bad answer: {brief(body)}") + expect(len(answer.get("probabilities", {})) == 27, + f"{len(answer.get('probabilities', {}))} options, want 27") + + +def invalid(ctx, server, model: str) -> None: + with ctx.case(NAME, "invalid", model, server): + status, body = post(server, {"questions": {"ok": {"type": "noul"}, "bad": {"type": "rank"}}}) + expect(status == 422, f"status {status}: {brief(body)}") + want = {"detail": {"invalid_questions": { + "bad": "unsupported question type 'rank' (choice, noul or score)"}}} + expect(body == want, f"reply {brief(body)}") + with ctx.case(NAME, "not-json", model, server): + status, body = post(server, b'{"questions": ') + expect(status == 422, f"status {status}: {brief(body)}") + expect(str(body.get("detail", "")).startswith("could not read the body as JSON"), + f"reply {brief(body)}") + + +def too_long(ctx, server, model: str) -> None: + with ctx.case(NAME, "too-long", model, server): + status, body = post(server, {"state": "word " * 33000, "questions": {"q": {"type": "noul"}}}) + expect(status == 200, f"status {status}: {brief(body)}") + answer = (body.get("answers") or {}).get("q", {}) + expect(answer.get("error") == "max_length_exceeded", f"answer {brief(answer)}") + expect((body.get("usage") or {}).get("input_tokens") == 0, + f"usage {body.get('usage')}: nothing should have run") + + +def image(ctx, server, model: str) -> None: + with ctx.case(NAME, "image", model, server) as case: + status, body = post(server, {"questions": {"bg": BACKGROUND_QUESTION}, "images": [IMAGE]}) + expect(status == 200, f"status {status}: {brief(body)}") + answer = (body.get("answers") or {}).get("bg", {}) + expect(answer_shape_ok(answer) and "error" not in answer, f"bad answer: {brief(body)}") + case.data["choice"] = answer.get("choice") + # The image tokens must have reached the prefill, not just the text. + expect((body.get("usage") or {}).get("input_tokens", 0) > 100, f"usage {body.get('usage')}") + with ctx.case(NAME, "image-broken", model, server): + status, body = post(server, {"questions": {"bg": BACKGROUND_QUESTION}, "images": [IMAGE[:120]]}) + expect(status == 422, f"status {status}: {brief(body)}") + expect(str(body.get("detail", "")).startswith("images[0]: "), f"reply {brief(body)}") + + +def box_ok(answer: dict) -> bool: + box = answer.get("box") or {} + return (list(answer) == ["type", "box", "confidence"] + and sorted(box) == ["bottom", "left", "right", "top"] + and all(isinstance(v, (int, float)) and 0.0 <= v <= 1.0 for v in box.values())) + + +def prefix_bidirectional(ctx, server, model: str) -> None: + """d3:turbo / d3:ultra: what /v1/systemone/models lists is what the build answers.""" + entry = None + with ctx.case(NAME, "models", model, server) as case: + data = server.client.get_json("/v1/systemone/models", timeout=30).get("data") or [] + expect(data and isinstance(data[0], dict), f"no model entry: {data!r}"[:200]) + entry = data[0] + case.data.update(question_types=entry.get("question_types"), + assets=entry.get("assets"), long_context=entry.get("long_context")) + if entry is None: + return + listed = entry.get("question_types") or [] + + with ctx.case(NAME, "box", model, server): + # Listed: the engine returns the four hidden rows a box reads. Not + # listed: refused inline, the rest of the request still answered. + status, body = post(server, {"state": STATE, "questions": { + "b": {"type": "box", "instructions": "The customer's name"}}}) + expect(status == 200, f"status {status}: {brief(body)}") + answer = (body.get("answers") or {}).get("b", {}) + if "box" in listed: + box = answer.get("box") or {} + expect(box_ok(answer) and box.get("top", 1) <= box.get("bottom", 0) + and box.get("left", 1) <= box.get("right", 0), f"bad box: {brief(answer)}") + else: + expect(answer.get("error") == "invalid_question" + and answer.get("reason") == "box_unsupported", f"answer {brief(answer)}") + + with ctx.case(NAME, "state-once-usage", model, server) as case: + # The state block is prefilled once, however many questions share it. + long_state = "The customer wrote: " + "the parcel is late, again. " * 40 + one = {"q1": {"type": "noul", "instructions": "Is the customer upset?"}} + four = {f"q{i}": {"type": "noul", "instructions": f"Is the customer upset? ({i})"} + for i in range(1, 5)} + s1, b1 = post(server, {"state": long_state, "questions": one}) + s4, b4 = post(server, {"state": long_state, "questions": four}) + expect(s1 == 200 and s4 == 200, f"status {s1} / {s4}") + n1 = (b1.get("usage") or {}).get("input_tokens", 0) + n4 = (b4.get("usage") or {}).get("input_tokens", 0) + case.data.update(one_question=n1, four_questions=n4) + expect(0 < n1 < n4 < 4 * n1, f"input_tokens 1 question {n1}, 4 questions {n4}") + + if "box" in listed: + with ctx.case(NAME, "box-image", model, server): + status, body = post(server, {"state": "A small test picture.", "images": [IMAGE], + "questions": {"b": {"type": "box", + "instructions": "The yellow rectangle"}}}) + expect(status == 200, f"status {status}: {brief(body)}") + answer = (body.get("answers") or {}).get("b", {}) + expect(box_ok(answer), f"bad box: {brief(answer)}") + expect((body.get("usage") or {}).get("input_tokens", 0) > 100, f"usage {body.get('usage')}") + + if entry.get("assets"): + with ctx.case(NAME, "assets", model, server): + status, body = post(server, {"state": "Picture: ", "assets": {"": IMAGE}, + "questions": {"bg": BACKGROUND_QUESTION}}) + expect(status == 200, f"status {status}: {brief(body)}") + answer = (body.get("answers") or {}).get("bg", {}) + expect(answer_shape_ok(answer) and "error" not in answer, f"bad answer: {brief(body)}") + expect((body.get("usage") or {}).get("input_tokens", 0) > 100, f"usage {body.get('usage')}") + + if "track" in listed: + with ctx.case(NAME, "track", model, server) as case: + # A track question needs an encoded video (a frame list is only read + # by plain questions). + clip = ctx.repo_root / TRACK_CLIP + if not clip.is_file(): + skip(f"{TRACK_CLIP} is not in this checkout") + video = "data:video/mp4;base64," + base64.b64encode(clip.read_bytes()).decode("ascii") + status, body = post(server, {"state": "A short clip.", "videos": [video], + "questions": {"t": {"type": "track", + "instructions": "The moving object", + "rate": 1}}}) + if status == 422 and "decoding backend not available" in json.dumps(body): + skip("no FFmpeg libraries on this machine") + expect(status == 200, f"status {status}: {brief(body)}") + answer = (body.get("answers") or {}).get("t", {}) + moments = answer.get("track") or [] + case.data["moments"] = len(moments) + expect(answer.get("type") == "track" and moments, f"no track: {brief(answer)}") + expect(all(isinstance(m, dict) and sorted(m) == ["box", "confidence", "time", "visible"] + and 0.0 <= m["visible"] <= 1.0 for m in moments), + f"bad track entries: {json.dumps(moments)[:200]}") + + with ctx.case(NAME, "long-context", model, server): + # Refused exactly when the model entry says long_context is unavailable. + status, body = post(server, {"state": STATE, "long_context": True, + "questions": {"q": {"type": "noul"}}}) + refused_listed = entry.get("long_context") is False + expect((status == 422) == refused_listed, + f"status {status} but long_context listed {entry.get('long_context')}: {brief(body)}") diff --git a/.github/script/npu_test/suites/embedding.py b/.github/script/npu_test/suites/embedding.py new file mode 100644 index 000000000..bca7a8f67 --- /dev/null +++ b/.github/script/npu_test/suites/embedding.py @@ -0,0 +1,115 @@ +"""The embedding model through /v1/embeddings. + +Well-formed vectors are not enough: a model whose vectors carry no meaning +passes the shape checks, so the similarity case asserts that a paraphrase +outscores an unrelated sentence. The server loads the embedding model only +with `-e 1`, and that one model answers whatever "model" a request names. +""" + +from __future__ import annotations + +import math + +from ..harness import FAIL, Result, ServerGone, expect + +NAME = "embedding" +DESCRIPTION = "/v1/embeddings: shape, batching, determinism and semantic similarity" + +SINGLE_INPUT = "Hi, everyone!" + +BATCH_INPUTS = [ + "The NPU accelerates matrix multiplication.", + "Quantized weights reduce memory bandwidth.", + "She poured the tea into a chipped porcelain cup.", +] + +# (anchor, paraphrase, unrelated) +SIMILARITY_TRIPLE = ( + "A man is playing a guitar on the street.", + "A busker performs guitar music on the sidewalk.", + "The quarterly earnings report exceeded analyst expectations.", +) +MIN_SIMILARITY_MARGIN = 0.05 +# The same text through the same weights; anything below this is not noise. +MIN_DETERMINISTIC_COSINE = 0.999 + + +def cosine(a: list, b: list) -> float: + dot = sum(x * y for x, y in zip(a, b)) + norm_a = math.sqrt(sum(x * x for x in a)) + norm_b = math.sqrt(sum(x * x for x in b)) + if norm_a == 0.0 or norm_b == 0.0: + return 0.0 + return dot / (norm_a * norm_b) + + +def embed(server, model: str, text_or_list) -> list[list[float]]: + """The vectors of one request, checked for shape, in the server's order. + + The handler reports its own failures as a 200 with an error body, so a + missing "data" list is the failure signal, not the status. + """ + body = server.client.post_json("/v1/embeddings", {"model": model, "input": text_or_list}) + data = body.get("data") if isinstance(body, dict) else None + expect(isinstance(data, list) and data, f"no embedding data in reply: {str(body)[:200]}") + vectors = [] + for i, item in enumerate(data): + vector = item.get("embedding") + expect(isinstance(vector, list) and vector, f"data[{i}] has no embedding vector") + expect(all(isinstance(x, (int, float)) and math.isfinite(x) for x in vector), + f"data[{i}] has a non-finite or non-numeric value") + expect(item.get("index") == i, f"data[{i}] has index {item.get('index')!r}") + vectors.append(vector) + return vectors + + +def run(ctx) -> None: + models = ctx.plan.suite(NAME) + if not models: + return + with ctx.server(NAME, ["-e", "1"]) as server: + for done, model in enumerate(models): + try: + check(ctx, server, model) + except ServerGone as exc: + ctx.record(Result(NAME, "server-gone", FAIL, model, str(exc))) + ctx.not_attempted(NAME, models[done + 1:], str(exc)) + return + + +def check(ctx, server, model: str) -> None: + with ctx.case(NAME, "single", model, server) as case: + vectors = embed(server, model, SINGLE_INPUT) + expect(len(vectors) == 1, f"expected 1 vector, got {len(vectors)}") + vector = vectors[0] + norm = math.sqrt(sum(x * x for x in vector)) + expect(norm > 0.0, "zero vector") + case.data.update(dim=len(vector), norm=round(norm, 4)) + + with ctx.case(NAME, "batch", model, server) as case: + vectors = embed(server, model, BATCH_INPUTS) + expect(len(vectors) == len(BATCH_INPUTS), + f"expected {len(BATCH_INPUTS)} vectors, got {len(vectors)}") + dims = sorted({len(v) for v in vectors}) + expect(len(dims) == 1, f"inconsistent dimensions across batch: {dims}") + case.data["dim"] = dims[0] + + with ctx.case(NAME, "similarity", model, server) as case: + vectors = embed(server, model, list(SIMILARITY_TRIPLE)) + expect(len(vectors) == 3, f"expected 3 vectors, got {len(vectors)}") + near = cosine(vectors[0], vectors[1]) + far = cosine(vectors[0], vectors[2]) + case.data.update(paraphrase=round(near, 4), unrelated=round(far, 4)) + expect(near - far >= MIN_SIMILARITY_MARGIN, + f"paraphrase {near:.4f} vs unrelated {far:.4f}: margin " + f"{near - far:.4f} below {MIN_SIMILARITY_MARGIN}") + + with ctx.case(NAME, "deterministic", model, server) as case: + first = embed(server, model, SINGLE_INPUT)[0] + second = embed(server, model, SINGLE_INPUT)[0] + expect(len(first) == len(second), + f"dimensions differ between runs: {len(first)} vs {len(second)}") + similarity = cosine(first, second) + case.data["cosine"] = round(similarity, 6) + expect(similarity > MIN_DETERMINISTIC_COSINE, + f"same input twice gave cosine {similarity:.6f}") diff --git a/.github/script/npu_test/suites/prefix_cache.py b/.github/script/npu_test/suites/prefix_cache.py new file mode 100644 index 000000000..509f34e9a --- /dev/null +++ b/.github/script/npu_test/suites/prefix_cache.py @@ -0,0 +1,226 @@ +"""The prefix cache: saved prompts are found again, and evicted by the +documented policy, across interleaved conversations. + +Each chat request reports what the cache did in usage.flm_prefix_cache +(lookup hit/miss, the slot it resumed from, what it saved and dropped) and +how many prompt tokens it skipped in usage.prompt_tokens_details.cached_tokens. +The server runs with two slots, so a third conversation must push one out, +and with a --prefix-min-gain larger than any follow-up turn here, so each +conversation keeps exactly one snapshot: that of its first turn. Without that, +a follow-up would replace its own conversation's older snapshot ("superseded") +and the case would no longer tell least-recently-used from first-in-first-out. + +A model whose responses carry no flm_prefix_cache has no slot cache and is +skipped; which models do is the engine's business, not a list kept here. +""" + +from __future__ import annotations + +from ..harness import ServerGone, expect, skip, warn +from ..server import ServerError + +NAME = "prefix_cache" +DESCRIPTION = "prefix-slot hits, LRU eviction with 2 slots, cached vs cold reply" + +SLOTS = 2 +# Far more tokens than a follow-up turn adds here, so follow-ups are never +# saved (see the module docstring). +MIN_GAIN = 1000 + +GREEDY = {"temperature": 0, "top_k": 1, "top_p": 1.0} +REPLY_TOKENS = 24 +THINKING_TOKENS = 512 + +# Distinct system prompts, so no conversation's snapshot is a prefix of +# another's. All are far below the 512-token minimum for a shared-head +# ("boundary") snapshot, so only end-of-prompt snapshots are saved. +CONVERSATIONS = { + "A": ("You are a geography tutor. Keep every answer to one short sentence.", + ["What is the capital of France?", "And the capital of Germany?", + "And the capital of Spain?"]), + "B": ("You are a zoology tutor. Keep every answer to one short sentence.", + ["Name one large mammal.", "Name one bird that cannot fly."]), + "C": ("You are a cooking tutor. Keep every answer to one short sentence.", + ["Name one citrus fruit."]), + "D": ("You are an astronomy tutor. Keep every answer to one short sentence.", + ["Name the planet closest to the Sun."]), +} + + +def run(ctx) -> None: + models = ctx.plan.suite(NAME) + if not models: + return + args = ["--prefix-slots", str(SLOTS), "--prefix-min-gain", str(MIN_GAIN)] + try: + with ctx.server(NAME, args) as server: + for index, model_id in enumerate(models): + try: + _model_cases(ctx, server, ctx.plan.model(model_id)) + except ServerGone as exc: + ctx.not_attempted(NAME, models[index + 1:], str(exc)) + raise + except ServerError as exc: + ctx.not_attempted(NAME, models, f"server did not start: {exc}") + raise + + +class _Conversation: + """One conversation's history, and the end snapshot the cache holds for it.""" + + def __init__(self, name: str, system: str, questions: list): + self.name = name + self.messages = [{"role": "system", "content": system}] + self.questions = list(questions) + self.asked: list = [] # the messages of each request sent, in order + self.replies: list = [] + self.slot = None # id of its newest end snapshot + + +class _Session: + def __init__(self, server, model): + self.server = server + self.model = model + self.conversations = {k: _Conversation(k, *v) for k, v in CONVERSATIONS.items()} + self.boundary_seen = False + self.trace: list = [] + + def ask(self, name: str, stream: bool = False, probe: bool = False): + """Sends the next question of a conversation; returns (reply, report). + + probe: the model's first request, which skips the model when the + engine reports no prefix cache at all. + """ + conv = self.conversations[name] + messages = conv.messages + [{"role": "user", "content": conv.questions[len(conv.asked)]}] + reply = self.send(messages, stream) + conv.asked.append(messages) + conv.replies.append(reply) + conv.messages = messages + [{"role": "assistant", + "content": reply.content or reply.reasoning[:300]}] + report = self.report(reply, f"{name}{len(conv.asked)}", probe) + for saved in report.get("saved") or []: + if saved.get("slot_type") == "end": + conv.slot = saved.get("slot") + return reply, report + + def send(self, messages: list, stream: bool = False): + # think=False and greedy on every request: FLM keeps both on the + # loaded model, and the think setting changes the rendered prompt. + budget = THINKING_TOKENS if self.model.always_thinks else REPLY_TOKENS + return self.server.client.chat(self.model.id, messages, stream=stream, think=False, + max_tokens=budget, **GREEDY) + + def report(self, reply, label: str, probe: bool = False) -> dict: + report = reply.usage.get("flm_prefix_cache") + if probe and not isinstance(report, dict): + skip("no usage.flm_prefix_cache: this model's engine has no prefix-slot cache") + expect(isinstance(report, dict), f"{label}: usage has no flm_prefix_cache") + if any(s.get("slot_type") == "boundary" for s in report.get("saved") or []): + self.boundary_seen = True + self.trace.append({ + "request": label, "lookup": report.get("lookup"), "slot": report.get("slot"), + "matched": report.get("matched_tokens"), "miss_reason": report.get("miss_reason"), + "saved": [s.get("slot") for s in report.get("saved") or []], + "dropped": report.get("dropped"), "slots_used": report.get("slots_used"), + "cached_tokens": reply.cached_tokens, "prompt_tokens": reply.usage.get("prompt_tokens"), + }) + _check_usage(reply, report, label) + return report + + +def _check_usage(reply, report: dict, label: str) -> None: + """cached_tokens and the cache's own report must tell the same story.""" + cached, prompt = reply.cached_tokens, reply.usage.get("prompt_tokens") + expect(report.get("lookup") in ("hit", "miss"), f"{label}: lookup is {report.get('lookup')!r}") + expect(isinstance(report.get("slots_used"), int) and report["slots_used"] <= SLOTS, + f"{label}: slots_used {report.get('slots_used')!r} with --prefix-slots {SLOTS}") + if report["lookup"] == "hit": + matched = report.get("matched_tokens") + expect(isinstance(matched, int) and matched > 0, f"{label}: hit with matched_tokens {matched!r}") + expect(cached == matched, f"{label}: cached_tokens {cached!r} != matched_tokens {matched}") + expect(isinstance(prompt, int) and cached < prompt, + f"{label}: cached_tokens {cached} not below prompt_tokens {prompt!r}") + else: + expect(report.get("miss_reason"), f"{label}: miss without a miss_reason") + expect(cached == 0, f"{label}: miss but cached_tokens is {cached!r}") + + +def _model_cases(ctx, server, model) -> None: + session = _Session(server, model) + later = ("hit-after-switch", "lru-eviction", "cached-matches-cold") + + with ctx.case(NAME, "cold-miss", model.id, server) as case: + if model.single_turn: + skip("single-turn model: no conversation to resume") + reports = {"A1": session.ask("A", probe=True)[1], "B1": session.ask("B")[1]} + case.data["trace"] = session.trace + for label, report in reports.items(): + expect(report["lookup"] == "miss", f"{label}: {report['lookup']} on a new conversation") + expect(report.get("slots_max") == SLOTS, + f"{label}: slots_max {report.get('slots_max')!r}, started with --prefix-slots {SLOTS}") + expect(session.conversations[label[0]].slot is not None, + f"{label} saved no end snapshot (saved: {report.get('saved')})") + expect(reports["B1"].get("slots_used") == 2, + f"B1: slots_used {reports['B1'].get('slots_used')}, expected 2") + if not case.ok: + reason = (ctx.results[-1].detail if case.status == "skip" else "needs cold-miss to pass") + for name in later: + with ctx.case(NAME, name, model.id, server): + skip(reason) + return + + a = session.conversations["A"] + with ctx.case(NAME, "hit-after-switch", model.id, server) as case: + a_slot = a.slot + reply, report = session.ask("A") + case.data["trace"] = session.trace[-1:] + expect(report["lookup"] == "hit", + f"A2 after B1: {report['lookup']} ({report.get('miss_reason')}), expected a hit on A1's snapshot") + expect(report.get("slot") == a_slot, + f"A2 resumed from slot {report.get('slot')}, A1's snapshot is slot {a_slot}") + expect(report.get("slot_type") == "end", f"A2 resumed from a {report.get('slot_type')} slot") + expect(reply.content.strip() or reply.reasoning.strip(), "A2: empty reply") + hit_ok = case.ok + + with ctx.case(NAME, "lru-eviction", model.id, server) as case: + if not hit_ok: + skip("needs hit-after-switch to pass") + if session.boundary_seen: + skip("the model saved a shared-head snapshot; this case assumes end snapshots only") + # Slots now: A1 (used by A2) and B1 (saved after A1, unused since). + # C1 needs room: least recently used is B1, though A1 is older. + a_slot, b_slot = a.slot, session.conversations["B"].slot + _, c1 = session.ask("C") + expect(c1["lookup"] == "miss", f"C1: {c1['lookup']} on a new conversation") + expect(c1.get("dropped") == 1, f"C1 dropped {c1.get('dropped')} snapshots, expected 1") + # Streamed, so the streamed usage block is checked as well. + _, a3 = session.ask("A", stream=True) + _, b2 = session.ask("B") + case.data["trace"] = session.trace[-3:] + expect(a3["lookup"] == "hit" and a3.get("slot") == a_slot, + f"A3: {a3['lookup']} slot {a3.get('slot')} ({a3.get('miss_reason')}); A was used " + f"more recently than B, so A's slot {a_slot} should have survived C1") + expect(not (b2["lookup"] == "hit" and b2.get("slot") == b_slot), + f"B2 resumed from B1's slot {b_slot}, which C1 should have evicted as least recently used") + expect(b2["lookup"] == "miss", f"B2: hit on slot {b2.get('slot')}, expected a miss") + + with ctx.case(NAME, "cached-matches-cold", model.id, server) as case: + if not hit_ok: + skip("needs hit-after-switch to pass") + # D1 pushes out A's snapshot (least recently used again), so replaying + # A2 prefills it from scratch. + session.ask("D") + cold = session.send(a.asked[1]) + cold_report = session.report(cold, "A2-replay") + cached = a.replies[1] + case.data.update(trace=session.trace[-2:], cached=cached.content[:200], cold=cold.content[:200]) + expect(cold.content.strip() or cold.reasoning.strip(), "A2 replay: empty reply") + if cold_report["lookup"] != "miss": + warn(f"the A2 replay resumed from slot {cold_report.get('slot')}; no cold run to compare") + if cold.content.strip() != cached.content.strip(): + # docs/.../prefix_cache.md: resuming from a snapshot is not + # bit-identical to a cold prefill; about 1 in 5 greedy answers + # differ in wording. Worth a look, not a failure. + warn(f"cached reply {cached.content[:80]!r} != cold reply {cold.content[:80]!r}") + diff --git a/.github/script/npu_test/suites/serving.py b/.github/script/npu_test/suites/serving.py new file mode 100644 index 000000000..675c2c0ca --- /dev/null +++ b/.github/script/npu_test/suites/serving.py @@ -0,0 +1,521 @@ +"""The server around the model: the NPU queue, cancellation, model switching, +and the endpoints that do not touch the NPU. + +FLM runs one NPU request at a time; the rest wait in a queue of -q/--q-len +entries, and a request that finds the queue full gets 503 at once +(WebServer::handle_request). The cases below pin that behaviour, and check +that a request that is cancelled, by POST /api/cancel or by its client going +away, gives the NPU back to the next one rather than holding it until its +max_tokens run out. +""" + +from __future__ import annotations + +import threading +import time +import uuid + +from ..client import HttpError +from ..harness import ServerGone, expect, skip, warn +from ..server import ServerError + +NAME = "serving" +DESCRIPTION = "NPU queue and 503, cancellation, client disconnect, model switch, /v1/models" + +# Queue length for this suite's server. With one request generating, a burst +# of BURST more gets exactly QUEUE_LEN queued and the rest 503. +QUEUE_LEN = 2 +BURST = QUEUE_LEN + 3 + +GREEDY = {"temperature": 0, "top_k": 1, "top_p": 1.0} + +# Long enough that a request which is not cancelled visibly keeps the NPU: +# 2048 tokens is well over ten seconds on any model this suite runs. +LONG_PROMPT = "Write the numbers from one to two thousand in words, one per line." +HOLD_TOKENS = 512 +LONG_TOKENS = 2048 +SHORT_TOKENS = 8 + +ANSWERS = [("What is the capital of France? Answer with one word.", "paris"), + ("What is the capital of Japan? Answer with one word.", "tokyo"), + ("What is the capital of Italy? Answer with one word.", "rome")] + +# Seconds to wait for a streamed request to start producing tokens (it may +# include a model load), and for the NPU to come free once a request has +# been cancelled. Cancellation is checked between tokens, so freeing takes +# well under a second; the margin is for a loaded runner. +FLOW_TIMEOUT = 300.0 +RELEASE_TIMEOUT = 5.0 +IDLE_TIMEOUT = 120.0 +STREAM_DEADLINE = 300.0 + +# Events read before the client disconnects: past the role chunk, so the +# request is decoding when the connection goes. +DISCONNECT_AFTER = 4 + +UNKNOWN_MODEL = "npu-test-no-such-model:1b" + +# Families /v1/models leaves out (get_all_models_openai in model_list.hpp): +# they are served by their own endpoints, not chat. +NOT_LISTED = {"whisper-v3", "embed-gemma", "d3"} + + +def run(ctx) -> None: + models = ctx.plan.suite(NAME) + if not models: + return + primary = models[0] + secondary = models[1] if len(models) > 1 else None + try: + with ctx.server(NAME, ["-q", str(QUEUE_LEN)]) as server: + _cases(ctx, server, primary, secondary) + except ServerError as exc: + ctx.not_attempted(NAME, models, f"server did not start: {exc}") + raise + except ServerGone as exc: + ctx.not_attempted(NAME, models, str(exc)) + raise + + +def _cases(ctx, server, primary: str, secondary: str | None) -> None: + client = server.client + + with ctx.case(NAME, "models-endpoint", server=server) as case: + listing = client.get_json("/v1/models", timeout=60) + expect(listing.get("object") == "list", f"object is {listing.get('object')!r}, not 'list'") + entries = listing.get("data") + expect(isinstance(entries, list) and entries, "data is empty or not a list") + ids = [e.get("id") for e in entries] + bad = [e for e in entries if e.get("object") != "model" or not e.get("id")] + expect(not bad, f"{len(bad)} entries without an id or object 'model', e.g. {bad[:1]}") + dupes = sorted({i for i in ids if ids.count(i) > 1}) + expect(not dupes, f"listed twice: {', '.join(dupes)}") + # The server lists the same catalog `flm list` printed, chat models only. + expected = {m for m in ctx.plan.catalog if m.split(":")[0] not in NOT_LISTED} + missing, extra = sorted(expected - set(ids)), sorted(set(ids) - expected) + case.data.update(listed=len(ids), missing=missing, extra=extra) + expect(not missing, f"catalog chat models missing: {', '.join(missing[:10])}") + expect(not extra, f"listed but not chat models of the catalog: {', '.join(extra[:10])}") + + with ctx.case(NAME, "load", primary, server) as case: + reply = _ask(client, primary, ANSWERS[0][0]) + case.data["load_duration"] = reply.usage.get("load_duration") + expect(reply.content.strip(), "empty content") + if not case.ok: + for name in ("queue-full", "concurrent-complete", "client-disconnect", "api-cancel", + "unknown-model", "model-switch"): + with ctx.case(NAME, name, primary, server): + skip(f"{primary} did not load") + return + + with ctx.case(NAME, "queue-full", primary, server) as case: + _queue_full(ctx, client, primary, case) + + with ctx.case(NAME, "concurrent-complete", primary, server) as case: + expect(_wait_idle(client, IDLE_TIMEOUT), "NPU still busy before the case started") + # Each prompt alone first, for its greedy reference reply; then all at + # once. Comparing against the model's own replies tests routing alone, + # whatever the model knows: a 0.6B model gets capitals and echoes + # wrong, but it gets them wrong the same way each time. + prompts = [prompt for prompt, _ in ANSWERS[:QUEUE_LEN + 1]] + reference = [_content(body).strip() for status, body, _ in + (_post(client, _body(primary, p, SHORT_TOKENS), ctx.request_timeout) + for p in prompts)] + expect(all(reference), f"empty reference reply: {reference}") + results = _fire(client, [_body(primary, p, SHORT_TOKENS) for p in prompts], + ctx.request_timeout) + case.data["reference"] = reference + case.data["results"] = [_summary(r) for r in results] + drifted = [] + for i, (prompt, (status, body, _)) in enumerate(zip(prompts, results)): + expect(status == 200, f"{prompt!r}: HTTP {status}: {str(body)[:200]}") + reply = _content(body).strip() + if reply == reference[i]: + continue + # Another request's reply is a mix-up, unless both prompts happen + # to get the same one. + swapped = [j for j, ref in enumerate(reference) if j != i and reply == ref + and ref != reference[i]] + if swapped: + raise AssertionError(f"{prompt!r} got the reply of " + f"{prompts[swapped[0]]!r}: {reply[:120]!r}") + drifted.append(f"{prompt!r}: {reference[i][:60]!r} alone, {reply[:60]!r} together") + if drifted: + # Greedy decoding on the NPU is not always bit-reproducible, so a + # reply that differs without matching another's is not a mix-up. + warn("replies differed from the same prompt sent alone: " + "; ".join(drifted)) + + with ctx.case(NAME, "client-disconnect", primary, server) as case: + _client_disconnect(ctx, server, primary, case) + _recover(ctx, server, client, primary, "client-disconnect") + + with ctx.case(NAME, "api-cancel", primary, server) as case: + _api_cancel(ctx, client, primary, case) + _recover(ctx, server, client, primary, "api-cancel") + + with ctx.case(NAME, "unknown-model", primary, server) as case: + try: + body = client.post_json("/v1/chat/completions", { + "model": UNKNOWN_MODEL, "stream": False, "max_tokens": SHORT_TOKENS, + "messages": [{"role": "user", "content": "Hi"}]}, timeout=120) + status = 200 + except HttpError as exc: + status, body = exc.status, exc.json() or exc.body + case.data.update(status=status, body=str(body)[:300]) + expect(isinstance(body, dict) and body.get("error"), f"no error body: {str(body)[:200]}") + expect(not body.get("choices"), "answered with choices for a model that does not exist") + # ensure_model_loaded refuses the name before releasing the loaded + # engine, so the next request runs on it without a reload. + after = _ask(client, primary, ANSWERS[0][0]) + case.data["after_load_duration"] = after.usage.get("load_duration") + expect(after.content.strip(), f"{primary} gave no answer after the unknown-model request") + if status == 200: + # The handler's error has no "code", so send_response keeps 200. + # OpenAI answers 404; until the server sets a code this is a + # warning, so the gap stays visible without failing every run. + warn(f"unknown model answered HTTP 200 with {str(body)[:120]}; expected a 4xx") + expect(400 <= status < 500, f"unknown model answered HTTP {status}") + + with ctx.case(NAME, "model-switch", primary, server) as case: + if not secondary: + skip("one model in the plan; a switch needs two") + prompt, word = ANSWERS[0] + steps = [] + for model_id in (primary, secondary, primary): + reply = _ask(client, model_id, prompt) + steps.append({"model": model_id, "load_duration": reply.usage.get("load_duration"), + "content": reply.content[:80]}) + case.data["steps"] = steps + expect(word in reply.content.lower(), + f"{model_id} (step {len(steps)}): answer was {reply.content[:120]!r}") + served = (reply.raw or {}).get("model") + expect(served == model_id, f"asked for {model_id}, response names {served!r}") + + +# -- cases ---------------------------------------------------------------------- + + +def _queue_full(ctx, client, model: str, case) -> None: + expect(_wait_idle(client, IDLE_TIMEOUT), "NPU still busy before the case started") + holder_id = _request_id("hold") + holder = _Stream(client, _body(model, LONG_PROMPT, HOLD_TOKENS, stream=True, + request_id=holder_id)) + holder.start() + try: + expect(holder.wait_flowing(FLOW_TIMEOUT), + f"the long request produced no tokens: {holder.error or holder.finish_reason}") + burst = _Burst(client, [_body(model, ANSWERS[0][0], SHORT_TOKENS)] * BURST, + ctx.request_timeout) + # The rejections come back at once; the queued requests only finish + # after the long one, so wait for the rejections alone. + burst.wait_for(BURST - QUEUE_LEN, timeout=60) + holder_busy = not holder.done.is_set() + finally: + # Ends the long request now rather than after its max_tokens; if + # /api/cancel is broken, api-cancel fails and this only costs time. + _cancel(client, holder_id) + holder.join(STREAM_DEADLINE) + results = burst.join(ctx.request_timeout) + statuses = [r[0] for r in results] + case.data.update(statuses=statuses, holder_busy=holder_busy, + holder_finish=holder.finish_reason, results=[_summary(r) for r in results]) + + unexpected = [_summary(r) for r in results if r[0] not in (200, 503)] + expect(not unexpected, f"expected only 200 or 503, got {unexpected[:1]}") + for status, body, _ in results: + if status == 503: + message = body.get("error") if isinstance(body, dict) else None + expect(isinstance(message, str) and "queue is full" in message, + f"503 without the queue-full error: {str(body)[:200]}") + else: + expect(_content(body).strip(), "a queued request answered 200 with no content") + expect(holder.error is None, f"the long request failed: {holder.error}") + + rejected = statuses.count(503) + after = _ask(client, model, ANSWERS[0][0]) + expect(after.content.strip(), "no answer after the burst") + if holder_busy: + expect(rejected == BURST - QUEUE_LEN, + f"{BURST} requests behind a running one with -q {QUEUE_LEN}: " + f"expected {BURST - QUEUE_LEN} rejected, got {rejected}") + elif rejected == 0: + # The long request ended before the burst was in, so the queue drained. + warn("the long request finished before the burst arrived; queue-full not exercised") + + +def _client_disconnect(ctx, server, model: str, case) -> None: + client = server.client + expect(_wait_idle(client, IDLE_TIMEOUT), "NPU still busy before the case started") + log_offset = _log_size(server) + stream = _Stream(client, _body(model, LONG_PROMPT, LONG_TOKENS, stream=True), + close_after=DISCONNECT_AFTER) + stream.start() + stream.join(FLOW_TIMEOUT) + expect(stream.closed_at is not None, + f"the stream ended after {stream.events} events before the client could leave: " + f"{stream.error or stream.finish_reason}") + released = _wait_idle(client, RELEASE_TIMEOUT) + release_seconds = round(time.monotonic() - stream.closed_at, 2) + case.data.update(events=stream.events, release_seconds=release_seconds) + expect(released, f"NPU still busy {RELEASE_TIMEOUT:.0f}s after the client disconnected " + f"from a {LONG_TOKENS}-token generation") + started = time.monotonic() + after = _ask(client, model, ANSWERS[0][0]) + case.data["next_seconds"] = round(time.monotonic() - started, 2) + expect(after.content.strip(), "no answer after the disconnect") + log = _log_since(server, log_offset) + if "cancel" not in log.lower(): + # The NPU came free in time but the server did not log a cancellation: + # the generation may have ended by itself, which this case cannot tell + # from a cancel. + warn("NPU freed, but the server log shows no cancellation for the disconnect") + + +def _api_cancel(ctx, client, model: str, case) -> None: + # Off until the server bug is fixed: handle_request registers every request + # under the request_id in its body, /api/cancel included, so a cancel by id + # cancels itself and never reaches its target (src/server/server.cpp, + # process_task). Delete this line to turn the case back on. + skip("disabled: /api/cancel cancels itself rather than the request it names (server bug)") + # /api/cancel finds a request by the request_id of its JSON body + # (WebServer::handle_request registers it), which an OpenAI chat body + # may carry like any other field. + reply = client.post_json("/api/cancel", {"request_id": _request_id("none")}, timeout=30) + if reply.get("cancelled") is True: + # Known server bug: handle_request registers every request under the + # request_id in its body, /api/cancel included, so the cancel request + # replaces the target's entry and then cancels itself + # (src/server/server.cpp, process_task). A cancel by id never reaches + # its target until that is fixed; this turns back into a failure then. + warn("known server bug: /api/cancel cancels itself rather than the request " + "it names (server.cpp registers the cancel request under that request_id)") + expect(reply.get("cancelled") is False, f"cancelling an unknown id answered {reply}") + + expect(_wait_idle(client, IDLE_TIMEOUT), "NPU still busy before the case started") + request_id = _request_id("cancel") + stream = _Stream(client, _body(model, LONG_PROMPT, LONG_TOKENS, stream=True, + request_id=request_id)) + stream.start() + expect(stream.wait_flowing(FLOW_TIMEOUT), + f"the request produced no tokens: {stream.error or stream.finish_reason}") + reply = client.post_json("/api/cancel", {"request_id": request_id}, timeout=30) + cancelled_at = time.monotonic() + stream.join(STREAM_DEADLINE) + end_seconds = round(time.monotonic() - cancelled_at, 2) + case.data.update(reply=reply, events=stream.events, finish_reason=stream.finish_reason, + end_seconds=end_seconds) + if reply.get("cancelled") is not True and stream.finish_reason in ("stop", "length"): + # Nothing left to cancel: the model ended its answer first. + warn(f"the generation ended ({stream.finish_reason}) before the cancel arrived") + expect(reply.get("cancelled") is True, f"/api/cancel answered {reply}") + expect(not stream.is_alive(), f"the stream was still open {STREAM_DEADLINE:.0f}s after the cancel") + expect(stream.error is None, f"the stream failed: {stream.error}") + expect(stream.finish_reason == "cancel", + f"finish_reason {stream.finish_reason!r} after the cancel, expected 'cancel'") + expect(end_seconds <= RELEASE_TIMEOUT, + f"the stream took {end_seconds}s to end after the cancel") + expect(_wait_idle(client, RELEASE_TIMEOUT), "NPU still busy after the cancelled stream ended") + + +# -- helpers -------------------------------------------------------------------- + + +def _recover(ctx, server, client, model: str, after: str) -> None: + """Restarts a server whose NPU never came free after a cancellation case. + + A cancel that leaves the NPU lock held makes every later request queue + forever, so the remaining cases would only time out one after another. + That case has already failed; this keeps the rest of the suite meaningful. + """ + if _wait_idle(client, RELEASE_TIMEOUT): + return + with ctx.case(NAME, "npu-stuck", model, server) as case: + case.data["after"] = after + restarted = server.restart() + case.data["restarted"] = restarted + raise AssertionError(f"the NPU stayed busy after {after}; " + + ("restarted the server for the remaining cases" if restarted + else "the server could not be restarted")) + + +def _body(model: str, prompt: str, max_tokens: int, *, stream: bool = False, + request_id: str | None = None) -> dict: + body = {"model": model, "messages": [{"role": "user", "content": prompt}], + "stream": stream, "max_tokens": max_tokens, "think": False, **GREEDY} + if stream: + body["stream_options"] = {"include_usage": True} + if request_id: + body["request_id"] = request_id + return body + + +def _ask(client, model: str, prompt: str): + return client.chat(model, [{"role": "user", "content": prompt}], stream=False, + max_tokens=SHORT_TOKENS, think=False, **GREEDY) + + +def _request_id(kind: str) -> str: + return f"npu-test-{kind}-{uuid.uuid4().hex[:12]}" + + +def _content(body) -> str: + if not isinstance(body, dict): + return "" + choice = (body.get("choices") or [{}])[0] + return (choice.get("message") or {}).get("content") or "" + + +def _post(client, body: dict, timeout: float) -> tuple: + """One non-streamed chat as (status, body, seconds); never raises. + + status is None when the connection itself failed (refused or reset), + which a full queue must never cause. + """ + started = time.monotonic() + try: + status, out = 200, client.post_json("/v1/chat/completions", body, timeout=timeout) + except HttpError as exc: + status, out = exc.status, exc.json() or exc.body + except Exception as exc: + status, out = None, f"{type(exc).__name__}: {exc}" + return status, out, round(time.monotonic() - started, 2) + + +def _summary(result) -> dict: + status, body, seconds = result + text = _content(body) if status == 200 else str(body) + return {"status": status, "seconds": seconds, "body": text[:120]} + + +class _Burst: + """Sends requests from one thread each, released together by a barrier.""" + + def __init__(self, client, bodies: list, timeout: float): + self.results: list = [None] * len(bodies) + self._arrived = threading.Semaphore(0) + barrier = threading.Barrier(len(bodies)) + + def send(i: int, body: dict) -> None: + try: + barrier.wait(timeout=30) + except threading.BrokenBarrierError: + pass + self.results[i] = _post(client, body, timeout) + self._arrived.release() + + self._threads = [threading.Thread(target=send, args=(i, b), daemon=True) + for i, b in enumerate(bodies)] + for thread in self._threads: + thread.start() + + def wait_for(self, count: int, timeout: float) -> int: + """Waits until count responses are in; returns how many arrived.""" + end = time.monotonic() + timeout + got = 0 + while got < count and self._arrived.acquire(timeout=max(0.0, end - time.monotonic())): + got += 1 + for _ in range(got): + self._arrived.release() + return got + + def join(self, timeout: float) -> list: + end = time.monotonic() + timeout + for thread in self._threads: + thread.join(max(0.0, end - time.monotonic())) + return [r if r is not None else (None, "no response in time", 0.0) for r in self.results] + + +def _fire(client, bodies: list, timeout: float) -> list: + return _Burst(client, bodies, timeout).join(timeout) + + +class _Stream(threading.Thread): + """Reads one streamed chat in the background. + + wait_flowing() returns once a few events are in, i.e. the request holds + the NPU and is decoding. close_after closes the connection after that many + events, as a client that gives up would. + """ + + def __init__(self, client, body: dict, close_after: int | None = None, + ready_after: int = 3): + super().__init__(daemon=True) + self.client = client + self.body = body + self.close_after = close_after + self.ready_after = ready_after + self.events = 0 + self.content = "" + self.finish_reason = None + self.error = None + self.closed_at = None + self.flowing = threading.Event() + self.done = threading.Event() + + def run(self) -> None: + events = self.client.stream_events("/v1/chat/completions", self.body, + timeout=STREAM_DEADLINE, deadline=STREAM_DEADLINE) + try: + for event in events: + self.events += 1 + if "error" in event: + self.error = str(event["error"])[:300] + for choice in event.get("choices") or []: + self.content += (choice.get("delta") or {}).get("content") or "" + if choice.get("finish_reason"): + self.finish_reason = choice["finish_reason"] + if self.events >= self.ready_after: + self.flowing.set() + if self.close_after and self.events >= self.close_after: + break + except Exception as exc: + self.error = f"{type(exc).__name__}: {exc}" + finally: + events.close() # closes the socket, which is what the server watches for + if self.close_after and self.events >= self.close_after: + self.closed_at = time.monotonic() + self.done.set() + + def wait_flowing(self, timeout: float) -> bool: + end = time.monotonic() + timeout + while time.monotonic() < end and not self.done.is_set(): + if self.flowing.wait(0.05): + return True + return self.flowing.is_set() + + +def _cancel(client, request_id: str) -> None: + try: + client.post_json("/api/cancel", {"request_id": request_id}, timeout=30) + except Exception: + pass + + +def _wait_idle(client, timeout: float) -> bool: + """True once /api/npu/status reports the NPU free (it is not an NPU route).""" + end = time.monotonic() + timeout + while True: + try: + if client.get_json("/api/npu/status", timeout=10).get("npu_available"): + return True + except Exception: + pass + if time.monotonic() >= end: + return False + time.sleep(0.1) + + +def _log_size(server) -> int: + try: + return server.log_path.stat().st_size + except OSError: + return 0 + + +def _log_since(server, offset: int) -> str: + try: + with open(server.log_path, "rb") as log: + log.seek(offset) + return log.read().decode("utf-8", "replace") + except OSError: + return "" diff --git a/.github/script/build_asr_eval.py b/.github/script/npu_test/tools/build_asr_eval.py similarity index 98% rename from .github/script/build_asr_eval.py rename to .github/script/npu_test/tools/build_asr_eval.py index 9ba6b070d..d609989ae 100644 --- a/.github/script/build_asr_eval.py +++ b/.github/script/npu_test/tools/build_asr_eval.py @@ -112,7 +112,7 @@ def main() -> None: parser.add_argument("--revision", required=True, help="FLEURS Parquet revision the files came from, recorded in the manifest") parser.add_argument("--out", type=Path, - default=Path(__file__).resolve().parent / "assets" / "asr_eval") + default=Path(__file__).resolve().parent.parent / "assets" / "asr_eval") args = parser.parse_args() args.out.mkdir(parents=True, exist_ok=True) diff --git a/.github/script/npu_test/tools/make_vision_assets.py b/.github/script/npu_test/tools/make_vision_assets.py new file mode 100644 index 000000000..30d9df35c --- /dev/null +++ b/.github/script/npu_test/tools/make_vision_assets.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Write the synthetic images in assets/vision/ that the chat suite asks about. + +A photo only supports "the model said something"; a large solid square on a +white background has one right answer, so a vision encoder that hands the +language model garbage fails the colour check instead of passing it. Two +colours, so a model that always answers "red" still fails. + +The images are committed; rerun this only to change them: + + python .github/script/npu_test/tools/make_vision_assets.py + +Standard library only, and deterministic: the same bytes on every run. +""" + +from __future__ import annotations + +import struct +import zlib +from pathlib import Path + +OUT_DIR = Path(__file__).resolve().parents[1] / "assets" / "vision" + +# 448 is a whole number of patches for the Qwen-VL (28 px) and Gemma (14 px) +# encoders, so no resize blurs the square's edges. +SIZE = 448 +MARGIN = 80 +WHITE = (255, 255, 255) + +SQUARES = { + "red_square.png": (220, 20, 20), + "blue_square.png": (20, 40, 220), +} + + +def png(width: int, height: int, pixel) -> bytes: + """An 8-bit RGB PNG; pixel(x, y) returns an (r, g, b) tuple.""" + rows = bytearray() + for y in range(height): + rows.append(0) # filter type: none + for x in range(width): + rows.extend(pixel(x, y)) + + def chunk(kind: bytes, data: bytes) -> bytes: + return (struct.pack(">I", len(data)) + kind + data + + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)) + + header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + + chunk(b"IDAT", zlib.compress(bytes(rows), 9)) + chunk(b"IEND", b"")) + + +def square(color): + inside = range(MARGIN, SIZE - MARGIN) + return lambda x, y: color if x in inside and y in inside else WHITE + + +def main() -> None: + OUT_DIR.mkdir(parents=True, exist_ok=True) + for name, color in SQUARES.items(): + path = OUT_DIR / name + path.write_bytes(png(SIZE, SIZE, square(color))) + print(f"wrote {path} ({path.stat().st_size} bytes)") + + +if __name__ == "__main__": + main() diff --git a/.github/script/resolve_build.sh b/.github/script/resolve_build.sh deleted file mode 100755 index 4cbbbf57b..000000000 --- a/.github/script/resolve_build.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env bash -# Decides where one platform's portable builds come from for the model sweep, -# and does not return until those builds actually exist. Writes these step -# outputs, each left unset when that build is unavailable: -# -# run_id the CI run holding the artifacts (run mode only) -# ready the default portable build -# hrx_ready the HRX portable build -# models_from_source_ready the models-from-source build (run mode only) -# -# Usage: resolve_build.sh -# -# Environment: -# MODE "release" or "run", decided by the resolve-source job -# RELEASE_TAG the release to check, in release mode -# CANDIDATE_SHAS commits whose build run to wait for, in run mode -# FINISH_MINUTES how long to wait for that run to complete, in run mode -# GH_TOKEN, GITHUB_REPOSITORY, GITHUB_OUTPUT as set by Actions -set -euo pipefail - -platform="$1" -case "${platform}" in - linux) - workflow=debian-portable.yml - label=Linux - # Release assets are anchored at the start of the name so an - # HRX_-prefixed asset is never mistaken for the default build. - release_asset='^fastflowlm_.*_linux\.tar\.gz$' - release_hrx_asset='^HRX_fastflowlm_.*_linux\.tar\.gz$' - artifact='^portable$' - hrx_artifact='^HRX-portable$' - source_artifact='^models-from-source-portable$' - ;; - windows) - workflow=windows-build.yml - label=Windows - release_asset='^fastflowlm_.*_windows_amd64\.zip$' - release_hrx_asset='^HRX_fastflowlm_.*_windows_amd64\.zip$' - # Fully anchored, so "fastflowlm-windows-" picks up neither the - # HRX- prefixed artifact nor the MSI job's "fastflowlm-windows-msi-". - artifact='^fastflowlm-windows-[0-9a-f]+$' - hrx_artifact='^HRX-fastflowlm-windows-[0-9a-f]+$' - source_artifact='^models-from-source-fastflowlm-windows-[0-9a-f]+$' - ;; - *) - echo "::error::Unknown platform '${platform}'." - exit 2 - ;; -esac - -# Sets =true when matches one of , and otherwise -# reports the build as missing at the given severity. -check() { - local names="$1" pattern="$2" flag="$3" what="$4" severity="$5" - if printf '%s\n' "${names}" | grep -qE "${pattern}"; then - echo "${flag}=true" >> "$GITHUB_OUTPUT" - echo " ${what}: ready" - else - echo "::${severity}::No ${what}; skipping that sweep." - fi -} - -# --- Release mode: a dispatch sweeps a shipped build ------------------------ -if [ "${MODE}" = "release" ]; then - # Fails the job on a typo'd tag, which is the right outcome: the operator - # asked for something specific that does not exist. - assets=$(gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${RELEASE_TAG}" \ - --jq '.assets[].name') - echo "Assets on ${RELEASE_TAG}:" - printf '%s\n' "${assets}" | sed 's/^/ /' - - # A release can legitimately ship one platform and not the other, so a - # missing asset skips that sweep instead of failing. - check "${assets}" "${release_asset}" ready \ - "${label} portable asset on ${RELEASE_TAG}" warning - # HRX assets ship from v1.0.8 on, and a failed HRX build leaves a release - # without them, so their absence is a notice, not a warning. - check "${assets}" "${release_hrx_asset}" hrx_ready \ - "${label} HRX portable asset on ${RELEASE_TAG}" notice - exit 0 -fi - -# --- Run mode: a PR or a src/** push sweeps this commit's build ------------- -POLL_SECONDS=20 -# Separate windows: a build that never starts is a lost cause well before one -# that is merely slow or queued behind other runs. -APPEAR_DEADLINE=$(( SECONDS + 10 * 60 )) -FINISH_DEADLINE=$(( SECONDS + FINISH_MINUTES * 60 )) - -# Newest run for whichever candidate SHA matches, or empty. -find_run() { - local sha run_id - for sha in ${CANDIDATE_SHAS}; do - run_id=$(gh api \ - "repos/${GITHUB_REPOSITORY}/actions/workflows/${workflow}/runs?head_sha=${sha}&per_page=100" \ - --jq 'if (.workflow_runs | length) == 0 then "" else (.workflow_runs | sort_by(.run_number) | last | .id) end') - if [ -n "${run_id}" ]; then - printf '%s' "${run_id}" - return 0 - fi - done -} - -while :; do - run_id=$(find_run) - [ -n "${run_id}" ] && break - if [ "${SECONDS}" -ge "${APPEAR_DEADLINE}" ]; then - echo "::warning::No ${workflow} run appeared for this commit; skipping the ${label} sweeps." - exit 0 - fi - echo "Waiting for a ${workflow} run to appear..." - sleep "${POLL_SECONDS}" -done - -# Waits for the whole run rather than the individual build jobs: artifacts -# are only listable once their job has uploaded them, and the run completing -# is the one event that guarantees that for all of them. -while :; do - status=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}" --jq '.status') - [ "${status}" = "completed" ] && break - if [ "${SECONDS}" -ge "${FINISH_DEADLINE}" ]; then - echo "::warning::${workflow} run ${run_id} is still ${status} after ${FINISH_MINUTES} minutes; skipping the ${label} sweeps." - exit 0 - fi - echo "${workflow} run ${run_id} is ${status}; checking again in ${POLL_SECONDS}s." - sleep "${POLL_SECONDS}" -done - -artifacts=$(gh api --paginate \ - "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/artifacts?per_page=100" \ - --jq '.artifacts[] | select(.expired | not) | .name') -echo "${workflow} run ${run_id} published:" -printf '%s\n' "${artifacts}" | sed 's/^/ /' - -# Written even when no artifact landed; the ready flags are what gate the -# sweeps, and an unused run id costs nothing. -echo "run_id=${run_id}" >> "$GITHUB_OUTPUT" - -# The upload step is the last thing each build job does and carries no -# "if: always()", so an artifact exists only if that job got all the way -# through. That makes artifact presence a better readiness signal than the -# run's conclusion, which is a single verdict over every job in the workflow: -# a failed HRX build, or a failed .deb build, would otherwise take the default -# portable sweep down with it. -check "${artifacts}" "${artifact}" ready \ - "${label} portable artifact in run ${run_id}" warning -check "${artifacts}" "${hrx_artifact}" hrx_ready \ - "${label} HRX portable artifact in run ${run_id}" warning -check "${artifacts}" "${source_artifact}" models_from_source_ready \ - "${label} models-from-source artifact in run ${run_id}" warning diff --git a/.github/script/sweep_audio.py b/.github/script/sweep_audio.py deleted file mode 100644 index 50ef12fd5..000000000 --- a/.github/script/sweep_audio.py +++ /dev/null @@ -1,184 +0,0 @@ -#!/usr/bin/env python3 -"""Sweep the audio path through /v1/audio/transcriptions. - -Whisper is absent from /v1/models, so the server needs `-a 1` and the model -set comes from `flm list`. A transcript is only accepted if it comes back -non-empty, since the server answers with an empty body when no ASR model is -loaded rather than reporting an error. - -Some chat models accept audio inline via `input_audio` instead of going -through the transcription endpoint. That path is off by default because it is -a different feature with different model support. Turn it on with ---include-chat-audio. -""" - -from __future__ import annotations - -import base64 -import sys -import time -from pathlib import Path - -from sweep_common import SweepTask, run_task - -CHAT_AUDIO_PROMPT = "Describe what you hear in this audio." - -AUDIO_FORMAT_BY_SUFFIX = { - ".mp3": "mp3", - ".wav": "wav", - ".m4a": "m4a", - ".flac": "flac", - ".ogg": "ogg", -} - - -class AudioSweep(SweepTask): - name = "audio" - needs_asr = True - csv_header = [ - "Model", "Mode", "Input", "Output Text", "Characters", - "Latency (s)", "Status", "Error", - ] - - @staticmethod - def add_arguments(parser) -> None: - parser.add_argument( - "--audio", - default=None, - metavar="PATH", - help="Audio clip to transcribe. Defaults to the bundled test clip.", - ) - parser.add_argument( - "--include-chat-audio", - action="store_true", - help="Also send the clip inline to audio-capable chat models.", - ) - - def audio_path(self) -> Path: - if self.args.audio: - path = Path(self.args.audio) - if not path.is_file(): - raise FileNotFoundError(f"Audio file not found: {path}") - return path - return self.asset("test_audio.mp3") - - def select_models(self, catalog: list[dict]) -> list[str]: - models = [e["id"] for e in catalog if e["id"].startswith("whisper-v3")] - if self.args.include_chat_audio: - models += [ - e["id"] for e in catalog - if e["asr"] and not e["id"].startswith("whisper-v3") - ] - if not models: - print( - "[audio] No audio models found. The /v1/models fallback hides " - "Whisper, so pass --models explicitly if 'flm list' failed." - ) - return models - - def _is_whisper(self, model: str) -> bool: - return model.startswith("whisper-v3") - - def _transcribe(self, writer, model: str, path: Path) -> None: - print(f" transcription: {path.name}") - started = time.monotonic() - try: - with open(path, "rb") as handle: - response = self.client.audio.transcriptions.create( - model=model, file=handle - ) - text = (getattr(response, "text", None) or "").strip() - if not text: - # The server returns an empty body when ASR is not loaded, so - # an empty transcript means a broken run, not a silent clip. - raise RuntimeError( - "empty transcript; the ASR model may not have loaded" - ) - except Exception as exc: - elapsed = f"{time.monotonic() - started:.2f}" - print(f" ERROR: {exc}") - self.record( - writer, - [model, "transcription", path.name, "", 0, elapsed, "error", str(exc)], - model=model, - error=f"transcription: {exc}", - ) - return - - elapsed = f"{time.monotonic() - started:.2f}" - self.record( - writer, - [model, "transcription", path.name, text, len(text), elapsed, "ok", ""], - model=model, - ) - print(f" {len(text)} chars in {elapsed}s: {text[:80]!r}") - - def _chat_audio(self, writer, model: str, path: Path) -> None: - audio_format = AUDIO_FORMAT_BY_SUFFIX.get(path.suffix.lower(), "mp3") - encoded = base64.b64encode(path.read_bytes()).decode("utf-8") - print(f" chat audio: {path.name}") - started = time.monotonic() - try: - _, output = self.chat( - model, - [ - { - "role": "user", - "content": [ - {"type": "text", "text": CHAT_AUDIO_PROMPT}, - { - "type": "input_audio", - "input_audio": { - "data": encoded, - "format": audio_format, - }, - }, - ], - } - ], - stream=True, - ) - output = (output or "").strip() - if not output: - raise RuntimeError("empty response") - except Exception as exc: - elapsed = f"{time.monotonic() - started:.2f}" - print(f" ERROR: {exc}") - self.record( - writer, - [model, "chat-audio", path.name, "", 0, elapsed, "error", str(exc)], - model=model, - error=f"chat-audio: {exc}", - ) - return - - elapsed = f"{time.monotonic() - started:.2f}" - self.record( - writer, - [model, "chat-audio", path.name, output, len(output), elapsed, "ok", ""], - model=model, - ) - print(f" {len(output)} chars in {elapsed}s") - - def run(self) -> None: - path = self.audio_path() - print(f"[audio] Clip: {path}") - - models = self.resolve_models() - handle, writer = self.open_csv() - try: - for model in self.iter_models(models): - print(f"\n--- Audio: {model} ---") - if self._is_whisper(model): - self._transcribe(writer, model, path) - else: - self._chat_audio(writer, model, path) - time.sleep(1) - finally: - handle.close() - print(f"\nAudio sweep complete. Saved to {self.csv_path}") - self.write_summary(models) - - -if __name__ == "__main__": - sys.exit(run_task(AudioSweep)) diff --git a/.github/script/sweep_common.py b/.github/script/sweep_common.py deleted file mode 100644 index db2543f4e..000000000 --- a/.github/script/sweep_common.py +++ /dev/null @@ -1,892 +0,0 @@ -"""Shared plumbing for the FLM model sweep scripts. - -The sweep is split into four independent entry points so that a failure in one -modality does not block or invalidate the others, and so a single failed run -can be restarted on its own: - - sweep_llm.py text chat completions - sweep_vision.py chat completions with images - sweep_embedding.py the /v1/embeddings endpoint - sweep_audio.py the /v1/audio/transcriptions endpoint - -Each entry point owns its own `flm serve` process, discovers the models it -cares about, writes one CSV, and exits non-zero if any model failed. Nothing is -shared between them at runtime. -""" - -from __future__ import annotations - -import argparse -import csv -import json -import os -import signal -import subprocess -import sys -import time -import urllib.error -import urllib.request -from datetime import datetime, timezone -from pathlib import Path - -# A sweep runs for tens of minutes and its progress output *is* the live status -# on a CI runner. Python block-buffers stdout whenever it is not a terminal, so -# under Actions every print would otherwise sit in an 8 KB buffer and arrive in -# one burst when the job ends -- no way to tell which model is being tested, or -# whether anything is happening at all. Line buffering here covers every print -# in every task script, so individual calls do not have to remember flush=True. -if hasattr(sys.stdout, "reconfigure"): - sys.stdout.reconfigure(line_buffering=True) -if hasattr(sys.stderr, "reconfigure"): - sys.stderr.reconfigure(line_buffering=True) - -try: - from openai import OpenAI -except ImportError: # pragma: no cover - surfaced immediately at runtime - print("The 'openai' package is required. Install it with: pip install openai") - raise - -SCRIPT_DIR = Path(__file__).resolve().parent -ASSETS_DIR = SCRIPT_DIR / "assets" - -# Matches utils::get_server_port() in src/common/utils.cpp. -DEFAULT_PORT = 52625 - -IS_WINDOWS = os.name == "nt" - -# Per-request timeouts, in seconds, for models too slow for --request-timeout. -# A model listed here gets whichever of the two is longer, so raising -# --request-timeout still raises it for these models too. -# -# qwen3.8-mtp:27b decodes at ~1.7 tok/s on the NPU, so a reply that uses the -# whole default 1024-token budget takes ~600 s, plus ~50 s to load the model -# on the first round. -REQUEST_TIMEOUT_OVERRIDES = { - "qwen3.8-mtp:27b": 1200.0, -} - - -def usage_to_dict(usage) -> dict: - """Flattens an SDK usage object into a plain dict. - - FLM returns non-standard fields alongside the OpenAI ones -- the prefill - and decode durations and rates, KV occupancy. The SDK's models allow extra - fields, so model_dump() keeps them, but going through a dict means nothing - here depends on that or on any particular SDK version. - """ - if usage is None: - return {} - if isinstance(usage, dict): - return usage - for method in ("model_dump", "to_dict", "dict"): - dump = getattr(usage, method, None) - if callable(dump): - try: - result = dump() - except Exception: - continue - if isinstance(result, dict): - return result - return dict(getattr(usage, "__dict__", {}) or {}) - - -# -------------------------------------------------------------------------- -# Arguments -# -------------------------------------------------------------------------- - -def build_parser(description: str) -> argparse.ArgumentParser: - """Argument parser carrying the options every sweep script understands.""" - parser = argparse.ArgumentParser(description=description) - - parser.add_argument( - "--flm-bin", - default=os.environ.get("FLM_BIN", "flm"), - help="Path to the flm binary (default: $FLM_BIN, else 'flm' on PATH).", - ) - parser.add_argument( - "--model-path", - default=None, - help="Value to export as FLM_MODEL_PATH before starting the server.", - ) - parser.add_argument( - "--platform", - default="windows" if IS_WINDOWS else "linux", - # Build variants use separate labels so their results do not land in - # the same filename or summary row. - choices=[ - "linux", - "windows", - "linux-hrx", - "windows-hrx", - "linux-models-from-source", - "windows-models-from-source", - ], - help="Label recorded in the output filename (default: autodetected).", - ) - parser.add_argument("--host", default="127.0.0.1", help="Server bind address.") - parser.add_argument( - "--port", - type=int, - default=int(os.environ.get("FLM_SERVE_PORT", DEFAULT_PORT)), - help=f"Server port (default: $FLM_SERVE_PORT, else {DEFAULT_PORT}).", - ) - parser.add_argument( - "--output-dir", - default="sweep-results", - help="Directory for the CSV, summary JSON and server log.", - ) - - parser.add_argument( - "--models", - nargs="+", - default=None, - metavar="TAG", - help="Test only these model tags instead of the discovered set.", - ) - parser.add_argument( - "--skip-models", - nargs="+", - default=[], - metavar="TAG", - help="Model tags to exclude from the discovered set.", - ) - parser.add_argument( - "--installed-only", - action="store_true", - help="Only test already-downloaded models, so the sweep pulls nothing.", - ) - parser.add_argument( - "--limit", - type=int, - default=0, - help="Stop after this many models. 0 means no cap.", - ) - - parser.add_argument( - "--gen-lim", - type=int, - default=1024, - help="max_completion_tokens per request. 0 or less leaves it unset.", - ) - parser.add_argument( - "--request-timeout", - type=float, - default=600.0, - help=( - "Per-request timeout in seconds (default: 600). Applied both as the " - "HTTP timeout and as a wall-clock deadline on streamed responses. " - "Models in REQUEST_TIMEOUT_OVERRIDES get a longer one." - ), - ) - parser.add_argument( - "--server-timeout", - type=float, - default=300.0, - help="How long to wait for the server to answer /v1/version.", - ) - parser.add_argument( - "--max-server-restarts", - type=int, - default=3, - help=( - "How many times to restart the server after it dies mid-sweep " - "(default: 3). 0 disables restarting." - ), - ) - parser.add_argument( - "--wedged-after", - type=int, - default=3, - help=( - "Treat the server as wedged once this many consecutive models fail " - "while it is still running, and restart it (default: 3). " - "0 disables the check." - ), - ) - parser.add_argument( - "--no-server", - action="store_true", - help="Attach to an already-running server instead of starting one.", - ) - - return parser - - -# -------------------------------------------------------------------------- -# Base task -# -------------------------------------------------------------------------- - -class SweepTask: - """Owns a server process, a model selection and one CSV of results.""" - - # Overridden by subclasses. - name = "sweep" - needs_asr = False - needs_embed = False - # A decision model tag (e.g. "d3:lite") served with --decision; empty for none. - needs_decision = "" - csv_header: list[str] = [] - - @staticmethod - def add_arguments(parser: argparse.ArgumentParser) -> None: - """Subclasses override this to register task-specific flags.""" - - def __init__(self, args: argparse.Namespace): - self.args = args - self.base_url = f"http://{args.host}:{args.port}/v1" - self.output_dir = Path(args.output_dir) - self.output_dir.mkdir(parents=True, exist_ok=True) - - if args.model_path: - os.environ["FLM_MODEL_PATH"] = str(args.model_path) - - self.server: subprocess.Popen | None = None - self._server_log_handle = None - self.version = "unknown" - self.rows_written = 0 - self.failures: list[str] = [] - # Models the sweep gave up on after the server stopped coming back. - # Distinct from failures: these were never actually exercised. - self.not_attempted: list[str] = [] - self.server_restarts = 0 - # Kept on the instance so a summary can still be written if run() dies. - self.models: list[str] = [] - # Audit trail of every server-lifecycle decision, with its reason. - self.events: list[dict] = [] - self._started_at = time.monotonic() - self._consecutive_failures = 0 - # Usage block from the most recent chat(), consumed by format_usage(). - self.last_usage: dict = {} - self._catalog_cache: list[dict] | None = None - self._labels_by_model: dict[str, list[str]] | None = None - - # -- lifecycle --------------------------------------------------------- - - def __enter__(self) -> "SweepTask": - self.start_server() - self.version = self.fetch_version() - self.client = OpenAI( - base_url=self.base_url, - api_key="flm", - timeout=self.args.request_timeout, - max_retries=0, - ) - return self - - def __exit__(self, exc_type, exc, tb) -> bool: - self.stop_server() - return False - - @property - def server_log_path(self) -> Path: - return self.output_dir / f"{self.name}_server.log" - - def log_event(self, event: str, *, reason: str = "", detail: str = "", **fields) -> None: - """Records one lifecycle decision, on stdout and in the summary JSON. - - Console output dies with the runner's log retention and cannot be - correlated with the CSV, so anything the sweep decides on its own -- - above all why it restarted the server -- has to explain itself in the - artifact as well. - """ - entry = { - "event": event, - "at": datetime.now(timezone.utc).isoformat(timespec="seconds"), - "elapsed_s": round(time.monotonic() - self._started_at, 1), - } - if reason: - entry["reason"] = reason - if detail: - entry["detail"] = detail - entry.update(fields) - self.events.append(entry) - - message = f"[{self.name}] {event}" - if reason: - message += f" ({reason})" - if detail: - message += f": {detail}" - print(message, flush=True) - - def start_server(self, append_log: bool = False) -> None: - if self.args.no_server: - print(f"[{self.name}] Attaching to existing server at {self.base_url}") - self.wait_until_ready() - return - - cmd = [ - self.args.flm_bin, - "serve", - "--host", self.args.host, - "--port", str(self.args.port), - "-a", "1" if self.needs_asr else "0", - "-e", "1" if self.needs_embed else "0", - ] - if self.needs_decision: - cmd += ["--decision", self.needs_decision] - self.log_event( - "server_start", - detail=" ".join(cmd), - attempt=self.server_restarts, - log=str(self.server_log_path), - ) - - # Appending on a restart keeps the log of the crash being recovered from. - self._server_log_handle = open(self.server_log_path, "ab" if append_log else "wb") - if append_log: - banner = f"\n=== server restart {self.server_restarts} ===\n" - self._server_log_handle.write(banner.encode("utf-8")) - self._server_log_handle.flush() - - popen_kwargs: dict = { - "stdout": self._server_log_handle, - "stderr": subprocess.STDOUT, - } - if IS_WINDOWS: - # Needed so we can deliver Ctrl-Break for a graceful shutdown - # without also killing this script. - popen_kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP - - self.server = subprocess.Popen(cmd, **popen_kwargs) - self.wait_until_ready() - - def wait_until_ready(self) -> None: - deadline = time.monotonic() + self.args.server_timeout - while time.monotonic() < deadline: - if self.server is not None and self.server.poll() is not None: - raise RuntimeError( - f"flm serve exited with code {self.server.returncode} " - f"before becoming ready. See {self.server_log_path}" - ) - try: - with urllib.request.urlopen(f"{self.base_url}/version", timeout=5): - print(f"[{self.name}] Server is ready at {self.base_url}") - return - except (urllib.error.URLError, OSError): - time.sleep(2) - raise TimeoutError( - f"Server did not respond within {self.args.server_timeout:.0f}s. " - f"See {self.server_log_path}" - ) - - def stop_server(self) -> None: - if self.server is not None and self.server.poll() is None: - try: - if IS_WINDOWS: - self.server.send_signal(signal.CTRL_BREAK_EVENT) - else: - self.server.send_signal(signal.SIGINT) - self.server.wait(timeout=30) - self.log_event("server_stop", reason="graceful", - detail=f"exit code {self.server.returncode}") - except (subprocess.TimeoutExpired, OSError, ValueError) as exc: - # Worth recording: a server that will not answer Ctrl-C is a - # different kind of broken from one that exits on request. - self.log_event("server_stop", reason="killed", - detail=f"graceful stop failed: {exc}") - self.server.kill() - try: - self.server.wait(timeout=15) - except subprocess.TimeoutExpired: - self.log_event("server_stop", reason="unresponsive", - detail="still running 15s after SIGKILL") - if self._server_log_handle is not None: - self._server_log_handle.close() - self._server_log_handle = None - - def server_died(self) -> bool: - """True when this script started the server and it has since exited. - - Only catches the server going away. A server that is still running but - wedged looks alive here, and is caught by the consecutive-failure check - in iter_models instead. - """ - return self.server is not None and self.server.poll() is not None - - def await_server_exit(self, timeout: float = 5.0) -> bool: - """Waits briefly for a server that may still be on its way out. - - A process killed by the request that just failed is often mid-exit when - poll() first runs. Returning "alive" there would blame the *next* model - for the crash, so give it a moment to actually go away before deciding. - """ - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - if self.server_died(): - return True - time.sleep(0.2) - return self.server_died() - - def restart_server(self, reason: str, detail: str = "") -> bool: - """Brings the server back mid-sweep. False once the budget is spent.""" - if self.args.no_server: - self.log_event("restart_declined", reason="--no-server", - detail="the server is managed outside this script") - return False - if self.server_restarts >= self.args.max_server_restarts: - self.log_event( - "restart_declined", - reason="budget exhausted", - detail=f"--max-server-restarts is {self.args.max_server_restarts}", - ) - return False - - self.server_restarts += 1 - # A wedged server is still running, so take it down before starting - # another one on the same port. Already-dead servers no-op here. - self.stop_server() - try: - self.start_server(append_log=True) - except Exception as exc: - # A restart can fail for reasons the sweep cannot fix, such as the - # old process still holding the port. Give up cleanly so the rows - # collected so far still get written. - self.log_event("server_restart", reason=reason, detail=detail, - attempt=self.server_restarts, outcome=f"failed: {exc}") - return False - - self.log_event("server_restart", reason=reason, detail=detail, - attempt=self.server_restarts, outcome="ready") - return True - - def iter_models(self, models: list[str]): - """Yields models, keeping the server alive for the rest of the sweep. - - Two ways the server ruins a sweep, both handled the same way. It can - exit, which poll() sees directly. Or it can stay up and stop answering, - which looks exactly like every model failing in turn -- so a run of - consecutive failed models is treated as the server being wedged and - earns a forced restart. - - Either way the sweep resumes at the *next* model: the culprit is not - retried, since it would just break the server again. Once restarts run - out the remaining models are recorded as not attempted rather than - reported as failures they never got to be. Every decision here lands in - self.events with its reason. - """ - total = len(models) - for index, model in enumerate(models): - # Every task funnels through here, so one progress line covers all - # four. Position and elapsed time are what a watcher actually wants - # from a sweep that runs for tens of minutes: which model is up now, - # and how much of the list is left. - elapsed = int(time.monotonic() - self._started_at) - print( - f"[{self.name}] ({index + 1}/{total}) {model} " - f"-- {elapsed // 60}m{elapsed % 60:02d}s elapsed", - flush=True, - ) - failures_before = len(self.failures) - yield model - - model_failed = len(self.failures) > failures_before - if model_failed: - self._consecutive_failures += 1 - else: - self._consecutive_failures = 0 - - # Only pay the grace period when something actually went wrong. - crashed = self.await_server_exit() if model_failed else self.server_died() - wedged = ( - not crashed - and self.args.wedged_after > 0 - and self._consecutive_failures >= self.args.wedged_after - ) - if not (crashed or wedged): - continue - - if crashed: - returncode = self.server.returncode if self.server else "unknown" - reason = "crash" - detail = f"exited with code {returncode} while testing {model}" - self.failures.append(f"{model}: flm server {detail}") - self.log_event("server_crashed", reason="process exited", - detail=detail, model=model) - else: - culprits = models[index - self._consecutive_failures + 1: index + 1] - reason = "wedged" - detail = ( - f"{self._consecutive_failures} consecutive models failed " - f"while the server was still running" - ) - self.log_event("server_wedged", reason="consecutive failures", - detail=detail, models=culprits) - - self._consecutive_failures = 0 - remaining = models[index + 1:] - if not remaining: - return - if not self.restart_server(reason=reason, detail=detail): - self.log_event( - "models_not_attempted", - reason=f"server unavailable after {reason}", - detail=f"{len(remaining)} model(s) skipped", - models=remaining, - ) - self.not_attempted.extend(remaining) - return - - # -- discovery --------------------------------------------------------- - - def fetch_version(self) -> str: - try: - with urllib.request.urlopen(f"{self.base_url}/version", timeout=10) as resp: - version = json.loads(resp.read().decode("utf-8")).get("version") - if version: - print(f"[{self.name}] flm version {version}") - return str(version) - except (urllib.error.URLError, OSError, json.JSONDecodeError, KeyError) as exc: - print(f"[{self.name}] Could not read version: {exc}") - return "unknown" - - def catalog(self) -> list[dict]: - """Model catalog, loaded once per run.""" - if self._catalog_cache is None: - self._catalog_cache = self._load_catalog() - return self._catalog_cache - - def _load_catalog(self) -> list[dict]: - """Full model catalog with the vlm/asr flags the sweep filters on. - - `flm list --json` is the only source that carries those flags; - /v1/models omits them and hides the embedding and Whisper models - entirely. Falls back to /v1/models when the binary is unavailable. - """ - list_filter = "installed" if self.args.installed_only else "all" - cmd = [self.args.flm_bin, "list", "--json", "--filter", list_filter] - try: - proc = subprocess.run( - cmd, capture_output=True, text=True, timeout=300, check=True - ) - # Tolerate banner lines printed before the JSON document. - start = proc.stdout.find("{") - if start < 0: - raise ValueError("no JSON object in 'flm list' output") - entries = json.loads(proc.stdout[start:]).get("models", []) - return [ - { - "id": entry.get("model") or entry.get("name"), - "vlm": bool(entry.get("vlm", False)), - "asr": bool(entry.get("asr", False)), - "family": (entry.get("details") or {}).get("family", ""), - # model_list.json's label array, passed through verbatim by - # `flm list --json`. Carries "single-turn" among others. - "labels": list(entry.get("label") or []), - "installed": bool(entry.get("installed", True)), - } - for entry in entries - if entry.get("model") or entry.get("name") - ] - except (subprocess.SubprocessError, OSError, ValueError, json.JSONDecodeError) as exc: - print(f"[{self.name}] 'flm list' unavailable ({exc}); falling back to /v1/models") - - try: - with urllib.request.urlopen(f"{self.base_url}/models", timeout=15) as resp: - data = json.loads(resp.read().decode("utf-8")).get("data", []) - return [ - {"id": m["id"], "vlm": False, "asr": False, "family": "", - "labels": [], "installed": True} - for m in data - if m.get("id") - ] - except (urllib.error.URLError, OSError, json.JSONDecodeError, KeyError) as exc: - print(f"[{self.name}] Could not list models: {exc}") - return [] - - def select_models(self, catalog: list[dict]) -> list[str]: - """Subclasses narrow the catalog to the models they exercise.""" - raise NotImplementedError - - def model_labels(self, model: str) -> list[str]: - """Catalog labels for one model, empty when the catalog had none.""" - if self._labels_by_model is None: - self._labels_by_model = { - entry["id"]: entry.get("labels") or [] - for entry in self.catalog() - if entry.get("id") - } - return self._labels_by_model.get(model, []) - - def is_single_turn(self, model: str) -> bool: - """True for models model_list.json labels "single-turn". - - These are built for one exchange -- dedicated translation models, the - flash variants -- and cap context at 1k. Sending a follow-up turn gets - rejected by design, so a sweep that counted that as a failure would be - reporting the model working correctly as a defect. - - Unknown models are treated as multi-turn: /v1/models carries no labels, - so guessing here would silently drop the follow-up for every model - whenever the flm binary was unavailable. - """ - return "single-turn" in self.model_labels(model) - - def resolve_models(self) -> list[str]: - if self.args.models: - models = list(self.args.models) - print(f"[{self.name}] Using {len(models)} model(s) from --models") - else: - models = self.select_models(self.catalog()) - - skip = set(self.args.skip_models) - if skip: - models = [m for m in models if m not in skip] - if self.args.limit > 0: - models = models[: self.args.limit] - - # Warmed here so a catalog that cannot be read complains during setup - # rather than in the middle of the first model's output. - single_turn = [m for m in models if self.is_single_turn(m)] - - print(f"[{self.name}] {len(models)} model(s) selected:") - for index, model in enumerate(models, 1): - suffix = " [single-turn]" if model in single_turn else "" - print(f" {index:>3}. {model}{suffix}") - if single_turn: - print( - f"[{self.name}] {len(single_turn)} model(s) are single-turn; " - "their follow-up round is skipped, not failed." - ) - self.models = models - return models - - # -- output ------------------------------------------------------------ - - @property - def csv_path(self) -> Path: - return self.output_dir / ( - f"{self.name}_{self.args.platform}_v{self.version}.csv" - ) - - @property - def summary_path(self) -> Path: - return self.output_dir / ( - f"{self.name}_{self.args.platform}_v{self.version}_summary.json" - ) - - def open_csv(self): - handle = open(self.csv_path, "w", newline="", encoding="utf-8") - writer = csv.writer(handle) - writer.writerow(self.csv_header) - return handle, writer - - def record(self, writer, row: list, *, model: str, error: str | None = None) -> None: - writer.writerow(row) - self.rows_written += 1 - if error is not None: - self.failures.append(f"{model}: {error}") - - def write_summary(self, models: list[str]) -> None: - summary = { - "task": self.name, - "platform": self.args.platform, - "flm_version": self.version, - "models_tested": len(models) - len(self.not_attempted), - "models_selected": len(models), - "rows_written": self.rows_written, - "failures": self.failures, - "not_attempted": self.not_attempted, - # Models whose follow-up round was deliberately not sent. Recorded - # so a reader can tell a deliberately shortened test from one that - # silently lost a round. - "single_turn": [m for m in models if self.is_single_turn(m)], - "server_restarts": self.server_restarts, - "events": self.events, - "csv": self.csv_path.name, - "server_log": self.server_log_path.name, - } - self.summary_path.write_text(json.dumps(summary, indent=2), encoding="utf-8") - - # -- helpers ----------------------------------------------------------- - - def completion_kwargs(self) -> dict: - """Token cap, omitted entirely when the caller disabled it.""" - if self.args.gen_lim and self.args.gen_lim > 0: - return {"max_completion_tokens": self.args.gen_lim} - return {} - - def request_timeout(self, model: str) -> float: - """Per-request timeout for one model, see REQUEST_TIMEOUT_OVERRIDES.""" - return max(self.args.request_timeout, REQUEST_TIMEOUT_OVERRIDES.get(model, 0.0)) - - def collect_stream(self, response, timeout: float) -> tuple[str, str]: - """Accumulates streamed chunks into (reasoning_content, output_content). - - The HTTP timeout only bounds the wait for the next chunk, so a model - that keeps emitting tokens can stream well past it. This enforces the - request timeout as a deadline on the whole response. - """ - reasoning_content, output_content = "", "" - usage = None - deadline = time.monotonic() + timeout - for chunk in response: - if time.monotonic() > deadline: - try: - response.close() - except Exception: - pass - raise TimeoutError( - f"stream exceeded the {timeout:.0f}s " - f"request timeout after {len(output_content)} characters" - ) - # With include_usage the usage arrives on its own terminal chunk - # with no choices, so read it before the choices guard below. - if getattr(chunk, "usage", None): - usage = chunk.usage - if not chunk.choices: - continue - delta = chunk.choices[0].delta - if getattr(delta, "reasoning_content", None): - reasoning_content += delta.reasoning_content - if delta.content: - output_content += delta.content - return reasoning_content, output_content, usage - - def chat(self, model: str, messages: list, stream: bool = True) -> tuple[str, str]: - """One chat completion, returned as (reasoning_content, output_content). - - The server's usage block lands on self.last_usage rather than in the - return value: callers unpack a 2-tuple, and timing data is something - only the reporting cares about. - """ - self.last_usage = {} - timeout = self.request_timeout(model) - response = self.client.chat.completions.create( - model=model, - messages=messages, - stream=stream, - # a stream reports usage only when asked, as OpenAI's does - **({"stream_options": {"include_usage": True}} if stream else {}), - **self.completion_kwargs(), - timeout=timeout, - ) - if stream: - reasoning, output, usage = self.collect_stream(response, timeout) - else: - message = response.choices[0].message - reasoning = getattr(message, "reasoning_content", None) or "" - output = message.content or "" - usage = getattr(response, "usage", None) - self.last_usage = usage_to_dict(usage) - return reasoning, output - - def format_usage(self) -> str: - """The server's own prefill/decode accounting, as one line. - - FLM reports far more than the OpenAI-standard token counts: it breaks - the request into prefill and decode, each with a duration and a rate, - and says how much of the prompt the KV cache served. That is the part - worth seeing per request, since wall-clock latency alone cannot - distinguish a slow prefill from a slow decode. - - Everything is optional. A server that reports only the standard fields - still gets a token count, and one that reports no usage at all gets an - empty string, leaving the caller's plain timing line as it was. - """ - usage = self.last_usage - if not usage: - return "" - - def num(key): - value = usage.get(key) - return value if isinstance(value, (int, float)) else None - - segments = [] - - prompt = num("prompt_tokens") - if prompt is not None: - cached = (usage.get("prompt_tokens_details") or {}).get("cached_tokens") - part = f"prefill {prompt} tok" - if isinstance(cached, (int, float)) and cached: - part += f" ({cached} cached)" - ttft = num("prefill_duration_ttft") - if ttft is not None: - part += f" in {ttft:.2f}s" - rate = num("prefill_speed_tps") - if rate is not None: - part += f" @ {rate:.1f} tps" - segments.append(part) - - completion = num("completion_tokens") - if completion is not None: - part = f"decode {completion} tok" - duration = num("decoding_duration") - if duration is not None: - part += f" in {duration:.2f}s" - rate = num("decoding_speed_tps") - if rate is not None: - part += f" @ {rate:.1f} tps" - segments.append(part) - - # Only meaningful on the first request against a model, where it is - # the difference between "the model is slow" and "it was still loading". - load = num("load_duration") - if load: - segments.append(f"load {load:.2f}s") - - occupancy = num("kv_token_occupancy_rate_percentage") - if occupancy is not None: - active = num("active_kv_tokens") - capacity = num("max_kv_token_capacity") - if active is not None and capacity: - segments.append(f"kv {active}/{capacity} ({occupancy:.0f}%)") - - return " | ".join(segments) - - @staticmethod - def asset(filename: str) -> Path: - path = ASSETS_DIR / filename - if not path.is_file(): - raise FileNotFoundError(f"Test asset missing: {path}") - return path - - # -- entry point ------------------------------------------------------- - - def run(self) -> None: - raise NotImplementedError - - -def run_task(task_cls, argv: list[str] | None = None) -> int: - """Parses arguments, runs one sweep task, and maps failures to an exit code.""" - parser = build_parser(task_cls.__doc__ or task_cls.name) - task_cls.add_arguments(parser) - args = parser.parse_args(argv) - - task = task_cls(args) - try: - try: - with task: - task.run() - except KeyboardInterrupt: - task.log_event("aborted", reason="interrupted") - return 130 - except Exception as exc: - task.log_event("aborted", reason="fatal error", detail=str(exc)) - return 1 - finally: - # Rewritten here rather than only at the end of run(), so the summary - # also captures how the server was shut down and any fatal error that - # cut the sweep short. write_summary overwrites, so the extra call is - # just the authoritative one. - try: - task.write_summary(task.models) - except Exception as summary_exc: - print(f"[{task.name}] Could not write summary: {summary_exc}") - - if task.not_attempted: - print( - f"\n[{task.name}] {len(task.not_attempted)} model(s) not attempted " - f"after the server stopped coming back:" - ) - for model in task.not_attempted: - print(f" - {model}") - - if task.failures: - print(f"\n[{task.name}] {len(task.failures)} failure(s):") - for failure in task.failures: - print(f" - {failure}") - print(f"[{task.name}] Results: {task.csv_path}") - return 1 - - print(f"\n[{task.name}] All checks passed. Results: {task.csv_path}") - return 0 diff --git a/.github/script/sweep_decision.py b/.github/script/sweep_decision.py deleted file mode 100644 index c92e484cf..000000000 --- a/.github/script/sweep_decision.py +++ /dev/null @@ -1,324 +0,0 @@ -#!/usr/bin/env python3 -"""Sweep the decision models through POST /v1/systemone. - -Per model (served with `--decision `): - goldens stored requests answered by the upstream BF16 model (assets/d3_goldens.jsonl): every answer has - the upstream shape, top-1 agrees on at least --min-agreement of them, mean |dp| at most --max-dp - types one choice, one noul, one score, and all three in one request - two-letter a 27-option choice (the 27th option's code is "AA") - invalid a malformed question -> 422 {"detail": {"invalid_questions": ...}}, nothing run - too-long a prompt over the input limit -> max_length_exceeded inside a 200 reply - image a request with a PNG image (yellow box on blue) answered, and a broken image -> 422 "images[0]: ..." - docs the example request of docs/docs/models/d3.md: the server's reply must decide what the documented - reply decides (choice, noul side, most likely score), so the docs cannot drift from the server - -Decision models are absent from /v1/models, so the model set comes from `flm list` (family "d3"). Each decision -model gets its own server (--decision takes one tag). -""" - -from __future__ import annotations - -import base64 -import json -import re -import time -import urllib.error -import urllib.request -from pathlib import Path - -from sweep_common import ASSETS_DIR, SweepTask, run_task - -REPO = Path(__file__).resolve().parents[2] -STATE = "The order arrived damaged yesterday. The customer has a receipt and asks for a replacement today." - - -def answer_shape_ok(answer: dict) -> bool: - kind = answer.get("type") - if "error" in answer: - return set(answer) == {"type", "error", "message"} - if kind == "choice": - return list(answer) == ["type", "choice", "probabilities", "confidence"] - if kind == "noul": - return list(answer) == ["type", "noul"] - if kind == "score": - return list(answer) == ["type", "score", "probabilities", "confidence", "legend"] - return False - - -class DecisionSweep(SweepTask): - name = "decision" - csv_header = ["Model", "Case", "Detail", "Latency (s)", "Status", "Error"] - - def __init__(self, args): - super().__init__(args) - # One server per decision model: the tag is the server's --decision argument. run() restarts - # the server for every further model. - self.needs_decision = (args.models or ["d3:lite"])[0] - - @staticmethod - def add_arguments(parser) -> None: - parser.add_argument("--min-agreement", type=float, default=0.85, - help="Share of golden answers whose top-1 must match the upstream BF16 model.") - parser.add_argument("--max-dp", type=float, default=0.05, - help="Largest allowed mean |probability difference| against the upstream BF16 model.") - - def select_models(self, catalog: list[dict]) -> list[str]: - return [entry["id"] for entry in catalog if entry["family"] == "d3"] or ["d3:lite"] - - # -- http ------------------------------------------------------------- - - @staticmethod - def _json_or_raw(raw: bytes) -> dict: - """The reply body as JSON; a body that is not JSON (a proxy's or crash page's 5xx) becomes - {"_raw": its start}, so the case fails instead of the whole sweep.""" - text = raw.decode("utf-8", errors="replace") - try: - body = json.loads(text or "{}") - except ValueError: - return {"_raw": text[:200]} - return body if isinstance(body, dict) else {"_raw": text[:200]} - - def post(self, body) -> tuple[int, dict, float]: - data = body if isinstance(body, bytes) else json.dumps(body).encode("utf-8") - req = urllib.request.Request(f"{self.base_url}/systemone", data=data, - headers={"Content-Type": "application/json"}, method="POST") - started = time.monotonic() - try: - with urllib.request.urlopen(req, timeout=self.args.request_timeout) as resp: - return resp.status, self._json_or_raw(resp.read()), time.monotonic() - started - except urllib.error.HTTPError as err: - return err.code, self._json_or_raw(err.read()), time.monotonic() - started - - def case(self, writer, model: str, case: str, ok: bool, detail: str, elapsed: float, error: str = "") -> None: - self.record(writer, [model, case, detail, f"{elapsed:.2f}", "ok" if ok else "error", "" if ok else error], - model=model, error=None if ok else f"{case}: {error or detail}") - print(f" {case}: {detail} -> {'ok' if ok else 'FAILED ' + error}") - - # -- cases ------------------------------------------------------------ - - def _goldens(self, writer, model: str) -> None: - agree = total = 0 - dps: list[float] = [] - bad_shape: list[str] = [] - elapsed = 0.0 - # every d3 size has its own BF16 answers: d3_goldens.jsonl is d3:lite's, d3_goldens_.jsonl the others' - size = model.split(":", 1)[1] - goldens = "d3_goldens.jsonl" if size == "lite" else f"d3_goldens_{size}.jsonl" - if not (ASSETS_DIR / goldens).is_file(): - # e.g. d3_goldens_turbo.jsonl exists once the upstream BF16 reference has answered the requests - self.case(writer, model, "goldens", False, f"{goldens} is missing", 0.0, f"no BF16 goldens for {model} ({goldens})") - return - for line in open(self.asset(goldens), encoding="utf-8"): - rec = json.loads(line) - status, body, t = self.post(rec["request"]) - elapsed += t - if status != 200: - bad_shape.append(f"{rec['id']}: status {status}") - continue - for key, want in rec["response"]["answers"].items(): - got = body["answers"].get(key, {}) - if not answer_shape_ok(got) or list(body["answers"]) != list(rec["response"]["answers"]): - bad_shape.append(f"{rec['id']}/{key}") - continue - if "probabilities" in want: - p, q = list(want["probabilities"].values()), list(got["probabilities"].values()) - dps.append(sum(abs(a - b) for a, b in zip(p, q)) / len(p)) - agree += max(range(len(p)), key=p.__getitem__) == max(range(len(q)), key=q.__getitem__) - total += 1 - elif "noul" in want: - dps.append(abs(want["noul"] - got["noul"])) - agree += (want["noul"] >= 0.5) == (got["noul"] >= 0.5) - total += 1 - share = agree / max(total, 1) - mdp = sum(dps) / max(len(dps), 1) - ok = not bad_shape and share >= self.args.min_agreement and mdp <= self.args.max_dp - self.case(writer, model, "goldens", ok, f"top-1 {agree}/{total}, mean |dp| {mdp:.4f}", elapsed, - "; ".join(bad_shape[:5]) or f"agreement {share:.3f} / mean |dp| {mdp:.4f} outside limits") - - def _types(self, writer, model: str) -> None: - questions = { - "route": {"type": "choice", "instructions": "Which team should handle this?", - "criteria": {"returns": "Refunds and replacements", "billing": "Payments", "general": None}}, - "urgent": {"type": "noul", "instructions": "Does this need a reply today?"}, - "anger": {"type": "score", "instructions": "How upset is the customer?", "criteria": ["calm", "annoyed", "angry"]}, - } - status, body, t = self.post({"state": STATE, "questions": questions}) - ok = (status == 200 and list(body.get("answers", {})) == list(questions) - and all(answer_shape_ok(a) and "error" not in a for a in body["answers"].values()) - and body["usage"]["input_tokens"] > 0 and body["usage"]["output_tokens"] == 0) - self.case(writer, model, "types", ok, f"status {status}, {len(body.get('answers', {}))} answers", t, json.dumps(body)[:200]) - - def _two_letter(self, writer, model: str) -> None: - criteria = {f"option{i:02d}": f"Option number {i}" for i in range(27)} - status, body, t = self.post({"state": STATE, "questions": {"q": {"type": "choice", "criteria": criteria}}}) - a = body.get("answers", {}).get("q", {}) - ok = status == 200 and answer_shape_ok(a) and len(a.get("probabilities", {})) == 27 - self.case(writer, model, "two-letter", ok, f"status {status}, {len(a.get('probabilities', {}))} options", t, json.dumps(body)[:200]) - - def _invalid(self, writer, model: str) -> None: - status, body, t = self.post({"questions": {"ok": {"type": "noul"}, "bad": {"type": "rank"}}}) - ok = status == 422 and body == {"detail": {"invalid_questions": { - "bad": "unsupported question type 'rank' (choice, noul or score)"}}} - self.case(writer, model, "invalid", ok, f"status {status}", t, json.dumps(body)[:200]) - status, body, t = self.post(b'{"questions": ') - ok = status == 422 and str(body.get("detail", "")).startswith("could not read the body as JSON") - self.case(writer, model, "not-json", ok, f"status {status}", t, json.dumps(body)[:200]) - - def _too_long(self, writer, model: str) -> None: - status, body, t = self.post({"state": "word " * 33000, "questions": {"q": {"type": "noul"}}}) - a = body.get("answers", {}).get("q", {}) - ok = status == 200 and a.get("error") == "max_length_exceeded" and body["usage"]["input_tokens"] == 0 - self.case(writer, model, "too-long", ok, f"status {status}, {a.get('error')}", t, json.dumps(body)[:200]) - - def _prefix_bidir(self, writer, model: str) -> None: - """d3:turbo only: a box question is answered when /v1/systemone/models lists `box` (the engine returns the - four hidden rows a box reads) and refused inline (`box_unsupported`) when it does not; input_tokens counts - the state block once (4 questions cost far less than 4 single-question requests).""" - try: - with urllib.request.urlopen(f"{self.base_url}/systemone/models", timeout=30) as resp: - entry = json.loads(resp.read())["data"][0] - listed = entry.get("question_types") or [] - except (urllib.error.URLError, ValueError, KeyError, IndexError) as err: - self.case(writer, model, "models", False, "GET /v1/systemone/models failed", 0.0, str(err)) - return - status, body, t = self.post({"state": STATE, "questions": {"b": {"type": "box", "instructions": "The customer's name"}}}) - a = body.get("answers", {}).get("b", {}) - if "box" in listed: - box = a.get("box", {}) - ok = (status == 200 and list(a) == ["type", "box", "confidence"] and sorted(box) == ["bottom", "left", "right", "top"] - and all(0.0 <= v <= 1.0 for v in box.values()) and box["top"] <= box["bottom"] and box["left"] <= box["right"]) - else: - ok = status == 200 and a.get("error") == "invalid_question" and a.get("reason") == "box_unsupported" - self.case(writer, model, "box", ok, f"status {status}, box listed: {'box' in listed}", t, json.dumps(body)[:200]) - long_state = "The customer wrote: " + "the parcel is late, again. " * 40 - one = {"q1": {"type": "noul", "instructions": "Is the customer upset?"}} - four = {f"q{i}": {"type": "noul", "instructions": f"Is the customer upset? ({i})"} for i in range(1, 5)} - s1, b1, t1 = self.post({"state": long_state, "questions": one}) - s4, b4, t4 = self.post({"state": long_state, "questions": four}) - n1 = b1.get("usage", {}).get("input_tokens", 0) - n4 = b4.get("usage", {}).get("input_tokens", 0) - ok = s1 == 200 and s4 == 200 and n1 > 0 and n1 < n4 < 4 * n1 - self.case(writer, model, "state-once-usage", ok, f"1 question {n1} tokens, 4 questions {n4}", t1 + t4, - f"input_tokens {n1} / {n4}") - # what the build lists is what it answers: boxes on an image, assets, a track over an encoded video - # (needs FFmpeg), long_context refused - if "box" in listed: - status, body, t = self.post({"state": "A small test picture.", "images": [self.IMAGE], - "questions": {"b": {"type": "box", "instructions": "The yellow rectangle"}}}) - a = body.get("answers", {}).get("b", {}) - box = a.get("box", {}) - ok = (status == 200 and list(a) == ["type", "box", "confidence"] and sorted(box) == ["bottom", "left", "right", "top"] - and all(0.0 <= v <= 1.0 for v in box.values()) and body["usage"]["input_tokens"] > 100) - self.case(writer, model, "box-image", ok, f"status {status}, box {box}", t, json.dumps(body)[:200]) - if entry.get("assets"): - status, body, t = self.post({"state": "Picture: ", "assets": {"": self.IMAGE}, - "questions": {"bg": {"type": "choice", "instructions": "What is the main colour of the background?", - "criteria": {"blue": "Blue", "red": "Red", "green": "Green"}}}}) - a = body.get("answers", {}).get("bg", {}) - ok = status == 200 and answer_shape_ok(a) and "error" not in a and body["usage"]["input_tokens"] > 100 - self.case(writer, model, "assets", ok, f"status {status}, choice {a.get('choice')}", t, json.dumps(body)[:200]) - if "track" in listed: - # a track question needs an encoded video (a frame list is only read by plain questions): the 4 s, - # 48 x 32 H.264 clip of the video tests. Without the FFmpeg libraries the server refuses encoded - # videos with 422 "video decoding backend not available": the case is then skipped, not failed. - clip = REPO / "src/test/d3_video/fixtures/h264.mp4" - if not clip.is_file(): - self.case(writer, model, "track", True, "skipped: the clip fixture is not in this checkout", 0.0) - else: - video = "data:video/mp4;base64," + base64.b64encode(clip.read_bytes()).decode("ascii") - status, body, t = self.post({"state": "A short clip.", "videos": [video], - "questions": {"t": {"type": "track", "instructions": "The moving object", "rate": 1}}}) - a = body.get("answers", {}).get("t", {}) - entries = a.get("track") or [] - if status == 422 and "decoding backend not available" in json.dumps(body): - self.case(writer, model, "track", True, "skipped: no FFmpeg libraries on this machine", t) - else: - ok = (status == 200 and a.get("type") == "track" and len(entries) >= 1 - and all(sorted(e) == ["box", "confidence", "time", "visible"] and 0.0 <= e["visible"] <= 1.0 for e in entries)) - self.case(writer, model, "track", ok, f"status {status}, {len(entries)} moments", t, json.dumps(body)[:200]) - status, body, t = self.post({"state": STATE, "long_context": True, "questions": {"q": {"type": "noul"}}}) - ok = (status == 422) == (entry.get("long_context") is False) - self.case(writer, model, "long-context", ok, f"status {status}, listed long_context {entry.get('long_context')}", t, - json.dumps(body)[:200]) - - # 96 x 64 PNG: a yellow box on a blue background - IMAGE = ("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGAAAABACAIAAABqVuVZAAAAiElEQVR42u3bMRXAIAxAQcKrjk5V1QkZlYAM" - "5FQEYmoApi4Z7u8ZuJeMxHm/RfsqAkCAAAECBAgQIK06/gzPp+V81dWHDXJigAABAiRAgAABAgQIECBAAgQIECBAgAABEiBAgAABAgQIkAAB" - "AgQIECBAgABpXfiSaYMAAQIECBAgQAIECBCgbH3ZCwYBFiTfPwAAAABJRU5ErkJggg==") - - def _image(self, writer, model: str) -> None: - q = {"type": "choice", "instructions": "What is the main colour of the background?", - "criteria": {"blue": "Blue", "red": "Red", "green": "Green"}} - status, body, t = self.post({"questions": {"bg": q}, "images": [self.IMAGE]}) - a = body.get("answers", {}).get("bg", {}) - ok = status == 200 and answer_shape_ok(a) and "error" not in a and body["usage"]["input_tokens"] > 100 - self.case(writer, model, "image", ok, f"status {status}, choice {a.get('choice')}", t, json.dumps(body)[:200]) - status, body, t = self.post({"questions": {"bg": q}, "images": [self.IMAGE[:120]]}) - ok = status == 422 and str(body.get("detail", "")).startswith("images[0]: ") - self.case(writer, model, "image-broken", ok, f"status {status}", t, json.dumps(body)[:200]) - - @staticmethod - def _decision(answer: dict): - """What an answer decides: the choice, the side of a noul, the most likely score level.""" - if answer.get("type") == "noul": - return answer.get("noul", 0) >= 0.5 - probabilities = answer.get("probabilities") or {} - return max(probabilities, key=probabilities.get) if probabilities else None - - def _same_decision(self, documented: dict, got: dict) -> bool: - """choice / noul: exactly the documented decision. score: the documented reply is one model's and its example is - a borderline one (levels 0 and 1 near each other for another model), so the neighbouring level may rank first; - the documented level must still carry at least 0.3 of the server's probability.""" - if self._decision(documented) == self._decision(got): - return True - if documented.get("type") == "score" and got.get("type") == "score": - level = self._decision(documented) - return got.get("probabilities", {}).get(level, 0.0) >= 0.3 - return False - - def _docs(self, writer, model: str) -> None: - doc = (REPO / "docs/docs/models/d3.md").read_text(encoding="utf-8") - m = re.search(r"-d '(\{.*?\})'", doc, re.S) - r = re.search(r"```json\s*(\{.*?\})\s*```", doc[m.end():], re.S) if m else None - if not m or not r: - self.case(writer, model, "docs", False, "no example request and reply in docs/docs/models/d3.md", 0.0, "missing") - return - request, documented = json.loads(m.group(1)), json.loads(r.group(1)) - status, body, t = self.post(request) - answers = body.get("answers", {}) - ok = (status == 200 and list(answers) == list(request["questions"]) - and all(answer_shape_ok(a) and "error" not in a for a in answers.values())) - # the documented reply must decide what the server decides; its numbers are rounded - drift = [k for k, a in documented.get("answers", {}).items() if not self._same_decision(a, answers.get(k, {}))] - ok = ok and not drift - self.case(writer, model, "docs", ok, f"status {status}" + (f", docs decide differently on {drift}" if drift else ""), - t, json.dumps(body)[:200]) - - def run(self) -> None: - models = self.resolve_models() - if self.args.no_server and len(models) > 1: - raise SystemExit("--no-server attaches to one server, which serves one decision model; pass --models ") - handle, writer = self.open_csv() - try: - for model in self.iter_models(models): - if model != self.needs_decision: # each decision model needs its own server - self.stop_server() - self.needs_decision = model - self.start_server(append_log=True) - print(f"\n--- Decision: {model} ---") - self._goldens(writer, model) - self._types(writer, model) - self._two_letter(writer, model) - self._invalid(writer, model) - self._too_long(writer, model) - if model in ("d3:turbo", "d3:ultra"): - self._prefix_bidir(writer, model) - self._image(writer, model) - self._docs(writer, model) - finally: - handle.close() - self.write_summary(models) - - -if __name__ == "__main__": - raise SystemExit(run_task(DecisionSweep)) diff --git a/.github/script/sweep_embedding.py b/.github/script/sweep_embedding.py deleted file mode 100644 index 14698d81d..000000000 --- a/.github/script/sweep_embedding.py +++ /dev/null @@ -1,209 +0,0 @@ -#!/usr/bin/env python3 -"""Sweep the embedding models through the /v1/embeddings endpoint. - -Covers three things per model: a single-string request, a batched request, -and a semantic sanity check. The sanity check embeds a sentence alongside a -paraphrase and an unrelated sentence, then asserts the paraphrase scores -higher. A model that returns well-formed vectors carrying no meaning passes -the first two checks and fails this one. - -Embedding models are absent from /v1/models, so the server needs `-e 1` and -the model set comes from `flm list`. -""" - -from __future__ import annotations - -import math -import sys -import time - -from sweep_common import SweepTask, run_task - -SINGLE_INPUT = "Hi, everyone!" - -BATCH_INPUTS = [ - "The NPU accelerates matrix multiplication.", - "Quantized weights reduce memory bandwidth.", - "She poured the tea into a chipped porcelain cup.", -] - -# (anchor, paraphrase, unrelated) -SIMILARITY_TRIPLE = ( - "A man is playing a guitar on the street.", - "A busker performs guitar music on the sidewalk.", - "The quarterly earnings report exceeded analyst expectations.", -) - - -def cosine(a: list[float], b: list[float]) -> float: - dot = sum(x * y for x, y in zip(a, b)) - norm_a = math.sqrt(sum(x * x for x in a)) - norm_b = math.sqrt(sum(x * x for x in b)) - if norm_a == 0.0 or norm_b == 0.0: - return 0.0 - return dot / (norm_a * norm_b) - - -class EmbeddingSweep(SweepTask): - name = "embedding" - needs_embed = True - csv_header = [ - "Model", "Case", "Input", "Dimensions", "L2 Norm", - "Detail", "Latency (s)", "Status", "Error", - ] - - @staticmethod - def add_arguments(parser) -> None: - parser.add_argument( - "--min-similarity-margin", - type=float, - default=0.05, - help="How far the paraphrase must outscore the unrelated sentence.", - ) - - def select_models(self, catalog: list[dict]) -> list[str]: - models = [entry["id"] for entry in catalog if entry["family"] == "embed-gemma"] - if not models: - print( - "[embedding] No embedding models found. The /v1/models fallback " - "hides them, so pass --models explicitly if 'flm list' failed." - ) - return models - - def _embed(self, model: str, text_or_list): - response = self.client.embeddings.create(model=model, input=text_or_list) - if not response.data: - raise RuntimeError("server returned no embedding data") - return [item.embedding for item in response.data] - - def _case_single(self, writer, model: str) -> None: - print(f" single input: {SINGLE_INPUT!r}") - started = time.monotonic() - try: - vectors = self._embed(model, SINGLE_INPUT) - except Exception as exc: - self._fail(writer, model, "single", SINGLE_INPUT, started, exc) - return - - elapsed = f"{time.monotonic() - started:.2f}" - vector = vectors[0] - norm = math.sqrt(sum(x * x for x in vector)) - preview = ", ".join(f"{x:.4f}" for x in vector[:8]) - self.record( - writer, - [model, "single", SINGLE_INPUT, len(vector), f"{norm:.4f}", - f"first 8: [{preview}]", elapsed, "ok", ""], - model=model, - ) - print(f" {len(vector)} dims, norm {norm:.4f}, {elapsed}s") - - def _case_batch(self, writer, model: str) -> None: - label = f"{len(BATCH_INPUTS)} inputs" - print(f" batch: {label}") - started = time.monotonic() - try: - vectors = self._embed(model, BATCH_INPUTS) - except Exception as exc: - self._fail(writer, model, "batch", label, started, exc) - return - - elapsed = f"{time.monotonic() - started:.2f}" - if len(vectors) != len(BATCH_INPUTS): - error = f"expected {len(BATCH_INPUTS)} vectors, got {len(vectors)}" - self.record( - writer, - [model, "batch", label, "", "", "", elapsed, "error", error], - model=model, - error=f"batch: {error}", - ) - print(f" ERROR: {error}") - return - - dims = {len(v) for v in vectors} - if len(dims) != 1: - error = f"inconsistent dimensions across batch: {sorted(dims)}" - self.record( - writer, - [model, "batch", label, "", "", "", elapsed, "error", error], - model=model, - error=f"batch: {error}", - ) - print(f" ERROR: {error}") - return - - dim = dims.pop() - self.record( - writer, - [model, "batch", label, dim, "", - f"{len(vectors)} vectors returned", elapsed, "ok", ""], - model=model, - ) - print(f" {len(vectors)} vectors, {dim} dims, {elapsed}s") - - def _case_similarity(self, writer, model: str) -> None: - anchor, paraphrase, unrelated = SIMILARITY_TRIPLE - print(" similarity: paraphrase should outscore unrelated") - started = time.monotonic() - try: - vectors = self._embed(model, list(SIMILARITY_TRIPLE)) - except Exception as exc: - self._fail(writer, model, "similarity", anchor, started, exc) - return - - elapsed = f"{time.monotonic() - started:.2f}" - if len(vectors) != 3: - error = f"expected 3 vectors, got {len(vectors)}" - self.record( - writer, - [model, "similarity", anchor, "", "", "", elapsed, "error", error], - model=model, - error=f"similarity: {error}", - ) - print(f" ERROR: {error}") - return - - near = cosine(vectors[0], vectors[1]) - far = cosine(vectors[0], vectors[2]) - margin = near - far - detail = f"paraphrase {near:.4f} vs unrelated {far:.4f}, margin {margin:.4f}" - passed = margin >= self.args.min_similarity_margin - - self.record( - writer, - [model, "similarity", anchor, len(vectors[0]), "", - detail, elapsed, "ok" if passed else "error", - "" if passed else f"margin below {self.args.min_similarity_margin}"], - model=model, - error=None if passed else f"similarity margin {margin:.4f} too small", - ) - print(f" {detail} -> {'ok' if passed else 'FAILED'}") - - def _fail(self, writer, model: str, case: str, label: str, - started: float, exc: Exception) -> None: - elapsed = f"{time.monotonic() - started:.2f}" - print(f" ERROR: {exc}") - self.record( - writer, - [model, case, label, "", "", "", elapsed, "error", str(exc)], - model=model, - error=f"{case}: {exc}", - ) - - def run(self) -> None: - models = self.resolve_models() - handle, writer = self.open_csv() - try: - for model in self.iter_models(models): - print(f"\n--- Embedding: {model} ---") - self._case_single(writer, model) - self._case_batch(writer, model) - self._case_similarity(writer, model) - time.sleep(1) - finally: - handle.close() - print(f"\nEmbedding sweep complete. Saved to {self.csv_path}") - self.write_summary(models) - - -if __name__ == "__main__": - sys.exit(run_task(EmbeddingSweep)) diff --git a/.github/script/sweep_llm.py b/.github/script/sweep_llm.py deleted file mode 100644 index 98f9ebdc3..000000000 --- a/.github/script/sweep_llm.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env python3 -"""Sweep every chat model with a two-round text conversation. - -Each model gets a first prompt and a follow-up that depends on the first -answer, so multi-turn context handling is covered as well as single-shot -generation. Runs in streaming mode by default; pass --modes to add or swap in -the non-streaming path. - -Models the catalog labels "single-turn" only get the first round: refusing a -follow-up is correct behaviour for them, not a failure. -""" - -from __future__ import annotations - -import sys -import time - -from sweep_common import SweepTask, run_task - -PROMPTS = { - "stream": ("Tell me a joke.", "Explain why it is funny."), - "non-stream": ("Teach me Maxwell's equations.", "Summarize your answer."), -} - - -class LLMSweep(SweepTask): - name = "llm" - csv_header = [ - "Model", "Mode", "Round", "Input", - "Reasoning Content", "Output Content", - "Latency (s)", "Status", "Error", - ] - - @staticmethod - def add_arguments(parser) -> None: - parser.add_argument( - "--modes", - nargs="+", - default=["stream"], - choices=["stream", "non-stream"], - help="Which completion modes to exercise (default: stream).", - ) - - def select_models(self, catalog: list[dict]) -> list[str]: - # Every chat model, vision-capable ones included: they all accept a - # plain text conversation. The embedding and Whisper models are not - # chat models and are handled by their own sweeps. - return [ - entry["id"] - for entry in catalog - if entry["family"] != "embed-gemma" - and not entry["id"].startswith("whisper-v3") - ] - - def _round(self, writer, model: str, mode: str, round_no: int, - messages: list, prompt: str) -> str | None: - """Runs one turn. Returns the assistant text, or None on failure.""" - print(f" [{mode}] round {round_no}: {prompt}") - started = time.monotonic() - try: - reasoning, output = self.chat(model, messages, stream=(mode == "stream")) - except Exception as exc: - elapsed = f"{time.monotonic() - started:.2f}" - print(f" ERROR: {exc}") - self.record( - writer, - [model, mode, round_no, prompt, "", "", elapsed, "error", str(exc)], - model=model, - error=f"{mode} round {round_no}: {exc}", - ) - return None - - elapsed = f"{time.monotonic() - started:.2f}" - self.record( - writer, - [model, mode, round_no, prompt, reasoning, output, elapsed, "ok", ""], - model=model, - ) - print(f" done in {elapsed}s, {len(output)} chars") - # Empty unless the server reported usage, so this adds a line only - # when there is something in it. - breakdown = self.format_usage() - if breakdown: - print(f" {breakdown}") - return output - - def run(self) -> None: - models = self.resolve_models() - handle, writer = self.open_csv() - try: - for model in self.iter_models(models): - print(f"\n--- LLM: {model} ---") - for mode in self.args.modes: - prompt, followup = PROMPTS[mode] - messages = [{"role": "user", "content": prompt}] - - output = self._round(writer, model, mode, 1, messages, prompt) - if output is None: - # The follow-up needs the first answer for context, so - # skip it rather than send a conversation with a hole. - continue - - if self.is_single_turn(model): - # Rejecting a second turn is what these models are - # supposed to do, so the round is recorded as skipped - # rather than sent and counted as a failure. - self.record( - writer, - [model, mode, 2, followup, "", "", "", "skipped", - "single-turn model: follow-up not supported"], - model=model, - ) - print(" round 2 skipped: model is labelled single-turn") - time.sleep(1) - continue - - messages.append({"role": "assistant", "content": output}) - messages.append({"role": "user", "content": followup}) - self._round(writer, model, mode, 2, messages, followup) - time.sleep(1) - finally: - handle.close() - print(f"\nLLM sweep complete. Saved to {self.csv_path}") - self.write_summary(models) - - -if __name__ == "__main__": - sys.exit(run_task(LLMSweep)) diff --git a/.github/script/sweep_summary.py b/.github/script/sweep_summary.py deleted file mode 100644 index 26fa3883e..000000000 --- a/.github/script/sweep_summary.py +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env python3 -"""Render the sweep summary JSON files as a Markdown report. - -Reads every *_summary.json written by the sweep scripts under the given -directory and prints a GitHub-flavoured Markdown report to stdout. Intended -for $GITHUB_STEP_SUMMARY, but works fine on a terminal. -""" - -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path - - -def load_summaries(root: Path) -> list[dict]: - summaries = [] - for path in sorted(root.rglob("*_summary.json")): - try: - summaries.append(json.loads(path.read_text(encoding="utf-8"))) - except (OSError, json.JSONDecodeError) as exc: - print(f"") - return summaries - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("directory", help="Directory holding the downloaded artifacts.") - parser.add_argument("--linux-result", default="", help="Job result for the Linux sweep.") - parser.add_argument("--windows-result", default="", help="Job result for the Windows sweep.") - parser.add_argument( - "--linux-hrx-result", default="", help="Job result for the Linux HRX sweep." - ) - parser.add_argument( - "--windows-hrx-result", default="", help="Job result for the Windows HRX sweep." - ) - parser.add_argument( - "--linux-models-from-source-result", - default="", - help="Job result for the Linux models-from-source sweep.", - ) - parser.add_argument( - "--windows-models-from-source-result", - default="", - help="Job result for the Windows models-from-source sweep.", - ) - args = parser.parse_args(argv) - - out: list[str] = ["# Model Sweep Results", ""] - - # A platform whose sweep never ran contributes an empty result and is left - # out of the table rather than shown as a blank row. - overall = [ - ("Linux", args.linux_result), - ("Linux (HRX)", args.linux_hrx_result), - ("Windows", args.windows_result), - ("Windows (HRX)", args.windows_hrx_result), - ("Linux (models from source)", args.linux_models_from_source_result), - ("Windows (models from source)", args.windows_models_from_source_result), - ] - if any(result for _, result in overall): - out += ["| Platform | Overall |", "| --- | --- |"] - for label, result in overall: - if result: - out.append(f"| {label} | {result} |") - out.append("") - - summaries = load_summaries(Path(args.directory)) - if not summaries: - out += ["_No sweep summaries were produced._", ""] - print("\n".join(out)) - return 0 - - out += [ - "| Task | Platform | FLM | Models | Rows | Failures | Not run | Restarts |", - "| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |", - ] - for item in summaries: - tested = item.get("models_tested", 0) - selected = item.get("models_selected", tested) - out.append( - "| {} | {} | {} | {} | {} | {} | {} | {} |".format( - item.get("task", "?"), - item.get("platform", "?"), - item.get("flm_version", "?"), - f"{tested}/{selected}" if tested != selected else tested, - item.get("rows_written", 0), - len(item.get("failures", [])), - len(item.get("not_attempted", [])), - item.get("server_restarts", 0), - ) - ) - out.append("") - - failing = [item for item in summaries if item.get("failures")] - out += ["## Failures", ""] - if not failing: - out += ["_None._", ""] - for item in failing: - out.append(f"**{item.get('task', '?')} / {item.get('platform', '?')}**") - out.append("") - for failure in item["failures"]: - # Collapse newlines so a multi-line server error stays one bullet. - out.append("- " + " ".join(str(failure).split())) - out.append("") - - # Models the sweep never got to, because the server stopped coming back. - # Reported apart from failures so they are not mistaken for tested models. - skipped = [item for item in summaries if item.get("not_attempted")] - if skipped: - out += ["## Not attempted", ""] - for item in skipped: - out.append(f"**{item.get('task', '?')} / {item.get('platform', '?')}**") - out.append("") - for model in item["not_attempted"]: - out.append(f"- {model}") - out.append("") - - # Not a problem, but worth stating: these models were tested with one - # round instead of two, by design rather than by accident. - single_turn = [item for item in summaries if item.get("single_turn")] - if single_turn: - out += [ - "## Single-turn models", - "", - "Follow-up round skipped by design; refusing a second turn is " - "correct behaviour for these.", - "", - ] - for item in single_turn: - listed = ", ".join(f"`{m}`" for m in item["single_turn"]) - out.append( - f"- **{item.get('task', '?')} / {item.get('platform', '?')}**: {listed}" - ) - out.append("") - - # Anything the sweep decided on its own, with the reason it decided it. - # Routine start/stop is omitted; a restart is never routine. - NOISE = {"server_start", "server_stop"} - eventful = [ - (item, [e for e in item.get("events", []) if e.get("event") not in NOISE]) - for item in summaries - ] - eventful = [(item, events) for item, events in eventful if events] - if eventful: - out += ["## Server events", ""] - for item, events in eventful: - out.append(f"**{item.get('task', '?')} / {item.get('platform', '?')}**") - out.append("") - out += ["| At | Event | Reason | Detail |", "| --- | --- | --- | --- |"] - for event in events: - detail = " ".join(str(event.get("detail", "")).split()) - models = event.get("models") or event.get("model") - if models: - listed = models if isinstance(models, str) else ", ".join(models) - detail = f"{detail} ({listed})" if detail else listed - out.append( - "| +{}s | {} | {} | {} |".format( - event.get("elapsed_s", "?"), - event.get("event", "?"), - event.get("reason", ""), - detail, - ) - ) - out.append("") - out.append(f"Full server output: `{item.get('server_log', 'server log')}`") - out.append("") - - print("\n".join(out)) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/script/sweep_vision.py b/.github/script/sweep_vision.py deleted file mode 100644 index 6d00c350d..000000000 --- a/.github/script/sweep_vision.py +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env python3 -"""Sweep every vision-capable model with a two-image conversation. - -Both test images go in the first message, so multi-image prompts are covered -as well as basic image understanding. The follow-up asks the model to relate -the two, which only works if the image context survived the first turn. -""" - -from __future__ import annotations - -import base64 -import sys -import time -from pathlib import Path - -from sweep_common import SweepTask, run_task - -PROMPT = "Describe these two images in detail." -FOLLOWUP = "Write a short story that connects the two images together." - -MIME_BY_SUFFIX = { - ".jpg": "image/jpeg", - ".jpeg": "image/jpeg", - ".png": "image/png", - ".webp": "image/webp", - ".gif": "image/gif", - ".bmp": "image/bmp", -} - - -class VisionSweep(SweepTask): - name = "vision" - csv_header = [ - "Model", "Round", "Input", - "Reasoning Content", "Output Content", - "Latency (s)", "Status", "Error", - ] - - @staticmethod - def add_arguments(parser) -> None: - parser.add_argument( - "--images", - nargs="+", - default=None, - metavar="PATH", - help="Images to send. Defaults to the two bundled test images.", - ) - - def image_paths(self) -> list[Path]: - if self.args.images: - paths = [Path(p) for p in self.args.images] - missing = [str(p) for p in paths if not p.is_file()] - if missing: - raise FileNotFoundError(f"Image(s) not found: {', '.join(missing)}") - return paths - return [self.asset("test_image1.jpg"), self.asset("test_image2.jpg")] - - @staticmethod - def _as_data_url(path: Path) -> str: - mime = MIME_BY_SUFFIX.get(path.suffix.lower(), "image/jpeg") - encoded = base64.b64encode(path.read_bytes()).decode("utf-8") - return f"data:{mime};base64,{encoded}" - - def select_models(self, catalog: list[dict]) -> list[str]: - vlms = [entry["id"] for entry in catalog if entry["vlm"]] - if not vlms: - # The fallback catalog from /v1/models carries no vlm flag, so an - # empty result there means "unknown", not "none exist". - print( - "[vision] No vision-capable models found. If the flm binary was " - "unavailable, pass --models explicitly." - ) - return vlms - - def _round(self, writer, model: str, round_no: int, - messages: list, prompt: str) -> str | None: - print(f" round {round_no}: {prompt}") - started = time.monotonic() - try: - reasoning, output = self.chat(model, messages, stream=True) - except Exception as exc: - elapsed = f"{time.monotonic() - started:.2f}" - print(f" ERROR: {exc}") - self.record( - writer, - [model, round_no, prompt, "", "", elapsed, "error", str(exc)], - model=model, - error=f"round {round_no}: {exc}", - ) - return None - - elapsed = f"{time.monotonic() - started:.2f}" - self.record( - writer, - [model, round_no, prompt, reasoning, output, elapsed, "ok", ""], - model=model, - ) - print(f" done in {elapsed}s, {len(output)} chars") - breakdown = self.format_usage() - if breakdown: - print(f" {breakdown}") - return output - - def run(self) -> None: - paths = self.image_paths() - print(f"[vision] Images: {', '.join(str(p) for p in paths)}") - image_parts = [ - {"type": "image_url", "image_url": {"url": self._as_data_url(p)}} - for p in paths - ] - - models = self.resolve_models() - handle, writer = self.open_csv() - try: - for model in self.iter_models(models): - print(f"\n--- Vision: {model} ---") - messages = [ - { - "role": "user", - "content": [{"type": "text", "text": PROMPT}] + image_parts, - } - ] - - output = self._round(writer, model, 1, messages, PROMPT) - if output is None: - continue - - if self.is_single_turn(model): - # Expected behaviour for these models, not a defect; see - # SweepTask.is_single_turn. - self.record( - writer, - [model, 2, FOLLOWUP, "", "", "", "skipped", - "single-turn model: follow-up not supported"], - model=model, - ) - print(" round 2 skipped: model is labelled single-turn") - time.sleep(1) - continue - - messages.append({"role": "assistant", "content": output}) - messages.append({"role": "user", "content": FOLLOWUP}) - self._round(writer, model, 2, messages, FOLLOWUP) - time.sleep(1) - finally: - handle.close() - print(f"\nVision sweep complete. Saved to {self.csv_path}") - self.write_summary(models) - - -if __name__ == "__main__": - sys.exit(run_task(VisionSweep)) diff --git a/.github/workflows/asr-test.yml b/.github/workflows/asr-test.yml deleted file mode 100644 index 81e70d4a5..000000000 --- a/.github/workflows/asr-test.yml +++ /dev/null @@ -1,594 +0,0 @@ -name: ASR Test - -# Measures speech recognition on real NPU hardware whenever ASR code changes. -# Nothing is rebuilt here: like the model sweep, this tests the portable builds -# that "Build Linux Portable" and "Build Windows Packages" produce for the same -# commit, default and HRX alike. .github/script/asr_test.py holds the checks -# and .github/script/assets/asr_eval/ the FLEURS evaluation set. -# -# On a pull request, main's latest build runs first on the same machine and -# becomes the baseline. The run fails if the pooled error rate rises by more -# than 1 point, any one set's by more than 5, or the number of runaway -# transcripts grows; the report also shows how far the transcripts moved. -# Pushes to main and the weekly run have no baseline build, so they compare -# against the per-platform rates in asr_eval/expected.json instead, with -# 2 points of slack on the pooled rate (those rates come from one machine per -# platform) and the same 5 points per set. -# -# The trigger paths are the Whisper engine, its NPU kernels and prebuilt NPU -# libraries, the audio decoders, front end and FFT code, the audio front ends of -# the chat models that transcribe, and the server files that host -# /v1/audio/transcriptions. Those server files also hold every other endpoint, -# so any change to them runs this test too. The pull_request and push path -# lists must stay identical. Shared files such as src/CMakeLists.txt and -# lm_config.hpp are left out, since every change touches them. -# -# Because rest_handler.cpp and server.cpp are shared, the `gate` job below looks -# at what a pull request or push really changed before any NPU machine is -# reserved: .github/script/asr_gate.sh runs the test when an ASR-only file -# changed, or when a changed line in those two server files is about audio or -# speech, and skips it otherwise (for example a model family adding a tag to -# the REST handler). The weekly run and manual dispatches always run. -# -# The weekly run catches what no commit changes: Whisper weights republished -# on Hugging Face, and NPU driver or firmware updates on the runners. -# -# A manual dispatch tests the dispatched ref's commit, or a pull request's -# builds when given its number. - -on: - pull_request: - branches: [ main ] - paths: - - 'src/common/whisper/**' - - 'src/include/whisper/**' - - 'src/xclbins/Whisper-V3-Turbo-NPU2/**' - - 'src/lib/xrt/*whisper_npu*' - - 'src/lib/hrx/*whisper_npu*' - - 'src/common/audio_process_utils/**' - - 'src/include/audio_process_utils/**' - - 'src/common/audio/**' - - 'src/include/audio/**' - - 'src/common/AutoModel/*_audio.cpp' - - 'src/detail/gemma4e_npu/gemma4e_audio*' - - 'src/include/pocketfft/**' - - 'src/include/dr_libs/**' - - 'src/cmake/FlmAudioCodecs.cmake' - - 'src/server/rest_handler.cpp' - - 'src/server/server.cpp' - - 'src/server/multipart.cpp' - - 'src/server/multipart.hpp' - - 'src/test/whisper_npu/**' - - '.github/script/asr_test.py' - - '.github/script/asr_summary.py' - - '.github/script/asr_gate.sh' - - '.github/script/sweep_common.py' - - '.github/script/assets/asr_eval/**' - - '.github/workflows/asr-test.yml' - push: - branches: [ main ] - paths: - - 'src/common/whisper/**' - - 'src/include/whisper/**' - - 'src/xclbins/Whisper-V3-Turbo-NPU2/**' - - 'src/lib/xrt/*whisper_npu*' - - 'src/lib/hrx/*whisper_npu*' - - 'src/common/audio_process_utils/**' - - 'src/include/audio_process_utils/**' - - 'src/common/audio/**' - - 'src/include/audio/**' - - 'src/common/AutoModel/*_audio.cpp' - - 'src/detail/gemma4e_npu/gemma4e_audio*' - - 'src/include/pocketfft/**' - - 'src/include/dr_libs/**' - - 'src/cmake/FlmAudioCodecs.cmake' - - 'src/server/rest_handler.cpp' - - 'src/server/server.cpp' - - 'src/server/multipart.cpp' - - 'src/server/multipart.hpp' - - 'src/test/whisper_npu/**' - - '.github/script/asr_test.py' - - '.github/script/asr_summary.py' - - '.github/script/asr_gate.sh' - - '.github/script/sweep_common.py' - - '.github/script/assets/asr_eval/**' - - '.github/workflows/asr-test.yml' - schedule: - - cron: '23 4 * * 1' # Mondays 04:23 UTC - workflow_dispatch: - inputs: - pr: - description: "Pull request number whose builds to test (blank = the dispatched ref)" - type: string - default: "" - require: - description: "Feature groups that must pass, space-separated (blank = the default below)" - type: string - default: "" - max_drift: - description: "Fail a set whose transcripts differ from the baseline's by more than this fraction (blank = report only)" - type: string - default: "" - -permissions: - contents: read - # Needed to list and download artifacts from the build workflow runs. - actions: read - -# Only the newest commit on a ref is worth an NPU machine. The event name keeps -# the weekly run and a push to main from cancelling each other. -concurrency: - group: asr-test-${{ github.ref }}-${{ github.event_name }}-${{ inputs.pr }} - cancel-in-progress: true - -env: - # Feature groups main passes on every platform, so a pull request that - # breaks one fails. Add a group in the pull request that makes main pass it: - # window_tail, language, quiet_log. The rest are measured and reported as - # warnings. window_tail is not here yet because main's Linux HRX build loops - # on the long-form file; a pull request that adds repeats is still caught by - # the baseline check. - ASR_REQUIRE: ${{ inputs.require || 'prompt verbose_json timestamp_rules' }} - ASR_MAX_DRIFT: ${{ inputs.max_drift || '' }} - -jobs: - # Cheap hosted-runner check that the change is about ASR (see the header). Every - # later job hangs off resolve-build, so skipping that one skips the whole run, - # including the summary. Anything the script cannot decide counts as "run". - gate: - runs-on: ubuntu-latest - timeout-minutes: 10 - outputs: - run: ${{ steps.gate.outputs.run }} - steps: - - name: Checkout (history without file contents) - if: github.event_name == 'pull_request' || github.event_name == 'push' - uses: actions/checkout@v4 - with: - fetch-depth: 0 - filter: blob:none - - name: Decide - id: gate - env: - EVENT: ${{ github.event_name }} - PR_BASE: ${{ github.event.pull_request.base.sha }} - PR_HEAD: ${{ github.event.pull_request.head.sha }} - PUSH_BEFORE: ${{ github.event.before }} - PUSH_AFTER: ${{ github.sha }} - run: | - case "$EVENT" in - pull_request) range="${PR_BASE}...${PR_HEAD}" ;; - push) range="${PUSH_BEFORE}..${PUSH_AFTER}" ;; - *) range="" ;; # schedule, workflow_dispatch: always run - esac - bash .github/script/asr_gate.sh "$range" | tee -a "$GITHUB_OUTPUT" - - # Waits on a hosted runner for the build runs under test, the same way the - # model sweep's resolve-build does, and pins each to a run id so a stale - # artifact from an earlier push can never be tested. For a pull request it - # also finds main's latest successful build as the baseline. Outputs one - # JSON list of builds per platform, which the test jobs expand into their - # matrix. - resolve-build: - needs: gate - if: needs.gate.outputs.run == 'true' - runs-on: ubuntu-latest - timeout-minutes: 90 - outputs: - linux_run_id: ${{ steps.resolve.outputs.linux_run_id }} - windows_run_id: ${{ steps.resolve.outputs.windows_run_id }} - linux_base_run_id: ${{ steps.resolve.outputs.linux_base_run_id }} - windows_base_run_id: ${{ steps.resolve.outputs.windows_base_run_id }} - linux_builds: ${{ steps.resolve.outputs.linux_builds }} - windows_builds: ${{ steps.resolve.outputs.windows_builds }} - - steps: - - name: Wait for the portable builds - id: resolve - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_NUMBER: ${{ inputs.pr }} - # A pull_request run records the PR head commit in head_sha, while - # github.sha is the synthetic merge commit. Try both. On a push, a - # dispatch or the weekly run only the second is set; for the weekly - # run it is main's latest commit, built by that commit's push. - CANDIDATE_SHAS: ${{ github.event.pull_request.head.sha }} ${{ github.sha }} - WANT_BASELINE: ${{ github.event_name == 'pull_request' || inputs.pr != '' }} - run: | - set -euo pipefail - - POLL_SECONDS=20 - APPEAR_DEADLINE=$(( SECONDS + 10 * 60 )) - FINISH_DEADLINE=$(( SECONDS + 80 * 60 )) - linux_builds='[]' - windows_builds='[]' - - if [ -n "${PR_NUMBER}" ]; then - CANDIDATE_SHAS=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq '.head.sha') - echo "Testing pull request #${PR_NUMBER} at ${CANDIDATE_SHAS}." - fi - - # Newest run for whichever candidate SHA matches, or empty. - find_run() { - local workflow="$1" sha run_id - for sha in ${CANDIDATE_SHAS}; do - run_id=$(gh api \ - "repos/${GITHUB_REPOSITORY}/actions/workflows/${workflow}/runs?head_sha=${sha}&per_page=100" \ - --jq 'if (.workflow_runs | length) == 0 then "" else (.workflow_runs | sort_by(.run_number) | last | .id) end') - if [ -n "${run_id}" ]; then - printf '%s' "${run_id}" - return 0 - fi - done - } - - list_artifacts() { - gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/$1/artifacts?per_page=100" \ - --jq '.artifacts[] | select(.expired | not) | .name' - } - - # main's newest successful push build that still has its artifacts. - # Sets base_run_id and base_artifacts, both empty when there is none. - # Not called in $(...), which would lose the assignments. - base_run_id="" - base_artifacts="" - find_base() { - local workflow="$1" run_id - base_run_id="" - base_artifacts="" - for run_id in $(gh api \ - "repos/${GITHUB_REPOSITORY}/actions/workflows/${workflow}/runs?branch=main&event=push&status=success&per_page=10" \ - --jq '.workflow_runs[].id'); do - base_artifacts=$(list_artifacts "${run_id}") - if [ -n "${base_artifacts}" ]; then - base_run_id="${run_id}" - return 0 - fi - done - } - - # Appends {artifact, base_artifact, platform} to the named list when - # the run published an artifact matching the pattern. An artifact - # exists only if its build job got all the way through, so a failed - # HRX build drops just the HRX test. base_artifact is empty when - # there is no baseline build of that kind. - add_build() { - local list="$1" artifacts="$2" pattern="$3" platform="$4" name base - name=$(printf '%s\n' "${artifacts}" | grep -E "${pattern}" | head -n 1 || true) - if [ -z "${name}" ]; then - echo "::warning::No ${platform} portable artifact was published; skipping that test." - return 0 - fi - base=$(printf '%s\n' "${base_artifacts}" | grep -E "${pattern}" | head -n 1 || true) - echo " ${platform}: ${name} (baseline: ${base:-none})" - printf -v "${list}" '%s' "$(jq -c --arg a "${name}" --arg b "${base}" --arg p "${platform}" \ - '. + [{artifact: $a, base_artifact: $b, platform: $p}]' <<< "${!list}")" - } - - wait_for_build() { - local workflow="$1" platform="$2" default_pattern="$3" hrx_pattern="$4" - local run_id="" status artifacts - - while :; do - run_id=$(find_run "${workflow}") - [ -n "${run_id}" ] && break - if [ "${SECONDS}" -ge "${APPEAR_DEADLINE}" ]; then - echo "::warning::No ${workflow} run appeared for this commit; skipping ${platform}." - return 1 - fi - echo "Waiting for a ${workflow} run to appear..." - sleep "${POLL_SECONDS}" - done - - while :; do - status=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}" --jq '.status') - [ "${status}" = "completed" ] && break - if [ "${SECONDS}" -ge "${FINISH_DEADLINE}" ]; then - echo "::warning::${workflow} run ${run_id} is still ${status}; skipping ${platform}." - return 1 - fi - echo "${workflow} run ${run_id} is ${status}; checking again in ${POLL_SECONDS}s." - sleep "${POLL_SECONDS}" - done - - artifacts=$(list_artifacts "${run_id}") - echo "${workflow} run ${run_id} published:" - printf '%s\n' "${artifacts}" | sed 's/^/ /' - echo "${platform}_run_id=${run_id}" >> "$GITHUB_OUTPUT" - - base_artifacts="" - if [ "${WANT_BASELINE}" = "true" ]; then - find_base "${workflow}" - if [ -n "${base_run_id}" ]; then - echo "Baseline: main's ${workflow} run ${base_run_id}." - echo "${platform}_base_run_id=${base_run_id}" >> "$GITHUB_OUTPUT" - else - echo "::warning::No ${workflow} build of main has artifacts left; ${platform} runs without a baseline." - fi - fi - - # Fully anchored, so neither pattern matches the other's artifact, - # the models-from-source one, or "fastflowlm-windows-msi-". - add_build "${platform}_builds" "${artifacts}" "${default_pattern}" "${platform}" - add_build "${platform}_builds" "${artifacts}" "${hrx_pattern}" "${platform}-hrx" - } - - wait_for_build debian-portable.yml linux \ - '^portable$' '^HRX-portable$' || true - wait_for_build windows-build.yml windows \ - '^fastflowlm-windows-[0-9a-f]+$' '^HRX-fastflowlm-windows-[0-9a-f]+$' || true - - { - echo "linux_builds=${linux_builds}" - echo "windows_builds=${windows_builds}" - } >> "$GITHUB_OUTPUT" - - # One missing platform is a warning above; nothing at all to test - # would otherwise leave a green check that tested nothing. - if [ "${linux_builds}" = "[]" ] && [ "${windows_builds}" = "[]" ]; then - # The Linux portable artifact is kept for 7 days, so a weekly run - # after a quiet week on main can find nothing left to download. - echo "::error::No portable build was found, so nothing was tested. Build artifacts expire, so for an older commit, rerun the build workflows first." - exit 1 - fi - - asr-linux: - name: asr (${{ matrix.build.platform }}) - needs: resolve-build - if: needs.resolve-build.outputs.linux_builds != '[]' - runs-on: [self-hosted, linux, x64, npu, sweep] - timeout-minutes: 120 - - strategy: - fail-fast: false - matrix: - build: ${{ fromJSON(needs.resolve-build.outputs.linux_builds) }} - - env: - # Weights are expected at $FLM_MODEL_PATH/models/ - FLM_MODEL_PATH: /scratch - - steps: - # Only the scripts and their test assets; this job compiles nothing. - - name: Checkout test scripts - uses: actions/checkout@v4 - with: - sparse-checkout: .github/script - filter: blob:none - - - name: Download ${{ matrix.build.platform }} portable build - uses: dawidd6/action-download-artifact@v25 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - run_id: ${{ needs.resolve-build.outputs.linux_run_id }} - name: ${{ matrix.build.artifact }} - path: build-artifact - if_no_artifact_found: fail - - - name: Download baseline build - if: matrix.build.base_artifact != '' - uses: dawidd6/action-download-artifact@v25 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - run_id: ${{ needs.resolve-build.outputs.linux_base_run_id }} - name: ${{ matrix.build.base_artifact }} - path: base-artifact - if_no_artifact_found: fail - - - name: Unpack portable builds - run: | - set -euo pipefail - for build in build base; do - [ -d "${build}-artifact" ] || continue - mkdir -p "flm-${build}" - tarball=$(ls "${build}"-artifact/*_linux.tar.gz) - echo "Using tarball: ${tarball}" - tar xzf "${tarball}" -C "flm-${build}" - chmod +x "flm-${build}/flm" "flm-${build}/flm-real" - "flm-${build}/flm" version - done - echo "FLM_BIN=${GITHUB_WORKSPACE}/flm-build/flm" >> "$GITHUB_ENV" - - # No-ops once the weights are on the runner. - - name: Pull models - run: | - set -euo pipefail - "${FLM_BIN}" pull whisper-v3:turbo - "${FLM_BIN}" pull gemma4-it:e2b - "${FLM_BIN}" pull gemma4-it:12b - - - name: Set up Python virtualenv - run: | - set -euo pipefail - python3 -m venv .venv - .venv/bin/python -m pip install --upgrade pip - for attempt in 1 2 3; do - .venv/bin/python -m pip install --no-cache-dir openai && break - if [ "${attempt}" -eq 3 ]; then - exit 1 - fi - done - - # The baseline's own failures are reported in its results, not here: - # what matters is how the build under test compares with it. - - name: Run ASR test on the baseline - if: matrix.build.base_artifact != '' - run: | - .venv/bin/python .github/script/asr_test.py \ - --flm-bin "${GITHUB_WORKSPACE}/flm-base/flm" \ - --platform "${{ matrix.build.platform }}" \ - --output-dir asr-baseline || true - - - name: Run ASR test - run: | - set -euo pipefail - args=(--flm-bin "${FLM_BIN}" --platform "${{ matrix.build.platform }}" - --output-dir asr-results) - baseline=$(ls asr-baseline/*_results.json 2>/dev/null || true) - if [ -n "${baseline}" ]; then - args+=(--baseline "${baseline}") - else - # Expected rates come from one machine, so the pooled rate gets - # 2 points of slack instead of 1; sets keep their 5 points. - args+=(--expected .github/script/assets/asr_eval/expected.json --tolerance 2) - fi - [ -n "${ASR_REQUIRE}" ] && args+=(--require ${ASR_REQUIRE}) - [ -n "${ASR_MAX_DRIFT}" ] && args+=(--max-drift "${ASR_MAX_DRIFT}") - .venv/bin/python .github/script/asr_test.py "${args[@]}" - - - name: Upload results - if: always() - uses: actions/upload-artifact@v4 - with: - name: asr-test-${{ matrix.build.platform }}-${{ github.run_number }} - path: | - asr-results/ - asr-baseline/ - if-no-files-found: warn - retention-days: 30 - - asr-windows: - name: asr (${{ matrix.build.platform }}) - needs: resolve-build - if: needs.resolve-build.outputs.windows_builds != '[]' - runs-on: [self-hosted, windows, x64, npu, sweep] - timeout-minutes: 120 - - strategy: - fail-fast: false - matrix: - build: ${{ fromJSON(needs.resolve-build.outputs.windows_builds) }} - - steps: - - name: Checkout test scripts - uses: actions/checkout@v4 - with: - sparse-checkout: .github/script - filter: blob:none - - - name: Resolve model path - shell: powershell - run: | - $modelPath = Join-Path $env:USERPROFILE ".flm" - New-Item -ItemType Directory -Force $modelPath | Out-Null - "FLM_MODEL_PATH=$modelPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - - # The artifacts hold flm.exe and its DLLs at the top level. - - name: Download ${{ matrix.build.platform }} portable build - uses: dawidd6/action-download-artifact@v25 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - run_id: ${{ needs.resolve-build.outputs.windows_run_id }} - name: ${{ matrix.build.artifact }} - path: flm-build - if_no_artifact_found: fail - - - name: Download baseline build - if: matrix.build.base_artifact != '' - uses: dawidd6/action-download-artifact@v25 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - run_id: ${{ needs.resolve-build.outputs.windows_base_run_id }} - name: ${{ matrix.build.base_artifact }} - path: flm-base - if_no_artifact_found: fail - - - name: Verify FLM binaries run - shell: powershell - run: | - $ErrorActionPreference = "Stop" - foreach ($build in "flm-build", "flm-base") { - $flm = Join-Path $env:GITHUB_WORKSPACE "$build\flm.exe" - if (Test-Path $flm) { & $flm version } - elseif ($build -eq "flm-build") { throw "flm.exe not found at $flm" } - } - "FLM_BIN=$(Join-Path $env:GITHUB_WORKSPACE 'flm-build\flm.exe')" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - - # No-ops once the weights are on the runner. - - name: Pull models - shell: powershell - run: | - foreach ($model in "whisper-v3:turbo", "gemma4-it:e2b", "gemma4-it:12b") { - & $env:FLM_BIN pull $model - # A native command's exit code does not trip ErrorActionPreference. - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - } - - - name: Set up Python virtualenv - shell: powershell - run: | - $ErrorActionPreference = "Stop" - python -m venv .venv - .\.venv\Scripts\python.exe -m pip install --upgrade pip - .\.venv\Scripts\python.exe -m pip install openai - - # See the Linux job: the baseline's own failures do not fail this job. - - name: Run ASR test on the baseline - if: matrix.build.base_artifact != '' - shell: powershell - run: | - & ".\.venv\Scripts\python.exe" .github\script\asr_test.py ` - --flm-bin (Join-Path $env:GITHUB_WORKSPACE "flm-base\flm.exe") ` - --platform "${{ matrix.build.platform }}" ` - --output-dir asr-baseline - exit 0 - - - name: Run ASR test - shell: powershell - run: | - $ErrorActionPreference = "Stop" - $cmdArgs = @( - ".github\script\asr_test.py", - "--flm-bin", $env:FLM_BIN, - "--platform", "${{ matrix.build.platform }}", - "--output-dir", "asr-results" - ) - $baseline = Get-ChildItem asr-baseline\*_results.json -ErrorAction SilentlyContinue | Select-Object -First 1 - if ($baseline) { - $cmdArgs += @("--baseline", $baseline.FullName) - } else { - # Expected rates come from one machine, so the pooled rate gets - # 2 points of slack instead of 1; sets keep their 5 points. - $cmdArgs += @("--expected", ".github\script\assets\asr_eval\expected.json", "--tolerance", "2") - } - if ($env:ASR_REQUIRE) { $cmdArgs += @("--require") + ($env:ASR_REQUIRE -split '\s+') } - if ($env:ASR_MAX_DRIFT) { $cmdArgs += @("--max-drift", $env:ASR_MAX_DRIFT) } - & ".\.venv\Scripts\python.exe" $cmdArgs - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - - - name: Upload results - if: always() - uses: actions/upload-artifact@v4 - with: - name: asr-test-${{ matrix.build.platform }}-${{ github.run_number }} - path: | - asr-results/ - asr-baseline/ - if-no-files-found: warn - retention-days: 30 - - # One report for the whole run, a section per model with the platforms side - # by side, so it is clear which model failed where. Each job's own report is - # in its artifact. Reporting only, so it stays on a GitHub-hosted runner. - summary: - runs-on: ubuntu-latest - needs: [resolve-build, asr-linux, asr-windows] - if: always() && needs.resolve-build.result == 'success' - steps: - - name: Checkout test scripts - uses: actions/checkout@v4 - with: - sparse-checkout: .github/script - filter: blob:none - - - name: Download results - uses: actions/download-artifact@v4 - continue-on-error: true - with: - pattern: asr-test-*-${{ github.run_number }} - path: asr-artifacts - - - name: Write job summary - run: | - set -euo pipefail - python3 .github/script/asr_summary.py asr-artifacts >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml new file mode 100644 index 000000000..d7a511d22 --- /dev/null +++ b/.github/workflows/build-and-test.yml @@ -0,0 +1,528 @@ +name: Build and Test + +# Builds every FastFlowLM package and tests each one on the NPU runners as soon +# as its own build finishes. +# +# Linux XRT portable ─ test HRX portable ─ test Source portable ─ test +# XRT .deb ─ install test (off) HRX .deb ─ install test (off) +# Windows XRT zip ─ test HRX zip ─ test Source zip ─ test +# XRT MSI ─ install test (off) +# +# Each package is its own build call, so its tests do not wait on the other +# packages' builds, and XRT and HRX are tested side by side. HRX skips the api +# and serving suites: they test server logic the backend does not touch, and +# the XRT run covers it. +# +# A pull request runs only the suites its changes can affect (an ASR-only +# change runs asr, a d3-only change decision); see .github/script/ci/changes.py. +# +# Tiers (.github/script/npu_test/plan.py): pull requests and pushes run the pr +# tier, one model per engine family plus every model of a family the change +# touches; the nightly run tests every model. A change that only touches docs +# builds and tests nothing. +# +# release.yml calls this with release: true before publishing a tag. +on: + push: + branches: + - main + pull_request: + branches: + - main + schedule: + - cron: '17 6 * * *' # nightly, full tier + workflow_dispatch: + inputs: + tier: + description: Test tier + type: choice + options: [pr, full, install] + default: pr + workflow_call: + inputs: + tier: + description: Test tier (install, pr or full). + type: string + default: pr + release: + description: Build and test only what a release ships (no models-from-source). + type: boolean + default: false + outputs: + xrt-passed: + description: '"true" when every XRT package built and passed its tests.' + value: ${{ jobs.summary.outputs.xrt-passed }} + hrx-passed: + description: '"true" when every HRX package built and passed its tests.' + value: ${{ jobs.summary.outputs.hrx-passed }} + deb-distros: + description: How many distributions each .deb variant was built for. + value: ${{ jobs.build-lin-xrt-deb.outputs.deb_distros }} + +# A superseded pull request commit only holds the NPU runners up, so a new +# push cancels its run. Pushes to main, nightly runs and tags each get their +# own group and always finish. +concurrency: + group: build-and-test-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + changes: + runs-on: ubuntu-latest + outputs: + run: ${{ steps.detect.outputs.run }} + families: ${{ steps.detect.outputs.families }} + asr-full: ${{ steps.detect.outputs.asr-full }} + tier: ${{ steps.detect.outputs.tier }} + short-sha: ${{ steps.detect.outputs.short-sha }} + suites: ${{ steps.detect.outputs.suites }} + hrx-suites: ${{ steps.detect.outputs.hrx-suites }} + source: ${{ steps.detect.outputs.source }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + filter: blob:none + + - name: Detect changes + id: detect + uses: ./.github/actions/detect-changes + with: + tier: ${{ inputs.tier }} + + # ── Linux builds ───────────────────────────────────────────────────────── + + build-lin-xrt-portable: + name: Linux XRT portable + needs: changes + if: needs.changes.outputs.run == 'true' + uses: ./.github/workflows/lin-build.yml + with: + variants: XRT-portable + + build-lin-hrx-portable: + name: Linux HRX portable + needs: changes + if: needs.changes.outputs.run == 'true' + uses: ./.github/workflows/lin-build.yml + with: + variants: HRX-portable + + build-lin-source-portable: + name: Linux models-from-source portable + needs: changes + if: needs.changes.outputs.run == 'true' && !inputs.release + uses: ./.github/workflows/lin-build.yml + with: + variants: Source-portable + + build-lin-xrt-deb: + name: Linux XRT .deb + needs: changes + if: needs.changes.outputs.run == 'true' + uses: ./.github/workflows/lin-build.yml + with: + variants: XRT-deb + + build-lin-hrx-deb: + name: Linux HRX .deb + needs: changes + if: needs.changes.outputs.run == 'true' + uses: ./.github/workflows/lin-build.yml + with: + variants: HRX-deb + + # ── Windows builds ─────────────────────────────────────────────────────── + + build-win-xrt: + name: Windows XRT + needs: changes + if: needs.changes.outputs.run == 'true' + uses: ./.github/workflows/win-build.yml + with: + variants: XRT + msi: true + + build-win-hrx: + name: Windows HRX + needs: changes + if: needs.changes.outputs.run == 'true' + uses: ./.github/workflows/win-build.yml + with: + variants: HRX + msi: false + + build-win-source: + name: Windows models-from-source + needs: changes + if: needs.changes.outputs.run == 'true' && !inputs.release + uses: ./.github/workflows/win-build.yml + with: + variants: Source + msi: false + + # ── Linux tests ────────────────────────────────────────────────────────── + # Only the test scripts and their assets are checked out, never the sources: + # these jobs run a prebuilt package and compile nothing. + + test-lin-xrt-portable: + name: Test Linux XRT portable + needs: [changes, build-lin-xrt-portable] + runs-on: [self-hosted, linux, x64, npu, sweep] + timeout-minutes: 120 + steps: + - &checkout-tests + name: Checkout test scripts + uses: actions/checkout@v4 + with: + sparse-checkout: | + .github/actions + .github/script + src/test/openai_api_test + src/test/openai_usage + src/test/d3_video/fixtures + filter: blob:none + - name: Download package + uses: actions/download-artifact@v4 + with: + name: portable + path: package + - name: Prepare package + id: package + uses: ./.github/actions/install-package + with: + kind: portable + path: package + - name: Test + uses: ./.github/actions/npu-test + with: + flm-bin: ${{ steps.package.outputs.flm-bin }} + label: linux-xrt-portable + tier: ${{ needs.changes.outputs.tier }} + suites: ${{ needs.changes.outputs.suites }} + families: ${{ needs.changes.outputs.families }} + asr-full: ${{ needs.changes.outputs.asr-full }} + asr-expected-key: linux + + test-lin-hrx-portable: + name: Test Linux HRX portable + needs: [changes, build-lin-hrx-portable] + if: needs.changes.outputs.hrx-suites != '' + runs-on: [self-hosted, linux, x64, npu, sweep] + timeout-minutes: 120 + steps: + - *checkout-tests + - name: Download package + uses: actions/download-artifact@v4 + with: + name: HRX-portable + path: package + - name: Prepare package + id: package + uses: ./.github/actions/install-package + with: + kind: portable + path: package + - name: Test + uses: ./.github/actions/npu-test + with: + flm-bin: ${{ steps.package.outputs.flm-bin }} + label: linux-hrx-portable + tier: ${{ needs.changes.outputs.tier }} + suites: ${{ needs.changes.outputs.hrx-suites }} + families: ${{ needs.changes.outputs.families }} + asr-full: ${{ needs.changes.outputs.asr-full }} + asr-expected-key: linux-hrx + + # Only the engines differ from the XRT portable build, and only the Gemma 4 + # engines are built from source so far. + test-lin-source-portable: + name: Test Linux models-from-source portable + needs: [changes, build-lin-source-portable] + if: needs.changes.outputs.source == 'true' + runs-on: [self-hosted, linux, x64, npu, sweep] + timeout-minutes: 60 + steps: + - *checkout-tests + - name: Download package + uses: actions/download-artifact@v4 + with: + name: models-from-source-portable + path: package + - name: Prepare package + id: package + uses: ./.github/actions/install-package + with: + kind: portable + path: package + - name: Test + uses: ./.github/actions/npu-test + with: + flm-bin: ${{ steps.package.outputs.flm-bin }} + label: linux-source-portable + tier: ${{ needs.changes.outputs.tier }} + suites: chat + models: gemma4-it:e2b,gemma4-it:e4b + + # The .deb and MSI install tests are off unless the repository variable + # FLM_INSTALL_TESTS is "true": the runners cannot install yet (no + # passwordless sudo on Linux, an FLM already installed on Windows), so they + # only tested the unpacked payload. When turning them on, also add them to + # the summary's verdict, so a release waits for them. + # + # The NPU runners run Ubuntu 24.04, so that is the .deb that can be + # installed; the other distributions' builds are covered by their build. + install-lin-xrt-deb: + name: Install test Linux XRT .deb + needs: [changes, build-lin-xrt-deb] + if: vars.FLM_INSTALL_TESTS == 'true' + runs-on: [self-hosted, linux, x64, npu, sweep] + timeout-minutes: 60 + steps: + - *checkout-tests + - name: Download package + uses: actions/download-artifact@v4 + with: + name: flm-ubuntu24.04-${{ needs.changes.outputs.short-sha }}-deb + path: package + - name: Install package + id: package + uses: ./.github/actions/install-package + with: + kind: deb + path: package + - name: Test + uses: ./.github/actions/npu-test + with: + flm-bin: ${{ steps.package.outputs.flm-bin }} + label: linux-xrt-deb-${{ steps.package.outputs.mode }} + tier: install + - name: Remove package + if: always() + uses: ./.github/actions/remove-package + with: + kind: deb + mode: ${{ steps.package.outputs.mode }} + + install-lin-hrx-deb: + name: Install test Linux HRX .deb + needs: [changes, build-lin-hrx-deb] + if: vars.FLM_INSTALL_TESTS == 'true' + runs-on: [self-hosted, linux, x64, npu, sweep] + timeout-minutes: 60 + steps: + - *checkout-tests + - name: Download package + uses: actions/download-artifact@v4 + with: + name: HRX-flm-ubuntu24.04-${{ needs.changes.outputs.short-sha }}-deb + path: package + - name: Install package + id: package + uses: ./.github/actions/install-package + with: + kind: deb + path: package + - name: Test + uses: ./.github/actions/npu-test + with: + flm-bin: ${{ steps.package.outputs.flm-bin }} + label: linux-hrx-deb-${{ steps.package.outputs.mode }} + tier: install + - name: Remove package + if: always() + uses: ./.github/actions/remove-package + with: + kind: deb + mode: ${{ steps.package.outputs.mode }} + + # ── Windows tests ──────────────────────────────────────────────────────── + + test-win-xrt: + name: Test Windows XRT + needs: [changes, build-win-xrt] + runs-on: [self-hosted, windows, x64, npu, sweep] + timeout-minutes: 120 + steps: + - *checkout-tests + - name: Download package + uses: actions/download-artifact@v4 + with: + name: fastflowlm-windows-${{ github.sha }} + path: package + - name: Prepare package + id: package + uses: ./.github/actions/install-package + with: + kind: portable + path: package + - name: Test + uses: ./.github/actions/npu-test + with: + flm-bin: ${{ steps.package.outputs.flm-bin }} + label: windows-xrt + tier: ${{ needs.changes.outputs.tier }} + suites: ${{ needs.changes.outputs.suites }} + families: ${{ needs.changes.outputs.families }} + asr-full: ${{ needs.changes.outputs.asr-full }} + asr-expected-key: windows + + test-win-hrx: + name: Test Windows HRX + needs: [changes, build-win-hrx] + if: needs.changes.outputs.hrx-suites != '' + runs-on: [self-hosted, windows, x64, npu, sweep] + timeout-minutes: 120 + steps: + - *checkout-tests + - name: Download package + uses: actions/download-artifact@v4 + with: + name: HRX-fastflowlm-windows-${{ github.sha }} + path: package + - name: Prepare package + id: package + uses: ./.github/actions/install-package + with: + kind: portable + path: package + - name: Test + uses: ./.github/actions/npu-test + with: + flm-bin: ${{ steps.package.outputs.flm-bin }} + label: windows-hrx + tier: ${{ needs.changes.outputs.tier }} + suites: ${{ needs.changes.outputs.hrx-suites }} + families: ${{ needs.changes.outputs.families }} + asr-full: ${{ needs.changes.outputs.asr-full }} + asr-expected-key: windows-hrx + + test-win-source: + name: Test Windows models-from-source + needs: [changes, build-win-source] + if: needs.changes.outputs.source == 'true' + runs-on: [self-hosted, windows, x64, npu, sweep] + timeout-minutes: 60 + steps: + - *checkout-tests + - name: Download package + uses: actions/download-artifact@v4 + with: + name: models-from-source-fastflowlm-windows-${{ github.sha }} + path: package + - name: Prepare package + id: package + uses: ./.github/actions/install-package + with: + kind: portable + path: package + - name: Test + uses: ./.github/actions/npu-test + with: + flm-bin: ${{ steps.package.outputs.flm-bin }} + label: windows-source + tier: ${{ needs.changes.outputs.tier }} + suites: chat + models: gemma4-it:e2b,gemma4-it:e4b + + install-win-msi: + name: Install test Windows MSI + needs: [changes, build-win-xrt] + if: vars.FLM_INSTALL_TESTS == 'true' + runs-on: [self-hosted, windows, x64, npu, sweep] + timeout-minutes: 60 + steps: + - *checkout-tests + - name: Download package + uses: actions/download-artifact@v4 + with: + name: fastflowlm-windows-msi-${{ github.sha }} + path: package + - name: Install package + id: package + uses: ./.github/actions/install-package + with: + kind: msi + path: package + - name: Test + uses: ./.github/actions/npu-test + with: + flm-bin: ${{ steps.package.outputs.flm-bin }} + label: windows-msi-${{ steps.package.outputs.mode }} + tier: install + - name: Remove package + if: always() + uses: ./.github/actions/remove-package + with: + kind: msi + mode: ${{ steps.package.outputs.mode }} + path: package + + # ── Summary ────────────────────────────────────────────────────────────── + + summary: + needs: + - changes + - build-lin-xrt-portable + - build-lin-hrx-portable + - build-lin-xrt-deb + - build-lin-hrx-deb + - build-win-xrt + - build-win-hrx + - test-lin-xrt-portable + - test-lin-hrx-portable + - test-lin-source-portable + - install-lin-xrt-deb + - install-lin-hrx-deb + - test-win-xrt + - test-win-hrx + - test-win-source + - install-win-msi + if: always() + runs-on: ubuntu-latest + outputs: + xrt-passed: ${{ steps.verdict.outputs.xrt }} + hrx-passed: ${{ steps.verdict.outputs.hrx }} + steps: + - name: Checkout scripts + uses: actions/checkout@v4 + with: + sparse-checkout: .github/script/ci + filter: blob:none + + - name: Download reports + uses: actions/download-artifact@v4 + with: + pattern: npu-test-* + path: reports + + - name: Summarize + run: python3 .github/script/ci/merge_reports.py reports >> "$GITHUB_STEP_SUMMARY" + + # What release.yml publishes on: every package of a backend built and + # passed its tests. A test job skipped because the change did not touch + # its suites counts as passed; one skipped because its build failed does + # not, since the build's own failure is in the list too. + - name: Verdict + id: verdict + env: + XRT: >- + ${{ needs.build-lin-xrt-portable.result }} ${{ needs.build-lin-xrt-deb.result }} + ${{ needs.build-win-xrt.result }} + ${{ needs.test-lin-xrt-portable.result }} ${{ needs.test-win-xrt.result }} + HRX: >- + ${{ needs.build-lin-hrx-portable.result }} ${{ needs.build-lin-hrx-deb.result }} + ${{ needs.build-win-hrx.result }} + ${{ needs.test-lin-hrx-portable.result }} ${{ needs.test-win-hrx.result }} + run: | + verdict() { + for result in $1; do + case "${result}" in success|skipped) ;; *) echo false; return ;; esac + done + echo true + } + echo "xrt=$(verdict "${XRT}")" >> "$GITHUB_OUTPUT" + echo "hrx=$(verdict "${HRX}")" >> "$GITHUB_OUTPUT" + echo "XRT: ${XRT} -> $(verdict "${XRT}")" + echo "HRX: ${HRX} -> $(verdict "${HRX}")" diff --git a/.github/workflows/cpu-tests.yml b/.github/workflows/cpu-tests.yml new file mode 100644 index 000000000..a1a9cf0c2 --- /dev/null +++ b/.github/workflows/cpu-tests.yml @@ -0,0 +1,40 @@ +name: CPU Tests + +# Unit tests that build and run on a plain hosted runner, no NPU. +on: + push: + branches: + - main + tags: + - '*' + pull_request: + branches: + - main + workflow_dispatch: + +jobs: + # The d3 System One contract (src/common/decision) against fixtures written by the upstream + # package: prompt bytes, request/question errors and answer objects. CPU only, no NPU. + d3-contract: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Build and run the d3 contract test + run: | + cmake -S src/test/d3_contract -B build-d3-contract -DCMAKE_BUILD_TYPE=Release + cmake --build build-d3-contract -j"$(nproc)" + ctest --test-dir build-d3-contract --output-on-failure + + # continue_final_message (src/common/chat): where the prompt is cut on the template shapes the + # models ship, and how the output parser reads the continued text. CPU only, no NPU. + chat-continue: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Build and run the chat continue test + run: | + cmake -S src/test/chat_continue -B build-chat-continue -DCMAKE_BUILD_TYPE=Release + cmake --build build-chat-continue -j"$(nproc)" + ctest --test-dir build-chat-continue --output-on-failure diff --git a/.github/workflows/debian-portable.yml b/.github/workflows/debian-portable.yml deleted file mode 100644 index 39e21fb1d..000000000 --- a/.github/workflows/debian-portable.yml +++ /dev/null @@ -1,329 +0,0 @@ -name: Build Linux Portable - -on: - push: - branches: [ main ] - tags: - - '*' - pull_request: - branches: [ main ] - workflow_dispatch: - -jobs: - build-portable: - runs-on: ubuntu-latest - strategy: - matrix: - include: - - distro: "debian" - version: "13" - - container: - image: ghcr.io/rocm/fastflowlm/build-environment:${{ matrix.distro }}${{ matrix.version }} - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - submodules: recursive - fetch-depth: 0 - - - name: Configure git safe directory - run: git config --global --add safe.directory $(pwd) - - - name: Get version from git describe - id: get_version - run: | - VERSION=$(git describe --tags --always | sed 's/^v//') - echo "version=${VERSION}" >> $GITHUB_OUTPUT - - - name: Get short SHA - id: short_sha - run: echo "sha=$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_OUTPUT - - - name: Upgrade Rust toolchain - run: | - apt-get update - apt-get install -y uuid-dev nasm patchelf - apt-get install -y --no-install-recommends curl ca-certificates - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ - sh -s -- -y --profile minimal --default-toolchain stable - echo "$HOME/.cargo/bin" >> $GITHUB_PATH - - - name: Build FLM portable distribution - run: | - cd src - cmake --preset linux-portable - cmake --build build -j$(nproc) - - - name: Create installation package - run: | - cd src/build - DESTDIR=$PWD/install cmake --install . - - # Rename binary and wrapper for portable layout - mv install/opt/fastflowlm/flm install/opt/fastflowlm/flm-real - mv install/opt/fastflowlm/flm-wrapper.sh install/opt/fastflowlm/flm - chmod +x install/opt/fastflowlm/flm - - # Strip debug symbols to reduce binary size (111MB → 33MB) - # --strip-debug removes debug info while keeping stack trace capability - strip --strip-debug install/opt/fastflowlm/flm-real - - - name: Create portable tarball - env: - VERSION: ${{ steps.get_version.outputs.version }} - run: | - cd src/build/install/opt/fastflowlm - tar czf $GITHUB_WORKSPACE/fastflowlm_${VERSION}_linux.tar.gz \ - flm \ - flm-real \ - lib/ \ - xclbins/ \ - model_list.json \ - model_info.json \ - LICENSE_RUNTIME.txt \ - LICENSE_BINARY.txt \ - THIRD_PARTY_NOTICES.txt - echo "Created tarball with symlinks preserved" - # The default backend is XRT, so confirm libxrt is bundled. - tar tzf $GITHUB_WORKSPACE/fastflowlm_${VERSION}_linux.tar.gz | grep libxrt - $GITHUB_WORKSPACE/.github/script/check_no_copyleft_libs.sh \ - $GITHUB_WORKSPACE/fastflowlm_${VERSION}_linux.tar.gz - - - name: Upload portable build artifact - uses: actions/upload-artifact@v4 - with: - name: portable - path: fastflowlm_${{ steps.get_version.outputs.version }}_linux.tar.gz - retention-days: 7 - if-no-files-found: error - - - name: Build models from source - run: | - cd src - cmake --preset linux-portable \ - -B build-models-from-source \ - -DFLM_BUILD_GEMMA4E=ON - cmake --build build-models-from-source --parallel 2 - - - name: Create models-from-source package - env: - VERSION: ${{ steps.get_version.outputs.version }} - run: | - cd src/build-models-from-source - rm -rf install-models-from-source - DESTDIR=$PWD/install-models-from-source cmake --install . - - package_root=install-models-from-source/opt/fastflowlm - mv "${package_root}/flm" "${package_root}/flm-real" - mv "${package_root}/flm-wrapper.sh" "${package_root}/flm" - chmod +x "${package_root}/flm" - strip --strip-debug "${package_root}/flm-real" - - staged_engine="${package_root}/lib/libgemma4e_npu.so" - test -f "${staged_engine}" - if cmp -s "$GITHUB_WORKSPACE/src/lib/xrt/libgemma4e_npu.so" "${staged_engine}"; then - echo "The staged Gemma 4 engine matches the prebuilt library." >&2 - exit 1 - fi - readelf -n "${staged_engine}" | grep -A1 'Build ID' - tar czf "$GITHUB_WORKSPACE/models-from-source_fastflowlm_${VERSION}_linux.tar.gz" \ - -C "${package_root}" \ - flm \ - flm-real \ - lib/ \ - xclbins/ \ - model_list.json \ - model_info.json \ - LICENSE_RUNTIME.txt \ - LICENSE_BINARY.txt \ - THIRD_PARTY_NOTICES.txt - "$GITHUB_WORKSPACE/.github/script/check_no_copyleft_libs.sh" \ - "$GITHUB_WORKSPACE/models-from-source_fastflowlm_${VERSION}_linux.tar.gz" - - - name: Upload models-from-source build artifact - uses: actions/upload-artifact@v4 - with: - name: models-from-source-portable - path: models-from-source_fastflowlm_${{ steps.get_version.outputs.version }}_linux.tar.gz - retention-days: 7 - if-no-files-found: error - - build-HRX-portable: - runs-on: ubuntu-latest - strategy: - matrix: - include: - - distro: "debian" - version: "13" - - container: - image: ghcr.io/rocm/fastflowlm/build-environment:${{ matrix.distro }}${{ matrix.version }} - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - submodules: recursive - fetch-depth: 0 - - - name: Configure git safe directory - run: git config --global --add safe.directory $(pwd) - - - name: Get version from git describe - id: get_version - run: | - VERSION=$(git describe --tags --always | sed 's/^v//') - echo "version=${VERSION}" >> $GITHUB_OUTPUT - - - name: Upgrade Rust toolchain - run: | - apt-get update - apt-get install -y uuid-dev nasm patchelf zstd - apt-get install -y --no-install-recommends curl ca-certificates - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ - sh -s -- -y --profile minimal --default-toolchain stable - echo "$HOME/.cargo/bin" >> $GITHUB_PATH - - - name: Fetch pinned HRX release - id: hrx - run: ./hrx-integration/fetch-hrx-release.sh - - - name: Build FLM HRX portable distribution - run: | - cd src - cmake --preset linux-portable \ - -DFLM_USE_HRX=ON \ - -DCMAKE_PREFIX_PATH="${{ steps.hrx.outputs.prefix }}" - cmake --build build -j$(nproc) - - - name: Create installation package - run: | - cd src/build - DESTDIR=$PWD/install cmake --install . - - # Rename binary and wrapper for portable layout - mv install/opt/fastflowlm/flm install/opt/fastflowlm/flm-real - mv install/opt/fastflowlm/flm-wrapper.sh install/opt/fastflowlm/flm - chmod +x install/opt/fastflowlm/flm - - # Strip debug symbols to reduce binary size (111MB → 33MB) - # --strip-debug removes debug info while keeping stack trace capability - strip --strip-debug install/opt/fastflowlm/flm-real - - - name: Create portable tarball - env: - VERSION: ${{ steps.get_version.outputs.version }} - run: | - cd src/build/install/opt/fastflowlm - tar czf $GITHUB_WORKSPACE/HRX_fastflowlm_${VERSION}_linux.tar.gz \ - flm \ - flm-real \ - lib/ \ - xclbins/ \ - model_list.json \ - model_info.json \ - LICENSE_RUNTIME.txt \ - LICENSE_BINARY.txt \ - THIRD_PARTY_NOTICES.txt - echo "Created tarball with symlinks preserved" - # This build selects the HRX backend, so confirm libhrx is bundled. - tar tzf $GITHUB_WORKSPACE/HRX_fastflowlm_${VERSION}_linux.tar.gz | grep libhrx - $GITHUB_WORKSPACE/.github/script/check_no_copyleft_libs.sh \ - $GITHUB_WORKSPACE/HRX_fastflowlm_${VERSION}_linux.tar.gz - - - name: Upload portable build artifact - uses: actions/upload-artifact@v4 - with: - name: HRX-portable - path: HRX_fastflowlm_${{ steps.get_version.outputs.version }}_linux.tar.gz - retention-days: 7 - if-no-files-found: error - - test-summary: - runs-on: ubuntu-latest - needs: build-portable - if: always() - steps: - - name: Summary - run: | - echo "# Portable Build Test Results" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "✅ FLM built successfully with portable distribution" >> $GITHUB_STEP_SUMMARY - echo "✅ No FFmpeg, GPL codec, FFTW or readline libraries shipped or linked" >> $GITHUB_STEP_SUMMARY - echo "✅ XRT and XDNA plugin bundled with distribution" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "## Build Configuration" >> $GITHUB_STEP_SUMMARY - echo "- **XRT**: Dynamically linked, bundled in lib/" >> $GITHUB_STEP_SUMMARY - echo "- **XDNA plugin**: Bundled in lib/" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "## Usage" >> $GITHUB_STEP_SUMMARY - echo "The binary is portable and includes all necessary libraries in the lib/ directory." >> $GITHUB_STEP_SUMMARY - echo "RPATH is set to \$ORIGIN/../lib for automatic library discovery." >> $GITHUB_STEP_SUMMARY - - release: - permissions: - contents: write - runs-on: ubuntu-latest - needs: build-portable - if: github.ref_type == 'tag' - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Get version from git describe - id: get_version - run: | - VERSION=$(git describe --tags --always | sed 's/^v//') - echo "version=${VERSION}" >> $GITHUB_OUTPUT - - # By name: this run also publishes the models-from-source and HRX - # tarballs, and neither belongs in the default release asset set. - - name: Download portable artifact - uses: actions/download-artifact@v4 - with: - name: portable - path: portable - - - name: Upload Binaries to Release - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - tag: ${{ github.ref }} - file_glob: true - file: portable/fastflowlm_*_linux.tar.gz - overwrite: true - - # The HRX build ships as a preview, so it gets its own release job: a failed - # HRX build leaves the release without the HRX asset instead of holding back - # the default one. - release-hrx: - permissions: - contents: write - runs-on: ubuntu-latest - needs: build-HRX-portable - if: github.ref_type == 'tag' - steps: - - name: Download HRX portable artifact - uses: actions/download-artifact@v4 - with: - name: HRX-portable - path: portable - - - name: Upload HRX Binaries to Release - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - tag: ${{ github.ref }} - file_glob: true - file: portable/HRX_fastflowlm_*_linux.tar.gz - overwrite: true diff --git a/.github/workflows/lin-build.yml b/.github/workflows/lin-build.yml new file mode 100644 index 000000000..3857ae02b --- /dev/null +++ b/.github/workflows/lin-build.yml @@ -0,0 +1,210 @@ +name: Linux Build + +# Builds, packages and uploads one or more Linux variants, each inside the +# fastflowlm build-environment container for its distribution. The artifact +# names are what build-and-test.yml and release.yml download, so they are fixed +# per variant: +# +# XRT-deb flm---deb (every distro) +# HRX-deb HRX-flm---deb (every distro) +# XRT-portable portable (Debian 13) +# HRX-portable HRX-portable (Debian 13) +# Source-portable models-from-source-portable (Debian 13) +# +# Build and Test (build-and-test.yml) calls this once per package on every pull +# request, push and tag; dispatch it to build packages without testing them. +on: + workflow_call: + inputs: + variants: + description: > + XRT-deb, HRX-deb, XRT-portable, HRX-portable, Source-portable, a + comma-separated list of them, or "all". + type: string + default: all + outputs: + deb_distros: + description: How many distributions each .deb variant is built for. + value: ${{ jobs.plan.outputs.deb_distros }} + workflow_dispatch: + inputs: + variants: + description: Variant to build + type: choice + options: [all, XRT-deb, HRX-deb, XRT-portable, HRX-portable, Source-portable] + default: all + +# No concurrency group: build-and-test.yml calls this several times in one +# run, and the calls would cancel each other. The caller owns cancellation. + +jobs: + plan: + runs-on: ubuntu-latest + outputs: + include: ${{ steps.plan.outputs.include }} + deb_distros: ${{ steps.plan.outputs.deb_distros }} + steps: + - name: Expand variants + id: plan + env: + VARIANTS: ${{ inputs.variants || 'all' }} + SHA: ${{ github.sha }} + run: | + set -euo pipefail + # The .debs ship for each of these; the portable tarballs are built + # on the oldest glibc among them so they run on all. + deb_distros='[ + {"distro": "ubuntu", "version": "24.04", "codename": "noble"}, + {"distro": "ubuntu", "version": "25.10", "codename": "questing"}, + {"distro": "ubuntu", "version": "26.04", "codename": "resolute"}, + {"distro": "debian", "version": "13", "codename": "trixie"} + ]' + portable_distro='{"distro": "debian", "version": "13", "codename": "trixie"}' + short_sha="${SHA:0:7}" + + variants="${VARIANTS//,/ }" + if [ "$(echo ${variants})" = "all" ]; then + variants="XRT-deb HRX-deb XRT-portable HRX-portable Source-portable" + fi + + include='[]' + for variant in ${variants}; do + case "${variant}" in + XRT-deb|HRX-deb) + prefix="" + [ "${variant}" = "HRX-deb" ] && prefix="HRX-" + include=$(jq -c --arg v "${variant}" --arg p "${prefix}" --arg s "${short_sha}" \ + --argjson d "${deb_distros}" \ + '. + [$d[] | . + {variant: $v, retention: "", + artifact: "\($p)flm-\(.distro)\(.version)-\($s)-deb"}]' <<< "${include}") + ;; + XRT-portable|HRX-portable|Source-portable) + case "${variant}" in + XRT-portable) artifact=portable ;; + HRX-portable) artifact=HRX-portable ;; + Source-portable) artifact=models-from-source-portable ;; + esac + include=$(jq -c --arg v "${variant}" --arg a "${artifact}" \ + --argjson d "${portable_distro}" \ + '. + [$d + {variant: $v, retention: "7", artifact: $a}]' <<< "${include}") + ;; + *) + echo "::error::Unknown Linux build variant '${variant}'." + exit 1 + ;; + esac + done + if [ "${include}" = '[]' ]; then + echo "::error::No Linux build variant given." + exit 1 + fi + + jq . <<< "${include}" + echo "include=${include}" >> "$GITHUB_OUTPUT" + echo "deb_distros=$(jq length <<< "${deb_distros}")" >> "$GITHUB_OUTPUT" + + build: + name: build (${{ matrix.variant }}, ${{ matrix.distro }}${{ matrix.version }}) + needs: plan + runs-on: ubuntu-latest + strategy: + # Each variant and distro ships on its own; one failing must not cancel + # the others. + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.include) }} + + container: + image: ghcr.io/rocm/fastflowlm/build-environment:${{ matrix.distro }}${{ matrix.version }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + fetch-tags: true + + - name: Build + id: build + uses: ./.github/actions/lin-build + with: + variant: ${{ matrix.variant }} + distro: ${{ matrix.distro }} + distro-version: ${{ matrix.version }} + codename: ${{ matrix.codename }} + + - name: Package portable distribution + id: portable + if: endsWith(matrix.variant, '-portable') + env: + VARIANT: ${{ matrix.variant }} + VERSION: ${{ steps.build.outputs.version }} + BUILD_DIR: ${{ steps.build.outputs.build-dir }} + run: | + set -euo pipefail + case "${VARIANT}" in + XRT-portable) name_prefix="" ;; + HRX-portable) name_prefix="HRX_" ;; + Source-portable) name_prefix="models-from-source_" ;; + esac + tarball="${name_prefix}fastflowlm_${VERSION}_linux.tar.gz" + + cd "${GITHUB_WORKSPACE}/${BUILD_DIR}" + rm -rf install + DESTDIR="$PWD/install" cmake --install . + + # Rename binary and wrapper for portable layout + package_root=install/opt/fastflowlm + mv "${package_root}/flm" "${package_root}/flm-real" + mv "${package_root}/flm-wrapper.sh" "${package_root}/flm" + chmod +x "${package_root}/flm" + + # Strip debug symbols to reduce binary size (111MB → 33MB) + # --strip-debug removes debug info while keeping stack trace capability + strip --strip-debug "${package_root}/flm-real" + + if [ "${VARIANT}" = "Source-portable" ]; then + staged_engine="${package_root}/lib/libgemma4e_npu.so" + test -f "${staged_engine}" + if cmp -s "${GITHUB_WORKSPACE}/src/lib/xrt/libgemma4e_npu.so" "${staged_engine}"; then + echo "The staged Gemma 4 engine matches the prebuilt library." >&2 + exit 1 + fi + readelf -n "${staged_engine}" | grep -A1 'Build ID' + fi + + # Symlinks are preserved. + tar czf "${GITHUB_WORKSPACE}/${tarball}" \ + -C "${package_root}" \ + flm \ + flm-real \ + lib/ \ + xclbins/ \ + model_list.json \ + model_info.json \ + LICENSE_RUNTIME.txt \ + LICENSE_BINARY.txt \ + THIRD_PARTY_NOTICES.txt + + # Confirm the runtime the build selects is bundled. + case "${VARIANT}" in + XRT-portable) tar tzf "${GITHUB_WORKSPACE}/${tarball}" | grep libxrt ;; + HRX-portable) tar tzf "${GITHUB_WORKSPACE}/${tarball}" | grep libhrx ;; + esac + "${GITHUB_WORKSPACE}/.github/script/ci/check_no_copyleft_libs.sh" \ + "${GITHUB_WORKSPACE}/${tarball}" + + echo "tarball=${tarball}" >> "$GITHUB_OUTPUT" + + - name: Upload build artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.artifact }} + path: ${{ steps.build.outputs.package || steps.portable.outputs.tarball }} + retention-days: ${{ matrix.retention }} + if-no-files-found: error + diff --git a/.github/workflows/model-sweep.yml b/.github/workflows/model-sweep.yml deleted file mode 100644 index aeb59731c..000000000 --- a/.github/workflows/model-sweep.yml +++ /dev/null @@ -1,845 +0,0 @@ -name: Model Sweep - -# Runs the cross-model sweep against real NPU hardware. Nothing is rebuilt -# here; the sweep always consumes a portable build someone else produced. -# Where that build comes from depends on how the sweep was triggered: -# -# pull_request / push to main the "Build Linux Portable" and "Build Windows -# Packages" runs for this same commit -# workflow_dispatch the portable assets attached to a GitHub -# Release (the latest one unless a tag is given) -# -# A PR or a src/** push is asking "did this change break a model?", so it has -# to sweep that commit's binary. A manual dispatch is asking "how does a -# shipped build behave?", so it sweeps a release and does not need CI at all. -# -# The sweep is split by modality (llm, vision, embedding, audio) into one -# matrix job each, so a failure in one modality does not hide the others and -# "Re-run failed jobs" restarts only what actually broke. -# -# Each platform is swept against the default portable build and the HRX build; -# releases ship both, the HRX one as a preview under an HRX_ prefix. Commit -# sweeps also run Gemma 4 against separate Linux and Windows packages that -# build model engines from source. Releases do not ship those. - -on: - pull_request: - branches: [ main ] - # Merges land on main without ever having been swept when a PR skipped CI, - # and src/** is what changes the binary the sweep exercises. Doc- and - # workflow-only pushes produce an identical build, so they are not worth an - # NPU machine. - push: - branches: [ main ] - paths: - - 'src/**' - workflow_dispatch: - inputs: - release_tag: - description: "Release tag to sweep (blank = latest release)" - type: string - default: "" - gen_lim: - description: "max_completion_tokens per request (0 or less leaves it unset)" - type: string - default: "1024" - request_timeout: - description: "Per-request timeout in seconds" - type: string - default: "600" - limit: - description: "Test only the first N models per task (0 = all). Use a small N to smoke-test the pipeline without downloading every model." - type: string - default: "0" - -permissions: - contents: read - # Needed to list and download artifacts from the sibling build workflow runs. - actions: read - -# The sweep occupies a physical NPU machine, so only keep the newest run per -# ref. The event name is part of the group so that dispatching a release sweep -# does not cancel the commit sweep that a push to main just started, and vice -# versa: those two answer different questions and both are worth finishing. -concurrency: - group: model-sweep-${{ github.ref }}-${{ github.event_name }} - cancel-in-progress: true - -env: - GEN_LIM: ${{ inputs.gen_lim || '1024' }} - REQUEST_TIMEOUT: ${{ inputs.request_timeout || '600' }} - # 0 unless dispatched, so the cap only exists when someone asks for one. - MODEL_LIMIT: ${{ inputs.limit || '0' }} - -jobs: - # Decides what the sweep runs against. On a dispatch that means resolving - # a release tag. The tag is resolved once here rather than letting each - # platform (or the downloader) ask for "latest", so both platforms are - # guaranteed to sweep the same release even if one is published mid-run, - # and so the tag appears in the log. On a commit sweep there is nothing to - # look up and this finishes in seconds. - resolve-source: - runs-on: ubuntu-latest - timeout-minutes: 5 - outputs: - # "run" = artifacts from a CI run, "release" = assets from a release. - mode: ${{ steps.resolve.outputs.mode }} - release_tag: ${{ steps.resolve.outputs.release_tag }} - - steps: - - name: Resolve the build source - id: resolve - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - RELEASE_TAG: ${{ inputs.release_tag }} - run: | - set -euo pipefail - if [ "${GITHUB_EVENT_NAME}" != "workflow_dispatch" ]; then - echo "mode=run" >> "$GITHUB_OUTPUT" - exit 0 - fi - tag="${RELEASE_TAG}" - if [ -z "${tag}" ]; then - tag=$(gh api "repos/${GITHUB_REPOSITORY}/releases/latest" --jq '.tag_name') - echo "No tag given; latest release is ${tag}." - fi - { - echo "mode=release" - echo "release_tag=${tag}" - } >> "$GITHUB_OUTPUT" - - # Each platform's portable builds are resolved by their own job, which does - # not finish until those builds actually exist. Everything downstream reads - # that job's ready flags and only has to know the answer, not how it was - # reached; .github/script/resolve_build.sh has the details. - # - # On a dispatch that means confirming the release carries the asset each - # sweep needs. On a commit sweep it means waiting. Both build workflows - # trigger on the same event as this one, so they are still compiling when - # the sweep is queued, and GitHub has no cross-workflow "needs:". Waiting - # from a hosted runner keeps the NPU machines free until there is something - # to sweep. Each build resolves to a concrete run id; pinning the download - # to that id is what stops a sweep from silently picking up a stale artifact - # built from an earlier push on the same branch. - # - # The platforms are resolved by separate jobs so that neither waits on the - # other: the Linux sweeps start as soon as the hosted Linux build is done, - # and the Windows build, which queues for a shared self-hosted runner, gets - # its own, longer deadline instead of whatever the Linux wait left over. - # - # Readiness is per build, not per workflow run. Each artifact has its own - # readiness flag. A failed optional build does not block the other sweeps. - # The build workflow reports its failure, so this workflow skips the missing - # artifact. - resolve-linux: - needs: resolve-source - runs-on: ubuntu-latest - timeout-minutes: 90 - outputs: - mode: ${{ needs.resolve-source.outputs.mode }} - release_tag: ${{ needs.resolve-source.outputs.release_tag }} - run_id: ${{ steps.resolve.outputs.run_id }} - ready: ${{ steps.resolve.outputs.ready }} - hrx_ready: ${{ steps.resolve.outputs.hrx_ready }} - models_from_source_ready: ${{ steps.resolve.outputs.models_from_source_ready }} - - steps: - - name: Checkout sweep scripts - uses: actions/checkout@v4 - with: - sparse-checkout: .github/script - filter: blob:none - - - name: Resolve the Linux portable builds - id: resolve - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - MODE: ${{ needs.resolve-source.outputs.mode }} - RELEASE_TAG: ${{ needs.resolve-source.outputs.release_tag }} - # A pull_request run records the PR head commit in head_sha, while - # github.sha is the synthetic merge commit. Try both, so the lookup - # does not depend on which one the API reports. On a push only the - # second is set, and it is the one the build workflow records. - CANDIDATE_SHAS: ${{ github.event.pull_request.head.sha }} ${{ github.sha }} - # "Build Linux Portable" runs on hosted runners and takes ~30 min. - FINISH_MINUTES: "80" - run: bash .github/script/resolve_build.sh linux - - resolve-windows: - needs: resolve-source - runs-on: ubuntu-latest - timeout-minutes: 190 - outputs: - mode: ${{ needs.resolve-source.outputs.mode }} - release_tag: ${{ needs.resolve-source.outputs.release_tag }} - run_id: ${{ steps.resolve.outputs.run_id }} - ready: ${{ steps.resolve.outputs.ready }} - hrx_ready: ${{ steps.resolve.outputs.hrx_ready }} - models_from_source_ready: ${{ steps.resolve.outputs.models_from_source_ready }} - - steps: - - name: Checkout sweep scripts - uses: actions/checkout@v4 - with: - sparse-checkout: .github/script - filter: blob:none - - - name: Resolve the Windows portable builds - id: resolve - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - MODE: ${{ needs.resolve-source.outputs.mode }} - RELEASE_TAG: ${{ needs.resolve-source.outputs.release_tag }} - CANDIDATE_SHAS: ${{ github.event.pull_request.head.sha }} ${{ github.sha }} - # The build itself takes ~30 min, but both portable jobs share one - # self-hosted runner and queue behind other PRs' builds, which has - # pushed a run past 85 min. Waiting longer only costs hosted-runner - # time; giving up skips the Windows sweep entirely. - FINISH_MINUTES: "180" - run: bash .github/script/resolve_build.sh windows - - sweep-linux: - needs: resolve-linux - # Unset when no Linux build could be resolved, whichever mode we are in. - if: needs.resolve-linux.outputs.ready == 'true' - runs-on: [self-hosted, linux, x64, npu, sweep] - timeout-minutes: 120 - - strategy: - # One job per modality, so they can spread across however many runners - # carry the labels above, and "Re-run failed jobs" restarts only what - # broke. fail-fast stays off so one modality failing does not cancel the - # others mid-sweep. - fail-fast: false - matrix: - # sweep_embedding.py, sweep_audio.py and sweep_decision.py are ready and tested, but stay - # out of the matrix until llm and vision have proven themselves against - # real hardware. Add them back here to enable them; nothing else needs - # to change. - task: [llm, vision] - - env: - # Weights are expected at $FLM_MODEL_PATH/models/ - FLM_MODEL_PATH: /scratch - - steps: - # Only the sweep scripts and their test assets, never the sources: this - # job runs a prebuilt binary and compiles nothing. src/ is ~880 MB of - # code and test media that every sweep job would otherwise copy onto an - # NPU machine for no reason. Cone mode also brings the root files along, - # which are small enough not to matter. - - name: Checkout sweep scripts - uses: actions/checkout@v4 - with: - sparse-checkout: .github/script - # sparse-checkout alone still downloads every blob at this depth. - # blob:none makes git fetch only the blobs the sparse paths need. - filter: blob:none - - # Exactly one of the next two steps runs. Both land the same - # fastflowlm__linux.tar.gz in build-artifact/, so the unpack - # step below does not care which one it was. - - # Pinned to the exact run that resolve-linux confirmed, so this cannot - # resolve to an older green build of the same branch. The artifact is - # named "portable" and holds a single tarball. - - name: Download Linux portable build (CI run) - if: needs.resolve-linux.outputs.mode == 'run' - uses: dawidd6/action-download-artifact@v25 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - run_id: ${{ needs.resolve-linux.outputs.run_id }} - name: portable - path: build-artifact - if_no_artifact_found: fail - - # A JavaScript action, so it needs nothing installed on the self-hosted - # runner beyond the node the runner already ships. extract stays off: - # the unpack step below handles the tarball, and it also has to handle - # the CI-artifact case. - - name: Download Linux portable build (release ${{ needs.resolve-linux.outputs.release_tag }}) - if: needs.resolve-linux.outputs.mode == 'release' - uses: robinraju/release-downloader@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - tag: ${{ needs.resolve-linux.outputs.release_tag }} - fileName: "fastflowlm_*_linux.tar.gz" - out-file-path: build-artifact - - - name: Unpack Linux portable build - run: | - set -euo pipefail - mkdir -p flm-linux - tarball=$(ls build-artifact/fastflowlm_*_linux.tar.gz) - echo "Using tarball: ${tarball}" - tar xzf "${tarball}" -C flm-linux - chmod +x flm-linux/flm flm-linux/flm-real - echo "FLM_BIN=${GITHUB_WORKSPACE}/flm-linux/flm" >> "$GITHUB_ENV" - - - name: Verify FLM binary runs - run: | - set -euo pipefail - "${FLM_BIN}" version - - - name: Set up Python virtualenv - run: | - set -euo pipefail - python3 -m venv .venv - .venv/bin/python -m pip install --upgrade pip - for attempt in 1 2 3; do - .venv/bin/python -m pip install --no-cache-dir openai && break - if [ "${attempt}" -eq 3 ]; then - exit 1 - fi - done - - # Each script starts and stops its own flm server, picks its own models, - # and exits non-zero if any model failed. - - name: Run ${{ matrix.task }} sweep - run: | - set -euo pipefail - .venv/bin/python .github/script/sweep_${{ matrix.task }}.py \ - --flm-bin "${FLM_BIN}" \ - --platform linux \ - --gen-lim "${GEN_LIM}" \ - --request-timeout "${REQUEST_TIMEOUT}" \ - --limit "${MODEL_LIMIT}" \ - --output-dir sweep-results - - # Includes the CSV, the summary JSON and the captured server log, so a - # failed sweep can be diagnosed without access to the runner. - - name: Upload ${{ matrix.task }} results - if: always() - uses: actions/upload-artifact@v4 - with: - name: model-sweep-linux-${{ matrix.task }}-${{ github.run_number }} - path: sweep-results/ - if-no-files-found: warn - retention-days: 30 - - sweep-linux-models-from-source: - needs: resolve-linux - if: needs.resolve-linux.outputs.models_from_source_ready == 'true' - runs-on: [self-hosted, linux, x64, npu, sweep] - timeout-minutes: 60 - - strategy: - fail-fast: false - matrix: - task: [llm, vision] - - env: - FLM_MODEL_PATH: /scratch - - steps: - - name: Checkout sweep scripts - uses: actions/checkout@v4 - with: - sparse-checkout: .github/script - filter: blob:none - - - name: Download Linux models-from-source build - uses: dawidd6/action-download-artifact@v25 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - run_id: ${{ needs.resolve-linux.outputs.run_id }} - name: models-from-source-portable - path: build-artifact - if_no_artifact_found: fail - - - name: Unpack Linux models-from-source build - run: | - set -euo pipefail - mkdir -p flm-linux-models-from-source - tarball=$(ls build-artifact/models-from-source_fastflowlm_*_linux.tar.gz) - echo "Using tarball: ${tarball}" - tar xzf "${tarball}" -C flm-linux-models-from-source - chmod +x flm-linux-models-from-source/flm flm-linux-models-from-source/flm-real - echo "FLM_BIN=${GITHUB_WORKSPACE}/flm-linux-models-from-source/flm" >> "$GITHUB_ENV" - - - name: Verify FLM binary runs - run: | - set -euo pipefail - "${FLM_BIN}" version - - - name: Set up Python virtualenv - run: | - set -euo pipefail - python3 -m venv .venv - .venv/bin/python -m pip install --upgrade pip - for attempt in 1 2 3; do - .venv/bin/python -m pip install --no-cache-dir openai && break - if [ "${attempt}" -eq 3 ]; then - exit 1 - fi - done - - - name: Run Gemma 4 ${{ matrix.task }} sweep - run: | - set -euo pipefail - .venv/bin/python .github/script/sweep_${{ matrix.task }}.py \ - --flm-bin "${FLM_BIN}" \ - --platform linux-models-from-source \ - --models gemma4-it:e2b gemma4-it:e4b \ - --gen-lim "${GEN_LIM}" \ - --request-timeout "${REQUEST_TIMEOUT}" \ - --output-dir sweep-results - - - name: Upload Gemma 4 ${{ matrix.task }} results - if: always() - uses: actions/upload-artifact@v4 - with: - name: model-sweep-linux-models-from-source-${{ matrix.task }}-${{ github.run_number }} - path: sweep-results/ - if-no-files-found: warn - retention-days: 30 - - # The Linux sweep again, against the HRX build. "Build Linux Portable" - # produces both from the same run, so this shares its run id with - # sweep-linux but has its own readiness flag: whichever of the two builds - # succeeded gets swept, independently of the other. - sweep-linux-hrx: - needs: resolve-linux - # Unset when no Linux HRX build could be resolved, whichever mode we are in. - if: needs.resolve-linux.outputs.hrx_ready == 'true' - runs-on: [self-hosted, linux, x64, npu, sweep] - timeout-minutes: 120 - - strategy: - fail-fast: false - matrix: - # Kept in step with sweep-linux; see the note there about embedding - # and audio. - task: [llm, vision] - - env: - # Weights are expected at $FLM_MODEL_PATH/models/ - FLM_MODEL_PATH: /scratch - - steps: - # Only the sweep scripts and their test assets, never the sources: this - # job runs a prebuilt binary and compiles nothing. src/ is ~880 MB of - # code and test media that every sweep job would otherwise copy onto an - # NPU machine for no reason. Cone mode also brings the root files along, - # which are small enough not to matter. - - name: Checkout sweep scripts - uses: actions/checkout@v4 - with: - sparse-checkout: .github/script - # sparse-checkout alone still downloads every blob at this depth. - # blob:none makes git fetch only the blobs the sparse paths need. - filter: blob:none - - # Exactly one of the next two steps runs. Both land the same - # HRX_fastflowlm__linux.tar.gz in build-artifact/. - - # Same shape as the default artifact, under the HRX- name and holding a - # single HRX_-prefixed tarball. - - name: Download Linux HRX portable build (CI run) - if: needs.resolve-linux.outputs.mode == 'run' - uses: dawidd6/action-download-artifact@v25 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - run_id: ${{ needs.resolve-linux.outputs.run_id }} - name: HRX-portable - path: build-artifact - if_no_artifact_found: fail - - - name: Download Linux HRX portable build (release ${{ needs.resolve-linux.outputs.release_tag }}) - if: needs.resolve-linux.outputs.mode == 'release' - uses: robinraju/release-downloader@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - tag: ${{ needs.resolve-linux.outputs.release_tag }} - fileName: "HRX_fastflowlm_*_linux.tar.gz" - out-file-path: build-artifact - - - name: Unpack Linux HRX portable build - run: | - set -euo pipefail - mkdir -p flm-linux - tarball=$(ls build-artifact/HRX_fastflowlm_*_linux.tar.gz) - echo "Using tarball: ${tarball}" - tar xzf "${tarball}" -C flm-linux - chmod +x flm-linux/flm flm-linux/flm-real - echo "FLM_BIN=${GITHUB_WORKSPACE}/flm-linux/flm" >> "$GITHUB_ENV" - - - name: Verify FLM binary runs - run: | - set -euo pipefail - "${FLM_BIN}" version - - - name: Set up Python virtualenv - run: | - set -euo pipefail - python3 -m venv .venv - .venv/bin/python -m pip install --upgrade pip - for attempt in 1 2 3; do - .venv/bin/python -m pip install --no-cache-dir openai && break - if [ "${attempt}" -eq 3 ]; then - exit 1 - fi - done - - # --platform is only a label, but it is the one that keeps these results - # apart from the default build's in the filenames and the summary table. - - name: Run ${{ matrix.task }} sweep - run: | - set -euo pipefail - .venv/bin/python .github/script/sweep_${{ matrix.task }}.py \ - --flm-bin "${FLM_BIN}" \ - --platform linux-hrx \ - --gen-lim "${GEN_LIM}" \ - --request-timeout "${REQUEST_TIMEOUT}" \ - --limit "${MODEL_LIMIT}" \ - --output-dir sweep-results - - - name: Upload ${{ matrix.task }} results - if: always() - uses: actions/upload-artifact@v4 - with: - name: model-sweep-linux-hrx-${{ matrix.task }}-${{ github.run_number }} - path: sweep-results/ - if-no-files-found: warn - retention-days: 30 - - sweep-windows: - needs: resolve-windows - # Unset when no Windows build could be resolved, whichever mode we are in. - if: needs.resolve-windows.outputs.ready == 'true' - runs-on: [self-hosted, windows, x64, npu, sweep] - timeout-minutes: 120 - - strategy: - fail-fast: false - matrix: - # sweep_embedding.py, sweep_audio.py and sweep_decision.py are ready and tested, but stay - # out of the matrix until llm and vision have proven themselves against - # real hardware. Add them back here to enable them; nothing else needs - # to change. - task: [llm, vision] - - steps: - # Only the sweep scripts and their test assets, never the sources: this - # job runs a prebuilt binary and compiles nothing. src/ is ~880 MB of - # code and test media that every sweep job would otherwise copy onto an - # NPU machine for no reason. Cone mode also brings the root files along, - # which are small enough not to matter. - - name: Checkout sweep scripts - uses: actions/checkout@v4 - with: - sparse-checkout: .github/script - # sparse-checkout alone still downloads every blob at this depth. - # blob:none makes git fetch only the blobs the sparse paths need. - filter: blob:none - - - name: Resolve model path - shell: powershell - run: | - $modelPath = Join-Path $env:USERPROFILE ".flm" - New-Item -ItemType Directory -Force $modelPath | Out-Null - "FLM_MODEL_PATH=$modelPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - - # Exactly one of the next two steps runs. Both leave flm.exe and its DLLs - # at the top level of flm-windows/, which is where the verify step looks. - - # "Build Windows Packages" publishes the portable payload under - # fastflowlm-windows-. Both workflows run off the same event, so - # github.sha matches; the run id from resolve-windows pins which run it - # comes from. - - name: Download Windows portable build (CI run) - if: needs.resolve-windows.outputs.mode == 'run' - uses: dawidd6/action-download-artifact@v25 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - run_id: ${{ needs.resolve-windows.outputs.run_id }} - name: fastflowlm-windows-${{ github.sha }} - path: flm-windows - if_no_artifact_found: fail - - # The release zip is that same artifact's contents zipped from inside the - # directory, so extracting it in place reproduces the CI layout. The zip - # itself stays alongside, which is harmless. - - name: Download Windows portable build (release ${{ needs.resolve-windows.outputs.release_tag }}) - if: needs.resolve-windows.outputs.mode == 'release' - uses: robinraju/release-downloader@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - tag: ${{ needs.resolve-windows.outputs.release_tag }} - fileName: "fastflowlm_*_windows_amd64.zip" - out-file-path: flm-windows - extract: true - - - name: Verify FLM binary runs - shell: powershell - run: | - $ErrorActionPreference = "Stop" - $flm = Join-Path $env:GITHUB_WORKSPACE "flm-windows\flm.exe" - if (-not (Test-Path $flm)) { - throw "flm.exe not found at $flm" - } - "FLM_BIN=$flm" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - & $flm version - - - name: Set up Python virtualenv - shell: powershell - run: | - $ErrorActionPreference = "Stop" - python -m venv .venv - .\.venv\Scripts\python.exe -m pip install --upgrade pip - .\.venv\Scripts\python.exe -m pip install openai - - - name: Run ${{ matrix.task }} sweep - shell: powershell - run: | - $ErrorActionPreference = "Stop" - $cmdArgs = @( - ".github\script\sweep_${{ matrix.task }}.py", - "--flm-bin", $env:FLM_BIN, - "--platform", "windows", - "--gen-lim", $env:GEN_LIM, - "--request-timeout", $env:REQUEST_TIMEOUT, - "--limit", $env:MODEL_LIMIT, - "--output-dir", "sweep-results" - ) - & ".\.venv\Scripts\python.exe" $cmdArgs - # A native command's exit code does not trip ErrorActionPreference. - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - - - name: Upload ${{ matrix.task }} results - if: always() - uses: actions/upload-artifact@v4 - with: - name: model-sweep-windows-${{ matrix.task }}-${{ github.run_number }} - path: sweep-results/ - if-no-files-found: warn - retention-days: 30 - - sweep-windows-models-from-source: - needs: resolve-windows - if: needs.resolve-windows.outputs.models_from_source_ready == 'true' - runs-on: [self-hosted, windows, x64, npu, sweep] - timeout-minutes: 60 - - strategy: - fail-fast: false - matrix: - task: [llm, vision] - - steps: - - name: Checkout sweep scripts - uses: actions/checkout@v4 - with: - sparse-checkout: .github/script - filter: blob:none - - - name: Resolve model path - shell: powershell - run: | - $modelPath = Join-Path $env:USERPROFILE ".flm" - New-Item -ItemType Directory -Force $modelPath | Out-Null - "FLM_MODEL_PATH=$modelPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - - - name: Download Windows models-from-source build - uses: dawidd6/action-download-artifact@v25 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - run_id: ${{ needs.resolve-windows.outputs.run_id }} - name: models-from-source-fastflowlm-windows-${{ github.sha }} - path: flm-windows-models-from-source - if_no_artifact_found: fail - - - name: Verify FLM binary runs - shell: powershell - run: | - $ErrorActionPreference = "Stop" - $flm = Join-Path $env:GITHUB_WORKSPACE "flm-windows-models-from-source\flm.exe" - if (-not (Test-Path $flm)) { - throw "flm.exe not found at $flm" - } - "FLM_BIN=$flm" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - & $flm version - - - name: Set up Python virtualenv - shell: powershell - run: | - $ErrorActionPreference = "Stop" - python -m venv .venv - .\.venv\Scripts\python.exe -m pip install --upgrade pip - .\.venv\Scripts\python.exe -m pip install openai - - - name: Run models-from-source ${{ matrix.task }} sweep - shell: powershell - run: | - $ErrorActionPreference = "Stop" - $cmdArgs = @( - ".github\script\sweep_${{ matrix.task }}.py", - "--flm-bin", $env:FLM_BIN, - "--platform", "windows-models-from-source", - "--models", "gemma4-it:e2b", "gemma4-it:e4b", - "--gen-lim", $env:GEN_LIM, - "--request-timeout", $env:REQUEST_TIMEOUT, - "--output-dir", "sweep-results" - ) - & ".\.venv\Scripts\python.exe" $cmdArgs - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - - - name: Upload models-from-source ${{ matrix.task }} results - if: always() - uses: actions/upload-artifact@v4 - with: - name: model-sweep-windows-models-from-source-${{ matrix.task }}-${{ github.run_number }} - path: sweep-results/ - if-no-files-found: warn - retention-days: 30 - - # The Windows sweep again, against the HRX build. "Build Windows Packages" - # produces both from the same run, so this shares its run id with - # sweep-windows but has its own readiness flag. - sweep-windows-hrx: - needs: resolve-windows - # Unset when no Windows HRX build could be resolved, whichever mode we are in. - if: needs.resolve-windows.outputs.hrx_ready == 'true' - runs-on: [self-hosted, windows, x64, npu, sweep] - timeout-minutes: 120 - - strategy: - fail-fast: false - matrix: - # Kept in step with sweep-windows; see the note there about embedding - # and audio. - task: [llm, vision] - - steps: - # Only the sweep scripts and their test assets, never the sources: this - # job runs a prebuilt binary and compiles nothing. src/ is ~880 MB of - # code and test media that every sweep job would otherwise copy onto an - # NPU machine for no reason. Cone mode also brings the root files along, - # which are small enough not to matter. - - name: Checkout sweep scripts - uses: actions/checkout@v4 - with: - sparse-checkout: .github/script - # sparse-checkout alone still downloads every blob at this depth. - # blob:none makes git fetch only the blobs the sparse paths need. - filter: blob:none - - - name: Resolve model path - shell: powershell - run: | - $modelPath = Join-Path $env:USERPROFILE ".flm" - New-Item -ItemType Directory -Force $modelPath | Out-Null - "FLM_MODEL_PATH=$modelPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - - # Exactly one of the next two steps runs. Both leave flm.exe and its DLLs - # at the top level of flm-windows/, which is where the verify step looks. - - # Same payload as the default Windows portable artifact, under the HRX- - # prefixed name: flm.exe and its DLLs at the top level, with the HRX op - # libraries and hrx.dll overlaid. - - name: Download Windows HRX portable build (CI run) - if: needs.resolve-windows.outputs.mode == 'run' - uses: dawidd6/action-download-artifact@v25 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - run_id: ${{ needs.resolve-windows.outputs.run_id }} - name: HRX-fastflowlm-windows-${{ github.sha }} - path: flm-windows - if_no_artifact_found: fail - - # The HRX release zip is that same artifact zipped from inside the - # directory, so extracting it in place reproduces the CI layout. - - name: Download Windows HRX portable build (release ${{ needs.resolve-windows.outputs.release_tag }}) - if: needs.resolve-windows.outputs.mode == 'release' - uses: robinraju/release-downloader@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - tag: ${{ needs.resolve-windows.outputs.release_tag }} - fileName: "HRX_fastflowlm_*_windows_amd64.zip" - out-file-path: flm-windows - extract: true - - - name: Verify FLM binary runs - shell: powershell - run: | - $ErrorActionPreference = "Stop" - $flm = Join-Path $env:GITHUB_WORKSPACE "flm-windows\flm.exe" - if (-not (Test-Path $flm)) { - throw "flm.exe not found at $flm" - } - "FLM_BIN=$flm" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - & $flm version - - - name: Set up Python virtualenv - shell: powershell - run: | - $ErrorActionPreference = "Stop" - python -m venv .venv - .\.venv\Scripts\python.exe -m pip install --upgrade pip - .\.venv\Scripts\python.exe -m pip install openai - - # --platform is only a label, but it is the one that keeps these results - # apart from the default build's in the filenames and the summary table. - - name: Run ${{ matrix.task }} sweep - shell: powershell - run: | - $ErrorActionPreference = "Stop" - $cmdArgs = @( - ".github\script\sweep_${{ matrix.task }}.py", - "--flm-bin", $env:FLM_BIN, - "--platform", "windows-hrx", - "--gen-lim", $env:GEN_LIM, - "--request-timeout", $env:REQUEST_TIMEOUT, - "--limit", $env:MODEL_LIMIT, - "--output-dir", "sweep-results" - ) - & ".\.venv\Scripts\python.exe" $cmdArgs - # A native command's exit code does not trip ErrorActionPreference. - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - - - name: Upload ${{ matrix.task }} results - if: always() - uses: actions/upload-artifact@v4 - with: - name: model-sweep-windows-hrx-${{ matrix.task }}-${{ github.run_number }} - path: sweep-results/ - if-no-files-found: warn - retention-days: 30 - - # Reporting only, no NPU needed, so this one stays on a GitHub-hosted runner. - summary: - runs-on: ubuntu-latest - needs: [sweep-linux, sweep-linux-models-from-source, sweep-windows, sweep-windows-models-from-source, sweep-linux-hrx, sweep-windows-hrx] - if: always() - steps: - # Only the sweep scripts and their test assets, never the sources: this - # job runs a prebuilt binary and compiles nothing. src/ is ~880 MB of - # code and test media that every sweep job would otherwise copy onto an - # NPU machine for no reason. Cone mode also brings the root files along, - # which are small enough not to matter. - - name: Checkout sweep scripts - uses: actions/checkout@v4 - with: - sparse-checkout: .github/script - # sparse-checkout alone still downloads every blob at this depth. - # blob:none makes git fetch only the blobs the sparse paths need. - filter: blob:none - - - name: Download sweep artifacts - uses: actions/download-artifact@v4 - continue-on-error: true - with: - pattern: model-sweep-* - path: sweep-results - - - name: Write job summary - run: | - set -euo pipefail - python3 .github/script/sweep_summary.py sweep-results \ - --linux-result "${{ needs.sweep-linux.result }}" \ - --windows-result "${{ needs.sweep-windows.result }}" \ - --linux-hrx-result "${{ needs.sweep-linux-hrx.result }}" \ - --windows-hrx-result "${{ needs.sweep-windows-hrx.result }}" \ - --linux-models-from-source-result "${{ needs.sweep-linux-models-from-source.result }}" \ - --windows-models-from-source-result "${{ needs.sweep-windows-models-from-source.result }}" \ - >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..45f89f87e --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,157 @@ +name: Release + +# Builds and tests the tagged commit, then publishes it to the tag's GitHub +# Release. Built here rather than taken from main's CI artifacts because the +# version comes from git describe at build time: only a build of the tag itself +# gets the tag's version in its .deb metadata and tarball names. +# +# Assets: the XRT and HRX builds for both platforms (Windows zip and MSI, the +# .debs for every distro, the Linux tarball). The models-from-source builds +# are not released. Each backend is published only when all of its packages +# passed their NPU tests. +on: + push: + tags: + - '*' + workflow_dispatch: + +jobs: + build-and-test: + uses: ./.github/workflows/build-and-test.yml + with: + tier: pr + release: true + + # Each publish job downloads only the artifacts it ships, by name, and runs + # only when that backend passed: a failed HRX build or test leaves the release + # without the HRX assets instead of holding back the default ones. + release: + needs: build-and-test + if: ${{ !cancelled() && github.ref_type == 'tag' && needs.build-and-test.outputs.xrt-passed == 'true' }} + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Download Windows portable artifact + uses: actions/download-artifact@v4 + with: + name: fastflowlm-windows-${{ github.sha }} + path: windows-dist + + - name: Download Windows MSI artifact + uses: actions/download-artifact@v4 + with: + name: fastflowlm-windows-msi-${{ github.sha }} + path: msi-dist + + - name: Download Linux portable artifact + uses: actions/download-artifact@v4 + with: + name: portable + path: linux-dist + + # Not the HRX- prefixed ones, which release-hrx publishes. + - name: Download deb artifacts + uses: actions/download-artifact@v4 + with: + path: debs + pattern: 'flm-*-deb' + + # A pattern download takes whatever is there, so a distro whose build + # failed would otherwise just be missing from the release. + - name: Check every distro's deb is present + env: + EXPECTED: ${{ needs.build-and-test.outputs.deb-distros }} + run: | + set -euo pipefail + ls debs/*/*.deb + found=$(ls debs/*/*.deb | wc -l) + if [ "${found}" -ne "${EXPECTED}" ]; then + echo "::error::Expected ${EXPECTED} XRT .debs, found ${found}." + exit 1 + fi + + - name: Stage release assets + run: | + set -euo pipefail + version="${GITHUB_REF_NAME#v}" + mkdir assets + + # flm.exe and its DLLs at the top level of the zip. + (cd windows-dist && zip -r "../assets/fastflowlm_${version}_windows_amd64.zip" .) + cp msi-dist/flm-setup.msi assets/ + cp linux-dist/fastflowlm_*_linux.tar.gz assets/ + cp debs/*/*.deb assets/ + ls -lah assets + + - name: Upload assets to GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ github.ref_name }} + name: Release ${{ github.ref_name }} + files: assets/* + overwrite_files: true + fail_on_unmatched_files: true + + # The HRX build ships as a preview, so it gets its own publish job. It runs + # after the default one, whatever that one's outcome, so the two never race + # to create the release. + release-hrx: + needs: [build-and-test, release] + if: ${{ !cancelled() && github.ref_type == 'tag' && needs.build-and-test.outputs.hrx-passed == 'true' }} + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Download Windows HRX portable artifact + uses: actions/download-artifact@v4 + with: + name: HRX-fastflowlm-windows-${{ github.sha }} + path: windows-dist + + - name: Download Linux HRX portable artifact + uses: actions/download-artifact@v4 + with: + name: HRX-portable + path: linux-dist + + - name: Download HRX deb artifacts + uses: actions/download-artifact@v4 + with: + path: debs + pattern: 'HRX-flm-*-deb' + + - name: Check every distro's HRX deb is present + env: + EXPECTED: ${{ needs.build-and-test.outputs.deb-distros }} + run: | + set -euo pipefail + ls debs/*/*.deb + found=$(ls debs/*/*.deb | wc -l) + if [ "${found}" -ne "${EXPECTED}" ]; then + echo "::error::Expected ${EXPECTED} HRX .debs, found ${found}." + exit 1 + fi + + - name: Stage HRX release assets + run: | + set -euo pipefail + version="${GITHUB_REF_NAME#v}" + mkdir assets + + # Same layout as the default zip: flm.exe and its DLLs at the top level. + (cd windows-dist && zip -r "../assets/HRX_fastflowlm_${version}_windows_amd64.zip" .) + cp linux-dist/HRX_fastflowlm_*_linux.tar.gz assets/ + cp debs/*/*.deb assets/ + ls -lah assets + + - name: Upload HRX assets to GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ github.ref_name }} + name: Release ${{ github.ref_name }} + files: assets/* + overwrite_files: true + fail_on_unmatched_files: true diff --git a/.github/workflows/ubuntu-build.yml b/.github/workflows/ubuntu-build.yml deleted file mode 100644 index b2a1e87f7..000000000 --- a/.github/workflows/ubuntu-build.yml +++ /dev/null @@ -1,165 +0,0 @@ -name: Build Deb Packages - -on: - push: - branches: - - main - tags: - - '*' - pull_request: - branches: - - main - workflow_dispatch: - -jobs: - # The d3 System One contract (src/common/decision) against fixtures written by the upstream - # package: prompt bytes, request/question errors and answer objects. CPU only, no NPU. - d3-contract: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Build and run the d3 contract test - run: | - cmake -S src/test/d3_contract -B build-d3-contract -DCMAKE_BUILD_TYPE=Release - cmake --build build-d3-contract -j"$(nproc)" - ctest --test-dir build-d3-contract --output-on-failure - - # continue_final_message (src/common/chat): where the prompt is cut on the template shapes the - # models ship, and how the output parser reads the continued text. CPU only, no NPU. - chat-continue: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Build and run the chat continue test - run: | - cmake -S src/test/chat_continue -B build-chat-continue -DCMAKE_BUILD_TYPE=Release - cmake --build build-chat-continue -j"$(nproc)" - ctest --test-dir build-chat-continue --output-on-failure - - package-deb: - runs-on: ubuntu-latest - strategy: - matrix: - include: - - version: "24.04" - codename: "noble" - distro: "ubuntu" - - version: "25.10" - codename: "questing" - distro: "ubuntu" - - version: "26.04" - codename: "resolute" - distro: "ubuntu" - - version: "13" - codename: "trixie" - distro: "debian" - - container: - image: ghcr.io/rocm/fastflowlm/build-environment:${{ matrix.distro }}${{ matrix.version }} - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - submodules: recursive - fetch-depth: 0 - fetch-tags: true - - - name: Configure git safe directory - run: git config --global --add safe.directory $(pwd) - - - name: Get version from git describe - id: get_version - run: | - VERSION=$(git describe --tags --always) - # Strip leading 'v' for Debian version compatibility - DEB_VERSION=$(echo "${VERSION}" | sed 's/^v//') - echo "version=${VERSION}" >> $GITHUB_OUTPUT - echo "deb_version=${DEB_VERSION}" >> $GITHUB_OUTPUT - - - name: Get short SHA - id: short_sha - run: echo "sha=$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_OUTPUT - - - name: Create Debian packaging metadata - working-directory: ${{ github.workspace }} - run: | - sed -e "s|@@DEB_VERSION@@|${{ steps.get_version.outputs.deb_version }}|g" \ - -e "s|@@DEB_CODENAME@@|${{ matrix.codename }}|g" \ - -e "s|@@DEB_DATE@@|$(date -R)|g" \ - debian/changelog.in > debian/changelog - - chmod +x debian/rules - - - name: Upgrade Rust toolchain - run: | - apt-get update - apt-get install -y --no-install-recommends curl ca-certificates - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ - sh -s -- -y --profile minimal --default-toolchain stable - echo "$HOME/.cargo/bin" >> $GITHUB_PATH - rustc --version - - # The build-environment image is rebuilt nightly from main's Dockerfile, so - # a Build-Depends change reaches it only after merge. Install the ones the - # image may still lack so dpkg-checkbuilddeps passes in the meantime. - - name: Install build dependencies missing from the image - run: | - apt-get install -y --no-install-recommends libedit-dev - - - name: Build Debian package - working-directory: ${{ github.workspace }} - env: - DEBEMAIL: noreply@example.com - DEBFULLNAME: fastflowlm - DEB_VERSION: ${{ steps.get_version.outputs.deb_version }} - run: | - dpkg-buildpackage -b -us -uc - mv ../fastflowlm_${DEB_VERSION}_amd64.deb ./fastflowlm_${DEB_VERSION}_${{ matrix.distro }}${{ matrix.version }}_amd64.deb - ls -lah ./fastflowlm_${DEB_VERSION}_${{ matrix.distro }}${{ matrix.version }}_amd64.deb - .github/script/check_no_copyleft_libs.sh \ - ./fastflowlm_${DEB_VERSION}_${{ matrix.distro }}${{ matrix.version }}_amd64.deb - - - name: Upload Debian package artifact - uses: actions/upload-artifact@v4 - with: - name: flm-${{ matrix.distro }}${{ matrix.version }}-${{ steps.short_sha.outputs.sha }}-deb - path: ./fastflowlm_${{ steps.get_version.outputs.deb_version }}_${{ matrix.distro }}${{ matrix.version }}_amd64.deb - if-no-files-found: error - - release: - permissions: - contents: write - runs-on: ubuntu-latest - needs: package-deb - if: github.ref_type == 'tag' - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Get version from git describe - id: get_version - run: | - VERSION=$(git describe --tags --always) - echo "version=${VERSION}" >> $GITHUB_OUTPUT - - - name: Download all deb artifacts - uses: actions/download-artifact@v4 - with: - path: debs - pattern: '*-deb' - - - name: Upload Binaries to Release - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - tag: ${{ github.ref }} - file_glob: true - file: debs/*/*.deb - overwrite: true diff --git a/.github/workflows/win-build.yml b/.github/workflows/win-build.yml new file mode 100644 index 000000000..ed20dae16 --- /dev/null +++ b/.github/workflows/win-build.yml @@ -0,0 +1,204 @@ +name: Windows Build + +# Builds, packages and uploads one or more Windows variants. The artifact names +# are what build-and-test.yml and release.yml download, so they are fixed per +# variant: +# +# XRT fastflowlm-windows- (+ fastflowlm-windows-msi-) +# HRX HRX-fastflowlm-windows- +# Source models-from-source-fastflowlm-windows- +# +# Build and Test (build-and-test.yml) calls this once per package on every pull +# request, push and tag; dispatch it to build packages without testing them. +on: + workflow_call: + inputs: + variants: + description: XRT, HRX, Source, a comma-separated list of them, or "all". + type: string + default: all + msi: + description: Also build the MSI installer when the XRT variant is built. + type: boolean + default: true + workflow_dispatch: + inputs: + variants: + description: Variant to build + type: choice + options: [all, XRT, HRX, Source] + default: all + msi: + description: Also build the MSI installer when the XRT variant is built + type: boolean + default: true + +# No concurrency group: build-and-test.yml calls this several times in one +# run, and the calls would cancel each other. The caller owns cancellation. + +jobs: + plan: + runs-on: ubuntu-latest + outputs: + include: ${{ steps.plan.outputs.include }} + msi: ${{ steps.plan.outputs.msi }} + steps: + - name: Expand variants + id: plan + env: + VARIANTS: ${{ inputs.variants || 'all' }} + MSI: ${{ inputs.msi }} + SHA: ${{ github.sha }} + run: | + set -euo pipefail + variants="${VARIANTS//,/ }" + if [ "$(echo ${variants})" = "all" ]; then + variants="XRT HRX Source" + fi + + include='[]' + has_xrt=false + for variant in ${variants}; do + case "${variant}" in + XRT) artifact="fastflowlm-windows-${SHA}"; has_xrt=true ;; + HRX) artifact="HRX-fastflowlm-windows-${SHA}" ;; + Source) artifact="models-from-source-fastflowlm-windows-${SHA}" ;; + *) + echo "::error::Unknown Windows build variant '${variant}'." + exit 1 + ;; + esac + include=$(jq -c --arg v "${variant}" --arg a "${artifact}" \ + '. + [{variant: $v, artifact: $a}]' <<< "${include}") + done + if [ "${include}" = '[]' ]; then + echo "::error::No Windows build variant given." + exit 1 + fi + + msi=false + if [ "${has_xrt}" = "true" ] && [ "${MSI}" != "false" ]; then + msi=true + fi + + echo "include=${include}" >> "$GITHUB_OUTPUT" + echo "msi=${msi}" >> "$GITHUB_OUTPUT" + + # The builds run on GitHub-hosted runners, outside the corporate proxy that + # intermittently blocks third-party downloads on the self-hosted ones. + # windows-2025-vs2026 carries Visual Studio 2026 for the windows-vs18 preset. + build: + name: build (${{ matrix.variant }}) + needs: plan + runs-on: windows-2025-vs2026 + strategy: + # Each variant ships on its own; one failing must not cancel the others. + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.include) }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Build + id: build + uses: ./.github/actions/win-build + with: + variant: ${{ matrix.variant }} + + - name: Prepare package files + shell: powershell + working-directory: src + env: + VARIANT: ${{ matrix.variant }} + BUILD_DIR: ${{ steps.build.outputs.build-dir }} + HRX_RUNTIME: ${{ steps.build.outputs.hrx-runtime }} + run: | + $ErrorActionPreference = "Stop" + $build = Join-Path $env:GITHUB_WORKSPACE $env:BUILD_DIR + $dist = "dist" + + if (Test-Path $dist) { + Remove-Item $dist -Recurse -Force + } + + New-Item -ItemType Directory -Force $dist | Out-Null + + # Main executable + Copy-Item -Path $build\flm.exe -Destination $dist\ -Force + + # Required data assets + Copy-Item -Path .\xclbins -Destination $dist\ -Recurse -Force + Copy-Item -Path .\model_list.json -Destination $dist\ -Force + Copy-Item -Path .\model_info.json -Destination $dist\ -Force + Copy-Item -Path ..\LICENSE_RUNTIME.txt -Destination $dist\ -Force + Copy-Item -Path ..\LICENSE_BINARY.txt -Destination $dist\ -Force + Copy-Item -Path ..\THIRD_PARTY_NOTICES.txt -Destination $dist\ -Force + + if ($env:VARIANT -eq 'HRX') { + Copy-Item -Path .\lib\*.dll -Destination $dist\ -Force + # HRX-variant op libraries override the default ones + Copy-Item -Path .\lib\hrx\*.dll -Destination $dist\ -Force + Copy-Item -Path $env:HRX_RUNTIME -Destination $dist\ -Force + } else { + Copy-Item -Path .\lib\xrt\*.dll -Destination $dist\ -Force + Copy-Item -Path .\lib\*.dll -Destination $dist\ -Force + } + + if ($env:VARIANT -eq 'Source') { + # Source-built engines override the prebuilt ones from lib\xrt. + Copy-Item -Path $build\engines\*.dll -Destination $dist\ -Force + + $sourceEngine = Get-FileHash $build\engines\gemma4e_npu.dll -Algorithm SHA256 + $prebuiltEngine = Get-FileHash .\lib\xrt\gemma4e_npu.dll -Algorithm SHA256 + $stagedEngine = Get-FileHash $dist\gemma4e_npu.dll -Algorithm SHA256 + if ($sourceEngine.Hash -eq $prebuiltEngine.Hash) { + throw "The source-built Gemma 4 DLL matches the prebuilt DLL." + } + if ($sourceEngine.Hash -ne $stagedEngine.Hash) { + throw "The staged Gemma 4 DLL does not match the source-built DLL." + } + } + + - name: Upload build artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.artifact }} + path: src/dist/ + if-no-files-found: error + + # Packages the flm.exe from the XRT build instead of building it again, so + # the MSI and the portable zip ship the same binary. Runs after a failed + # sibling variant too; it only needs the XRT artifact, and fails without it. + msi: + needs: [plan, build] + if: ${{ !cancelled() && needs.plan.outputs.msi == 'true' }} + runs-on: windows-2025 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download portable build artifact + uses: actions/download-artifact@v4 + with: + name: fastflowlm-windows-${{ github.sha }} + path: portable-dist + + - name: Build MSI installer + id: msi + uses: ./.github/actions/win-msi + with: + flm-exe: portable-dist/flm.exe + + - name: Upload MSI artifact + uses: actions/upload-artifact@v4 + with: + name: fastflowlm-windows-msi-${{ github.sha }} + path: ${{ steps.msi.outputs.msi }} + if-no-files-found: error + diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml deleted file mode 100644 index f92e97de1..000000000 --- a/.github/workflows/windows-build.yml +++ /dev/null @@ -1,543 +0,0 @@ -name: Build Windows Packages - -on: - push: - branches: - - main - - windows_zip_workflow - tags: - - '*' - pull_request: - branches: - - main - workflow_dispatch: - -# A build for a commit that has since been superseded on the same PR only -# delays that PR's model sweep, which waits for this whole run. Only -# pull_request runs share a group; every other run gets its own, so branch and -# tag builds are never cancelled. -concurrency: - group: windows-build-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: true - -# The builds run on GitHub-hosted runners, outside the corporate proxy that -# intermittently blocks third-party downloads on the self-hosted ones. -# windows-2025-vs2026 carries Visual Studio 2026 for the windows-vs18 preset; -# setup_windows_deps.ps1 lays out the C:\dev dependencies the self-hosted -# runner has, cached between runs. -jobs: - build-windows-portable: - runs-on: windows-2025-vs2026 - - defaults: - run: - working-directory: src - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - # checkout's fetch-depth: 0 also applies to submodules, which pulls the - # full msgpack-c history (~95 MB) and keeps getting reset mid-transfer on - # the runner network. The pinned commits alone are under 1 MB. - - name: Checkout submodules - shell: powershell - working-directory: . - run: | - for ($i = 1; $i -le 3; $i++) { - git submodule update --init --recursive --depth 1 --jobs 4 - if ($LASTEXITCODE -eq 0) { exit 0 } - Write-Host "Submodule fetch failed (attempt $i), retrying..." - Start-Sleep -Seconds 15 - } - exit 1 - - - name: Restore build dependencies - uses: actions/cache@v4 - with: - path: C:\dev - key: windows-deps-${{ hashFiles('.github/script/setup_windows_deps.ps1', '.github/windows-deps/**') }} - - - name: Set up build dependencies - shell: powershell - working-directory: . - run: .github\script\setup_windows_deps.ps1 - - - name: Configure CMake - shell: powershell - run: | - cmake --preset windows-vs18 - - - name: Build - shell: powershell - run: | - cmake --build --preset windows-default --config Release --parallel - - - name: Prepare package files - shell: powershell - run: | - $ErrorActionPreference = "Stop" - $dist = "dist" - - if (Test-Path $dist) { - Remove-Item $dist -Recurse -Force - } - - New-Item -ItemType Directory -Force $dist | Out-Null - - # Main executable - Copy-Item -Path .\build\flm.exe -Destination $dist\ -Force - - # Required data assets - Copy-Item -Path .\xclbins -Destination $dist\ -Recurse -Force - Copy-Item -Path .\model_list.json -Destination $dist\ -Force - Copy-Item -Path .\model_info.json -Destination $dist\ -Force - Copy-Item -Path ..\LICENSE_RUNTIME.txt -Destination $dist\ -Force - Copy-Item -Path ..\LICENSE_BINARY.txt -Destination $dist\ -Force - Copy-Item -Path ..\THIRD_PARTY_NOTICES.txt -Destination $dist\ -Force - Copy-Item -Path .\lib\xrt\*.dll -Destination $dist\ -Force - Copy-Item -Path .\lib\*.dll -Destination $dist\ -Force - - - name: Upload build artifact - uses: actions/upload-artifact@v4 - with: - name: fastflowlm-windows-${{ github.sha }} - path: src/dist/ - if-no-files-found: error - - # A second, independent configure that builds the model engines from source - # instead of using the prebuilt DLLs in lib\xrt. Its own job so the portable - # build (and the MSI, which packages it) does not wait on it. - build-windows-models-from-source: - runs-on: windows-2025-vs2026 - - defaults: - run: - working-directory: src - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - # checkout's fetch-depth: 0 also applies to submodules, which pulls the - # full msgpack-c history (~95 MB) and keeps getting reset mid-transfer on - # the runner network. The pinned commits alone are under 1 MB. - - name: Checkout submodules - shell: powershell - working-directory: . - run: | - for ($i = 1; $i -le 3; $i++) { - git submodule update --init --recursive --depth 1 --jobs 4 - if ($LASTEXITCODE -eq 0) { exit 0 } - Write-Host "Submodule fetch failed (attempt $i), retrying..." - Start-Sleep -Seconds 15 - } - exit 1 - - - name: Restore build dependencies - uses: actions/cache@v4 - with: - path: C:\dev - key: windows-deps-${{ hashFiles('.github/script/setup_windows_deps.ps1', '.github/windows-deps/**') }} - - - name: Set up build dependencies - shell: powershell - working-directory: . - run: .github\script\setup_windows_deps.ps1 - - - name: Configure models-from-source build - shell: powershell - run: | - cmake --preset windows-vs18 ` - -B build-models-from-source ` - -DFLM_BUILD_GEMMA4E=ON - - - name: Build models from source - shell: powershell - run: | - cmake --build build-models-from-source --config Release --parallel 2 - - - name: Prepare models-from-source package files - shell: powershell - run: | - $ErrorActionPreference = "Stop" - $dist = "dist-models-from-source" - - if (Test-Path $dist) { - Remove-Item $dist -Recurse -Force - } - - New-Item -ItemType Directory -Force $dist | Out-Null - Copy-Item -Path .\build-models-from-source\flm.exe -Destination $dist\ -Force - Copy-Item -Path .\xclbins -Destination $dist\ -Recurse -Force - Copy-Item -Path .\model_list.json -Destination $dist\ -Force - Copy-Item -Path .\model_info.json -Destination $dist\ -Force - Copy-Item -Path ..\LICENSE_RUNTIME.txt -Destination $dist\ -Force - Copy-Item -Path ..\LICENSE_BINARY.txt -Destination $dist\ -Force - Copy-Item -Path ..\THIRD_PARTY_NOTICES.txt -Destination $dist\ -Force - Copy-Item -Path .\lib\xrt\*.dll -Destination $dist\ -Force - Copy-Item -Path .\lib\*.dll -Destination $dist\ -Force - Copy-Item -Path .\build-models-from-source\engines\*.dll -Destination $dist\ -Force - - $sourceEngine = Get-FileHash .\build-models-from-source\engines\gemma4e_npu.dll -Algorithm SHA256 - $prebuiltEngine = Get-FileHash .\lib\xrt\gemma4e_npu.dll -Algorithm SHA256 - $stagedEngine = Get-FileHash $dist\gemma4e_npu.dll -Algorithm SHA256 - if ($sourceEngine.Hash -eq $prebuiltEngine.Hash) { - throw "The source-built Gemma 4 DLL matches the prebuilt DLL." - } - if ($sourceEngine.Hash -ne $stagedEngine.Hash) { - throw "The staged Gemma 4 DLL does not match the source-built DLL." - } - - - name: Upload models-from-source build artifact - uses: actions/upload-artifact@v4 - with: - name: models-from-source-fastflowlm-windows-${{ github.sha }} - path: src/dist-models-from-source/ - if-no-files-found: error - - # Packages the flm.exe built by build-windows-portable instead of building it - # again, so the MSI and the portable zip ship the same binary. Everything - # else the installer stages (DLLs, xclbins, model lists) comes from the - # checkout, so no submodules or history are needed here. - build-windows-msi: - runs-on: windows-2025 - needs: build-windows-portable - - defaults: - run: - working-directory: src - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Download portable build artifact - uses: actions/download-artifact@v4 - with: - name: fastflowlm-windows-${{ github.sha }} - path: src/portable-dist - - # get_files.bat (inno and wix) stage flm.exe from build\, where a local - # build puts it. - - name: Place flm.exe from portable build - shell: powershell - run: | - $ErrorActionPreference = "Stop" - New-Item -ItemType Directory -Force build | Out-Null - Copy-Item -Path .\portable-dist\flm.exe -Destination .\build\flm.exe -Force - - - name: Stage installer package files - shell: powershell - run: | - cd inno - .\get_files.bat - - - # The image only has WiX 3; flm.wxs is WiX 5. Same version as the - # self-hosted runner had. - - name: Install WiX - shell: powershell - run: | - $ErrorActionPreference = "Stop" - dotnet tool install --global wix --version 5.0.2 - if ($LASTEXITCODE -ne 0) { throw "dotnet tool install wix failed" } - $wixToolsPath = Join-Path $env:USERPROFILE ".dotnet\tools" - & "$wixToolsPath\wix.exe" extension add --global WixToolset.UI.wixext/5.0.2 - if ($LASTEXITCODE -ne 0) { throw "wix extension add failed" } - - $wixToolsPath | Out-File ` - -FilePath $env:GITHUB_PATH ` - -Encoding utf8 ` - -Append - - - name: Verify WiX - shell: powershell - run: | - $wix = Get-Command wix -ErrorAction Stop - Write-Host "Using WiX: $($wix.Source)" - wix --version - - - - name: Build MSI installer - shell: powershell - run: | - .\wix\build.ps1 - - - name: Upload MSI artifact - uses: actions/upload-artifact@v4 - with: - name: fastflowlm-windows-msi-${{ github.sha }} - path: src/wix/output/flm-setup.msi - if-no-files-found: error - - build-windows-portable-hrx: - runs-on: windows-2025-vs2026 - - defaults: - run: - working-directory: src - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - # checkout's fetch-depth: 0 also applies to submodules, which pulls the - # full msgpack-c history (~95 MB) and keeps getting reset mid-transfer on - # the runner network. The pinned commits alone are under 1 MB. - - name: Checkout submodules - shell: powershell - working-directory: . - run: | - for ($i = 1; $i -le 3; $i++) { - git submodule update --init --recursive --depth 1 --jobs 4 - if ($LASTEXITCODE -eq 0) { exit 0 } - Write-Host "Submodule fetch failed (attempt $i), retrying..." - Start-Sleep -Seconds 15 - } - exit 1 - - # HRX needs the same Boost and curl as the XRT build; the XRT parts of - # the tree go unused here. - - name: Restore build dependencies - uses: actions/cache@v4 - with: - path: C:\dev - key: windows-deps-${{ hashFiles('.github/script/setup_windows_deps.ps1', '.github/windows-deps/**') }} - - - name: Set up build dependencies - shell: powershell - working-directory: . - run: .github\script\setup_windows_deps.ps1 - - - name: Configure CMake (HRX) - shell: powershell - run: | - $ErrorActionPreference = "Stop" - $lines = & '..\hrx-integration\fetch-hrx-release.ps1' 6>&1 | ForEach-Object { $_.ToString() } - $lines | ForEach-Object { Write-Host $_ } - $prefix = ($lines | Where-Object { $_ -like 'HRX_CMAKE_PREFIX=*' } | Select-Object -First 1) -replace 'HRX_CMAKE_PREFIX=', '' - if (-not $prefix) { - throw 'failed to determine HRX_CMAKE_PREFIX from fetch-hrx-release.ps1 output' - } - cmake --preset windows-vs18 -DFLM_USE_HRX=ON "-DCMAKE_PREFIX_PATH=$prefix" - - - name: Build - shell: powershell - run: | - cmake --build --preset windows-default --config Release --parallel - - - name: Prepare package files - shell: powershell - run: | - $ErrorActionPreference = "Stop" - $dist = "dist" - - if (Test-Path $dist) { - Remove-Item $dist -Recurse -Force - } - - New-Item -ItemType Directory -Force $dist | Out-Null - - # Main executable - Copy-Item -Path .\build\flm.exe -Destination $dist\ -Force - - # Required data assets - Copy-Item -Path .\xclbins -Destination $dist\ -Recurse -Force - Copy-Item -Path .\model_list.json -Destination $dist\ -Force - Copy-Item -Path .\model_info.json -Destination $dist\ -Force - Copy-Item -Path ..\LICENSE_RUNTIME.txt -Destination $dist\ -Force - Copy-Item -Path ..\LICENSE_BINARY.txt -Destination $dist\ -Force - Copy-Item -Path ..\THIRD_PARTY_NOTICES.txt -Destination $dist\ -Force - Copy-Item -Path .\lib\*.dll -Destination $dist\ -Force - - # HRX-variant op libraries override the default ones - Copy-Item -Path .\lib\hrx\*.dll -Destination $dist\ -Force - - # HRX runtime, fetched by the Configure CMake (HRX) step - $hrxDll = Get-ChildItem -Path ..\hrx-integration\.hrx-release -Recurse -Filter 'hrx.dll' -ErrorAction SilentlyContinue | Select-Object -First 1 - if (-not $hrxDll) { - throw 'hrx.dll not found under hrx-integration\.hrx-release; HRX fetch step may have failed' - } - Copy-Item -Path $hrxDll.FullName -Destination $dist\ -Force - - - name: Upload build artifact - uses: actions/upload-artifact@v4 - with: - name: HRX-fastflowlm-windows-${{ github.sha }} - path: src/dist/ - if-no-files-found: error - - # build-windows-msi-hrx: - # runs-on: [self-hosted, windows, x64, msi] - - # defaults: - # run: - # working-directory: src - - # steps: - # - name: Checkout code - # uses: actions/checkout@v4 - # with: - # submodules: recursive - # fetch-depth: 0 - # fetch-tags: true - - # - name: Configure CMake (HRX) - # shell: powershell - # run: | - # $ErrorActionPreference = "Stop" - # $lines = & '..\hrx-integration\fetch-hrx-release.ps1' 6>&1 | ForEach-Object { $_.ToString() } - # $lines | ForEach-Object { Write-Host $_ } - # $prefix = ($lines | Where-Object { $_ -like 'HRX_CMAKE_PREFIX=*' } | Select-Object -First 1) -replace 'HRX_CMAKE_PREFIX=', '' - # if (-not $prefix) { - # throw 'failed to determine HRX_CMAKE_PREFIX from fetch-hrx-release.ps1 output' - # } - # cmake --preset windows-vs18 -DFLM_USE_HRX=ON "-DCMAKE_PREFIX_PATH=$prefix" - - # - name: Build - # shell: powershell - # run: | - # cmake --build --preset windows-default --config Release --parallel - - # - name: Stage installer package files - # shell: powershell - # run: | - # cd inno - # .\get_files.bat - - # - name: Overlay HRX runtime files - # shell: powershell - # run: | - # $ErrorActionPreference = "Stop" - - # # HRX-variant op libraries override the default ones staged above - # Copy-Item -Path .\lib\hrx\*.dll -Destination .\inno\ -Force - - # # HRX runtime, fetched by the Configure CMake (HRX) step - # $hrxDll = Get-ChildItem -Path ..\hrx-integration\.hrx-release -Recurse -Filter 'hrx.dll' -ErrorAction SilentlyContinue | Select-Object -First 1 - # if (-not $hrxDll) { - # throw 'hrx.dll not found under hrx-integration\.hrx-release; HRX fetch step may have failed' - # } - # Copy-Item -Path $hrxDll.FullName -Destination .\inno\hrx.dll -Force - - # - name: Add WiX to PATH - # shell: powershell - # run: | - # $wixToolsPath = "C:\Users\zani\.dotnet\tools" - - # if (-not (Test-Path "$wixToolsPath\wix.exe")) { - # throw "WiX was not found at $wixToolsPath\wix.exe" - # } - - # $wixToolsPath | Out-File ` - # -FilePath $env:GITHUB_PATH ` - # -Encoding utf8 ` - # -Append - - # - name: Verify WiX - # shell: powershell - # run: | - # Write-Host "Runner identity: $(whoami)" - # Write-Host "Runner profile: $env:USERPROFILE" - - # $wix = Get-Command wix -ErrorAction Stop - # Write-Host "Using WiX: $($wix.Source)" - # wix --version - - # - name: Build MSI installer - # shell: powershell - # run: | - # .\wix\build.ps1 -Hrx - - # - name: Upload MSI artifact - # uses: actions/upload-artifact@v4 - # with: - # name: HRX-fastflowlm-windows-msi-${{ github.sha }} - # path: src/wix/output/flm-setup.msi - # if-no-files-found: error - - release: - runs-on: ubuntu-latest - needs: [build-windows-portable, build-windows-msi] - if: github.ref_type == 'tag' - permissions: - contents: write - - steps: - - name: Download artifacts - uses: actions/download-artifact@v4 - with: - path: release-assets - - - name: Create release zip - run: | - set -euo pipefail - version="${GITHUB_REF_NAME#v}" - zip_name="fastflowlm_${version}_windows_amd64.zip" - - # Zip the portable dist payload once for the GitHub Release asset. - cd "release-assets/fastflowlm-windows-${{ github.sha }}" - zip -r "../../${zip_name}" . - - - name: Stage MSI installer - run: | - set -euo pipefail - cp "release-assets/fastflowlm-windows-msi-${{ github.sha }}/flm-setup.msi" ./flm-setup.msi - - - name: Upload assets to GitHub Release - uses: softprops/action-gh-release@v2 - with: - tag_name: ${{ github.ref_name }} - name: Release ${{ github.ref_name }} - files: | - fastflowlm_*.zip - flm-setup.msi - overwrite_files: true - fail_on_unmatched_files: true - - # The HRX build ships as a preview, so it gets its own release job: a failed - # HRX build leaves the release without the HRX asset instead of holding back - # the default ones. - release-hrx: - runs-on: ubuntu-latest - needs: build-windows-portable-hrx - if: github.ref_type == 'tag' - permissions: - contents: write - - steps: - - name: Download HRX portable artifact - uses: actions/download-artifact@v4 - with: - name: HRX-fastflowlm-windows-${{ github.sha }} - path: hrx-dist - - - name: Create HRX release zip - run: | - set -euo pipefail - version="${GITHUB_REF_NAME#v}" - zip_name="HRX_fastflowlm_${version}_windows_amd64.zip" - - # Same layout as the default zip: flm.exe and its DLLs at the top level. - cd hrx-dist - zip -r "../${zip_name}" . - - - name: Upload HRX assets to GitHub Release - uses: softprops/action-gh-release@v2 - with: - tag_name: ${{ github.ref_name }} - name: Release ${{ github.ref_name }} - files: | - HRX_fastflowlm_*.zip - overwrite_files: true - fail_on_unmatched_files: true diff --git a/debian/control b/debian/control index f39bdb13c..0b0f23195 100644 --- a/debian/control +++ b/debian/control @@ -21,6 +21,6 @@ Standards-Version: 4.7.3 Package: fastflowlm Architecture: amd64 -Depends: ${shlibs:Depends}, ${misc:Depends}, libxrt-npu2 +Depends: ${shlibs:Depends}, ${misc:Depends}, ${flm:RuntimeDepends} Description: FastFlowLM CLI FastFlowLM runtime and CLI. diff --git a/debian/rules b/debian/rules index 60d66f44e..96d8d11cf 100755 --- a/debian/rules +++ b/debian/rules @@ -1,4 +1,17 @@ #!/usr/bin/make -f + +# Backend selection, read from the environment so CI can build the HRX package +# from the same tree: FLM_USE_HRX=ON plus FLM_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=". +FLM_USE_HRX ?= OFF +FLM_CMAKE_ARGS ?= + +# The XRT package needs the system XRT runtime; the HRX one bundles libhrx. +ifeq ($(FLM_USE_HRX),ON) +FLM_RUNTIME_DEPENDS := +else +FLM_RUNTIME_DEPENDS := libxrt-npu2 +endif + %: dh $@ @@ -9,7 +22,7 @@ override_dh_dwz: # dh_dwz is skipped because it causes issues with the binary structure override_dh_auto_configure: - cmake -S src -B build --preset linux-default + cmake -S src -B build --preset linux-default -DFLM_USE_HRX=$(FLM_USE_HRX) $(FLM_CMAKE_ARGS) override_dh_auto_build: cmake --build build @@ -22,3 +35,6 @@ override_dh_auto_install: # Remove include directories rm -rf debian/fastflowlm/opt/fastflowlm/include + +override_dh_gencontrol: + dh_gencontrol -- -Vflm:RuntimeDepends="$(FLM_RUNTIME_DEPENDS)" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index bdf6a4711..48f118bed 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -599,6 +599,17 @@ if(FLM_USE_HRX) if(NOT WIN32) set_target_properties(flm PROPERTIES BUILD_RPATH "$;${FLM_ENGINE_LIB_DIR}") + # The HRX package ships its own libtinfo (and libedit, curl, ...) next to + # libhrx, and the linker looks in BUILD_RPATH for the dependencies of + # shared libraries. The system libedit linked below for non-portable + # builds would then resolve its libtinfo to the HRX copy, which lacks the + # NCURSES6_TINFO symbol versions, and the link fails. -rpath-link is + # searched first and only at link time, so the system copies win and the + # installed RPATH is unchanged. + if(NOT FLM_PORTABLE_BUILD AND CMAKE_LIBRARY_ARCHITECTURE) + target_link_options(flm PRIVATE + "LINKER:-rpath-link,/usr/lib/${CMAKE_LIBRARY_ARCHITECTURE}") + endif() endif() else() # XRT: link xrt_coreutil (+ aiebu for the ELF assembler used by npu_utils_xrt).