From e52e3a29bec9115c4736f3c5dfc4f852d1fb2c1a Mon Sep 17 00:00:00 2001 From: scama Date: Tue, 15 Sep 2026 07:49:36 +1000 Subject: [PATCH 1/9] build: make accelerators repository defaults --- .../manage-ci/references/current-inventory.md | 44 ++--- .cargo/config.toml | 33 +++- .../actions/compute-changes/derive-outputs.sh | 6 +- .github/actions/setup-macos-lld/action.yml | 21 +-- .../setup-windows-short-paths/action.yml | 21 +++ .github/workflows/cache-warm-sccache.yml | 3 +- .github/workflows/ci-linux-host-slice.yml | 3 +- .github/workflows/ci-linux-runtime-slice.yml | 2 +- .../workflows/ci-platform-checks-slice.yml | 2 + .github/workflows/ci-quality-slice.yml | 13 +- .github/workflows/ci-rust-tests-slice.yml | 6 +- .github/workflows/ci-windows-host-slice.yml | 1 + .../workflows/ci-windows-runtime-slice.yml | 1 + .github/workflows/depot-canary.yml | 4 +- .github/workflows/hf-download-smoke.yml | 7 - .github/workflows/llama-upstream-canary.yml | 48 ++++- .github/workflows/nightly-kv-coverage.yml | 1 - .github/workflows/node-sdk-addon-artifact.yml | 3 +- .github/workflows/release.yml | 17 +- .github/workflows/sdk-smoke.yml | 13 +- .github/workflows/windows-warm-caches.yml | 2 + CONTRIBUTING.md | 28 ++- README.md | 8 +- ci/ci.md | 16 ++ ci/runner-images.json | 7 +- just/build.just | 62 ++----- scripts/bootstrap-build-tools | 41 +++++ scripts/bootstrap-build-tools.ps1 | 30 ++++ scripts/build-host.sh | 48 ----- scripts/build-llama.sh | 4 + scripts/build-windows.ps1 | 4 + scripts/cargo-linker | 165 ++++++++++++++++++ scripts/cargo-linker.cmd | 26 +++ scripts/lib/lld.sh | 3 +- scripts/package-native-runtime.sh | 19 +- .../tests/test_build_accelerator_defaults.py | 82 +++++++++ scripts/tests/test_ci_artifact_actions.py | 2 +- .../tests/test_ci_native_toolchain_routing.py | 2 +- scripts/tests/test_justfile_layout.py | 2 +- .../test_kv_nightly_workflow_contract.py | 1 + .../test_llama_upstream_canary_contract.py | 10 +- scripts/tests/test_lld_lib.py | 34 ++-- scripts/tests/test_package_native_runtime.py | 20 +-- scripts/tests/test_sccache_evidence.py | 5 +- 44 files changed, 599 insertions(+), 271 deletions(-) create mode 100644 .github/actions/setup-windows-short-paths/action.yml create mode 100755 scripts/bootstrap-build-tools create mode 100644 scripts/bootstrap-build-tools.ps1 create mode 100755 scripts/cargo-linker create mode 100644 scripts/cargo-linker.cmd create mode 100644 scripts/tests/test_build_accelerator_defaults.py diff --git a/.agents/skills/manage-ci/references/current-inventory.md b/.agents/skills/manage-ci/references/current-inventory.md index 30b9d98850..8c90129343 100644 --- a/.agents/skills/manage-ci/references/current-inventory.md +++ b/.agents/skills/manage-ci/references/current-inventory.md @@ -458,29 +458,14 @@ bump `@playwright/test` in `crates/mesh-llm-ui/package.json`. ### `setup-macos-lld` composite -`.github/actions/setup-macos-lld` replaces per-callsite -`brew install lld` plus a hand-rolled `PATH`/`RUSTFLAGS` export with one -composite: install lld via brew, resolve `$(brew --prefix lld)/bin`, link a -real `edition = "2024"` probe binary with `-Clink-arg=-fuse-ld=lld`, then -export `CARGO_ENCODED_RUSTFLAGS=-Clink-arg=-fuse-ld=lld` and the resolved bin -directory. `CARGO_ENCODED_RUSTFLAGS` **replaces** any -`target..rustflags` from a checked-in `.cargo/config.toml` rather -than merging with it -- confirmed safe here because every call site is -macOS-gated and no call site touches an `android` target (the repo's -`.cargo/config.toml` android entries carry a `-Wl,-z,max-page-size=16384` -flag that would otherwise silently stop applying). Seven call sites: +`.github/actions/setup-macos-lld` installs lld through Homebrew, adds its +resolved bin directory to the job path, and invokes the checked-in +`scripts/cargo-linker` probe. It does not export Rust flags. The repository +Cargo config owns every later link, so Android target flags and caller flags +continue to compose normally. Seven call sites: `ci-platform-checks-slice.yml`, `ci-macos-host-slice.yml`, `swift-sdk-artifact.yml`, `native-sdk-artifact.yml`, -`node-sdk-addon-artifact.yml`, and two in `release.yml`. Only the first -three were reachable by the temporary branch-head harness used to validate -PR #1380 (no macOS row in -`native-sdk-artifact.yml`/`node-sdk-addon-artifact.yml` ran there, and -`release.yml` only runs on an actual release cut) -- the other four are -statically cleared (macOS-gated, no android target on any of them) rather -than proven by a real run. `node-sdk-addon-artifact.yml`'s own -`Validate macOS x64 cross-linker` step is a deliberate near-duplicate of the -composite's probe, not dead code: it passes `--target x86_64-apple-darwin` -where the composite only probes the host target. +`node-sdk-addon-artifact.yml`, and two in `release.yml`. ### A pinned digest is a frozen artifact @@ -549,6 +534,23 @@ fail-open policy. ## Artifact and cache owners +Repository Cargo defaults require `sccache` and select a target-specific +linker driver. Full Linux runner images provide mold as the primary linker and +lld as the compatibility control. macOS jobs install lld through the shared +setup action; an installed ld64.lld that fails the active SDK/target probe +falls back to Apple ld. Windows resolves rust-lld or lld-link. The Unix probe +cache includes the linker, compiler, SDK, host and target-driver identity. + +The persistent macOS llama canary keeps a stable arm64-only sccache directory +and adds an explicit C/C++ cache buster over architecture, backend, profile, +compiler/SDK identity and native recipe inputs. Its CMake build directory +remains run-unique, every staged archive is checked with `lipo`, and cache +statistics are retained in the job log. The nightly KV workflow no longer +clears the repository Rust wrapper. Every managed Windows compile job uses +sccache with short `C:\\s` and `C:\\t` roots, and every Windows native backend +asks CMake to hash object paths at 180 characters before the legacy MAX_PATH +boundary. + - `restore-release-ui` / `scripts/ui-distribution.py`: verify the shared release console's source SHA, version, complete file hashes and built JavaScript entry before platform-specific Rust compilation or SDK resource packaging. The diff --git a/.cargo/config.toml b/.cargo/config.toml index ff5e5605d4..55a4762d96 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,3 +1,24 @@ +[build] +rustc-wrapper = "sccache" + +[target.aarch64-apple-darwin] +linker = "scripts/cargo-linker" + +[target.x86_64-apple-darwin] +linker = "scripts/cargo-linker" + +[target.aarch64-unknown-linux-gnu] +linker = "scripts/cargo-linker" + +[target.x86_64-unknown-linux-gnu] +linker = "scripts/cargo-linker" + +[target.aarch64-pc-windows-msvc] +linker = "scripts/cargo-linker.cmd" + +[target.x86_64-pc-windows-msvc] +linker = "scripts/cargo-linker.cmd" + [target.aarch64-linux-android] rustflags = ["-C", "link-arg=-Wl,-z,max-page-size=16384"] @@ -7,10 +28,8 @@ rustflags = ["-C", "link-arg=-Wl,-z,max-page-size=16384"] [target.x86_64-linux-android] rustflags = ["-C", "link-arg=-Wl,-z,max-page-size=16384"] -# No macOS linker is configured here on purpose. A checked-in absolute -# `-fuse-ld=/opt/homebrew/bin/ld64.lld` applied to every cargo invocation, -# hardcoded an Apple-Silicon Homebrew path, and could not check that the -# linker works first. When lld fell behind the macOS SDK, every `cargo build` -# and `cargo test` failed to link while `cargo check` stayed green, because -# check never links. lld is now resolved and probed at build time by -# scripts/lib/lld.sh, which `just with-lld` and scripts/build-host.sh use. +# The repository linker drivers resolve and probe the active platform linker +# before they execute it. Linux prefers mold; macOS prefers Homebrew ld64.lld +# and falls back to Apple ld when the installed lld cannot parse the active +# SDK. Windows resolves rust-lld/lld-link and fails with setup instructions +# when neither is installed. diff --git a/.github/actions/compute-changes/derive-outputs.sh b/.github/actions/compute-changes/derive-outputs.sh index 62c238248b..c0cd3f3d82 100644 --- a/.github/actions/compute-changes/derive-outputs.sh +++ b/.github/actions/compute-changes/derive-outputs.sh @@ -349,7 +349,7 @@ BACKEND_CHANGED="false" if [[ "$ALL_RUST" == "true" ]]; then BACKEND_CHANGED="true" elif [[ -n "$CHANGED_FILES" ]]; then - BACKEND_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^third_party/llama\.cpp/|^crates/skippy-ffi/|^scripts/(build-llama|prepare-llama|build-linux|build-linux-rocm|build-mac|build-windows|install-windows-sdk|build-host|build-release|package-release|package-native-runtime|verify-native-runtime-package|verify-checksum-sidecar|safe-extract-tar|compose-product-bundle|ci-compose-product-input|ci-client-readiness-smoke)\.|^\.github/actions/(prepare-host-input|prepare-windows-host-input|prepare-native-runtime-input|compose-product-input|resolve-native-toolchain-epoch|restore-smoke-inputs|restore-windows-abi-cache|save-and-verify-actions-cache|setup-windows-rocm-sdk)/|^\.github/workflows/(ci|main_[a-z]+|pr_[a-z]+|release|sdk-smoke|smoke)\.yml$|^\.github/cache-version\.txt$)' || true) + BACKEND_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^third_party/llama\.cpp/|^crates/skippy-ffi/|^scripts/(build-llama|prepare-llama|build-linux|build-linux-rocm|build-mac|build-windows|install-windows-sdk|build-host|build-release|package-release|package-native-runtime|verify-native-runtime-package|verify-checksum-sidecar|safe-extract-tar|compose-product-bundle|ci-compose-product-input|ci-client-readiness-smoke)\.|^\.github/actions/(prepare-host-input|prepare-windows-host-input|prepare-native-runtime-input|compose-product-input|resolve-native-toolchain-epoch|restore-smoke-inputs|restore-windows-abi-cache|save-and-verify-actions-cache|setup-windows-rocm-sdk|setup-windows-short-paths)/|^\.github/workflows/(ci|main_[a-z]+|pr_[a-z]+|release|sdk-smoke|smoke)\.yml$|^\.github/cache-version\.txt$)' || true) if [[ -n "$BACKEND_INPUTS" ]] || [[ "$BACKEND_RECIPE_CHANGED" == "true" ]]; then BACKEND_CHANGED="true" fi @@ -361,8 +361,8 @@ if [[ "$FORCE_ALL" == "true" ]]; then WINDOWS_CPU_BUILD_REQUIRED="true" WINDOWS_GPU_BUILD_REQUIRED="true" elif [[ -n "$CHANGED_FILES" ]]; then - WINDOWS_CPU_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^crates/mesh-llm-release-footer/|^crates/mesh-llm-nodejs/|^crates/skippy-ffi/|^scripts/(build-windows|package-release)\.ps1$|^scripts/verify-host-dependencies\.py$|^scripts/(package-native-runtime|verify-native-runtime-package|verify-checksum-sidecar|safe-extract-tar|compose-product-bundle|ci-compose-product-input|ci-client-readiness-smoke)\.|^third_party/llama\.cpp/|^Cargo\.toml$|^Cargo\.lock$|^\.github/cache-version\.txt$|^\.github/workflows/(ci|main_[a-z]+|pr_[a-z]+|release|windows-warm-caches)\.yml$|^\.github/actions/(compute-changes/|prepare-windows-host-input/|prepare-native-runtime-input/|compose-product-input/|resolve-native-toolchain-epoch/|restore-windows-abi-cache/|save-and-verify-actions-cache/))' || true) - WINDOWS_GPU_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^crates/skippy-ffi/|^scripts/(build-windows|install-windows-sdk|package-release)\.ps1$|^scripts/verify-host-dependencies\.py$|^scripts/(package-native-runtime|verify-native-runtime-package|verify-checksum-sidecar|safe-extract-tar|compose-product-bundle|ci-compose-product-input|ci-client-readiness-smoke)\.|^scripts/windows-native-runtime-deps\.py$|^scripts/tests/test_windows_native_runtime_deps\.py$|^third_party/llama\.cpp/|^\.github/cache-version\.txt$|^\.github/workflows/(ci|main_[a-z]+|pr_[a-z]+|release|windows-warm-caches)\.yml$|^\.github/actions/(compute-changes/|prepare-windows-host-input/|prepare-native-runtime-input/|compose-product-input/|resolve-native-toolchain-epoch/|restore-windows-abi-cache/|save-and-verify-actions-cache/|setup-windows-rocm-sdk/))' || true) + WINDOWS_CPU_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^crates/mesh-llm-release-footer/|^crates/mesh-llm-nodejs/|^crates/skippy-ffi/|^scripts/(build-windows|package-release)\.ps1$|^scripts/verify-host-dependencies\.py$|^scripts/(package-native-runtime|verify-native-runtime-package|verify-checksum-sidecar|safe-extract-tar|compose-product-bundle|ci-compose-product-input|ci-client-readiness-smoke)\.|^third_party/llama\.cpp/|^Cargo\.toml$|^Cargo\.lock$|^\.github/cache-version\.txt$|^\.github/workflows/(ci|main_[a-z]+|pr_[a-z]+|release|windows-warm-caches)\.yml$|^\.github/actions/(compute-changes/|prepare-windows-host-input/|prepare-native-runtime-input/|compose-product-input/|resolve-native-toolchain-epoch/|restore-windows-abi-cache/|save-and-verify-actions-cache/|setup-windows-short-paths/))' || true) + WINDOWS_GPU_INPUTS=$(echo "$CHANGED_FILES" | grep -E '(^crates/skippy-ffi/|^scripts/(build-windows|install-windows-sdk|package-release)\.ps1$|^scripts/verify-host-dependencies\.py$|^scripts/(package-native-runtime|verify-native-runtime-package|verify-checksum-sidecar|safe-extract-tar|compose-product-bundle|ci-compose-product-input|ci-client-readiness-smoke)\.|^scripts/windows-native-runtime-deps\.py$|^scripts/tests/test_windows_native_runtime_deps\.py$|^third_party/llama\.cpp/|^\.github/cache-version\.txt$|^\.github/workflows/(ci|main_[a-z]+|pr_[a-z]+|release|windows-warm-caches)\.yml$|^\.github/actions/(compute-changes/|prepare-windows-host-input/|prepare-native-runtime-input/|compose-product-input/|resolve-native-toolchain-epoch/|restore-windows-abi-cache/|save-and-verify-actions-cache/|setup-windows-rocm-sdk/|setup-windows-short-paths/))' || true) if echo "$CHANGED_FILES" | grep -Eq '^\.github/workflows/(main|pr)_[a-z]+\.yml$'; then WINDOWS_CPU_INPUTS="ci-entry-workflow" WINDOWS_GPU_INPUTS="ci-entry-workflow" diff --git a/.github/actions/setup-macos-lld/action.yml b/.github/actions/setup-macos-lld/action.yml index bf37860a37..507598e989 100644 --- a/.github/actions/setup-macos-lld/action.yml +++ b/.github/actions/setup-macos-lld/action.yml @@ -1,8 +1,7 @@ name: Set up macOS lld linker description: > - Install and verify Homebrew's lld on a macOS runner. Always probe-links a - throwaway crate before any caller trusts the result -- a bad install must - fail loudly, not build silently against the wrong (or no) linker. + Install Homebrew's lld on a macOS runner and run the repository linker + probe. Cargo links keep using the same repository driver afterwards. Deliberately uncached: mesh-llm's Actions cache is already over the 10GB per-repo quota (see gh cache list), so a ~1.9GB lld/llvm/z3 Cellar entry here would evict the far more expensive macOS Rust target/ cache for a @@ -16,23 +15,11 @@ runs: shell: bash run: brew install lld - - name: Verify lld links a real binary + - name: Verify repository linker selection shell: bash run: | set -euo pipefail lld_bin="$(brew --prefix lld)/bin" test -x "$lld_bin/ld64.lld" - probe="$RUNNER_TEMP/mesh-setup-macos-lld-probe" - mkdir -p "$probe/src" - printf '[package]\nname = "mesh-setup-macos-lld-probe"\nversion = "0.0.0"\nedition = "2024"\n' > "$probe/Cargo.toml" - printf 'fn main() {}\n' > "$probe/src/main.rs" - PATH="$lld_bin:$PATH" CARGO_ENCODED_RUSTFLAGS='-Clink-arg=-fuse-ld=lld' \ - cargo rustc --quiet --manifest-path "$probe/Cargo.toml" echo "$lld_bin" >> "$GITHUB_PATH" - # Job-global via GITHUB_ENV, and cargo prefers CARGO_ENCODED_RUSTFLAGS - # over RUSTFLAGS -- any later step in the same job that sets RUSTFLAGS - # is silently ignored, and any macOS job that cross-builds an android - # target loses .cargo/config.toml's -Wl,-z,max-page-size=16384 for - # that job. Verified safe today: no other RUSTFLAGS setter and no - # android targets on any of this action's macOS call sites. - echo 'CARGO_ENCODED_RUSTFLAGS=-Clink-arg=-fuse-ld=lld' >> "$GITHUB_ENV" + PATH="$lld_bin:$PATH" scripts/cargo-linker --mesh-probe >/dev/null diff --git a/.github/actions/setup-windows-short-paths/action.yml b/.github/actions/setup-windows-short-paths/action.yml new file mode 100644 index 0000000000..b5bc34921e --- /dev/null +++ b/.github/actions/setup-windows-short-paths/action.yml @@ -0,0 +1,21 @@ +name: Configure short Windows build paths +description: Keep compiler cache and temporary files below Windows toolchain path limits. + +runs: + using: composite + steps: + - name: Create short compiler paths + shell: pwsh + run: | + $ErrorActionPreference = "Stop" + $paths = @{ + SCCACHE_DIR = "C:\s" + TEMP = "C:\t" + TMP = "C:\t" + } + New-Item -ItemType Directory -Force -Path $paths.SCCACHE_DIR, $paths.TEMP | Out-Null + foreach ($entry in $paths.GetEnumerator()) { + "$($entry.Key)=$($entry.Value)" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + } + Write-Host "Windows compiler cache: $($paths.SCCACHE_DIR)" + Write-Host "Windows temporary root: $($paths.TEMP)" diff --git a/.github/workflows/cache-warm-sccache.yml b/.github/workflows/cache-warm-sccache.yml index 8ed24181eb..8467fa1dc4 100644 --- a/.github/workflows/cache-warm-sccache.yml +++ b/.github/workflows/cache-warm-sccache.yml @@ -26,7 +26,6 @@ jobs: CARGO_INCREMENTAL: "0" MESH_LLM_SKIP_UI: "1" RUSTC_WRAPPER: sccache - RUSTFLAGS: "-C link-arg=-fuse-ld=lld" SCCACHE_GHA_ENABLED: "false" SCCACHE_CACHE_SIZE: 2G steps: @@ -41,7 +40,7 @@ jobs: shell: bash run: | set -euo pipefail - key="mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', 'Justfile', 'just/**') }}" + key="mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/lib/lld.sh', 'Justfile', 'just/**') }}" echo "key=$key" >> "$GITHUB_OUTPUT" - name: Check for an existing exact seed id: restore diff --git a/.github/workflows/ci-linux-host-slice.yml b/.github/workflows/ci-linux-host-slice.yml index 22cf7e8269..9e657eadde 100644 --- a/.github/workflows/ci-linux-host-slice.yml +++ b/.github/workflows/ci-linux-host-slice.yml @@ -108,7 +108,6 @@ jobs: MESH_LLM_SKIP_UI: "1" MESH_LLM_REQUIRE_SCCACHE: "1" RUSTC_WRAPPER: sccache - RUSTFLAGS: "-C link-arg=-fuse-ld=lld" steps: - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@ed07043b84d720aab30e75ed2f038f7042576f16 with: @@ -126,7 +125,7 @@ jobs: uses: ./.github/actions/restore-sccache-seed with: allow_trusted_seed: ${{ needs.runner_policy.outputs.allow_trusted_sccache_seed == 'true' && matrix.host.architecture == 'x86_64' && 'true' || 'false' }} - cache_key: mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', 'Justfile', 'just/**') }} + cache_key: mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/lib/lld.sh', 'Justfile', 'just/**') }} - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.runner_policy.outputs.allow_depot_remote_cache }} diff --git a/.github/workflows/ci-linux-runtime-slice.yml b/.github/workflows/ci-linux-runtime-slice.yml index c0b78c59fe..5204af94de 100644 --- a/.github/workflows/ci-linux-runtime-slice.yml +++ b/.github/workflows/ci-linux-runtime-slice.yml @@ -134,7 +134,7 @@ jobs: with: # Three verified warm samples in run 34272984200/1 observed zero reuse. allow_trusted_seed: "false" - cache_key: mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', 'Justfile', 'just/**') }} + cache_key: mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/lib/lld.sh', 'Justfile', 'just/**') }} - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.runner_policy.outputs.allow_depot_remote_cache }} diff --git a/.github/workflows/ci-platform-checks-slice.yml b/.github/workflows/ci-platform-checks-slice.yml index e60cef7471..d4d722a73e 100644 --- a/.github/workflows/ci-platform-checks-slice.yml +++ b/.github/workflows/ci-platform-checks-slice.yml @@ -111,6 +111,8 @@ jobs: with: ref: ${{ inputs.source_sha || github.sha }} persist-credentials: false + - uses: ./.github/actions/setup-windows-short-paths + if: ${{ matrix.check.platform == 'windows' }} - uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable 2026-08-20 - name: Configure macOS linker if: ${{ matrix.check.platform == 'macos' }} diff --git a/.github/workflows/ci-quality-slice.yml b/.github/workflows/ci-quality-slice.yml index 9f98b88ba2..013727c517 100644 --- a/.github/workflows/ci-quality-slice.yml +++ b/.github/workflows/ci-quality-slice.yml @@ -164,11 +164,6 @@ jobs: with: ref: ${{ inputs.source_sha || github.sha }} persist-credentials: false - - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: ./.github/actions/configure-sccache-gha - with: - allow_depot_remote_cache: ${{ needs.runner_policy.outputs.allow_depot_remote_cache }} - allow_native_github_cache: ${{ needs.runner_policy.outputs.allow_native_github_cache }} - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.12" @@ -238,7 +233,6 @@ jobs: run: shell: bash env: - RUSTFLAGS: "-C link-arg=-fuse-ld=lld" RUSTC_WRAPPER: sccache steps: - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@ed07043b84d720aab30e75ed2f038f7042576f16 @@ -257,7 +251,7 @@ jobs: uses: ./.github/actions/restore-sccache-seed with: allow_trusted_seed: ${{ needs.runner_policy.outputs.allow_trusted_sccache_seed }} - cache_key: mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', 'Justfile', 'just/**') }} + cache_key: mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/lib/lld.sh', 'Justfile', 'just/**') }} - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.runner_policy.outputs.allow_depot_remote_cache }} @@ -309,11 +303,6 @@ jobs: cache: npm cache-dependency-path: website/package-lock.json - uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable 2026-08-20 - - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: ./.github/actions/configure-sccache-gha - with: - allow_depot_remote_cache: ${{ needs.runner_policy.outputs.allow_depot_remote_cache }} - allow_native_github_cache: ${{ needs.runner_policy.outputs.allow_native_github_cache }} - uses: taiki-e/install-action@3d23c1bbdafe696dfccad2664945a04f47d03dc3 # just - name: Install website dependencies working-directory: website diff --git a/.github/workflows/ci-rust-tests-slice.yml b/.github/workflows/ci-rust-tests-slice.yml index 369d5581a0..2d082d4d99 100644 --- a/.github/workflows/ci-rust-tests-slice.yml +++ b/.github/workflows/ci-rust-tests-slice.yml @@ -106,7 +106,6 @@ jobs: LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-static MESH_LLM_SKIP_UI: "1" MESH_LLM_REQUIRE_SCCACHE: "1" - RUSTFLAGS: "-C link-arg=-fuse-ld=lld" RUSTC_WRAPPER: sccache steps: - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@ed07043b84d720aab30e75ed2f038f7042576f16 @@ -129,7 +128,7 @@ jobs: uses: ./.github/actions/restore-sccache-seed with: allow_trusted_seed: ${{ needs.runner_policy.outputs.allow_trusted_sccache_seed }} - cache_key: mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', 'Justfile', 'just/**') }} + cache_key: mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/lib/lld.sh', 'Justfile', 'just/**') }} - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.runner_policy.outputs.allow_depot_remote_cache }} @@ -222,7 +221,6 @@ jobs: LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-static MESH_LLM_SKIP_UI: "1" MESH_LLM_REQUIRE_SCCACHE: "1" - RUSTFLAGS: "-C link-arg=-fuse-ld=lld" RUSTC_WRAPPER: sccache steps: - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@ed07043b84d720aab30e75ed2f038f7042576f16 @@ -245,7 +243,7 @@ jobs: uses: ./.github/actions/restore-sccache-seed with: allow_trusted_seed: ${{ needs.runner_policy.outputs.allow_trusted_sccache_seed }} - cache_key: mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', 'Justfile', 'just/**') }} + cache_key: mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/lib/lld.sh', 'Justfile', 'just/**') }} - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.runner_policy.outputs.allow_depot_remote_cache }} diff --git a/.github/workflows/ci-windows-host-slice.yml b/.github/workflows/ci-windows-host-slice.yml index f42a4693cc..476ac6c4a3 100644 --- a/.github/workflows/ci-windows-host-slice.yml +++ b/.github/workflows/ci-windows-host-slice.yml @@ -107,6 +107,7 @@ jobs: with: ref: ${{ inputs.source_sha || github.sha }} persist-credentials: false + - uses: ./.github/actions/setup-windows-short-paths - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" diff --git a/.github/workflows/ci-windows-runtime-slice.yml b/.github/workflows/ci-windows-runtime-slice.yml index 48dbb87723..a56dbc67e7 100644 --- a/.github/workflows/ci-windows-runtime-slice.yml +++ b/.github/workflows/ci-windows-runtime-slice.yml @@ -111,6 +111,7 @@ jobs: with: ref: ${{ inputs.source_sha }} persist-credentials: false + - uses: ./.github/actions/setup-windows-short-paths - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" diff --git a/.github/workflows/depot-canary.yml b/.github/workflows/depot-canary.yml index fb5ed29806..4c41f26757 100644 --- a/.github/workflows/depot-canary.yml +++ b/.github/workflows/depot-canary.yml @@ -677,7 +677,7 @@ jobs: runtime_seed: name: CPU runtime seed (${{ matrix.pair }} / ${{ matrix.arm }}) - if: github.repository == 'Mesh-LLM/mesh-llm' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.mode == 'runtime-seed' + if: github.repository == 'Mesh-LLM/mesh-llm' && github.event_name == 'workflow_dispatch' && inputs.mode == 'runtime-seed' runs-on: ubuntu-24.04 timeout-minutes: 75 permissions: @@ -722,7 +722,7 @@ jobs: id: seed_key env: GH_TOKEN: ${{ github.token }} - CANARY_KEY: mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', 'Justfile', 'just/**') }} + CANARY_KEY: mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/lib/lld.sh', 'Justfile', 'just/**') }} run: python3 scripts/runtime-seed-canary.py preflight runtime-seed-evidence - name: Start restore timer run: python3 scripts/runtime-seed-canary.py restore_start runtime-seed-evidence diff --git a/.github/workflows/hf-download-smoke.yml b/.github/workflows/hf-download-smoke.yml index b7fdb852c9..b161bb7408 100644 --- a/.github/workflows/hf-download-smoke.yml +++ b/.github/workflows/hf-download-smoke.yml @@ -56,13 +56,6 @@ jobs: - uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable 2026-08-20 - name: Verify prebuilt smoke environment run: verify-runner-image public cpu - - name: Configure Linux Rust linker - run: | - mkdir -p .cargo - cat > .cargo/config.toml <<'EOF' - [target.x86_64-unknown-linux-gnu] - rustflags = ["-C", "link-arg=-fuse-ld=lld"] - EOF - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: ./.github/actions/configure-sccache-gha with: diff --git a/.github/workflows/llama-upstream-canary.yml b/.github/workflows/llama-upstream-canary.yml index 6ca2646042..4db7b69ae7 100644 --- a/.github/workflows/llama-upstream-canary.yml +++ b/.github/workflows/llama-upstream-canary.yml @@ -42,11 +42,8 @@ jobs: contents: read env: LLAMA_UPSTREAM_CANARY_SMOKE: ${{ vars.LLAMA_UPSTREAM_CANARY_SMOKE || '1' }} - # Certification must compile the native closure for this arm64 runner. - # Both the former GitHub backend and the persistent runner-local cache - # have served Intel objects for AppleClang's implicit host target. - RUSTC_WRAPPER: "" - LLAMA_STAGE_USE_SCCACHE: "0" + MESH_LLM_REQUIRE_SCCACHE: "1" + SCCACHE_DIR: /Users/lab/Library/Caches/mesh-llm/sccache/llama-canary-arm64 # A persistent runner can retain a CMake cache from a different host # architecture. Build every canary in a run-unique directory, and feed # that exact native closure to all Rust, smoke, and battery commands. @@ -140,6 +137,22 @@ jobs: exit 1 fi + - name: Isolate compiler cache identity + run: | + set -euo pipefail + mkdir -p "$SCCACHE_DIR" + cache_buster="$({ + printf '%s\n' 'platform=macos' 'architecture=arm64' 'backend=metal' 'profile=release' 'cmake-osx-architectures=arm64' + rustc -vV + cc --version + xcrun --show-sdk-path + xcrun --show-sdk-build-version + git ls-files -s -- .cargo/config.toml scripts/build-llama.sh scripts/package-native-runtime.sh third_party/llama.cpp/upstream.txt third_party/llama.cpp/patches + } | shasum -a 256 | awk '{print $1}')" + echo "SCCACHE_C_CUSTOM_CACHE_BUSTER=$cache_buster" >> "$GITHUB_ENV" + sccache --start-server >/dev/null 2>&1 || true + sccache --zero-stats + - name: Resolve requested llama.cpp upstream id: sha env: @@ -298,6 +311,7 @@ jobs: echo "canary native archive must be arm64, got: $arches" >&2 exit 1 fi + sccache --show-stats - name: Verify generated model-family patch id: family_patch @@ -432,8 +446,8 @@ jobs: permissions: contents: read env: - RUSTC_WRAPPER: "" - LLAMA_STAGE_USE_SCCACHE: "0" + MESH_LLM_REQUIRE_SCCACHE: "1" + SCCACHE_DIR: /Users/lab/Library/Caches/mesh-llm/sccache/llama-canary-arm64 LLAMA_BUILD_DIR: ${{ github.workspace }}/.deps/llama-canary-verify-${{ github.run_id }}-${{ github.run_attempt }} LLAMA_STAGE_BUILD_DIR: ${{ github.workspace }}/.deps/llama-canary-verify-${{ github.run_id }}-${{ github.run_attempt }} LLAMA_STAGE_BACKEND: metal @@ -470,6 +484,22 @@ jobs: fi echo "SKIPPY_REWRITER_LLVM_PREFIX=$llvm_prefix" >> "$GITHUB_ENV" + - name: Isolate compiler cache identity + run: | + set -euo pipefail + mkdir -p "$SCCACHE_DIR" + cache_buster="$({ + printf '%s\n' 'platform=macos' 'architecture=arm64' 'backend=metal' 'profile=release' 'cmake-osx-architectures=arm64' + rustc -vV + cc --version + xcrun --show-sdk-path + xcrun --show-sdk-build-version + git ls-files -s -- .cargo/config.toml scripts/build-llama.sh scripts/package-native-runtime.sh third_party/llama.cpp/upstream.txt third_party/llama.cpp/patches + } | shasum -a 256 | awk '{print $1}')" + echo "SCCACHE_C_CUSTOM_CACHE_BUSTER=$cache_buster" >> "$GITHUB_ENV" + sccache --start-server >/dev/null 2>&1 || true + sccache --zero-stats + - name: Download agent candidate uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -488,6 +518,10 @@ jobs: UPSTREAM_SHA_INPUT: ${{ needs.latest-upstream.outputs.candidate_upstream }} run: scripts/llama-canary-agent-repair.sh + - name: Show compiler cache stats + if: ${{ success() || failure() }} + run: sccache --show-stats + - name: Upload independently certified candidate id: certified_upload if: ${{ !cancelled() && steps.verify.outcome == 'success' }} diff --git a/.github/workflows/nightly-kv-coverage.yml b/.github/workflows/nightly-kv-coverage.yml index 8312356a86..10026ffe82 100644 --- a/.github/workflows/nightly-kv-coverage.yml +++ b/.github/workflows/nightly-kv-coverage.yml @@ -25,7 +25,6 @@ jobs: image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:f499b79bc52dc7492d57397fdbec9f890c6f6bb1d8c1fcde9c1c97d45c0541a7 env: CARGO_INCREMENTAL: "0" - RUSTC_WRAPPER: "" RUST_BACKTRACE: "1" SKIPPY_CACHE_STATE_MACHINE_SEEDS: ${{ vars.MESH_NIGHTLY_KV_STATE_MACHINE_SEEDS || '256' }} SKIPPY_CACHE_STATE_MACHINE_STEPS: ${{ vars.MESH_NIGHTLY_KV_STATE_MACHINE_STEPS || '10000' }} diff --git a/.github/workflows/node-sdk-addon-artifact.yml b/.github/workflows/node-sdk-addon-artifact.yml index 6397f8a31a..06d535e270 100644 --- a/.github/workflows/node-sdk-addon-artifact.yml +++ b/.github/workflows/node-sdk-addon-artifact.yml @@ -206,7 +206,7 @@ jobs: mkdir -p "$probe/src" printf '[package]\nname = "mesh-cargo-linker-check"\nversion = "0.0.0"\nedition = "2024"\n' > "$probe/Cargo.toml" printf 'fn main() {}\n' > "$probe/src/main.rs" - PATH="$lld_bin:$PATH" CARGO_ENCODED_RUSTFLAGS='-Clink-arg=-fuse-ld=lld' \ + PATH="$lld_bin:$PATH" \ cargo rustc --quiet --manifest-path "$probe/Cargo.toml" --target x86_64-apple-darwin - name: Build, smoke, and stage immutable addon env: @@ -293,6 +293,7 @@ jobs: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false + - uses: ./.github/actions/setup-windows-short-paths - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 24.20.0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e38a594c7a..2e91ee74af 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -263,7 +263,7 @@ jobs: - name: Install Linux dependencies if: runner.os == 'Linux' - run: sudo apt-get update && sudo apt-get install -y build-essential cmake ninja-build pkg-config libssl-dev libdbus-1-dev curl lld + run: sudo apt-get update && sudo apt-get install -y build-essential cmake ninja-build pkg-config libssl-dev libdbus-1-dev curl mold lld - name: Install macOS dependencies if: runner.os == 'macOS' @@ -522,7 +522,7 @@ jobs: - name: Install Linux dependencies if: runner.os == 'Linux' - run: sudo apt-get update && sudo apt-get install -y build-essential cmake ninja-build pkg-config libssl-dev libdbus-1-dev curl lld patchelf + run: sudo apt-get update && sudo apt-get install -y build-essential cmake ninja-build pkg-config libssl-dev libdbus-1-dev curl mold lld patchelf - name: Install macOS dependencies if: runner.os == 'macOS' @@ -859,7 +859,7 @@ jobs: - uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable 2026-08-20 - uses: taiki-e/install-action@3d23c1bbdafe696dfccad2664945a04f47d03dc3 # just - name: Install dependencies - run: sudo apt-get update && sudo apt-get install -y build-essential cmake ninja-build pkg-config libssl-dev libdbus-1-dev curl lld + run: sudo apt-get update && sudo apt-get install -y build-essential cmake ninja-build pkg-config libssl-dev libdbus-1-dev curl mold lld - name: Prepare dispatched release version if: github.event_name == 'workflow_dispatch' env: @@ -1310,6 +1310,7 @@ jobs: with: ref: ${{ needs.metadata.outputs.source_sha }} persist-credentials: false + - uses: ./.github/actions/setup-windows-short-paths - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 24.20.0 @@ -1493,6 +1494,7 @@ jobs: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false + - uses: ./.github/actions/setup-windows-short-paths - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.x' @@ -1545,6 +1547,7 @@ jobs: WINDOWS_VULKAN_SDK_VERSION: ${{ vars.VULKAN_SDK_VERSION || '1.4.328.1' }} LLAMA_STAGE_BUILD_DIR: .deps/llama-build/windows/build-stage-abi-${{ matrix.backend }} MESH_NATIVE_RUNTIME_TARGET: x86_64-pc-windows-msvc + MESH_LLM_REQUIRE_SCCACHE: "1" strategy: fail-fast: false matrix: @@ -1557,12 +1560,6 @@ jobs: backend: rocm rocm_architectures: gfx90a;gfx942;gfx1100;gfx1101;gfx1102;gfx1103;gfx1151;gfx1200;gfx1201 artifact_name: release-native-runtime-windows-x86_64-rocm - # sccache + hipcc is broken on Windows: HIP's multi-offload-arch - # compile pipeline plus the long arch-derived build directory pushes - # object paths past MAX_PATH, and sccache fails to persist its temp - # file (os error 3). Disable the compiler launcher for this row only; - # CUDA/Vulkan keep sccache. - use_sccache: '0' - name: Vulkan backend: vulkan artifact_name: release-native-runtime-windows-x86_64-vulkan @@ -1570,6 +1567,7 @@ jobs: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false + - uses: ./.github/actions/setup-windows-short-paths - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.x' @@ -1656,7 +1654,6 @@ jobs: LLAMA_STAGE_BACKEND: ${{ matrix.backend }} LLAMA_STAGE_CUDA_ARCHITECTURES: ${{ matrix.cuda_architectures }} LLAMA_STAGE_AMDGPU_TARGETS: ${{ matrix.rocm_architectures }} - LLAMA_STAGE_USE_SCCACHE: ${{ matrix.use_sccache || '1' }} MESH_LLM_CUDA_TOOLKIT_MAJOR: '12' uses: ./.github/actions/prepare-native-runtime-input with: diff --git a/.github/workflows/sdk-smoke.yml b/.github/workflows/sdk-smoke.yml index b2e78b523d..34e10ada7b 100644 --- a/.github/workflows/sdk-smoke.yml +++ b/.github/workflows/sdk-smoke.yml @@ -96,7 +96,7 @@ env: SDK_RUST_TARGET: x86_64-unknown-linux-gnu SDK_RUST_IMAGE_DIGEST: f499b79bc52dc7492d57397fdbec9f890c6f6bb1d8c1fcde9c1c97d45c0541a7 SDK_RUST_TOOLCHAIN_EPOCH: rust-stable-2026-08-20-4360b52568e2003a75bf9bc1d59f33a8e3fc893c - SDK_RUST_PROFILE_LINKER: debug-fuse-ld-lld + SDK_RUST_PROFILE_LINKER: debug-repository-mold-driver jobs: sdk_smoke: @@ -252,7 +252,7 @@ jobs: # carries the boundaries that the action cannot infer: the pinned # Linux image, toolchain epoch, test profile/linker recipe, and the # workflow/scripts/cache-version inputs that define this smoke lane. - prefix-key: ${{ format('{0}-sdk-rust-cargo-v1-{1}-{2}-{3}-image-{4}-{5}-{6}-{7}', env.CACHE_NAMESPACE, runner.os, runner.arch, env.SDK_RUST_TARGET, env.SDK_RUST_IMAGE_DIGEST, env.SDK_RUST_TOOLCHAIN_EPOCH, env.SDK_RUST_PROFILE_LINKER, hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', '**/Cargo.toml', 'scripts/ci-rust-sdk-smoke.sh', 'scripts/ci-sdk-fixture.sh', 'scripts/ci-prepare-native-runtime.sh', 'scripts/package-sdk-console-assets.sh', 'scripts/check-sdk-contract.sh', 'scripts/verify-sdk-console-assets.sh', '.github/workflows/sdk-smoke.yml')) }} + prefix-key: ${{ format('{0}-sdk-rust-cargo-v1-{1}-{2}-{3}-image-{4}-{5}-{6}-{7}', env.CACHE_NAMESPACE, runner.os, runner.arch, env.SDK_RUST_TARGET, env.SDK_RUST_IMAGE_DIGEST, env.SDK_RUST_TOOLCHAIN_EPOCH, env.SDK_RUST_PROFILE_LINKER, hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/lib/lld.sh', '**/Cargo.toml', 'scripts/ci-rust-sdk-smoke.sh', 'scripts/ci-sdk-fixture.sh', 'scripts/ci-prepare-native-runtime.sh', 'scripts/package-sdk-console-assets.sh', 'scripts/check-sdk-contract.sh', 'scripts/verify-sdk-console-assets.sh', '.github/workflows/sdk-smoke.yml')) }} shared-key: ci-sdk-smoke-rust save-if: ${{ github.ref == 'refs/heads/main' && github.event.inputs.original_event_name != 'pull_request' && github.event.inputs.original_event_name != 'pull_request_target' }} @@ -260,15 +260,6 @@ jobs: if: ${{ inputs.sdk_kind != 'swift' }} run: verify-runner-image public cpu - - name: Configure Linux Rust linker - if: ${{ runner.os == 'Linux' && inputs.sdk_kind == 'rust' }} - run: | - mkdir -p .cargo - cat > .cargo/config.toml <<'EOF' - [target.x86_64-unknown-linux-gnu] - rustflags = ["-C", "link-arg=-fuse-ld=lld"] - EOF - - uses: ./.github/actions/restore-smoke-inputs id: model_inputs with: diff --git a/.github/workflows/windows-warm-caches.yml b/.github/workflows/windows-warm-caches.yml index a811de0b63..74147014be 100644 --- a/.github/workflows/windows-warm-caches.yml +++ b/.github/workflows/windows-warm-caches.yml @@ -57,6 +57,7 @@ jobs: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false + - uses: ./.github/actions/setup-windows-short-paths - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.x' @@ -147,6 +148,7 @@ jobs: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: persist-credentials: false + - uses: ./.github/actions/setup-windows-short-paths - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.x' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7b84ec1393..77b535b4a0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,11 +9,33 @@ This file covers local build and development workflows for this repository. - `just` - `cmake` - Rust toolchain (`cargo`) +- `sccache` (required by the repository Cargo configuration) - Node.js 24 + pnpm 10 or newer (for UI development). The UI lockfile keeps `overrides` in `pnpm-workspace.yaml`, which pnpm 9 does not read, so pnpm 9 cannot install it. `corepack pnpm@10` is enough if your host pnpm is older. -**macOS**: Apple Silicon. Metal is used automatically. +Install the pinned compiler cache and the platform linkers with: + +```bash +just bootstrap-build-tools +``` + +The bootstrap installs sccache 0.16.0. On Linux it also installs mold and lld +with the detected system package manager; on macOS it installs lld with +Homebrew. On Windows it installs the Rust LLVM tools with rustup. Set +`MESH_LLM_SCCACHE_VERSION` only when deliberately testing a newer cache binary. + +**macOS**: Apple Silicon. Metal is used automatically. Install the accelerated +linker with `brew install lld`; the repository probes `ld64.lld` against the +active SDK and uses Apple ld when that installed version is incompatible. + +**Linux**: install `mold` and `lld`. Cargo uses mold by default and retains lld +as the diagnosed compatibility fallback. On Ubuntu/Debian: + +```bash +sudo apt-get update +sudo apt-get install -y mold lld +``` **Linux NVIDIA**: x86_64 with an NVIDIA GPU. Requires the CUDA toolkit (`nvcc` in your `PATH`). On Arch Linux, CUDA is typically at `/opt/cuda`; on Ubuntu/Debian it's at `/usr/local/cuda`. Auto-detection finds the right SM architecture for your GPU. @@ -22,7 +44,9 @@ This file covers local build and development workflows for this repository. **Linux Vulkan**: Vulkan is supported when the Vulkan development files and `glslc` are installed. On Ubuntu/Debian, install `libvulkan-dev glslc`. On Arch Linux, install `vulkan-headers shaderc`. **Windows**: native runtime builds support `cuda`, `hip`/`rocm`, `vulkan`, or -`cpu`. Metal is not supported on Windows. +`cpu`. Metal is not supported on Windows. Install the Rust lld tools with +`rustup component add llvm-tools-preview`, or install LLVM with `winget install +LLVM.LLVM`. ## Build from source diff --git a/README.md b/README.md index 3444d9a5e4..81a06a1140 100644 --- a/README.md +++ b/README.md @@ -221,8 +221,12 @@ cd mesh-llm just build ``` -Source builds require `just`, `cmake`, Rust, and Node.js 24 + npm. To exercise -the release boundary locally, build the neutral host and one runtime: +Source builds require `just`, `cmake`, Rust, `sccache`, the platform fast linker +(`mold` on Linux and lld on macOS/Windows), and Node.js 24 + npm. Cargo uses +these defaults from the checked-in configuration, including plain `cargo +build`. See [CONTRIBUTING.md](CONTRIBUTING.md) for installation commands and +linker fallback behavior. To exercise the release boundary locally, build the +neutral host and one runtime: ```bash just release-host-build diff --git a/ci/ci.md b/ci/ci.md index 66cdd6b6dd..ae7831f576 100644 --- a/ci/ci.md +++ b/ci/ci.md @@ -570,6 +570,22 @@ from being duplicated into every composed product artifact. ## Provider and cache policy +The checked-in Cargo configuration is the repository-wide Rust accelerator +owner: `sccache` is mandatory, Linux final links use the probed mold driver, +macOS uses a probed ld64.lld with Apple ld fallback for SDK incompatibility, +and Windows resolves rust-lld/lld-link. Workflows must not clear +`RUSTC_WRAPPER`, synthesize a replacement Cargo linker config, or inject a +direct `-fuse-ld` flag. Full Linux runner-image verification performs a real +mold link before the image is eligible for a pinned consumer digest. + +Native llama builds keep C/C++ plus CUDA/HIP compiler launchers under +`scripts/build-llama.sh`. The macOS family canary isolates its persistent +cache by arm64 toolchain, SDK, backend, profile and recipe identity while +retaining run-unique CMake state and `lipo` archive checks. Every managed +Windows compile job uses short cache/temp roots, and every Windows native +backend sets `CMAKE_OBJECT_PATH_MAX=180`, so even the nine-target ROCm +release row no longer disables sccache. + `.github/actions/select-ci-runners` maps semantic roles to approved labels. Fork pull requests use GitHub-hosted runners. Eligible same-repository PRs may use Depot while the repository-wide gate and time-bounded cache-risk exception diff --git a/ci/runner-images.json b/ci/runner-images.json index e2b673eb60..b7df571fde 100644 --- a/ci/runner-images.json +++ b/ci/runner-images.json @@ -781,6 +781,9 @@ "recipe_inputs": [ "Cargo.lock", ".github/cache-version.txt", + ".cargo/config.toml", + "scripts/cargo-linker", + "scripts/lib/lld.sh", "Justfile", "just/**" ], @@ -804,12 +807,14 @@ "target": "x86_64-unknown-linux-gnu", "rust_action_ref": "dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c", "toolchain_epoch": "rust-stable-2026-08-20-4360b52568e2003a75bf9bc1d59f33a8e3fc893c", - "profile_linker": "debug-fuse-ld-lld", + "profile_linker": "debug-repository-mold-driver", "cache_namespace": "mesh-llm", "cache_recipe_inputs": [ "Cargo.lock", ".github/cache-version.txt", ".cargo/config.toml", + "scripts/cargo-linker", + "scripts/lib/lld.sh", "**/Cargo.toml", "scripts/ci-rust-sdk-smoke.sh", "scripts/ci-sdk-fixture.sh", diff --git a/just/build.just b/just/build.just index c506abd27a..6c1e91d8df 100644 --- a/just/build.just +++ b/just/build.just @@ -5,66 +5,30 @@ qa-logging-console-e2e: @just build @scripts/qa-logging-console-e2e.sh --current-binary ./target/debug/mesh-llm +[unix] +bootstrap-build-tools: + @scripts/bootstrap-build-tools + +[windows] +bootstrap-build-tools: + @powershell -NoProfile -ExecutionPolicy Bypass -File scripts/bootstrap-build-tools.ps1 + [private] [unix] with-lld *COMMAND: #!/usr/bin/env bash set -euo pipefail - source scripts/lib/lld.sh - lld="$(find_lld)" - if [[ -z "$lld" ]]; then - case "$(uname -s)" in - Linux) - cat >&2 <<'EOF' - Error: LLVM ld.lld was not found. - - lld is required for faster Rust builds (measured up to 26% faster locally). - - Install lld, then rerun the just command. Common Linux packages: - Ubuntu/Debian: sudo apt-get update && sudo apt-get install -y lld - Fedora: sudo dnf install lld - Arch Linux: sudo pacman -S lld - openSUSE: sudo zypper install lld - - The build requires ld.lld to be available on PATH. - EOF - ;; - Darwin) - cat >&2 <<'EOF' - Error: LLVM ld64.lld was not found. - - lld is required for faster Rust builds (measured up to 26% faster locally). - - Install lld, then rerun the just command: - brew install lld - - If Homebrew installed lld but it is not on PATH, Mesh-LLM also checks: - $(brew --prefix lld)/bin/ld64.lld - /opt/homebrew/opt/lld/bin/ld64.lld - /usr/local/opt/lld/bin/ld64.lld - EOF - ;; - *) - echo "Unsupported OS for lld linker setup: $(uname -s)" >&2 - ;; - esac + command -v sccache >/dev/null 2>&1 || { + echo "Error: sccache is required; install it and retry." >&2 exit 1 - fi - # Installed but unable to link (typically the SDK is newer than lld) is a - # slower build, not a broken one: fall back to the platform linker and say - # why, rather than fail a build that would otherwise succeed. See - # scripts/lib/lld.sh. - if lld_links "$lld"; then - export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-C link-arg=-fuse-ld=$lld" - else - report_lld_fallback "$lld" - fi + } + scripts/cargo-linker --mesh-probe >/dev/null exec {{ COMMAND }} [private] [windows] with-lld *COMMAND: - @powershell -NoProfile -ExecutionPolicy Bypass -Command "$$ErrorActionPreference = 'Stop'; $$linker = $$null; try { $$sysroot = (& rustc --print sysroot).Trim(); foreach ($$target in @('x86_64-pc-windows-msvc', 'aarch64-pc-windows-msvc')) { $$candidate = Join-Path $$sysroot \"lib\rustlib\$$target\bin\rust-lld.exe\"; if (Test-Path $$candidate) { $$linker = $$candidate; break } } } catch {}; if (-not $$linker) { foreach ($$name in @('rust-lld.exe', 'lld-link.exe')) { $$command = Get-Command $$name -ErrorAction SilentlyContinue; if ($$command) { $$linker = $$command.Source; break } } }; if (-not $$linker) { Write-Error \"LLVM lld was not found for the Windows MSVC target.`n`nlld is required for faster Rust builds (measured up to 26% faster locally).`n`nInstall one of these, then rerun the just command:`n rustup component add llvm-tools-preview`n`nOr install LLVM lld-link:`n winget install LLVM.LLVM`n choco install llvm`n`nThe build requires lld. It looks for rust-lld.exe in the active Rust sysroot first, then falls back to rust-lld.exe or lld-link.exe on PATH.\"; exit 1 }; $$env:CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER = $$linker; $$env:CARGO_TARGET_AARCH64_PC_WINDOWS_MSVC_LINKER = $$linker; Invoke-Expression '{{ COMMAND }}'" + @powershell -NoProfile -ExecutionPolicy Bypass -Command "$$ErrorActionPreference = 'Stop'; if (-not (Get-Command sccache -ErrorAction SilentlyContinue)) { throw 'sccache is required; install it and retry.' }; Invoke-Expression '{{ COMMAND }}'" # Build a local product for the current platform. This is always a diff --git a/scripts/bootstrap-build-tools b/scripts/bootstrap-build-tools new file mode 100755 index 0000000000..33d63730e9 --- /dev/null +++ b/scripts/bootstrap-build-tools @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCCACHE_VERSION="${MESH_LLM_SCCACHE_VERSION:-0.16.0}" + +case "$(uname -s)" in + Darwin) + command -v brew >/dev/null 2>&1 || { + echo "Homebrew is required to install ld64.lld." >&2 + exit 1 + } + brew install lld + ;; + Linux) + if command -v apt-get >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install -y mold lld + elif command -v dnf >/dev/null 2>&1; then + sudo dnf install -y mold lld + elif command -v pacman >/dev/null 2>&1; then + sudo pacman -S --needed mold lld + elif command -v zypper >/dev/null 2>&1; then + sudo zypper install -y mold lld + else + echo "Install mold and lld with the system package manager, then retry." >&2 + exit 1 + fi + ;; + *) + echo "Unsupported Unix platform: $(uname -s)" >&2 + exit 1 + ;; +esac + +installed_version="$(sccache --version 2>/dev/null | awk '{print $2}' || true)" +if [[ "$installed_version" != "$SCCACHE_VERSION" ]]; then + RUSTC_WRAPPER='' cargo install sccache --version "$SCCACHE_VERSION" --locked --force +fi + +sccache --version +"$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/cargo-linker" --mesh-probe diff --git a/scripts/bootstrap-build-tools.ps1 b/scripts/bootstrap-build-tools.ps1 new file mode 100644 index 0000000000..8bf176e5b1 --- /dev/null +++ b/scripts/bootstrap-build-tools.ps1 @@ -0,0 +1,30 @@ +$ErrorActionPreference = "Stop" + +$sccacheVersion = if ($env:MESH_LLM_SCCACHE_VERSION) { + $env:MESH_LLM_SCCACHE_VERSION +} else { + "0.16.0" +} + +& rustup component add llvm-tools-preview +if ($LASTEXITCODE -ne 0) { + throw "Failed to install the Rust LLVM linker tools." +} + +$installedVersion = "" +if (Get-Command sccache -ErrorAction SilentlyContinue) { + $installedVersion = ((& sccache --version) -split '\s+')[-1] +} +if ($installedVersion -ne $sccacheVersion) { + $env:RUSTC_WRAPPER = "" + & cargo install sccache --version $sccacheVersion --locked --force + if ($LASTEXITCODE -ne 0) { + throw "Failed to install sccache $sccacheVersion." + } +} + +& sccache --version +& (Join-Path $PSScriptRoot "cargo-linker.cmd") --mesh-probe +if ($LASTEXITCODE -ne 0) { + throw "The Windows lld linker probe failed." +} diff --git a/scripts/build-host.sh b/scripts/build-host.sh index e17d4d91de..b490c7f67c 100755 --- a/scripts/build-host.sh +++ b/scripts/build-host.sh @@ -9,9 +9,6 @@ REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" UI_DIR="$REPO_ROOT/crates/mesh-llm-ui" BUILD_PROFILE="${MESH_LLM_BUILD_PROFILE:-debug}" -# shellcheck source=scripts/lib/lld.sh -source "$SCRIPT_DIR/lib/lld.sh" - usage() { echo "usage: scripts/build-host.sh [--profile debug|dev|release]" >&2 } @@ -41,48 +38,6 @@ case "$BUILD_PROFILE" in ;; esac -append_rustflag() { - local flag="$1" - case " ${RUSTFLAGS:-} " in - *" $flag "*) ;; - *) export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }$flag" ;; - esac -} - -configure_lld_linker() { - case "$(uname -s)" in - Linux | Darwin) ;; - *) - echo "unsupported OS for a dynamic host build: $(uname -s)" >&2 - exit 1 - ;; - esac - local lld - lld="$(find_lld)" - if [[ -z "$lld" ]]; then - if [[ "$(uname -s)" == "Darwin" ]]; then - echo "Error: LLVM ld64.lld was not found; run 'brew install lld'." >&2 - else - echo "Error: LLVM ld.lld was not found; install lld and retry." >&2 - fi - exit 1 - fi - # Installed but unable to link is a slower build, not a broken one; see - # scripts/lib/lld.sh. - if lld_links "$lld"; then - append_rustflag "-C link-arg=-fuse-ld=$lld" - else - report_lld_fallback "$lld" - fi -} - -configure_rust_cache() { - if [[ -z "${RUSTC_WRAPPER:-}" ]] && command -v sccache >/dev/null 2>&1; then - RUSTC_WRAPPER="$(command -v sccache)" - export RUSTC_WRAPPER - fi -} - stamp_build_version() { local release_version="" local pkgid="" @@ -142,9 +97,6 @@ if [[ "${MESH_LLM_DYNAMIC_NATIVE_RUNTIME:-1}" != "1" ]]; then exit 1 fi -configure_lld_linker -configure_rust_cache - echo "Building backend-neutral MeshLLM host (profile: $BUILD_PROFILE)." if [[ "${MESH_LLM_SKIP_UI:-0}" != "1" ]]; then MESH_LLM_BUILD_PROFILE="$BUILD_PROFILE" "$SCRIPT_DIR/build-ui.sh" "$UI_DIR" diff --git a/scripts/build-llama.sh b/scripts/build-llama.sh index 8075fd9f7b..b7ed9757bb 100755 --- a/scripts/build-llama.sh +++ b/scripts/build-llama.sh @@ -237,6 +237,10 @@ if command -v ninja >/dev/null 2>&1; then echo "using CMake generator: Ninja" fi +case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) CMAKE_ARGS+=(-DCMAKE_OBJECT_PATH_MAX=180) ;; +esac + case "$LLAMA_BACKEND" in cuda) if ! configure_cuda_toolkit_env; then diff --git a/scripts/build-windows.ps1 b/scripts/build-windows.ps1 index 34a0ea5d8a..4cb1f75daa 100755 --- a/scripts/build-windows.ps1 +++ b/scripts/build-windows.ps1 @@ -1147,6 +1147,10 @@ Invoke-InRepo { "-DLLAMA_BUILD_EXAMPLES=OFF", "-DLLAMA_BUILD_TESTS=OFF", "-DGGML_BUILD_TESTS=OFF", + # Keep every Windows native build below the shortest tool in the + # compiler chain, regardless of backend. CMake hashes long object + # paths once this limit is reached. + "-DCMAKE_OBJECT_PATH_MAX=180", # mtmd video pulls in the ffmpeg subprocess path (sheredom/subprocess.h) # that mesh-llm does not use; keep it off to match build-llama.sh. "-DMTMD_VIDEO=OFF" diff --git a/scripts/cargo-linker b/scripts/cargo-linker new file mode 100755 index 0000000000..d971d48ac3 --- /dev/null +++ b/scripts/cargo-linker @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +# shellcheck source=scripts/lib/lld.sh +source "$SCRIPT_DIR/lib/lld.sh" + +link_args=("$@") +probe_args=() +for ((index = 0; index < ${#link_args[@]}; index++)); do + argument="${link_args[$index]}" + case "$argument" in + -arch|-target|-isysroot) + if ((index + 1 < ${#link_args[@]})); then + probe_args+=("$argument" "${link_args[$((index + 1))]}") + index=$((index + 1)) + fi + ;; + --target=*|-m32|-m64) + probe_args+=("$argument") + ;; + esac +done + +sha256_stdin() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum | awk '{print $1}' + else + shasum -a 256 | awk '{print $1}' + fi +} + +target_identity() { + if [[ -n "${MESH_LLM_LINK_TARGET:-}" ]]; then + printf '%s\n' "$MESH_LLM_LINK_TARGET" + return + fi + for ((index = 0; index < ${#probe_args[@]}; index++)); do + case "${probe_args[$index]}" in + -arch|-target) + if ((index + 1 < ${#probe_args[@]})); then + printf '%s\n' "${probe_args[$((index + 1))]}" + return + fi + ;; + --target=*) + printf '%s\n' "${probe_args[$index]#--target=}" + return + ;; + esac + done + case "$(uname -s):$(uname -m)" in + Darwin:arm64) printf 'aarch64-apple-darwin\n' ;; + Darwin:x86_64) printf 'x86_64-apple-darwin\n' ;; + Linux:aarch64|Linux:arm64) printf 'aarch64-unknown-linux-gnu\n' ;; + Linux:x86_64) printf 'x86_64-unknown-linux-gnu\n' ;; + *) printf '%s:%s\n' "$(uname -s)" "$(uname -m)" ;; + esac +} + +linker_identity() { + local linker="$1" + local linker_path="" + linker_path="$(command -v "$linker" 2>/dev/null || true)" + { + printf 'os=%s\n' "$(uname -s)" + printf 'machine=%s\n' "$(uname -m)" + printf 'target=%s\n' "$(target_identity)" + printf 'cc=%s\n' "$(command -v cc)" + cc --version 2>&1 || true + printf 'linker=%s\n' "$linker_path" + "$linker_path" --version 2>&1 || true + printf 'probe-arg=%s\n' "${probe_args[@]}" + if [[ "$(uname -s)" == Darwin ]] && command -v xcrun >/dev/null 2>&1; then + xcrun --show-sdk-path 2>&1 || true + xcrun --show-sdk-version 2>&1 || true + xcrun --show-sdk-build-version 2>&1 || true + fi + } | sha256_stdin +} + +probe_linker_cached() { + local linker="$1" + local identity cache_root status_file diagnostics_file status temporary + identity="$(linker_identity "$linker")" + cache_root="${MESH_LLM_LINKER_PROBE_CACHE_DIR:-$REPO_ROOT/.deps/linker-probes}" + mkdir -p "$cache_root" + status_file="$cache_root/$identity.status" + diagnostics_file="$cache_root/$identity.log" + + if [[ -r "$status_file" ]]; then + status="$(cat "$status_file")" + if [[ "$status" == usable ]]; then + return 0 + fi + if [[ "$status" == incompatible ]]; then + LLD_PROBE_OUTPUT="$(cat "$diagnostics_file" 2>/dev/null || true)" + return 1 + fi + fi + + if lld_links "$linker" "${probe_args[@]}"; then + status=usable + else + status=incompatible + fi + temporary="$(mktemp "$cache_root/.probe.XXXXXX")" + printf '%s\n' "$status" > "$temporary" + mv -f "$temporary" "$status_file" + temporary="$(mktemp "$cache_root/.probe-log.XXXXXX")" + printf '%s\n' "${LLD_PROBE_OUTPUT:-}" > "$temporary" + mv -f "$temporary" "$diagnostics_file" + [[ "$status" == usable ]] +} + +selected_linker="" +case "$(uname -s)" in + Linux) + if ! command -v mold >/dev/null 2>&1; then + echo "Error: mold is required for Linux Rust links. Install mold and retry." >&2 + exit 1 + fi + if probe_linker_cached mold; then + selected_linker=mold + else + cat >&2 <&2 + exit 1 + fi + if ! probe_linker_cached "$selected_linker"; then + report_lld_fallback "$selected_linker" + selected_linker="" + fi + ;; + *) + echo "unsupported Unix platform for Cargo linker driver: $(uname -s)" >&2 + exit 1 + ;; +esac + +if [[ "${1:-}" == --mesh-probe ]]; then + printf '%s\n' "${selected_linker:-platform}" + exit 0 +fi + +if [[ -n "$selected_linker" ]]; then + exec cc "-fuse-ld=$selected_linker" "${link_args[@]}" +fi +exec cc "${link_args[@]}" diff --git a/scripts/cargo-linker.cmd b/scripts/cargo-linker.cmd new file mode 100644 index 0000000000..c1585e240a --- /dev/null +++ b/scripts/cargo-linker.cmd @@ -0,0 +1,26 @@ +@echo off +setlocal EnableExtensions EnableDelayedExpansion + +set "MESH_LLD=" +for /f "usebackq delims=" %%S in (`rustc --print sysroot 2^>nul`) do set "MESH_RUST_SYSROOT=%%S" +if defined MESH_RUST_SYSROOT ( + for %%T in (x86_64-pc-windows-msvc aarch64-pc-windows-msvc) do ( + if not defined MESH_LLD if exist "!MESH_RUST_SYSROOT!\lib\rustlib\%%T\bin\rust-lld.exe" set "MESH_LLD=!MESH_RUST_SYSROOT!\lib\rustlib\%%T\bin\rust-lld.exe" + ) +) +if not defined MESH_LLD for %%L in (rust-lld.exe lld-link.exe) do ( + if not defined MESH_LLD for /f "delims=" %%P in ('where %%L 2^>nul') do if not defined MESH_LLD set "MESH_LLD=%%P" +) +if not defined MESH_LLD ( + >&2 echo Error: LLVM lld-link is required for Windows Rust links. + >&2 echo Run "rustup component add llvm-tools-preview" or install LLVM.LLVM with winget. + exit /b 1 +) + +if "%~1"=="--mesh-probe" ( + echo %MESH_LLD% + exit /b 0 +) + +"%MESH_LLD%" %* +exit /b %ERRORLEVEL% diff --git a/scripts/lib/lld.sh b/scripts/lib/lld.sh index dc77a88907..79ce3de4b9 100755 --- a/scripts/lib/lld.sh +++ b/scripts/lib/lld.sh @@ -67,6 +67,7 @@ _find_lld_darwin() { # reason instead of guessing one. lld_links() { local linker="$1" + shift LLD_PROBE_OUTPUT="" if ! command -v cc >/dev/null 2>&1; then LLD_PROBE_OUTPUT="no C compiler driver (cc) on PATH to probe with" @@ -76,7 +77,7 @@ lld_links() { probe_dir="$(mktemp -d)" || return 1 printf 'int main(void) { return 0; }\n' >"$probe_dir/probe.c" LLD_PROBE_OUTPUT="$( - cc "-fuse-ld=$linker" "$probe_dir/probe.c" -o "$probe_dir/probe" 2>&1 + cc "$@" "-fuse-ld=$linker" "$probe_dir/probe.c" -o "$probe_dir/probe" 2>&1 )" || status=1 rm -rf "$probe_dir" return "$status" diff --git a/scripts/package-native-runtime.sh b/scripts/package-native-runtime.sh index cb4ecfe8d1..d3811776c3 100755 --- a/scripts/package-native-runtime.sh +++ b/scripts/package-native-runtime.sh @@ -6,9 +6,6 @@ REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" # shellcheck disable=SC1091 source "$SCRIPT_DIR/lib/cuda-toolkit.sh" -# shellcheck source=scripts/lib/lld.sh -source "$SCRIPT_DIR/lib/lld.sh" - BUILD=0 OUT_DIR="$REPO_ROOT/dist/native-runtimes" BACKEND="${LLAMA_STAGE_BACKEND:-${SKIPPY_LLAMA_BACKEND:-cpu}}" @@ -338,7 +335,7 @@ build_model_package_tool() { return 0 fi - local tool_rel tool_path source_path configured cargo_target_dir macos_lld + local tool_rel tool_path source_path configured cargo_target_dir local -a cargo_env=( "LLAMA_STAGE_LINK_MODE=dynamic" "LLAMA_STAGE_LIB_DIR=$stage_dir/lib" @@ -357,20 +354,6 @@ build_model_package_tool() { fi source_path="$configured" else - if [[ "$runtime_os" == "macos" ]]; then - # Use lld only when it is installed AND links against the active - # SDK. A protected reusable workflow may not include the - # repository setup action at all, and on a developer machine lld - # can fall behind the SDK; both take the platform linker. The - # explicitly empty encoded flag set still overrides any RUSTFLAGS - # a caller exported. See scripts/lib/lld.sh. - macos_lld="$(resolve_usable_lld)" - if [[ -n "$macos_lld" ]]; then - cargo_env+=("CARGO_ENCODED_RUSTFLAGS=-Clink-arg=-fuse-ld=$macos_lld") - else - cargo_env+=("CARGO_ENCODED_RUSTFLAGS=") - fi - fi env "${cargo_env[@]}" \ cargo build --release --locked --target "$TARGET_TRIPLE" \ -p skippy-model-package diff --git a/scripts/tests/test_build_accelerator_defaults.py b/scripts/tests/test_build_accelerator_defaults.py new file mode 100644 index 0000000000..0c806b6d61 --- /dev/null +++ b/scripts/tests/test_build_accelerator_defaults.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +from pathlib import Path +import unittest + + +ROOT = Path(__file__).resolve().parents[2] + + +class BuildAcceleratorDefaultsTests(unittest.TestCase): + def test_workflows_do_not_bypass_repository_defaults(self) -> None: + findings: list[str] = [] + for path in sorted((ROOT / ".github").rglob("*.yml")): + text = path.read_text(encoding="utf-8") + for forbidden in ( + 'RUSTC_WRAPPER: ""', + "RUSTC_WRAPPER: ''", + 'LLAMA_STAGE_USE_SCCACHE: "0"', + "LLAMA_STAGE_USE_SCCACHE: '0'", + "fuse-ld=lld", + ): + if forbidden in text: + findings.append(f"{path.relative_to(ROOT)}: {forbidden}") + self.assertEqual(findings, []) + + def test_linux_and_windows_release_paths_keep_sccache_enabled(self) -> None: + release = (ROOT / ".github/workflows/release.yml").read_text(encoding="utf-8") + short_paths = ( + ROOT / ".github/actions/setup-windows-short-paths/action.yml" + ).read_text(encoding="utf-8") + self.assertNotIn("use_sccache", release) + self.assertIn('SCCACHE_DIR = "C:\\s"', short_paths) + self.assertIn('TEMP = "C:\\t"', short_paths) + self.assertIn('MESH_LLM_REQUIRE_SCCACHE: "1"', release) + self.assertIn("New-Item -ItemType Directory -Force", short_paths) + + def test_all_managed_windows_compile_jobs_configure_short_paths(self) -> None: + workflows = { + "ci-platform-checks-slice.yml": 1, + "ci-windows-host-slice.yml": 1, + "ci-windows-runtime-slice.yml": 1, + "node-sdk-addon-artifact.yml": 1, + "release.yml": 3, + "windows-warm-caches.yml": 2, + } + for name, expected in workflows.items(): + with self.subTest(workflow=name): + source = (ROOT / ".github/workflows" / name).read_text(encoding="utf-8") + self.assertEqual(source.count("uses: ./.github/actions/setup-windows-short-paths"), expected) + + def test_every_windows_native_backend_hashes_long_object_paths(self) -> None: + bash_build = (ROOT / "scripts/build-llama.sh").read_text(encoding="utf-8") + powershell_build = (ROOT / "scripts/build-windows.ps1").read_text(encoding="utf-8") + self.assertIn("-DCMAKE_OBJECT_PATH_MAX=180", bash_build) + self.assertIn('"-DCMAKE_OBJECT_PATH_MAX=180"', powershell_build) + rocm_case = powershell_build.rsplit('\n "rocm" {', maxsplit=1)[1] + self.assertNotIn('"-DCMAKE_OBJECT_PATH_MAX=180"', rocm_case) + + def test_linker_drivers_encode_platform_policy(self) -> None: + unix = (ROOT / "scripts/cargo-linker").read_text(encoding="utf-8") + windows = (ROOT / "scripts/cargo-linker.cmd").read_text(encoding="utf-8") + self.assertIn("command -v mold", unix) + self.assertIn("probe_linker_cached mold", unix) + self.assertIn("report_lld_fallback", unix) + self.assertIn("xcrun --show-sdk-build-version", unix) + self.assertIn("rust-lld.exe", windows) + self.assertIn("lld-link.exe", windows) + + def test_developer_bootstrap_pins_sccache_and_installs_linkers(self) -> None: + unix = (ROOT / "scripts/bootstrap-build-tools").read_text(encoding="utf-8") + windows = (ROOT / "scripts/bootstrap-build-tools.ps1").read_text(encoding="utf-8") + recipes = (ROOT / "just/build.just").read_text(encoding="utf-8") + self.assertIn('MESH_LLM_SCCACHE_VERSION:-0.16.0', unix) + self.assertIn("mold lld", unix) + self.assertIn("brew install lld", unix) + self.assertIn('"0.16.0"', windows) + self.assertIn("rustup component add llvm-tools-preview", windows) + self.assertEqual(recipes.count("bootstrap-build-tools:"), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_ci_artifact_actions.py b/scripts/tests/test_ci_artifact_actions.py index 1ba4e9cdce..bd784042f1 100644 --- a/scripts/tests/test_ci_artifact_actions.py +++ b/scripts/tests/test_ci_artifact_actions.py @@ -1078,7 +1078,7 @@ def test_sccache_seed_keys_include_imported_just_sources(self) -> None: with self.subTest(workflow=workflow): source = (workflow_dir / workflow).read_text(encoding="utf-8") self.assertIn( - "hashFiles('Cargo.lock', '.github/cache-version.txt', 'Justfile', 'just/**')", + "hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/lib/lld.sh', 'Justfile', 'just/**')", source, ) diff --git a/scripts/tests/test_ci_native_toolchain_routing.py b/scripts/tests/test_ci_native_toolchain_routing.py index f91668aa70..23d05fc6da 100644 --- a/scripts/tests/test_ci_native_toolchain_routing.py +++ b/scripts/tests/test_ci_native_toolchain_routing.py @@ -345,7 +345,7 @@ def test_sccache_seed_keys_include_imported_just_sources(self) -> None: with self.subTest(workflow=workflow): source = (workflow_dir / workflow).read_text(encoding="utf-8") self.assertIn( - "hashFiles('Cargo.lock', '.github/cache-version.txt', 'Justfile', 'just/**')", + "hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/lib/lld.sh', 'Justfile', 'just/**')", source, ) diff --git a/scripts/tests/test_justfile_layout.py b/scripts/tests/test_justfile_layout.py index fd8fa650a1..450abf35e0 100644 --- a/scripts/tests/test_justfile_layout.py +++ b/scripts/tests/test_justfile_layout.py @@ -22,7 +22,7 @@ ) RECIPES_BY_FILE: Final = { "just/build.just": { - "build", "build-dev", "build-linux", "build-mac", "build-runtime", + "bootstrap-build-tools", "build", "build-dev", "build-linux", "build-mac", "build-runtime", "qa-logging-console-e2e", "with-lld", }, "just/release-build.just": { diff --git a/scripts/tests/test_kv_nightly_workflow_contract.py b/scripts/tests/test_kv_nightly_workflow_contract.py index 32ac8c308e..89e2e04464 100644 --- a/scripts/tests/test_kv_nightly_workflow_contract.py +++ b/scripts/tests/test_kv_nightly_workflow_contract.py @@ -21,6 +21,7 @@ def test_ownership_schedule_is_trusted_hosted_and_reproducible(self) -> None: self.assertIn("persist-credentials: false", workflow) self.assertIn("verify-runner-image public cpu", workflow) self.assertIn("cargo test --locked -p skippy-cache", workflow) + self.assertNotIn('RUSTC_WRAPPER: ""', workflow) self.assertIn("SKIPPY_CACHE_STATE_MACHINE_SEEDS", workflow) self.assertIn("SKIPPY_CACHE_STATE_MACHINE_STEPS", workflow) self.assertNotIn("secrets.", workflow) diff --git a/scripts/tests/test_llama_upstream_canary_contract.py b/scripts/tests/test_llama_upstream_canary_contract.py index b0d2705300..3d02da12ce 100644 --- a/scripts/tests/test_llama_upstream_canary_contract.py +++ b/scripts/tests/test_llama_upstream_canary_contract.py @@ -127,10 +127,12 @@ def test_workflow_builds_binaries_before_skipping_per_lane_builds(self) -> None: self.assertIn("LLAMA_STAGE_BACKEND: metal", workflow) self.assertNotIn("mozilla-actions/sccache-action", workflow) self.assertNotIn("SCCACHE_GHA_ENABLED", workflow) - self.assertNotIn("SCCACHE_C_CUSTOM_CACHE_BUSTER", workflow) - self.assertIn('RUSTC_WRAPPER: ""', workflow) - self.assertIn('LLAMA_STAGE_USE_SCCACHE: "0"', workflow) - self.assertNotIn("Show sccache stats", workflow) + self.assertIn("SCCACHE_C_CUSTOM_CACHE_BUSTER", workflow) + self.assertIn('MESH_LLM_REQUIRE_SCCACHE: "1"', workflow) + self.assertIn("SCCACHE_DIR: /Users/lab/Library/Caches/mesh-llm/sccache/llama-canary-arm64", workflow) + self.assertNotIn('RUSTC_WRAPPER: ""', workflow) + self.assertNotIn('LLAMA_STAGE_USE_SCCACHE: "0"', workflow) + self.assertIn("Show compiler cache stats", workflow) native_build = _step_block(workflow, "Build patched llama.cpp ABI") self.assertIn( diff --git a/scripts/tests/test_lld_lib.py b/scripts/tests/test_lld_lib.py index 72c21fb605..d0be813a42 100644 --- a/scripts/tests/test_lld_lib.py +++ b/scripts/tests/test_lld_lib.py @@ -108,29 +108,21 @@ def test_an_unusable_linker_never_fails_the_caller(self) -> None: class CallSiteTests(unittest.TestCase): """Every place that hands cargo an lld must probe it first.""" - def test_the_with_lld_recipe_probes_before_exporting_the_linker(self) -> None: + def test_the_with_lld_recipe_delegates_to_repository_defaults(self) -> None: source = read_justfile_source(JUSTFILE) start = source.index("[unix]\nwith-lld *COMMAND:") end = source.index("[windows]\nwith-lld *COMMAND:", start) recipe = source[start:end] - self.assertIn("source scripts/lib/lld.sh", recipe) - self.assertIn('lld="$(find_lld)"', recipe) - self.assertIn('if lld_links "$lld"; then', recipe) - self.assertIn('report_lld_fallback "$lld"', recipe) - # Not installed stays fatal with install instructions; only an - # installed-but-unusable lld degrades to a note. - self.assertIn("brew install lld", recipe) - self.assertIn("exit 1", recipe) - - def test_build_host_probes_before_exporting_the_linker(self) -> None: + self.assertIn("scripts/cargo-linker --mesh-probe", recipe) + self.assertIn("command -v sccache", recipe) + self.assertNotIn("RUSTFLAGS", recipe) + + def test_build_host_does_not_override_repository_cargo_defaults(self) -> None: script = BUILD_HOST.read_text(encoding="utf-8") - self.assertIn('source "$SCRIPT_DIR/lib/lld.sh"', script) - start = script.index("configure_lld_linker() {") - end = script.index("configure_rust_cache() {", start) - function = script[start:end] - self.assertIn('if lld_links "$lld"; then', function) - self.assertIn('report_lld_fallback "$lld"', function) - self.assertNotIn("command -v ld64.lld", function) + self.assertNotIn("configure_lld_linker", script) + self.assertNotIn("configure_rust_cache", script) + self.assertNotIn("RUSTFLAGS", script) + self.assertNotIn("RUSTC_WRAPPER", script) def test_no_unprobed_linker_directive_in_cargo_config(self) -> None: """A checked-in `-fuse-ld=` applies to every cargo invocation with no @@ -143,6 +135,12 @@ def test_no_unprobed_linker_directive_in_cargo_config(self) -> None: ] self.assertEqual(directives, [], f"unexpected linker directives: {directives}") + def test_cargo_config_owns_cache_and_platform_linker_drivers(self) -> None: + config = (ROOT / ".cargo" / "config.toml").read_text(encoding="utf-8") + self.assertIn('rustc-wrapper = "sccache"', config) + self.assertEqual(config.count('linker = "scripts/cargo-linker"'), 4) + self.assertEqual(config.count('linker = "scripts/cargo-linker.cmd"'), 2) + if __name__ == "__main__": unittest.main() diff --git a/scripts/tests/test_package_native_runtime.py b/scripts/tests/test_package_native_runtime.py index b7ac69dbb8..89d46141a9 100644 --- a/scripts/tests/test_package_native_runtime.py +++ b/scripts/tests/test_package_native_runtime.py @@ -219,26 +219,14 @@ def test_linux_cuda_target_skips_license_without_cuda_library(self) -> None: self.assertEqual(result.returncode, 0, result.stderr) - def test_macos_model_package_tool_uses_only_a_probed_linker(self) -> None: - """Installed is not enough: lld must also link against the active - SDK, and a protected reusable workflow may not have installed it at - all. Both cases take an explicitly empty encoded flag set.""" + def test_model_package_tool_keeps_repository_cargo_defaults(self) -> None: script = SCRIPT.read_text(encoding="utf-8") - self.assertIn('source "$SCRIPT_DIR/lib/lld.sh"', script) start = script.index("build_model_package_tool() {") end = script.index("collect_runtime_libraries() {", start) function = script[start:end] - self.assertNotIn('command -v ld64.lld', function) - self.assertIn('macos_lld="$(resolve_usable_lld)"', function) - self.assertIn( - 'CARGO_ENCODED_RUSTFLAGS=-Clink-arg=-fuse-ld=$macos_lld', - function, - ) - self.assertIn('cargo_env+=("CARGO_ENCODED_RUSTFLAGS=")', function) - self.assertNotIn( - "LLVM ld64.lld is required to build the macOS model package tool", - function, - ) + self.assertNotIn("CARGO_ENCODED_RUSTFLAGS", function) + self.assertNotIn("RUSTFLAGS", function) + self.assertIn("cargo build --release --locked", function) def test_windows_package_skips_dynamic_model_package_tool(self) -> None: script = SCRIPT.read_text(encoding="utf-8") diff --git a/scripts/tests/test_sccache_evidence.py b/scripts/tests/test_sccache_evidence.py index eb6c5594a4..6b67e602a0 100644 --- a/scripts/tests/test_sccache_evidence.py +++ b/scripts/tests/test_sccache_evidence.py @@ -38,7 +38,8 @@ SEED_WARMER = ROOT / ".github" / "workflows" / "cache-warm-sccache.yml" SEED_KEY_PATTERN = re.compile( r"mesh-llm-sccache-seed-[^\n]+-\$\{\{ hashFiles\('" - r"Cargo\.lock', '\.github/cache-version\.txt', 'Justfile', 'just/\*\*'\) \}\}" + r"Cargo\.lock', '\.github/cache-version\.txt', '\.cargo/config\.toml', " + r"'scripts/cargo-linker', 'scripts/lib/lld\.sh', 'Justfile', 'just/\*\*'\) \}\}" ) SEED_IMAGE = ( "ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:" @@ -528,6 +529,8 @@ def test_rust_sdk_cache_key_covers_smoke_compatibility_boundaries(self) -> None: "'Cargo.lock'", "'.github/cache-version.txt'", "'.cargo/config.toml'", + "'scripts/cargo-linker'", + "'scripts/lib/lld.sh'", "'**/Cargo.toml'", "'scripts/ci-rust-sdk-smoke.sh'", "'scripts/ci-sdk-fixture.sh'", From 1011f00963ce98ac78234198bb7e0d5caa4e4890 Mon Sep 17 00:00:00 2001 From: scama Date: Tue, 15 Sep 2026 08:32:48 +1000 Subject: [PATCH 2/9] fix: address accelerator review findings --- .../manage-ci/references/current-inventory.md | 9 ++--- .cargo/config.toml | 4 +-- .github/workflows/cache-warm-sccache.yml | 2 +- .github/workflows/ci-linux-host-slice.yml | 2 +- .github/workflows/ci-linux-runtime-slice.yml | 2 +- .github/workflows/ci-quality-slice.yml | 2 +- .github/workflows/ci-rust-tests-slice.yml | 4 +-- .github/workflows/depot-canary.yml | 4 +-- .github/workflows/release.yml | 1 + .github/workflows/sdk-smoke.yml | 2 +- ci/ci.md | 25 +++++++------ ci/runner-images.json | 2 ++ scripts/bootstrap-build-tools.ps1 | 18 +++++++--- scripts/cargo-linker | 36 ++++++++++++++++--- scripts/cargo-linker-linux-aarch64 | 6 ++++ scripts/cargo-linker-linux-x86_64 | 6 ++++ scripts/cargo-linker.cmd | 1 + scripts/lib/lld.sh | 9 ++--- .../tests/test_build_accelerator_defaults.py | 7 ++++ scripts/tests/test_ci_artifact_actions.py | 2 +- .../tests/test_ci_native_toolchain_routing.py | 2 +- scripts/tests/test_lld_lib.py | 13 ++++++- scripts/tests/test_sccache_evidence.py | 4 +-- 23 files changed, 120 insertions(+), 43 deletions(-) create mode 100755 scripts/cargo-linker-linux-aarch64 create mode 100755 scripts/cargo-linker-linux-x86_64 diff --git a/.agents/skills/manage-ci/references/current-inventory.md b/.agents/skills/manage-ci/references/current-inventory.md index 8c90129343..470f62f9c0 100644 --- a/.agents/skills/manage-ci/references/current-inventory.md +++ b/.agents/skills/manage-ci/references/current-inventory.md @@ -809,14 +809,15 @@ other historical receipts and CPU seed workload coverage remain unknown. See [CI topology](../../../../ci/ci.md#qualified-lean-ui-consumers) for admission scope and the required candidate-branch lane execution before merge. -### Existing-seed CPU runtime canary +### CPU runtime seed canary -`depot-canary.yml` has an isolated manual `runtime-seed` mode with three cold/warm -pairs on fresh GitHub-hosted CPU jobs. It restores only the admitted main seed, +`depot-canary.yml` has an isolated default-branch-only manual `runtime-seed` mode +with three cold/warm pairs on fresh GitHub-hosted CPU jobs. It restores only the +current image-bound, recipe-bound main seed, never saves caches or changes production eligibility, and retains negative or inconclusive results. The catalog tracks this qualification restore separately from the five production restore-action bindings, including the explicitly -disabled runtime binding. See [CI topology](../../../../ci/ci.md#existing-seed-cpu-runtime-canary) +disabled runtime binding. See [CI topology](../../../../ci/ci.md#cpu-runtime-seed-canary) for identity, measurements and the completed qualification limits. Runtime exclusion evidence: run `34272984200/1`, source diff --git a/.cargo/config.toml b/.cargo/config.toml index 55a4762d96..deb1efd090 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -8,10 +8,10 @@ linker = "scripts/cargo-linker" linker = "scripts/cargo-linker" [target.aarch64-unknown-linux-gnu] -linker = "scripts/cargo-linker" +linker = "scripts/cargo-linker-linux-aarch64" [target.x86_64-unknown-linux-gnu] -linker = "scripts/cargo-linker" +linker = "scripts/cargo-linker-linux-x86_64" [target.aarch64-pc-windows-msvc] linker = "scripts/cargo-linker.cmd" diff --git a/.github/workflows/cache-warm-sccache.yml b/.github/workflows/cache-warm-sccache.yml index 8467fa1dc4..4648de00c0 100644 --- a/.github/workflows/cache-warm-sccache.yml +++ b/.github/workflows/cache-warm-sccache.yml @@ -40,7 +40,7 @@ jobs: shell: bash run: | set -euo pipefail - key="mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/lib/lld.sh', 'Justfile', 'just/**') }}" + key="mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/cargo-linker-linux-*', 'scripts/lib/lld.sh', 'Justfile', 'just/**') }}" echo "key=$key" >> "$GITHUB_OUTPUT" - name: Check for an existing exact seed id: restore diff --git a/.github/workflows/ci-linux-host-slice.yml b/.github/workflows/ci-linux-host-slice.yml index 9e657eadde..66b925c10d 100644 --- a/.github/workflows/ci-linux-host-slice.yml +++ b/.github/workflows/ci-linux-host-slice.yml @@ -125,7 +125,7 @@ jobs: uses: ./.github/actions/restore-sccache-seed with: allow_trusted_seed: ${{ needs.runner_policy.outputs.allow_trusted_sccache_seed == 'true' && matrix.host.architecture == 'x86_64' && 'true' || 'false' }} - cache_key: mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/lib/lld.sh', 'Justfile', 'just/**') }} + cache_key: mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/cargo-linker-linux-*', 'scripts/lib/lld.sh', 'Justfile', 'just/**') }} - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.runner_policy.outputs.allow_depot_remote_cache }} diff --git a/.github/workflows/ci-linux-runtime-slice.yml b/.github/workflows/ci-linux-runtime-slice.yml index 5204af94de..67e08b9b87 100644 --- a/.github/workflows/ci-linux-runtime-slice.yml +++ b/.github/workflows/ci-linux-runtime-slice.yml @@ -134,7 +134,7 @@ jobs: with: # Three verified warm samples in run 34272984200/1 observed zero reuse. allow_trusted_seed: "false" - cache_key: mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/lib/lld.sh', 'Justfile', 'just/**') }} + cache_key: mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/cargo-linker-linux-*', 'scripts/lib/lld.sh', 'Justfile', 'just/**') }} - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.runner_policy.outputs.allow_depot_remote_cache }} diff --git a/.github/workflows/ci-quality-slice.yml b/.github/workflows/ci-quality-slice.yml index 013727c517..6307d50d33 100644 --- a/.github/workflows/ci-quality-slice.yml +++ b/.github/workflows/ci-quality-slice.yml @@ -251,7 +251,7 @@ jobs: uses: ./.github/actions/restore-sccache-seed with: allow_trusted_seed: ${{ needs.runner_policy.outputs.allow_trusted_sccache_seed }} - cache_key: mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/lib/lld.sh', 'Justfile', 'just/**') }} + cache_key: mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/cargo-linker-linux-*', 'scripts/lib/lld.sh', 'Justfile', 'just/**') }} - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.runner_policy.outputs.allow_depot_remote_cache }} diff --git a/.github/workflows/ci-rust-tests-slice.yml b/.github/workflows/ci-rust-tests-slice.yml index 2d082d4d99..c1091365bc 100644 --- a/.github/workflows/ci-rust-tests-slice.yml +++ b/.github/workflows/ci-rust-tests-slice.yml @@ -128,7 +128,7 @@ jobs: uses: ./.github/actions/restore-sccache-seed with: allow_trusted_seed: ${{ needs.runner_policy.outputs.allow_trusted_sccache_seed }} - cache_key: mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/lib/lld.sh', 'Justfile', 'just/**') }} + cache_key: mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/cargo-linker-linux-*', 'scripts/lib/lld.sh', 'Justfile', 'just/**') }} - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.runner_policy.outputs.allow_depot_remote_cache }} @@ -243,7 +243,7 @@ jobs: uses: ./.github/actions/restore-sccache-seed with: allow_trusted_seed: ${{ needs.runner_policy.outputs.allow_trusted_sccache_seed }} - cache_key: mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/lib/lld.sh', 'Justfile', 'just/**') }} + cache_key: mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/cargo-linker-linux-*', 'scripts/lib/lld.sh', 'Justfile', 'just/**') }} - uses: ./.github/actions/configure-sccache-gha with: allow_depot_remote_cache: ${{ needs.runner_policy.outputs.allow_depot_remote_cache }} diff --git a/.github/workflows/depot-canary.yml b/.github/workflows/depot-canary.yml index 4c41f26757..38e73ee1be 100644 --- a/.github/workflows/depot-canary.yml +++ b/.github/workflows/depot-canary.yml @@ -677,7 +677,7 @@ jobs: runtime_seed: name: CPU runtime seed (${{ matrix.pair }} / ${{ matrix.arm }}) - if: github.repository == 'Mesh-LLM/mesh-llm' && github.event_name == 'workflow_dispatch' && inputs.mode == 'runtime-seed' + if: github.repository == 'Mesh-LLM/mesh-llm' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.mode == 'runtime-seed' runs-on: ubuntu-24.04 timeout-minutes: 75 permissions: @@ -722,7 +722,7 @@ jobs: id: seed_key env: GH_TOKEN: ${{ github.token }} - CANARY_KEY: mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/lib/lld.sh', 'Justfile', 'just/**') }} + CANARY_KEY: mesh-llm-sccache-seed-linux-x86_64-img-f499b79b-epoch-f499b79b-v3-${{ hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/cargo-linker-linux-*', 'scripts/lib/lld.sh', 'Justfile', 'just/**') }} run: python3 scripts/runtime-seed-canary.py preflight runtime-seed-evidence - name: Start restore timer run: python3 scripts/runtime-seed-canary.py restore_start runtime-seed-evidence diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2e91ee74af..446563a691 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -858,6 +858,7 @@ jobs: package-manager-cache: false - uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable 2026-08-20 - uses: taiki-e/install-action@3d23c1bbdafe696dfccad2664945a04f47d03dc3 # just + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - name: Install dependencies run: sudo apt-get update && sudo apt-get install -y build-essential cmake ninja-build pkg-config libssl-dev libdbus-1-dev curl mold lld - name: Prepare dispatched release version diff --git a/.github/workflows/sdk-smoke.yml b/.github/workflows/sdk-smoke.yml index 34e10ada7b..e087e1f211 100644 --- a/.github/workflows/sdk-smoke.yml +++ b/.github/workflows/sdk-smoke.yml @@ -252,7 +252,7 @@ jobs: # carries the boundaries that the action cannot infer: the pinned # Linux image, toolchain epoch, test profile/linker recipe, and the # workflow/scripts/cache-version inputs that define this smoke lane. - prefix-key: ${{ format('{0}-sdk-rust-cargo-v1-{1}-{2}-{3}-image-{4}-{5}-{6}-{7}', env.CACHE_NAMESPACE, runner.os, runner.arch, env.SDK_RUST_TARGET, env.SDK_RUST_IMAGE_DIGEST, env.SDK_RUST_TOOLCHAIN_EPOCH, env.SDK_RUST_PROFILE_LINKER, hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/lib/lld.sh', '**/Cargo.toml', 'scripts/ci-rust-sdk-smoke.sh', 'scripts/ci-sdk-fixture.sh', 'scripts/ci-prepare-native-runtime.sh', 'scripts/package-sdk-console-assets.sh', 'scripts/check-sdk-contract.sh', 'scripts/verify-sdk-console-assets.sh', '.github/workflows/sdk-smoke.yml')) }} + prefix-key: ${{ format('{0}-sdk-rust-cargo-v1-{1}-{2}-{3}-image-{4}-{5}-{6}-{7}', env.CACHE_NAMESPACE, runner.os, runner.arch, env.SDK_RUST_TARGET, env.SDK_RUST_IMAGE_DIGEST, env.SDK_RUST_TOOLCHAIN_EPOCH, env.SDK_RUST_PROFILE_LINKER, hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/cargo-linker-linux-*', 'scripts/lib/lld.sh', '**/Cargo.toml', 'scripts/ci-rust-sdk-smoke.sh', 'scripts/ci-sdk-fixture.sh', 'scripts/ci-prepare-native-runtime.sh', 'scripts/package-sdk-console-assets.sh', 'scripts/check-sdk-contract.sh', 'scripts/verify-sdk-console-assets.sh', '.github/workflows/sdk-smoke.yml')) }} shared-key: ci-sdk-smoke-rust save-if: ${{ github.ref == 'refs/heads/main' && github.event.inputs.original_event_name != 'pull_request' && github.event.inputs.original_event_name != 'pull_request_target' }} diff --git a/ci/ci.md b/ci/ci.md index ae7831f576..ea76cdfb93 100644 --- a/ci/ci.md +++ b/ci/ci.md @@ -903,21 +903,24 @@ Protected default-branch PR slices alone can still execute old image definitions their success is not candidate-image qualification. Release-tag selection must also retain its full-web image and release-only preparation guard. -### Existing-seed CPU runtime canary +### CPU runtime seed canary -`depot-canary.yml` adds an isolated manual `runtime-seed` mode. Existing audit and -cache-authority modes retain their behavior. Six fresh GitHub-hosted Ubuntu jobs -run three cold/warm pairs in the unchanged CPU `8d93…` image, preserving the real -CPU native build directory and `prepare-native-runtime-input` action. The canary +`depot-canary.yml` adds an isolated default-branch-only manual `runtime-seed` mode. +Existing audit and cache-authority modes retain their behavior. Six fresh +GitHub-hosted Ubuntu jobs run three cold/warm pairs in the current CPU image, +preserving the real CPU native build directory and `prepare-native-runtime-input` +action. The canary registers its image and seed restore separately from production seed consumers. It never saves a cache or changes production eligibility, provider policy or ARC placement. The compiler cache remains disk-only and capped at 2 GiB. -The exact main cache is ID `7456497330`, version `6e0f5d94…`, key suffix `9522c1c3…`, -from trusted publisher run `34230668171` at `a6487dd`. A miss, branch shadow, -metadata drift, nonfresh outputs, cache error or incomparable host makes evidence -inconclusive. The warm 1% floor still fails the worker; complete below-floor data -is retained as a negative qualification result. C/C++ improvements are compared +The canary accepts only the current image-bound, recipe-bound key emitted by the +trusted warmer and requires exactly one matching cache on `refs/heads/main`. +It records and validates the cache ID, version and bounded size returned by GitHub; +a missing seed, branch shadow, malformed metadata, nonfresh outputs, cache error or +incomparable host makes evidence inconclusive. The warm 1% floor still fails the +worker; complete below-floor data is retained as a negative qualification result. +C/C++ improvements are compared against each cold partner separately from assembler and Rust packaging hits. Restore-step elapsed, unchanged action elapsed and total measured path are primary timings. Optional native-preparation/packaging splits may be derived from timestamped @@ -925,7 +928,7 @@ job logs at `built patched llama.cpp`; missing markers leave that split unavaila No automatic result grants eligibility. The completed qualification below retains its negative coverage result and inconclusive full-cohort timing. -The existing CPU seed is deliberately excluded from production runtime restore. +The prior CPU seed was deliberately excluded from production runtime restore. [Run 34272984200/1](https://github.com/Mesh-LLM/mesh-llm/actions/runs/34272984200), source `1f4545616e98db715e37c57e1196cbdc975a010e`, completed all six real build/package verifications. Every warm sample restored the exact main seed and diff --git a/ci/runner-images.json b/ci/runner-images.json index b7df571fde..14c3c4e05c 100644 --- a/ci/runner-images.json +++ b/ci/runner-images.json @@ -783,6 +783,7 @@ ".github/cache-version.txt", ".cargo/config.toml", "scripts/cargo-linker", + "scripts/cargo-linker-linux-*", "scripts/lib/lld.sh", "Justfile", "just/**" @@ -814,6 +815,7 @@ ".github/cache-version.txt", ".cargo/config.toml", "scripts/cargo-linker", + "scripts/cargo-linker-linux-*", "scripts/lib/lld.sh", "**/Cargo.toml", "scripts/ci-rust-sdk-smoke.sh", diff --git a/scripts/bootstrap-build-tools.ps1 b/scripts/bootstrap-build-tools.ps1 index 8bf176e5b1..dc8550d4ef 100644 --- a/scripts/bootstrap-build-tools.ps1 +++ b/scripts/bootstrap-build-tools.ps1 @@ -16,10 +16,20 @@ if (Get-Command sccache -ErrorAction SilentlyContinue) { $installedVersion = ((& sccache --version) -split '\s+')[-1] } if ($installedVersion -ne $sccacheVersion) { - $env:RUSTC_WRAPPER = "" - & cargo install sccache --version $sccacheVersion --locked --force - if ($LASTEXITCODE -ne 0) { - throw "Failed to install sccache $sccacheVersion." + $hadRustcWrapper = Test-Path Env:RUSTC_WRAPPER + $previousRustcWrapper = $env:RUSTC_WRAPPER + try { + $env:RUSTC_WRAPPER = "" + & cargo install sccache --version $sccacheVersion --locked --force + if ($LASTEXITCODE -ne 0) { + throw "Failed to install sccache $sccacheVersion." + } + } finally { + if ($hadRustcWrapper) { + $env:RUSTC_WRAPPER = $previousRustcWrapper + } else { + Remove-Item Env:RUSTC_WRAPPER -ErrorAction SilentlyContinue + } } } diff --git a/scripts/cargo-linker b/scripts/cargo-linker index d971d48ac3..12c08955b3 100755 --- a/scripts/cargo-linker +++ b/scripts/cargo-linker @@ -60,6 +60,34 @@ target_identity() { esac } +compiler_driver() { + local target host + target="$(target_identity)" + host="$(uname -m)" + case "$(uname -s):$target:$host" in + Linux:x86_64-unknown-linux-gnu:x86_64|Linux:aarch64-unknown-linux-gnu:aarch64|Linux:aarch64-unknown-linux-gnu:arm64|Darwin:*) + printf 'cc\n' + ;; + Linux:aarch64-unknown-linux-gnu:x86_64) + printf 'aarch64-linux-gnu-gcc\n' + ;; + Linux:x86_64-unknown-linux-gnu:aarch64|Linux:x86_64-unknown-linux-gnu:arm64) + printf 'x86_64-linux-gnu-gcc\n' + ;; + *) + printf 'unsupported host/target linker pair: %s -> %s\n' "$host" "$target" >&2 + return 1 + ;; + esac +} + +compiler="$(compiler_driver)" || exit 1 +if ! command -v "$compiler" >/dev/null 2>&1; then + echo "Error: $compiler is required to link target $(target_identity) from host $(uname -m)." >&2 + exit 1 +fi +export MESH_LLM_CC="$compiler" + linker_identity() { local linker="$1" local linker_path="" @@ -68,8 +96,8 @@ linker_identity() { printf 'os=%s\n' "$(uname -s)" printf 'machine=%s\n' "$(uname -m)" printf 'target=%s\n' "$(target_identity)" - printf 'cc=%s\n' "$(command -v cc)" - cc --version 2>&1 || true + printf 'cc=%s\n' "$(command -v "$compiler")" + "$compiler" --version 2>&1 || true printf 'linker=%s\n' "$linker_path" "$linker_path" --version 2>&1 || true printf 'probe-arg=%s\n' "${probe_args[@]}" @@ -160,6 +188,6 @@ if [[ "${1:-}" == --mesh-probe ]]; then fi if [[ -n "$selected_linker" ]]; then - exec cc "-fuse-ld=$selected_linker" "${link_args[@]}" + exec "$compiler" "-fuse-ld=$selected_linker" "${link_args[@]}" fi -exec cc "${link_args[@]}" +exec "$compiler" "${link_args[@]}" diff --git a/scripts/cargo-linker-linux-aarch64 b/scripts/cargo-linker-linux-aarch64 new file mode 100755 index 0000000000..f36328d9ba --- /dev/null +++ b/scripts/cargo-linker-linux-aarch64 @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +export MESH_LLM_LINK_TARGET=aarch64-unknown-linux-gnu +exec "$SCRIPT_DIR/cargo-linker" "$@" diff --git a/scripts/cargo-linker-linux-x86_64 b/scripts/cargo-linker-linux-x86_64 new file mode 100755 index 0000000000..65ac90fcc6 --- /dev/null +++ b/scripts/cargo-linker-linux-x86_64 @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +export MESH_LLM_LINK_TARGET=x86_64-unknown-linux-gnu +exec "$SCRIPT_DIR/cargo-linker" "$@" diff --git a/scripts/cargo-linker.cmd b/scripts/cargo-linker.cmd index c1585e240a..8911d82e04 100644 --- a/scripts/cargo-linker.cmd +++ b/scripts/cargo-linker.cmd @@ -22,5 +22,6 @@ if "%~1"=="--mesh-probe" ( exit /b 0 ) +setlocal DisableDelayedExpansion "%MESH_LLD%" %* exit /b %ERRORLEVEL% diff --git a/scripts/lib/lld.sh b/scripts/lib/lld.sh index 79ce3de4b9..8f3c872b98 100755 --- a/scripts/lib/lld.sh +++ b/scripts/lib/lld.sh @@ -61,23 +61,24 @@ _find_lld_darwin() { return 0 } -# Link a one-line C program with `-fuse-ld=$1` through `cc`, the same driver +# Link a one-line C program with `-fuse-ld=$1` through the same compiler driver # rustc uses. One invocation, tens of milliseconds. The linker's own # diagnostics are left in LLD_PROBE_OUTPUT so a caller can report the real # reason instead of guessing one. lld_links() { local linker="$1" + local compiler="${MESH_LLM_CC:-cc}" shift LLD_PROBE_OUTPUT="" - if ! command -v cc >/dev/null 2>&1; then - LLD_PROBE_OUTPUT="no C compiler driver (cc) on PATH to probe with" + if ! command -v "$compiler" >/dev/null 2>&1; then + LLD_PROBE_OUTPUT="no C compiler driver ($compiler) on PATH to probe with" return 1 fi local probe_dir status=0 probe_dir="$(mktemp -d)" || return 1 printf 'int main(void) { return 0; }\n' >"$probe_dir/probe.c" LLD_PROBE_OUTPUT="$( - cc "$@" "-fuse-ld=$linker" "$probe_dir/probe.c" -o "$probe_dir/probe" 2>&1 + "$compiler" "$@" "-fuse-ld=$linker" "$probe_dir/probe.c" -o "$probe_dir/probe" 2>&1 )" || status=1 rm -rf "$probe_dir" return "$status" diff --git a/scripts/tests/test_build_accelerator_defaults.py b/scripts/tests/test_build_accelerator_defaults.py index 0c806b6d61..4b4d818dcb 100644 --- a/scripts/tests/test_build_accelerator_defaults.py +++ b/scripts/tests/test_build_accelerator_defaults.py @@ -1,6 +1,7 @@ from __future__ import annotations from pathlib import Path +import re import unittest @@ -18,9 +19,13 @@ def test_workflows_do_not_bypass_repository_defaults(self) -> None: 'LLAMA_STAGE_USE_SCCACHE: "0"', "LLAMA_STAGE_USE_SCCACHE: '0'", "fuse-ld=lld", + "-C linker=", + "-Clinker=", ): if forbidden in text: findings.append(f"{path.relative_to(ROOT)}: {forbidden}") + if re.search(r"CARGO_TARGET_[A-Z0-9_]+_LINKER", text): + findings.append(f"{path.relative_to(ROOT)}: target linker environment override") self.assertEqual(findings, []) def test_linux_and_windows_release_paths_keep_sccache_enabled(self) -> None: @@ -65,6 +70,8 @@ def test_linker_drivers_encode_platform_policy(self) -> None: self.assertIn("xcrun --show-sdk-build-version", unix) self.assertIn("rust-lld.exe", windows) self.assertIn("lld-link.exe", windows) + self.assertIn("aarch64-linux-gnu-gcc", unix) + self.assertIn("x86_64-linux-gnu-gcc", unix) def test_developer_bootstrap_pins_sccache_and_installs_linkers(self) -> None: unix = (ROOT / "scripts/bootstrap-build-tools").read_text(encoding="utf-8") diff --git a/scripts/tests/test_ci_artifact_actions.py b/scripts/tests/test_ci_artifact_actions.py index bd784042f1..2b6cf467d3 100644 --- a/scripts/tests/test_ci_artifact_actions.py +++ b/scripts/tests/test_ci_artifact_actions.py @@ -1078,7 +1078,7 @@ def test_sccache_seed_keys_include_imported_just_sources(self) -> None: with self.subTest(workflow=workflow): source = (workflow_dir / workflow).read_text(encoding="utf-8") self.assertIn( - "hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/lib/lld.sh', 'Justfile', 'just/**')", + "hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/cargo-linker-linux-*', 'scripts/lib/lld.sh', 'Justfile', 'just/**')", source, ) diff --git a/scripts/tests/test_ci_native_toolchain_routing.py b/scripts/tests/test_ci_native_toolchain_routing.py index 23d05fc6da..4f9126d722 100644 --- a/scripts/tests/test_ci_native_toolchain_routing.py +++ b/scripts/tests/test_ci_native_toolchain_routing.py @@ -345,7 +345,7 @@ def test_sccache_seed_keys_include_imported_just_sources(self) -> None: with self.subTest(workflow=workflow): source = (workflow_dir / workflow).read_text(encoding="utf-8") self.assertIn( - "hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/lib/lld.sh', 'Justfile', 'just/**')", + "hashFiles('Cargo.lock', '.github/cache-version.txt', '.cargo/config.toml', 'scripts/cargo-linker', 'scripts/cargo-linker-linux-*', 'scripts/lib/lld.sh', 'Justfile', 'just/**')", source, ) diff --git a/scripts/tests/test_lld_lib.py b/scripts/tests/test_lld_lib.py index d0be813a42..7165ace9eb 100644 --- a/scripts/tests/test_lld_lib.py +++ b/scripts/tests/test_lld_lib.py @@ -78,6 +78,15 @@ def test_the_probe_passes_the_same_fuse_ld_value_cargo_will_get(self) -> None: result = run_with_stub_cc(recording_cc, 'lld_links /opt/x/ld64.lld; printf %s "$LLD_PROBE_OUTPUT"') self.assertIn("-fuse-ld=/opt/x/ld64.lld", result.stdout) + def test_the_probe_uses_the_selected_target_compiler(self) -> None: + recording_cc = "#!/bin/sh\necho \"$0 $@\" >&2\nexit 0\n" + result = run_with_stub_cc( + recording_cc, + 'cp "$(command -v cc)" "$(dirname "$(command -v cc)")/target-cc"\n' + 'MESH_LLM_CC=target-cc lld_links lld; printf %s "$LLD_PROBE_OUTPUT"', + ) + self.assertIn("target-cc", result.stdout) + def test_resolve_prints_nothing_and_explains_when_the_probe_fails(self) -> None: result = run_with_stub_cc(BROKEN_CC, 'printf "[%s]" "$(resolve_usable_lld)"') self.assertEqual(result.stdout.strip(), "[]") @@ -138,7 +147,9 @@ def test_no_unprobed_linker_directive_in_cargo_config(self) -> None: def test_cargo_config_owns_cache_and_platform_linker_drivers(self) -> None: config = (ROOT / ".cargo" / "config.toml").read_text(encoding="utf-8") self.assertIn('rustc-wrapper = "sccache"', config) - self.assertEqual(config.count('linker = "scripts/cargo-linker"'), 4) + self.assertEqual(config.count('linker = "scripts/cargo-linker"'), 2) + self.assertIn('linker = "scripts/cargo-linker-linux-aarch64"', config) + self.assertIn('linker = "scripts/cargo-linker-linux-x86_64"', config) self.assertEqual(config.count('linker = "scripts/cargo-linker.cmd"'), 2) diff --git a/scripts/tests/test_sccache_evidence.py b/scripts/tests/test_sccache_evidence.py index 6b67e602a0..db45edceaa 100644 --- a/scripts/tests/test_sccache_evidence.py +++ b/scripts/tests/test_sccache_evidence.py @@ -39,7 +39,7 @@ SEED_KEY_PATTERN = re.compile( r"mesh-llm-sccache-seed-[^\n]+-\$\{\{ hashFiles\('" r"Cargo\.lock', '\.github/cache-version\.txt', '\.cargo/config\.toml', " - r"'scripts/cargo-linker', 'scripts/lib/lld\.sh', 'Justfile', 'just/\*\*'\) \}\}" + r"'scripts/cargo-linker', 'scripts/cargo-linker-linux-\*', 'scripts/lib/lld\.sh', 'Justfile', 'just/\*\*'\) \}\}" ) SEED_IMAGE = ( "ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:" @@ -529,7 +529,7 @@ def test_rust_sdk_cache_key_covers_smoke_compatibility_boundaries(self) -> None: "'Cargo.lock'", "'.github/cache-version.txt'", "'.cargo/config.toml'", - "'scripts/cargo-linker'", + "'scripts/cargo-linker', 'scripts/cargo-linker-linux-*'", "'scripts/lib/lld.sh'", "'**/Cargo.toml'", "'scripts/ci-rust-sdk-smoke.sh'", From 38b748f02dcc9b80f8fb52e6397dbdf911796dc2 Mon Sep 17 00:00:00 2001 From: scama Date: Tue, 15 Sep 2026 08:44:46 +1000 Subject: [PATCH 3/9] fix: select Windows rust-lld flavor --- ci/ci.md | 3 ++- scripts/cargo-linker.cmd | 5 ++++- scripts/tests/test_build_accelerator_defaults.py | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ci/ci.md b/ci/ci.md index ea76cdfb93..66ac3dd6b5 100644 --- a/ci/ci.md +++ b/ci/ci.md @@ -571,7 +571,8 @@ from being duplicated into every composed product artifact. ## Provider and cache policy The checked-in Cargo configuration is the repository-wide Rust accelerator -owner: `sccache` is mandatory, Linux final links use the probed mold driver, +owner: `sccache` is mandatory, Linux final links prefer the probed mold driver +and fall back to a compatible lld or the platform linker when that probe fails, macOS uses a probed ld64.lld with Apple ld fallback for SDK incompatibility, and Windows resolves rust-lld/lld-link. Workflows must not clear `RUSTC_WRAPPER`, synthesize a replacement Cargo linker config, or inject a diff --git a/scripts/cargo-linker.cmd b/scripts/cargo-linker.cmd index 8911d82e04..5d37dbb4ec 100644 --- a/scripts/cargo-linker.cmd +++ b/scripts/cargo-linker.cmd @@ -17,11 +17,14 @@ if not defined MESH_LLD ( exit /b 1 ) +set "MESH_LLD_FLAVOR=" +for %%F in ("!MESH_LLD!") do if /I "%%~nxF"=="rust-lld.exe" set "MESH_LLD_FLAVOR=-flavor link" + if "%~1"=="--mesh-probe" ( echo %MESH_LLD% exit /b 0 ) setlocal DisableDelayedExpansion -"%MESH_LLD%" %* +"%MESH_LLD%" %MESH_LLD_FLAVOR% %* exit /b %ERRORLEVEL% diff --git a/scripts/tests/test_build_accelerator_defaults.py b/scripts/tests/test_build_accelerator_defaults.py index 4b4d818dcb..ec9d77cc59 100644 --- a/scripts/tests/test_build_accelerator_defaults.py +++ b/scripts/tests/test_build_accelerator_defaults.py @@ -70,6 +70,7 @@ def test_linker_drivers_encode_platform_policy(self) -> None: self.assertIn("xcrun --show-sdk-build-version", unix) self.assertIn("rust-lld.exe", windows) self.assertIn("lld-link.exe", windows) + self.assertIn('MESH_LLD_FLAVOR=-flavor link', windows) self.assertIn("aarch64-linux-gnu-gcc", unix) self.assertIn("x86_64-linux-gnu-gcc", unix) From 7db1e69a3b8e60e94870920711193787a7d30846 Mon Sep 17 00:00:00 2001 From: scama Date: Tue, 15 Sep 2026 08:57:34 +1000 Subject: [PATCH 4/9] fix: bridge Linux linker image rollout --- .cargo/config.toml | 9 ++-- ci/ci.md | 3 +- scripts/cargo-linker | 37 +++++++++----- .../tests/test_build_accelerator_defaults.py | 51 ++++++++++++++++++- 4 files changed, 80 insertions(+), 20 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index deb1efd090..4ba5f55986 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -29,7 +29,8 @@ rustflags = ["-C", "link-arg=-Wl,-z,max-page-size=16384"] rustflags = ["-C", "link-arg=-Wl,-z,max-page-size=16384"] # The repository linker drivers resolve and probe the active platform linker -# before they execute it. Linux prefers mold; macOS prefers Homebrew ld64.lld -# and falls back to Apple ld when the installed lld cannot parse the active -# SDK. Windows resolves rust-lld/lld-link and fails with setup instructions -# when neither is installed. +# before they execute it. Linux prefers mold and falls back to a compatible +# lld or the platform linker; macOS prefers Homebrew ld64.lld and falls back +# to Apple ld when the installed lld cannot parse the active SDK. Windows +# resolves rust-lld/lld-link and fails with setup instructions when neither is +# installed. diff --git a/ci/ci.md b/ci/ci.md index 66ac3dd6b5..c549f84d48 100644 --- a/ci/ci.md +++ b/ci/ci.md @@ -572,7 +572,8 @@ from being duplicated into every composed product artifact. The checked-in Cargo configuration is the repository-wide Rust accelerator owner: `sccache` is mandatory, Linux final links prefer the probed mold driver -and fall back to a compatible lld or the platform linker when that probe fails, +and fall back to a compatible lld or the platform linker when mold is absent or +its probe fails, macOS uses a probed ld64.lld with Apple ld fallback for SDK incompatibility, and Windows resolves rust-lld/lld-link. Workflows must not clear `RUSTC_WRAPPER`, synthesize a replacement Cargo linker config, or inject a diff --git a/scripts/cargo-linker b/scripts/cargo-linker index 12c08955b3..c77b748544 100755 --- a/scripts/cargo-linker +++ b/scripts/cargo-linker @@ -7,19 +7,31 @@ REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" # shellcheck source=scripts/lib/lld.sh source "$SCRIPT_DIR/lib/lld.sh" -link_args=("$@") +link_args=() probe_args=() -for ((index = 0; index < ${#link_args[@]}; index++)); do - argument="${link_args[$index]}" +raw_link_args=("$@") +for ((index = 0; index < ${#raw_link_args[@]}; index++)); do + argument="${raw_link_args[$index]}" case "$argument" in + -fuse-ld=*) + # The repository driver owns linker selection. Recent rustc + # versions may still inject their default lld preference. + continue + ;; -arch|-target|-isysroot) - if ((index + 1 < ${#link_args[@]})); then - probe_args+=("$argument" "${link_args[$((index + 1))]}") + link_args+=("$argument") + if ((index + 1 < ${#raw_link_args[@]})); then + probe_args+=("$argument" "${raw_link_args[$((index + 1))]}") + link_args+=("${raw_link_args[$((index + 1))]}") index=$((index + 1)) fi ;; --target=*|-m32|-m64) probe_args+=("$argument") + link_args+=("$argument") + ;; + *) + link_args+=("$argument") ;; esac done @@ -146,19 +158,18 @@ probe_linker_cached() { selected_linker="" case "$(uname -s)" in Linux) - if ! command -v mold >/dev/null 2>&1; then - echo "Error: mold is required for Linux Rust links. Install mold and retry." >&2 - exit 1 - fi - if probe_linker_cached mold; then - selected_linker=mold + mold_linker="${MESH_LLM_MOLD:-mold}" + if command -v "$mold_linker" >/dev/null 2>&1 && probe_linker_cached "$mold_linker"; then + selected_linker="$mold_linker" else cat >&2 < None: def test_linker_drivers_encode_platform_policy(self) -> None: unix = (ROOT / "scripts/cargo-linker").read_text(encoding="utf-8") windows = (ROOT / "scripts/cargo-linker.cmd").read_text(encoding="utf-8") - self.assertIn("command -v mold", unix) - self.assertIn("probe_linker_cached mold", unix) + self.assertIn('command -v "$mold_linker"', unix) + self.assertIn('probe_linker_cached "$mold_linker"', unix) self.assertIn("report_lld_fallback", unix) self.assertIn("xcrun --show-sdk-build-version", unix) self.assertIn("rust-lld.exe", windows) @@ -74,6 +77,50 @@ def test_linker_drivers_encode_platform_policy(self) -> None: self.assertIn("aarch64-linux-gnu-gcc", unix) self.assertIn("x86_64-linux-gnu-gcc", unix) + def test_linux_driver_falls_back_and_removes_rustc_linker_override(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + binaries = root / "bin" + binaries.mkdir() + log = root / "cc.log" + + stubs = { + "uname": "#!/bin/sh\ncase \"$1\" in -s) echo Linux ;; -m) echo x86_64 ;; *) echo Linux ;; esac\n", + "cc": "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$MESH_TEST_CC_LOG\"\nexit 0\n", + "ld.lld": "#!/bin/sh\nexit 0\n", + } + for name, body in stubs.items(): + path = binaries / name + path.write_text(body, encoding="utf-8") + path.chmod(0o755) + + env = dict(os.environ) + env.update( + PATH=f"{binaries}{os.pathsep}{env['PATH']}", + MESH_LLM_MOLD="missing-mold-for-test", + MESH_LLM_LINKER_PROBE_CACHE_DIR=str(root / "probes"), + MESH_TEST_CC_LOG=str(log), + ) + result = subprocess.run( + [ + str(ROOT / "scripts" / "cargo-linker-linux-x86_64"), + "-m64", + "-fuse-ld=rustc-injected", + "input.o", + ], + capture_output=True, + text=True, + env=env, + check=False, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("mold is unavailable", result.stderr) + final_invocation = log.read_text(encoding="utf-8").splitlines()[-1] + self.assertIn("input.o", final_invocation) + self.assertIn("-fuse-ld=lld", final_invocation) + self.assertNotIn("rustc-injected", final_invocation) + def test_developer_bootstrap_pins_sccache_and_installs_linkers(self) -> None: unix = (ROOT / "scripts/bootstrap-build-tools").read_text(encoding="utf-8") windows = (ROOT / "scripts/bootstrap-build-tools.ps1").read_text(encoding="utf-8") From 2241a1e5f3407ddfc4e18982e853870c3c97b619 Mon Sep 17 00:00:00 2001 From: scama Date: Tue, 15 Sep 2026 09:19:26 +1000 Subject: [PATCH 5/9] fix: keep planner metadata wrapper-free --- scripts/affected-crates.sh | 42 +++++------------------------------ scripts/tests/test_plan_ci.py | 11 +++++++++ 2 files changed, 17 insertions(+), 36 deletions(-) diff --git a/scripts/affected-crates.sh b/scripts/affected-crates.sh index 38785aa01b..4d4c4e18ac 100755 --- a/scripts/affected-crates.sh +++ b/scripts/affected-crates.sh @@ -228,48 +228,18 @@ EOF crate_to_dir["$crate_name"]="$dir" done < <(echo "$metadata_no_deps" | jq -r '.packages[] | "\(.name)|\(.manifest_path)"') - # Step 2: Build reverse-dep graph: for each crate, list which crates depend on it + # Step 2: Build the reverse-dependency graph from the same no-deps metadata. + # Package dependency declarations are sufficient for conservative CI + # selection and avoid cargo invoking rustc merely to resolve target details. local -A reverse_deps=() - local metadata_json - metadata_json=$(cargo metadata --format-version=1 2>/dev/null) || fail_open - - # Build name→id mapping - local -A name_to_id=() - while IFS='|' read -r name id; do - [[ -z "$name" ]] && continue - name_to_id["$name"]="$id" - done < <(echo "$metadata_json" | jq -r '.packages[] | "\(.name)|\(.id)"') - - # Build reverse deps: for each node, add it as a reverse dep of its dependencies - while IFS='|' read -r node_id dep_id; do - [[ -z "$node_id" ]] && continue - # Find crate name for dep_id - local dep_name="" - for name in "${!name_to_id[@]}"; do - if [[ "${name_to_id[$name]}" == "$dep_id" ]]; then - dep_name="$name" - break - fi - done - [[ -z "$dep_name" ]] && continue - - # Find crate name for node_id - local node_name="" - for name in "${!name_to_id[@]}"; do - if [[ "${name_to_id[$name]}" == "$node_id" ]]; then - node_name="$name" - break - fi - done - [[ -z "$node_name" ]] && continue - - # Add node_name as reverse dep of dep_name + while IFS='|' read -r node_name dep_name; do + [[ -z "$node_name" || -z "$dep_name" || -z "${crate_to_dir[$dep_name]:-}" ]] && continue if [[ -z "${reverse_deps[$dep_name]:-}" ]]; then reverse_deps["$dep_name"]="$node_name" else reverse_deps["$dep_name"]="${reverse_deps[$dep_name]} $node_name" fi - done < <(echo "$metadata_json" | jq -r '.resolve.nodes[] | "\(.id)|\(.dependencies[]?)"') + done < <(echo "$metadata_no_deps" | jq -r '.packages[] | .name as $node | .dependencies[]? | "\($node)|\(.name)"') # Step 3: Match changed files to owning crates local -a test_crates=() diff --git a/scripts/tests/test_plan_ci.py b/scripts/tests/test_plan_ci.py index 57124f0d68..5008afba1c 100644 --- a/scripts/tests/test_plan_ci.py +++ b/scripts/tests/test_plan_ci.py @@ -4,6 +4,7 @@ import importlib.util import json from fnmatch import fnmatchcase +import os from pathlib import Path import shutil import subprocess @@ -217,6 +218,16 @@ def test_docs_only_selects_the_fast_quality_slice(self) -> None: }, ) + def test_planning_metadata_does_not_invoke_the_rustc_wrapper(self) -> None: + with mock.patch.dict( + os.environ, + {"RUSTC_WRAPPER": "missing-wrapper-for-metadata-test"}, + ): + plan = PLANNER.build_plan(fixture("docs-only.json"), root=ROOT) + + self.assertEqual(plan["domains"], ["docs"]) + self.assertEqual(plan["affected_crates"], []) + def test_draft_profile_skips_build_slices_for_regular_changes(self) -> None: payload = fixture("runtime.json") payload.update({"profile": "pr-draft", "event_name": "pull_request"}) From c602711981ee105db2e3c4248cb476521b057e45 Mon Sep 17 00:00:00 2001 From: scama Date: Tue, 15 Sep 2026 09:28:46 +1000 Subject: [PATCH 6/9] fix: provision sccache for quality jobs --- .github/workflows/ci-quality-slice.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/ci-quality-slice.yml b/.github/workflows/ci-quality-slice.yml index 6307d50d33..d4c4a6efeb 100644 --- a/.github/workflows/ci-quality-slice.yml +++ b/.github/workflows/ci-quality-slice.yml @@ -164,6 +164,11 @@ jobs: with: ref: ${{ inputs.source_sha || github.sha }} persist-credentials: false + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.runner_policy.outputs.allow_depot_remote_cache }} + allow_native_github_cache: ${{ needs.runner_policy.outputs.allow_native_github_cache }} - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.12" @@ -303,6 +308,11 @@ jobs: cache: npm cache-dependency-path: website/package-lock.json - uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable 2026-08-20 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + - uses: ./.github/actions/configure-sccache-gha + with: + allow_depot_remote_cache: ${{ needs.runner_policy.outputs.allow_depot_remote_cache }} + allow_native_github_cache: ${{ needs.runner_policy.outputs.allow_native_github_cache }} - uses: taiki-e/install-action@3d23c1bbdafe696dfccad2664945a04f47d03dc3 # just - name: Install website dependencies working-directory: website From b15106b8ca889a8dcfdd79acd3ced82b46c4b0bc Mon Sep 17 00:00:00 2001 From: scama Date: Tue, 15 Sep 2026 12:13:02 +1000 Subject: [PATCH 7/9] fix: support macOS system bash in linker probe --- scripts/cargo-linker | 12 ++++--- .../tests/test_build_accelerator_defaults.py | 33 +++++++++++++++++++ 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/scripts/cargo-linker b/scripts/cargo-linker index c77b748544..22f3061851 100755 --- a/scripts/cargo-linker +++ b/scripts/cargo-linker @@ -112,7 +112,9 @@ linker_identity() { "$compiler" --version 2>&1 || true printf 'linker=%s\n' "$linker_path" "$linker_path" --version 2>&1 || true - printf 'probe-arg=%s\n' "${probe_args[@]}" + if ((${#probe_args[@]} > 0)); then + printf 'probe-arg=%s\n' "${probe_args[@]}" + fi if [[ "$(uname -s)" == Darwin ]] && command -v xcrun >/dev/null 2>&1; then xcrun --show-sdk-path 2>&1 || true xcrun --show-sdk-version 2>&1 || true @@ -141,7 +143,9 @@ probe_linker_cached() { fi fi - if lld_links "$linker" "${probe_args[@]}"; then + # Bash 3.2 treats an empty quoted array expansion as unbound under `set + # -u`. macOS still ships that version, so use the compatible expansion. + if lld_links "$linker" ${probe_args[@]+"${probe_args[@]}"}; then status=usable else status=incompatible @@ -199,6 +203,6 @@ if [[ "${1:-}" == --mesh-probe ]]; then fi if [[ -n "$selected_linker" ]]; then - exec "$compiler" "${link_args[@]}" "-fuse-ld=$selected_linker" + exec "$compiler" ${link_args[@]+"${link_args[@]}"} "-fuse-ld=$selected_linker" fi -exec "$compiler" "${link_args[@]}" +exec "$compiler" ${link_args[@]+"${link_args[@]}"} diff --git a/scripts/tests/test_build_accelerator_defaults.py b/scripts/tests/test_build_accelerator_defaults.py index a12731d816..0a75421036 100644 --- a/scripts/tests/test_build_accelerator_defaults.py +++ b/scripts/tests/test_build_accelerator_defaults.py @@ -121,6 +121,39 @@ def test_linux_driver_falls_back_and_removes_rustc_linker_override(self) -> None self.assertIn("-fuse-ld=lld", final_invocation) self.assertNotIn("rustc-injected", final_invocation) + def test_macos_probe_accepts_no_explicit_target_flags(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + binaries = root / "bin" + binaries.mkdir() + + stubs = { + "uname": "#!/bin/sh\ncase \"$1\" in -s) echo Darwin ;; -m) echo arm64 ;; *) echo Darwin ;; esac\n", + "cc": "#!/bin/sh\nexit 0\n", + "ld64.lld": "#!/bin/sh\nexit 0\n", + "xcrun": "#!/bin/sh\necho test-sdk\n", + } + for name, body in stubs.items(): + path = binaries / name + path.write_text(body, encoding="utf-8") + path.chmod(0o755) + + env = dict(os.environ) + env.update( + PATH=f"{binaries}{os.pathsep}{env['PATH']}", + MESH_LLM_LINKER_PROBE_CACHE_DIR=str(root / "probes"), + ) + result = subprocess.run( + ["/bin/bash", str(ROOT / "scripts" / "cargo-linker"), "--mesh-probe"], + capture_output=True, + text=True, + env=env, + check=False, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), str(binaries / "ld64.lld")) + def test_developer_bootstrap_pins_sccache_and_installs_linkers(self) -> None: unix = (ROOT / "scripts/bootstrap-build-tools").read_text(encoding="utf-8") windows = (ROOT / "scripts/bootstrap-build-tools.ps1").read_text(encoding="utf-8") From af30f76c04c91f96f1c250844dcdd589d1726896 Mon Sep 17 00:00:00 2001 From: scama Date: Tue, 15 Sep 2026 12:23:11 +1000 Subject: [PATCH 8/9] fix: provision sccache in macOS setup action --- .github/actions/setup-macos-lld/action.yml | 36 ++++++++++++++++++---- scripts/tests/test_lld_lib.py | 12 ++++++++ 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/.github/actions/setup-macos-lld/action.yml b/.github/actions/setup-macos-lld/action.yml index 507598e989..5798061fb8 100644 --- a/.github/actions/setup-macos-lld/action.yml +++ b/.github/actions/setup-macos-lld/action.yml @@ -1,7 +1,8 @@ -name: Set up macOS lld linker +name: Set up macOS build accelerators description: > - Install Homebrew's lld on a macOS runner and run the repository linker - probe. Cargo links keep using the same repository driver afterwards. + Install the required sccache and Homebrew lld tools on a macOS runner, + then run the repository linker probe. Cargo links keep using the same + repository driver afterwards. Deliberately uncached: mesh-llm's Actions cache is already over the 10GB per-repo quota (see gh cache list), so a ~1.9GB lld/llvm/z3 Cellar entry here would evict the far more expensive macOS Rust target/ cache for a @@ -11,14 +12,37 @@ description: > runs: using: composite steps: - - name: Install lld + - name: Install required build tools shell: bash - run: brew install lld + run: | + set -euo pipefail + brew install lld + if ! command -v sccache >/dev/null 2>&1; then + brew install sccache + fi - - name: Verify repository linker selection + - name: Verify repository build accelerators shell: bash run: | set -euo pipefail + command -v sccache >/dev/null 2>&1 + sccache --version + if [[ "${SCCACHE_GHA_ENABLED:-}" == "true" && + -z "${ACTIONS_CACHE_URL:-}" && + -z "${ACTIONS_RESULTS_URL:-}" ]]; then + cache_dir="${SCCACHE_DIR:-$RUNNER_TEMP/mesh-llm-sccache}" + mkdir -p "$cache_dir" + echo "SCCACHE_DIR=$cache_dir" >> "$GITHUB_ENV" + echo "SCCACHE_GHA_ENABLED=false" >> "$GITHUB_ENV" + echo "SCCACHE_MULTILEVEL_CHAIN=disk" >> "$GITHUB_ENV" + echo "SCCACHE_IGNORE_SERVER_IO_ERROR=1" >> "$GITHUB_ENV" + export SCCACHE_GHA_ENABLED=false + export SCCACHE_DIR="$cache_dir" + export SCCACHE_MULTILEVEL_CHAIN=disk + export SCCACHE_IGNORE_SERVER_IO_ERROR=1 + sccache --stop-server >/dev/null 2>&1 || true + sccache --start-server + fi lld_bin="$(brew --prefix lld)/bin" test -x "$lld_bin/ld64.lld" echo "$lld_bin" >> "$GITHUB_PATH" diff --git a/scripts/tests/test_lld_lib.py b/scripts/tests/test_lld_lib.py index 7165ace9eb..7d46361a0b 100644 --- a/scripts/tests/test_lld_lib.py +++ b/scripts/tests/test_lld_lib.py @@ -20,6 +20,7 @@ LIB = ROOT / "scripts" / "lib" / "lld.sh" JUSTFILE = ROOT / "Justfile" BUILD_HOST = ROOT / "scripts" / "build-host.sh" +MACOS_SETUP = ROOT / ".github" / "actions" / "setup-macos-lld" / "action.yml" WORKING_CC = "#!/bin/sh\nexit 0\n" # Mirrors the real failure: lld rejects the SDK's text-based stub, after which @@ -126,6 +127,17 @@ def test_the_with_lld_recipe_delegates_to_repository_defaults(self) -> None: self.assertIn("command -v sccache", recipe) self.assertNotIn("RUSTFLAGS", recipe) + def test_macos_setup_provisions_both_required_accelerators(self) -> None: + action = MACOS_SETUP.read_text(encoding="utf-8") + self.assertIn("brew install lld", action) + self.assertIn("brew install sccache", action) + self.assertIn("command -v sccache", action) + self.assertIn('"${SCCACHE_GHA_ENABLED:-}" == "true"', action) + self.assertIn('SCCACHE_GHA_ENABLED=false', action) + self.assertIn('SCCACHE_MULTILEVEL_CHAIN=disk', action) + self.assertIn('$RUNNER_TEMP/mesh-llm-sccache', action) + self.assertIn("scripts/cargo-linker --mesh-probe", action) + def test_build_host_does_not_override_repository_cargo_defaults(self) -> None: script = BUILD_HOST.read_text(encoding="utf-8") self.assertNotIn("configure_lld_linker", script) From 762ac2f74ef47031186ab10c78b4699e6893c0ed Mon Sep 17 00:00:00 2001 From: scama Date: Tue, 15 Sep 2026 12:56:49 +1000 Subject: [PATCH 9/9] fix: satisfy Rust 1.98 clippy --- crates/skippy-model/src/gguf_writer_tests/conversion.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/skippy-model/src/gguf_writer_tests/conversion.rs b/crates/skippy-model/src/gguf_writer_tests/conversion.rs index 18b1b31ab0..a93a8d1f93 100644 --- a/crates/skippy-model/src/gguf_writer_tests/conversion.rs +++ b/crates/skippy-model/src/gguf_writer_tests/conversion.rs @@ -176,8 +176,10 @@ fn assert_f32_tensor(bytes: &[u8], parsed: &ParsedGguf, name: &str, expected: &[ let tensor = parsed.tensor(name); assert_eq!(tensor.ggml_type, GGML_TYPE_F32); let actual = bytes[tensor.absolute_offset..tensor.absolute_offset + expected.len() * 4] - .chunks_exact(4) - .map(|chunk| f32::from_le_bytes(chunk.try_into().unwrap())) + .as_chunks::<4>() + .0 + .iter() + .map(|chunk| f32::from_le_bytes(*chunk)) .collect::>(); assert_eq!(actual, expected); }