From 0acf6f635fede350db3a3248aa7a86b0988728b9 Mon Sep 17 00:00:00 2001 From: Fadion Dashi Date: Thu, 19 Nov 2026 23:14:15 +0100 Subject: [PATCH 1/2] ci: compile DuckDB's C++ through sccache A cold build recompiles the bundled DuckDB's C++ amalgamation, and cold is every release and every CI run after a new rustc: v0.40.0's runs took 43 min on Windows CI and 34-50 min per release leg, and the first run under Rust 1.100 took an hour. libduckdb-sys builds DuckDB through CMake and forwards CMAKE_C/CXX_COMPILER_LAUNCHER, so setting those to sccache (GitHub Actions cache backend) wraps exactly DuckDB's compiles and leaves rustc to rust-cache. A new local action, .github/actions/sccache-cpp, does it for the three CI jobs and the release. On Windows it also loads the MSVC developer environment, because CMake ignores a launcher under the Visual Studio generator and libduckdb-sys picks Ninja only when ninja is on PATH; the job fails if it is not. Unmeasured until the probe in the next commit has run: whether a release build hits the objects CI's dev build stored. --- .github/actions/sccache-cpp/action.yml | 45 ++++++++++++++++++++++++++ .github/workflows/ci.yml | 5 +++ .github/workflows/release.yml | 7 ++++ 3 files changed, 57 insertions(+) create mode 100644 .github/actions/sccache-cpp/action.yml diff --git a/.github/actions/sccache-cpp/action.yml b/.github/actions/sccache-cpp/action.yml new file mode 100644 index 00000000..a1c0b6ef --- /dev/null +++ b/.github/actions/sccache-cpp/action.yml @@ -0,0 +1,45 @@ +name: sccache for DuckDB's C++ +description: >- + Route the bundled DuckDB's C and C++ compiles through sccache with the GitHub + Actions cache backend, so a cold Rust build (a new rustc, an evicted cache, a + tag) reuses the objects an earlier run compiled. + +# **Only the C/C++ compiles, never rustc.** `libduckdb-sys` builds DuckDB through +# CMake and forwards `CMAKE_C_COMPILER_LAUNCHER` / `CMAKE_CXX_COMPILER_LAUNCHER` +# to it, so setting those two wraps exactly DuckDB's compiler calls. Rust stays +# with rust-cache: wrapping rustc too (`RUSTC_WRAPPER`) would store every crate a +# second time in the repository's 10 GB of cache, where rust-cache already keeps +# `target/`. +# +# **Windows needs Ninja and the MSVC environment.** CMake ignores a compiler +# launcher under its default Visual Studio generator. `libduckdb-sys` switches to +# Ninja by itself when `ninja` is on PATH, and the developer environment puts +# Visual Studio's bundled Ninja and `cl` there; the step after it fails the job +# if it did not, rather than letting the launcher be silently ignored. + +runs: + using: composite + steps: + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + with: + version: v0.19.0 + - name: Point CMake's compiler launchers at sccache + shell: bash + run: | + { + echo "SCCACHE_GHA_ENABLED=true" + echo "CMAKE_C_COMPILER_LAUNCHER=sccache" + echo "CMAKE_CXX_COMPILER_LAUNCHER=sccache" + } >> "$GITHUB_ENV" + - name: MSVC developer environment (Windows) + if: runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 + - name: Require Ninja (Windows) + if: runner.os == 'Windows' + shell: bash + run: | + if ! command -v ninja >/dev/null; then + echo "::error::ninja is not on PATH, so CMake would use the Visual Studio generator and ignore the sccache launcher" + exit 1 + fi + ninja --version diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a626609f..cdc65675 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,9 @@ jobs: - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2 with: save-if: ${{ github.ref == 'refs/heads/main' }} + # DuckDB's C++ through sccache, here and in the two jobs below — see the + # action for why only the C/C++ compiles and not rustc. + - uses: ./.github/actions/sccache-cpp - name: Formatting run: cargo fmt --all -- --check # `--features schemaic-db/live-tests` is what compiles the live engine @@ -140,6 +143,7 @@ jobs: - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2 with: save-if: ${{ github.ref == 'refs/heads/main' }} + - uses: ./.github/actions/sccache-cpp # The lint job runs on Linux, where `#[cfg(windows)]` and # `#[cfg(target_os = "macos")]` code is never compiled — so clippy had # never seen it: Windows-only code was linted only when someone happened @@ -265,6 +269,7 @@ jobs: - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2 with: save-if: ${{ github.ref == 'refs/heads/main' }} + - uses: ./.github/actions/sccache-cpp # **Contained database authentication on**, so SQL Server's contained-user # round trip runs rather than reporting itself skipped. It is a server-wide # setting, which is why the test only reads it and this step — on a diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 126b69c2..22c69318 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -167,6 +167,13 @@ jobs: - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2 with: save-if: false + # **DuckDB's C++ from `main`'s sccache.** A tag run can read caches saved + # from the default branch, and the dev profile builds dependencies at the + # release profile's `opt-level` and `debug`, so CMake should invoke the + # compiler identically and the release can reuse the objects CI compiled. + # Linux is the expected exception: zigbuild compiles with `zig cc`, not + # CI's gcc, so its objects are its own. + - uses: ./.github/actions/sccache-cpp # Linux: build against an older glibc (2.31 = Ubuntu 20.04 / Debian 11) with # cargo-zigbuild, so the binary runs on distros older than the CI runner From 04a19073f36e9c9afeb1a889a35f63dd634310cf Mon Sep 17 00:00:00 2001 From: Fadion Dashi Date: Thu, 19 Nov 2026 23:14:15 +0100 Subject: [PATCH 2/2] ci: add a temporary sccache hit-rate probe Builds schemaic-db cold in the dev profile, then in release, and on Linux with cargo-zigbuild as well, printing sccache's stats after each, with rust-cache off so DuckDB is compiled every time. The release build's hit rate is the number docs/roadmap.md says to measure before relying on sccache. Delete before merging. --- .github/workflows/sccache-probe.yml | 66 +++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 .github/workflows/sccache-probe.yml diff --git a/.github/workflows/sccache-probe.yml b/.github/workflows/sccache-probe.yml new file mode 100644 index 00000000..af13ce4b --- /dev/null +++ b/.github/workflows/sccache-probe.yml @@ -0,0 +1,66 @@ +name: sccache probe (temporary) + +# **Delete before merging.** Measures the one thing docs/roadmap.md says to +# measure before relying on sccache: whether a cold `--release` build reuses the +# DuckDB objects a dev build compiled. No rust-cache here on purpose, so DuckDB +# is compiled every time and the sccache stats are the whole answer. Read the +# hit rate in each "stats" step, not the wall-clock time. +on: + pull_request: + +concurrency: + group: sccache-probe-${{ github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + +jobs: + probe: + name: Probe (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable + with: + toolchain: stable + - uses: ./.github/actions/sccache-cpp + - name: Dev build (fills sccache) + shell: bash + run: | + sccache --zero-stats + cargo build -p schemaic-db + - name: Stats after the dev build + shell: bash + run: sccache --show-stats + - name: Release build (should hit) + shell: bash + run: | + sccache --zero-stats + cargo build --release -p schemaic-db + - name: Stats after the release build + shell: bash + run: sccache --show-stats + - name: Install Zig (Linux) + if: runner.os == 'Linux' + uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2 + with: + version: 0.13.0 + - name: Install cargo-zigbuild (Linux) + if: runner.os == 'Linux' + run: cargo install --locked cargo-zigbuild + - name: Zigbuild release (Linux, expected to miss) + if: runner.os == 'Linux' + shell: bash + run: | + sccache --zero-stats + cargo zigbuild --release -p schemaic-db --target x86_64-unknown-linux-gnu.2.31 + - name: Stats after the zigbuild + if: runner.os == 'Linux' + shell: bash + run: sccache --show-stats