diff --git a/FORK.md b/FORK.md index c81c2c78d..47979d0cb 100644 --- a/FORK.md +++ b/FORK.md @@ -47,16 +47,21 @@ and documentation. `git log upstream/main..main` is the authoritative list. - **crossdirect Rust's native chroot gets build dependencies only, from binary repositories**: runtime depends, or a newer pmaports fork of a library, made pmbootstrap build whole packages for the native arch first. +- **Packages carry the commit they were built from**: abuild looked for git in + the copy of the aport inside the chroot, so every apk said + `commit = -dirty` and got the build time as its date. pmbootstrap now reads + the aport's checkout and passes `ABUILD_LAST_COMMIT` and + `SOURCE_DATE_EPOCH`. - **One sccache server per chroot**: sccache's default TCP port is shared by every chroot and work dir on the host, so one build's server compiled another work dir's crates against the wrong root and failed them. -The changes that make crossdirect handle more of Rust live in pmaports, -because crossdirect is an aport (`cross/crossdirect`, 5.3.1-r4 in the -porthole-dev pmaports): rustc called directly by meson compiles target crates -for the target and proc-macros natively, `cargo auditable build` no longer -falls back to QEMU, and `bindgen` runs natively. See -[docs/cross_compiling.md](docs/cross_compiling.md). +The changes that make crossdirect faster live in pmaports, because crossdirect +is an aport (`cross/crossdirect` in the porthole-dev pmaports): GCC links, +including LTO, run natively instead of under QEMU; rustc called directly by +meson compiles target crates for the target and proc-macros natively; `cargo +auditable build` no longer falls back to QEMU; and `bindgen` runs natively. +See [docs/cross_compiling.md](docs/cross_compiling.md). ## Update from upstream diff --git a/docs/build_caches.md b/docs/build_caches.md new file mode 100644 index 000000000..3aef595c2 --- /dev/null +++ b/docs/build_caches.md @@ -0,0 +1,83 @@ +# Build caches + +Everything pmbootstrap caches lives in the work directory, next to the +chroots. Only the caches are worth keeping between builds or restoring in CI; +the chroots are rebuilt from packages in seconds. + + +| Directory | What it holds | Written by | +|---------------------------|--------------------------------------------------------|---------------------------------| +| `cache_ccache_$ARCH` | ccache: compiled C/C++ objects | every package build | +| `cache_sccache` | sccache: compiled Rust crates | Rust builds without crossdirect | +| `cache_rust` | cargo registry and git checkouts | Rust builds | +| `cache_go` | `GOCACHE` and `GOMODCACHE` | Go builds | +| `cache_distfiles` | source tarballs abuild downloaded | every package build | +| `cache_apk_$ARCH` | apks downloaded from the mirrors | every chroot | +| `cache_git`, `cache_http` | git clones (pmaports, aports), APKINDEX and apk.static | pmbootstrap itself | + + +`$ARCH` is the architecture of the *chroot the build runs in*, which is not +always the architecture of the package. A crossdirect build of an aarch64 +package runs the native cross compiler, but from inside the foreign chroot, +so its objects land in `cache_ccache_aarch64` next to the ones a QEMU-only +build of the same package writes. They do not collide -- ccache hashes the +compiler binary -- but they do not share either: moving a package between +QEMU-only and crossdirect starts from a cold cache. A cross-native build (the +kernel) runs in the native chroot, and its objects go to +`cache_ccache_x86_64`, not to the target's cache. + +## Sharing one cache between work directories + +Two work directories (a second checkout, a test work dir, a `--work` +elsewhere) each have their own caches and share nothing. Point them at one +directory with a symlink, before the first build: + +```shell +ln -sfn /srv/pmb-cache/ccache_aarch64 "$WORK/cache_ccache_aarch64" +ln -sfn /srv/pmb-cache/distfiles "$WORK/cache_distfiles" +``` + +pmbootstrap bind-mounts these into the chroots and follows the symlink. +Measured: libcamera built in a second, empty work directory (new chroots, +freshly installed compilers) took 2m17s with 268 of 274 cacheable compiles +served from the first work directory's ccache, against 5m00s cold. + +ccache is safe for several builds using one cache directory at the same time. +sccache is not: it keeps one server per cache directory and accounts for the +cache size in that server, so share `cache_sccache` only between work +directories that do not build at the same time. + +## CI + +Restore and save the caches, not the work directory. A job that builds one +package at a time wants one cache entry per package, because that keeps each +entry small enough for a cache service with a size limit (GitHub: 10 GB per +repository, least recently used entries are evicted): + +```yaml +- uses: actions/cache@v6 + with: + path: | + ${{ runner.temp }}/work/cache_ccache_aarch64 + ${{ runner.temp }}/work/cache_sccache + ${{ runner.temp }}/work/cache_rust + ${{ runner.temp }}/work/cache_go + ${{ runner.temp }}/work/cache_distfiles + key: pmb-build-aarch64-${{ matrix.package }}-${{ github.run_id }} + restore-keys: pmb-build-aarch64-${{ matrix.package }}- +``` + +A key that ends in the run id is never restored, only saved, and +`restore-keys` then picks the newest entry of the same package: every run +saves a fresh entry and starts from the previous one. Keep the apk cache in +its own entry (`pmb-apk-aarch64-...`), because every package's job fills it +with the same downloads. + +Two things make a restored cache useless, and both are easy to miss: + +* **Ownership.** Builds run as uid 12345 inside the chroots, while the cache + comes back owned by the runner. `chown -R 12345:12345` the ccache and + distfiles directories after restoring them. +* **Size.** `ccache_size` is 5G per architecture by default, which one large + package can fill and which is half of a repository's whole cache budget. + `pmbootstrap config ccache_size 2G` keeps an entry restorable. diff --git a/docs/cross_compiling.md b/docs/cross_compiling.md index 7466ba4f5..486f890e2 100644 --- a/docs/cross_compiling.md +++ b/docs/cross_compiling.md @@ -37,9 +37,22 @@ with other methods. This is the default method. This method works for almost all packages, and gives a good speed improvement -over running everything in QEMU. However only the cross compilers run natively. -Linkers and all other commands used during the build still need to run through -QEMU and so these are still very slow. +over running everything in QEMU. The cross compilers run natively, and with +crossdirect 5.3.1-r6 or later (porthole-dev pmaports) so do GCC's link steps: +`collect2`, `ld` and, for a package built with LTO, `lto-wrapper` and `lto1`. +The rest of the build -- the build system, generators, `meson`, `python3` -- +still runs through QEMU and is still slow. + +Link steps get `--sysroot=/` plus `-B/usr/lib/gcc/` and `-B/usr/lib/`, so the +driver takes the target's own GCC runtime (`crtbegin*.o`, `libgcc`, +`libstdc++`, spec files such as `libgomp.spec`) instead of the cross +toolchain's copies. The linked output is then byte-for-byte what the target's +GCC produces under QEMU, apart from the build ID. Invocations that are not +links keep running in QEMU, because their output has to describe the target's +GCC: `-E`, `-S`, `-M`, `-v`, `--version` and `-print-*` (libtool runs the +linker path that `-print-prog-name=ld` gives it, and the cross `ld` cannot run +outside crossdirect's environment), and any compiler call under `fakeroot`, +i.e. in `package()`. The native chroot gets mounted in the foreign arch chroot at `/native`. The [crossdirect](https://gitlab.postmarketos.org/postmarketOS/pmaports/-/blob/main/cross/crossdirect/APKBUILD) diff --git a/docs/index.rst b/docs/index.rst index 44e893882..adf40df42 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,6 +17,7 @@ In case of any problems, have a look at the `issue-tracker`_. chroot debugging cross_compiling + build_caches ssh-keys mirrors environment_variables diff --git a/pmb/build/backend.py b/pmb/build/backend.py index 50433b35b..2374d186d 100644 --- a/pmb/build/backend.py +++ b/pmb/build/backend.py @@ -5,6 +5,7 @@ from pathlib import Path import pmb.chroot +import pmb.helpers.pmaports import pmb.helpers.run from pmb.core import Context from pmb.core.arch import Arch @@ -185,6 +186,37 @@ def handle_csum_failure(apkbuild: Apkbuild, chroot: Chroot) -> None: raise RuntimeError(f"Remote checksum mismatch for {apkbuild['pkgname']}") +def git_provenance(aport: Path) -> Env: + """ABUILD_LAST_COMMIT and SOURCE_DATE_EPOCH for an aport in a git checkout. + + abuild derives both from git: the last commit that touched the aport, + "-dirty" if the aport has uncommitted changes, and that commit's date as + SOURCE_DATE_EPOCH unless it is dirty. It runs git in the copy under + /home/pmos/build, though, whose .git link (see link_to_git_dir()) makes + the pmaports tree look moved to the chroot's build directory, and cannot + work at all for a pmaports git worktree. Every apk then said + "commit = -dirty" and got the copied APKBUILD's mtime, the time of the + build, as its date. Ask the checkout itself, the way abuild would. + + :returns: an empty dict if the aport is not in a git checkout + """ + + def git(*args: str) -> str: + return pmb.helpers.run.user_output( + ["git", *args], aport, output=RunOutputTypeDefault.NULL, check=False + ).strip() + + if git("rev-parse", "--is-inside-work-tree") != "true": + return {} + commit = "" + if git("ls-files", "--", "APKBUILD"): + commit = git("rev-list", "-n", "1", "HEAD", "--", ".") + if git("status", "--porcelain", "--", "."): + return {"ABUILD_LAST_COMMIT": f"{commit}-dirty"} + epoch = git("log", "-1", "--format=%cd", "--date=unix", commit, "--", ".") + return {"ABUILD_LAST_COMMIT": commit, "SOURCE_DATE_EPOCH": epoch} + + def rust_cross_native2_env(arch: Arch, sysroot: str = "/mnt/sysroot") -> Env: """Environment for cargo to cross-compile for arch in the native chroot. @@ -331,6 +363,7 @@ def run_abuild( ) env = abuild_env(context, arch, cross, bootstrap_stage) + env.update(git_provenance(pmb.helpers.pmaports.find(apkbuild["pkgname"]))) # Build the abuild command # Since we install dependencies with pmb, disable dependency handling in abuild. diff --git a/test/build/test_provenance.py b/test/build/test_provenance.py new file mode 100644 index 000000000..4b486be13 --- /dev/null +++ b/test/build/test_provenance.py @@ -0,0 +1,62 @@ +# Copyright 2026 Giuseppe Maggio +# SPDX-License-Identifier: GPL-3.0-or-later +import subprocess +import tempfile +from pathlib import Path + +from pmb.build.backend import git_provenance + + +def _git(repo: Path, *args: str) -> str: + env = { + "GIT_AUTHOR_NAME": "t", + "GIT_AUTHOR_EMAIL": "t@t", + "GIT_COMMITTER_NAME": "t", + "GIT_COMMITTER_EMAIL": "t@t", + "GIT_COMMITTER_DATE": "@1700000000 +0000", + "GIT_AUTHOR_DATE": "@1700000000 +0000", + "PATH": "/usr/bin:/bin", + } + return subprocess.run( + ["git", "-C", repo, *args], env=env, check=True, capture_output=True, text=True + ).stdout.strip() + + +def test_git_provenance(pmb_args: None, tmp_path: Path) -> None: + repo = tmp_path / "pmaports" + aport = repo / "main/hello" + other = repo / "main/other" + aport.mkdir(parents=True) + other.mkdir(parents=True) + _git(repo, "init", "-q") + (aport / "APKBUILD").write_text("pkgname=hello\n") + _git(repo, "add", ".") + _git(repo, "commit", "-q", "-m", "hello") + commit = _git(repo, "rev-parse", "HEAD") + + # The last commit that touched the aport, not HEAD + (other / "APKBUILD").write_text("pkgname=other\n") + _git(repo, "add", ".") + _git(repo, "commit", "-q", "-m", "other") + assert git_provenance(aport) == { + "ABUILD_LAST_COMMIT": commit, + "SOURCE_DATE_EPOCH": "1700000000", + } + + # Uncommitted changes: abuild's "-dirty", and no date from git + (aport / "APKBUILD").write_text("pkgname=hello\npkgrel=1\n") + assert git_provenance(aport) == {"ABUILD_LAST_COMMIT": f"{commit}-dirty"} + + # A new aport that was never committed + new = repo / "main/new" + new.mkdir() + (new / "APKBUILD").write_text("pkgname=new\n") + assert git_provenance(new) == {"ABUILD_LAST_COMMIT": "-dirty"} + + +def test_git_provenance_outside_git(pmb_args: None) -> None: + # Not tmp_path: pytest puts that inside this git checkout, and an aport + # there is in a checkout, just not its own. + with tempfile.TemporaryDirectory(dir="/tmp") as outside: + (Path(outside) / "APKBUILD").write_text("pkgname=hello\n") + assert git_provenance(Path(outside)) == {}