From 37caa06eac0f0b9527095cea4474614f95bf778c Mon Sep 17 00:00:00 2001 From: Giuseppe Maggio Date: Wed, 16 Sep 2026 01:38:43 +0000 Subject: [PATCH 1/2] pmb.build: pass the aport's commit and date to abuild abuild puts the last commit that touched an aport in the apk's .PKGINFO, and uses that commit's date as SOURCE_DATE_EPOCH so a rebuild is reproducible. It runs git in $startdir, which under pmbootstrap is the copy of the aport in the chroot, and link_to_git_dir() points that copy's .git at pmaports so it can: but then the APKBUILD is not the one git knows at that path ("main/foo/APKBUILD" against "APKBUILD"), so git_last_commit() finds nothing, git_dirty() sees a tree full of changes, and the aport being a git worktree -- which pmaports is in the porthole workspace -- breaks it a second way, since the .git file it links to names a path that does not exist in the chroot. Every package pmbootstrap has ever built therefore says commit = -dirty and falls back to the mtime of the copied APKBUILD, i.e. the time of the copy, as its date: the build is not reproducible and the apk does not say what it was built from. Ask the aport's own checkout instead, with the same three git commands abuild would run, and pass ABUILD_LAST_COMMIT and SOURCE_DATE_EPOCH in the build environment. A dirty aport keeps abuild's meaning -- "-dirty" and no date from git -- and an aport outside a git checkout is left to abuild. Verified: hello-world built twice in a row is byte-identical as an apk, and its .PKGINFO names the commit that last touched main/hello-world. Before, the two apks differed and both said "-dirty". Assisted-by: Claude Signed-off-by: Giuseppe Maggio --- FORK.md | 5 +++ pmb/build/backend.py | 33 +++++++++++++++++++ test/build/test_provenance.py | 62 +++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) create mode 100644 test/build/test_provenance.py diff --git a/FORK.md b/FORK.md index c81c2c78d..8c4ab1dd4 100644 --- a/FORK.md +++ b/FORK.md @@ -47,6 +47,11 @@ and documentation. `git log upstream/main..main` is the authoritative list. - **crossdirect Rust's native chroot gets build dependencies only, from binary repositories**: runtime depends, or a newer pmaports fork of a library, made pmbootstrap build whole packages for the native arch first. +- **Packages carry the commit they were built from**: abuild looked for git in + the copy of the aport inside the chroot, so every apk said + `commit = -dirty` and got the build time as its date. pmbootstrap now reads + the aport's checkout and passes `ABUILD_LAST_COMMIT` and + `SOURCE_DATE_EPOCH`. - **One sccache server per chroot**: sccache's default TCP port is shared by every chroot and work dir on the host, so one build's server compiled another work dir's crates against the wrong root and failed them. diff --git a/pmb/build/backend.py b/pmb/build/backend.py index 50433b35b..2374d186d 100644 --- a/pmb/build/backend.py +++ b/pmb/build/backend.py @@ -5,6 +5,7 @@ from pathlib import Path import pmb.chroot +import pmb.helpers.pmaports import pmb.helpers.run from pmb.core import Context from pmb.core.arch import Arch @@ -185,6 +186,37 @@ def handle_csum_failure(apkbuild: Apkbuild, chroot: Chroot) -> None: raise RuntimeError(f"Remote checksum mismatch for {apkbuild['pkgname']}") +def git_provenance(aport: Path) -> Env: + """ABUILD_LAST_COMMIT and SOURCE_DATE_EPOCH for an aport in a git checkout. + + abuild derives both from git: the last commit that touched the aport, + "-dirty" if the aport has uncommitted changes, and that commit's date as + SOURCE_DATE_EPOCH unless it is dirty. It runs git in the copy under + /home/pmos/build, though, whose .git link (see link_to_git_dir()) makes + the pmaports tree look moved to the chroot's build directory, and cannot + work at all for a pmaports git worktree. Every apk then said + "commit = -dirty" and got the copied APKBUILD's mtime, the time of the + build, as its date. Ask the checkout itself, the way abuild would. + + :returns: an empty dict if the aport is not in a git checkout + """ + + def git(*args: str) -> str: + return pmb.helpers.run.user_output( + ["git", *args], aport, output=RunOutputTypeDefault.NULL, check=False + ).strip() + + if git("rev-parse", "--is-inside-work-tree") != "true": + return {} + commit = "" + if git("ls-files", "--", "APKBUILD"): + commit = git("rev-list", "-n", "1", "HEAD", "--", ".") + if git("status", "--porcelain", "--", "."): + return {"ABUILD_LAST_COMMIT": f"{commit}-dirty"} + epoch = git("log", "-1", "--format=%cd", "--date=unix", commit, "--", ".") + return {"ABUILD_LAST_COMMIT": commit, "SOURCE_DATE_EPOCH": epoch} + + def rust_cross_native2_env(arch: Arch, sysroot: str = "/mnt/sysroot") -> Env: """Environment for cargo to cross-compile for arch in the native chroot. @@ -331,6 +363,7 @@ def run_abuild( ) env = abuild_env(context, arch, cross, bootstrap_stage) + env.update(git_provenance(pmb.helpers.pmaports.find(apkbuild["pkgname"]))) # Build the abuild command # Since we install dependencies with pmb, disable dependency handling in abuild. diff --git a/test/build/test_provenance.py b/test/build/test_provenance.py new file mode 100644 index 000000000..4b486be13 --- /dev/null +++ b/test/build/test_provenance.py @@ -0,0 +1,62 @@ +# Copyright 2026 Giuseppe Maggio +# SPDX-License-Identifier: GPL-3.0-or-later +import subprocess +import tempfile +from pathlib import Path + +from pmb.build.backend import git_provenance + + +def _git(repo: Path, *args: str) -> str: + env = { + "GIT_AUTHOR_NAME": "t", + "GIT_AUTHOR_EMAIL": "t@t", + "GIT_COMMITTER_NAME": "t", + "GIT_COMMITTER_EMAIL": "t@t", + "GIT_COMMITTER_DATE": "@1700000000 +0000", + "GIT_AUTHOR_DATE": "@1700000000 +0000", + "PATH": "/usr/bin:/bin", + } + return subprocess.run( + ["git", "-C", repo, *args], env=env, check=True, capture_output=True, text=True + ).stdout.strip() + + +def test_git_provenance(pmb_args: None, tmp_path: Path) -> None: + repo = tmp_path / "pmaports" + aport = repo / "main/hello" + other = repo / "main/other" + aport.mkdir(parents=True) + other.mkdir(parents=True) + _git(repo, "init", "-q") + (aport / "APKBUILD").write_text("pkgname=hello\n") + _git(repo, "add", ".") + _git(repo, "commit", "-q", "-m", "hello") + commit = _git(repo, "rev-parse", "HEAD") + + # The last commit that touched the aport, not HEAD + (other / "APKBUILD").write_text("pkgname=other\n") + _git(repo, "add", ".") + _git(repo, "commit", "-q", "-m", "other") + assert git_provenance(aport) == { + "ABUILD_LAST_COMMIT": commit, + "SOURCE_DATE_EPOCH": "1700000000", + } + + # Uncommitted changes: abuild's "-dirty", and no date from git + (aport / "APKBUILD").write_text("pkgname=hello\npkgrel=1\n") + assert git_provenance(aport) == {"ABUILD_LAST_COMMIT": f"{commit}-dirty"} + + # A new aport that was never committed + new = repo / "main/new" + new.mkdir() + (new / "APKBUILD").write_text("pkgname=new\n") + assert git_provenance(new) == {"ABUILD_LAST_COMMIT": "-dirty"} + + +def test_git_provenance_outside_git(pmb_args: None) -> None: + # Not tmp_path: pytest puts that inside this git checkout, and an aport + # there is in a checkout, just not its own. + with tempfile.TemporaryDirectory(dir="/tmp") as outside: + (Path(outside) / "APKBUILD").write_text("pkgname=hello\n") + assert git_provenance(Path(outside)) == {} From 86b311eee73308323c1c68bfae57130871eccf85 Mon Sep 17 00:00:00 2001 From: Giuseppe Maggio Date: Wed, 16 Sep 2026 01:38:54 +0000 Subject: [PATCH 2/2] docs: build caches, and what crossdirect now links natively Two things were undocumented. crossdirect 5.3.1-r6 (porthole-dev pmaports) runs GCC's link steps natively, including lto-wrapper and lto1, and leaves qemu only what has to answer for the target's GCC; cross_compiling.md still said that everything but the compiler goes through qemu. And nothing described the caches at all: which directory holds what, that a crossdirect build and a QEMU-only build of the same package write to the same cache but cannot share entries, how to point two work directories at one cache (measured: 268 of 274 compiles served across work directories), and what CI has to restore, with the two things that silently make a restored cache useless -- the chroot's uid 12345 owning it, and a 5G per-arch limit against a 10G cache budget. Assisted-by: Claude Signed-off-by: Giuseppe Maggio --- FORK.md | 12 +++--- docs/build_caches.md | 83 +++++++++++++++++++++++++++++++++++++++++ docs/cross_compiling.md | 19 ++++++++-- docs/index.rst | 1 + 4 files changed, 106 insertions(+), 9 deletions(-) create mode 100644 docs/build_caches.md diff --git a/FORK.md b/FORK.md index 8c4ab1dd4..47979d0cb 100644 --- a/FORK.md +++ b/FORK.md @@ -56,12 +56,12 @@ and documentation. `git log upstream/main..main` is the authoritative list. every chroot and work dir on the host, so one build's server compiled another work dir's crates against the wrong root and failed them. -The changes that make crossdirect handle more of Rust live in pmaports, -because crossdirect is an aport (`cross/crossdirect`, 5.3.1-r4 in the -porthole-dev pmaports): rustc called directly by meson compiles target crates -for the target and proc-macros natively, `cargo auditable build` no longer -falls back to QEMU, and `bindgen` runs natively. See -[docs/cross_compiling.md](docs/cross_compiling.md). +The changes that make crossdirect faster live in pmaports, because crossdirect +is an aport (`cross/crossdirect` in the porthole-dev pmaports): GCC links, +including LTO, run natively instead of under QEMU; rustc called directly by +meson compiles target crates for the target and proc-macros natively; `cargo +auditable build` no longer falls back to QEMU; and `bindgen` runs natively. +See [docs/cross_compiling.md](docs/cross_compiling.md). ## Update from upstream diff --git a/docs/build_caches.md b/docs/build_caches.md new file mode 100644 index 000000000..3aef595c2 --- /dev/null +++ b/docs/build_caches.md @@ -0,0 +1,83 @@ +# Build caches + +Everything pmbootstrap caches lives in the work directory, next to the +chroots. Only the caches are worth keeping between builds or restoring in CI; +the chroots are rebuilt from packages in seconds. + + +| Directory | What it holds | Written by | +|---------------------------|--------------------------------------------------------|---------------------------------| +| `cache_ccache_$ARCH` | ccache: compiled C/C++ objects | every package build | +| `cache_sccache` | sccache: compiled Rust crates | Rust builds without crossdirect | +| `cache_rust` | cargo registry and git checkouts | Rust builds | +| `cache_go` | `GOCACHE` and `GOMODCACHE` | Go builds | +| `cache_distfiles` | source tarballs abuild downloaded | every package build | +| `cache_apk_$ARCH` | apks downloaded from the mirrors | every chroot | +| `cache_git`, `cache_http` | git clones (pmaports, aports), APKINDEX and apk.static | pmbootstrap itself | + + +`$ARCH` is the architecture of the *chroot the build runs in*, which is not +always the architecture of the package. A crossdirect build of an aarch64 +package runs the native cross compiler, but from inside the foreign chroot, +so its objects land in `cache_ccache_aarch64` next to the ones a QEMU-only +build of the same package writes. They do not collide -- ccache hashes the +compiler binary -- but they do not share either: moving a package between +QEMU-only and crossdirect starts from a cold cache. A cross-native build (the +kernel) runs in the native chroot, and its objects go to +`cache_ccache_x86_64`, not to the target's cache. + +## Sharing one cache between work directories + +Two work directories (a second checkout, a test work dir, a `--work` +elsewhere) each have their own caches and share nothing. Point them at one +directory with a symlink, before the first build: + +```shell +ln -sfn /srv/pmb-cache/ccache_aarch64 "$WORK/cache_ccache_aarch64" +ln -sfn /srv/pmb-cache/distfiles "$WORK/cache_distfiles" +``` + +pmbootstrap bind-mounts these into the chroots and follows the symlink. +Measured: libcamera built in a second, empty work directory (new chroots, +freshly installed compilers) took 2m17s with 268 of 274 cacheable compiles +served from the first work directory's ccache, against 5m00s cold. + +ccache is safe for several builds using one cache directory at the same time. +sccache is not: it keeps one server per cache directory and accounts for the +cache size in that server, so share `cache_sccache` only between work +directories that do not build at the same time. + +## CI + +Restore and save the caches, not the work directory. A job that builds one +package at a time wants one cache entry per package, because that keeps each +entry small enough for a cache service with a size limit (GitHub: 10 GB per +repository, least recently used entries are evicted): + +```yaml +- uses: actions/cache@v6 + with: + path: | + ${{ runner.temp }}/work/cache_ccache_aarch64 + ${{ runner.temp }}/work/cache_sccache + ${{ runner.temp }}/work/cache_rust + ${{ runner.temp }}/work/cache_go + ${{ runner.temp }}/work/cache_distfiles + key: pmb-build-aarch64-${{ matrix.package }}-${{ github.run_id }} + restore-keys: pmb-build-aarch64-${{ matrix.package }}- +``` + +A key that ends in the run id is never restored, only saved, and +`restore-keys` then picks the newest entry of the same package: every run +saves a fresh entry and starts from the previous one. Keep the apk cache in +its own entry (`pmb-apk-aarch64-...`), because every package's job fills it +with the same downloads. + +Two things make a restored cache useless, and both are easy to miss: + +* **Ownership.** Builds run as uid 12345 inside the chroots, while the cache + comes back owned by the runner. `chown -R 12345:12345` the ccache and + distfiles directories after restoring them. +* **Size.** `ccache_size` is 5G per architecture by default, which one large + package can fill and which is half of a repository's whole cache budget. + `pmbootstrap config ccache_size 2G` keeps an entry restorable. diff --git a/docs/cross_compiling.md b/docs/cross_compiling.md index 7466ba4f5..486f890e2 100644 --- a/docs/cross_compiling.md +++ b/docs/cross_compiling.md @@ -37,9 +37,22 @@ with other methods. This is the default method. This method works for almost all packages, and gives a good speed improvement -over running everything in QEMU. However only the cross compilers run natively. -Linkers and all other commands used during the build still need to run through -QEMU and so these are still very slow. +over running everything in QEMU. The cross compilers run natively, and with +crossdirect 5.3.1-r6 or later (porthole-dev pmaports) so do GCC's link steps: +`collect2`, `ld` and, for a package built with LTO, `lto-wrapper` and `lto1`. +The rest of the build -- the build system, generators, `meson`, `python3` -- +still runs through QEMU and is still slow. + +Link steps get `--sysroot=/` plus `-B/usr/lib/gcc/` and `-B/usr/lib/`, so the +driver takes the target's own GCC runtime (`crtbegin*.o`, `libgcc`, +`libstdc++`, spec files such as `libgomp.spec`) instead of the cross +toolchain's copies. The linked output is then byte-for-byte what the target's +GCC produces under QEMU, apart from the build ID. Invocations that are not +links keep running in QEMU, because their output has to describe the target's +GCC: `-E`, `-S`, `-M`, `-v`, `--version` and `-print-*` (libtool runs the +linker path that `-print-prog-name=ld` gives it, and the cross `ld` cannot run +outside crossdirect's environment), and any compiler call under `fakeroot`, +i.e. in `package()`. The native chroot gets mounted in the foreign arch chroot at `/native`. The [crossdirect](https://gitlab.postmarketos.org/postmarketOS/pmaports/-/blob/main/cross/crossdirect/APKBUILD) diff --git a/docs/index.rst b/docs/index.rst index 44e893882..adf40df42 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,6 +17,7 @@ In case of any problems, have a look at the `issue-tracker`_. chroot debugging cross_compiling + build_caches ssh-keys mirrors environment_variables