From a77fd20b5e22ad01413ecc3f9f2269b84e2ea537 Mon Sep 17 00:00:00 2001 From: DevomB Date: Sat, 19 Sep 2026 23:35:16 -0700 Subject: [PATCH 1/3] gcc pass 1 never sees a C library's headers, even when the sysroot came out of a cache Run 35492373903 (794c481) failed in stage 02, in gcc pass 2's libstdc++: "cannot convert '' to 'unsigned int'" at __GTHREAD_COND_INIT. It was called a from-scratch rebuild and was not one. The workflow's last restore key is a bare "work02-", so with every input changed it still restored the previous run's sysroot, and the toolchain was rebuilt on top of it. gcc's fixincludes keeps a private copy of every header it "fixes", and pthread.h is one (pthread_incomplete_struct_argument). Pass 1 normally runs before any C library exists and finds nothing. Here it found the old sysroot's glibc 2.40 headers and kept that pthread.h. glibc was then rebuilt from the release branch, which changes struct __pthread_cond_s and PTHREAD_COND_INITIALIZER together. The pass-1 compiler went on reading its private, old pthread.h beside the sysroot's new thread-shared-types.h, and the old initializer does not fit the new struct. The build log says all of it: "Fixing directory .../sysroot/usr/include", "Applying pthread_incomplete_struct_argument to pthread.h" in gcc-pass1.log, and the cache line "restored from key: work02-...-35482105602". So pass 1 removes usr/include when a C library's headers are there. The step only runs when the toolchain is being rebuilt, and linux-headers (which already clears the directory) and glibc put the headers back right after it. An incremental rebuild is then the same as a from-scratch one in the respect that broke, and a later glibc header change cannot meet a stale private copy, because pass 1 no longer has one. Neither the new gcc nor the glibc branch was at fault by itself; the order they were built in, over an old tree, was. --- build/stages/01-toolchain.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/build/stages/01-toolchain.sh b/build/stages/01-toolchain.sh index 444b32b..9dd5c34 100755 --- a/build/stages/01-toolchain.sh +++ b/build/stages/01-toolchain.sh @@ -101,6 +101,18 @@ s_binutils_pass1() { } s_gcc_pass1() { + # Pass 1 is built before any C library exists, and must be even when the + # sysroot came out of a cache with the last toolchain's headers in it. + # gcc's fixincludes keeps a private copy of every header it "fixes", and + # pthread.h is one. Built over an old sysroot, pass 1 kept glibc 2.40's + # pthread.h; glibc was then rebuilt with a changed pthread_cond_t, and + # pass 2 compiled libstdc++ with the old initializer against the new + # struct. This step only runs when the toolchain is being rebuilt, and + # linux-headers and glibc put the headers back right after it. + if [[ -e "${LFS}/usr/include/features.h" ]]; then + echo "removing the previous toolchain's headers from ${LFS}/usr/include" + rm -rf "${LFS}/usr/include" + fi local src src="$(unpack "gcc-${V_GCC}.tar.xz" "gcc-${V_GCC}")" cd "$src" From 8a4b9db822f77eebf75107b0133a2f63ddfe0b7e Mon Sep 17 00:00:00 2001 From: DevomB Date: Sat, 19 Sep 2026 23:39:02 -0700 Subject: [PATCH 2/3] A cross toolchain is never rebuilt over another one's sysroot: the sysroot records which toolchain built it The first version of this fix removed usr/include before gcc pass 1 when glibc's headers were there. Two readers' worth of looking said that was one header too narrow. A tree with the kernel's headers and no glibc would keep them, and fixincludes freezes private copies of dozens of linux/*.h too. And the headers are only what was shown to matter: a restored sysroot also keeps the old compiler's directory under tools and superseded libraries beside their replacements. So the rule is the general one and sits before the first step. The sysroot holds a marker: a hash of the binutils, gcc, glibc, kernel-header, mpfr, gmp and mpc versions and of glibc's patch set. A populated sysroot whose marker is missing or different was built by another toolchain: every stamp is archived, the sysroot is removed, and stage 01 starts from layout. It refuses if anything is mounted under the sysroot, and removes with --one-file-system, because stage 03 binds /dev, /proc and this repository in there. A sysroot with no headers gets this toolchain's marker. Stamps could not have caught this: the header that went stale is not an input of any step. It is the third time something outside a step's inputs has decided a result (the microcode beside the kernel tree, the release keys outside the cache), and the first where the cure is a record of what the tree was built by. The workflow keeps its bare work02- restore key on purpose. build/config/** is in that cache's key, so without the fallback every kernel fragment edit would rebuild the toolchain and the base system from nothing; with this rule the fallback is safe, because a restored tree from another toolchain clears itself. Exercised outside the build against six fake sysroots: fresh (marker written), same toolchain (left alone), gcc moved (cleared, two stamps archived), a cached tree with no marker (cleared), kernel headers and no glibc (cleared), something mounted under it (refused, nothing deleted). --- build/stages/01-toolchain.sh | 41 +++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/build/stages/01-toolchain.sh b/build/stages/01-toolchain.sh index 9dd5c34..a26ffdb 100755 --- a/build/stages/01-toolchain.sh +++ b/build/stages/01-toolchain.sh @@ -101,18 +101,6 @@ s_binutils_pass1() { } s_gcc_pass1() { - # Pass 1 is built before any C library exists, and must be even when the - # sysroot came out of a cache with the last toolchain's headers in it. - # gcc's fixincludes keeps a private copy of every header it "fixes", and - # pthread.h is one. Built over an old sysroot, pass 1 kept glibc 2.40's - # pthread.h; glibc was then rebuilt with a changed pthread_cond_t, and - # pass 2 compiled libstdc++ with the old initializer against the new - # struct. This step only runs when the toolchain is being rebuilt, and - # linux-headers and glibc put the headers back right after it. - if [[ -e "${LFS}/usr/include/features.h" ]]; then - echo "removing the previous toolchain's headers from ${LFS}/usr/include" - rm -rf "${LFS}/usr/include" - fi local src src="$(unpack "gcc-${V_GCC}.tar.xz" "gcc-${V_GCC}")" cd "$src" @@ -356,6 +344,35 @@ Run 'make check' for the full host requirement list." } preflight +# --- a cross toolchain is never rebuilt over another one's sysroot ------------- +# +# Pass 1 is built before any C library exists. Over a sysroot that came out of +# a cache it is not: gcc's fixincludes kept a private copy of the OLD glibc's +# pthread.h, glibc was then rebuilt with a changed pthread_cond_t, and pass 2 +# compiled libstdc++ with the old initializer against the new struct (run +# 35492373903). Stamps cannot see this: the header is not an input of any +# step. So the sysroot records which toolchain built it, and one that was +# built by another is cleared, with every stamp, before the first step. +toolchain_id="$({ printf '%s\n' "$V_BINUTILS" "$V_GCC" "$V_GLIBC" "$V_LINUX" "$V_MPFR" "$V_GMP" "$V_MPC" + cat "${KRYPTIK_ROOT}/build/patches/glibc-${V_GLIBC}/SHA256SUMS" 2>/dev/null; } | sha256_of_stdin)" +toolchain_marker="${LFS}/.kryptik-toolchain" +if [[ -d "${LFS}/usr/include" && "$(cat "$toolchain_marker" 2>/dev/null)" != "$toolchain_id" ]]; then + warn "the sysroot was built by a different toolchain (or by none this script recorded)." + # Never across a mount: stage 03 binds /dev, /proc and this repository in. + if grep -q " ${LFS}/" /proc/mounts; then + die "something is mounted under ${LFS}; unmount it (make chroot-umount), then run this again" + fi + old="${STAMPS}/legacy/toolchain-changed-$(date +%Y%m%dT%H%M%S)" + mkdir -p "$old" + find "$STAMPS" -maxdepth 1 -type f -exec mv -f {} "$old/" \; + rm -rf --one-file-system "${LFS:?}" + mkdir -p "$LFS" + warn "cleared ${LFS}; the stamps are archived under ${old}/. Everything is built again." +fi + +# A sysroot with no headers in it is about to be built by this toolchain. +[[ -d "${LFS}/usr/include" ]] || { mkdir -p "$LFS"; printf '%s\n' "$toolchain_id" > "$toolchain_marker"; } + step layout s_layout step binutils-pass1 s_binutils_pass1 step gcc-pass1 s_gcc_pass1 From c4a9bf1d02880ffe961cdaabffba0e4dd41c3916 Mon Sep 17 00:00:00 2001 From: DevomB Date: Sat, 19 Sep 2026 23:50:48 -0700 Subject: [PATCH 3/3] The sysroot is never removed across a mount it cannot see, and its record lives with the stamps Review of the rule found that its removal could reach the source tree. Stage 03 binds this repository into the sysroot. A bind mount from the same filesystem has the sysroot's own device number, so rm --one-file-system walks into it: that guard does not exist for exactly the mount that matters. That left `grep " ${LFS}/" /proc/mounts` as the only thing between a stale record and an rm -rf of the checkout, and it is blind three ways, each failing open: /proc/mounts writes a space as \040 (this checkout is called "Linux Distro"), it names the resolved path where $LFS may hold a symlink, and grep reads $LFS as a pattern, so a bracket in it misses or errors. The test now resolves the path, escapes it the way /proc/mounts does, and compares each mountpoint as a fixed string from its start; awk gets the string through ENVIRON, because -v would turn \040 back into a space. A mounts file that cannot be read counts as mounted. And the question is asked a second way that does not depend on parsing anything: the four directories stage 03 binds the repository, the sources and the work tree onto are empty when nothing is mounted, so any of them holding a file is a refusal. The record moves from the top of the sysroot to the stamps directory. Stage 06's root image is a copy of the sysroot with an exclude list that did not name it, so every verified root would have carried a build-host file at /. The cache restores the stamps and the sysroot together, a lost stamps directory reads as no record and clears, and the archive step moves the old record aside with the other stamps. The identity also hashes gcc and binutils patch sets, for the day the tree has any. Exercised outside the build: the six sysroots again, the sixth now with a file standing in for the repository (refused, and the file survived), and the mount test against a mounts file holding a plain path, a path with a space, one with brackets, one reached through a symlink, and a sibling whose name merely starts the same; all eight answers right, and an unreadable mounts file answers mounted. --- build/stages/01-toolchain.sh | 42 +++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/build/stages/01-toolchain.sh b/build/stages/01-toolchain.sh index a26ffdb..e2eefbd 100755 --- a/build/stages/01-toolchain.sh +++ b/build/stages/01-toolchain.sh @@ -346,22 +346,44 @@ preflight # --- a cross toolchain is never rebuilt over another one's sysroot ------------- # -# Pass 1 is built before any C library exists. Over a sysroot that came out of -# a cache it is not: gcc's fixincludes kept a private copy of the OLD glibc's +# Pass 1 is built before any header exists. Over a sysroot that came out of a +# cache it is not: gcc's fixincludes kept a private copy of the OLD glibc's # pthread.h, glibc was then rebuilt with a changed pthread_cond_t, and pass 2 # compiled libstdc++ with the old initializer against the new struct (run # 35492373903). Stamps cannot see this: the header is not an input of any -# step. So the sysroot records which toolchain built it, and one that was -# built by another is cleared, with every stamp, before the first step. +# step. So the tree records which toolchain built it, and one built by another +# is cleared, with every stamp, before the first step. The record lives with +# the stamps, not in the sysroot: the root image is a copy of the sysroot. toolchain_id="$({ printf '%s\n' "$V_BINUTILS" "$V_GCC" "$V_GLIBC" "$V_LINUX" "$V_MPFR" "$V_GMP" "$V_MPC" - cat "${KRYPTIK_ROOT}/build/patches/glibc-${V_GLIBC}/SHA256SUMS" 2>/dev/null; } | sha256_of_stdin)" -toolchain_marker="${LFS}/.kryptik-toolchain" + cat "${KRYPTIK_ROOT}"/build/patches/{glibc,gcc,binutils}-*/SHA256SUMS 2>/dev/null; } | sha256_of_stdin)" +toolchain_marker="${STAMPS}/toolchain-id" + +# Anything mounted under DIR? Stage 03 binds /dev, /proc and THIS REPOSITORY +# into the sysroot, and a bind mount from the same filesystem is not stopped +# by rm --one-file-system, so this test is what stands between a stale record +# and the source tree. /proc/mounts names the resolved path and writes a +# space as \040; the mountpoint is compared as a fixed string from its start +# (ENVIRON, because awk -v would turn \040 back into a space). Unreadable +# means yes. +mounted_under() { + local real esc + real="$(realpath -m -- "$1")" || return 0 + [[ -r /proc/mounts ]] || return 0 + esc="$(printf '%s' "$real" | sed -e 's/\\/\\134/g' -e 's/ /\\040/g' -e 's/\t/\\011/g')" + P="${esc}/" awk 'index($2, ENVIRON["P"]) == 1 { found = 1 } END { exit !found }' /proc/mounts +} + if [[ -d "${LFS}/usr/include" && "$(cat "$toolchain_marker" 2>/dev/null)" != "$toolchain_id" ]]; then warn "the sysroot was built by a different toolchain (or by none this script recorded)." - # Never across a mount: stage 03 binds /dev, /proc and this repository in. - if grep -q " ${LFS}/" /proc/mounts; then + if mounted_under "$LFS"; then die "something is mounted under ${LFS}; unmount it (make chroot-umount), then run this again" fi + # The same question asked a second way, of the four places stage 03 binds + # the repository, the sources and the work tree: unmounted, they are empty. + for d in kryptik kryptik-sources kryptik-work kryptik-kryptikd; do + [[ -z "$(ls -A "${LFS}/${d}" 2>/dev/null)" ]] \ + || die "${LFS}/${d} is not empty: it looks mounted. Refusing to remove the sysroot" + done old="${STAMPS}/legacy/toolchain-changed-$(date +%Y%m%dT%H%M%S)" mkdir -p "$old" find "$STAMPS" -maxdepth 1 -type f -exec mv -f {} "$old/" \; @@ -370,8 +392,8 @@ if [[ -d "${LFS}/usr/include" && "$(cat "$toolchain_marker" 2>/dev/null)" != "$t warn "cleared ${LFS}; the stamps are archived under ${old}/. Everything is built again." fi -# A sysroot with no headers in it is about to be built by this toolchain. -[[ -d "${LFS}/usr/include" ]] || { mkdir -p "$LFS"; printf '%s\n' "$toolchain_id" > "$toolchain_marker"; } +# A tree with no headers in it is about to be built by this toolchain. +[[ -d "${LFS}/usr/include" ]] || { mkdir -p "$STAMPS"; printf '%s\n' "$toolchain_id" > "$toolchain_marker"; } step layout s_layout step binutils-pass1 s_binutils_pass1