From 538003fec69fddf536e38aed6f9d9ffe6cc06e4a Mon Sep 17 00:00:00 2001 From: virant Date: Mon, 2 Nov 2026 13:10:31 +0100 Subject: [PATCH 1/2] fix(packaging): build DKMS modules when kernel headers are configured bindeb-pkg's linux-headers package has no maintainer scripts. When apt configures linux-image before linux-headers, the image's DKMS hook skips the kernel ("kernel headers ... do not seem to be installed") and nothing reruns it, so the kernel boots without its DKMS modules. A SmoothNAS box upgraded to 7.2.14-83 that way booted without ZFS: its pool didn't import and SMB and NFS failed on the shares stored on it. Give the headers package a postinst that runs /etc/kernel/header_postinst.d on configure, as Debian's headers packages do. The build fails if bindeb-pkg ever ships its own headers postinst, rather than silently overwriting it. Test: make check-headers-dkms-order (a new release CI step) installs openzfs-zfs-dkms, then linux-image alone, then linux-headers in a Debian 13 container. ZFS must be absent after the image (negative control) and built for the kernel, with its vermagic, after the headers. --- .github/workflows/release.yml | 6 +++ Makefile | 6 ++- docs/DKMS.md | 8 ++++ recipes/build-kernel.sh | 39 +++++++++++++++++++ recipes/check-headers-dkms-order.sh | 59 +++++++++++++++++++++++++++++ 5 files changed, 117 insertions(+), 1 deletion(-) create mode 100755 recipes/check-headers-dkms-order.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 60f8ef7..b2a724a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -240,6 +240,12 @@ jobs: - name: Validate packaged ZFS DKMS against the built kernel run: sudo make ENV_FILE=examples/smooth.env check-zfs-dkms DEB_ARCH=${{ matrix.deb_arch }} BUILD_THREADS="$(nproc)" + # apt may configure linux-image before linux-headers; the image's DKMS + # hook then skips the kernel, and only the headers' postinst can build + # the modules. Install in that order and require ZFS to get built. + - name: Headers configured after the image still build ZFS + run: make check-headers-dkms-order DEB_ARCH=${{ matrix.deb_arch }} + # Independent of the checks inside build-kernel.sh: this re-derives the # answer from the finished .deb, so a bug in the recipe's own verification # cannot wave a bad artifact through. diff --git a/Makefile b/Makefile index d4dd67c..6e0a72e 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ NOBARA_PATCHSET ?= nobara-picks POST_NOBARA_PATCHSET ?= post-nobara-$(KERNEL_VERSION) OUT_DIR ?= $(CURDIR)/out -.PHONY: help kernel kernel-all kernel-config-update kernel-config-update-all check-rocm-config check-router-config check-zfs-dkms zfs clean show +.PHONY: help kernel kernel-all kernel-config-update kernel-config-update-all check-rocm-config check-router-config check-zfs-dkms check-headers-dkms-order zfs clean show help: @echo "smoothkernel — Smooth* shared kernel build harness" @@ -38,6 +38,7 @@ help: @echo " check-router-config Assert every arch keeps the SmoothRouter netfilter/PPPoE symbols" @echo " zfs Build OpenZFS .debs (zfs-dkms + libs + utils)" @echo " check-zfs-dkms Rebuild packaged ZFS against the built kernel (native)" + @echo " check-headers-dkms-order Install image before headers; headers must build ZFS (docker)" @echo " clean Remove build trees + out/" @echo " show Print the resolved build environment" @echo "" @@ -99,6 +100,9 @@ check-router-config: check-zfs-dkms: @$(CURDIR)/recipes/check-zfs-dkms.sh +check-headers-dkms-order: + @$(CURDIR)/recipes/check-headers-dkms-order.sh + zfs: @$(CURDIR)/recipes/build-zfs.sh diff --git a/docs/DKMS.md b/docs/DKMS.md index 6e67f42..655e4c7 100644 --- a/docs/DKMS.md +++ b/docs/DKMS.md @@ -96,6 +96,14 @@ Flavor meta packages may expose stable dependencies, but DKMS ultimately uses: /lib/modules/$(uname -r)/build ``` +apt may configure `linux-image-*` before `linux-headers-*`. The image's +postinst runs `/etc/kernel/postinst.d/dkms`, which skips a kernel whose headers +aren't there yet, so the headers package's postinst reruns +`/etc/kernel/header_postinst.d` (DKMS's hook) as Debian's headers packages do. +`bindeb-pkg` ships no headers postinst; `recipes/build-kernel.sh` adds it, and +`make check-headers-dkms-order` installs the packages in that order and +requires ZFS to get built. + ## Secure Boot DKMS-built modules are signed on the target machine by the diff --git a/recipes/build-kernel.sh b/recipes/build-kernel.sh index d6403ac..5b129ea 100755 --- a/recipes/build-kernel.sh +++ b/recipes/build-kernel.sh @@ -172,11 +172,50 @@ add_headers_virtual_provides() { sed -i "/^Version: /a Provides: ${provides}" "$control" fi + add_headers_postinst "$tmp" + dpkg-deb -b "$tmp" "${deb}.patched" >/dev/null mv "${deb}.patched" "$deb" rm -rf "$tmp" } +# bindeb-pkg's headers package has no maintainer scripts. The image's postinst +# runs /etc/kernel/postinst.d/dkms, which skips a kernel whose headers aren't +# configured yet, and apt is free to configure the image first. Debian's +# headers packages rerun /etc/kernel/header_postinst.d (DKMS's hook) when they +# are configured; without that, nothing ever builds DKMS modules for the new +# kernel. A SmoothNAS box upgraded that way booted 7.2.14-83 without ZFS. +add_headers_postinst() { + local root="$1" + local package krel postinst + + package="$(sed -n 's/^Package: //p' "$root/DEBIAN/control" | head -1)" + krel="${package#linux-headers-}" + if [[ -z "$krel" || "$krel" == "$package" ]]; then + echo "ERROR: cannot derive the kernel release from headers package '$package'" >&2 + exit 1 + fi + postinst="$root/DEBIAN/postinst" + if [[ -e "$postinst" ]]; then + echo "ERROR: bindeb-pkg now ships a headers postinst; merge the DKMS hook into it" >&2 + exit 1 + fi + cat > "$postinst" <, the boot-newest script, the /etc/kernel hooks that run # it (before GRUB's zz-update-grub) and the maintainer scripts that apply it # when the package itself is installed or removed. diff --git a/recipes/check-headers-dkms-order.sh b/recipes/check-headers-dkms-order.sh new file mode 100755 index 0000000..58d3ab3 --- /dev/null +++ b/recipes/check-headers-dkms-order.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Install this build's packages in a Debian 13 container in the order that +# left a SmoothNAS box without ZFS: OpenZFS DKMS first, then linux-image on +# its own (its DKMS hook skips a kernel without headers), then the headers. +# The headers' postinst must rerun the DKMS hook and build ZFS. +# +# Needs docker. The negative control (no zfs.ko after the image alone) makes +# sure the last check can't pass because something else built the module. +set -euo pipefail +DEB_ARCH="${DEB_ARCH:-amd64}" +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +OUT_DIR="${OUT_DIR:-$ROOT/out}" +IMAGE="${CHECK_IMAGE:-debian:trixie}" + +shopt -s nullglob +images=("$OUT_DIR"/linux-image-*-smoothkernel_*_"$DEB_ARCH".deb) +headers=("$OUT_DIR"/linux-headers-*-smoothkernel_*_"$DEB_ARCH".deb) +zfs=("$OUT_DIR"/openzfs-zfs-dkms_*.deb) +[[ ${#images[@]} -eq 1 && ${#headers[@]} -eq 1 && ${#zfs[@]} -eq 1 ]] || { + echo "Expected one linux-image, one linux-headers and one openzfs-zfs-dkms .deb in $OUT_DIR" >&2 + ls -la "$OUT_DIR" >&2 + exit 1 +} +krel="$(dpkg-deb -f "${headers[0]}" Package)" +krel="${krel#linux-headers-}" + +docker run --rm -v "$OUT_DIR:/out:ro" -e KREL="$krel" \ + -e IMG="/out/$(basename "${images[0]}")" \ + -e HDR="/out/$(basename "${headers[0]}")" \ + -e ZFS="/out/$(basename "${zfs[0]}")" \ + "$IMAGE" bash -euo pipefail -c ' +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq +zfs_installed() { dkms status -k "$KREL" | grep -Eq "^zfs/.*: installed"; } + +echo "==> OpenZFS DKMS, before any SmoothKernel" +apt-get install -y -qq "$ZFS" >/dev/null + +echo "==> linux-image $KREL without its headers" +apt-get install -y -qq --no-install-recommends "$IMG" >/dev/null +if zfs_installed || [ -n "$(find "/lib/modules/$KREL" -name "zfs.ko*")" ]; then + echo "ZFS was built before the headers were installed; this check proves nothing" >&2 + exit 1 +fi + +echo "==> linux-headers $KREL" +apt-get install -y -qq "$HDR" >/dev/null +if ! zfs_installed; then + dkms status >&2 + find /var/lib/dkms -name make.log -exec tail -n 40 {} \; >&2 + echo "configuring linux-headers-$KREL did not build ZFS for it" >&2 + exit 1 +fi +vermagic="$(modinfo -k "$KREL" -F vermagic zfs)" +case "$vermagic" in + "$KREL "*) echo "==> zfs.ko built for $KREL: $vermagic" ;; + *) echo "zfs.ko has the wrong vermagic: $vermagic" >&2; exit 1 ;; +esac +' From 8d090a4e6d83a59e180943bd8b6ae0e55f3c0a04 Mon Sep 17 00:00:00 2001 From: virant Date: Mon, 2 Nov 2026 14:08:12 +0100 Subject: [PATCH 2/2] fix(packaging): declare the libraries the headers' host tools link The new headers postinst builds DKMS modules when the headers are configured, and check-headers-dkms-order failed on amd64 doing it: dpkg could not configure linux-headers-7.2.15-86-smoothkernel. The headers ship a prebuilt tools/objtool/objtool, which links libelf.so.1 and runs in every x86 out-of-tree build, but bindeb-pkg declares no Depends, and neither dkms nor openzfs-zfs-dkms pulls libelf in. arm64 passed because it has no objtool. SmoothNAS boxes only worked because the smoothnas package depends on libelf-dev. - build-kernel.sh reads what each ELF tool in the headers package links and adds the matching Depends (libelf1t64 | libelf1, libssl3t64 | libssl3 for sign-file); a library with no package mapping fails the build instead of shipping a package that cannot build modules. - check-headers-dkms-order.sh no longer discards apt output: a failed step prints its apt/dpkg output, dkms status and every DKMS make.log, so the CI log says why. Its container stays minimal on purpose. --- docs/DKMS.md | 7 ++++ recipes/build-kernel.sh | 55 +++++++++++++++++++++++++++++ recipes/check-headers-dkms-order.sh | 41 ++++++++++++++++----- 3 files changed, 94 insertions(+), 9 deletions(-) diff --git a/docs/DKMS.md b/docs/DKMS.md index 655e4c7..5b9c717 100644 --- a/docs/DKMS.md +++ b/docs/DKMS.md @@ -104,6 +104,13 @@ aren't there yet, so the headers package's postinst reruns `make check-headers-dkms-order` installs the packages in that order and requires ZFS to get built. +The headers package also ships prebuilt host tools that every module build +runs: `objtool` (x86) links libelf and `sign-file` links libcrypto. +`bindeb-pkg` declares no Depends for them, so `build-kernel.sh` adds the +libraries they link, and fails the build if one links a library it has no +package mapping for. The `check-headers-dkms-order` container is deliberately +minimal, so it fails if the headers package doesn't bring what it needs. + ## Secure Boot DKMS-built modules are signed on the target machine by the diff --git a/recipes/build-kernel.sh b/recipes/build-kernel.sh index 5b129ea..e340cba 100755 --- a/recipes/build-kernel.sh +++ b/recipes/build-kernel.sh @@ -173,6 +173,7 @@ add_headers_virtual_provides() { fi add_headers_postinst "$tmp" + add_headers_tool_depends "$tmp" dpkg-deb -b "$tmp" "${deb}.patched" >/dev/null mv "${deb}.patched" "$deb" @@ -216,6 +217,60 @@ EOF chmod 755 "$postinst" } +# The headers package ships prebuilt host tools that every out-of-tree module +# build runs (objtool on x86 links libelf; sign-file links libcrypto), but +# bindeb-pkg declares no Depends. On a system without libelf, DKMS can't build +# a single module. Declare the libraries those tools link, and fail the build +# if one links a library this map doesn't know, rather than ship a package +# that installs but can't build. +add_headers_tool_depends() { + local root="$1" + local control="$root/DEBIAN/control" + local -A pkg_for=( + [libelf.so.1]="libelf1t64 | libelf1" + [libcrypto.so.3]="libssl3t64 | libssl3" + [libz.so.1]="zlib1g" + [libzstd.so.1]="libzstd1" + [liblzma.so.5]="liblzma5" + ) + local -A needed=() + local f lib unknown="" + + while IFS= read -r -d '' f; do + [[ "$(head -c 4 "$f" 2>/dev/null | od -An -tx1 | tr -d ' \n')" == 7f454c46 ]] || continue + while read -r lib; do + case "$lib" in + libc.so.*|libm.so.*|libpthread.so.*|libdl.so.*|librt.so.*|ld-linux*) ;; + *) + if [[ -n "${pkg_for[$lib]:-}" ]]; then + needed[$lib]=1 + else + unknown+=" ${f#"$root"/} needs $lib;" + fi + ;; + esac + done < <(readelf -d "$f" 2>/dev/null | sed -n 's/.*(NEEDED).*\[\(.*\)\]/\1/p') + done < <(find "$root" -path "$root/DEBIAN" -prune -o -type f -perm -u+x -print0) + + if [[ -n "$unknown" ]]; then + echo "ERROR: headers host tools link libraries with no package mapping:$unknown" >&2 + echo " add them to add_headers_tool_depends in recipes/build-kernel.sh" >&2 + exit 1 + fi + [[ ${#needed[@]} -gt 0 ]] || return 0 + + local depends="" l + for l in $(printf '%s\n' "${!needed[@]}" | sort); do + depends+="${depends:+, }${pkg_for[$l]}" + done + echo " headers Depends: $depends" + if grep -q '^Depends:' "$control"; then + sed -i "s/^Depends: /Depends: ${depends}, /" "$control" + else + sed -i "/^Version: /a Depends: ${depends}" "$control" + fi +} + # Lay out, under , the boot-newest script, the /etc/kernel hooks that run # it (before GRUB's zz-update-grub) and the maintainer scripts that apply it # when the package itself is installed or removed. diff --git a/recipes/check-headers-dkms-order.sh b/recipes/check-headers-dkms-order.sh index 58d3ab3..7a18774 100755 --- a/recipes/check-headers-dkms-order.sh +++ b/recipes/check-headers-dkms-order.sh @@ -30,24 +30,47 @@ docker run --rm -v "$OUT_DIR:/out:ro" -e KREL="$krel" \ -e ZFS="/out/$(basename "${zfs[0]}")" \ "$IMAGE" bash -euo pipefail -c ' export DEBIAN_FRONTEND=noninteractive -apt-get update -qq zfs_installed() { dkms status -k "$KREL" | grep -Eq "^zfs/.*: installed"; } -echo "==> OpenZFS DKMS, before any SmoothKernel" -apt-get install -y -qq "$ZFS" >/dev/null +# Show why a step failed: its apt/dpkg output, DKMS state and build logs. +diagnose() { + echo "---- output of: $1" >&2 + cat /tmp/step.log >&2 || true + if command -v dkms >/dev/null; then + echo "---- dkms status" >&2 + dkms status >&2 || true + fi + for log in /var/lib/dkms/*/*/build/make.log /var/lib/dkms/*/*/*/*/log/make.log; do + [ -f "$log" ] || continue + echo "---- $log (last 80 lines)" >&2 + tail -n 80 "$log" >&2 + done +} +# Run a step quietly; on failure print everything above, then fail. +step() { + local what="$1"; shift + echo "==> $what" + if ! "$@" > /tmp/step.log 2>&1; then + diagnose "$what" + echo "FAILED: $what" >&2 + exit 1 + fi +} -echo "==> linux-image $KREL without its headers" -apt-get install -y -qq --no-install-recommends "$IMG" >/dev/null +step "apt-get update" apt-get update +step "OpenZFS DKMS, before any SmoothKernel" apt-get install -y "$ZFS" +step "linux-image $KREL without its headers" apt-get install -y --no-install-recommends "$IMG" if zfs_installed || [ -n "$(find "/lib/modules/$KREL" -name "zfs.ko*")" ]; then echo "ZFS was built before the headers were installed; this check proves nothing" >&2 exit 1 fi -echo "==> linux-headers $KREL" -apt-get install -y -qq "$HDR" >/dev/null +# The headers must bring the libraries their host tools need (objtool links +# libelf), so the container stays minimal on purpose. +step "linux-headers $KREL" apt-get install -y "$HDR" if ! zfs_installed; then - dkms status >&2 - find /var/lib/dkms -name make.log -exec tail -n 40 {} \; >&2 + : > /tmp/step.log + diagnose "linux-headers $KREL" echo "configuring linux-headers-$KREL did not build ZFS for it" >&2 exit 1 fi