quiz-kernel

#!/usr/bin/env bash
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Copyright (c) 2023-2025, Rob Norris <robn@despairlabs.com>

set -uo pipefail

usage() {
  cat <<EOF
compile and manage quiz custom kernels
usage: quiz-kernel [opts]

options:
  -k <kernel>     kernel to build
                    (can be used multiple times)
  -a <arch>       architecture to build for
  -e/-d/-m <key>  set kernel config key to enable/disable/module (Y/N/M)
                    (can be used multiple times)
  -u              update standard config for this kernel minor version
  -L              build with LLVM/Clang
  -T              initialise missing config from tinyconfig
  -C              don't install kernel config before build
  -B              don't build kernel
  -K              rebuild all installed kernels
                    (equivalent to -k <v1> -k <v2> ...)
  -U              upgrade kernels (build latest for minor series)
  -X              remove old kernels in same minor series
  -h              this help
EOF
  exit 1
}

trace() {
  local STAMP=$(date +%Y%m%d-%H:%M:%S)
  echo "[quiz-kernel] $STAMP $@" >&2
}

fail() {
  local TEXT=${@:-}
  if [[ -z $TEXT ]] ; then
    TEXT="[${BASH_SOURCE[0]}:${BASH_LINENO[0]}: $BASH_COMMAND]"
  fi
  trace "FATAL $TEXT"
  exit 1
}
trap fail ERR

RUNDIR=$(realpath $(dirname $0))
source $RUNDIR/quiz-config
source $RUNDIR/quiz-lib

opt_arch=""
opt_kernel=""
opt_init_config=""
opt_no_config=""
opt_no_build=""
opt_llvm=""
opt_update_config=""
opt_modify_config=""
opt_all_kernels=""
opt_upgrade=""
opt_upgrade_cleanup=""

OPTIND=1
while getopts "a:k:ICBe:d:m:uLKUXh" opt
do
  case "$opt" in
    'a') opt_arch=$OPTARG ;;
    'k') opt_kernel="$opt_kernel $OPTARG" ;;
    'I') opt_init_config=1 ;;
    'C') opt_no_config=1 ;;
    'B') opt_no_build=1 ;;
    'e') opt_modify_config="$opt_modify_config -e $OPTARG" ;;
    'd') opt_modify_config="$opt_modify_config -d $OPTARG" ;;
    'm') opt_modify_config="$opt_modify_config -m $OPTARG" ;;
    'u') opt_update_config=1 ;;
    'L') opt_llvm=1 ;;
    'K') opt_all_kernels=1 ;;
    'U') opt_upgrade=1 ;;
    'X') opt_upgrade_cleanup=1 ;;
    'h') usage ;;
    *) exit 1 ;;
  esac
done
shift $(expr $OPTIND - 1)

quiz_arch_vars ${opt_arch:-$QUIZ_ARCH}

cd $RUNDIR

if [[ -n $opt_kernel && -n $opt_all_kernels ]] ; then
  fail "invalid kernel version options; can't use -k and -K together"
fi
if [[ -z $opt_kernel && -z $opt_all_kernels ]] ; then
  fail "no kernel version supplied; one of -k or -K required"
fi

if [[ -n $opt_all_kernels ]] ; then
    opt_kernel=$(ls system/kernel/$_quiz_arch_kernel/kernel-* | cut -d- -f2 | sort -V)
    if [[ -z $opt_kernel ]] ; then
      fail "no kernels found in system/kernel/$_quiz_arch_kernel"
    fi
fi

# dedicate half the cpus, rounded up, to building kernels
ncpus=$((($(nproc)+1)/2))
trace "using $ncpus cpus for kernel builds"

# makeopts are switches for make. makevars are variables added to the end
makeopts="-j$ncpus"
makevars=""

if [[ -n $opt_llvm ]] ; then
  IFS=:
  best_clang=$(find $PATH -maxdepth 1 -executable -name "clang-??" -print | \
    sed -Ee 's/^.+-//' | sort -r | head -1)
  unset IFS
  if [[ -z "$best_clang" ]] ; then
    trace "no specific clang found, setting LLVM=1"
    best_clang=1
  else
    trace "found clang-$best_clang, setting LLVM=-$best_clang"
    best_clang="-$best_clang"
  fi
  makevars="$makevars LLVM=$best_clang"
fi

if [[ -z $_quiz_arch_native ]] ; then
  trace "setting up for cross-compile: $_quiz_arch_kernel"
  makevars="$makevars CROSS_COMPILE=${_quiz_arch_prefix}- ARCH=$_quiz_arch_kernel"
fi

mkdir -p build/kernel

declare -A kernel_sumfile

kernel_versions=""
for kver in $opt_kernel ; do
  _quiz_kernel_vars $kver
  if [[ $_quiz_kmajor && $_quiz_kminor && ( -z $_quiz_kpatch || -n $opt_upgrade ) ]] ; then
    if [[ -z ${kernel_sumfile[$_quiz_kmajor]:-} ]] ; then
      sumfile=build/kernel/sha256sums.$_quiz_kmajor
      trace "fetching list of available kernels for v$_quiz_kmajor.x"
      if [[ ! -e $sumfile ]] ; then
        touch --date=@0 $sumfile
      fi
      curl -# -f -o $sumfile -z $sumfile -R \
        https://cdn.kernel.org/pub/linux/kernel/v$_quiz_kmajor.x/sha256sums.asc
      kernel_sumfile[$_quiz_kmajor]=$sumfile
    fi
    latest=$(cat ${kernel_sumfile[$_quiz_kmajor]} | \
      sed -Ene "/linux-.+\.tar\.xz/ { s/.*linux-($_quiz_kmajor\.$_quiz_kminor(\.[0-9]+)?)\.tar\.xz/\1/ p }" | tail -1)
    if [[ -z $latest ]] ; then
      trace "no latest kernel matching $_quiz_kmajor.$_quiz_kminor"
      kver=""
    elif [[ -n $opt_upgrade ]] ; then
      _quiz_kernel_vars $latest
      if [[ -z $_quiz_kpatch ]] ; then
        latest="$_quiz_kmajor.$_quiz_kminor.0"
      fi
      if [[ $latest == $kver ]] ; then
        trace "$kver already latest, not rebuilding"
        kver=""
      else
        trace "will upgrade $kver to $latest"
        kver=$latest
      fi
    else
      trace "will build $latest for $kver"
      kver=$latest
    fi
  fi
  if [[ $kver ]] ; then
    kernel_versions="$kernel_versions $kver"
  fi
done

kernel_versions=$(echo $kernel_versions | xargs -n1 | sort -V | uniq | xargs)
trace "will build kernel versions: $kernel_versions"

kdest=$RUNDIR/system/kernel/$_quiz_arch_kernel

build_kernel() {
  local kver=$1

  quiz_kernel_vars $kver

  case $_quiz_ktype in
  release)
    ksrctype=tar
    karchive=linux-$_quiz_ksrcver.tar.xz
    kurl=https://cdn.kernel.org/pub/linux/kernel/v$_quiz_kmajor.x/$karchive
    kzopt=J
    kdir=linux-$_quiz_ksrcver
    ;;
  rc)
    ksrctype=tar
    karchive=linux-$_quiz_ksrcver.tar.gz
    kurl=https://git.kernel.org/torvalds/t/$karchive
    kzopt=z
    kdir=linux-$_quiz_ksrcver
    ;;
  next)
    ksrctype=tar
    karchive=next-$_quiz_ktype_patch.tar.gz
    kurl=https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/snapshot/linux-next-$karchive
    kzopt=z
    kdir=linux-next-next-$_quiz_ktype_patch
    ;;
  master)
    ksrctype=git
    kgittag=master
    kgiturl=https://github.com/torvalds/linux.git
    kdir=linux-master-git
    ;;
  *)
    fail "unknown kernel version type: $_quiz_ktype"
  esac

  trace "preparing build: $_quiz_kver ($_quiz_ktype)"

  ksrc=$RUNDIR/build/kernel/$kdir

  if [[ -d $ksrc ]] ; then
    trace "using existing source dir: $ksrc"
    if [[ $ksrctype == 'git' ]] ; then
      trace "cleaning up git dir: $ksrc"
      # XXX could we be less destructive if we're building the same again?
      #     (maybe check localversion-quiz?) -- robn, 2026-02-13
      git -C $ksrc clean -fdxq
      # XXX is this the right place to update? should it require -u?
      #       -- robn, 2026-02-13
      git -C $ksrc pull
    fi
  else
    trace "setting up for build in: $ksrc"
    pushd build/kernel > /dev/null
    case $ksrctype in
    tar|zip)
      if [[ -f $karchive ]] ; then
        trace "using existing archive: $karchive"
      else
        trace "downloading: $kurl"
        mkdir -p tmp
        curl \
          --progress-bar \
          --output-dir tmp \
          --output $karchive \
          --continue-at - \
          --location \
          --fail \
            $kurl
        mv tmp/$karchive $karchive
      fi
      if [[ $ksrctype == 'tar' ]] ; then
        tar xf$kzopt $karchive
      elif [[ $ksrctype == 'zip' ]] ; then
        unzip $karchive
      fi
      ;;
    git)
      trace "cloning: $kgittag @ $kgiturl"
      # XXX obviously --depth=1 would be preferable, but we need the history
      #     to generate the version, unless we decide to do our own thing
      #       -- robn, 2026-02-13
      git clone \
        --config advice.detachedHead=false \
        --branch $kgittag \
          $kgiturl $kdir
      ;;
    *)
      fail "unknown kernel source type: $ksrctype"
      ;;
    esac
    popd > /dev/null
  fi

  if [[ -f $ksrc/.config ]] ; then
    if ! grep -q -m1 "^# Linux/$_quiz_arch_kernel" $ksrc/.config ; then
      trace "existing source dir was for a different architecture, cleaning it"
      $QUIZ_BUILD_CMD_PREFIX make $makeopts -s -C $ksrc distclean $makevars
    fi
  fi

  _do_mrproper=0
  if [[ $_quiz_ktype = "next" || $_quiz_ktype = "master" ]] ; then
    # get rid of -rcX or similar. this is already too complicated
    sed -i -E 's/^(EXTRAVERSION =) .*/\1/' $ksrc/Makefile

    if [[ ! -f $ksrc/include/config/auto.conf ]] ; then
      # sigh, circular dependency. we need a config set up to be able to
      # query the version. which we need for a config
      trace "prepping source tree for pre-config"
      $QUIZ_BUILD_CMD_PREFIX make $makeopts -s -C $ksrc tinyconfig prepare $makevars
      _do_mrproper=1
    fi

    if [[ $ksrctype = "git" ]] ; then
      # for git builds, setup LOCALVERSION=-<type>-<hash>
      echo -$_quiz_ktype-$(git -C $ksrc rev-parse --short HEAD) > $ksrc/localversion-quiz
    fi
    quiz_kernel_vars $(make -s -C $ksrc kernelrelease)
    trace "updated version from makefile: $_quiz_kver"

    if [[ $_do_mrproper ]] ; then
      # and then cleanup from above, so it'll start fresh with the new config
      $QUIZ_BUILD_CMD_PREFIX make $makeopts -s -C $ksrc mrproper $makevars
    fi
  fi

  kcfg=config-$_quiz_kmajor.$_quiz_kminor
  if [[ -z $opt_no_config ]] ; then
    found_kcfg=""
    if [[ -f kernel/$_quiz_arch_kernel/$kcfg ]] ; then
      found_kcfg=$kcfg
    else
      found_kcfg=$(find kernel/$_quiz_arch_kernel/ -name "config-$_quiz_kmajor.*" | \
        cut -f3 -d/ | sort -V | tail -1)
      if [[ -z $found_kcfg ]] ; then
        found_kcfg=$(find kernel/$_quiz_arch_kernel/ -name "config-*" | \
          cut -f3 -d/ | sort -V | tail -1)
      fi
    fi
    if [[ -z $found_kcfg ]] ; then
      if [[ -z $opt_init_config ]] ; then
        fail "no usable config found!"
      fi
      trace "no usable config found, bootstrapping from tinyconfig"
      $QUIZ_BUILD_CMD_PREFIX make $makeopts -s -C $ksrc tinyconfig $makevars
      # XXX encode other essential config here -- robn, 2025-11-12
    else
      if [[ $kcfg != $found_kcfg ]] ; then
        trace "specific config not found, using fallback: $found_kcfg"
      fi

      trace "installing config: $found_kcfg"
      cp kernel/$_quiz_arch_kernel/$found_kcfg $ksrc/.config
    fi
  else
    trace "NOT installing config as requested"
  fi

  pushd $ksrc > /dev/null

  if [[ -n $opt_modify_config ]] ; then
    trace "updating config from commandline switches"
    scripts/config $opt_modify_config
  fi

  trace "compiling config: $_quiz_ksrcver"
  $QUIZ_BUILD_CMD_PREFIX make $makeopts -s olddefconfig prepare $makevars

  if [[ -z $opt_no_build ]] ; then

    # XXX image_name actually returns the name of the image produced by the
    #     last completed build. this has bitten me when bootstrapping a new
    #     architecture, when I realise that I'm taking a compressed image
    #     but the bootloader wants a raw image. the next run after I realise
    #     fixes it, because the _previous_ run did an uncompressed image.
    #
    #     it's not clear to me what, if any, solution is needed here. doing
    #     a mrproper round like above definitely fixes it, but we lose all
    #     the output from the previous run. maybe I should burn all images
    #     from the previous run too? or maybe doesn't matter, and this is just
    #     a "note to self" that I'll find when I'm ripping my hair out next
    #     time.
    #
    #       -- robn, 2025-11-12 

    kimagepath=$(make -s $makeopts image_name $makevars)
    kimage=$(basename $kimagepath)

    trace "building kernel: $_quiz_ksrcver ($kimage)"
    $QUIZ_BUILD_CMD_PREFIX make $makeopts $kimage modules $makevars

    trace "installing completed kernel: $_quiz_kver"

    mkdir -p $kdest

    # merged /usr setup
    mkdir -p $kdest/usr/lib
    ln -sf usr/lib $kdest/lib

    # install everything we need. the kernel's own install stuff doesn't quite
    # do everything the way we want, so we do a few things ourselves

    # XXX delete the old stuff first? -- robn, 2025-01-13

    # kernel and system map
    cp -vf $ksrc/$kimagepath $kdest/kernel-$_quiz_kver
    cp -vf $ksrc/System.map $kdest/System.map-$_quiz_kver

    # modules tree
    $QUIZ_BUILD_CMD_PREFIX make $makeopts -C $ksrc modules_install INSTALL_MOD_PATH=$kdest $makevars
    /sbin/depmod -b $kdest $_quiz_kver

    # extern module build, so we're not dependent on kernel source to rebuild
    # this is roughly what you find symlinked from /lib/modules/XXX/build
    # on stock Linux systems. other architectures may require more/different
    # things, refer to builddeb/buildrpm/install-extmod-build in the kernel
    # source tree for details
    mkdir -p $kdest/kbuild/$_quiz_kver
    (
      cd $ksrc
      archdir=arch/$_quiz_arch_kernel
      find Makefile vmlinux include -type f -o -type l
      find scripts tools -executable -type f -o -type l
      find $archdir -maxdepth 1 -name 'Makefile*'
      find $archdir -name include -o -name scripts -type d
      find $archdir/include Module.symvers include scripts -type f
    ) | tar -cf - -C $ksrc -T - | tar -xf - -C $kdest/kbuild/$_quiz_kver

  else
    trace "NOT building kernel as requested"
  fi

  popd > /dev/null

  if [[ -n $opt_update_config ]] ; then
    trace "updating standard config: $kcfg"
    cp $ksrc/.config kernel/$_quiz_arch_kernel/$kcfg
  fi

  trace "build finished: $_quiz_kver"

  if [[ -n $opt_upgrade_cleanup ]] ; then
    trace "removing old kernels in minor series: $_quiz_kmajor.$_quiz_kminor"
    find $kdest \
      -name "*$_quiz_kmajor.$_quiz_kminor.*" -a \
      ! -name "*$_quiz_kver" \
    | xargs -r rm -r
  fi
}

kver_failed=()

for kver in $kernel_versions ; do
  # capture errors in the build_kernel function by calling it in a subshell
  # we have to disable our error trap first, then reenable them inside the
  # subshell so that the function still early aborts, but the subshell doesn't
  # trigger our error trap.
  trap - ERR
  ( trap fail ERR ; build_kernel $kver )
  rc=$?
  trap fail ERR

  if [[ $rc -ne 0 ]] ; then
    kver_failed+=($kver)
  fi
done

if [[ ${#kver_failed[*]} -ne 0 ]] ; then
  trace "kernels failed: ${kver_failed[*]}"
fi

# vim: ft=bash