feat(buildah): add qemu-binfmt daemonset + fix 0.8.0 multi-arch

.tekton/qemu-binfmt-image.yaml

@@ -0,0 +1,115 @@
+---
+apiVersion: tekton.dev/v1
+kind: PipelineRun
+metadata:
+  name: qemu-binfmt-image
+  annotations:
+    pipelinesascode.tekton.dev/max-keep-runs: "1"
+    pipelinesascode.tekton.dev/on-cel-expression: |
+      event == "pull_request" && target_branch == "main" && "containers/qemu-binfmt/containerfile".pathChanged()
+    pipelinesascode.tekton.dev/target-namespace: "homelab"
+    pipelinesascode.tekton.dev/task: "https://raw.githubusercontent.com/ArthurVardevanyan/HomeLab/main/tekton/tasks/git-clone/0.9.1/git-clone.yaml"
+    pipelinesascode.tekton.dev/task-1: "tekton/tasks/buildah/0.7.1/buildah.yaml"
+    pipelinesascode.tekton.dev/task-2: "tekton/base/clair-action/clair-action-task.yaml"
+spec:
+  params:
+    - name: git-url
+      value: "{{ repo_url }}"
+    - name: git-commit
+      value: "{{ revision }}"
+    - name: DOCKERFILE
+      value: "./containers/qemu-binfmt/containerfile"
+    - name: IMAGE
+      value: "registry.arthurvardevanyan.com/homelab/qemu-binfmt:not_latest"
+    - name: GH_TOKEN
+      value: ""
+
+  pipelineSpec:
+    params:
+      - name: git-url
+        description: Repository URL to clone from.
+        type: string
+      - name: git-commit
+        type: string
+      - name: IMAGE
+        description: Reference of the image buildah will produce.
+      - name: DOCKERFILE
+        description: Path to the Dockerfile to build.
+        type: string
+        default: ./Dockerfile
+      - name: GH_TOKEN
+        type: string
+        description: GitHub token for authenticated API calls during image build.
+
+    results:
+      - description: The common vulnerabilities and exposures (CVE) result
+        name: SCAN_OUTPUT
+        value: $(tasks.clair-action.results.SCAN_OUTPUT)
+        type: string
+
+    workspaces:
+      - name: data
+      - name: git_auth_secret
+
+    tasks:
+      - name: git-clone
+        taskRef:
+          name: git-clone
+          kind: Task
+        params:
+          - name: url
+            value: $(params.git-url)
+          - name: revision
+            value: $(params.git-commit)
+        workspaces:
+          - name: output
+            workspace: data
+          - name: basic-auth
+            workspace: git_auth_secret
+
+      - name: buildah
+        runAfter:
+          - git-clone
+        taskRef:
+          name: buildah
+          kind: Task
+        params:
+          - name: IMAGE
+            value: $(params.IMAGE)
+          - name: DOCKERFILE
+            value: $(params.DOCKERFILE)
+          - name: BUILD_EXTRA_ARGS
+            value: "--build-arg GH_TOKEN=$(params.GH_TOKEN)"
+        workspaces:
+          - name: source
+            workspace: data
+
+      - name: clair-action
+        runAfter:
+          - buildah
+        taskRef:
+          name: clair-action
+          kind: Task
+        params:
+          - name: IMAGE
+            value: $(params.IMAGE)
+
+  taskRunTemplate:
+    serviceAccountName: pipeline
+  workspaces:
+    - name: data
+      volumeClaimTemplate:
+        apiVersion: v1
+        kind: PersistentVolumeClaim
+        metadata:
+          name: data
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: "100Mi"
+          storageClassName: rook-ceph-block-ci
+    - name: git_auth_secret
+      secret:
+        secretName: "{{ git_auth_secret }}"

.vscode/settings.json

@@ -86,6 +86,7 @@
     "baremetal",
     "Battlemage",
     "benjojo",
+    "binfmt",
     "bitfield",
     "bitnami",
     "bitwarden",

containers/qemu-binfmt/containerfile

@@ -0,0 +1,16 @@
+---
+# syntax=docker/dockerfile-1
+FROM quay.io/centos/centos:stream10-minimal
+
+# renovate: datasource=repology depName=centos_stream_10/qemu-user-static
+ENV QEMU_USER_STATIC_VERSION=10.2.0
+RUN microdnf -y update && \
+    microdnf -y install --nodocs --setopt=install_weak_deps=0 \
+    qemu-user-static-${QEMU_USER_STATIC_VERSION} bash util-linux && \
+    microdnf clean all && rm -rf /var/cache/* && \
+    rm -rf /usr/share/doc/* /usr/share/man/* /usr/share/info/*
+
+COPY manage-binfmt.sh /usr/local/bin/manage-binfmt.sh
+RUN chmod 755 /usr/local/bin/manage-binfmt.sh
+
+CMD ["sleep", "infinity"]

containers/qemu-binfmt/manage-binfmt.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+BINFMT=/proc/sys/fs/binfmt_misc
+DEST=/opt/qemu-user-static
+INTERP=$DEST/qemu-aarch64-static
+
+# Fallback: if the host's binfmt is not mounted (or not writable), mount it.
+# In the per-userns design (6.12+), a mount inside the pod's init userns
+# lands in the shared init binfmt instance, making entries visible to all pods.
+if [ ! -w "$BINFMT/register" ]; then
+  echo "binfmt_misc not writable, mounting..." >&2
+  mount -t binfmt_misc binfmt_misc "$BINFMT"
+fi
+
+mkdir -p "$DEST"
+
+# Install the qemu binary onto the node filesystem (survives pod restarts).
+# This lets the F-flag pinned entry survive a temporary DaemonSet gap.
+install -m 0755 /usr/bin/qemu-aarch64-static "$DEST/"
+
+register() {
+  # Remove stale entry from a previous incarnation (its pinned file is dead).
+  [ -e "$BINFMT/qemu-aarch64" ] && rm -f "$BINFMT/qemu-aarch64"
+  # shellcheck disable=SC2028
+  echo ':qemu-aarch64:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7\x00:\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xfe:'"$INTERP"':F' \
+    > "$BINFMT/register"
+}
+
+register
+
+echo "binfmt aarch64 handler registered (flags: F, interp: $INTERP)" >&2
+
+# Self-heal: re-register every 60s if the entry disappeared or was disabled.
+while :; do
+  sleep 60
+  if [ ! -e "$BINFMT/qemu-aarch64" ] || \
+     [ "$(cat "$BINFMT/qemu-aarch64")" != "enabled" ]; then
+    echo "Handler missing/disabled, re-registering..." >&2
+    register
+  fi
+done

kubernetes/argocd/applications/kustomization.yaml

@@ -76,6 +76,7 @@ resources:
   - postgres.yaml
   - prometheus.yaml
   - pr-agent.yaml
+  - qemu-binfmt.yaml
   - quay.yaml
   - registry.yaml
   - renovate.yaml

kubernetes/argocd/applications/qemu-binfmt.yaml

@@ -0,0 +1,28 @@
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: qemu-binfmt
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+    notifications.argoproj.io/subscribe.on-sync-succeeded.gh-cluster: ""
+    notifications.argoproj.io/subscribe.on-sync-failed.gh-cluster: ""
+    notifications.argoproj.io/subscribe.on-sync-status-unknown.gh-cluster: ""
+    notifications.argoproj.io/subscribe.on-health-degraded.gh-cluster: ""
+  labels:
+    app.kubernetes.io/instance: argocd
+spec:
+  destination:
+    namespace: qemu-binfmt
+    server: https://kubernetes.default.svc
+  project: default
+  source:
+    path: kubernetes/qemu-binfmt/overlays/okd
+    repoURL: https://git.arthurvardevanyan.com/ArthurVardevanyan/HomeLab
+    targetRevision: HEAD
+  syncPolicy:
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true

kubernetes/qemu-binfmt/base/daemonset.yaml

@@ -0,0 +1,71 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: qemu-binfmt
+  namespace: qemu-bgit add infmt
+  labels:
+    app: qemu-binfmt
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+    gitops-ci.k8s.io/exempt-image-checksum: "registry.arthurvardevanyan.com/homelab/qemu-binfmt:not_latest"
+spec:
+  selector:
+    matchLabels:
+      app: qemu-binfmt
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: qemu-binfmt
+    spec:
+      serviceAccountName: qemu-binfmt-sa
+      nodeSelector:
+        kubernetes.io/arch: amd64
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+      restartPolicy: Always
+      securityContext:
+        runAsNonRoot: false
+      containers:
+        - name: qemu-binfmt
+          image: registry.arthurvardevanyan.com/homelab/qemu-binfmt:not_latest
+          imagePullPolicy: Always
+          command: ["/usr/local/bin/manage-binfmt.sh"]
+          securityContext:
+            privileged: true
+            readOnlyRootFilesystem: false
+          volumeMounts:
+            - name: binfmt-misc
+              mountPath: /proc/sys/fs/binfmt_misc
+            - name: qemu-host
+              mountPath: /opt/qemu-user-static
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              cpu: 500m
+              memory: 256Mi
+          livenessProbe:
+            exec:
+              command:
+                - /bin/sh
+                - -c
+                - "[ -e /proc/sys/fs/binfmt_misc/qemu-aarch64 ]"
+            initialDelaySeconds: 30
+            periodSeconds: 60
+            failureThreshold: 3
+      volumes:
+        - name: binfmt-misc
+          hostPath:
+            path: /proc/sys/fs/binfmt_misc
+            type: Directory
+        - name: qemu-host
+          hostPath:
+            path: /opt/qemu-user-static
+            type: DirectoryOrCreate

kubernetes/qemu-binfmt/base/kustomization.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - service-account.yaml
+  - daemonset.yaml

kubernetes/qemu-binfmt/base/namespace.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: qemu-binfmt
+  labels:
+    app.kubernetes.io/name: qemu-binfmt
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/enforce-version: latest
+    pod-security.kubernetes.io/warn: privileged
+    pod-security.kubernetes.io/warn-version: latest
+    pod-security.kubernetes.io/audit: privileged
+    pod-security.kubernetes.io/audit-version: latest

kubernetes/qemu-binfmt/base/service-account.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: qemu-binfmt-sa
+  namespace: qemu-binfmt
+  labels:
+    app.kubernetes.io/instance: qemu-binfmt

kubernetes/qemu-binfmt/components/openshift/kustomization.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Component
+resources:
+  - ./rbac.yaml

kubernetes/qemu-binfmt/components/openshift/rbac.yaml

@@ -0,0 +1,31 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: qemu-binfmt-cr
+  labels:
+    app.kubernetes.io/instance: qemu-binfmt
+rules:
+  - apiGroups:
+      - security.openshift.io
+    resourceNames:
+      - privileged
+    resources:
+      - securitycontextconstraints
+    verbs:
+      - use
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: qemu-binfmt-crb
+  labels:
+    app.kubernetes.io/instance: qemu-binfmt
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: qemu-binfmt-cr
+subjects:
+  - kind: ServiceAccount
+    name: qemu-binfmt-sa
+    namespace: qemu-binfmt

kubernetes/qemu-binfmt/overlays/okd/kustomization.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../../base
+components:
+  - ../../components/openshift