From 1b4d69d58931b52829eaa14a5beb7c9e043d72e8 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 24 Sep 2026 11:50:32 +0000 Subject: [PATCH] feat(buildah): multi-arch --- .tekton/qemu-binfmt-image.yaml | 115 +++++++ .vscode/settings.json | 1 + containers/qemu-binfmt/containerfile | 16 + containers/qemu-binfmt/manage-binfmt.sh | 42 +++ .../argocd/applications/kustomization.yaml | 1 + .../argocd/applications/qemu-binfmt.yaml | 28 ++ kubernetes/qemu-binfmt/base/daemonset.yaml | 71 ++++ .../qemu-binfmt/base/kustomization.yaml | 7 + kubernetes/qemu-binfmt/base/namespace.yaml | 13 + .../qemu-binfmt/base/service-account.yaml | 8 + .../components/openshift/kustomization.yaml | 5 + .../components/openshift/rbac.yaml | 31 ++ .../overlays/okd/kustomization.yaml | 7 + .../tasks/buildah/{0.7.1 => 0.8.0}/README.md | 60 +++- tekton/tasks/buildah/0.8.0/buildah.yaml | 316 ++++++++++++++++++ 15 files changed, 715 insertions(+), 6 deletions(-) create mode 100644 .tekton/qemu-binfmt-image.yaml create mode 100644 containers/qemu-binfmt/containerfile create mode 100644 containers/qemu-binfmt/manage-binfmt.sh create mode 100644 kubernetes/argocd/applications/qemu-binfmt.yaml create mode 100644 kubernetes/qemu-binfmt/base/daemonset.yaml create mode 100644 kubernetes/qemu-binfmt/base/kustomization.yaml create mode 100644 kubernetes/qemu-binfmt/base/namespace.yaml create mode 100644 kubernetes/qemu-binfmt/base/service-account.yaml create mode 100644 kubernetes/qemu-binfmt/components/openshift/kustomization.yaml create mode 100644 kubernetes/qemu-binfmt/components/openshift/rbac.yaml create mode 100644 kubernetes/qemu-binfmt/overlays/okd/kustomization.yaml rename tekton/tasks/buildah/{0.7.1 => 0.8.0}/README.md (58%) create mode 100644 tekton/tasks/buildah/0.8.0/buildah.yaml diff --git a/.tekton/qemu-binfmt-image.yaml b/.tekton/qemu-binfmt-image.yaml new file mode 100644 index 000000000..3655388c5 --- /dev/null +++ b/.tekton/qemu-binfmt-image.yaml @@ -0,0 +1,115 @@ +--- +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + name: qemu-binfmt-image + annotations: + pipelinesascode.tekton.dev/max-keep-runs: "1" + pipelinesascode.tekton.dev/on-cel-expression: | + event == "pull_request" && target_branch == "main" && "containers/qemu-binfmt/containerfile".pathChanged() + pipelinesascode.tekton.dev/target-namespace: "homelab" + pipelinesascode.tekton.dev/task: "https://raw.githubusercontent.com/ArthurVardevanyan/HomeLab/main/tekton/tasks/git-clone/0.9.1/git-clone.yaml" + pipelinesascode.tekton.dev/task-1: "tekton/tasks/buildah/0.7.1/buildah.yaml" + pipelinesascode.tekton.dev/task-2: "tekton/base/clair-action/clair-action-task.yaml" +spec: + params: + - name: git-url + value: "{{ repo_url }}" + - name: git-commit + value: "{{ revision }}" + - name: DOCKERFILE + value: "./containers/qemu-binfmt/containerfile" + - name: IMAGE + value: "registry.arthurvardevanyan.com/homelab/qemu-binfmt:not_latest" + - name: GH_TOKEN + value: "" + + pipelineSpec: + params: + - name: git-url + description: Repository URL to clone from. + type: string + - name: git-commit + type: string + - name: IMAGE + description: Reference of the image buildah will produce. + - name: DOCKERFILE + description: Path to the Dockerfile to build. + type: string + default: ./Dockerfile + - name: GH_TOKEN + type: string + description: GitHub token for authenticated API calls during image build. + + results: + - description: The common vulnerabilities and exposures (CVE) result + name: SCAN_OUTPUT + value: $(tasks.clair-action.results.SCAN_OUTPUT) + type: string + + workspaces: + - name: data + - name: git_auth_secret + + tasks: + - name: git-clone + taskRef: + name: git-clone + kind: Task + params: + - name: url + value: $(params.git-url) + - name: revision + value: $(params.git-commit) + workspaces: + - name: output + workspace: data + - name: basic-auth + workspace: git_auth_secret + + - name: buildah + runAfter: + - git-clone + taskRef: + name: buildah + kind: Task + params: + - name: IMAGE + value: $(params.IMAGE) + - name: DOCKERFILE + value: $(params.DOCKERFILE) + - name: BUILD_EXTRA_ARGS + value: "--build-arg GH_TOKEN=$(params.GH_TOKEN)" + workspaces: + - name: source + workspace: data + + - name: clair-action + runAfter: + - buildah + taskRef: + name: clair-action + kind: Task + params: + - name: IMAGE + value: $(params.IMAGE) + + taskRunTemplate: + serviceAccountName: pipeline + workspaces: + - name: data + volumeClaimTemplate: + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "100Mi" + storageClassName: rook-ceph-block-ci + - name: git_auth_secret + secret: + secretName: "{{ git_auth_secret }}" diff --git a/.vscode/settings.json b/.vscode/settings.json index 2b351663a..bd9ed4847 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -86,6 +86,7 @@ "baremetal", "Battlemage", "benjojo", + "binfmt", "bitfield", "bitnami", "bitwarden", diff --git a/containers/qemu-binfmt/containerfile b/containers/qemu-binfmt/containerfile new file mode 100644 index 000000000..9f9434bfe --- /dev/null +++ b/containers/qemu-binfmt/containerfile @@ -0,0 +1,16 @@ +--- +# syntax=docker/dockerfile-1 +FROM quay.io/centos/centos:stream10-minimal + +# renovate: datasource=repology depName=centos_stream_10/qemu-user-static +ENV QEMU_USER_STATIC_VERSION=10.2.0 +RUN microdnf -y update && \ + microdnf -y install --nodocs --setopt=install_weak_deps=0 \ + qemu-user-static-${QEMU_USER_STATIC_VERSION} bash util-linux && \ + microdnf clean all && rm -rf /var/cache/* && \ + rm -rf /usr/share/doc/* /usr/share/man/* /usr/share/info/* + +COPY manage-binfmt.sh /usr/local/bin/manage-binfmt.sh +RUN chmod 755 /usr/local/bin/manage-binfmt.sh + +CMD ["sleep", "infinity"] diff --git a/containers/qemu-binfmt/manage-binfmt.sh b/containers/qemu-binfmt/manage-binfmt.sh new file mode 100644 index 000000000..09e0d6a73 --- /dev/null +++ b/containers/qemu-binfmt/manage-binfmt.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail + +BINFMT=/proc/sys/fs/binfmt_misc +DEST=/opt/qemu-user-static +INTERP=$DEST/qemu-aarch64-static + +# Fallback: if the host's binfmt is not mounted (or not writable), mount it. +# In the per-userns design (6.12+), a mount inside the pod's init userns +# lands in the shared init binfmt instance, making entries visible to all pods. +if [ ! -w "$BINFMT/register" ]; then + echo "binfmt_misc not writable, mounting..." >&2 + mount -t binfmt_misc binfmt_misc "$BINFMT" +fi + +mkdir -p "$DEST" + +# Install the qemu binary onto the node filesystem (survives pod restarts). +# This lets the F-flag pinned entry survive a temporary DaemonSet gap. +install -m 0755 /usr/bin/qemu-aarch64-static "$DEST/" + +register() { + # Remove stale entry from a previous incarnation (its pinned file is dead). + [ -e "$BINFMT/qemu-aarch64" ] && rm -f "$BINFMT/qemu-aarch64" + # shellcheck disable=SC2028 + echo ':qemu-aarch64:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7\x00:\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xfe:'"$INTERP"':F' \ + > "$BINFMT/register" +} + +register + +echo "binfmt aarch64 handler registered (flags: F, interp: $INTERP)" >&2 + +# Self-heal: re-register every 60s if the entry disappeared or was disabled. +while :; do + sleep 60 + if [ ! -e "$BINFMT/qemu-aarch64" ] || \ + [ "$(cat "$BINFMT/qemu-aarch64")" != "enabled" ]; then + echo "Handler missing/disabled, re-registering..." >&2 + register + fi +done diff --git a/kubernetes/argocd/applications/kustomization.yaml b/kubernetes/argocd/applications/kustomization.yaml index 6a0c1a487..5bc9ba711 100644 --- a/kubernetes/argocd/applications/kustomization.yaml +++ b/kubernetes/argocd/applications/kustomization.yaml @@ -76,6 +76,7 @@ resources: - postgres.yaml - prometheus.yaml - pr-agent.yaml + - qemu-binfmt.yaml - quay.yaml - registry.yaml - renovate.yaml diff --git a/kubernetes/argocd/applications/qemu-binfmt.yaml b/kubernetes/argocd/applications/qemu-binfmt.yaml new file mode 100644 index 000000000..e4b9ef3d1 --- /dev/null +++ b/kubernetes/argocd/applications/qemu-binfmt.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: qemu-binfmt + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "1" + argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true + notifications.argoproj.io/subscribe.on-sync-succeeded.gh-cluster: "" + notifications.argoproj.io/subscribe.on-sync-failed.gh-cluster: "" + notifications.argoproj.io/subscribe.on-sync-status-unknown.gh-cluster: "" + notifications.argoproj.io/subscribe.on-health-degraded.gh-cluster: "" + labels: + app.kubernetes.io/instance: argocd +spec: + destination: + namespace: qemu-binfmt + server: https://kubernetes.default.svc + project: default + source: + path: kubernetes/qemu-binfmt/overlays/okd + repoURL: https://git.arthurvardevanyan.com/ArthurVardevanyan/HomeLab + targetRevision: HEAD + syncPolicy: + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/kubernetes/qemu-binfmt/base/daemonset.yaml b/kubernetes/qemu-binfmt/base/daemonset.yaml new file mode 100644 index 000000000..215ddea9c --- /dev/null +++ b/kubernetes/qemu-binfmt/base/daemonset.yaml @@ -0,0 +1,71 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: qemu-binfmt + namespace: qemu-bgit add infmt + labels: + app: qemu-binfmt + annotations: + argocd.argoproj.io/sync-wave: "0" + gitops-ci.k8s.io/exempt-image-checksum: "registry.arthurvardevanyan.com/homelab/qemu-binfmt:not_latest" +spec: + selector: + matchLabels: + app: qemu-binfmt + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: qemu-binfmt + spec: + serviceAccountName: qemu-binfmt-sa + nodeSelector: + kubernetes.io/arch: amd64 + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + restartPolicy: Always + securityContext: + runAsNonRoot: false + containers: + - name: qemu-binfmt + image: registry.arthurvardevanyan.com/homelab/qemu-binfmt:not_latest + imagePullPolicy: Always + command: ["/usr/local/bin/manage-binfmt.sh"] + securityContext: + privileged: true + readOnlyRootFilesystem: false + volumeMounts: + - name: binfmt-misc + mountPath: /proc/sys/fs/binfmt_misc + - name: qemu-host + mountPath: /opt/qemu-user-static + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + livenessProbe: + exec: + command: + - /bin/sh + - -c + - "[ -e /proc/sys/fs/binfmt_misc/qemu-aarch64 ]" + initialDelaySeconds: 30 + periodSeconds: 60 + failureThreshold: 3 + volumes: + - name: binfmt-misc + hostPath: + path: /proc/sys/fs/binfmt_misc + type: Directory + - name: qemu-host + hostPath: + path: /opt/qemu-user-static + type: DirectoryOrCreate diff --git a/kubernetes/qemu-binfmt/base/kustomization.yaml b/kubernetes/qemu-binfmt/base/kustomization.yaml new file mode 100644 index 000000000..e3250ef52 --- /dev/null +++ b/kubernetes/qemu-binfmt/base/kustomization.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - service-account.yaml + - daemonset.yaml diff --git a/kubernetes/qemu-binfmt/base/namespace.yaml b/kubernetes/qemu-binfmt/base/namespace.yaml new file mode 100644 index 000000000..79c11d56c --- /dev/null +++ b/kubernetes/qemu-binfmt/base/namespace.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: qemu-binfmt + labels: + app.kubernetes.io/name: qemu-binfmt + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/enforce-version: latest + pod-security.kubernetes.io/warn: privileged + pod-security.kubernetes.io/warn-version: latest + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/audit-version: latest diff --git a/kubernetes/qemu-binfmt/base/service-account.yaml b/kubernetes/qemu-binfmt/base/service-account.yaml new file mode 100644 index 000000000..574f15071 --- /dev/null +++ b/kubernetes/qemu-binfmt/base/service-account.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: qemu-binfmt-sa + namespace: qemu-binfmt + labels: + app.kubernetes.io/instance: qemu-binfmt diff --git a/kubernetes/qemu-binfmt/components/openshift/kustomization.yaml b/kubernetes/qemu-binfmt/components/openshift/kustomization.yaml new file mode 100644 index 000000000..5c848613e --- /dev/null +++ b/kubernetes/qemu-binfmt/components/openshift/kustomization.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Component +resources: + - ./rbac.yaml diff --git a/kubernetes/qemu-binfmt/components/openshift/rbac.yaml b/kubernetes/qemu-binfmt/components/openshift/rbac.yaml new file mode 100644 index 000000000..099f710e0 --- /dev/null +++ b/kubernetes/qemu-binfmt/components/openshift/rbac.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: qemu-binfmt-cr + labels: + app.kubernetes.io/instance: qemu-binfmt +rules: + - apiGroups: + - security.openshift.io + resourceNames: + - privileged + resources: + - securitycontextconstraints + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: qemu-binfmt-crb + labels: + app.kubernetes.io/instance: qemu-binfmt +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: qemu-binfmt-cr +subjects: + - kind: ServiceAccount + name: qemu-binfmt-sa + namespace: qemu-binfmt diff --git a/kubernetes/qemu-binfmt/overlays/okd/kustomization.yaml b/kubernetes/qemu-binfmt/overlays/okd/kustomization.yaml new file mode 100644 index 000000000..2aab800f3 --- /dev/null +++ b/kubernetes/qemu-binfmt/overlays/okd/kustomization.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../../base +components: + - ../../components/openshift diff --git a/tekton/tasks/buildah/0.7.1/README.md b/tekton/tasks/buildah/0.8.0/README.md similarity index 58% rename from tekton/tasks/buildah/0.7.1/README.md rename to tekton/tasks/buildah/0.8.0/README.md index 4d0b786c5..c440cb93b 100644 --- a/tekton/tasks/buildah/0.7.1/README.md +++ b/tekton/tasks/buildah/0.8.0/README.md @@ -14,20 +14,19 @@ to assemble a container image, then pushes that image to a container registry. ## Install the Task ```bash -kubectl apply -f https://tekton-hub-api-openshift-pipelines.apps.okd.homelab.arthurvardevanyan.com/v1/resource/homelab/task/buildah/0.7.1/raw +kubectl apply -f https://tekton-hub-api-openshift-pipelines.apps.okd.homelab.arthurvardevanyan.com/v1/resource/homelab/task/buildah/0.8.0/raw ``` ## Parameters - **IMAGE**: The name (reference) of the image to build. -- **BUILDER_IMAGE:**: The name of the image containing the Buildah tool. See - note below. (_default:_ quay.io/buildah/stable:v1.23.3) +- **BUILDER_IMAGE:**: The name of the image containing the Buildah tool. (_default:_ quay.io/buildah/stable:v1.23.3) - **DOCKERFILE**: The path to the `Dockerfile` to execute (_default:_ `./Dockerfile`) - **CONTEXT**: Path to the directory to use as context (_default:_ `.`) -- **TLSVERIFY**: Verify the TLS on the registry endpoint (for push/pull to a - non-TLS registry) (_default:_ `true`) +- **TLSVERIFY**: Verify the TLS on the registry endpoint for push/pull to a + non-TLS registry (_default:_ `true`) - **FORMAT**: The format of the built container, oci or docker (_default:_ `oci`) - **BUILD_EXTRA_ARGS**: Extra parameters passed for the build command when @@ -35,6 +34,17 @@ kubectl apply -f https://tekton-hub-api-openshift-pipelines.apps.okd.homelab.art - **PUSH_EXTRA_ARGS**: Extra parameters passed for the push command when pushing images. (_default:_ `""`) - **SKIP_PUSH**: Skip pushing the built image (_default:_ `false`) +- **IMAGE_REPOSITORIES**: Comma-separated list of image repositories to push + the manifest to (e.g. `ghcr.io/ArthurVardevanyan/k8s-gitops-ci`). When + set, the task builds multi-arch (per `PLATFORMS`), assembles a manifest + list, and pushes the complete manifest to each repository. When empty, + falls back to single-arch build-and-push (backward-compatible). +- **PLATFORMS**: Comma-separated list of platforms to build for (e.g. + `linux/amd64,linux/arm64`). Ignored when `IMAGE_REPOSITORIES` is empty. +- **IMAGE_TAG_SUFFIX**: Suffix appended to each per-arch intermediate image + name to avoid collisions during multi-arch builds (e.g. `${ARCH}`). + Ignored in single-arch mode. Defaults to empty (buildah auto-generates + unique tags). ## Workspaces @@ -45,7 +55,45 @@ kubectl apply -f https://tekton-hub-api-openshift-pipelines.apps.okd.homelab.art ## Platforms -The Task can be run on `linux/amd64`, `linux/s390x`, `linux/arm64` and `linux/ppc64le` platforms. +The Task can be run on `linux/amd64`, `linux/s390x`, `linux/arm64` and +`linux/ppc64le` platforms. It supports multi-arch builds when +`IMAGE_REPOSITORIES` is set — it builds each platform separately, assembles +a manifest list with `buildah manifest`, and pushes the complete manifest +(all of the per-arch images + the manifest list) to each `IMAGE_REPOSITORY` +in a single `buildah manifest push` operation. + +## Multi-Arch Usage + +When `IMAGE_REPOSITORIES` is set, the task builds a multi-arch manifest +list: + +```yaml +apiVersion: tekton.dev/v1 +kind: TaskRun +metadata: + name: buildah-build-multiarch +spec: + taskRef: + name: buildah + params: + - name: IMAGE + value: ghcr.io/ArthurVardevanyan/k8s-gitops-ci:latest + - name: IMAGE_REPOSITORIES + value: "ghcr.io/ArthurVardevanyan/k8s-gitops-ci" + - name: PLATFORMS + value: "linux/amd64,linux/arm64" + - name: IMAGE_TAG_SUFFIX + value: "-${ARCH}" + workspaces: + - name: source + persistentVolumeClaim: + claimName: my-source +``` + +This builds both `linux/amd64` and `linux/arm64` images, creates a manifest +list tagged `ghcr.io/ArthurVardevanyan/k8s-gitops-ci:latest`, and pushes the +complete multi-arch manifest to the registry. The intermediate per-arch +images are cleaned up after the push. ## Usage diff --git a/tekton/tasks/buildah/0.8.0/buildah.yaml b/tekton/tasks/buildah/0.8.0/buildah.yaml new file mode 100644 index 000000000..e17205dc1 --- /dev/null +++ b/tekton/tasks/buildah/0.8.0/buildah.yaml @@ -0,0 +1,316 @@ +--- +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: buildah + namespace: homelab + labels: + app.kubernetes.io/version: "0.8.0" + annotations: + argocd.argoproj.io/sync-wave: "2" + argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true + tekton.dev/categories: Image Build + tekton.dev/pipelines.minVersion: "0.17.0" + tekton.dev/tags: image-build + tekton.dev/platforms: "linux/amd64,linux/s390x,linux/ppc64le,linux/arm64" + tekton.dev/displayName: buildah +spec: + description: >- + Buildah task builds source into a container image and + then pushes it to a container registry. + + Buildah Task builds source into a container image using Project Atomic's + Buildah build tool. It uses Buildah's support for building from + Dockerfiles, using its buildah bud command. This command executes the + directives in the Dockerfile to assemble a container image, then pushes + that image to a container registry. + + Supports multi-arch image builds via manifest lists when IMAGE_REPOS is + set — builds each platform separately, assembles a manifest list with + `buildah manifest`, and pushes the complete manifest (all arch images + + the manifest list) to each IMAGE_REPOSITORY in one `buildah manifest push`. + + params: + - name: IMAGE + description: Reference of the image buildah will produce. + - name: BUILDER_IMAGE + description: The location of the buildah builder image. + default: quay.io/buildah/stable:v1.43.1-immutable@sha256:fc649e1899c4e09a8e19d6fbdbda178d2008fa4186d5a280211f8ffa59e77e32 + - name: STORAGE_DRIVER + description: "Buildah storage driver (default: overlay with fuse-overlayfs as mount_program for improved performance)" + default: overlay + - name: DOCKERFILE + description: Path to the Dockerfile to build. + default: ./Dockerfile + - name: CONTEXT + description: Path to the directory to use as context. + default: . + - name: TLSVERIFY + description: Verify the TLS on the registry endpoint (for push/pull to a non-TLS registry) + default: "true" + - name: FORMAT + description: The format of the built container, oci or docker + default: "oci" + - name: BUILD_EXTRA_ARGS + description: Extra parameters passed for the build command when building images. + default: "" + - name: PUSH_EXTRA_ARGS + description: Extra parameters passed for the push command when pushing images. + type: string + default: "" + - name: SKIP_PUSH + description: Skip pushing the built image + default: "false" + - name: IMAGE_REPOSITORIES + description: "Comma-separated list of image repositories to push the manifest to. When set, builds multi-arch (PLATFORMS) and pushes a manifest list; when empty, falls back to single-arch build-and-push (backward-compatible)." + default: "" + - name: PLATFORMS + description: "Comma-separated list of platforms to build for (e.g. linux/amd64,linux/arm64). Ignored when IMAGE_REPOSITORIES is empty (single-arch fallback)." + default: "linux/amd64,linux/arm64" + - name: IMAGE_TAG_SUFFIX + description: "Suffix appended to each per-arch intermediate image name to avoid collisions during multi-arch builds (e.g. ${ARCH}). Ignored in single-arch mode." + default: "" + workspaces: + - name: source + - name: sslcertdir + optional: true + - name: dockerconfig + description: >- + An optional workspace that allows providing a .docker/config.json file + for Buildah to access the container registry. + The file should be placed at the root of the Workspace with name config.json. + optional: true + results: + - name: IMAGE_DIGEST + description: Digest of the image just built. + - name: IMAGE_URL + description: Image repository where the built image would be pushed to + steps: + - name: build-and-push + computeResources: + requests: + memory: 500Mi + ephemeral-storage: 5Gi + cpu: 1000m + limits: + cpu: "4000m" + ephemeral-storage: 75Gi + memory: 5Gi + image: $(params.BUILDER_IMAGE) + workingDir: $(workspaces.source.path) + script: | + mkdir -p /home/build/.local/share/containers + mkdir -p /home/build/rundir/libpod + rm -rf $(workspaces.source.path)/lost+found + + [ "$(workspaces.sslcertdir.bound)" = "true" ] && CERT_DIR_FLAG="--cert-dir=$(workspaces.sslcertdir.path)" + [ "$(workspaces.dockerconfig.bound)" = "true" ] && DOCKER_CONFIG="$(workspaces.dockerconfig.path)" && export DOCKER_CONFIG + + MULTI_ARCH=false + if [ -n "$(params.IMAGE_REPOSITORIES)" ]; then + MULTI_ARCH=true + fi + + # ── Fail-fast: require node-wide binfmt handler ────────────────── + # The qemu-binfmt DaemonSet (namespace: qemu-binfmt) registers + # aarch64 binfmt handlers with the F flag on every node. Without + # it cross-arch RUN steps silently fail with "exec format error". + if [ "${MULTI_ARCH}" = "true" ]; then + IFS=',' read -ra PLATFORMS <<< "$(params.PLATFORMS)" + for platform in "${PLATFORMS[@]}"; do + arch="${platform##*/}" + case "$arch" in + amd64|x86_64) ;; # native on this (amd64) cluster + *) + handler="qemu-$arch" + if [ ! -f /proc/sys/fs/binfmt_misc/$handler ] || \ + [ "$(cat /proc/sys/fs/binfmt_misc/$handler 2>/dev/null | head -1)" != "enabled" ]; then + echo "ERROR: no enabled binfmt handler for ${arch} on this node." >&2 + echo "The qemu-binfmt DaemonSet (namespace qemu-binfmt) must be running." >&2 + exit 1 + fi + ;; + esac + done + fi + + # Resolve the IMAGE param into a shell variable so string operations + # like ${IMAGE%:*} work correctly. + IMAGE="$(params.IMAGE)" + + if [ "${MULTI_ARCH}" = "true" ]; then + # ── Multi-arch mode ────────────────────────────────────────────── + # 1. Parse platforms and images into parallel arrays + IFS=',' read -ra PLATFORMS <<< "$(params.PLATFORMS)" + IMAGES=() + for i in "${!PLATFORMS[@]}"; do + PLATFORM="${PLATFORMS[$i]}" + ARCH="${PLATFORM##*/}" + # Extract image name (without tag), use :ARCH as the tag + # No placeholder trickery — ARCH is already available in this loop + IMAGE_NAME="${IMAGE%:*}" + # IMAGE has a tag (e.g. "registry.example.com/repo/image:0.1.0") + # -> IMAGE_NAME = "registry.example.com/repo/image" + # If IMAGE has no tag, IMAGE_NAME == IMAGE which is fine + IMAGES+=("${IMAGE_NAME}:${ARCH}") + done + + # 2. Build each architecture + for i in "${!PLATFORMS[@]}"; do + PLATFORM="${PLATFORMS[$i]}" + IMAGE="${IMAGES[$i]}" + ARCH="${PLATFORM##*/}" + echo "Building ${IMAGE} for ${PLATFORM}..." + buildah ${CERT_DIR_FLAG} "--storage-driver=$(params.STORAGE_DRIVER)" bud \ + --arch="${ARCH}" --override-arch="${ARCH}" $(params.BUILD_EXTRA_ARGS) \ + "--format=$(params.FORMAT)" "--tls-verify=$(params.TLSVERIFY)" \ + -f "$(params.DOCKERFILE)" -t "${IMAGE}" "$(params.CONTEXT)" + done + + # 3. Create and populate manifest list + MANIFEST_IMAGE="${IMAGES[0]%%:*}" + MANIFEST_TAG="${IMAGES[0]##*:}" + echo "Creating manifest ${MANIFEST_IMAGE}:${MANIFEST_TAG}..." + buildah manifest create "${MANIFEST_IMAGE}:${MANIFEST_TAG}" + for i in "${!PLATFORMS[@]}"; do + PLATFORM="${PLATFORMS[$i]}" + IMAGE="${IMAGES[$i]}" + ARCH="${PLATFORM##*/}" + echo "Adding ${IMAGE} (${ARCH}) to manifest..." + buildah manifest add "${MANIFEST_IMAGE}:${MANIFEST_TAG}" "docker://${IMAGE}" + done + + # 4. Push manifest to all repositories + RETRIES=5 + IFS=',' read -ra REPOS <<< "$(params.IMAGE_REPOSITORIES)" + for repo in "${REPOS[@]}"; do + echo "Pushing manifest to ${repo}..." + for i in $(seq 1 $RETRIES); do + if buildah ${CERT_DIR_FLAG} "--storage-driver=$(params.STORAGE_DRIVER)" push \ + $(params.PUSH_EXTRA_ARGS) "--tls-verify=$(params.TLSVERIFY)" \ + --digestfile /tmp/image-digest \ + "manifests:${MANIFEST_IMAGE}:${MANIFEST_TAG}" "docker://${repo}"; then + break + fi + [ "$i" -eq "$RETRIES" ] && echo "Push failed after $RETRIES attempts" >&2 && exit 1 + echo "Push attempt $i/$RETRIES failed, retrying..." >&2 + sleep 5 + done + tee "$(results.IMAGE_DIGEST.path)" < /tmp/image-digest + printf '%s' "${repo}:${MANIFEST_TAG}" | tee "$(results.IMAGE_URL.path)" + done + + # 5. Cleanup intermediate images + for i in "${!PLATFORMS[@]}"; do + IMAGE="${IMAGES[$i]}" + buildah rm "${IMAGE}" 2>/dev/null || true + done + + else + # ── Single-arch mode (backward-compatible) ─────────────────────── + # shellcheck disable=SC2046,SC2086 + buildah ${CERT_DIR_FLAG} "--storage-driver=$(params.STORAGE_DRIVER)" bud $(params.BUILD_EXTRA_ARGS) \ + "--format=$(params.FORMAT)" "--tls-verify=$(params.TLSVERIFY)" \ + -f "$(params.DOCKERFILE)" -t "$(params.IMAGE)" "$(params.CONTEXT)" + [ "$(params.SKIP_PUSH)" = "true" ] && echo "Push skipped" && exit 0 + # push the image with retries (registry 500s are transient) + RETRIES=5 + for i in $(seq 1 $RETRIES); do + if buildah ${CERT_DIR_FLAG} "--storage-driver=$(params.STORAGE_DRIVER)" push $(params.PUSH_EXTRA_ARGS) \ + "--tls-verify=$(params.TLSVERIFY)" --digestfile /tmp/image-digest "$(params.IMAGE)" \ + "docker://$(params.IMAGE)"; then + break + fi + [ "$i" -eq "$RETRIES" ] && echo "Push failed after $RETRIES attempts" >&2 && exit 1 + echo "Push attempt $i/$RETRIES failed, retrying..." >&2 + sleep 5 + done + tee "$(results.IMAGE_DIGEST.path)" < /tmp/image-digest + printf '%s' "$(params.IMAGE)" | tee "$(results.IMAGE_URL.path)" + fi + volumeMounts: + - name: varlibcontainers + mountPath: /var/lib/containers + - name: tmp + mountPath: /tmp + - name: var-tmp + mountPath: /var/tmp + - name: home + mountPath: /home + - name: root + mountPath: /root + - name: run + mountPath: /var/run + - name: cache + mountPath: /var/cache + - name: networks # TODO Circle Back to This. + mountPath: /etc/containers/networks + - name: binfmt-misc + mountPath: /proc/sys/fs/binfmt_misc + securityContext: + # runAsNonRoot: true + runAsUser: 0 #1000 # 65532 + privileged: false + readOnlyRootFilesystem: false + allowPrivilegeEscalation: true + # seccompProfile: + # type: RuntimeDefault + capabilities: + add: + - SETFCAP + - SYS_ADMIN + drop: + - MKNOD + - KILL + # - SETUID + # - SETGID + # - ALL + volumes: + - name: varlibcontainers + emptyDir: + sizeLimit: 75Gi + # ephemeral: + # volumeClaimTemplate: + # metadata: + # creationTimestamp: null + # spec: + # accessModes: ["ReadWriteOnce"] + # storageClassName: "rook-ceph-block-ci" + # resources: + # requests: + # storage: 30Gi + - name: home + emptyDir: + sizeLimit: 1Mi + - name: tmp + emptyDir: + sizeLimit: 2Mi + - name: var-tmp + emptyDir: + sizeLimit: 30Gi + # ephemeral: + # volumeClaimTemplate: + # metadata: + # creationTimestamp: null + # spec: + # accessModes: ["ReadWriteOnce"] + # storageClassName: "rook-ceph-block-ci" + # resources: + # requests: + # storage: 30Gi + - name: root + emptyDir: + sizeLimit: 1Mi + - name: run + emptyDir: + sizeLimit: 1Mi + - name: cache + emptyDir: + sizeLimit: 1Mi + - name: networks # TODO Circle Back to This. + emptyDir: + sizeLimit: 1Mi + - name: binfmt-misc + hostPath: + path: /proc/sys/fs/binfmt_misc + type: Directory