From b4774e509a349104a13b1b82e6ca7f9cd5ddc2bf Mon Sep 17 00:00:00 2001 From: Elron Bandel Date: Mon, 29 Jun 2026 12:11:22 +0300 Subject: [PATCH] =?UTF-8?q?feat(release):=20incremental=20combos=20?= =?UTF-8?q?=E2=80=94=20skip=20unchanged=20via=20a=20content-hash=20tag?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every release rebuilt all ~12,800 combos+standalones (the bulk of the ~6h wall clock) even when nothing changed. Add a content-hash skip on the existing buildx bake — no account, no SaaS, no Dockerfile rewrite (git + buildx + bash): - containers/scripts/combo-src-hash.sh: a combo's hash = its build source + the manifest digest of every base it's FROM (benchmark, agent, gosu, otel, process-compose, model). Identical hash -> skip; a changed base -> new digest -> new hash -> rebuild. That last property is the cascade, for free. - combos job: compute the hash, skip (under skip_published) if the evals/--:src- tag already exists, and stamp that tag on a successful build so the next run skips this exact source. - tests/build/hash-cascade.sweep.sh + a Rust wrapper (no docker, every cargo test) prove the hash is deterministic, cascades over all 6 parents, and is source-sensitive. imagetools can't read config labels (verified both on the manifest list and the per-arch sub-manifest), so the hash lives in a TAG, not a label. Signed-off-by: Elron Bandel --- .github/workflows/release-images.yml | 15 +++++++++-- containers/scripts/combo-src-hash.sh | 36 +++++++++++++++++++++++++ tests/build/hash-cascade.sweep.sh | 39 ++++++++++++++++++++++++++++ tests/build/test.rs | 24 +++++++++++++++++ 4 files changed, 112 insertions(+), 2 deletions(-) create mode 100755 containers/scripts/combo-src-hash.sh create mode 100755 tests/build/hash-cascade.sweep.sh diff --git a/.github/workflows/release-images.yml b/.github/workflows/release-images.yml index 5b262170..fba58260 100644 --- a/.github/workflows/release-images.yml +++ b/.github/workflows/release-images.yml @@ -588,6 +588,13 @@ jobs: done } fails=0 + # Incremental skip: a combo's content hash folds its build source + the + # digest of every base it's FROM. Identical hash → its :src- tag + # already exists → skip; a changed base → new digest → new hash → rebuild + # (the cascade). gosu/otel/process-compose/model are shared — resolve once. + dg() { docker buildx imagetools inspect "$1" --format '{{.Manifest.Digest}}' 2>/dev/null || echo absent; } + GOSU_D=$(dg "${REGISTRY}/core/gosu:${TAG}"); OTEL_D=$(dg "${REGISTRY}/core/otel:${TAG}") + PC_D=$(dg "${REGISTRY}/core/process-compose:${TAG}"); MODEL_D=$(dg "${REGISTRY}/models/bifrost:${TAG}") while read -r it; do IFS=$'\t' read -r B A TASK < <(jq -r '[.b,.a,.task]|@tsv' <<< "$it") if [ -n "$TASK" ]; then # per-task combo: evals/--- @@ -598,7 +605,9 @@ jobs: eb="$B"; bench_img="${REGISTRY}/benchmarks/${B}:${TAG}" PLAT="linux/amd64,linux/arm64" # light overlay → multi-arch is fine fi - if [ "${{ inputs.skip_published }}" = "true" ] && docker buildx imagetools inspect "${REGISTRY}/evals/${eb}--${A}:${TAG}" >/dev/null 2>&1; then echo "skip-published: evals/${eb}--${A}"; continue; fi + HASH=$(containers/scripts/combo-src-hash.sh "$(dg "$bench_img")" "$(dg "${REGISTRY}/agents/${A}:${TAG}")" "$GOSU_D" "$OTEL_D" "$PC_D" "$MODEL_D" "standalone=${STANDALONE}") + ht="${REGISTRY}/evals/${eb}--${A}:src-${HASH}" + if [ "${{ inputs.skip_published }}" = "true" ] && docker buildx imagetools inspect "$ht" >/dev/null 2>&1; then echo "skip-unchanged: evals/${eb}--${A} (src-${HASH})"; continue; fi # eval = lean base (sidecar mode); eval-standalone = single-container # bundle (gateway+otelcol+process-compose in-image). bake builds eval # once, then layers standalone on it via the eval-base context. @@ -610,7 +619,9 @@ jobs: BENCHMARK_IMAGE="$bench_img" AGENT_IMAGE="${REGISTRY}/agents/${A}:${TAG}" \ retry docker buildx bake -f containers/docker-bake.hcl -f containers/core/combination.docker-bake.hcl \ --set "*.platform=${PLAT}" \ - "${ACT[@]}"; then :; \ + "${ACT[@]}"; then + # stamp the content-hash tag so a later skip_published run skips this exact source. + [ "$DRY" = "true" ] || docker buildx imagetools create -t "$ht" "${REGISTRY}/evals/${eb}--${A}:${TAG}" 2>/dev/null || true else echo "::error::combo failed: evals/${eb}--${A}"; fails=$((fails+1)); fi echo "::endgroup::" done < <(jq -c '.[]' <<< "$ITEMS") diff --git a/containers/scripts/combo-src-hash.sh b/containers/scripts/combo-src-hash.sh new file mode 100755 index 00000000..22a989e0 --- /dev/null +++ b/containers/scripts/combo-src-hash.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Content hash for an eval combo (evals/-- + its -standalone). +# +# Folds the combo's OWN build source together with the image digest of every +# base it is FROM. Same source + same parent digests -> identical hash, so the +# combos job can skip it (its :src- tag already exists). Change any base +# -> new parent digest -> new hash -> rebuild. That last property is the +# cascade: a base rebuild ripples into every combo on top of it, for free. +# +# Usage: +# combo-src-hash.sh +# where each <*_d> is a parent image's manifest digest, e.g. +# docker buildx imagetools inspect --format '{{.Manifest.Digest}}' +# +# Env: +# COMBO_SRC_ROOT dir holding the combo build source (default: containers/core). +set -euo pipefail +cd "$(dirname "$0")/../.." # repo root (this script lives in containers/scripts/) +root="${COMBO_SRC_ROOT:-containers/core}" + +# sha256 of stdin — portable across Linux (sha256sum, the CI runners) and macOS +# (shasum). Used only in pipes, never as an xargs target (xargs can't call a +# shell function), so the source hash below cats the files into it. +sha() { if command -v sha256sum >/dev/null 2>&1; then sha256sum; else shasum -a 256; fi; } + +# The combo's build inputs, shared across every combo: the two Dockerfiles, the +# framework scripts they COPY, and the bake graph that wires them. Hash the +# concatenated content (sorted by path) so any edit changes the result. +paths=("$root/combination.Dockerfile" "$root/standalone.Dockerfile" \ + "$root/runner" "$root/entrypoint" "$root/combination.docker-bake.hcl") +n=$(find "${paths[@]}" -type f 2>/dev/null | wc -l | tr -d ' ') +[ "$n" -gt 0 ] || { echo "combo-src-hash: no source files under $root" >&2; exit 1; } +src=$(find "${paths[@]}" -type f 2>/dev/null | sort | xargs cat | sha | cut -c1-12) + +# Fold the source hash with every parent digest -> the combo's content hash. +printf '%s|%s' "$src" "$*" | sha | cut -c1-16 diff --git a/tests/build/hash-cascade.sweep.sh b/tests/build/hash-cascade.sweep.sh new file mode 100755 index 00000000..24dcedcd --- /dev/null +++ b/tests/build/hash-cascade.sweep.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Contract test for containers/scripts/combo-src-hash.sh — the hash that drives +# the combos job's :src- skip. Asserts it is: +# - deterministic (same inputs -> same hash, so unchanged combos skip) +# - cascading (any parent digest changes -> hash changes, so a base +# rebuild rebuilds everything on top of it) +# - source-sensitive (a combo-source edit -> hash changes, so a Dockerfile +# change rebuilds) +# Pure hashing — no docker, no network — so it runs on every `cargo test`. +set -euo pipefail +cd "$(dirname "$0")/../.." # repo root +H=containers/scripts/combo-src-hash.sh +fail=0 +neq() { if [ "$2" = "$3" ]; then echo "FAIL: $1 — both '$2'"; fail=1; fi; } +eq() { if [ "$2" != "$3" ]; then echo "FAIL: $1 — '$2' != '$3'"; fail=1; fi; } + +b=$("$H" benchD agentD gosuD otelD pcD modelD) +eq "deterministic" "$b" "$("$H" benchD agentD gosuD otelD pcD modelD)" +# Each of the 6 parents must, when its digest changes, flip the combo hash. +neq "bench cascade" "$b" "$("$H" BENCH2 agentD gosuD otelD pcD modelD)" +neq "agent cascade" "$b" "$("$H" benchD AGENT2 gosuD otelD pcD modelD)" +neq "gosu cascade" "$b" "$("$H" benchD agentD GOSU2 otelD pcD modelD)" +neq "otel cascade" "$b" "$("$H" benchD agentD gosuD OTEL2 pcD modelD)" +neq "pc cascade" "$b" "$("$H" benchD agentD gosuD otelD PC2 modelD)" +neq "model cascade" "$b" "$("$H" benchD agentD gosuD otelD pcD MODEL2)" + +# Source-sensitivity: an edit to a combo-source file flips the hash. Use a temp +# copy so the repo working tree is never touched. +tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT +cp -R containers/core "$tmp/core" +s=$(COMBO_SRC_ROOT="$tmp/core" "$H" benchD agentD gosuD otelD pcD modelD) +printf '\n# hash-cascade test edit\n' >> "$tmp/core/combination.Dockerfile" +neq "source sensitive" "$s" "$(COMBO_SRC_ROOT="$tmp/core" "$H" benchD agentD gosuD otelD pcD modelD)" + +if [ "$fail" = 0 ]; then + echo "PASS: combo hash is deterministic, cascades over all 6 parents, and is source-sensitive" +else + exit 1 +fi diff --git a/tests/build/test.rs b/tests/build/test.rs index cf7780f9..04ad172c 100644 --- a/tests/build/test.rs +++ b/tests/build/test.rs @@ -1038,3 +1038,27 @@ fn eval_local_resolves_from_full_graph() { ); } } + +// ─── combo source-hash contract (incremental :src- skip) ──────── +// +// The combos job tags each built combo `:src-` and, on a re-run with +// skip_published, skips any combo whose tag already exists. The hash +// (containers/scripts/combo-src-hash.sh) must be deterministic, cascade over +// every parent digest, and be source-sensitive — or the skip is wrong. This +// guard is pure hashing (no docker), so unlike the bake/build checks above it +// runs on every `cargo test`, not just the build lane. +#[test] +fn combo_src_hash_cascade() { + let root = test_support::repo_root(); + let out = Command::new("bash") + .arg(root.join("tests/build/hash-cascade.sweep.sh")) + .current_dir(&root) + .output() + .expect("run tests/build/hash-cascade.sweep.sh"); + assert!( + out.status.success(), + "combo source-hash contract failed:\n--- stdout ---\n{}--- stderr ---\n{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr), + ); +}