perf(ci): reduce repeated image builds and test time (phase 2)

.github/workflows/compose-ci.yml

@@ -270,8 +270,63 @@ jobs:
           pnpm --dir frontend exec playwright install --with-deps chromium
       - name: Validate Compose configuration
         run: docker compose config --quiet
+      - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
+      - name: Verify build checkout
+        env:
+          BUILD_SHA: ${{ github.sha }}
+        run: test "$(git rev-parse HEAD)" = "${BUILD_SHA}"
+      - name: Build Compose images
+        uses: docker/bake-action@018cb6412ab401ebaa809aa5f85966b74628600f # v7.4.0
+        env:
+          FLOWTEST_CI_SHA: ${{ github.sha }}
+          FLOWTEST_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+          DOCKER_BUILD_RECORD_UPLOAD: "false"
+        with:
+          source: .
+          files: |
+            ./compose.yaml
+            ./backend/docker-bake.ci.hcl
+          targets: ci-compose
+          load: true
+      - name: Verify loaded Compose image identities
+        env:
+          BUILD_SHA: ${{ github.sha }}
+        run: |
+          for image in \
+            flowtest-backend flowtest-worker flowtest-worker-data flowtest-worker-ai flowtest-beat \
+            flowtest-frontend flowtest-worker-performance flowtest-worker-environment \
+            flowtest-environment-docker \
+            flowtest-mock-target flowtest-grpc-target flowtest-postgres-walg:17.6-v3.0.8; do
+            test "$(docker image inspect "${image}" --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}')" = "${BUILD_SHA}"
+            test "$(docker image inspect "${image}" --format '{{.Os}}/{{.Architecture}}')" = 'linux/amd64'
+            docker image inspect "${image}" --format '{{.RepoTags}} {{.Id}} {{.Os}}/{{.Architecture}}'
+          done
       - name: Start FlowTest stack
-        run: docker compose up -d --build --wait
+        run: docker compose up -d --no-build --wait
+      - name: Verify running containers use loaded images
+        run: |
+          python3 - <<'PY'
+          import json
+          import subprocess
+
+          def docker(*arguments: str) -> str:
+              return subprocess.check_output(("docker", *arguments), text=True).strip()
+
+          compose = json.loads(docker("compose", "config", "--format", "json"))
+          services = compose["services"]
+          for service_name, service in services.items():
+              if "build" not in service:
+                  continue
+              container_id = docker("compose", "ps", "-q", service_name)
+              if not container_id:
+                  raise SystemExit(f"Missing running container: {service_name}")
+              image = service.get("image", f"{compose['name']}-{service_name}")
+              expected = docker("image", "inspect", image, "--format", "{{.Id}}")
+              actual = docker("inspect", container_id, "--format", "{{.Image}}")
+              if actual != expected:
+                  raise SystemExit(f"Image mismatch for {service_name}: {actual} != {expected}")
+              print(f"{service_name}: {actual}")
+          PY
       - name: Verify public endpoints
         run: |
           curl --fail --retry 10 --retry-delay 2 http://localhost:8000/api/v1/live

.github/workflows/security-ci.yml

@@ -56,15 +56,38 @@ jobs:
             shellcheck -x -P deploy/compact \
               scripts/backup.sh scripts/restore.sh scripts/verify_restore.sh \
               scripts/verify_v2_v3_upgrade.sh deploy/compact/*.sh
+      - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
+      - name: Verify build checkout
+        env:
+          BUILD_SHA: ${{ github.sha }}
+        run: test "$(git rev-parse HEAD)" = "${BUILD_SHA}"
       - name: Build release images
+        uses: docker/bake-action@018cb6412ab401ebaa809aa5f85966b74628600f # v7.4.0
+        env:
+          FLOWTEST_CI_SHA: ${{ github.sha }}
+          FLOWTEST_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+          DOCKER_BUILD_RECORD_UPLOAD: "false"
+        with:
+          source: .
+          files: |
+            ./compose.yaml
+            ./backend/docker-bake.ci.hcl
+          targets: ci-security
+          load: true
+      - name: Verify scanned image identities
+        env:
+          BUILD_SHA: ${{ github.sha }}
+        run: |
+          for image in \
+            flowtest-backend:ci flowtest-frontend:ci flowtest-performance:ci \
+            flowtest-environment:ci flowtest-environment-daemon:ci \
+            flowtest-runner:ci flowtest-mock-target:ci; do
+            test "$(docker image inspect "${image}" --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}')" = "${BUILD_SHA}"
+            test "$(docker image inspect "${image}" --format '{{.Os}}/{{.Architecture}}')" = 'linux/amd64'
+            docker image inspect "${image}" --format '{{.RepoTags}} {{.Id}} {{.Os}}/{{.Architecture}}'
+          done
+      - name: Prepare pinned environment fixture
         run: |
-          docker build -t flowtest-backend:ci backend
-          docker build --target performance-runtime -t flowtest-performance:ci backend
-          docker build --target environment-runtime -t flowtest-environment:ci backend
-          docker build --target environment-daemon -t flowtest-environment-daemon:ci backend
-          docker build --target runner-runtime -t flowtest-runner:ci backend
-          docker build -t flowtest-frontend:ci frontend
-          docker build -t flowtest-mock-target:ci mock-target
           docker pull docker.io/nginxinc/nginx-unprivileged@sha256:123fb7283ffb4788e260d4e980005a978995fefedcdbb04d268077a19b84576d
           docker tag docker.io/nginxinc/nginx-unprivileged@sha256:123fb7283ffb4788e260d4e980005a978995fefedcdbb04d268077a19b84576d flowtest-environment-fixture:ci
       - name: Scan backend image

backend/Dockerfile

@@ -1,3 +1,8 @@
+ARG FLOWTEST_PYTHON_BASE=python-base-source
+ARG FLOWTEST_K6_BASE=k6-base-source
+ARG FLOWTEST_DOCKER_CLI_BASE=docker-cli-base-source
+ARG FLOWTEST_ENVIRONMENT_DAEMON_BASE=environment-daemon-source
+
 FROM ghcr.io/astral-sh/uv:0.11.32 AS uv
 FROM golang:1.26.6-alpine3.23@sha256:5978cc992ad5ef96a7469713c8af849c1433824761ce3be2c56381403cd8d9a3 AS go-toolchain
 
@@ -16,6 +21,9 @@ RUN --mount=type=cache,target=/go/pkg/mod \
     go version -m /out/k6 | grep -F 'google.golang.org/grpc' | grep -F 'v1.83.2'; \
     /out/k6 version | grep -F 'k6 v2.2.0'
 
+FROM scratch AS k6-base-source
+COPY --from=k6-builder /out/k6 /out/k6
+
 FROM go-toolchain AS docker-cli-builder
 WORKDIR /go/src/github.com/docker/cli
 RUN --mount=type=cache,target=/go/pkg/mod \
@@ -43,6 +51,11 @@ RUN --mount=type=cache,target=/go/pkg/mod \
     go version -m /out/docker | grep -F 'go1.26.6'; \
     /out/docker --version | grep -F 'Docker version 29.7.2'
 
+FROM scratch AS docker-cli-base-source
+COPY --from=docker-cli-builder /out/docker /out/docker
+
+FROM ${FLOWTEST_DOCKER_CLI_BASE} AS selected-docker-cli-base
+
 FROM go-toolchain AS dockerd-builder
 RUN apk add --no-cache bash build-base libseccomp-dev libseccomp-static linux-headers pkgconf
 WORKDIR /src/moby
@@ -116,8 +129,8 @@ RUN set -eux; \
         /usr/local/libexec/docker/cli-plugins/docker-buildx \
         /usr/local/libexec/docker/cli-plugins/docker-compose
 
-FROM environment-daemon-base AS environment-daemon
-COPY --from=docker-cli-builder /out/docker /usr/local/bin/docker
+FROM environment-daemon-base AS environment-daemon-source
+COPY --from=selected-docker-cli-base /out/docker /usr/local/bin/docker
 COPY --from=dockerd-builder /out/bundles/binary-daemon/dockerd /usr/local/bin/dockerd
 COPY --from=containerd-builder /src/containerd/bin/containerd /usr/local/bin/containerd
 COPY --from=containerd-builder /src/containerd/bin/ctr /usr/local/bin/ctr
@@ -126,7 +139,9 @@ RUN set -eux; \
     containerd --version; \
     ctr --version
 
-FROM debian:trixie-slim AS python-runtime
+FROM ${FLOWTEST_ENVIRONMENT_DAEMON_BASE} AS environment-daemon
+
+FROM debian:trixie-slim AS python-base-source
 
 COPY patches/cpython-3.13-cve-2026-82049.patch /tmp/cpython-3.13-cve-2026-82049.patch
 
@@ -228,6 +243,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1 \
     PIP_NO_CACHE_DIR=1
 
+FROM ${FLOWTEST_PYTHON_BASE} AS python-runtime
 WORKDIR /app
 
 COPY --from=uv /uv /uvx /bin/
@@ -244,8 +260,10 @@ EXPOSE 8000
 
 CMD ["sh", "-c", "alembic upgrade head && uvicorn app.main:app --host 0.0.0.0 --port 8000"]
 
+FROM ${FLOWTEST_K6_BASE} AS selected-k6-base
+
 FROM python-runtime AS performance-runtime
-COPY --from=k6-builder /out/k6 /usr/local/bin/k6
+COPY --from=selected-k6-base /out/k6 /usr/local/bin/k6
 RUN set -eux; \
     groupadd --gid 65532 flowtest; \
     useradd --uid 65532 --gid 65532 --no-create-home --home-dir /tmp \
@@ -255,7 +273,7 @@ ENV HOME=/tmp
 USER flowtest
 
 FROM python-runtime AS environment-runtime
-COPY --from=docker-cli-builder /out/docker /usr/local/bin/docker
+COPY --from=selected-docker-cli-base /out/docker /usr/local/bin/docker
 RUN set -eux; \
     groupadd --gid 65532 flowtest; \
     useradd --uid 65532 --gid 65532 --no-create-home --home-dir /tmp \

backend/docker-bake.ci.hcl

@@ -0,0 +1,81 @@
+// CI builds the unique Compose image targets once, then loads every service tag.
+// Cache scopes are separated by Dockerfile target and platform. BuildKit still
+// validates Dockerfile, lockfile, patch, and source inputs within each scope.
+variable "FLOWTEST_CI_SHA" {
+  default = "local"
+}
+
+variable "FLOWTEST_PR_HEAD_SHA" {
+  default = "local"
+}
+
+target "_ci_common" {
+  platforms = ["linux/amd64"]
+  labels = {
+    "org.opencontainers.image.revision" = FLOWTEST_CI_SHA
+    "com.flowtest.pr-head"             = FLOWTEST_PR_HEAD_SHA
+  }
+}
+
+target "backend" {
+  inherits   = ["_ci_common"]
+  tags       = ["flowtest-backend", "flowtest-backend:ci", "flowtest-worker", "flowtest-worker-data", "flowtest-worker-ai", "flowtest-beat"]
+  cache-from = ["type=gha,scope=flowtest-backend-app-linux-amd64"]
+  cache-to   = ["type=gha,scope=flowtest-backend-app-linux-amd64,mode=max,ignore-error=true"]
+}
+
+target "frontend" {
+  inherits   = ["_ci_common"]
+  tags       = ["flowtest-frontend", "flowtest-frontend:ci"]
+  cache-from = ["type=gha,scope=flowtest-frontend-linux-amd64"]
+  cache-to   = ["type=gha,scope=flowtest-frontend-linux-amd64,mode=max,ignore-error=true"]
+}
+
+target "worker-performance" {
+  inherits   = ["_ci_common"]
+  tags       = ["flowtest-worker-performance", "flowtest-performance:ci"]
+  cache-from = ["type=gha,scope=flowtest-performance-linux-amd64"]
+  cache-to   = ["type=gha,scope=flowtest-performance-linux-amd64,mode=max,ignore-error=true"]
+}
+
+target "worker-environment" {
+  inherits   = ["_ci_common"]
+  tags       = ["flowtest-worker-environment", "flowtest-environment:ci"]
+  cache-from = ["type=gha,scope=flowtest-environment-linux-amd64"]
+  cache-to   = ["type=gha,scope=flowtest-environment-linux-amd64,mode=max,ignore-error=true"]
+}
+
+target "environment-docker" {
+  inherits   = ["_ci_common"]
+  tags       = ["flowtest-environment-docker", "flowtest-environment-daemon:ci"]
+  cache-from = ["type=gha,scope=flowtest-environment-daemon-linux-amd64"]
+  cache-to   = ["type=gha,scope=flowtest-environment-daemon-linux-amd64,mode=max,ignore-error=true"]
+}
+
+target "runner-agent-a" {
+  inherits   = ["_ci_common"]
+  tags       = ["flowtest-runner-agent-a", "flowtest-runner-agent-b", "flowtest-runner:ci"]
+  cache-from = ["type=gha,scope=flowtest-runner-linux-amd64"]
+  cache-to   = ["type=gha,scope=flowtest-runner-linux-amd64,mode=max,ignore-error=true"]
+}
+
+target "mock-target" {
+  inherits   = ["_ci_common"]
+  tags       = ["flowtest-mock-target", "flowtest-grpc-target", "flowtest-mock-target:ci"]
+  cache-from = ["type=gha,scope=flowtest-mock-target-linux-amd64"]
+  cache-to   = ["type=gha,scope=flowtest-mock-target-linux-amd64,mode=max,ignore-error=true"]
+}
+
+target "postgres" {
+  inherits   = ["_ci_common"]
+  cache-from = ["type=gha,scope=flowtest-postgres-walg-linux-amd64"]
+  cache-to   = ["type=gha,scope=flowtest-postgres-walg-linux-amd64,mode=max,ignore-error=true"]
+}
+
+group "ci-security" {
+  targets = ["backend", "frontend", "worker-performance", "worker-environment", "environment-docker", "runner-agent-a", "mock-target"]
+}
+
+group "ci-compose" {
+  targets = ["backend", "frontend", "worker-performance", "worker-environment", "environment-docker", "mock-target", "postgres"]
+}

backend/tests/test_environment_lab.py

@@ -606,7 +606,7 @@ async def test_fixed_docker_command_maps_failure_and_timeout(
     assert failed.value.code == "ENVIRONMENT_RUNTIME_FAILED"
     assert await environment_runner._run_docker("version", timeout_seconds=2, check=False) == ""
 
-    executable.write_text("#!/bin/sh\nsleep 5\n", encoding="utf-8")
+    executable.write_text("#!/bin/sh\nexec sleep 5\n", encoding="utf-8")
     with pytest.raises(EnvironmentRuntimeError) as timed_out:
         await environment_runner._run_docker("version", timeout_seconds=1)
     assert timed_out.value.code == "ENVIRONMENT_RUNTIME_TIMEOUT"

backend/tests/test_performance_lab.py

@@ -299,7 +299,7 @@ async def test_k6_process_runner_accepts_threshold_exit_and_rejects_missing_summ
         )
 
     sleepy = tmp_path / "sleepy-k6"
-    sleepy.write_text("#!/bin/sh\nsleep 5\n", encoding="utf-8")
+    sleepy.write_text("#!/bin/sh\nexec sleep 5\n", encoding="utf-8")
     sleepy.chmod(0o700)
     with pytest.raises(K6ExecutionError, match="K6_TIMEOUT"):
         await K6ProcessRunner(str(sleepy)).run(