Only build and push images which were touched

.github/workflows/build-and-push.yaml

@@ -1,70 +1,104 @@
 name: Build and Push Docker Images
+
 on:
   pull_request:
     branches:
       - main
+  push:
+    branches:
+      - main
   workflow_dispatch:
     inputs:
       package:
-        description: 'Package name to build (leave empty for all)'
+        description: "Package name to build (leave empty for auto detection)"
         required: false
-        default: ''
+        default: ""
+
 env:
   REGISTRY: ghcr.io
   IMAGE_PREFIX: ghcr.io/urgap
+
 jobs:
   build-matrix:
     name: Generate Build Matrix
     runs-on: ubuntu-latest
+
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
       has_packages: ${{ steps.set-matrix.outputs.has_packages }}
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: "3.11"
+
+      - name: Detect changed packages
+        id: detect
+        run: |
+          CHANGED=$(python helpers/detect_changed_packages.py package-information.json)
+          echo "changed=$CHANGED" >> $GITHUB_OUTPUT
+
       - name: Generate matrix
         id: set-matrix
         run: |
+
           if [ -n "${{ github.event.inputs.package }}" ]; then
             MATRIX=$(python helpers/parse_packages.py package-information.json "${{ github.event.inputs.package }}")
+
+          elif [ -n "${{ steps.detect.outputs.changed }}" ]; then
+            MATRIX=$(python helpers/parse_packages.py package-information.json ${{ steps.detect.outputs.changed }})
+
           else
-            MATRIX=$(python helpers/parse_packages.py package-information.json)
+            MATRIX='{"include":[]}'
           fi
+
           echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
 
-          # Check if matrix has packages
           INCLUDE_COUNT=$(echo "$MATRIX" | python -c "import sys, json; print(len(json.load(sys.stdin).get('include', [])))")
+
           if [ "$INCLUDE_COUNT" -gt 0 ]; then
             echo "has_packages=true" >> $GITHUB_OUTPUT
           else
             echo "has_packages=false" >> $GITHUB_OUTPUT
           fi
+
   build-test-push:
     name: Build ${{ matrix.name }}:${{ matrix.version }}
+
     needs: build-matrix
     if: needs.build-matrix.outputs.has_packages == 'true'
+
     runs-on: ubuntu-latest
+
     permissions:
       contents: read
       packages: write
+
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }}
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
+
       - name: Log in to GitHub Container Registry
+        if: github.event_name == 'push'
         uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Prepare Docker tags
         id: tags
         run: |
@@ -76,6 +110,7 @@ jobs:
           fi
 
           echo "tags=$TAGS" >> $GITHUB_OUTPUT
+
       - name: Build Docker image
         uses: docker/build-push-action@v5
         with:
@@ -86,21 +121,19 @@ jobs:
           tags: ${{ steps.tags.outputs.tags }}
           build-args: |
             BASEIMAGE=${{ matrix.base_image }}
-            URGAP=wheel/${{ steps.download-wheel.outputs.wheel_name }}
+            URGAP=wheel/${{ matrix.urgap_version }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
+
       - name: Run tests
         run: |
           IMAGE_TAG="${{ env.IMAGE_PREFIX }}/${{ matrix.name }}:${{ matrix.version }}"
           CONTAINER_NAME="${{ matrix.name }}-test-$$"
 
-          # Start container
           docker run -d --name "$CONTAINER_NAME" "$IMAGE_TAG"
 
-          # Copy tests into container
           docker cp ${{ matrix.name }}/tests "$CONTAINER_NAME":/home/nonroot/tests
 
-          # Run tests based on separate_venv setting
           if [ "${{ matrix.separate_venv }}" == "true" ]; then
             docker exec "$CONTAINER_NAME" /home/nonroot/venv/bin/pip install pytest
             docker exec "$CONTAINER_NAME" /home/nonroot/venv/bin/pytest /home/nonroot/tests/
@@ -109,18 +142,12 @@ jobs:
             docker exec "$CONTAINER_NAME" pytest /home/nonroot/tests/
           fi
 
-          # Cleanup
           docker stop "$CONTAINER_NAME"
           docker rm "$CONTAINER_NAME"
+
       - name: Push Docker image
-        uses: docker/build-push-action@v5
-        with:
-          context: ${{ matrix.name }}
-          file: ${{ matrix.name }}/Dockerfile
-          push: true
-          tags: ${{ steps.tags.outputs.tags }}
-          build-args: |
-            BASEIMAGE=${{ matrix.base_image }}
-            URGAP=wheel/${{ steps.download-wheel.outputs.wheel_name }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+        if: github.event_name == 'push'
+        run: |
+          for tag in $(echo "${{ steps.tags.outputs.tags }}" | tr ',' ' '); do
+            docker push $tag
+          done

helpers/detect_changed_packages.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+"""Detect which packages changed relative to main."""
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+
+def git_show_file(ref: str, path: str) -> str | None:
+    """Return file content from git reference."""
+    try:
+        return subprocess.check_output(
+            ["git", "show", f"{ref}:{path}"],
+            text=True,
+        )
+    except subprocess.CalledProcessError:
+        return None
+
+
+def get_changed_files(base: str) -> list[str]:
+    """Return list of files changed relative to base."""
+    diff = subprocess.check_output(
+        ["git", "diff", "--name-only", base],
+        text=True,
+    )
+    return diff.splitlines()
+
+
+def get_merge_base() -> str:
+    """Return merge base with origin/main."""
+    return subprocess.check_output(
+        ["git", "merge-base", "origin/main", "HEAD"],
+        text=True,
+    ).strip()
+
+
+def detect_changed_packages(json_path: str) -> list[str]:
+    """
+    Determine which packages changed based on git diff.
+
+    Args:
+        json_path: Path to package-information.json
+
+    Returns:
+        List of changed package names
+    """
+
+    base = get_merge_base()
+
+    with open(json_path) as f:
+        current = json.load(f)
+
+    previous_raw = git_show_file(base, json_path)
+    previous = json.loads(previous_raw) if previous_raw else {"packages": []}
+
+    current_packages = {p["name"]: p for p in current.get("packages", [])}
+    previous_packages = {p["name"]: p for p in previous.get("packages", [])}
+
+    changed_packages = set()
+
+    # detect added / modified packages
+    for name, pkg in current_packages.items():
+
+        if name not in previous_packages:
+            changed_packages.add(name)
+            continue
+
+        if pkg != previous_packages[name]:
+            changed_packages.add(name)
+
+    # detect directory changes
+    changed_files = get_changed_files(base)
+
+    for file in changed_files:
+        path = Path(file)
+
+        if path.parts and path.parts[0] in current_packages:
+            changed_packages.add(path.parts[0])
+
+    return sorted(changed_packages)
+
+
+def main():
+
+    if len(sys.argv) != 2:
+        print("Usage: detect_changed_packages.py <json_path>", file=sys.stderr)
+        sys.exit(1)
+
+    json_path = sys.argv[1]
+
+    packages = detect_changed_packages(json_path)
+
+    print(" ".join(packages))
+
+
+if __name__ == "__main__":
+    main()

helpers/parse_packages.py

@@ -6,13 +6,13 @@
 from pathlib import Path
 
 
-def parse_packages(json_path: str, package_filter: str | None = None) -> dict:
+def parse_packages(json_path: str, package_filters: list[str] | None = None) -> dict:
     """
     Parse package-information.json and return GitHub Actions matrix.
 
     Args:
         json_path: Path to package-information.json
-        package_filter: Optional package name to filter (for manual dispatch)
+        package_filters: Optional package names to filter
 
     Returns:
         Dictionary with 'include' key containing list of matrix entries
@@ -29,11 +29,13 @@ def parse_packages(json_path: str, package_filter: str | None = None) -> dict:
     matrix_include = []
     errors = []
 
+    filters = set(package_filters) if package_filters else None
+
     for package in packages:
         name = package["name"]
 
-        # Skip if filter specified and doesn't match
-        if package_filter and name != package_filter:
+        # Skip if filters specified and doesn't match
+        if filters and name not in filters:
             continue
 
         # Validation: check package directory exists
@@ -90,17 +92,17 @@ def parse_packages(json_path: str, package_filter: str | None = None) -> dict:
 
 def main():
     if len(sys.argv) < 2:
-        print("Usage: parse_packages.py <json_path> [package_filter]", file=sys.stderr)
+        print("Usage: parse_packages.py <json_path> [package_filters...]", file=sys.stderr)
         sys.exit(1)
 
     json_path = sys.argv[1]
-    package_filter = sys.argv[2] if len(sys.argv) > 2 else None
+    package_filters = sys.argv[2:] if len(sys.argv) > 2 else None
 
     if not Path(json_path).exists():
         print(f"Error: {json_path} not found", file=sys.stderr)
         sys.exit(1)
 
-    matrix = parse_packages(json_path, package_filter)
+    matrix = parse_packages(json_path, package_filters)
 
     # Output for GitHub Actions
     print(json.dumps(matrix))