diff --git a/.github/workflows/build-and-push.yaml b/.github/workflows/build-and-push.yaml index 174c9b9..aa750d6 100644 --- a/.github/workflows/build-and-push.yaml +++ b/.github/workflows/build-and-push.yaml @@ -1,70 +1,104 @@ name: Build and Push Docker Images + on: pull_request: branches: - main + push: + branches: + - main workflow_dispatch: inputs: package: - description: 'Package name to build (leave empty for all)' + description: "Package name to build (leave empty for auto detection)" required: false - default: '' + default: "" + env: REGISTRY: ghcr.io IMAGE_PREFIX: ghcr.io/urgap + jobs: build-matrix: name: Generate Build Matrix runs-on: ubuntu-latest + outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} has_packages: ${{ steps.set-matrix.outputs.has_packages }} + steps: - name: Checkout repository uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: "3.11" + + - name: Detect changed packages + id: detect + run: | + CHANGED=$(python helpers/detect_changed_packages.py package-information.json) + echo "changed=$CHANGED" >> $GITHUB_OUTPUT + - name: Generate matrix id: set-matrix run: | + if [ -n "${{ github.event.inputs.package }}" ]; then MATRIX=$(python helpers/parse_packages.py package-information.json "${{ github.event.inputs.package }}") + + elif [ -n "${{ steps.detect.outputs.changed }}" ]; then + MATRIX=$(python helpers/parse_packages.py package-information.json ${{ steps.detect.outputs.changed }}) + else - MATRIX=$(python helpers/parse_packages.py package-information.json) + MATRIX='{"include":[]}' fi + echo "matrix=$MATRIX" >> $GITHUB_OUTPUT - # Check if matrix has packages INCLUDE_COUNT=$(echo "$MATRIX" | python -c "import sys, json; print(len(json.load(sys.stdin).get('include', [])))") + if [ "$INCLUDE_COUNT" -gt 0 ]; then echo "has_packages=true" >> $GITHUB_OUTPUT else echo "has_packages=false" >> $GITHUB_OUTPUT fi + build-test-push: name: Build ${{ matrix.name }}:${{ matrix.version }} + needs: build-matrix if: needs.build-matrix.outputs.has_packages == 'true' + runs-on: ubuntu-latest + permissions: contents: read packages: write + strategy: fail-fast: false matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} + steps: - name: Checkout repository uses: actions/checkout@v4 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Log in to GitHub Container Registry + if: github.event_name == 'push' uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Prepare Docker tags id: tags run: | @@ -76,6 +110,7 @@ jobs: fi echo "tags=$TAGS" >> $GITHUB_OUTPUT + - name: Build Docker image uses: docker/build-push-action@v5 with: @@ -86,21 +121,19 @@ jobs: tags: ${{ steps.tags.outputs.tags }} build-args: | BASEIMAGE=${{ matrix.base_image }} - URGAP=wheel/${{ steps.download-wheel.outputs.wheel_name }} + URGAP=wheel/${{ matrix.urgap_version }} cache-from: type=gha cache-to: type=gha,mode=max + - name: Run tests run: | IMAGE_TAG="${{ env.IMAGE_PREFIX }}/${{ matrix.name }}:${{ matrix.version }}" CONTAINER_NAME="${{ matrix.name }}-test-$$" - # Start container docker run -d --name "$CONTAINER_NAME" "$IMAGE_TAG" - # Copy tests into container docker cp ${{ matrix.name }}/tests "$CONTAINER_NAME":/home/nonroot/tests - # Run tests based on separate_venv setting if [ "${{ matrix.separate_venv }}" == "true" ]; then docker exec "$CONTAINER_NAME" /home/nonroot/venv/bin/pip install pytest docker exec "$CONTAINER_NAME" /home/nonroot/venv/bin/pytest /home/nonroot/tests/ @@ -109,18 +142,12 @@ jobs: docker exec "$CONTAINER_NAME" pytest /home/nonroot/tests/ fi - # Cleanup docker stop "$CONTAINER_NAME" docker rm "$CONTAINER_NAME" + - name: Push Docker image - uses: docker/build-push-action@v5 - with: - context: ${{ matrix.name }} - file: ${{ matrix.name }}/Dockerfile - push: true - tags: ${{ steps.tags.outputs.tags }} - build-args: | - BASEIMAGE=${{ matrix.base_image }} - URGAP=wheel/${{ steps.download-wheel.outputs.wheel_name }} - cache-from: type=gha - cache-to: type=gha,mode=max + if: github.event_name == 'push' + run: | + for tag in $(echo "${{ steps.tags.outputs.tags }}" | tr ',' ' '); do + docker push $tag + done diff --git a/helpers/detect_changed_packages.py b/helpers/detect_changed_packages.py new file mode 100644 index 0000000..c3bf75d --- /dev/null +++ b/helpers/detect_changed_packages.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Detect which packages changed relative to main.""" + +import json +import subprocess +import sys +from pathlib import Path + + +def git_show_file(ref: str, path: str) -> str | None: + """Return file content from git reference.""" + try: + return subprocess.check_output( + ["git", "show", f"{ref}:{path}"], + text=True, + ) + except subprocess.CalledProcessError: + return None + + +def get_changed_files(base: str) -> list[str]: + """Return list of files changed relative to base.""" + diff = subprocess.check_output( + ["git", "diff", "--name-only", base], + text=True, + ) + return diff.splitlines() + + +def get_merge_base() -> str: + """Return merge base with origin/main.""" + return subprocess.check_output( + ["git", "merge-base", "origin/main", "HEAD"], + text=True, + ).strip() + + +def detect_changed_packages(json_path: str) -> list[str]: + """ + Determine which packages changed based on git diff. + + Args: + json_path: Path to package-information.json + + Returns: + List of changed package names + """ + + base = get_merge_base() + + with open(json_path) as f: + current = json.load(f) + + previous_raw = git_show_file(base, json_path) + previous = json.loads(previous_raw) if previous_raw else {"packages": []} + + current_packages = {p["name"]: p for p in current.get("packages", [])} + previous_packages = {p["name"]: p for p in previous.get("packages", [])} + + changed_packages = set() + + # detect added / modified packages + for name, pkg in current_packages.items(): + + if name not in previous_packages: + changed_packages.add(name) + continue + + if pkg != previous_packages[name]: + changed_packages.add(name) + + # detect directory changes + changed_files = get_changed_files(base) + + for file in changed_files: + path = Path(file) + + if path.parts and path.parts[0] in current_packages: + changed_packages.add(path.parts[0]) + + return sorted(changed_packages) + + +def main(): + + if len(sys.argv) != 2: + print("Usage: detect_changed_packages.py ", file=sys.stderr) + sys.exit(1) + + json_path = sys.argv[1] + + packages = detect_changed_packages(json_path) + + print(" ".join(packages)) + + +if __name__ == "__main__": + main() diff --git a/helpers/parse_packages.py b/helpers/parse_packages.py index 3519436..a3e180f 100644 --- a/helpers/parse_packages.py +++ b/helpers/parse_packages.py @@ -6,13 +6,13 @@ from pathlib import Path -def parse_packages(json_path: str, package_filter: str | None = None) -> dict: +def parse_packages(json_path: str, package_filters: list[str] | None = None) -> dict: """ Parse package-information.json and return GitHub Actions matrix. Args: json_path: Path to package-information.json - package_filter: Optional package name to filter (for manual dispatch) + package_filters: Optional package names to filter Returns: Dictionary with 'include' key containing list of matrix entries @@ -29,11 +29,13 @@ def parse_packages(json_path: str, package_filter: str | None = None) -> dict: matrix_include = [] errors = [] + filters = set(package_filters) if package_filters else None + for package in packages: name = package["name"] - # Skip if filter specified and doesn't match - if package_filter and name != package_filter: + # Skip if filters specified and doesn't match + if filters and name not in filters: continue # Validation: check package directory exists @@ -90,17 +92,17 @@ def parse_packages(json_path: str, package_filter: str | None = None) -> dict: def main(): if len(sys.argv) < 2: - print("Usage: parse_packages.py [package_filter]", file=sys.stderr) + print("Usage: parse_packages.py [package_filters...]", file=sys.stderr) sys.exit(1) json_path = sys.argv[1] - package_filter = sys.argv[2] if len(sys.argv) > 2 else None + package_filters = sys.argv[2:] if len(sys.argv) > 2 else None if not Path(json_path).exists(): print(f"Error: {json_path} not found", file=sys.stderr) sys.exit(1) - matrix = parse_packages(json_path, package_filter) + matrix = parse_packages(json_path, package_filters) # Output for GitHub Actions print(json.dumps(matrix))