diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..74160975 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,224 @@ +# reusable workflow triggered by other actions +name: CI + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + schedule: + - cron: '0 8 * * TUE' + # Triggered on push by .github/workflows/release.yaml + workflow_call: + outputs: + artifact-prefix: + description: build_charm.yaml `artifact-prefix` output + value: ${{ jobs.build.outputs.artifact-prefix }} + charm-paths: + description: paths for all charms in this repo + value: ${{ jobs.get-charm-paths-track.outputs.charm-paths }} + track: + description: Charmhub track determined from branch name + value: ${{ jobs.get-charm-paths-track.outputs.track }} + +jobs: + get-charm-paths-track: + name: Get charm paths and track + runs-on: ubuntu-latest + outputs: + charm-paths: ${{ steps.get-charm-paths.outputs.charm-paths }} + track: ${{ steps.determine-track.outputs.track }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Get paths for all charms in this repo + id: get-charm-paths + uses: canonical/kubeflow-ci/actions/get-charm-paths@main + - name: Determine track + id: determine-track + shell: python + run: | + import os + + if "${{ github.event_name }}" == "pull_request": + ref = "${{ github.base_ref }}" + else: + ref = "${{ github.ref_name }}" + + if ref.startswith("track/"): + track = ref.removeprefix("track/") + else: + track = "latest" + + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"track={track}\n") + + print(f"Track: {track}") + + lint: + name: Lint + runs-on: ubuntu-24.04 + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Install dependencies + run: pipx install tox + + - name: Lint code + run: tox -vve lint + + unit: + name: Unit + runs-on: ubuntu-24.04 + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Install dependencies + run: pipx install tox + + - name: Run unit tests + run: tox -e unit + + - name: Upload coverage artifacts + uses: actions/upload-artifact@v4 + with: + name: coverage + path: training-operator-cov_html + if: failure() + + terraform-checks: + name: Terraform + uses: canonical/charmed-kubeflow-workflows/.github/workflows/terraform-checks.yaml@main + with: + charm-path: . + + build: + strategy: + matrix: + charm: ${{ fromJSON(needs.get-charm-paths-track.outputs.charm-paths) }} + name: Build charm | ${{ matrix.charm }} + needs: + - get-charm-paths-track + uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v49.0.1 + with: + path-to-charm-directory: ${{ matrix.charm }} + cache: true + charmcraft-snap-channel: 3.x/stable + permissions: + actions: read # Needed for GitHub API call to get workflow version (for private repositories) + contents: read + + release: + strategy: + matrix: + charm: ${{ fromJSON(needs.get-charm-paths-track.outputs.charm-paths) }} + name: Release charm to Charmhub branch | ${{ matrix.charm }} + if: ${{ github.event_name == 'pull_request' }} + needs: + - get-charm-paths-track + - build + uses: canonical/data-platform-workflows/.github/workflows/release_charm_pr.yaml@v49.0.1 + with: + track: ${{ needs.get-charm-paths-track.outputs.track }} + artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} + path-to-charm-directory: ${{ matrix.charm }} + secrets: + charmhub-token: ${{ secrets.CHARMCRAFT_CREDENTIALS }} + permissions: + actions: read # Needed for GitHub API call to get workflow version (for private repositories) + contents: read + + integration: + name: Integration + needs: + - build + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + tox-environment: + - integration + - integration-ambient + - integration-with-profiles + steps: + - name: Maximise GH runner space + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be + + - name: Check out code + uses: actions/checkout@v4 + + - name: Install dependencies + run: pipx install tox + + - name: Setup environment + run: | + sudo apt-get remove -y docker-ce docker-ce-cli containerd.io + sudo rm -rf /run/containerd + sudo snap install concierge --classic + sudo concierge prepare --trace + + - name: Configure Cilium for Canonical K8s + if: matrix.tox-environment == 'integration-ambient' + run: | + # for context, see https://docs.cilium.io/en/stable/network/servicemesh/istio/ + kubectl -n kube-system patch configmap cilium-config --type merge --patch '{"data":{"bpf-lb-sock-hostns-only":"true"}}' + kubectl -n kube-system patch configmap cilium-config --type merge --patch '{"data":{"cni-exclusive":"false"}}' + kubectl -n kube-system rollout restart daemonset cilium + + - name: Fetch charm + uses: actions/download-artifact@v4 + with: + pattern: ${{ needs.build.outputs.artifact-prefix }}-* + merge-multiple: true + path: built/ + + - name: Get charm path + id: charm-path + run: echo "charm_path=$(find built/ -name '*.charm' -type f -print)" >> $GITHUB_OUTPUT + + - name: Run integration tests + run: tox -e ${{ matrix.tox-environment }} -- --model testing --charm-path="${{ steps.charm-path.outputs.charm_path }}" + + - name: Capture k8s resources on failure + run: | + set -eux + kubectl get all -A + kubectl get pods -n testing --show-labels + kubectl get crds + if: failure() + + - name: Get juju status + run: juju status + if: always() + + - name: Get validatingwebhookconfigurations + run: kubectl get validatingwebhookconfigurations validator.training-operator.kubeflow.org -oyaml + if: failure() + + - name: Get secret + run: kubectl get secret -n testing training-operator-webhook-cert -oyaml + if: failure() + + - name: Describe operator pod + run: kubectl describe pod -n testing -l app.kubernetes.io/name=training-operator + if: failure() + + - name: Describe workload pod + run: kubectl describe pod -n testing -l control-plane=testing-training-operator + if: failure() + + - name: Get pods + run: kubectl get pods -A + if: failure() + + - name: Get operator logs + run: kubectl logs --tail 100 -n testing -l app.kubernetes.io/name=training-operator -c charm + if: failure() + + - name: Get workload logs + run: kubectl logs --tail 100 -n testing -l control-plane=testing-training-operator -c training-operator + if: failure() diff --git a/.github/workflows/get-charm-paths.sh b/.github/workflows/get-charm-paths.sh deleted file mode 100644 index 1110d59c..00000000 --- a/.github/workflows/get-charm-paths.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -x - -# Finds the charms in this repo, outputting them as JSON -# Will return one of: -# * the relative paths of the directories listed in `./charms`, if that directory exists -# * "./", if the root directory has a "metadata.yaml" file -# * otherwise, error -# -# Modified from: https://stackoverflow.com/questions/63517732/github-actions-build-matrix-for-lambda-functions/63736071#63736071 -CHARMS_DIR="./charms" -if [ -d "$CHARMS_DIR" ]; -then - CHARM_PATHS=$(find $CHARMS_DIR -maxdepth 1 -type d -not -path '*/\.*' -not -path "$CHARMS_DIR") -else - if [ -f "./metadata.yaml" ] - then - CHARM_PATHS="./" - else - echo "Cannot find valid charm directories - aborting" - exit 1 - fi -fi - -# Convert output to JSON string format -# { charm_paths: [...] } -CHARM_PATHS_LIST=$(echo "$CHARM_PATHS" | jq -c --slurp --raw-input 'split("\n")[:-1]') - -echo "Found CHARM_PATHS_LIST: $CHARM_PATHS_LIST" - -echo "::set-output name=CHARM_PATHS_LIST::$CHARM_PATHS_LIST" diff --git a/.github/workflows/integrate.yaml b/.github/workflows/integrate.yaml deleted file mode 100644 index ca42b52f..00000000 --- a/.github/workflows/integrate.yaml +++ /dev/null @@ -1,133 +0,0 @@ -# reusable workflow triggered by other actions -name: CI - -on: - workflow_call: - secrets: - CHARMCRAFT_CREDENTIALS: - required: true - -jobs: - lint: - name: Lint - runs-on: ubuntu-24.04 - steps: - - name: Check out code - uses: actions/checkout@v4 - - - name: Install dependencies - run: pipx install tox - - - name: Lint code - run: tox -vve lint - - unit-test: - name: Unit - runs-on: ubuntu-24.04 - steps: - - name: Check out code - uses: actions/checkout@v4 - - name: Install dependencies - run: pipx install tox - - - name: Run unit tests - run: tox -e unit - - - name: Upload coverage artifacts - uses: actions/upload-artifact@v4 - with: - name: coverage - path: training-operator-cov_html - if: failure() - - terraform-checks: - name: Terraform - uses: canonical/charmed-kubeflow-workflows/.github/workflows/terraform-checks.yaml@main - with: - charm-path: . - - integration-test: - name: Integration - runs-on: ubuntu-24.04 - strategy: - matrix: - tox-environment: - - integration - - integration-ambient - - integration-with-profiles - steps: - - name: Maximise GH runner space - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be - - - name: Check out code - uses: actions/checkout@v4 - - name: Install dependencies - run: pipx install tox - - - name: Setup environment - run: | - sudo apt-get remove -y docker-ce docker-ce-cli containerd.io - sudo rm -rf /run/containerd - sudo snap install concierge --classic - sudo concierge prepare --trace - - - name: Configure Cilium for Canonical K8s - if: matrix.tox-environment == 'integration-ambient' - run: | - # Configure Cilium for Canonical K8s to work with Charmed Istio (Ambient mode) - # See https://canonical-service-mesh-documentation.readthedocs-hosted.com/en/latest/how-to/use-charmed-istio-with-canonical-kubernetes/ - kubectl -n kube-system patch configmap cilium-config --type merge --patch '{"data":{"bpf-lb-sock-hostns-only":"true"}}' - kubectl -n kube-system rollout restart daemonset cilium - - - name: Fetch charm - uses: actions/download-artifact@v5 - with: - name: built-charm - path: built/ - - - name: Get charm path - id: charm-path - run: echo "charm_path=$(find built/ -name '*.charm' -type f -print)" >> $GITHUB_OUTPUT - - - name: Run integration tests - run: tox -e ${{ matrix.tox-environment }} -- --model testing --charm-path="${{ steps.charm-path.outputs.charm_path }}" - - - name: Capture k8s resources on failure - run: | - set -eux - kubectl get all -A - kubectl get pods -n testing --show-labels - kubectl get crds - if: failure() - - - name: Get juju status - run: juju status - if: failure() - - - name: Get validatingwebhookconfigurations - run: kubectl get validatingwebhookconfigurations validator.training-operator.kubeflow.org -oyaml - if: failure() - - - name: Get secret - run: kubectl get secret -n testing training-operator-webhook-cert -oyaml - if: failure() - - - name: Describe operator pod - run: kubectl describe pod -n testing -l app.kubernetes.io/name=training-operator - if: failure() - - - name: Describe workload pod - run: kubectl describe pod -n testing -l control-plane=testing-training-operator - if: failure() - - - name: Get pods - run: kubectl get pods -A - if: failure() - - - name: Get operator logs - run: kubectl logs --tail 100 -n testing -l app.kubernetes.io/name=training-operator -c charm - if: failure() - - - name: Get workload logs - run: kubectl logs --tail 100 -n testing -l control-plane=testing-training-operator -c training-operator - if: failure() diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml index e55eba96..e61df83e 100644 --- a/.github/workflows/on_pull_request.yaml +++ b/.github/workflows/on_pull_request.yaml @@ -1,8 +1,7 @@ name: On Pull Request # On pull_request, we: -# * always publish to charmhub at latest/edge/branchname -# * always run tests +# * create backport labels if it is against main, only when the PR is opened/reopened on: pull_request: @@ -17,43 +16,3 @@ jobs: with: track_file_path: ".github/automatic_backport_tracks.yaml" label_prefix: "backport " - build-charm: - name: Build charm - runs-on: ubuntu-24.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup LXD - uses: canonical/setup-lxd@main - with: - channel: 5.21/stable - - - name: Install charmcraft - run: sudo snap install charmcraft --classic - - - name: Build charm under test - run: charmcraft pack --verbose - - - name: Archive charm - uses: actions/upload-artifact@v4 - with: - name: built-charm - path: "*.charm" - retention-days: 5 - - tests: - name: Run Tests - needs: - - build-charm - uses: ./.github/workflows/integrate.yaml - secrets: inherit - - # publish runs in parallel with tests, as we always publish in this situation - publish-charm: - name: Publish Charm - needs: - - build-charm - uses: ./.github/workflows/publish.yaml - secrets: inherit - diff --git a/.github/workflows/on_push.yaml b/.github/workflows/on_push.yaml deleted file mode 100644 index fae1e5a7..00000000 --- a/.github/workflows/on_push.yaml +++ /dev/null @@ -1,54 +0,0 @@ -name: On Push - -# On push to a "special" branch, we: -# * always publish to charmhub at latest/edge/branchname -# * always run tests -# where a "special" branch is one of main or track/**, as -# by convention these branches are the source for a corresponding -# charmhub edge channel. - -on: - push: - branches: - - main - - track/** - -jobs: - build-charm: - name: Build charm - runs-on: ubuntu-24.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup LXD - uses: canonical/setup-lxd@main - with: - channel: 5.21/stable - - - name: Install charmcraft - run: sudo snap install charmcraft --classic - - - name: Build charm under test - run: charmcraft pack --verbose - - - name: Archive charm - uses: actions/upload-artifact@v4 - with: - name: built-charm - path: "*.charm" - retention-days: 5 - - tests: - name: Run Tests - needs: - - build-charm - uses: ./.github/workflows/integrate.yaml - secrets: inherit - - # publish runs in series with tests, and only publishes if tests passes - publish-charm: - name: Publish Charm - needs: tests - uses: ./.github/workflows/publish.yaml - secrets: inherit diff --git a/.github/workflows/promote.yaml b/.github/workflows/promote.yaml new file mode 100644 index 00000000..45c2882d --- /dev/null +++ b/.github/workflows/promote.yaml @@ -0,0 +1,32 @@ +# reusable workflow triggered manually +name: Promote charm to other tracks and channels + +on: + workflow_dispatch: + inputs: + destination-channel: + description: 'Destination Channel' + required: true + origin-channel: + description: 'Origin Channel' + required: true + charm-name: + description: 'Charm subdirectory name' + required: true + +jobs: + promote-charm: + name: Promote charm + runs-on: ubuntu-24.04 + env: + CHARMCRAFT_AUTH: ${{ secrets.CHARMCRAFT_CREDENTIALS }} + steps: + - name: Install charmcraft + run: | + sudo snap install charmcraft --classic --channel latest/stable + - name: Run charmcraft promote + run: | + charmcraft promote --name ${{ github.event.inputs.charm-name }} \ + --from-channel ${{ github.event.inputs.origin-channel }} \ + --to-channel ${{ github.event.inputs.destination-channel }} \ + --yes diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml deleted file mode 100644 index 6a3414fa..00000000 --- a/.github/workflows/publish.yaml +++ /dev/null @@ -1,112 +0,0 @@ -# reusable workflow for publishing all charms in this repo -name: Publish - -on: - workflow_call: - inputs: - source_branch: - description: Github branch from this repo to publish. If blank, will use the default branch - default: '' - required: false - type: string - secrets: - CHARMCRAFT_CREDENTIALS: - required: true - workflow_dispatch: - inputs: - destination_channel: - description: CharmHub channel to publish to - required: false - default: 'latest/edge' - type: string - source_branch: - description: Github branch from this repo to publish. If blank, will use the default branch - required: false - default: '' - type: string - -jobs: - get-charm-paths: - name: Generate the Charm Matrix - runs-on: ubuntu-24.04 - outputs: - charm_paths_list: ${{ steps.get-charm-paths.outputs.CHARM_PATHS_LIST }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ inputs.source_branch }} - - name: Get paths for all charms in repo - id: get-charm-paths - run: bash .github/workflows/get-charm-paths.sh - - - publish-charm: - name: Publish Charm - runs-on: ubuntu-24.04 - needs: get-charm-paths - strategy: - fail-fast: false - matrix: - charm-path: ${{ fromJson(needs.get-charm-paths.outputs.charm_paths_list) }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ inputs.source_branch }} - - - name: Select charmhub channel - uses: canonical/charming-actions/channel@2.6.2 - id: select-channel - if: ${{ inputs.destination_channel == '' }} - - # Combine inputs from different sources to a single canonical value so later steps don't - # need logic for picking the right one - - name: Parse and combine inputs - id: parse-inputs - run: | - # destination_channel - destination_channel="${{ inputs.destination_channel || steps.select-channel.outputs.name }}" - echo "setting output of destination_channel=$destination_channel" - echo "::set-output name=destination_channel::$destination_channel" - - # tag_prefix - # if charm_path = ./ --> tag_prefix = '' (null) - # if charm_path != ./some-charm (eg: a charm in a ./charms dir) --> tag_prefix = 'some-charm' - if [ ${{ matrix.charm-path }} == './' ]; then - tag_prefix='' - else - tag_prefix=$(basename ${{ matrix.charm-path }} ) - fi - echo "setting output of tag_prefix=$tag_prefix" - echo "::set-output name=tag_prefix::$tag_prefix" - - # Required to charmcraft pack in non-destructive mode - - name: Setup lxd - uses: canonical/setup-lxd@v0.1.2 - with: - channel: latest/stable - - - name: Fetch charm - uses: actions/download-artifact@v5 - with: - name: built-charm - path: built/ - - - name: Get charm path - id: charm-path - run: echo "charm_path=$(find built/ -name '*.charm' -type f -print)" >> $GITHUB_OUTPUT - - - name: Upload charm to charmhubpip-tools - uses: canonical/charming-actions/upload-charm@2.6.2 - with: - credentials: ${{ secrets.CHARMCRAFT_CREDENTIALS }} - github-token: ${{ secrets.GITHUB_TOKEN }} - charm-path: ${{ matrix.charm-path }} - built-charm-path: ${{ steps.charm-path.outputs.charm_path }} - channel: ${{ steps.parse-inputs.outputs.destination_channel }} - tag-prefix: ${{ steps.parse-inputs.outputs.tag_prefix }} - charmcraft-channel: 3.x/stable - destructive-mode: false diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9d3752b6..069eb562 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,27 +1,33 @@ -# reusable workflow triggered manually -name: Release charm to other tracks and channels +name: Release to Charmhub on: - workflow_dispatch: - inputs: - destination-channel: - description: 'Destination Channel' - required: true - origin-channel: - description: 'Origin Channel' - required: true + push: + branches: + - main + - track/** jobs: - promote-charm: - name: Promote charm - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - name: Release charm to channel - uses: canonical/charming-actions/release-charm@2.6.2 - with: - credentials: ${{ secrets.CHARMCRAFT_CREDENTIALS }} - github-token: ${{ secrets.GITHUB_TOKEN }} - destination-channel: ${{ github.event.inputs.destination-channel }} - origin-channel: ${{ github.event.inputs.origin-channel }} - base-channel: "24.04" + ci-tests: + uses: ./.github/workflows/ci.yaml + secrets: inherit + permissions: + actions: read + contents: read + + release: + strategy: + matrix: + charm: ${{ fromJSON(needs.ci-tests.outputs.charm-paths) }} + name: Release charm | ${{ matrix.charm }} + needs: + - ci-tests + uses: canonical/data-platform-workflows/.github/workflows/release_charm_edge.yaml@v49.0.1 + with: + track: ${{ needs.ci-tests.outputs.track }} + artifact-prefix: ${{ needs.ci-tests.outputs.artifact-prefix }} + path-to-charm-directory: ${{ matrix.charm }} + secrets: + charmhub-token: ${{ secrets.CHARMCRAFT_CREDENTIALS }} + permissions: + actions: read + contents: write # Needed to create git tags diff --git a/.github/workflows/weekly_ci.yaml b/.github/workflows/weekly_ci.yaml deleted file mode 100644 index 71a3c788..00000000 --- a/.github/workflows/weekly_ci.yaml +++ /dev/null @@ -1,12 +0,0 @@ -name: Run weekly tests - -on: - schedule: - - cron: '0 8 * * TUE' - -jobs: - tests: - name: Run Tests - uses: ./.github/workflows/integrate.yaml - secrets: - charmcraft-credentials: '${{ secrets.CHARMCRAFT_CREDENTIALS }}'