diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..9e0c3da --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,185 @@ +name: CI + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + schedule: + - cron: '0 8 * * TUE' + # Triggered on push by .github/workflows/release.yaml + workflow_call: + outputs: + artifact-prefix: + description: build_charm.yaml `artifact-prefix` output + value: ${{ jobs.build.outputs.artifact-prefix }} + charm-paths: + description: paths for all charms in this repo + value: ${{ jobs.get-charm-paths-track.outputs.charm-paths }} + track: + description: Charmhub track determined from branch name + value: ${{ jobs.get-charm-paths-track.outputs.track }} + +jobs: + get-charm-paths-track: + name: Get charm paths and track + runs-on: ubuntu-latest + outputs: + charm-paths: ${{ steps.get-charm-paths.outputs.charm-paths }} + track: ${{ steps.determine-track.outputs.track }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Get paths for all charms in this repo + id: get-charm-paths + uses: canonical/kubeflow-ci/actions/get-charm-paths@main + - name: Determine track + id: determine-track + shell: python + run: | + import os + + if "${{ github.event_name }}" == "pull_request": + ref = "${{ github.base_ref }}" + else: + ref = "${{ github.ref_name }}" + + if ref.startswith("track/"): + track = ref.removeprefix("track/") + else: + track = "latest" + + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"track={track}\n") + + print(f"Track: {track}") + + lint: + name: Lint + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - run: pipx install tox + - run: tox -vve lint + + unit: + name: Unit tests + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - run: pipx install tox + - run: tox -e unit + + terraform-checks: + name: Terraform + needs: + - get-charm-paths-track + uses: canonical/charmed-kubeflow-workflows/.github/workflows/terraform-checks.yaml@main + strategy: + matrix: + charm: ${{ fromJSON(needs.get-charm-paths-track.outputs.charm-paths) }} + with: + charm-path: ${{ matrix.charm }} + + build: + strategy: + matrix: + charm: ${{ fromJSON(needs.get-charm-paths-track.outputs.charm-paths) }} + name: Build charm | ${{ matrix.charm }} + needs: + - get-charm-paths-track + uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v49.0.1 + with: + path-to-charm-directory: ${{ matrix.charm }} + cache: true + permissions: + actions: read # Needed for GitHub API call to get workflow version (for private repositories) + contents: read + + release: + strategy: + matrix: + charm: ${{ fromJSON(needs.get-charm-paths-track.outputs.charm-paths) }} + name: Release charm to Charmhub branch | ${{ matrix.charm }} + if: ${{ github.event_name == 'pull_request' }} + needs: + - get-charm-paths-track + - build + uses: canonical/data-platform-workflows/.github/workflows/release_charm_pr.yaml@v49.0.1 + with: + track: ${{ needs.get-charm-paths-track.outputs.track }} + artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} + path-to-charm-directory: ${{ matrix.charm }} + secrets: + charmhub-token: ${{ secrets.CHARMCRAFT_CREDENTIALS }} + permissions: + actions: read # Needed for GitHub API call to get workflow version (for private repositories) + contents: read + + integration: + name: Integration tests + needs: + - build + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + charm: [mlmd] + test-type: [integration] + steps: + - name: Maximise GH runner space + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be + + - uses: actions/checkout@v4 + + - name: Install dependencies + run: pipx install tox + + - name: Setup environment + run: | + sudo apt-get remove -y docker-ce docker-ce-cli containerd.io + sudo rm -rf /run/containerd + sudo snap install concierge --classic + sudo concierge prepare --trace + + - name: Download packed charm(s) + id: download-charms + timeout-minutes: 5 + uses: actions/download-artifact@v4 + with: + pattern: ${{ needs.build.outputs.artifact-prefix }}-* + merge-multiple: true + + - name: Integration tests + run: | + # Pass the path where the charm artefact is downloaded to the tox command + # FIXME: Right now the complete path is half hardcoded to _ubuntu@24.04-amd64.charm + # We need to find a better way to dynamically get this value + tox -vve ${{ matrix.test-type }} -- --model testing --charm-path=${{ github.workspace }}/${{ matrix.charm }}_ubuntu@24.04-amd64.charm + + # On failure, capture debugging resources + - name: Get all + run: kubectl get all -A + if: failure() + + - name: Describe deployments + run: kubectl describe deployments -A + if: failure() + + - name: Describe replicasets + run: kubectl describe replicasets -A + if: failure() + + - name: Get juju status + run: juju status + if: always() + + - name: Get application logs + run: kubectl logs -n testing --tail 1000 -lapp.kubernetes.io/name=mlmd + if: failure() + + - name: Get application operator logs + run: kubectl logs -n testing --tail 1000 -ljuju-operator=mlmd + if: failure() diff --git a/.github/workflows/get-charm-paths.sh b/.github/workflows/get-charm-paths.sh deleted file mode 100644 index 1110d59..0000000 --- a/.github/workflows/get-charm-paths.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -x - -# Finds the charms in this repo, outputting them as JSON -# Will return one of: -# * the relative paths of the directories listed in `./charms`, if that directory exists -# * "./", if the root directory has a "metadata.yaml" file -# * otherwise, error -# -# Modified from: https://stackoverflow.com/questions/63517732/github-actions-build-matrix-for-lambda-functions/63736071#63736071 -CHARMS_DIR="./charms" -if [ -d "$CHARMS_DIR" ]; -then - CHARM_PATHS=$(find $CHARMS_DIR -maxdepth 1 -type d -not -path '*/\.*' -not -path "$CHARMS_DIR") -else - if [ -f "./metadata.yaml" ] - then - CHARM_PATHS="./" - else - echo "Cannot find valid charm directories - aborting" - exit 1 - fi -fi - -# Convert output to JSON string format -# { charm_paths: [...] } -CHARM_PATHS_LIST=$(echo "$CHARM_PATHS" | jq -c --slurp --raw-input 'split("\n")[:-1]') - -echo "Found CHARM_PATHS_LIST: $CHARM_PATHS_LIST" - -echo "::set-output name=CHARM_PATHS_LIST::$CHARM_PATHS_LIST" diff --git a/.github/workflows/integrate.yaml b/.github/workflows/integrate.yaml deleted file mode 100644 index ebb10a3..0000000 --- a/.github/workflows/integrate.yaml +++ /dev/null @@ -1,84 +0,0 @@ -# reusable workflow triggered by other actions -name: CI - -on: - workflow_call: - secrets: - CHARMCRAFT_CREDENTIALS: - required: true - -jobs: - lint: - name: Lint - runs-on: ubuntu-24.04 - - steps: - - name: Check out code - uses: actions/checkout@v4 - - name: Install dependencies - run: pipx install tox - - name: Lint code - run: tox -vve lint - - unit: - name: Unit Tests - runs-on: ubuntu-24.04 - steps: - - name: Check out code - uses: actions/checkout@v4 - - name: Install dependencies - run: pipx install tox - - name: Run unit tests - run: tox -vve unit - - terraform-checks: - name: Terraform - uses: canonical/charmed-kubeflow-workflows/.github/workflows/terraform-checks.yaml@main - with: - charm-path: . - - integration: - name: Integration Tests - runs-on: ubuntu-24.04 - steps: - - name: Maximise GH runner space - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be - - name: Check out code - uses: actions/checkout@v4 - - name: Install dependencies - run: pipx install tox - - - name: Setup environment - run: | - sudo apt-get remove -y docker-ce docker-ce-cli containerd.io - sudo rm -rf /run/containerd - sudo snap install concierge --classic - sudo concierge prepare --trace - - - name: Test - run: tox -vve integration -- --model testing - - # On failure, capture debugging resources - - name: Get all - run: kubectl get all -A - if: failure() - - - name: Describe deployments - run: kubectl describe deployments -A - if: failure() - - - name: Describe replicasets - run: kubectl describe replicasets -A - if: failure() - - - name: Get juju status - run: juju status - if: failure() - - - name: Get application logs - run: kubectl logs -n testing --tail 1000 -lapp.kubernetes.io/name=mlmd - if: failure() - - - name: Get application operator logs - run: kubectl logs -n testing --tail 1000 -ljuju-operator=mlmd - if: failure() diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml index d5477af..b64e808 100644 --- a/.github/workflows/on_pull_request.yaml +++ b/.github/workflows/on_pull_request.yaml @@ -2,8 +2,6 @@ name: On Pull Request # On pull_request, we: # * create backport labels if it is against main, only when the PR is opened/reopened -# * always publish to charmhub at latest/edge/branchname -# * always run tests on: pull_request: @@ -19,13 +17,3 @@ jobs: track_file_path: ".github/automatic_backport_tracks.yaml" label_prefix: "backport " - tests: - name: Run Tests - uses: ./.github/workflows/integrate.yaml - secrets: inherit - - # publish runs in parallel with tests, as we always publish in this situation - publish-charm: - name: Publish Charm - uses: ./.github/workflows/publish.yaml - secrets: inherit diff --git a/.github/workflows/on_push.yaml b/.github/workflows/on_push.yaml deleted file mode 100644 index 31c42da..0000000 --- a/.github/workflows/on_push.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: On Push - -# On push to a "special" branch, we: -# * always publish to charmhub at latest/edge/branchname -# * always run tests -# where a "special" branch is one of main or track/**, as -# by convention these branches are the source for a corresponding -# charmhub edge channel. - -on: - push: - branches: - - main - - track/** - -jobs: - - tests: - name: Run Tests - uses: ./.github/workflows/integrate.yaml - secrets: inherit - - # publish runs in series with tests, and only publishes if tests passes - publish-charm: - name: Publish Charm - needs: tests - uses: ./.github/workflows/publish.yaml - secrets: inherit diff --git a/.github/workflows/promote.yaml b/.github/workflows/promote.yaml new file mode 100644 index 0000000..45c2882 --- /dev/null +++ b/.github/workflows/promote.yaml @@ -0,0 +1,32 @@ +# reusable workflow triggered manually +name: Promote charm to other tracks and channels + +on: + workflow_dispatch: + inputs: + destination-channel: + description: 'Destination Channel' + required: true + origin-channel: + description: 'Origin Channel' + required: true + charm-name: + description: 'Charm subdirectory name' + required: true + +jobs: + promote-charm: + name: Promote charm + runs-on: ubuntu-24.04 + env: + CHARMCRAFT_AUTH: ${{ secrets.CHARMCRAFT_CREDENTIALS }} + steps: + - name: Install charmcraft + run: | + sudo snap install charmcraft --classic --channel latest/stable + - name: Run charmcraft promote + run: | + charmcraft promote --name ${{ github.event.inputs.charm-name }} \ + --from-channel ${{ github.event.inputs.origin-channel }} \ + --to-channel ${{ github.event.inputs.destination-channel }} \ + --yes diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml deleted file mode 100644 index deb9bea..0000000 --- a/.github/workflows/publish.yaml +++ /dev/null @@ -1,101 +0,0 @@ -# reusable workflow for publishing all charms in this repo -name: Publish - -on: - workflow_call: - inputs: - source_branch: - description: Github branch from this repo to publish. If blank, will use the default branch - default: '' - required: false - type: string - secrets: - CHARMCRAFT_CREDENTIALS: - required: true - workflow_dispatch: - inputs: - destination_channel: - description: CharmHub channel to publish to - required: false - default: 'latest/edge' - type: string - source_branch: - description: Github branch from this repo to publish. If blank, will use the default branch - required: false - default: '' - type: string - -jobs: - get-charm-paths: - name: Generate the Charm Matrix - runs-on: ubuntu-24.04 - outputs: - charm_paths_list: ${{ steps.get-charm-paths.outputs.CHARM_PATHS_LIST }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ inputs.source_branch }} - - name: Get paths for all charms in repo - id: get-charm-paths - run: bash .github/workflows/get-charm-paths.sh - - - publish-charm: - name: Publish Charm - runs-on: ubuntu-24.04 - needs: get-charm-paths - strategy: - fail-fast: false - matrix: - charm-path: ${{ fromJson(needs.get-charm-paths.outputs.charm_paths_list) }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ inputs.source_branch }} - - - name: Select charmhub channel - uses: canonical/charming-actions/channel@2.6.2 - id: select-channel - if: ${{ inputs.destination_channel == '' }} - - # Combine inputs from different sources to a single canonical value so later steps don't - # need logic for picking the right one - - name: Parse and combine inputs - id: parse-inputs - run: | - # destination_channel - destination_channel="${{ inputs.destination_channel || steps.select-channel.outputs.name }}" - echo "setting output of destination_channel=$destination_channel" - echo "::set-output name=destination_channel::$destination_channel" - - # tag_prefix - # if charm_path = ./ --> tag_prefix = '' (null) - # if charm_path != ./some-charm (eg: a charm in a ./charms dir) --> tag_prefix = 'some-charm' - if [ ${{ matrix.charm-path }} == './' ]; then - tag_prefix='' - else - tag_prefix=$(basename ${{ matrix.charm-path }} ) - fi - echo "setting output of tag_prefix=$tag_prefix" - echo "::set-output name=tag_prefix::$tag_prefix" - - # Required to charmcraft pack in non-destructive mode - - name: Setup lxd - uses: canonical/setup-lxd@v0.1.2 - with: - channel: latest/stable - - - name: Upload charm to charmhub - uses: canonical/charming-actions/upload-charm@2.6.2 - with: - credentials: ${{ secrets.CHARMCRAFT_CREDENTIALS }} - github-token: ${{ secrets.GITHUB_TOKEN }} - charm-path: ${{ matrix.charm-path }} - channel: ${{ steps.parse-inputs.outputs.destination_channel }} - tag-prefix: ${{ steps.parse-inputs.outputs.tag_prefix }} - charmcraft-channel: 3.x/stable - destructive-mode: false diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9d3752b..069eb56 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,27 +1,33 @@ -# reusable workflow triggered manually -name: Release charm to other tracks and channels +name: Release to Charmhub on: - workflow_dispatch: - inputs: - destination-channel: - description: 'Destination Channel' - required: true - origin-channel: - description: 'Origin Channel' - required: true + push: + branches: + - main + - track/** jobs: - promote-charm: - name: Promote charm - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - name: Release charm to channel - uses: canonical/charming-actions/release-charm@2.6.2 - with: - credentials: ${{ secrets.CHARMCRAFT_CREDENTIALS }} - github-token: ${{ secrets.GITHUB_TOKEN }} - destination-channel: ${{ github.event.inputs.destination-channel }} - origin-channel: ${{ github.event.inputs.origin-channel }} - base-channel: "24.04" + ci-tests: + uses: ./.github/workflows/ci.yaml + secrets: inherit + permissions: + actions: read + contents: read + + release: + strategy: + matrix: + charm: ${{ fromJSON(needs.ci-tests.outputs.charm-paths) }} + name: Release charm | ${{ matrix.charm }} + needs: + - ci-tests + uses: canonical/data-platform-workflows/.github/workflows/release_charm_edge.yaml@v49.0.1 + with: + track: ${{ needs.ci-tests.outputs.track }} + artifact-prefix: ${{ needs.ci-tests.outputs.artifact-prefix }} + path-to-charm-directory: ${{ matrix.charm }} + secrets: + charmhub-token: ${{ secrets.CHARMCRAFT_CREDENTIALS }} + permissions: + actions: read + contents: write # Needed to create git tags diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000..09b9311 --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,11 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +from _pytest.config.argparsing import Parser + + +def pytest_addoption(parser: Parser): + parser.addoption( + "--charm-path", + help="Path to charm file for performing tests on.", + ) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index d54f19e..296dd44 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -35,15 +35,19 @@ def lightkube_client() -> Client: @pytest.mark.abort_on_fail -async def test_build_and_deploy(ops_test: OpsTest): - built_charm_path = await ops_test.build_charm(".") - log.info(f"Built charm {built_charm_path}") +async def test_build_and_deploy(ops_test: OpsTest, request): + entity_url = ( + await ops_test.build_charm(".") + if not (entity_url := request.config.getoption("--charm-path")) + else entity_url + ) + log.info(f"Built charm {entity_url}") image_path = METADATA["resources"]["oci-image"]["upstream-source"] resources = {"oci-image": image_path} await ops_test.model.deploy( - entity_url=built_charm_path, + entity_url=entity_url, resources=resources, trust=True, )