From 83bb06a9c2a9c96fb09879b63cd874894dc779af Mon Sep 17 00:00:00 2001 From: Manos Vlassis <57320708+mvlassis@users.noreply.github.com> Date: Thu, 28 May 2026 12:30:40 +0300 Subject: [PATCH 1/2] chore: update CI to use charmcraftcache --- .github/workflows/ci.yaml | 224 +++++++++++++++++++++++++ .github/workflows/get-charm-paths.sh | 30 ---- .github/workflows/on_pull_request.yaml | 15 +- .github/workflows/on_push.yaml | 54 ------ .github/workflows/promote.yaml | 32 ++++ .github/workflows/publish.yaml | 112 ------------- .github/workflows/release.yaml | 52 +++--- .github/workflows/weekly_ci.yaml | 12 -- 8 files changed, 298 insertions(+), 233 deletions(-) create mode 100644 .github/workflows/ci.yaml delete mode 100644 .github/workflows/get-charm-paths.sh delete mode 100644 .github/workflows/on_push.yaml create mode 100644 .github/workflows/promote.yaml delete mode 100644 .github/workflows/publish.yaml delete mode 100644 .github/workflows/weekly_ci.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..74160975 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,224 @@ +# reusable workflow triggered by other actions +name: CI + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + schedule: + - cron: '0 8 * * TUE' + # Triggered on push by .github/workflows/release.yaml + workflow_call: + outputs: + artifact-prefix: + description: build_charm.yaml `artifact-prefix` output + value: ${{ jobs.build.outputs.artifact-prefix }} + charm-paths: + description: paths for all charms in this repo + value: ${{ jobs.get-charm-paths-track.outputs.charm-paths }} + track: + description: Charmhub track determined from branch name + value: ${{ jobs.get-charm-paths-track.outputs.track }} + +jobs: + get-charm-paths-track: + name: Get charm paths and track + runs-on: ubuntu-latest + outputs: + charm-paths: ${{ steps.get-charm-paths.outputs.charm-paths }} + track: ${{ steps.determine-track.outputs.track }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Get paths for all charms in this repo + id: get-charm-paths + uses: canonical/kubeflow-ci/actions/get-charm-paths@main + - name: Determine track + id: determine-track + shell: python + run: | + import os + + if "${{ github.event_name }}" == "pull_request": + ref = "${{ github.base_ref }}" + else: + ref = "${{ github.ref_name }}" + + if ref.startswith("track/"): + track = ref.removeprefix("track/") + else: + track = "latest" + + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"track={track}\n") + + print(f"Track: {track}") + + lint: + name: Lint + runs-on: ubuntu-24.04 + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Install dependencies + run: pipx install tox + + - name: Lint code + run: tox -vve lint + + unit: + name: Unit + runs-on: ubuntu-24.04 + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Install dependencies + run: pipx install tox + + - name: Run unit tests + run: tox -e unit + + - name: Upload coverage artifacts + uses: actions/upload-artifact@v4 + with: + name: coverage + path: training-operator-cov_html + if: failure() + + terraform-checks: + name: Terraform + uses: canonical/charmed-kubeflow-workflows/.github/workflows/terraform-checks.yaml@main + with: + charm-path: . + + build: + strategy: + matrix: + charm: ${{ fromJSON(needs.get-charm-paths-track.outputs.charm-paths) }} + name: Build charm | ${{ matrix.charm }} + needs: + - get-charm-paths-track + uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v49.0.1 + with: + path-to-charm-directory: ${{ matrix.charm }} + cache: true + charmcraft-snap-channel: 3.x/stable + permissions: + actions: read # Needed for GitHub API call to get workflow version (for private repositories) + contents: read + + release: + strategy: + matrix: + charm: ${{ fromJSON(needs.get-charm-paths-track.outputs.charm-paths) }} + name: Release charm to Charmhub branch | ${{ matrix.charm }} + if: ${{ github.event_name == 'pull_request' }} + needs: + - get-charm-paths-track + - build + uses: canonical/data-platform-workflows/.github/workflows/release_charm_pr.yaml@v49.0.1 + with: + track: ${{ needs.get-charm-paths-track.outputs.track }} + artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} + path-to-charm-directory: ${{ matrix.charm }} + secrets: + charmhub-token: ${{ secrets.CHARMCRAFT_CREDENTIALS }} + permissions: + actions: read # Needed for GitHub API call to get workflow version (for private repositories) + contents: read + + integration: + name: Integration + needs: + - build + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + tox-environment: + - integration + - integration-ambient + - integration-with-profiles + steps: + - name: Maximise GH runner space + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be + + - name: Check out code + uses: actions/checkout@v4 + + - name: Install dependencies + run: pipx install tox + + - name: Setup environment + run: | + sudo apt-get remove -y docker-ce docker-ce-cli containerd.io + sudo rm -rf /run/containerd + sudo snap install concierge --classic + sudo concierge prepare --trace + + - name: Configure Cilium for Canonical K8s + if: matrix.tox-environment == 'integration-ambient' + run: | + # for context, see https://docs.cilium.io/en/stable/network/servicemesh/istio/ + kubectl -n kube-system patch configmap cilium-config --type merge --patch '{"data":{"bpf-lb-sock-hostns-only":"true"}}' + kubectl -n kube-system patch configmap cilium-config --type merge --patch '{"data":{"cni-exclusive":"false"}}' + kubectl -n kube-system rollout restart daemonset cilium + + - name: Fetch charm + uses: actions/download-artifact@v4 + with: + pattern: ${{ needs.build.outputs.artifact-prefix }}-* + merge-multiple: true + path: built/ + + - name: Get charm path + id: charm-path + run: echo "charm_path=$(find built/ -name '*.charm' -type f -print)" >> $GITHUB_OUTPUT + + - name: Run integration tests + run: tox -e ${{ matrix.tox-environment }} -- --model testing --charm-path="${{ steps.charm-path.outputs.charm_path }}" + + - name: Capture k8s resources on failure + run: | + set -eux + kubectl get all -A + kubectl get pods -n testing --show-labels + kubectl get crds + if: failure() + + - name: Get juju status + run: juju status + if: always() + + - name: Get validatingwebhookconfigurations + run: kubectl get validatingwebhookconfigurations validator.training-operator.kubeflow.org -oyaml + if: failure() + + - name: Get secret + run: kubectl get secret -n testing training-operator-webhook-cert -oyaml + if: failure() + + - name: Describe operator pod + run: kubectl describe pod -n testing -l app.kubernetes.io/name=training-operator + if: failure() + + - name: Describe workload pod + run: kubectl describe pod -n testing -l control-plane=testing-training-operator + if: failure() + + - name: Get pods + run: kubectl get pods -A + if: failure() + + - name: Get operator logs + run: kubectl logs --tail 100 -n testing -l app.kubernetes.io/name=training-operator -c charm + if: failure() + + - name: Get workload logs + run: kubectl logs --tail 100 -n testing -l control-plane=testing-training-operator -c training-operator + if: failure() diff --git a/.github/workflows/get-charm-paths.sh b/.github/workflows/get-charm-paths.sh deleted file mode 100644 index 1110d59c..00000000 --- a/.github/workflows/get-charm-paths.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -x - -# Finds the charms in this repo, outputting them as JSON -# Will return one of: -# * the relative paths of the directories listed in `./charms`, if that directory exists -# * "./", if the root directory has a "metadata.yaml" file -# * otherwise, error -# -# Modified from: https://stackoverflow.com/questions/63517732/github-actions-build-matrix-for-lambda-functions/63736071#63736071 -CHARMS_DIR="./charms" -if [ -d "$CHARMS_DIR" ]; -then - CHARM_PATHS=$(find $CHARMS_DIR -maxdepth 1 -type d -not -path '*/\.*' -not -path "$CHARMS_DIR") -else - if [ -f "./metadata.yaml" ] - then - CHARM_PATHS="./" - else - echo "Cannot find valid charm directories - aborting" - exit 1 - fi -fi - -# Convert output to JSON string format -# { charm_paths: [...] } -CHARM_PATHS_LIST=$(echo "$CHARM_PATHS" | jq -c --slurp --raw-input 'split("\n")[:-1]') - -echo "Found CHARM_PATHS_LIST: $CHARM_PATHS_LIST" - -echo "::set-output name=CHARM_PATHS_LIST::$CHARM_PATHS_LIST" diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml index a59378ad..6cc62c10 100644 --- a/.github/workflows/on_pull_request.yaml +++ b/.github/workflows/on_pull_request.yaml @@ -1,13 +1,13 @@ name: On Pull Request # On pull_request, we: -# * always publish to charmhub at latest/edge/branchname -# * always run tests +# * create backport labels if it is against main, only when the PR is opened/reopened on: pull_request: jobs: +<<<<<<< HEAD build-charm: name: Build charm runs-on: ubuntu-24.04 @@ -48,3 +48,14 @@ jobs: uses: ./.github/workflows/publish.yaml secrets: inherit +======= + + populate-labels: + name: Populate labels + if: github.base_ref == 'main' && (github.event.action == 'opened' || github.event.action == 'reopened') + uses: canonical/charmed-kubeflow-workflows/.github/workflows/populate-labels.yaml@main + secrets: inherit + with: + track_file_path: ".github/automatic_backport_tracks.yaml" + label_prefix: "backport " +>>>>>>> f98583d (chore: Update CI to use charmcraftcache (#305)) diff --git a/.github/workflows/on_push.yaml b/.github/workflows/on_push.yaml deleted file mode 100644 index fae1e5a7..00000000 --- a/.github/workflows/on_push.yaml +++ /dev/null @@ -1,54 +0,0 @@ -name: On Push - -# On push to a "special" branch, we: -# * always publish to charmhub at latest/edge/branchname -# * always run tests -# where a "special" branch is one of main or track/**, as -# by convention these branches are the source for a corresponding -# charmhub edge channel. - -on: - push: - branches: - - main - - track/** - -jobs: - build-charm: - name: Build charm - runs-on: ubuntu-24.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup LXD - uses: canonical/setup-lxd@main - with: - channel: 5.21/stable - - - name: Install charmcraft - run: sudo snap install charmcraft --classic - - - name: Build charm under test - run: charmcraft pack --verbose - - - name: Archive charm - uses: actions/upload-artifact@v4 - with: - name: built-charm - path: "*.charm" - retention-days: 5 - - tests: - name: Run Tests - needs: - - build-charm - uses: ./.github/workflows/integrate.yaml - secrets: inherit - - # publish runs in series with tests, and only publishes if tests passes - publish-charm: - name: Publish Charm - needs: tests - uses: ./.github/workflows/publish.yaml - secrets: inherit diff --git a/.github/workflows/promote.yaml b/.github/workflows/promote.yaml new file mode 100644 index 00000000..45c2882d --- /dev/null +++ b/.github/workflows/promote.yaml @@ -0,0 +1,32 @@ +# reusable workflow triggered manually +name: Promote charm to other tracks and channels + +on: + workflow_dispatch: + inputs: + destination-channel: + description: 'Destination Channel' + required: true + origin-channel: + description: 'Origin Channel' + required: true + charm-name: + description: 'Charm subdirectory name' + required: true + +jobs: + promote-charm: + name: Promote charm + runs-on: ubuntu-24.04 + env: + CHARMCRAFT_AUTH: ${{ secrets.CHARMCRAFT_CREDENTIALS }} + steps: + - name: Install charmcraft + run: | + sudo snap install charmcraft --classic --channel latest/stable + - name: Run charmcraft promote + run: | + charmcraft promote --name ${{ github.event.inputs.charm-name }} \ + --from-channel ${{ github.event.inputs.origin-channel }} \ + --to-channel ${{ github.event.inputs.destination-channel }} \ + --yes diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml deleted file mode 100644 index 6a3414fa..00000000 --- a/.github/workflows/publish.yaml +++ /dev/null @@ -1,112 +0,0 @@ -# reusable workflow for publishing all charms in this repo -name: Publish - -on: - workflow_call: - inputs: - source_branch: - description: Github branch from this repo to publish. If blank, will use the default branch - default: '' - required: false - type: string - secrets: - CHARMCRAFT_CREDENTIALS: - required: true - workflow_dispatch: - inputs: - destination_channel: - description: CharmHub channel to publish to - required: false - default: 'latest/edge' - type: string - source_branch: - description: Github branch from this repo to publish. If blank, will use the default branch - required: false - default: '' - type: string - -jobs: - get-charm-paths: - name: Generate the Charm Matrix - runs-on: ubuntu-24.04 - outputs: - charm_paths_list: ${{ steps.get-charm-paths.outputs.CHARM_PATHS_LIST }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ inputs.source_branch }} - - name: Get paths for all charms in repo - id: get-charm-paths - run: bash .github/workflows/get-charm-paths.sh - - - publish-charm: - name: Publish Charm - runs-on: ubuntu-24.04 - needs: get-charm-paths - strategy: - fail-fast: false - matrix: - charm-path: ${{ fromJson(needs.get-charm-paths.outputs.charm_paths_list) }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ inputs.source_branch }} - - - name: Select charmhub channel - uses: canonical/charming-actions/channel@2.6.2 - id: select-channel - if: ${{ inputs.destination_channel == '' }} - - # Combine inputs from different sources to a single canonical value so later steps don't - # need logic for picking the right one - - name: Parse and combine inputs - id: parse-inputs - run: | - # destination_channel - destination_channel="${{ inputs.destination_channel || steps.select-channel.outputs.name }}" - echo "setting output of destination_channel=$destination_channel" - echo "::set-output name=destination_channel::$destination_channel" - - # tag_prefix - # if charm_path = ./ --> tag_prefix = '' (null) - # if charm_path != ./some-charm (eg: a charm in a ./charms dir) --> tag_prefix = 'some-charm' - if [ ${{ matrix.charm-path }} == './' ]; then - tag_prefix='' - else - tag_prefix=$(basename ${{ matrix.charm-path }} ) - fi - echo "setting output of tag_prefix=$tag_prefix" - echo "::set-output name=tag_prefix::$tag_prefix" - - # Required to charmcraft pack in non-destructive mode - - name: Setup lxd - uses: canonical/setup-lxd@v0.1.2 - with: - channel: latest/stable - - - name: Fetch charm - uses: actions/download-artifact@v5 - with: - name: built-charm - path: built/ - - - name: Get charm path - id: charm-path - run: echo "charm_path=$(find built/ -name '*.charm' -type f -print)" >> $GITHUB_OUTPUT - - - name: Upload charm to charmhubpip-tools - uses: canonical/charming-actions/upload-charm@2.6.2 - with: - credentials: ${{ secrets.CHARMCRAFT_CREDENTIALS }} - github-token: ${{ secrets.GITHUB_TOKEN }} - charm-path: ${{ matrix.charm-path }} - built-charm-path: ${{ steps.charm-path.outputs.charm_path }} - channel: ${{ steps.parse-inputs.outputs.destination_channel }} - tag-prefix: ${{ steps.parse-inputs.outputs.tag_prefix }} - charmcraft-channel: 3.x/stable - destructive-mode: false diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9d3752b6..069eb562 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,27 +1,33 @@ -# reusable workflow triggered manually -name: Release charm to other tracks and channels +name: Release to Charmhub on: - workflow_dispatch: - inputs: - destination-channel: - description: 'Destination Channel' - required: true - origin-channel: - description: 'Origin Channel' - required: true + push: + branches: + - main + - track/** jobs: - promote-charm: - name: Promote charm - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - name: Release charm to channel - uses: canonical/charming-actions/release-charm@2.6.2 - with: - credentials: ${{ secrets.CHARMCRAFT_CREDENTIALS }} - github-token: ${{ secrets.GITHUB_TOKEN }} - destination-channel: ${{ github.event.inputs.destination-channel }} - origin-channel: ${{ github.event.inputs.origin-channel }} - base-channel: "24.04" + ci-tests: + uses: ./.github/workflows/ci.yaml + secrets: inherit + permissions: + actions: read + contents: read + + release: + strategy: + matrix: + charm: ${{ fromJSON(needs.ci-tests.outputs.charm-paths) }} + name: Release charm | ${{ matrix.charm }} + needs: + - ci-tests + uses: canonical/data-platform-workflows/.github/workflows/release_charm_edge.yaml@v49.0.1 + with: + track: ${{ needs.ci-tests.outputs.track }} + artifact-prefix: ${{ needs.ci-tests.outputs.artifact-prefix }} + path-to-charm-directory: ${{ matrix.charm }} + secrets: + charmhub-token: ${{ secrets.CHARMCRAFT_CREDENTIALS }} + permissions: + actions: read + contents: write # Needed to create git tags diff --git a/.github/workflows/weekly_ci.yaml b/.github/workflows/weekly_ci.yaml deleted file mode 100644 index 71a3c788..00000000 --- a/.github/workflows/weekly_ci.yaml +++ /dev/null @@ -1,12 +0,0 @@ -name: Run weekly tests - -on: - schedule: - - cron: '0 8 * * TUE' - -jobs: - tests: - name: Run Tests - uses: ./.github/workflows/integrate.yaml - secrets: - charmcraft-credentials: '${{ secrets.CHARMCRAFT_CREDENTIALS }}' From 1427eb489b7b72a1eff98c3c37979284f0524aca Mon Sep 17 00:00:00 2001 From: mvlassis Date: Fri, 29 May 2026 18:30:48 +0300 Subject: [PATCH 2/2] chore: remove ambient integration tests --- .github/workflows/ci.yaml | 9 - .github/workflows/integrate.yaml | 9 - tests/integration/test_charm_ambient.py | 404 ------------------------ 3 files changed, 422 deletions(-) delete mode 100644 tests/integration/test_charm_ambient.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 74160975..f04d068e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -142,7 +142,6 @@ jobs: matrix: tox-environment: - integration - - integration-ambient - integration-with-profiles steps: - name: Maximise GH runner space @@ -161,14 +160,6 @@ jobs: sudo snap install concierge --classic sudo concierge prepare --trace - - name: Configure Cilium for Canonical K8s - if: matrix.tox-environment == 'integration-ambient' - run: | - # for context, see https://docs.cilium.io/en/stable/network/servicemesh/istio/ - kubectl -n kube-system patch configmap cilium-config --type merge --patch '{"data":{"bpf-lb-sock-hostns-only":"true"}}' - kubectl -n kube-system patch configmap cilium-config --type merge --patch '{"data":{"cni-exclusive":"false"}}' - kubectl -n kube-system rollout restart daemonset cilium - - name: Fetch charm uses: actions/download-artifact@v4 with: diff --git a/.github/workflows/integrate.yaml b/.github/workflows/integrate.yaml index 8296adf9..cd65abf9 100644 --- a/.github/workflows/integrate.yaml +++ b/.github/workflows/integrate.yaml @@ -60,7 +60,6 @@ jobs: matrix: tox-environment: - integration - - integration-ambient - integration-with-profiles steps: - name: Maximise GH runner space @@ -88,14 +87,6 @@ jobs: id: charm-path run: echo "charm_path=$(find built/ -name '*.charm' -type f -print)" >> $GITHUB_OUTPUT - - name: Configure Cilium for Canonical K8s - if: matrix.tox-environment == 'integration-ambient' - run: | - # Configure Cilium for Canonical K8s to work with Charmed Istio (Ambient mode) - # See https://canonical-service-mesh-documentation.readthedocs-hosted.com/en/latest/how-to/use-charmed-istio-with-canonical-kubernetes/ - kubectl -n kube-system patch configmap cilium-config --type merge --patch '{"data":{"bpf-lb-sock-hostns-only":"true"}}' - kubectl -n kube-system rollout restart daemonset cilium - - name: Run integration tests run: tox -e ${{ matrix.tox-environment }} -- --model testing --charm-path="${{ steps.charm-path.outputs.charm_path }}" diff --git a/tests/integration/test_charm_ambient.py b/tests/integration/test_charm_ambient.py deleted file mode 100644 index 69623042..00000000 --- a/tests/integration/test_charm_ambient.py +++ /dev/null @@ -1,404 +0,0 @@ -# Copyright 2021 Canonical Ltd. -# See LICENSE file for licensing details. - -import glob -import logging -from pathlib import Path - -import lightkube -import lightkube.codecs -import lightkube.generic_resource -import pytest -import tenacity -import yaml -from charmed_kubeflow_chisme.testing import ( - assert_alert_rules, - assert_metrics_endpoint, - assert_security_context, - deploy_and_assert_grafana_agent, - deploy_and_integrate_service_mesh_charms, - generate_container_securitycontext_map, - get_alert_rules, - get_pod_names, -) -from jinja2 import Template -from lightkube import Client -from lightkube.resources.apiextensions_v1 import CustomResourceDefinition -from lightkube.resources.rbac_authorization_v1 import ClusterRole -from pytest_operator.plugin import OpsTest - -logger = logging.getLogger(__name__) - -METADATA = yaml.safe_load(Path("./metadata.yaml").read_text()) -DEPLOYMENT_FILE = Path("./src/templates/deployment.yaml.j2").read_text() -APP_NAME = METADATA["name"] -CHARM_LOCATION = None -APP_PREVIOUS_CHANNEL = "1.7/stable" -METRICS_PATH = "/metrics" -METRICS_PORT = 8080 -WEBHOOK_TARGET_PORT = "9443" -DEPLOYMENT_YAML = yaml.safe_load( - Template(DEPLOYMENT_FILE).render( - **{ - "app_name": APP_NAME, - "metrics_port": METRICS_PORT, - "webhook_target_port": WEBHOOK_TARGET_PORT, - } - ) -) - - -@pytest.fixture(scope="session") -def lightkube_client() -> Client: - """Returns lightkube Kubernetes client""" - client = Client(field_manager=f"{APP_NAME}") - return client - - -@pytest.mark.abort_on_fail -async def test_build_and_deploy(ops_test: OpsTest): - """Build the charm and deploy it and deploy its dependencies. - - Assert on the unit status. - """ - charm_under_test = await ops_test.build_charm(".") - - await ops_test.model.deploy(charm_under_test, application_name=APP_NAME, trust=True) - await ops_test.model.wait_for_idle( - apps=[APP_NAME], status="active", raise_on_blocked=True, timeout=60 * 10 - ) - assert ops_test.model.applications[APP_NAME].units[0].workload_status == "active" - - # store charm location in global to be used in other tests - global CHARM_LOCATION - CHARM_LOCATION = charm_under_test - - # Deploy grafana-agent for COS integration tests - await deploy_and_assert_grafana_agent(ops_test.model, APP_NAME, metrics=True) - - # Wait for the training-operator workload Pod to run and the operator to start - await ensure_training_operator_is_running(ops_test) - - # Deploy and integrate service mesh charms - await deploy_and_integrate_service_mesh_charms( - APP_NAME, ops_test.model, relate_to_ingress_route_endpoint=False - ) - - -@tenacity.retry( - wait=tenacity.wait_exponential(multiplier=1, min=1, max=30), - stop=tenacity.stop_after_delay(30), - reraise=True, -) -async def ensure_training_operator_is_running(ops_test: OpsTest) -> None: - """Waits until the training-operator workload Pod's status is Running.""" - # The training-operator workload Pod gets a random name, the easiest way - # to wait for it to be ready is using kubectl directly - await ops_test.run( - "kubectl", - "wait", - "--for=condition=ready", - "pod", - "-lapp.kubernetes.io/name=training-operator", - f"-n{ops_test.model_name}", - "--timeout=10m", - check=True, - ) - - _, out, err = await ops_test.run( - "kubectl", - "get", - "pods", - f"-n{ops_test.model_name}", - "--field-selector", - "status.phase!=Running", - check=True, - ) - assert "training-operator" not in out - - -def lightkube_create_global_resources() -> dict: - """Returns a dict with GenericNamespacedResource as value for each CRD key.""" - crds_kinds = [ - crd["spec"]["names"] - for crd in yaml.safe_load_all(Path("./src/templates/crds_manifests.yaml.j2").read_text()) - ] - jobs_classes = {} - for kind in crds_kinds: - job_class = lightkube.generic_resource.create_namespaced_resource( - group="kubeflow.org", version="v1", kind=kind["kind"], plural=kind["plural"] - ) - jobs_classes[kind["kind"]] = job_class - return jobs_classes - - -# TODO: Kubeflow upstream MXNet examples use GPU. -# Not testing MXNetjobs until we have a CPU mxjob example. -JOBS_CLASSES = lightkube_create_global_resources() - - -@pytest.mark.parametrize("example", glob.glob("examples/*.yaml")) -def test_create_training_jobs(ops_test: OpsTest, example: str): - """Validates that a training job can be created and is running. - - Asserts on the *Job status. - """ - namespace = ops_test.model_name - lightkube_client = lightkube.Client() - - # Set up for creating an object of kind *Job - job_yaml = yaml.safe_load(Path(example).read_text()) - job_object = lightkube.codecs.load_all_yaml(yaml.dump(job_yaml))[0] - job_class = JOBS_CLASSES[job_object.kind] - - @tenacity.retry( - wait=tenacity.wait_exponential(multiplier=1, min=1, max=15), - stop=tenacity.stop_after_delay(30), - reraise=True, - ) - def create_training_job(): - """Create the training job. - - Retry if there is an error when creating the Job. - The training-operator may not be ready (even though the Pod shows a Running status, - this retry allows the create command to fail a couple times to allow the operator to - start all validatingwebhooks for each training job type. - """ - # Create *Job and check if it exists where expected - lightkube_client.create(job_object, namespace=namespace) - - # Allow the resource to be created - @tenacity.retry( - wait=tenacity.wait_exponential(multiplier=1, min=1, max=15), - stop=tenacity.stop_after_delay(30), - reraise=True, - ) - def assert_get_job(): - """Asserts on the job. - - Retries multiple times using tenacity to allow time for the training job - to be created. - """ - job = lightkube_client.get(job_class, name=job_object.metadata.name, namespace=namespace) - - assert job is not None, f"{job_object.metadata.name} does not exist" - - # Wait for the *Job to have a status - # TODO: change this workaround after - # we have an implementation in lightkube - @tenacity.retry( - wait=tenacity.wait_exponential(multiplier=2, min=1, max=30), - stop=tenacity.stop_after_attempt(30), - reraise=True, - ) - def assert_job_status_running_success(): - """Asserts on the job status. - - Retries multiple times using tenacity to allow time for the training job - to change its status from None -> Created -> Running/Succeeded. - """ - job_status = lightkube_client.get( - job_class.Status, name=job_object.metadata.name, namespace=namespace - ).status["conditions"][-1]["type"] - - # Check whether the last status of *Job is Running/Success - assert job_status in [ - "Running", - "Succeeded", - ], f"{job_object.metadata.name} was not running or did not succeed (status == {job_status})" - - create_training_job() - assert_get_job() - assert_job_status_running_success() - - -async def test_alert_rules(ops_test: OpsTest): - """Test check charm alert rules and rules defined in relation data bag.""" - app = ops_test.model.applications[APP_NAME] - alert_rules = get_alert_rules() - logger.info("found alert_rules: %s", alert_rules) - await assert_alert_rules(app, alert_rules) - - -async def test_metrics_endpoint(ops_test: OpsTest): - """Test metrics_endpoints are defined in relation data bag and their accessibility. - - This function gets all the metrics_endpoints from the relation data bag, checks if - they are available from the grafana-agent-k8s charm and finally compares them with the - ones provided to the function. - """ - app = ops_test.model.applications[APP_NAME] - # metrics_target should be the same as the one defined in the charm code when instantiating - # the MetricsEndpointProvider. It is set to the training-operator Service name because this - # charm is not a sidecar, once this is re-written in sidecar pattern, this value can be * - await assert_metrics_endpoint(app, metrics_port=METRICS_PORT, metrics_path=METRICS_PATH) - - -def build_pod_container_map(model_name: str, deployment_template: dict) -> dict[str, dict]: - """Build full map of pods:containers belonging to this charm. - - This function builds a custom mapping of security context for pods and containers, - necessary because some pods are not directly spawned by juju but are defined in - `src/templates/deployment.yaml.j2`. - """ - charm_pods: list = get_pod_names(model_name, APP_NAME) - deployment_pods: list = get_pod_names(model_name, f"{APP_NAME}-manager") - deployment_container_name = deployment_template["spec"]["template"]["spec"]["containers"][0][ - "name" - ] - deployment_container_security_context = deployment_template["spec"]["template"]["spec"][ - "containers" - ][0]["securityContext"] - pod_container_map = {} - - for charm_pod in charm_pods: - pod_container_map[charm_pod] = generate_container_securitycontext_map(METADATA) - for pod in deployment_pods: - pod_container_map[pod] = {deployment_container_name: deployment_container_security_context} - return pod_container_map - - -async def test_container_security_context( - ops_test: OpsTest, - lightkube_client: Client, -): - """Test container security context is correctly set. - - Verify that container spec defines the security context with correct - user ID and group ID. - """ - failed_checks = [] - pod_container_map = build_pod_container_map(ops_test.model_name, DEPLOYMENT_YAML) - for pod, pod_containers in pod_container_map.items(): - for container in pod_containers.keys(): - try: - logger.info("Checking security context for container %s (pod: %s)", container, pod) - assert_security_context( - lightkube_client, - pod, - container, - pod_containers, - ops_test.model_name, - ) - except AssertionError as err: - failed_checks.append(f"{pod}/{container}: {err}") - assert failed_checks == [] - - -@pytest.mark.abort_on_fail -async def test_remove_with_resources_present(ops_test: OpsTest): - """Test remove with all resources deployed. - - Verify that all deployed resources that need to be removed are removed. - - This test should be next before before test_upgrade(), because it removes deployed charm. - """ - # remove deployed charm and verify that it is removed - await ops_test.model.remove_application(app_name=APP_NAME, block_until_done=True) - assert APP_NAME not in ops_test.model.applications - - # verify that all resources that were deployed are removed - lightkube_client = lightkube.Client() - crd_list = lightkube_client.list( - CustomResourceDefinition, - labels=[("app.juju.is/created-by", "training-operator")], - namespace=ops_test.model_name, - ) - # testing for empty list (iterator) - _last = object() - assert next(crd_list, _last) is _last - - -@pytest.mark.skip("Due to https://github.com/canonical/training-operator/issues/170") -@pytest.mark.abort_on_fail -async def test_upgrade(ops_test: OpsTest): - """Test upgrade. - - Verify that all upgrade process succeeds. - - There should be no charm with APP_NAME deployed (after test_remove_with_resources_present()), - because it deploys stable version of this charm and performs upgrade. - """ - - # deploy stable version of the charm - await ops_test.model.deploy(entity_url=APP_NAME, channel=APP_PREVIOUS_CHANNEL, trust=True) - await ops_test.model.wait_for_idle( - apps=[APP_NAME], status="active", raise_on_blocked=True, timeout=60 * 10 - ) - - # refresh (upgrade) using charm built in test_build_and_deploy() - # NOTE: using ops_test.juju() because there is no functionality to refresh in ops_test - await ops_test.juju( - "refresh", - APP_NAME, - f"--path={CHARM_LOCATION}", - "--trust", - ) - await ops_test.model.wait_for_idle( - apps=[APP_NAME], status="active", raise_on_blocked=True, timeout=60 * 10 - ) - - # verify that all CRDs are installed - lightkube_client = lightkube.Client() - crd_list = lightkube_client.list( - CustomResourceDefinition, - labels=[("app.juju.is/created-by", "training-operator")], - namespace=ops_test.model_name, - ) - # testing for non empty list (iterator) - _last = object() - assert not next(crd_list, _last) is _last - - # check that all CRDs are installed and versions are correct - test_crd_list = [] - for crd in yaml.safe_load_all(Path("./src/templates/crds_manifests.yaml.j2").read_text()): - test_crd_list.append( - ( - crd["metadata"]["name"], - crd["metadata"]["annotations"]["controller-gen.kubebuilder.io/version"], - ) - ) - for crd in crd_list: - assert ( - (crd.metadata.name, crd.metadata.annotations["controller-gen.kubebuilder.io/version"]) - ) in test_crd_list - - # verify that if ClusterRole is installed and parameters are correct - cluster_role = lightkube_client.get( - ClusterRole, - name=f"{ops_test.model_name}-{APP_NAME}-charm", - namespace=ops_test.model_name, - ) - for rule in cluster_role.rules: - if rule.apiGroups == "kubeflow.org": - assert "paddlejobs" in rule.resources - - -@pytest.mark.skip("Due to https://github.com/canonical/training-operator/issues/170") -@pytest.mark.abort_on_fail -async def test_remove_without_resources(ops_test: OpsTest): - """Test remove when no resources are present. - - Verify that application is removed and not stuck in error state. - - This test should be last in the test suite after test_upgrade(), because it removes deployed - charm. - """ - - # remove all CRDs - lightkube_client = lightkube.Client() - crd_list = lightkube_client.list( - CustomResourceDefinition, - labels=[("app.juju.is/created-by", "training-operator")], - namespace=ops_test.model_name, - ) - for crd in crd_list: - lightkube_client.delete( - CustomResourceDefinition, - name=crd.metadata.name, - namespace=ops_test.model_name, - ) - - # remove deployed charm and verify that it is removed successfully - await ops_test.model.remove_application(app_name=APP_NAME, block_until_done=True) - assert APP_NAME not in ops_test.model.applications