From 18bfa60d0b6cf7865c308c65064802b46d661030 Mon Sep 17 00:00:00 2001 From: Andrew McDermott Date: Wed, 9 Sep 2026 14:21:16 +0100 Subject: [PATCH 1/2] Give the Azure e2e teardown room to wait Azure out Cancelling a job does not cancel what Azure is already doing. Measured on 2026-09-09: killing a Route Server create 45 seconds in left it running server-side, and every delete was refused with AnotherOperationInProgress until it finished, about fifteen minutes later. The teardown itself then took a further seven minutes. That is the case a grace period exists for, and thirty minutes does not cover it. The job would be killed part way through, leaving a Route Server, the public IP allocated to it, the subnet it holds open and an address prefix added to the cluster's own vnet, all of which then block the deprovision. --- .../bgp-cloud-connector/openshift-bgp-cloud-connector-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-operator/config/openshift/bgp-cloud-connector/openshift-bgp-cloud-connector-main.yaml b/ci-operator/config/openshift/bgp-cloud-connector/openshift-bgp-cloud-connector-main.yaml index 797bd20898a9a..ffcd61ba8e5f8 100644 --- a/ci-operator/config/openshift/bgp-cloud-connector/openshift-bgp-cloud-connector-main.yaml +++ b/ci-operator/config/openshift/bgp-cloud-connector/openshift-bgp-cloud-connector-main.yaml @@ -212,7 +212,7 @@ tests: cli: latest commands: hack/ci-e2e-azure.sh from: src - grace_period: 30m0s + grace_period: 1h0m0s resources: requests: cpu: 100m From 10c2429aedcfdd53a010bea132742156da6bc46e Mon Sep 17 00:00:00 2001 From: Andrew McDermott Date: Wed, 9 Sep 2026 15:07:44 +0100 Subject: [PATCH 2/2] Build the Azure e2e job an image that has az in it e2e-azure-operator runs hack/ci-e2e-azure.sh, and those scripts drive Azure through az. The build root carries none, and unlike the aws CLI there is no standalone binary to fetch: az is a Python distribution. So it is installed from Microsoft's own repository into an image built FROM src, which keeps the pull request's checkout and the Go toolchain the suite will need. kata-containers and stolostron/capi-tests both do the same, so this follows what already works rather than inventing. Two details are measured rather than copied across. The build root wraps dnf with ART's wrapper, which ignores /etc/yum.repos.d, so the repository has to be written where the wrapper looks as well or the install fails with "Unable to find a match: azure-cli". And packages-microsoft-prod.rpm is what configures the RHEL 9 repository: the older yumrepos/azure-cli path resolves to azure-cli 2.38 from 2022, where this resolves to 2.90. azure-cli-image proves it, with no cluster and no lease, so rehearsing that one job demonstrates the image in seconds rather than forty minutes into an e2e run. It checks the flags the scripts actually pass rather than a couple of command groups, because a renamed flag is the failure worth catching. Two things the first rehearsal taught, both folded in here. az keeps its configuration under $HOME/.azure and a prow container runs as a random uid that does not own its home, so without AZURE_CONFIG_DIR az dies with "PermissionError: [Errno 13] Permission denied: '/.azure'" before doing anything; hack/azure/ci.sh sets it for the same reason, and that reason is now observed in CI rather than inferred. And piping help into grep -q makes grep exit on its first match, az die of SIGPIPE, and under pipefail the pipeline report 120, so a flag that is present reads as missing -- the help is captured and searched instead. All ten checks pass against az 2.90 in an image built locally from this config. --- .../openshift-bgp-cloud-connector-main.yaml | 77 ++++++++++++++++++- ...t-bgp-cloud-connector-main-presubmits.yaml | 66 ++++++++++++++++ 2 files changed, 142 insertions(+), 1 deletion(-) diff --git a/ci-operator/config/openshift/bgp-cloud-connector/openshift-bgp-cloud-connector-main.yaml b/ci-operator/config/openshift/bgp-cloud-connector/openshift-bgp-cloud-connector-main.yaml index ffcd61ba8e5f8..a230fbb29f725 100644 --- a/ci-operator/config/openshift/bgp-cloud-connector/openshift-bgp-cloud-connector-main.yaml +++ b/ci-operator/config/openshift/bgp-cloud-connector/openshift-bgp-cloud-connector-main.yaml @@ -17,6 +17,29 @@ images: items: - dockerfile_path: Dockerfile to: bgp-cloud-connector + - dockerfile_literal: | + FROM src + # The build root carries no az and there is no standalone binary to + # fetch, so it is installed from Microsoft's own repository, as + # kata-containers and stolostron/capi-tests both do. + # + # The two awkward bits are measured rather than copied. The build + # root wraps dnf with ART's wrapper, which ignores + # /etc/yum.repos.d, so the repository has to be put where the + # wrapper looks as well. And packages-microsoft-prod.rpm is what + # configures the RHEL 9 repository: pointing at the older + # yumrepos/azure-cli path instead yields azure-cli 2.38 from 2022, + # where this yields 2.90. + ENV ART_DNF_WRAPPER_POLICY=append + RUN rpm --import https://packages.microsoft.com/keys/microsoft.asc && \ + dnf install -y https://packages.microsoft.com/config/rhel/9.0/packages-microsoft-prod.rpm && \ + if [ -d /etc/yum.repos.art/ci ]; then \ + cp /etc/yum.repos.d/microsoft-prod.repo /etc/yum.repos.art/ci/ || true; \ + fi && \ + dnf install -y azure-cli && \ + dnf clean all + from: src + to: azure-e2e-runner operator: bundles: - as: bgp-cloud-connector-bundle @@ -58,6 +81,58 @@ tests: container: from: src skip_if_only_changed: ^docs/|\.md$|^(?:.*/)?(?:\.gitignore|OWNERS|PROJECT|LICENSE)$ +- as: azure-cli-image + commands: | + # Proves the image the e2e job runs in actually carries a usable az, + # in seconds and without a cluster. The alternative is finding out + # forty minutes into e2e-azure-operator, after an install has been + # paid for. + set -euo pipefail + + # az keeps its config under $HOME/.azure, and a prow test container + # runs as a random uid whose home it does not own: without this it + # fails with "PermissionError: [Errno 13] Permission denied: + # '/.azure'" before doing anything at all. hack/azure/ci.sh points it + # at the run's scratch directory for the same reason. + AZURE_CONFIG_DIR="$(mktemp -d)" + export AZURE_CONFIG_DIR + + az version + + # The flags the scripts actually pass, rather than just the command + # groups. A release of az that renames one is the failure worth + # catching here, and a group's help would still succeed. + # The help is captured and then searched, rather than piped into + # grep -q. Piped, grep exits on its first match, az dies of SIGPIPE, + # and under pipefail the pipeline reports 120 -- so a flag that is + # present reads as missing. Measured against az 2.90. + check() { + local flag="$1"; shift + local help + if ! help="$(az "$@" --help 2>/dev/null)"; then + echo "MISSING: az $* (its help would not run)" >&2 + return 1 + fi + if grep -q -- "${flag}" <<<"${help}"; then + echo "ok: az $* ${flag}" + else + echo "MISSING: az $* ${flag}" >&2 + return 1 + fi + } + + check --service-principal login + check --subscription account set + check --address-prefixes network vnet update + check --address-prefixes network vnet subnet create + check --sku network public-ip create + check --hosted-subnet network routeserver create + check --public-ip-address network routeserver create + check --peer-ip network routeserver peering create + check --peer-asn network routeserver peering create + container: + from: azure-e2e-runner + skip_if_only_changed: ^docs/|\.md$|^(?:.*/)?(?:\.gitignore|OWNERS|PROJECT|LICENSE)$ - as: fips-image-scan skip_if_only_changed: ^docs/|\.md$|^(?:.*/)?(?:\.gitignore|OWNERS|PROJECT|LICENSE)$ steps: @@ -211,7 +286,7 @@ tests: - as: test cli: latest commands: hack/ci-e2e-azure.sh - from: src + from: azure-e2e-runner grace_period: 1h0m0s resources: requests: diff --git a/ci-operator/jobs/openshift/bgp-cloud-connector/openshift-bgp-cloud-connector-main-presubmits.yaml b/ci-operator/jobs/openshift/bgp-cloud-connector/openshift-bgp-cloud-connector-main-presubmits.yaml index 412f239436d60..119dc3198a959 100644 --- a/ci-operator/jobs/openshift/bgp-cloud-connector/openshift-bgp-cloud-connector-main-presubmits.yaml +++ b/ci-operator/jobs/openshift/bgp-cloud-connector/openshift-bgp-cloud-connector-main-presubmits.yaml @@ -1,5 +1,71 @@ presubmits: openshift/bgp-cloud-connector: + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build10 + context: ci/prow/azure-cli-image + decorate: true + decoration_config: + sparse_checkout_files: + - .ci-operator.yaml + - Dockerfile + labels: + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-bgp-cloud-connector-main-azure-cli-image + rerun_command: /test azure-cli-image + skip_if_only_changed: ^docs/|\.md$|^(?:.*/)?(?:\.gitignore|OWNERS|PROJECT|LICENSE)$ + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --report-credentials-file=/etc/report/credentials + - --target=azure-cli-image + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )azure-cli-image,?($|\s.*) - agent: kubernetes always_run: true branches: