diff --git a/.gitignore b/.gitignore index b2f4c77a..6df2887e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ catalog-renders/ **/catalog.yaml **/catalog.json +__pycache__/ .DS_Store \ No newline at end of file diff --git a/.tekton/integration-tests/pipelines/catalog-argocd-e2e.yaml b/.tekton/integration-tests/pipelines/catalog-argocd-e2e.yaml new file mode 100644 index 00000000..e0619166 --- /dev/null +++ b/.tekton/integration-tests/pipelines/catalog-argocd-e2e.yaml @@ -0,0 +1,386 @@ +--- +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: gitops-catalog-argocd-e2e +spec: + description: | + ArgoCD E2E integration test which provisions an ephemeral cluster, + extracts ArgoCD server image from catalog, deploys ArgoCD standalone, + and runs upstream ArgoCD E2E tests. + params: + - description: Snapshot of the application + name: SNAPSHOT + default: '{"components": [{"name":"catalog-main", "containerImage": "quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/catalog:latest"}]}' + type: string + - description: Git URL of catalog repository (for task definitions) + name: CATALOG_TASK_URL + default: "https://github.com/rh-gitops-midstream/catalog" + type: string + - description: Git revision for task definitions + name: CATALOG_TASK_REVISION + default: "konflux-integration" + type: string + - description: OpenShift version to provision + name: OPENSHIFT_VERSION + default: "4.20" + type: string + - description: Operator channel to query in catalog + name: OPERATOR_CHANNEL + default: "latest" + type: string + - description: Enable FIPS mode for the ephemeral cluster + name: FIPS_ENABLED + default: "false" + type: string + - description: ArgoCD version for upstream manifests + name: ARGOCD_VERSION + default: "v2.14.1" + type: string + - description: Git URL of the ArgoCD test repository + name: TEST_REPO_URL + default: "https://github.com/argoproj/argo-cd.git" + type: string + - description: Git branch or revision of the ArgoCD test repository + name: TEST_REPO_BRANCH + default: "v2.14.1" + type: string + - description: AWS instance type for the ephemeral cluster + name: CLUSTER_INSTANCE_TYPE + default: "m6g.xlarge" + type: string + - description: PR label required to run tests (empty = no gating, always run). When set, push events always proceed but pull_request events require this label on the PR. + name: GATE_LABEL + default: "" + type: string + + finally: + - name: pipeline-wrapup + taskRef: + resolver: git + params: + - name: url + value: $(params.CATALOG_TASK_URL) + - name: revision + value: $(params.CATALOG_TASK_REVISION) + - name: pathInRepo + value: .tekton/tasks/pipeline-wrapup.yaml + params: + - name: eaasSpaceSecretRef + value: $(tasks.provision-eaas-space.results.secretRef) + - name: clusterName + value: "$(tasks.provision-cluster.results.clusterName)" + - name: pipelineRunName + value: "$(context.pipelineRun.name)" + - name: testImageUrl + value: "$(tasks.overlay-test-scripts.results.image-url)" + - name: aggregateStatus + value: "$(tasks.status)" + - name: logUrl + value: "https://konflux-ui.apps.stone-prd-rh01.pg1f.p1.openshiftapps.com/ns/rh-openshift-gitops-tenant/applications/gitops-catalog/pipelineruns/$(context.pipelineRun.name)" + - name: pipelineName + value: "argocd-e2e" + - name: namespace + value: "argocd" + - name: taskNames + value: "extract-argocd-image deploy-argocd test-argocd" + - name: openshiftVersion + value: "$(params.OPENSHIFT_VERSION)" + - name: resolvedOpenshiftVersion + value: "$(tasks.provision-cluster.results.resolvedVersion)" + - name: operatorChannel + value: "$(params.OPERATOR_CHANNEL)" + - name: fipsEnabled + value: "$(params.FIPS_ENABLED)" + - name: argocdVersion + value: "$(params.ARGOCD_VERSION)" + + results: + - name: TEST_OUTPUT + value: $(tasks.test-argocd.results.TEST_OUTPUT) + + tasks: + - name: parse-metadata + taskRef: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/integration-examples + - name: revision + value: main + - name: pathInRepo + value: tasks/test_metadata.yaml + params: + - name: SNAPSHOT + value: $(params.SNAPSHOT) + + - name: check-gate + runAfter: + - parse-metadata + taskSpec: + params: + - name: gate-label + type: string + - name: source-git-url + type: string + - name: source-git-revision + type: string + results: + - name: proceed + description: "true if tests should run, false to skip" + steps: + - name: check + image: registry.access.redhat.com/ubi9/ubi-minimal:latest + env: + - name: GATE_LABEL + value: $(params.gate-label) + - name: SOURCE_GIT_URL + value: $(params.source-git-url) + - name: SOURCE_GIT_REVISION + value: $(params.source-git-revision) + script: | + #!/bin/sh + set -eu + + PROCEED=true + + if [ -z "${GATE_LABEL}" ]; then + echo "No gate label configured, proceeding" + echo -n "${PROCEED}" > $(results.proceed.path) + exit 0 + fi + + echo "Gate label: ${GATE_LABEL}" + + REPO_SLUG="" + if [ -n "${SOURCE_GIT_URL}" ]; then + REPO_SLUG=$(echo "${SOURCE_GIT_URL}" | sed 's|.*github.com/||' | sed 's|\.git$||') + fi + + # Find PR by commit SHA via GitHub API + PR_NUMBER="" + if [ -n "${REPO_SLUG}" ] && [ -n "${SOURCE_GIT_REVISION}" ]; then + echo "Looking up PRs for commit ${SOURCE_GIT_REVISION}..." + COMMIT_PRS=$(curl -sf "https://api.github.com/repos/${REPO_SLUG}/commits/${SOURCE_GIT_REVISION}/pulls" || true) + if [ -n "${COMMIT_PRS}" ] && echo "${COMMIT_PRS}" | grep -q '"number"'; then + PR_NUMBER=$(echo "${COMMIT_PRS}" | grep -o '"number": *[0-9]*' | head -1 | grep -o '[0-9]*' || true) + fi + fi + + if [ -z "${PR_NUMBER}" ]; then + echo "No PR found for this commit, proceeding (push event)" + echo -n "${PROCEED}" > $(results.proceed.path) + exit 0 + fi + + echo "Found PR #${PR_NUMBER}, checking labels..." + LABELS=$(curl -sf "https://api.github.com/repos/${REPO_SLUG}/pulls/${PR_NUMBER}" \ + | grep -o '"name": *"[^"]*"' | sed 's/"name": *"//;s/"$//' || true) + echo "PR labels: ${LABELS:-none}" + + if echo "${LABELS}" | grep -qx "${GATE_LABEL}"; then + echo "Label '${GATE_LABEL}' found, proceeding" + else + echo "Label '${GATE_LABEL}' NOT found, skipping tests" + PROCEED=false + fi + + echo -n "${PROCEED}" > $(results.proceed.path) + echo "Result: proceed=${PROCEED}" + params: + - name: gate-label + value: $(params.GATE_LABEL) + - name: source-git-url + value: $(tasks.parse-metadata.results.source-git-url) + - name: source-git-revision + value: $(tasks.parse-metadata.results.source-git-revision) + + - name: build-test-image + runAfter: + - check-gate + when: + - input: $(tasks.check-gate.results.proceed) + operator: in + values: ["true"] + taskRef: + resolver: git + params: + - name: url + value: $(params.CATALOG_TASK_URL) + - name: revision + value: $(params.CATALOG_TASK_REVISION) + - name: pathInRepo + value: .tekton/tasks/build-ginkgo-test-image.yaml + params: + - name: SOURCE_URL + value: $(params.CATALOG_TASK_URL) + - name: SOURCE_REVISION + value: $(params.CATALOG_TASK_REVISION) + - name: IMAGE_EXPIRES_AFTER + value: "7d" + + - name: overlay-test-scripts + runAfter: + - build-test-image + when: + - input: $(tasks.check-gate.results.proceed) + operator: in + values: ["true"] + taskRef: + resolver: git + params: + - name: url + value: $(params.CATALOG_TASK_URL) + - name: revision + value: $(params.CATALOG_TASK_REVISION) + - name: pathInRepo + value: .tekton/tasks/overlay-test-scripts.yaml + params: + - name: BASE_IMAGE_URL + value: $(tasks.build-test-image.results.IMAGE_URL) + - name: SOURCE_URL + value: $(params.CATALOG_TASK_URL) + - name: SOURCE_REVISION + value: $(params.CATALOG_TASK_REVISION) + + - name: provision-eaas-space + runAfter: + - overlay-test-scripts + when: + - input: $(tasks.check-gate.results.proceed) + operator: in + values: ["true"] + taskRef: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: task/eaas-provision-space/0.1/eaas-provision-space.yaml + params: + - name: ownerName + value: $(context.pipelineRun.name) + - name: ownerUid + value: $(context.pipelineRun.uid) + + - name: provision-cluster + runAfter: + - provision-eaas-space + taskRef: + resolver: git + params: + - name: url + value: $(params.CATALOG_TASK_URL) + - name: revision + value: $(params.CATALOG_TASK_REVISION) + - name: pathInRepo + value: .tekton/tasks/provision-cluster.yaml + params: + - name: eaasSpaceSecretRef + value: $(tasks.provision-eaas-space.results.secretRef) + - name: catalogSourceUrl + value: $(params.CATALOG_TASK_URL) + - name: catalogSourceRevision + value: $(params.CATALOG_TASK_REVISION) + - name: catalogTaskUrl + value: $(params.CATALOG_TASK_URL) + - name: catalogTaskRevision + value: $(params.CATALOG_TASK_REVISION) + - name: openshiftVersion + value: $(params.OPENSHIFT_VERSION) + - name: fipsEnabled + value: $(params.FIPS_ENABLED) + - name: clusterInstanceType + value: $(params.CLUSTER_INSTANCE_TYPE) + + - name: extract-argocd-image + runAfter: + - overlay-test-scripts + taskRef: + resolver: git + params: + - name: url + value: $(params.CATALOG_TASK_URL) + - name: revision + value: $(params.CATALOG_TASK_REVISION) + - name: pathInRepo + value: .tekton/tasks/extract-argocd-image.yaml + params: + - name: catalogImage + value: $(tasks.parse-metadata.results.component-container-image) + - name: operatorChannel + value: $(params.OPERATOR_CHANNEL) + - name: testImageUrl + value: $(tasks.overlay-test-scripts.results.image-url) + - name: pipelineRunName + value: $(context.pipelineRun.name) + + - name: deploy-argocd + runAfter: + - extract-argocd-image + taskRef: + resolver: git + params: + - name: url + value: $(params.CATALOG_TASK_URL) + - name: revision + value: $(params.CATALOG_TASK_REVISION) + - name: pathInRepo + value: .tekton/tasks/deploy-argocd.yaml + params: + - name: argoCDImage + value: $(tasks.extract-argocd-image.results.argoCDImage) + - name: argoCDVersion + value: $(params.ARGOCD_VERSION) + - name: testImageUrl + value: $(tasks.overlay-test-scripts.results.image-url) + - name: eaasSpaceSecretRef + value: $(tasks.provision-eaas-space.results.secretRef) + - name: clusterName + value: $(tasks.provision-cluster.results.clusterName) + - name: pipelineRunName + value: $(context.pipelineRun.name) + + - name: test-argocd + runAfter: + - deploy-argocd + taskRef: + resolver: git + params: + - name: url + value: $(params.CATALOG_TASK_URL) + - name: revision + value: $(params.CATALOG_TASK_REVISION) + - name: pathInRepo + value: .tekton/tasks/test-argocd.yaml + params: + - name: argoCDImage + value: $(tasks.extract-argocd-image.results.argoCDImage) + - name: testRepoUrl + value: $(params.TEST_REPO_URL) + - name: testRepoBranch + value: $(params.TEST_REPO_BRANCH) + - name: testImageUrl + value: $(tasks.overlay-test-scripts.results.image-url) + - name: eaasSpaceSecretRef + value: $(tasks.provision-eaas-space.results.secretRef) + - name: clusterName + value: $(tasks.provision-cluster.results.clusterName) + - name: pipelineRunName + value: $(context.pipelineRun.name) + - name: argoCDNamespace + value: $(tasks.deploy-argocd.results.namespace) + - name: argoCDServer + value: $(tasks.deploy-argocd.results.server) + - name: argoCDAdminPassword + value: $(tasks.deploy-argocd.results.adminPassword) + - name: argoCDServerName + value: $(tasks.deploy-argocd.results.serverName) + - name: argoCDRepoServerName + value: $(tasks.deploy-argocd.results.repoServerName) + - name: argoCDApplicationControllerName + value: $(tasks.deploy-argocd.results.applicationControllerName) + - name: argoCDRedisName + value: $(tasks.deploy-argocd.results.redisName) diff --git a/.tekton/integration-tests/pipelines/catalog-gitops-operator-dast.yaml b/.tekton/integration-tests/pipelines/catalog-gitops-operator-dast.yaml new file mode 100644 index 00000000..6d82021c --- /dev/null +++ b/.tekton/integration-tests/pipelines/catalog-gitops-operator-dast.yaml @@ -0,0 +1,343 @@ +--- +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: gitops-catalog-operator-dast +spec: + description: | + DAST (Dynamic Application Security Testing) pipeline for the GitOps operator. + Provisions an ephemeral HyperShift cluster, installs the GitOps operator from + catalog, then runs RapidAST/ZAP against the deployed ArgoCD REST API. + + Alert thresholds and false-positive suppression rules are configured in + .tekton/test-image/config/dast-false-positives.json in this repo. + params: + - name: SNAPSHOT + description: Snapshot of the application + default: '{"components": [{"name":"gitops-operator-bundle-main", "containerImage": "quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/gitops-operator-bundle:latest"}]}' + type: string + - name: CATALOG_TASK_URL + description: Git URL of catalog repository (used to resolve task definitions) + default: "https://github.com/rh-gitops-midstream/catalog" + type: string + - name: CATALOG_TASK_REVISION + description: Git revision for task definitions + default: "konflux-integration" + type: string + - name: OPENSHIFT_VERSION + description: OpenShift version to provision (e.g. "4.20") + default: "4.20" + type: string + - name: OPERATOR_CHANNEL + description: OLM channel to install the operator from + default: "latest" + type: string + - name: OPERATOR_VERSION + description: Specific operator version to install (empty = latest on channel) + default: "" + type: string + - name: INSTALL_TIMEOUT + description: Duration to wait for operator installation to complete + default: "25m" + type: string + - name: CLUSTER_INSTANCE_TYPE + description: AWS instance type for the ephemeral cluster + default: "m6g.large" + type: string + - name: GATE_LABEL + description: PR label required to run this pipeline (empty = always run) + default: "" + type: string + + finally: + - name: pipeline-wrapup + taskRef: + resolver: git + params: + - name: url + value: $(params.CATALOG_TASK_URL) + - name: revision + value: $(params.CATALOG_TASK_REVISION) + - name: pathInRepo + value: .tekton/tasks/pipeline-wrapup.yaml + params: + - name: eaasSpaceSecretRef + value: $(tasks.provision-eaas-space.results.secretRef) + - name: clusterName + value: "$(tasks.provision-cluster.results.clusterName)" + - name: pipelineRunName + value: "$(context.pipelineRun.name)" + - name: testImageUrl + value: "$(tasks.overlay-test-scripts.results.image-url)" + - name: aggregateStatus + value: "$(tasks.status)" + - name: logUrl + value: "https://konflux-ui.apps.stone-prd-rh01.pg1f.p1.openshiftapps.com/ns/rh-openshift-gitops-tenant/applications/gitops-main/pipelineruns/$(context.pipelineRun.name)" + - name: pipelineName + value: "gitops-operator-dast" + - name: namespace + value: "openshift-gitops-operator" + - name: taskNames + value: "install-operator dast" + - name: openshiftVersion + value: "$(params.OPENSHIFT_VERSION)" + - name: resolvedOpenshiftVersion + value: "$(tasks.provision-cluster.results.resolvedVersion)" + - name: operatorChannel + value: "$(params.OPERATOR_CHANNEL)" + - name: fipsEnabled + value: "false" + - name: installedCSV + value: "$(tasks.install-operator.results.installedCSV)" + - name: testScript + value: "dast-scan" + - name: upgrade + value: "false" + + tasks: + - name: parse-metadata + taskRef: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/integration-examples + - name: revision + value: main + - name: pathInRepo + value: tasks/test_metadata.yaml + params: + - name: SNAPSHOT + value: $(params.SNAPSHOT) + + - name: check-gate + runAfter: + - parse-metadata + taskSpec: + params: + - name: gate-label + type: string + - name: source-git-url + type: string + - name: source-git-revision + type: string + results: + - name: proceed + description: "true if tests should run, false to skip" + steps: + - name: check + image: registry.access.redhat.com/ubi9/ubi-minimal:latest + env: + - name: GATE_LABEL + value: $(params.gate-label) + - name: SOURCE_GIT_URL + value: $(params.source-git-url) + - name: SOURCE_GIT_REVISION + value: $(params.source-git-revision) + script: | + #!/bin/sh + set -eu + + PROCEED=true + + if [ -z "${GATE_LABEL}" ]; then + echo "No gate label configured, proceeding" + echo -n "${PROCEED}" > $(results.proceed.path) + exit 0 + fi + + echo "Gate label: ${GATE_LABEL}" + + REPO_SLUG="" + if [ -n "${SOURCE_GIT_URL}" ]; then + REPO_SLUG=$(echo "${SOURCE_GIT_URL}" | sed 's|.*github.com/||' | sed 's|\.git$||') + fi + + PR_NUMBER="" + if [ -n "${REPO_SLUG}" ] && [ -n "${SOURCE_GIT_REVISION}" ]; then + echo "Looking up PRs for commit ${SOURCE_GIT_REVISION}..." + COMMIT_PRS=$(curl -sf "https://api.github.com/repos/${REPO_SLUG}/commits/${SOURCE_GIT_REVISION}/pulls" || true) + if [ -n "${COMMIT_PRS}" ] && echo "${COMMIT_PRS}" | grep -q '"number"'; then + PR_NUMBER=$(echo "${COMMIT_PRS}" | grep -o '"number": *[0-9]*' | head -1 | grep -o '[0-9]*' || true) + fi + fi + + if [ -z "${PR_NUMBER}" ]; then + echo "No PR found for this commit, proceeding (push event)" + echo -n "${PROCEED}" > $(results.proceed.path) + exit 0 + fi + + echo "Found PR #${PR_NUMBER}, checking labels..." + LABELS=$(curl -sf "https://api.github.com/repos/${REPO_SLUG}/pulls/${PR_NUMBER}" \ + | grep -o '"name": *"[^"]*"' | sed 's/"name": *"//;s/"$//' || true) + echo "PR labels: ${LABELS:-none}" + + if echo "${LABELS}" | grep -qx "${GATE_LABEL}"; then + echo "Label '${GATE_LABEL}' found, proceeding" + else + echo "Label '${GATE_LABEL}' NOT found, skipping tests" + PROCEED=false + fi + + echo -n "${PROCEED}" > $(results.proceed.path) + echo "Result: proceed=${PROCEED}" + params: + - name: gate-label + value: $(params.GATE_LABEL) + - name: source-git-url + value: $(tasks.parse-metadata.results.source-git-url) + - name: source-git-revision + value: $(tasks.parse-metadata.results.source-git-revision) + + - name: build-test-image + runAfter: + - check-gate + when: + - input: $(tasks.check-gate.results.proceed) + operator: in + values: ["true"] + taskRef: + resolver: git + params: + - name: url + value: $(params.CATALOG_TASK_URL) + - name: revision + value: $(params.CATALOG_TASK_REVISION) + - name: pathInRepo + value: .tekton/tasks/build-ginkgo-test-image.yaml + params: + - name: SOURCE_URL + value: $(params.CATALOG_TASK_URL) + - name: SOURCE_REVISION + value: $(params.CATALOG_TASK_REVISION) + - name: IMAGE_EXPIRES_AFTER + value: "7d" + + - name: overlay-test-scripts + runAfter: + - build-test-image + when: + - input: $(tasks.check-gate.results.proceed) + operator: in + values: ["true"] + taskRef: + resolver: git + params: + - name: url + value: $(params.CATALOG_TASK_URL) + - name: revision + value: $(params.CATALOG_TASK_REVISION) + - name: pathInRepo + value: .tekton/tasks/overlay-test-scripts.yaml + params: + - name: BASE_IMAGE_URL + value: $(tasks.build-test-image.results.IMAGE_URL) + - name: SOURCE_URL + value: $(params.CATALOG_TASK_URL) + - name: SOURCE_REVISION + value: $(params.CATALOG_TASK_REVISION) + + - name: provision-eaas-space + runAfter: + - overlay-test-scripts + when: + - input: $(tasks.check-gate.results.proceed) + operator: in + values: ["true"] + taskRef: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: task/eaas-provision-space/0.1/eaas-provision-space.yaml + params: + - name: ownerName + value: $(context.pipelineRun.name) + - name: ownerUid + value: $(context.pipelineRun.uid) + + - name: provision-cluster + runAfter: + - provision-eaas-space + taskRef: + resolver: git + params: + - name: url + value: $(params.CATALOG_TASK_URL) + - name: revision + value: $(params.CATALOG_TASK_REVISION) + - name: pathInRepo + value: .tekton/tasks/provision-cluster.yaml + params: + - name: eaasSpaceSecretRef + value: $(tasks.provision-eaas-space.results.secretRef) + - name: catalogSourceUrl + value: $(tasks.parse-metadata.results.source-git-url) + - name: catalogSourceRevision + value: $(tasks.parse-metadata.results.source-git-revision) + - name: catalogTaskUrl + value: $(params.CATALOG_TASK_URL) + - name: catalogTaskRevision + value: $(params.CATALOG_TASK_REVISION) + - name: openshiftVersion + value: $(params.OPENSHIFT_VERSION) + - name: fipsEnabled + value: "false" + - name: clusterInstanceType + value: $(params.CLUSTER_INSTANCE_TYPE) + + - name: install-operator + runAfter: + - provision-cluster + taskRef: + resolver: git + params: + - name: url + value: $(params.CATALOG_TASK_URL) + - name: revision + value: $(params.CATALOG_TASK_REVISION) + - name: pathInRepo + value: .tekton/tasks/install-operator.yaml + params: + - name: testImageUrl + value: "$(tasks.overlay-test-scripts.results.image-url)" + - name: eaasSpaceSecretRef + value: $(tasks.provision-eaas-space.results.secretRef) + - name: clusterName + value: "$(tasks.provision-cluster.results.clusterName)" + - name: pipelineRunName + value: "$(context.pipelineRun.name)" + - name: openshiftVersion + value: "$(params.OPENSHIFT_VERSION)" + - name: installTimeout + value: "$(params.INSTALL_TIMEOUT)" + - name: operatorChannel + value: "$(params.OPERATOR_CHANNEL)" + - name: operatorVersion + value: "$(params.OPERATOR_VERSION)" + + - name: test-dast + timeout: "1h30m" + runAfter: + - install-operator + taskRef: + resolver: git + params: + - name: url + value: $(params.CATALOG_TASK_URL) + - name: revision + value: $(params.CATALOG_TASK_REVISION) + - name: pathInRepo + value: .tekton/tasks/test-dast.yaml + params: + - name: testImageUrl + value: "$(tasks.overlay-test-scripts.results.image-url)" + - name: eaasSpaceSecretRef + value: $(tasks.provision-eaas-space.results.secretRef) + - name: clusterName + value: "$(tasks.provision-cluster.results.clusterName)" + - name: pipelineRunName + value: "$(context.pipelineRun.name)" diff --git a/.tekton/integration-tests/pipelines/catalog-gitops-operator-e2e.yaml b/.tekton/integration-tests/pipelines/catalog-gitops-operator-e2e.yaml new file mode 100644 index 00000000..3829e84c --- /dev/null +++ b/.tekton/integration-tests/pipelines/catalog-gitops-operator-e2e.yaml @@ -0,0 +1,392 @@ +--- +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: gitops-catalog-operator-e2e +spec: + description: | + An integration test which provisions an ephemeral Hypershift cluster, deploys GitOps Operator from catalog, and runs parallel e2e tests. + params: + - description: Snapshot of the application + name: SNAPSHOT + default: '{"components": [{"name":"gitops-operator-bundle-main", "containerImage": "quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/gitops-operator-bundle:latest"}]}' + type: string + - description: Namespace where the Operator bundle will be deployed. + name: NAMESPACE + default: default + type: string + - description: Duration to wait for bundle installation to complete before failing. + name: INSTALL_TIMEOUT + default: 25m + type: string + - description: Git URL of catalog repository (used to resolve task definitions only; source URL for builds is auto-derived from SNAPSHOT) + name: CATALOG_TASK_URL + default: "https://github.com/rh-gitops-midstream/catalog" + type: string + - description: Git revision for task definitions (used to resolve task definitions only; source revision for builds is auto-derived from SNAPSHOT) + name: CATALOG_TASK_REVISION + default: "konflux-integration" + type: string + - description: OpenShift version to provision (minor version e.g. "4.14", or full patch version e.g. "4.14.57") + name: OPENSHIFT_VERSION + default: "4.14" + type: string + - description: Operator channel to use for installation (e.g., latest, stable). For upgrade testing, set to the pre-upgrade channel. + name: OPERATOR_CHANNEL + default: "latest" + type: string + - description: Enable FIPS mode for the ephemeral cluster + name: FIPS_ENABLED + default: "false" + type: string + - description: Specific GitOps operator version to install (e.g., "1.14.0"). Leave empty to install the latest available on the channel. + name: OPERATOR_VERSION + default: "" + type: string + - description: Enable auto-upgrade on the operator subscription + name: AUTO_UPGRADE + default: "false" + type: string + - description: Perform an upgrade after initial installation by switching to UPGRADE_TO_CHANNEL + name: UPGRADE + default: "false" + type: string + - description: Channel to switch to for upgrade testing (only used when UPGRADE is "true") + name: UPGRADE_TO_CHANNEL + default: "latest" + type: string + - description: Git URL of the test repository + name: TEST_REPO_URL + default: "https://github.com/redhat-developer/gitops-operator.git" + type: string + - description: Git branch or revision of the test repository + name: TEST_REPO_BRANCH + default: "master" + type: string + - description: Test runner script name (e.g., run-parallel-tests.sh, run-sequential-tests.sh, run-rollouts-tests.sh) + name: TEST_SCRIPT + default: "run-parallel-tests.sh" + type: string + - description: AWS instance type for the ephemeral cluster (e.g., m6g.large, m6g.xlarge, m6g.2xlarge) + name: CLUSTER_INSTANCE_TYPE + default: "m6g.large" + type: string + - description: PR label required to run tests (empty = no gating, always run). When set, push events always proceed but pull_request events require this label on the PR. + name: GATE_LABEL + default: "" + type: string + + finally: + - name: pipeline-wrapup + taskRef: + resolver: git + params: + - name: url + value: $(params.CATALOG_TASK_URL) + - name: revision + value: $(params.CATALOG_TASK_REVISION) + - name: pathInRepo + value: .tekton/tasks/pipeline-wrapup.yaml + params: + - name: eaasSpaceSecretRef + value: $(tasks.provision-eaas-space.results.secretRef) + - name: clusterName + value: "$(tasks.provision-cluster.results.clusterName)" + - name: pipelineRunName + value: "$(context.pipelineRun.name)" + - name: testImageUrl + value: "$(tasks.overlay-test-scripts.results.image-url)" + - name: aggregateStatus + value: "$(tasks.status)" + - name: logUrl + value: "https://konflux-ui.apps.stone-prd-rh01.pg1f.p1.openshiftapps.com/ns/rh-openshift-gitops-tenant/applications/gitops-main/pipelineruns/$(context.pipelineRun.name)" + - name: pipelineName + value: "gitops-operator-e2e" + - name: namespace + value: "openshift-gitops-operator" + - name: taskNames + value: "install-operator upgrade-operator test-operator logs" + - name: openshiftVersion + value: "$(params.OPENSHIFT_VERSION)" + - name: resolvedOpenshiftVersion + value: "$(tasks.provision-cluster.results.resolvedVersion)" + - name: operatorChannel + value: "$(params.OPERATOR_CHANNEL)" + - name: fipsEnabled + value: "$(params.FIPS_ENABLED)" + - name: installedCSV + value: "$(tasks.install-operator.results.installedCSV)" + - name: testScript + value: "$(params.TEST_SCRIPT)" + - name: testRepoUrl + value: "$(params.TEST_REPO_URL)" + - name: testRepoBranch + value: "$(params.TEST_REPO_BRANCH)" + - name: upgrade + value: "$(params.UPGRADE)" + + tasks: + - name: parse-metadata + taskRef: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/integration-examples + - name: revision + value: main + - name: pathInRepo + value: tasks/test_metadata.yaml + params: + - name: SNAPSHOT + value: $(params.SNAPSHOT) + + - name: check-gate + runAfter: + - parse-metadata + taskSpec: + params: + - name: gate-label + type: string + - name: source-git-url + type: string + - name: source-git-revision + type: string + results: + - name: proceed + description: "true if tests should run, false to skip" + steps: + - name: check + image: registry.access.redhat.com/ubi9/ubi-minimal:latest + env: + - name: GATE_LABEL + value: $(params.gate-label) + - name: SOURCE_GIT_URL + value: $(params.source-git-url) + - name: SOURCE_GIT_REVISION + value: $(params.source-git-revision) + script: | + #!/bin/sh + set -eu + + PROCEED=true + + if [ -z "${GATE_LABEL}" ]; then + echo "No gate label configured, proceeding" + echo -n "${PROCEED}" > $(results.proceed.path) + exit 0 + fi + + echo "Gate label: ${GATE_LABEL}" + + REPO_SLUG="" + if [ -n "${SOURCE_GIT_URL}" ]; then + REPO_SLUG=$(echo "${SOURCE_GIT_URL}" | sed 's|.*github.com/||' | sed 's|\.git$||') + fi + + # Find PR by commit SHA via GitHub API + PR_NUMBER="" + if [ -n "${REPO_SLUG}" ] && [ -n "${SOURCE_GIT_REVISION}" ]; then + echo "Looking up PRs for commit ${SOURCE_GIT_REVISION}..." + COMMIT_PRS=$(curl -sf "https://api.github.com/repos/${REPO_SLUG}/commits/${SOURCE_GIT_REVISION}/pulls" || true) + if [ -n "${COMMIT_PRS}" ] && echo "${COMMIT_PRS}" | grep -q '"number"'; then + PR_NUMBER=$(echo "${COMMIT_PRS}" | grep -o '"number": *[0-9]*' | head -1 | grep -o '[0-9]*' || true) + fi + fi + + if [ -z "${PR_NUMBER}" ]; then + echo "No PR found for this commit, proceeding (push event)" + echo -n "${PROCEED}" > $(results.proceed.path) + exit 0 + fi + + echo "Found PR #${PR_NUMBER}, checking labels..." + LABELS=$(curl -sf "https://api.github.com/repos/${REPO_SLUG}/pulls/${PR_NUMBER}" \ + | grep -o '"name": *"[^"]*"' | sed 's/"name": *"//;s/"$//' || true) + echo "PR labels: ${LABELS:-none}" + + if echo "${LABELS}" | grep -qx "${GATE_LABEL}"; then + echo "Label '${GATE_LABEL}' found, proceeding" + else + echo "Label '${GATE_LABEL}' NOT found, skipping tests" + PROCEED=false + fi + + echo -n "${PROCEED}" > $(results.proceed.path) + echo "Result: proceed=${PROCEED}" + params: + - name: gate-label + value: $(params.GATE_LABEL) + - name: source-git-url + value: $(tasks.parse-metadata.results.source-git-url) + - name: source-git-revision + value: $(tasks.parse-metadata.results.source-git-revision) + + - name: build-test-image + runAfter: + - check-gate + when: + - input: $(tasks.check-gate.results.proceed) + operator: in + values: ["true"] + taskRef: + resolver: git + params: + - name: url + value: $(params.CATALOG_TASK_URL) + - name: revision + value: $(params.CATALOG_TASK_REVISION) + - name: pathInRepo + value: .tekton/tasks/build-ginkgo-test-image.yaml + params: + - name: SOURCE_URL + value: $(params.CATALOG_TASK_URL) + - name: SOURCE_REVISION + value: $(params.CATALOG_TASK_REVISION) + - name: IMAGE_EXPIRES_AFTER + value: "7d" + + - name: overlay-test-scripts + runAfter: + - build-test-image + when: + - input: $(tasks.check-gate.results.proceed) + operator: in + values: ["true"] + taskRef: + resolver: git + params: + - name: url + value: $(params.CATALOG_TASK_URL) + - name: revision + value: $(params.CATALOG_TASK_REVISION) + - name: pathInRepo + value: .tekton/tasks/overlay-test-scripts.yaml + params: + - name: BASE_IMAGE_URL + value: $(tasks.build-test-image.results.IMAGE_URL) + - name: SOURCE_URL + value: $(params.CATALOG_TASK_URL) + - name: SOURCE_REVISION + value: $(params.CATALOG_TASK_REVISION) + + - name: provision-eaas-space + runAfter: + - overlay-test-scripts + when: + - input: $(tasks.check-gate.results.proceed) + operator: in + values: ["true"] + taskRef: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: task/eaas-provision-space/0.1/eaas-provision-space.yaml + params: + - name: ownerName + value: $(context.pipelineRun.name) + - name: ownerUid + value: $(context.pipelineRun.uid) + + - name: provision-cluster + runAfter: + - provision-eaas-space + taskRef: + resolver: git + params: + - name: url + value: $(params.CATALOG_TASK_URL) + - name: revision + value: $(params.CATALOG_TASK_REVISION) + - name: pathInRepo + value: .tekton/tasks/provision-cluster.yaml + params: + - name: eaasSpaceSecretRef + value: $(tasks.provision-eaas-space.results.secretRef) + - name: catalogSourceUrl + value: $(tasks.parse-metadata.results.source-git-url) + - name: catalogSourceRevision + value: $(tasks.parse-metadata.results.source-git-revision) + - name: catalogTaskUrl + value: $(params.CATALOG_TASK_URL) + - name: catalogTaskRevision + value: $(params.CATALOG_TASK_REVISION) + - name: openshiftVersion + value: $(params.OPENSHIFT_VERSION) + - name: fipsEnabled + value: $(params.FIPS_ENABLED) + - name: clusterInstanceType + value: $(params.CLUSTER_INSTANCE_TYPE) + + - name: install-operator + runAfter: + - provision-cluster + taskRef: + resolver: git + params: + - name: url + value: $(params.CATALOG_TASK_URL) + - name: revision + value: $(params.CATALOG_TASK_REVISION) + - name: pathInRepo + value: .tekton/tasks/install-operator.yaml + params: + - name: testImageUrl + value: "$(tasks.overlay-test-scripts.results.image-url)" + - name: eaasSpaceSecretRef + value: $(tasks.provision-eaas-space.results.secretRef) + - name: clusterName + value: "$(tasks.provision-cluster.results.clusterName)" + - name: pipelineRunName + value: "$(context.pipelineRun.name)" + - name: openshiftVersion + value: "$(params.OPENSHIFT_VERSION)" + - name: installTimeout + value: "$(params.INSTALL_TIMEOUT)" + - name: operatorChannel + value: "$(params.OPERATOR_CHANNEL)" + - name: operatorVersion + value: "$(params.OPERATOR_VERSION)" + - name: autoUpgrade + value: "$(params.AUTO_UPGRADE)" + - name: upgrade + value: "$(params.UPGRADE)" + - name: upgradeToChannel + value: "$(params.UPGRADE_TO_CHANNEL)" + + - name: test-operator + runAfter: + - install-operator + taskRef: + resolver: git + params: + - name: url + value: $(params.CATALOG_TASK_URL) + - name: revision + value: $(params.CATALOG_TASK_REVISION) + - name: pathInRepo + value: .tekton/tasks/test-operator.yaml + params: + - name: testImageUrl + value: "$(tasks.overlay-test-scripts.results.image-url)" + - name: eaasSpaceSecretRef + value: $(tasks.provision-eaas-space.results.secretRef) + - name: clusterName + value: "$(tasks.provision-cluster.results.clusterName)" + - name: pipelineRunName + value: "$(context.pipelineRun.name)" + - name: testRepoUrl + value: "$(params.TEST_REPO_URL)" + - name: testRepoBranch + value: "$(params.TEST_REPO_BRANCH)" + - name: testScript + value: "$(params.TEST_SCRIPT)" + - name: openshiftVersion + value: "$(params.OPENSHIFT_VERSION)" + - name: catalogUrl + value: "$(params.CATALOG_TASK_URL)" + - name: catalogRevision + value: "$(params.CATALOG_TASK_REVISION)" diff --git a/.tekton/integration-tests/scenarios/gitops-argocd-tests.yaml b/.tekton/integration-tests/scenarios/gitops-argocd-tests.yaml new file mode 100644 index 00000000..51f35160 --- /dev/null +++ b/.tekton/integration-tests/scenarios/gitops-argocd-tests.yaml @@ -0,0 +1,66 @@ +--- +apiVersion: appstudio.redhat.com/v1beta2 +kind: IntegrationTestScenario +metadata: + name: gitops-catalog-argocd-e2e + namespace: rh-openshift-gitops-tenant + labels: + test.appstudio.openshift.io/optional: "true" +spec: + application: catalog-4-20 + contexts: + - description: execute on component build + name: component_catalog-4-20 + params: + - name: TEST_IMAGE_URL + value: quay.io/devtools_gitops/test_image:konflux_v1.21.0 + - name: OPENSHIFT_VERSION + value: "4.20" + - name: OPERATOR_CHANNEL + value: latest + - name: GATE_LABEL + value: rc-argocd-check + resolverRef: + resolver: git + resourceKind: pipeline + params: + - name: url + value: https://github.com/rh-gitops-midstream/catalog + - name: revision + value: konflux-integration + - name: pathInRepo + value: .tekton/integration-tests/pipelines/catalog-argocd-e2e.yaml +--- +apiVersion: appstudio.redhat.com/v1beta2 +kind: IntegrationTestScenario +metadata: + name: gitops-catalog-argocd-e2e-fips + namespace: rh-openshift-gitops-tenant + labels: + test.appstudio.openshift.io/optional: "true" +spec: + application: catalog-4-20 + contexts: + - description: execute on component build + name: component_catalog-4-20 + params: + - name: TEST_IMAGE_URL + value: quay.io/devtools_gitops/test_image:konflux_v1.21.0 + - name: OPENSHIFT_VERSION + value: "4.20" + - name: OPERATOR_CHANNEL + value: latest + - name: FIPS_ENABLED + value: "true" + - name: GATE_LABEL + value: rc-argocd-check + resolverRef: + resolver: git + resourceKind: pipeline + params: + - name: url + value: https://github.com/rh-gitops-midstream/catalog + - name: revision + value: konflux-integration + - name: pathInRepo + value: .tekton/integration-tests/pipelines/catalog-argocd-e2e.yaml diff --git a/.tekton/integration-tests/scenarios/gitops-dast.yaml b/.tekton/integration-tests/scenarios/gitops-dast.yaml new file mode 100644 index 00000000..79b32983 --- /dev/null +++ b/.tekton/integration-tests/scenarios/gitops-dast.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: appstudio.redhat.com/v1beta2 +kind: IntegrationTestScenario +metadata: + name: gitops-catalog-operator-dast + namespace: rh-openshift-gitops-tenant + labels: + test.appstudio.openshift.io/optional: "true" +spec: + application: catalog-4-20 + contexts: + - description: execute on component build + name: component_catalog-4-20 + params: + - name: OPENSHIFT_VERSION + value: "4.20" + - name: OPERATOR_CHANNEL + value: latest + - name: GATE_LABEL + value: run-dast + resolverRef: + resolver: git + resourceKind: pipeline + params: + - name: url + value: https://github.com/rh-gitops-midstream/catalog + - name: revision + value: konflux-integration + - name: pathInRepo + value: .tekton/integration-tests/pipelines/catalog-gitops-operator-dast.yaml diff --git a/.tekton/integration-tests/scenarios/gitops-operator-tests.yaml b/.tekton/integration-tests/scenarios/gitops-operator-tests.yaml new file mode 100644 index 00000000..6ef79dc9 --- /dev/null +++ b/.tekton/integration-tests/scenarios/gitops-operator-tests.yaml @@ -0,0 +1,248 @@ +--- +apiVersion: appstudio.redhat.com/v1beta2 +kind: IntegrationTestScenario +metadata: + name: gitops-catalog-operator-parallel + namespace: rh-openshift-gitops-tenant + labels: + test.appstudio.openshift.io/optional: "true" +spec: + application: catalog-4-20 + contexts: + - description: execute on component build + name: component_catalog-4-20 + params: + - name: TEST_IMAGE_URL + value: quay.io/devtools_gitops/test_image:konflux_v1.21.0 + - name: TEST_SCRIPT + value: run-parallel-tests.sh + - name: OPENSHIFT_VERSION + value: "4.20" + - name: OPERATOR_CHANNEL + value: latest + - name: GATE_LABEL + value: rc-operator-check + resolverRef: + resolver: git + resourceKind: pipeline + params: + - name: url + value: https://github.com/rh-gitops-midstream/catalog + - name: revision + value: konflux-integration + - name: pathInRepo + value: .tekton/integration-tests/pipelines/catalog-gitops-operator-e2e.yaml +--- +apiVersion: appstudio.redhat.com/v1beta2 +kind: IntegrationTestScenario +metadata: + name: gitops-catalog-operator-parallel-fips + namespace: rh-openshift-gitops-tenant + labels: + test.appstudio.openshift.io/optional: "true" +spec: + application: catalog-4-20 + contexts: + - description: execute on component build + name: component_catalog-4-20 + params: + - name: TEST_IMAGE_URL + value: quay.io/devtools_gitops/test_image:konflux_v1.21.0 + - name: TEST_SCRIPT + value: run-parallel-tests.sh + - name: OPENSHIFT_VERSION + value: "4.20" + - name: OPERATOR_CHANNEL + value: latest + - name: FIPS_ENABLED + value: "true" + - name: GATE_LABEL + value: rc-operator-check + resolverRef: + resolver: git + resourceKind: pipeline + params: + - name: url + value: https://github.com/rh-gitops-midstream/catalog + - name: revision + value: konflux-integration + - name: pathInRepo + value: .tekton/integration-tests/pipelines/catalog-gitops-operator-e2e.yaml +--- +apiVersion: appstudio.redhat.com/v1beta2 +kind: IntegrationTestScenario +metadata: + name: gitops-catalog-operator-sequential-s1 + namespace: rh-openshift-gitops-tenant + labels: + test.appstudio.openshift.io/optional: "true" +spec: + application: catalog-4-20 + contexts: + - description: execute on component build + name: component_catalog-4-20 + params: + - name: TEST_IMAGE_URL + value: quay.io/devtools_gitops/test_image:konflux_v1.21.0 + - name: TEST_SCRIPT + value: run-sequential-tests-shard1.sh + - name: OPENSHIFT_VERSION + value: "4.20" + - name: OPERATOR_CHANNEL + value: latest + - name: GATE_LABEL + value: rc-operator-check + resolverRef: + resolver: git + resourceKind: pipeline + params: + - name: url + value: https://github.com/rh-gitops-midstream/catalog + - name: revision + value: konflux-integration + - name: pathInRepo + value: .tekton/integration-tests/pipelines/catalog-gitops-operator-e2e.yaml +--- +apiVersion: appstudio.redhat.com/v1beta2 +kind: IntegrationTestScenario +metadata: + name: gitops-catalog-operator-sequential-s2 + namespace: rh-openshift-gitops-tenant + labels: + test.appstudio.openshift.io/optional: "true" +spec: + application: catalog-4-20 + contexts: + - description: execute on component build + name: component_catalog-4-20 + params: + - name: TEST_IMAGE_URL + value: quay.io/devtools_gitops/test_image:konflux_v1.21.0 + - name: TEST_SCRIPT + value: run-sequential-tests-shard2.sh + - name: OPENSHIFT_VERSION + value: "4.20" + - name: OPERATOR_CHANNEL + value: latest + - name: GATE_LABEL + value: rc-operator-check + resolverRef: + resolver: git + resourceKind: pipeline + params: + - name: url + value: https://github.com/rh-gitops-midstream/catalog + - name: revision + value: konflux-integration + - name: pathInRepo + value: .tekton/integration-tests/pipelines/catalog-gitops-operator-e2e.yaml +--- +apiVersion: appstudio.redhat.com/v1beta2 +kind: IntegrationTestScenario +metadata: + name: gitops-catalog-operator-rollouts + namespace: rh-openshift-gitops-tenant + labels: + test.appstudio.openshift.io/optional: "true" +spec: + application: catalog-4-20 + contexts: + - description: execute on component build + name: component_catalog-4-20 + params: + - name: TEST_IMAGE_URL + value: quay.io/devtools_gitops/test_image:konflux_v1.21.0 + - name: TEST_SCRIPT + value: run-rollouts-tests.sh + - name: OPENSHIFT_VERSION + value: "4.20" + - name: OPERATOR_CHANNEL + value: latest + - name: GATE_LABEL + value: rc-operator-check + resolverRef: + resolver: git + resourceKind: pipeline + params: + - name: url + value: https://github.com/rh-gitops-midstream/catalog + - name: revision + value: konflux-integration + - name: pathInRepo + value: .tekton/integration-tests/pipelines/catalog-gitops-operator-e2e.yaml +--- +apiVersion: appstudio.redhat.com/v1beta2 +kind: IntegrationTestScenario +metadata: + name: gitops-catalog-operator-parallel-upgrade + namespace: rh-openshift-gitops-tenant + labels: + test.appstudio.openshift.io/optional: "true" +spec: + application: catalog-4-20 + contexts: + - description: execute on component build + name: component_catalog-4-20 + params: + - name: TEST_IMAGE_URL + value: quay.io/devtools_gitops/test_image:konflux_v1.21.0 + - name: TEST_SCRIPT + value: run-parallel-tests.sh + - name: OPENSHIFT_VERSION + value: "4.20" + - name: OPERATOR_CHANNEL + value: gitops-1.20 + - name: UPGRADE_TO_CHANNEL + value: latest + - name: UPGRADE + value: "true" + - name: GATE_LABEL + value: rc-operator-check + resolverRef: + resolver: git + resourceKind: pipeline + params: + - name: url + value: https://github.com/rh-gitops-midstream/catalog + - name: revision + value: konflux-integration + - name: pathInRepo + value: .tekton/integration-tests/pipelines/catalog-gitops-operator-e2e.yaml +--- +apiVersion: appstudio.redhat.com/v1beta2 +kind: IntegrationTestScenario +metadata: + name: gitops-catalog-operator-sequential-s1-upgrade + namespace: rh-openshift-gitops-tenant + labels: + test.appstudio.openshift.io/optional: "true" +spec: + application: catalog-4-20 + contexts: + - description: execute on component build + name: component_catalog-4-20 + params: + - name: TEST_IMAGE_URL + value: quay.io/devtools_gitops/test_image:konflux_v1.21.0 + - name: TEST_SCRIPT + value: run-sequential-tests-shard1.sh + - name: OPENSHIFT_VERSION + value: "4.20" + - name: OPERATOR_CHANNEL + value: gitops-1.20 + - name: UPGRADE_TO_CHANNEL + value: latest + - name: UPGRADE + value: "true" + - name: GATE_LABEL + value: rc-operator-check + resolverRef: + resolver: git + resourceKind: pipeline + params: + - name: url + value: https://github.com/rh-gitops-midstream/catalog + - name: revision + value: konflux-integration + - name: pathInRepo + value: .tekton/integration-tests/pipelines/catalog-gitops-operator-e2e.yaml diff --git a/.tekton/integration-tests/scenarios/gitops-sanity-tests.yaml b/.tekton/integration-tests/scenarios/gitops-sanity-tests.yaml new file mode 100644 index 00000000..96efcd98 --- /dev/null +++ b/.tekton/integration-tests/scenarios/gitops-sanity-tests.yaml @@ -0,0 +1,148 @@ +--- +apiVersion: appstudio.redhat.com/v1beta2 +kind: IntegrationTestScenario +metadata: + name: gitops-catalog-sanity + namespace: rh-openshift-gitops-tenant + labels: + test.appstudio.openshift.io/optional: "true" +spec: + application: catalog-4-20 + contexts: + - description: execute on component build + name: component_catalog-4-20 + params: + - name: TEST_IMAGE_URL + value: quay.io/devtools_gitops/test_image:konflux_v1.21.0 + - name: TEST_SCRIPT + value: run-sanity-tests.sh + - name: OPENSHIFT_VERSION + value: "4.20" + - name: OPERATOR_CHANNEL + value: latest + - name: GATE_LABEL + value: rc-sanity-check + resolverRef: + resolver: git + resourceKind: pipeline + params: + - name: url + value: https://github.com/rh-gitops-midstream/catalog + - name: revision + value: konflux-integration + - name: pathInRepo + value: .tekton/integration-tests/pipelines/catalog-gitops-operator-e2e.yaml +--- +apiVersion: appstudio.redhat.com/v1beta2 +kind: IntegrationTestScenario +metadata: + name: gitops-catalog-sanity-fips + namespace: rh-openshift-gitops-tenant + labels: + test.appstudio.openshift.io/optional: "true" +spec: + application: catalog-4-20 + contexts: + - description: execute on component build + name: component_catalog-4-20 + params: + - name: TEST_IMAGE_URL + value: quay.io/devtools_gitops/test_image:konflux_v1.21.0 + - name: TEST_SCRIPT + value: run-sanity-tests.sh + - name: OPENSHIFT_VERSION + value: "4.20" + - name: OPERATOR_CHANNEL + value: latest + - name: FIPS_ENABLED + value: "true" + - name: GATE_LABEL + value: rc-sanity-check + resolverRef: + resolver: git + resourceKind: pipeline + params: + - name: url + value: https://github.com/rh-gitops-midstream/catalog + - name: revision + value: konflux-integration + - name: pathInRepo + value: .tekton/integration-tests/pipelines/catalog-gitops-operator-e2e.yaml +--- +apiVersion: appstudio.redhat.com/v1beta2 +kind: IntegrationTestScenario +metadata: + name: gitops-catalog-sanity-upgrade + namespace: rh-openshift-gitops-tenant + labels: + test.appstudio.openshift.io/optional: "true" +spec: + application: catalog-4-20 + contexts: + - description: execute on component build + name: component_catalog-4-20 + params: + - name: TEST_IMAGE_URL + value: quay.io/devtools_gitops/test_image:konflux_v1.21.0 + - name: TEST_SCRIPT + value: run-sanity-tests.sh + - name: OPENSHIFT_VERSION + value: "4.20" + - name: OPERATOR_CHANNEL + value: gitops-1.20 + - name: UPGRADE_TO_CHANNEL + value: latest + - name: UPGRADE + value: "true" + - name: GATE_LABEL + value: rc-sanity-check + resolverRef: + resolver: git + resourceKind: pipeline + params: + - name: url + value: https://github.com/rh-gitops-midstream/catalog + - name: revision + value: konflux-integration + - name: pathInRepo + value: .tekton/integration-tests/pipelines/catalog-gitops-operator-e2e.yaml +--- +apiVersion: appstudio.redhat.com/v1beta2 +kind: IntegrationTestScenario +metadata: + name: gitops-catalog-sanity-upgrade-fips + namespace: rh-openshift-gitops-tenant + labels: + test.appstudio.openshift.io/optional: "true" +spec: + application: catalog-4-20 + contexts: + - description: execute on component build + name: component_catalog-4-20 + params: + - name: TEST_IMAGE_URL + value: quay.io/devtools_gitops/test_image:konflux_v1.21.0 + - name: TEST_SCRIPT + value: run-sanity-tests.sh + - name: OPENSHIFT_VERSION + value: "4.20" + - name: OPERATOR_CHANNEL + value: gitops-1.20 + - name: UPGRADE_TO_CHANNEL + value: latest + - name: UPGRADE + value: "true" + - name: FIPS_ENABLED + value: "true" + - name: GATE_LABEL + value: rc-sanity-check + resolverRef: + resolver: git + resourceKind: pipeline + params: + - name: url + value: https://github.com/rh-gitops-midstream/catalog + - name: revision + value: konflux-integration + - name: pathInRepo + value: .tekton/integration-tests/pipelines/catalog-gitops-operator-e2e.yaml diff --git a/.tekton/integration-tests/scenarios/gitops-ui-tests.yaml b/.tekton/integration-tests/scenarios/gitops-ui-tests.yaml new file mode 100644 index 00000000..05461958 --- /dev/null +++ b/.tekton/integration-tests/scenarios/gitops-ui-tests.yaml @@ -0,0 +1,34 @@ +--- +apiVersion: appstudio.redhat.com/v1beta2 +kind: IntegrationTestScenario +metadata: + name: gitops-catalog-ui-e2e + namespace: rh-openshift-gitops-tenant + labels: + test.appstudio.openshift.io/optional: "true" +spec: + application: catalog-4-20 + contexts: + - description: execute on component build + name: component_catalog-4-20 + params: + - name: TEST_IMAGE_URL + value: quay.io/devtools_gitops/test_image:konflux_v1.21.0 + - name: TEST_SCRIPT + value: run-ui-e2e-tests.sh + - name: OPENSHIFT_VERSION + value: "4.20" + - name: OPERATOR_CHANNEL + value: latest + - name: GATE_LABEL + value: rc-ui-check + resolverRef: + resolver: git + resourceKind: pipeline + params: + - name: url + value: https://github.com/rh-gitops-midstream/catalog + - name: revision + value: konflux-integration + - name: pathInRepo + value: .tekton/integration-tests/pipelines/catalog-gitops-operator-e2e.yaml diff --git a/.tekton/stepactions/extract-image-content-sources.yaml b/.tekton/stepactions/extract-image-content-sources.yaml new file mode 100644 index 00000000..0291dd9b --- /dev/null +++ b/.tekton/stepactions/extract-image-content-sources.yaml @@ -0,0 +1,49 @@ +apiVersion: tekton.dev/v1beta1 +kind: StepAction +metadata: + name: extract-image-content-sources +spec: + params: + - name: gitUrl + type: string + description: Git repository URL containing the images-mirror-set.yaml file + - name: gitRevision + type: string + description: Git revision (branch, tag, or commit SHA) to checkout + - name: mirrorSetPath + type: string + description: Path to the ImageDigestMirrorSet YAML file within the repository + default: .tekton/images-mirror-set.yaml + - name: outputPath + type: string + description: Path where the JSON output should be written + default: /workspace/imageContentSources.json + env: + - name: GIT_URL + value: $(params.gitUrl) + - name: GIT_REVISION + value: $(params.gitRevision) + - name: MIRROR_SET_PATH + value: $(params.mirrorSetPath) + - name: OUTPUT_PATH + value: $(params.outputPath) + image: docker.io/python:3-alpine + script: | + #!/bin/sh + set -e + apk add --no-cache git >/dev/null 2>&1 + pip install pyyaml --quiet >/dev/null 2>&1 + + WORK=$(mktemp -d) + git clone "$GIT_URL" "$WORK/repo" + cd "$WORK/repo" + git checkout "$GIT_REVISION" + + python3 -c " + import yaml, json, sys + with open('$MIRROR_SET_PATH') as f: + data = yaml.safe_load(f) + json.dump(data['spec']['imageDigestMirrors'], sys.stdout) + " > "$OUTPUT_PATH" + + echo "Successfully extracted imageContentSources from $GIT_URL" diff --git a/.tekton/stepactions/resolve-openshift-version.yaml b/.tekton/stepactions/resolve-openshift-version.yaml new file mode 100644 index 00000000..2e56d0af --- /dev/null +++ b/.tekton/stepactions/resolve-openshift-version.yaml @@ -0,0 +1,52 @@ +apiVersion: tekton.dev/v1beta1 +kind: StepAction +metadata: + name: resolve-openshift-version +spec: + params: + - name: version + type: string + description: OpenShift version (e.g., "4.17" or "4.17.5") + results: + - name: resolvedVersion + description: Resolved OpenShift version (full x.y.z format) + env: + - name: OPENSHIFT_VERSION + value: $(params.version) + - name: RESULT_PATH + value: $(step.results.resolvedVersion.path) + image: docker.io/python:3-alpine + script: | + #!/usr/bin/env python3 + import urllib.request, json, re, os + + version = os.environ["OPENSHIFT_VERSION"] + result_path = os.environ["RESULT_PATH"] + + if re.match(r'^\d+\.\d+\.\d+$', version): + with open(result_path, "w") as f: + f.write(version) + print(f"Using exact version: {version}") + raise SystemExit(0) + + url = f"https://quay.io/api/v1/repository/openshift-release-dev/ocp-release/tag/?filter_tag_name=like:{version}.&limit=100&onlyActiveTags=true" + data = json.loads(urllib.request.urlopen(url).read()) + + candidates = [] + for tag in data.get("tags", []): + name = tag["name"] + if name.endswith("-multi"): + bare = name.removesuffix("-multi") + if re.match(rf'^{re.escape(version)}\.\d+$', bare): + candidates.append(bare) + + if not candidates: + print(f"ERROR: Could not resolve minor version {version} to a patch release") + raise SystemExit(1) + + candidates.sort(key=lambda v: list(map(int, v.split(".")))) + latest = candidates[-1] + + with open(result_path, "w") as f: + f.write(latest) + print(f"Resolved {version} -> {latest}") diff --git a/.tekton/tasks/build-ginkgo-test-image.yaml b/.tekton/tasks/build-ginkgo-test-image.yaml new file mode 100644 index 00000000..9734de4f --- /dev/null +++ b/.tekton/tasks/build-ginkgo-test-image.yaml @@ -0,0 +1,240 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: build-ginkgo-test-image +spec: + description: | + Builds the gitops-ginkgo-test-runner image in three layers: + 1. Base image (Dockerfile.base-*) - CLI tools and Go toolchain + 2. Testsuites image (Dockerfile.testsuites) - pre-compiled Ginkgo tests + 3. Final image (Dockerfile) - helper scripts, rebuilt per commit + + Each layer is tagged by a content hash and skipped if already present. + Use BASE_DOCKERFILE to select the correct base for the operator version under test. + params: + - name: SOURCE_URL + type: string + description: Git URL of the catalog repo (from pipeline context) + - name: SOURCE_REVISION + type: string + description: Git commit SHA - used for tagging + - name: IMAGE_DESTINATION + type: string + default: "quay.io/devtools_gitops/test_image" + description: Base image repository (without tag) + - name: DOCKERFILE_PATH + type: string + default: ".tekton/test-image/Dockerfile" + - name: CONTEXT_DIR + type: string + default: ".tekton/test-image" + - name: BASE_DOCKERFILE + type: string + default: "Dockerfile.base-v1.21" + description: "Base Dockerfile name (e.g., Dockerfile.base-v1.21). Different operator versions may need different Go toolchains." + - name: IMAGE_EXPIRES_AFTER + type: string + default: "7d" + description: "Quay expiration label (e.g., 1h, 7d, never)" + results: + - name: IMAGE_DIGEST + description: Digest of the built image + - name: IMAGE_URL + description: Full image reference with tag (repo:tag) + - name: IMAGE_TAG + description: Just the tag portion (short SHA) + volumes: + - name: source + emptyDir: {} + - name: docker-credentials + secret: + secretName: gitops-test-runner-image-push + - name: cache-state + emptyDir: {} + steps: + - name: clone-source + image: docker.io/alpine/git:latest + volumeMounts: + - name: source + mountPath: /workspace/source + script: | + #!/bin/sh + set -e + echo "Cloning $(params.SOURCE_URL) @ $(params.SOURCE_REVISION)" + git clone "$(params.SOURCE_URL)" /workspace/source + cd /workspace/source + git checkout "$(params.SOURCE_REVISION)" + + COMMIT_SHA=$(git rev-parse HEAD) + echo "Commit SHA: ${COMMIT_SHA}" + git log -1 --oneline + echo -n "${COMMIT_SHA}" > /workspace/source/COMMIT_SHA + + - name: check-cache + image: quay.io/skopeo/stable:latest + volumeMounts: + - name: source + mountPath: /workspace/source + - name: docker-credentials + mountPath: /credentials + - name: cache-state + mountPath: /cache-state + script: | + #!/bin/bash + set -euo pipefail + + COMMIT_SHA=$(cat /workspace/source/COMMIT_SHA) + SHORT_SHA=$(echo "${COMMIT_SHA}" | cut -c1-8) + CONTEXT="/workspace/source/$(params.CONTEXT_DIR)" + AUTH="--authfile=/credentials/.dockerconfigjson" + BUILD_ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') + + BASE_HASH=$(sha256sum "${CONTEXT}/$(params.BASE_DOCKERFILE)" | cut -c1-12) + BASE_IMAGE="$(params.IMAGE_DESTINATION):base-${BUILD_ARCH}-${BASE_HASH}" + TESTSUITES_HASH=$(cat "${CONTEXT}/$(params.BASE_DOCKERFILE)" "${CONTEXT}/Dockerfile.testsuites" | sha256sum | cut -c1-12) + TESTSUITES_IMAGE="$(params.IMAGE_DESTINATION):testsuites-${BUILD_ARCH}-${TESTSUITES_HASH}" + + # Final image tag includes scripts content hash to ensure rebuilds when scripts change + SCRIPTS_HASH=$(find "${CONTEXT}/scripts" -type f -exec sha256sum {} \; | sort | sha256sum | cut -c1-12) + FINAL_HASH=$(echo "${TESTSUITES_HASH}${SCRIPTS_HASH}" | sha256sum | cut -c1-12) + DESTINATION="$(params.IMAGE_DESTINATION):final-${BUILD_ARCH}-${FINAL_HASH}" + + echo "Cache check — base: ${BASE_IMAGE}" + echo "Cache check — testsuites: ${TESTSUITES_IMAGE}" + echo "Final image tag: ${DESTINATION}" + + # Note: We do NOT cache-check the final image - it's always rebuilt to pick up script changes + echo "miss" > /cache-state/final + + # Check base image + if skopeo inspect ${AUTH} "docker://${BASE_IMAGE}" >/dev/null 2>&1; then + echo "Base image exists — will skip base build" + echo "hit" > /cache-state/base + else + echo "Base image not found — will rebuild" + echo "miss" > /cache-state/base + fi + + # Check testsuites image + if skopeo inspect ${AUTH} "docker://${TESTSUITES_IMAGE}" >/dev/null 2>&1; then + echo "Testsuites image exists — will skip testsuites build" + echo "hit" > /cache-state/testsuites + else + echo "Testsuites image not found — will rebuild" + echo "miss" > /cache-state/testsuites + fi + + - name: build-and-push + image: quay.io/buildah/stable:latest + workingDir: /workspace/source + volumeMounts: + - name: source + mountPath: /workspace/source + - name: docker-credentials + mountPath: /credentials + - name: cache-state + mountPath: /cache-state + securityContext: + capabilities: + add: + - SETFCAP + computeResources: + requests: + memory: "4Gi" + cpu: "1000m" + limits: + memory: "8Gi" + cpu: "2000m" + script: | + #!/bin/bash + set -euo pipefail + + COMMIT_SHA=$(cat /workspace/source/COMMIT_SHA) + CONTEXT="/workspace/source/$(params.CONTEXT_DIR)" + + export REGISTRY_AUTH_FILE=/credentials/.dockerconfigjson + BUILD_ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') + + # --- Layer 1: Base image (tools + Go) --- + BASE_DOCKERFILE="${CONTEXT}/$(params.BASE_DOCKERFILE)" + BASE_HASH=$(sha256sum "${BASE_DOCKERFILE}" | cut -c1-12) + BASE_IMAGE="$(params.IMAGE_DESTINATION):base-${BUILD_ARCH}-${BASE_HASH}" + + if [ "$(cat /cache-state/base)" = "hit" ]; then + echo "Base image up to date, skipping base build." + else + echo "Building base image (tag: base-${BUILD_ARCH}-${BASE_HASH})" + buildah bud \ + --storage-driver=vfs \ + --format=oci \ + --file="${BASE_DOCKERFILE}" \ + --tag="${BASE_IMAGE}" \ + "${CONTEXT}" + + buildah push \ + --storage-driver=vfs \ + --authfile="${REGISTRY_AUTH_FILE}" \ + "${BASE_IMAGE}" + + echo "Base image pushed: ${BASE_IMAGE}" + fi + + # --- Layer 2: Testsuites image (pre-compiled tests) --- + TESTSUITES_DOCKERFILE="${CONTEXT}/Dockerfile.testsuites" + TESTSUITES_HASH=$(cat "${BASE_DOCKERFILE}" "${TESTSUITES_DOCKERFILE}" | sha256sum | cut -c1-12) + TESTSUITES_IMAGE="$(params.IMAGE_DESTINATION):testsuites-${BUILD_ARCH}-${TESTSUITES_HASH}" + + if [ "$(cat /cache-state/testsuites)" = "hit" ]; then + echo "Testsuites image up to date, skipping testsuites build." + else + echo "Building testsuites image (tag: testsuites-${BUILD_ARCH}-${TESTSUITES_HASH})" + buildah bud \ + --storage-driver=vfs \ + --format=oci \ + --build-arg="BASE_IMAGE=${BASE_IMAGE}" \ + --file="${TESTSUITES_DOCKERFILE}" \ + --tag="${TESTSUITES_IMAGE}" \ + "${CONTEXT}" + + buildah push \ + --storage-driver=vfs \ + --authfile="${REGISTRY_AUTH_FILE}" \ + "${TESTSUITES_IMAGE}" + + echo "Testsuites image pushed: ${TESTSUITES_IMAGE}" + fi + + # --- Layer 3: Final image (scripts overlay) --- + # Calculate final image tag based on testsuites + scripts content + BASE_DOCKERFILE="${CONTEXT}/$(params.BASE_DOCKERFILE)" + TESTSUITES_DOCKERFILE="${CONTEXT}/Dockerfile.testsuites" + TESTSUITES_HASH=$(cat "${BASE_DOCKERFILE}" "${TESTSUITES_DOCKERFILE}" | sha256sum | cut -c1-12) + SCRIPTS_HASH=$(find "${CONTEXT}/scripts" -type f -exec sha256sum {} \; | sort | sha256sum | cut -c1-12) + FINAL_HASH=$(echo "${TESTSUITES_HASH}${SCRIPTS_HASH}" | sha256sum | cut -c1-12) + DESTINATION="$(params.IMAGE_DESTINATION):final-${BUILD_ARCH}-${FINAL_HASH}" + + echo "Building final image: ${DESTINATION}" + buildah bud \ + --storage-driver=vfs \ + --format=oci \ + --build-arg="BASE_IMAGE=${TESTSUITES_IMAGE}" \ + --file="/workspace/source/$(params.DOCKERFILE_PATH)" \ + --tag="${DESTINATION}" \ + --label="quay.expires-after=$(params.IMAGE_EXPIRES_AFTER)" \ + --label="git.commit=${COMMIT_SHA}" \ + --label="base.image=${BASE_IMAGE}" \ + --label="testsuites.image=${TESTSUITES_IMAGE}" \ + --label="scripts.hash=${SCRIPTS_HASH}" \ + "${CONTEXT}" + + buildah push \ + --storage-driver=vfs \ + --authfile="${REGISTRY_AUTH_FILE}" \ + --digestfile=/tekton/results/IMAGE_DIGEST \ + "${DESTINATION}" + + echo -n "${DESTINATION}" > /tekton/results/IMAGE_URL + echo -n "final-${BUILD_ARCH}-${FINAL_HASH}" > /tekton/results/IMAGE_TAG + + echo "Done: ${DESTINATION}" + echo "Digest: $(cat /tekton/results/IMAGE_DIGEST)" diff --git a/.tekton/tasks/deploy-argocd.yaml b/.tekton/tasks/deploy-argocd.yaml new file mode 100644 index 00000000..bf43f2a6 --- /dev/null +++ b/.tekton/tasks/deploy-argocd.yaml @@ -0,0 +1,88 @@ +--- +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: deploy-argocd +spec: + description: | + Deploys ArgoCD standalone on an ephemeral cluster using the extracted server image. + params: + - name: argoCDImage + type: string + - name: argoCDVersion + type: string + - name: testImageUrl + type: string + - name: eaasSpaceSecretRef + type: string + - name: clusterName + type: string + - name: pipelineRunName + type: string + - name: quayRepo + type: string + default: "quay.io/devtools_gitops/test_image" + results: + - name: namespace + description: Namespace where ArgoCD is deployed + - name: server + description: ArgoCD server service DNS name + - name: adminPassword + description: ArgoCD admin password + - name: serverName + description: ArgoCD server deployment name + - name: repoServerName + description: ArgoCD repo-server deployment name + - name: applicationControllerName + description: ArgoCD application-controller deployment name + - name: redisName + description: ArgoCD redis deployment name + volumes: + - name: credentials + emptyDir: {} + - name: quay-credentials + secret: + secretName: gitops-test-runner-image-push + steps: + - name: get-kubeconfig + ref: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: stepactions/eaas-get-ephemeral-cluster-credentials/0.1/eaas-get-ephemeral-cluster-credentials.yaml + params: + - name: eaasSpaceSecretRef + value: $(params.eaasSpaceSecretRef) + - name: clusterName + value: $(params.clusterName) + - name: credentials + value: credentials + - name: deploy + image: $(params.testImageUrl) + env: + - name: ARGOCD_SERVER_IMAGE + value: $(params.argoCDImage) + - name: ARGOCD_VERSION + value: $(params.argoCDVersion) + - name: NAMESPACE + value: argocd-e2e + - name: KUBECONFIG + value: "/credentials/$(steps.get-kubeconfig.results.kubeconfig)" + - name: TASK_LOG_NAME + value: deploy-argocd + - name: PIPELINE_RUN_NAME + value: $(params.pipelineRunName) + - name: QUAY_REPO + value: $(params.quayRepo) + command: + - /usr/local/bin/run-and-save-logs.sh + - /usr/local/bin/deploy-argocd-standalone.sh + volumeMounts: + - name: credentials + mountPath: /credentials + - name: quay-credentials + mountPath: /quay-credentials diff --git a/.tekton/tasks/extract-argocd-image.yaml b/.tekton/tasks/extract-argocd-image.yaml new file mode 100644 index 00000000..6f08ce00 --- /dev/null +++ b/.tekton/tasks/extract-argocd-image.yaml @@ -0,0 +1,79 @@ +--- +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: extract-argocd-image +spec: + description: | + Extracts the ArgoCD server image from the operator catalog for a given channel. + params: + - name: catalogImage + type: string + - name: operatorChannel + type: string + - name: testImageUrl + type: string + - name: pipelineRunName + type: string + - name: quayRepo + type: string + default: "quay.io/devtools_gitops/test_image" + results: + - name: argoCDImage + description: ArgoCD server image extracted from catalog + volumes: + - name: shared-data + emptyDir: {} + - name: quay-credentials + secret: + secretName: gitops-test-runner-image-push + - name: quay-pull-credentials + secret: + secretName: gitops-quay-pull-secret-merged + steps: + - name: extract-image + image: $(params.testImageUrl) + env: + - name: CATALOG_IMAGE + value: $(params.catalogImage) + - name: OPERATOR_CHANNEL + value: $(params.operatorChannel) + - name: TASK_LOG_NAME + value: extract-argocd-image + - name: PIPELINE_RUN_NAME + value: $(params.pipelineRunName) + - name: QUAY_REPO + value: $(params.quayRepo) + command: + - /usr/local/bin/run-and-save-logs.sh + - python3 + - /usr/local/bin/extract-argocd-image-from-catalog.py + volumeMounts: + - name: shared-data + mountPath: /shared + - name: quay-credentials + mountPath: /quay-credentials + - name: quay-pull-credentials + mountPath: /quay-pull-credentials + readOnly: true + - name: save-result + image: $(params.testImageUrl) + script: | + #!/bin/bash + if [ ! -f /shared/argocd-image.txt ]; then + echo "ERROR: /shared/argocd-image.txt not found" + echo "ArgoCD extraction may have failed" + exit 1 + fi + + ARGOCD_IMAGE=$(cat /shared/argocd-image.txt) + if [ -z "$ARGOCD_IMAGE" ]; then + echo "ERROR: ArgoCD image is empty" + exit 1 + fi + + echo -n "$ARGOCD_IMAGE" > $(results.argoCDImage.path) + echo "ArgoCD image saved: $ARGOCD_IMAGE" + volumeMounts: + - name: shared-data + mountPath: /shared diff --git a/.tekton/tasks/install-operator.yaml b/.tekton/tasks/install-operator.yaml new file mode 100644 index 00000000..3d69144a --- /dev/null +++ b/.tekton/tasks/install-operator.yaml @@ -0,0 +1,156 @@ +--- +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: install-operator +spec: + description: | + Installs the GitOps operator from the catalog on an ephemeral cluster, + optionally performs an upgrade, and reports the installed CSV. + params: + - name: testImageUrl + type: string + - name: eaasSpaceSecretRef + type: string + - name: clusterName + type: string + - name: pipelineRunName + type: string + - name: openshiftVersion + type: string + - name: namespace + type: string + default: "openshift-gitops-operator" + - name: installTimeout + type: string + default: "25m" + - name: operatorChannel + type: string + - name: operatorVersion + type: string + default: "" + - name: autoUpgrade + type: string + default: "false" + - name: upgrade + type: string + default: "false" + - name: upgradeToChannel + type: string + default: "latest" + - name: quayRepo + type: string + default: "quay.io/devtools_gitops/test_image" + results: + - name: installedCSV + value: "$(steps.get-installed-version.results.installedCSV)" + volumes: + - name: credentials + emptyDir: {} + - name: quay-credentials + secret: + secretName: gitops-test-runner-image-push + - name: quay-pull-credentials + secret: + secretName: gitops-quay-pull-secret-merged + steps: + - name: get-kubeconfig + ref: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: stepactions/eaas-get-ephemeral-cluster-credentials/0.1/eaas-get-ephemeral-cluster-credentials.yaml + params: + - name: eaasSpaceSecretRef + value: $(params.eaasSpaceSecretRef) + - name: clusterName + value: $(params.clusterName) + - name: credentials + value: credentials + - name: install-operator + image: $(params.testImageUrl) + volumeMounts: + - name: credentials + mountPath: /credentials + - name: quay-credentials + mountPath: /quay-credentials + readOnly: true + - name: quay-pull-credentials + mountPath: /quay-pull-credentials + readOnly: true + env: + - name: OPENSHIFT_VERSION + value: $(params.openshiftVersion) + - name: NAMESPACE + value: $(params.namespace) + - name: INSTALL_TIMEOUT + value: $(params.installTimeout) + - name: OPERATOR_CHANNEL + value: $(params.operatorChannel) + - name: OPERATOR_VERSION + value: $(params.operatorVersion) + - name: AUTO_UPGRADE + value: $(params.autoUpgrade) + - name: KUBECONFIG + value: "/credentials/$(steps.get-kubeconfig.results.kubeconfig)" + - name: TASK_LOG_NAME + value: "install-operator" + - name: PIPELINE_RUN_NAME + value: $(params.pipelineRunName) + - name: QUAY_REPO + value: $(params.quayRepo) + command: + - /bin/bash + - -c + - | + /usr/local/bin/print-cluster-login-info.sh + /usr/local/bin/run-and-save-logs.sh /usr/local/bin/install-operator.sh + - name: upgrade-operator + onError: continue + image: $(params.testImageUrl) + volumeMounts: + - name: credentials + mountPath: /credentials + - name: quay-credentials + mountPath: /quay-credentials + readOnly: true + env: + - name: NAMESPACE + value: $(params.namespace) + - name: INSTALL_TIMEOUT + value: $(params.installTimeout) + - name: UPGRADE + value: $(params.upgrade) + - name: UPGRADE_TO_CHANNEL + value: $(params.upgradeToChannel) + - name: KUBECONFIG + value: "/credentials/$(steps.get-kubeconfig.results.kubeconfig)" + - name: TASK_LOG_NAME + value: "upgrade-operator" + - name: PIPELINE_RUN_NAME + value: $(params.pipelineRunName) + - name: QUAY_REPO + value: $(params.quayRepo) + command: + - /usr/local/bin/run-and-save-logs.sh + - /usr/local/bin/upgrade-operator.sh + - name: get-installed-version + image: $(params.testImageUrl) + results: + - name: installedCSV + volumeMounts: + - name: credentials + mountPath: /credentials + env: + - name: KUBECONFIG + value: "/credentials/$(steps.get-kubeconfig.results.kubeconfig)" + - name: NAMESPACE + value: $(params.namespace) + - name: RESULT_PATH + value: "$(step.results.installedCSV.path)" + command: + - /usr/local/bin/get-installed-version.sh diff --git a/.tekton/tasks/overlay-test-scripts.yaml b/.tekton/tasks/overlay-test-scripts.yaml new file mode 100644 index 00000000..823b5611 --- /dev/null +++ b/.tekton/tasks/overlay-test-scripts.yaml @@ -0,0 +1,151 @@ +--- +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: overlay-test-scripts +spec: + description: | + Builds a thin scripts-overlay image on top of a pre-built test runner + base image. Clones the catalog repo, hashes the scripts/ and config/ + directories, and skips the build when an image with that hash already + exists in the registry. + params: + - name: BASE_IMAGE_URL + type: string + description: Pre-built test runner image to overlay scripts onto + - name: SOURCE_URL + type: string + description: Git URL of the catalog repo + - name: SOURCE_REVISION + type: string + description: Git branch or SHA to clone + - name: IMAGE_DESTINATION + type: string + default: "quay.io/devtools_gitops/test_image" + description: Registry repo for overlay images (without tag) + - name: IMAGE_EXPIRES_AFTER + type: string + default: "7d" + description: "Quay expiration label (e.g., 7d, 30d, never)" + results: + - name: image-url + description: Full overlay image reference (repo:tag) + volumes: + - name: source + emptyDir: {} + - name: docker-credentials + secret: + secretName: gitops-test-runner-image-push + - name: cache-state + emptyDir: {} + steps: + - name: clone-source + image: docker.io/alpine/git:latest + volumeMounts: + - name: source + mountPath: /workspace/source + script: | + #!/bin/sh + set -e + echo "Cloning $(params.SOURCE_URL) @ $(params.SOURCE_REVISION)" + git clone --depth 1 --branch "$(params.SOURCE_REVISION)" \ + "$(params.SOURCE_URL)" /workspace/source + cd /workspace/source + COMMIT_SHA=$(git rev-parse HEAD) + echo "Commit: ${COMMIT_SHA}" + echo -n "${COMMIT_SHA}" > /workspace/source/COMMIT_SHA + + - name: check-cache + image: quay.io/skopeo/stable:latest + volumeMounts: + - name: source + mountPath: /workspace/source + - name: docker-credentials + mountPath: /credentials + - name: cache-state + mountPath: /cache-state + script: | + #!/bin/bash + set -euo pipefail + + CONTEXT="/workspace/source/.tekton/test-image" + AUTH="--authfile=/credentials/.dockerconfigjson" + BUILD_ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') + + SCRIPTS_HASH=$(find "${CONTEXT}/scripts" "${CONTEXT}/config" \ + -type f -exec sha256sum {} \; | sort | sha256sum | cut -c1-12) + BASE_IMAGE_URL="$(params.BASE_IMAGE_URL)" + BASE_TAG="${BASE_IMAGE_URL##*:}" + COMBINED_HASH=$(echo "${SCRIPTS_HASH}${BASE_TAG}" | sha256sum | cut -c1-12) + DESTINATION="$(params.IMAGE_DESTINATION):scripts-${BUILD_ARCH}-${COMBINED_HASH}" + + echo "Scripts hash: ${SCRIPTS_HASH}" + echo "Base image tag: ${BASE_TAG}" + echo "Overlay image: ${DESTINATION}" + echo -n "${DESTINATION}" > /cache-state/destination + + if skopeo inspect ${AUTH} "docker://${DESTINATION}" >/dev/null 2>&1; then + echo "Overlay image exists — cache hit" + echo "hit" > /cache-state/result + else + echo "Overlay image not found — will build" + echo "miss" > /cache-state/result + fi + + - name: build-overlay + image: quay.io/buildah/stable:latest + volumeMounts: + - name: source + mountPath: /workspace/source + - name: docker-credentials + mountPath: /credentials + - name: cache-state + mountPath: /cache-state + securityContext: + capabilities: + add: + - SETFCAP + script: | + #!/bin/bash + set -euo pipefail + + DESTINATION=$(cat /cache-state/destination) + + if [ "$(cat /cache-state/result)" = "hit" ]; then + echo "Cache hit — using existing image: ${DESTINATION}" + echo -n "${DESTINATION}" > $(results.image-url.path) + exit 0 + fi + + COMMIT_SHA=$(cat /workspace/source/COMMIT_SHA) + CONTEXT="/workspace/source/.tekton/test-image" + export REGISTRY_AUTH_FILE=/credentials/.dockerconfigjson + + cat > /tmp/Containerfile <<'CEOF' + ARG BASE_IMAGE + FROM ${BASE_IMAGE} + COPY scripts/ /usr/local/bin/ + COPY config/ /usr/local/config/ + RUN chmod +x /usr/local/bin/*.sh /usr/local/bin/*.py /usr/local/bin/lib/*.sh + CEOF + + echo "Building overlay on top of $(params.BASE_IMAGE_URL)..." + buildah bud \ + --storage-driver=vfs \ + --format=oci \ + --build-arg="BASE_IMAGE=$(params.BASE_IMAGE_URL)" \ + --file=/tmp/Containerfile \ + --tag="${DESTINATION}" \ + --label="quay.expires-after=$(params.IMAGE_EXPIRES_AFTER)" \ + --label="git.commit=${COMMIT_SHA}" \ + --label="base.image=$(params.BASE_IMAGE_URL)" \ + "${CONTEXT}" + + echo "Pushing ${DESTINATION}..." + buildah push \ + --storage-driver=vfs \ + --authfile="${REGISTRY_AUTH_FILE}" \ + "${DESTINATION}" + + echo -n "${DESTINATION}" > $(results.image-url.path) + echo "Done: ${DESTINATION}" diff --git a/.tekton/tasks/pipeline-wrapup.yaml b/.tekton/tasks/pipeline-wrapup.yaml new file mode 100644 index 00000000..b09d1622 --- /dev/null +++ b/.tekton/tasks/pipeline-wrapup.yaml @@ -0,0 +1,204 @@ +--- +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: pipeline-wrapup +spec: + description: | + Collects logs, publishes results, and sends Slack notification. + Consolidates upload-logs, publish-results, and send-slack-notification + into a single task with shared volumes between steps. + params: + - name: eaasSpaceSecretRef + type: string + - name: clusterName + type: string + - name: pipelineRunName + type: string + - name: testImageUrl + type: string + - name: aggregateStatus + type: string + - name: logUrl + type: string + - name: pipelineName + type: string + - name: namespace + type: string + - name: taskNames + type: string + - name: openshiftVersion + type: string + - name: resolvedOpenshiftVersion + type: string + - name: operatorChannel + type: string + - name: fipsEnabled + type: string + - name: argocdVersion + type: string + default: "" + - name: installedCSV + type: string + default: "" + - name: testScript + type: string + default: "" + - name: testRepoUrl + type: string + default: "" + - name: testRepoBranch + type: string + default: "" + - name: upgrade + type: string + default: "" + - name: quayRepo + type: string + default: "quay.io/devtools_gitops/test_image" + - name: quayCredentialsPath + type: string + default: "/quay-credentials/.dockerconfigjson" + volumes: + - name: credentials + emptyDir: {} + - name: shared + emptyDir: {} + - name: quay-credentials + secret: + secretName: gitops-test-runner-image-push + - name: deploy-key + secret: + secretName: catalog-results-deploy-key + defaultMode: 0400 + steps: + - name: get-kubeconfig + onError: continue + ref: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: stepactions/eaas-get-ephemeral-cluster-credentials/0.1/eaas-get-ephemeral-cluster-credentials.yaml + params: + - name: eaasSpaceSecretRef + value: $(params.eaasSpaceSecretRef) + - name: clusterName + value: $(params.clusterName) + - name: credentials + value: credentials + + - name: collect-and-upload-logs + onError: continue + image: $(params.testImageUrl) + env: + - name: PIPELINE_RUN_NAME + value: $(params.pipelineRunName) + - name: NAMESPACE + value: $(params.namespace) + - name: QUAY_REPO + value: $(params.quayRepo) + - name: QUAY_CREDENTIALS_PATH + value: $(params.quayCredentialsPath) + - name: TASK_NAMES + value: $(params.taskNames) + - name: SHARED_DIR + value: /shared + command: + - /usr/local/bin/collect-and-upload-logs.sh + volumeMounts: + - name: credentials + mountPath: /credentials + - name: quay-credentials + mountPath: /quay-credentials + - name: shared + mountPath: /shared + + - name: publish-results + onError: continue + image: $(params.testImageUrl) + env: + - name: PIPELINE_NAME + value: $(params.pipelineName) + - name: PIPELINE_RUN_NAME + value: $(params.pipelineRunName) + - name: AGGREGATE_STATUS + value: $(params.aggregateStatus) + - name: OPENSHIFT_VERSION + value: $(params.openshiftVersion) + - name: RESOLVED_OPENSHIFT_VERSION + value: $(params.resolvedOpenshiftVersion) + - name: ARGOCD_VERSION + value: $(params.argocdVersion) + - name: OPERATOR_CHANNEL + value: $(params.operatorChannel) + - name: INSTALLED_CSV + value: $(params.installedCSV) + - name: TEST_SCRIPT + value: $(params.testScript) + - name: FIPS_ENABLED + value: $(params.fipsEnabled) + - name: UPGRADE + value: $(params.upgrade) + - name: LOG_URL + value: $(params.logUrl) + - name: LOGS_ARTIFACT + value: "$(params.quayRepo):$(params.pipelineRunName)-logs" + - name: SHARED_DIR + value: /shared + command: + - /usr/local/bin/publish-results.sh + volumeMounts: + - name: deploy-key + mountPath: /deploy-key + readOnly: true + - name: shared + mountPath: /shared + + - name: send-slack-notification + onError: continue + image: $(params.testImageUrl) + env: + - name: SLACK_WEBHOOK_URL + valueFrom: + secretKeyRef: + name: slack-gitops-test-notification-url + key: hook-url + - name: PIPELINE_RUN_NAME + value: $(params.pipelineRunName) + - name: AGGREGATE_STATUS + value: $(params.aggregateStatus) + - name: LOG_URL + value: $(params.logUrl) + - name: QUAY_REPO + value: $(params.quayRepo) + - name: QUAY_CREDENTIALS_PATH + value: $(params.quayCredentialsPath) + - name: TASK_NAMES + value: $(params.taskNames) + - name: TEST_SCRIPT + value: $(params.testScript) + - name: TEST_REPO_URL + value: $(params.testRepoUrl) + - name: TEST_REPO_BRANCH + value: $(params.testRepoBranch) + - name: OPENSHIFT_VERSION + value: $(params.openshiftVersion) + - name: OPERATOR_CHANNEL + value: $(params.operatorChannel) + - name: FIPS_ENABLED + value: $(params.fipsEnabled) + - name: SHARED_DIR + value: /shared + command: + - /usr/local/bin/send-slack-message.py + volumeMounts: + - name: credentials + mountPath: /credentials + - name: quay-credentials + mountPath: /quay-credentials + - name: shared + mountPath: /shared diff --git a/.tekton/tasks/provision-cluster.yaml b/.tekton/tasks/provision-cluster.yaml new file mode 100644 index 00000000..fa350275 --- /dev/null +++ b/.tekton/tasks/provision-cluster.yaml @@ -0,0 +1,98 @@ +--- +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: provision-cluster +spec: + description: | + Provisions an ephemeral HyperShift cluster on AWS via EaaS. + Resolves the OpenShift version, extracts image content sources from the catalog, + and creates the cluster with optional FIPS mode. + params: + - name: eaasSpaceSecretRef + type: string + - name: catalogSourceUrl + type: string + - name: catalogSourceRevision + type: string + - name: catalogTaskUrl + type: string + - name: catalogTaskRevision + type: string + - name: openshiftVersion + type: string + - name: fipsEnabled + type: string + - name: clusterInstanceType + type: string + default: "m6g.xlarge" + results: + - name: clusterName + value: "$(steps.create-cluster.results.clusterName)" + - name: resolvedVersion + value: "$(steps.resolve-openshift-version.results.resolvedVersion)" + steps: + - name: get-supported-versions + ref: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: stepactions/eaas-get-supported-ephemeral-cluster-versions/0.1/eaas-get-supported-ephemeral-cluster-versions.yaml + params: + - name: eaasSpaceSecretRef + value: $(params.eaasSpaceSecretRef) + - name: extract-image-content-sources + ref: + resolver: git + params: + - name: url + value: $(params.catalogTaskUrl) + - name: revision + value: $(params.catalogTaskRevision) + - name: pathInRepo + value: .tekton/stepactions/extract-image-content-sources.yaml + params: + - name: gitUrl + value: $(params.catalogSourceUrl) + - name: gitRevision + value: $(params.catalogSourceRevision) + - name: resolve-openshift-version + ref: + resolver: git + params: + - name: url + value: $(params.catalogTaskUrl) + - name: revision + value: $(params.catalogTaskRevision) + - name: pathInRepo + value: .tekton/stepactions/resolve-openshift-version.yaml + params: + - name: version + value: $(params.openshiftVersion) + - name: create-cluster + ref: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: stepactions/eaas-create-ephemeral-cluster-hypershift-aws/0.1/eaas-create-ephemeral-cluster-hypershift-aws.yaml + params: + - name: eaasSpaceSecretRef + value: $(params.eaasSpaceSecretRef) + - name: imageContentSourcesFile + value: "/workspace/imageContentSources.json" + - name: timeout + value: "60m" + - name: version + value: "$(steps.resolve-openshift-version.results.resolvedVersion)" + - name: fips + value: "$(params.fipsEnabled)" + - name: instanceType + value: "$(params.clusterInstanceType)" diff --git a/.tekton/tasks/test-argocd.yaml b/.tekton/tasks/test-argocd.yaml new file mode 100644 index 00000000..27af0bc3 --- /dev/null +++ b/.tekton/tasks/test-argocd.yaml @@ -0,0 +1,148 @@ +--- +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: test-argocd +spec: + description: | + Runs upstream ArgoCD E2E tests on an ephemeral cluster with a deployed ArgoCD instance. + Produces a TEST_OUTPUT result in Konflux format. + params: + - name: argoCDImage + type: string + - name: testRepoUrl + type: string + - name: testRepoBranch + type: string + - name: testImageUrl + type: string + - name: eaasSpaceSecretRef + type: string + - name: clusterName + type: string + - name: pipelineRunName + type: string + - name: argoCDNamespace + type: string + - name: argoCDServer + type: string + - name: argoCDAdminPassword + type: string + - name: argoCDServerName + type: string + - name: argoCDRepoServerName + type: string + - name: argoCDApplicationControllerName + type: string + - name: argoCDRedisName + type: string + - name: testRunnerImage + type: string + default: "registry.access.redhat.com/ubi9/go-toolset:latest" + - name: quayRepo + type: string + default: "quay.io/devtools_gitops/test_image" + results: + - name: TEST_OUTPUT + description: Test results in Tekton format + volumes: + - name: credentials + emptyDir: {} + - name: quay-credentials + secret: + secretName: gitops-test-runner-image-push + steps: + - name: get-kubeconfig + ref: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: stepactions/eaas-get-ephemeral-cluster-credentials/0.1/eaas-get-ephemeral-cluster-credentials.yaml + params: + - name: eaasSpaceSecretRef + value: $(params.eaasSpaceSecretRef) + - name: clusterName + value: $(params.clusterName) + - name: credentials + value: credentials + - name: run-tests + image: $(params.testImageUrl) + env: + - name: ARGOCD_SERVER_IMAGE + value: $(params.argoCDImage) + - name: TEST_REPO_URL + value: $(params.testRepoUrl) + - name: BRANCH + value: $(params.testRepoBranch) + - name: KUBECONFIG + value: "/credentials/$(steps.get-kubeconfig.results.kubeconfig)" + - name: TASK_LOG_NAME + value: test-argocd + - name: PIPELINE_RUN_NAME + value: $(params.pipelineRunName) + - name: QUAY_REPO + value: $(params.quayRepo) + - name: ARGOCD_NAMESPACE + value: $(params.argoCDNamespace) + - name: ARGOCD_SERVER + value: $(params.argoCDServer) + - name: ARGOCD_ADMIN_PASSWORD + value: $(params.argoCDAdminPassword) + - name: ARGOCD_SERVER_NAME + value: $(params.argoCDServerName) + - name: ARGOCD_REPO_SERVER_NAME + value: $(params.argoCDRepoServerName) + - name: ARGOCD_APPLICATION_CONTROLLER_NAME + value: $(params.argoCDApplicationControllerName) + - name: ARGOCD_REDIS_NAME + value: $(params.argoCDRedisName) + - name: TEST_RUNNER_IMAGE + value: $(params.testRunnerImage) + command: + - /usr/local/bin/run-and-save-logs.sh + - /usr/local/bin/run-argocd-e2e-tests-in-pod.sh + volumeMounts: + - name: credentials + mountPath: /credentials + - name: quay-credentials + mountPath: /quay-credentials + - name: generate-test-output + image: $(params.testImageUrl) + script: | + #!/usr/bin/env python3 + import json, os, datetime + + junit = "/tmp/task-logs/junit-results.xml" + results_json = "/tmp/task-logs/test-results.json" + + if os.path.isfile(junit): + os.system(f"python3 /usr/local/bin/parse-test-results.py {junit} {results_json}") + + if os.path.isfile(results_json): + with open(results_json) as f: + tr = json.load(f) + result = "SUCCESS" if tr["failed"] == 0 and tr["errors"] == 0 else "FAILURE" + output = { + "result": result, + "timestamp": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S+00:00"), + "successes": tr["passed"], + "failures": tr["failed"] + tr["errors"], + "warnings": 0, + } + else: + output = { + "result": "ERROR", + "timestamp": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S+00:00"), + "successes": 0, + "failures": 1, + "warnings": 0, + } + + with open("$(results.TEST_OUTPUT.path)", "w") as f: + json.dump(output, f) + + print(json.dumps(output, indent=2)) diff --git a/.tekton/tasks/test-dast.yaml b/.tekton/tasks/test-dast.yaml new file mode 100644 index 00000000..810f931f --- /dev/null +++ b/.tekton/tasks/test-dast.yaml @@ -0,0 +1,288 @@ +--- +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: test-dast +spec: + description: | + Runs a DAST (Dynamic Application Security Testing) scan against the + ArgoCD REST API deployed by the GitOps operator on an ephemeral cluster. + + Steps: + 1. get-kubeconfig — fetch EaaS cluster credentials + 2. get-cluster-info — extract API URL, apps domain, kubeadmin password; + wait for ArgoCD server to be Available + 3. run-dast — authenticate to ArgoCD via OC OAuth, run RapidAST/ZAP + 4. collect-results — parse ZAP JSON to JUnit XML, upload task artifact + + The scan result is determined by dast-false-positives.json (baked into + the overlay image at /usr/local/config/dast-false-positives.json). + Alerts exceeding their configured threshold → JUnit failure → task fails. + + GCP credentials (for uploading raw findings to GCS) must be in a secret + named gcp with key key.json. + params: + - name: testImageUrl + type: string + - name: eaasSpaceSecretRef + type: string + - name: clusterName + type: string + - name: pipelineRunName + type: string + - name: rapidastImage + type: string + default: "quay.io/redhatproductsecurity/rapidast@sha256:b6eba7ca96e4c775f965e12eebd2853e8aa11d5894eb0867aba94d436789604d" + - name: quayRepo + type: string + default: "quay.io/devtools_gitops/test_image" + - name: namespace + type: string + default: "openshift-gitops-operator" + results: + - name: LOG_ARTIFACT_TAG + description: Tag of the uploaded log artifact + volumes: + - name: credentials + emptyDir: {} + - name: dast-data + emptyDir: {} + - name: quay-credentials + secret: + secretName: gitops-test-runner-image-push + - name: gcp-key + secret: + secretName: gcp + steps: + - name: get-kubeconfig + image: $(params.testImageUrl) + onError: continue + env: + - name: CLUSTER_NAME + value: $(params.clusterName) + - name: KUBECONFIG_VALUE + valueFrom: + secretKeyRef: + name: $(params.eaasSpaceSecretRef) + key: kubeconfig + volumeMounts: + - name: credentials + mountPath: /credentials + script: | + #!/bin/bash + set -eo pipefail + + echo "${KUBECONFIG_VALUE}" > /tmp/eaas-kubeconfig + export KUBECONFIG=/tmp/eaas-kubeconfig + + CLUSTER_KUBECONFIG="/credentials/${CLUSTER_NAME}-kubeconfig" + + SECRET=$(oc get cti "${CLUSTER_NAME}" -o=jsonpath='{.status.kubeconfig.name}') + echo "Found kubeconfig secret: ${SECRET}" + oc get secret "${SECRET}" -o go-template --template='{{.data.kubeconfig|base64decode}}' \ + > "${CLUSTER_KUBECONFIG}" + echo "Wrote kubeconfig to ${CLUSTER_KUBECONFIG}" + + - name: get-cluster-info + image: $(params.testImageUrl) + volumeMounts: + - name: credentials + mountPath: /credentials + - name: dast-data + mountPath: /dast-data + script: | + #!/bin/bash + set -euo pipefail + + KUBECONFIG=$(find /credentials -name "*kubeconfig" -type f 2>/dev/null | head -1) + if [[ -z "${KUBECONFIG:-}" ]]; then + echo "ERROR: no kubeconfig found in /credentials" + ls -la /credentials/ 2>/dev/null || true + exit 1 + fi + export KUBECONFIG + + echo "Cluster: $(oc whoami --show-server)" + + APIURL=$(oc whoami --show-server) + CONSOLE_HOST=$(oc get route console -n openshift-console \ + -o jsonpath='{.spec.host}' 2>/dev/null) + APPSURL="${CONSOLE_HOST#console-openshift-console.}" + echo "Apps domain: ${APPSURL}" + + KPWD=$(oc get secret kubeadmin -n kube-system \ + -o jsonpath='{.data.kubeadmin}' | base64 -d) + + echo "Waiting for ArgoCD server to be Available (timeout 5m)..." + oc wait deployment/openshift-gitops-server \ + -n openshift-gitops \ + --for=condition=Available \ + --timeout=5m || { + echo "WARNING: ArgoCD server not Available after 5m, proceeding anyway" + oc get pods -n openshift-gitops || true + } + + printf 'export APIURL=%q\nexport APPSURL=%q\nexport KPWD=%q\n' \ + "${APIURL}" "${APPSURL}" "${KPWD}" > /dast-data/cluster-info.env + echo "Cluster info written to /dast-data/cluster-info.env" + + - name: run-dast + image: $(params.rapidastImage) + workingDir: /dast-data + volumeMounts: + - name: dast-data + mountPath: /dast-data + - name: gcp-key + mountPath: /gcp + readOnly: true + env: + - name: _JAVA_OPTIONS + value: "-DmaxYamlCodePoints=99999999" + script: | + #!/bin/bash + set -euo pipefail + + source /dast-data/cluster-info.env + + echo "Authenticating to cluster at ${APIURL}..." + OC_TOKEN=$(curl -u kubeadmin:"${KPWD}" -k -s -i \ + "https://oauth-openshift.${APPSURL}/oauth/authorize?client_id=openshift-challenging-client&response_type=token" \ + | grep -oP "access_token=\K[^&]*") + if [[ -z "${OC_TOKEN}" ]]; then + echo "ERROR: failed to obtain OC OAuth token" + exit 1 + fi + + echo "Fetching ArgoCD admin password from cluster secret..." + ARGO_PWD=$(curl -sk -H "Authorization: Bearer ${OC_TOKEN}" \ + "${APIURL}/api/v1/namespaces/openshift-gitops/secrets/openshift-gitops-cluster" \ + | grep -oP '"admin\.password": "\K[^"]*' | base64 -d) + if [[ -z "${ARGO_PWD}" ]]; then + echo "ERROR: failed to read ArgoCD admin password" + exit 1 + fi + + ARGO_URL="https://openshift-gitops-server-openshift-gitops.${APPSURL}" + echo "ArgoCD server: ${ARGO_URL}" + + echo "Obtaining ArgoCD session token..." + ARGO_TOKEN=$(curl -sk -H "Content-Type: application/json" \ + "${ARGO_URL}/api/v1/session" \ + -d "{\"username\":\"admin\",\"password\":\"${ARGO_PWD}\"}" \ + | grep -oP '"token":"\K[^"]*') + if [[ -z "${ARGO_TOKEN}" ]]; then + echo "ERROR: failed to obtain ArgoCD session token" + exit 1 + fi + + cat > /tmp/rapidast-config.yaml << RAPIDASTEOF + config: + configVersion: 6 + googleCloudStorage: + keyFile: "/gcp/key.json" + bucketName: "gitops-results" + application: + shortName: "gitops" + url: "${ARGO_URL}" + general: + authentication: + type: http_header + parameters: + name: "Authorization" + value: "Bearer ${ARGO_TOKEN}" + scanners: + zap: + apiScan: + apis: + apiUrl: "${ARGO_URL}/swagger.json" + report: + format: ["json", "html", "xml"] + activeScan: + policy: "API-scan-minimal" + maxScanDurationInMins: 30 + passiveScan: {} + miscOptions: + additionalAddons: "ascanrulesBeta" + zapPort: 8080 + memMaxHeap: "2048m" + maxRuleDurationInMins: 5 + RAPIDASTEOF + + echo "Starting RapidAST scan..." + rapidast.py --log-level info --config /tmp/rapidast-config.yaml 2>&1 | tee /dast-data/dast.log + echo "RapidAST scan complete" + + - name: collect-results + image: $(params.testImageUrl) + volumeMounts: + - name: dast-data + mountPath: /dast-data + - name: quay-credentials + mountPath: /quay-credentials + readOnly: true + env: + - name: PIPELINE_RUN_NAME + value: $(params.pipelineRunName) + - name: QUAY_REPO + value: $(params.quayRepo) + - name: QUAY_CREDENTIALS_PATH + value: /quay-credentials/.dockerconfigjson + script: | + #!/bin/bash + set -euo pipefail + + # shellcheck source=./lib/oras-helpers.sh + source /usr/local/bin/lib/oras-helpers.sh + + LOG_DIR="/tmp/dast-logs" + mkdir -p "${LOG_DIR}" + + # Copy ZAP results and the rapidast run log + cp /dast-data/dast.log "${LOG_DIR}/" 2>/dev/null || echo "WARNING: dast.log not found" + cp -r /dast-data/results/ "${LOG_DIR}/" 2>/dev/null || echo "WARNING: results/ not found" + + # Parse ZAP output → JUnit XML + echo "Parsing ZAP results..." + JUNIT="${LOG_DIR}/junit-dast.xml" + python3 /usr/local/bin/parse-dast-results.py /dast-data/results/ "${JUNIT}" + PARSE_EXIT=$? + + # Upload task artifact regardless of parse outcome + echo "Uploading task artifact..." + if setup_oras_auth "${QUAY_CREDENTIALS_PATH}"; then + ARTIFACT_TAG="${PIPELINE_RUN_NAME}-task-dast" + if oras_push_tarball "${LOG_DIR}" "${QUAY_REPO}" "${ARTIFACT_TAG}" \ + "application/vnd.konflux.logs.v1+tar" "dast-logs"; then + echo "Artifact pushed: ${QUAY_REPO}:${ARTIFACT_TAG}" + echo -n "${ARTIFACT_TAG}" > /tekton/results/LOG_ARTIFACT_TAG + else + echo "WARNING: artifact upload failed" + fi + else + echo "WARNING: Quay credentials not found, skipping artifact upload" + fi + + exit "${PARSE_EXIT}" + + sidecars: + - name: log-collector + image: $(params.testImageUrl) + workingDir: /var/log-workspace + env: + - name: KUBECONFIG_NAME + value: "auto" + - name: PIPELINE_RUN_NAME + value: $(params.pipelineRunName) + - name: BRANCH_NAME + value: "logs" + - name: NAMESPACE + value: $(params.namespace) + - name: QUAY_REPO + value: $(params.quayRepo) + volumeMounts: + - name: credentials + mountPath: /credentials + - name: quay-credentials + mountPath: /quay-credentials + readOnly: true + command: ["/bin/bash", "-c", "/usr/local/bin/collect-logs-sidecar.sh"] diff --git a/.tekton/tasks/test-operator.yaml b/.tekton/tasks/test-operator.yaml new file mode 100644 index 00000000..b86e6fb6 --- /dev/null +++ b/.tekton/tasks/test-operator.yaml @@ -0,0 +1,146 @@ +--- +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: test-operator +spec: + description: | + Runs GitOps operator E2E tests (Ginkgo) on an ephemeral cluster. + Includes a log-collector sidecar that uploads logs to Quay. + params: + - name: testImageUrl + type: string + - name: eaasSpaceSecretRef + type: string + - name: clusterName + type: string + - name: pipelineRunName + type: string + - name: testRepoUrl + type: string + - name: testRepoBranch + type: string + - name: testScript + type: string + - name: openshiftVersion + type: string + - name: quayRepo + type: string + default: "quay.io/devtools_gitops/test_image" + - name: namespace + type: string + default: "openshift-gitops-operator" + - name: catalogUrl + type: string + default: "" + description: "Catalog repo URL (for smoke test app source)" + - name: catalogRevision + type: string + default: "" + description: "Catalog repo revision (for smoke test app source)" + results: + - name: LOG_ARTIFACT_TAG + description: Tag of uploaded log artifact from sidecar + volumes: + - name: credentials + emptyDir: {} + - name: quay-credentials + secret: + secretName: gitops-test-runner-image-push + - name: confluence-credentials + secret: + secretName: confluence-api-credentials + optional: true + steps: + - name: get-kubeconfig + onError: continue + ref: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: stepactions/eaas-get-ephemeral-cluster-credentials/0.1/eaas-get-ephemeral-cluster-credentials.yaml + params: + - name: eaasSpaceSecretRef + value: $(params.eaasSpaceSecretRef) + - name: clusterName + value: $(params.clusterName) + - name: credentials + value: credentials + - name: run-tests + image: $(params.testImageUrl) + volumeMounts: + - name: credentials + mountPath: /credentials + - name: quay-credentials + mountPath: /quay-credentials + readOnly: true + - name: confluence-credentials + mountPath: /confluence-credentials + readOnly: true + env: + - name: CI + value: "konflux" + - name: TEST_REPO_URL + value: $(params.testRepoUrl) + - name: BRANCH + value: $(params.testRepoBranch) + - name: TASK_LOG_NAME + value: "test-operator" + - name: PIPELINE_RUN_NAME + value: $(params.pipelineRunName) + - name: QUAY_REPO + value: $(params.quayRepo) + - name: OPENSHIFT_VERSION + value: $(params.openshiftVersion) + - name: TEST_SCRIPT + value: $(params.testScript) + - name: CATALOG_URL + value: $(params.catalogUrl) + - name: CATALOG_REVISION + value: $(params.catalogRevision) + computeResources: + requests: + memory: "4Gi" + cpu: "1000m" + limits: + memory: "8Gi" + cpu: "2000m" + command: + - /bin/bash + - -c + - | + KUBECONFIG=$(find /credentials -name "*kubeconfig" -type f 2>/dev/null | head -1) + if [[ -z "$KUBECONFIG" ]]; then + echo "ERROR: No kubeconfig found in /credentials" + ls -la /credentials/ 2>/dev/null || true + exit 1 + fi + export KUBECONFIG + /usr/local/bin/print-cluster-login-info.sh + /usr/local/bin/run-and-save-logs.sh /usr/local/bin/"$TEST_SCRIPT" + sidecars: + - name: log-collector + image: $(params.testImageUrl) + workingDir: /var/log-workspace + env: + - name: KUBECONFIG_NAME + value: "auto" + - name: PIPELINE_RUN_NAME + value: $(params.pipelineRunName) + - name: BRANCH_NAME + value: "logs" + - name: NAMESPACE + value: $(params.namespace) + - name: QUAY_REPO + value: $(params.quayRepo) + volumeMounts: + - name: credentials + mountPath: /credentials + - name: quay-credentials + mountPath: /quay-credentials + readOnly: true + command: ["/bin/bash", "-c", "/usr/local/bin/collect-logs-sidecar.sh"] diff --git a/.tekton/test-image/.hadolint.yaml b/.tekton/test-image/.hadolint.yaml new file mode 100644 index 00000000..0612c537 --- /dev/null +++ b/.tekton/test-image/.hadolint.yaml @@ -0,0 +1,6 @@ +ignored: + - DL3003 + - DL3013 + - DL3041 + - DL3059 + - SC1091 diff --git a/.tekton/test-image/COMPONENT-PLAN.md b/.tekton/test-image/COMPONENT-PLAN.md new file mode 100644 index 00000000..c6ca2bf3 --- /dev/null +++ b/.tekton/test-image/COMPONENT-PLAN.md @@ -0,0 +1,71 @@ +# Test Image Konflux Component Plan + +**Status:** Blocked — wait for current release to complete before creating. + +## Goal + +Replace manual `build-and-push.sh` runs with an automatic Konflux component that builds the test runner image on push. Contributors update scripts or Dockerfiles in a PR and the pipeline transparently picks up the new image from the snapshot. + +## Component Definition + +```yaml +apiVersion: appstudio.redhat.com/v1alpha1 +kind: Component +metadata: + name: gitops-test-runner + namespace: rh-openshift-gitops-tenant +spec: + application: catalog-4-20 + componentName: gitops-test-runner + containerImage: quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/gitops-test-runner + source: + git: + url: https://github.com/rh-gitops-midstream/catalog.git + revision: main + context: .tekton/test-image + dockerfileUrl: Dockerfile +``` + +## Merged Multi-Stage Dockerfile + +Collapse the current 3-layer build (`Dockerfile.base` / `Dockerfile.testsuites` / `Dockerfile`) into a single multi-stage Dockerfile for automatic layer caching: + +``` +Stage 1: base — UBI9, dnf packages, oc, oras, yq (changes: rarely) +Stage 2: testsuites — git clone + ginkgo build + argocd (changes: version bumps) +Stage 3: final — node.js, COPY scripts/, COPY config/ (changes: frequently) +``` + +### Cache behavior + +| What changed | Cached stages | Rebuild cost | +|---------------------------|--------------------|--------------| +| `scripts/` or `config/` | base + testsuites | ~1 min | +| Dockerfile test versions | base | ~15 min | +| Dockerfile base tools | nothing | ~20 min | + +Requires `--cache-from` pointing to the previous image so Buildah reuses layers across builds. + +## Pipeline Integration + +Two options for how e2e pipelines consume the image: + +1. **Snapshot-based** (preferred): The test image component is part of the same application as the catalog. When a snapshot is created, the e2e IntegrationTestScenarios receive a SNAPSHOT containing both the catalog image and the test runner image. The pipeline extracts the test runner image from the snapshot instead of using a hardcoded `TEST_IMAGE_URL`. + +2. **Default parameter**: Keep the current `TEST_IMAGE_URL` pipeline parameter and update its default whenever the test image is rebuilt. Simpler but requires manual default updates. + +## Prerequisites + +- [ ] Release in progress must complete +- [ ] `konflux-integration` branch pushed to `rh-gitops-midstream/catalog` +- [ ] Merged multi-stage Dockerfile created and tested locally +- [ ] Verify `--cache-from` works with `docker-build-oci-ta` pipeline +- [ ] Create Application + Component resources on cluster +- [ ] Update IntegrationTestScenarios to consume image from snapshot (if using option 1) + +## Files + +- `Dockerfile` — replace with merged multi-stage version +- `Dockerfile.base` — keep for local `build-and-push.sh` fallback, or remove +- `Dockerfile.testsuites` — same as above +- `build-and-push.sh` — keep as local dev fallback, or remove once component is working diff --git a/.tekton/test-image/Dockerfile b/.tekton/test-image/Dockerfile new file mode 100644 index 00000000..a434455e --- /dev/null +++ b/.tekton/test-image/Dockerfile @@ -0,0 +1,23 @@ +ARG BASE_IMAGE=quay.io/devtools_gitops/test_image:testsuites +FROM ${BASE_IMAGE} + +LABEL name="gitops-ginkgo-test-runner" \ + description="GitOps integration test runner with helper scripts" \ + maintainer="GitOps Team" \ + io.openshift.tags="gitops,testing,ginkgo,integration" + +# Node.js + Playwright for UI E2E tests +ARG NODE_VERSION=v20.18.0 +RUN ARCH=$(uname -m) && \ + case "$ARCH" in x86_64) NODE_ARCH="x64";; aarch64) NODE_ARCH="arm64";; esac && \ + curl -fsSL "https://nodejs.org/dist/${NODE_VERSION}/node-${NODE_VERSION}-linux-${NODE_ARCH}.tar.gz" \ + | tar -xz -C /usr/local --strip-components=1 && \ + node --version && npm --version + +# Copy helper scripts and config files +COPY scripts/ /usr/local/bin/ +COPY config/ /usr/local/config/ +RUN chmod +x /usr/local/bin/*.sh /usr/local/bin/*.py /usr/local/bin/lib/*.sh + +WORKDIR /workspace +ENTRYPOINT ["/bin/bash"] diff --git a/.tekton/test-image/Dockerfile.base-v1.21 b/.tekton/test-image/Dockerfile.base-v1.21 new file mode 100644 index 00000000..64a143fe --- /dev/null +++ b/.tekton/test-image/Dockerfile.base-v1.21 @@ -0,0 +1,57 @@ +FROM registry.access.redhat.com/ubi9/ubi:9.8 + +LABEL name="gitops-ginkgo-test-runner-base" \ + description="Base image with CLI tools and Go toolchain" \ + maintainer="GitOps Team" + +ARG GO_VERSION=1.26.2 +ARG OC_VERSION=4.14.67 +ARG ORAS_VERSION=1.2.0 +ARG YQ_VERSION=4.44.3 + +RUN dnf -y install \ + jq \ + git \ + python3-pyyaml \ + skopeo \ + make \ + tar \ + gzip \ + file \ + && dnf clean all \ + && rm -rf /var/cache/dnf + +# Install Go from official tarball (UBI9 dnf only provides older versions) +RUN ARCH=$(case $(uname -m) in x86_64) echo -n amd64 ;; aarch64) echo -n arm64 ;; *) echo -n $(uname -m) ;; esac) \ + && curl -Lo /tmp/go.tar.gz \ + "https://go.dev/dl/go${GO_VERSION}.linux-${ARCH}.tar.gz" \ + && tar -xzf /tmp/go.tar.gz -C /usr/local \ + && rm /tmp/go.tar.gz \ + && ln -s /usr/local/go/bin/go /usr/local/bin/go \ + && ln -s /usr/local/go/bin/gofmt /usr/local/bin/gofmt \ + && go version + +# Install OpenShift CLI (oc) +RUN curl -Lo /tmp/openshift-client.tar.gz \ + https://mirror.openshift.com/pub/openshift-v4/clients/ocp/${OC_VERSION}/openshift-client-linux.tar.gz \ + && tar -xzf /tmp/openshift-client.tar.gz -C /usr/local/bin/ oc kubectl \ + && rm /tmp/openshift-client.tar.gz \ + && oc version --client + +# Install ORAS for log artifact upload +RUN ARCH=$(case $(uname -m) in x86_64) echo -n amd64 ;; aarch64) echo -n arm64 ;; *) echo -n $(uname -m) ;; esac) \ + && curl -Lo /tmp/oras.tar.gz \ + https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/oras_${ORAS_VERSION}_linux_${ARCH}.tar.gz \ + && tar -xzf /tmp/oras.tar.gz -C /usr/local/bin/ oras \ + && rm /tmp/oras.tar.gz \ + && oras version + +# Install yq for YAML parsing +RUN ARCH=$(case $(uname -m) in x86_64) echo -n amd64 ;; aarch64) echo -n arm64 ;; *) echo -n $(uname -m) ;; esac) \ + && curl -Lo /usr/local/bin/yq \ + https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${ARCH} \ + && chmod +x /usr/local/bin/yq \ + && yq --version + +WORKDIR /workspace +ENTRYPOINT ["/bin/bash"] diff --git a/.tekton/test-image/Dockerfile.testsuites b/.tekton/test-image/Dockerfile.testsuites new file mode 100644 index 00000000..17a0af1b --- /dev/null +++ b/.tekton/test-image/Dockerfile.testsuites @@ -0,0 +1,60 @@ +ARG BASE_IMAGE=quay.io/devtools_gitops/test_image:base +FROM ${BASE_IMAGE} + +LABEL name="gitops-ginkgo-test-runner-testsuites" \ + description="Base image with pre-compiled Ginkgo test suites" \ + maintainer="GitOps Team" + +ARG TEST_REPO_URL=https://github.com/redhat-developer/gitops-operator.git +ARG TEST_REPO_BRANCH=master + +WORKDIR /testsuites + +RUN git clone --depth 1 --branch ${TEST_REPO_BRANCH} ${TEST_REPO_URL} gitops-operator + +WORKDIR /testsuites/gitops-operator + +RUN make ginkgo && go mod download + +RUN ./bin/ginkgo build -r ./test/openshift/e2e/ginkgo/parallel/ + +RUN ./bin/ginkgo build -r ./test/openshift/e2e/ginkgo/sequential/ + +RUN go clean -cache \ + && rm -rf /root/go/pkg/mod/cache/download \ + && rm -rf /tmp/* \ + && rm -rf .git \ + && git init && git remote add origin ${TEST_REPO_URL} + +# Pre-compile ArgoCD E2E tests for common versions +# Note: These are compiled for the image's native architecture (TARGETARCH) +# For cross-arch testing, the test script will recompile if needed +WORKDIR /testsuites/argocd + +ARG TARGETARCH +ENV GOARCH=${TARGETARCH} + +# ArgoCD v2.14.x (stable) +ARG ARGOCD_VERSION_2_14=v2.14.1 +RUN git clone --depth 1 --branch ${ARGOCD_VERSION_2_14} https://github.com/argoproj/argo-cd.git v2.14 && \ + cd v2.14 && \ + go mod download && \ + CLIENT_VERSION=$(cat VERSION 2>/dev/null || echo ${ARGOCD_VERSION_2_14}) && \ + CLIENT_VERSION="${CLIENT_VERSION#v}" && \ + MODULE_PATH=$(head -1 go.mod | awk '{print $2}') && \ + mkdir -p dist && \ + GOOS=linux GOARCH=${TARGETARCH} go test -c \ + -ldflags "-X ${MODULE_PATH}/common.version=${CLIENT_VERSION}" \ + -o e2e.test ./test/e2e && \ + # dist/argocd is a fallback — at runtime, scripts prefer the argocd CLI + # extracted from the deployed release-candidate image + GOOS=linux GOARCH=${TARGETARCH} go build \ + -ldflags "-X ${MODULE_PATH}/common.version=${CLIENT_VERSION}" \ + -o dist/argocd ./cmd && \ + rm -rf .git /root/.cache/go-build /root/go/pkg/mod/cache + +# ArgoCD master (latest) +# Note: Skipped for now — will compile from source at runtime if master branch is requested + +WORKDIR /workspace +ENTRYPOINT ["/bin/bash"] diff --git a/.tekton/test-image/README.md b/.tekton/test-image/README.md new file mode 100644 index 00000000..15d295bf --- /dev/null +++ b/.tekton/test-image/README.md @@ -0,0 +1,253 @@ +# GitOps Test Image + +This directory contains the test image used for running integration tests in Konflux pipelines. + +## Structure + +``` +.tekton/test-image/ +├── Dockerfile # Final layer: copies scripts + ArgoCD manifests +├── Dockerfile.base # Layer 1: CLI tools + Go toolchain +├── Dockerfile.testsuites # Layer 2: Pre-compiled Ginkgo + ArgoCD E2E tests +├── argocd-v2.14.1-install.yaml # Bundled ArgoCD install manifest (templated) +├── scripts/ # Test helper scripts +│ ├── deploy-argocd-standalone.sh +│ ├── run-argocd-e2e-tests.sh +│ ├── run-parallel-tests.sh +│ └── ... +└── config/ # Test skip lists + ├── skip-parallel.txt + ├── skip-argocd.txt + └── ... +``` + +## ArgoCD Install Manifest + +### Current Version + +- **File:** `argocd-v2.14.1-install.yaml` +- **Source:** https://raw.githubusercontent.com/argoproj/argo-cd/v2.14.1/manifests/install.yaml +- **Modifications:** Namespace references replaced with `ARGOCD_NAMESPACE_PLACEHOLDER` + +### Updating ArgoCD Version + +To update to a new ArgoCD version (e.g., v2.15.0): + +1. **Download upstream install.yaml:** + ```bash + VERSION=v2.15.0 + curl -sSL https://raw.githubusercontent.com/argoproj/argo-cd/${VERSION}/manifests/install.yaml \ + > /tmp/argocd-install.yaml + ``` + +2. **Replace namespace references:** + ```bash + sed 's/namespace: argocd$/namespace: ARGOCD_NAMESPACE_PLACEHOLDER/g' \ + /tmp/argocd-install.yaml \ + > .tekton/test-image/argocd-${VERSION}-install.yaml + ``` + +3. **Verify substitution:** + ```bash + grep "ARGOCD_NAMESPACE_PLACEHOLDER" .tekton/test-image/argocd-${VERSION}-install.yaml + # Should show 3 matches (ClusterRoleBinding subjects) + ``` + +4. **Update deploy script:** + Edit `scripts/deploy-argocd-standalone.sh`: + ```bash + INSTALL_TEMPLATE="/usr/local/argocd-install/argocd-v2.15.0-install.yaml" + ``` + +5. **Update Dockerfile:** + Edit `Dockerfile`: + ```dockerfile + COPY argocd-v2.15.0-install.yaml /usr/local/argocd-install/argocd-v2.15.0-install.yaml + ``` + +6. **Update pipeline default:** + Edit `.tekton/integration-tests/pipelines/catalog-argocd-e2e.yaml`: + ```yaml + - description: ArgoCD version for upstream manifests + name: ARGOCD_VERSION + default: "v2.15.0" + ``` + +7. **Test locally:** + ```bash + podman build -f Dockerfile.base -t test-base . + podman build -f Dockerfile.testsuites --build-arg BASE_IMAGE=test-base -t test-suites . + podman build -f Dockerfile --build-arg BASE_IMAGE=test-suites -t test-final . + + # Verify file exists in image + podman run --rm test-final ls -lh /usr/local/argocd-install/ + ``` + +8. **Clean up old version (optional):** + ```bash + git rm .tekton/test-image/argocd-v2.14.1-install.yaml + ``` + +### Why Bundled Manifest? + +**Benefits:** +- ✅ No runtime network dependency on GitHub +- ✅ Exact ArgoCD version controlled in git +- ✅ Faster deployment (no curl download) +- ✅ Works in air-gapped environments +- ✅ Can test manifest changes locally +- ✅ Clear versioning (filename = ArgoCD version) + +**Tradeoffs:** +- Large file in git (~26K lines) +- Manual update process when upgrading ArgoCD + +### Namespace Placeholder + +The template uses `ARGOCD_NAMESPACE_PLACEHOLDER` which is substituted at deployment time: + +```bash +sed "s/ARGOCD_NAMESPACE_PLACEHOLDER/${NAMESPACE}/g" \ + /usr/local/argocd-install/argocd-v2.14.1-install.yaml \ + > /tmp/argocd-install.yaml +``` + +This allows deploying ArgoCD to any namespace (e.g., `argocd-e2e` for tests, `argocd` for production). + +Only 3 occurrences need substitution (ClusterRoleBinding subjects that reference ServiceAccounts). + +## Building the Test Image + +The image is built in 3 layers for efficient caching: + +### Layer 1: Base (tools + Go) +```bash +podman build -f Dockerfile.base -t quay.io/devtools_gitops/test_image:base-amd64- . +``` + +**Contains:** +- OpenShift CLI (oc) +- jq, yq, git, skopeo +- Go toolchain +- ORAS (for artifact upload) + +**Rebuilt when:** Dockerfile.base changes + +### Layer 2: Testsuites (pre-compiled tests) +```bash +podman build -f Dockerfile.testsuites \ + --build-arg BASE_IMAGE=quay.io/devtools_gitops/test_image:base-amd64- \ + -t quay.io/devtools_gitops/test_image:testsuites-amd64- . +``` + +**Contains:** +- Pre-compiled GitOps operator Ginkgo tests +- Pre-compiled ArgoCD v2.14.1 E2E tests +- Go module cache + +**Rebuilt when:** Dockerfile.base OR Dockerfile.testsuites changes + +### Layer 3: Final (scripts) +```bash +podman build -f Dockerfile \ + --build-arg BASE_IMAGE=quay.io/devtools_gitops/test_image:testsuites-amd64- \ + -t quay.io/devtools_gitops/test_image:final-amd64- . +``` + +**Contains:** +- All test scripts +- ArgoCD install manifests +- Test skip lists + +**Rebuilt when:** Any Dockerfile, script, or config changes + +## Test Scripts + +### deploy-argocd-standalone.sh + +Deploys ArgoCD in standalone mode (no GitOps operator): + +**Inputs (env vars):** +- `ARGOCD_SERVER_IMAGE` - ArgoCD server image to deploy +- `ARGOCD_VERSION` - ArgoCD version (for manifest selection) +- `NAMESPACE` - Target namespace (default: `argocd-e2e`) +- `KUBECONFIG` - Path to kubeconfig + +**Outputs (task results):** +- `namespace` - Namespace where ArgoCD is deployed +- `server` - ArgoCD server service DNS name +- `adminPassword` - Admin password from secret +- `serverName` - Server deployment name +- `repoServerName` - Repo-server deployment name +- `applicationControllerName` - App controller deployment name +- `redisName` - Redis deployment name + +**What it does:** +1. Create namespace +2. Apply ArgoCD manifests (with namespace substitution) +3. Apply OpenShift patches (SCC, redis secret) +4. Patch server deployment image +5. Wait for deployments to be ready +6. Extract admin password +7. Write task results + +### run-argocd-e2e-tests.sh + +Runs upstream ArgoCD E2E tests against deployed ArgoCD: + +**Inputs (env vars):** +- `ARGOCD_NAMESPACE` - Where ArgoCD is deployed +- `ARGOCD_SERVER` - ArgoCD server service DNS +- `ARGOCD_ADMIN_PASSWORD` - Admin password +- `ARGOCD_SERVER_NAME` - Server deployment name (for finding pods) +- `TEST_REPO_URL` - ArgoCD git repo URL +- `BRANCH` - ArgoCD version/branch to test +- `KUBECONFIG` - Path to kubeconfig + +**What it does:** +1. Check for pre-compiled tests (in `/testsuites/argocd/`) +2. Clone and compile if needed +3. Create test namespaces (`argocd-e2e-external`) +4. Deploy git-server pod (for test repos) +5. Set E2E environment variables +6. Run `./e2e.test -test.v` + +**Key environment variables set:** +- `ARGOCD_E2E_REMOTE=true` - Run tests remotely +- `ARGOCD_SERVER=:80` - ArgoCD server address +- `ARGOCD_E2E_GIT_SERVICE=git://git-server...` - Git daemon for tests +- `ARGOCD_E2E_NAMESPACE=argocd-e2e` - Test namespace +- `DIST_DIR=/tmp/.../dist` - ArgoCD CLI location + +**Does NOT:** +- Deploy ArgoCD (expects it already running) +- Install CRDs (expects them installed) +- Configure operator (standalone mode) + +## Pre-compiled Tests + +To save ~7 minutes per test run, we pre-compile test binaries in the testsuites layer. + +### GitOps Operator Tests +- Location: `/testsuites/gitops-operator/` +- Built from: https://github.com/rh-gitops-release-qa/gitops-operator.git +- Binaries: `parallel.test`, `sequential.test` + +### ArgoCD E2E Tests +- Location: `/testsuites/argocd/v2.14/` +- Built from: https://github.com/argoproj/argo-cd.git @ v2.14.1 +- Binary: `e2e.test` +- CLI: `dist/argocd` + +**Architecture detection:** Tests are compiled for the build platform (amd64 or arm64). The test script checks binary architecture and recompiles if there's a mismatch. + +## Skip Lists + +Test skip lists in `config/`: + +- `skip-parallel.txt` - GitOps operator parallel tests to skip +- `skip-sequential.txt` - GitOps operator sequential tests to skip +- `skip-argocd.txt` - ArgoCD E2E tests to skip +- `skip-ui-e2e.txt` - UI E2E tests to skip + +Format: One test name per line, regex supported, `#` for comments. diff --git a/.tekton/test-image/build-and-push.sh b/.tekton/test-image/build-and-push.sh new file mode 100755 index 00000000..a76be52a --- /dev/null +++ b/.tekton/test-image/build-and-push.sh @@ -0,0 +1,71 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CONTEXT="${SCRIPT_DIR}" +IMAGE_REPO="${IMAGE_REPO:-quay.io/devtools_gitops/test_image}" +BASE_DOCKERFILE="${BASE_DOCKERFILE:-Dockerfile.base-v1.21}" + +BUILD_ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') + +BASE_HASH=$(sha256sum "${CONTEXT}/${BASE_DOCKERFILE}" | cut -c1-12) +BASE_IMAGE="${IMAGE_REPO}:base-${BUILD_ARCH}-${BASE_HASH}" + +TESTSUITES_HASH=$(cat "${CONTEXT}/${BASE_DOCKERFILE}" "${CONTEXT}/Dockerfile.testsuites" | sha256sum | cut -c1-12) +TESTSUITES_IMAGE="${IMAGE_REPO}:testsuites-${BUILD_ARCH}-${TESTSUITES_HASH}" + +SCRIPTS_HASH=$(find "${CONTEXT}/scripts" -type f -exec sha256sum {} \; | sort | sha256sum | cut -c1-12) +FINAL_HASH=$(echo "${TESTSUITES_HASH}${SCRIPTS_HASH}" | sha256sum | cut -c1-12) +FINAL_IMAGE="${IMAGE_REPO}:final-${BUILD_ARCH}-${FINAL_HASH}" + +echo "Image repo: ${IMAGE_REPO}" +echo "Base Dockerfile: ${BASE_DOCKERFILE}" +echo "Architecture: ${BUILD_ARCH}" +echo "Base: ${BASE_IMAGE}" +echo "Testsuites: ${TESTSUITES_IMAGE}" +echo "Final: ${FINAL_IMAGE}" +echo "" + +# Layer 1: Base +if skopeo inspect "docker://${BASE_IMAGE}" >/dev/null 2>&1; then + echo "Base image exists, skipping." +else + echo "Building base image..." + podman build \ + --format=oci \ + -f "${CONTEXT}/${BASE_DOCKERFILE}" \ + -t "${BASE_IMAGE}" \ + "${CONTEXT}" + podman push "${BASE_IMAGE}" + echo "Pushed: ${BASE_IMAGE}" +fi + +# Layer 2: Testsuites +if skopeo inspect "docker://${TESTSUITES_IMAGE}" >/dev/null 2>&1; then + echo "Testsuites image exists, skipping." +else + echo "Building testsuites image..." + podman build \ + --format=oci \ + --build-arg="BASE_IMAGE=${BASE_IMAGE}" \ + -f "${CONTEXT}/Dockerfile.testsuites" \ + -t "${TESTSUITES_IMAGE}" \ + "${CONTEXT}" + podman push "${TESTSUITES_IMAGE}" + echo "Pushed: ${TESTSUITES_IMAGE}" +fi + +# Layer 3: Final (scripts overlay) +echo "Building final image..." +podman build \ + --format=oci \ + --build-arg="BASE_IMAGE=${TESTSUITES_IMAGE}" \ + -f "${CONTEXT}/Dockerfile" \ + -t "${FINAL_IMAGE}" \ + "${CONTEXT}" +podman push "${FINAL_IMAGE}" +echo "" +echo "Pushed: ${FINAL_IMAGE}" +echo "" +echo "Use this image in pipelines:" +echo " TEST_IMAGE_URL=${FINAL_IMAGE}" diff --git a/.tekton/test-image/config/dast-false-positives.json b/.tekton/test-image/config/dast-false-positives.json new file mode 100644 index 00000000..01ad14c5 --- /dev/null +++ b/.tekton/test-image/config/dast-false-positives.json @@ -0,0 +1,41 @@ +{ + "_description": "DAST scan suppression rules and alert thresholds. Edit this file to tune what the pipeline reports as FAIL vs. suppressed. Restart a scan to pick up changes (the file is baked into the overlay image).", + + "thresholds": { + "high": 0, + "medium": 10, + "low": 9999, + "informational": 9999 + }, + + "falsePositives": [ + { + "alertRef": "10015", + "reason": "ArgoCD REST API endpoints intentionally omit Cache-Control — API clients must not cache by convention" + }, + { + "alertRef": "10024", + "reason": "OAuth authorize endpoint encodes the access_token in the redirect URI — this is per-spec and expected" + }, + { + "alertRef": "10027", + "reason": "Information disclosure via URL is expected for REST API path segments" + }, + { + "alertRef": "10054", + "reason": "ArgoCD session cookie lacks SameSite — known upstream limitation, tracked separately" + }, + { + "alertRef": "10096", + "reason": "Timestamp disclosure in API responses is not sensitive in this context" + }, + { + "alertRef": "10109", + "reason": "Modern web application heuristic does not apply to ArgoCD's REST API" + }, + { + "alertRef": "10112", + "reason": "Session management rule not applicable to token-based API authentication" + } + ] +} diff --git a/.tekton/test-image/config/skip-argocd.txt b/.tekton/test-image/config/skip-argocd.txt new file mode 100644 index 00000000..7bc5dc71 --- /dev/null +++ b/.tekton/test-image/config/skip-argocd.txt @@ -0,0 +1,228 @@ +# Tests to skip when running upstream ArgoCD E2E tests on EaaS HyperShift clusters. +# One test function name per line. Blank lines and lines starting with # are ignored. +# These are combined into a Go -test.skip regex via '|'. +# +# Source: release pipeline run argocd-e2e-tests-120-414 (v1.20 / OCP 4.14) +# Tests are grouped by failure category. + +# ============================================================================ +# Account tests — configmap propagation race +# The server restarts when argocd-cm is patched to add accounts, and tests call +# update-password before the restart finishes. TestCreateAndUseAccount also +# panics with a nil pointer dereference, killing the entire test binary. +# ============================================================================ +TestCreateAndUseAccount +TestCanIGetLogs +TestAccountSessionToken + +# ============================================================================ +# Tests that fail in the release pipeline (65 failures) +# Skipping these to match the release pipeline's baseline. +# ============================================================================ + +# --- CMP (Config Management Plugin) — requires sidecar containers --- +TestCMPDiscoverWithFileName +TestCMPDiscoverWithFindCommandWithEnv +TestCMPDiscoverWithFindGlob +TestCMPDiscoverWithPluginName +TestCMPWithSymlinkFiles +TestCMPWithSymlinkFolder +TestCMPWithSymlinkPartialFiles +TestPreserveFileModeForCMP +TestPruneResourceFromCMP + +# --- Helm chart deploy — e2e-server chart fixtures fail current Helm validation --- +TestHelmRepo +TestHelmRepoDiffLocal +TestHelmWithDependencies +TestHelmWithDependenciesLegacyRepo +TestHelmWithMultipleDependencies + +# --- OCI — require local OCI registry (ports 5000/5001, not available remotely) --- +TestOCIImage +TestOCIImageWithOutOfBoundsSymlink +TestOCIWithAuthedOCIHelmRegistryDeps +TestOCIWithOCIHelmRegistryDependencies + +# --- Hydrator — requires authenticated HTTPS git --- +TestHydrateTo +TestHydratorNoOp +TestHydratorWithAuthenticatedRepo +TestHydratorWithDirectory +TestHydratorWithHelm +TestHydratorWithKustomize +TestHydratorWithPlugin +TestSimpleHydrator + +# --- SSH / private repos — require custom tools or generator features --- +TestCustomToolWithSSHGitCreds +TestCustomToolWithSSHGitCredsDisabled +TestGetRepoWithInheritedCreds +TestGitGeneratorPrivateRepoWithTemplatedProjectAndProjectScopedRepo + +# --- Server-side diff — known failures --- +TestServerSideDiffCommand +TestServerSideDiffErrorHandling +TestServerSideDiffWithLocal +TestServerSideDiffWithRevision +TestServerSideDiffWithSyncedApp +TestResourceDiffing +TestHookDiff + +# --- ApplicationSet generators — require external API access --- +TestClusterMatrixGenerator +TestClusterMergeGenerator +TestListMatrixGenerator +TestListMergeGenerator +TestMatrixTerminalMatrixGeneratorSelector +TestMatrixTerminalMergeGeneratorSelector +TestMergeTerminalMergeGeneratorSelector +TestSimplePullRequestGenerator +TestSimplePullRequestGeneratorGoTemplate +TestSimpleSCMProviderGenerator +TestSimpleSCMProviderGeneratorGoTemplate +TestSimpleSCMProviderGeneratorTokenRefStrictKo +TestSimpleSCMProviderGeneratorTokenRefStrictOk +TestSimpleListGeneratorExternalNamespace +TestPullRequestGeneratorNotAllowedSCMProvider +TestSCMProviderGeneratorSCMProviderNotAllowed + +# --- Cluster tests — crash via log.Fatalf on upsert failure --- +# TestClusterListDenied calls cluster upsert which triggers log.Fatalf +# (fixture/cluster/actions.go:63), killing the entire test binary. +TestClusterListDenied + +# --- Cluster tests — K8s version string format mismatch --- +# Tests expect short version (e.g. "1.34") but OCP reports full "v1.34.4". +# fixture.GetVersions() returns the full semver on OCP 4.21+. +TestClusterAdd +TestClusterAddAllowed +TestClusterGet + +# --- Helm tests — crash via log.Fatal in fixture/RPC EOF --- +# EnsureCleanState triggers "rpc error: code = Unavailable desc = error reading +# from server: EOF", killing the binary. Hooks crash via log.Fatal in fixture code. +TestHelmHooksAreCreated +TestHelmHookWeight +TestHelmHookDeletePolicy +TestDeclarativeHelm +TestHelmValues +TestHelmIgnoreMissingValueFiles +TestHelmCrdHook +TestHelmSet +TestHelmReleaseName + +# --- HTTPS credential template — requires insecure-skip-verify credential setup --- +TestCanAddAppFromInsecurePrivateRepoWithCredCfg + +# --- Deployment tests — require kubeconfig file with current context --- +# extractKubeConfigValues() reads ~/.kube/config to get API URL; the test-runner +# pod uses in-cluster auth which has no kubeconfig file. +TestDeployToKubernetesAPIURLWithQueryParameter +TestArgoCDSupportsMultipleServiceAccountsWithDifferingRBACOnSameCluster + +# --- Backup export — TLS cert leaks into export, breaks YAML unmarshal --- +TestBackupExportImport + +# --- Sync timeout — DeletionConfirmation consistently times out on OCP --- +TestDeletionConfirmation + +# --- Helm multi-source — also fails in downstream CI --- +TestMultiSourceAppWithHelmExternalValueFiles + +# --- ClusterDecisionResource generator — requires OCM PlacementDecision CRD --- +# PlacementDecision is an Open Cluster Management resource not installed on OCP. +TestSimpleClusterDecisionResourceGenerator +TestSimpleClusterDecisionResourceGeneratorAddingCluster +TestSimpleClusterDecisionResourceGeneratorDeletingClusterFromResource +TestSimpleClusterDecisionResourceGeneratorDeletingClusterSecret +TestSimpleClusterDecisionResourceGeneratorExternalNamespace + +# --- ApplicationSet cluster generator timeout --- +TestSyncPolicyCreateUpdate + +# --- Misc failures --- +TestAddingApp +TestClusterDelete +TestKubectlMetrics +TestManagedByURLFallbackToCurrentInstance +TestManagedByURLWithAnnotation +TestNotificationsHealthcheck + +# --- Disabled in release pipeline via source rename --- +TestSimpleGitFilesPreserveResourcesOnDeletion + +# ============================================================================ +# Tests already skipped in the release pipeline via SkipOnEnv (57 skipped) +# Adding here for completeness — prevents wasted time if env-based skip fails. +# ============================================================================ + +# --- GPG signing --- +TestSyncToSignedBranchWithKnownKey +TestSyncToSignedBranchWithUnknownKey +TestSyncToSignedCommitWithKnownKey +TestSyncToSignedCommitWithoutKnownKey +TestSyncToSignedTagWithKnownKey +TestSyncToSignedTagWithUnknownKey +TestSyncToUnsignedBranch +TestSyncToUnsignedCommit +TestSyncToUnsignedTag +TestSimpleGitDirectoryGeneratorGPGEnabledUnsignedCommits +TestSimpleGitDirectoryGeneratorGPGEnabledWithoutKnownKeys +TestSimpleGitFilesGeneratorGPGEnabledUnsignedCommits +TestSimpleGitFilesGeneratorGPGEnabledWithoutKnownKeys +TestNamespacedSyncToSignedCommitKWKK +TestNamespacedSyncToSignedCommitWKK +TestNamespacedSyncToUnsignedCommit + +# --- Helm OCI (patched with SkipOnEnv in release pipeline) --- +TestHelmOCIRegistry +TestHelmOCIRegistryWithDependencies +TestGitWithHelmOCIRegistryDependencies +TestTemplatesGitWithHelmOCIDependencies +TestTemplatesHelmOCIWithDependencies + +# --- Custom tools (patched with SkipOnEnv in release pipeline) --- +TestCustomToolSyncAndDiffLocal +TestCustomToolWithEnv +TestCustomToolWithGitCreds +TestCustomToolWithGitCredsTemplate +TestKustomizeVersionOverride + +# --- App management --- +TestAppWithSecrets +TestAutomaticallyNamingUnnamedHook +TestSyncWithSkipHook +TestSyncOptionsValidateTrue +TestImmutableChange +TestNamespacedImmutableChange + +# --- Namespace management --- +TestNamespaceAutoCreation +TestNamespaceCreationWithSSA +TestNamespacedNamespaceAutoCreation +TestNamespacedNamespaceAutoCreationWithMetadata +TestNamespacedNamespaceAutoCreationWithMetadataAndNsManifest +TestNamespacedNamespaceAutoCreationWithPreexistingNs + +# --- Logs --- +TestAppLogs +TestGetLogsAllow +TestGetLogsDeny +TestNamespacedAppLogs +TestNamespacedGetLogsAllowNS +TestNamespacedGetLogsDeny + +# --- Resource listing / orphaned --- +TestListResource +TestNamespacedListResource +TestOrphanedResource +TestNamespacedOrphanedResource +TestCRDs + +# --- ApplicationSet progressive sync --- +TestApplicationSetProgressiveSyncStep +TestNoApplicationStatusWhenNoApplications +TestNoApplicationStatusWhenNoSteps +TestProgressiveSyncHealthGating +TestProgressiveSyncMultipleAppsPerStep diff --git a/.tekton/test-image/config/skip-parallel.txt b/.tekton/test-image/config/skip-parallel.txt new file mode 100644 index 00000000..6a7f6f3a --- /dev/null +++ b/.tekton/test-image/config/skip-parallel.txt @@ -0,0 +1,4 @@ +# Tests to skip when running parallel e2e tests. +# One regex pattern per line. Blank lines and lines starting with # are ignored. +# These are passed to ginkgo --skip via a combined '|' regex. + diff --git a/.tekton/test-image/config/skip-sequential.txt b/.tekton/test-image/config/skip-sequential.txt new file mode 100644 index 00000000..55cf8634 --- /dev/null +++ b/.tekton/test-image/config/skip-sequential.txt @@ -0,0 +1,37 @@ +# Tests to skip when running sequential e2e tests on EaaS HyperShift clusters. +# One regex pattern per line. Blank lines and lines starting with # are ignored. +# These are passed to ginkgo --skip via a combined '|' regex. + +# MachineConfig tests require MCO which is not available on HyperShift. +# Application stuck OutOfSync because MachineConfig resources cannot be applied. +1-006_validate_machine_config + +# ArgoCD CLI login fails on ephemeral HyperShift clusters due to route/TLS issues. +1-064_validate_tcp_reset_error + +# Route TLS passthrough verification fails — route does not converge to expected +# state after updating ArgoCD CR on HyperShift. +1-111_validate_default_argocd_route + +# NodeSelector test sets key1:value1 but EaaS HyperShift nodes have no custom labels, +# causing all pods to get stuck in FailedScheduling. +1-071_validate_node_selectors + +# Agent/principal test requires LoadBalancer ingress which is not available on EaaS +# clusters. The argocd-hub-agent-principal deployment is never created. +1-053_validate_argocd_agent_principal_connected + +# Pod-security labels (pod-security.kubernetes.io/enforce: restricted) are not +# applied to the openshift-gitops namespace on HyperShift clusters. +1-110_validate_podsecurity_alerts + +# AllowManagedBy=false is not enforced on HyperShift — the app stays Synced +# instead of going OutOfSync. The AfterEach cleanup then hangs forever waiting +# for the ALLOW_NAMESPACE_MANAGEMENT_IN_NAMESPACE_SCOPED_INSTANCES env var to +# be removed from the operator deployment, consuming all remaining pipeline time. +1-113_validate_namespacemanagement + +# The operator bundles the openshift route plugin as a sidecar (file:/plugins/...), +# but the test expects a GitHub release download URL. This is a packaging difference, +# not a bug. +1-112_validate_rollout_plugin_support diff --git a/.tekton/test-image/config/skip-ui-e2e.txt b/.tekton/test-image/config/skip-ui-e2e.txt new file mode 100644 index 00000000..37efa406 --- /dev/null +++ b/.tekton/test-image/config/skip-ui-e2e.txt @@ -0,0 +1,3 @@ +# Tests to skip when running UI E2E tests on EaaS HyperShift clusters. +# One regex pattern per line. Blank lines and lines starting with # are ignored. +# These are passed to playwright --grep-invert as a combined '|' regex. diff --git a/.tekton/test-image/config/smoke-app/configmap.yaml b/.tekton/test-image/config/smoke-app/configmap.yaml new file mode 100644 index 00000000..789d4016 --- /dev/null +++ b/.tekton/test-image/config/smoke-app/configmap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: sanity-smoke-test +data: + purpose: "ArgoCD sync smoke test - validates that ArgoCD can sync resources from git" diff --git a/.tekton/test-image/scripts/cleanup-argocd-e2e.sh b/.tekton/test-image/scripts/cleanup-argocd-e2e.sh new file mode 100755 index 00000000..1fd2581d --- /dev/null +++ b/.tekton/test-image/scripts/cleanup-argocd-e2e.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -euo pipefail + +# Clean up all resources from an ArgoCD E2E test run. +# Safe to run multiple times. + +NAMESPACE="${NAMESPACE:-argocd-e2e}" + +echo "==========================================" +echo "ArgoCD E2E Cleanup" +echo "==========================================" +echo "Namespace: ${NAMESPACE}" +echo "" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [[ -f /usr/local/bin/lib/argocd-e2e-cleanup.sh ]]; then + source /usr/local/bin/lib/argocd-e2e-cleanup.sh +else + source "${SCRIPT_DIR}/lib/argocd-e2e-cleanup.sh" +fi +cleanup_argocd_e2e "${NAMESPACE}" + +# Delete the main namespace (this removes ArgoCD and everything in it) +echo "Deleting namespace ${NAMESPACE}..." +oc delete namespace "$NAMESPACE" --ignore-not-found --wait=false 2>/dev/null || true + +# Wait for namespace to be fully gone +echo "Waiting for namespace deletion..." +for _i in $(seq 1 60); do + if ! oc get namespace "$NAMESPACE" 2>/dev/null; then + echo "Cleanup complete" + exit 0 + fi + sleep 2 +done + +echo "WARNING: namespace ${NAMESPACE} still terminating after 2 minutes" +echo "Run 'oc get namespace ${NAMESPACE}' to check status" diff --git a/.tekton/test-image/scripts/collect-and-upload-logs.sh b/.tekton/test-image/scripts/collect-and-upload-logs.sh new file mode 100644 index 00000000..10ff3ce3 --- /dev/null +++ b/.tekton/test-image/scripts/collect-and-upload-logs.sh @@ -0,0 +1,309 @@ +#!/bin/bash +set -u + +# Environment variables expected: +# - PIPELINE_RUN_NAME +# - NAMESPACE +# - QUAY_REPO +# - QUAY_CREDENTIALS_PATH (path to .dockerconfigjson) +# - TASK_NAMES (space-separated list of task names whose logs to pull, e.g. "install-operator test-operator") +# - KUBECONFIG (optional, will be auto-detected if not set) + +# shellcheck source=./lib/oras-helpers.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib/oras-helpers.sh" +# shellcheck source=./lib/collect-pod-logs.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib/collect-pod-logs.sh" + +# Find kubeconfig — get-kubeconfig step may have failed (cluster deprovisioned) +if [ -z "${KUBECONFIG:-}" ]; then + KUBECONFIG=$(find /credentials -name "*kubeconfig" -type f 2>/dev/null | head -1) + export KUBECONFIG="${KUBECONFIG:-}" +fi + +LOGS_DIR="logs" +IMAGE_TAG="${PIPELINE_RUN_NAME}-logs" +TASK_NAMES="${TASK_NAMES:-}" +ERRORS=() + +collect_error() { + ERRORS+=("$1") + echo "WARNING: $1" +} + +echo "==========================================" +echo "Collecting logs and debug info" +echo "Pipeline: ${PIPELINE_RUN_NAME}" +echo "Namespace: ${NAMESPACE}" +echo "==========================================" + +# Configure Docker credentials for oras (needed for both pulling task logs and final push) +if ! setup_oras_auth "${QUAY_CREDENTIALS_PATH}"; then + collect_error "Quay credentials not found at ${QUAY_CREDENTIALS_PATH}" +fi + +# Create logs directory structure +mkdir -p "${LOGS_DIR}/tasks" +mkdir -p "${LOGS_DIR}/cluster-pods" +mkdir -p "${LOGS_DIR}/debug" +mkdir -p "${LOGS_DIR}/results" + +# ----------------------------------------------------------- +# 1. Pull per-task log artifacts uploaded by earlier tasks +# ----------------------------------------------------------- +if [ -n "${TASK_NAMES}" ]; then + echo "" + echo "==========================================" + echo "Pulling per-task log artifacts" + echo "==========================================" + for TASK_NAME in ${TASK_NAMES}; do + TASK_TAG="${PIPELINE_RUN_NAME}-task-${TASK_NAME}" + TASK_DIR="${LOGS_DIR}/tasks/${TASK_NAME}" + mkdir -p "${TASK_DIR}" + + echo "Pulling task logs: ${QUAY_REPO}:${TASK_TAG}" + if oras_pull_tarball "${QUAY_REPO}" "${TASK_TAG}" "${TASK_DIR}"; then + # Move test result files to the results directory + find "${TASK_DIR}" -name "*.xml" -exec cp {} "${LOGS_DIR}/results/" \; 2>/dev/null || true + find "${TASK_DIR}" -name "*.json" -exec cp {} "${LOGS_DIR}/results/" \; 2>/dev/null || true + else + collect_error "Could not pull logs for task ${TASK_NAME} (may not have uploaded)" + fi + done +fi + +# ----------------------------------------------------------- +# 2-4. Collect cluster debug info (only if cluster is reachable) +# ----------------------------------------------------------- +CLUSTER_REACHABLE=false +if [ -n "${KUBECONFIG:-}" ] && [ -f "${KUBECONFIG:-}" ]; then + export KUBECONFIG + if oc whoami --request-timeout=10s &>/dev/null; then + CLUSTER_REACHABLE=true + else + collect_error "Cluster unreachable (kubeconfig exists but oc commands fail)" + fi +else + collect_error "Kubeconfig not available (file: ${KUBECONFIG:-unset})" +fi + +if [ "$CLUSTER_REACHABLE" = true ]; then + echo "" + echo "Collecting cluster debug information..." + + { + echo "--- Cluster Version ---" + oc version 2>&1 || true + echo "" + echo "--- Cluster Operators ---" + oc get co 2>&1 || true + echo "" + echo "--- Nodes ---" + oc get nodes -o wide 2>&1 || true + } > "${LOGS_DIR}/debug/cluster-info.txt" 2>&1 || collect_error "Failed to collect cluster info" + + echo "Collecting namespace debug information..." + + { + echo "--- All Resources in ${NAMESPACE} ---" + oc get all -n "${NAMESPACE}" -o wide 2>&1 || true + echo "" + echo "--- Subscriptions ---" + oc get subscriptions -n "${NAMESPACE}" -o yaml 2>&1 || true + echo "" + echo "--- ClusterServiceVersions ---" + oc get csv -n "${NAMESPACE}" -o yaml 2>&1 || true + echo "" + echo "--- InstallPlans ---" + oc get installplans -n "${NAMESPACE}" -o yaml 2>&1 || true + } > "${LOGS_DIR}/debug/namespace-resources.txt" 2>&1 || collect_error "Failed to collect namespace resources" + + { + echo "--- Events ---" + oc get events -n "${NAMESPACE}" --sort-by='.lastTimestamp' 2>&1 || true + } > "${LOGS_DIR}/debug/events.txt" 2>&1 || collect_error "Failed to collect events" + + { + echo "--- CatalogSource ---" + oc get catalogsource -n openshift-marketplace -o yaml 2>&1 || true + } > "${LOGS_DIR}/debug/catalogsource.txt" 2>&1 || collect_error "Failed to collect catalogsource" + + echo "Collecting pod logs..." + if ! collect_pod_logs "${NAMESPACE}" "${LOGS_DIR}/cluster-pods" true; then + collect_error "Failed to collect pod logs" + fi +else + echo "Skipping cluster debug collection (cluster unreachable)" +fi + +# ----------------------------------------------------------- +# 4.5. Collect build metadata (component versions) +# ----------------------------------------------------------- +SHARED_DIR="${SHARED_DIR:-/shared}" +if [ "$CLUSTER_REACHABLE" = true ]; then + echo "" + echo "Collecting build metadata..." + /usr/local/bin/collect-build-metadata.sh "${SHARED_DIR}/build-metadata.json" || \ + collect_error "Failed to collect build metadata" +fi + +# ----------------------------------------------------------- +# 5. Parse JUnit test results (if available) +# ----------------------------------------------------------- +TEST_SUMMARY="" +JUNIT_FILE=$(find "${LOGS_DIR}/results" -name "*.xml" -type f 2>/dev/null | head -1) +if [ -f "${JUNIT_FILE:-}" ]; then + echo "Parsing test results from ${JUNIT_FILE}..." + if python3 /usr/local/bin/parse-test-results.py \ + "${JUNIT_FILE}" "${SHARED_DIR}/test-results.json" 2>&1; then + TEST_SUMMARY="Tests: $(python3 -c "import json; print(json.load(open('${SHARED_DIR}/test-results.json'))['summary'])")" + else + echo "WARNING: Failed to parse JUnit XML" + fi +fi + +# ----------------------------------------------------------- +# 6. Create README +# ----------------------------------------------------------- +{ + echo "Pipeline Run Logs - ${PIPELINE_RUN_NAME}" + echo "Namespace - ${NAMESPACE}" + echo "Collected - $(date -u +"%Y-%m-%d %H:%M:%S UTC")" + echo "" + if [ -n "$TEST_SUMMARY" ]; then + echo "Test Summary: ${TEST_SUMMARY}" + echo "" + fi + echo "Structure:" + echo " - tasks/ : stdout/stderr from each pipeline task step" + echo " - results/ : test result files (JUnit XML, JSON reports)" + echo " - cluster-pods/ : pod logs from the ephemeral cluster" + echo " - debug/ : cluster and namespace debug information" + echo "" + echo "Files:" + find "${LOGS_DIR}/" -type f | sort | sed 's/^/ - /' + echo "" + if [ ${#ERRORS[@]} -gt 0 ]; then + echo "Collection warnings:" + for err in "${ERRORS[@]}"; do + echo " - ${err}" + done + echo "" + fi + echo "To extract these logs:" + echo " oras pull ${QUAY_REPO}:${IMAGE_TAG}" + echo " tar xzf ${PIPELINE_RUN_NAME}-logs.tar.gz" +} > "${LOGS_DIR}/README.txt" + +# ----------------------------------------------------------- +# 7. Create CLAUDE.md for self-documenting analysis +# ----------------------------------------------------------- +cat > "${LOGS_DIR}/CLAUDE.md" << 'CLAUDE_EOF' +# GitOps Operator Integration Test Logs + +These logs are from a Konflux pipeline that installs the GitOps operator +on an ephemeral EaaS HyperShift cluster and runs e2e tests. + +## Quick diagnosis + +1. **Test results**: Check `results/junit-results.xml` for pass/fail summary. + Count failures: `grep -c 'failure message' results/junit-results.xml` + +2. **Test output**: Check `tasks/test-operator/test-operator.log` for test + stdout/stderr. Search for `FAIL` or `--- FAIL` to find failing tests. + +3. **Operator install**: Check `tasks/install-operator/install-operator.log` + for operator deployment issues. + +4. **Pod health**: Check `cluster-pods/` for ArgoCD component logs. + +5. **Cluster events**: Check `debug/events.txt` for scheduling, image pull, + or crash loop issues. + +## Common failure patterns + +| Symptom | Where to look | Likely cause | +|---------|--------------|--------------| +| `ImagePullBackOff` in events | debug/events.txt, install-operator.log | Pull secret not propagated to HyperShift nodes | +| `exec format error` | test-operator.log | Architecture mismatch (ARM image on x86 or vice versa) | +| Test timeout (no results) | test-operator.log (last test name) | A test hung — check which test was running last | +| `FailedScheduling` | debug/events.txt | Node selector mismatch or insufficient resources | +| `MachineConfig` failures | test-operator.log | MCO not available on HyperShift — should be in skip list | +| 464/470 argo tests fail | tasks/test-operator/argocd-e2e.log | `argocd-delete` plugin missing — kubectl is wrong binary | +| `connection refused` | test-operator.log | ArgoCD server not ready or port-forward failed | + +## File structure + +``` +logs/ +├── CLAUDE.md ← you are here +├── README.txt ← pipeline run metadata and test summary +├── tasks/ ← per-task stdout/stderr from pipeline steps +│ ├── install-operator/ +│ │ ├── install-operator.log ← stdout/stderr +│ │ ├── env.sh ← environment variables at execution time +│ │ ├── kubeconfig ← cluster credentials (if present) +│ │ └── reproduce.sh ← script showing how to reproduce the run +│ └── test-operator/ +│ ├── test-operator.log +│ ├── env.sh +│ ├── kubeconfig +│ ├── reproduce.sh +│ └── *.xml, *.json ← test results (JUnit, JSON) +├── results/ ← copies of JUnit XML and JSON reports +├── cluster-pods/ ← pod logs from the ephemeral test cluster +└── debug/ ← cluster state: events, resources, catalog +``` + +## Reproducing a task locally + +Each task directory includes: +- **env.sh**: Environment variables (credentials filtered out) +- **kubeconfig**: Cluster credentials (if task had cluster access) +- **reproduce.sh**: Instructions for reproducing the task execution + +To reproduce a failed task: +```bash +cd tasks/install-operator/ +source env.sh +export KUBECONFIG=kubeconfig +cat reproduce.sh # Review the original command +``` + +## Analysis workflow + +1. Read `README.txt` for the test summary line +2. If tests failed, read the test log to identify which tests failed and why +3. If operator install failed, check install log for image pull or timeout issues +4. Cross-reference with cluster events and pod logs for infrastructure problems +5. Check if failures match known HyperShift limitations (skip list candidates) +6. Use env.sh + kubeconfig to reproduce the task execution locally +CLAUDE_EOF + +# ----------------------------------------------------------- +# 8. Upload combined logs to Quay +# ----------------------------------------------------------- +echo "" +echo "==========================================" +echo "Uploading combined logs to Quay" +echo "==========================================" + +echo "Uploading logs to ${QUAY_REPO}:${IMAGE_TAG}..." + +if UPLOADED_REF=$(oras_push_tarball "${LOGS_DIR}" "${QUAY_REPO}" "${IMAGE_TAG}" \ + "application/vnd.konflux.logs.v1+tar" "${PIPELINE_RUN_NAME}"); then + echo "Successfully pushed combined log artifact to ${UPLOADED_REF}" +else + echo "ERROR: Failed to push log artifact to ${QUAY_REPO}:${IMAGE_TAG}" + echo "Logs were collected locally but could not be uploaded." +fi + +echo "" +echo "Contents:" +find "${LOGS_DIR}/" -type f | sort | head -50 +echo "" +echo "Total size:" +du -sh "${LOGS_DIR}/" +if [ ${#ERRORS[@]} -gt 0 ]; then + echo "" + echo "Collection completed with ${#ERRORS[@]} warning(s)." +fi diff --git a/.tekton/test-image/scripts/collect-build-metadata.sh b/.tekton/test-image/scripts/collect-build-metadata.sh new file mode 100755 index 00000000..e3dcd32d --- /dev/null +++ b/.tekton/test-image/scripts/collect-build-metadata.sh @@ -0,0 +1,82 @@ +#!/bin/bash +set -uo pipefail + +# Collects component versions from a running GitOps operator cluster. +# Usage: collect-build-metadata.sh [output-json-path] +# Requires: KUBECONFIG set, cluster reachable, oc available + +OUTPUT="${1:-/shared/build-metadata.json}" +NS="openshift-gitops" +OP_NS="openshift-gitops-operator" + +find_pod() { + local prefix="$1" + oc get pods -n "$NS" --field-selector=status.phase=Running -o name 2>/dev/null \ + | grep "/${prefix}" \ + | head -1 \ + | sed 's|^pod/||' +} + +echo "Collecting build metadata from cluster..." + +BUILD=$(oc get csv -n "$OP_NS" -o jsonpath='{.items[0].spec.version}' 2>/dev/null || echo "") + +SERVER_POD=$(find_pod "openshift-gitops-server") +DEX_POD=$(find_pod "openshift-gitops-dex-server") +REDIS_POD=$(find_pod "openshift-gitops-redis") + +ARGOCD="" +KUSTOMIZE="" +HELM="" +GIT_LFS="" +DEX="" +REDIS="" +AGENT="" + +if [ -n "$SERVER_POD" ]; then + echo " Server pod: $SERVER_POD" + ARGOCD=$(oc exec -n "$NS" "$SERVER_POD" -- argocd version --client --short 2>/dev/null | sed 's/argocd: //' || true) + KUSTOMIZE=$(oc exec -n "$NS" "$SERVER_POD" -- kustomize version 2>/dev/null | tr -d '[:space:]' || true) + HELM=$(oc exec -n "$NS" "$SERVER_POD" -- helm version --short 2>/dev/null | sed 's/+.*//' || true) + GIT_LFS=$(oc exec -n "$NS" "$SERVER_POD" -- git-lfs version 2>/dev/null | grep -oP 'git-lfs/\K[^ ]+' || true) +else + echo " WARNING: No running openshift-gitops-server pod found" +fi + +if [ -n "$DEX_POD" ]; then + echo " Dex pod: $DEX_POD" + DEX=$(oc exec -n "$NS" "$DEX_POD" -- dex version 2>/dev/null | grep 'Dex Version:' | sed 's/.*: //' | tr -d '[:space:]' || true) +else + echo " WARNING: No running dex pod found" +fi + +if [ -n "$REDIS_POD" ]; then + echo " Redis pod: $REDIS_POD" + REDIS=$(oc exec -n "$NS" "$REDIS_POD" -- redis-server -v 2>/dev/null | grep -oP 'v=\K[^ ]+' || true) +else + echo " WARNING: No running redis pod found" +fi + +AGENT_IMAGE=$(oc get deployment -n "$NS" -l app.kubernetes.io/component=agent-principal \ + -o jsonpath='{.items[0].spec.template.spec.containers[0].image}' 2>/dev/null || true) +if [ -n "$AGENT_IMAGE" ]; then + AGENT=$(echo "$AGENT_IMAGE" | grep -oP ':\K.*' || true) +fi + +python3 -c " +import json +data = { + 'build': '''${BUILD}''', + 'argocd': '''${ARGOCD}''', + 'dex': '''${DEX}''', + 'redis': '''${REDIS}''', + 'kustomize': '''${KUSTOMIZE}''', + 'helm': '''${HELM}''', + 'gitLfs': '''${GIT_LFS}''', + 'agent': '''${AGENT}''', +} +data = {k: v for k, v in data.items() if v} +with open('''${OUTPUT}''', 'w') as f: + json.dump(data, f, indent=2) +print('Build metadata:', json.dumps(data, separators=(', ', ': '))) +" diff --git a/.tekton/test-image/scripts/collect-logs-sidecar.sh b/.tekton/test-image/scripts/collect-logs-sidecar.sh new file mode 100644 index 00000000..82b8dddd --- /dev/null +++ b/.tekton/test-image/scripts/collect-logs-sidecar.sh @@ -0,0 +1,135 @@ +#!/bin/bash +set -u + +# Environment variables expected: +# - KUBECONFIG_NAME +# - PIPELINE_RUN_NAME +# - BRANCH_NAME (used as the task artifact name, e.g. "logs" → tag: ${PIPELINE_RUN_NAME}-task-logs) +# - NAMESPACE +# - QUAY_REPO + +# shellcheck source=./lib/oras-helpers.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib/oras-helpers.sh" +# shellcheck source=./lib/collect-pod-logs.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib/collect-pod-logs.sh" + +LOGS_DIR="logs" +COLLECT_INTERVAL=30 +UPLOAD_EVERY=10 # upload every 10 snapshots (10 * 30s = 5 min) + +TIMEOUT=300 +ELAPSED=0 + +if [ "${KUBECONFIG_NAME}" = "auto" ]; then + while [ $ELAPSED -lt $TIMEOUT ]; do + KUBECONFIG_PATH=$(find /credentials -name "*kubeconfig" -type f 2>/dev/null | head -1) + if [ -n "${KUBECONFIG_PATH:-}" ]; then break; fi + sleep 5 + ELAPSED=$((ELAPSED + 5)) + done +else + KUBECONFIG_PATH="/credentials/${KUBECONFIG_NAME}" + while [ ! -f "${KUBECONFIG_PATH}" ] && [ $ELAPSED -lt $TIMEOUT ]; do + sleep 5 + ELAPSED=$((ELAPSED + 5)) + done +fi + +CLUSTER_AVAILABLE=false +if [ -n "${KUBECONFIG_PATH:-}" ] && [ -f "${KUBECONFIG_PATH:-}" ]; then + export KUBECONFIG="${KUBECONFIG_PATH}" + if oc whoami --request-timeout=10s &>/dev/null; then + CLUSTER_AVAILABLE=true + else + echo "WARNING: Kubeconfig found but cluster is unreachable" + fi +else + echo "WARNING: Kubeconfig not found after ${TIMEOUT}s — will skip cluster log collection" + echo "Contents of /credentials:" + ls -la /credentials/ 2>/dev/null || true +fi + +# Configure Docker credentials for oras (needed for periodic uploads) +setup_oras_auth + +mkdir -p "${LOGS_DIR}" + +# --- Helper functions --- + +collect_snapshot() { + if [ "$CLUSTER_AVAILABLE" != true ]; then return 0; fi + if ! oc get pods -n "${NAMESPACE}" &>/dev/null; then + return 0 + fi + + local timestamp + timestamp=$(date +%s) + local snapshot_dir="${LOGS_DIR}/snapshot-${timestamp}" + + collect_pod_logs_with_tail "${NAMESPACE}" "${snapshot_dir}" 100 +} + +collect_final() { + if [ "$CLUSTER_AVAILABLE" != true ]; then return 0; fi + + FINAL_DIR="${LOGS_DIR}/final" + collect_pod_logs "${NAMESPACE}" "${FINAL_DIR}" +} + +generate_readme() { + { + echo "Sidecar Logs - ${PIPELINE_RUN_NAME}" + echo "Namespace - ${NAMESPACE}" + echo "Collected - $(date -u +"%Y-%m-%d %H:%M:%S UTC")" + echo "Cluster available: ${CLUSTER_AVAILABLE}" + echo "Upload #${UPLOAD_COUNT}" + echo "" + echo "Structure:" + echo " - snapshot-* directories: periodic snapshots during test execution" + echo " - final/ directory: complete logs at test completion (if present)" + echo "" + echo "Files:" + find "${LOGS_DIR}/" -type f -name "*.log" 2>/dev/null | sort | sed 's/^/ - /' + echo "" + echo "To extract these logs:" + echo " oras pull ${QUAY_REPO}:${PIPELINE_RUN_NAME}-task-${BRANCH_NAME}" + echo " tar xzf ${PIPELINE_RUN_NAME}-task-${BRANCH_NAME}-logs.tar.gz" + } > "${LOGS_DIR}/README.txt" +} + +upload_logs() { + UPLOAD_COUNT=$((UPLOAD_COUNT + 1)) + generate_readme + + local tag="${PIPELINE_RUN_NAME}-task-${BRANCH_NAME}" + local size=$(du -sh "${LOGS_DIR}" 2>/dev/null | cut -f1) + + if oras_push_tarball "${LOGS_DIR}" "${QUAY_REPO}" "${tag}" \ + "application/vnd.konflux.logs.v1+tar" "${tag}" >/dev/null; then + echo "Sidecar upload #${UPLOAD_COUNT} pushed to ${QUAY_REPO}:${tag} (${size})" + else + echo "WARNING: Sidecar upload #${UPLOAD_COUNT} failed" + fi +} + +# --- Main loop --- + +SNAPSHOT_COUNT=0 +UPLOAD_COUNT=0 + +while true; do + collect_snapshot + SNAPSHOT_COUNT=$((SNAPSHOT_COUNT + 1)) + + if (( SNAPSHOT_COUNT % UPLOAD_EVERY == 0 )); then + upload_logs + fi + + sleep ${COLLECT_INTERVAL} & + wait $! || break +done + +# Final comprehensive collection + upload (best effort) +echo "Sidecar exiting — collecting final logs and uploading" +collect_final +upload_logs diff --git a/.tekton/test-image/scripts/deploy-argocd-standalone.sh b/.tekton/test-image/scripts/deploy-argocd-standalone.sh new file mode 100755 index 00000000..57fe6fb3 --- /dev/null +++ b/.tekton/test-image/scripts/deploy-argocd-standalone.sh @@ -0,0 +1,146 @@ +#!/bin/bash +set -euo pipefail + +# Deploy ArgoCD in standalone mode (without the GitOps operator). +# Uses upstream ArgoCD manifests but overrides the server image to test a specific build. +# +# Environment variables expected: +# - ARGOCD_SERVER_IMAGE: ArgoCD server image to deploy +# - ARGOCD_VERSION: ArgoCD version for upstream manifests (default: v2.14.1) +# - NAMESPACE: Namespace to deploy ArgoCD (default: argocd) +# - KUBECONFIG: Path to kubeconfig + +ARGOCD_SERVER_IMAGE="${ARGOCD_SERVER_IMAGE:?ARGOCD_SERVER_IMAGE must be set}" +ARGOCD_VERSION="${ARGOCD_VERSION:-v2.14.1}" +NAMESPACE="${NAMESPACE:-argocd}" + +echo "==========================================" +echo "Deploying ArgoCD standalone" +echo "==========================================" +echo "ArgoCD version: ${ARGOCD_VERSION}" +echo "Server image: ${ARGOCD_SERVER_IMAGE}" +echo "Namespace: ${NAMESPACE}" +echo "" + +# Create namespace +echo "Creating namespace ${NAMESPACE}..." +oc create namespace "$NAMESPACE" --dry-run=client -o yaml | oc apply -f - + +# Download upstream ArgoCD manifests for the requested version +echo "Downloading ArgoCD ${ARGOCD_VERSION} manifests from upstream..." +UPSTREAM_URL="https://raw.githubusercontent.com/argoproj/argo-cd/${ARGOCD_VERSION}/manifests/install.yaml" +curl -sSL --fail "$UPSTREAM_URL" -o /tmp/argocd-upstream.yaml + +# Replace hardcoded namespace in ClusterRoleBinding subjects +sed "s/namespace: argocd/namespace: ${NAMESPACE}/g" /tmp/argocd-upstream.yaml > /tmp/argocd-install.yaml + +# Apply manifests +echo "Applying ArgoCD manifests to namespace ${NAMESPACE}..." +oc apply -n "$NAMESPACE" -f /tmp/argocd-install.yaml + +# OpenShift-specific fixes +echo "Applying OpenShift-specific patches..." + +# Create argocd-redis secret with a real password (empty string causes Redis --requirepass to fail) +if ! oc get secret argocd-redis -n "$NAMESPACE" &>/dev/null; then + echo " Creating argocd-redis secret..." + oc create secret generic argocd-redis \ + --from-literal=auth="argocd-e2e-redis-password" \ + -n "$NAMESPACE" +fi + +# Grant anyuid SCC to service accounts to allow running as UID 999 +# Upstream ArgoCD manifests use hardcoded UIDs that don't match OpenShift's allocated ranges +echo " Granting anyuid SCC to ArgoCD service accounts..." +for sa in argocd-application-controller argocd-server argocd-repo-server argocd-dex-server argocd-redis; do + oc adm policy add-scc-to-user anyuid -z "$sa" -n "$NAMESPACE" 2>/dev/null || true +done + +# Remove seccompProfile from Redis deployment — upstream sets RuntimeDefault which +# OpenShift's anyuid SCC rejects +echo " Patching Redis deployment to remove seccompProfile..." +oc patch deployment argocd-redis -n "$NAMESPACE" --type json -p '[ + {"op": "remove", "path": "/spec/template/spec/securityContext/seccompProfile"}, + {"op": "remove", "path": "/spec/template/spec/initContainers/0/securityContext/seccompProfile"} +]' 2>/dev/null || true + +# Patch argocd-server deployment to use custom image +echo "Patching argocd-server to use image: ${ARGOCD_SERVER_IMAGE}" +oc set image deployment/argocd-server \ + argocd-server="$ARGOCD_SERVER_IMAGE" \ + -n "$NAMESPACE" + +# Wait for deployments to be ready +echo "Waiting for ArgoCD deployments to become ready..." +for deploy in argocd-redis argocd-server argocd-repo-server argocd-dex-server; do + echo " Waiting for $deploy..." + if ! oc wait --for=condition=Available deployment/"$deploy" -n "$NAMESPACE" --timeout=10m; then + echo "ERROR: deployment/$deploy did not become Available" + oc get deployment "$deploy" -n "$NAMESPACE" -o wide 2>/dev/null || true + oc get pods -n "$NAMESPACE" -o wide 2>/dev/null || true + oc get events -n "$NAMESPACE" --sort-by='.lastTimestamp' 2>/dev/null | tail -30 || true + exit 1 + fi +done + +# Wait for application-controller statefulset +echo " Waiting for argocd-application-controller..." +if ! oc rollout status statefulset/argocd-application-controller -n "$NAMESPACE" --timeout=10m; then + echo "ERROR: statefulset/argocd-application-controller did not become ready" + oc get pods -n "$NAMESPACE" -o wide 2>/dev/null || true + oc get events -n "$NAMESPACE" --sort-by='.lastTimestamp' 2>/dev/null | tail -30 || true + exit 1 +fi + +echo "" +echo "==========================================" +echo "ArgoCD deployed successfully" +echo "==========================================" +echo "" + +# Show deployment status +oc get deployments,statefulsets,pods -n "$NAMESPACE" -o wide + +# Create Route to expose ArgoCD server externally (for cross-cluster access from Konflux) +echo "" +echo "Creating external Route for ArgoCD server..." +oc create route passthrough argocd-server --service=argocd-server --port=https -n "$NAMESPACE" 2>/dev/null || \ + echo " Route already exists" + +# Get external Route URL +ARGOCD_SERVER_URL=$(oc get route argocd-server -n "$NAMESPACE" -o jsonpath='{.spec.host}') + +if [ -z "$ARGOCD_SERVER_URL" ]; then + echo "ERROR: Failed to get ArgoCD server route URL" + exit 1 +fi + +echo "ArgoCD server route: https://${ARGOCD_SERVER_URL}" + +# Extract admin password +ADMIN_PASSWORD=$(oc get secret argocd-initial-admin-secret -n "$NAMESPACE" -o jsonpath='{.data.password}' 2>/dev/null | base64 -d || echo "") + +if [ -z "$ADMIN_PASSWORD" ]; then + echo "WARNING: Could not extract admin password from argocd-initial-admin-secret" + # Try cluster secret (some ArgoCD versions use this) + ADMIN_PASSWORD=$(oc get secret argocd-cluster -n "$NAMESPACE" -o jsonpath='{.data.admin\.password}' 2>/dev/null | base64 -d || echo "password") +fi + +echo "" +echo "ArgoCD server URL: https://${ARGOCD_SERVER_URL}" +echo "Admin username: admin" +echo "Admin password: ${ADMIN_PASSWORD:0:8}..." # Print first 8 chars only + +# Write task results for use by test task +# These will be available as $(tasks.deploy-argocd.results.xxx) +if [ -d /tekton/results ]; then + echo -n "$NAMESPACE" > /tekton/results/namespace + echo -n "$ARGOCD_SERVER_URL" > /tekton/results/server + echo -n "$ADMIN_PASSWORD" > /tekton/results/adminPassword + echo -n "argocd-server" > /tekton/results/serverName + echo -n "argocd-repo-server" > /tekton/results/repoServerName + echo -n "argocd-application-controller" > /tekton/results/applicationControllerName + echo -n "argocd-redis" > /tekton/results/redisName + + echo "Task results written to /tekton/results/" +fi diff --git a/.tekton/test-image/scripts/deploy-e2e-server.sh b/.tekton/test-image/scripts/deploy-e2e-server.sh new file mode 100755 index 00000000..4bc9d7aa --- /dev/null +++ b/.tekton/test-image/scripts/deploy-e2e-server.sh @@ -0,0 +1,99 @@ +#!/bin/bash +set -euo pipefail + +# Deploy the upstream ArgoCD E2E test server (argocd-e2e-server). +# Provides git repos over HTTP, HTTPS (basic auth), HTTPS (client cert), +# SSH, and Helm chart repos — matching upstream test/remote infrastructure. + +NAMESPACE="${ARGOCD_NAMESPACE:-argocd-e2e}" +E2E_SERVER_IMAGE="${E2E_SERVER_IMAGE:-quay.io/redhat-developer/argocd-e2e-cluster:latest}" + +echo "Deploying argocd-e2e-server in namespace ${NAMESPACE}..." +echo " Image: ${E2E_SERVER_IMAGE}" + +cat </dev/null 2>&1 || PKGS="$PKGS git" + command -v gpg >/dev/null 2>&1 || PKGS="$PKGS gnupg2" + command -v make >/dev/null 2>&1 || PKGS="$PKGS make" + if [[ -n "$PKGS" ]]; then + echo " Installing: $PKGS" + dnf install -y $PKGS >/dev/null 2>&1 + else + echo " All required packages already installed" + fi + + # Install kubectl if not available (needed by ArgoCD E2E test framework) + if ! command -v kubectl >/dev/null 2>&1; then + echo " Installing kubectl..." + ARCH=$(uname -m) + case "${ARCH}" in + x86_64) ARCH="amd64" ;; + aarch64) ARCH="arm64" ;; + esac + curl -sLo /tmp/bin/kubectl "https://dl.k8s.io/release/$(curl -sL https://dl.k8s.io/release/stable.txt)/bin/linux/${ARCH}/kubectl" + chmod +x /tmp/bin/kubectl + echo " kubectl installed at /tmp/bin/kubectl" + fi + + # Install helm if not available (needed by Helm E2E tests) + if ! command -v helm >/dev/null 2>&1; then + echo " Installing helm..." + curl -sL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | HELM_INSTALL_DIR=/tmp/bin bash + echo " helm installed at /tmp/bin/helm" + fi + + # Install kustomize if not available (needed by local sync and diffing tests) + if ! command -v kustomize >/dev/null 2>&1; then + echo " Installing kustomize..." + ARCH=$(uname -m) + case "${ARCH}" in + x86_64) ARCH="amd64" ;; + aarch64) ARCH="arm64" ;; + esac + curl -sLo /tmp/kustomize.tar.gz "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv5.6.0/kustomize_v5.6.0_linux_${ARCH}.tar.gz" + tar -xzf /tmp/kustomize.tar.gz -C /tmp/bin kustomize + rm /tmp/kustomize.tar.gz + chmod +x /tmp/bin/kustomize + echo " kustomize installed at /tmp/bin/kustomize" + fi +' + +echo "e2e-test-runner pod is ready" +oc get pod e2e-test-runner -n "${NAMESPACE}" -o wide diff --git a/.tekton/test-image/scripts/extract-argocd-image-from-catalog.py b/.tekton/test-image/scripts/extract-argocd-image-from-catalog.py new file mode 100644 index 00000000..3bc9d45d --- /dev/null +++ b/.tekton/test-image/scripts/extract-argocd-image-from-catalog.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +"""Extract ArgoCD server image from an operator catalog. + +Parses a File-Based Catalog (FBC) to find the latest bundle for +openshift-gitops-operator, then extracts the ArgoCD server image +from the bundle's relatedImages in its ClusterServiceVersion. + +Environment variables: + CATALOG_IMAGE (required) Full catalog image reference + OPERATOR_CHANNEL Operator channel (default: latest) + OPERATOR_NAME Package name (default: openshift-gitops-operator) + +Output: + Writes image ref to /shared/argocd-image.txt and prints to stdout. +""" + +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +import yaml + +PULL_CREDS = Path("/quay-pull-credentials/.dockerconfigjson") + + +def run_cmd(cmd, *, env=None): + result = subprocess.run( + cmd, shell=True, capture_output=True, text=True, timeout=300, env=env, + ) + if result.returncode != 0: + print(f"Command failed: {cmd}", file=sys.stderr) + if result.stdout: + print(result.stdout, file=sys.stderr) + if result.stderr: + print(result.stderr, file=sys.stderr) + return result + + +def setup_registry_auth(work_dir): + if not PULL_CREDS.is_file(): + print("WARNING: Pull credentials not found at", PULL_CREDS) + return None + + print("Configuring registry authentication...") + auth_dir = Path(work_dir) / "docker-auth" + auth_dir.mkdir() + shutil.copy2(PULL_CREDS, auth_dir / "config.json") + + with open(PULL_CREDS) as f: + creds = json.load(f) + registries = list(creds.get("auths", {}).keys()) + if "registry.redhat.io" in registries: + print(" Found registry.redhat.io credentials in pull secret") + else: + print(" WARNING: registry.redhat.io NOT found in pull secret") + print(" Available registries:", ", ".join(registries)) + + return str(auth_dir) + + +def extract_catalog_json(catalog_image, operator_name, work_dir, docker_config): + extract_dir = Path(work_dir) / "extract" + extract_dir.mkdir() + + print("Extracting catalog.json...") + env = dict(os.environ) + if docker_config: + env["DOCKER_CONFIG"] = docker_config + + result = run_cmd( + f"oc image extract {catalog_image}" + f" --path /configs/{operator_name}/catalog.json:{extract_dir}", + env=env, + ) + if result.returncode != 0: + print(f"ERROR: Failed to extract catalog.json from {catalog_image}") + sys.exit(1) + + catalog_json = extract_dir / "catalog.json" + if not catalog_json.is_file() or catalog_json.stat().st_size == 0: + print("ERROR: catalog.json not found or empty") + sys.exit(1) + + print(f"Successfully extracted catalog.json ({catalog_json.stat().st_size} bytes)") + return catalog_json + + +def parse_fbc_entries(catalog_json): + """Parse FBC catalog into a list of dicts. + + Handles both NDJSON (one object per line) and pretty-printed + multi-document JSON (multiple objects concatenated across lines). + """ + with open(catalog_json) as f: + raw = f.read() + + decoder = json.JSONDecoder() + entries = [] + pos = 0 + length = len(raw) + while pos < length: + while pos < length and raw[pos] in " \t\n\r": + pos += 1 + if pos >= length: + break + try: + obj, end = decoder.raw_decode(raw, pos) + entries.append(obj) + pos = end + except json.JSONDecodeError: + pos += 1 + + return entries + + +def find_bundle_in_catalog(catalog_json, operator_name, channel): + print(f"Parsing catalog for package: {operator_name}, channel: {channel}") + entries = parse_fbc_entries(catalog_json) + + channel_entries = [ + e for e in entries + if e.get("schema") == "olm.channel" + and e.get("package") == operator_name + and e.get("name") == channel + ] + + if not channel_entries: + print(f"ERROR: Channel {channel} not found for package {operator_name}") + available = [ + e["name"] for e in entries + if e.get("schema") == "olm.channel" and e.get("package") == operator_name + ] + if available: + print("Available channels:", ", ".join(available)) + sys.exit(1) + + channel_entry = channel_entries[0] + entry_list = channel_entry.get("entries", []) + if not entry_list: + print("ERROR: No entries in channel") + sys.exit(1) + + bundle_name = entry_list[-1].get("name") or entry_list[0].get("name") + if not bundle_name: + print("ERROR: Channel entries have no bundle name") + sys.exit(1) + print(f"Found latest bundle: {bundle_name}") + + bundle_entries = [ + e for e in entries + if e.get("schema") == "olm.bundle" and e.get("name") == bundle_name + ] + if not bundle_entries: + print(f"ERROR: Could not find bundle entry for {bundle_name}") + sys.exit(1) + + bundle_image = bundle_entries[0].get("image") + if not bundle_image: + print(f"ERROR: No image field in bundle {bundle_name}") + sys.exit(1) + + print(f"Found bundle image from catalog: {bundle_image}") + return bundle_name, bundle_image + + +def remap_bundle_image(bundle_name, bundle_image): + if not bundle_image.startswith("registry.redhat.io/"): + return bundle_image + + print("Remapping bundle from registry.redhat.io to Quay...") + match = re.search(r"\.(v\d+\.\d+\.\d+)", bundle_name) + if match: + version = match.group(1) + else: + print(f"WARNING: Could not extract version from: {bundle_name}") + version = bundle_name + + quay_bundle = ( + f"quay.io/redhat-user-workloads/rh-openshift-gitops-tenant" + f"/gitops-operator-bundle:{version}" + ) + print(f" Original: {bundle_image}") + print(f" Remapped: {quay_bundle}") + return quay_bundle + + +def extract_argocd_image_from_bundle(bundle_image, work_dir, docker_config): + bundle_dir = Path(work_dir) / "bundle-extract" + bundle_dir.mkdir() + + print(f"Extracting bundle from: {bundle_image}") + env = dict(os.environ) + if docker_config: + env["DOCKER_CONFIG"] = docker_config + + result = run_cmd( + f"oc image extract {bundle_image} --path /:{bundle_dir}", env=env, + ) + if result.returncode != 0: + print(f"ERROR: Failed to extract bundle from {bundle_image}") + sys.exit(1) + + manifests = bundle_dir / "manifests" + if not manifests.is_dir(): + print("ERROR: /manifests directory not found in bundle image") + sys.exit(1) + + csv_files = list(manifests.glob("*.clusterserviceversion.yaml")) + if not csv_files: + print("ERROR: No ClusterServiceVersion found in bundle") + sys.exit(1) + + csv_file = csv_files[0] + print(f"Found CSV: {csv_file}") + + with open(csv_file) as f: + csv_data = yaml.safe_load(f) + + related = csv_data.get("spec", {}).get("relatedImages", []) + if not related: + print("ERROR: No relatedImages in CSV") + sys.exit(1) + + print("Extracting ArgoCD server image from CSV...") + + # Try exact name match first + for img in related: + if img.get("name") in ("argocd-server", "argocd"): + return img["image"] + + # Fall back to image path containing "argocd-rhel" but not agent/extension + exclude = {"agent", "extension"} + for img in related: + image = img.get("image", "") + if "argocd-rhel" in image and not any(x in image for x in exclude): + return image + + # Last resort: name contains "argocd" but not agent/extension/rollouts + exclude_name = {"agent", "extension", "rollouts"} + for img in related: + name = img.get("name", "") + if "argocd" in name and not any(x in name for x in exclude_name): + return img["image"] + + print("ERROR: Could not extract ArgoCD server image from bundle") + print("Related images in CSV:") + for img in related: + print(f" {img.get('name', '?')}: {img.get('image', '?')}") + sys.exit(1) + + +def main(): + catalog_image = os.environ.get("CATALOG_IMAGE") + if not catalog_image: + print("ERROR: CATALOG_IMAGE must be set", file=sys.stderr) + sys.exit(1) + + channel = os.environ.get("OPERATOR_CHANNEL", "latest") + operator_name = os.environ.get("OPERATOR_NAME", "openshift-gitops-operator") + + print(f"Extracting ArgoCD image from catalog: {catalog_image}") + print(f" Channel: {channel}") + print(f" Package: {operator_name}") + + work_dir = tempfile.mkdtemp() + try: + docker_config = setup_registry_auth(work_dir) + + catalog_json = extract_catalog_json( + catalog_image, operator_name, work_dir, docker_config, + ) + + bundle_name, bundle_image = find_bundle_in_catalog( + catalog_json, operator_name, channel, + ) + + bundle_image = remap_bundle_image(bundle_name, bundle_image) + + argocd_image = extract_argocd_image_from_bundle( + bundle_image, work_dir, docker_config, + ) + + print(f"Successfully extracted ArgoCD image: {argocd_image}") + + shared = Path("/shared") + if shared.is_dir(): + (shared / "argocd-image.txt").write_text(argocd_image) + print("Wrote ArgoCD image to /shared/argocd-image.txt") + else: + print("WARNING: /shared directory not found, save-result step may fail") + + print(argocd_image) + finally: + shutil.rmtree(work_dir, ignore_errors=True) + + +if __name__ == "__main__": + main() diff --git a/.tekton/test-image/scripts/get-installed-version.sh b/.tekton/test-image/scripts/get-installed-version.sh new file mode 100755 index 00000000..1ca1f9d1 --- /dev/null +++ b/.tekton/test-image/scripts/get-installed-version.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Get the installed CSV version +# Environment variables expected: +# - NAMESPACE (default: openshift-gitops-operator) +# - KUBECONFIG +# - RESULT_PATH (path to write the result) +# - USE_SUBSCRIPTION (optional: "true" to get from subscription, "false" to get from CSV directly) + +USE_SUBSCRIPTION="${USE_SUBSCRIPTION:-true}" + +if [[ "$USE_SUBSCRIPTION" == "true" ]]; then + # Catalog-based install: get CSV from subscription status + CSV=$(oc get subscription -n "$NAMESPACE" -o jsonpath='{.items[0].status.installedCSV}' 2>/dev/null || echo "unknown") +else + # Bundle-based install: get CSV directly + CSV=$(oc get csv -n "$NAMESPACE" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "unknown") +fi + +printf "%s" "$CSV" > "$RESULT_PATH" +echo "Installed CSV: $CSV" diff --git a/.tekton/test-image/scripts/go-cache.sh b/.tekton/test-image/scripts/go-cache.sh new file mode 100644 index 00000000..ef953292 --- /dev/null +++ b/.tekton/test-image/scripts/go-cache.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# Go build cache backed by OCI registry via oras. +# Source this file, then call go_cache_pull / go_cache_push. +# +# Usage: +# source /usr/local/bin/go-cache.sh +# go_cache_pull "argocd-v2.14" +# # ... compile ... +# go_cache_push "argocd-v2.14" +# +# Expects: QUAY_REPO env var, quay credentials at /quay-credentials/.dockerconfigjson + +QUAY_REPO="${QUAY_REPO:-quay.io/devtools_gitops/test_image}" + +# shellcheck source=./lib/oras-helpers.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib/oras-helpers.sh" + +_go_cache_tag() { + local suffix="${1:-default}" + local arch + arch=$(go env GOARCH 2>/dev/null || echo "amd64") + + local sum_hash="unknown" + if [ -f "go.sum" ]; then + sum_hash=$(sha256sum go.sum | cut -c1-12) + fi + + echo "go-cache-${arch}-${suffix}-${sum_hash}" +} + +go_cache_pull() { + local tag + tag="${QUAY_REPO}:$(_go_cache_tag "${1:-default}")" + + setup_oras_auth + + local tmpdir + tmpdir=$(mktemp -d) + if (cd "$tmpdir" && oras pull --no-tty "$tag" 2>/dev/null); then + if [ -f "$tmpdir/go-cache.tar.gz" ]; then + tar xzf "$tmpdir/go-cache.tar.gz" -C / 2>/dev/null || true + echo "Go cache restored from ${tag}" + fi + else + echo "No Go cache found at ${tag}, building from scratch" + fi + rm -rf "$tmpdir" +} + +go_cache_push() { + local tag + tag="${QUAY_REPO}:$(_go_cache_tag "${1:-default}")" + + setup_oras_auth + + local gocache gomodcache + gocache=$(go env GOCACHE 2>/dev/null) + gomodcache=$(go env GOMODCACHE 2>/dev/null) + + local paths=() + [ -d "$gocache" ] && paths+=("${gocache#/}") + [ -d "$gomodcache" ] && paths+=("${gomodcache#/}") + + if [ ${#paths[@]} -eq 0 ]; then + return 0 + fi + + local tmpdir + tmpdir=$(mktemp -d) + tar czf "$tmpdir/go-cache.tar.gz" -C / "${paths[@]}" 2>/dev/null || true + + (cd "$tmpdir" && oras push --no-tty \ + --artifact-type "application/vnd.go-cache.v1+tar" \ + "$tag" \ + "go-cache.tar.gz" 2>/dev/null) && \ + echo "Go cache pushed to ${tag}" || \ + echo "WARNING: Failed to push Go cache" + + rm -rf "$tmpdir" +} diff --git a/.tekton/test-image/scripts/install-operator.sh b/.tekton/test-image/scripts/install-operator.sh new file mode 100644 index 00000000..5d094c85 --- /dev/null +++ b/.tekton/test-image/scripts/install-operator.sh @@ -0,0 +1,337 @@ +#!/bin/bash +set -ex + +# Environment variables expected: +# - OPENSHIFT_VERSION (e.g. "4.20" or "4.20.19") +# - NAMESPACE +# - INSTALL_TIMEOUT +# - KUBECONFIG +# - OPERATOR_CHANNEL (default: latest) +# - OPERATOR_VERSION (optional, pins to a specific CSV version) + +# shellcheck source=./lib/wait-for-resources.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib/wait-for-resources.sh" + +MINOR_VERSION=$(echo "${OPENSHIFT_VERSION}" | grep -oP '^\d+\.\d+') +CATALOG_IMAGE="quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/catalog:v${MINOR_VERSION}" + +echo "Installing GitOps Operator from catalog: ${CATALOG_IMAGE}" +echo "OpenShift version: ${OPENSHIFT_VERSION} (minor: ${MINOR_VERSION})" +echo "Target namespace: ${NAMESPACE}" + +# 1. Inject quay pull credentials into cluster +if [[ -f "/quay-pull-credentials/.dockerconfigjson" ]]; then + # 1a. Patch global pull-secret (may take time to propagate on HyperShift) + echo "Injecting quay pull credentials into cluster global pull-secret..." + EXISTING=$(oc get secret pull-secret -n openshift-config -o jsonpath='{.data.\.dockerconfigjson}' | base64 -d) + MERGED=$(echo "$EXISTING" | python3 -c " +import json, sys +existing = json.load(sys.stdin) +with open('/quay-pull-credentials/.dockerconfigjson') as f: + extra = json.load(f) +existing.setdefault('auths', {}).update(extra.get('auths', {})) +print(json.dumps(existing)) +") + oc set data secret/pull-secret -n openshift-config --from-literal=.dockerconfigjson="$MERGED" + echo "Injected $(echo "$MERGED" | python3 -c "import json,sys; print(len(json.load(sys.stdin)['auths']))" 2>/dev/null) registry credentials into cluster pull-secret" + + # 1b. Create additional-pull-secret in kube-system (HyperShift-native mechanism). + # The Hosted Cluster Config Operator detects this secret and deploys a DaemonSet + # that writes credentials to /var/lib/kubelet/config.json on each node. + echo "Creating additional-pull-secret in kube-system for HyperShift node credential injection..." + oc create secret generic additional-pull-secret \ + -n kube-system \ + --from-file=.dockerconfigjson=/quay-pull-credentials/.dockerconfigjson \ + --type=kubernetes.io/dockerconfigjson \ + --dry-run=client -o yaml | oc apply -f - + + # Wait for the syncer DaemonSet to appear and propagate to all nodes + echo "Waiting for pull-secret syncer DaemonSet..." + SYNC_TIMEOUT=300 + SYNC_START=$(date +%s) + while true; do + DS_NAME=$(oc get daemonset -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null \ + | grep -i 'pull-secret' || true) + + if [[ -n "$DS_NAME" ]]; then + DESIRED=$(oc get ds "$DS_NAME" -n kube-system -o jsonpath='{.status.desiredNumberScheduled}' 2>/dev/null || echo "0") + READY=$(oc get ds "$DS_NAME" -n kube-system -o jsonpath='{.status.numberReady}' 2>/dev/null || echo "0") + if [[ "$DESIRED" -gt 0 && "$DESIRED" == "$READY" ]]; then + echo "Pull-secret syncer DaemonSet $DS_NAME is ready ($READY/$DESIRED nodes)" + break + fi + echo " Syncer DaemonSet $DS_NAME: $READY/$DESIRED nodes ready..." + fi + + ELAPSED=$(( $(date +%s) - SYNC_START )) + if [[ $ELAPSED -ge $SYNC_TIMEOUT ]]; then + echo "WARNING: Pull-secret syncer not fully ready within ${SYNC_TIMEOUT}s, continuing anyway" + oc get daemonset -n kube-system 2>/dev/null || true + break + fi + sleep 15 + done +else + echo "WARNING: No quay pull credentials found at /quay-pull-credentials/.dockerconfigjson" +fi + +# 2. Ensure the operator namespace exists +oc create namespace "${NAMESPACE}" --dry-run=client -o yaml | oc apply -f - + +# 3. Create CatalogSource +cat </dev/null; then + # Ensure pull secret exists in the namespace + oc create secret generic quay-mirror-pull \ + --from-file=.dockerconfigjson=/quay-pull-credentials/.dockerconfigjson \ + --type=kubernetes.io/dockerconfigjson \ + -n openshift-gitops \ + --dry-run=client -o yaml | oc apply -f - &>/dev/null + + # Link to all SAs that don't already have it + for sa in $(oc get sa -n openshift-gitops -o jsonpath='{.items[*].metadata.name}' 2>/dev/null); do + if ! oc get sa "$sa" -n openshift-gitops -o jsonpath='{.imagePullSecrets[*].name}' 2>/dev/null | grep -q quay-mirror-pull; then + oc patch sa "$sa" -n openshift-gitops --type=json \ + -p '[{"op":"add","path":"/imagePullSecrets/-","value":{"name":"quay-mirror-pull"}}]' &>/dev/null || true + fi + done + + # Restart pods stuck on image pull errors so they pick up the new SA credentials + STUCK=$(oc get pods -n openshift-gitops 2>/dev/null | grep -E 'ImagePullBackOff|ErrImagePull' | awk '{print $1}') + for pod in $STUCK; do + echo " [pull-secret-injector] Restarting stuck pod: $pod" + oc delete pod "$pod" -n openshift-gitops --grace-period=0 &>/dev/null || true + done + fi + sleep 10 + done + ) & + SA_PATCH_PID=$! + echo "Background pull-secret injection started (PID: $SA_PATCH_PID)" +fi + +# 8. Verify all related images are available at mirrors +echo "" +echo "==========================================" +echo "Verifying related images are available" +echo "==========================================" +python3 /usr/local/bin/verify-images.py || { + echo "WARNING: Some images are not available at their mirror locations." + echo "ArgoCD pods may fail with ImagePullBackOff." +} + +echo "" +echo "==========================================" +echo "DEBUG INFO: Post-Installation State" +echo "==========================================" +echo "" + +echo "--- CatalogSource Status ---" +oc get catalogsource gitops-stage -n openshift-marketplace -o yaml || true +echo "" + +echo "--- Subscription Status ---" +oc get subscription gitops-operator-konflux -n "${NAMESPACE}" -o yaml || true +echo "" + +echo "--- ClusterServiceVersion Status ---" +oc get csv "${CSV_NAME}" -n "${NAMESPACE}" -o yaml || true +echo "" + +echo "--- All Pods in ${NAMESPACE} ---" +oc get pods -n "${NAMESPACE}" -o wide || true +echo "" + +echo "--- Events in ${NAMESPACE} ---" +oc get events -n "${NAMESPACE}" --sort-by='.lastTimestamp' || true +echo "" + +echo "--- Operator Deployment ---" +oc get deployments -n "${NAMESPACE}" -o wide || true +echo "" + +echo "==========================================" + +# 9. Verify default ArgoCD instance is ready +echo "" +echo "==========================================" +echo "Verifying default ArgoCD instance" +echo "==========================================" + +echo "Waiting for openshift-gitops namespace and ArgoCD deployments to appear..." +for _ in {1..60}; do + if oc get deployment openshift-gitops-server -n openshift-gitops &>/dev/null; then + break + fi + sleep 10 +done + +if ! oc get deployment openshift-gitops-server -n openshift-gitops &>/dev/null; then + echo "ERROR: ArgoCD deployments not created after 10 minutes" + oc get ns openshift-gitops 2>/dev/null || echo "Namespace openshift-gitops does not exist" + oc get argocd -n openshift-gitops 2>/dev/null || true + oc get pods -n openshift-gitops -o wide 2>/dev/null || true + oc get events -n openshift-gitops --sort-by='.lastTimestamp' 2>/dev/null | tail -30 || true + echo "--- IDMS on cluster ---" + oc get imagedigestmirrorset 2>/dev/null || echo "No IDMS" + echo "--- Operator logs ---" + oc logs deployment/openshift-gitops-operator-controller-manager -n openshift-gitops-operator -c manager --tail=50 2>/dev/null || true + exit 1 +fi + +echo "ArgoCD deployments found, waiting for them to become available..." +for deploy in openshift-gitops-server openshift-gitops-repo-server; do + if ! wait_for_deployment "$deploy" openshift-gitops 600s; then + exit 1 + fi + echo "$deploy is ready" +done + +# application-controller is a StatefulSet, not a Deployment +if oc get statefulset openshift-gitops-application-controller -n openshift-gitops &>/dev/null; then + if ! wait_for_statefulset openshift-gitops-application-controller openshift-gitops 600s; then + exit 1 + fi + echo "openshift-gitops-application-controller is ready" +else + echo "WARNING: openshift-gitops-application-controller statefulset not found, skipping" +fi + +echo "ArgoCD instance is ready" + +# Stop background pull-secret injection +if [[ -n "${SA_PATCH_PID}" ]]; then + kill $SA_PATCH_PID 2>/dev/null || true + wait $SA_PATCH_PID 2>/dev/null || true + echo "Stopped background pull-secret injection" +fi + +# 10. Collect cluster-wide debug info (on success) +echo "" +echo "==========================================" +echo "DEBUG INFO: Cluster Image Configuration" +echo "==========================================" + +echo "--- ImageDigestMirrorSet ---" +oc get imagedigestmirrorset -o yaml 2>/dev/null || echo "No IDMS found" +echo "" + +echo "--- ImageContentSourcePolicy ---" +oc get imagecontentsourcepolicy -o yaml 2>/dev/null || echo "No ICSP found" +echo "" + +echo "--- openshift-gitops namespace pods ---" +oc get pods -n openshift-gitops -o wide 2>/dev/null || true +echo "" + +echo "--- openshift-gitops namespace events (last 5 min) ---" +oc get events -n openshift-gitops --sort-by='.lastTimestamp' 2>/dev/null | tail -40 || true +echo "" + +echo "--- openshift-gitops pod descriptions (non-Running) ---" +for pod in $(oc get pods -n openshift-gitops -o jsonpath='{range .items[?(@.status.phase!="Running")]}{.metadata.name}{"\n"}{end}' 2>/dev/null); do + echo "=== Pod: $pod ===" + oc describe pod "$pod" -n openshift-gitops 2>/dev/null | grep -A5 -E 'State:|Image:|Warning|Error|Back-off|ImagePull' || true + echo "" +done + +echo "==========================================" + +# 11. Verify pull-secret propagated to nodes +if [[ -f "/quay-pull-credentials/.dockerconfigjson" ]]; then + echo "" + echo "==========================================" + echo "Verifying pull-secret propagation to nodes" + echo "==========================================" + EXPECTED_REPOS=$(python3 -c " +import json +with open('/quay-pull-credentials/.dockerconfigjson') as f: + d = json.load(f) +for k in sorted(d.get('auths', {})): + print(k) +" 2>/dev/null | head -3) + CLUSTER_SECRET=$(oc get secret pull-secret -n openshift-config -o jsonpath='{.data.\.dockerconfigjson}' 2>/dev/null | base64 -d) + MISSING=0 + while IFS= read -r repo; do + if echo "$CLUSTER_SECRET" | python3 -c "import json,sys; d=json.load(sys.stdin); sys.exit(0 if '$repo' in d.get('auths',{}) else 1)" 2>/dev/null; then + echo " OK $repo" + else + echo " MISS $repo" + MISSING=$((MISSING + 1)) + fi + done <<< "$EXPECTED_REPOS" + if [[ $MISSING -gt 0 ]]; then + echo "WARNING: $MISSING repo(s) missing from cluster pull-secret" + else + echo "Pull-secret contains injected credentials" + fi +fi diff --git a/.tekton/test-image/scripts/lib/argocd-e2e-cleanup.sh b/.tekton/test-image/scripts/lib/argocd-e2e-cleanup.sh new file mode 100644 index 00000000..987f9e79 --- /dev/null +++ b/.tekton/test-image/scripts/lib/argocd-e2e-cleanup.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Shared cleanup for ArgoCD E2E test resources. +# Source this file and call cleanup_argocd_e2e from your trap. + +cleanup_argocd_e2e() { + local namespace="${1:-${ARGOCD_NAMESPACE:-argocd-e2e}}" + + echo "Cleaning up ArgoCD E2E resources (namespace: ${namespace})..." + + for ns in argocd-e2e-external argocd-e2e-external-2; do + if oc get ns "$ns" >/dev/null 2>&1; then + oc get applications -n "$ns" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null | \ + xargs -n1 -I{} oc patch application {} -n "$ns" --type merge \ + -p '{"metadata":{"finalizers":[]}}' 2>/dev/null || true + fi + done + + if oc get ns "$namespace" >/dev/null 2>&1; then + oc get applications -n "$namespace" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null | \ + xargs -n1 -I{} oc patch application {} -n "$namespace" --type merge \ + -p '{"metadata":{"finalizers":[]}}' 2>/dev/null || true + fi + + oc delete ns -l e2e.argoproj.io=true --ignore-not-found --wait=false 2>/dev/null || true + oc delete project argocd-e2e-external argocd-e2e-external-2 \ + --ignore-not-found --wait=false 2>/dev/null || true + + oc delete pod e2e-test-runner -n "$namespace" --ignore-not-found --wait=false 2>/dev/null || true + oc delete deployment argocd-e2e-cluster -n "$namespace" --ignore-not-found --wait=false 2>/dev/null || true + oc delete service argocd-e2e-server -n "$namespace" --ignore-not-found --wait=false 2>/dev/null || true +} diff --git a/.tekton/test-image/scripts/lib/collect-pod-logs.sh b/.tekton/test-image/scripts/lib/collect-pod-logs.sh new file mode 100644 index 00000000..1a85e4fa --- /dev/null +++ b/.tekton/test-image/scripts/lib/collect-pod-logs.sh @@ -0,0 +1,181 @@ +#!/bin/bash +# Utilities for collecting pod logs from Kubernetes/OpenShift namespaces. +# Source this file to use these functions in log collection scripts. +# +# Usage: +# source "$(dirname "${BASH_SOURCE[0]}")/lib/collect-pod-logs.sh" +# collect_pod_logs openshift-gitops /tmp/logs +# collect_pod_logs_with_tail openshift-gitops /tmp/snapshots 100 + +# Collect full logs from all pods in a namespace. +# Creates numbered log files for each pod with all container logs. +# +# Args: +# $1 - namespace: Namespace to collect logs from +# $2 - output_dir: Directory where log files should be written +# $3 - include_description: Set to "true" to include pod description (optional, default: false) +# +# Returns: +# 0 on success (always succeeds, errors are non-fatal) +# +# Output files: +# /01-.log +# /02-.log +# ... +# +# File format: +# === Pod: === +# === Namespace: === +# === Collection Time: === +# +# [--- Pod Description --- (if include_description=true)] +# +# --- Container: --- +# +# +# --- Container: (previous) --- +# +# +# Example: +# collect_pod_logs openshift-gitops /tmp/cluster-logs +# collect_pod_logs openshift-gitops /tmp/cluster-logs true # with descriptions +collect_pod_logs() { + local namespace=$1 + local output_dir=$2 + local include_description=${3:-false} + + if [ -z "$namespace" ] || [ -z "$output_dir" ]; then + echo "ERROR: collect_pod_logs requires namespace and output_dir" >&2 + return 1 + fi + + mkdir -p "$output_dir" + + local count=0 + oc get pods -n "$namespace" -o json 2>/dev/null | jq -r '.items[].metadata.name' 2>/dev/null | while read -r pod; do + count=$((count + 1)) + local log_file="${output_dir}/$(printf '%02d' ${count})-${pod}.log" + + { + echo "=== Pod: ${pod} ===" + echo "=== Namespace: ${namespace} ===" + echo "=== Collection Time: $(date -u +"%Y-%m-%d %H:%M:%S UTC") ===" + echo "" + + if [ "$include_description" = "true" ]; then + echo "--- Pod Description ---" + oc describe pod "$pod" -n "$namespace" 2>&1 || true + echo "" + fi + + oc get pod "$pod" -n "$namespace" -o json 2>/dev/null | jq -r '.spec.containers[]?.name' 2>/dev/null | while read -r container; do + echo "--- Container: ${container} ---" + oc logs "$pod" -c "$container" -n "$namespace" 2>&1 || echo "(no logs available)" + echo "" + + echo "--- Container: ${container} (previous) ---" + oc logs "$pod" -c "$container" -n "$namespace" --previous 2>&1 || echo "(no previous logs)" + echo "" + done + } > "$log_file" + done || true + + return 0 +} + +# Collect tail of logs from all pods in a namespace. +# Similar to collect_pod_logs but only retrieves the last N lines per container. +# Useful for periodic snapshots where full logs would be too large. +# +# Args: +# $1 - namespace: Namespace to collect logs from +# $2 - output_dir: Directory where log files should be written +# $3 - tail_lines: Number of lines to retrieve (default: 100) +# +# Returns: +# 0 on success (always succeeds, errors are non-fatal) +# +# Example: +# collect_pod_logs_with_tail openshift-gitops /tmp/snapshot-123 50 +collect_pod_logs_with_tail() { + local namespace=$1 + local output_dir=$2 + local tail_lines=${3:-100} + + if [ -z "$namespace" ] || [ -z "$output_dir" ]; then + echo "ERROR: collect_pod_logs_with_tail requires namespace and output_dir" >&2 + return 1 + fi + + mkdir -p "$output_dir" + + oc get pods -n "$namespace" -o json 2>/dev/null | jq -r '.items[].metadata.name' 2>/dev/null | while read -r pod; do + local log_file="${output_dir}/${pod}.log" + + { + echo "=== Pod: ${pod} ===" + echo "=== Timestamp: $(date -u +"%Y-%m-%d %H:%M:%S UTC") ===" + echo "" + + oc get pod "$pod" -n "$namespace" -o json 2>/dev/null | jq -r '.spec.containers[]?.name' 2>/dev/null | while read -r container; do + echo "--- Container: ${container} ---" + oc logs "$pod" -c "$container" -n "$namespace" --tail="$tail_lines" 2>&1 || echo "(no logs available)" + echo "" + done + } > "$log_file" + done || true + + return 0 +} + +# Collect description and logs for a specific pod. +# Includes full pod description (events, status, etc.) plus container logs. +# +# Args: +# $1 - pod_name: Name of the pod +# $2 - namespace: Namespace containing the pod +# $3 - output_file: Path to output file +# +# Returns: +# 0 on success, 1 if pod not found +# +# Example: +# collect_single_pod_logs openshift-gitops-server-abc123 openshift-gitops /tmp/server.log +collect_single_pod_logs() { + local pod_name=$1 + local namespace=$2 + local output_file=$3 + + if [ -z "$pod_name" ] || [ -z "$namespace" ] || [ -z "$output_file" ]; then + echo "ERROR: collect_single_pod_logs requires pod_name, namespace, and output_file" >&2 + return 1 + fi + + if ! oc get pod "$pod_name" -n "$namespace" &>/dev/null; then + echo "ERROR: Pod $pod_name not found in namespace $namespace" >&2 + return 1 + fi + + { + echo "=== Pod: ${pod_name} ===" + echo "=== Namespace: ${namespace} ===" + echo "=== Collection Time: $(date -u +"%Y-%m-%d %H:%M:%S UTC") ===" + echo "" + + echo "--- Pod Description ---" + oc describe pod "$pod_name" -n "$namespace" 2>&1 || true + echo "" + + oc get pod "$pod_name" -n "$namespace" -o json 2>/dev/null | jq -r '.spec.containers[]?.name' 2>/dev/null | while read -r container; do + echo "--- Container: ${container} ---" + oc logs "$pod_name" -c "$container" -n "$namespace" 2>&1 || echo "(no logs available)" + echo "" + + echo "--- Container: ${container} (previous) ---" + oc logs "$pod_name" -c "$container" -n "$namespace" --previous 2>&1 || echo "(no previous logs)" + echo "" + done + } > "$output_file" + + return 0 +} diff --git a/.tekton/test-image/scripts/lib/load-skip-patterns.sh b/.tekton/test-image/scripts/lib/load-skip-patterns.sh new file mode 100644 index 00000000..450f6541 --- /dev/null +++ b/.tekton/test-image/scripts/lib/load-skip-patterns.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# Utilities for loading test skip patterns from config files. +# Source this file to use these functions in test wrapper scripts. +# +# Usage: +# source "$(dirname "${BASH_SOURCE[0]}")/lib/load-skip-patterns.sh" +# load_ginkgo_skip_patterns /usr/local/config/skip-sequential.txt +# load_playwright_skip_patterns /usr/local/config/skip-ui-e2e.txt + +# Load skip patterns from a config file and append to GINKGO_SKIP env var. +# Config file format: one pattern per line, # for comments, blank lines ignored. +# Patterns are joined with '|' to form a regex for ginkgo's --skip flag. +# +# Args: +# $1 - skip_file: Path to skip patterns config file +# +# Sets: +# GINKGO_SKIP environment variable (appends if already set) +# +# Returns: +# 0 on success, 1 if file not found +# +# Example: +# load_ginkgo_skip_patterns /usr/local/config/skip-sequential.txt +# # GINKGO_SKIP is now set to "pattern1|pattern2|pattern3" +load_ginkgo_skip_patterns() { + local skip_file=$1 + + if [[ ! -f "$skip_file" ]]; then + return 1 + fi + + local skip_pattern + skip_pattern=$(grep -v '^\s*#' "$skip_file" | grep -v '^\s*$' | paste -sd '|') + + if [[ -n "$skip_pattern" ]]; then + if [[ -n "${GINKGO_SKIP:-}" ]]; then + export GINKGO_SKIP="${GINKGO_SKIP}|${skip_pattern}" + else + export GINKGO_SKIP="$skip_pattern" + fi + fi + + return 0 +} + +# Load skip patterns from a config file for Playwright's --grep-invert flag. +# Config file format: one pattern per line, # for comments, blank lines ignored. +# Patterns are joined with '|' to form a regex. +# +# Args: +# $1 - skip_file: Path to skip patterns config file +# +# Returns: +# An array of playwright arguments: (--grep-invert "pattern1|pattern2") +# Empty array if no patterns found or file doesn't exist +# +# Example: +# PLAYWRIGHT_ARGS=($(load_playwright_skip_patterns /usr/local/config/skip-ui-e2e.txt)) +# npx playwright test "${PLAYWRIGHT_ARGS[@]}" +load_playwright_skip_patterns() { + local skip_file=$1 + + if [[ ! -f "$skip_file" ]]; then + return 0 + fi + + local skip_pattern + skip_pattern=$(grep -v '^\s*#' "$skip_file" | grep -v '^\s*$' | paste -sd '|') + + if [[ -n "$skip_pattern" ]]; then + echo "--grep-invert" + echo "$skip_pattern" + fi + + return 0 +} diff --git a/.tekton/test-image/scripts/lib/oras-helpers.sh b/.tekton/test-image/scripts/lib/oras-helpers.sh new file mode 100644 index 00000000..fcd777e8 --- /dev/null +++ b/.tekton/test-image/scripts/lib/oras-helpers.sh @@ -0,0 +1,151 @@ +#!/bin/bash +# Shared utilities for oras operations and authentication setup. +# Source this file to use these functions in other scripts. +# +# Usage: +# source "$(dirname "${BASH_SOURCE[0]}")/lib/oras-helpers.sh" +# setup_oras_auth +# oras_push_tarball /path/to/logs quay.io/repo my-tag + +# Configure Docker credentials for oras. +# Copies the .dockerconfigjson to a temporary directory and exports DOCKER_CONFIG. +# +# Args: +# $1 - Path to .dockerconfigjson (default: /quay-credentials/.dockerconfigjson) +# +# Returns: +# 0 if credentials were set up successfully +# 1 if credentials file not found +# +# Sets: +# DOCKER_CONFIG environment variable +setup_oras_auth() { + local auth_path="${1:-/quay-credentials/.dockerconfigjson}" + + # Skip if already configured + if [ -n "${DOCKER_CONFIG:-}" ] && [ -f "${DOCKER_CONFIG}/config.json" ]; then + return 0 + fi + + if [ -f "$auth_path" ]; then + local temp_config + temp_config=$(mktemp -d) + cp "$auth_path" "$temp_config/config.json" + export DOCKER_CONFIG="$temp_config" + return 0 + else + echo "WARNING: Oras credentials not found at ${auth_path}" >&2 + return 1 + fi +} + +# Push a directory as a gzipped tarball to an OCI registry using oras. +# The tarball is created in /tmp and cleaned up after push. +# +# Args: +# $1 - source_dir: Directory to tar and push +# $2 - quay_repo: OCI repository (e.g., quay.io/org/repo) +# $3 - tag: Image tag +# $4 - artifact_type: OCI artifact type (optional, default: application/vnd.konflux.logs.v1+tar) +# $5 - tarball_prefix: Prefix for paths inside tarball (optional, default: tag name) +# +# Returns: +# 0 on success, 1 on failure +# +# Prints: +# Full OCI reference on success +# +# Example: +# oras_push_tarball /tmp/logs quay.io/my/repo pipeline-123-logs +oras_push_tarball() { + local source_dir=$1 + local quay_repo=$2 + local tag=$3 + local artifact_type=${4:-"application/vnd.konflux.logs.v1+tar"} + local tarball_prefix=${5:-"$tag"} + + if [ ! -d "$source_dir" ]; then + echo "ERROR: Source directory does not exist: ${source_dir}" >&2 + return 1 + fi + + local tarball_name="${tag}.tar.gz" + local full_ref="${quay_repo}:${tag}" + + # Create tarball with path transformation + if ! tar czf "/tmp/${tarball_name}" --transform "s,^,${tarball_prefix}/," -C "$source_dir" .; then + echo "ERROR: Failed to create tarball" >&2 + return 1 + fi + + # Push to registry + if ( cd /tmp && oras push --no-tty \ + --artifact-type "$artifact_type" \ + "$full_ref" \ + "$tarball_name" ); then + rm -f "/tmp/${tarball_name}" + echo "$full_ref" + return 0 + else + echo "ERROR: Failed to push tarball to ${full_ref}" >&2 + rm -f "/tmp/${tarball_name}" + return 1 + fi +} + +# Pull an OCI artifact from a registry and extract any tarballs found. +# Automatically handles .tar.gz extraction and cleanup. +# +# Args: +# $1 - quay_repo: OCI repository (e.g., quay.io/org/repo) +# $2 - tag: Image tag +# $3 - output_dir: Directory where contents should be extracted +# +# Returns: +# 0 on success, 1 on failure +# +# Example: +# oras_pull_tarball quay.io/my/repo pipeline-123-logs /tmp/extracted +oras_pull_tarball() { + local quay_repo=$1 + local tag=$2 + local output_dir=$3 + + if [ -z "$quay_repo" ] || [ -z "$tag" ] || [ -z "$output_dir" ]; then + echo "ERROR: Missing required arguments to oras_pull_tarball" >&2 + return 1 + fi + + mkdir -p "$output_dir" + + local ref="${quay_repo}:${tag}" + local tmpdir + tmpdir=$(mktemp -d) + + # Pull artifact + if ! oras pull --no-tty -o "$tmpdir" "$ref" 2>/dev/null; then + rm -rf "$tmpdir" + return 1 + fi + + # Extract any tarballs found + local found_tarball=false + for tarball in "$tmpdir"/*.tar.gz; do + if [ -f "$tarball" ]; then + tar xzf "$tarball" -C "$output_dir" 2>/dev/null || true + found_tarball=true + fi + done + + # Copy any remaining non-tarball files + find "$tmpdir" -type f ! -name "*.tar.gz" -exec cp {} "$output_dir/" \; 2>/dev/null || true + + rm -rf "$tmpdir" + + if [ "$found_tarball" = false ]; then + # No tarball found, but pull succeeded - might be individual files + return 0 + fi + + return 0 +} diff --git a/.tekton/test-image/scripts/lib/wait-for-resources.sh b/.tekton/test-image/scripts/lib/wait-for-resources.sh new file mode 100644 index 00000000..a124203e --- /dev/null +++ b/.tekton/test-image/scripts/lib/wait-for-resources.sh @@ -0,0 +1,262 @@ +#!/bin/bash +# Utilities for waiting on Kubernetes/OpenShift resources. +# Source this file to use these functions in operator installation/upgrade scripts. +# +# Usage: +# source "$(dirname "${BASH_SOURCE[0]}")/lib/wait-for-resources.sh" +# wait_for_deployment openshift-gitops-server openshift-gitops 600s +# wait_for_operator_pods openshift-gitops-operator + +# Wait for a deployment to become available. +# Uses 'oc wait' with condition=Available. +# +# Args: +# $1 - deployment_name: Name of the deployment +# $2 - namespace: Namespace containing the deployment +# $3 - timeout: Timeout duration (default: 600s) +# +# Returns: +# 0 on success, 1 on timeout or failure +# +# Prints: +# Debug information on failure (pod status, events, IDMS) +# +# Example: +# wait_for_deployment openshift-gitops-server openshift-gitops 10m +wait_for_deployment() { + local deployment_name=$1 + local namespace=$2 + local timeout=${3:-600s} + + if ! oc wait --for=condition=Available "deployment/$deployment_name" -n "$namespace" --timeout="$timeout"; then + echo "ERROR: deployment/$deployment_name did not become Available within $timeout" + echo "--- Deployment status ---" + oc get deployment "$deployment_name" -n "$namespace" -o wide 2>/dev/null || true + echo "--- Pods ---" + oc get pods -n "$namespace" -o wide 2>/dev/null || true + echo "--- Pod details ---" + for pod in $(oc get pods -n "$namespace" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null); do + echo "=== $pod ===" + oc get pod "$pod" -n "$namespace" -o jsonpath='{range .status.containerStatuses[*]}container={.name} ready={.ready} state={.state}{"\n"}{end}' 2>/dev/null || true + done + echo "--- Events ---" + oc get events -n "$namespace" --sort-by='.lastTimestamp' 2>/dev/null | tail -30 || true + echo "--- IDMS ---" + oc get imagedigestmirrorset 2>/dev/null || echo "No IDMS" + return 1 + fi + + return 0 +} + +# Wait for a statefulset to complete rollout. +# Uses 'oc rollout status'. +# +# Args: +# $1 - statefulset_name: Name of the statefulset +# $2 - namespace: Namespace containing the statefulset +# $3 - timeout: Timeout duration (default: 600s) +# +# Returns: +# 0 on success, 1 on timeout or failure +# +# Prints: +# Debug information on failure +# +# Example: +# wait_for_statefulset openshift-gitops-application-controller openshift-gitops +wait_for_statefulset() { + local statefulset_name=$1 + local namespace=$2 + local timeout=${3:-600s} + + if ! oc rollout status "statefulset/$statefulset_name" -n "$namespace" --timeout="$timeout"; then + echo "ERROR: statefulset/$statefulset_name did not become ready within $timeout" + oc get pods -n "$namespace" -o wide 2>/dev/null || true + oc get events -n "$namespace" --sort-by='.lastTimestamp' 2>/dev/null | tail -30 || true + return 1 + fi + + return 0 +} + +# Wait for operator pods to be running and ready. +# Polls until all pods matching a selector are Running with all containers ready. +# +# Args: +# $1 - namespace: Namespace to check +# $2 - pod_selector: Substring to match in pod names (default: "openshift-gitops-operator-controller-manager") +# $3 - max_attempts: Maximum polling attempts (default: 150, ~5 minutes at 2s intervals) +# +# Returns: +# 0 on success, 1 on timeout +# +# Example: +# wait_for_operator_pods openshift-gitops-operator +# wait_for_operator_pods my-namespace my-operator 60 +wait_for_operator_pods() { + local namespace=$1 + local pod_selector=${2:-"openshift-gitops-operator-controller-manager"} + local max_attempts=${3:-150} + + echo "Waiting for operator pods in namespace $namespace to be ready (selector: $pod_selector)" + + for attempt in $(seq 1 "$max_attempts"); do + local pods + pods=$(oc get pods --no-headers -n "$namespace" 2>/dev/null | grep "$pod_selector" || true) + + if [ -z "$pods" ]; then + sleep 2 + continue + fi + + # Check if all pods are Running (ignore Completed) + local not_running + not_running=$(echo "$pods" | grep -v Running | grep -vc Completed || true) + if [ "$not_running" -ne 0 ]; then + sleep 2 + continue + fi + + # Check readiness: all containers in each pod must be ready + local all_ready=true + while IFS= read -r pod; do + local current + current=$(echo "$pod" | awk '{split($2,a,"/"); print a[1]}') + local total + total=$(echo "$pod" | awk '{split($2,a,"/"); print a[2]}') + if [ "$current" != "$total" ] || [ "$current" -lt 1 ]; then + all_ready=false + break + fi + done <<< "$pods" + + if $all_ready; then + echo "All pods are ready (attempt $attempt/$max_attempts)" + return 0 + fi + + sleep 2 + done + + echo "ERROR: timeout waiting for pods to become ready after $max_attempts attempts" + oc get pods -n "$namespace" 2>/dev/null || true + return 1 +} + +# Wait for a ClusterServiceVersion to reach Succeeded phase. +# Polls subscription status to get CSV name, then waits for CSV to succeed. +# +# Args: +# $1 - subscription_name: Name of the subscription +# $2 - namespace: Namespace containing the subscription +# $3 - timeout: Timeout duration for CSV wait (default: 25m) +# $4 - max_poll_attempts: Max attempts to find CSV name (default: 30) +# +# Returns: +# 0 on success, 1 on failure +# +# Sets: +# CSV_NAME variable with the installed CSV name +# +# Example: +# wait_for_csv gitops-operator-konflux openshift-gitops-operator +# echo "Installed CSV: $CSV_NAME" +wait_for_csv() { + local subscription_name=$1 + local namespace=$2 + local timeout=${3:-25m} + local max_poll_attempts=${4:-30} + + echo "Waiting for ClusterServiceVersion from subscription $subscription_name..." + sleep 30 + + CSV_NAME="" + for attempt in $(seq 1 "$max_poll_attempts"); do + CSV_NAME=$(oc get sub "$subscription_name" -n "$namespace" -o jsonpath='{.status.installedCSV}' 2>/dev/null || true) + if [ -n "$CSV_NAME" ]; then + echo "Found CSV: $CSV_NAME (attempt $attempt)" + break + fi + echo "Waiting for subscription status to be updated (attempt $attempt/$max_poll_attempts)..." + sleep 10 + done + + if [ -z "$CSV_NAME" ]; then + echo "ERROR: CSV name not found in subscription after $max_poll_attempts attempts" + oc get sub "$subscription_name" -n "$namespace" -o yaml 2>/dev/null || true + return 1 + fi + + if ! oc wait --for=jsonpath='{.status.phase}'=Succeeded "csv/$CSV_NAME" -n "$namespace" --timeout="$timeout"; then + echo "ERROR: CSV $CSV_NAME did not reach Succeeded phase within $timeout" + oc get csv "$CSV_NAME" -n "$namespace" -o yaml 2>/dev/null || true + return 1 + fi + + echo "CSV $CSV_NAME is in Succeeded phase" + return 0 +} + +# Wait for ArgoCD workloads to be updated after an operator upgrade. +# Polls until the ArgoCD server container image changes (indicating +# the new operator has reconciled), then waits for all workload +# rollouts to complete. +# +# Args: +# $1 - operator_ns: Operator namespace (default: openshift-gitops-operator) +# $2 - gitops_ns: ArgoCD instance namespace (default: openshift-gitops) +# $3 - timeout: Max seconds to wait for image change (default: 300) +# +# Returns: +# 0 on success, 1 on rollout failure +# +# Example: +# wait_for_argocd_reconciliation openshift-gitops-operator openshift-gitops +wait_for_argocd_reconciliation() { + local operator_ns=${1:-openshift-gitops-operator} + local gitops_ns=${2:-openshift-gitops} + local timeout=${3:-300} + + local old_image + old_image=$(oc get deployment openshift-gitops-server -n "$gitops_ns" \ + -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || true) + echo "Current ArgoCD server image: ${old_image:-unknown}" + + echo "Waiting for operator controller pod to restart..." + wait_for_operator_pods "$operator_ns" + + echo "Waiting for operator to reconcile ArgoCD workloads..." + local deadline=$(($(date +%s) + timeout)) + while [[ -n "$old_image" ]]; do + local new_image + new_image=$(oc get deployment openshift-gitops-server -n "$gitops_ns" \ + -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || true) + if [[ "$new_image" != "$old_image" ]]; then + echo "ArgoCD server image updated: ${new_image}" + break + fi + if [[ $(date +%s) -ge "$deadline" ]]; then + echo "WARNING: ArgoCD server image unchanged after ${timeout}s — operator may not need to update workloads" + break + fi + sleep 10 + done + + local deployments="openshift-gitops-server openshift-gitops-repo-server openshift-gitops-applicationset-controller openshift-gitops-redis openshift-gitops-dex-server" + for deploy in $deployments; do + if oc get deployment "$deploy" -n "$gitops_ns" &>/dev/null; then + echo " Waiting for $deploy..." + wait_for_deployment "$deploy" "$gitops_ns" 300s || return 1 + fi + done + + local statefulset="openshift-gitops-application-controller" + if oc get statefulset "$statefulset" -n "$gitops_ns" &>/dev/null; then + echo " Waiting for $statefulset..." + wait_for_statefulset "$statefulset" "$gitops_ns" 300s || return 1 + fi + + echo "All ArgoCD workloads reconciled after upgrade" + return 0 +} diff --git a/.tekton/test-image/scripts/parse-dast-results.py b/.tekton/test-image/scripts/parse-dast-results.py new file mode 100644 index 00000000..a0976d55 --- /dev/null +++ b/.tekton/test-image/scripts/parse-dast-results.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +"""Parse RapidAST/ZAP JSON scan results into JUnit XML. + +Usage: parse-dast-results.py + +Reads /usr/local/config/dast-false-positives.json for alert thresholds and +suppression rules. Exits 0 if all unsuppressed alerts are within threshold, +1 if any exceed the threshold. + +Output JUnit XML format: + One testcase per ZAP alert type. + Suppressed (false-positive) alerts → skipped. + Alerts within threshold → pass. + Alerts exceeding threshold → failure. +""" + +import glob +import json +import os +import sys +import xml.etree.ElementTree as ET + +CONFIG_PATH = "/usr/local/config/dast-false-positives.json" + +DEFAULT_THRESHOLDS = { + "high": 0, + "medium": 10, + "low": 9999, + "informational": 9999, +} + +RISK_NAMES = {0: "informational", 1: "low", 2: "medium", 3: "high"} + + +def load_config(): + if not os.path.exists(CONFIG_PATH): + print(f"WARNING: {CONFIG_PATH} not found, using defaults", file=sys.stderr) + return {"thresholds": DEFAULT_THRESHOLDS, "falsePositives": []} + with open(CONFIG_PATH) as f: + cfg = json.load(f) + thresholds = cfg.get("thresholds", {}) + for key, default in DEFAULT_THRESHOLDS.items(): + thresholds.setdefault(key, default) + cfg["thresholds"] = thresholds + cfg.setdefault("falsePositives", []) + return cfg + + +def is_false_positive(alert, fp_rules): + alert_ref = str(alert.get("alertRef", alert.get("pluginid", ""))) + for rule in fp_rules: + if str(rule.get("alertRef", "")) != alert_ref: + continue + url_filter = rule.get("url", "") + if not url_filter: + return True, rule.get("reason", "suppressed") + for inst in alert.get("instances", []): + if url_filter in inst.get("uri", ""): + return True, rule.get("reason", "suppressed") + return False, "" + + +def find_zap_json(results_dir): + for pattern in [ + os.path.join(results_dir, "rapidast-*", "zap", "zap-report.json"), + os.path.join(results_dir, "*", "zap-report.json"), + os.path.join(results_dir, "zap-report.json"), + ]: + matches = sorted(glob.glob(pattern)) + if matches: + return matches[-1] + return None + + +def parse_alerts(zap_json_path, fp_rules, thresholds): + with open(zap_json_path) as f: + data = json.load(f) + + all_alerts = [] + for site in data.get("site", []): + all_alerts.extend(site.get("alerts", [])) + + results = [] + for alert in all_alerts: + risk_code = int(alert.get("riskcode", 0)) + risk_name = RISK_NAMES.get(risk_code, "informational") + threshold = thresholds.get(risk_name, 9999) + count = int(alert.get("count", len(alert.get("instances", [])))) + fp, fp_reason = is_false_positive(alert, fp_rules) + results.append({ + "name": alert.get("name", alert.get("alert", "Unknown")), + "alertRef": str(alert.get("alertRef", alert.get("pluginid", ""))), + "riskName": risk_name, + "count": count, + "threshold": threshold, + "isFalsePositive": fp, + "fpReason": fp_reason, + "fails": not fp and count > threshold, + "instances": [i.get("uri", "") for i in alert.get("instances", [])[:5]], + }) + return results + + +def write_junit(results, output_path): + total = len(results) + failures = sum(1 for r in results if r["fails"]) + skipped = sum(1 for r in results if r["isFalsePositive"]) + + root = ET.Element("testsuites") + suite = ET.SubElement(root, "testsuite", { + "name": "DAST Scan", + "tests": str(total), + "failures": str(failures), + "errors": "0", + "skipped": str(skipped), + }) + + for r in results: + tc = ET.SubElement(suite, "testcase", { + "name": f"[{r['riskName'].upper()}] {r['name']} (alertRef={r['alertRef']})", + "classname": f"dast.{r['riskName']}", + }) + if r["isFalsePositive"]: + ET.SubElement(tc, "skipped", message=f"Suppressed: {r['fpReason']}") + elif r["fails"]: + detail = ( + f"Alert count {r['count']} exceeds threshold {r['threshold']}\n" + f"Risk: {r['riskName'].upper()} AlertRef: {r['alertRef']}\n" + f"Instances (first {len(r['instances'])}):\n" + + "\n".join(f" {u}" for u in r["instances"]) + ) + ET.SubElement(tc, "failure", { + "message": f"count={r['count']} > threshold={r['threshold']}", + }).text = detail + + ET.ElementTree(root).write(output_path, encoding="unicode", xml_declaration=True) + return failures + + +def write_error_junit(output_path, message): + root = ET.Element("testsuites") + suite = ET.SubElement(root, "testsuite", + {"name": "DAST Scan", "tests": "1", "failures": "1", + "errors": "0", "skipped": "0"}) + tc = ET.SubElement(suite, "testcase", + {"name": "ZAP results", "classname": "dast"}) + ET.SubElement(tc, "failure", {"message": message}).text = message + ET.ElementTree(root).write(output_path, encoding="unicode", xml_declaration=True) + + +def main(): + if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(1) + + results_dir, output_path = sys.argv[1], sys.argv[2] + + cfg = load_config() + thresholds = cfg["thresholds"] + fp_rules = cfg["falsePositives"] + print(f"Thresholds: {thresholds}") + print(f"Suppression rules: {len(fp_rules)}") + + zap_json = find_zap_json(results_dir) + if not zap_json: + msg = f"No zap-report.json found in {results_dir}" + print(f"ERROR: {msg}", file=sys.stderr) + write_error_junit(output_path, msg) + sys.exit(1) + + print(f"Parsing: {zap_json}") + results = parse_alerts(zap_json, fp_rules, thresholds) + failures = write_junit(results, output_path) + + passed = sum(1 for r in results if not r["fails"] and not r["isFalsePositive"]) + suppressed = sum(1 for r in results if r["isFalsePositive"]) + print(f"Alert types: {len(results)} total — " + f"{passed} within threshold, {failures} exceeded, {suppressed} suppressed") + + if failures > 0: + print(f"\nFailed alerts ({failures}):") + for r in results: + if r["fails"]: + print(f" [{r['riskName'].upper()}] {r['name']} " + f"count={r['count']} threshold={r['threshold']}") + + sys.exit(0 if failures == 0 else 1) + + +if __name__ == "__main__": + main() diff --git a/.tekton/test-image/scripts/parse-test-results.py b/.tekton/test-image/scripts/parse-test-results.py new file mode 100644 index 00000000..5b80f3f0 --- /dev/null +++ b/.tekton/test-image/scripts/parse-test-results.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Parse JUnit XML test results into a structured JSON file. + +Usage: parse-test-results.py + +Output JSON format: + { + "total": 45, + "passed": 43, + "failed": 2, + "skipped": 0, + "errors": 0, + "failedTests": ["TestFoo", "TestBar/subtest"], + "summary": "45 total, 43 passed, 2 failed, 0 skipped, 0 errors" + } +""" + +import json +import sys +import xml.etree.ElementTree as ET + + +def parse_junit(junit_path): + tree = ET.parse(junit_path) + root = tree.getroot() + + if root.tag == "testsuites": + suites = list(root) + elif root.tag == "testsuite": + suites = [root] + else: + suites = [root] + + total = 0 + failures = 0 + errors = 0 + skipped = 0 + failed_tests = [] + + for suite in suites: + total += int(suite.get("tests", 0)) + failures += int(suite.get("failures", 0)) + errors += int(suite.get("errors", 0)) + skipped += int(suite.get("skipped", 0)) + + for tc in suite.iter("testcase"): + has_failure = tc.find("failure") is not None + has_error = tc.find("error") is not None + if has_failure or has_error: + name = tc.get("name", "unknown") + classname = tc.get("classname", "") + if classname and classname != name: + failed_tests.append(f"{classname}/{name}") + else: + failed_tests.append(name) + + passed = total - failures - errors - skipped + if passed < 0: + passed = 0 + + summary = ( + f"{total} total, {passed} passed, {failures} failed, " + f"{skipped} skipped, {errors} errors" + ) + + return { + "total": total, + "passed": passed, + "failed": failures, + "skipped": skipped, + "errors": errors, + "failedTests": failed_tests, + "summary": summary, + } + + +def main(): + if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(1) + + junit_path = sys.argv[1] + output_path = sys.argv[2] + + try: + results = parse_junit(junit_path) + except ET.ParseError as e: + print(f"ERROR: Failed to parse JUnit XML: {e}", file=sys.stderr) + results = { + "total": 0, "passed": 0, "failed": 0, "skipped": 0, "errors": 0, + "failedTests": [], "summary": "0 total (XML parse error)", + } + + with open(output_path, "w") as f: + json.dump(results, f, indent=2) + + print(f"Test results: {results['summary']}") + if results["failedTests"]: + print(f"Failed tests ({len(results['failedTests'])}):") + for name in results["failedTests"][:20]: + print(f" - {name}") + if len(results["failedTests"]) > 20: + print(f" ... and {len(results['failedTests']) - 20} more") + + +if __name__ == "__main__": + main() diff --git a/.tekton/test-image/scripts/print-cluster-login-info.sh b/.tekton/test-image/scripts/print-cluster-login-info.sh new file mode 100755 index 00000000..cdace351 --- /dev/null +++ b/.tekton/test-image/scripts/print-cluster-login-info.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Print debug login command for the EaaS test cluster + +API_SERVER=$(oc whoami --show-server 2>/dev/null || true) +PASS_FILE=$(find /credentials -name "*password" -type f 2>/dev/null | head -1) + +if [[ -n "$API_SERVER" && -n "$PASS_FILE" ]]; then + echo "========================================" + echo "DEBUG: To log in to the test cluster:" + echo " oc login $API_SERVER -u kubeadmin -p $(cat "$PASS_FILE") --insecure-skip-tls-verify" + echo "========================================" +fi diff --git a/.tekton/test-image/scripts/publish-results.sh b/.tekton/test-image/scripts/publish-results.sh new file mode 100755 index 00000000..44437f9f --- /dev/null +++ b/.tekton/test-image/scripts/publish-results.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +set -euo pipefail + +RESULTS_REPO="git@github.com:rh-gitops-release-qa/catalog-results.git" +RESULTS_FILE="results.jsonl" + +echo "Publishing pipeline results to ${RESULTS_REPO}..." + +mkdir -p ~/.ssh +cp /deploy-key/ssh-privatekey ~/.ssh/id_ed25519 +chmod 600 ~/.ssh/id_ed25519 +if [[ -f /deploy-key/known-hosts ]]; then + cp /deploy-key/known-hosts ~/.ssh/known_hosts +else + ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null +fi + +REPO_DIR=$(mktemp -d) +git clone --depth 1 "${RESULTS_REPO}" "${REPO_DIR}" + +SHARED_DIR="${SHARED_DIR:-/shared}" + +python3 -c ' +import json, os, datetime + +record = { + "pipeline": os.environ.get("PIPELINE_NAME", ""), + "pipelineRun": os.environ.get("PIPELINE_RUN_NAME", ""), + "timestamp": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "status": os.environ.get("AGGREGATE_STATUS", ""), + "openshiftVersion": os.environ.get("OPENSHIFT_VERSION", ""), + "resolvedOpenshiftVersion": os.environ.get("RESOLVED_OPENSHIFT_VERSION", ""), + "operatorChannel": os.environ.get("OPERATOR_CHANNEL", ""), + "installedCSV": os.environ.get("INSTALLED_CSV", ""), + "argocdVersion": os.environ.get("ARGOCD_VERSION", ""), + "testScript": os.environ.get("TEST_SCRIPT", ""), + "fipsEnabled": os.environ.get("FIPS_ENABLED", ""), + "upgrade": os.environ.get("UPGRADE", ""), + "logUrl": os.environ.get("LOG_URL", ""), + "logsArtifact": os.environ.get("LOGS_ARTIFACT", ""), +} + +test_results = os.path.join(os.environ.get("SHARED_DIR", "/shared"), "test-results.json") +if os.path.isfile(test_results): + with open(test_results) as f: + tr = json.load(f) + record["testsTotal"] = tr.get("total", 0) + record["testsPassed"] = tr.get("passed", 0) + record["testsFailed"] = tr.get("failed", 0) + record["testsSkipped"] = tr.get("skipped", 0) + record["testsErrors"] = tr.get("errors", 0) + record["failedTests"] = tr.get("failedTests", []) + # Derive status from actual test results, not pipeline aggregate + if tr.get("total", 0) > 0 and tr.get("failed", 0) == 0 and tr.get("errors", 0) == 0: + record["status"] = "Succeeded" + elif tr.get("failed", 0) > 0 or tr.get("errors", 0) > 0: + record["status"] = "Failed" + +build_metadata = os.path.join(os.environ.get("SHARED_DIR", "/shared"), "build-metadata.json") +if os.path.isfile(build_metadata): + with open(build_metadata) as f: + bm = json.load(f) + record["buildMetadata"] = bm + +print(json.dumps(record, separators=(",", ":"))) +' >> "${REPO_DIR}/${RESULTS_FILE}" + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +if [[ -f "${SCRIPT_DIR}/render-results.py" ]]; then + echo "Rendering dashboard..." + python3 "${SCRIPT_DIR}/render-results.py" "${REPO_DIR}" +fi + +cd "${REPO_DIR}" +git add -A +git config user.name "Konflux Pipeline" +git config user.email "noreply@konflux-ci.dev" +git commit -m "Add results: ${PIPELINE_RUN_NAME:-unknown}" + +for attempt in 1 2 3; do + if git push; then + echo "Results published successfully" + break + fi + echo "Push failed (attempt ${attempt}/3), rebasing and retrying..." + git pull --rebase +done + +rm -rf "${REPO_DIR}" diff --git a/.tekton/test-image/scripts/render-results.py b/.tekton/test-image/scripts/render-results.py new file mode 100755 index 00000000..7b7ab1df --- /dev/null +++ b/.tekton/test-image/scripts/render-results.py @@ -0,0 +1,419 @@ +#!/usr/bin/env python3 +"""Render results.jsonl into a navigable directory of Markdown files. + +Four levels of README are generated so any directory in GitHub shows a +useful at-a-glance summary: + + README.md top-level overview + {product}/README.md per-version summary + {product}/{version}/README.md per-OCP summary + {product}/{version}/ocp-{ocp}/README.md per-config summary + {product}/{version}/ocp-{ocp}/{variant}/README.md full run history (leaf) +""" +import json +import os +import re +import shutil +import sys +from collections import defaultdict + +MAX_LEAF_RUNS = 10 +MAX_HISTORY_ICONS = 4 +FAIL_DETAIL_THRESHOLD = 3 # show test names when fewer than this many failures + +PRODUCT_DIRS = { + "gitops-operator-e2e": "gitops-operator", + "gitops-operator-dast": "gitops-operator-dast", + "argocd-e2e": "argocd", +} + +VARIANTS = ["default", "upgrade", "fips", "fips-upgrade"] + + +# ── Data loading and grouping ───────────────────────────────────────────────── + +def load_records(repo_dir): + path = os.path.join(repo_dir, "results.jsonl") + if not os.path.exists(path): + return [] + records = [] + with open(path) as f: + for line in f: + line = line.strip() + if not line: + continue + try: + records.append(json.loads(line)) + except json.JSONDecodeError: + continue + return records + + +def get_version(record): + csv = record.get("installedCSV", "") + if csv: + parts = csv.rsplit(".", 1) + if len(parts) == 2 and parts[1].startswith("v"): + return parts[1] + return csv + return record.get("argocdVersion", "unknown") + + +def get_variant(record): + fips = record.get("fipsEnabled", "false") == "true" + upgrade = record.get("upgrade", "false") == "true" + if fips and upgrade: + return "fips-upgrade" + if fips: + return "fips" + if upgrade: + return "upgrade" + return "default" + + +def get_product_dir(record): + return PRODUCT_DIRS.get(record.get("pipeline", ""), record.get("pipeline", "unknown")) + + +def group_records(records): + """Return dict (product, version, ocp, variant) -> [records newest-first].""" + groups = defaultdict(list) + for r in records: + key = ( + get_product_dir(r), + get_version(r), + r.get("openshiftVersion", "unknown"), + get_variant(r), + ) + groups[key].append(r) + for key in groups: + groups[key].sort(key=lambda r: r.get("timestamp", ""), reverse=True) + return groups + + +# ── Status helpers ──────────────────────────────────────────────────────────── + +def short_test_name(full_name): + """Extract a compact identifier from a long Ginkgo test name.""" + m = re.search(r"(\d+-\d+[a-zA-Z0-9_]+)", full_name) + if m: + return m.group(1) + parts = re.split(r"[/: ]+", full_name) + last = parts[-1].strip() + return (last[:35] + "…") if len(last) > 35 else last + + +def status_cell(record): + """Rich status cell: shows fail count and test names when < FAIL_DETAIL_THRESHOLD.""" + status = record.get("status", "") + failed_count = (record.get("testsFailed") or 0) + (record.get("testsErrors") or 0) + failed_tests = record.get("failedTests", []) + + if status == "Succeeded": + passed = record.get("testsPassed") + return f"✅ {passed} pass" if passed is not None else "✅ pass" + + if not failed_count: + return "❌ ERROR" + + if failed_tests and failed_count < FAIL_DETAIL_THRESHOLD: + names = ", ".join(short_test_name(t) for t in failed_tests[:failed_count]) + return f"❌ {failed_count} fail: {names}" + + return f"❌ {failed_count} fail" + + +def status_icon(record): + """Single icon for history sparklines.""" + return "✅" if record.get("status") == "Succeeded" else "❌" + + +def history_icons(records, skip=1, n=MAX_HISTORY_ICONS): + """Compact sparkline of the last n runs (skipping the most recent).""" + icons = [status_icon(r) for r in records[skip : skip + n]] + return " ".join(icons) if icons else "—" + + +def build_meta_line(record): + """One-liner component version string from buildMetadata.""" + bm = record.get("buildMetadata") or {} + labels = { + "build": "Build", "argocd": "Argo CD", "dex": "Dex", + "redis": "Redis", "kustomize": "Kustomize", "helm": "Helm", + "gitLfs": "git-lfs", "agent": "Agent", + } + parts = [f"**{labels.get(k, k)}:** {v}" for k, v in bm.items() if v] + return ("*Component versions:* " + " | ".join(parts)) if parts else "" + + +def version_sort_key(v): + return [int(x) if x.isdigit() else x for x in re.split(r"[.\-]", v.lstrip("v"))] + + +# ── Leaf README (full run history) ─────────────────────────────────────────── + +def render_leaf_readme(records, product, version, ocp, variant): + parts = [product, version, f"OCP {ocp}"] + if variant != "default": + parts.append(variant.upper()) + + lines = [f"# {' / '.join(parts)}", ""] + lines += [ + "| Date | Status | Passed | Failed | Skipped | Channel | Logs |", + "|------|--------|--------|--------|---------|---------|------|", + ] + for r in records[:MAX_LEAF_RUNS]: + ts = r.get("timestamp", "")[:10] + st = status_cell(r) + passed = str(r["testsPassed"]) if "testsPassed" in r else "-" + failed = str(r["testsFailed"]) if "testsFailed" in r else "-" + skipped = str(r["testsSkipped"]) if "testsSkipped" in r else "-" + channel = r.get("operatorChannel", "") + log_parts = [] + if r.get("logUrl"): + log_parts.append(f"[UI]({r['logUrl']})") + if r.get("logsArtifact"): + log_parts.append(f"`oras pull {r['logsArtifact']}`") + lines.append( + f"| {ts} | {st} | {passed} | {failed} | {skipped} | {channel} | {' / '.join(log_parts)} |" + ) + + meta = build_meta_line(records[0]) + if meta: + lines += ["", meta] + if len(records) > MAX_LEAF_RUNS: + lines += ["", f"*Showing {MAX_LEAF_RUNS} of {len(records)} runs.*"] + lines.append("") + return "\n".join(lines) + + +# ── OCP-level README (config summary for one OCP version) ──────────────────── + +def render_ocp_readme(variant_map, product, version, ocp): + """variant_map: {variant: [records newest-first]}""" + lines = [f"# {product} / {version} / OCP {ocp}", ""] + + for v in VARIANTS: + if variant_map.get(v): + meta = build_meta_line(variant_map[v][0]) + if meta: + lines += [meta, ""] + break + + lines += [ + "| Config | Channel | Updated | Latest Result | History |", + "|--------|---------|---------|---------------|---------|", + ] + for variant in VARIANTS: + recs = variant_map.get(variant, []) + if not recs: + lines.append(f"| [{variant}](./{variant}/) | — | — | — | — |") + continue + latest = recs[0] + ts = latest.get("timestamp", "")[:10] + channel = latest.get("operatorChannel", "") + st = status_cell(latest) + hist = history_icons(recs) + lines.append(f"| [{variant}](./{variant}/) | {channel} | {ts} | {st} | {hist} |") + + lines.append("") + return "\n".join(lines) + + +# ── Version-level README (OCP summary for one operator version) ─────────────── + +def render_version_readme(ocp_variant_map, product, version): + """ocp_variant_map: {(ocp, variant): [records newest-first]}""" + lines = [f"# {product} / {version}", ""] + + for recs in ocp_variant_map.values(): + if recs: + meta = build_meta_line(recs[0]) + if meta: + lines += [meta, ""] + break + + ocps = sorted({ocp for ocp, _ in ocp_variant_map}, reverse=True) + present_variants = [v for v in VARIANTS if any((ocp, v) in ocp_variant_map for ocp in ocps)] + + col_hdr = " | ".join(f"**{v}**" for v in present_variants) + sep = " | ".join(["---"] * (2 + len(present_variants))) + lines += [f"| OCP | {col_hdr} | Updated |", f"| {sep} |"] + + for ocp in ocps: + cells, latest_ts = [], "" + for variant in present_variants: + recs = ocp_variant_map.get((ocp, variant), []) + if not recs: + cells.append("—") + else: + cells.append(status_cell(recs[0])) + ts = recs[0].get("timestamp", "") + if ts > latest_ts: + latest_ts = ts + lines.append(f"| [{ocp}](./ocp-{ocp}/) | {' | '.join(cells)} | {latest_ts[:10]} |") + + lines.append("") + return "\n".join(lines) + + +# ── Product-level README (version summary) ──────────────────────────────────── + +def render_product_readme(prod_groups, product): + """prod_groups: {(version, ocp, variant): [records newest-first]}""" + lines = [f"# {product}", ""] + + versions = sorted( + {version for version, _, _ in prod_groups}, + key=version_sort_key, + reverse=True, + ) + + for version in versions: + ocps = sorted( + {ocp for v, ocp, _ in prod_groups if v == version}, + reverse=True, + ) + present_variants = [ + var for var in VARIANTS + if any((version, ocp, var) in prod_groups for ocp in ocps) + ] + + col_hdr = " | ".join(f"**{v}**" for v in present_variants) + sep = " | ".join(["---"] * (3 + len(present_variants))) + + lines += [ + f"## [{version}](./{version}/)", + "", + f"| OCP | {col_hdr} | ArgoCD | Updated |", + f"| {sep} |", + ] + + for ocp in ocps: + cells, latest_ts, argocd_ver = [], "", "" + for variant in present_variants: + recs = prod_groups.get((version, ocp, variant), []) + if not recs: + cells.append("—") + else: + cells.append(status_cell(recs[0])) + ts = recs[0].get("timestamp", "") + if ts > latest_ts: + latest_ts = ts + if not argocd_ver: + argocd_ver = (recs[0].get("buildMetadata") or {}).get("argocd", "") + lines.append( + f"| [{ocp}](./{version}/ocp-{ocp}/) | {' | '.join(cells)} | {argocd_ver} | {latest_ts[:10]} |" + ) + + lines.append("") + + lines += ["---", "*Auto-generated by Konflux pipeline.*", ""] + return "\n".join(lines) + + +# ── Top-level README ────────────────────────────────────────────────────────── + +def render_top_readme(all_groups): + lines = ["# Catalog Test Results", ""] + + # Nest: product -> version -> ocp -> variant -> records + tree = defaultdict(lambda: defaultdict(lambda: defaultdict(dict))) + for (product, version, ocp, variant), recs in all_groups.items(): + tree[product][version][ocp][variant] = recs + + for product in sorted(tree): + lines.append(f"## [{product}](./{product}/)") + lines.append("") + versions = sorted(tree[product], key=version_sort_key, reverse=True) + for version in versions[:3]: + for ocp in sorted(tree[product][version], reverse=True): + variant_data = tree[product][version][ocp] + parts, latest_ts = [], "" + for var in VARIANTS: + recs = variant_data.get(var, []) + if recs: + parts.append(f"{var}: {status_icon(recs[0])}") + ts = recs[0].get("timestamp", "") + if ts > latest_ts: + latest_ts = ts + summary = " · ".join(parts) + lines.append( + f"- **[{version} / OCP {ocp}](./{product}/{version}/ocp-{ocp}/)** " + f"— {summary} *(updated {latest_ts[:10]})*" + ) + lines.append("") + + lines += ["---", "*Auto-generated by Konflux pipeline.*", ""] + return "\n".join(lines) + + +# ── Main ────────────────────────────────────────────────────────────────────── + +def clean_generated_dirs(repo_dir): + for dirname in set(PRODUCT_DIRS.values()): + path = os.path.join(repo_dir, dirname) + if os.path.isdir(path): + shutil.rmtree(path) + + +def main(): + if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} ") + sys.exit(1) + + repo_dir = sys.argv[1] + records = load_records(repo_dir) + if not records: + print("No records found in results.jsonl") + return + + all_groups = group_records(records) + clean_generated_dirs(repo_dir) + + by_product = defaultdict(dict) + for (product, version, ocp, variant), recs in all_groups.items(): + by_product[product][(version, ocp, variant)] = recs + + total_groups = 0 + for product, prod_groups in by_product.items(): + # Leaf READMEs + for (version, ocp, variant), recs in prod_groups.items(): + leaf_dir = os.path.join(repo_dir, product, version, f"ocp-{ocp}", variant) + os.makedirs(leaf_dir, exist_ok=True) + with open(os.path.join(leaf_dir, "README.md"), "w") as f: + f.write(render_leaf_readme(recs, product, version, ocp, variant)) + total_groups += 1 + + # OCP-level READMEs + ocp_buckets = defaultdict(dict) + for (version, ocp, variant), recs in prod_groups.items(): + ocp_buckets[(version, ocp)][variant] = recs + for (version, ocp), variant_map in ocp_buckets.items(): + ocp_dir = os.path.join(repo_dir, product, version, f"ocp-{ocp}") + with open(os.path.join(ocp_dir, "README.md"), "w") as f: + f.write(render_ocp_readme(variant_map, product, version, ocp)) + + # Version-level READMEs + ver_buckets = defaultdict(dict) + for (version, ocp, variant), recs in prod_groups.items(): + ver_buckets[version][(ocp, variant)] = recs + for version, ocp_variant_map in ver_buckets.items(): + ver_dir = os.path.join(repo_dir, product, version) + with open(os.path.join(ver_dir, "README.md"), "w") as f: + f.write(render_version_readme(ocp_variant_map, product, version)) + + # Product-level README + prod_dir = os.path.join(repo_dir, product) + with open(os.path.join(prod_dir, "README.md"), "w") as f: + f.write(render_product_readme(prod_groups, product)) + + # Top-level README + with open(os.path.join(repo_dir, "README.md"), "w") as f: + f.write(render_top_readme(all_groups)) + + print(f"Rendered {total_groups} result groups from {len(records)} records") + + +if __name__ == "__main__": + main() diff --git a/.tekton/test-image/scripts/run-and-save-logs.sh b/.tekton/test-image/scripts/run-and-save-logs.sh new file mode 100644 index 00000000..b0cad8e5 --- /dev/null +++ b/.tekton/test-image/scripts/run-and-save-logs.sh @@ -0,0 +1,98 @@ +#!/bin/bash +set -o pipefail + +# Wrapper that runs a command, tees output to a log file, and uploads to Quay. +# The finally step later pulls these per-task artifacts and combines them. +# +# Environment variables expected: +# - TASK_LOG_NAME - name for the log artifact (e.g. "install-operator") +# - PIPELINE_RUN_NAME - pipeline run name, used in the artifact tag +# - QUAY_REPO - quay repository for log uploads +# - QUAY_CREDENTIALS_PATH - path to .dockerconfigjson (default: /quay-credentials/.dockerconfigjson) +# +# Usage: run-and-save-logs.sh [args...] + +# shellcheck source=./lib/oras-helpers.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib/oras-helpers.sh" + +QUAY_CREDENTIALS_PATH="${QUAY_CREDENTIALS_PATH:-/quay-credentials/.dockerconfigjson}" +LOG_DIR="/tmp/task-logs" +LOG_FILE="${LOG_DIR}/${TASK_LOG_NAME}.log" + +mkdir -p "${LOG_DIR}" + +# Save execution context before running the command +{ + echo "# Environment variables captured at $(date -u +"%Y-%m-%d %H:%M:%S UTC")" + echo "# Pipeline: ${PIPELINE_RUN_NAME}" + echo "# Task: ${TASK_LOG_NAME}" + echo "# Command: $*" + echo "" + + # Export all environment variables (excluding credentials and sensitive data) + env | grep -v -E '^(PATH=|HOME=|USER=|HOSTNAME=|PWD=|OLDPWD=|LS_COLORS=|DOCKER_CONFIG=|.*PASSWORD.*=|.*SECRET.*=|.*TOKEN.*=|.*KEY.*=)' | sort +} > "${LOG_DIR}/env.sh" + +# Copy KUBECONFIG if it exists and is a file +if [[ -n "${KUBECONFIG:-}" && -f "${KUBECONFIG}" ]]; then + cp "${KUBECONFIG}" "${LOG_DIR}/kubeconfig" + echo "Saved KUBECONFIG to ${LOG_DIR}/kubeconfig" +fi + +# Create a reproduce.sh script +{ + echo '#!/bin/bash' + echo '# Script to help reproduce this task execution' + echo '#' + echo "# Pipeline: ${PIPELINE_RUN_NAME}" + echo "# Task: ${TASK_LOG_NAME}" + echo "# Captured: $(date -u +"%Y-%m-%d %H:%M:%S UTC")" + echo '#' + echo '# To use this script:' + echo '# 1. Extract the logs artifact: oras pull ' + echo '# tar xzf -logs.tar.gz' + echo '# 2. Source the environment: source env.sh' + echo '# 3. Set KUBECONFIG: export KUBECONFIG=kubeconfig (if present)' + echo '# 4. Run the command below (adjust paths as needed)' + echo '' + echo 'set -x' + echo '' + echo "# Original command:" + echo "$*" +} > "${LOG_DIR}/reproduce.sh" +chmod +x "${LOG_DIR}/reproduce.sh" + +# Run the command, tee to log file, preserve exit code +"$@" 2>&1 | tee "${LOG_FILE}" +EXIT_CODE=${PIPESTATUS[0]} + +# Collect any test result files (JUnit XML, JSON reports) produced by the command +for pattern in /tmp/task-logs/*.xml /tmp/task-logs/*.json; do + if [ -f "$pattern" ]; then + echo "Found test result file: $pattern" + fi +done + +# Upload logs to Quay if credentials are available +if [ -n "${QUAY_REPO}" ] && [ -n "${PIPELINE_RUN_NAME}" ] && [ -n "${TASK_LOG_NAME}" ]; then + echo "" + echo "==========================================" + echo "Uploading task logs: ${TASK_LOG_NAME}" + echo "==========================================" + + if ! setup_oras_auth "${QUAY_CREDENTIALS_PATH}"; then + echo "Warning: Quay credentials not available, skipping upload" + exit "${EXIT_CODE}" + fi + + IMAGE_TAG="${PIPELINE_RUN_NAME}-task-${TASK_LOG_NAME}" + + if UPLOADED_REF=$(oras_push_tarball "${LOG_DIR}" "${QUAY_REPO}" "${IMAGE_TAG}" \ + "application/vnd.konflux.logs.v1+tar" "${TASK_LOG_NAME}-logs"); then + echo "Task logs uploaded to ${UPLOADED_REF}" + else + echo "Warning: failed to upload task logs" + fi +fi + +exit "${EXIT_CODE}" diff --git a/.tekton/test-image/scripts/run-argocd-e2e-full.sh b/.tekton/test-image/scripts/run-argocd-e2e-full.sh new file mode 100755 index 00000000..2632c7a8 --- /dev/null +++ b/.tekton/test-image/scripts/run-argocd-e2e-full.sh @@ -0,0 +1,318 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Full end-to-end ArgoCD E2E test execution +# This script orchestrates the complete test flow: +# 1. Deploy ArgoCD standalone +# 2. Deploy test infrastructure (git-server, test-runner pod) +# 3. Clone, compile, and run tests inside the pod +# +# Compilation happens inside the test-runner pod (not locally), so only +# small scripts (~10KB) are copied over the network. Go build caching +# via Quay (go-cache.sh) speeds up repeat compilations. +# +# Can be run: +# - Locally for fast iteration +# - In Konflux pipeline +# +# Expected env vars: +# - ARGOCD_SERVER_IMAGE: ArgoCD server image to test +# - ARGOCD_VERSION: ArgoCD version (default: v2.14.1) +# - NAMESPACE: Target namespace (default: argocd-e2e) +# - TEST_REPO_URL: ArgoCD git repo (default: https://github.com/argoproj/argo-cd.git) +# - BRANCH: Test branch (default: v2.14.1) +# - KUBECONFIG: Path to kubeconfig + +# Configuration +ARGOCD_SERVER_IMAGE="${ARGOCD_SERVER_IMAGE:?ARGOCD_SERVER_IMAGE must be set}" +ARGOCD_VERSION="${ARGOCD_VERSION:-v2.14.1}" +NAMESPACE="${NAMESPACE:-argocd-e2e}" +TEST_REPO_URL="${TEST_REPO_URL:-https://github.com/argoproj/argo-cd.git}" +BRANCH="${BRANCH:-v2.14.1}" +TEST_RUN_FILTER="${TEST_RUN_FILTER:-}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +RESULTS_DIR="${RESULTS_DIR:-/tmp/task-logs}" +mkdir -p "${RESULTS_DIR}" + +TAG="${BRANCH}" +if [[ "${BRANCH}" =~ ^v ]]; then + TAG="${BRANCH%%+*}" +fi + +# Cleanup on exit +if [[ -f /usr/local/bin/lib/argocd-e2e-cleanup.sh ]]; then + source /usr/local/bin/lib/argocd-e2e-cleanup.sh +else + source "${SCRIPT_DIR}/lib/argocd-e2e-cleanup.sh" +fi +cleanup_resources() { + local exit_code=$? + cleanup_argocd_e2e "${NAMESPACE}" + exit "$exit_code" +} +trap cleanup_resources EXIT INT TERM + +# --- Step 1: Deploy ArgoCD --- + +echo "" +echo "==========================================" +echo "Step 1: Deploy ArgoCD Standalone" +echo "==========================================" +echo "" + +export ARGOCD_SERVER_IMAGE +export ARGOCD_VERSION +export NAMESPACE +export KUBECONFIG + +if [ -f "/usr/local/bin/deploy-argocd-standalone.sh" ]; then + /usr/local/bin/deploy-argocd-standalone.sh +else + "${SCRIPT_DIR}/deploy-argocd-standalone.sh" +fi + +# Extract results +if [ -f /tekton/results/namespace ]; then + ARGOCD_NAMESPACE=$(cat /tekton/results/namespace) + ARGOCD_SERVER=$(cat /tekton/results/server) + ARGOCD_ADMIN_PASSWORD=$(cat /tekton/results/adminPassword) + ARGOCD_SERVER_NAME=$(cat /tekton/results/serverName) + ARGOCD_REPO_SERVER_NAME=$(cat /tekton/results/repoServerName) + ARGOCD_APPLICATION_CONTROLLER_NAME=$(cat /tekton/results/applicationControllerName) + ARGOCD_REDIS_NAME=$(cat /tekton/results/redisName) +else + ARGOCD_NAMESPACE="${NAMESPACE}" + ARGOCD_SERVER=$(oc get route argocd-server -n "${NAMESPACE}" -o jsonpath='{.spec.host}' 2>/dev/null || echo "argocd-server.${NAMESPACE}.svc.cluster.local") + ARGOCD_ADMIN_PASSWORD=$(oc get secret argocd-initial-admin-secret -n "${NAMESPACE}" -o jsonpath='{.data.password}' 2>/dev/null | base64 -d || echo "password") + ARGOCD_SERVER_NAME="argocd-server" + ARGOCD_REPO_SERVER_NAME="argocd-repo-server" + ARGOCD_APPLICATION_CONTROLLER_NAME="argocd-application-controller" + ARGOCD_REDIS_NAME="argocd-redis" +fi + +export ARGOCD_NAMESPACE +export ARGOCD_SERVER +export ARGOCD_ADMIN_PASSWORD +export ARGOCD_SERVER_NAME +export ARGOCD_REPO_SERVER_NAME +export ARGOCD_APPLICATION_CONTROLLER_NAME +export ARGOCD_REDIS_NAME + +echo "" +echo "ArgoCD deployed:" +echo " Namespace: ${ARGOCD_NAMESPACE}" +echo " Server: ${ARGOCD_SERVER}" +echo " Admin password: ${ARGOCD_ADMIN_PASSWORD:0:8}..." + +# --- Step 2: Deploy Test Infrastructure --- + +echo "" +echo "==========================================" +echo "Step 2: Deploy Test Infrastructure" +echo "==========================================" +echo "" + +# Create test namespaces +# External namespaces must NOT have the e2e.argoproj.io=true label — upstream +# EnsureCleanState() deletes any namespace with that label, and external +# namespaces are expected to persist throughout the test suite. +echo "Creating test namespaces..." +oc create namespace argocd-e2e --dry-run=client -o yaml | oc apply -f - 2>/dev/null || true +oc create namespace argocd-e2e-external --dry-run=client -o yaml | oc apply -f - +oc create namespace argocd-e2e-external-2 --dry-run=client -o yaml | oc apply -f - + +# Grant privileges +oc -n argocd-e2e adm policy add-scc-to-user privileged -z default 2>/dev/null || true +oc adm policy add-cluster-role-to-user cluster-admin -z default -n argocd-e2e 2>/dev/null || true + +# Configure ArgoCD to manage Applications in external namespaces. +# Controllers get namespace config from env vars populated via valueFrom +# referencing argocd-cmd-params-cm. Patch the configmap, then restart. +EXTERNAL_NS="argocd-e2e-external,argocd-e2e-external-2" +echo "Configuring ArgoCD for external namespaces: ${EXTERNAL_NS}" +oc patch configmap argocd-cmd-params-cm -n "${ARGOCD_NAMESPACE}" --type merge -p "{ + \"data\": { + \"application.namespaces\": \"${EXTERNAL_NS}\", + \"applicationsetcontroller.namespaces\": \"${EXTERNAL_NS}\", + \"applicationsetcontroller.enable.scm.providers\": \"false\" + } +}" + +# Grant ArgoCD service accounts cluster-admin so they can manage resources +# in external namespaces (matches downstream CI's appset_cluster_role_bindings.yaml) +echo "Creating RBAC for ArgoCD in external namespaces..." +for sa in argocd-application-controller argocd-applicationset-controller argocd-server; do + oc adm policy add-cluster-role-to-user cluster-admin \ + -z "${sa}" -n "${ARGOCD_NAMESPACE}" 2>/dev/null || true +done + +# Restart controllers to pick up configmap changes (env vars from valueFrom +# are only read at pod startup) +oc rollout restart deployment/argocd-server -n "${ARGOCD_NAMESPACE}" +oc rollout restart statefulset/argocd-application-controller -n "${ARGOCD_NAMESPACE}" +oc rollout restart deployment/argocd-applicationset-controller -n "${ARGOCD_NAMESPACE}" +oc rollout restart deployment/argocd-notifications-controller -n "${ARGOCD_NAMESPACE}" +oc rollout status deployment/argocd-server -n "${ARGOCD_NAMESPACE}" --timeout=5m +oc rollout status statefulset/argocd-application-controller -n "${ARGOCD_NAMESPACE}" --timeout=5m +oc rollout status deployment/argocd-applicationset-controller -n "${ARGOCD_NAMESPACE}" --timeout=5m +oc rollout status deployment/argocd-notifications-controller -n "${ARGOCD_NAMESPACE}" --timeout=5m + +# Deploy argocd-e2e-server (git over HTTP/HTTPS/SSH + Helm repos) +if [ -f "/usr/local/bin/deploy-e2e-server.sh" ]; then + /usr/local/bin/deploy-e2e-server.sh +else + "${SCRIPT_DIR}/deploy-e2e-server.sh" +fi + +# Deploy test-runner pod (Go-capable image) +if [ -f "/usr/local/bin/deploy-test-runner-pod.sh" ]; then + /usr/local/bin/deploy-test-runner-pod.sh +else + "${SCRIPT_DIR}/deploy-test-runner-pod.sh" +fi + +# --- Step 2b: Extract argocd CLI from release-candidate image --- + +echo "" +echo "==========================================" +echo "Extracting ArgoCD CLI from Release Candidate" +echo "==========================================" + +ARGOCD_SERVER_POD=$(oc get pods -n "${ARGOCD_NAMESPACE}" \ + -l app.kubernetes.io/name=argocd-server \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + +EXTRACTED_RC=false +if [[ -n "${ARGOCD_SERVER_POD}" ]]; then + echo "Copying argocd CLI from pod ${ARGOCD_SERVER_POD}..." + oc exec -n "${ARGOCD_NAMESPACE}" e2e-test-runner -- mkdir -p /tmp/rc-argocd + + TEMP_ARGOCD=$(mktemp) + for bin_path in /usr/local/bin/argocd /usr/bin/argocd; do + if oc cp "${ARGOCD_NAMESPACE}/${ARGOCD_SERVER_POD}:${bin_path}" "${TEMP_ARGOCD}" \ + -c argocd-server 2>&1; then + if [[ -s "${TEMP_ARGOCD}" ]]; then + oc cp "${TEMP_ARGOCD}" \ + "${ARGOCD_NAMESPACE}/e2e-test-runner:/tmp/rc-argocd/argocd" + oc exec -n "${ARGOCD_NAMESPACE}" e2e-test-runner -- \ + chmod +x /tmp/rc-argocd/argocd + if oc exec -n "${ARGOCD_NAMESPACE}" e2e-test-runner -- \ + /tmp/rc-argocd/argocd version --client --short 2>/dev/null; then + EXTRACTED_RC=true + echo "Release-candidate argocd CLI extracted successfully" + break + fi + fi + fi + done + rm -f "${TEMP_ARGOCD}" + + if [[ "${EXTRACTED_RC}" != "true" ]]; then + echo "WARNING: Could not extract argocd CLI from release-candidate image" + echo " Tests will fall back to source-compiled argocd CLI" + fi +else + echo "WARNING: argocd-server pod not found in ${ARGOCD_NAMESPACE}" + echo " Tests will fall back to source-compiled argocd CLI" +fi + +# --- Step 3: Copy Helper Scripts to Pod --- + +echo "" +echo "==========================================" +echo "Step 3: Copy Helper Scripts to Pod" +echo "==========================================" +echo "" + +# Detect script locations (container vs local) +if [[ -f /usr/local/bin/go-cache.sh ]]; then + GO_CACHE_SCRIPT=/usr/local/bin/go-cache.sh + ORAS_HELPERS_SCRIPT=/usr/local/bin/lib/oras-helpers.sh +else + GO_CACHE_SCRIPT="${SCRIPT_DIR}/go-cache.sh" + ORAS_HELPERS_SCRIPT="${SCRIPT_DIR}/lib/oras-helpers.sh" +fi + +# Detect skip file +if [ -f "/usr/local/config/skip-argocd.txt" ]; then + SKIP_FILE=/usr/local/config/skip-argocd.txt +else + SKIP_FILE="${SCRIPT_DIR}/../config/skip-argocd.txt" +fi + +# Build skip pattern +SKIP_FROM_FILE="" +if [[ -f "$SKIP_FILE" ]]; then + SKIP_FROM_FILE=$(grep -v '^\s*#' "$SKIP_FILE" | grep -v '^\s*$' | paste -sd '|') +fi +SKIP_FROM_FILE="${SKIP_FROM_FILE:-TestCreateAndUseAccount|TestCanIGetLogs|TestAccountSessionToken}" + +if [[ -n "${ARGOCD_E2E_SKIP:-}" && -n "${SKIP_FROM_FILE}" ]]; then + ARGOCD_E2E_SKIP="${SKIP_FROM_FILE}|${ARGOCD_E2E_SKIP}" +else + ARGOCD_E2E_SKIP="${SKIP_FROM_FILE}" +fi + +# Copy go-cache scripts to pod (small files, ~5KB total) +oc exec -n "${NAMESPACE}" e2e-test-runner -- mkdir -p /opt/e2e-test/lib +if [[ -f "${GO_CACHE_SCRIPT}" && -f "${ORAS_HELPERS_SCRIPT}" ]]; then + echo "Copying go-cache scripts to pod..." + oc cp "${GO_CACHE_SCRIPT}" "${NAMESPACE}/e2e-test-runner:/opt/e2e-test/go-cache.sh" + oc cp "${ORAS_HELPERS_SCRIPT}" "${NAMESPACE}/e2e-test-runner:/opt/e2e-test/lib/oras-helpers.sh" +else + echo "Go-cache scripts not found locally, skipping (compilation will run without cache)" +fi + +# Copy Quay credentials if available (for go-cache push/pull) +if [[ -f /quay-credentials/.dockerconfigjson ]]; then + echo "Copying Quay credentials to pod..." + oc exec -n "${NAMESPACE}" e2e-test-runner -- mkdir -p /opt/e2e-test/quay-credentials + oc cp /quay-credentials/.dockerconfigjson "${NAMESPACE}/e2e-test-runner:/opt/e2e-test/quay-credentials/.dockerconfigjson" +fi + +echo "Helper scripts copied" + +# --- Step 4: Build and Run Tests in Pod --- + +echo "" +echo "==========================================" +echo "Step 4: Build and Run Tests in Pod" +echo "==========================================" +echo "" + +# Copy inner test script to pod and execute with env vars forwarded +if [[ -f /usr/local/bin/run-argocd-e2e-in-pod-inner.sh ]]; then + INNER_SCRIPT=/usr/local/bin/run-argocd-e2e-in-pod-inner.sh +else + INNER_SCRIPT="${SCRIPT_DIR}/run-argocd-e2e-in-pod-inner.sh" +fi + +oc cp "${INNER_SCRIPT}" "${NAMESPACE}/e2e-test-runner:/tmp/run_test.sh" + +echo "Executing tests inside pod..." +oc exec -n "${NAMESPACE}" e2e-test-runner -- \ + env \ + TEST_REPO_URL="${TEST_REPO_URL}" \ + BRANCH="${BRANCH}" \ + TAG="${TAG}" \ + ARGOCD_NAMESPACE="${ARGOCD_NAMESPACE}" \ + ARGOCD_ADMIN_PASSWORD="${ARGOCD_ADMIN_PASSWORD}" \ + ARGOCD_SERVER_NAME="${ARGOCD_SERVER_NAME}" \ + ARGOCD_REDIS_NAME="${ARGOCD_REDIS_NAME}" \ + ARGOCD_REPO_SERVER_NAME="${ARGOCD_REPO_SERVER_NAME}" \ + ARGOCD_APPLICATION_CONTROLLER_NAME="${ARGOCD_APPLICATION_CONTROLLER_NAME}" \ + ARGOCD_E2E_SKIP="${ARGOCD_E2E_SKIP}" \ + TEST_RUN_FILTER="${TEST_RUN_FILTER}" \ + USE_RC_ARGOCD_CLI="${EXTRACTED_RC}" \ + bash /tmp/run_test.sh + +TEST_EXIT_CODE=$? + +echo "" +echo "==========================================" +echo "Tests completed with exit code: ${TEST_EXIT_CODE}" +echo "==========================================" + +exit ${TEST_EXIT_CODE} diff --git a/.tekton/test-image/scripts/run-argocd-e2e-in-pod-inner.sh b/.tekton/test-image/scripts/run-argocd-e2e-in-pod-inner.sh new file mode 100644 index 00000000..19f327e1 --- /dev/null +++ b/.tekton/test-image/scripts/run-argocd-e2e-in-pod-inner.sh @@ -0,0 +1,283 @@ +#!/bin/bash +set -euo pipefail + +# ArgoCD E2E test runner — executed inside the e2e-test-runner pod. +# All configuration is passed via environment variables from the outer script. + +: "${TEST_REPO_URL:?TEST_REPO_URL must be set}" +: "${BRANCH:?BRANCH must be set}" +: "${TAG:?TAG must be set}" +: "${ARGOCD_NAMESPACE:?ARGOCD_NAMESPACE must be set}" +: "${ARGOCD_ADMIN_PASSWORD:?ARGOCD_ADMIN_PASSWORD must be set}" +: "${ARGOCD_SERVER_NAME:?ARGOCD_SERVER_NAME must be set}" +: "${ARGOCD_REDIS_NAME:?ARGOCD_REDIS_NAME must be set}" +: "${ARGOCD_REPO_SERVER_NAME:?ARGOCD_REPO_SERVER_NAME must be set}" +: "${ARGOCD_APPLICATION_CONTROLLER_NAME:?ARGOCD_APPLICATION_CONTROLLER_NAME must be set}" +: "${ARGOCD_E2E_SKIP:?ARGOCD_E2E_SKIP must be set}" +TEST_RUN_FILTER="${TEST_RUN_FILTER:-}" + +echo "==========================================" +echo "ArgoCD E2E Tests (Inside Pod)" +echo "==========================================" +echo "Pod: $(hostname)" +echo "" + +# --- Phase 1: Clone and Compile --- + +WORKDIR=/opt/e2e-test +ARGO_CD_DIR="${WORKDIR}/argo-cd" +export HOME="${WORKDIR}" +export GOCACHE="${WORKDIR}/go-cache" +export GOMODCACHE="${WORKDIR}/go-mod" +export GODEBUG="tarinsecurepath=0,zipinsecurepath=0" +export GOTOOLCHAIN=auto +mkdir -p "${GOCACHE}" "${GOMODCACHE}" + +git config --global user.name "E2E Test" +git config --global user.email "e2e@example.com" +git config --global --add safe.directory "*" + +TARGET_ARCH=$(uname -m) +case "${TARGET_ARCH}" in + x86_64) TARGET_ARCH="amd64" ;; + aarch64) TARGET_ARCH="arm64" ;; +esac + +# Check for pre-built tests (pipeline image has /testsuites/argocd/v2.14/) +if [[ "${TAG}" =~ ^v2\.14 ]] && [[ -d /testsuites/argocd/v2.14 ]]; then + PREBUILT_DIR="/testsuites/argocd/v2.14" + if [[ -f "${PREBUILT_DIR}/e2e.test" && -f "${PREBUILT_DIR}/dist/argocd" ]]; then + BINARY_ARCH=$(file "${PREBUILT_DIR}/e2e.test" | grep -oP '(x86-64|aarch64|ARM aarch64)' | head -1) + if [[ ("${BINARY_ARCH}" == "x86-64" && "${TARGET_ARCH}" == "amd64") || \ + (("${BINARY_ARCH}" == "aarch64" || "${BINARY_ARCH}" == "ARM aarch64") && "${TARGET_ARCH}" == "arm64") ]]; then + echo "Using pre-built tests from ${PREBUILT_DIR}" + mkdir -p "${ARGO_CD_DIR}" + cp -a "${PREBUILT_DIR}"/* "${ARGO_CD_DIR}/" + if [[ "${USE_RC_ARGOCD_CLI:-false}" == "true" && -f /tmp/rc-argocd/argocd ]]; then + echo "Overriding pre-built argocd CLI with release-candidate binary" + cp /tmp/rc-argocd/argocd "${ARGO_CD_DIR}/dist/argocd" + chmod +x "${ARGO_CD_DIR}/dist/argocd" + fi + fi + fi +fi + +# Clone and compile if no pre-built tests +if [[ ! -f "${ARGO_CD_DIR}/e2e.test" ]]; then + echo "Cloning ArgoCD ${BRANCH}..." + git clone --branch "${BRANCH}" --depth 1 "${TEST_REPO_URL}" "${ARGO_CD_DIR}" + cd "${ARGO_CD_DIR}" + + # Pull go-cache (best-effort) + if [[ -f /opt/e2e-test/go-cache.sh ]]; then + if [[ -f /opt/e2e-test/quay-credentials/.dockerconfigjson ]]; then + mkdir -p /quay-credentials + ln -sf /opt/e2e-test/quay-credentials/.dockerconfigjson /quay-credentials/.dockerconfigjson + fi + source /opt/e2e-test/go-cache.sh + go_cache_pull "argocd-${TAG}" || true + fi + + go mod download + + CLIENT_VERSION=$(cat VERSION 2>/dev/null || echo "${TAG}") + CLIENT_VERSION="${CLIENT_VERSION#v}" + MODULE_PATH=$(head -1 go.mod | awk '{print $2}') + + echo "Compiling E2E test binary (${TARGET_ARCH})..." + ( while true; do echo "still compiling..."; sleep 60; done ) & + HEARTBEAT_PID=$! + + GOOS=linux GOARCH="${TARGET_ARCH}" go test -c \ + -ldflags "-X ${MODULE_PATH}/common.version=${CLIENT_VERSION}" \ + -o e2e.test ./test/e2e + + if [[ "${USE_RC_ARGOCD_CLI:-false}" == "true" && -f /tmp/rc-argocd/argocd ]]; then + echo "Using release-candidate argocd CLI from deployed image" + mkdir -p dist + cp /tmp/rc-argocd/argocd dist/argocd + chmod +x dist/argocd + echo "RC argocd version: $(dist/argocd version --client --short 2>/dev/null || echo 'unknown')" + else + echo "Building ArgoCD CLI from source..." + GOOS=linux GOARCH="${TARGET_ARCH}" go build \ + -ldflags "-X ${MODULE_PATH}/common.version=${CLIENT_VERSION}" \ + -o dist/argocd ./cmd + fi + + kill "${HEARTBEAT_PID}" 2>/dev/null || true + + # Push updated go-cache (best-effort) + if [[ -f /opt/e2e-test/go-cache.sh ]]; then + go_cache_push "argocd-${TAG}" || true + fi +fi + +echo "Test assets ready:" +ls -lh "${ARGO_CD_DIR}/e2e.test" +ls -lh "${ARGO_CD_DIR}/dist/argocd" + +# --- Phase 2: Run Tests --- + +export ARGOCD_E2E_REMOTE=true +export ARGOCD_SERVER="argocd-server.${ARGOCD_NAMESPACE}.svc.cluster.local" +export ARGOCD_E2E_ADMIN_USERNAME=admin +export ARGOCD_E2E_ADMIN_PASSWORD="${ARGOCD_ADMIN_PASSWORD}" +export ARGOCD_E2E_NAMESPACE="${ARGOCD_NAMESPACE}" +export ARGOCD_E2E_APP_NAMESPACE=argocd-e2e-external +export ARGOCD_APPLICATION_NAMESPACES="argocd-e2e-external,argocd-e2e-external-2" +export ARGOCD_E2E_SERVER_NAME="${ARGOCD_SERVER_NAME}" +export ARGOCD_E2E_REDIS_NAME="${ARGOCD_REDIS_NAME}" +export ARGOCD_E2E_REPO_SERVER_NAME="${ARGOCD_REPO_SERVER_NAME}" +export ARGOCD_E2E_APPLICATION_CONTROLLER_NAME="${ARGOCD_APPLICATION_CONTROLLER_NAME}" +# Push URLs (unauthenticated HTTP for CI to push test fixtures) +export ARGOCD_E2E_GIT_SERVICE="http://argocd-e2e-server:9081/argo-e2e/testdata.git" +export ARGOCD_E2E_HELM_SERVICE="http://argocd-e2e-server:9081/helm-repo" +export ARGOCD_E2E_GIT_SERVICE_SUBMODULE="http://argocd-e2e-server:9081/argo-e2e/submodule.git" +export ARGOCD_E2E_GIT_SERVICE_SUBMODULE_PARENT="http://argocd-e2e-server:9081/argo-e2e/submoduleParent.git" +# Test URLs (what ArgoCD uses to fetch repos) +export ARGOCD_E2E_REPO_SSH="ssh://root@argocd-e2e-server:2222/tmp/argo-e2e/testdata.git" +export ARGOCD_E2E_REPO_SSH_SUBMODULE="ssh://root@argocd-e2e-server:2222/tmp/argo-e2e/submodule.git" +export ARGOCD_E2E_REPO_SSH_SUBMODULE_PARENT="ssh://root@argocd-e2e-server:2222/tmp/argo-e2e/submoduleParent.git" +export ARGOCD_E2E_REPO_HTTPS="https://argocd-e2e-server:9443/argo-e2e/testdata.git" +export ARGOCD_E2E_REPO_HTTPS_CLIENT_CERT="https://argocd-e2e-server:9444/argo-e2e/testdata.git" +export ARGOCD_E2E_REPO_HTTPS_SUBMODULE="https://argocd-e2e-server:9443/argo-e2e/submodule.git" +export ARGOCD_E2E_REPO_HTTPS_SUBMODULE_PARENT="https://argocd-e2e-server:9443/argo-e2e/submoduleParent.git" +export ARGOCD_E2E_REPO_HELM="https://argocd-e2e-server:9444/helm-repo" +export ARGOCD_E2E_REPO_DEFAULT="http://argocd-e2e-server:9081/argo-e2e/testdata.git" +# Skip flags +export ARGOCD_E2E_SKIP_GPG=true +export ARGOCD_E2E_SKIP_OPENSHIFT=true +export ARGOCD_E2E_SKIP_HELM=false +export ARGOCD_E2E_K3S=true +export ARGOCD_E2E_DEFAULT_TIMEOUT=30 +export ARGOCD_GPG_ENABLED=true +export NO_PROXY="*" +export ARGOCD_E2E_SKIP="${ARGOCD_E2E_SKIP}" +export PATH="${ARGO_CD_DIR}/dist:/tmp/bin:${PATH}" + +# Ensure kubectl is available (go-toolset image only has oc) +mkdir -p /tmp/bin +if ! command -v kubectl >/dev/null 2>&1; then + if command -v oc >/dev/null 2>&1; then + ln -sf "$(which oc)" /tmp/bin/kubectl + echo "Symlinked kubectl -> $(which oc)" + else + echo "WARNING: neither kubectl nor oc found" + fi +fi + +echo "" +echo "Connectivity checks:" +getent hosts "argocd-server.${ARGOCD_NAMESPACE}.svc.cluster.local" || echo " WARNING: ArgoCD DNS failed" +getent hosts "argocd-e2e-server" || echo " WARNING: argocd-e2e-server DNS failed" + +# Run from test/e2e/ (upstream convention — relative paths depend on this) +cd "${ARGO_CD_DIR}/test/e2e" + +# Crash-resilient test runner: upstream fixture code contains log.Fatal() calls +# that kill the entire test binary when certain operations fail (repo add, cluster +# upsert). This loop detects crashes, identifies the offending test, adds it to +# the skip list, and re-runs the remaining tests. +MAX_CRASH_RETRIES=5 +CRASH_RETRY=0 +CRASH_SKIP="" +TOTAL_PASSED=0 +TOTAL_FAILED=0 +TOTAL_SKIPPED=0 +FINAL_EXIT=0 +TEST_LOG="/tmp/e2e-test-run.log" + +while true; do + FULL_SKIP="${ARGOCD_E2E_SKIP}" + if [[ -n "${CRASH_SKIP}" ]]; then + FULL_SKIP="${FULL_SKIP:+${FULL_SKIP}|}${CRASH_SKIP}" + fi + + if [[ ${CRASH_RETRY} -gt 0 ]]; then + echo "" + echo "==========================================" + echo "Crash recovery retry ${CRASH_RETRY}/${MAX_CRASH_RETRIES}" + echo " Crashed tests skipped: ${CRASH_SKIP}" + echo "==========================================" + # Recreate external namespaces (crash or EnsureCleanState may have removed them) + kubectl create namespace argocd-e2e-external --dry-run=client -o yaml | kubectl apply -f - 2>/dev/null || true + kubectl create namespace argocd-e2e-external-2 --dry-run=client -o yaml | kubectl apply -f - 2>/dev/null || true + fi + echo "" + echo "Running: ${ARGO_CD_DIR}/e2e.test -test.v -test.timeout 60m" + [[ -n "${TEST_RUN_FILTER}" ]] && echo " Run: ${TEST_RUN_FILTER}" + echo " Skip: ${FULL_SKIP}" + echo "" + + set +e + ${ARGO_CD_DIR}/e2e.test -test.v -test.timeout 60m \ + ${TEST_RUN_FILTER:+-test.run "${TEST_RUN_FILTER}"} \ + ${FULL_SKIP:+-test.skip "${FULL_SKIP}"} 2>&1 | tee "${TEST_LOG}" + EXIT_CODE=${PIPESTATUS[0]} + set -e + + RUN_PASSED=$(grep -c '^--- PASS:' "${TEST_LOG}" 2>/dev/null || true) + RUN_FAILED=$(grep -c '^--- FAIL:' "${TEST_LOG}" 2>/dev/null || true) + RUN_SKIPPED=$(grep -c '^--- SKIP:' "${TEST_LOG}" 2>/dev/null || true) + TOTAL_PASSED=$((TOTAL_PASSED + RUN_PASSED)) + TOTAL_FAILED=$((TOTAL_FAILED + RUN_FAILED)) + TOTAL_SKIPPED=$((TOTAL_SKIPPED + RUN_SKIPPED)) + + if [[ ${EXIT_CODE} -eq 0 ]]; then + break + fi + + # Check if the binary completed normally (last lines contain "FAIL" or "ok") + if tail -5 "${TEST_LOG}" | grep -qE '^(FAIL|ok\s)'; then + FINAL_EXIT=${EXIT_CODE} + break + fi + + # Binary crashed mid-suite + CRASH_RETRY=$((CRASH_RETRY + 1)) + + # Extract the top-level test that was running when the crash happened + CRASHED_TEST=$(grep '^=== RUN ' "${TEST_LOG}" | tail -1 \ + | sed 's/=== RUN *//' | awk '{print $1}' | cut -d/ -f1) + + if [[ -z "${CRASHED_TEST}" ]]; then + echo "ERROR: Binary crashed but could not identify the crashing test" + FINAL_EXIT=${EXIT_CODE} + break + fi + + TOTAL_FAILED=$((TOTAL_FAILED + 1)) + + echo "" + echo "==========================================" + echo "CRASH DETECTED during: ${CRASHED_TEST}" + echo " Exit code: ${EXIT_CODE}" + echo " Tests completed before crash: $((RUN_PASSED + RUN_FAILED + RUN_SKIPPED))" + echo "==========================================" + + if [[ ${CRASH_RETRY} -gt ${MAX_CRASH_RETRIES} ]]; then + echo "Max crash retries (${MAX_CRASH_RETRIES}) exceeded" + FINAL_EXIT=${EXIT_CODE} + break + fi + + CRASH_SKIP="${CRASH_SKIP:+${CRASH_SKIP}|}${CRASHED_TEST}" + echo "Skipping ${CRASHED_TEST}, retrying remaining tests..." +done + +echo "" +echo "==========================================" +echo "Final Results" +echo "==========================================" +echo " Passed: ${TOTAL_PASSED}" +echo " Failed: ${TOTAL_FAILED}" +echo " Skipped: ${TOTAL_SKIPPED}" +if [[ ${CRASH_RETRY} -gt 0 ]]; then + echo " Crash retries: ${CRASH_RETRY}" + echo " Crashed tests: ${CRASH_SKIP}" +fi + +if [[ ${TOTAL_FAILED} -gt 0 || ${FINAL_EXIT} -ne 0 ]]; then + exit 1 +fi diff --git a/.tekton/test-image/scripts/run-argocd-e2e-tests-in-pod.sh b/.tekton/test-image/scripts/run-argocd-e2e-tests-in-pod.sh new file mode 100755 index 00000000..19282a77 --- /dev/null +++ b/.tekton/test-image/scripts/run-argocd-e2e-tests-in-pod.sh @@ -0,0 +1,232 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Run ArgoCD E2E tests inside e2e-test-runner pod in test cluster +# This script: +# 1. Deploys test infrastructure (git-server, test-runner pod) +# 2. Copies helper scripts to pod (~10KB) +# 3. Pod clones, compiles (with go-cache), and runs tests +# +# Compilation happens inside the pod, not in the Konflux task container. +# Go build caching via Quay (go-cache.sh) speeds up repeat compilations. +# Pre-built test suites at /testsuites/ are detected and used if available. +# +# Expected env vars (from deploy-argocd task results): +# - ARGOCD_NAMESPACE: Namespace where ArgoCD is deployed +# - ARGOCD_ADMIN_PASSWORD: ArgoCD admin password +# - ARGOCD_SERVER_NAME: ArgoCD server deployment name +# - ARGOCD_REPO_SERVER_NAME: ArgoCD repo-server deployment name +# - ARGOCD_APPLICATION_CONTROLLER_NAME: ArgoCD application-controller deployment name +# - ARGOCD_REDIS_NAME: ArgoCD redis deployment name +# +# Other expected env vars: +# - TEST_REPO_URL: ArgoCD git repo URL (default: https://github.com/argoproj/argo-cd.git) +# - BRANCH: ArgoCD version tag or branch (default: v2.14.1) +# - KUBECONFIG: Path to kubeconfig + +RESULTS_DIR="${RESULTS_DIR:-/tmp/task-logs}" +mkdir -p "${RESULTS_DIR}" + +TEST_REPO_URL="${TEST_REPO_URL:-https://github.com/argoproj/argo-cd.git}" +BRANCH="${BRANCH:-v2.14.1}" + +SKIP_FILE=/usr/local/config/skip-argocd.txt +SKIP_FROM_FILE="" +if [[ -f "$SKIP_FILE" ]]; then + SKIP_FROM_FILE=$(grep -v '^\s*#' "$SKIP_FILE" | grep -v '^\s*$' | paste -sd '|') +fi +SKIP_FROM_FILE="${SKIP_FROM_FILE:-TestCreateAndUseAccount|TestCanIGetLogs|TestAccountSessionToken}" + +if [[ -n "${ARGOCD_E2E_SKIP:-}" && -n "${SKIP_FROM_FILE}" ]]; then + ARGOCD_E2E_SKIP="${SKIP_FROM_FILE}|${ARGOCD_E2E_SKIP}" +else + ARGOCD_E2E_SKIP="${SKIP_FROM_FILE}" +fi + +TAG="${BRANCH}" +if [[ "${BRANCH}" =~ ^v ]]; then + TAG="${BRANCH%%+*}" +fi + +# Cleanup on exit +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [[ -f /usr/local/bin/lib/argocd-e2e-cleanup.sh ]]; then + source /usr/local/bin/lib/argocd-e2e-cleanup.sh +else + source "${SCRIPT_DIR}/lib/argocd-e2e-cleanup.sh" +fi +cleanup_resources() { + local exit_code=$? + cleanup_argocd_e2e "${ARGOCD_NAMESPACE}" + exit "$exit_code" +} +trap cleanup_resources EXIT INT TERM + +# --- Step 1: Deploy test infrastructure --- + +echo "" +echo "==========================================" +echo "Deploying Test Infrastructure" +echo "==========================================" + +# Create test namespaces +# External namespaces must NOT have the e2e.argoproj.io=true label — upstream +# EnsureCleanState() deletes any namespace with that label, and external +# namespaces are expected to persist throughout the test suite. +echo "Creating test namespaces..." +oc create namespace argocd-e2e --dry-run=client -o yaml | oc apply -f - +oc create namespace argocd-e2e-external --dry-run=client -o yaml | oc apply -f - +oc create namespace argocd-e2e-external-2 --dry-run=client -o yaml | oc apply -f - + +# Grant test namespace privileges +oc -n argocd-e2e adm policy add-scc-to-user privileged -z default 2>/dev/null || true +oc adm policy add-cluster-role-to-user cluster-admin -z default -n argocd-e2e 2>/dev/null || true + +# Configure ArgoCD to manage Applications in external namespaces. +# Controllers get namespace config from env vars populated via valueFrom +# referencing argocd-cmd-params-cm. Patch the configmap, then restart. +EXTERNAL_NS="argocd-e2e-external,argocd-e2e-external-2" +echo "Configuring ArgoCD for external namespaces: ${EXTERNAL_NS}" +oc patch configmap argocd-cmd-params-cm -n "${ARGOCD_NAMESPACE}" --type merge -p "{ + \"data\": { + \"application.namespaces\": \"${EXTERNAL_NS}\", + \"applicationsetcontroller.namespaces\": \"${EXTERNAL_NS}\", + \"applicationsetcontroller.enable.scm.providers\": \"false\" + } +}" + +# Grant ArgoCD service accounts cluster-admin so they can manage resources +# in external namespaces (matches downstream CI's appset_cluster_role_bindings.yaml) +echo "Creating RBAC for ArgoCD in external namespaces..." +for sa in argocd-application-controller argocd-applicationset-controller argocd-server; do + oc adm policy add-cluster-role-to-user cluster-admin \ + -z "${sa}" -n "${ARGOCD_NAMESPACE}" 2>/dev/null || true +done + +# Restart controllers to pick up configmap changes +oc rollout restart deployment/argocd-server -n "${ARGOCD_NAMESPACE}" +oc rollout restart statefulset/argocd-application-controller -n "${ARGOCD_NAMESPACE}" +oc rollout restart deployment/argocd-applicationset-controller -n "${ARGOCD_NAMESPACE}" +oc rollout restart deployment/argocd-notifications-controller -n "${ARGOCD_NAMESPACE}" +oc rollout status deployment/argocd-server -n "${ARGOCD_NAMESPACE}" --timeout=5m +oc rollout status statefulset/argocd-application-controller -n "${ARGOCD_NAMESPACE}" --timeout=5m +oc rollout status deployment/argocd-applicationset-controller -n "${ARGOCD_NAMESPACE}" --timeout=5m +oc rollout status deployment/argocd-notifications-controller -n "${ARGOCD_NAMESPACE}" --timeout=5m + +# Deploy argocd-e2e-server (git over HTTP/HTTPS/SSH + Helm repos) +echo "Deploying argocd-e2e-server..." +/usr/local/bin/deploy-e2e-server.sh + +# Deploy test-runner pod (Go-capable image) +/usr/local/bin/deploy-test-runner-pod.sh + +# --- Step 1b: Extract argocd CLI from release-candidate image --- + +echo "" +echo "==========================================" +echo "Extracting ArgoCD CLI from Release Candidate" +echo "==========================================" + +# The argocd-server pod runs the release-candidate image. Copy the argocd +# binary from it to the test-runner pod. Both pods run on the same cluster +# (same architecture), so no cross-arch issues. +ARGOCD_SERVER_POD=$(oc get pods -n "${ARGOCD_NAMESPACE}" \ + -l app.kubernetes.io/name=argocd-server \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + +EXTRACTED_RC=false +if [[ -n "${ARGOCD_SERVER_POD}" ]]; then + echo "Copying argocd CLI from pod ${ARGOCD_SERVER_POD}..." + oc exec -n "${ARGOCD_NAMESPACE}" e2e-test-runner -- mkdir -p /tmp/rc-argocd + + TEMP_ARGOCD=$(mktemp) + for bin_path in /usr/local/bin/argocd /usr/bin/argocd; do + if oc cp "${ARGOCD_NAMESPACE}/${ARGOCD_SERVER_POD}:${bin_path}" "${TEMP_ARGOCD}" \ + -c argocd-server 2>&1; then + if [[ -s "${TEMP_ARGOCD}" ]]; then + oc cp "${TEMP_ARGOCD}" \ + "${ARGOCD_NAMESPACE}/e2e-test-runner:/tmp/rc-argocd/argocd" + oc exec -n "${ARGOCD_NAMESPACE}" e2e-test-runner -- \ + chmod +x /tmp/rc-argocd/argocd + if oc exec -n "${ARGOCD_NAMESPACE}" e2e-test-runner -- \ + /tmp/rc-argocd/argocd version --client --short 2>/dev/null; then + EXTRACTED_RC=true + echo "Release-candidate argocd CLI extracted successfully" + break + fi + fi + fi + done + rm -f "${TEMP_ARGOCD}" + + if [[ "${EXTRACTED_RC}" != "true" ]]; then + echo "WARNING: Could not extract argocd CLI from release-candidate image" + echo " Tests will fall back to source-compiled argocd CLI" + fi +else + echo "WARNING: argocd-server pod not found in ${ARGOCD_NAMESPACE}" + echo " Tests will fall back to source-compiled argocd CLI" +fi + +# --- Step 2: Copy helper scripts to pod --- + +echo "" +echo "==========================================" +echo "Copying Helper Scripts to Pod" +echo "==========================================" + +oc exec -n "${ARGOCD_NAMESPACE}" e2e-test-runner -- mkdir -p /opt/e2e-test/lib + +# Copy go-cache scripts if available (pipeline image has oras; multi-arch go-toolset does not) +if [[ -f /usr/local/bin/go-cache.sh && -f /usr/local/bin/lib/oras-helpers.sh ]]; then + echo "Copying go-cache scripts to pod..." + oc cp /usr/local/bin/go-cache.sh "${ARGOCD_NAMESPACE}/e2e-test-runner:/opt/e2e-test/go-cache.sh" + oc cp /usr/local/bin/lib/oras-helpers.sh "${ARGOCD_NAMESPACE}/e2e-test-runner:/opt/e2e-test/lib/oras-helpers.sh" + + if [[ -f /quay-credentials/.dockerconfigjson ]]; then + echo "Copying Quay credentials to pod..." + oc exec -n "${ARGOCD_NAMESPACE}" e2e-test-runner -- mkdir -p /quay-credentials + oc cp /quay-credentials/.dockerconfigjson "${ARGOCD_NAMESPACE}/e2e-test-runner:/quay-credentials/.dockerconfigjson" + fi +else + echo "Go-cache scripts not found, skipping (compilation will run without cache)" +fi + +echo "Helper scripts copied" + +# --- Step 3: Build and run tests inside pod --- + +echo "" +echo "==========================================" +echo "Running ArgoCD E2E Tests in Pod" +echo "==========================================" + +# Copy inner test script to pod and execute with env vars forwarded +oc cp /usr/local/bin/run-argocd-e2e-in-pod-inner.sh \ + "${ARGOCD_NAMESPACE}/e2e-test-runner:/tmp/run_test.sh" + +echo "Executing tests inside pod..." +oc exec -n "${ARGOCD_NAMESPACE}" e2e-test-runner -- \ + env \ + TEST_REPO_URL="${TEST_REPO_URL}" \ + BRANCH="${BRANCH}" \ + TAG="${TAG}" \ + ARGOCD_NAMESPACE="${ARGOCD_NAMESPACE}" \ + ARGOCD_ADMIN_PASSWORD="${ARGOCD_ADMIN_PASSWORD}" \ + ARGOCD_SERVER_NAME="${ARGOCD_SERVER_NAME}" \ + ARGOCD_REDIS_NAME="${ARGOCD_REDIS_NAME}" \ + ARGOCD_REPO_SERVER_NAME="${ARGOCD_REPO_SERVER_NAME}" \ + ARGOCD_APPLICATION_CONTROLLER_NAME="${ARGOCD_APPLICATION_CONTROLLER_NAME}" \ + ARGOCD_E2E_SKIP="${ARGOCD_E2E_SKIP}" \ + TEST_RUN_FILTER="${TEST_RUN_FILTER:-}" \ + USE_RC_ARGOCD_CLI="${EXTRACTED_RC}" \ + bash /tmp/run_test.sh + +TEST_EXIT_CODE=$? + +echo "" +echo "==========================================" +echo "Tests completed with exit code: ${TEST_EXIT_CODE}" +echo "==========================================" + +exit ${TEST_EXIT_CODE} diff --git a/.tekton/test-image/scripts/run-argocd-e2e-tests.sh b/.tekton/test-image/scripts/run-argocd-e2e-tests.sh new file mode 100644 index 00000000..66c48a30 --- /dev/null +++ b/.tekton/test-image/scripts/run-argocd-e2e-tests.sh @@ -0,0 +1,411 @@ +#!/usr/bin/env bash +set -u -o pipefail + +# Upstream ArgoCD E2E tests - expects ArgoCD already deployed +# This script sets up test infrastructure and runs E2E tests against existing ArgoCD +# +# Expected env vars (from deploy-argocd task results): +# - ARGOCD_NAMESPACE: Namespace where ArgoCD is deployed +# - ARGOCD_SERVER: ArgoCD server external Route hostname +# - ARGOCD_ADMIN_PASSWORD: ArgoCD admin password +# - ARGOCD_SERVER_NAME: ArgoCD server deployment name +# - ARGOCD_REPO_SERVER_NAME: ArgoCD repo-server deployment name +# - ARGOCD_APPLICATION_CONTROLLER_NAME: ArgoCD application-controller deployment name +# - ARGOCD_REDIS_NAME: ArgoCD redis deployment name +# +# Other expected env vars: +# - TEST_REPO_URL: ArgoCD git repo URL (default: https://github.com/argoproj/argo-cd.git) +# - BRANCH: ArgoCD version tag or branch (default: v2.14.1) +# - KUBECONFIG: Path to kubeconfig + +RESULTS_DIR="${RESULTS_DIR:-/tmp/task-logs}" +mkdir -p "${RESULTS_DIR}" + +ROOT_DIR=$(mktemp -d) +TEST_REPO_URL="${TEST_REPO_URL:-https://github.com/argoproj/argo-cd.git}" +BRANCH="${BRANCH:-v2.14.1}" + +SKIP_FILE=/usr/local/config/skip-argocd.txt +if [[ -f "$SKIP_FILE" ]]; then + ARGOCD_E2E_SKIP=$(grep -v '^\s*#' "$SKIP_FILE" | grep -v '^\s*$' | paste -sd '|') +fi +ARGOCD_E2E_SKIP="${ARGOCD_E2E_SKIP:-TestCreateAndUseAccount|TestCanIGetLogs|TestAccountSessionToken}" + +ARGO_CD_DIR="${ROOT_DIR}/argo-cd" +export HOME="$ROOT_DIR" +export GIT_HTTP_LOW_SPEED_LIMIT=1000 +export GIT_TERMINAL_PROMPT=0 +export GODEBUG="tarinsecurepath=0,zipinsecurepath=0" +export GOTOOLCHAIN=auto + +COMPILE_HEARTBEAT_PID="" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [[ -f /usr/local/bin/lib/argocd-e2e-cleanup.sh ]]; then + source /usr/local/bin/lib/argocd-e2e-cleanup.sh +else + source "${SCRIPT_DIR}/lib/argocd-e2e-cleanup.sh" +fi +cleanup_resources() { + local exit_code=$? + echo "Cleaning up..." + if [[ -n "${COMPILE_HEARTBEAT_PID}" ]]; then + kill "${COMPILE_HEARTBEAT_PID}" 2>/dev/null || true + fi + cleanup_argocd_e2e "${ARGOCD_NAMESPACE:-argocd-e2e}" + exit "$exit_code" +} +trap cleanup_resources EXIT INT TERM + +# --- Clone and compile test suite --- + +git config --global user.name "Tekton Pipeline" +git config --global user.email "tekton@example.com" +git config --global --add safe.directory "*" + +# Debug: Check if pre-compiled test suites exist in image +echo "Checking for pre-compiled test suites..." +if [ -d /testsuites ]; then + echo " /testsuites directory exists" + ls -la /testsuites/ 2>&1 | head -10 + if [ -d /testsuites/argocd ]; then + echo " ArgoCD test suites:" + ls -la /testsuites/argocd/ 2>&1 + fi +else + echo " WARNING: /testsuites directory not found - will compile from source" +fi + +# Detect test runner architecture (where this pod/container is running) +# Note: Tests execute in the Konflux pod, not on the target cluster nodes +TARGET_ARCH=$(uname -m) +case "${TARGET_ARCH}" in + x86_64) TARGET_ARCH="amd64" ;; + aarch64) TARGET_ARCH="arm64" ;; +esac +echo "Test runner architecture: ${TARGET_ARCH}" + +TAG="${BRANCH}" +if [[ "${BRANCH}" =~ ^v ]]; then + TAG="${BRANCH%%+*}" +fi + +IMAGE_TAG="${TAG}" +if [[ "${IMAGE_TAG}" == "master" || "${IMAGE_TAG}" == "main" ]]; then + IMAGE_TAG="latest" +fi + +# Check for pre-built ArgoCD E2E tests in test image +# Match by version prefix (v2.14.1 → v2.14) +# Note: master is not pre-compiled (requires Go 1.26+) +PREBUILT_BASE="/testsuites/argocd" +PREBUILT_DIR="" + +echo "Checking for pre-built tests (TAG=${TAG}, TARGET_ARCH=${TARGET_ARCH})" + +if [[ "${TAG}" =~ ^v2\.14 ]]; then + PREBUILT_DIR="${PREBUILT_BASE}/v2.14" + echo " Looking for pre-built v2.14 tests at: ${PREBUILT_DIR}" +fi + +# Verify pre-built binaries exist and match target architecture +if [[ -n "${PREBUILT_DIR}" ]]; then + echo " Checking if directory exists: ${PREBUILT_DIR}" + ls -la "${PREBUILT_DIR}" 2>&1 || echo " Directory not found" + + if [[ -f "${PREBUILT_DIR}/e2e.test" && -f "${PREBUILT_DIR}/dist/argocd" ]]; then + echo " Found pre-built binaries, checking architecture..." + # Check if the binary architecture matches target cluster architecture + BINARY_ARCH=$(file "${PREBUILT_DIR}/e2e.test" | grep -oP '(x86-64|aarch64|ARM aarch64)' | head -1) + echo " Binary arch: ${BINARY_ARCH}, Target arch: ${TARGET_ARCH}" + + if [[ ("${BINARY_ARCH}" == "x86-64" && "${TARGET_ARCH}" == "amd64") || \ + (("${BINARY_ARCH}" == "aarch64" || "${BINARY_ARCH}" == "ARM aarch64") && "${TARGET_ARCH}" == "arm64") ]]; then + echo "Using pre-built artifacts from ${PREBUILT_DIR} (${TARGET_ARCH})" + mkdir -p "${ARGO_CD_DIR}" + cp -a "${PREBUILT_DIR}"/* "${ARGO_CD_DIR}/" + cd "${ARGO_CD_DIR}" || exit 1 + # Override pre-built argocd CLI with release-candidate binary + if [[ -n "${ARGOCD_SERVER_IMAGE:-}" ]]; then + echo "Extracting release-candidate argocd CLI from ${ARGOCD_SERVER_IMAGE}..." + EXTRACT_AUTH_DIR=$(mktemp -d) + oc get secret pull-secret -n openshift-config \ + -o jsonpath='{.data.\.dockerconfigjson}' | \ + base64 -d > "${EXTRACT_AUTH_DIR}/config.json" 2>/dev/null || true + for bin_path in /usr/local/bin/argocd /usr/bin/argocd; do + RC_TMP=$(mktemp -d) + if DOCKER_CONFIG="${EXTRACT_AUTH_DIR}" oc image extract "${ARGOCD_SERVER_IMAGE}" \ + --filter-by-os="linux/${TARGET_ARCH}" \ + --path "${bin_path}:${RC_TMP}/" --confirm 2>&1; then + if [[ -f "${RC_TMP}/argocd" ]]; then + chmod +x "${RC_TMP}/argocd" + if "${RC_TMP}/argocd" version --client --short 2>/dev/null; then + cp "${RC_TMP}/argocd" "${ARGO_CD_DIR}/dist/argocd" + echo "Pre-built argocd CLI overridden with release-candidate" + rm -rf "${RC_TMP}" + break + fi + fi + fi + rm -rf "${RC_TMP}" + done + rm -rf "${EXTRACT_AUTH_DIR}" + fi + else + echo "Pre-built binary architecture (${BINARY_ARCH:-unknown}) doesn't match target (${TARGET_ARCH})" + echo "Will compile from source" + fi + else + echo " Pre-built binaries not found in ${PREBUILT_DIR}" + fi +else + echo " No pre-built directory for tag ${TAG}" +fi + +# If we haven't copied pre-built artifacts, clone and compile +if [[ ! -d "${ARGO_CD_DIR}" ]]; then + echo "Pre-built artifacts not available (want ${TAG}/${TARGET_ARCH})" + echo "Cloning argo-cd from ${TEST_REPO_URL} @ ${BRANCH}" + + git clone --depth 1 "${TEST_REPO_URL}" "${ARGO_CD_DIR}" 2>&1 + cd "${ARGO_CD_DIR}" || exit 1 + + if [[ "${BRANCH}" =~ ^v ]]; then + git fetch --depth 1 origin "tags/$TAG" 2>&1 + git checkout FETCH_HEAD 2>&1 + fi + + mkdir -p "${ROOT_DIR}/go-cache" "${ROOT_DIR}/go-mod" + export GOCACHE="${ROOT_DIR}/go-cache" + export GOMODCACHE="${ROOT_DIR}/go-mod" + export GOARCH="${TARGET_ARCH}" + export GOOS="linux" + + # Seed from image-baked caches if available + if [[ -d /usr/local/go-cache/build ]]; then + cp -a /usr/local/go-cache/build/* "${GOCACHE}/" 2>/dev/null || true + fi + if [[ -d /usr/local/go-cache/mod ]]; then + cp -a /usr/local/go-cache/mod/* "${GOMODCACHE}/" 2>/dev/null || true + fi + + # shellcheck source=/dev/null + source /usr/local/bin/go-cache.sh + go_cache_pull "argocd-${TAG}" + + go mod download + + CLIENT_VERSION=$(cat VERSION 2>/dev/null || echo "${TAG}") + CLIENT_VERSION="${CLIENT_VERSION#v}" + + echo "Compiling E2E test binary..." + ( while true; do echo "still compiling..."; sleep 60; done ) & + COMPILE_HEARTBEAT_PID=$! + + if ! go test -c -ldflags "-X github.com/argoproj/argo-cd/v3/common.version=${CLIENT_VERSION}" \ + -o e2e.test ./test/e2e 2>&1 | tee "${RESULTS_DIR}/compile.log"; then + kill "$COMPILE_HEARTBEAT_PID" 2>/dev/null || true + echo "ERROR: test compilation failed" + exit 1 + fi + kill "$COMPILE_HEARTBEAT_PID" 2>/dev/null || true + + # Try to use release-candidate argocd CLI from deployed image + RC_ARGOCD=false + if [[ -n "${ARGOCD_SERVER_IMAGE:-}" ]]; then + echo "Extracting argocd CLI from release-candidate image: ${ARGOCD_SERVER_IMAGE}" + EXTRACT_AUTH_DIR=$(mktemp -d) + oc get secret pull-secret -n openshift-config \ + -o jsonpath='{.data.\.dockerconfigjson}' | \ + base64 -d > "${EXTRACT_AUTH_DIR}/config.json" 2>/dev/null || true + + mkdir -p "${ARGO_CD_DIR}/dist" + for bin_path in /usr/local/bin/argocd /usr/bin/argocd; do + if DOCKER_CONFIG="${EXTRACT_AUTH_DIR}" oc image extract "${ARGOCD_SERVER_IMAGE}" \ + --filter-by-os="linux/${TARGET_ARCH}" \ + --path "${bin_path}:${ARGO_CD_DIR}/dist/" --confirm 2>&1; then + if [[ -f "${ARGO_CD_DIR}/dist/argocd" ]]; then + chmod +x "${ARGO_CD_DIR}/dist/argocd" + if "${ARGO_CD_DIR}/dist/argocd" version --client --short 2>/dev/null; then + RC_ARGOCD=true + echo "Using release-candidate argocd CLI" + break + else + rm -f "${ARGO_CD_DIR}/dist/argocd" + fi + fi + fi + done + rm -rf "${EXTRACT_AUTH_DIR}" + fi + + if [[ "${RC_ARGOCD}" != "true" ]]; then + echo "Falling back to building argocd CLI from source..." + go build -ldflags "-X github.com/argoproj/argo-cd/v3/common.version=${CLIENT_VERSION}" \ + -o "${ARGO_CD_DIR}/dist/argocd" ./cmd 2>&1 + fi + + go_cache_push "argocd-${TAG}" +fi + +# --- Setup test infrastructure --- + +echo "Setting up test infrastructure..." +echo "ArgoCD already deployed in namespace: ${ARGOCD_NAMESPACE}" +echo "ArgoCD server: ${ARGOCD_SERVER}" + +# Create test namespaces +echo "Creating test namespaces..." +# Note: argocd-e2e already created by deploy-argocd task, just ensure it exists +oc create namespace argocd-e2e --dry-run=client -o yaml | oc apply -f - + +# Only label the external namespaces for cleanup (not argocd-e2e where ArgoCD runs) +oc create namespace argocd-e2e-external --dry-run=client -o yaml | oc apply -f - +oc label namespace argocd-e2e-external e2e.argoproj.io=true --overwrite 2>/dev/null || true + +oc create namespace argocd-e2e-external-2 --dry-run=client -o yaml | oc apply -f - +oc label namespace argocd-e2e-external-2 e2e.argoproj.io=true --overwrite 2>/dev/null || true + +# Grant test namespace privileges +oc -n argocd-e2e adm policy add-scc-to-user privileged -z default 2>/dev/null || true +oc adm policy add-cluster-role-to-user cluster-admin -z default -n argocd-e2e 2>/dev/null || true + +# Deploy git-server for test repos +echo "Deploying git-server..." +cat <&1 || true + +# Verify ArgoCD Route is accessible +echo "Verifying ArgoCD Route..." +echo " ArgoCD server URL: ${ARGOCD_SERVER}" + +# Test Route connectivity +echo " Testing Route connectivity..." +if ! curl -k -s -o /dev/null -w "%{http_code}" "https://${ARGOCD_SERVER}/healthz" | grep -q "200"; then + echo "WARNING: ArgoCD Route healthcheck returned non-200 status" + echo " Checking Route resource..." + oc get route argocd-server -n "${ARGOCD_NAMESPACE}" -o wide || true + echo " Checking ArgoCD pods..." + oc get pods -n "${ARGOCD_NAMESPACE}" -l app.kubernetes.io/name=argocd-server || true + echo " Continuing anyway - server might still be starting..." +fi +echo " Route verification complete" + +# --- Set ArgoCD E2E environment variables --- + +echo "Configuring E2E test environment..." + +# Execution mode - remote (not local goreman) +export ARGOCD_E2E_REMOTE=true + +# ArgoCD connection (Route URL uses HTTPS on default port 443) +export ARGOCD_SERVER="${ARGOCD_SERVER}" +export ARGOCD_SERVER_INSECURE=true # Route uses self-signed cert +export ARGOCD_E2E_ADMIN_USERNAME=admin +export ARGOCD_E2E_ADMIN_PASSWORD="${ARGOCD_ADMIN_PASSWORD}" + +# Namespaces +export ARGOCD_E2E_NAMESPACE=argocd-e2e +export ARGOCD_E2E_APP_NAMESPACE=argocd-e2e-external +export ARGOCD_APPLICATION_NAMESPACES="argocd-e2e-external,argocd-e2e-external-2" + +# Component names (for finding deployments/pods) +export ARGOCD_E2E_SERVER_NAME="${ARGOCD_SERVER_NAME}" +export ARGOCD_E2E_REDIS_NAME="${ARGOCD_REDIS_NAME}" +export ARGOCD_E2E_REPO_SERVER_NAME="${ARGOCD_REPO_SERVER_NAME}" +export ARGOCD_E2E_APPLICATION_CONTROLLER_NAME="${ARGOCD_APPLICATION_CONTROLLER_NAME}" + +# Git service +export ARGOCD_E2E_GIT_SERVICE="git://git-server.argocd-e2e.svc.cluster.local:9418/testdata.git" +export ARGOCD_E2E_REPO_DEFAULT="git://git-server.argocd-e2e.svc.cluster.local:9418/testdata.git" + +# Working directory +export ARGOCD_E2E_DIR=/tmp/argo-e2e + +# CLI binary location +export DIST_DIR="${ARGO_CD_DIR}/dist" + +echo "Environment variables set:" +echo " ARGOCD_E2E_REMOTE=${ARGOCD_E2E_REMOTE}" +echo " ARGOCD_SERVER=${ARGOCD_SERVER}" +echo " ARGOCD_E2E_NAMESPACE=${ARGOCD_E2E_NAMESPACE}" +echo " ARGOCD_E2E_GIT_SERVICE=${ARGOCD_E2E_GIT_SERVICE}" +echo " DIST_DIR=${DIST_DIR}" + +# Verify CLI binary exists +if [[ ! -f "${DIST_DIR}/argocd" ]]; then + echo "ERROR: ArgoCD CLI not found at ${DIST_DIR}/argocd" + exit 1 +fi + +echo "ArgoCD CLI version:" +"${DIST_DIR}/argocd" version --client 2>&1 || true + +# --- Run E2E tests --- + +echo "" +echo "==========================================" +echo "Running ArgoCD E2E Tests" +echo "==========================================" +echo "" + +cd "${ARGO_CD_DIR}/test/e2e" || exit 1 + +# Save KUBECONFIG for tests +export KUBECONFIG="${KUBECONFIG:-${HOME}/.kube/config}" +cp "$KUBECONFIG" "${RESULTS_DIR}/kubeconfig" 2>/dev/null || true + +# Run tests +./../../e2e.test -test.v -test.timeout 60m \ + ${ARGOCD_E2E_SKIP:+-test.skip "$ARGOCD_E2E_SKIP"} 2>&1 | tee "${RESULTS_DIR}/test.log" + +TEST_EXIT_CODE=${PIPESTATUS[0]} + +echo "" +echo "==========================================" +echo "Tests completed with exit code: ${TEST_EXIT_CODE}" +echo "==========================================" + +exit "${TEST_EXIT_CODE}" diff --git a/.tekton/test-image/scripts/run-e2e-tests.sh b/.tekton/test-image/scripts/run-e2e-tests.sh new file mode 100644 index 00000000..d7ed9070 --- /dev/null +++ b/.tekton/test-image/scripts/run-e2e-tests.sh @@ -0,0 +1,189 @@ +#!/bin/bash +set -x + +# Environment variables expected: +# - TEST_REPO_URL (optional, defaults to the pre-baked repo remote) +# - BRANCH +# - TEST_DIR +# - TIMEOUT +# - PROCS +# - KUBECONFIG + +RESULTS_DIR="${RESULTS_DIR:-/tmp/task-logs}" +mkdir -p "${RESULTS_DIR}" + +CACHE_DIR=$(mktemp -d) +export GOCACHE="${CACHE_DIR}/go-cache" +export GOMODCACHE="${CACHE_DIR}/go-mod" +mkdir -p "$GOCACHE" "$GOMODCACHE" + +oc status + +# --- Ensure argocd CLI is available (some tests call `argocd login` etc.) --- +# Extract the release-candidate argocd binary from the deployed operator image. +# The Konflux task container is x86_64 while cluster nodes may be arm64 (m6g), +# so we use oc image extract with --filter-by-os to get the correct arch. +if ! command -v argocd &>/dev/null; then + ARGOCD_IMAGE=$(oc get deployment openshift-gitops-repo-server -n openshift-gitops \ + -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || true) + ARGOCD_BIN_DIR=$(mktemp -d) + ARGOCD_EXTRACTED=false + + if [[ -n "$ARGOCD_IMAGE" ]]; then + echo "Extracting argocd CLI from ${ARGOCD_IMAGE}..." + EXTRACT_AUTH_DIR=$(mktemp -d) + oc get secret pull-secret -n openshift-config \ + -o jsonpath='{.data.\.dockerconfigjson}' | \ + base64 -d > "${EXTRACT_AUTH_DIR}/config.json" 2>/dev/null || true + + for bin_path in /usr/local/bin/argocd /usr/bin/argocd; do + if DOCKER_CONFIG="${EXTRACT_AUTH_DIR}" oc image extract "${ARGOCD_IMAGE}" \ + --filter-by-os=linux/amd64 \ + --path "${bin_path}:${ARGOCD_BIN_DIR}/" --confirm 2>&1; then + if [[ -f "${ARGOCD_BIN_DIR}/argocd" ]]; then + chmod +x "${ARGOCD_BIN_DIR}/argocd" + if "${ARGOCD_BIN_DIR}/argocd" version --client --short 2>/dev/null; then + ARGOCD_EXTRACTED=true + break + else + echo "Extracted binary not executable on this arch, trying next path..." + file "${ARGOCD_BIN_DIR}/argocd" 2>/dev/null || true + rm -f "${ARGOCD_BIN_DIR}/argocd" + fi + fi + fi + done + rm -rf "${EXTRACT_AUTH_DIR}" + else + echo "openshift-gitops-repo-server deployment not found" + fi + + # Fallback 2: oc cp from running argocd-server pod (works when same arch) + if [[ "$ARGOCD_EXTRACTED" != "true" ]]; then + echo "oc image extract failed, trying oc cp from argocd-server pod..." + ARGOCD_SERVER_POD=$(oc get pods -n openshift-gitops \ + -l app.kubernetes.io/name=openshift-gitops-server \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + if [[ -z "$ARGOCD_SERVER_POD" ]]; then + ARGOCD_SERVER_POD=$(oc get pods -n openshift-gitops \ + -l app.kubernetes.io/part-of=argocd \ + -l app.kubernetes.io/component=server \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + fi + if [[ -n "$ARGOCD_SERVER_POD" ]]; then + for bin_path in /usr/local/bin/argocd /usr/bin/argocd; do + if oc cp "openshift-gitops/${ARGOCD_SERVER_POD}:${bin_path}" \ + "${ARGOCD_BIN_DIR}/argocd" 2>&1; then + if [[ -f "${ARGOCD_BIN_DIR}/argocd" ]]; then + chmod +x "${ARGOCD_BIN_DIR}/argocd" + if "${ARGOCD_BIN_DIR}/argocd" version --client --short 2>/dev/null; then + ARGOCD_EXTRACTED=true + break + else + echo "Copied binary not executable on this arch" + file "${ARGOCD_BIN_DIR}/argocd" 2>/dev/null || true + rm -f "${ARGOCD_BIN_DIR}/argocd" + fi + fi + fi + done + fi + fi + + # Fallback 3: skopeo copy with --override-arch to local OCI layout, then extract + if [[ "$ARGOCD_EXTRACTED" != "true" && -n "${ARGOCD_IMAGE:-}" ]] && command -v skopeo &>/dev/null; then + echo "Trying skopeo to copy image and extract argocd binary..." + SKOPEO_DIR=$(mktemp -d) + SKOPEO_AUTH_DIR=$(mktemp -d) + oc get secret pull-secret -n openshift-config \ + -o jsonpath='{.data.\.dockerconfigjson}' | \ + base64 -d > "${SKOPEO_AUTH_DIR}/auth.json" 2>/dev/null || true + + if skopeo copy --override-arch amd64 --override-os linux \ + --authfile "${SKOPEO_AUTH_DIR}/auth.json" \ + "docker://${ARGOCD_IMAGE}" \ + "dir:${SKOPEO_DIR}/image" 2>&1; then + # Use skopeo's dir output — the layer tarballs are in the directory. + # Extract argocd binary by searching through layers. + for layer in "${SKOPEO_DIR}"/image/*.tar "${SKOPEO_DIR}"/image/*.tar.gz; do + [[ -f "$layer" ]] || continue + if tar -tf "$layer" 2>/dev/null | grep -qE '(usr/local/bin/argocd|usr/bin/argocd)$'; then + tar -xf "$layer" -C "${ARGOCD_BIN_DIR}/" --strip-components=3 \ + usr/local/bin/argocd 2>/dev/null || \ + tar -xf "$layer" -C "${ARGOCD_BIN_DIR}/" --strip-components=2 \ + usr/bin/argocd 2>/dev/null || true + if [[ -f "${ARGOCD_BIN_DIR}/argocd" ]]; then + chmod +x "${ARGOCD_BIN_DIR}/argocd" + if "${ARGOCD_BIN_DIR}/argocd" version --client --short 2>/dev/null; then + ARGOCD_EXTRACTED=true + break + else + rm -f "${ARGOCD_BIN_DIR}/argocd" + fi + fi + fi + done + fi + rm -rf "${SKOPEO_DIR}" "${SKOPEO_AUTH_DIR}" + fi + + # Fallback 4: pre-compiled argocd from test image (may be older than deployed version) + if [[ "$ARGOCD_EXTRACTED" != "true" ]]; then + echo "Trying pre-compiled argocd from test image..." + for pre_built in /testsuites/argocd/*/dist/argocd; do + if [[ -x "$pre_built" ]] && "$pre_built" version --client --short 2>/dev/null; then + ARGOCD_BIN_DIR=$(dirname "$pre_built") + ARGOCD_EXTRACTED=true + echo "Using pre-compiled argocd from test image (version may differ from deployed)" + break + fi + done + fi + + if [[ "$ARGOCD_EXTRACTED" == "true" ]]; then + export PATH="${ARGOCD_BIN_DIR}:${PATH}" + echo "argocd CLI available: $(argocd version --client --short)" + else + echo "WARNING: argocd CLI is NOT available — tests requiring it will fail" + rm -rf "${ARGOCD_BIN_DIR}" + fi +fi + +cd /testsuites/gitops-operator/ || exit 1 +TEST_REPO_URL="${TEST_REPO_URL:-https://github.com/redhat-developer/gitops-operator.git}" +git remote set-url origin "${TEST_REPO_URL}" 2>/dev/null || git remote add origin "${TEST_REPO_URL}" +git fetch origin +git clean -fd +git checkout -B "${BRANCH}" "origin/${BRANCH}" + +# shellcheck source=/dev/null +source /usr/local/bin/go-cache.sh +go_cache_pull "operator-${BRANCH}" + +GINKGO_ARGS=() +if [[ -n "${GINKGO_SKIP:-}" ]]; then + GINKGO_ARGS+=("--skip=${GINKGO_SKIP}") + echo "Skipping tests matching: ${GINKGO_SKIP}" +fi + +if [[ -n "${GINKGO_FOCUS_FILE:-}" ]]; then + GINKGO_ARGS+=("--focus-file=${GINKGO_FOCUS_FILE}") + echo "Focusing on files matching: ${GINKGO_FOCUS_FILE}" +fi + +# Enable parallel mode only when PROCS > 1 +PARALLEL_FLAG="" +if [[ "${PROCS:-1}" -gt 1 ]]; then + PARALLEL_FLAG="-p" +fi + +TEST_EXIT=0 +/testsuites/gitops-operator/bin/ginkgo -timeout "${TIMEOUT}" ${PARALLEL_FLAG} -procs="${PROCS}" --no-color -v --trace -r \ + "${GINKGO_ARGS[@]}" \ + --junit-report="${RESULTS_DIR}/junit-results.xml" \ + --json-report="${RESULTS_DIR}/test-results.json" \ + "${TEST_DIR}/." || TEST_EXIT=$? + +go_cache_push "operator-${BRANCH}" + +exit $TEST_EXIT diff --git a/.tekton/test-image/scripts/run-parallel-tests.sh b/.tekton/test-image/scripts/run-parallel-tests.sh new file mode 100644 index 00000000..daacd9d2 --- /dev/null +++ b/.tekton/test-image/scripts/run-parallel-tests.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -x + +# Parallel ginkgo tests for the gitops-operator. +# Env vars expected: TEST_REPO_URL, BRANCH, KUBECONFIG + +# shellcheck source=./lib/load-skip-patterns.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib/load-skip-patterns.sh" + +export TEST_DIR="${TEST_DIR:-./test/openshift/e2e/ginkgo/parallel}" +export PROCS="${PROCS:-4}" +export TIMEOUT="${TIMEOUT:-90m}" + +load_ginkgo_skip_patterns /usr/local/config/skip-parallel.txt + +/usr/local/bin/run-e2e-tests.sh +exit $? diff --git a/.tekton/test-image/scripts/run-rollouts-tests.sh b/.tekton/test-image/scripts/run-rollouts-tests.sh new file mode 100644 index 00000000..940b41ad --- /dev/null +++ b/.tekton/test-image/scripts/run-rollouts-tests.sh @@ -0,0 +1,192 @@ +#!/bin/bash +set -ex + +# Argo Rollouts E2E tests adapted from downstream-CI z-stream pipeline. +# Runs three test suites: +# 1. argo-rollouts-manager E2E (cluster-scoped + namespace-scoped) +# 2. upstream argoproj/argo-rollouts E2E +# 3. rollouts-plugin-trafficrouter-openshift E2E +# +# Env vars expected: KUBECONFIG +# Env vars optional: TEST_REPO_URL, BRANCH (used to resolve commit pins from gitops-operator go.mod) + +# shellcheck source=./lib/wait-for-resources.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib/wait-for-resources.sh" + +RESULTS_DIR="${RESULTS_DIR:-/tmp/task-logs}" +mkdir -p "${RESULTS_DIR}" + +exit_code=0 +failed=0 + +OPERATOR_NAMESPACE=$(oc get deployment openshift-gitops-operator-controller-manager \ + -n openshift-gitops-operator -o jsonpath='{.metadata.namespace}' --ignore-not-found) +OPERATOR_NAMESPACE=${OPERATOR_NAMESPACE:-"openshift-operators"} + +SUBSCRIPTION_NAME=$(oc get subscription -n "$OPERATOR_NAMESPACE" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "openshift-gitops-operator") +echo "Using subscription: ${SUBSCRIPTION_NAME} in ${OPERATOR_NAMESPACE}" + +# --- Helper functions --- + +enable_rollouts_cluster_scoped() { + oc patch -n "$OPERATOR_NAMESPACE" subscription "$SUBSCRIPTION_NAME" \ + --type merge --patch '{"spec": {"config": {"env": [{"name": "CLUSTER_SCOPED_ARGO_ROLLOUTS_NAMESPACES", "value": "argo-rollouts,test-rom-ns-1,rom-ns-1"}]}}}' + + for _ in {1..30}; do + if oc get deployment openshift-gitops-operator-controller-manager -n "$OPERATOR_NAMESPACE" \ + -o jsonpath='{.spec.template.spec.containers[0].env}' | grep -q 'CLUSTER_SCOPED_ARGO_ROLLOUTS_NAMESPACES'; then + break + fi + echo "Waiting for CLUSTER_SCOPED_ARGO_ROLLOUTS_NAMESPACES to be set" + sleep 5 + done + wait_for_operator_pods "$OPERATOR_NAMESPACE" +} + +enable_rollouts_namespace_scoped() { + oc patch -n "$OPERATOR_NAMESPACE" subscription "$SUBSCRIPTION_NAME" \ + --type merge --patch '{"spec": {"config": {"env": [{"name": "CLUSTER_SCOPED_ARGO_ROLLOUTS_NAMESPACES", "value": ""}]}}}' + oc patch -n "$OPERATOR_NAMESPACE" subscription "$SUBSCRIPTION_NAME" \ + --type merge --patch '{"spec": {"config": {"env": [{"name": "NAMESPACE_SCOPED_ARGO_ROLLOUTS", "value": "true"}]}}}' + + for _ in {1..30}; do + if oc get deployment openshift-gitops-operator-controller-manager -n "$OPERATOR_NAMESPACE" \ + -o jsonpath='{.spec.template.spec.containers[0].env}' | grep -q 'NAMESPACE_SCOPED_ARGO_ROLLOUTS'; then + break + fi + echo "Waiting for NAMESPACE_SCOPED_ARGO_ROLLOUTS to be set" + sleep 5 + done + wait_for_operator_pods "$OPERATOR_NAMESPACE" +} + +disable_rollouts_config() { + oc patch -n "$OPERATOR_NAMESPACE" subscription "$SUBSCRIPTION_NAME" \ + --type json --patch '[{"op": "remove", "path": "/spec/config"}]' || true + wait_for_operator_pods "$OPERATOR_NAMESPACE" +} + +cleanup() { + disable_rollouts_config + oc delete rollouts -A --all 2>/dev/null || true + oc delete rolloutmanager -A --all 2>/dev/null || true +} +trap cleanup EXIT + +# --- Resolve commit pins from gitops-operator go.mod --- + +ROLLOUTS_TMP_DIR=$(mktemp -d) +cd "$ROLLOUTS_TMP_DIR" + +TEST_REPO_URL="${TEST_REPO_URL:-https://github.com/redhat-developer/gitops-operator.git}" +BRANCH="${BRANCH:-master}" + +echo "Resolving rollouts commit pins from ${TEST_REPO_URL} @ ${BRANCH}" +git clone --depth 1 --branch "${BRANCH}" "${TEST_REPO_URL}" gitops-operator-src + +TARGET_ROLLOUT_MANAGER_COMMIT=$(grep 'argoproj-labs/argo-rollouts-manager' \ + gitops-operator-src/go.mod | awk '{print $2}' | sed 's/.*-//' | head -1) + +if [[ -z "$TARGET_ROLLOUT_MANAGER_COMMIT" ]]; then + echo "ERROR: Could not resolve argo-rollouts-manager commit from go.mod" + exit 1 +fi + +echo "argo-rollouts-manager commit: ${TARGET_ROLLOUT_MANAGER_COMMIT}" + +# --- 1. argo-rollouts-manager E2E tests --- + +git clone https://github.com/argoproj-labs/argo-rollouts-manager +cd "$ROLLOUTS_TMP_DIR/argo-rollouts-manager" +git checkout "$TARGET_ROLLOUT_MANAGER_COMMIT" + +TARGET_PLUGIN_COMMIT=$(grep 'rollouts-plugin-trafficrouter-openshift' \ + go.mod | awk '{print $2}' | sed 's/.*-//' | head -1 || true) +if [[ -z "$TARGET_PLUGIN_COMMIT" ]]; then + TARGET_PLUGIN_COMMIT="main" + echo "rollouts-plugin commit: not pinned in go.mod, using main" +else + echo "rollouts-plugin commit (from rollouts-manager go.mod): ${TARGET_PLUGIN_COMMIT}" +fi + +export GOCACHE="${ROLLOUTS_TMP_DIR}/go-cache" +export GOMODCACHE="${ROLLOUTS_TMP_DIR}/go-mod" +mkdir -p "$GOCACHE" "$GOMODCACHE" + +# shellcheck source=/dev/null +source /usr/local/bin/go-cache.sh +go_cache_pull "rollouts-${TARGET_ROLLOUT_MANAGER_COMMIT}" + +enable_rollouts_cluster_scoped + +echo "=== Running cluster-scoped E2E tests ===" +DISABLE_METRICS=true make test-e2e-cluster-scoped 2>&1 | tee "${RESULTS_DIR}/rollout-manager-cluster-scoped.log" || exit_code=$? +if [[ $exit_code != 0 ]]; then + failed=$exit_code + exit_code=0 +fi + +kubectl delete rolloutmanagers --all -n test-rom-ns-1 || true + +enable_rollouts_namespace_scoped + +echo "=== Running namespace-scoped E2E tests ===" +DISABLE_METRICS=true make test-e2e-namespace-scoped 2>&1 | tee "${RESULTS_DIR}/rollout-manager-namespace-scoped.log" || exit_code=$? +if [[ $exit_code != 0 ]]; then + failed=$exit_code + exit_code=0 +fi + +kubectl delete rolloutmanagers --all -n test-rom-ns-1 || true + +# --- 2. Upstream argo-rollouts E2E tests --- + +enable_rollouts_cluster_scoped + +cd "$ROLLOUTS_TMP_DIR/argo-rollouts-manager" + +echo "=== Running upstream argo-rollouts E2E tests ===" +SKIP_RUN_STEP=true hack/run-upstream-argo-rollouts-e2e-tests.sh 2>&1 | tee "${RESULTS_DIR}/argo-rollouts-upstream.log" || exit_code=$? +if [[ $exit_code != 0 ]]; then + failed=$exit_code + exit_code=0 +fi + +# --- 3. rollouts-plugin-trafficrouter-openshift E2E tests --- + +echo "=== Running rollouts OpenShift route plugin E2E tests ===" + +kubectl delete ns argo-rollouts 2>/dev/null || true +kubectl wait --timeout=5m --for=delete namespace/argo-rollouts 2>/dev/null || true +kubectl create ns argo-rollouts +kubectl config set-context --current --namespace=argo-rollouts + +cat <&1 | tee "${RESULTS_DIR}/rollouts-plugin.log" || exit_code=$? +if [[ $exit_code != 0 ]]; then + failed=$exit_code +fi + +# --- Done --- + +go_cache_push "rollouts-${TARGET_ROLLOUT_MANAGER_COMMIT}" + +if [[ $failed != 0 ]]; then + echo "ERROR: One or more rollouts test suites failed" + exit 1 +fi + +echo "All rollouts tests passed" diff --git a/.tekton/test-image/scripts/run-sanity-tests.sh b/.tekton/test-image/scripts/run-sanity-tests.sh new file mode 100644 index 00000000..f230b4c6 --- /dev/null +++ b/.tekton/test-image/scripts/run-sanity-tests.sh @@ -0,0 +1,438 @@ +#!/bin/bash +set -euo pipefail + +# Release sanity checks for the gitops-operator (replaces manual GITOPS-6448 checklist). +# Validates: CSV health, operator pods, toolchain versions, basic app sync. +# +# Env vars expected: +# KUBECONFIG - path to cluster kubeconfig +# Env vars optional: +# NAMESPACE - operator namespace (default: openshift-gitops-operator) +# GITOPS_NS - ArgoCD instance namespace (default: openshift-gitops) +# CONFLUENCE_USERNAME - Confluence API username (for component matrix lookup) +# CONFLUENCE_TOKEN - Confluence API token +# CONFLUENCE_PAGE_ID - Component matrix page ID (default: 265652015) + +# shellcheck source=./lib/wait-for-resources.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib/wait-for-resources.sh" + +RESULTS_DIR="${RESULTS_DIR:-/tmp/task-logs}" +mkdir -p "${RESULTS_DIR}" + +NAMESPACE="${NAMESPACE:-openshift-gitops-operator}" +GITOPS_NS="${GITOPS_NS:-openshift-gitops}" +if [[ -z "${CONFLUENCE_USERNAME:-}" && -f /confluence-credentials/username ]]; then + CONFLUENCE_USERNAME=$(cat /confluence-credentials/username) +fi +if [[ -z "${CONFLUENCE_TOKEN:-}" && -f /confluence-credentials/token ]]; then + CONFLUENCE_TOKEN=$(cat /confluence-credentials/token) +fi +CONFLUENCE_USERNAME="${CONFLUENCE_USERNAME:-}" +CONFLUENCE_TOKEN="${CONFLUENCE_TOKEN:-}" +CONFLUENCE_PAGE_ID="${CONFLUENCE_PAGE_ID:-265652015}" + +failures=0 +fail() { echo "FAIL: $1"; failures=$((failures + 1)); } +pass() { echo "PASS: $1"; } + +# --- Fetch expected versions from Confluence Component Matrix --- +fetch_expected_versions() { + if [[ -z "${CONFLUENCE_USERNAME}" || -z "${CONFLUENCE_TOKEN}" ]]; then + echo "Confluence credentials not configured, skipping version assertions" + return 1 + fi + + local page_url="https://redhat.atlassian.net/wiki/rest/api/content/${CONFLUENCE_PAGE_ID}?expand=body.storage" + local body + body=$(curl -sf -u "${CONFLUENCE_USERNAME}:${CONFLUENCE_TOKEN}" \ + --url "${page_url}" --header "Accept: application/json" 2>/dev/null || true) + + if [[ -z "$body" ]]; then + echo "WARNING: Could not fetch Component Matrix from Confluence" + return 1 + fi + + # Extract installed operator version to look up in the matrix + local operator_version + operator_version=$(echo "${CSV_NAME:-}" | grep -oP '\d+\.\d+\.\d+' || true) + if [[ -z "$operator_version" ]]; then + echo "WARNING: Could not determine operator version from CSV name '${CSV_NAME:-}'" + return 1 + fi + + echo "Looking up expected versions for operator ${operator_version} in Component Matrix..." + + # Parse the HTML table and extract versions for our operator version + # Columns: 0=Version, 5=Helm, 6=Kustomize, 8=ArgoCD, 15=Dex + local parsed + parsed=$(echo "$body" | python3 -c " +import json, sys, re +from html.parser import HTMLParser + +target = '${operator_version}' + +class P(HTMLParser): + def __init__(self): + super().__init__() + self.in_t = self.in_r = self.in_c = False + self.rows, self.row, self.cell = [], [], '' + def handle_starttag(self, t, a): + if t == 'table': self.in_t = True + elif t == 'tr' and self.in_t: self.in_r = True; self.row = [] + elif t in ('td','th') and self.in_r: self.in_c = True; self.cell = '' + def handle_endtag(self, t): + if t in ('td','th') and self.in_c: self.in_c = False; self.row.append(self.cell.strip()) + elif t == 'tr' and self.in_r: self.in_r = False; self.rows.append(self.row) if self.row else None + elif t == 'table': self.in_t = False + def handle_data(self, d): + if self.in_c: self.cell += d + +p = P() +d = json.load(sys.stdin) +p.feed(d['body']['storage']['value']) +for r in p.rows: + if len(r) >= 16 and r[0] == target: + print(f'HELM={r[5]}') + print(f'KUSTOMIZE={r[6]}') + print(f'ARGOCD={r[8]}') + print(f'DEX={r[15]}') + break +" 2>/dev/null || true) + + if [[ -z "$parsed" ]]; then + echo "WARNING: Operator version ${operator_version} not found in Component Matrix" + return 1 + fi + + EXPECTED_HELM="" EXPECTED_KUSTOMIZE="" EXPECTED_ARGOCD="" EXPECTED_DEX="" + while IFS='=' read -r key val; do + # Only accept known keys with safe values + val="${val//[^a-zA-Z0-9._-]/}" + case "$key" in + HELM) EXPECTED_HELM="$val" ;; + KUSTOMIZE) EXPECTED_KUSTOMIZE="$val" ;; + ARGOCD) EXPECTED_ARGOCD="$val" ;; + DEX) EXPECTED_DEX="$val" ;; + esac + done <<< "$parsed" + echo " Expected Helm: ${EXPECTED_HELM}" + echo " Expected Kustomize: ${EXPECTED_KUSTOMIZE}" + echo " Expected ArgoCD: ${EXPECTED_ARGOCD}" + echo " Expected Dex: ${EXPECTED_DEX}" + return 0 +} + +EXPECTED_HELM="" EXPECTED_KUSTOMIZE="" EXPECTED_ARGOCD="" EXPECTED_DEX="" +HAS_EXPECTED=false + +# ============================================================ +# 1. Bundle / CSV validation +# ============================================================ +echo "" +echo "==========================================" +echo "1. Bundle / CSV Validation" +echo "==========================================" + +CSV_NAME=$(oc get csv -n "${NAMESPACE}" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) +if [[ -z "$CSV_NAME" ]]; then + fail "No ClusterServiceVersion found in ${NAMESPACE}" +else + echo "Installed CSV: ${CSV_NAME}" + + CSV_PHASE=$(oc get csv "${CSV_NAME}" -n "${NAMESPACE}" -o jsonpath='{.status.phase}' 2>/dev/null || true) + if [[ "$CSV_PHASE" == "Succeeded" ]]; then + pass "CSV phase is Succeeded" + else + fail "CSV phase is '${CSV_PHASE}', expected 'Succeeded'" + fi + + RELATED_COUNT=$(oc get csv "${CSV_NAME}" -n "${NAMESPACE}" \ + -o jsonpath='{.spec.relatedImages}' 2>/dev/null | python3 -c "import json,sys; print(len(json.load(sys.stdin)))" 2>/dev/null || echo "0") + if [[ "$RELATED_COUNT" -gt 0 ]]; then + pass "CSV has ${RELATED_COUNT} relatedImages" + echo " Related images:" + oc get csv "${CSV_NAME}" -n "${NAMESPACE}" \ + -o jsonpath='{range .spec.relatedImages[*]} - {.name}: {.image}{"\n"}{end}' 2>/dev/null || true + else + fail "CSV has no relatedImages" + fi +fi + +if fetch_expected_versions; then + HAS_EXPECTED=true +fi + +# ============================================================ +# 2. Operator health check +# ============================================================ +echo "" +echo "==========================================" +echo "2. Operator Health Check" +echo "==========================================" + +for deploy in openshift-gitops-server openshift-gitops-repo-server \ + openshift-gitops-applicationset-controller openshift-gitops-redis; do + if oc get deployment "${deploy}" -n "${GITOPS_NS}" &>/dev/null; then + if wait_for_deployment "${deploy}" "${GITOPS_NS}" 60s; then + pass "Deployment ${deploy} is Available" + else + fail "Deployment ${deploy} is NOT Available" + fi + else + fail "Deployment ${deploy} not found in ${GITOPS_NS}" + fi +done + +CONTROLLER="openshift-gitops-application-controller" +if oc get statefulset "${CONTROLLER}" -n "${GITOPS_NS}" &>/dev/null; then + if wait_for_statefulset "${CONTROLLER}" "${GITOPS_NS}" 60s; then + pass "StatefulSet ${CONTROLLER} is ready" + else + fail "StatefulSet ${CONTROLLER} is NOT ready" + fi +else + fail "StatefulSet ${CONTROLLER} not found in ${GITOPS_NS}" +fi + +BAD_PODS=$(oc get pods -n "${GITOPS_NS}" --no-headers 2>/dev/null \ + | grep -E 'CrashLoopBackOff|ImagePullBackOff|ErrImagePull|Error' || true) +if [[ -z "$BAD_PODS" ]]; then + pass "No pods in error state" +else + fail "Pods in error state:" + echo "$BAD_PODS" +fi + +# ============================================================ +# 3. Toolchain version report +# ============================================================ +echo "" +echo "==========================================" +echo "3. Toolchain Version Report" +echo "==========================================" + +SERVER_POD=$(oc get pods -n "${GITOPS_NS}" --no-headers \ + -l app.kubernetes.io/name=openshift-gitops-server \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) +DEX_POD=$(oc get pods -n "${GITOPS_NS}" --no-headers \ + -l app.kubernetes.io/name=openshift-gitops-dex-server \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) +REDIS_POD=$(oc get pods -n "${GITOPS_NS}" --no-headers \ + -l app.kubernetes.io/name=openshift-gitops-redis \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + +declare -A versions + +if [[ -n "$SERVER_POD" ]]; then + versions[kustomize]=$(oc exec -n "${GITOPS_NS}" "${SERVER_POD}" -- kustomize version 2>/dev/null || echo "N/A") + versions[helm]=$(oc exec -n "${GITOPS_NS}" "${SERVER_POD}" -- helm version --short 2>/dev/null \ + | sed 's/+.*//' || echo "N/A") + versions[argocd]=$(oc exec -n "${GITOPS_NS}" "${SERVER_POD}" -- argocd version --client --short 2>/dev/null \ + | grep -oP 'v[\d.]+' | head -1 || echo "N/A") +else + echo "WARNING: argocd-server pod not found, skipping kustomize/helm/argocd version checks" + versions[kustomize]="N/A"; versions[helm]="N/A"; versions[argocd]="N/A" +fi + +if [[ -n "$DEX_POD" ]]; then + versions[dex]=$(oc exec -n "${GITOPS_NS}" "${DEX_POD}" -- dex version 2>&1 \ + | grep -i 'version' | head -1 | awk -F': ' '{print $2}' || echo "N/A") +else + echo "WARNING: dex pod not found" + versions[dex]="N/A" +fi + +if [[ -n "$REDIS_POD" ]]; then + versions[redis]=$(oc exec -n "${GITOPS_NS}" "${REDIS_POD}" -- redis-server -v 2>/dev/null \ + | awk -F'=' '{print $2}' | cut -d' ' -f1 || echo "N/A") +else + echo "WARNING: redis pod not found" + versions[redis]="N/A" +fi + +echo "" +echo " Component versions:" +for component in kustomize helm argocd dex redis; do + printf " %-12s %s\n" "${component}:" "${versions[$component]}" +done + +# Assert versions against Component Matrix if available +if [[ "$HAS_EXPECTED" == "true" ]]; then + echo "" + echo " Checking against Component Matrix:" + check_version() { + local name=$1 actual=$2 expected=$3 + if [[ -z "$expected" ]]; then + return + fi + # Strip leading 'v' for comparison + local actual_clean="${actual#v}" + local expected_clean="${expected#v}" + if [[ "$actual_clean" == "$expected_clean" ]]; then + pass "${name} version matches (${actual})" + else + fail "${name} version mismatch: deployed=${actual}, expected=${expected}" + fi + } + check_version "Helm" "${versions[helm]}" "${EXPECTED_HELM}" + check_version "Kustomize" "${versions[kustomize]}" "${EXPECTED_KUSTOMIZE}" + check_version "ArgoCD" "${versions[argocd]}" "${EXPECTED_ARGOCD}" + check_version "Dex" "${versions[dex]}" "${EXPECTED_DEX}" +else + echo " (No expected versions available — report only, no assertions)" +fi + +# ============================================================ +# 4. ArgoCD login test +# ============================================================ +echo "" +echo "==========================================" +echo "4. ArgoCD Login Test" +echo "==========================================" + +ARGOCD_ROUTE=$(oc get route openshift-gitops-server -n "${GITOPS_NS}" \ + -o jsonpath='{.spec.host}' 2>/dev/null || true) +ARGOCD_PASSWORD=$(oc get secret openshift-gitops-cluster -n "${GITOPS_NS}" \ + -o jsonpath='{.data.admin\.password}' 2>/dev/null | base64 -d 2>/dev/null || true) + +if [[ -z "$ARGOCD_ROUTE" ]]; then + fail "ArgoCD route not found in ${GITOPS_NS}" +elif [[ -z "$ARGOCD_PASSWORD" ]]; then + fail "ArgoCD admin password not found" +else + echo "ArgoCD URL: https://${ARGOCD_ROUTE}" + + LOGIN_RESPONSE=$(curl -sk -o /dev/null -w "%{http_code}" \ + -X POST "https://${ARGOCD_ROUTE}/api/v1/session" \ + -H "Content-Type: application/json" \ + -d "{\"username\":\"admin\",\"password\":\"${ARGOCD_PASSWORD}\"}" 2>/dev/null || echo "000") + + if [[ "$LOGIN_RESPONSE" == "200" ]]; then + pass "ArgoCD admin login via API succeeded (HTTP 200)" + else + fail "ArgoCD admin login failed (HTTP ${LOGIN_RESPONSE})" + fi + + # Test OpenShift SSO endpoint is reachable + DEX_RESPONSE=$(curl -sk -o /dev/null -w "%{http_code}" \ + "https://${ARGOCD_ROUTE}/api/dex/.well-known/openid-configuration" 2>/dev/null || echo "000") + + if [[ "$DEX_RESPONSE" == "200" ]]; then + pass "Dex/SSO OpenID configuration endpoint reachable (HTTP 200)" + else + fail "Dex/SSO OpenID configuration endpoint not reachable (HTTP ${DEX_RESPONSE})" + fi +fi + +# ============================================================ +# 5. App sync smoke test +# ============================================================ +echo "" +echo "==========================================" +echo "5. App Sync Smoke Test" +echo "==========================================" + +TEST_APP_NS="sanity-test-smoke" +TEST_APP_NAME="sanity-smoke" +TEST_APP_REPO="${CATALOG_URL:-https://github.com/rh-gitops-midstream/catalog.git}" +TEST_APP_REVISION="${CATALOG_REVISION:-HEAD}" +TEST_APP_PATH=".tekton/test-image/config/smoke-app" + +cleanup_smoke_test() { + oc delete application "${TEST_APP_NAME}" -n "${GITOPS_NS}" --ignore-not-found 2>/dev/null || true + oc delete namespace "${TEST_APP_NS}" --ignore-not-found 2>/dev/null || true +} +trap cleanup_smoke_test EXIT + +oc create namespace "${TEST_APP_NS}" --dry-run=client -o yaml | oc apply -f - 2>/dev/null +oc label namespace "${TEST_APP_NS}" "argocd.argoproj.io/managed-by=${GITOPS_NS}" --overwrite + +echo "Waiting for operator to create RBAC in ${TEST_APP_NS}..." +for _rbac_wait in $(seq 1 30); do + if oc get rolebinding -n "${TEST_APP_NS}" 2>/dev/null | grep -q "${GITOPS_NS}"; then + echo "RBAC ready in ${TEST_APP_NS}" + break + fi + sleep 2 +done + +cat </dev/null || true) + HEALTH_STATUS=$(oc get application "${TEST_APP_NAME}" -n "${GITOPS_NS}" \ + -o jsonpath='{.status.health.status}' 2>/dev/null || true) + + if [[ "$SYNC_STATUS" == "Synced" && "$HEALTH_STATUS" == "Healthy" ]]; then + SYNC_OK=true + break + fi + sleep 5 +done + +if [[ "$SYNC_OK" == "true" ]]; then + pass "Smoke app synced and healthy" +else + fail "Smoke app did not reach Synced/Healthy (sync=${SYNC_STATUS:-unknown}, health=${HEALTH_STATUS:-unknown})" + oc get application "${TEST_APP_NAME}" -n "${GITOPS_NS}" -o yaml 2>/dev/null || true +fi + +cleanup_smoke_test +trap - EXIT + +# ============================================================ +# Summary +# ============================================================ +echo "" +echo "==========================================" +echo "Sanity Test Summary" +echo "==========================================" + +cat > "${RESULTS_DIR}/sanity-results.json" <&1 + +UI_TEST_DIR="${ROOT_DIR}/gitops-operator/test/ui-e2e" +if [[ ! -d "$UI_TEST_DIR" ]]; then + echo "ERROR: test/ui-e2e directory not found in ${TEST_REPO_URL} @ ${BRANCH}" + exit 1 +fi +cd "${UI_TEST_DIR}" || exit 1 + +# --- Install dependencies --- + +echo "Installing npm dependencies..." +npm ci 2>&1 + +echo "Installing Playwright browser dependencies..." +if command -v dnf &>/dev/null; then + dnf -y install \ + alsa-lib atk at-spi2-atk cups-libs libdrm mesa-libgbm \ + gtk3 nss libXcomposite libXdamage libXrandr pango \ + libxkbcommon libXScrnSaver 2>&1 || true +elif command -v apt-get &>/dev/null; then + npx playwright install-deps chromium 2>&1 || true +fi + +echo "Installing Playwright Chromium..." +npx playwright install chromium 2>&1 + +# --- Discover cluster URLs --- + +if [[ -z "${CONSOLE_URL:-}" ]]; then + CONSOLE_HOST=$(oc get route console -n openshift-console \ + -o jsonpath='{.spec.host}' 2>/dev/null || true) + if [[ -n "$CONSOLE_HOST" ]]; then + CONSOLE_URL="https://${CONSOLE_HOST}" + fi +fi + +if [[ -z "${ARGOCD_URL:-}" ]]; then + ARGOCD_HOST=$(oc get route -n openshift-gitops openshift-gitops-server \ + -o jsonpath='{.spec.host}' 2>/dev/null || true) + if [[ -n "$ARGOCD_HOST" ]]; then + ARGOCD_URL="https://${ARGOCD_HOST}" + fi +fi + +if [[ -z "${ARGOCD_URL:-}" ]]; then + echo "ERROR: Could not discover ArgoCD URL. Set ARGOCD_URL or check the route." + oc get routes -n openshift-gitops 2>/dev/null || true + exit 1 +fi + +if [[ -z "${CONSOLE_URL:-}" ]]; then + echo "WARNING: Could not discover OpenShift Console URL. SSO login tests may fail." +fi + +# --- Get cluster credentials --- + +CLUSTER_USER="${CLUSTER_USER:-kubeadmin}" +if [[ -z "${CLUSTER_PASSWORD:-}" ]]; then + PASS_FILE=$(find /credentials -name "*password" -type f 2>/dev/null | head -1) + if [[ -n "$PASS_FILE" ]]; then + CLUSTER_PASSWORD=$(cat "$PASS_FILE") + echo "Discovered cluster password from ${PASS_FILE}" + fi +fi + +if [[ -z "${CLUSTER_PASSWORD:-}" ]]; then + CLUSTER_PASSWORD=$(oc get secret kubeadmin -n kube-system \ + -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null || true) +fi + +if [[ -z "${CLUSTER_PASSWORD:-}" ]]; then + echo "ERROR: CLUSTER_PASSWORD not set and could not be auto-discovered." + echo "Set CLUSTER_PASSWORD env var, ensure /credentials contains a password file," + echo "or ensure kubeadmin secret exists in kube-system." + exit 1 +fi + +echo "Console URL: ${CONSOLE_URL:-unset}" +echo "ArgoCD URL: ${ARGOCD_URL}" +echo "Cluster user: ${CLUSTER_USER}" + +# --- Handle skip patterns --- + +readarray -t PLAYWRIGHT_EXTRA_ARGS < <(load_playwright_skip_patterns /usr/local/config/skip-ui-e2e.txt) +if [ ${#PLAYWRIGHT_EXTRA_ARGS[@]} -gt 0 ]; then + echo "Skipping tests matching: ${PLAYWRIGHT_EXTRA_ARGS[1]}" +fi + +# --- Run Playwright tests --- + +export CONSOLE_URL="${CONSOLE_URL:-}" +export ARGOCD_URL +export CLUSTER_USER +export CLUSTER_PASSWORD +if [[ -z "${IDP:-}" ]]; then + IDP_NAME=$(oc get oauth cluster -o jsonpath='{.spec.identityProviders[0].name}' 2>/dev/null || true) + IDP="${IDP_NAME:-kube:admin}" +fi +export IDP +export CI="konflux" +export PLAYWRIGHT_JUNIT_OUTPUT_NAME="${RESULTS_DIR}/junit-results.xml" + +# Clean stale browser state +rm -f .auth/storageState.json + +echo "Running UI E2E tests..." +npx playwright test \ + --project=chromium \ + --reporter=list,junit \ + "${PLAYWRIGHT_EXTRA_ARGS[@]}" \ + 2>&1 | tee "${RESULTS_DIR}/ui-e2e.log" +TEST_EXIT_CODE=${PIPESTATUS[0]} + +# --- Collect artifacts --- + +for dir in playwright-report test-results; do + if [[ -d "$dir" ]]; then + cp -r "$dir" "${RESULTS_DIR}/" 2>/dev/null || true + fi +done + +if [[ "$TEST_EXIT_CODE" -ne 0 ]]; then + echo "UI E2E tests failed (exit code ${TEST_EXIT_CODE})" + exit 1 +fi + +echo "All UI E2E tests passed." diff --git a/.tekton/test-image/scripts/send-slack-message.py b/.tekton/test-image/scripts/send-slack-message.py new file mode 100755 index 00000000..52d2227b --- /dev/null +++ b/.tekton/test-image/scripts/send-slack-message.py @@ -0,0 +1,444 @@ +#!/usr/bin/env python3 +"""Send Slack notification with task details for GitOps Catalog E2E tests. + +Environment variables: + SLACK_WEBHOOK_URL - Slack incoming webhook URL + PIPELINE_RUN_NAME - Tekton PipelineRun name + AGGREGATE_STATUS - Overall pipeline status (Succeeded/Failed/etc.) + LOG_URL - Konflux UI link for this pipeline run + QUAY_REPO - OCI repo for log artifacts + TASK_NAMES - Space-separated task names that have log artifacts + SHARED_DIR - Path to shared volume with test-results.json (default: /shared) +""" +import json +import logging +import os +import subprocess +import sys +import urllib.request +from datetime import datetime + + +def run_cmd(cmd, timeout=30, verbose=False): + """Run a shell command and return stdout, or None on failure.""" + try: + result = subprocess.run( + cmd, shell=True, capture_output=True, text=True, timeout=timeout + ) + if verbose or result.returncode != 0: + logging.info(f"CMD: {cmd}") + logging.info(f" RC: {result.returncode}") + if result.stdout: + logging.info(f" STDOUT: {result.stdout[:500]}") + if result.stderr: + logging.info(f" STDERR: {result.stderr[:500]}") + return result.stdout.strip() if result.returncode == 0 else None + except Exception as e: + logging.error(f"CMD exception: {cmd}: {e}") + return None + + +def get_test_results(): + """Read test results from the shared volume (written by collect-and-upload-logs).""" + shared_dir = os.environ.get("SHARED_DIR", "/shared") + path = os.path.join(shared_dir, "test-results.json") + if not os.path.isfile(path): + return None + try: + with open(path) as f: + return json.load(f) + except (json.JSONDecodeError, OSError) as e: + logging.warning(f"Failed to read test results from {path}: {e}") + return None + + +def get_build_metadata(): + """Read build metadata from the shared volume (written by collect-build-metadata.sh).""" + shared_dir = os.environ.get("SHARED_DIR", "/shared") + path = os.path.join(shared_dir, "build-metadata.json") + if not os.path.isfile(path): + return None + try: + with open(path) as f: + return json.load(f) + except (json.JSONDecodeError, OSError) as e: + logging.warning(f"Failed to read build metadata from {path}: {e}") + return None + + +def get_task_runs(pipeline_run_name): + """Query TaskRuns for timing, status, and result information.""" + raw = run_cmd( + f"oc get taskruns -l tekton.dev/pipelineRun={pipeline_run_name} -o json" + ) + if not raw: + return {} + + try: + data = json.loads(raw) + except json.JSONDecodeError: + return {} + + tasks = {} + for item in data.get("items", []): + labels = item.get("metadata", {}).get("labels", {}) + task_name = labels.get("tekton.dev/pipelineTask", "unknown") + + status = item.get("status", {}) + start_time = status.get("startTime") + completion_time = status.get("completionTime") + + conditions = status.get("conditions", []) + task_status = "Unknown" + if conditions: + reason = conditions[0].get("reason", "") + cond_status = conditions[0].get("status", "") + if cond_status == "True": + task_status = "Succeeded" + elif reason in ("Failed", "TaskRunTimeout"): + task_status = "Failed" + else: + task_status = reason or "Unknown" + + duration = "" + if start_time and completion_time: + try: + start = datetime.fromisoformat(start_time.replace("Z", "+00:00")) + end = datetime.fromisoformat(completion_time.replace("Z", "+00:00")) + total_secs = int((end - start).total_seconds()) + if total_secs >= 3600: + duration = f"{total_secs // 3600}h {(total_secs % 3600) // 60}m {total_secs % 60}s" + elif total_secs >= 60: + duration = f"{total_secs // 60}m {total_secs % 60}s" + else: + duration = f"{total_secs}s" + except Exception: + pass + elif start_time: + duration = "running" + + results = {} + for r in status.get("results", []): + results[r.get("name", "")] = r.get("value", "") + + tasks[task_name] = { + "status": task_status, + "duration": duration, + "results": results, + } + + return tasks + + +def get_failed_task_log_tail(quay_repo, pipeline_run_name, task_name, lines=20): + """Pull a failed task's log artifact and return the tail.""" + ref = f"{quay_repo}:{pipeline_run_name}-task-{task_name}" + tmpdir = f"/tmp/slack-logs-{task_name}" + run_cmd(f"mkdir -p {tmpdir}") + + if run_cmd(f"oras pull --no-tty -o {tmpdir} {ref} 2>/dev/null") is None: + run_cmd(f"rm -rf {tmpdir}") + return None + + log_file = run_cmd(f"find {tmpdir} -name '*.log' -type f 2>/dev/null | head -1") + if not log_file: + run_cmd(f"rm -rf {tmpdir}") + return None + + tail = run_cmd(f"tail -n {lines} {log_file}") + run_cmd(f"rm -rf {tmpdir}") + return tail + + +def build_config_block(task_runs): + """Build a block showing pipeline configuration and actual runtime values.""" + test_script = os.environ.get("TEST_SCRIPT", "") + test_repo_url = os.environ.get("TEST_REPO_URL", "") + test_repo_branch = os.environ.get("TEST_REPO_BRANCH", "") + ocp_requested = os.environ.get("OPENSHIFT_VERSION", "") + operator_channel = os.environ.get("OPERATOR_CHANNEL", "") + fips = os.environ.get("FIPS_ENABLED", "") + + ocp_actual = ( + task_runs.get("provision-cluster", {}).get("results", {}).get("resolvedVersion") + ) + installed_csv = ( + task_runs.get("install-operator", {}).get("results", {}).get("installedCSV") + ) + + lines = [] + if test_script: + lines.append(f"*Test suite:* `{test_script}`") + if test_repo_url: + repo_short = test_repo_url.rstrip("/").rsplit("/", 2)[-2:] + repo_label = "/".join(repo_short).replace(".git", "") + branch_part = f" @ `{test_repo_branch}`" if test_repo_branch else "" + lines.append(f"*Test repo:* `{repo_label}`{branch_part}") + if ocp_requested: + if ocp_actual and ocp_actual != ocp_requested: + lines.append(f"*OpenShift:* `{ocp_requested}` -> `{ocp_actual}`") + else: + lines.append(f"*OpenShift:* `{ocp_requested}`") + if installed_csv: + lines.append(f"*Operator:* `{installed_csv}`") + if operator_channel: + lines.append(f"*Channel:* `{operator_channel}`") + if fips == "true": + lines.append("*FIPS:* enabled") + + if not lines: + return None + + return { + "type": "section", + "text": {"type": "mrkdwn", "text": "\n".join(lines)}, + } + + +def build_blocks( + pipeline_run_name, aggregate_status, log_url, quay_repo, task_runs, loggable_tasks +): + """Build Slack Block Kit blocks for the notification.""" + # Derive status from actual test results when available, not pipeline aggregate + test_data = get_test_results() + if test_data is not None: + if test_data.get("total", 0) > 0 and test_data.get("failed", 0) == 0 and test_data.get("errors", 0) == 0: + aggregate_status = "Succeeded" + elif test_data.get("failed", 0) > 0 or test_data.get("errors", 0) > 0: + aggregate_status = "Failed" + + status_emoji = ( + ":white_check_mark:" if aggregate_status == "Succeeded" else ":x:" + ) + + blocks = [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": f"GitOps Catalog E2E: {pipeline_run_name}", + }, + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": f"{status_emoji} Pipeline finished: *{aggregate_status}*", + }, + }, + ] + + # Add test summary from shared volume + if test_data: + summary = test_data.get("summary", "") + if summary: + blocks.append( + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": f":test_tube: *Test Results:* {summary}", + }, + } + ) + failed_tests = test_data.get("failedTests", []) + if failed_tests: + names = failed_tests[:15] + text = ":x: *Failed tests:*\n" + "\n".join(f"• `{t}`" for t in names) + if len(failed_tests) > 15: + text += f"\n_...and {len(failed_tests) - 15} more_" + blocks.append( + { + "type": "section", + "text": {"type": "mrkdwn", "text": text}, + } + ) + + config_block = build_config_block(task_runs) + if config_block: + blocks.append(config_block) + + build_meta = get_build_metadata() + if build_meta: + labels = { + "build": "Build", "argocd": "Argo CD", "dex": "Dex", + "redis": "Redis", "kustomize": "Kustomize", "helm": "Helm", + "gitLfs": "git-lfs", "agent": "Agent", + } + parts = [f"*{labels.get(k, k)}:* `{v}`" for k, v in build_meta.items() if v] + if parts: + blocks.append({ + "type": "section", + "text": {"type": "mrkdwn", "text": ":package: " + " | ".join(parts)}, + }) + + # Task list with status and timing + if task_runs: + task_order = [ + "parse-metadata", + "build-ginkgo-test-image", + "provision-eaas-space", + "provision-cluster", + "install-operator", + "test-operator", + ] + + lines = [] + failed_tasks = [] + seen = set() + + for name in task_order: + info = task_runs.get(name) + if not info: + continue + seen.add(name) + icon = { + "Succeeded": ":white_check_mark:", + "Failed": ":x:", + }.get(info["status"], ":hourglass_flowing_sand:") + if info["status"] == "Failed": + failed_tasks.append(name) + dur = f" ({info['duration']})" if info["duration"] else "" + lines.append(f"{icon} `{name}`{dur}") + + # Any tasks not in our predefined order + for name, info in sorted(task_runs.items()): + if name in seen: + continue + icon = { + "Succeeded": ":white_check_mark:", + "Failed": ":x:", + }.get(info["status"], ":hourglass_flowing_sand:") + if info["status"] == "Failed": + failed_tasks.append(name) + dur = f" ({info['duration']})" if info["duration"] else "" + lines.append(f"{icon} `{name}`{dur}") + + blocks.append( + { + "type": "section", + "text": {"type": "mrkdwn", "text": "*Tasks:*\n" + "\n".join(lines)}, + } + ) + + # Log tails for failed tasks that have artifacts + for task_name in failed_tasks: + if task_name not in loggable_tasks or not quay_repo: + continue + tail = get_failed_task_log_tail(quay_repo, pipeline_run_name, task_name) + if not tail: + continue + # Slack block text limit is ~3000 chars + if len(tail) > 2800: + tail = tail[-2800:] + blocks.append({"type": "divider"}) + blocks.append( + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": f":page_facing_up: *`{task_name}` (last 20 lines):*\n```\n{tail}\n```", + }, + } + ) + + # Links + blocks.append({"type": "divider"}) + links_parts = [] + if log_url: + links_parts.append(f":technologist: <{log_url}|View in Konflux UI>") + if quay_repo and pipeline_run_name: + links_parts.append( + f":open_file_folder: `oras pull {quay_repo}:{pipeline_run_name}-logs`" + ) + if links_parts: + blocks.append( + { + "type": "section", + "text": {"type": "mrkdwn", "text": "\n".join(links_parts)}, + } + ) + + return blocks + + +def send_slack_message(webhook_url, blocks, fallback_text): + """Post a Slack message via incoming webhook.""" + msg = {"text": fallback_text, "blocks": blocks} + req = urllib.request.Request( + webhook_url, + data=json.dumps(msg).encode(), + headers={"Content-type": "application/json"}, + method="POST", + ) + try: + resp = urllib.request.urlopen(req, timeout=10) + return resp.read().decode() + except Exception as e: + logging.error(f"Failed to send Slack message: {e}") + return "" + + +def main(): + # Configure logging to stderr (will appear in pod logs) + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s [%(levelname)s] %(message)s', + stream=sys.stderr + ) + + logging.info("=== Starting send-slack-message.py ===") + + webhook_url = os.environ.get("SLACK_WEBHOOK_URL", "") + pipeline_run_name = os.environ.get("PIPELINE_RUN_NAME", "") + aggregate_status = os.environ.get("AGGREGATE_STATUS", "Unknown") + log_url = os.environ.get("LOG_URL", "") + quay_repo = os.environ.get("QUAY_REPO", "") + task_names_str = os.environ.get("TASK_NAMES", "") + + logging.info(f"Environment:") + logging.info(f" PIPELINE_RUN_NAME: {pipeline_run_name}") + logging.info(f" AGGREGATE_STATUS: {aggregate_status}") + logging.info(f" QUAY_REPO: {quay_repo}") + logging.info(f" TASK_NAMES: {task_names_str}") + logging.info(f" LOG_URL: {log_url}") + + if not webhook_url: + logging.error("SLACK_WEBHOOK_URL is not set") + return 1 + + loggable_tasks = set(task_names_str.split()) if task_names_str else set() + logging.info(f"Loggable tasks: {loggable_tasks}") + + # Setup oras credentials for pulling per-task log artifacts + quay_creds = os.environ.get("QUAY_CREDENTIALS_PATH", "/quay-credentials/.dockerconfigjson") + if quay_repo and os.path.isfile(quay_creds): + import shutil + import tempfile + tmpdir = tempfile.mkdtemp() + shutil.copy2(quay_creds, os.path.join(tmpdir, "config.json")) + os.environ["DOCKER_CONFIG"] = tmpdir + + logging.info("Fetching task runs from cluster...") + task_runs = get_task_runs(pipeline_run_name) + logging.info(f"Found {len(task_runs)} task runs") + + logging.info("Building Slack blocks...") + blocks = build_blocks( + pipeline_run_name, aggregate_status, log_url, quay_repo, task_runs, loggable_tasks + ) + logging.info(f"Built {len(blocks)} blocks") + + fallback = f"GitOps Catalog E2E {pipeline_run_name}: {aggregate_status}" + logging.info(f"Sending Slack message...") + ret = send_slack_message(webhook_url, blocks, fallback) + logging.info(f"Slack API response: {ret}") + if ret: + print(ret) + + logging.info("=== send-slack-message.py completed ===") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.tekton/test-image/scripts/upgrade-operator.sh b/.tekton/test-image/scripts/upgrade-operator.sh new file mode 100755 index 00000000..c3636da3 --- /dev/null +++ b/.tekton/test-image/scripts/upgrade-operator.sh @@ -0,0 +1,73 @@ +#!/bin/bash +set -e + +# Environment variables expected: +# - UPGRADE (true/false) +# - UPGRADE_TO_CHANNEL (target channel) +# - NAMESPACE (default: openshift-gitops-operator) +# - INSTALL_TIMEOUT (e.g., "25m") +# - KUBECONFIG + +# shellcheck source=./lib/wait-for-resources.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib/wait-for-resources.sh" + +if [[ "$UPGRADE" != "true" ]]; then + echo "UPGRADE is not enabled, skipping upgrade step" + exit 0 +fi + +if [[ -z "$UPGRADE_TO_CHANNEL" ]]; then + echo "ERROR: UPGRADE is enabled but UPGRADE_TO_CHANNEL is not set" + exit 1 +fi + +SUBSCRIPTION_NAME=$(oc get subscription -n "$NAMESPACE" -o jsonpath='{.items[0].metadata.name}') +if [[ -z "$SUBSCRIPTION_NAME" ]]; then + echo "ERROR: No subscription found in namespace ${NAMESPACE}" + exit 1 +fi + +CURRENT_CHANNEL=$(oc get subscription "$SUBSCRIPTION_NAME" -n "$NAMESPACE" -o jsonpath='{.spec.channel}') +PRE_UPGRADE_CSV=$(oc get subscription "$SUBSCRIPTION_NAME" -n "$NAMESPACE" -o jsonpath='{.status.installedCSV}') +echo "Current channel: ${CURRENT_CHANNEL}, installed CSV: ${PRE_UPGRADE_CSV}" +echo "Upgrading to channel: ${UPGRADE_TO_CHANNEL}" + +oc patch subscription "$SUBSCRIPTION_NAME" -n "$NAMESPACE" --type merge \ + -p "{\"spec\":{\"channel\":\"${UPGRADE_TO_CHANNEL}\",\"installPlanApproval\":\"Automatic\"}}" + +echo "Waiting for upgrade to complete..." +if [[ "$INSTALL_TIMEOUT" =~ ^([0-9]+)m$ ]]; then + TIMEOUT_SECONDS=$(( BASH_REMATCH[1] * 60 )) +elif [[ "$INSTALL_TIMEOUT" =~ ^([0-9]+)s$ ]]; then + TIMEOUT_SECONDS=${BASH_REMATCH[1]} +else + TIMEOUT_SECONDS=1500 +fi +DEADLINE=$(($(date +%s) + TIMEOUT_SECONDS)) + +while true; do + CSV=$(oc get subscription "$SUBSCRIPTION_NAME" -n "$NAMESPACE" -o jsonpath='{.status.installedCSV}' 2>/dev/null || true) + if [[ -n "$CSV" && "$CSV" != "$PRE_UPGRADE_CSV" ]]; then + PHASE=$(oc get csv "$CSV" -n "$NAMESPACE" -o jsonpath='{.status.phase}' 2>/dev/null || true) + echo "CSV: $CSV, Phase: $PHASE" + if [[ "$PHASE" == "Succeeded" ]]; then + echo "Upgrade completed successfully: ${PRE_UPGRADE_CSV} -> ${CSV}" + + GITOPS_NS="${GITOPS_NS:-openshift-gitops}" + echo "Waiting for ArgoCD workloads to reconcile after upgrade..." + wait_for_argocd_reconciliation "$NAMESPACE" "$GITOPS_NS" 300 + break + fi + else + echo "Waiting for new CSV (current: ${CSV:-none}, pre-upgrade: ${PRE_UPGRADE_CSV})" + fi + + if [[ $(date +%s) -ge $DEADLINE ]]; then + echo "ERROR: Upgrade timed out after ${INSTALL_TIMEOUT}" + oc get subscription "$SUBSCRIPTION_NAME" -n "$NAMESPACE" -o yaml + oc get csv -n "$NAMESPACE" + exit 1 + fi + + sleep 30 +done diff --git a/.tekton/test-image/scripts/verify-images.py b/.tekton/test-image/scripts/verify-images.py new file mode 100644 index 00000000..280948b0 --- /dev/null +++ b/.tekton/test-image/scripts/verify-images.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""Verify that all images referenced by the installed CSV are available +at their mirror locations (from IDMS on the cluster). + +Environment variables: + KUBECONFIG (required) + NAMESPACE Operator namespace (default: openshift-gitops-operator) + TARGET_ARCH e.g. arm64, amd64 (auto-detected from cluster if unset) + IDMS_FILE Path to images-mirror-set.yaml (falls back to cluster IDMS) + +Exit codes: + 0 All images verified (or only skips) + 1 One or more images failed verification +""" + +import json +import os +import subprocess +import sys +from pathlib import Path + +try: + import yaml +except ImportError: + yaml = None + + +def run_cmd(cmd): + return subprocess.run( + cmd, shell=True, capture_output=True, text=True, timeout=120, + ) + + +def detect_target_arch(): + arch = os.environ.get("TARGET_ARCH") + if arch: + return arch + result = run_cmd( + "oc get nodes -o jsonpath='{.items[0].status.nodeInfo.architecture}'", + ) + return result.stdout.strip("'\" \n") if result.returncode == 0 else "amd64" + + +def get_installed_csv(namespace): + result = run_cmd( + f"oc get subscription -n {namespace}" + f" -o jsonpath='{{.items[0].status.installedCSV}}'", + ) + csv_name = result.stdout.strip("'\" \n") + if result.returncode != 0 or not csv_name: + print(f"ERROR: No installed CSV found in namespace {namespace}") + sys.exit(1) + return csv_name + + +def get_related_images(namespace, csv_name): + result = run_cmd( + f"oc get csv {csv_name} -n {namespace}" + f" -o jsonpath='{{range .spec.relatedImages[*]}}{{.image}}{{\"\\n\"}}{{end}}'", + ) + images = [line.strip("'\" ") for line in result.stdout.strip().splitlines() if line.strip()] + return images + + +def build_mirror_map(namespace, idms_file): + mirror_map = {} + + if idms_file and Path(idms_file).is_file(): + if yaml is None: + print("ERROR: PyYAML required for IDMS_FILE parsing but not installed") + sys.exit(1) + print(f"Loading mirrors from file: {idms_file}") + with open(idms_file) as f: + data = yaml.safe_load(f) + for entry in data.get("spec", {}).get("imageDigestMirrors", []): + mirrors = entry.get("mirrors", []) + if mirrors: + mirror_map[entry["source"]] = mirrors[0] + else: + print("Loading mirrors from cluster IDMS...") + result = run_cmd("oc get imagedigestmirrorset -o json") + if result.returncode == 0 and result.stdout.strip(): + try: + data = json.loads(result.stdout) + for item in data.get("items", []): + for entry in item.get("spec", {}).get("imageDigestMirrors", []): + mirrors = entry.get("mirrors", []) + if mirrors: + mirror_map[entry["source"]] = mirrors[0] + except json.JSONDecodeError: + print("WARNING: Could not parse IDMS JSON from cluster") + + return mirror_map + + +def find_auth_file(): + for path in [ + "/quay-pull-credentials/.dockerconfigjson", + "/quay-credentials/.dockerconfigjson", + ]: + if Path(path).is_file(): + return path + return None + + +def check_manifest_arch(image_ref, auth_file, target_arch): + """Inspect a manifest and return (is_ok, not_found, detail_string).""" + auth_args = f"--authfile={auth_file}" if auth_file else "" + result = run_cmd(f"skopeo inspect --raw {auth_args} docker://{image_ref}") + if result.returncode != 0: + return False, True, "image not found at mirror" + + try: + manifest = json.loads(result.stdout) + except json.JSONDecodeError: + return True, False, "single-arch manifest" + + media_type = str(manifest.get("mediaType", manifest.get("schemaVersion", ""))) + if "manifest.list" in media_type or "image.index" in media_type: + archs = [ + p.get("platform", {}).get("architecture", "?") + for p in manifest.get("manifests", []) + ] + if target_arch in archs: + return True, False, f"{target_arch} in: {','.join(archs)}" + return False, False, f"missing {target_arch} (available: {','.join(archs)})" + + return True, False, "single-arch manifest" + + +def main(): + namespace = os.environ.get("NAMESPACE", "openshift-gitops-operator") + idms_file = os.environ.get("IDMS_FILE") + + target_arch = detect_target_arch() + print(f"Target architecture: {target_arch}") + + csv_name = get_installed_csv(namespace) + print(f"Installed CSV: {csv_name}") + + images = get_related_images(namespace, csv_name) + if not images: + print(f"WARNING: No relatedImages found in CSV {csv_name}") + sys.exit(0) + print(f"Found {len(images)} related images in CSV") + + mirror_map = build_mirror_map(namespace, idms_file) + if not mirror_map: + print("WARNING: No mirror mappings found") + print("Images will be pulled directly from source registries") + print(f"\nMirror mappings loaded: {len(mirror_map)} entries") + + auth_file = find_auth_file() + + passed = 0 + failed = 0 + skipped = 0 + + for image in images: + repo, _, digest = image.partition("@") + + mirror_repo = mirror_map.get(repo) + if not mirror_repo: + print(f" SKIP [no mirror] {repo}") + skipped += 1 + continue + + check_ref = f"{mirror_repo}@{digest}" + ok, not_found, detail = check_manifest_arch(check_ref, auth_file, target_arch) + + if ok: + print(f" OK [mirror] {repo} ({detail})") + passed += 1 + else: + print(f" FAIL [mirror] {repo} — {detail}") + print(f" ref: {check_ref}") + if not_found: + print(f" source: {image}") + failed += 1 + + print() + print("=" * 42) + print(f"Image verification: {passed} OK, {failed} FAILED, {skipped} SKIPPED (no mirror)") + print("=" * 42) + + if failed > 0: + print(f"ERROR: {failed} image(s) are not available at their expected locations.") + print("The operator will fail to create workload pods for these images.") + sys.exit(1) + + +if __name__ == "__main__": + main()