diff --git a/.github/workflows/ostests-e2e.yaml b/.github/workflows/ostests-e2e.yaml index 9d7882551c93..2dc982bdd622 100644 --- a/.github/workflows/ostests-e2e.yaml +++ b/.github/workflows/ostests-e2e.yaml @@ -16,6 +16,10 @@ on: type: string description: The selector for the e2e tests to be run. default: \[Conformance\] + fail-fast: + type: boolean + description: Stop the e2e test suite after the first test failure. + default: false os: type: string description: The operating system to test. @@ -64,6 +68,7 @@ jobs: env: E2E_FOCUS: ${{ inputs.e2e-focus }} + E2E_FAIL_FAST: ${{ inputs.fail-fast }} TF_VAR_os: ${{ inputs.os }} TF_VAR_additional_tags: '{ "ostests.k0sproject.io/github-run-id"="${{ github.run_id }}", @@ -168,12 +173,16 @@ jobs: E2E_CONCURRENCY_LEVEL: ${{ inputs.e2e-concurrency-level }} run: | make bin/sonobuoy + ginkgoArgs="-v --timeout=120m --procs=$E2E_CONCURRENCY_LEVEL" + if [ "$E2E_FAIL_FAST" = true ]; then + ginkgoArgs="$ginkgoArgs --fail-fast" + fi bin/sonobuoy run -p e2e --wait=150 \ --kubernetes-version=v"$KUBERNETES_VERSION" \ --plugin-env=e2e.E2E_PARALLEL=true \ --plugin-env=e2e.E2E_FOCUS="$E2E_FOCUS" \ --plugin-env=e2e.E2E_SKIP='\[Serial\]' \ - --plugin-env=e2e.E2E_EXTRA_GINKGO_ARGS="-v --timeout=120m --procs=$E2E_CONCURRENCY_LEVEL" + --plugin-env=e2e.E2E_EXTRA_GINKGO_ARGS="$ginkgoArgs" - name: "e2e tests :: Retrieve parallel results" id: e2e-retrieve-parallel @@ -193,14 +202,21 @@ jobs: - name: "e2e tests :: Run serial tests" working-directory: inttest - timeout-minutes: 180 # three hours + timeout-minutes: ${{ inputs.e2e-concurrency-level > 1 && 60 || 240 }} # 1 hour in parallel mode, 4 hours in serial mode + env: + K0S_SONOBUOY_WAIT: "${{ inputs.e2e-concurrency-level > 1 && 45 || 210 }}" # 45 mins in parallel mode, 3.5 hours in serial mode + K0S_GINKGO_TIMEOUT: "${{ inputs.e2e-concurrency-level > 1 && '30m' || '3h' }}" # 30 mins in parallel mode, 3 hours in serial mode run: | make bin/sonobuoy - bin/sonobuoy run -p e2e --wait=150 \ + ginkgoArgs="-v --timeout=$K0S_GINKGO_TIMEOUT" + if [ "$E2E_FAIL_FAST" = true ]; then + ginkgoArgs="$ginkgoArgs --fail-fast" + fi + bin/sonobuoy run -p e2e --wait="$K0S_SONOBUOY_WAIT" \ --kubernetes-version=v"$KUBERNETES_VERSION" \ --plugin-env=e2e.E2E_FOCUS="$E2E_FOCUS" \ --plugin-env=e2e.E2E_SKIP='' \ - --plugin-env=e2e.E2E_EXTRA_GINKGO_ARGS='-v --timeout=120m' + --plugin-env=e2e.E2E_EXTRA_GINKGO_ARGS="$ginkgoArgs" - name: "e2e tests :: Retrieve serial results" id: e2e-retrieve-serial @@ -231,11 +247,18 @@ jobs: run: | fail=0 for f in sonobuoy-e2e-*.tar.gz; do - echo "::group::$f" - bin/sonobuoy results "$f" - numNotPassedOrSkipped=$(bin/sonobuoy results "$f" -p=e2e --mode=detailed | jq --slurp '[.[] | select(.status != "passed" and .status != "skipped")] | length') - echo "Number of tests that didn't pass and weren't skipped: $numNotPassedOrSkipped" - echo ::endgroup:: + { + echo "### $f" + numNotPassedOrSkipped=$(bin/sonobuoy results "$f" -p=e2e --mode=detailed | jq --slurp '[.[] | select(.status != "passed" and .status != "skipped")] | length') + echo '
' + echo '' + echo "Number of tests that didn't pass and weren't skipped: $numNotPassedOrSkipped" + echo '' + echo + echo '```text' + bin/sonobuoy results "$f" + echo '```' + } >>"$GITHUB_STEP_SUMMARY" [ "$numNotPassedOrSkipped" = 0 ] || fail=1 done [ "$fail" = 0 ] || exit 1 diff --git a/.github/workflows/ostests-matrix.yaml b/.github/workflows/ostests-matrix.yaml index f759aba19d25..ce2e3730ae17 100644 --- a/.github/workflows/ostests-matrix.yaml +++ b/.github/workflows/ostests-matrix.yaml @@ -20,6 +20,10 @@ on: type: string description: The selector for the e2e tests to be run. default: \[Conformance\] + fail-fast: + type: boolean + description: Stop each e2e test suite after the first test failure. + default: false oses: type: string description: The operating systems to test. @@ -89,6 +93,7 @@ jobs: k0sctl-version: ${{ inputs.k0sctl-version }} e2e-concurrency-level: ${{ fromJSON(inputs.e2e-concurrency-level) }} # infamous GH workflows bug that looses type information (actions/runner#2206) e2e-focus: ${{ inputs.e2e-focus }} + fail-fast: ${{ inputs.fail-fast }} os: ${{ matrix.os }} arch: ${{ inputs.arch }} network-provider: ${{ matrix.network-provider }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5f3a1407f872..b47189523274 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -656,6 +656,7 @@ jobs: os: debian_12 arch: ${{ matrix.arch }} network-provider: kuberouter + fail-fast: true secrets: aws-access-key-id: ${{ secrets.AWS_TERRAFORM_ID }} aws-secret-access-key: ${{ secrets.AWS_TERRAFORM_KEY }} diff --git a/hack/ostests/README.md b/hack/ostests/README.md index 22d7c1606117..f4db7838ec50 100644 --- a/hack/ostests/README.md +++ b/hack/ostests/README.md @@ -195,6 +195,9 @@ $ gh workflow run ostests-matrix.yaml --ref some/experimental/branch \ To see runs for this workflow, try: gh run list --workflow=ostests-matrix.yaml ``` +Add `-f fail-fast=true` to the above command line to stop the e2e test suite +after the first test failure has been recorded. + [gh]: https://github.com/cli/cli ## TODO