Regression Detection Suite #5
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Regression Detection Suite | |
| # | |
| # This workflow runs under the following conditions: | |
| # - Once per week, based on a schedule | |
| # - On demand, triggered from https://github.com/vectordotdev/vector/actions/workflows/regression.yml | |
| # - On demand, via the `/ci-run-regression` (or `/ci-run-all`) PR review comment, | |
| # which invokes this workflow via `workflow_call` from `ci-review-trigger.yml`. | |
| # | |
| # The workflow accepts two optional inputs: | |
| # - The baseline ref: | |
| # - If not specified, the SHA from 7 days ago on origin/master is used. | |
| # - The comparison ref: | |
| # - If not specified, the current HEAD of origin/master is used. | |
| # | |
| # Both inputs accept any git ref that `git rev-parse` can resolve: a full commit | |
| # SHA, a short SHA, a branch name (e.g. `origin/master`), or a tag. | |
| # | |
| # This workflow runs regression detection experiments, performing relative | |
| # evaluations of the baseline SHA and comparison SHA. The exact SHAs are determined | |
| # by how the workflow is triggered. | |
| # | |
| # The goal is to provide quick feedback on Vector's performance across a variety | |
| # of configurations, checking if throughput performance has degraded or become | |
| # more variable in the comparison SHA relative to the baseline SHA. | |
| # | |
| # Docker image tags are based on the resolved SHAs. | |
| name: Regression Detection Suite | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| baseline-sha: | |
| description: "Baseline git ref (full/short SHA, branch such as `origin/master`, or tag). Defaults to the SHA from 7 days ago." | |
| required: false | |
| comparison-sha: | |
| description: "Comparison git ref (full/short SHA, branch such as `origin/master`, or tag). Defaults to the current HEAD of origin/master." | |
| required: false | |
| workflow_call: | |
| inputs: | |
| baseline-sha: | |
| description: "Baseline git ref (full/short SHA, branch such as `origin/master`, or tag). Defaults to the SHA from 7 days ago." | |
| required: false | |
| type: string | |
| comparison-sha: | |
| description: "Comparison git ref (full/short SHA, branch such as `origin/master`, or tag). Defaults to the current HEAD of origin/master." | |
| required: false | |
| type: string | |
| schedule: | |
| - cron: "0 7 * * 1" # Runs at 7 AM UTC on Mondays | |
| # Workflow-level permissions - read access to repository contents | |
| permissions: | |
| contents: read # Required to checkout code | |
| env: | |
| SINGLE_MACHINE_PERFORMANCE_API: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_API }} | |
| SMP_WARMUP_SECONDS: 70 # default is 45 seconds | |
| SMP_REPLICAS: 100 # default is 10 | |
| jobs: | |
| resolve-inputs: | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| baseline-sha: ${{ steps.set_and_validate_shas.outputs.BASELINE_SHA }} | |
| comparison-sha: ${{ steps.set_and_validate_shas.outputs.COMPARISON_SHA }} | |
| baseline-tag: ${{ steps.set_and_validate_shas.outputs.BASELINE_TAG }} | |
| comparison-tag: ${{ steps.set_and_validate_shas.outputs.COMPARISON_TAG }} | |
| smp-version: ${{ steps.experimental-meta.outputs.SMP_CRATE_VERSION }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| fetch-depth: 0 # need to pull repository history to find merge bases | |
| - name: Set and Validate SHAs | |
| id: set_and_validate_shas | |
| env: | |
| BASELINE_INPUT: ${{ inputs.baseline-sha }} | |
| COMPARISON_INPUT: ${{ inputs.comparison-sha }} | |
| run: | | |
| # Resolve baseline ref to a full commit SHA | |
| if [ -z "${BASELINE_INPUT}" ]; then | |
| BASELINE_SHA=$(git rev-list -n 1 --before="7 days ago" origin/master) | |
| echo "Using baseline SHA from 7 days ago: ${BASELINE_SHA}" | |
| else | |
| if ! BASELINE_SHA=$(git rev-parse --verify "${BASELINE_INPUT}^{commit}" 2>/dev/null); then | |
| echo "Invalid baseline ref: '${BASELINE_INPUT}'. Expected a full/short SHA, branch (e.g. origin/master), or tag." | |
| exit 1 | |
| fi | |
| echo "Resolved baseline input '${BASELINE_INPUT}' to SHA: ${BASELINE_SHA}" | |
| fi | |
| # Resolve comparison ref to a full commit SHA | |
| if [ -z "${COMPARISON_INPUT}" ]; then | |
| COMPARISON_SHA=$(git rev-parse origin/master) | |
| echo "Using current HEAD of origin/master as comparison SHA: ${COMPARISON_SHA}" | |
| else | |
| if ! COMPARISON_SHA=$(git rev-parse --verify "${COMPARISON_INPUT}^{commit}" 2>/dev/null); then | |
| echo "Invalid comparison ref: '${COMPARISON_INPUT}'. Expected a full/short SHA, branch (e.g. origin/master), or tag." | |
| exit 1 | |
| fi | |
| echo "Resolved comparison input '${COMPARISON_INPUT}' to SHA: ${COMPARISON_SHA}" | |
| fi | |
| # Set tags and export them | |
| BASELINE_TAG="workflow_dispatch-${COMPARISON_SHA}-${BASELINE_SHA}" | |
| COMPARISON_TAG="workflow_dispatch-${COMPARISON_SHA}-${COMPARISON_SHA}" | |
| echo "BASELINE_SHA=${BASELINE_SHA}" >> $GITHUB_OUTPUT | |
| echo "COMPARISON_SHA=${COMPARISON_SHA}" >> $GITHUB_OUTPUT | |
| echo "BASELINE_TAG=${BASELINE_TAG}" >> $GITHUB_OUTPUT | |
| echo "COMPARISON_TAG=${COMPARISON_TAG}" >> $GITHUB_OUTPUT | |
| - name: Set SMP version | |
| id: experimental-meta | |
| run: | | |
| export SMP_CRATE_VERSION="0.26.1" | |
| echo "smp crate version: ${SMP_CRATE_VERSION}" | |
| echo "SMP_CRATE_VERSION=${SMP_CRATE_VERSION}" >> $GITHUB_OUTPUT | |
| # Only run this workflow if files changed in areas that could possibly introduce a regression. | |
| check-source-changed: | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 5 | |
| needs: resolve-inputs | |
| outputs: | |
| source_changed: ${{ steps.filter.outputs.SOURCE_CHANGED }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Collect file changes | |
| id: changes | |
| uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 | |
| with: | |
| base: ${{ needs.resolve-inputs.outputs.baseline-sha }} | |
| ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} | |
| list-files: shell | |
| filters: | | |
| all_changed: | |
| - added|deleted|modified: "**" | |
| ignore: | |
| - "./.github/**/!(regression.yml)" | |
| - "./.gitignore" | |
| - "distribution/**" | |
| - "rust-doc/**" | |
| - "docs/**" | |
| - "rfcs/**" | |
| - "testing/**" | |
| - "tilt/**" | |
| - "website/**" | |
| - "*.md" | |
| - "Tiltfile" | |
| - "NOTICE" | |
| - "LICENSE-3rdparty.csv" | |
| - "LICENSE" | |
| - "lib/codecs/tests/data/**" | |
| # This step allows us to conservatively run the tests if we added a new | |
| # file or directory for source code, but forgot to add it to this workflow. | |
| # Instead, we may unnecessarily run the test on new file or dir additions that | |
| # wouldn't likely introduce regressions. | |
| - name: Determine if should not run due to irrelevant file changes | |
| id: filter | |
| env: | |
| ALL: ${{ steps.changes.outputs.all_changed_files }} | |
| IGNORE: ${{ steps.changes.outputs.ignore_files }} | |
| run: | | |
| echo "ALL='${{ env.ALL }}'" | |
| echo "IGNORE='${{ env.IGNORE }}'" | |
| export SOURCE_CHANGED=$(comm -2 -3 <(printf "%s\n" "${{ env.ALL }}") <(printf "%s\n" "${{ env.IGNORE }}")) | |
| echo "SOURCE_CHANGED='${SOURCE_CHANGED}'" | |
| if [ "${SOURCE_CHANGED}" == "" ]; then | |
| export SOURCE_CHANGED="false" | |
| else | |
| export SOURCE_CHANGED="true" | |
| fi | |
| echo "SOURCE_CHANGED='${SOURCE_CHANGED}'" | |
| echo "SOURCE_CHANGED=${SOURCE_CHANGED}" >> $GITHUB_OUTPUT | |
| should-run-gate: | |
| runs-on: ubuntu-24.04 | |
| needs: check-source-changed | |
| if: ${{ needs.check-source-changed.outputs.source_changed }} | |
| steps: | |
| - name: Gate check passed | |
| run: echo "Source code changes detected, proceeding with regression tests" | |
| ## | |
| ## BUILD | |
| ## | |
| build-baseline: | |
| name: Build baseline Vector container | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 60 | |
| needs: | |
| - should-run-gate | |
| - resolve-inputs | |
| steps: | |
| - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| ref: ${{ needs.resolve-inputs.outputs.baseline-sha }} | |
| path: baseline-vector | |
| - name: Set up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 | |
| - name: Build 'vector' target image | |
| uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 | |
| with: | |
| context: baseline-vector/ | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| file: baseline-vector/regression/Dockerfile | |
| builder: ${{ steps.buildx.outputs.name }} | |
| outputs: type=docker,dest=${{ runner.temp }}/baseline-image.tar | |
| tags: | | |
| vector:${{ needs.resolve-inputs.outputs.baseline-tag }} | |
| - name: Upload image as artifact | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: baseline-image | |
| path: "${{ runner.temp }}/baseline-image.tar" | |
| build-comparison: | |
| name: Build comparison Vector container | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 60 | |
| needs: | |
| - should-run-gate | |
| - resolve-inputs | |
| steps: | |
| - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} | |
| path: comparison-vector | |
| - name: Set up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 | |
| - name: Build 'vector' target image | |
| uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 | |
| with: | |
| context: comparison-vector/ | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| file: comparison-vector/regression/Dockerfile | |
| builder: ${{ steps.buildx.outputs.name }} | |
| outputs: type=docker,dest=${{ runner.temp }}/comparison-image.tar | |
| tags: | | |
| vector:${{ needs.resolve-inputs.outputs.comparison-tag }} | |
| - name: Upload image as artifact | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: comparison-image | |
| path: "${{ runner.temp }}/comparison-image.tar" | |
| confirm-valid-credentials: | |
| name: Confirm AWS credentials are minimally valid | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 5 | |
| needs: | |
| - should-run-gate | |
| - resolve-inputs | |
| # Job level permissions for downloading SMP binary | |
| permissions: | |
| id-token: write # Required for GitHub OIDC token exchange | |
| steps: | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 | |
| with: | |
| role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc | |
| aws-region: us-west-2 | |
| role-duration-seconds: 14400 # 4 hours | |
| - name: Download SMP binary | |
| run: | | |
| aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-musl/smp ${{ runner.temp }}/bin/smp | |
| ## | |
| ## SUBMIT | |
| ## | |
| upload-baseline-image-to-ecr: | |
| name: Upload baseline images to ECR | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 5 | |
| needs: | |
| - should-run-gate | |
| - resolve-inputs | |
| - confirm-valid-credentials | |
| - build-baseline | |
| # Job level permissions for uploading baseline image to SMP ECR | |
| permissions: | |
| id-token: write # Required for GitHub OIDC token exchange | |
| steps: | |
| - name: "Download baseline image" | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: baseline-image | |
| - name: Load baseline image | |
| run: | | |
| docker load --input baseline-image.tar | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 | |
| with: | |
| role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc | |
| aws-region: us-west-2 | |
| role-duration-seconds: 14400 # 4 hours | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 | |
| - name: Docker Login to ECR | |
| uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 | |
| with: | |
| registry: ${{ steps.login-ecr.outputs.registry }} | |
| - name: Tag & push baseline image | |
| run: | | |
| docker tag vector:${{ needs.resolve-inputs.outputs.baseline-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.baseline-tag }} | |
| docker push ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.baseline-tag }} | |
| upload-comparison-image-to-ecr: | |
| name: Upload comparison images to ECR | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 5 | |
| needs: | |
| - should-run-gate | |
| - resolve-inputs | |
| - confirm-valid-credentials | |
| - build-comparison | |
| # Job level permissions for uploading comparison image to SMP ECR | |
| permissions: | |
| id-token: write # Required for GitHub OIDC token exchange | |
| steps: | |
| - name: "Download comparison image" | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: comparison-image | |
| - name: Load comparison image | |
| run: | | |
| docker load --input comparison-image.tar | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 | |
| with: | |
| role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc | |
| aws-region: us-west-2 | |
| role-duration-seconds: 14400 # 4 hours | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 | |
| - name: Docker Login to ECR | |
| uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 | |
| with: | |
| registry: ${{ steps.login-ecr.outputs.registry }} | |
| - name: Tag & push comparison image | |
| run: | | |
| docker tag vector:${{ needs.resolve-inputs.outputs.comparison-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.comparison-tag }} | |
| docker push ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.comparison-tag }} | |
| submit-job: | |
| name: Submit regression job | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 120 | |
| needs: | |
| - should-run-gate | |
| - resolve-inputs | |
| - upload-baseline-image-to-ecr | |
| - upload-comparison-image-to-ecr | |
| permissions: | |
| contents: read # Required to checkout code | |
| id-token: write # Required for GitHub OIDC token exchange | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 | |
| with: | |
| role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc | |
| aws-region: us-west-2 | |
| role-duration-seconds: 14400 # 4 hours | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 | |
| - name: Download SMP binary | |
| run: | | |
| aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-musl/smp ${{ runner.temp }}/bin/smp | |
| - name: Submit job | |
| env: | |
| RUST_LOG: info | |
| run: | | |
| chmod +x ${{ runner.temp }}/bin/smp | |
| ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job submit \ | |
| --baseline-image ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.baseline-tag }} \ | |
| --comparison-image ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.comparison-tag }} \ | |
| --baseline-sha ${{ needs.resolve-inputs.outputs.baseline-sha }} \ | |
| --comparison-sha ${{ needs.resolve-inputs.outputs.comparison-sha }} \ | |
| --target-config-dir ${{ github.workspace }}/regression/ \ | |
| --warmup-seconds ${{ env.SMP_WARMUP_SECONDS }} \ | |
| --submission-metadata ${{ runner.temp }}/submission-metadata \ | |
| --replicas ${{ env.SMP_REPLICAS }} | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: vector-submission-metadata | |
| path: ${{ runner.temp }}/submission-metadata | |
| - name: Await job | |
| timeout-minutes: 120 | |
| env: | |
| RUST_LOG: info | |
| run: | | |
| chmod +x ${{ runner.temp }}/bin/smp | |
| ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job status \ | |
| --wait \ | |
| --wait-delay-seconds 60 \ | |
| --wait-timeout-minutes 120 \ | |
| --submission-metadata ${{ runner.temp }}/submission-metadata | |
| - name: Handle cancellation if necessary | |
| if: ${{ cancelled() }} | |
| env: | |
| RUST_LOG: info | |
| run: | | |
| chmod +x ${{ runner.temp }}/bin/smp | |
| ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job cancel \ | |
| --submission-metadata ${{ runner.temp }}/submission-metadata | |
| ## | |
| ## ANALYZE | |
| ## | |
| detect-regression: | |
| name: Determine regression status | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 5 | |
| needs: | |
| - submit-job | |
| - should-run-gate | |
| - resolve-inputs | |
| # Job level permissions for downloading SMP results | |
| permissions: | |
| contents: read # Required to checkout code | |
| id-token: write # Required for GitHub OIDC token exchange | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 | |
| with: | |
| role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc | |
| aws-region: us-west-2 | |
| role-duration-seconds: 14400 # 4 hours | |
| - name: Download SMP binary | |
| run: | | |
| aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-musl/smp ${{ runner.temp }}/bin/smp | |
| - name: Download submission metadata | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: vector-submission-metadata | |
| path: ${{ runner.temp }}/ | |
| - name: Determine SMP job result | |
| env: | |
| RUST_LOG: info | |
| run: | | |
| chmod +x ${{ runner.temp }}/bin/smp | |
| ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job result \ | |
| --submission-metadata ${{ runner.temp }}/submission-metadata | |
| analyze-experiment: | |
| name: Download regression analysis & upload report | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 5 | |
| needs: | |
| - should-run-gate | |
| - submit-job | |
| - resolve-inputs | |
| permissions: | |
| contents: read # Required to checkout code | |
| id-token: write # Required for GitHub OIDC token exchange | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 | |
| with: | |
| role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc | |
| aws-region: us-west-2 | |
| role-duration-seconds: 14400 # 4 hours | |
| - name: Download SMP binary | |
| run: | | |
| aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-musl/smp ${{ runner.temp }}/bin/smp | |
| - name: Download submission metadata | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: vector-submission-metadata | |
| path: ${{ runner.temp }}/ | |
| - name: Sync regression report to local system | |
| env: | |
| RUST_LOG: info | |
| run: | | |
| chmod +x ${{ runner.temp }}/bin/smp | |
| ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job sync \ | |
| --submission-metadata ${{ runner.temp }}/submission-metadata \ | |
| --output-path "${{ runner.temp }}/outputs" | |
| - name: Read regression report | |
| id: read-analysis | |
| uses: juliangruber/read-file-action@271ff311a4947af354c6abcd696a306553b9ec18 # v1.1.8 | |
| with: | |
| path: ${{ runner.temp }}/outputs/report.md | |
| - name: Upload regression report to artifacts | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: capture-artifacts | |
| path: ${{ runner.temp }}/outputs/* | |
| regression-detection-suite: | |
| name: Set final result | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| if: always() | |
| needs: | |
| - should-run-gate | |
| - resolve-inputs | |
| - build-baseline | |
| - build-comparison | |
| - confirm-valid-credentials | |
| - upload-baseline-image-to-ecr | |
| - upload-comparison-image-to-ecr | |
| - submit-job | |
| - detect-regression | |
| - analyze-experiment | |
| steps: | |
| - name: Download capture-artifacts | |
| continue-on-error: true | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: capture-artifacts | |
| - name: Display Markdown Summary | |
| continue-on-error: true | |
| run: | | |
| REPORT_MD=report.md | |
| if [ -f ${REPORT_MD} ]; then | |
| cat ${REPORT_MD} >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "Did not find ${REPORT_MD} file." | |
| fi | |
| - name: Check all jobs status | |
| run: | | |
| if [[ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" == "true" ]]; then | |
| echo "One or more jobs failed or were cancelled" | |
| exit 1 | |
| else | |
| echo "All jobs completed successfully" | |
| fi |