Skip to content

Regression Detection Suite #5

Regression Detection Suite

Regression Detection Suite #5

Workflow file for this run

# Regression Detection Suite
#
# This workflow runs under the following conditions:
# - Once per week, based on a schedule
# - On demand, triggered from https://github.com/vectordotdev/vector/actions/workflows/regression.yml
# - On demand, via the `/ci-run-regression` (or `/ci-run-all`) PR review comment,
# which invokes this workflow via `workflow_call` from `ci-review-trigger.yml`.
#
# The workflow accepts two optional inputs:
# - The baseline ref:
# - If not specified, the SHA from 7 days ago on origin/master is used.
# - The comparison ref:
# - If not specified, the current HEAD of origin/master is used.
#
# Both inputs accept any git ref that `git rev-parse` can resolve: a full commit
# SHA, a short SHA, a branch name (e.g. `origin/master`), or a tag.
#
# This workflow runs regression detection experiments, performing relative
# evaluations of the baseline SHA and comparison SHA. The exact SHAs are determined
# by how the workflow is triggered.
#
# The goal is to provide quick feedback on Vector's performance across a variety
# of configurations, checking if throughput performance has degraded or become
# more variable in the comparison SHA relative to the baseline SHA.
#
# Docker image tags are based on the resolved SHAs.
name: Regression Detection Suite
on:
workflow_dispatch:
inputs:
baseline-sha:
description: "Baseline git ref (full/short SHA, branch such as `origin/master`, or tag). Defaults to the SHA from 7 days ago."
required: false
comparison-sha:
description: "Comparison git ref (full/short SHA, branch such as `origin/master`, or tag). Defaults to the current HEAD of origin/master."
required: false
workflow_call:
inputs:
baseline-sha:
description: "Baseline git ref (full/short SHA, branch such as `origin/master`, or tag). Defaults to the SHA from 7 days ago."
required: false
type: string
comparison-sha:
description: "Comparison git ref (full/short SHA, branch such as `origin/master`, or tag). Defaults to the current HEAD of origin/master."
required: false
type: string
schedule:
- cron: "0 7 * * 1" # Runs at 7 AM UTC on Mondays
# Workflow-level permissions - read access to repository contents
permissions:
contents: read # Required to checkout code
env:
SINGLE_MACHINE_PERFORMANCE_API: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_API }}
SMP_WARMUP_SECONDS: 70 # default is 45 seconds
SMP_REPLICAS: 100 # default is 10
jobs:
resolve-inputs:
runs-on: ubuntu-24.04
outputs:
baseline-sha: ${{ steps.set_and_validate_shas.outputs.BASELINE_SHA }}
comparison-sha: ${{ steps.set_and_validate_shas.outputs.COMPARISON_SHA }}
baseline-tag: ${{ steps.set_and_validate_shas.outputs.BASELINE_TAG }}
comparison-tag: ${{ steps.set_and_validate_shas.outputs.COMPARISON_TAG }}
smp-version: ${{ steps.experimental-meta.outputs.SMP_CRATE_VERSION }}
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 0 # need to pull repository history to find merge bases
- name: Set and Validate SHAs
id: set_and_validate_shas
env:
BASELINE_INPUT: ${{ inputs.baseline-sha }}
COMPARISON_INPUT: ${{ inputs.comparison-sha }}
run: |
# Resolve baseline ref to a full commit SHA
if [ -z "${BASELINE_INPUT}" ]; then
BASELINE_SHA=$(git rev-list -n 1 --before="7 days ago" origin/master)
echo "Using baseline SHA from 7 days ago: ${BASELINE_SHA}"
else
if ! BASELINE_SHA=$(git rev-parse --verify "${BASELINE_INPUT}^{commit}" 2>/dev/null); then
echo "Invalid baseline ref: '${BASELINE_INPUT}'. Expected a full/short SHA, branch (e.g. origin/master), or tag."
exit 1
fi
echo "Resolved baseline input '${BASELINE_INPUT}' to SHA: ${BASELINE_SHA}"
fi
# Resolve comparison ref to a full commit SHA
if [ -z "${COMPARISON_INPUT}" ]; then
COMPARISON_SHA=$(git rev-parse origin/master)
echo "Using current HEAD of origin/master as comparison SHA: ${COMPARISON_SHA}"
else
if ! COMPARISON_SHA=$(git rev-parse --verify "${COMPARISON_INPUT}^{commit}" 2>/dev/null); then
echo "Invalid comparison ref: '${COMPARISON_INPUT}'. Expected a full/short SHA, branch (e.g. origin/master), or tag."
exit 1
fi
echo "Resolved comparison input '${COMPARISON_INPUT}' to SHA: ${COMPARISON_SHA}"
fi
# Set tags and export them
BASELINE_TAG="workflow_dispatch-${COMPARISON_SHA}-${BASELINE_SHA}"
COMPARISON_TAG="workflow_dispatch-${COMPARISON_SHA}-${COMPARISON_SHA}"
echo "BASELINE_SHA=${BASELINE_SHA}" >> $GITHUB_OUTPUT
echo "COMPARISON_SHA=${COMPARISON_SHA}" >> $GITHUB_OUTPUT
echo "BASELINE_TAG=${BASELINE_TAG}" >> $GITHUB_OUTPUT
echo "COMPARISON_TAG=${COMPARISON_TAG}" >> $GITHUB_OUTPUT
- name: Set SMP version
id: experimental-meta
run: |
export SMP_CRATE_VERSION="0.26.1"
echo "smp crate version: ${SMP_CRATE_VERSION}"
echo "SMP_CRATE_VERSION=${SMP_CRATE_VERSION}" >> $GITHUB_OUTPUT
# Only run this workflow if files changed in areas that could possibly introduce a regression.
check-source-changed:
runs-on: ubuntu-24.04
timeout-minutes: 5
needs: resolve-inputs
outputs:
source_changed: ${{ steps.filter.outputs.SOURCE_CHANGED }}
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Collect file changes
id: changes
uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
with:
base: ${{ needs.resolve-inputs.outputs.baseline-sha }}
ref: ${{ needs.resolve-inputs.outputs.comparison-sha }}
list-files: shell
filters: |
all_changed:
- added|deleted|modified: "**"
ignore:
- "./.github/**/!(regression.yml)"
- "./.gitignore"
- "distribution/**"
- "rust-doc/**"
- "docs/**"
- "rfcs/**"
- "testing/**"
- "tilt/**"
- "website/**"
- "*.md"
- "Tiltfile"
- "NOTICE"
- "LICENSE-3rdparty.csv"
- "LICENSE"
- "lib/codecs/tests/data/**"
# This step allows us to conservatively run the tests if we added a new
# file or directory for source code, but forgot to add it to this workflow.
# Instead, we may unnecessarily run the test on new file or dir additions that
# wouldn't likely introduce regressions.
- name: Determine if should not run due to irrelevant file changes
id: filter
env:
ALL: ${{ steps.changes.outputs.all_changed_files }}
IGNORE: ${{ steps.changes.outputs.ignore_files }}
run: |
echo "ALL='${{ env.ALL }}'"
echo "IGNORE='${{ env.IGNORE }}'"
export SOURCE_CHANGED=$(comm -2 -3 <(printf "%s\n" "${{ env.ALL }}") <(printf "%s\n" "${{ env.IGNORE }}"))
echo "SOURCE_CHANGED='${SOURCE_CHANGED}'"
if [ "${SOURCE_CHANGED}" == "" ]; then
export SOURCE_CHANGED="false"
else
export SOURCE_CHANGED="true"
fi
echo "SOURCE_CHANGED='${SOURCE_CHANGED}'"
echo "SOURCE_CHANGED=${SOURCE_CHANGED}" >> $GITHUB_OUTPUT
should-run-gate:
runs-on: ubuntu-24.04
needs: check-source-changed
if: ${{ needs.check-source-changed.outputs.source_changed }}
steps:
- name: Gate check passed
run: echo "Source code changes detected, proceeding with regression tests"
##
## BUILD
##
build-baseline:
name: Build baseline Vector container
runs-on: ubuntu-24.04
timeout-minutes: 60
needs:
- should-run-gate
- resolve-inputs
steps:
- uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ needs.resolve-inputs.outputs.baseline-sha }}
path: baseline-vector
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
- name: Build 'vector' target image
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2
with:
context: baseline-vector/
cache-from: type=gha
cache-to: type=gha,mode=max
file: baseline-vector/regression/Dockerfile
builder: ${{ steps.buildx.outputs.name }}
outputs: type=docker,dest=${{ runner.temp }}/baseline-image.tar
tags: |
vector:${{ needs.resolve-inputs.outputs.baseline-tag }}
- name: Upload image as artifact
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: baseline-image
path: "${{ runner.temp }}/baseline-image.tar"
build-comparison:
name: Build comparison Vector container
runs-on: ubuntu-24.04
timeout-minutes: 60
needs:
- should-run-gate
- resolve-inputs
steps:
- uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ needs.resolve-inputs.outputs.comparison-sha }}
path: comparison-vector
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
- name: Build 'vector' target image
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2
with:
context: comparison-vector/
cache-from: type=gha
cache-to: type=gha,mode=max
file: comparison-vector/regression/Dockerfile
builder: ${{ steps.buildx.outputs.name }}
outputs: type=docker,dest=${{ runner.temp }}/comparison-image.tar
tags: |
vector:${{ needs.resolve-inputs.outputs.comparison-tag }}
- name: Upload image as artifact
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: comparison-image
path: "${{ runner.temp }}/comparison-image.tar"
confirm-valid-credentials:
name: Confirm AWS credentials are minimally valid
runs-on: ubuntu-22.04
timeout-minutes: 5
needs:
- should-run-gate
- resolve-inputs
# Job level permissions for downloading SMP binary
permissions:
id-token: write # Required for GitHub OIDC token exchange
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0
with:
role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc
aws-region: us-west-2
role-duration-seconds: 14400 # 4 hours
- name: Download SMP binary
run: |
aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-musl/smp ${{ runner.temp }}/bin/smp
##
## SUBMIT
##
upload-baseline-image-to-ecr:
name: Upload baseline images to ECR
runs-on: ubuntu-22.04
timeout-minutes: 5
needs:
- should-run-gate
- resolve-inputs
- confirm-valid-credentials
- build-baseline
# Job level permissions for uploading baseline image to SMP ECR
permissions:
id-token: write # Required for GitHub OIDC token exchange
steps:
- name: "Download baseline image"
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: baseline-image
- name: Load baseline image
run: |
docker load --input baseline-image.tar
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0
with:
role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc
aws-region: us-west-2
role-duration-seconds: 14400 # 4 hours
- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1
- name: Docker Login to ECR
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
with:
registry: ${{ steps.login-ecr.outputs.registry }}
- name: Tag & push baseline image
run: |
docker tag vector:${{ needs.resolve-inputs.outputs.baseline-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.baseline-tag }}
docker push ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.baseline-tag }}
upload-comparison-image-to-ecr:
name: Upload comparison images to ECR
runs-on: ubuntu-22.04
timeout-minutes: 5
needs:
- should-run-gate
- resolve-inputs
- confirm-valid-credentials
- build-comparison
# Job level permissions for uploading comparison image to SMP ECR
permissions:
id-token: write # Required for GitHub OIDC token exchange
steps:
- name: "Download comparison image"
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: comparison-image
- name: Load comparison image
run: |
docker load --input comparison-image.tar
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0
with:
role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc
aws-region: us-west-2
role-duration-seconds: 14400 # 4 hours
- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1
- name: Docker Login to ECR
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
with:
registry: ${{ steps.login-ecr.outputs.registry }}
- name: Tag & push comparison image
run: |
docker tag vector:${{ needs.resolve-inputs.outputs.comparison-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.comparison-tag }}
docker push ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.comparison-tag }}
submit-job:
name: Submit regression job
runs-on: ubuntu-22.04
timeout-minutes: 120
needs:
- should-run-gate
- resolve-inputs
- upload-baseline-image-to-ecr
- upload-comparison-image-to-ecr
permissions:
contents: read # Required to checkout code
id-token: write # Required for GitHub OIDC token exchange
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ needs.resolve-inputs.outputs.comparison-sha }}
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0
with:
role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc
aws-region: us-west-2
role-duration-seconds: 14400 # 4 hours
- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1
- name: Download SMP binary
run: |
aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-musl/smp ${{ runner.temp }}/bin/smp
- name: Submit job
env:
RUST_LOG: info
run: |
chmod +x ${{ runner.temp }}/bin/smp
${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job submit \
--baseline-image ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.baseline-tag }} \
--comparison-image ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.comparison-tag }} \
--baseline-sha ${{ needs.resolve-inputs.outputs.baseline-sha }} \
--comparison-sha ${{ needs.resolve-inputs.outputs.comparison-sha }} \
--target-config-dir ${{ github.workspace }}/regression/ \
--warmup-seconds ${{ env.SMP_WARMUP_SECONDS }} \
--submission-metadata ${{ runner.temp }}/submission-metadata \
--replicas ${{ env.SMP_REPLICAS }}
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: vector-submission-metadata
path: ${{ runner.temp }}/submission-metadata
- name: Await job
timeout-minutes: 120
env:
RUST_LOG: info
run: |
chmod +x ${{ runner.temp }}/bin/smp
${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job status \
--wait \
--wait-delay-seconds 60 \
--wait-timeout-minutes 120 \
--submission-metadata ${{ runner.temp }}/submission-metadata
- name: Handle cancellation if necessary
if: ${{ cancelled() }}
env:
RUST_LOG: info
run: |
chmod +x ${{ runner.temp }}/bin/smp
${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job cancel \
--submission-metadata ${{ runner.temp }}/submission-metadata
##
## ANALYZE
##
detect-regression:
name: Determine regression status
runs-on: ubuntu-22.04
timeout-minutes: 5
needs:
- submit-job
- should-run-gate
- resolve-inputs
# Job level permissions for downloading SMP results
permissions:
contents: read # Required to checkout code
id-token: write # Required for GitHub OIDC token exchange
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0
with:
role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc
aws-region: us-west-2
role-duration-seconds: 14400 # 4 hours
- name: Download SMP binary
run: |
aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-musl/smp ${{ runner.temp }}/bin/smp
- name: Download submission metadata
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: vector-submission-metadata
path: ${{ runner.temp }}/
- name: Determine SMP job result
env:
RUST_LOG: info
run: |
chmod +x ${{ runner.temp }}/bin/smp
${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job result \
--submission-metadata ${{ runner.temp }}/submission-metadata
analyze-experiment:
name: Download regression analysis & upload report
runs-on: ubuntu-22.04
timeout-minutes: 5
needs:
- should-run-gate
- submit-job
- resolve-inputs
permissions:
contents: read # Required to checkout code
id-token: write # Required for GitHub OIDC token exchange
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ needs.resolve-inputs.outputs.comparison-sha }}
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0
with:
role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc
aws-region: us-west-2
role-duration-seconds: 14400 # 4 hours
- name: Download SMP binary
run: |
aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-musl/smp ${{ runner.temp }}/bin/smp
- name: Download submission metadata
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: vector-submission-metadata
path: ${{ runner.temp }}/
- name: Sync regression report to local system
env:
RUST_LOG: info
run: |
chmod +x ${{ runner.temp }}/bin/smp
${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job sync \
--submission-metadata ${{ runner.temp }}/submission-metadata \
--output-path "${{ runner.temp }}/outputs"
- name: Read regression report
id: read-analysis
uses: juliangruber/read-file-action@271ff311a4947af354c6abcd696a306553b9ec18 # v1.1.8
with:
path: ${{ runner.temp }}/outputs/report.md
- name: Upload regression report to artifacts
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: capture-artifacts
path: ${{ runner.temp }}/outputs/*
regression-detection-suite:
name: Set final result
runs-on: ubuntu-latest
timeout-minutes: 5
if: always()
needs:
- should-run-gate
- resolve-inputs
- build-baseline
- build-comparison
- confirm-valid-credentials
- upload-baseline-image-to-ecr
- upload-comparison-image-to-ecr
- submit-job
- detect-regression
- analyze-experiment
steps:
- name: Download capture-artifacts
continue-on-error: true
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: capture-artifacts
- name: Display Markdown Summary
continue-on-error: true
run: |
REPORT_MD=report.md
if [ -f ${REPORT_MD} ]; then
cat ${REPORT_MD} >> $GITHUB_STEP_SUMMARY
else
echo "Did not find ${REPORT_MD} file."
fi
- name: Check all jobs status
run: |
if [[ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
echo "One or more jobs failed or were cancelled"
exit 1
else
echo "All jobs completed successfully"
fi