From a6baa31c8978e2b68b74fca873a80f6c58654efe Mon Sep 17 00:00:00 2001 From: Volodymyr Yavdoshenko Date: Fri, 20 Feb 2026 18:54:22 +0200 Subject: [PATCH 1/6] ci: workflow updated --- .github/actions/ensure-build/action.yml | 115 +++++++++ .github/actions/fetch-build/action.yml | 42 ++++ .github/actions/regression-tests/action.yml | 23 +- .github/workflows/ci-shared-build.yml | 246 ++++++++++++++++++++ 4 files changed, 422 insertions(+), 4 deletions(-) create mode 100644 .github/actions/ensure-build/action.yml create mode 100644 .github/actions/fetch-build/action.yml create mode 100644 .github/workflows/ci-shared-build.yml diff --git a/.github/actions/ensure-build/action.yml b/.github/actions/ensure-build/action.yml new file mode 100644 index 000000000000..c46d5b19e132 --- /dev/null +++ b/.github/actions/ensure-build/action.yml @@ -0,0 +1,115 @@ +name: Ensure Build +description: > + Check S3 for a pre-built artifact for the current SHA+config. + If missing, build from source and upload. If present, skip build. + +inputs: + build-id: + required: true + type: string + description: "Unique ID for this build config (e.g., ubuntu20-gcc14-Debug)" + ninja-target: + required: true + type: string + description: "Ninja target: 'src/all' (full, for ctest) or 'dragonfly' (minimal, for pytest only)" + cmake-args: + required: true + type: string + description: "Full cmake args string passed through to cmake" + s3-bucket: + required: true + type: string + description: "S3 bucket name for storing build artifacts" + +runs: + using: "composite" + steps: + - name: Check S3 for existing artifact + id: check-s3 + shell: bash + run: | + S3_KEY="ci-builds/${GITHUB_SHA}/${{ inputs.build-id }}/artifact.tar.zst" + echo "s3_key=${S3_KEY}" >> $GITHUB_OUTPUT + + if aws s3api head-object --bucket "${{ inputs.s3-bucket }}" --key "${S3_KEY}" > /dev/null 2>&1; then + echo "found=true" >> $GITHUB_OUTPUT + echo "::notice::Artifact found for ${GITHUB_SHA}/${{ inputs.build-id }}, skipping build" + else + echo "found=false" >> $GITHUB_OUTPUT + echo "::notice::No artifact for ${GITHUB_SHA}/${{ inputs.build-id }}, building from source" + fi + + - name: Free disk space + if: steps.check-s3.outputs.found == 'false' + shell: bash + run: | + echo "=== Before freeing up space ===" + df -h + rm -rf /hostroot/usr/share/dotnet 2>/dev/null || true + rm -rf /hostroot/usr/local/share/boost 2>/dev/null || true + rm -rf /hostroot/usr/local/lib/android 2>/dev/null || true + rm -rf /hostroot/opt/ghc 2>/dev/null || true + echo "=== After freeing up space ===" + df -h + + - name: Configure CMake + if: steps.check-s3.outputs.found == 'false' + shell: bash + run: | + cmake -B /build ${{ inputs.cmake-args }} -L + + - name: Build + if: steps.check-s3.outputs.found == 'false' + shell: bash + run: | + cd /build + ninja ${{ inputs.ninja-target }} + echo "=== Build complete ===" + df -h + + - name: Package artifact + if: steps.check-s3.outputs.found == 'false' + shell: bash + run: | + cd /build + + if [ "${{ inputs.ninja-target }}" = "dragonfly" ]; then + # Minimal package: just the dragonfly binary + tar --dereference -cf - dragonfly | zstd -3 -o /tmp/artifact.tar.zst + else + # Full package: dragonfly + test binaries + CTestTestfile tree + runfiles + BINARIES="dragonfly" + + # Collect test binaries (top-level *_test files) + TEST_BINS=$(find . -maxdepth 1 -name '*_test' -type f | sed 's|^\./||') + if [ -n "$TEST_BINS" ]; then + BINARIES="$BINARIES $TEST_BINS" + fi + + # Collect CTestTestfile.cmake files (needed for ctest -L DFLY) + CTEST_FILES=$(find . -name 'CTestTestfile.cmake' | sed 's|^\./||') + + # Collect runfiles directories (test data symlinks resolved by --dereference) + RUNFILES=$(find . -name '*.runfiles' -type d | sed 's|^\./||') + + # Build the file list + FILE_LIST="" + for f in $BINARIES $CTEST_FILES $RUNFILES; do + FILE_LIST="$FILE_LIST $f" + done + + echo "=== Packaging $(echo $FILE_LIST | wc -w) items ===" + echo "$FILE_LIST" | tr ' ' '\n' | sort + tar --dereference -cf - $FILE_LIST | zstd -3 -o /tmp/artifact.tar.zst + fi + + ls -lh /tmp/artifact.tar.zst + + - name: Upload to S3 + if: steps.check-s3.outputs.found == 'false' + shell: bash + run: | + export AWS_MAX_ATTEMPTS=3 + aws s3 cp --no-progress /tmp/artifact.tar.zst \ + "s3://${{ inputs.s3-bucket }}/${{ steps.check-s3.outputs.s3_key }}" + echo "::notice::Artifact uploaded to s3://${{ inputs.s3-bucket }}/${{ steps.check-s3.outputs.s3_key }}" diff --git a/.github/actions/fetch-build/action.yml b/.github/actions/fetch-build/action.yml new file mode 100644 index 000000000000..bfc70d04ca44 --- /dev/null +++ b/.github/actions/fetch-build/action.yml @@ -0,0 +1,42 @@ +name: Fetch Build +description: > + Download and extract a pre-built artifact from S3. + Expects the artifact to have been uploaded by the ensure-build action. + +inputs: + build-id: + required: true + type: string + description: "Same build-id used in ensure-build (e.g., ubuntu20-gcc14-Debug)" + s3-bucket: + required: true + type: string + description: "S3 bucket name for storing build artifacts" + +runs: + using: "composite" + steps: + - name: Download artifact from S3 + shell: bash + run: | + export AWS_MAX_ATTEMPTS=3 + S3_KEY="ci-builds/${GITHUB_SHA}/${{ inputs.build-id }}/artifact.tar.zst" + echo "Downloading s3://${{ inputs.s3-bucket }}/${S3_KEY}" + aws s3 cp --no-progress "s3://${{ inputs.s3-bucket }}/${S3_KEY}" /tmp/artifact.tar.zst + ls -lh /tmp/artifact.tar.zst + + - name: Extract artifact + shell: bash + run: | + mkdir -p /build + cd /build + zstd -d /tmp/artifact.tar.zst --stdout | tar xf - + rm -f /tmp/artifact.tar.zst + + # Ensure binaries are executable + chmod +x /build/dragonfly 2>/dev/null || true + find /build -maxdepth 1 -name '*_test' -type f -exec chmod +x {} \; + + echo "=== Artifact extracted to /build ===" + ls -lh /build/dragonfly + echo "Test binaries: $(find /build -maxdepth 1 -name '*_test' -type f | wc -l)" diff --git a/.github/actions/regression-tests/action.yml b/.github/actions/regression-tests/action.yml index 425c06765086..0eff03df290d 100644 --- a/.github/actions/regression-tests/action.yml +++ b/.github/actions/regression-tests/action.yml @@ -33,6 +33,10 @@ inputs: epoll: required: false type: string + dragonfly-path: + required: false + type: string + description: "Absolute path to dragonfly binary. If set, overrides build-folder-name/dfly-executable." df-arg: default: "" required: false @@ -87,7 +91,11 @@ runs: echo "=== Running S3 snapshot tests with local MinIO ===" cd ${GITHUB_WORKSPACE}/tests - export DRAGONFLY_PATH="${GITHUB_WORKSPACE}/${{inputs.build-folder-name}}/${{inputs.dfly-executable}}" + if [ -n "${{ inputs.dragonfly-path }}" ]; then + export DRAGONFLY_PATH="${{ inputs.dragonfly-path }}" + else + export DRAGONFLY_PATH="${GITHUB_WORKSPACE}/${{inputs.build-folder-name}}/${{inputs.dfly-executable}}" + fi # MinIO binary is downloaded and started by conftest.py when MINIO_S3_ENDPOINT is set MINIO_S3_ENDPOINT=http://localhost:9000 timeout 10m pytest -k "s3" --timeout=300 --color=yes dragonfly/snapshot_test.py --log-cli-level=INFO -v @@ -100,7 +108,11 @@ runs: cd ${GITHUB_WORKSPACE}/tests echo "Current commit is ${{github.sha}}" # used by PyTests - export DRAGONFLY_PATH="${GITHUB_WORKSPACE}/${{inputs.build-folder-name}}/${{inputs.dfly-executable}}" + if [ -n "${{ inputs.dragonfly-path }}" ]; then + export DRAGONFLY_PATH="${{ inputs.dragonfly-path }}" + else + export DRAGONFLY_PATH="${GITHUB_WORKSPACE}/${{inputs.build-folder-name}}/${{inputs.dfly-executable}}" + fi export ROOT_DIR="${GITHUB_WORKSPACE}/tests/dragonfly/valkey_search" export UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1 # to crash on errors @@ -186,6 +198,9 @@ runs: if: failure() && contains(runner.labels, 'self-hosted') shell: bash run: | - cd ${GITHUB_WORKSPACE}/build timestamp=$(date +%Y-%m-%d_%H:%M:%S) - mv ./dragonfly /var/crash/dragonfly_${timestamp} + if [ -n "${{ inputs.dragonfly-path }}" ]; then + mv "${{ inputs.dragonfly-path }}" /var/crash/dragonfly_${timestamp} + else + mv "${GITHUB_WORKSPACE}/${{ inputs.build-folder-name }}/dragonfly" /var/crash/dragonfly_${timestamp} + fi diff --git a/.github/workflows/ci-shared-build.yml b/.github/workflows/ci-shared-build.yml new file mode 100644 index 000000000000..69e3bcaf7c25 --- /dev/null +++ b/.github/workflows/ci-shared-build.yml @@ -0,0 +1,246 @@ +name: ci-shared-build + +on: + pull_request: + branches: [main] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + #-------------------------------------------------------------------- + # BUILD JOBS — run on LARGE runners, upload artifacts to S3 + #-------------------------------------------------------------------- + build: + runs-on: CI-LARGE-86 + container: + image: ghcr.io/romange/ubuntu-dev:20-gcc14 + options: --security-opt seccomp=unconfined --sysctl "net.ipv6.conf.all.disable_ipv6=0" + volumes: + - /:/hostroot + - /mnt:/mnt + credentials: + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + permissions: + id-token: write + contents: read + concurrency: + group: ensure-build-${{ github.sha }}-ubuntu20-gcc14-Debug + cancel-in-progress: false + steps: + - uses: aws-actions/configure-aws-credentials@v5 + with: + role-to-assume: ${{ secrets.AWS_CI_S3_ROLE_ARN }} + aws-region: us-east-1 + - uses: actions/checkout@v6 + with: + submodules: true + - uses: ./.github/actions/ensure-build + with: + build-id: ubuntu20-gcc14-Debug + ninja-target: src/all + cmake-args: >- + -DCMAKE_BUILD_TYPE=Debug -GNinja + -DWITH_AWS=OFF -DWITH_GCP=OFF -DWITH_UNWIND=OFF -DWITH_GPERF=OFF + -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ + -DCMAKE_CXX_FLAGS="-Werror -no-pie" + s3-bucket: ${{ secrets.CI_BUILD_BUCKET }} + + build-arm: + runs-on: CI-LARGE-ARM + container: + image: ghcr.io/romange/ubuntu-dev:20-gcc14 + options: --security-opt seccomp=unconfined --sysctl "net.ipv6.conf.all.disable_ipv6=0" + volumes: + - /var/crash:/var/crash + - /:/hostroot + - /mnt:/mnt + permissions: + id-token: write + contents: read + concurrency: + group: ensure-build-${{ github.sha }}-arm64-gcc14-Release + cancel-in-progress: false + steps: + - uses: aws-actions/configure-aws-credentials@v5 + with: + role-to-assume: ${{ secrets.AWS_CI_S3_ROLE_ARN }} + aws-region: us-east-1 + - uses: actions/checkout@v6 + with: + submodules: true + - uses: ./.github/actions/ensure-build + with: + build-id: arm64-gcc14-Release + ninja-target: dragonfly + cmake-args: >- + -DCMAKE_BUILD_TYPE=Release -GNinja + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + -DPRINT_STACKTRACES_ON_SIGNAL=ON + -DCMAKE_CXX_FLAGS=-no-pie + s3-bucket: ${{ secrets.CI_BUILD_BUCKET }} + + #-------------------------------------------------------------------- + # TEST JOBS — download artifacts, run tests + #-------------------------------------------------------------------- + test-ctest: + needs: [build] + runs-on: ubuntu-latest + container: + image: ghcr.io/romange/ubuntu-dev:20-gcc14 + options: --security-opt seccomp=unconfined --sysctl "net.ipv6.conf.all.disable_ipv6=0" + credentials: + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v6 + - uses: aws-actions/configure-aws-credentials@v5 + with: + role-to-assume: ${{ secrets.AWS_CI_S3_ROLE_ARN }} + aws-region: us-east-1 + - uses: ./.github/actions/fetch-build + with: + build-id: ubuntu20-gcc14-Debug + s3-bucket: ${{ secrets.CI_BUILD_BUCKET }} + + - name: C++ Unit Tests - IoUring + run: | + echo "Run ctest -V -L DFLY" + + GLOG_alsologtostderr=1 GLOG_vmodule=rdb_load=1,rdb_save=1,snapshot=1,op_manager=1,op_manager_test=1 \ + FLAGS_fiber_safety_margin=4096 timeout 20m ctest -V -L DFLY -E allocation_tracker_test + + # Run allocation tracker test separately without alsologtostderr because it generates a TON of logs. + FLAGS_fiber_safety_margin=4096 timeout 5m ./allocation_tracker_test + + timeout 5m ./dragonfly_test + timeout 5m ./json_family_test --jsonpathv2=false + timeout 5m ./tiered_storage_test --vmodule=db_slice=2 --logtostderr + timeout 5m ./search_test --use_numeric_range_tree=false + timeout 5m ./search_family_test --use_numeric_range_tree=false + working-directory: /build + + - name: C++ Unit Tests - Epoll + run: | + # Create a rule that automatically prints stacktrace upon segfault + cat > ./init.gdb < Date: Fri, 20 Feb 2026 19:09:36 +0200 Subject: [PATCH 2/6] ci: workflow updated --- .github/actions/ensure-build/action.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.github/actions/ensure-build/action.yml b/.github/actions/ensure-build/action.yml index c46d5b19e132..ac7d36bbf313 100644 --- a/.github/actions/ensure-build/action.yml +++ b/.github/actions/ensure-build/action.yml @@ -39,19 +39,6 @@ runs: echo "::notice::No artifact for ${GITHUB_SHA}/${{ inputs.build-id }}, building from source" fi - - name: Free disk space - if: steps.check-s3.outputs.found == 'false' - shell: bash - run: | - echo "=== Before freeing up space ===" - df -h - rm -rf /hostroot/usr/share/dotnet 2>/dev/null || true - rm -rf /hostroot/usr/local/share/boost 2>/dev/null || true - rm -rf /hostroot/usr/local/lib/android 2>/dev/null || true - rm -rf /hostroot/opt/ghc 2>/dev/null || true - echo "=== After freeing up space ===" - df -h - - name: Configure CMake if: steps.check-s3.outputs.found == 'false' shell: bash From 1e55c9a6d068aa008046a0c0df90e5abce65124d Mon Sep 17 00:00:00 2001 From: Volodymyr Yavdoshenko Date: Mon, 23 Feb 2026 13:37:34 +0200 Subject: [PATCH 3/6] ci: workflow updated --- .github/actions/ensure-build/action.yml | 5 +++++ .github/actions/fetch-build/action.yml | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/.github/actions/ensure-build/action.yml b/.github/actions/ensure-build/action.yml index ac7d36bbf313..1f18e6d4b36a 100644 --- a/.github/actions/ensure-build/action.yml +++ b/.github/actions/ensure-build/action.yml @@ -54,6 +54,11 @@ runs: echo "=== Build complete ===" df -h + - name: Install zstd + if: steps.check-s3.outputs.found == 'false' + shell: bash + run: apt-get install -y -qq zstd > /dev/null + - name: Package artifact if: steps.check-s3.outputs.found == 'false' shell: bash diff --git a/.github/actions/fetch-build/action.yml b/.github/actions/fetch-build/action.yml index bfc70d04ca44..2fe98c873c28 100644 --- a/.github/actions/fetch-build/action.yml +++ b/.github/actions/fetch-build/action.yml @@ -16,6 +16,10 @@ inputs: runs: using: "composite" steps: + - name: Install zstd + shell: bash + run: apt-get install -y -qq zstd > /dev/null + - name: Download artifact from S3 shell: bash run: | From 801378681d8799e78389c1fb42a190d3f60c2175 Mon Sep 17 00:00:00 2001 From: Volodymyr Yavdoshenko Date: Mon, 23 Feb 2026 13:53:20 +0200 Subject: [PATCH 4/6] ci: workflow updated --- .github/actions/ensure-build/action.yml | 2 +- .github/actions/fetch-build/action.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/ensure-build/action.yml b/.github/actions/ensure-build/action.yml index 1f18e6d4b36a..cbf441eb1f78 100644 --- a/.github/actions/ensure-build/action.yml +++ b/.github/actions/ensure-build/action.yml @@ -57,7 +57,7 @@ runs: - name: Install zstd if: steps.check-s3.outputs.found == 'false' shell: bash - run: apt-get install -y -qq zstd > /dev/null + run: apt-get update -qq && apt-get install -y -qq zstd > /dev/null - name: Package artifact if: steps.check-s3.outputs.found == 'false' diff --git a/.github/actions/fetch-build/action.yml b/.github/actions/fetch-build/action.yml index 2fe98c873c28..b6802957ba8c 100644 --- a/.github/actions/fetch-build/action.yml +++ b/.github/actions/fetch-build/action.yml @@ -18,7 +18,7 @@ runs: steps: - name: Install zstd shell: bash - run: apt-get install -y -qq zstd > /dev/null + run: apt-get update -qq && apt-get install -y -qq zstd > /dev/null - name: Download artifact from S3 shell: bash From 2e39fbc6a4b6feef016bef382f16e23529100133 Mon Sep 17 00:00:00 2001 From: Volodymyr Yavdoshenko Date: Mon, 23 Feb 2026 14:10:36 +0200 Subject: [PATCH 5/6] ci: workflow updated --- .github/actions/ensure-build/action.yml | 9 ++++----- .github/actions/fetch-build/action.yml | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/actions/ensure-build/action.yml b/.github/actions/ensure-build/action.yml index cbf441eb1f78..b776acec8a09 100644 --- a/.github/actions/ensure-build/action.yml +++ b/.github/actions/ensure-build/action.yml @@ -24,6 +24,10 @@ inputs: runs: using: "composite" steps: + - name: Install dependencies + shell: bash + run: apt-get update -qq && apt-get install -y -qq awscli zstd > /dev/null + - name: Check S3 for existing artifact id: check-s3 shell: bash @@ -54,11 +58,6 @@ runs: echo "=== Build complete ===" df -h - - name: Install zstd - if: steps.check-s3.outputs.found == 'false' - shell: bash - run: apt-get update -qq && apt-get install -y -qq zstd > /dev/null - - name: Package artifact if: steps.check-s3.outputs.found == 'false' shell: bash diff --git a/.github/actions/fetch-build/action.yml b/.github/actions/fetch-build/action.yml index b6802957ba8c..545bfff036a9 100644 --- a/.github/actions/fetch-build/action.yml +++ b/.github/actions/fetch-build/action.yml @@ -16,9 +16,9 @@ inputs: runs: using: "composite" steps: - - name: Install zstd + - name: Install dependencies shell: bash - run: apt-get update -qq && apt-get install -y -qq zstd > /dev/null + run: apt-get update -qq && apt-get install -y -qq awscli zstd > /dev/null - name: Download artifact from S3 shell: bash From 4e6db44ad2c2809445cd557d5a9ead8d69fdd13d Mon Sep 17 00:00:00 2001 From: Volodymyr Yavdoshenko Date: Mon, 23 Feb 2026 14:24:57 +0200 Subject: [PATCH 6/6] ci: workflow updated --- .github/actions/ensure-build/action.yml | 4 +++- .github/actions/fetch-build/action.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/actions/ensure-build/action.yml b/.github/actions/ensure-build/action.yml index b776acec8a09..398119ba94e4 100644 --- a/.github/actions/ensure-build/action.yml +++ b/.github/actions/ensure-build/action.yml @@ -26,7 +26,9 @@ runs: steps: - name: Install dependencies shell: bash - run: apt-get update -qq && apt-get install -y -qq awscli zstd > /dev/null + run: | + apt-get update -qq && apt-get install -y -qq zstd > /dev/null + pip3 install -q awscli 2>/dev/null || pip3 install -q --break-system-packages awscli - name: Check S3 for existing artifact id: check-s3 diff --git a/.github/actions/fetch-build/action.yml b/.github/actions/fetch-build/action.yml index 545bfff036a9..1aa0cc563fce 100644 --- a/.github/actions/fetch-build/action.yml +++ b/.github/actions/fetch-build/action.yml @@ -18,7 +18,9 @@ runs: steps: - name: Install dependencies shell: bash - run: apt-get update -qq && apt-get install -y -qq awscli zstd > /dev/null + run: | + apt-get update -qq && apt-get install -y -qq zstd > /dev/null + pip3 install -q awscli 2>/dev/null || pip3 install -q --break-system-packages awscli - name: Download artifact from S3 shell: bash