diff --git a/.github/actions/ensure-build/action.yml b/.github/actions/ensure-build/action.yml new file mode 100644 index 000000000000..398119ba94e4 --- /dev/null +++ b/.github/actions/ensure-build/action.yml @@ -0,0 +1,108 @@ +name: Ensure Build +description: > + Check S3 for a pre-built artifact for the current SHA+config. + If missing, build from source and upload. If present, skip build. + +inputs: + build-id: + required: true + type: string + description: "Unique ID for this build config (e.g., ubuntu20-gcc14-Debug)" + ninja-target: + required: true + type: string + description: "Ninja target: 'src/all' (full, for ctest) or 'dragonfly' (minimal, for pytest only)" + cmake-args: + required: true + type: string + description: "Full cmake args string passed through to cmake" + s3-bucket: + required: true + type: string + description: "S3 bucket name for storing build artifacts" + +runs: + using: "composite" + steps: + - name: Install dependencies + shell: bash + run: | + apt-get update -qq && apt-get install -y -qq zstd > /dev/null + pip3 install -q awscli 2>/dev/null || pip3 install -q --break-system-packages awscli + + - name: Check S3 for existing artifact + id: check-s3 + shell: bash + run: | + S3_KEY="ci-builds/${GITHUB_SHA}/${{ inputs.build-id }}/artifact.tar.zst" + echo "s3_key=${S3_KEY}" >> $GITHUB_OUTPUT + + if aws s3api head-object --bucket "${{ inputs.s3-bucket }}" --key "${S3_KEY}" > /dev/null 2>&1; then + echo "found=true" >> $GITHUB_OUTPUT + echo "::notice::Artifact found for ${GITHUB_SHA}/${{ inputs.build-id }}, skipping build" + else + echo "found=false" >> $GITHUB_OUTPUT + echo "::notice::No artifact for ${GITHUB_SHA}/${{ inputs.build-id }}, building from source" + fi + + - name: Configure CMake + if: steps.check-s3.outputs.found == 'false' + shell: bash + run: | + cmake -B /build ${{ inputs.cmake-args }} -L + + - name: Build + if: steps.check-s3.outputs.found == 'false' + shell: bash + run: | + cd /build + ninja ${{ inputs.ninja-target }} + echo "=== Build complete ===" + df -h + + - name: Package artifact + if: steps.check-s3.outputs.found == 'false' + shell: bash + run: | + cd /build + + if [ "${{ inputs.ninja-target }}" = "dragonfly" ]; then + # Minimal package: just the dragonfly binary + tar --dereference -cf - dragonfly | zstd -3 -o /tmp/artifact.tar.zst + else + # Full package: dragonfly + test binaries + CTestTestfile tree + runfiles + BINARIES="dragonfly" + + # Collect test binaries (top-level *_test files) + TEST_BINS=$(find . -maxdepth 1 -name '*_test' -type f | sed 's|^\./||') + if [ -n "$TEST_BINS" ]; then + BINARIES="$BINARIES $TEST_BINS" + fi + + # Collect CTestTestfile.cmake files (needed for ctest -L DFLY) + CTEST_FILES=$(find . -name 'CTestTestfile.cmake' | sed 's|^\./||') + + # Collect runfiles directories (test data symlinks resolved by --dereference) + RUNFILES=$(find . -name '*.runfiles' -type d | sed 's|^\./||') + + # Build the file list + FILE_LIST="" + for f in $BINARIES $CTEST_FILES $RUNFILES; do + FILE_LIST="$FILE_LIST $f" + done + + echo "=== Packaging $(echo $FILE_LIST | wc -w) items ===" + echo "$FILE_LIST" | tr ' ' '\n' | sort + tar --dereference -cf - $FILE_LIST | zstd -3 -o /tmp/artifact.tar.zst + fi + + ls -lh /tmp/artifact.tar.zst + + - name: Upload to S3 + if: steps.check-s3.outputs.found == 'false' + shell: bash + run: | + export AWS_MAX_ATTEMPTS=3 + aws s3 cp --no-progress /tmp/artifact.tar.zst \ + "s3://${{ inputs.s3-bucket }}/${{ steps.check-s3.outputs.s3_key }}" + echo "::notice::Artifact uploaded to s3://${{ inputs.s3-bucket }}/${{ steps.check-s3.outputs.s3_key }}" diff --git a/.github/actions/fetch-build/action.yml b/.github/actions/fetch-build/action.yml new file mode 100644 index 000000000000..1aa0cc563fce --- /dev/null +++ b/.github/actions/fetch-build/action.yml @@ -0,0 +1,48 @@ +name: Fetch Build +description: > + Download and extract a pre-built artifact from S3. + Expects the artifact to have been uploaded by the ensure-build action. + +inputs: + build-id: + required: true + type: string + description: "Same build-id used in ensure-build (e.g., ubuntu20-gcc14-Debug)" + s3-bucket: + required: true + type: string + description: "S3 bucket name for storing build artifacts" + +runs: + using: "composite" + steps: + - name: Install dependencies + shell: bash + run: | + apt-get update -qq && apt-get install -y -qq zstd > /dev/null + pip3 install -q awscli 2>/dev/null || pip3 install -q --break-system-packages awscli + + - name: Download artifact from S3 + shell: bash + run: | + export AWS_MAX_ATTEMPTS=3 + S3_KEY="ci-builds/${GITHUB_SHA}/${{ inputs.build-id }}/artifact.tar.zst" + echo "Downloading s3://${{ inputs.s3-bucket }}/${S3_KEY}" + aws s3 cp --no-progress "s3://${{ inputs.s3-bucket }}/${S3_KEY}" /tmp/artifact.tar.zst + ls -lh /tmp/artifact.tar.zst + + - name: Extract artifact + shell: bash + run: | + mkdir -p /build + cd /build + zstd -d /tmp/artifact.tar.zst --stdout | tar xf - + rm -f /tmp/artifact.tar.zst + + # Ensure binaries are executable + chmod +x /build/dragonfly 2>/dev/null || true + find /build -maxdepth 1 -name '*_test' -type f -exec chmod +x {} \; + + echo "=== Artifact extracted to /build ===" + ls -lh /build/dragonfly + echo "Test binaries: $(find /build -maxdepth 1 -name '*_test' -type f | wc -l)" diff --git a/.github/actions/regression-tests/action.yml b/.github/actions/regression-tests/action.yml index 425c06765086..0eff03df290d 100644 --- a/.github/actions/regression-tests/action.yml +++ b/.github/actions/regression-tests/action.yml @@ -33,6 +33,10 @@ inputs: epoll: required: false type: string + dragonfly-path: + required: false + type: string + description: "Absolute path to dragonfly binary. If set, overrides build-folder-name/dfly-executable." df-arg: default: "" required: false @@ -87,7 +91,11 @@ runs: echo "=== Running S3 snapshot tests with local MinIO ===" cd ${GITHUB_WORKSPACE}/tests - export DRAGONFLY_PATH="${GITHUB_WORKSPACE}/${{inputs.build-folder-name}}/${{inputs.dfly-executable}}" + if [ -n "${{ inputs.dragonfly-path }}" ]; then + export DRAGONFLY_PATH="${{ inputs.dragonfly-path }}" + else + export DRAGONFLY_PATH="${GITHUB_WORKSPACE}/${{inputs.build-folder-name}}/${{inputs.dfly-executable}}" + fi # MinIO binary is downloaded and started by conftest.py when MINIO_S3_ENDPOINT is set MINIO_S3_ENDPOINT=http://localhost:9000 timeout 10m pytest -k "s3" --timeout=300 --color=yes dragonfly/snapshot_test.py --log-cli-level=INFO -v @@ -100,7 +108,11 @@ runs: cd ${GITHUB_WORKSPACE}/tests echo "Current commit is ${{github.sha}}" # used by PyTests - export DRAGONFLY_PATH="${GITHUB_WORKSPACE}/${{inputs.build-folder-name}}/${{inputs.dfly-executable}}" + if [ -n "${{ inputs.dragonfly-path }}" ]; then + export DRAGONFLY_PATH="${{ inputs.dragonfly-path }}" + else + export DRAGONFLY_PATH="${GITHUB_WORKSPACE}/${{inputs.build-folder-name}}/${{inputs.dfly-executable}}" + fi export ROOT_DIR="${GITHUB_WORKSPACE}/tests/dragonfly/valkey_search" export UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1 # to crash on errors @@ -186,6 +198,9 @@ runs: if: failure() && contains(runner.labels, 'self-hosted') shell: bash run: | - cd ${GITHUB_WORKSPACE}/build timestamp=$(date +%Y-%m-%d_%H:%M:%S) - mv ./dragonfly /var/crash/dragonfly_${timestamp} + if [ -n "${{ inputs.dragonfly-path }}" ]; then + mv "${{ inputs.dragonfly-path }}" /var/crash/dragonfly_${timestamp} + else + mv "${GITHUB_WORKSPACE}/${{ inputs.build-folder-name }}/dragonfly" /var/crash/dragonfly_${timestamp} + fi diff --git a/.github/workflows/ci-shared-build.yml b/.github/workflows/ci-shared-build.yml new file mode 100644 index 000000000000..69e3bcaf7c25 --- /dev/null +++ b/.github/workflows/ci-shared-build.yml @@ -0,0 +1,246 @@ +name: ci-shared-build + +on: + pull_request: + branches: [main] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + #-------------------------------------------------------------------- + # BUILD JOBS — run on LARGE runners, upload artifacts to S3 + #-------------------------------------------------------------------- + build: + runs-on: CI-LARGE-86 + container: + image: ghcr.io/romange/ubuntu-dev:20-gcc14 + options: --security-opt seccomp=unconfined --sysctl "net.ipv6.conf.all.disable_ipv6=0" + volumes: + - /:/hostroot + - /mnt:/mnt + credentials: + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + permissions: + id-token: write + contents: read + concurrency: + group: ensure-build-${{ github.sha }}-ubuntu20-gcc14-Debug + cancel-in-progress: false + steps: + - uses: aws-actions/configure-aws-credentials@v5 + with: + role-to-assume: ${{ secrets.AWS_CI_S3_ROLE_ARN }} + aws-region: us-east-1 + - uses: actions/checkout@v6 + with: + submodules: true + - uses: ./.github/actions/ensure-build + with: + build-id: ubuntu20-gcc14-Debug + ninja-target: src/all + cmake-args: >- + -DCMAKE_BUILD_TYPE=Debug -GNinja + -DWITH_AWS=OFF -DWITH_GCP=OFF -DWITH_UNWIND=OFF -DWITH_GPERF=OFF + -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ + -DCMAKE_CXX_FLAGS="-Werror -no-pie" + s3-bucket: ${{ secrets.CI_BUILD_BUCKET }} + + build-arm: + runs-on: CI-LARGE-ARM + container: + image: ghcr.io/romange/ubuntu-dev:20-gcc14 + options: --security-opt seccomp=unconfined --sysctl "net.ipv6.conf.all.disable_ipv6=0" + volumes: + - /var/crash:/var/crash + - /:/hostroot + - /mnt:/mnt + permissions: + id-token: write + contents: read + concurrency: + group: ensure-build-${{ github.sha }}-arm64-gcc14-Release + cancel-in-progress: false + steps: + - uses: aws-actions/configure-aws-credentials@v5 + with: + role-to-assume: ${{ secrets.AWS_CI_S3_ROLE_ARN }} + aws-region: us-east-1 + - uses: actions/checkout@v6 + with: + submodules: true + - uses: ./.github/actions/ensure-build + with: + build-id: arm64-gcc14-Release + ninja-target: dragonfly + cmake-args: >- + -DCMAKE_BUILD_TYPE=Release -GNinja + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + -DPRINT_STACKTRACES_ON_SIGNAL=ON + -DCMAKE_CXX_FLAGS=-no-pie + s3-bucket: ${{ secrets.CI_BUILD_BUCKET }} + + #-------------------------------------------------------------------- + # TEST JOBS — download artifacts, run tests + #-------------------------------------------------------------------- + test-ctest: + needs: [build] + runs-on: ubuntu-latest + container: + image: ghcr.io/romange/ubuntu-dev:20-gcc14 + options: --security-opt seccomp=unconfined --sysctl "net.ipv6.conf.all.disable_ipv6=0" + credentials: + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v6 + - uses: aws-actions/configure-aws-credentials@v5 + with: + role-to-assume: ${{ secrets.AWS_CI_S3_ROLE_ARN }} + aws-region: us-east-1 + - uses: ./.github/actions/fetch-build + with: + build-id: ubuntu20-gcc14-Debug + s3-bucket: ${{ secrets.CI_BUILD_BUCKET }} + + - name: C++ Unit Tests - IoUring + run: | + echo "Run ctest -V -L DFLY" + + GLOG_alsologtostderr=1 GLOG_vmodule=rdb_load=1,rdb_save=1,snapshot=1,op_manager=1,op_manager_test=1 \ + FLAGS_fiber_safety_margin=4096 timeout 20m ctest -V -L DFLY -E allocation_tracker_test + + # Run allocation tracker test separately without alsologtostderr because it generates a TON of logs. + FLAGS_fiber_safety_margin=4096 timeout 5m ./allocation_tracker_test + + timeout 5m ./dragonfly_test + timeout 5m ./json_family_test --jsonpathv2=false + timeout 5m ./tiered_storage_test --vmodule=db_slice=2 --logtostderr + timeout 5m ./search_test --use_numeric_range_tree=false + timeout 5m ./search_family_test --use_numeric_range_tree=false + working-directory: /build + + - name: C++ Unit Tests - Epoll + run: | + # Create a rule that automatically prints stacktrace upon segfault + cat > ./init.gdb <