diff --git a/.github/actions/docker-compose-healthy/action.yml b/.github/actions/docker-compose-healthy/action.yml new file mode 100644 index 0000000000..3160d7ab10 --- /dev/null +++ b/.github/actions/docker-compose-healthy/action.yml @@ -0,0 +1,27 @@ +name: Docker Compose Healthy +description: Wait for Docker Compose services to become healthy. + +inputs: + compose_file: + description: Docker Compose file to use. + required: true + services: + description: Whitespace-delimited list of services to wait for. + default: "" + +runs: + using: composite + steps: + - name: Wait for services + shell: bash + env: + COMPOSE_FILE: ${{ inputs.compose_file }} + SERVICES: ${{ inputs.services }} + run: | + if [[ -z "${SERVICES//[[:space:]]/}" ]]; then + echo "No services configured for docker compose healthy." + exit 0 + fi + + read -r -a services <<< "$SERVICES" + docker compose -f "$COMPOSE_FILE" up --wait "${services[@]}" diff --git a/.github/actions/docker-compose-pull/action.yml b/.github/actions/docker-compose-pull/action.yml deleted file mode 100644 index 4d29cf4696..0000000000 --- a/.github/actions/docker-compose-pull/action.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Docker Compose Start -description: Pull Docker Compose services with retries, then start them. - -inputs: - compose-file: - description: Docker Compose file to use. - required: true - services: - description: Whitespace-delimited list of services to pull. - required: true - attempts: - description: Number of pull attempts. - default: "5" - delay-seconds: - description: Delay between failed attempts. - default: "5" - timeout-minutes: - description: Timeout for each pull attempt. - default: "1" - down-flags: - description: Additional options to pass to docker compose down. - default: "" - -runs: - using: composite - steps: - - name: Pull services - uses: nick-fields/retry@v3 - env: - COMPOSE_FILE: ${{ inputs['compose-file'] }} - SERVICES: ${{ inputs.services }} - with: - max_attempts: ${{ inputs.attempts }} - retry_wait_seconds: ${{ inputs['delay-seconds'] }} - timeout_minutes: ${{ inputs['timeout-minutes'] }} - shell: bash - command: | - if [[ -z "${SERVICES//[[:space:]]/}" ]]; then - echo "No services configured for docker compose pull." - exit 0 - fi - docker compose -f "$COMPOSE_FILE" pull $SERVICES - - - name: Start services - uses: hoverkraft-tech/compose-action@v2.0.1 - with: - compose-file: ${{ inputs['compose-file'] }} - services: ${{ inputs.services }} - down-flags: ${{ inputs['down-flags'] }} diff --git a/.github/actions/docker-compose-start/action.yml b/.github/actions/docker-compose-start/action.yml new file mode 100644 index 0000000000..e0cc44cc62 --- /dev/null +++ b/.github/actions/docker-compose-start/action.yml @@ -0,0 +1,131 @@ +name: Docker Compose Start +description: Restore Docker image cache, pull Docker Compose services with retries, then start them. + +inputs: + compose_file: + description: Docker Compose file to use. + required: true + services: + description: Whitespace-delimited list of services to start. + default: "" + attempts: + description: Number of pull attempts. + default: "5" + delay_seconds: + description: Delay between failed attempts. + default: "5" + timeout_minutes: + description: Timeout for each pull attempt. + default: "1" + flags: + description: Additional options to pass to docker compose up. + default: "" + cache: + description: Set to "true" to cache Docker images. + default: "false" + cache_path: + description: Directory used to store the Docker image archive. + default: ${{ runner.temp }}/docker-image-cache + +runs: + using: composite + steps: + - name: Prepare Docker image cache + id: cache + if: ${{ inputs.cache == 'true' }} + shell: bash + env: + COMPOSE_HASH: ${{ hashFiles(inputs.compose_file) }} + SERVICES: ${{ inputs.services }} + run: | + services_hash="$(printf '%s' "$SERVICES" | shasum -a 256 | cut -d ' ' -f 1)" + + echo "key=docker-${RUNNER_OS}${RUNNER_ARCH}-${COMPOSE_HASH}-${services_hash}" >> "$GITHUB_OUTPUT" + + - name: Restore Docker image cache + id: restore-cache + if: ${{ inputs.cache == 'true' }} + uses: actions/cache/restore@v5 + with: + path: ${{ inputs.cache_path }} + key: ${{ steps.cache.outputs.key }} + + - name: Load Docker images + if: ${{ inputs.cache == 'true' && steps.restore-cache.outputs.cache-hit == 'true' }} + shell: bash + env: + CACHE_PATH: ${{ inputs.cache_path }} + run: | + if [[ -f "$CACHE_PATH/images.tar" ]]; then + docker load --input "$CACHE_PATH/images.tar" + else + echo "Docker image cache hit did not include images.tar." + fi + + - name: Pull services + if: ${{ steps.restore-cache.outputs.cache-hit != 'true' }} + uses: nick-fields/retry@v4 + env: + COMPOSE_FILE: ${{ inputs.compose_file }} + SERVICES: ${{ inputs.services }} + with: + max_attempts: ${{ inputs.attempts }} + retry_wait_seconds: ${{ inputs.delay_seconds }} + timeout_minutes: ${{ inputs.timeout_minutes }} + shell: bash + command: | + if [[ -z "${SERVICES//[[:space:]]/}" ]]; then + echo "No services configured for docker compose pull." + exit 0 + fi + docker compose -f "$COMPOSE_FILE" pull $SERVICES + + - name: Start services + shell: bash + env: + COMPOSE_FILE: ${{ inputs.compose_file }} + SERVICES: ${{ inputs.services }} + FLAGS: ${{ inputs.flags }} + run: | + if [[ -z "${SERVICES//[[:space:]]/}" ]]; then + echo "No services configured for docker compose up." + exit 0 + fi + docker compose -f "$COMPOSE_FILE" up -d $FLAGS $SERVICES + + - name: Save Docker images to archive + id: archive + if: ${{ inputs.cache == 'true' && steps.restore-cache.outputs.cache-hit != 'true' }} + shell: bash + env: + CACHE_PATH: ${{ inputs.cache_path }} + COMPOSE_FILE: ${{ inputs.compose_file }} + SERVICES: ${{ inputs.services }} + run: | + mkdir -p "$CACHE_PATH" + images_file="$CACHE_PATH/images.txt" + : > "$images_file" + + # Only cache images that were actually pulled for this job's services. + while IFS= read -r image; do + if docker image inspect "$image" >/dev/null 2>&1; then + printf '%s\n' "$image" >> "$images_file" + fi + done < <(docker compose -f "$COMPOSE_FILE" config --images $SERVICES | sort -u) + + if [[ ! -s "$images_file" ]]; then + echo "No Docker images are available to cache." + echo "cacheable=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + mapfile -t images < "$images_file" + docker save --output "$CACHE_PATH/images.tar" "${images[@]}" + echo "cacheable=true" >> "$GITHUB_OUTPUT" + + - name: Save Docker image cache + if: ${{ inputs.cache == 'true' && steps.archive.outputs.cacheable == 'true' }} + uses: actions/cache/save@v5 + with: + path: ${{ inputs.cache_path }} + key: ${{ steps.cache.outputs.key }} diff --git a/.github/actions/docker-compose-stop/action.yml b/.github/actions/docker-compose-stop/action.yml new file mode 100644 index 0000000000..a5bb7cef78 --- /dev/null +++ b/.github/actions/docker-compose-stop/action.yml @@ -0,0 +1,21 @@ +name: Docker Compose Stop +description: Stop Docker Compose services. + +inputs: + compose_file: + description: Docker Compose file to use. + required: true + flags: + description: Additional options to pass to docker compose down. + default: "" + +runs: + using: composite + steps: + - name: Stop services + shell: bash + env: + COMPOSE_FILE: ${{ inputs.compose_file }} + FLAGS: ${{ inputs.flags }} + run: | + docker compose -f "$COMPOSE_FILE" down $FLAGS diff --git a/.github/actions/post-test-reporting/action.yml b/.github/actions/post-test-reporting/action.yml index 08d18faa9c..7b9e02d3e5 100644 --- a/.github/actions/post-test-reporting/action.yml +++ b/.github/actions/post-test-reporting/action.yml @@ -1,18 +1,18 @@ name: Post Test Reporting description: Generate test reports and upload post-test artifacts inputs: - artifact-name-suffix: + artifact_name_suffix: description: Suffix to use for uploaded artifact names required: true - codecov-flag: + codecov_flag: description: Codecov flag for coverage and test result uploads required: false default: "" - debug-log-path: + debug_log_path: description: Optional debug log path to upload required: false default: "" - job-name: + job_name: description: Exact job name to resolve for artifact names required: true runs: @@ -44,8 +44,8 @@ runs: if: ${{ !cancelled() }} shell: bash env: - CODECOV_FLAG: ${{ inputs.codecov-flag }} - ARTIFACT_NAME_SUFFIX: ${{ inputs.artifact-name-suffix }} + CODECOV_FLAG: ${{ inputs.codecov_flag }} + ARTIFACT_NAME_SUFFIX: ${{ inputs.artifact_name_suffix }} run: | flag="$CODECOV_FLAG" if [ -z "$flag" ]; then @@ -74,7 +74,7 @@ runs: id: get_job_id uses: ./.github/actions/get-job-id with: - job_name: ${{ inputs.job-name }} + job_name: ${{ inputs.job_name }} run_id: ${{ github.run_id }} - name: Upload test results to GitHub @@ -82,7 +82,7 @@ runs: uses: actions/upload-artifact@v6 if: ${{ !cancelled() }} with: - name: junit-xml--${{ github.run_id }}--${{ steps.get_job_id.outputs.job_id }}--${{ github.run_attempt }}--${{ inputs.artifact-name-suffix }} + name: junit-xml--${{ github.run_id }}--${{ steps.get_job_id.outputs.job_id }}--${{ github.run_attempt }}--${{ inputs.artifact_name_suffix }} path: ./.testoutput/junit.*.xml include-hidden-files: true retention-days: 28 @@ -91,16 +91,16 @@ runs: uses: actions/upload-artifact@v6 if: ${{ !cancelled() }} with: - name: test-summary-json--${{ github.run_id }}--${{ steps.get_job_id.outputs.job_id }}--${{ github.run_attempt }}--${{ inputs.artifact-name-suffix }} + name: test-summary-json--${{ github.run_id }}--${{ steps.get_job_id.outputs.job_id }}--${{ github.run_attempt }}--${{ inputs.artifact_name_suffix }} path: ./.testoutput/test-summary.json if-no-files-found: ignore retention-days: 28 - name: Upload debug logs uses: actions/upload-artifact@v6 - if: ${{ !cancelled() && inputs.debug-log-path != '' }} + if: ${{ !cancelled() && inputs.debug_log_path != '' }} with: - name: debug-logs--${{ github.run_id }}--${{ steps.get_job_id.outputs.job_id }}--${{ github.run_attempt }}--${{ inputs.artifact-name-suffix }} - path: ${{ inputs.debug-log-path }} + name: debug-logs--${{ github.run_id }}--${{ steps.get_job_id.outputs.job_id }}--${{ github.run_attempt }}--${{ inputs.artifact_name_suffix }} + path: ${{ inputs.debug_log_path }} if-no-files-found: ignore retention-days: 14 diff --git a/.github/workflows/check-pr-placeholders.yml b/.github/workflows/check-pr-placeholders.yml index 51ade33a61..807e5f74b2 100644 --- a/.github/workflows/check-pr-placeholders.yml +++ b/.github/workflows/check-pr-placeholders.yml @@ -13,7 +13,7 @@ jobs: steps: - name: Validate PR description for placeholder lines or empty sections - uses: actions/github-script@v8 + uses: actions/github-script@v9 with: script: | const pr = await github.rest.pulls.get({ diff --git a/.github/workflows/promote-docker-image.yml b/.github/workflows/promote-docker-image.yml index e545aa7502..37859b4567 100644 --- a/.github/workflows/promote-docker-image.yml +++ b/.github/workflows/promote-docker-image.yml @@ -21,6 +21,9 @@ on: DOCKERHUB_TOKEN: required: true +permissions: + contents: read + jobs: validate-inputs: runs-on: ubuntu-latest @@ -30,7 +33,7 @@ jobs: steps: - name: Validate input tags id: validate - uses: actions/github-script@v8 + uses: actions/github-script@v9 env: SOURCE_TAG: ${{ inputs.source-tag }} TARGET_TAGS: ${{ inputs.target-tags }} @@ -94,7 +97,7 @@ jobs: sudo mv crane /usr/local/bin/ - name: Promote image - uses: actions/github-script@v8 + uses: actions/github-script@v9 env: SOURCE_TAG: ${{ needs.validate-inputs.outputs.source-tag-safe }} TARGET_TAGS: ${{ needs.validate-inputs.outputs.target-tags-safe }} diff --git a/.github/workflows/run-single-test.yml b/.github/workflows/run-single-test.yml index b033f8a994..a0040bdb41 100644 --- a/.github/workflows/run-single-test.yml +++ b/.github/workflows/run-single-test.yml @@ -124,6 +124,7 @@ jobs: env: PERSISTENCE_TYPE: ${{ matrix.persistence_type }} PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }} + CONTAINERS: ${{ join(matrix.containers, ' ') }} steps: - uses: actions/checkout@v6 if: ${{ contains(fromJSON(inputs.test_dbs), matrix.name) }} @@ -132,12 +133,18 @@ jobs: ref: ${{ env.COMMIT }} - name: Start containerized dependencies - if: ${{ contains(fromJSON(inputs.test_dbs), matrix.name) && toJson(matrix.containers) != '[]' }} - uses: hoverkraft-tech/compose-action@v2.0.1 + if: ${{ contains(fromJSON(inputs.test_dbs), matrix.name) }} + uses: ./.github/actions/docker-compose-start + with: + compose_file: ${{ env.DOCKER_COMPOSE_FILE }} + services: ${{ env.CONTAINERS }} + + - name: Wait for containerized dependencies to be healthy + if: ${{ contains(fromJSON(inputs.test_dbs), matrix.name) }} + uses: ./.github/actions/docker-compose-healthy with: - compose-file: ${{ env.DOCKER_COMPOSE_FILE }} - services: "${{ join(matrix.containers, '\n') }}" - down-flags: -v + compose_file: ${{ env.DOCKER_COMPOSE_FILE }} + services: ${{ env.CONTAINERS }} - uses: actions/setup-go@v6 if: ${{ contains(fromJSON(inputs.test_dbs), matrix.name) }} @@ -151,3 +158,10 @@ jobs: env: TEST_ARGS: "-run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}" TEST_TIMEOUT: "${{ inputs.timeout_minutes }}m" + + - name: Stop containerized dependencies + if: ${{ always() && contains(fromJSON(inputs.test_dbs), matrix.name) }} + uses: ./.github/actions/docker-compose-stop + with: + compose_file: ${{ env.DOCKER_COMPOSE_FILE }} + flags: -v diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 307e7dd4b5..680159e2c1 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -324,13 +324,15 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: - job-name: Unit test - artifact-name-suffix: unit-test + job_name: Unit test + artifact_name_suffix: unit-test integration-test: name: Integration test needs: [pre-build, test-setup] runs-on: ${{ needs.test-setup.outputs.runner_arm }} + env: + CONTAINERS: cassandra mysql postgresql steps: - uses: actions/checkout@v6 with: @@ -338,14 +340,10 @@ jobs: ref: ${{ env.COMMIT }} - name: Start containerized dependencies - uses: ./.github/actions/docker-compose-pull + uses: ./.github/actions/docker-compose-start with: - compose-file: ${{ env.DOCKER_COMPOSE_FILE }} - services: | - cassandra - mysql - postgresql - down-flags: -v + compose_file: ${{ env.DOCKER_COMPOSE_FILE }} + services: ${{ env.CONTAINERS }} - uses: actions/setup-go@v6 with: @@ -365,10 +363,10 @@ jobs: key: go-${{ runner.os }}${{ runner.arch }}-build-${{ env.COMMIT }} - name: Wait for containerized dependencies to be healthy - run: | - # Word splitting is intentional here. - # shellcheck disable=SC2046 - docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} up --wait $(docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} ps --services) + uses: ./.github/actions/docker-compose-healthy + with: + compose_file: ${{ env.DOCKER_COMPOSE_FILE }} + services: ${{ env.CONTAINERS }} - name: Run integration test timeout-minutes: 15 @@ -380,13 +378,15 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: - job-name: Integration test - artifact-name-suffix: integration-test + job_name: Integration test + artifact_name_suffix: integration-test - - name: Tear down docker compose + - name: Stop containerized dependencies if: ${{ always() }} - run: | - docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v + uses: ./.github/actions/docker-compose-stop + with: + compose_file: ${{ env.DOCKER_COMPOSE_FILE }} + flags: -v # Root job name includes matrix details so it is unique per job variant. # This MUST stay in sync with the `job_name` passed to the job-id action below. @@ -403,23 +403,19 @@ jobs: PERSISTENCE_TYPE: ${{ matrix.persistence_type }} PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }} TEST_TIMEOUT: ${{ matrix.test_timeout }} + CONTAINERS: ${{ join(matrix.containers, ' ') }} steps: - - uses: ScribeMD/docker-cache@0.3.7 - with: - key: docker-${{ runner.os }}${{ runner.arch }}-${{ hashFiles(env.DOCKER_COMPOSE_FILE) }} - - uses: actions/checkout@v6 with: token: ${{ secrets.GITHUB_TOKEN }} ref: ${{ env.COMMIT }} - name: Start containerized dependencies - if: ${{ toJson(matrix.containers) != '[]' }} - uses: ./.github/actions/docker-compose-pull + uses: ./.github/actions/docker-compose-start with: - compose-file: ${{ env.DOCKER_COMPOSE_FILE }} - services: "${{ join(matrix.containers, '\n') }}" - down-flags: -v + compose_file: ${{ env.DOCKER_COMPOSE_FILE }} + services: ${{ env.CONTAINERS }} + cache: true - uses: actions/setup-go@v6 with: @@ -442,11 +438,10 @@ jobs: run: echo "::notice::${{ matrix.display_name == 'smoke' && 'This is a smoke test. Add the test-all-dbs label to run all tests on all DBs.' || needs.test-setup.outputs.full_test_reason }}" - name: Wait for containerized dependencies to be healthy - if: ${{ toJson(matrix.containers) != '[]' }} - run: | - # Word splitting is intentional here. - # shellcheck disable=SC2046 - docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} up --wait $(docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} ps --services) + uses: ./.github/actions/docker-compose-healthy + with: + compose_file: ${{ env.DOCKER_COMPOSE_FILE }} + services: ${{ env.CONTAINERS }} - name: Run functional test timeout-minutes: ${{ matrix.github_timeout }} @@ -468,10 +463,17 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: - job-name: Functional test (${{ matrix.name }}, ${{ matrix.display_name }}) - artifact-name-suffix: ${{ matrix.name }}--${{ matrix.display_name }}--functional-test - codecov-flag: functional-test - debug-log-path: ${{ github.workspace }}/.testoutput/debug.log + job_name: Functional test (${{ matrix.name }}, ${{ matrix.display_name }}) + artifact_name_suffix: ${{ matrix.name }}--${{ matrix.display_name }}--functional-test + codecov_flag: functional-test + debug_log_path: ${{ github.workspace }}/.testoutput/debug.log + + - name: Stop containerized dependencies + if: always() + uses: ./.github/actions/docker-compose-stop + with: + compose_file: ${{ env.DOCKER_COMPOSE_FILE }} + flags: -v mixed-brain-test: name: Mixed brain test @@ -484,11 +486,10 @@ jobs: ref: ${{ env.COMMIT }} - name: Start PostgreSQL - uses: ./.github/actions/docker-compose-pull + uses: ./.github/actions/docker-compose-start with: - compose-file: ${{ env.DOCKER_COMPOSE_FILE }} + compose_file: ${{ env.DOCKER_COMPOSE_FILE }} services: postgresql - down-flags: -v - uses: actions/setup-go@v6 with: @@ -534,6 +535,13 @@ jobs: if: always() run: cat .testoutput/mixedbrain_omes.log || true + - name: Stop containerized dependencies + if: always() + uses: ./.github/actions/docker-compose-stop + with: + compose_file: ${{ env.DOCKER_COMPOSE_FILE }} + flags: -v + test-status: if: always() name: Test Status