diff --git a/.github/actions/clean-disk/action.yml b/.github/actions/clean-disk/action.yml new file mode 100644 index 0000000000000..fae68b86da9e8 --- /dev/null +++ b/.github/actions/clean-disk/action.yml @@ -0,0 +1,73 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: clean disk +description: makes some more space available on the disk by removing files +inputs: + mode: + description: "Use 'full' to clean as much as possible" + required: false +runs: + using: composite + steps: + - run: | + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + directories=(/usr/local/lib/android /opt/ghc) + if [[ "${{ inputs.mode }}" == "full" ]]; then + # remove these directories only when mode is 'full' + directories+=(/usr/share/dotnet /opt/hostedtoolcache/CodeQL) + fi + emptydir=/tmp/empty$$/ + mkdir $emptydir + echo "::group::Available diskspace" + time df -BM / /mnt + echo "::endgroup::" + for directory in "${directories[@]}"; do + echo "::group::Removing $directory" + # fast way to delete a lot of files on linux + time sudo eatmydata rsync -a --delete $emptydir ${directory}/ + time sudo eatmydata rm -rf ${directory} + time df -BM / /mnt + echo "::endgroup::" + done + if [[ "${{ inputs.mode }}" == "full" ]]; then + echo "::group::Moving /var/lib/docker to /mnt/docker" + sudo systemctl stop docker + if sudo test ! -f /etc/docker/daemon.json; then + # create daemon.json file + echo '{"data-root": "/mnt/docker"}' | sudo tee /etc/docker/daemon.json + else + # modify existing daemon.json file + sudo jq '.["data-root"]="/mnt/docker"' /etc/docker/daemon.json | sudo tee /tmp/daemon_temp.json$$ + sudo mv /tmp/daemon_temp.json$$ /etc/docker/daemon.json + fi + sudo mv /var/lib/docker /mnt/docker + sudo systemctl start docker + time df -BM / /mnt + echo "::endgroup::" + fi + echo "::group::Cleaning apt state" + time sudo bash -c "apt-get clean; apt-get autoclean; apt-get -y --purge autoremove" + time df -BM / /mnt + echo "::endgroup::" + fi + echo "::group::Available diskspace" + time df -BM / /mnt + echo "::endgroup::" + shell: bash diff --git a/.github/actions/copy-test-reports/action.yml b/.github/actions/copy-test-reports/action.yml new file mode 100644 index 0000000000000..7c49afacc60dc --- /dev/null +++ b/.github/actions/copy-test-reports/action.yml @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Copy test reports +description: Aggregates all test reports to ./test-reports and ./surefire-reports directories +runs: + using: composite + steps: + - run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh move_test_reports + shell: bash diff --git a/.github/actions/ssh-access/action.yml b/.github/actions/ssh-access/action.yml new file mode 100644 index 0000000000000..b4ad4a8555efc --- /dev/null +++ b/.github/actions/ssh-access/action.yml @@ -0,0 +1,161 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: ssh access +description: Sets up SSH access to build VM with upterm +inputs: + action: + description: | + Action to perform: options are "start" and "wait" + "start" will install, configure and start upterm. + "wait" will wait until a connection is established to upterm and will continue to wait until the session is closed. + required: false + default: 'start' + limit-access-to-actor: + description: 'If only the public SSH keys of the user triggering the workflow should be authorized' + required: false + default: 'false' + limit-access-to-users: + description: 'If only the public SSH keys of the listed GitHub users should be authorized. Comma separate list of GitHub user names.' + required: false + default: '' + secure-access: + description: | + Set to false for allowing public access when limit-access-to-actor and limit-access-to-users are unset. + required: false + default: 'true' + timeout: + description: 'When action=wait, the timeout in seconds to wait for the user to connect' + required: false + default: '300' +runs: + using: composite + steps: + - run: | + if [[ "${{ inputs.action }}" == "start" ]]; then + echo "::group::Installing upterm & tmux" + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + # install upterm + curl -sL https://github.com/owenthereal/upterm/releases/download/v0.7.6/upterm_linux_amd64.tar.gz | tar zxvf - -C /tmp upterm && sudo install /tmp/upterm /usr/local/bin/ && rm -rf /tmp/upterm + + # install tmux if it's not present + if ! command -v tmux &>/dev/null; then + sudo apt-get -y install tmux + fi + elif [[ "$OSTYPE" == "darwin"* ]]; then + brew install owenthereal/upterm/upterm + # install tmux if it's not present + if ! command -v tmux &>/dev/null; then + brew install tmux + fi + else + echo "Unsupported $OSTYPE" + exit 0 + fi + echo '::endgroup::' + echo "::group::Configuring ssh and ssh keys" + # generate ssh key + mkdir -p ~/.ssh + chmod 0700 ~/.ssh + if [ ! -f ~/.ssh/id_rsa ]; then + ssh-keygen -q -t rsa -N "" -f ~/.ssh/id_rsa + fi + if [ ! -f ~/.ssh/id_ed25519 ]; then + ssh-keygen -q -t ed25519 -N "" -f ~/.ssh/id_ed25519 + fi + # configure ssh + echo -e "Host *\nStrictHostKeyChecking no\nCheckHostIP no\nTCPKeepAlive yes\nServerAliveInterval 30\nServerAliveCountMax 180\nVerifyHostKeyDNS yes\nUpdateHostKeys yes\n" > ~/.ssh/config + # Auto-generate ~/.ssh/known_hosts by attempting connection to uptermd.upterm.dev + ssh -i ~/.ssh/id_ed25519 uptermd.upterm.dev || true + # @cert-authority entry is a mandatory entry when connecting to upterm. generate the entry based on the known_hosts entry key + cat <(cat ~/.ssh/known_hosts | awk '{ print "@cert-authority * " $2 " " $3 }') >> ~/.ssh/known_hosts + authorizedKeysParameter="" + authorizedKeysFile=${HOME}/.ssh/authorized_keys + if [[ "${{ inputs.secure-access }}" != "false" ]]; then + ssh-keygen -q -t ed25519 -N "$(echo $RANDOM | md5sum | awk '{ print $1 }')" -C "Prevent public access" -f /tmp/dummykey$$ + cat /tmp/dummykey$$.pub >> $authorizedKeysFile + rm /tmp/dummykey$$ /tmp/dummykey$$.pub + fi + limit_access_to_actor="${{ inputs.limit-access-to-actor }}" + if [[ "${limit_access_to_actor}" == "true" ]]; then + echo "Adding ${GITHUB_ACTOR} to allowed users (identified by ssh key registered in GitHub)" + curl -s https://github.com/${GITHUB_ACTOR}.keys >> $authorizedKeysFile + fi + limit_access_to_users="${{ inputs.limit-access-to-users }}" + for github_user in ${limit_access_to_users//,/ }; do + if [[ -n "${github_user}" ]]; then + echo "Adding ${github_user} to allowed users (identified by ssh key registered in GitHub)" + curl -s https://github.com/${github_user}.keys >> $authorizedKeysFile + fi + done + if [ -f $authorizedKeysFile ]; then + chmod 0600 $authorizedKeysFile + authorizedKeysParameter="-a $authorizedKeysFile" + echo -e "Using $authorizedKeysFile\nContent:\n---------------------------" + cat $authorizedKeysFile + echo "---------------------------" + fi + echo '::endgroup::' + echo "::group::Starting terminal session and connecting to server" + tmux new -d -s upterm-wrapper -x 132 -y 43 "upterm host ${authorizedKeysParameter} --force-command 'tmux attach -t upterm' -- tmux new -s upterm -x 132 -y 43" + sleep 2 + tmux send-keys -t upterm-wrapper q C-m + sleep 1 + tmux set -t upterm-wrapper window-size largest + tmux set -t upterm window-size largest + echo '::endgroup::' + echo -e "\nSSH connection information" + # wait up to 10 seconds for upterm admin socket to appear + for i in {1..10}; do + ADMIN_SOCKET=$(find $HOME/.upterm -name "*.sock") + if [ ! -S "$ADMIN_SOCKET" ]; then + echo "Waiting for upterm admin socket to appear in ~/.upterm/*.sock ..." + sleep 1 + else + echo "upterm admin socket available in $ADMIN_SOCKET" + break + fi + done + shopt -s nullglob + upterm session current --admin-socket ~/.upterm/*.sock || { + echo "Starting upterm failed." + exit 0 + } + elif [[ "${{ inputs.action }}" == "wait" ]]; then + # only wait if upterm was installed + if command -v upterm &>/dev/null; then + shopt -s nullglob + echo "SSH connection information" + upterm session current --admin-socket ~/.upterm/*.sock || { + echo "upterm isn't running. Not waiting any longer." + exit 0 + } + timeout=${{ inputs.timeout }} + echo "Waiting $timeout seconds..." + sleep $timeout + echo "Keep waiting as long as there's a connected session" + while upterm session current --admin-socket ~/.upterm/*.sock|grep Connected &>/dev/null; do + sleep 30 + done + echo "No session is connected. Not waiting any longer." + else + echo "upterm isn't installed" + fi + fi + shell: bash diff --git a/.github/actions/tune-runner-vm/action.yml b/.github/actions/tune-runner-vm/action.yml index 4c3d5b4e3892e..7e5f77f9a83fe 100644 --- a/.github/actions/tune-runner-vm/action.yml +++ b/.github/actions/tune-runner-vm/action.yml @@ -24,6 +24,7 @@ runs: steps: - run: | if [[ "$OSTYPE" == "linux-gnu"* ]]; then + echo "::group::Configure and tune OS" # Ensure that reverse lookups for current hostname are handled properly # Add the current IP address, long hostname and short hostname record to /etc/hosts file echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts @@ -32,18 +33,23 @@ runs: # consumption is high. # Set vm.swappiness=1 to avoid swapping and allow high RAM usage echo 1 | sudo tee /proc/sys/vm/swappiness - # Set swappiness to 1 for all cgroups and sub-groups - for swappiness_dir in /sys/fs/cgroup/memory/*/ /sys/fs/cgroup/memory/*/*/; do - if [ -d "swappiness_dir" ]; then - echo 1 | sudo tee $(swappiness_dir)memory.swappiness > /dev/null - fi - done + ( + shopt -s nullglob + # Set swappiness to 1 for all cgroups and sub-groups + for swappiness_file in /sys/fs/cgroup/memory/*/memory.swappiness /sys/fs/cgroup/memory/*/*/memory.swappiness; do + echo 1 | sudo tee $swappiness_file > /dev/null + done + ) || true # use "madvise" Linux Transparent HugePages (THP) setting # https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html # "madvise" is generally a better option than the default "always" setting + # Based on Azul instructions from https://docs.azul.com/prime/Enable-Huge-Pages#transparent-huge-pages-thp echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled - + echo advise | sudo tee /sys/kernel/mm/transparent_hugepage/shmem_enabled + echo defer+madvise | sudo tee /sys/kernel/mm/transparent_hugepage/defrag + echo 1 | sudo tee /sys/kernel/mm/transparent_hugepage/khugepaged/defrag + # tune filesystem mount options, https://www.kernel.org/doc/Documentation/filesystems/ext4.txt # commit=999999, effectively disables automatic syncing to disk (default is every 5 seconds) # nobarrier/barrier=0, loosen data consistency on system crash (no negative impact to empheral CI nodes) @@ -80,11 +86,17 @@ runs: echo '::endgroup::' # show memory + echo "::group::Available Memory" free -m + echo '::endgroup::' # show disk - df -h + echo "::group::Available diskspace" + df -BM + echo "::endgroup::" # show cggroup - echo "/actions_job cgroup settings:" - sudo cgget actions_job + echo "::group::Cgroup settings for current cgroup $CURRENT_CGGROUP" + CURRENT_CGGROUP=$(cat /proc/self/cgroup | grep '0::' | awk -F: '{ print $3 }') + sudo cgget -a $CURRENT_CGGROUP || true + echo '::endgroup::' fi shell: bash diff --git a/.github/actions/upload-coverage/action.yml b/.github/actions/upload-coverage/action.yml new file mode 100644 index 0000000000000..0ba73e94a8389 --- /dev/null +++ b/.github/actions/upload-coverage/action.yml @@ -0,0 +1,100 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Upload to Codecov with retries +description: | + Checks that the current repository is public and then + uploads to codecov with multiple retries as a workaround + for these issues + - https://github.com/codecov/codecov-action/issues/598 + - https://github.com/codecov/codecov-action/issues/837 +inputs: + flags: + # see https://github.com/codecov/codecov-action#arguments + description: 'Flag the upload to group coverage metrics. Multiple flags are separated by a comma.' +runs: + using: composite + steps: + - name: "Check that repository is public" + id: repo-check + shell: bash + run: | + if [[ "${{ github.server_url }}" != "https://github.com" ]]; then + echo "Not using github.com server ('${{ github.server_url }}'). Skipping uploading of coverage metrics." + echo "passed=false" >> $GITHUB_OUTPUT + exit 0 + fi + REPO_URL="${{ github.server_url }}/${{ github.repository }}" + { + # public repository url will respond to http HEAD request + curl -X HEAD -fs "$REPO_URL" && echo "passed=true" >> $GITHUB_OUTPUT + } || { + echo "$REPO_URL isn't a public repository. Skipping uploading of coverage metrics." + echo "passed=false" >> $GITHUB_OUTPUT + } + - name: "Upload to Codecov (attempt #1)" + id: codecov-upload-1 + if: steps.repo-check.outputs.passed == 'true' + uses: codecov/codecov-action@v4 + continue-on-error: true + with: + flags: ${{ inputs.flags }} + fail_ci_if_error: true + verbose: true + - name: "Wait 15 seconds before next attempt" + if: steps.codecov-upload-1.outcome == 'failure' + shell: bash + run: sleep 15 + - name: "Upload to Codecov (attempt #2)" + id: codecov-upload-2 + if: steps.codecov-upload-1.outcome == 'failure' + uses: codecov/codecov-action@v4 + continue-on-error: true + with: + flags: ${{ inputs.flags }} + fail_ci_if_error: true + verbose: true + - name: "Wait 60 seconds before next attempt" + if: steps.codecov-upload-2.outcome == 'failure' + shell: bash + run: sleep 60 + - name: "Upload to Codecov (attempt #3)" + id: codecov-upload-3 + if: steps.codecov-upload-2.outcome == 'failure' + uses: codecov/codecov-action@v4 + # fail on last attempt + continue-on-error: false + with: + flags: ${{ inputs.flags }} + fail_ci_if_error: true + verbose: true + - name: "Show link to Codecov report" + shell: bash + run: | + if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then + head_sha=$(jq -r '.pull_request.head.sha' "${GITHUB_EVENT_PATH}") + else + head_sha=$(git rev-parse HEAD) + fi + tee -a "$GITHUB_STEP_SUMMARY" < 0 }}" - - - name: Cache Maven dependencies - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-all- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - - name: build package - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B clean install -DskipTests -T 1C diff --git a/.github/workflows/ci-cancel-duplicate-workflows.yaml b/.github/workflows/ci-cancel-duplicate-workflows.yaml deleted file mode 100644 index d48a1210c2411..0000000000000 --- a/.github/workflows/ci-cancel-duplicate-workflows.yaml +++ /dev/null @@ -1,189 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Cancel duplicate workflows -on: - workflow_run: - # this could be any workflow that is always executed by PUSH/PR operation - workflows: ["CI - Unit"] - types: ['requested'] - -jobs: - - cancel-workflow-runs: - runs-on: ubuntu-22.04 - steps: - # the potiuk/cancel-workflow-run action has been allow-listed by - # the Apache Infrastructure - - name: cancel duplicate ci-build-macos.yaml - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-build-macos.yaml - - name: cancel duplicate ci-go-functions-style.yaml - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-go-functions-style.yaml - - name: cancel duplicate ci-go-functions-test.yaml - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-go-functions-test.yaml - - name: cancel duplicate ci-integration-backwards-compatibility.yaml - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-integration-backwards-compatibility.yaml - - name: cancel duplicate ci-integration-cli.yaml - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-integration-cli.yaml - - name: cancel duplicate ci-integration-function-state.yaml - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-integration-function-state.yaml - - name: cancel duplicate ci-integration-messaging.yaml - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-integration-messaging.yaml - - name: cancel duplicate ci-integration-process.yaml - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-integration-process.yaml - - name: cancel duplicate ci-integration-schema.yaml - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-integration-schema.yaml - - name: cancel duplicate ci-integration-sql.yaml - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-integration-sql.yaml - - name: cancel duplicate ci-integration-standalone.yaml - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-integration-standalone.yaml - - name: cancel duplicate ci-integration-thread.yaml - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-integration-thread.yaml - - name: cancel duplicate - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-integration-tiered-filesystem.yaml - - name: cancel duplicate - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-integration-tiered-filesystem.yaml - - name: cancel duplicate ci-license.yaml - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-license.yaml - - name: cancel duplicate ci-integration-tiered-jcloud.yaml - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-integration-tiered-jcloud.yaml - - name: cancel duplicate ci-integration-transaction.yaml - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-integration-transaction.yaml - - name: cancel duplicate ci-pulsar-website-build.yaml - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-pulsar-website-build.yaml - - name: cancel duplicate ci-shade-test.yaml - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-shade-test.yaml - - name: cancel duplicate ci-unit.yaml - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-unit.yaml - - name: cancel duplicate ci-unit-broker-broker-gp1.yaml - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-unit-broker-broker-gp1.yaml - - name: cancel duplicate ci-unit-broker-broker-gp2.yaml - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-unit-broker-broker-gp2.yaml - - name: cancel duplicate ci-unit-broker-client-api.yaml - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-unit-broker-client-api.yaml - - name: cancel duplicate ci-unit-broker-client-impl.yaml - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-unit-broker-client-impl.yaml - - name: cancel duplicate ci-unit-broker-other.yaml - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-unit-broker-other.yaml - - name: cancel duplicate ci-unit-proxy.yaml - uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: ci-unit-proxy.yaml diff --git a/.github/workflows/ci-documentbot.yaml b/.github/workflows/ci-documentbot.yml similarity index 75% rename from .github/workflows/ci-documentbot.yaml rename to .github/workflows/ci-documentbot.yml index ee38f539038c7..1006661b60d2a 100644 --- a/.github/workflows/ci-documentbot.yaml +++ b/.github/workflows/ci-documentbot.yml @@ -28,29 +28,18 @@ on: - unlabeled concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.number }} cancel-in-progress: true jobs: label: - if: ${{ github.repository == 'apache/pulsar' }} + if: (github.repository == 'apache/pulsar') && (github.event.pull_request.state == 'open') permissions: - pull-requests: write + pull-requests: write runs-on: ubuntu-22.04 steps: - - name: Checkout action - uses: actions/checkout@v3 - with: - repository: apache/pulsar-test-infra - ref: master - - - name: Set up Go - uses: actions/setup-go@v3 - with: - go-version: 1.18 - - name: Labeling - uses: ./docbot + uses: apache/pulsar-test-infra/docbot@master env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} LABEL_WATCH_LIST: 'doc,doc-required,doc-not-needed,doc-complete' diff --git a/.github/workflows/ci-go-functions-test.yaml b/.github/workflows/ci-go-functions-test.yaml deleted file mode 100644 index 265688a24fefc..0000000000000 --- a/.github/workflows/ci-go-functions-test.yaml +++ /dev/null @@ -1,76 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Go Functions Tests -on: - pull_request: - branches: - - branch-* - paths: - - '.github/workflows/**' - - 'pulsar-function-go/**' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - - go-functions-tests: - name: Run Go Tests - strategy: - matrix: - go-version: [1.11.x, 1.12.x, 1.13.x, 1.14.x] - platform: [ubuntu-22.04] - runs-on: ${{ matrix.platform }} - timeout-minutes: 120 - - steps: - - name: Check out code into the Go module directory - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Set up Go - uses: actions/setup-go@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - go-version: ${{ matrix.go-version }} - id: go - - - name: Run tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: | - cd pulsar-function-go - go test -v $(go list ./... | grep -v examples) diff --git a/.github/workflows/ci-go-functions-style.yaml b/.github/workflows/ci-go-functions.yaml similarity index 65% rename from .github/workflows/ci-go-functions-style.yaml rename to .github/workflows/ci-go-functions.yaml index 375982279402f..e2529dbe07339 100644 --- a/.github/workflows/ci-go-functions-style.yaml +++ b/.github/workflows/ci-go-functions.yaml @@ -17,14 +17,15 @@ # under the License. # -name: CI - Go Functions style check +name: CI - Go Functions on: pull_request: branches: - - branch-* + - branch-as-* paths: - '.github/workflows/**' - 'pulsar-function-go/**' + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -34,8 +35,42 @@ env: MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 jobs: - check-style: + preconditions: + name: Preconditions + runs-on: ubuntu-22.04 + outputs: + docs_only: ${{ steps.check_changes.outputs.docs_only }} + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Detect changed files + id: changes + uses: apache/pulsar-test-infra/paths-filter@master + with: + filters: .github/changes-filter.yaml + list-files: csv + + - name: Check changed files + id: check_changes + run: | + if [[ "${GITHUB_EVENT_NAME}" != "schedule" ]]; then + echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT + else + echo docs_only=false >> $GITHUB_OUTPUT + fi + - name: Check if the PR has been approved for testing + if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }} + env: + GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} + GITHUB_TOKEN: ${{ github.token }} + run: | + build/pulsar_ci_tool.sh check_ready_to_test + + check-style: + needs: preconditions + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} name: Go ${{ matrix.go-version }} Functions style check runs-on: ubuntu-22.04 strategy: @@ -44,44 +79,31 @@ jobs: steps: - name: Check out code into the Go module directory - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - name: Set up Go - uses: actions/setup-go@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} + uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} id: go - name: InstallTool - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} run: | cd pulsar-function-go wget -O - -q https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v1.18.0 ./bin/golangci-lint --version - name: Build - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} run: | cd pulsar-function-go + export GO111MODULE=on go build ./... - - name: CheckStyle - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} + - name: Test run: | cd pulsar-function-go + export GO111MODULE=on ./bin/golangci-lint run -c ./golangci.yml ./pf diff --git a/.github/workflows/ci-integration-backwards-compatibility.yaml b/.github/workflows/ci-integration-backwards-compatibility.yaml deleted file mode 100644 index d74a36d8c97de..0000000000000 --- a/.github/workflows/ci-integration-backwards-compatibility.yaml +++ /dev/null @@ -1,130 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Integration - Backwards Compatibility -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/ - -jobs: - - backwards-compatibility: - name: - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: clean disk - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: | - echo "::group::Available diskspace" - time df -BM / /mnt - echo "::endgroup::" - sudo swapoff -a - sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL - echo "::group::Cleaning apt state" - time sudo bash -c "apt-get clean; apt-get autoclean; apt-get -y --purge autoremove" - time df -BM / /mnt - echo "::endgroup::" - docker rmi $(docker images -q) -f - echo "::group::Available diskspace" - time df -BM / /mnt - echo "::endgroup::" - df -h - - - name: run install by skip tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -q -B -ntp clean install -DskipTests -T 1C - - - name: build pulsar image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build pulsar-all image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build artifacts and docker image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests - - - name: run integration tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh BACKWARDS_COMPAT - - - name: Upload container logs - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: container-logs - path: tests/integration/target/container-logs - - - name: Upload surefire-reports - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: surefire-reports - path: tests/integration/target/surefire-reports diff --git a/.github/workflows/ci-integration-cli.yaml b/.github/workflows/ci-integration-cli.yaml deleted file mode 100644 index 50ff739e3be53..0000000000000 --- a/.github/workflows/ci-integration-cli.yaml +++ /dev/null @@ -1,120 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Integration - Cli -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/ - -jobs: - - cli: - name: - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: clean disk - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - sudo apt clean - docker rmi $(docker images -q) -f - df -h - - - name: run install by skip tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -q -B -ntp clean install -DskipTests -T 1C - - - name: build pulsar image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build pulsar-all image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build artifacts and docker image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests - - - name: run integration tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh CLI - - - name: Upload container logs - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: container-logs - path: tests/integration/target/container-logs - - - name: Upload surefire-reports - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: surefire-reports - path: tests/integration/target/surefire-reports diff --git a/.github/workflows/ci-integration-function.yaml b/.github/workflows/ci-integration-function.yaml deleted file mode 100644 index e7831702c27e1..0000000000000 --- a/.github/workflows/ci-integration-function.yaml +++ /dev/null @@ -1,120 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Integration - Function & IO -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/ - -jobs: - - function-state: - name: - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: clean disk - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - sudo apt clean - docker rmi $(docker images -q) -f - df -h - - - name: run install by skip tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -q -B -ntp clean install -DskipTests -T 1C - - - name: build pulsar image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build pulsar-all image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build artifacts and docker image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests - - - name: run integration tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh FUNCTION - - - name: Upload container logs - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: container-logs - path: tests/integration/target/container-logs - - - name: Upload surefire-reports - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: surefire-reports - path: tests/integration/target/surefire-reports diff --git a/.github/workflows/ci-integration-messaging.yaml b/.github/workflows/ci-integration-messaging.yaml deleted file mode 100644 index 85b95a3fb5271..0000000000000 --- a/.github/workflows/ci-integration-messaging.yaml +++ /dev/null @@ -1,120 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Integration - Messaging -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/ - -jobs: - - messaging: - name: - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: clean disk - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - sudo apt clean - docker rmi $(docker images -q) -f - df -h - - - name: run install by skip tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -q -B -ntp clean install -DskipTests -T 1C - - - name: build pulsar image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build pulsar-all image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build artifacts and docker image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests - - - name: run integration tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh MESSAGING - - - name: Upload container logs - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: container-logs - path: tests/integration/target/container-logs - - - name: Upload surefire-reports - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: surefire-reports - path: tests/integration/target/surefire-reports diff --git a/.github/workflows/ci-integration-process.yaml b/.github/workflows/ci-integration-process.yaml deleted file mode 100644 index 922065a5356ec..0000000000000 --- a/.github/workflows/ci-integration-process.yaml +++ /dev/null @@ -1,124 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Integration - Process -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/ - -jobs: - - process: - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: clean disk - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - sudo apt clean - docker rmi $(docker images -q) -f - df -h - - - name: run install by skip tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -q -B -ntp clean install -DskipTests -T 1C - - - name: build pulsar image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build pulsar-all image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build artifacts and docker image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests - - - name: run integration tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh PULSAR_CONNECTORS_PROCESS - - - name: Log dmesg when failed - if: ${{ failure() }} - continue-on-error: true - run: sudo dmesg - - - name: Upload container logs - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: container-logs - path: tests/integration/target/container-logs - - - name: Upload surefire-reports - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: surefire-reports - path: tests/integration/target/surefire-reports diff --git a/.github/workflows/ci-integration-pulsar-io-ora.yaml b/.github/workflows/ci-integration-pulsar-io-ora.yaml deleted file mode 100644 index 4ebc69717d022..0000000000000 --- a/.github/workflows/ci-integration-pulsar-io-ora.yaml +++ /dev/null @@ -1,130 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Integration - Pulsar-IO Oracle Source -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/ - -jobs: - - pulsar-io: - name: - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: clean disk - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: | - echo "::group::Available diskspace" - time df -BM / /mnt - echo "::endgroup::" - sudo swapoff -a - sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL - echo "::group::Cleaning apt state" - time sudo bash -c "apt-get clean; apt-get autoclean; apt-get -y --purge autoremove" - time df -BM / /mnt - echo "::endgroup::" - docker rmi $(docker images -q) -f - echo "::group::Available diskspace" - time df -BM / /mnt - echo "::endgroup::" - df -h - - - name: run install by skip tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -q -B -ntp clean install -DskipTests -T 1C - - - name: build pulsar image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build pulsar-all image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build artifacts and docker image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests - - - name: run integration tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh PULSAR_IO_ORA - - - name: Upload container logs - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: container-logs - path: tests/integration/target/container-logs - - - name: Upload surefire-reports - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: surefire-reports - path: tests/integration/target/surefire-reports diff --git a/.github/workflows/ci-integration-pulsar-io.yaml b/.github/workflows/ci-integration-pulsar-io.yaml deleted file mode 100644 index d92ab1f04e0b3..0000000000000 --- a/.github/workflows/ci-integration-pulsar-io.yaml +++ /dev/null @@ -1,130 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Integration - Pulsar-IO Sinks and Sources -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/ - -jobs: - - pulsar-io: - name: - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: clean disk - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: | - echo "::group::Available diskspace" - time df -BM / /mnt - echo "::endgroup::" - sudo swapoff -a - sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL - echo "::group::Cleaning apt state" - time sudo bash -c "apt-get clean; apt-get autoclean; apt-get -y --purge autoremove" - time df -BM / /mnt - echo "::endgroup::" - docker rmi $(docker images -q) -f - echo "::group::Available diskspace" - time df -BM / /mnt - echo "::endgroup::" - df -h - - - name: run install by skip tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -q -B -ntp clean install -DskipTests -T 1C - - - name: build pulsar image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build pulsar-all image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build artifacts and docker image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests - - - name: run integration tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh PULSAR_IO - - - name: Upload container logs - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: container-logs - path: tests/integration/target/container-logs - - - name: Upload surefire-reports - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: surefire-reports - path: tests/integration/target/surefire-reports diff --git a/.github/workflows/ci-integration-schema.yaml b/.github/workflows/ci-integration-schema.yaml deleted file mode 100644 index ec0c72a9ddf45..0000000000000 --- a/.github/workflows/ci-integration-schema.yaml +++ /dev/null @@ -1,116 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Integration - Schema -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/ - -jobs: - - schema: - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: clean disk - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - sudo apt clean - docker rmi $(docker images -q) -f - df -h - - - name: run install by skip tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -q -B -ntp clean install -DskipTests -T 1C - - name: build pulsar image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - name: build pulsar-all image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - name: build artifacts and docker image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests - - - name: run integration tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh SCHEMA - - - name: Upload container logs - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: container-logs - path: tests/integration/target/container-logs - - - name: Upload surefire-reports - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: surefire-reports - path: tests/integration/target/surefire-reports diff --git a/.github/workflows/ci-integration-sql.yaml b/.github/workflows/ci-integration-sql.yaml deleted file mode 100644 index 5076dc855f4f2..0000000000000 --- a/.github/workflows/ci-integration-sql.yaml +++ /dev/null @@ -1,125 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Integration - Sql -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/ - MALLOC_ARENA_MAX: "1" - -jobs: - - sql: - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: clean disk - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - sudo apt clean - docker rmi $(docker images -q) -f - df -h - - - name: run install by skip tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -q -B -ntp clean install -DskipTests -T 1C - - - name: build pulsar image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build pulsar-all image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build artifacts and docker pulsar latest test image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests - - - name: run integration tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh SQL - - - name: Log dmesg when failed - if: ${{ failure() }} - continue-on-error: true - run: sudo dmesg - - - name: Upload container logs - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: container-logs - path: tests/integration/target/container-logs - - - name: Upload surefire-reports - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: surefire-reports - path: tests/integration/target/surefire-reports diff --git a/.github/workflows/ci-integration-standalone.yaml b/.github/workflows/ci-integration-standalone.yaml deleted file mode 100644 index a6f6cc7a677e6..0000000000000 --- a/.github/workflows/ci-integration-standalone.yaml +++ /dev/null @@ -1,119 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Integration - Standalone -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/ - -jobs: - - standalone: - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: clean disk - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - sudo apt clean - docker rmi $(docker images -q) -f - df -h - - - name: run install by skip tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -q -B -ntp clean install -DskipTests -T 1C - - - name: build pulsar image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build pulsar-all image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build artifacts and docker image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests - - - name: run integration tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh STANDALONE - - - name: Upload container logs - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: container-logs - path: tests/integration/target/container-logs - - - name: Upload surefire-reports - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: surefire-reports - path: tests/integration/target/surefire-reports diff --git a/.github/workflows/ci-integration-thread.yaml b/.github/workflows/ci-integration-thread.yaml deleted file mode 100644 index c52281fc38953..0000000000000 --- a/.github/workflows/ci-integration-thread.yaml +++ /dev/null @@ -1,119 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Integration - Thread -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/ - -jobs: - - thread: - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: clean disk - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - sudo apt clean - docker rmi $(docker images -q) -f - df -h - - - name: run install by skip tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -q -B -ntp clean install -DskipTests -T 1C - - - name: build pulsar image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build pulsar-all image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build artifacts and docker image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests - - - name: run integration function - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh PULSAR_CONNECTORS_THREAD - - - name: Upload container logs - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: container-logs - path: tests/integration/target/container-logs - - - name: Upload surefire-reports - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: surefire-reports - path: tests/integration/target/surefire-reports diff --git a/.github/workflows/ci-integration-tiered-filesystem.yaml b/.github/workflows/ci-integration-tiered-filesystem.yaml deleted file mode 100644 index eda75299e5467..0000000000000 --- a/.github/workflows/ci-integration-tiered-filesystem.yaml +++ /dev/null @@ -1,119 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Integration - Tiered FileSystem -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/ - -jobs: - - tiered-filesystem: - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: clean disk - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - sudo apt clean - docker rmi $(docker images -q) -f - df -h - - - name: run install by skip tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -q -B -ntp clean install -DskipTests -T 1C - - - name: build pulsar image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build pulsar-all image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build artifacts and docker image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests - - - name: run integration tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh TIERED_FILESYSTEM - - - name: Upload container logs - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: container-logs - path: tests/integration/target/container-logs - - - name: Upload surefire-reports - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: surefire-reports - path: tests/integration/target/surefire-reports diff --git a/.github/workflows/ci-integration-tiered-jcloud.yaml b/.github/workflows/ci-integration-tiered-jcloud.yaml deleted file mode 100644 index b101acbc0977c..0000000000000 --- a/.github/workflows/ci-integration-tiered-jcloud.yaml +++ /dev/null @@ -1,119 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Integration - Tiered JCloud -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/ - -jobs: - - tiered-jcloud: - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: clean disk - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - sudo apt clean - docker rmi $(docker images -q) -f - df -h - - - name: run install by skip tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -q -B -ntp clean install -DskipTests -T 1C - - - name: build pulsar image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build pulsar-all image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build artifacts and docker image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests - - - name: run integration tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh TIERED_JCLOUD - - - name: Upload container logs - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: container-logs - path: tests/integration/target/container-logs - - - name: Upload surefire-reports - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: surefire-reports - path: tests/integration/target/surefire-reports diff --git a/.github/workflows/ci-integration-transaction.yaml b/.github/workflows/ci-integration-transaction.yaml deleted file mode 100644 index b85fc372f2ba6..0000000000000 --- a/.github/workflows/ci-integration-transaction.yaml +++ /dev/null @@ -1,116 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Integration - Transaction -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/ - -jobs: - - transaction: - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: clean disk - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - sudo apt clean - docker rmi $(docker images -q) -f - df -h - - - name: run install by skip tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -q -B -ntp clean install -DskipTests -T 1C - - name: build pulsar image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - name: build pulsar-all image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - name: build artifacts and docker image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests - - - name: run integration tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh TRANSACTION - - - name: Upload container logs - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: container-logs - path: tests/integration/target/container-logs - - - name: Upload surefire-reports - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: surefire-reports - path: tests/integration/target/surefire-reports diff --git a/.github/workflows/ci-license.yaml b/.github/workflows/ci-license.yaml deleted file mode 100644 index 4804cd6e2e1aa..0000000000000 --- a/.github/workflows/ci-license.yaml +++ /dev/null @@ -1,83 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Misc -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 - -jobs: - - license-check: - name: License check - runs-on: ubuntu-22.04 - timeout-minutes: 60 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: build and check license - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -q -B -ntp -DskipTests apache-rat:check initialize license:check install - - - name: license check - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz diff --git a/.github/workflows/ci-maven-cache-update.yaml b/.github/workflows/ci-maven-cache-update.yaml deleted file mode 100644 index d92850bdf8028..0000000000000 --- a/.github/workflows/ci-maven-cache-update.yaml +++ /dev/null @@ -1,112 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# This workflow keeps the GitHub Action Caches up-to-date in the repository. -# -# A pull request build cannot update the cache of the upstream repository. Pull -# requests have a cache in the context of the fork of the upstream repository. -# A pull request build will lookup cache entries from the cache of the upstream -# repository in the case that a cache entry is missing in the pull request source -# repository's cache. -# To reduce cache misses for pull request builds, it is necessary that the -# caches in the upstream repository are up-to-date. -# If the cache entry already exists, the cache won't be updated. This will keep the -# update job very efficient and the downloading and updating will only run if one of the pom.xml -# files has been modified or the cache entry expires. -# - -name: CI - Maven Dependency Cache Update -on: - # trigger on every commit to given branches - push: - branches: - - master - # trigger on a schedule so that the cache will be rebuilt if it happens to expire - schedule: - - cron: '30 */12 * * *' - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - update-maven-dependencies-cache: - name: Update Maven dependency cache for ${{ matrix.name }} - runs-on: ${{ matrix.runs-on }} - timeout-minutes: 45 - - strategy: - fail-fast: false - matrix: - include: - - name: all modules - runs-on: ubuntu-22.04 - cache_name: 'm2-dependencies-all' - mvn_arguments: '' - - - name: all modules - macos - runs-on: macos-latest - cache_name: 'm2-dependencies-all' - - - name: core-modules - runs-on: ubuntu-22.04 - cache_name: 'm2-dependencies-core-modules' - mvn_arguments: '-Pcore-modules,-main' - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - if: ${{ github.event_name != 'schedule' }} - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: | - poms: - - 'pom.xml' - - '**/pom.xml' - - - name: Cache local Maven repository - if: ${{ github.event_name == 'schedule' || steps.changes.outputs.poms == 'true' }} - id: cache - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-${{ matrix.cache_name }}-${{ hashFiles('**/pom.xml') }} - # there is no restore-keys here so that the cache size doesn't keep - # on growing from old entries which wouldn't never expire if the old - # cache would be used as the starting point for a new cache entry - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ (github.event_name == 'schedule' || steps.changes.outputs.poms == 'true') && steps.cache.outputs.cache-hit != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: Download dependencies - if: ${{ (github.event_name == 'schedule' || steps.changes.outputs.poms == 'true') && steps.cache.outputs.cache-hit != 'true' }} - run: | - # download dependencies, ignore errors - mvn -B -fn -ntp ${{ matrix.mvn_arguments }} dependency:go-offline \ No newline at end of file diff --git a/.github/workflows/ci-owasp-dep-check.yaml b/.github/workflows/ci-owasp-dep-check.yaml deleted file mode 100644 index 37f2e172ab4f5..0000000000000 --- a/.github/workflows/ci-owasp-dep-check.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Misc - OWASP Dependency Check -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - - owasp-dep-check: - name: - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed pom files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: | - poms: - - 'pom.xml' - - '**/pom.xml' - - 'src/owasp-dependency-check-false-positives.xml' - - 'src/owasp-dependency-check-suppressions.xml' - - - name: Cache local Maven repository - if: ${{ steps.changes.outputs.poms == 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.changes.outputs.poms == 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: clean disk - if: ${{ steps.changes.outputs.poms == 'true' }} - run: | - sudo swapoff -a - sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc - sudo apt clean - docker rmi $(docker images -q) -f - df -h - - # Projects dependent on flume, hdfs, hbase, and presto currently excluded from the scan. - - name: run "clean install verify" to trigger dependency check - if: ${{ steps.changes.outputs.poms == 'true' }} - run: mvn -q -B -ntp clean install verify -PskipDocker,owasp-dependency-check -DskipTests -pl '!pulsar-sql,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs' - - - name: Upload report - uses: actions/upload-artifact@v4 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: dependency report - path: target/dependency-check-report.html diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml deleted file mode 100644 index 2322fd7dcccaa..0000000000000 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ /dev/null @@ -1,90 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - OWASP Dependency Check -on: - schedule: - - cron: '15 0 * * *' - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - run-owasp-dependency-check: - if: ${{ github.repository == 'apache/pulsar' }} - name: Run OWASP Dependency Check - runs-on: ubuntu-22.04 - timeout-minutes: 45 - strategy: - fail-fast: false - matrix: - include: - - name: master - checkout_branch: 'master' - - name: branch-2.9 - checkout_branch: 'branch-2.9' - - name: branch-2.8 - checkout_branch: 'branch-2.8' - - steps: - - name: checkout - uses: actions/checkout@v2 - with: - ref: ${{ matrix.checkout_branch }} - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Cache local Maven repository - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-owasp-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: 11 - - - name: run install by skip tests - run: mvn -q -B -ntp clean install -DskipTests -T 1C - - - name: run OWASP Dependency Check for distribution/server (-DfailBuildOnAnyVulnerability=true) - run: mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl distribution/server -DfailBuildOnAnyVulnerability=true - - - name: run OWASP Dependency Check for distribution/offloaders, distribution/io and pulsar-sql/presto-distribution - run: mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl distribution/offloaders,distribution/io,pulsar-sql/presto-distribution - - - name: Upload OWASP Dependency Check reports - uses: actions/upload-artifact@v4 - if: always() - with: - name: owasp-dependency-check-reports-${{ matrix.checkout_branch }} - path: | - distribution/server/target/dependency-check-report.html - distribution/offloaders/target/dependency-check-report.html - distribution/io/target/dependency-check-report.html - pulsar-sql/presto-distribution/target/dependency-check-report.html diff --git a/.github/workflows/ci-pulsar-website-build.yaml b/.github/workflows/ci-pulsar-website-build.yaml deleted file mode 100644 index 535de8f5d194a..0000000000000 --- a/.github/workflows/ci-pulsar-website-build.yaml +++ /dev/null @@ -1,86 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Pulsar Website build -on: - schedule: - - cron: '0 */6 * * *' - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - build-website: - if: ${{ github.repository == 'apache/pulsar' }} - name: Build and publish pulsar website - runs-on: ubuntu-22.04 - timeout-minutes: 180 - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Cache local Maven repository - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-website-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: 11 - - - name: clean disk - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - sudo apt clean - docker rmi $(docker images -q) -f - df -h - - - name: run install by skip tests - run: mvn -q -B -ntp install -Pcore-modules,-main -DskipTests -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true - - - name: generate swagger json file - run: mvn -B -ntp -pl pulsar-broker install -DskipTests -Pswagger - - - name: publish - env: - GH_TOKEN: ${{ secrets.PULSARBOT_TOKEN }} - CROWDIN_DOCUSAURUS_API_KEY: ${{ secrets.PULSAR_CROWDIN_DOCUSAURUS_API_KEY }} - run: | - export CROWDIN_DOCUSAURUS_PROJECT_ID=apache-pulsar - echo "Copying swagger json file ..." - mkdir -p site2/website/static/swagger/master/ - cp pulsar-broker/target/docs/swagger*.json site2/website/static/swagger/master - echo "Copied swagger json file." - echo "Building Website" - # Build the new website - site2/tools/docker-build-site.sh - # Script was initially made for travis - bash -e site2/tools/publish-website.sh diff --git a/.github/workflows/ci-pulsarbot.yaml b/.github/workflows/ci-pulsarbot.yaml index 70c4613180e2b..4ea83404856d2 100644 --- a/.github/workflows/ci-pulsarbot.yaml +++ b/.github/workflows/ci-pulsarbot.yaml @@ -21,29 +21,15 @@ name: Pulsar Bot on: issue_comment: types: [created] - pull_request: - types: [closed] - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 jobs: - - action-runner: - name: + pulsarbot: runs-on: ubuntu-22.04 - timeout-minutes: 120 - + timeout-minutes: 10 + if: github.event_name == 'issue_comment' && contains(github.event.comment.body, '/pulsarbot') steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - name: Execute pulsarbot command - id: pulsarbot - if: github.event_name == 'issue_comment' && startsWith(github.event.comment.body, '/pulsarbot') + id: pulsarbot env: GITHUB_TOKEN: ${{ secrets.PULSARBOT_TOKEN }} uses: apache/pulsar-test-infra/pulsarbot@master diff --git a/.github/workflows/ci-semantic-pull-request.yml b/.github/workflows/ci-semantic-pull-request.yml new file mode 100644 index 0000000000000..28246eb2b819c --- /dev/null +++ b/.github/workflows/ci-semantic-pull-request.yml @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +name: Semantic Pull Request + +on: + pull_request: + types: + - opened + - reopened + - edited + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.number }} + cancel-in-progress: true + +jobs: + main: + name: Check pull request title + runs-on: ubuntu-latest + steps: + - uses: amannn/action-semantic-pull-request@v5.0.2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + # Type abbreviation comments: + # feat -> feature + types: | + cleanup + feat + fix + improve + refactor + revert + # Scope abbreviation comments: + # cli -> command line interface + # fn -> Pulsar Functions + # io -> Pulsar Connectors + # offload -> tiered storage + # sec -> security + # sql -> Pulsar Trino Plugin + # txn -> transaction + # ws -> websocket + # ml -> managed ledger + # zk -> zookeeper + # bk -> bookkeeper + scopes: | + admin + bk + broker + build + ci + cli + client + doc + fn + io + meta + misc + ml + monitor + offload + proxy + schema + sec + site + sql + storage + test + txn + ws + zk + # The pull request's title should be fulfilled the following pattern: + # + # [][] + # + # ... where valid types and scopes can be found above; for example: + # + # [fix][test] flaky test V1_ProxyAuthenticationTest.anonymousSocketTest + headerPattern: '^\[(\w*?)\](?:\[(.*?)\])?(?:\s*)(.*)$' + headerPatternCorrespondence: type, scope, subject diff --git a/.github/workflows/ci-shade-test.yaml b/.github/workflows/ci-shade-test.yaml deleted file mode 100644 index e533d7aa80a51..0000000000000 --- a/.github/workflows/ci-shade-test.yaml +++ /dev/null @@ -1,104 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Shade - Test -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/ - -jobs: - - shade-check: - name: - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: clean disk - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - sudo apt clean - docker rmi $(docker images -q) -f - df -h - - - name: run install by skip tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -q -B -ntp clean install -DskipTests -T 1C - - - name: build pulsar image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build pulsar-all image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: build artifacts and docker pulsar latest test image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests -DUBUNTU_MIRROR=http://azure.archive.ubuntu.com/ubuntu/ - - - name: run shade tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh SHADE diff --git a/.github/workflows/ci-unit-broker-broker-gp.yaml b/.github/workflows/ci-unit-broker-broker-gp.yaml deleted file mode 100644 index 938d153aaec9e..0000000000000 --- a/.github/workflows/ci-unit-broker-broker-gp.yaml +++ /dev/null @@ -1,113 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Unit - Brokers - Broker Group -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - - unit-tests: - name: pulsar-ci-test - runs-on: ubuntu-22.04 - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - group: ["group1", "group2", "group3"] - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: build modules - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -ntp -q clean install -Pcore-modules,-main -DskipTests -T 1C - - - name: run unit test 'BROKER_GROUP_1' - if: ${{ steps.check_changes.outputs.docs_only != 'true' && matrix.group == 'group1' }} - run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_unit_group.sh BROKER_GROUP_1 - - - name: run unit test 'BROKER_GROUP_2' - if: ${{ steps.check_changes.outputs.docs_only != 'true' && matrix.group == 'group2' }} - run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_unit_group.sh BROKER_GROUP_2 - - - name: run unit test 'BROKER_GROUP_3' - if: ${{ steps.check_changes.outputs.docs_only != 'true' && matrix.group == 'group3' }} - run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_unit_group.sh BROKER_GROUP_3 - - - name: print JVM thread dumps when cancelled - if: cancelled() - run: ./build/pulsar_ci_tool.sh print_thread_dumps - - - name: package surefire artifacts - if: failure() - run: | - rm -rf artifacts - mkdir artifacts - find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \; - zip -r artifacts.zip artifacts - - - uses: actions/upload-artifact@master - name: upload surefire-artifacts - if: failure() - with: - name: surefire-artifacts - path: artifacts.zip diff --git a/.github/workflows/ci-unit-broker-client-api.yaml b/.github/workflows/ci-unit-broker-client-api.yaml deleted file mode 100644 index 5274305848fea..0000000000000 --- a/.github/workflows/ci-unit-broker-client-api.yaml +++ /dev/null @@ -1,101 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Unit - Brokers - Client Api -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - - unit-tests: - name: - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: build modules - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -ntp -q install -Pcore-modules,-main -DskipTests -T 1C - - - name: run unit test 'BROKER_CLIENT_API' - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_unit_group.sh BROKER_CLIENT_API - - - name: print JVM thread dumps when cancelled - if: cancelled() - run: ./build/pulsar_ci_tool.sh print_thread_dumps - - - name: package surefire artifacts - if: failure() - run: | - rm -rf artifacts - mkdir artifacts - find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \; - zip -r artifacts.zip artifacts - - - uses: actions/upload-artifact@master - name: upload surefire-artifacts - if: failure() - with: - name: surefire-artifacts - path: artifacts.zip diff --git a/.github/workflows/ci-unit-broker-client-impl.yaml b/.github/workflows/ci-unit-broker-client-impl.yaml deleted file mode 100644 index 760f8f6de4d6f..0000000000000 --- a/.github/workflows/ci-unit-broker-client-impl.yaml +++ /dev/null @@ -1,101 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Unit - Brokers - Client Impl -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - - unit-tests: - name: - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: build modules - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -ntp -q install -Pcore-modules,-main -DskipTests -T 1C - - - name: run unit test 'BROKER_CLIENT_IMPL' - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_unit_group.sh BROKER_CLIENT_IMPL - - - name: print JVM thread dumps when cancelled - if: cancelled() - run: ./build/pulsar_ci_tool.sh print_thread_dumps - - - name: package surefire artifacts - if: failure() - run: | - rm -rf artifacts - mkdir artifacts - find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \; - zip -r artifacts.zip artifacts - - - uses: actions/upload-artifact@master - name: upload surefire-artifacts - if: failure() - with: - name: surefire-artifacts - path: artifacts.zip diff --git a/.github/workflows/ci-unit-broker-jdk8.yaml b/.github/workflows/ci-unit-broker-jdk8.yaml deleted file mode 100644 index c38fb5402adc5..0000000000000 --- a/.github/workflows/ci-unit-broker-jdk8.yaml +++ /dev/null @@ -1,101 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Unit - Broker - JDK8 -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - - unit-tests: - name: - runs-on: ubuntu-22.04 - timeout-minutes: 60 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 8 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 8 - - - name: build modules - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -ntp -q clean install -Pcore-modules,-main -DskipTests -T 1C - - - name: run unit test 'BROKER_GROUP_1' - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_unit_group.sh BROKER_JDK8 - - - name: print JVM thread dumps when cancelled - if: cancelled() - run: ./build/pulsar_ci_tool.sh print_thread_dumps - - - name: package surefire artifacts - if: failure() - run: | - rm -rf artifacts - mkdir artifacts - find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \; - zip -r artifacts.zip artifacts - - - uses: actions/upload-artifact@master - name: upload surefire-artifacts - if: failure() - with: - name: surefire-artifacts - path: artifacts.zip diff --git a/.github/workflows/ci-unit-broker-other.yaml b/.github/workflows/ci-unit-broker-other.yaml deleted file mode 100644 index 326f9b52c03e2..0000000000000 --- a/.github/workflows/ci-unit-broker-other.yaml +++ /dev/null @@ -1,101 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Unit - Brokers - Other -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - - unit-tests: - name: - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: build modules - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -ntp -q clean install -Pcore-modules,-main -DskipTests -T 1C - - - name: run unit test 'BROKER_FLAKY' - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_unit_group.sh BROKER_FLAKY - - - name: print JVM thread dumps when cancelled - if: cancelled() - run: ./build/pulsar_ci_tool.sh print_thread_dumps - - - name: package surefire artifacts - if: failure() - run: | - rm -rf artifacts - mkdir artifacts - find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \; - zip -r artifacts.zip artifacts - - - uses: actions/upload-artifact@master - name: upload surefire-artifacts - if: failure() - with: - name: surefire-artifacts - path: artifacts.zip diff --git a/.github/workflows/ci-unit-proxy.yaml b/.github/workflows/ci-unit-proxy.yaml deleted file mode 100644 index 7bd37e754ecc1..0000000000000 --- a/.github/workflows/ci-unit-proxy.yaml +++ /dev/null @@ -1,101 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Unit - Proxy -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - - unit-tests: - name: - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: build modules pulsar-proxy - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -ntp -q install -Pcore-modules,-main -DskipTests -T 1C - - - name: run unit test 'PROXY' - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_unit_group.sh PROXY - - - name: print JVM thread dumps when cancelled - if: cancelled() - run: ./build/pulsar_ci_tool.sh print_thread_dumps - - - name: package surefire artifacts - if: failure() - run: | - rm -rf artifacts - mkdir artifacts - find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \; - zip -r artifacts.zip artifacts - - - uses: actions/upload-artifact@master - name: upload surefire-artifacts - if: failure() - with: - name: surefire-artifacts - path: artifacts.zip diff --git a/.github/workflows/ci-unit.yaml b/.github/workflows/ci-unit.yaml deleted file mode 100644 index 225c16f6e78ef..0000000000000 --- a/.github/workflows/ci-unit.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Unit -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xss1500k -Xmx2048m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - - unit-tests: - name: - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: run unit test 'OTHER' - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_unit_group.sh OTHER - - - name: print JVM thread dumps when cancelled - if: cancelled() - run: ./build/pulsar_ci_tool.sh print_thread_dumps - - - name: package surefire artifacts - if: failure() - run: | - rm -rf artifacts - mkdir artifacts - find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \; - zip -r artifacts.zip artifacts - - - uses: actions/upload-artifact@master - name: upload surefire-artifacts - if: failure() - with: - name: surefire-artifacts - path: artifacts.zip diff --git a/.github/workflows/pulsar-ci-flaky.yaml b/.github/workflows/pulsar-ci-flaky.yaml new file mode 100644 index 0000000000000..7087bc0538e5f --- /dev/null +++ b/.github/workflows/pulsar-ci-flaky.yaml @@ -0,0 +1,191 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Pulsar CI Flaky +on: + pull_request: + branches: + - branch-as-* + schedule: + - cron: '0 12 * * *' + workflow_dispatch: + inputs: + collect_coverage: + description: 'Collect test coverage and upload to Codecov' + required: true + default: 'true' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + MAVEN_OPTS: -Xss1500k -Xmx2048m -XX:+UnlockDiagnosticVMOptions -XX:GCLockerRetryAllocationCount=100 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job + # it's possible to rerun individual failed jobs when the build artifacts are available + # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR + ARTIFACT_RETENTION_DAYS: 3 + +jobs: + preconditions: + name: Preconditions + runs-on: ubuntu-22.04 + if: (github.event_name != 'schedule') || (github.repository == 'apache/pulsar') + outputs: + docs_only: ${{ steps.check_changes.outputs.docs_only }} + changed_tests: ${{ steps.changes.outputs.tests_files }} + collect_coverage: ${{ steps.check_coverage.outputs.collect_coverage }} + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Detect changed files + id: changes + uses: apache/pulsar-test-infra/paths-filter@master + with: + filters: .github/changes-filter.yaml + list-files: csv + + - name: Check changed files + id: check_changes + run: | + if [[ "${GITHUB_EVENT_NAME}" != "schedule" && "${GITHUB_EVENT_NAME}" != "workflow_dispatch" ]]; then + echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT + else + echo docs_only=false >> $GITHUB_OUTPUT + fi + + - name: Check if coverage should be collected + id: check_coverage + run: | + echo "collect_coverage=${{ + (steps.check_changes.outputs.docs_only != 'true' && github.event_name != 'workflow_dispatch' + && (github.base_ref == 'master' || github.ref_name == 'master')) + || (github.event_name == 'workflow_dispatch' && github.event.inputs.collect_coverage == 'true') + }}" >> $GITHUB_OUTPUT + + - name: Check if the PR has been approved for testing + if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }} + env: + GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} + GITHUB_TOKEN: ${{ github.token }} + run: | + build/pulsar_ci_tool.sh check_ready_to_test + + build-and-test: + needs: preconditions + name: Flaky tests suite + env: + JOB_NAME: Flaky tests suite + COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}" + runs-on: ubuntu-22.04 + timeout-minutes: 100 + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Cache local Maven repository + uses: actions/cache@v4 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 11 + + - name: Build core-modules + run: | + mvn -B -T 1C -ntp -Pcore-modules,-main clean install -DskipTests -Dlicense.skip=true -Drat.skip=true + + - name: Run unit test group BROKER_FLAKY + run: | + CHANGED_TESTS="${{ needs.preconditions.outputs.tests_files }}" ./build/run_unit_group.sh BROKER_FLAKY + + - name: print JVM thread dumps when cancelled + if: cancelled() + run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps + + - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories + if: ${{ always() }} + uses: ./.github/actions/copy-test-reports + + - name: Create Jacoco reports + if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report + cd $GITHUB_WORKSPACE/target + zip -qr jacoco_test_coverage_report_flaky.zip jacoco_test_coverage_report || true + + - name: Upload Jacoco report files to build artifacts + if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} + uses: actions/upload-artifact@v4 + with: + name: Jacoco-coverage-report-flaky + path: target/jacoco_test_coverage_report_flaky.zip + retention-days: 3 + + - name: Upload to Codecov + if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} + uses: ./.github/actions/upload-coverage + with: + flags: unittests + + - name: Publish Test Report + uses: apache/pulsar-test-infra/action-junit-report@master + if: ${{ always() }} + with: + report_paths: 'test-reports/TEST-*.xml' + annotate_only: 'true' + + - name: Upload Surefire reports + uses: actions/upload-artifact@v4 + if: ${{ !success() }} + with: + name: Unit-BROKER_FLAKY-surefire-reports + path: surefire-reports + retention-days: 7 + + - name: Upload possible heap dump + uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: Unit-BROKER_FLAKY-heapdump + path: /tmp/*.hprof + retention-days: 7 + if-no-files-found: ignore diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml new file mode 100644 index 0000000000000..b334e86b1b68c --- /dev/null +++ b/.github/workflows/pulsar-ci.yaml @@ -0,0 +1,1297 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Pulsar CI +on: + pull_request: + branches: + - branch-as-* + schedule: + - cron: '0 12 * * *' + workflow_dispatch: + inputs: + collect_coverage: + description: 'Collect test coverage and upload to Codecov' + required: true + default: 'true' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + MAVEN_OPTS: -Xss1500k -Xmx2048m -XX:+UnlockDiagnosticVMOptions -XX:GCLockerRetryAllocationCount=100 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job + # it's possible to rerun individual failed jobs when the build artifacts are available + # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR + ARTIFACT_RETENTION_DAYS: 3 + +jobs: + preconditions: + name: Preconditions + runs-on: ubuntu-22.04 + if: (github.event_name != 'schedule') || (github.repository == 'apache/pulsar') + outputs: + docs_only: ${{ steps.check_changes.outputs.docs_only }} + changed_tests: ${{ steps.changes.outputs.tests_files }} + need_owasp: ${{ steps.changes.outputs.need_owasp }} + collect_coverage: ${{ steps.check_coverage.outputs.collect_coverage }} + + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Detect changed files + id: changes + uses: apache/pulsar-test-infra/paths-filter@master + with: + filters: .github/changes-filter.yaml + list-files: csv + + - name: Check changed files + id: check_changes + run: | + if [[ "${GITHUB_EVENT_NAME}" != "schedule" && "${GITHUB_EVENT_NAME}" != "workflow_dispatch" ]]; then + echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT + else + echo docs_only=false >> $GITHUB_OUTPUT + fi + + - name: Check if coverage should be collected + id: check_coverage + run: | + echo "collect_coverage=${{ + (steps.check_changes.outputs.docs_only != 'true' && github.event_name != 'workflow_dispatch' + && (github.base_ref == 'master' || github.ref_name == 'master')) + || (github.event_name == 'workflow_dispatch' && github.event.inputs.collect_coverage == 'true') + }}" >> $GITHUB_OUTPUT + + - name: Check if the PR has been approved for testing + if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }} + env: + GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} + GITHUB_TOKEN: ${{ github.token }} + run: | + build/pulsar_ci_tool.sh check_ready_to_test + + build-and-license-check: + needs: preconditions + name: Build and License check + env: + JOB_NAME: Build and License check + runs-on: ubuntu-22.04 + timeout-minutes: 60 + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Cache local Maven repository + uses: actions/cache@v4 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 11 + + - name: Check source code license headers + run: mvn -B -T 8 -ntp initialize apache-rat:check license:check + +# - name: Check source code style +# run: mvn -B -T 8 -ntp initialize checkstyle:check + + - name: Build core-modules + run: | + mvn -B -T 1C -ntp -Pcore-modules,-main clean install -DskipTests -Dlicense.skip=true -Drat.skip=true -Dcheckstyle.skip=true + + - name: Check binary licenses + run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Save maven build results to Github artifact cache so that the results can be reused + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-maven-repository-binaries \ + tar --exclude '.m2/repository/org/apache/pulsar/pulsar-*-distribution' \ + -I zstd -cf - .m2/repository/org/apache/pulsar + cd $GITHUB_WORKSPACE + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-server-distribution \ + tar -I zstd -cf - distribution/server/target/apache-pulsar-*-bin.tar.gz + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + continue-on-error: true + with: + action: wait + + + unit-tests: + name: CI - Unit - ${{ matrix.name }} + env: + JOB_NAME: CI - Unit - ${{ matrix.name }} + COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}" + runs-on: ubuntu-22.04 + timeout-minutes: ${{ matrix.timeout || 60 }} + needs: ['preconditions', 'build-and-license-check'] + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + strategy: + fail-fast: false + matrix: + include: + - name: Other + group: OTHER + timeout: 75 + - name: Brokers - Broker Group 1 + group: BROKER_GROUP_1 + - name: Brokers - Broker Group 2 + group: BROKER_GROUP_2 + - name: Brokers - Broker Group 3 + group: BROKER_GROUP_3 + - name: Brokers - Broker Group 4 + group: BROKER_GROUP_4 + - name: Brokers - Client Api + group: BROKER_CLIENT_API + - name: Brokers - Client Impl + group: BROKER_CLIENT_IMPL + - name: Proxy + group: PROXY + - name: Pulsar IO + group: PULSAR_IO + timeout: 75 + - name: Pulsar IO - Elastic Search + group: PULSAR_IO_ELASTIC + - name: Pulsar IO - Kafka Connect Adaptor + group: PULSAR_IO_KAFKA_CONNECT + - name: Pulsar Client + group: CLIENT + + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Cache Maven dependencies + uses: actions/cache@v4 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK ${{ matrix.jdk || '11' }} + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ matrix.jdk || '11' }} + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts + + - name: Run setup commands + run: | + ${{ matrix.setup }} + + - name: Run unit test group '${{ matrix.group }}' + run: | + CHANGED_TESTS="${{ needs.preconditions.outputs.tests_files }}" ./build/run_unit_group.sh ${{ matrix.group }} + + - name: Upload coverage to build artifacts + if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} + run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_unittest_coverage_files ${{ matrix.group }} + + - name: print JVM thread dumps when cancelled + if: cancelled() + run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps + + - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories + if: ${{ always() }} + uses: ./.github/actions/copy-test-reports + + - name: Publish Test Report + uses: apache/pulsar-test-infra/action-junit-report@master + if: ${{ always() }} + with: + report_paths: 'test-reports/TEST-*.xml' + annotate_only: 'true' + + - name: Upload Surefire reports + uses: actions/upload-artifact@v4 + if: ${{ !success() }} + with: + name: Unit-${{ matrix.group }}-surefire-reports + path: surefire-reports + retention-days: 7 + + - name: Upload possible heap dump, core dump or crash files + uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: Unit-${{ matrix.group }}-dumps + path: | + /tmp/*.hprof + **/hs_err_*.log + **/core.* + retention-days: 7 + if-no-files-found: ignore + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + continue-on-error: true + with: + action: wait + + + unit-tests-upload-coverage: + name: CI - Unit - Upload Coverage + runs-on: ubuntu-22.04 + timeout-minutes: 30 + needs: ['preconditions', 'unit-tests'] + if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} + + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Cache Maven dependencies + uses: actions/cache@v4 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK ${{ matrix.jdk || '11' }} + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ matrix.jdk || '11' }} + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts + + - name: Restore coverage files from build artifacts and create Jacoco reports + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_unittest_coverage_files + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report + cd $GITHUB_WORKSPACE/target + zip -qr jacoco_test_coverage_report_unittests.zip jacoco_test_coverage_report || true + + - name: Upload Jacoco report files to build artifacts + uses: actions/upload-artifact@v4 + with: + name: Jacoco-coverage-report-unittests + path: target/jacoco_test_coverage_report_unittests.zip + retention-days: 3 + + - name: Upload to Codecov + uses: ./.github/actions/upload-coverage + with: + flags: unittests + + - name: Delete coverage files from build artifacts + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_unittest_coverage_files + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + continue-on-error: true + with: + action: wait + + integration-tests: + name: CI - Integration - ${{ matrix.name }} + runs-on: ubuntu-22.04 + timeout-minutes: ${{ matrix.timeout || 60 }} + needs: ['preconditions', 'pulsar-test-latest-version-image'] + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + env: + JOB_NAME: CI - Integration - ${{ matrix.name }} + PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest + strategy: + fail-fast: false + matrix: + include: + - name: Backwards Compatibility + group: BACKWARDS_COMPAT + no_coverage: true + + - name: Cli + group: CLI + + - name: Messaging + group: MESSAGING + + - name: Shade on Java 8 + group: SHADE_RUN + upload_name: SHADE_RUN_8 + runtime_jdk: 8 + setup: ./build/run_integration_group.sh SHADE_BUILD + no_coverage: true + + - name: Shade on Java 11 + group: SHADE_RUN + upload_name: SHADE_RUN_11 + runtime_jdk: 11 + setup: ./build/run_integration_group.sh SHADE_BUILD + no_coverage: true + + - name: Standalone + group: STANDALONE + + - name: Transaction + group: TRANSACTION + + - name: SQL + group: SQL + + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Cache Maven dependencies + uses: actions/cache@v4 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 11 + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts + + - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image + + - name: Run setup commands + if: ${{ matrix.setup }} + run: | + ${{ matrix.setup }} + + - name: Set up runtime JDK ${{ matrix.runtime_jdk }} + uses: actions/setup-java@v4 + if: ${{ matrix.runtime_jdk }} + with: + distribution: 'temurin' + java-version: ${{ matrix.runtime_jdk }} + + - name: Run integration test group '${{ matrix.group }}' + run: | + if [[ "${{ matrix.no_coverage }}" != "true" && "${{ needs.preconditions.outputs.collect_coverage }}" == "true" ]]; then + coverage_args="--coverage" + fi + ./build/run_integration_group.sh ${{ matrix.group }} $coverage_args + + - name: Upload coverage to build artifacts + if: ${{ !matrix.no_coverage && needs.preconditions.outputs.collect_coverage == 'true' }} + run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_inttest_coverage_files ${{ matrix.group }} + + - name: print JVM thread dumps when cancelled + if: cancelled() + run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps + + - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories + if: ${{ always() }} + uses: ./.github/actions/copy-test-reports + + - name: Publish Test Report + uses: apache/pulsar-test-infra/action-junit-report@master + if: ${{ always() }} + with: + report_paths: 'test-reports/TEST-*.xml' + annotate_only: 'true' + + - name: Upload Surefire reports + uses: actions/upload-artifact@v4 + if: ${{ !success() }} + with: + name: Integration-${{ matrix.upload_name || matrix.group }}-surefire-reports + path: surefire-reports + retention-days: 7 + + - name: Upload container logs + uses: actions/upload-artifact@v4 + if: ${{ !success() }} + continue-on-error: true + with: + name: Integration-${{ matrix.upload_name || matrix.group }}-container-logs + path: tests/integration/target/container-logs + retention-days: 7 + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + continue-on-error: true + with: + action: wait + + integration-tests-upload-coverage: + name: CI - Integration - Upload Coverage + runs-on: ubuntu-22.04 + timeout-minutes: 30 + needs: ['preconditions', 'integration-tests'] + if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} + env: + PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Cache Maven dependencies + uses: actions/cache@v4 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 11 + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts + + - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image + + - name: Restore coverage files from build artifacts and create Jacoco reports + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_inttest_coverage_files + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_inttest_coverage_report + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report + cd $GITHUB_WORKSPACE/target + zip -qr jacoco_test_coverage_report_inttests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report || true + + - name: Upload Jacoco report files to build artifacts + uses: actions/upload-artifact@v4 + with: + name: Jacoco-coverage-report-inttests + path: target/jacoco_test_coverage_report_inttests.zip + retention-days: 3 + + - name: Upload to Codecov + uses: ./.github/actions/upload-coverage + with: + flags: inttests + + - name: Delete coverage files from build artifacts + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_inttest_coverage_files + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + continue-on-error: true + with: + action: wait + + delete-integration-test-docker-image-artifact: + name: "Delete integration test docker image artifact" + runs-on: ubuntu-22.04 + timeout-minutes: 10 + needs: [ + 'preconditions', + 'integration-tests', + 'integration-tests-upload-coverage' + ] + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Delete docker image from GitHub Actions Artifacts + run: | + gh-actions-artifact-client.js delete pulsar-test-latest-version-image.zst + + pulsar-test-latest-version-image: + name: Build Pulsar docker image + runs-on: ubuntu-22.04 + timeout-minutes: 60 + needs: ['preconditions', 'build-and-license-check'] + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Clean Disk + uses: ./.github/actions/clean-disk + with: + mode: full + + - name: Cache local Maven repository + uses: actions/cache@v4 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 11 + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + + - name: Pick ubuntu mirror for the docker image build + run: | + # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh pick_ubuntu_mirror + + - name: Build latest-version-image docker image + run: | + # build docker image + # include building of Pulsar SQL, Connectors, Offloaders and server distros + mvn -B -am -pl pulsar-sql/presto-distribution,distribution/io,distribution/offloaders,distribution/server,tests/docker-images/latest-version-image install \ + -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \ + -Pmain,docker -Dmaven.test.skip=true -Ddocker.squash=true \ + -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true + + - name: Clean up disk space + run: | + # release disk space since saving docker image consumes local disk space + # + echo "::group::Available diskspace before cleaning" + time df -BM / /mnt + echo "::endgroup::" + echo "::group::Clean build directory" + # docker build changes some files to root ownership, fix this before deleting files + sudo chown -R $USER:$GROUP . + # clean build directories + time git clean -fdx + echo "::endgroup::" + echo "::group::Available diskspace after cleaning build directory" + time df -BM / /mnt + echo "::endgroup::" + echo "::group::Delete maven repository" + # delete maven repository + time rm -rf ~/.m2/repository + echo "::endgroup::" + echo "::group::Available diskspace after cleaning maven repository" + time df -BM / /mnt + echo "::endgroup::" + + - name: save docker image apachepulsar/pulsar-test-latest-version:latest to Github artifact cache + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/pulsar-test-latest-version:latest pulsar-test-latest-version-image + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + continue-on-error: true + with: + action: wait + + system-tests: + name: CI - System - ${{ matrix.name }} + runs-on: ubuntu-22.04 + timeout-minutes: 60 + needs: ['preconditions', 'pulsar-test-latest-version-image'] + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + env: + JOB_NAME: CI - System - ${{ matrix.name }} + PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest + strategy: + fail-fast: false + matrix: + include: + - name: Tiered FileSystem + group: TIERED_FILESYSTEM + + - name: Tiered JCloud + group: TIERED_JCLOUD + + - name: Function + group: FUNCTION + + - name: Schema + group: SCHEMA + + - name: Pulsar Connectors - Thread + group: PULSAR_CONNECTORS_THREAD + + - name: Pulsar Connectors - Process + group: PULSAR_CONNECTORS_PROCESS + + - name: Pulsar IO + group: PULSAR_IO + clean_disk: true + + - name: Sql + group: SQL + + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Clean Disk when needed + if: ${{ matrix.clean_disk }} + uses: ./.github/actions/clean-disk + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Cache local Maven repository + uses: actions/cache@v4 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 11 + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts + + - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image + + - name: Run setup commands + if: ${{ matrix.setup }} + run: | + ${{ matrix.setup }} + + - name: Run system test group '${{ matrix.group }}' + run: | + if [[ "${{ matrix.no_coverage }}" != "true" && "${{ needs.preconditions.outputs.collect_coverage }}" == "true" ]]; then + coverage_args="--coverage" + fi + ./build/run_integration_group.sh ${{ matrix.group }} $coverage_args + + - name: Upload coverage to build artifacts + if: ${{ !matrix.no_coverage && needs.preconditions.outputs.collect_coverage == 'true' }} + run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_systest_coverage_files ${{ matrix.group }} + + - name: print JVM thread dumps when cancelled + if: cancelled() + run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps + + - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories + if: ${{ always() }} + uses: ./.github/actions/copy-test-reports + + - name: Publish Test Report + uses: apache/pulsar-test-infra/action-junit-report@master + if: ${{ always() }} + with: + report_paths: 'test-reports/TEST-*.xml' + annotate_only: 'true' + + - name: Upload container logs + uses: actions/upload-artifact@v4 + if: ${{ !success() }} + continue-on-error: true + with: + name: System-${{ matrix.group }}-container-logs + path: tests/integration/target/container-logs + retention-days: 7 + + - name: Upload Surefire reports + uses: actions/upload-artifact@v4 + if: ${{ !success() }} + with: + name: System-${{ matrix.name }}-surefire-reports + path: surefire-reports + retention-days: 7 + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + continue-on-error: true + with: + action: wait + + system-tests-upload-coverage: + name: CI - System - Upload Coverage + runs-on: ubuntu-22.04 + timeout-minutes: 30 + needs: ['preconditions', 'system-tests'] + if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} + env: + PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest + + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Cache local Maven repository + uses: actions/cache@v4 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 11 + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + + - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image + + - name: Restore coverage files from build artifacts and create Jacoco reports + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_systest_coverage_files + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_inttest_coverage_report + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report + cd $GITHUB_WORKSPACE/target + zip -qr jacoco_test_coverage_report_systests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report || true + + - name: Upload Jacoco report files to build artifacts + uses: actions/upload-artifact@v4 + with: + name: Jacoco-coverage-report-systests + path: target/jacoco_test_coverage_report_systests.zip + retention-days: 3 + + - name: Upload to Codecov + uses: ./.github/actions/upload-coverage + with: + flags: systests + + - name: Delete coverage files from build artifacts + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_systest_coverage_files + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + continue-on-error: true + with: + action: wait + + flaky-system-tests: + name: CI Flaky - System - ${{ matrix.name }} + runs-on: ubuntu-22.04 + timeout-minutes: 60 + needs: [ 'preconditions', 'pulsar-test-latest-version-image' ] + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + env: + JOB_NAME: CI Flaky - System - ${{ matrix.name }} + PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest + strategy: + fail-fast: false + matrix: + include: + - name: Pulsar IO - Oracle + group: PULSAR_IO_ORA + clean_disk: true + + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Clean Disk when needed + if: ${{ matrix.clean_disk }} + uses: ./.github/actions/clean-disk + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Cache local Maven repository + uses: actions/cache@v4 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 11 + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + + - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image + + - name: Run setup commands + if: ${{ matrix.setup }} + run: | + ${{ matrix.setup }} + + - name: Run system test group '${{ matrix.group }}' + run: | + ./build/run_integration_group.sh ${{ matrix.group }} + + - name: print JVM thread dumps when cancelled + if: cancelled() + run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps + + - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories + if: ${{ always() }} + uses: ./.github/actions/copy-test-reports + + - name: Publish Test Report + uses: apache/pulsar-test-infra/action-junit-report@master + if: ${{ always() }} + with: + report_paths: 'test-reports/TEST-*.xml' + annotate_only: 'true' + + - name: Upload container logs + uses: actions/upload-artifact@v4 + if: ${{ !success() }} + continue-on-error: true + with: + name: System-${{ matrix.group }}-container-logs + path: tests/integration/target/container-logs + retention-days: 7 + + - name: Upload Surefire reports + uses: actions/upload-artifact@v4 + if: ${{ !success() }} + with: + name: System-${{ matrix.name }}-surefire-reports + path: surefire-reports + retention-days: 7 + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + continue-on-error: true + with: + action: wait + + delete-system-test-docker-image-artifact: + name: "Delete system test docker image artifact" + runs-on: ubuntu-22.04 + timeout-minutes: 10 + needs: [ + 'preconditions', + 'system-tests', + 'system-tests-upload-coverage', + 'flaky-system-tests' + ] + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Delete docker image from GitHub Actions Artifacts + run: | + gh-actions-artifact-client.js delete pulsar-test-latest-version-image.zst + + macos-build: + name: Build Pulsar on MacOS + runs-on: macos-latest + timeout-minutes: 120 + needs: ['preconditions', 'integration-tests'] + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Cache Maven dependencies + uses: actions/cache@v4 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-all- + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 11 + + - name: build package + run: mvn -B clean package -DskipTests -T 1C -ntp + + owasp-dep-check: + name: OWASP dependency check + runs-on: ubuntu-22.04 + timeout-minutes: 120 + needs: [ 'preconditions', 'integration-tests' ] + if: ${{ needs.preconditions.outputs.need_owasp == 'true' }} + env: + NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }} + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Restore Maven repository cache + uses: actions/cache/restore@v4 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules- + - name: Set up JDK ${{ matrix.jdk || '11' }} + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ matrix.jdk || '11' }} + + - name: Clean Disk + uses: ./.github/actions/clean-disk + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + + - name: OWASP cache key weeknum + id: get-weeknum + run: | + echo "weeknum=$(date -u +"%Y-%U")" >> $GITHUB_OUTPUT + shell: bash + + - name: Restore OWASP Dependency Check data + id: restore-owasp-dependency-check-data + uses: actions/cache/restore@v4 + timeout-minutes: 5 + with: + path: ~/.m2/repository/org/owasp/dependency-check-data + key: owasp-dependency-check-data-${{ steps.get-weeknum.outputs.weeknum }} + enableCrossOsArchive: true + restore-keys: | + owasp-dependency-check-data- + + - name: Log warning when skipped + if: ${{ !steps.restore-owasp-dependency-check-data.outputs.cache-matched-key }} + run: | + echo "::warning::OWASP Dependency Check was skipped since the OWASP Dependency check data wasn't found in the cache. Run ci-owasp-dependency-check.yaml workflow to update the cache." + + # Projects dependent on flume, hdfs, hbase, and presto currently excluded from the scan. + - name: trigger dependency check + if: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-matched-key }} + run: | + mvn -B -ntp verify -PskipDocker,skip-all,owasp-dependency-check -Dcheckstyle.skip=true -DskipTests \ + -pl '!pulsar-sql,!distribution/server,!distribution/io,!distribution/offloaders,!pulsar-sql/presto-distribution,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb' + + - name: Upload report + uses: actions/upload-artifact@v4 + if: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-matched-key && (cancelled() || failure()) }} + continue-on-error: true + with: + name: dependency report + path: target/dependency-check-report.html + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + continue-on-error: true + with: + action: wait + + # This job is required for pulls to be merged. + # It depends on all other jobs in this workflow. + # It cleans up the binaries in the same job in order to not spin up another runner for basically doing nothing. + pulsar-ci-checks-completed: + name: "Pulsar CI checks completed" + if: ${{ always() && ((github.event_name != 'schedule') || (github.repository == 'apache/pulsar')) }} + runs-on: ubuntu-22.04 + timeout-minutes: 10 + needs: [ + 'preconditions', + 'unit-tests', + 'integration-tests', + 'system-tests', + 'flaky-system-tests', + 'macos-build', + 'unit-tests-upload-coverage', + 'integration-tests-upload-coverage', + 'system-tests-upload-coverage', + 'owasp-dep-check' + ] + steps: + - name: Check that all required jobs were completed successfully + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + run: | + if [[ ! ( \ + "${{ needs.unit-tests.result }}" == "success" \ + && "${{ needs.integration-tests.result }}" == "success" \ + && "${{ needs.system-tests.result }}" == "success" \ + && "${{ needs.macos-build.result }}" == "success" \ + ) ]]; then + echo "Required jobs haven't been completed successfully." + exit 1 + fi + + - name: checkout + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + uses: actions/checkout@v4 + + - name: Tune Runner VM + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + uses: ./.github/actions/tune-runner-vm + + - name: Install gh-actions-artifact-client.js + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Delete maven repository binaries from GitHub Actions Artifacts + if: ${{ needs.preconditions.outputs.docs_only != 'true' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }} + run: | + gh-actions-artifact-client.js delete pulsar-maven-repository-binaries.tar.zst || true + gh-actions-artifact-client.js delete pulsar-server-distribution.tar.zst || true diff --git a/build/pulsar_ci_tool.sh b/build/pulsar_ci_tool.sh index 17875e3cd6bf8..40b1e5d18c39c 100755 --- a/build/pulsar_ci_tool.sh +++ b/build/pulsar_ci_tool.sh @@ -20,12 +20,16 @@ # shell function library for Pulsar CI builds +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" + set -e set -o pipefail +ARTIFACT_RETENTION_DAYS="${ARTIFACT_RETENTION_DAYS:-3}" + # lists all available functions in this tool function ci_list_functions() { - declare -F | awk '{print $NF}' | sort | egrep '^ci_' | sed 's/^ci_//' + declare -F | awk '{print $NF}' | sort | grep -E '^ci_' | sed 's/^ci_//' } # prints thread dumps for all running JVMs @@ -40,6 +44,7 @@ function ci_print_thread_dumps() { return 0 } +# runs maven function _ci_mvn() { mvn -B -ntp -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \ "$@" @@ -50,6 +55,565 @@ function ci_dependency_check() { _ci_mvn -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl '!pulsar-client-tools-test' "$@" } +function ci_pick_ubuntu_mirror() { + echo "Choosing fastest up-to-date ubuntu mirror based on download speed..." + UBUNTU_MIRROR=$({ + # choose mirrors that are up-to-date by checking the Last-Modified header for + { + # randomly choose up to 10 mirrors using http:// protocol + # (https isn't supported in docker containers that don't have ca-certificates installed) + curl -s http://mirrors.ubuntu.com/mirrors.txt | grep '^http://' | shuf -n 10 + # also consider Azure's Ubuntu mirror + echo http://azure.archive.ubuntu.com/ubuntu/ + } | xargs -I {} sh -c 'echo "$(curl -m 5 -sI {}dists/$(lsb_release -c | cut -f2)-security/Contents-$(dpkg --print-architecture).gz|sed s/\\r\$//|grep Last-Modified|awk -F": " "{ print \$2 }" | LANG=C date -f- -u +%s)" "{}"' | sort -rg | awk '{ if (NR==1) TS=$1; if ($1 == TS) print $2 }' + } | xargs -I {} sh -c 'echo `curl -r 0-102400 -m 5 -s -w %{speed_download} -o /dev/null {}ls-lR.gz` {}' \ + |sort -g -r |head -1| awk '{ print $2 }') + if [ -z "$UBUNTU_MIRROR" ]; then + # fallback to full mirrors list + UBUNTU_MIRROR="mirror://mirrors.ubuntu.com/mirrors.txt" + fi + OLD_MIRROR=$(cat /etc/apt/sources.list | grep '^deb ' | head -1 | awk '{ print $2 }') + echo "Picked '$UBUNTU_MIRROR'. Current mirror is '$OLD_MIRROR'." + if [[ "$OLD_MIRROR" != "$UBUNTU_MIRROR" ]]; then + sudo sed -i "s|$OLD_MIRROR|$UBUNTU_MIRROR|g" /etc/apt/sources.list + sudo apt-get update + fi + # set the chosen mirror also in the UBUNTU_MIRROR and UBUNTU_SECURITY_MIRROR environment variables + # that can be used by docker builds + export UBUNTU_MIRROR + export UBUNTU_SECURITY_MIRROR=$UBUNTU_MIRROR + # make environment variables available for later GitHub Actions steps + if [ -n "$GITHUB_ENV" ]; then + echo "UBUNTU_MIRROR=$UBUNTU_MIRROR" >> $GITHUB_ENV + echo "UBUNTU_SECURITY_MIRROR=$UBUNTU_SECURITY_MIRROR" >> $GITHUB_ENV + fi +} + +# installs a tool executable if it's not found on the PATH +function ci_install_tool() { + local tool_executable=$1 + local tool_package=${2:-$1} + if ! command -v $tool_executable &>/dev/null; then + if [[ "$GITHUB_ACTIONS" == "true" ]]; then + echo "::group::Installing ${tool_package}" + sudo apt-get -y install ${tool_package} >/dev/null || { + echo "Installing the package failed. Switching the ubuntu mirror and retrying..." + ci_pick_ubuntu_mirror + # retry after picking the ubuntu mirror + sudo apt-get -y install ${tool_package} + } + echo '::endgroup::' + else + fail "$tool_executable wasn't found on PATH. You should first install $tool_package with your package manager." + fi + fi +} + +# outputs the given message to stderr and exits the shell script +function fail() { + echo "$*" >&2 + exit 1 +} + +# saves a given image (1st parameter) to the GitHub Actions Artifacts with the given name (2nd parameter) +function ci_docker_save_image_to_github_actions_artifacts() { + local image=$1 + local artifactname="${2}.zst" + ci_install_tool pv + echo "::group::Saving docker image ${image} with name ${artifactname} in GitHub Actions Artifacts" + # delete possible previous artifact that might exist when re-running + timeout 1m gh-actions-artifact-client.js delete "${artifactname}" &>/dev/null || true + docker save ${image} | zstd | pv -ft -i 5 | pv -Wbaf -i 5 | timeout 20m gh-actions-artifact-client.js upload --retentionDays=$ARTIFACT_RETENTION_DAYS "${artifactname}" + echo "::endgroup::" +} + +# loads a docker image from the GitHub Actions Artifacts with the given name (1st parameter) +function ci_docker_load_image_from_github_actions_artifacts() { + local artifactname="${1}.zst" + ci_install_tool pv + echo "::group::Loading docker image from name ${artifactname} in GitHub Actions Artifacts" + timeout 20m gh-actions-artifact-client.js download "${artifactname}" | pv -batf -i 5 | unzstd | docker load + echo "::endgroup::" +} + +# loads and extracts a zstd (.tar.zst) compressed tar file from the GitHub Actions Artifacts with the given name (1st parameter) +function ci_restore_tar_from_github_actions_artifacts() { + local artifactname="${1}.tar.zst" + ci_install_tool pv + echo "::group::Restoring tar from name ${artifactname} in GitHub Actions Artifacts to $PWD" + timeout 5m gh-actions-artifact-client.js download "${artifactname}" | pv -batf -i 5 | tar -I zstd -xf - + echo "::endgroup::" +} + +# stores a given command (with full arguments, specified after 1st parameter) output to GitHub Actions Artifacts with the given name (1st parameter) +function ci_store_tar_to_github_actions_artifacts() { + local artifactname="${1}.tar.zst" + shift + if [[ "$GITHUB_ACTIONS" == "true" ]]; then + ci_install_tool pv + echo "::group::Storing $1 tar command output to name ${artifactname} in GitHub Actions Artifacts" + # delete possible previous artifact that might exist when re-running + timeout 1m gh-actions-artifact-client.js delete "${artifactname}" &>/dev/null || true + "$@" | pv -ft -i 5 | pv -Wbaf -i 5 | timeout 10m gh-actions-artifact-client.js upload --retentionDays=$ARTIFACT_RETENTION_DAYS "${artifactname}" + echo "::endgroup::" + else + local artifactfile="$(mktemp -t artifact.XXXX)" + echo "Storing output for debugging in $artifactfile" + "$@" | pv -ft -i 5 | pv -Wbaf -i 5 > $artifactfile + fi +} + +# copies test reports into test-reports and surefire-reports directory +# subsequent runs of tests might overwrite previous reports. This ensures that all test runs get reported. +function ci_move_test_reports() { + ( + if [ -n "${GITHUB_WORKSPACE}" ]; then + cd "${GITHUB_WORKSPACE}" + mkdir -p test-reports + mkdir -p surefire-reports + fi + # aggregate all junit xml reports in a single directory + if [ -d test-reports ]; then + # copy test reports to single directory, rename duplicates + find . -path '*/target/surefire-reports/junitreports/TEST-*.xml' -print0 | xargs -0 -r -n 1 mv -t test-reports --backup=numbered + # rename possible duplicates to have ".xml" extension + ( + for f in test-reports/*~; do + mv -- "$f" "${f}.xml" + done 2>/dev/null + ) || true + fi + # aggregate all surefire-reports in a single directory + if [ -d surefire-reports ]; then + ( + find . -type d -path '*/target/surefire-reports' -not -path './surefire-reports/*' | + while IFS=$'\n' read -r directory; do + echo "Copying reports from $directory" + target_dir="surefire-reports/${directory}" + if [ -d "$target_dir" ]; then + # rotate backup directory names *~3 -> *~2, *~2 -> *~3, *~1 -> *~2, ... + ( command ls -vr1d "${target_dir}~"* 2> /dev/null | awk '{print "mv "$0" "substr($0,0,length-1)substr($0,length,1)+1}' | sh ) || true + # backup existing target directory, these are the results of the previous test run + mv "$target_dir" "${target_dir}~1" + fi + # copy files + cp -R --parents "$directory" surefire-reports + # remove the original directory + rm -rf "$directory" + done + ) + fi + ) +} + +function ci_check_ready_to_test() { + if [[ -z "$GITHUB_EVENT_PATH" ]]; then + >&2 echo "GITHUB_EVENT_PATH isn't set" + return 1 + fi + + PR_JSON_URL=$(jq -r '.pull_request.url' "${GITHUB_EVENT_PATH}") + echo "Refreshing $PR_JSON_URL..." + PR_JSON=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" "${PR_JSON_URL}") + + if printf "%s" "${PR_JSON}" | jq -e '.draft | select(. == true)' &> /dev/null; then + echo "PR is draft." + elif ! ( printf "%s" "${PR_JSON}" | jq -e '.mergeable | select(. == true)' &> /dev/null ); then + echo "PR isn't mergeable." + else + # check ready-to-test label + if printf "%s" "${PR_JSON}" | jq -e '.labels[] | .name | select(. == "ready-to-test")' &> /dev/null; then + echo "Found ready-to-test label." + return 0 + else + echo "There is no ready-to-test label on the PR." + fi + + # check if the PR has been approved + PR_NUM=$(jq -r '.pull_request.number' "${GITHUB_EVENT_PATH}") + REPO_FULL_NAME=$(jq -r '.repository.full_name' "${GITHUB_EVENT_PATH}") + REPO_NAME=$(basename "${REPO_FULL_NAME}") + REPO_OWNER=$(dirname "${REPO_FULL_NAME}") + # use graphql query to find out reviewDecision + PR_REVIEW_DECISION=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" -X POST -d '{"query": "query { repository(name: \"'${REPO_NAME}'\", owner: \"'${REPO_OWNER}'\") { pullRequest(number: '${PR_NUM}') { reviewDecision } } }"}' https://api.github.com/graphql |jq -r '.data.repository.pullRequest.reviewDecision') + echo "Review decision for PR #${PR_NUM} in repository ${REPO_OWNER}/${REPO_NAME} is ${PR_REVIEW_DECISION}" + if [[ "$PR_REVIEW_DECISION" == "APPROVED" ]]; then + return 0 + fi + fi + + FORK_REPO_URL=$(jq -r '.pull_request.head.repo.html_url' "$GITHUB_EVENT_PATH") + PR_BRANCH_LABEL=$(jq -r '.pull_request.head.label' "$GITHUB_EVENT_PATH") + PR_BASE_BRANCH=$(jq -r '.pull_request.base.ref' "$GITHUB_EVENT_PATH") + PR_URL=$(jq -r '.pull_request.html_url' "$GITHUB_EVENT_PATH") + FORK_PR_TITLE_URL_ENCODED=$(printf "%s" "${PR_JSON}" | jq -r '"[run-tests] " + .title | @uri') + FORK_PR_BODY_URL_ENCODED=$(jq -n -r "\"This PR is for running tests for upstream PR ${PR_URL}.\n\n\" | @uri") + if [[ "$PR_BASE_BRANCH" != "master" ]]; then + sync_non_master_fork_docs=$(cat <&2 tee -a "$GITHUB_STEP_SUMMARY" < /tmp/provided_pulsar_maven_artifacts + ) +} + +ci_upload_unittest_coverage_files() { + _ci_upload_coverage_files unittest "$1" +} + +ci_upload_inttest_coverage_files() { + _ci_upload_coverage_files_inttest inttest "$1" integration-tests +} + +ci_upload_systest_coverage_files() { + _ci_upload_coverage_files_inttest systest "$1" system-tests +} + +_ci_upload_coverage_files_inttest() { + local testtype="$1" + local testgroup="$2" + local job_name="$3" + local store_deps=0 + local firsttestgroup="$(_ci_list_testgroups_with_coverage "${job_name}" | head -1)" + if [[ "${firsttestgroup}" == "${testgroup}" ]]; then + store_deps=1 + fi + _ci_upload_coverage_files "${testtype}" "${testgroup}" $store_deps +} + +_ci_upload_coverage_files() { + ( + testtype="$1" + testgroup="$2" + store_deps="${3:-1}" + echo "::group::Uploading $testtype coverage files" + if [ -n "$GITHUB_WORKSPACE" ]; then + cd "$GITHUB_WORKSPACE" + else + fail "This script can only be run in GitHub Actions" + fi + + if [ ! -f /tmp/provided_pulsar_maven_artifacts ]; then + fail "It is necessary to run '$0 snapshot_pulsar_maven_artifacts' before running any tests." + fi + + set -x + + local classpathFile="target/classpath_${testtype}_${testgroup}" + + local execFiles=$(find . -path "*/target/jacoco*.exec" -printf "%P\n") + if [[ -n "$execFiles" ]]; then + # create temp file + local completeClasspathFile=$(mktemp -t tmp.classpath.XXXX) + + ci_install_tool xmlstarlet + + local projects=$({ + for execFile in $execFiles; do + echo $(dirname "$(dirname "$execFile")") + done + } | sort | uniq) + + # iterate the projects + for project in $projects; do + local artifactId=$(xmlstarlet sel -t -m _:project -v _:artifactId -n $project/pom.xml) + # find the test scope classpath for the project + mvn -f $project/pom.xml -DincludeScope=test -Dscan=false dependency:build-classpath -B | { grep 'Dependencies classpath:' -A1 || true; } | tail -1 \ + | sed 's/:/\n/g' | { grep 'org/apache/pulsar' || true; } \ + | { tee -a $completeClasspathFile || true; } > target/classpath_$artifactId || true + done + + cat $completeClasspathFile | sort | uniq > $classpathFile + # delete temp file + rm $completeClasspathFile + + # upload target/jacoco*.exec, target/classes and any dependent jar files that were built during the unit test execution + # transform jacoco exec filenames by appending "_${testtype}_${testgroup}" to the filename part to make the files unique + # so that they don't get overridden when all files are extracted to the same working directory before merging + ( + cd / + ci_store_tar_to_github_actions_artifacts coverage_and_deps_${testtype}_${testgroup} \ + tar -I zstd -cPf - \ + --transform="flags=r;s|\\(/jacoco.*\\).exec$|\\1_${testtype}_${testgroup}.exec|" \ + --transform="flags=r;s|\\(/tmp/jacocoDir/.*\\).exec$|\\1_${testtype}_${testgroup}.exec|" \ + --exclude="*/META-INF/bundled-dependencies/*" \ + --exclude="*/META-INF/versions/*" \ + $GITHUB_WORKSPACE/target/classpath_* \ + $(find "$GITHUB_WORKSPACE" -path "*/target/jacoco*.exec" -printf "%p\n%h/classes\n" | sort | uniq) \ + $([ -d /tmp/jacocoDir ] && echo "/tmp/jacocoDir" ) \ + $([[ $store_deps -eq 1 ]] && { cat $GITHUB_WORKSPACE/$classpathFile | sort | uniq | { grep -v -Fx -f /tmp/provided_pulsar_maven_artifacts || true; }; } || true) + ) + fi + echo "::endgroup::" + ) +} + +ci_restore_unittest_coverage_files() { + _ci_restore_coverage_files unittest unit-tests +} + +ci_restore_inttest_coverage_files() { + _ci_restore_coverage_files inttest integration-tests +} + +ci_restore_systest_coverage_files() { + _ci_restore_coverage_files systest system-tests +} + +_ci_restore_coverage_files() { + ( + test_type="$1" + job_name="$2" + cd / + for testgroup in $(_ci_list_testgroups_with_coverage "${job_name}"); do + ci_restore_tar_from_github_actions_artifacts coverage_and_deps_${test_type}_${testgroup} || true + done + ) +} + +_ci_list_testgroups_with_coverage() { + local job_name="$1" + yq e ".jobs.${job_name}.strategy.matrix.include.[] | select(.no_coverage != true) | .group" "$GITHUB_WORKSPACE/.github/workflows/pulsar-ci.yaml" +} + +ci_delete_unittest_coverage_files() { + _ci_delete_coverage_files unittest unit-tests +} + +ci_delete_inttest_coverage_files() { + _ci_delete_coverage_files inttest integration-tests +} + +ci_delete_systest_coverage_files() { + _ci_delete_coverage_files systest system-tests +} + +_ci_delete_coverage_files() { + ( + test_type="$1" + job_name="$2" + for testgroup in $(yq e ".jobs.${job_name}.strategy.matrix.include.[] | select(.no_coverage != true) | .group" "$GITHUB_WORKSPACE/.github/workflows/pulsar-ci.yaml"); do + timeout 1m gh-actions-artifact-client.js delete coverage_and_deps_${test_type}_${testgroup}.tar.zst || true + done + ) +} + +# creates an aggregated jacoco xml report for all projects that contain a target/jacoco*.exec file +# +# the default maven jacoco report has multiple problems: +# - by default, jacoco:report goal will only report coverage for the current project. it is not suitable for Pulsar's +# unit tests that test production code that resides in multiple modules. +# - there's jacoco:report-aggregate that is supposed to resolve this. It has 2 issues: +# - 0.8.8 version doesn't yet support the required "includeCurrentProject" feature +# - the dependent projects must be built as part of the same mvn execution and belong to the same maven "reactor" +# - this isn't compatible with the way how Pulsar CI builds in "Build and License check" job and reuses +# the build results to run unit tests. +# +# This solution resolves the problem by using the Jacoco command line tool to generate the report. +# It assumes that all projects that contain a target/jacoco*.exec file will also contain compiled classfiles. +ci_create_test_coverage_report() { + echo "::group::Create test coverage report" + if [ -n "$GITHUB_WORKSPACE" ]; then + cd "$GITHUB_WORKSPACE" + else + cd "$SCRIPT_DIR/.." + fi + + local execFiles=$(find . -path "*/target/jacoco*.exec" -size +10c -printf "%P\n" ) + if [[ -n "$execFiles" ]]; then + mkdir -p /tmp/jacocoDir + if [ ! -f /tmp/jacocoDir/jacococli.jar ]; then + local jacoco_version=$(mvn help:evaluate -Dscan=false -Dexpression=jacoco-maven-plugin.version -q -DforceStdout) + curl -sL -o /tmp/jacocoDir/jacococli.jar "https://repo1.maven.org/maven2/org/jacoco/org.jacoco.cli/${jacoco_version}/org.jacoco.cli-${jacoco_version}-nodeps.jar" + fi + + ci_install_tool xmlstarlet + # create mapping from project directory to project artifactId + local projectToArtifactIdMapping=$(find -name pom.xml -printf "%P\n" |xargs -I{} bash -c 'echo -n "$(dirname $1) "; xmlstarlet sel -t -m _:project -v _:artifactId -n $1' -- {}) + + # create temp files + local completeClasspathFile=$(mktemp -t tmp.classpath.XXXX) + local filterArtifactsFile=$(mktemp -t tmp.artifacts.XXXX) + local classesDir=$(mktemp -d -t tmp.classes.XXXX) + local sourcefilesFile=$(mktemp -t tmp.sources.XXXX) + + local projects=$({ + for execFile in $execFiles; do + echo $(dirname "$(dirname "$execFile")") + done + } | sort | uniq) + + # projects that aren't considered as production code and their own src/main/java source code shouldn't be analysed + local excludeProjectsPattern="testmocks|testclient|buildtools" + + # iterate projects + for project in $projects; do + local artifactId="$(printf "%s" "$projectToArtifactIdMapping" | grep -F "$project " | cut -d' ' -f2)" + if [[ -d "$project/target/classes" && -d "$project/src/main/java" ]]; then + mkdir -p "$classesDir/$project" + cp -Rl "$project/target/classes" "$classesDir/$project" + echo "$project/src/main/java" >> $sourcefilesFile + fi + echo "/$artifactId/" >> $filterArtifactsFile + if [[ -n "$(echo "$project" | grep -v -E "$excludeProjectsPattern")" ]]; then + if [ -f "target/classpath_$artifactId" ]; then + echo "Found cached classpath for $artifactId." + cat "target/classpath_$artifactId" >> $completeClasspathFile + else + echo "Resolving classpath for $project..." + # find the test scope classpath for the project + mvn -f $project/pom.xml -DincludeScope=test -Dscan=false dependency:build-classpath -B | { grep 'Dependencies classpath:' -A1 || true; } | tail -1 \ + | sed 's/:/\n/g' | { grep 'org/apache/pulsar' || true; } \ + >> $completeClasspathFile || true + fi + else + echo "Skipping analysing of $project" + fi + done + + # delete any possible embedded jar files in the classes directory + find "$classesDir" -name "*.jar" -print -delete + + local excludeJarsPattern="bouncy-castle-bc|tests|$excludeProjectsPattern" + + local classfilesArgs="--classfiles $({ + { + for classpathEntry in $(cat $completeClasspathFile | { grep -v -f $filterArtifactsFile || true; } | sort | uniq | { grep -v -E "$excludeJarsPattern" || true; }); do + if [[ -f $classpathEntry && -n "$(unzip -Z1C $classpathEntry 'META-INF/bundled-dependencies/*' 'META-INF/versions/*' 2>/dev/null)" ]]; then + # file must be processed by removing META-INF/bundled-dependencies and META-INF/versions + local jartempfile=$(mktemp -t jarfile.XXXX --suffix=.jar) + cp $classpathEntry $jartempfile + zip -q -d $jartempfile 'META-INF/bundled-dependencies/*' 'META-INF/versions/*' &> /dev/null + echo $jartempfile + else + echo $classpathEntry + fi + done + } + echo $classesDir + } | tr '\n' ':' | sed -e 's/:$//' -e 's/:/ --classfiles /g')" + + local sourcefilesArgs="--sourcefiles $({ + # find the source file folders for the pulsar .jar files that are on the classpath + for artifactId in $(cat $completeClasspathFile | sort | uniq | { grep -v -E "$excludeJarsPattern" || true; } | perl -p -e 's|.*/org/apache/pulsar/([^/]*)/.*|$1|'); do + local project="$(printf "%s" "$projectToArtifactIdMapping" | { grep $artifactId || true; } | cut -d' ' -f1)" + if [[ -n "$project" && -d "$project/src/main/java" ]]; then + echo "$project/src/main/java" + fi + done + cat $sourcefilesFile + } | tr '\n' ':' | sed -e 's/:$//' -e 's/:/ --sourcefiles /g')" + + rm $completeClasspathFile $filterArtifactsFile $sourcefilesFile + + set -x + mkdir -p target/jacoco_test_coverage_report/html + java -jar /tmp/jacocoDir/jacococli.jar report $execFiles \ + $classfilesArgs \ + --encoding UTF-8 --name "Apache Pulsar test coverage" \ + $sourcefilesArgs \ + --xml target/jacoco_test_coverage_report/jacoco.xml \ + --html target/jacoco_test_coverage_report/html \ + --csv target/jacoco_test_coverage_report/jacoco.csv + set +x + + rm -rf "$classesDir" + fi + echo "::endgroup::" +} + +# creates a jacoco xml report of the jacoco exec files produced in docker containers which have /tmp/jacocoDir mounted as /jacocoDir +# this is used to calculate test coverage for the apache/pulsar code that is run inside the containers in integration tests +# and system tests +ci_create_inttest_coverage_report() { + echo "::group::Create int test coverage in containers report" + if [[ -n "$(find /tmp/jacocoDir -name "*.exec" -print -quit)" ]]; then + cd "$GITHUB_WORKSPACE" + echo "Creating coverage report to target/jacoco_inttest_coverage_report" + set -x + mkdir -p target/jacoco_inttest_coverage_report + # install jacococli.jar command line tool + if [ ! -f /tmp/jacocoDir/jacococli.jar ]; then + local jacoco_version=$(mvn help:evaluate -Dexpression=jacoco-maven-plugin.version -Dscan=false -q -DforceStdout) + curl -sL -o /tmp/jacocoDir/jacococli.jar "https://repo1.maven.org/maven2/org/jacoco/org.jacoco.cli/${jacoco_version}/org.jacoco.cli-${jacoco_version}-nodeps.jar" + fi + # extract the Pulsar jar files from the docker image that was used to run the tests in the docker containers + # the class files used to produce the jacoco exec files are needed in the xml report generation + if [ ! -d /tmp/jacocoDir/pulsar_lib ]; then + mkdir /tmp/jacocoDir/pulsar_lib + docker run --rm -u "$UID:${GID:-"$(id -g)"}" -v /tmp/jacocoDir/pulsar_lib:/pulsar_lib:rw ${PULSAR_TEST_IMAGE_NAME:-apachepulsar/java-test-image:latest} bash -c "cp -p /pulsar/lib/org.apache.pulsar-* /pulsar_lib; [ -d /pulsar/connectors ] && cp -R /pulsar/connectors /pulsar_lib || true" + # remove jar file that causes duplicate classes issue + rm /tmp/jacocoDir/pulsar_lib/org.apache.pulsar-bouncy-castle* || true + # remove any bundled dependencies as part of .jar/.nar files + find /tmp/jacocoDir/pulsar_lib '(' -name "*.jar" -or -name "*.nar" ')' -exec echo "Processing {}" \; -exec zip -q -d {} 'META-INF/bundled-dependencies/*' 'META-INF/versions/*' \; |grep -E -v "Nothing to do|^$" || true + fi + # projects that aren't considered as production code and their own src/main/java source code shouldn't be analysed + local excludeProjectsPattern="testmocks|testclient|buildtools" + # produce jacoco XML coverage report from the exec files and using the extracted jar files + java -jar /tmp/jacocoDir/jacococli.jar report /tmp/jacocoDir/*.exec \ + --classfiles /tmp/jacocoDir/pulsar_lib --encoding UTF-8 --name "Pulsar Integration Tests - coverage in containers" \ + $(find -path "*/src/main/java" -printf "--sourcefiles %P " | grep -v -E "$excludeProjectsPattern") \ + --xml target/jacoco_inttest_coverage_report/jacoco.xml \ + --html target/jacoco_inttest_coverage_report/html \ + --csv target/jacoco_inttest_coverage_report/jacoco.csv + set +x + else + echo "No /tmp/jacocoDir/*.exec files to process" + fi + echo "::endgroup::" +} if [ -z "$1" ]; then echo "usage: $0 [ci_tool_function_name]" @@ -60,11 +624,11 @@ fi ci_function_name="ci_$1" shift -if [[ "$(LC_ALL=C type -t $ci_function_name)" == "function" ]]; then +if [[ "$(LC_ALL=C type -t "${ci_function_name}")" == "function" ]]; then eval "$ci_function_name" "$@" else echo "Invalid ci tool function" echo "Available ci tool functions:" ci_list_functions exit 1 -fi \ No newline at end of file +fi diff --git a/build/run_integration_group.sh b/build/run_integration_group.sh index 0c91647acd9c7..960386b138b19 100755 --- a/build/run_integration_group.sh +++ b/build/run_integration_group.sh @@ -24,18 +24,45 @@ set -e set -o pipefail set -o errexit -TEST_GROUP=$1 -if [ -z "$TEST_GROUP" ]; then - echo "usage: $0 [test_group]" - exit 1 -fi -shift +JAVA_MAJOR_VERSION="$(java -version 2>&1 |grep " version " | awk -F\" '{ print $2 }' | awk -F. '{ if ($1=="1") { print $2 } else { print $1 } }')" +# Used to shade run test on Java 8, because the latest TestNG requires Java 11 or higher. +TESTNG_VERSION="7.3.0" + +# lists all active maven modules with given parameters +# parses the modules from the "mvn initialize" output +# returns a CSV value +mvn_list_modules() { + ( + mvn -B -ntp -Dscan=false "$@" initialize \ + | grep -- "-< .* >-" \ + | sed -E 's/.*-< (.*) >-.*/\1/' \ + | tr '\n' ',' | sed 's/,$/\n/' + ) +} # runs integration tests +# 1. cds to "tests" directory and lists the active modules to be used as value +# for "-pl" parameter of later mvn commands +# 2. runs "mvn -pl [active_modules] -am install [given_params]" to build and install required dependencies +# 3. finally runs tests with "mvn -pl [active_modules] test [given_params]" mvn_run_integration_test() { - ( - set -x - + set +x + # skip test run if next parameter is "--build-only" + local build_only=0 + if [[ "$1" == "--build-only" ]]; then + build_only=1 + shift + fi + local skip_build_deps=0 + while [[ "$1" == "--skip-build-deps" ]]; do + skip_build_deps=1 + shift + done + local clean_arg="" + if [[ "$1" == "--clean" ]]; then + clean_arg="clean" + shift + fi local use_fail_fast=1 if [[ "$GITHUB_ACTIONS" == "true" && "$GITHUB_EVENT_NAME" != "pull_request" ]]; then use_fail_fast=0 @@ -50,8 +77,33 @@ mvn_run_integration_test() { else failfast_args="-DtestFailFast=false --fail-at-end" fi - # run the integration tests - mvn -B -ntp -DredirectTestOutputToFile=false $failfast_args -f tests/pom.xml test "$@" + local coverage_args="" + if [[ "$1" == "--coverage" ]]; then + if [ ! -d /tmp/jacocoDir ]; then + mkdir /tmp/jacocoDir + sudo chmod 0777 /tmp/jacocoDir || chmod 0777 /tmp/jacocoDir + fi + coverage_args="-Pcoverage -Dintegrationtest.coverage.enabled=true -Dintegrationtest.coverage.dir=/tmp/jacocoDir" + shift + fi + ( + cd "$SCRIPT_DIR"/../tests + local modules=$(mvn_list_modules -DskipDocker "$@") + cd .. + set -x + if [ $skip_build_deps -ne 1 ]; then + echo "::group::Build dependencies for $modules" + mvn -B -T 1C -ntp -pl "$modules" -DskipDocker -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true -Dmaven.test.skip=true -Dcheckstyle.skip=true -Drat.skip=true -am $clean_arg install "$@" + echo "::endgroup::" + fi + if [[ $build_only -ne 1 ]]; then + echo "::group::Run tests for " "$@" + # use "verify" instead of "test" + mvn -B -ntp -pl "$modules" $failfast_args $coverage_args -DskipDocker -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true $clean_arg verify "$@" + echo "::endgroup::" + set +x + "$SCRIPT_DIR/pulsar_ci_tool.sh" move_test_reports || true + fi ) } @@ -59,6 +111,23 @@ test_group_shade() { mvn_run_integration_test "$@" -DShadeTests -DtestForkCount=1 -DtestReuseFork=false } +test_group_shade_build() { + mvn_run_integration_test --build-only "$@" -DShadeTests -DtestForkCount=1 -DtestReuseFork=false +} + +test_group_shade_run() { + local additional_args + if [[ $JAVA_MAJOR_VERSION -gt 8 && $JAVA_MAJOR_VERSION -lt 17 ]]; then + additional_args="-Dmaven.compiler.source=$JAVA_MAJOR_VERSION -Dmaven.compiler.target=$JAVA_MAJOR_VERSION" + fi + + if [[ $JAVA_MAJOR_VERSION -ge 8 && $JAVA_MAJOR_VERSION -lt 11 ]]; then + additional_args="$additional_args -Dtestng.version=$TESTNG_VERSION" + fi + + mvn_run_integration_test --skip-build-deps --clean "$@" -Denforcer.skip=true -DShadeTests -DtestForkCount=1 -DtestReuseFork=false $additional_args +} + test_group_backwards_compat() { mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-backwards-compatibility.xml -DintegrationTests mvn_run_integration_test "$@" -DBackwardsCompatTests -DtestForkCount=1 -DtestReuseFork=false @@ -69,7 +138,7 @@ test_group_cli() { mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-cli.xml -DintegrationTests # run pulsar auth integration tests - mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-auth.xml -DintegrationTests + mvn_run_integration_test --skip-build-deps "$@" -DintegrationTestSuiteFile=pulsar-auth.xml -DintegrationTests } test_group_function() { @@ -80,9 +149,15 @@ test_group_messaging() { # run integration messaging tests mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-messaging.xml -DintegrationTests # run integration proxy tests - mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-proxy.xml -DintegrationTests - # run integration proxy with WebSocket tests - mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-proxy-websocket.xml -DintegrationTests + mvn_run_integration_test --skip-build-deps "$@" -DintegrationTestSuiteFile=pulsar-proxy.xml -DintegrationTests +} + +test_group_loadbalance() { + mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-loadbalance.xml -DintegrationTests +} + +test_group_plugin() { + mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-plugin.xml -DintegrationTests } test_group_schema() { @@ -110,10 +185,10 @@ test_group_pulsar_connectors_thread() { mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-thread.xml -DintegrationTests -Dgroups=function # run integration source - mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-thread.xml -DintegrationTests -Dgroups=source + mvn_run_integration_test --skip-build-deps "$@" -DintegrationTestSuiteFile=pulsar-thread.xml -DintegrationTests -Dgroups=source # run integration sink - mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-thread.xml -DintegrationTests -Dgroups=sink + mvn_run_integration_test --skip-build-deps "$@" -DintegrationTestSuiteFile=pulsar-thread.xml -DintegrationTests -Dgroups=sink } test_group_pulsar_connectors_process() { @@ -121,10 +196,10 @@ test_group_pulsar_connectors_process() { mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-process.xml -DintegrationTests -Dgroups=function # run integration source - mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-process.xml -DintegrationTests -Dgroups=source + mvn_run_integration_test --skip-build-deps "$@" -DintegrationTestSuiteFile=pulsar-process.xml -DintegrationTests -Dgroups=source - # run integraion sink - mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-process.xml -DintegrationTests -Dgroups=sink + # run integration sink + mvn_run_integration_test --skip-build-deps "$@" -DintegrationTestSuiteFile=pulsar-process.xml -DintegrationTests -Dgroups=sink } test_group_sql() { @@ -133,20 +208,33 @@ test_group_sql() { test_group_pulsar_io() { mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-io-sources.xml -DintegrationTests -Dgroups=source - mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-io-sinks.xml -DintegrationTests -Dgroups=sink + mvn_run_integration_test --skip-build-deps "$@" -DintegrationTestSuiteFile=pulsar-io-sinks.xml -DintegrationTests -Dgroups=sink } test_group_pulsar_io_ora() { mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-io-ora-source.xml -DintegrationTests -Dgroups=source -DtestRetryCount=0 } +list_test_groups() { + declare -F | awk '{print $NF}' | sort | grep -E '^test_group_' | sed 's/^test_group_//g' | tr '[:lower:]' '[:upper:]' +} +TEST_GROUP=$1 +if [ -z "$TEST_GROUP" ]; then + echo "usage: $0 [test_group]" + echo "Available test groups:" + list_test_groups + exit 1 +fi +shift echo "Test Group : $TEST_GROUP" test_group_function_name="test_group_$(echo "$TEST_GROUP" | tr '[:upper:]' '[:lower:]')" -if [[ "$(LC_ALL=C type -t $test_group_function_name)" == "function" ]]; then +if [[ "$(LC_ALL=C type -t "${test_group_function_name}")" == "function" ]]; then eval "$test_group_function_name" "$@" else echo "INVALID TEST GROUP" + echo "Available test groups:" + list_test_groups exit 1 fi diff --git a/build/run_unit_group.sh b/build/run_unit_group.sh index c81409b76c53c..de6f40407ccf6 100755 --- a/build/run_unit_group.sh +++ b/build/run_unit_group.sh @@ -18,22 +18,30 @@ # under the License. # +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" + set -e set -o pipefail set -o errexit -# solution for printing output in "set -x" trace mode without tracing the echo calls -shopt -s expand_aliases -echo_and_restore_trace() { - builtin echo "$@" - [ $trace_enabled -eq 1 ] && set -x || true -} -alias echo='{ [[ $- =~ .*x.* ]] && trace_enabled=1 || trace_enabled=0; set +x; } 2> /dev/null; echo_and_restore_trace' - -MVN_COMMAND='mvn -B -ntp' +MVN_TEST_OPTIONS='mvn -B -ntp -DskipSourceReleaseAssembly=true -DskipBuildDistribution=true -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true' function mvn_test() { ( + local clean_arg="" + if [[ "$1" == "--clean" ]]; then + clean_arg="clean" + shift + fi + local coverage_arg="-Pcoverage" + if [[ "${COLLECT_COVERAGE}" == "false" ]]; then + coverage_arg="" + fi + local target="verify" + if [[ "$1" == "--install" ]]; then + target="install" + shift + fi local use_fail_fast=1 if [[ "$GITHUB_ACTIONS" == "true" && "$GITHUB_EVENT_NAME" != "pull_request" ]]; then use_fail_fast=0 @@ -49,36 +57,50 @@ function mvn_test() { failfast_args="-DtestFailFast=false --fail-at-end" fi echo "::group::Run tests for " "$@" - $MVN_COMMAND test $failfast_args "$@" + # use "verify" instead of "test" to workaround MDEP-187 issue in pulsar-functions-worker and pulsar-broker projects with the maven-dependency-plugin's copy goal + # Error message was "Artifact has not been packaged yet. When used on reactor artifact, copy should be executed after packaging: see MDEP-187" + $MVN_TEST_OPTIONS $failfast_args $clean_arg $target $coverage_arg "$@" "${COMMANDLINE_ARGS[@]}" echo "::endgroup::" + set +x + "$SCRIPT_DIR/pulsar_ci_tool.sh" move_test_reports ) } -echo -n "Test Group : $TEST_GROUP" +# solution for printing output in "set -x" trace mode without tracing the echo calls +shopt -s expand_aliases +echo_and_restore_trace() { + builtin echo "$@" + [ $trace_enabled -eq 1 ] && set -x || true +} +alias echo='{ [[ $- =~ .*x.* ]] && trace_enabled=1 || trace_enabled=0; set +x; } 2> /dev/null; echo_and_restore_trace' # Test Groups -- start -- -function broker_group_1() { - mvn_test -pl pulsar-broker -Dgroups='broker' -DtestReuseFork=true -DskipAfterFailureCount=1 +function test_group_broker_group_1() { + mvn_test -pl pulsar-broker -Dgroups='broker' -DtestReuseFork=true } -function broker_group_2() { - mvn_test -pl pulsar-broker -Dgroups='schema,utils,functions-worker,broker-io,broker-discovery,broker-compaction,broker-naming,websocket,other' -DtestReuseFork=false -DfailIfNoTests=false +function test_group_broker_group_2() { + mvn_test -pl pulsar-broker -Dgroups='schema,utils,functions-worker,broker-io,broker-discovery,broker-compaction,broker-naming,websocket,other' } -function broker_group_3() { +function test_group_broker_group_3() { mvn_test -pl pulsar-broker -Dgroups='broker-admin' } -function broker_client_api() { +function test_group_broker_group_4() { + mvn_test -pl pulsar-broker -Dgroups='cluster-migration' +} + +function test_group_broker_client_api() { mvn_test -pl pulsar-broker -Dgroups='broker-api' } -function broker_client_impl() { +function test_group_broker_client_impl() { mvn_test -pl pulsar-broker -Dgroups='broker-impl' } -function broker_jdk8() { - mvn_test -pl pulsar-broker -Dgroups='broker-jdk8' -Dpulsar.allocator.pooled=true +function test_group_client() { + mvn_test -pl pulsar-client } # prints summaries of failed tests to console @@ -101,7 +123,7 @@ function print_testng_failures() { fi local test_report_file="${testng_report_dir}/${failed_test_class}.txt" if [ -f "${test_report_file}" ]; then - local test_report="$(cat "${test_report_file}" | egrep "^Tests run: " | perl -p -se 's/^(Tests run: .*) <<< FAILURE! - in (.*)$/::warning::$report_prefix $2 - $1/' -- -report_prefix="${report_prefix}")" + local test_report="$(cat "${test_report_file}" | grep -E "^Tests run: " | perl -p -se 's/^(Tests run: .*) <<< FAILURE! - in (.*)$/::warning::$report_prefix $2 - $1/' -- -report_prefix="${report_prefix}")" echo "$test_report" cat "${test_report_file}" fi @@ -110,7 +132,7 @@ function print_testng_failures() { ) } -function broker_flaky() { +function test_group_broker_flaky() { echo "::endgroup::" echo "::group::Running quarantined tests" mvn_test --no-fail-fast -pl pulsar-broker -Dgroups='quarantine' -DexcludedGroups='flaky' -DfailIfNoTests=false \ @@ -130,7 +152,7 @@ function broker_flaky() { fi } -function proxy() { +function test_group_proxy() { echo "::group::Running pulsar-proxy tests" mvn_test -pl pulsar-proxy -Dtest="org.apache.pulsar.proxy.server.ProxyServiceTlsStarterTest" mvn_test -pl pulsar-proxy -Dtest="org.apache.pulsar.proxy.server.ProxyServiceStarterTest" @@ -139,28 +161,25 @@ function proxy() { echo "::endgroup::" } -function other() { - $MVN_COMMAND clean install -PbrokerSkipTest \ - -Dexclude='org/apache/pulsar/proxy/**/*.java, - **/ManagedLedgerTest.java, - **/TestPulsarKeyValueSchemaHandler.java, - **/PrimitiveSchemaTest.java, - **/BlobStoreManagedLedgerOffloaderTest.java, - **/BlobStoreManagedLedgerOffloaderStreamingTest.java' +function test_group_other() { + mvn_test --clean --install \ + -pl '!org.apache.pulsar:distribution,!org.apache.pulsar:pulsar-offloader-distribution,!org.apache.pulsar:pulsar-server-distribution,!org.apache.pulsar:pulsar-io-distribution,!org.apache.pulsar:pulsar-all-docker-image' \ + -PskipTestsForUnitGroupOther -DdisableIoMainProfile=true -DdisableSqlMainProfile=true -DskipIntegrationTests \ + -Dexclude='**/ManagedLedgerTest.java, + **/OffloadersCacheTest.java + **/PrimitiveSchemaTest.java, + **/BlobStoreManagedLedgerOffloaderTest.java, + **/BlobStoreManagedLedgerOffloaderStreamingTest.java' mvn_test -pl managed-ledger -Dinclude='**/ManagedLedgerTest.java, **/OffloadersCacheTest.java' - mvn_test -pl pulsar-sql/presto-pulsar-plugin -Dinclude='**/TestPulsarKeyValueSchemaHandler.java' - - mvn_test -pl pulsar-client -Dinclude='**/PrimitiveSchemaTest.java' - mvn_test -pl tiered-storage/jcloud -Dinclude='**/BlobStoreManagedLedgerOffloaderTest.java' mvn_test -pl tiered-storage/jcloud -Dinclude='**/BlobStoreManagedLedgerOffloaderStreamingTest.java' echo "::endgroup::" local modules_with_quarantined_tests=$(git grep -l '@Test.*"quarantine"' | grep '/src/test/java/' | \ - awk -F '/src/test/java/' '{ print $1 }' | egrep -v 'pulsar-broker|pulsar-proxy' | sort | uniq | \ + awk -F '/src/test/java/' '{ print $1 }' | grep -v -E 'pulsar-broker|pulsar-proxy|pulsar-io|pulsar-sql|pulsar-client' | sort | uniq | \ perl -0777 -p -e 's/\n(\S)/,$1/g') if [ -n "${modules_with_quarantined_tests}" ]; then echo "::group::Running quarantined tests outside of pulsar-broker & pulsar-proxy (if any)" @@ -171,54 +190,56 @@ function other() { fi } -# Test Groups -- end -- - -TEST_GROUP=$1 - -echo "Test Group : $TEST_GROUP" - -set -x - -case $TEST_GROUP in - - BROKER_GROUP_1) - broker_group_1 - ;; - - BROKER_GROUP_2) - broker_group_2 - ;; - - BROKER_GROUP_3) - broker_group_3 - ;; +function test_group_pulsar_io() { + echo "::group::Running pulsar-io tests" + mvn_test --install -Ppulsar-io-tests,-main + echo "::endgroup::" - BROKER_CLIENT_API) - broker_client_api - ;; + echo "::group::Running pulsar-sql tests" + mvn_test --install -Ppulsar-sql-tests,-main -DtestForkCount=1 + echo "::endgroup::" +} - BROKER_CLIENT_IMPL) - broker_client_impl - ;; +function test_group_pulsar_io_elastic() { + echo "::group::Running elastic-search tests" + mvn_test --install -Ppulsar-io-elastic-tests,-main + echo "::endgroup::" +} - BROKER_FLAKY) - broker_flaky - ;; +function test_group_pulsar_io_kafka_connect() { + echo "::group::Running Pulsar IO Kafka connect adaptor tests" + mvn_test --install -Ppulsar-io-kafka-connect-tests,-main + echo "::endgroup::" +} - PROXY) - proxy - ;; +function list_test_groups() { + declare -F | awk '{print $NF}' | sort | grep -E '^test_group_' | sed 's/^test_group_//g' | tr '[:lower:]' '[:upper:]' +} - OTHER) - other - ;; +# Test Groups -- end -- - BROKER_JDK8) - broker_jdk8 - ;; +if [[ "$1" == "--list" ]]; then + list_test_groups + exit 0 +fi - *) - echo -n "INVALID TEST GROUP" - exit 1 - ;; -esac +TEST_GROUP=$1 +if [ -z "$TEST_GROUP" ]; then + echo "usage: $0 [test_group]" + echo "Available test groups:" + list_test_groups + exit 1 +fi +shift +COMMANDLINE_ARGS=("$@") +echo "Test Group : $TEST_GROUP" +test_group_function_name="test_group_$(echo "$TEST_GROUP" | tr '[:upper:]' '[:lower:]')" +if [[ "$(LC_ALL=C type -t "${test_group_function_name}")" == "function" ]]; then + set -x + eval "$test_group_function_name" +else + echo "INVALID TEST GROUP" + echo "Available test groups:" + list_test_groups + exit 1 +fi diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 720542b862259..0a56931bbf3a7 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -25,7 +25,7 @@ org.apache apache - 23 + 29 @@ -45,7 +45,7 @@ 1.8 1.8 - 3.0.0-M3 + 3.5.3 2.18.0 1.7.32 7.3.0 @@ -102,11 +102,6 @@ testng ${testng.version} - - junit - junit - 4.13.1 - org.apache.logging.log4j log4j-api @@ -135,12 +130,17 @@ - io.netty netty-common - 4.1.86.Final - test + ${netty.version} + provided + + + io.netty + netty-buffer + ${netty.version} + provided org.mockito @@ -189,11 +189,17 @@ ${test.additional.args} + + + org.apache.maven.surefire + surefire-testng + ${surefire.version} + + org.apache.maven.plugins maven-shade-plugin - ${maven-shade-plugin.version} true true @@ -253,7 +259,7 @@ org.apache.maven.wagon wagon-ssh-external - 2.10 + 3.5.3 diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/BetweenTestClassesListenerAdapter.java b/buildtools/src/main/java/org/apache/pulsar/tests/BetweenTestClassesListenerAdapter.java index 7e803f29ef16a..16f484c97c2d1 100644 --- a/buildtools/src/main/java/org/apache/pulsar/tests/BetweenTestClassesListenerAdapter.java +++ b/buildtools/src/main/java/org/apache/pulsar/tests/BetweenTestClassesListenerAdapter.java @@ -18,43 +18,34 @@ */ package org.apache.pulsar.tests; -import org.testng.IClassListener; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.testng.ITestClass; import org.testng.ITestContext; import org.testng.ITestListener; +import org.testng.ITestNGMethod; /** - * TestNG listener adapter for detecting when execution finishes in previous - * test class and starts in a new class. + * TestNG listener adapter that detects when execution finishes for a test class, + * assuming that a single test class is run in each context. + * This is the case when running tests with maven-surefire-plugin. */ -abstract class BetweenTestClassesListenerAdapter implements IClassListener, ITestListener { - Class lastTestClass; +abstract class BetweenTestClassesListenerAdapter implements ITestListener { + private static final Logger log = LoggerFactory.getLogger(BetweenTestClassesListenerAdapter.class); @Override - public void onBeforeClass(ITestClass testClass) { - checkIfTestClassChanged(testClass.getRealClass()); - } - - private void checkIfTestClassChanged(Class testClazz) { - if (lastTestClass != testClazz) { - onBetweenTestClasses(lastTestClass, testClazz); - lastTestClass = testClazz; - } - } - - @Override - public void onFinish(ITestContext context) { - if (lastTestClass != null) { - onBetweenTestClasses(lastTestClass, null); - lastTestClass = null; - } + public final void onFinish(ITestContext context) { + List testClasses = + Arrays.stream(context.getAllTestMethods()).map(ITestNGMethod::getTestClass).distinct() + .collect(Collectors.toList()); + onBetweenTestClasses(testClasses); } /** - * Call back hook for adding logic when test execution moves from test class to another. - * - * @param endedTestClass the test class which has finished execution. null if the started test class is the first - * @param startedTestClass the test class which has started execution. null if the ended test class is the last + * Call back hook for adding logic when test execution has completely finished for one or many test classes. */ - protected abstract void onBetweenTestClasses(Class endedTestClass, Class startedTestClass); + protected abstract void onBetweenTestClasses(List testClasses); } diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/ExtendedNettyLeakDetector.java b/buildtools/src/main/java/org/apache/pulsar/tests/ExtendedNettyLeakDetector.java new file mode 100644 index 0000000000000..376fdee1e7628 --- /dev/null +++ b/buildtools/src/main/java/org/apache/pulsar/tests/ExtendedNettyLeakDetector.java @@ -0,0 +1,258 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufAllocator; +import io.netty.util.ResourceLeakDetector; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import org.apache.logging.log4j.LogManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A custom Netty leak detector used in Pulsar tests that dumps the detected leaks to a file in a directory that is + * configured with the NETTY_LEAK_DUMP_DIR environment variable. This directory defaults to java.io.tmpdir. + * The files will be named netty_leak_YYYYMMDD-HHMMSS.SSS.txt. + */ +public class ExtendedNettyLeakDetector extends ResourceLeakDetector { + private static final Logger LOG = LoggerFactory.getLogger(ExtendedNettyLeakDetector.class); + public static final String NETTY_CUSTOM_LEAK_DETECTOR_SYSTEM_PROPERTY_NAME = "io.netty.customResourceLeakDetector"; + + private static final File DUMP_DIR = + new File(System.getenv().getOrDefault("NETTY_LEAK_DUMP_DIR", System.getProperty("java.io.tmpdir"))); + public static final String EXIT_JVM_ON_LEAK_SYSTEM_PROPERTY_NAME = + ExtendedNettyLeakDetector.class.getName() + ".exitJvmOnLeak"; + public static final String SLEEP_AFTER_GC_AND_FINALIZATION_MILLIS_SYSTEM_PROPERTY_NAME = + ExtendedNettyLeakDetector.class.getName() + ".sleepAfterGCAndFinalizationMillis"; + private static final long SLEEP_AFTER_GC_AND_FINALIZATION_MILLIS = Long.parseLong(System.getProperty( + SLEEP_AFTER_GC_AND_FINALIZATION_MILLIS_SYSTEM_PROPERTY_NAME, "10")); + private static boolean exitJvmOnLeak = Boolean.valueOf( + System.getProperty(EXIT_JVM_ON_LEAK_SYSTEM_PROPERTY_NAME, "false")); + private static final boolean DEFAULT_EXIT_JVM_ON_LEAK = exitJvmOnLeak; + public static final String EXIT_JVM_DELAY_MILLIS_SYSTEM_PROPERTY_NAME = + ExtendedNettyLeakDetector.class.getName() + ".exitJvmDelayMillis"; + private static final long EXIT_JVM_DELAY_MILLIS = + Long.parseLong(System.getProperty(EXIT_JVM_DELAY_MILLIS_SYSTEM_PROPERTY_NAME, "1000")); + public static final String USE_SHUTDOWN_HOOK_SYSTEM_PROPERTY_NAME = + ExtendedNettyLeakDetector.class.getName() + ".useShutdownHook"; + private static boolean useShutdownHook = Boolean.valueOf( + System.getProperty(USE_SHUTDOWN_HOOK_SYSTEM_PROPERTY_NAME, "false")); + static { + maybeRegisterShutdownHook(); + } + + private boolean exitThreadStarted; + private static volatile String initialHint; + + /** + * Triggers Netty leak detection. + * Passing "-XX:+UnlockExperimentalVMOptions -XX:ReferencesPerThread=0" to the JVM will help to detect leaks + * with a shorter latency. When ReferencesPerThread is set to 0, the JVM will use maximum parallelism for processing + * reference objects. Netty's leak detection relies on WeakReferences and this setting will help to process them + * faster. + */ + public static void triggerLeakDetection() { + if (!isEnabled()) { + return; + } + // run System.gc() to trigger finalization of objects and detection of possible Netty leaks + System.gc(); + System.runFinalization(); + try { + // wait for WeakReference collection to complete + Thread.sleep(SLEEP_AFTER_GC_AND_FINALIZATION_MILLIS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + triggerLeakReporting(); + } + + private static void triggerLeakReporting() { + // create a ByteBuf and release it to trigger leak detection + // this calls ResourceLeakDetector's reportLeak method when paranoid is enabled. + ByteBuf buffer = ByteBufAllocator.DEFAULT.directBuffer(); + // this triggers leak detection which forces tracking even when paranoid isn't enabled + // check the source code of ResourceLeakDetector to see how it works or step through it in a debugger + // This will call leak detector's trackForcibly method which will then call ResourceLeakDetector's reportLeak + // trackForcibly gets called in io.netty.buffer.SimpleLeakAwareByteBuf.unwrappedDerived + ByteBuf retainedSlice = buffer.retainedSlice(); + retainedSlice.release(); + buffer.release(); + } + + public ExtendedNettyLeakDetector(Class resourceType, int samplingInterval, long maxActive) { + super(resourceType, samplingInterval, maxActive); + } + + public ExtendedNettyLeakDetector(Class resourceType, int samplingInterval) { + super(resourceType, samplingInterval); + } + + public ExtendedNettyLeakDetector(String resourceType, int samplingInterval, long maxActive) { + super(resourceType, samplingInterval, maxActive); + } + + /** + * Set the initial hint to be used when reporting leaks. + * This hint will be printed alongside the stack trace of the creation of the resource in the Netty leak report. + * + * @see ResourceLeakDetector#getInitialHint(String) + * @param initialHint the initial hint + */ + public static void setInitialHint(String initialHint) { + ExtendedNettyLeakDetector.initialHint = initialHint; + } + + @Override + protected boolean needReport() { + return true; + } + + @Override + protected Object getInitialHint(String resourceType) { + String currentInitialHint = ExtendedNettyLeakDetector.initialHint; + if (currentInitialHint != null) { + return currentInitialHint; + } else { + return super.getInitialHint(resourceType); + } + } + + @Override + protected void reportTracedLeak(String resourceType, String records) { + super.reportTracedLeak(resourceType, records); + dumpToFile(resourceType, records); + maybeExitJVM(); + } + + @Override + protected void reportUntracedLeak(String resourceType) { + super.reportUntracedLeak(resourceType); + dumpToFile(resourceType, null); + maybeExitJVM(); + } + + private synchronized void maybeExitJVM() { + if (exitThreadStarted) { + return; + } + if (exitJvmOnLeak) { + new Thread(() -> { + LOG.error("Exiting JVM due to Netty resource leak. Check logs for more details. Dumped to {}", + DUMP_DIR.getAbsolutePath()); + System.err.println("Exiting JVM due to Netty resource leak. Check logs for more details. Dumped to " + + DUMP_DIR.getAbsolutePath()); + triggerLeakDetectionBeforeJVMExit(); + // shutdown log4j2 logging to prevent log truncation + LogManager.shutdown(); + // flush log buffers + System.err.flush(); + System.out.flush(); + // exit JVM immediately + Runtime.getRuntime().halt(1); + }).start(); + exitThreadStarted = true; + } + } + + private void dumpToFile(String resourceType, String records) { + try { + if (!DUMP_DIR.exists()) { + DUMP_DIR.mkdirs(); + } + String datetimePart = + DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss.SSS").format(ZonedDateTime.now()); + File nettyLeakDumpFile = + new File(DUMP_DIR, "netty_leak_" + datetimePart + ".txt"); + // Prefix the line to make it easier to show the errors in GitHub Actions as annotations + String linePrefix = exitJvmOnLeak ? "::error::" : "::warning::"; + try (PrintWriter out = new PrintWriter(new FileWriter(nettyLeakDumpFile, true))) { + out.print(linePrefix); + if (records != null) { + out.println("Traced leak detected " + resourceType); + out.println(records); + } else { + out.println("Untraced leak detected " + resourceType); + } + out.println(); + out.flush(); + } + } catch (IOException e) { + LOG.error("Cannot write thread leak dump", e); + } + } + + public static boolean isExtendedNettyLeakDetectorEnabled() { + return ExtendedNettyLeakDetector.class.getName() + .equals(System.getProperty(NETTY_CUSTOM_LEAK_DETECTOR_SYSTEM_PROPERTY_NAME)); + } + + /** + * Disable exit on leak. This is useful when exitJvmOnLeak is enabled and there's a test that is expected + * to leak resources. This method can be called before the test execution begins. + * This will not disable the leak detection itself, only the exit on leak behavior. + */ + public static void disableExitJVMOnLeak() { + exitJvmOnLeak = false; + } + + /** + * Restore exit on leak to original value. This is used to re-enable exitJvmOnLeak feature after it was + * disabled for the duration of a specific test using disableExitJVMOnLeak. + */ + public static void restoreExitJVMOnLeak() { + triggerLeakDetection(); + exitJvmOnLeak = DEFAULT_EXIT_JVM_ON_LEAK; + } + + /** + * Shutdown hook to trigger leak detection on JVM shutdown. + * This is useful when using the leak detector in actual production code or in system tests which + * don't use don't have a test listener that would be calling triggerLeakDetection before the JVM exits. + */ + private static void maybeRegisterShutdownHook() { + if (!exitJvmOnLeak && useShutdownHook) { + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + if (!isEnabled()) { + return; + } + triggerLeakDetectionBeforeJVMExit(); + }, ExtendedNettyLeakDetector.class.getSimpleName() + "ShutdownHook")); + } + } + + private static void triggerLeakDetectionBeforeJVMExit() { + triggerLeakDetection(); + // wait for a while + try { + Thread.sleep(EXIT_JVM_DELAY_MILLIS); + } catch (InterruptedException e) { + // ignore + } + // trigger leak detection again to increase the chances of detecting leaks + // this could be helpful if more objects were finalized asynchronously during the delay + triggerLeakDetection(); + } +} diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/FailFastNotifier.java b/buildtools/src/main/java/org/apache/pulsar/tests/FailFastNotifier.java index 2dbc4f9ef99ad..cdfc134ad07c0 100644 --- a/buildtools/src/main/java/org/apache/pulsar/tests/FailFastNotifier.java +++ b/buildtools/src/main/java/org/apache/pulsar/tests/FailFastNotifier.java @@ -106,7 +106,7 @@ public void onTestFailure(ITestResult result) { result.setThrowable(null); result.setStatus(ITestResult.SKIP); } else { - FailFastNotifier.FailFastEventsSingleton.getInstance().testFailed(result); + FailFastEventsSingleton.getInstance().testFailed(result); } } @@ -124,8 +124,7 @@ public void beforeInvocation(IInvokedMethod iInvokedMethod, ITestResult iTestRes || iTestNGMethod.isAfterTestConfiguration())) { throw new FailFastSkipException("Skipped after failure since testFailFast system property is set."); } - } - if (FAIL_FAST_KILLSWITCH_FILE != null && FAIL_FAST_KILLSWITCH_FILE.exists()) { + } else if (FAIL_FAST_KILLSWITCH_FILE != null && FAIL_FAST_KILLSWITCH_FILE.exists()) { throw new FailFastSkipException("Skipped after failure since kill switch file exists."); } } diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/FastThreadLocalCleanupListener.java b/buildtools/src/main/java/org/apache/pulsar/tests/FastThreadLocalCleanupListener.java index c40956a9e259e..6111a4a390d32 100644 --- a/buildtools/src/main/java/org/apache/pulsar/tests/FastThreadLocalCleanupListener.java +++ b/buildtools/src/main/java/org/apache/pulsar/tests/FastThreadLocalCleanupListener.java @@ -18,8 +18,10 @@ */ package org.apache.pulsar.tests; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testng.ITestClass; /** * Cleanup Thread Local state attach to Netty's FastThreadLocal. @@ -48,7 +50,7 @@ public class FastThreadLocalCleanupListener extends BetweenTestClassesListenerAd }); @Override - protected void onBetweenTestClasses(Class endedTestClass, Class startedTestClass) { + protected void onBetweenTestClasses(List testClasses) { if (FAST_THREAD_LOCAL_CLEANUP_ENABLED && FastThreadLocalStateCleaner.isEnabled()) { LOG.info("Cleaning up FastThreadLocal thread local state."); CLEANER.cleanupAllFastThreadLocals((thread, value) -> { diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/HeapDumpUtil.java b/buildtools/src/main/java/org/apache/pulsar/tests/HeapDumpUtil.java new file mode 100644 index 0000000000000..897399ea54aa0 --- /dev/null +++ b/buildtools/src/main/java/org/apache/pulsar/tests/HeapDumpUtil.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests; + +import com.sun.management.HotSpotDiagnosticMXBean; +import java.io.File; +import java.lang.management.ManagementFactory; +import javax.management.MBeanServer; + +public class HeapDumpUtil { + private static final String HOTSPOT_BEAN_NAME = "com.sun.management:type=HotSpotDiagnostic"; + + // Utility method to get the HotSpotDiagnosticMXBean + private static HotSpotDiagnosticMXBean getHotSpotDiagnosticMXBean() { + try { + MBeanServer server = ManagementFactory.getPlatformMBeanServer(); + return ManagementFactory.newPlatformMXBeanProxy(server, HOTSPOT_BEAN_NAME, HotSpotDiagnosticMXBean.class); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * Dump the heap of the JVM. + * + * @param file the system-dependent filename + * @param liveObjects if true dump only live objects i.e. objects that are reachable from others + */ + public static void dumpHeap(File file, boolean liveObjects) { + try { + HotSpotDiagnosticMXBean hotspotMBean = getHotSpotDiagnosticMXBean(); + hotspotMBean.dumpHeap(file.getAbsolutePath(), liveObjects); + } catch (Exception e) { + throw new RuntimeException("Error generating heap dump", e); + } + } +} diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/HeapHistogramUtil.java b/buildtools/src/main/java/org/apache/pulsar/tests/HeapHistogramUtil.java new file mode 100644 index 0000000000000..f4c92a264bc1d --- /dev/null +++ b/buildtools/src/main/java/org/apache/pulsar/tests/HeapHistogramUtil.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.tests; + +import java.lang.management.ManagementFactory; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import javax.management.JMException; +import javax.management.ObjectName; + +public class HeapHistogramUtil { + public static String buildHeapHistogram() { + StringBuilder dump = new StringBuilder(); + dump.append(String.format("Timestamp: %s", DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now()))); + dump.append("\n\n"); + try { + dump.append(callDiagnosticCommand("gcHeapInfo")); + } catch (Exception e) { + e.printStackTrace(); + return null; + } + dump.append("\n"); + try { + dump.append(callDiagnosticCommand("gcClassHistogram")); + } catch (Exception e) { + e.printStackTrace(); + return null; + } + return dump.toString(); + } + + /** + * Calls a diagnostic commands. + * The available operations are similar to what the jcmd commandline tool has, + * however the naming of the operations are different. The "help" operation can be used + * to find out the available operations. For example, the jcmd command "Thread.print" maps + * to "threadPrint" operation name. + */ + static String callDiagnosticCommand(String operationName, String... args) + throws JMException { + return (String) ManagementFactory.getPlatformMBeanServer() + .invoke(new ObjectName("com.sun.management:type=DiagnosticCommand"), + operationName, new Object[] {args}, new String[] {String[].class.getName()}); + } +} diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/JacocoDumpListener.java b/buildtools/src/main/java/org/apache/pulsar/tests/JacocoDumpListener.java new file mode 100644 index 0000000000000..f3202f2ef5a91 --- /dev/null +++ b/buildtools/src/main/java/org/apache/pulsar/tests/JacocoDumpListener.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests; + +import java.lang.management.ManagementFactory; +import java.util.concurrent.TimeUnit; +import javax.management.InstanceNotFoundException; +import javax.management.MBeanServer; +import javax.management.MBeanServerInvocationHandler; +import javax.management.MalformedObjectNameException; +import javax.management.ObjectName; +import org.testng.IExecutionListener; +import org.testng.ISuite; +import org.testng.ISuiteListener; + +/** + * A TestNG listener that dumps Jacoco coverage data to file using the Jacoco JMX interface. + * + * This ensures that coverage data is dumped even if the shutdown sequence of the Test JVM gets stuck. Coverage + * data will be dumped every 2 minutes by default and once all test suites have been run. + * Each test class runs in its own suite when run with maven-surefire-plugin. + */ +public class JacocoDumpListener implements ISuiteListener, IExecutionListener { + private final MBeanServer platformMBeanServer = ManagementFactory.getPlatformMBeanServer(); + private final ObjectName jacocoObjectName; + private final JacocoProxy jacocoProxy; + private final boolean enabled; + + private long lastDumpTime; + + private static final long DUMP_INTERVAL_MILLIS = TimeUnit.SECONDS.toMillis(120); + + public JacocoDumpListener() { + try { + jacocoObjectName = new ObjectName("org.jacoco:type=Runtime"); + } catch (MalformedObjectNameException e) { + // this won't happen since the ObjectName is static and valid + throw new RuntimeException(e); + } + enabled = checkEnabled(); + if (enabled) { + jacocoProxy = MBeanServerInvocationHandler.newProxyInstance(platformMBeanServer, jacocoObjectName, + JacocoProxy.class, false); + } else { + jacocoProxy = null; + } + lastDumpTime = System.currentTimeMillis(); + } + + private boolean checkEnabled() { + try { + platformMBeanServer.getObjectInstance(jacocoObjectName); + } catch (InstanceNotFoundException e) { + // jacoco jmx is not enabled + return false; + } + return true; + } + + public void onFinish(ISuite suite) { + // dump jacoco coverage data to file using the Jacoco JMX interface if more than DUMP_INTERVAL_MILLIS has passed + // since the last dump + if (enabled && System.currentTimeMillis() - lastDumpTime > DUMP_INTERVAL_MILLIS) { + // dump jacoco coverage data to file using the Jacoco JMX interface + triggerJacocoDump(); + } + } + @Override + public void onExecutionFinish() { + if (enabled) { + // dump jacoco coverage data to file using the Jacoco JMX interface when all tests have finished + triggerJacocoDump(); + } + } + + private void triggerJacocoDump() { + System.out.println("Dumping Jacoco coverage data to file..."); + long start = System.currentTimeMillis(); + jacocoProxy.dump(true); + lastDumpTime = System.currentTimeMillis(); + System.out.println("Completed in " + (lastDumpTime - start) + "ms."); + } + + public interface JacocoProxy { + void dump(boolean reset); + } +} diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/MockitoCleanupListener.java b/buildtools/src/main/java/org/apache/pulsar/tests/MockitoCleanupListener.java index de0ba280ace5e..babef9abb830d 100644 --- a/buildtools/src/main/java/org/apache/pulsar/tests/MockitoCleanupListener.java +++ b/buildtools/src/main/java/org/apache/pulsar/tests/MockitoCleanupListener.java @@ -18,9 +18,11 @@ */ package org.apache.pulsar.tests; +import java.util.List; import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testng.ITestClass; /** * Cleanup Mockito's Thread Local state that leaks memory @@ -39,7 +41,7 @@ public class MockitoCleanupListener extends BetweenTestClassesListenerAdapter { "Cleaning up Mockito's ThreadSafeMockingProgress.MOCKING_PROGRESS_PROVIDER thread local state."; @Override - protected void onBetweenTestClasses(Class endedTestClass, Class startedTestClass) { + protected void onBetweenTestClasses(List testClasses) { if (MOCKITO_CLEANUP_ENABLED) { try { if (MockitoThreadLocalStateCleaner.INSTANCE.isEnabled()) { diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/PulsarTestListener.java b/buildtools/src/main/java/org/apache/pulsar/tests/PulsarTestListener.java index 7c78fe961a0c6..26fcfe48ca203 100644 --- a/buildtools/src/main/java/org/apache/pulsar/tests/PulsarTestListener.java +++ b/buildtools/src/main/java/org/apache/pulsar/tests/PulsarTestListener.java @@ -19,16 +19,21 @@ package org.apache.pulsar.tests; import java.util.Arrays; +import org.testng.IExecutionListener; +import org.testng.ISuite; +import org.testng.ISuiteListener; import org.testng.ITestContext; import org.testng.ITestListener; import org.testng.ITestResult; import org.testng.SkipException; import org.testng.internal.thread.ThreadTimeoutException; -public class PulsarTestListener implements ITestListener { +public class PulsarTestListener implements ITestListener, IExecutionListener, ISuiteListener { @Override public void onTestStart(ITestResult result) { + ExtendedNettyLeakDetector.setInitialHint(String.format("Test: %s.%s", result.getTestClass().getName(), + result.getMethod().getMethodName())); System.out.format("------- Starting test %s.%s(%s)-------\n", result.getTestClass(), result.getMethod().getMethodName(), Arrays.toString(result.getParameters())); } @@ -37,6 +42,7 @@ public void onTestStart(ITestResult result) { public void onTestSuccess(ITestResult result) { System.out.format("------- SUCCESS -- %s.%s(%s)-------\n", result.getTestClass(), result.getMethod().getMethodName(), Arrays.toString(result.getParameters())); + ExtendedNettyLeakDetector.triggerLeakDetection(); } @Override @@ -44,33 +50,64 @@ public void onTestFailure(ITestResult result) { if (!(result.getThrowable() instanceof SkipException)) { System.out.format("!!!!!!!!! FAILURE-- %s.%s(%s)-------\n", result.getTestClass(), result.getMethod().getMethodName(), Arrays.toString(result.getParameters())); - } - if (result.getThrowable() != null) { - result.getThrowable().printStackTrace(); - if (result.getThrowable() instanceof ThreadTimeoutException) { - System.out.println("====== THREAD DUMPS ======"); - System.out.println(ThreadDumpUtil.buildThreadDiagnosticString()); + if (result.getThrowable() != null) { + result.getThrowable().printStackTrace(); + if (result.getThrowable() instanceof ThreadTimeoutException) { + System.out.println("====== THREAD DUMPS ======"); + System.out.println(ThreadDumpUtil.buildThreadDiagnosticString()); + } } } + ExtendedNettyLeakDetector.triggerLeakDetection(); } @Override public void onTestSkipped(ITestResult result) { - System.out.format("~~~~~~~~~ SKIPPED -- %s.%s(%s)-------\n", result.getTestClass(), - result.getMethod().getMethodName(), Arrays.toString(result.getParameters())); + if (!(result.getThrowable() instanceof SkipException)) { + System.out.format("~~~~~~~~~ SKIPPED -- %s.%s(%s)-------\n", result.getTestClass(), + result.getMethod().getMethodName(), Arrays.toString(result.getParameters())); + if (result.getThrowable() != null) { + result.getThrowable().printStackTrace(); + if (result.getThrowable() instanceof ThreadTimeoutException) { + System.out.println("====== THREAD DUMPS ======"); + System.out.println(ThreadDumpUtil.buildThreadDiagnosticString()); + } + } + } } @Override public void onTestFailedButWithinSuccessPercentage(ITestResult result) { - + ExtendedNettyLeakDetector.triggerLeakDetection(); } @Override public void onStart(ITestContext context) { - + ExtendedNettyLeakDetector.setInitialHint("Starting test: " + context.getName()); } @Override public void onFinish(ITestContext context) { + ExtendedNettyLeakDetector.triggerLeakDetection(); + ExtendedNettyLeakDetector.setInitialHint("Finished test: " + context.getName()); + } + + @Override + public void onFinish(ISuite suite) { + ExtendedNettyLeakDetector.setInitialHint("Finished suite: " + suite.getName()); + } + + @Override + public void onExecutionFinish() { + if (!ExtendedNettyLeakDetector.isEnabled()) { + return; + } + ExtendedNettyLeakDetector.triggerLeakDetection(); + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + ExtendedNettyLeakDetector.triggerLeakDetection(); } } diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/SingletonCleanerListener.java b/buildtools/src/main/java/org/apache/pulsar/tests/SingletonCleanerListener.java new file mode 100644 index 0000000000000..c5960a38081c2 --- /dev/null +++ b/buildtools/src/main/java/org/apache/pulsar/tests/SingletonCleanerListener.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.List; +import org.apache.commons.lang3.ClassUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.ITestClass; + +/** + * This TestNG listener contains cleanup for some singletons or caches. + */ +public class SingletonCleanerListener extends BetweenTestClassesListenerAdapter { + private static final Logger LOG = LoggerFactory.getLogger(SingletonCleanerListener.class); + private static final Method OBJECTMAPPERFACTORY_CLEARCACHES_METHOD; + private static final Method JSONSCHEMA_CLEARCACHES_METHOD; + + static { + Class objectMapperFactoryClazz = + null; + try { + objectMapperFactoryClazz = ClassUtils.getClass("org.apache.pulsar.common.util.ObjectMapperFactory"); + } catch (ClassNotFoundException e) { + LOG.warn("Cannot find ObjectMapperFactory class", e); + } + + Method clearCachesMethod = null; + try { + if (objectMapperFactoryClazz != null) { + clearCachesMethod = + objectMapperFactoryClazz + .getMethod("clearCaches"); + } + } catch (NoSuchMethodException e) { + LOG.warn("Cannot find method for clearing singleton ObjectMapper caches", e); + } + OBJECTMAPPERFACTORY_CLEARCACHES_METHOD = clearCachesMethod; + + + Class jsonSchemaClazz = null; + try { + jsonSchemaClazz = ClassUtils.getClass("org.apache.pulsar.client.impl.schema.JSONSchema"); + } catch (ClassNotFoundException e) { + LOG.warn("Cannot find JSONSchema class", e); + } + + Method jsonSchemaCleanCachesMethod = null; + try { + if (jsonSchemaClazz != null) { + jsonSchemaCleanCachesMethod = + jsonSchemaClazz + .getMethod("clearCaches"); + } + } catch (NoSuchMethodException e) { + LOG.warn("Cannot find method for clearing singleton JSONSchema caches", e); + } + JSONSCHEMA_CLEARCACHES_METHOD = jsonSchemaCleanCachesMethod; + } + + @Override + protected void onBetweenTestClasses(List testClasses) { + objectMapperFactoryClearCaches(); + jsonSchemaClearCaches(); + } + + // Call ObjectMapperFactory.clearCaches() using reflection to clear up classes held in + // the singleton Jackson ObjectMapper instances + private static void objectMapperFactoryClearCaches() { + if (OBJECTMAPPERFACTORY_CLEARCACHES_METHOD != null) { + try { + OBJECTMAPPERFACTORY_CLEARCACHES_METHOD.invoke(null); + } catch (IllegalAccessException | InvocationTargetException e) { + LOG.warn("Cannot clean singleton ObjectMapper caches", e); + } + } + } + + // Call JSONSchema.clearCaches() using reflection to clear up classes held in + // the singleton Jackson ObjectMapper instance of JSONSchema class + private static void jsonSchemaClearCaches() { + if (JSONSCHEMA_CLEARCACHES_METHOD != null) { + try { + JSONSCHEMA_CLEARCACHES_METHOD.invoke(null); + } catch (IllegalAccessException | InvocationTargetException e) { + LOG.warn("Cannot clean singleton JSONSchema caches", e); + } + } + } +} diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/ThreadDumpUtil.java b/buildtools/src/main/java/org/apache/pulsar/tests/ThreadDumpUtil.java index bf0d8bf41caf1..293a02b0d1eac 100644 --- a/buildtools/src/main/java/org/apache/pulsar/tests/ThreadDumpUtil.java +++ b/buildtools/src/main/java/org/apache/pulsar/tests/ThreadDumpUtil.java @@ -1,3 +1,4 @@ + /** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -25,7 +26,7 @@ import java.lang.management.MonitorInfo; import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; -import java.time.LocalDateTime; +import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.Map; import javax.management.JMException; @@ -65,7 +66,7 @@ static String buildThreadDump() { // fallback to using JMX for creating the thread dump StringBuilder dump = new StringBuilder(); - dump.append(String.format("Timestamp: %s", DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(LocalDateTime.now()))); + dump.append(String.format("Timestamp: %s", DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now()))); dump.append("\n\n"); Map stackTraces = Thread.getAllStackTraces(); diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/ThreadLeakDetectorListener.java b/buildtools/src/main/java/org/apache/pulsar/tests/ThreadLeakDetectorListener.java index 163591cf82554..702d0d13ae0fb 100644 --- a/buildtools/src/main/java/org/apache/pulsar/tests/ThreadLeakDetectorListener.java +++ b/buildtools/src/main/java/org/apache/pulsar/tests/ThreadLeakDetectorListener.java @@ -18,53 +18,285 @@ */ package org.apache.pulsar.tests; +import com.google.common.base.Charsets; +import com.google.common.io.Files; +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.lang.reflect.Field; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import java.util.Collections; import java.util.LinkedHashSet; +import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.concurrent.ForkJoinWorkerThread; import java.util.stream.Collectors; import org.apache.commons.lang3.ThreadUtils; +import org.apache.commons.lang3.mutable.MutableBoolean; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testng.ISuite; +import org.testng.ISuiteListener; +import org.testng.ITestClass; /** - * Detects new threads that have been created during the test execution. + * Detects new threads that have been created during the test execution. This is useful to detect thread leaks. + * Will create files to the configured directory if new threads are detected and THREAD_LEAK_DETECTOR_WAIT_MILLIS + * is set to a positive value. A recommended value is 10000 for THREAD_LEAK_DETECTOR_WAIT_MILLIS. This will ensure + * that any asynchronous operations should have completed before the detector determines that it has found a leak. */ -public class ThreadLeakDetectorListener extends BetweenTestClassesListenerAdapter { +public class ThreadLeakDetectorListener extends BetweenTestClassesListenerAdapter implements ISuiteListener { private static final Logger LOG = LoggerFactory.getLogger(ThreadLeakDetectorListener.class); + private static final long WAIT_FOR_THREAD_TERMINATION_MILLIS = + Long.parseLong(System.getenv().getOrDefault("THREAD_LEAK_DETECTOR_WAIT_MILLIS", "0")); + private static final File DUMP_DIR = + new File(System.getenv().getOrDefault("THREAD_LEAK_DETECTOR_DIR", "target/thread-leak-dumps")); + private static final long THREAD_TERMINATION_POLL_INTERVAL = + Long.parseLong(System.getenv().getOrDefault("THREAD_LEAK_DETECTOR_POLL_INTERVAL", "250")); + private static final boolean COLLECT_THREADDUMP = + Boolean.parseBoolean(System.getenv().getOrDefault("THREAD_LEAK_DETECTOR_COLLECT_THREADDUMP", "true")); private Set capturedThreadKeys; + private static final Field THREAD_TARGET_FIELD; + static { + Field targetField = null; + try { + targetField = Thread.class.getDeclaredField("target"); + targetField.setAccessible(true); + } catch (NoSuchFieldException e) { + // ignore this error. on Java 21, the field is not present + // TODO: add support for extracting the Runnable target on Java 21 + } + THREAD_TARGET_FIELD = targetField; + } + + @Override + public void onStart(ISuite suite) { + // capture the initial set of threads + detectLeakedThreads(Collections.emptyList()); + } + @Override - protected void onBetweenTestClasses(Class endedTestClass, Class startedTestClass) { + protected void onBetweenTestClasses(List testClasses) { + detectLeakedThreads(testClasses); + } + + private static String joinTestClassNames(List testClasses) { + return testClasses.stream() + .map(ITestClass::getRealClass) + .map(Class::getName) + .collect(Collectors.joining(", ")); + } + + private static String joinSimpleTestClassNames(List testClasses) { + return testClasses.stream() + .map(ITestClass::getRealClass) + .map(Class::getSimpleName) + .collect(Collectors.joining(", ")); + } + + private static String firstTestClassName(List testClasses) { + return testClasses.stream() + .findFirst() + .get() + .getRealClass().getName(); + } + + private void detectLeakedThreads(List testClasses) { LOG.info("Capturing identifiers of running threads."); - capturedThreadKeys = compareThreads(capturedThreadKeys, endedTestClass); + MutableBoolean differenceDetected = new MutableBoolean(); + Set currentThreadKeys = + compareThreads(capturedThreadKeys, testClasses, WAIT_FOR_THREAD_TERMINATION_MILLIS <= 0, + differenceDetected, null); + if (WAIT_FOR_THREAD_TERMINATION_MILLIS > 0 && !testClasses.isEmpty() && differenceDetected.booleanValue()) { + LOG.info("Difference detected in active threads. Waiting up to {} ms for threads to terminate.", + WAIT_FOR_THREAD_TERMINATION_MILLIS); + long endTime = System.currentTimeMillis() + WAIT_FOR_THREAD_TERMINATION_MILLIS; + while (System.currentTimeMillis() < endTime) { + try { + Thread.sleep(THREAD_TERMINATION_POLL_INTERVAL); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + differenceDetected.setFalse(); + currentThreadKeys = compareThreads(capturedThreadKeys, testClasses, false, differenceDetected, null); + if (!differenceDetected.booleanValue()) { + break; + } + } + if (differenceDetected.booleanValue()) { + String datetimePart = + DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss.SSS").format(ZonedDateTime.now()); + PrintWriter out = null; + String firstTestClassName = firstTestClassName(testClasses); + try { + if (!DUMP_DIR.exists()) { + DUMP_DIR.mkdirs(); + } + File threadleakdumpFile = + new File(DUMP_DIR, "threadleak" + datetimePart + firstTestClassName + ".txt"); + out = new PrintWriter(threadleakdumpFile); + } catch (IOException e) { + LOG.error("Cannot write thread leak dump", e); + } + currentThreadKeys = compareThreads(capturedThreadKeys, testClasses, true, null, out); + if (out != null) { + out.close(); + } + if (COLLECT_THREADDUMP) { + File threaddumpFile = + new File(DUMP_DIR, "threaddump" + datetimePart + firstTestClassName + ".txt"); + try { + Files.asCharSink(threaddumpFile, Charsets.UTF_8) + .write(ThreadDumpUtil.buildThreadDiagnosticString()); + } catch (IOException e) { + LOG.error("Cannot write thread dump", e); + } + } + } + } + capturedThreadKeys = currentThreadKeys; } - private static Set compareThreads(Set previousThreadKeys, Class endedTestClass) { + private static Set compareThreads(Set previousThreadKeys, List testClasses, + boolean logDifference, MutableBoolean differenceDetected, + PrintWriter out) { Set threadKeys = Collections.unmodifiableSet(ThreadUtils.getAllThreads().stream() + .filter(thread -> !shouldSkipThread(thread)) .map(ThreadKey::of) .collect(Collectors.>toCollection(LinkedHashSet::new))); - if (endedTestClass != null && previousThreadKeys != null) { + if (!testClasses.isEmpty() && previousThreadKeys != null) { int newThreadsCounter = 0; - LOG.info("Checking for new threads created by {}.", endedTestClass.getName()); for (ThreadKey threadKey : threadKeys) { if (!previousThreadKeys.contains(threadKey)) { newThreadsCounter++; - LOG.warn("Tests in class {} created thread id {} with name '{}'", endedTestClass.getSimpleName(), - threadKey.getThreadId(), threadKey.getThreadName()); + if (differenceDetected != null) { + differenceDetected.setTrue(); + } + if (logDifference || out != null) { + String message = String.format("Tests in class %s created thread id %d with name '%s'", + joinSimpleTestClassNames(testClasses), + threadKey.getThreadId(), threadKey.getThreadName()); + if (logDifference) { + LOG.warn(message); + } + if (out != null) { + out.println(message); + } + } } } - if (newThreadsCounter > 0) { - LOG.warn("Summary: Tests in class {} created {} new threads", endedTestClass.getName(), - newThreadsCounter); + if (newThreadsCounter > 0 && (logDifference || out != null)) { + String message = String.format( + "Summary: Tests in class %s created %d new threads. There are now %d threads in total.", + joinTestClassNames(testClasses), newThreadsCounter, threadKeys.size()); + if (logDifference) { + LOG.warn(message); + } + if (out != null) { + out.println(message); + } } } return threadKeys; } + private static boolean shouldSkipThread(Thread thread) { + // skip ForkJoinPool threads + if (thread instanceof ForkJoinWorkerThread) { + return true; + } + // skip Testcontainers threads + final ThreadGroup threadGroup = thread.getThreadGroup(); + if (threadGroup != null && "testcontainers".equals(threadGroup.getName())) { + return true; + } + String threadName = thread.getName(); + if (threadName != null) { + // skip ClientTestFixtures.SCHEDULER threads + if (threadName.startsWith("ClientTestFixtures-SCHEDULER-")) { + return true; + } + // skip JVM internal threads related to java.lang.Process + if (threadName.equals("process reaper")) { + return true; + } + // skip thread created by sun.net.www.http.KeepAliveCache + if (threadName.equals("Keep-Alive-Timer")) { + return true; + } + // skip JVM internal thread related to agent attach + if (threadName.equals("Attach Listener")) { + return true; + } + // skip JVM internal thread used for CompletableFuture.delayedExecutor + if (threadName.equals("CompletableFutureDelayScheduler")) { + return true; + } + // skip threadpool created in dev.failsafe.internal.util.DelegatingScheduler + if (threadName.equals("FailsafeDelayScheduler")) { + return true; + } + // skip Okio Watchdog thread and interrupt it + if (threadName.equals("Okio Watchdog")) { + return true; + } + // skip OkHttp TaskRunner thread + if (threadName.equals("OkHttp TaskRunner")) { + return true; + } + // skip JNA background thread + if (threadName.equals("JNA Cleaner")) { + return true; + } + // skip org.glassfish.grizzly.http.server.DefaultSessionManager thread pool + if (threadName.equals("Grizzly-HttpSession-Expirer")) { + return true; + } + // skip Hadoop LocalFileSystem stats thread + if (threadName.equals("org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner")) { + return true; + } + // Testcontainers AbstractWaitStrategy.EXECUTOR + if (threadName.startsWith("testcontainers-wait-")) { + return true; + } + // org.rnorth.ducttape.timeouts.Timeouts.EXECUTOR_SERVICE thread pool, used by Testcontainers + if (threadName.startsWith("ducttape-")) { + return true; + } + } + Runnable target = extractRunnableTarget(thread); + if (target != null) { + String targetClassName = target.getClass().getName(); + // ignore threads that contain a Runnable class under org.testcontainers package + if (targetClassName.startsWith("org.testcontainers.")) { + return true; + } + } + return false; + } + + // use reflection to extract the Runnable target from a thread so that we can detect threads created by + // Testcontainers based on the Runnable's class name. + private static Runnable extractRunnableTarget(Thread thread) { + if (THREAD_TARGET_FIELD == null) { + return null; + } + Runnable target = null; + try { + target = (Runnable) THREAD_TARGET_FIELD.get(thread); + } catch (IllegalAccessException e) { + LOG.warn("Cannot access target field in Thread.class", e); + } + return target; + } + /** * Unique key for a thread * Based on thread id and it's identity hash code diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/TraceTestResourceCleanupListener.java b/buildtools/src/main/java/org/apache/pulsar/tests/TraceTestResourceCleanupListener.java new file mode 100644 index 0000000000000..3080c02c701b5 --- /dev/null +++ b/buildtools/src/main/java/org/apache/pulsar/tests/TraceTestResourceCleanupListener.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.tests; + +import com.google.common.base.Charsets; +import com.google.common.io.Files; +import java.io.File; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import org.testng.IExecutionListener; + +/** + * A TestNG listener that traces test resource cleanup by creating a thread dump, heap histogram and heap dump + * (when mode is 'full') before the TestNG JVM exits. + * The heap dump could help detecting memory leaks in tests or the sources of resource leaks that cannot be + * detected with the ThreadLeakDetectorListener. + */ +public class TraceTestResourceCleanupListener implements IExecutionListener { + enum TraceTestResourceCleanupMode { + OFF, + ON, + FULL // includes heap dump + } + + private static final TraceTestResourceCleanupMode MODE = + TraceTestResourceCleanupMode.valueOf( + System.getenv().getOrDefault("TRACE_TEST_RESOURCE_CLEANUP", "off").toUpperCase()); + private static final File DUMP_DIR = new File( + System.getenv().getOrDefault("TRACE_TEST_RESOURCE_CLEANUP_DIR", "target/trace-test-resource-cleanup")); + private static final long WAIT_BEFORE_DUMP_MILLIS = + Long.parseLong(System.getenv().getOrDefault("TRACE_TEST_RESOURCE_CLEANUP_DELAY", "5000")); + + static { + if (MODE != TraceTestResourceCleanupMode.OFF) { + Runtime.getRuntime().addShutdownHook(new Thread(TraceTestResourceCleanupListener::createDumps)); + } + } + + static void createDumps() { + if (!DUMP_DIR.isDirectory()) { + DUMP_DIR.mkdirs(); + } + try { + Thread.sleep(WAIT_BEFORE_DUMP_MILLIS); + } catch (InterruptedException e) { + // ignore + } + + String datetimePart = + DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss.SSS").format(ZonedDateTime.now()); + try { + String threadDump = ThreadDumpUtil.buildThreadDiagnosticString(); + File threaddumpFile = new File(DUMP_DIR, "threaddump" + datetimePart + ".txt"); + Files.asCharSink(threaddumpFile, Charsets.UTF_8).write(threadDump); + } catch (Throwable t) { + System.err.println("Error dumping threads"); + t.printStackTrace(System.err); + } + + try { + String heapHistogram = HeapHistogramUtil.buildHeapHistogram(); + File heapHistogramFile = new File(DUMP_DIR, "heaphistogram" + datetimePart + ".txt"); + Files.asCharSink(heapHistogramFile, Charsets.UTF_8).write(heapHistogram); + } catch (Throwable t) { + System.err.println("Error dumping heap histogram"); + t.printStackTrace(System.err); + } + + if (MODE == TraceTestResourceCleanupMode.FULL) { + try { + File heapdumpFile = new File(DUMP_DIR, "heapdump" + datetimePart + ".hprof"); + HeapDumpUtil.dumpHeap(heapdumpFile, true); + } catch (Throwable t) { + System.err.println("Error dumping heap"); + t.printStackTrace(System.err); + } + } + } +} diff --git a/buildtools/src/test/java/org/apache/pulsar/tests/BetweenTestClassesListenerAdapterTest.java b/buildtools/src/test/java/org/apache/pulsar/tests/BetweenTestClassesListenerAdapterTest.java new file mode 100644 index 0000000000000..0b10b6674c68b --- /dev/null +++ b/buildtools/src/test/java/org/apache/pulsar/tests/BetweenTestClassesListenerAdapterTest.java @@ -0,0 +1,395 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import org.testng.Assert; +import org.testng.IClass; +import org.testng.ITestClass; +import org.testng.ITestListener; +import org.testng.ITestResult; +import org.testng.TestNG; +import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Factory; +import org.testng.annotations.Test; +import org.testng.internal.IParameterInfo; +import org.testng.xml.XmlClass; +import org.testng.xml.XmlSuite; +import org.testng.xml.XmlTest; + +public class BetweenTestClassesListenerAdapterTest { + private TestBetweenTestClassesListener listener; + + @BeforeMethod + public void setUp() { + listener = new TestBetweenTestClassesListener(); + listener.reset(); + } + + @Test + public void testListenerWithNoAfterClassMethods() { + runTestNGWithClass(NoAfterClassMethods.class); + verifyListenerCalledForClass(NoAfterClassMethods.class); + } + + @Test + public void testListenerWithOneAfterClassMethod() { + runTestNGWithClass(OneAfterClassMethod.class); + verifyListenerCalledForClass(OneAfterClassMethod.class); + } + + @Test + public void testListenerWithMultipleAfterClassMethods() { + runTestNGWithClass(MultipleAfterClassMethods.class); + verifyListenerCalledForClass(MultipleAfterClassMethods.class); + } + + @Test + public void testListenerWithDisabledAfterClassMethod() { + runTestNGWithClass(DisabledAfterClassMethod.class); + verifyListenerCalledForClass(DisabledAfterClassMethod.class); + } + + @Test + public void testListenerWithFailingAfterClassMethods() { + runTestNGWithClass(FailingAfterClassMethod.class); + verifyListenerCalledForClass(FailingAfterClassMethod.class); + } + + @Test + public void testListenerWithTimeoutAndAfterClassMethod() { + runTestNGWithClasses(1, TimeoutAndAfterClassMethod.class); + verifyListenerCalledForClass(TimeoutAndAfterClassMethod.class); + } + + @Test + public void testListenerWithFactoryMethod() { + runTestNGWithClass(FactoryMethodCase.class); + String className = FactoryMethodCase.class.getName(); + assertEquals(1, listener.getClassesCalled().size(), + "Listener should be called exactly once"); + assertEquals(className, listener.getClassesCalled().get(0).getName(), + "Listener should be called for the correct class"); + } + + @Test + public void testListenerWithFactoryMethodWithoutAfterClassMethods() { + runTestNGWithClass(FactoryMethodCaseWithoutAfterClass.class); + String className = FactoryMethodCaseWithoutAfterClass.class.getName(); + assertEquals(1, listener.getClassesCalled().size(), + "Listener should be called exactly once"); + assertEquals(className, listener.getClassesCalled().get(0).getName(), + "Listener should be called for the correct class"); + } + + @Test + public void testListenerWithMultipleTestClasses() { + runTestNGWithClasses(0, + NoAfterClassMethods.class, + OneAfterClassMethod.class, + FailingAfterClassMethod.class, + MultipleAfterClassMethods.class); + + assertEquals(4, listener.getClassesCalled().size()); + + List actualClassNames = listener.getClassesCalled().stream() + .map(IClass::getName) + .collect(Collectors.toList()); + + assertTrue(actualClassNames.contains(NoAfterClassMethods.class.getName())); + assertTrue(actualClassNames.contains(OneAfterClassMethod.class.getName())); + assertTrue(actualClassNames.contains(MultipleAfterClassMethods.class.getName())); + assertTrue(actualClassNames.contains(FailingAfterClassMethod.class.getName())); + } + + private void verifyListenerCalledForClass(Class clazz) { + String className = clazz.getName(); + assertEquals(1, listener.getClassesCalled().size(), + "Listener should be called exactly once"); + assertEquals(className, listener.getClassesCalled().get(0).getName(), + "Listener should be called for the correct class"); + } + + private void runTestNGWithClass(Class testClass) { + runTestNGWithClasses(0, testClass); + } + + // programmatically test TestNG listener with some classes + private void runTestNGWithClasses(int expectedFailureCount, Class... testClasses) { + XmlSuite suite = new XmlSuite(); + suite.setName("Programmatic Suite"); + + for (Class cls : testClasses) { + // create a new XmlTest for each class so that this simulates the behavior of maven-surefire-plugin + XmlTest test = new XmlTest(suite); + test.setName("Programmatic Test for " + cls.getName()); + List xmlClasses = new ArrayList<>(); + xmlClasses.add(new XmlClass(cls)); + test.setXmlClasses(xmlClasses); + } + + List suites = new ArrayList<>(); + suites.add(suite); + + TestNG tng = new TestNG(); + tng.setXmlSuites(suites); + tng.addListener(listener); + tng.addListener(new TestLoggingListener()); + // set verbose output for debugging + tng.setVerbose(2); + AtomicInteger failureCounter = new AtomicInteger(); + tng.addListener(new ITestListener() { + @Override + public void onTestFailure(ITestResult result) { + failureCounter.incrementAndGet(); + } + }); + tng.run(); + assertEquals(failureCounter.get(), expectedFailureCount, "TestNG run should complete successfully"); + } + + // Test implementation of the abstract listener + private class TestBetweenTestClassesListener extends BetweenTestClassesListenerAdapter { + private final List classesCalled = new ArrayList<>(); + + @Override + protected void onBetweenTestClasses(List testClasses) { + assertEquals(testClasses.size(), 1); + ITestClass testClass = testClasses.get(0); + System.out.println("onBetweenTestClasses " + testClass); + classesCalled.add(testClass); + closeTestInstance(testClass); + } + + private void closeTestInstance(ITestClass testClass) { + Arrays.stream(testClass.getInstances(false)) + .map(instance -> instance instanceof IParameterInfo + ? ((IParameterInfo) instance).getInstance() : instance) + .filter(AutoCloseable.class::isInstance) + .map(AutoCloseable.class::cast) + .forEach(autoCloseable -> { + try { + autoCloseable.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + public List getClassesCalled() { + return classesCalled; + } + + public void reset() { + classesCalled.clear(); + } + } + + private static class TestLoggingListener implements ITestListener { + @Override + public void onTestStart(ITestResult result) { + System.out.println("Test started: " + result.getName() + " in instance " + result.getInstance()); + } + } + + public static class TestRetryAnalyzer extends RetryAnalyzer { + public TestRetryAnalyzer() { + // retry once + setCount(1); + } + } + + private static class CloseableBase implements AutoCloseable { + protected boolean closed; + + protected void checkNotClosed() { + Assert.assertFalse(closed); + } + + public void close() { + closed = true; + } + } + + private static class Base extends CloseableBase { + int counter = 0; + + protected void failOnFirstExecution() { + if (counter++ == 0) { + throw new IllegalStateException("Simulated failure"); + } + } + + @Test(retryAnalyzer = TestRetryAnalyzer.class) + public void testMethod() { + checkNotClosed(); + failOnFirstExecution(); + } + + @AfterMethod(alwaysRun = true) + public void afterMethodInBase() { + checkNotClosed(); + } + + @AfterClass(alwaysRun = true) + public void afterClassInBase() { + checkNotClosed(); + } + } + + private static class NoAfterClassMethods extends Base { + + } + + private static class OneAfterClassMethod extends Base { + @AfterClass + public void afterClass() { + checkNotClosed(); + } + } + + private static class TimeoutAndAfterClassMethod extends Base { + @Override + @Test(timeOut = 100) + public void testMethod() { + checkNotClosed(); + try { + Thread.sleep(300); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + @AfterClass(alwaysRun = true) + public void afterClass() { + checkNotClosed(); + } + } + + private static class MultipleAfterClassMethods extends Base { + private static final AtomicInteger afterClassCounter = new AtomicInteger(0); + + @AfterClass + public void afterClass1() { + checkNotClosed(); + afterClassCounter.incrementAndGet(); + } + + @AfterClass + public void afterClass2() { + checkNotClosed(); + afterClassCounter.incrementAndGet(); + } + } + + private static class DisabledAfterClassMethod extends Base { + @AfterClass(enabled = false) + public void disabledAfterClass() {} + } + + private static class FailingAfterClassMethod extends Base { + @AfterClass + public void failingAfterClass() { + checkNotClosed(); + throw new RuntimeException("Simulated failure"); + } + } + + protected static class FactoryMethodCase extends Base { + private final int id; + + private FactoryMethodCase(int id) { + this.id = id; + } + + @Factory + public static Object[] createTestInstances() { + return new Object[]{ + new FactoryMethodCase(1), + new FactoryMethodCase(2) + }; + } + + @DataProvider + public Object[] idDataProvider() { + return new Object[]{ id }; + } + + @Test(dataProvider = "idDataProvider") + public void testWithDataProvider(int id) { + checkNotClosed(); + Assert.assertEquals(this.id, id); + } + + @AfterClass + public void afterClass() { + checkNotClosed(); + } + + @Override + public String toString() { + return "FactoryMethodCase{" + + "id=" + id + + '}'; + } + } + + protected static class FactoryMethodCaseWithoutAfterClass extends CloseableBase { + private final int id; + + private FactoryMethodCaseWithoutAfterClass(int id) { + this.id = id; + } + + @Factory + public static Object[] createTestInstances() { + return new Object[]{ + new FactoryMethodCaseWithoutAfterClass(1), + new FactoryMethodCaseWithoutAfterClass(2) + }; + } + + @DataProvider + public Object[] idDataProvider() { + return new Object[]{ id }; + } + + @Test(dataProvider = "idDataProvider") + public void testWithDataProvider(int id) { + checkNotClosed(); + Assert.assertEquals(this.id, id); + } + + @Override + public String toString() { + return "FactoryMethodCaseWithoutAfterClass{" + + "id=" + id + + '}'; + } + } +} \ No newline at end of file diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 4594c13f455b1..e3749a10a9160 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -383,14 +383,19 @@ The Apache Software License, Version 2.0 - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar * Prometheus client - - io.prometheus-simpleclient-0.5.0.jar - - io.prometheus-simpleclient_common-0.5.0.jar - - io.prometheus-simpleclient_hotspot-0.5.0.jar - - io.prometheus-simpleclient_servlet-0.5.0.jar - - io.prometheus-simpleclient_log4j2-0.5.0.jar - - io.prometheus-simpleclient_jetty-0.5.0.jar - - io.prometheus.jmx-collector-0.14.0.jar - - io.prometheus-simpleclient_caffeine-0.5.0.jar + - io.prometheus.jmx-collector-0.16.1.jar + - io.prometheus-simpleclient-0.16.0.jar + - io.prometheus-simpleclient_caffeine-0.16.0.jar + - io.prometheus-simpleclient_common-0.16.0.jar + - io.prometheus-simpleclient_hotspot-0.16.0.jar + - io.prometheus-simpleclient_httpserver-0.16.0.jar + - io.prometheus-simpleclient_jetty-0.16.0.jar + - io.prometheus-simpleclient_log4j2-0.16.0.jar + - io.prometheus-simpleclient_servlet-0.16.0.jar + - io.prometheus-simpleclient_servlet_common-0.16.0.jar + - io.prometheus-simpleclient_tracer_common-0.16.0.jar + - io.prometheus-simpleclient_tracer_otel-0.16.0.jar + - io.prometheus-simpleclient_tracer_otel_agent-0.16.0.jar * Jakarta Bean Validation API - jakarta.validation-jakarta.validation-api-2.0.2.jar - javax.validation-validation-api-1.1.0.Final.jar @@ -433,7 +438,7 @@ The Apache Software License, Version 2.0 - org.apache.bookkeeper-native-io-4.16.6.jar * Apache HTTP Client - org.apache.httpcomponents-httpclient-4.5.13.jar - - org.apache.httpcomponents-httpcore-4.4.13.jar + - org.apache.httpcomponents-httpcore-4.4.15.jar * AirCompressor - io.airlift-aircompressor-0.27.jar * AsyncHttpClient @@ -524,7 +529,7 @@ The Apache Software License, Version 2.0 - io.dropwizard.metrics-metrics-jvm-4.1.12.1.jar - io.dropwizard.metrics-metrics-jmx-4.1.12.1.jar * Prometheus - - io.prometheus-simpleclient_httpserver-0.5.0.jar + - io.prometheus-simpleclient_httpserver-0.16.0.jar * Java JSON WebTokens - io.jsonwebtoken-jjwt-api-0.11.1.jar - io.jsonwebtoken-jjwt-impl-0.11.1.jar diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 960010fe96571..7f9109d54a420 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -72,6 +72,7 @@ FROM ubuntu:20.04 ARG DEBIAN_FRONTEND=noninteractive ARG UBUNTU_MIRROR=http://archive.ubuntu.com/ubuntu/ ARG UBUNTU_PORTS_MIRROR=http://ports.ubuntu.com/ubuntu-ports/ +ARG DEFAULT_USERNAME=pulsar # Install some utilities RUN sed -i -e "s|http://archive\.ubuntu\.com/ubuntu/|${UBUNTU_MIRROR:-http://archive.ubuntu.com/ubuntu/}|g" \ @@ -115,4 +116,5 @@ ENV PULSAR_CLIENT_PYTHON_VERSION ${PULSAR_CLIENT_PYTHON_VERSION} RUN /pulsar/bin/install-pulsar-client.sh # The UID must be non-zero. Otherwise, it is arbitrary. No logic should rely on its specific value. +RUN useradd ${DEFAULT_USERNAME} -u 10000 -g 0 --no-create-home --home-dir /pulsar/data USER 10000 diff --git a/kafka-connect-avro-converter-shaded/pom.xml b/kafka-connect-avro-converter-shaded/pom.xml deleted file mode 100644 index b0f53aeff8aa3..0000000000000 --- a/kafka-connect-avro-converter-shaded/pom.xml +++ /dev/null @@ -1,118 +0,0 @@ - - - - 4.0.0 - - pulsar - org.apache.pulsar - 2.10.7-SNAPSHOT - .. - - - kafka-connect-avro-converter-shaded - Apache Pulsar :: Kafka Connect Avro Converter shaded - - - - - io.confluent - kafka-connect-avro-converter - ${confluent.version} - - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - ${shadePluginPhase} - - shade - - - - - true - true - - - - io.confluent:* - io.confluent:kafka-avro-serializer - io.confluent:kafka-schema-registry-client - io.confluent:common-config - io.confluent:common-utils - org.apache.avro:* - - org.codehaus.jackson:jackson-core-asl - org.codehaus.jackson:jackson-mapper-asl - com.thoughtworks.paranamer:paranamer - org.xerial.snappy:snappy-java - org.apache.commons:commons-compress - org.tukaani:xz - - - - - io.confluent - org.apache.pulsar.kafka.shade.io.confluent - - - org.apache.avro - org.apache.pulsar.kafka.shade.avro - - - org.codehaus.jackson - org.apache.pulsar.kafka.shade.org.codehaus.jackson - - - com.thoughtworks.paranamer - org.apache.pulsar.kafka.shade.com.thoughtworks.paranamer - - - org.xerial.snappy - org.apache.pulsar.kafka.shade.org.xerial.snappy - - - org.apache.commons - org.apache.pulsar.kafka.shade.org.apache.commons - - - org.tukaani - org.apache.pulsar.kafka.shade.org.tukaani - - - - - - - - - - - - diff --git a/pom.xml b/pom.xml index 04f2dc5ec0382..8a79b3f9097e5 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ org.apache apache - 23 + 29 org.apache.pulsar @@ -131,7 +131,7 @@ flexible messaging model and an intuitive client API. 2.5.2 2.34 1.10.50 - 0.5.0 + 0.16.0 4.5.15 7.9.2 1.7.32 @@ -188,7 +188,7 @@ flexible messaging model and an intuitive client API. 2.4.9 32.0.0-jre 1.0 - 0.14.0 + 0.16.1 7.0.1 5.3.0 5.3.0 @@ -265,23 +265,22 @@ flexible messaging model and an intuitive client API. 3.0.0 4.1 1.0 - 3.0.0-M3 + 3.5.0 - - 3.0.0-M3 - 3.3.0 - 3.9.0 - 3.1.2 + 3.5.3 + 3.7.1 + 3.14.0 + 3.8.1 2.3.0 3.4.0 3.0.0 3.6.0 1.0.0 - 1.5.0 + 1.5.1 3.1.2 4.9.10 - 3.4.3 + 3.5.3 1.4.1.Final 0.8.7 4.2.2 @@ -326,6 +325,7 @@ flexible messaging model and an intuitive client API. * + test @@ -377,6 +377,7 @@ flexible messaging model and an intuitive client API. org.powermock powermock-reflect ${powermock.version} + test @@ -1564,7 +1565,6 @@ flexible messaging model and an intuitive client API. UTF-8 true true - true false @@ -1611,10 +1611,17 @@ flexible messaging model and an intuitive client API. listener - org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.AnnotationListener,org.apache.pulsar.tests.FailFastNotifier,org.apache.pulsar.tests.MockitoCleanupListener,org.apache.pulsar.tests.FastThreadLocalCleanupListener,org.apache.pulsar.tests.ThreadLeakDetectorListener + org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.JacocoDumpListener,org.apache.pulsar.tests.TraceTestResourceCleanupListener,org.apache.pulsar.tests.AnnotationListener,org.apache.pulsar.tests.FailFastNotifier,org.apache.pulsar.tests.MockitoCleanupListener,org.apache.pulsar.tests.FastThreadLocalCleanupListener,org.apache.pulsar.tests.ThreadLeakDetectorListener,org.apache.pulsar.tests.SingletonCleanerListener + + + org.apache.maven.surefire + surefire-testng + ${surefire.version} + + @@ -1901,6 +1908,8 @@ flexible messaging model and an intuitive client API. **/*.dylib src/test/resources/*.txt src/test/resources/ssl/README.md + + **/.mvn/** @@ -1996,7 +2005,6 @@ flexible messaging model and an intuitive client API. org.apache.maven.plugins maven-shade-plugin - ${maven-shade-plugin} org.apache.maven.plugins @@ -2313,9 +2321,6 @@ flexible messaging model and an intuitive client API. pulsar-io - - kafka-connect-avro-converter-shaded - bouncy-castle @@ -2617,6 +2622,20 @@ flexible messaging model and an intuitive client API. osx-x86_64 + + + pulsar-io-tests + + pulsar-io + + + + + pulsar-sql-tests + + pulsar-sql + + diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java index c3225b66a50d3..0c5c290e9ec71 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java @@ -74,7 +74,7 @@ public class AuthenticationProviderToken implements AuthenticationProvider { static final String TOKEN = "token"; private static final Counter expiredTokenMetrics = Counter.build() - .name("pulsar_expired_token_count") + .name("pulsar_expired_token_total") .help("Pulsar expired token") .register(); diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/metrics/AuthenticationMetrics.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/metrics/AuthenticationMetrics.java index e2029bcfefc65..00cb01ed08a84 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/metrics/AuthenticationMetrics.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/metrics/AuthenticationMetrics.java @@ -22,12 +22,12 @@ public class AuthenticationMetrics { private static final Counter authSuccessMetrics = Counter.build() - .name("pulsar_authentication_success_count") + .name("pulsar_authentication_success_total") .help("Pulsar authentication success") .labelNames("provider_name", "auth_method") .register(); private static final Counter authFailuresMetrics = Counter.build() - .name("pulsar_authentication_failures_count") + .name("pulsar_authentication_failures_total") .help("Pulsar authentication failures") .labelNames("provider_name", "auth_method", "reason") .register(); diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorUtils.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorUtils.java index 4bf741c0f9482..658a65f02f628 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorUtils.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorUtils.java @@ -60,7 +60,7 @@ public static void generateSystemMetrics(SimpleTextOutputStream stream, String c Collector.MetricFamilySamples metricFamily = metricFamilySamples.nextElement(); // Write type of metric - stream.write("# TYPE ").write(metricFamily.name).write(' ') + stream.write("# TYPE ").write(metricFamily.name).write(getTypeNameSuffix(metricFamily.type)).write(' ') .write(getTypeStr(metricFamily.type)).write('\n'); for (int i = 0; i < metricFamily.samples.size(); i++) { @@ -95,19 +95,27 @@ public static void generateSystemMetrics(SimpleTextOutputStream stream, String c } } + static String getTypeNameSuffix(Collector.Type type) { + if (type.equals(Collector.Type.INFO)) { + return "_info"; + } + return ""; + } + static String getTypeStr(Collector.Type type) { switch (type) { case COUNTER: return "counter"; case GAUGE: + case INFO: return "gauge"; - case SUMMARY : + case SUMMARY: return "summary"; case HISTOGRAM: return "histogram"; - case UNTYPED: + case UNKNOWN: default: - return "untyped"; + return "unknown"; } } diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorUtilsTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorUtilsTest.java index b84636dc72aee..4fd06d944b8a2 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorUtilsTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorUtilsTest.java @@ -46,7 +46,7 @@ public void testGenerateSystemMetricsWithSpecifyCluster() throws Exception { PrometheusMetricsGeneratorUtils.generate(defaultClusterValue, out, Collections.emptyList()); System.out.println(("metrics 1 :" + out.toString())); assertTrue(out.toString().contains( - String.format("%s{cluster=\"%s\"} 1.0", metricsName, specifyClusterValue) + String.format("%s_total{cluster=\"%s\"} 1.0", metricsName, specifyClusterValue) )); // cleanup out.close(); @@ -70,7 +70,7 @@ public void testGenerateSystemMetricsWithDefaultCluster() throws Exception { PrometheusMetricsGeneratorUtils.generate(defaultClusterValue, out, Collections.emptyList()); System.out.println(("metrics 1 :" + out.toString())); assertTrue(out.toString().contains( - String.format("%s{cluster=\"%s\",%s=\"%s\"} 1.0", + String.format("%s_total{cluster=\"%s\",%s=\"%s\"} 1.0", metricsName, defaultClusterValue, labelName, labelValue) )); // cleanup @@ -92,7 +92,7 @@ public void testGenerateSystemMetricsWithoutCustomizedLabel() throws Exception { PrometheusMetricsGeneratorUtils.generate(defaultClusterValue, out, Collections.emptyList()); System.out.println(("metrics 1 :" + out.toString())); assertTrue(out.toString().contains( - String.format("%s{cluster=\"%s\"} 1.0", metricsName, defaultClusterValue) + String.format("%s_total{cluster=\"%s\"} 1.0", metricsName, defaultClusterValue) )); // cleanup out.close(); diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 9b7886c825e81..53877f3f969f1 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -860,7 +860,7 @@ - brokerSkipTest + skipTestsForUnitGroupOther diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroup.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroup.java index b67eea09fd039..da4ab78655cab 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroup.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroup.java @@ -901,7 +901,7 @@ public int hashCode() { .labelNames(resourceGroupMontoringclassRemotebrokerLabels) .register(); private static final Gauge rgRemoteUsageReportsBytes = Gauge.build() - .name("pulsar_resource_group_remote_usage_bytes_used") + .name("pulsar_resource_group_remote_usage_bytes_used_gauge") .help("Bytes used reported about this from a remote broker") .labelNames(resourceGroupMontoringclassRemotebrokerLabels) .register(); @@ -911,7 +911,7 @@ public int hashCode() { .labelNames(resourceGroupMontoringclassRemotebrokerLabels) .register(); private static final Gauge rgRemoteUsageReportsMessages = Gauge.build() - .name("pulsar_resource_group_remote_usage_messages_used") + .name("pulsar_resource_group_remote_usage_messages_used_gauge") .help("Messages used reported about this from a remote broker") .labelNames(resourceGroupMontoringclassRemotebrokerLabels) .register(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java index 6c620e7074fc0..cbe0d1e95f02e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java @@ -1040,7 +1040,7 @@ protected Cache newStatsCache(long durationMS) { .labelNames(resourceGroupMonitoringclassLabels) .register(); private static final Gauge rgCalculatedQuotaBytes = Gauge.build() - .name("pulsar_resource_group_calculated_bytes_quota") + .name("pulsar_resource_group_calculated_bytes_quota_gauge") .help("Bytes quota calculated for resource group") .labelNames(resourceGroupMonitoringclassLabels) .register(); @@ -1050,7 +1050,7 @@ protected Cache newStatsCache(long durationMS) { .labelNames(resourceGroupMonitoringclassLabels) .register(); private static final Gauge rgCalculatedQuotaMessages = Gauge.build() - .name("pulsar_resource_group_calculated_messages_quota") + .name("pulsar_resource_group_calculated_messages_quota_gauge") .help("Messages quota calculated for resource group") .labelNames(resourceGroupMonitoringclassLabels) .register(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ResourceGroupsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ResourceGroupsTest.java index 4f20c1b360556..7f083e2c6219f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ResourceGroupsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ResourceGroupsTest.java @@ -18,12 +18,19 @@ */ package org.apache.pulsar.broker.admin; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; -import org.apache.pulsar.broker.web.RestException; import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.broker.admin.v2.ResourceGroups; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.ResourceGroup; import org.apache.pulsar.common.policies.data.TenantInfoImpl; @@ -32,16 +39,8 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; - -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.spy; -import static org.testng.Assert.*; - +@Slf4j public class ResourceGroupsTest extends MockedPulsarServiceBaseTest { - private ResourceGroups resourcegroups; private List expectedRgNames = Lists.newArrayList(); private final String testCluster = "test"; private final String testTenant = "test-tenant"; @@ -52,14 +51,6 @@ public class ResourceGroupsTest extends MockedPulsarServiceBaseTest { @Override protected void setup() throws Exception { super.internalSetup(); - resourcegroups = spy(ResourceGroups.class); - resourcegroups.setServletContext(new MockServletContext()); - resourcegroups.setPulsar(pulsar); - doReturn(false).when(resourcegroups).isRequestHttps(); - doReturn("test").when(resourcegroups).clientAppId(); - doReturn(null).when(resourcegroups).originalPrincipal(); - doReturn(null).when(resourcegroups).clientAuthData(); - prepareData(); } @@ -72,16 +63,13 @@ protected void cleanup() throws Exception { @Test public void testCrudResourceGroups() throws Exception { // create with null resourcegroup should fail. - try { - resourcegroups.createOrUpdateResourceGroup("test-resourcegroup-invalid", null); - fail("should have failed"); - } catch (RestException e){ - //Ok. - } + assertThatThrownBy(() -> { + admin.resourcegroups().createResourceGroup("test-resourcegroup-invalid", null); + }).isInstanceOf(PulsarAdminException.class); // create resourcegroup with default values ResourceGroup testResourceGroupOne = new ResourceGroup(); - resourcegroups.createOrUpdateResourceGroup("test-resourcegroup-one", testResourceGroupOne); + admin.resourcegroups().createResourceGroup("test-resourcegroup-one", testResourceGroupOne); expectedRgNames.add("test-resourcegroup-one"); // create resourcegroup with non default values. @@ -91,16 +79,13 @@ public void testCrudResourceGroups() throws Exception { testResourceGroupTwo.setPublishRateInMsgs(100); testResourceGroupTwo.setPublishRateInBytes(10000L); - resourcegroups.createOrUpdateResourceGroup("test-resourcegroup-two", testResourceGroupTwo); + admin.resourcegroups().createResourceGroup("test-resourcegroup-two", testResourceGroupTwo); expectedRgNames.add("test-resourcegroup-two"); // null resourcegroup update should fail. - try { - resourcegroups.createOrUpdateResourceGroup("test-resourcegroup-one", null); - fail("should have failed"); - } catch (RestException e){ - //Ok. - } + assertThatThrownBy(() -> { + admin.resourcegroups().createResourceGroup("test-resourcegroup-one", null); + }).isInstanceOf(PulsarAdminException.class); // update with some real values ResourceGroup testResourceGroupOneUpdate = new ResourceGroup(); @@ -108,35 +93,29 @@ public void testCrudResourceGroups() throws Exception { testResourceGroupOneUpdate.setDispatchRateInBytes(5000L); testResourceGroupOneUpdate.setPublishRateInMsgs(10); testResourceGroupOneUpdate.setPublishRateInBytes(1000L); - resourcegroups.createOrUpdateResourceGroup("test-resourcegroup-one", testResourceGroupOneUpdate); + admin.resourcegroups().createResourceGroup("test-resourcegroup-one", testResourceGroupOneUpdate); // get a non existent resourcegroup - try { - resourcegroups.getResourceGroup("test-resourcegroup-invalid"); - fail("should have failed"); - } catch (RestException e) { - //Ok - } + assertThatThrownBy(() -> { + admin.resourcegroups().getResourceGroup("test-resourcegroup-invalid"); + }).isInstanceOf(PulsarAdminException.class); // get list of all resourcegroups - List gotRgNames = resourcegroups.getResourceGroups(); + List gotRgNames = admin.resourcegroups().getResourceGroups(); assertEquals(gotRgNames.size(), expectedRgNames.size()); Collections.sort(gotRgNames); Collections.sort(expectedRgNames); assertEquals(gotRgNames, expectedRgNames); // delete a non existent resourcegroup - try { - resourcegroups.deleteResourceGroup("test-resourcegroup-invalid"); - fail("should have failed"); - } catch (RestException e) { - //Ok - } + assertThatThrownBy(() -> { + admin.resourcegroups().getResourceGroup("test-resourcegroup-invalid"); + }).isInstanceOf(PulsarAdminException.class); // delete the ResourceGroups we created. Iterator rg_Iterator = expectedRgNames.iterator(); while (rg_Iterator.hasNext()) { - resourcegroups.deleteResourceGroup(rg_Iterator.next()); + admin.resourcegroups().deleteResourceGroup(rg_Iterator.next()); } } @@ -149,28 +128,28 @@ public void testNamespaceResourceGroup() throws Exception { testResourceGroupTwo.setPublishRateInMsgs(100); testResourceGroupTwo.setPublishRateInBytes(10000L); - resourcegroups.createOrUpdateResourceGroup("test-resourcegroup-three", testResourceGroupTwo); + admin.resourcegroups().createResourceGroup("test-resourcegroup-three", testResourceGroupTwo); admin.namespaces().createNamespace(testNameSpace); // set invalid ResourceGroup in namespace - try { + assertThatThrownBy(() -> { admin.namespaces().setNamespaceResourceGroup(testNameSpace, "test-resourcegroup-invalid"); - fail("should have failed"); - } catch (Exception e){ - //Ok. - } + }).isInstanceOf(PulsarAdminException.class); + // set resourcegroup in namespace admin.namespaces().setNamespaceResourceGroup(testNameSpace, "test-resourcegroup-three"); + Awaitility.await().untilAsserted(() -> assertNotNull(pulsar.getResourceGroupServiceManager() + .getNamespaceResourceGroup(NamespaceName.get(testNameSpace)))); // try deleting the resourcegroup, should fail - try { - resourcegroups.deleteResourceGroup("test-resourcegroup-three"); - } catch (RestException e) { - //Ok - } + assertThatThrownBy(() -> { + admin.resourcegroups().deleteResourceGroup("test-resourcegroup-three"); + }).isInstanceOf(PulsarAdminException.class); + // remove resourcegroup from namespace admin.namespaces().removeNamespaceResourceGroup(testNameSpace); - Awaitility.await().untilAsserted(() -> { - resourcegroups.deleteResourceGroup("test-resourcegroup-three"); - }); + Awaitility.await().untilAsserted(() -> assertNull(pulsar.getResourceGroupServiceManager() + .getNamespaceResourceGroup(NamespaceName.get(testNameSpace)))); + + admin.resourcegroups().deleteResourceGroup("test-resourcegroup-three"); } private void prepareData() throws PulsarAdminException { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java index 1c08a8b3a3909..6ddabf0897480 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java @@ -36,7 +36,6 @@ import io.netty.channel.nio.NioEventLoopGroup; import java.lang.reflect.Field; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -227,50 +226,6 @@ public void teardown() throws Exception { } } - @Test - public void testCanAcknowledgeAndCommitForTransaction() throws ExecutionException, InterruptedException { - doAnswer((invocationOnMock) -> { - ((AsyncCallbacks.DeleteCallback) invocationOnMock.getArguments()[1]) - .deleteComplete(invocationOnMock.getArguments()[2]); - return null; - }).when(cursorMock).asyncDelete(any(List.class), any(AsyncCallbacks.DeleteCallback.class), any()); - - List> positionsPair = new ArrayList<>(); - positionsPair.add(new MutablePair<>(new PositionImpl(1, 1), 0)); - positionsPair.add(new MutablePair<>(new PositionImpl(1, 3), 0)); - positionsPair.add(new MutablePair<>(new PositionImpl(1, 5), 0)); - - doAnswer((invocationOnMock) -> { - assertTrue(Arrays.deepEquals(((List)invocationOnMock.getArguments()[0]).toArray(), - positionsPair.toArray())); - ((AsyncCallbacks.MarkDeleteCallback) invocationOnMock.getArguments()[2]) - .markDeleteComplete(invocationOnMock.getArguments()[3]); - return null; - }).when(cursorMock).asyncMarkDelete(any(), any(), any(AsyncCallbacks.MarkDeleteCallback.class), any()); - - // Single ack for txn - persistentSubscription.transactionIndividualAcknowledge(txnID1, positionsPair); - - // Commit txn - persistentSubscription.endTxn(txnID1.getMostSigBits(), txnID1.getLeastSigBits(), TxnAction.COMMIT_VALUE, -1).get(); - - List positions = new ArrayList<>(); - positions.add(new PositionImpl(3, 100)); - - // Cumulative ack for txn - persistentSubscription.transactionCumulativeAcknowledge(txnID1, positions); - - doAnswer((invocationOnMock) -> { - assertEquals(((PositionImpl) invocationOnMock.getArguments()[0]).compareTo(new PositionImpl(3, 100)), 0); - ((AsyncCallbacks.MarkDeleteCallback) invocationOnMock.getArguments()[2]) - .markDeleteComplete(invocationOnMock.getArguments()[3]); - return null; - }).when(cursorMock).asyncMarkDelete(any(), any(), any(AsyncCallbacks.MarkDeleteCallback.class), any()); - - // Commit txn - persistentSubscription.endTxn(txnID1.getMostSigBits(), txnID1.getLeastSigBits(), TxnAction.COMMIT_VALUE, -1).get(); - } - @Test public void testCanAcknowledgeAndAbortForTransaction() throws Exception { List> positionsPair = new ArrayList<>(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java index 6ea2058b1f039..e91fbdf0c142d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java @@ -987,6 +987,16 @@ public void testDuplicateMetricTypeDefinitions() throws Exception { if (!typeDefs.containsKey(summaryMetricName)) { fail("Metric " + metricName + " does not have a corresponding summary type definition"); } + } else if (metricName.endsWith("_created")) { + String summaryMetricName = metricName.substring(0, metricName.indexOf("_created")); + if (!typeDefs.containsKey(summaryMetricName)) { + fail("Metric " + metricName + " does not have a corresponding summary type definition"); + } + } else if (metricName.endsWith("_total")) { + String summaryMetricName = metricName.substring(0, metricName.indexOf("_total")); + if (!typeDefs.containsKey(summaryMetricName)) { + fail("Metric " + metricName + " does not have a corresponding counter type definition"); + } } else { fail("Metric " + metricName + " does not have a type definition"); } @@ -1206,7 +1216,7 @@ public String getCommandData() { PrometheusMetricsGenerator.generate(pulsar, false, false, false, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); - List cm = (List) metrics.get("pulsar_authentication_success_count"); + List cm = (List) metrics.get("pulsar_authentication_success_total"); boolean haveSucceed = false; for (Metric metric : cm) { if (Objects.equals(metric.tags.get("auth_method"), "token") @@ -1216,7 +1226,7 @@ public String getCommandData() { } Assert.assertTrue(haveSucceed); - cm = (List) metrics.get("pulsar_authentication_failures_count"); + cm = (List) metrics.get("pulsar_authentication_failures_total"); boolean haveFailed = false; for (Metric metric : cm) { @@ -1267,7 +1277,7 @@ public String getCommandData() { PrometheusMetricsGenerator.generate(pulsar, false, false, false, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); - List cm = (List) metrics.get("pulsar_expired_token_count"); + List cm = (List) metrics.get("pulsar_expired_token_total"); assertEquals(cm.size(), 1); provider.close(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java index 372f62d5b9cb2..286163a93a5e2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java @@ -924,9 +924,9 @@ public void testMergeLookupRequests() throws Exception { } private int calculateLookupRequestCount() throws Exception { - int failures = CollectorRegistry.defaultRegistry.getSampleValue("pulsar_broker_lookup_failures") + int failures = CollectorRegistry.defaultRegistry.getSampleValue("pulsar_broker_lookup_failures_total") .intValue(); - int answers = CollectorRegistry.defaultRegistry.getSampleValue("pulsar_broker_lookup_answers") + int answers = CollectorRegistry.defaultRegistry.getSampleValue("pulsar_broker_lookup_answers_total") .intValue(); return failures + answers; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java index 7029dc222dbd5..f26f07b709087 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java @@ -946,7 +946,7 @@ private void testPulsarSinkLocalRun(String jarFilePathUrl, int parallelism, Stri assertFalse(metrics.isEmpty()); PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_sink_sink_exceptions_total"); if (m == null) { - m = metrics.get("pulsar_sink_sink_exceptions_total_1min"); + m = metrics.get("pulsar_sink_sink_exceptions_1min_total"); } assertEquals(m.value, 0); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarBatchSourceE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarBatchSourceE2ETest.java index d1cd1dc0614e3..e371190a0d0e5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarBatchSourceE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarBatchSourceE2ETest.java @@ -113,7 +113,7 @@ private void testPulsarBatchSourceStats(String jarFilePathUrl) throws Exception assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName)); assertTrue(m.value > 0.0); - m = metrics.get("pulsar_source_received_total_1min"); + m = metrics.get("pulsar_source_received_1min_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), sourceName); @@ -127,7 +127,7 @@ private void testPulsarBatchSourceStats(String jarFilePathUrl) throws Exception assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName)); assertTrue(m.value > 0.0); - m = metrics.get("pulsar_source_written_total_1min"); + m = metrics.get("pulsar_source_written_1min_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), sourceName); @@ -141,7 +141,7 @@ private void testPulsarBatchSourceStats(String jarFilePathUrl) throws Exception assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName)); assertEquals(m.value, 0.0); - m = metrics.get("pulsar_source_source_exceptions_total_1min"); + m = metrics.get("pulsar_source_source_exceptions_1min_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), sourceName); @@ -155,7 +155,7 @@ private void testPulsarBatchSourceStats(String jarFilePathUrl) throws Exception assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName)); assertEquals(m.value, 0.0); - m = metrics.get("pulsar_source_system_exceptions_total_1min"); + m = metrics.get("pulsar_source_system_exceptions_1min_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), sourceName); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionE2ETest.java index 09efd6e5296f9..9258563942ba5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionE2ETest.java @@ -344,7 +344,7 @@ public void testPulsarFunctionStats() throws Exception { assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); - m = metrics.get("pulsar_function_received_total_1min"); + m = metrics.get("pulsar_function_received_1min_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); @@ -358,7 +358,7 @@ public void testPulsarFunctionStats() throws Exception { assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); - m = metrics.get("pulsar_function_user_exceptions_total_1min"); + m = metrics.get("pulsar_function_user_exceptions_1min_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); @@ -386,7 +386,7 @@ public void testPulsarFunctionStats() throws Exception { assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); - m = metrics.get("pulsar_function_system_exceptions_total_1min"); + m = metrics.get("pulsar_function_system_exceptions_1min_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); @@ -407,7 +407,7 @@ public void testPulsarFunctionStats() throws Exception { assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); - m = metrics.get("pulsar_function_processed_successfully_total_1min"); + m = metrics.get("pulsar_function_processed_successfully_1min_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); @@ -504,7 +504,7 @@ public void testPulsarFunctionStats() throws Exception { assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, (double) totalMsgs); - m = metrics.get("pulsar_function_received_total_1min"); + m = metrics.get("pulsar_function_received_1min_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); @@ -518,7 +518,7 @@ public void testPulsarFunctionStats() throws Exception { assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); - m = metrics.get("pulsar_function_user_exceptions_total_1min"); + m = metrics.get("pulsar_function_user_exceptions_1min_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); @@ -546,7 +546,7 @@ public void testPulsarFunctionStats() throws Exception { assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); - m = metrics.get("pulsar_function_system_exceptions_total_1min"); + m = metrics.get("pulsar_function_system_exceptions_1min_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); @@ -567,7 +567,7 @@ public void testPulsarFunctionStats() throws Exception { assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, (double) totalMsgs); - m = metrics.get("pulsar_function_processed_successfully_total_1min"); + m = metrics.get("pulsar_function_processed_successfully_1min_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSinkE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSinkE2ETest.java index 8cb581669f8c1..aceb32c0e188c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSinkE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSinkE2ETest.java @@ -284,7 +284,7 @@ private void testPulsarSinkStats(String jarFilePathUrl, Function 0.0); - m = metrics.get("pulsar_source_received_total_1min"); + m = metrics.get("pulsar_source_received_1min_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), sourceName); @@ -131,7 +131,7 @@ private void testPulsarSourceStats(String jarFilePathUrl) throws Exception { assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName)); assertTrue(m.value > 0.0); - m = metrics.get("pulsar_source_written_total_1min"); + m = metrics.get("pulsar_source_written_1min_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), sourceName); @@ -145,7 +145,7 @@ private void testPulsarSourceStats(String jarFilePathUrl) throws Exception { assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName)); assertEquals(m.value, 0.0); - m = metrics.get("pulsar_source_source_exceptions_total_1min"); + m = metrics.get("pulsar_source_source_exceptions_1min_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), sourceName); @@ -159,7 +159,7 @@ private void testPulsarSourceStats(String jarFilePathUrl) throws Exception { assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName)); assertEquals(m.value, 0.0); - m = metrics.get("pulsar_source_system_exceptions_total_1min"); + m = metrics.get("pulsar_source_system_exceptions_1min_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), sourceName); diff --git a/pulsar-broker/src/test/resources/prometheus_metrics_sample.txt b/pulsar-broker/src/test/resources/prometheus_metrics_sample.txt index a35f89a5836c2..2022fbd800000 100644 --- a/pulsar-broker/src/test/resources/prometheus_metrics_sample.txt +++ b/pulsar-broker/src/test/resources/prometheus_metrics_sample.txt @@ -83,8 +83,8 @@ jvm_buffer_pool_used_buffers{cluster="use",pool="direct"} 97.0 jvm_buffer_pool_used_buffers{cluster="use",pool="mapped"} 0.0 # TYPE pulsar_broker_lookup_pending_requests gauge pulsar_broker_lookup_pending_requests{cluster="use"} 0.0 -# TYPE pulsar_authentication_success_count counter -pulsar_authentication_success_count{cluster="use",provider_name="AuthenticationProviderTls",auth_method="tls"} 850.0 +# TYPE pulsar_authentication_success_total counter +pulsar_authentication_success_total{cluster="use",provider_name="AuthenticationProviderTls",auth_method="tls"} 850.0 # TYPE pulsar_version_info gauge pulsar_version_info{cluster="use",version="2.8.0-SNAPSHOT",commit="e600b65a05e610bc7cbd874d4c446619d9d9606f"} 1.0 # TYPE zk_read_latency summary @@ -311,8 +311,8 @@ jvm_gc_collection_seconds_count{cluster="use",gc="G1 Old Generation"} 0.0 jvm_gc_collection_seconds_sum{cluster="use",gc="G1 Old Generation"} 0.0 # TYPE pulsar_broker_lookup_failures counter pulsar_broker_lookup_failures{cluster="use"} 0.0 -# TYPE pulsar_authentication_failures_count counter -pulsar_authentication_failures_count{cluster="use",provider_name="AuthenticationProviderTls",auth_method="tls",reason="Client unable to authenticate with TLS certificate"} 1.0 +# TYPE pulsar_authentication_failures_total counter +pulsar_authentication_failures_total{cluster="use",provider_name="AuthenticationProviderTls",auth_method="tls",reason="Client unable to authenticate with TLS certificate"} 1.0 # TYPE pulsar_broker_lookup_answers counter pulsar_broker_lookup_answers{cluster="use"} 134.0 # TYPE pulsar_topics_count gauge @@ -665,8 +665,8 @@ pulsar_function_worker_total_function_count{cluster="use",} 1 pulsar_function_worker_total_expected_instance_count{cluster="use",} 1 pulsar_function_worker_is_leader{cluster="use",} 1 pulsar_function_last_invocation{tenant="external-repl-prop",namespace="external-repl-prop/io",name="PulsarSink-test",instance_id="0",cluster="use",fqfn="external-repl-prop/io/PulsarSink-test",} 0.0 -pulsar_function_processed_successfully_total_1min{tenant="external-repl-prop",namespace="external-repl-prop/io",name="PulsarSink-test",instance_id="0",cluster="use",fqfn="external-repl-prop/io/PulsarSink-test",} 0.0 -pulsar_function_received_total_1min{tenant="external-repl-prop",namespace="external-repl-prop/io",name="PulsarSink-test",instance_id="0",cluster="use",fqfn="external-repl-prop/io/PulsarSink-test",} 0.0 +pulsar_function_processed_successfully_1min_total{tenant="external-repl-prop",namespace="external-repl-prop/io",name="PulsarSink-test",instance_id="0",cluster="use",fqfn="external-repl-prop/io/PulsarSink-test",} 0.0 +pulsar_function_received_1min_total{tenant="external-repl-prop",namespace="external-repl-prop/io",name="PulsarSink-test",instance_id="0",cluster="use",fqfn="external-repl-prop/io/PulsarSink-test",} 0.0 pulsar_function_user_exceptions_total{tenant="external-repl-prop",namespace="external-repl-prop/io",name="PulsarSink-test",instance_id="0",cluster="use",fqfn="external-repl-prop/io/PulsarSink-test",} 0.0 pulsar_function_process_latency_ms_1min{tenant="external-repl-prop",namespace="external-repl-prop/io",name="PulsarSink-test",instance_id="0",cluster="use",fqfn="external-repl-prop/io/PulsarSink-test",quantile="0.5",} NaN pulsar_function_process_latency_ms_1min{tenant="external-repl-prop",namespace="external-repl-prop/io",name="PulsarSink-test",instance_id="0",cluster="use",fqfn="external-repl-prop/io/PulsarSink-test",quantile="0.9",} NaN @@ -683,8 +683,8 @@ pulsar_function_process_latency_ms{tenant="external-repl-prop",namespace="extern pulsar_function_process_latency_ms{tenant="external-repl-prop",namespace="external-repl-prop/io",name="PulsarSink-test",instance_id="0",cluster="use",fqfn="external-repl-prop/io/PulsarSink-test",quantile="0.999",} NaN pulsar_function_process_latency_ms_count{tenant="external-repl-prop",namespace="external-repl-prop/io",name="PulsarSink-test",instance_id="0",cluster="use",fqfn="external-repl-prop/io/PulsarSink-test",} 0.0 pulsar_function_process_latency_ms_sum{tenant="external-repl-prop",namespace="external-repl-prop/io",name="PulsarSink-test",instance_id="0",cluster="use",fqfn="external-repl-prop/io/PulsarSink-test",} 0.0 -pulsar_function_system_exceptions_total_1min{tenant="external-repl-prop",namespace="external-repl-prop/io",name="PulsarSink-test",instance_id="0",cluster="use",fqfn="external-repl-prop/io/PulsarSink-test",} 0.0 -pulsar_function_user_exceptions_total_1min{tenant="external-repl-prop",namespace="external-repl-prop/io",name="PulsarSink-test",instance_id="0",cluster="use",fqfn="external-repl-prop/io/PulsarSink-test",} 0.0 +pulsar_function_system_exceptions_1min_total{tenant="external-repl-prop",namespace="external-repl-prop/io",name="PulsarSink-test",instance_id="0",cluster="use",fqfn="external-repl-prop/io/PulsarSink-test",} 0.0 +pulsar_function_user_exceptions_1min_total{tenant="external-repl-prop",namespace="external-repl-prop/io",name="PulsarSink-test",instance_id="0",cluster="use",fqfn="external-repl-prop/io/PulsarSink-test",} 0.0 # TYPE pulsar_ml_cache_evictions gauge pulsar_ml_cache_evictions{cluster="use"} 0 # TYPE pulsar_ml_cache_hits_rate gauge diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/FunctionStatsManager.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/FunctionStatsManager.java index 3db2ec9b4ba1a..ac80d1a113fcf 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/FunctionStatsManager.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/FunctionStatsManager.java @@ -52,13 +52,13 @@ public class FunctionStatsManager extends ComponentStatsManager{ public static final String LAST_INVOCATION = "last_invocation"; public static final String RECEIVED_TOTAL = "received_total"; - public static final String PROCESSED_SUCCESSFULLY_TOTAL_1min = "processed_successfully_total_1min"; - public static final String SYSTEM_EXCEPTIONS_TOTAL_1min = "system_exceptions_total_1min"; - public static final String USER_EXCEPTIONS_TOTAL_1min = "user_exceptions_total_1min"; - public static final String SOURCE_EXCEPTIONS_TOTAL_1min = "source_exceptions_total_1min"; - public static final String SINK_EXCEPTIONS_TOTAL_1min = "sink_exceptions_total_1min"; + public static final String PROCESSED_SUCCESSFULLY_TOTAL_1min = "processed_successfully_1min"; + public static final String SYSTEM_EXCEPTIONS_TOTAL_1min = "system_exceptions_1min"; + public static final String USER_EXCEPTIONS_TOTAL_1min = "user_exceptions_1min"; + public static final String SOURCE_EXCEPTIONS_TOTAL_1min = "source_exceptions_1min"; + public static final String SINK_EXCEPTIONS_TOTAL_1min = "sink_exceptions_1min"; public static final String PROCESS_LATENCY_MS_1min = "process_latency_ms_1min"; - public static final String RECEIVED_TOTAL_1min = "received_total_1min"; + public static final String RECEIVED_TOTAL_1min = "received_1min"; /** Declare Prometheus stats **/ diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/SinkStatsManager.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/SinkStatsManager.java index a0536ef8f23e2..f7785e1f7f79c 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/SinkStatsManager.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/SinkStatsManager.java @@ -40,10 +40,10 @@ public class SinkStatsManager extends ComponentStatsManager { public static final String RECEIVED_TOTAL = "received_total"; public static final String WRITTEN_TOTAL = "written_total"; - public static final String SYSTEM_EXCEPTIONS_TOTAL_1min = "system_exceptions_total_1min"; - public static final String SINK_EXCEPTIONS_TOTAL_1min = "sink_exceptions_total_1min"; - public static final String RECEIVED_TOTAL_1min = "received_total_1min"; - public static final String WRITTEN_TOTAL_1min = "written_total_1min"; + public static final String SYSTEM_EXCEPTIONS_TOTAL_1min = "system_exceptions_1min"; + public static final String SINK_EXCEPTIONS_TOTAL_1min = "sink_exceptions_1min"; + public static final String RECEIVED_TOTAL_1min = "received_1min"; + public static final String WRITTEN_TOTAL_1min = "written_1min"; /** Declare Prometheus stats **/ diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/SourceStatsManager.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/SourceStatsManager.java index fd67984a4fb31..3ac41f52de98d 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/SourceStatsManager.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/SourceStatsManager.java @@ -40,10 +40,10 @@ public class SourceStatsManager extends ComponentStatsManager { public static final String RECEIVED_TOTAL = "received_total"; public static final String WRITTEN_TOTAL = "written_total"; - public static final String SYSTEM_EXCEPTIONS_TOTAL_1min = "system_exceptions_total_1min"; - public static final String SOURCE_EXCEPTIONS_TOTAL_1min = "source_exceptions_total_1min"; - public static final String RECEIVED_TOTAL_1min = "received_total_1min"; - public static final String WRITTEN_TOTAL_1min = "written_total_1min"; + public static final String SYSTEM_EXCEPTIONS_TOTAL_1min = "system_exceptions_1min"; + public static final String SOURCE_EXCEPTIONS_TOTAL_1min = "source_exceptions_1min"; + public static final String RECEIVED_TOTAL_1min = "received_1min"; + public static final String WRITTEN_TOTAL_1min = "written_1min"; /** Declare Prometheus stats **/ diff --git a/pulsar-functions/instance/src/main/python/function_stats.py b/pulsar-functions/instance/src/main/python/function_stats.py index 8b54f75044de2..39246a83c4768 100644 --- a/pulsar-functions/instance/src/main/python/function_stats.py +++ b/pulsar-functions/instance/src/main/python/function_stats.py @@ -41,11 +41,11 @@ class Stats(object): LAST_INVOCATION = 'last_invocation' TOTAL_RECEIVED = 'received_total' - TOTAL_SUCCESSFULLY_PROCESSED_1min = 'processed_successfully_total_1min' - TOTAL_SYSTEM_EXCEPTIONS_1min = 'system_exceptions_total_1min' - TOTAL_USER_EXCEPTIONS_1min = 'user_exceptions_total_1min' + TOTAL_SUCCESSFULLY_PROCESSED_1min = 'processed_successfully_1min_total' + TOTAL_SYSTEM_EXCEPTIONS_1min = 'system_exceptions_1min_total' + TOTAL_USER_EXCEPTIONS_1min = 'user_exceptions_1min_total' PROCESS_LATENCY_MS_1min = 'process_latency_ms_1min' - TOTAL_RECEIVED_1min = 'received_total_1min' + TOTAL_RECEIVED_1min = 'received_1min_total' # Declare Prometheus stat_total_processed_successfully = Counter(PULSAR_FUNCTION_METRICS_PREFIX + TOTAL_SUCCESSFULLY_PROCESSED, diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml index c06e3e38686bc..91c48a8bd2b87 100644 --- a/pulsar-io/docs/pom.xml +++ b/pulsar-io/docs/pom.xml @@ -82,6 +82,16 @@ pulsar-io-debezium-postgres ${project.version} + + ${project.groupId} + pulsar-io-debezium-oracle + ${project.version} + + + ${project.groupId} + pulsar-io-debezium-mssql + ${project.version} + ${project.groupId} pulsar-io-dynamodb diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index b2b8db8fc48cc..560256ecfa14e 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -80,20 +80,11 @@ ${project.version} + - ${project.groupId} - kafka-connect-avro-converter-shaded - ${project.version} - - - io.confluent - * - - - org.apache.avro - * - - + io.confluent + kafka-connect-avro-converter + ${confluent.version} diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/AbstractKafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/AbstractKafkaConnectSource.java index 2364e1d62a6a7..4303251f9ae59 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/AbstractKafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/AbstractKafkaConnectSource.java @@ -20,6 +20,9 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import io.confluent.connect.avro.AvroConverter; +import io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient; +import io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; @@ -48,9 +51,6 @@ import org.apache.pulsar.io.core.Source; import org.apache.pulsar.io.core.SourceContext; import org.apache.pulsar.io.kafka.connect.schema.KafkaSchemaWrappedSchema; -import org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter; -import org.apache.pulsar.kafka.shade.io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient; -import org.apache.pulsar.kafka.shade.io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig; /** * A pulsar source that runs. diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java index d4f86924de595..c324ab3966808 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java @@ -20,6 +20,7 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import io.confluent.connect.avro.AvroData; import java.util.Base64; import java.util.Map; import java.util.Optional; @@ -34,7 +35,6 @@ import org.apache.pulsar.functions.api.KVRecord; import org.apache.pulsar.io.core.SourceContext; import org.apache.pulsar.io.kafka.connect.schema.KafkaSchemaWrappedSchema; -import org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroData; /** * A pulsar source that runs. diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java index e412d25127f21..6786ca2451455 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java @@ -36,10 +36,9 @@ @Slf4j public class KafkaSchemaWrappedSchema implements Schema, Serializable { - private SchemaInfo schemaInfo = null; + private final SchemaInfo schemaInfo; - public KafkaSchemaWrappedSchema(org.apache.pulsar.kafka.shade.avro.Schema schema, - Converter converter) { + public KafkaSchemaWrappedSchema(org.apache.avro.Schema schema, Converter converter) { Map props = new HashMap<>(); boolean isJsonConverter = converter instanceof JsonConverter; props.put(GenericAvroSchema.OFFSET_PROP, isJsonConverter ? "0" : "5"); diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java index 72d68610bdb82..90f92c2e278f0 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.ExecutionError; import com.google.common.util.concurrent.UncheckedExecutionException; +import io.confluent.connect.avro.AvroData; import java.nio.charset.StandardCharsets; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -33,7 +34,6 @@ import org.apache.kafka.connect.data.SchemaBuilder; import org.apache.pulsar.client.api.schema.KeyValueSchema; import org.apache.pulsar.common.schema.SchemaType; -import org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroData; @Slf4j public class PulsarSchemaToKafkaSchema { @@ -59,9 +59,8 @@ public class PulsarSchemaToKafkaSchema { } // Parse json to shaded schema - private static org.apache.pulsar.kafka.shade.avro.Schema parseAvroSchema(String schemaJson) { - final org.apache.pulsar.kafka.shade.avro.Schema.Parser parser = - new org.apache.pulsar.kafka.shade.avro.Schema.Parser(); + private static org.apache.avro.Schema parseAvroSchema(String schemaJson) { + final org.apache.avro.Schema.Parser parser = new org.apache.avro.Schema.Parser(); parser.setValidateDefaults(false); return parser.parse(schemaJson); } @@ -72,23 +71,22 @@ public static Schema getKafkaConnectSchema(org.apache.pulsar.client.api.Schema p return pulsarSchemaTypeToKafkaSchema.get(pulsarSchema.getSchemaInfo().getType()); } - try { - return schemaCache.get(pulsarSchema.getSchemaInfo().getSchema(), () -> { - if (pulsarSchema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) { - KeyValueSchema kvSchema = (KeyValueSchema) pulsarSchema; - return SchemaBuilder.map(getKafkaConnectSchema(kvSchema.getKeySchema()), - getKafkaConnectSchema(kvSchema.getValueSchema())) - .build(); - } - org.apache.pulsar.kafka.shade.avro.Schema avroSchema = - parseAvroSchema(new String(pulsarSchema.getSchemaInfo().getSchema(), - StandardCharsets.UTF_8)); - return avroData.toConnectSchema(avroSchema); - }); - } catch (ExecutionException | UncheckedExecutionException | ExecutionError ee) { - throw logAndThrowOnUnsupportedSchema(pulsarSchema, "Failed to convert to Kafka Schema.", ee); - } + try { + return schemaCache.get(pulsarSchema.getSchemaInfo().getSchema(), () -> { + if (pulsarSchema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) { + KeyValueSchema kvSchema = (KeyValueSchema) pulsarSchema; + return SchemaBuilder.map(getKafkaConnectSchema(kvSchema.getKeySchema()), + getKafkaConnectSchema(kvSchema.getValueSchema())) + .build(); + } + org.apache.avro.Schema avroSchema = parseAvroSchema( + new String(pulsarSchema.getSchemaInfo().getSchema(), StandardCharsets.UTF_8)); + return avroData.toConnectSchema(avroSchema); + }); + } catch (ExecutionException | UncheckedExecutionException | ExecutionError ee) { + throw logAndThrowOnUnsupportedSchema(pulsarSchema, "Failed to convert to Kafka Schema.", ee); } + } throw logAndThrowOnUnsupportedSchema(pulsarSchema, "Schema is required.", null); } diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml index 693930fa35599..96cb939c25a5e 100644 --- a/pulsar-io/pom.xml +++ b/pulsar-io/pom.xml @@ -76,6 +76,43 @@ + + pulsar-io-tests + + core + batch-discovery-triggerers + batch-data-generator + common + docs + aws + twitter + cassandra + aerospike + kafka + rabbitmq + kinesis + hdfs3 + jdbc + data-generator + elastic-search + kafka-connect-adaptor + kafka-connect-adaptor-nar + debezium + hdfs2 + canal + file + netty + hbase + mongo + flume + redis + solr + influxdb + dynamodb + nsq + + + core-modules diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index c0545b4b1b07f..44d5200cfe3e7 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -267,6 +267,20 @@ + + skipTestsForUnitGroupOther + + + + org.apache.maven.plugins + maven-surefire-plugin + + true + + + + + diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyPrometheusMetricsTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyPrometheusMetricsTest.java index 57af5e900d564..460abcdb85d97 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyPrometheusMetricsTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyPrometheusMetricsTest.java @@ -107,7 +107,7 @@ public void testMetrics() { Multimap metrics = parseMetrics(response); // Check that ProxyService metrics are present - List cm = (List) metrics.get("pulsar_proxy_binary_bytes"); + List cm = (List) metrics.get("pulsar_proxy_binary_bytes_total"); assertEquals(cm.size(), 1); assertEquals(cm.get(0).tags.get("cluster"), TEST_CLUSTER); @@ -117,7 +117,7 @@ public void testMetrics() { assertEquals(cm2.get(0).tags.get("label1"), "xyz"); // Check that PrometheusRawMetricsProvider metrics are present - List cm3 = (List) metrics.get("test_counter"); + List cm3 = (List) metrics.get("test_counter_total"); assertEquals(cm3.size(), 1); assertEquals(cm3.get(0).tags.get("cluster"), TEST_CLUSTER); } diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml index cddc8f4709669..ebb5fae9c42e6 100644 --- a/pulsar-sql/pom.xml +++ b/pulsar-sql/pom.xml @@ -31,13 +31,6 @@ pulsar-sql Pulsar SQL :: Parent - - presto-pulsar - presto-pulsar-plugin - java-version-trim-agent - presto-distribution - - 3.14.9 @@ -104,6 +97,31 @@ + + main + + + disableSqlMainProfile + + !true + + + + presto-pulsar + presto-pulsar-plugin + java-version-trim-agent + presto-distribution + + + + pulsar-sql-tests + + presto-pulsar + presto-pulsar-plugin + java-version-trim-agent + presto-distribution + + - - - org.apache.pulsar.tests - docker-images - 2.10.7-SNAPSHOT - - 4.0.0 - java-test-image - Apache Pulsar :: Tests :: Docker Images :: Java Test Image - pom - - - - docker - - target/pulsar-server-distribution-bin.tar.gz - ${env.UBUNTU_MIRROR} - - - - integrationTests - - - - - org.apache.pulsar.tests - java-test-functions - ${project.parent.version} - - - org.apache.pulsar - pulsar-server-distribution - ${project.parent.version} - bin - tar.gz - provided - - - * - * - - - - - - - - maven-dependency-plugin - - - copy-installed - package - - copy - - - - - org.apache.pulsar.tests - java-test-functions - ${project.parent.version} - jar - true - ${project.build.directory} - java-test-functions.jar - - - org.apache.pulsar - pulsar-server-distribution - ${project.parent.version} - bin - tar.gz - true - ${project.build.directory} - pulsar-server-distribution-bin.tar.gz - - - - - - - - maven-resources-plugin - - - copy-files - generate-resources - - copy-resources - - - ${project.build.directory} - true - - - ${pulsar.basedir}/docker/pulsar/scripts - scripts - false - - - ${project.basedir}/../latest-version-image/scripts - scripts - false - - - ${project.basedir}/../latest-version-image/ssl - ssl - false - - - ${project.basedir}/../latest-version-image/conf - conf - false - - - - - - - - io.fabric8 - docker-maven-plugin - - - default - package - - build - tag - - - - - ${docker.organization}/java-test-image:${docker.tag} - - - ${docker.organization}/${docker.image}:${project.version}-${git.commit.id.abbrev} - - ${project.basedir} - true - - - - - - - - - - - - diff --git a/tests/docker-images/latest-version-image/Dockerfile b/tests/docker-images/latest-version-image/Dockerfile index 028c9d9690fb6..43c7778fe2e44 100644 --- a/tests/docker-images/latest-version-image/Dockerfile +++ b/tests/docker-images/latest-version-image/Dockerfile @@ -59,10 +59,6 @@ FROM $PULSAR_IMAGE # However, any processes exec'ing into the containers will run as root, by default. USER root -# We need to define the user in order for supervisord to work correctly -# We don't need a user defined in the public docker image, though. -RUN adduser -u 10000 --gid 0 --disabled-login --disabled-password --gecos '' pulsar - RUN rm -rf /var/lib/apt/lists/* && apt update RUN apt-get clean && apt-get update && apt-get install -y supervisor vim procps curl diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml index dc841bfd60309..52ccc8a21e85a 100644 --- a/tests/docker-images/pom.xml +++ b/tests/docker-images/pom.xml @@ -49,7 +49,6 @@ java-test-functions latest-version-image - java-test-image diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java index 246b52b5dd9fc..d1f96ce8527a4 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java @@ -55,8 +55,7 @@ public void testDebeziumMySqlSourceJsonWithClientBuilder() throws Exception { @Test(groups = "source") public void testDebeziumMySqlSourceAvro() throws Exception { - testDebeziumMySqlConnect( - "org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter", false, false); + testDebeziumMySqlConnect("io.confluent.connect.avro.AvroConverter", false, false); } @Test(groups = "source")