Skip to content

Commit 5472dc7

Browse files
committed
[feat][ci] Refactor CI workflows
Signed-off-by: Zixuan Liu <nodeces@gmail.com>
1 parent 7316a6a commit 5472dc7

46 files changed

Lines changed: 2929 additions & 3535 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.
Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
name: clean disk
21+
description: makes some more space available on the disk by removing files
22+
inputs:
23+
mode:
24+
description: "Use 'full' to clean as much as possible"
25+
required: false
26+
runs:
27+
using: composite
28+
steps:
29+
- run: |
30+
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
31+
directories=(/usr/local/lib/android /opt/ghc)
32+
if [[ "${{ inputs.mode }}" == "full" ]]; then
33+
# remove these directories only when mode is 'full'
34+
directories+=(/usr/share/dotnet /opt/hostedtoolcache/CodeQL)
35+
fi
36+
emptydir=/tmp/empty$$/
37+
mkdir $emptydir
38+
echo "::group::Available diskspace"
39+
time df -BM / /mnt
40+
echo "::endgroup::"
41+
for directory in "${directories[@]}"; do
42+
echo "::group::Removing $directory"
43+
# fast way to delete a lot of files on linux
44+
time sudo eatmydata rsync -a --delete $emptydir ${directory}/
45+
time sudo eatmydata rm -rf ${directory}
46+
time df -BM / /mnt
47+
echo "::endgroup::"
48+
done
49+
if [[ "${{ inputs.mode }}" == "full" ]]; then
50+
echo "::group::Moving /var/lib/docker to /mnt/docker"
51+
sudo systemctl stop docker
52+
if sudo test ! -f /etc/docker/daemon.json; then
53+
# create daemon.json file
54+
echo '{"data-root": "/mnt/docker"}' | sudo tee /etc/docker/daemon.json
55+
else
56+
# modify existing daemon.json file
57+
sudo jq '.["data-root"]="/mnt/docker"' /etc/docker/daemon.json | sudo tee /tmp/daemon_temp.json$$
58+
sudo mv /tmp/daemon_temp.json$$ /etc/docker/daemon.json
59+
fi
60+
sudo mv /var/lib/docker /mnt/docker
61+
sudo systemctl start docker
62+
time df -BM / /mnt
63+
echo "::endgroup::"
64+
fi
65+
echo "::group::Cleaning apt state"
66+
time sudo bash -c "apt-get clean; apt-get autoclean; apt-get -y --purge autoremove"
67+
time df -BM / /mnt
68+
echo "::endgroup::"
69+
fi
70+
echo "::group::Available diskspace"
71+
time df -BM / /mnt
72+
echo "::endgroup::"
73+
shell: bash
Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
name: Copy test reports
21+
description: Aggregates all test reports to ./test-reports and ./surefire-reports directories
22+
runs:
23+
using: composite
24+
steps:
25+
- run: |
26+
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh move_test_reports
27+
shell: bash
Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
name: ssh access
21+
description: Sets up SSH access to build VM with upterm
22+
inputs:
23+
action:
24+
description: |
25+
Action to perform: options are "start" and "wait"
26+
"start" will install, configure and start upterm.
27+
"wait" will wait until a connection is established to upterm and will continue to wait until the session is closed.
28+
required: false
29+
default: 'start'
30+
limit-access-to-actor:
31+
description: 'If only the public SSH keys of the user triggering the workflow should be authorized'
32+
required: false
33+
default: 'false'
34+
limit-access-to-users:
35+
description: 'If only the public SSH keys of the listed GitHub users should be authorized. Comma separate list of GitHub user names.'
36+
required: false
37+
default: ''
38+
secure-access:
39+
description: |
40+
Set to false for allowing public access when limit-access-to-actor and limit-access-to-users are unset.
41+
required: false
42+
default: 'true'
43+
timeout:
44+
description: 'When action=wait, the timeout in seconds to wait for the user to connect'
45+
required: false
46+
default: '300'
47+
runs:
48+
using: composite
49+
steps:
50+
- run: |
51+
if [[ "${{ inputs.action }}" == "start" ]]; then
52+
echo "::group::Installing upterm & tmux"
53+
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
54+
# install upterm
55+
curl -sL https://github.com/owenthereal/upterm/releases/download/v0.7.6/upterm_linux_amd64.tar.gz | tar zxvf - -C /tmp upterm && sudo install /tmp/upterm /usr/local/bin/ && rm -rf /tmp/upterm
56+
57+
# install tmux if it's not present
58+
if ! command -v tmux &>/dev/null; then
59+
sudo apt-get -y install tmux
60+
fi
61+
elif [[ "$OSTYPE" == "darwin"* ]]; then
62+
brew install owenthereal/upterm/upterm
63+
# install tmux if it's not present
64+
if ! command -v tmux &>/dev/null; then
65+
brew install tmux
66+
fi
67+
else
68+
echo "Unsupported $OSTYPE"
69+
exit 0
70+
fi
71+
echo '::endgroup::'
72+
echo "::group::Configuring ssh and ssh keys"
73+
# generate ssh key
74+
mkdir -p ~/.ssh
75+
chmod 0700 ~/.ssh
76+
if [ ! -f ~/.ssh/id_rsa ]; then
77+
ssh-keygen -q -t rsa -N "" -f ~/.ssh/id_rsa
78+
fi
79+
if [ ! -f ~/.ssh/id_ed25519 ]; then
80+
ssh-keygen -q -t ed25519 -N "" -f ~/.ssh/id_ed25519
81+
fi
82+
# configure ssh
83+
echo -e "Host *\nStrictHostKeyChecking no\nCheckHostIP no\nTCPKeepAlive yes\nServerAliveInterval 30\nServerAliveCountMax 180\nVerifyHostKeyDNS yes\nUpdateHostKeys yes\n" > ~/.ssh/config
84+
# Auto-generate ~/.ssh/known_hosts by attempting connection to uptermd.upterm.dev
85+
ssh -i ~/.ssh/id_ed25519 uptermd.upterm.dev || true
86+
# @cert-authority entry is a mandatory entry when connecting to upterm. generate the entry based on the known_hosts entry key
87+
cat <(cat ~/.ssh/known_hosts | awk '{ print "@cert-authority * " $2 " " $3 }') >> ~/.ssh/known_hosts
88+
authorizedKeysParameter=""
89+
authorizedKeysFile=${HOME}/.ssh/authorized_keys
90+
if [[ "${{ inputs.secure-access }}" != "false" ]]; then
91+
ssh-keygen -q -t ed25519 -N "$(echo $RANDOM | md5sum | awk '{ print $1 }')" -C "Prevent public access" -f /tmp/dummykey$$
92+
cat /tmp/dummykey$$.pub >> $authorizedKeysFile
93+
rm /tmp/dummykey$$ /tmp/dummykey$$.pub
94+
fi
95+
limit_access_to_actor="${{ inputs.limit-access-to-actor }}"
96+
if [[ "${limit_access_to_actor}" == "true" ]]; then
97+
echo "Adding ${GITHUB_ACTOR} to allowed users (identified by ssh key registered in GitHub)"
98+
curl -s https://github.com/${GITHUB_ACTOR}.keys >> $authorizedKeysFile
99+
fi
100+
limit_access_to_users="${{ inputs.limit-access-to-users }}"
101+
for github_user in ${limit_access_to_users//,/ }; do
102+
if [[ -n "${github_user}" ]]; then
103+
echo "Adding ${github_user} to allowed users (identified by ssh key registered in GitHub)"
104+
curl -s https://github.com/${github_user}.keys >> $authorizedKeysFile
105+
fi
106+
done
107+
if [ -f $authorizedKeysFile ]; then
108+
chmod 0600 $authorizedKeysFile
109+
authorizedKeysParameter="-a $authorizedKeysFile"
110+
echo -e "Using $authorizedKeysFile\nContent:\n---------------------------"
111+
cat $authorizedKeysFile
112+
echo "---------------------------"
113+
fi
114+
echo '::endgroup::'
115+
echo "::group::Starting terminal session and connecting to server"
116+
tmux new -d -s upterm-wrapper -x 132 -y 43 "upterm host ${authorizedKeysParameter} --force-command 'tmux attach -t upterm' -- tmux new -s upterm -x 132 -y 43"
117+
sleep 2
118+
tmux send-keys -t upterm-wrapper q C-m
119+
sleep 1
120+
tmux set -t upterm-wrapper window-size largest
121+
tmux set -t upterm window-size largest
122+
echo '::endgroup::'
123+
echo -e "\nSSH connection information"
124+
# wait up to 10 seconds for upterm admin socket to appear
125+
for i in {1..10}; do
126+
ADMIN_SOCKET=$(find $HOME/.upterm -name "*.sock")
127+
if [ ! -S "$ADMIN_SOCKET" ]; then
128+
echo "Waiting for upterm admin socket to appear in ~/.upterm/*.sock ..."
129+
sleep 1
130+
else
131+
echo "upterm admin socket available in $ADMIN_SOCKET"
132+
break
133+
fi
134+
done
135+
shopt -s nullglob
136+
upterm session current --admin-socket ~/.upterm/*.sock || {
137+
echo "Starting upterm failed."
138+
exit 0
139+
}
140+
elif [[ "${{ inputs.action }}" == "wait" ]]; then
141+
# only wait if upterm was installed
142+
if command -v upterm &>/dev/null; then
143+
shopt -s nullglob
144+
echo "SSH connection information"
145+
upterm session current --admin-socket ~/.upterm/*.sock || {
146+
echo "upterm isn't running. Not waiting any longer."
147+
exit 0
148+
}
149+
timeout=${{ inputs.timeout }}
150+
echo "Waiting $timeout seconds..."
151+
sleep $timeout
152+
echo "Keep waiting as long as there's a connected session"
153+
while upterm session current --admin-socket ~/.upterm/*.sock|grep Connected &>/dev/null; do
154+
sleep 30
155+
done
156+
echo "No session is connected. Not waiting any longer."
157+
else
158+
echo "upterm isn't installed"
159+
fi
160+
fi
161+
shell: bash

.github/actions/tune-runner-vm/action.yml

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ runs:
2424
steps:
2525
- run: |
2626
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
27+
echo "::group::Configure and tune OS"
2728
# Ensure that reverse lookups for current hostname are handled properly
2829
# Add the current IP address, long hostname and short hostname record to /etc/hosts file
2930
echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
@@ -32,18 +33,23 @@ runs:
3233
# consumption is high.
3334
# Set vm.swappiness=1 to avoid swapping and allow high RAM usage
3435
echo 1 | sudo tee /proc/sys/vm/swappiness
35-
# Set swappiness to 1 for all cgroups and sub-groups
36-
for swappiness_dir in /sys/fs/cgroup/memory/*/ /sys/fs/cgroup/memory/*/*/; do
37-
if [ -d "swappiness_dir" ]; then
38-
echo 1 | sudo tee $(swappiness_dir)memory.swappiness > /dev/null
39-
fi
40-
done
36+
(
37+
shopt -s nullglob
38+
# Set swappiness to 1 for all cgroups and sub-groups
39+
for swappiness_file in /sys/fs/cgroup/memory/*/memory.swappiness /sys/fs/cgroup/memory/*/*/memory.swappiness; do
40+
echo 1 | sudo tee $swappiness_file > /dev/null
41+
done
42+
) || true
4143
4244
# use "madvise" Linux Transparent HugePages (THP) setting
4345
# https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html
4446
# "madvise" is generally a better option than the default "always" setting
47+
# Based on Azul instructions from https://docs.azul.com/prime/Enable-Huge-Pages#transparent-huge-pages-thp
4548
echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
46-
49+
echo advise | sudo tee /sys/kernel/mm/transparent_hugepage/shmem_enabled
50+
echo defer+madvise | sudo tee /sys/kernel/mm/transparent_hugepage/defrag
51+
echo 1 | sudo tee /sys/kernel/mm/transparent_hugepage/khugepaged/defrag
52+
4753
# tune filesystem mount options, https://www.kernel.org/doc/Documentation/filesystems/ext4.txt
4854
# commit=999999, effectively disables automatic syncing to disk (default is every 5 seconds)
4955
# nobarrier/barrier=0, loosen data consistency on system crash (no negative impact to empheral CI nodes)
@@ -80,11 +86,17 @@ runs:
8086
echo '::endgroup::'
8187
8288
# show memory
89+
echo "::group::Available Memory"
8390
free -m
91+
echo '::endgroup::'
8492
# show disk
85-
df -h
93+
echo "::group::Available diskspace"
94+
df -BM
95+
echo "::endgroup::"
8696
# show cggroup
87-
echo "/actions_job cgroup settings:"
88-
sudo cgget actions_job
97+
echo "::group::Cgroup settings for current cgroup $CURRENT_CGGROUP"
98+
CURRENT_CGGROUP=$(cat /proc/self/cgroup | grep '0::' | awk -F: '{ print $3 }')
99+
sudo cgget -a $CURRENT_CGGROUP || true
100+
echo '::endgroup::'
89101
fi
90102
shell: bash

0 commit comments

Comments
 (0)