Skip to content

Commit d614346

Browse files
vsochvsoch
committed
ci: test updated main build script to add macOS
Signed-off-by: vsoch <vsoch@users.noreply.github.com>
1 parent db430f2 commit d614346

2 files changed

Lines changed: 192 additions & 94 deletions

File tree

‎.github/scripts/setup.sh‎

Lines changed: 140 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -1,73 +1,150 @@
11
#!/bin/bash
22

3+
# Exit on error, unset variable, or pipe failure
34
set -euo pipefail
45

56
# This will be empty for nightly test, and we will clone master branch
67
FLUX_RELEASE_VERSION=${FLUX_RELEASE_VERSION:-0.68.0}
78
FLUX_VERSION=${FLUX_VERSION:-0.68.0}
89

9-
export DEBIAN_FRONTEND=noninteractive
10-
export TZ=UTC
11-
sudo ln -snf /usr/share/zoneinfo/$TZ /etc/localtime
12-
echo $TZ > /tmp/timezone
13-
sudo mv /tmp/timezone /etc/timezone
10+
if [[ "$(uname)" == "Darwin" ]]; then
11+
brew install \
12+
autoconf \
13+
automake \
14+
libtool \
15+
make \
16+
pkg-config \
17+
epoll-shim \
18+
libev \
19+
zeromq \
20+
jansson \
21+
lz4 \
22+
libarchive \
23+
hwloc \
24+
sqlite \
25+
lua \
26+
luarocks \
27+
python3 \
28+
cffi \
29+
libyaml \
30+
jq
1431

15-
# Prepare the version file
32+
# Possibility - install munge with MacPorts
33+
# sudo port install munge
34+
eval "$(/opt/homebrew/bin/brew shellenv)"
35+
36+
# Set the dynamic library path variable for macOS
37+
export DYLD_LIBRARY_PATH=/usr/local/lib
38+
PIP_INSTALL="python -m pip install"
39+
ln -s $(which glibtoolize) /usr/local/bin/libtoolize
40+
41+
CPPFLAGS="-I${HOMEBREW_PREFIX}/include/lua"
42+
CPPFLAGS="-I$(brew --prefix libev)/include ${CPPFLAGS}"
43+
CPPFLAGS="-I$(brew --prefix epoll-shim)/include/libepoll-shim ${CPPFLAGS}"
44+
LDFLAGS=-L${HOMEBREW_PREFIX}/lib
45+
PKG_CONFIG_PATH=$(pkg-config --variable pc_path pkg-config)
46+
PKG_CONFIG_PATH=$(brew --prefix libarchive)/lib/pkgconfig:${PKG_CONFIG_PATH}
47+
PATH=$(brew --prefix libtool)/libexec/gnubin:$PATH
48+
49+
# Linux block!
50+
else
51+
export DEBIAN_FRONTEND=noninteractive
52+
export TZ=UTC
53+
sudo ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ | sudo tee /etc/timezone
54+
55+
sudo apt-get update
56+
sudo apt-get install -y \
57+
libjson-glib-dev \
58+
wget \
59+
curl \
60+
automake \
61+
libsodium-dev \
62+
libzmq3-dev \
63+
libczmq-dev \
64+
libjansson-dev \
65+
libmunge-dev \
66+
libncursesw5-dev \
67+
lua5.3 \
68+
liblua5.3-dev \
69+
liblz4-dev \
70+
libsqlite3-dev \
71+
uuid-dev \
72+
libhwloc-dev \
73+
libmpich-dev \
74+
libs3-dev \
75+
libevent-dev \
76+
libarchive-dev \
77+
python3-sphinx \
78+
python3-cffi \
79+
libtool \
80+
git \
81+
build-essential \
82+
libpam-dev
83+
84+
sudo rm -rf /var/lib/apt/lists/*
85+
sudo ldconfig
86+
87+
# Set the dynamic library path variable for Linux. The conda path is for the container.
88+
export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:/opt/conda/envs/build/lib
89+
PIP_INSTALL="/opt/conda/envs/build/bin/python3 -m pip install"
90+
fi
91+
92+
# Here we can start common setup (we hope)
1693
echo "Flux Version for pypi is ${FLUX_VERSION}"
17-
sed -i "s/package_version = \"develop\"/package_version = \"$FLUX_VERSION\"/" setup.py
94+
if [[ "$(uname)" == "Darwin" ]]; then
95+
sed -i '' "s/package_version = \"develop\"/package_version = \"$FLUX_VERSION\"/" setup.py
96+
else
97+
sed -i "s/package_version = \"develop\"/package_version = \"$FLUX_VERSION\"/" setup.py
98+
fi
1899

100+
# Store current directory
19101
here=$(pwd)
20-
sudo apt-get update
21-
sudo apt-get install -y \
22-
libjson-glib-dev \
23-
wget \
24-
curl \
25-
automake \
26-
libsodium-dev \
27-
libzmq3-dev \
28-
libczmq-dev \
29-
libjansson-dev \
30-
libmunge-dev \
31-
libncursesw5-dev \
32-
lua5.3 \
33-
liblua5.3-dev \
34-
liblz4-dev \
35-
libsqlite3-dev \
36-
uuid-dev \
37-
libhwloc-dev \
38-
libmpich-dev \
39-
libs3-dev \
40-
libevent-dev \
41-
libarchive-dev \
42-
python3 \
43-
python3-dev \
44-
python3-pip \
45-
python3-sphinx \
46-
libtool \
47-
git \
48-
build-essential \
49-
libpam-dev
50-
51-
sudo ldconfig
52-
sudo rm -rf /var/lib/apt/lists/*
53-
54-
# noble and later requires --break-system-packages
55-
PIP_INSTALL="sudo /opt/conda/envs/build/bin/python3 -m pip install"
56-
${PIP_INSTALL} IPython || ${PIP_INSTALL} IPython --break-system-packages
57-
${PIP_INSTALL} -r .github/scripts/requirements-dev.txt || ${PIP_INSTALL} -r .github/scripts/requirements-dev.txt --break-system-packages
58-
export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:/opt/conda/envs/build/lib
59102

103+
# Install Python requirements using the OS-specific pip command - allow for ubuntu to be 24.04 or not.
104+
# I like how "--break-system-packages" is analogous with --i-acknowledge-i-am-a-terrible-person
105+
if [[ "$(uname)" == "Darwin" ]]; then
106+
${PIP_INSTALL} IPython
107+
${PIP_INSTALL} -r .github/scripts/requirements-dev.txt
108+
109+
# Build flux core first for the libmissing
110+
cd "${here}"
111+
echo "Flux Release Version is ${FLUX_RELEASE_VERSION}"
112+
wget https://github.com/flux-framework/flux-core/releases/download/v${FLUX_RELEASE_VERSION}/flux-core-${FLUX_RELEASE_VERSION}.tar.gz
113+
tar -xzvf flux-core-${FLUX_RELEASE_VERSION}.tar.gz
114+
sudo mv flux-core-${FLUX_RELEASE_VERSION} /code
115+
sudo chown -R $(id -u) /code
116+
cd /code
117+
chmod +x etc/gen-cmdhelp.py
118+
./autogen.sh || echo "No autogen here"
119+
CPPFLAGS="$CPPFLAGS" LDFLAGS=$LDFLAGS PKG_CONFIG_PATH=$PKG_CONFIG_PATH \
120+
./configure --with-external-libev --prefix=/usr/local
121+
else
122+
${PIP_INSTALL} IPython || ${PIP_INSTALL} IPython --break-system-packages
123+
${PIP_INSTALL} -r .github/scripts/requirements-dev.txt || ${PIP_INSTALL} -r .github/scripts/requirements-dev.txt --break-system-packages
124+
fi
125+
126+
127+
128+
# Flux Security --
60129
git clone https://github.com/flux-framework/flux-security ~/security
61130
cd ~/security
62131
./autogen.sh
63-
PYTHON=/opt/conda/envs/build/bin/python3 ./configure --prefix=/usr/local
64-
make
132+
133+
if [[ "$(uname)" == "Darwin" ]]; then
134+
CPPFLAGS="$CPPFLAGS" LDFLAGS=$LDFLAGS PKG_CONFIG_PATH=$PKG_CONFIG_PATH \
135+
./configure --with-external-libev --prefix=/usr/local
136+
else
137+
./configure --prefix=/usr/local
138+
fi
139+
make
65140
sudo make install
66-
sudo ldconfig
141+
if [[ "$(uname)" != "Darwin" ]]; then
142+
sudo ldconfig
143+
fi
67144

68-
# This is a build from a release
145+
# Flux Core --
146+
cd "${here}"
69147
echo "Flux Release Version is ${FLUX_RELEASE_VERSION}"
70-
echo "Flux version requested to build is ${FLUX_RELEASE_VERSION}"
71148
wget https://github.com/flux-framework/flux-core/releases/download/v${FLUX_RELEASE_VERSION}/flux-core-${FLUX_RELEASE_VERSION}.tar.gz
72149
tar -xzvf flux-core-${FLUX_RELEASE_VERSION}.tar.gz
73150
sudo mv flux-core-${FLUX_RELEASE_VERSION} /code
@@ -76,15 +153,20 @@ sudo chown -R $(id -u) /code
76153
cd /code
77154
chmod +x etc/gen-cmdhelp.py
78155

79-
# This is only needed for non-releases
80156
./autogen.sh || echo "No autogen here"
81-
PYTHON=/opt/conda/envs/build/bin/python3 ./configure --prefix=/usr/local
82157

83-
# We don't really care about the version here -just building python bindings
84-
make VERBOSE=1
158+
if [[ "$(uname)" == "Darwin" ]]; then
159+
CPPFLAGS="$CPPFLAGS" LDFLAGS=$LDFLAGS PKG_CONFIG_PATH=$PKG_CONFIG_PATH \
160+
./configure --with-external-libev --prefix=/usr/local
161+
else
162+
./configure --prefix=/usr/local
163+
fi
85164

86-
# This sometimes fails the first time but then works ok? Weird
165+
make VERBOSE=1
87166
sudo make install || true
88167
sudo make install
89-
sudo ldconfig
90-
cd ${here}
168+
if [[ "$(uname)" != "Darwin" ]]; then
169+
sudo ldconfig
170+
fi
171+
172+
cd "${here}"

‎.github/workflows/build-release.yaml‎

Lines changed: 52 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: build flux-python
1+
name: flux-python
22

33
on:
44
pull_request: {}
@@ -20,90 +20,106 @@ on:
2020
default: "https://github.com/flux-framework/flux-core"
2121

2222
jobs:
23-
build-manual:
24-
runs-on: ubuntu-latest
23+
build:
24+
name: ${{ matrix.os }} Python ${{ matrix.python }}
25+
runs-on: ${{ matrix.os }}
2526
strategy:
2627
fail-fast: false
2728
matrix:
29+
# The matrix now includes both os and python
30+
os: [ubuntu-latest, macos-latest]
2831
python: ["3.8", "3.9", "3.10", "3.11"]
29-
container:
30-
image: fluxrm/testenv:jammy
32+
3133
steps:
3234
- uses: actions/checkout@v4
33-
- name: Setup Python
35+
- name: Setup Python (Linux)
36+
if: runner.os == 'Linux'
3437
env:
3538
python_version: ${{ matrix.python }}
36-
# Installs to /opt/conda/envs/build/bin/python3
37-
run: /bin/bash ./docker/install-mamba.sh ${python_version}
39+
run: /bin/bash ./docker/install-mamba.sh ${{ matrix.python }}
40+
41+
# Use micromamba for MacOS and hope someone else figured this out :)
42+
- name: Setup Python (macOS)
43+
if: runner.os == 'macOS'
44+
uses: mamba-org/setup-micromamba@v1
45+
with:
46+
environment-name: build-env
47+
create-args: python=${{ matrix.python }}
3848

49+
# The login shell ensures the mamba env is active
3950
- name: Build Flux Core Branch
51+
shell: bash -l {0}
4052
env:
4153
FLUX_RELEASE_VERSION: ${{ inputs.release_version }}
4254
FLUX_VERSION: ${{ inputs.version }}
4355
run: /bin/bash .github/scripts/setup.sh
4456

4557
- name: Build Python Bindings
58+
shell: bash -l {0}
4659
env:
4760
FLUX_BRANCH: ${{ inputs.branch }}
4861
FLUX_REPO: ${{ inputs.repo }}
49-
shell: bash
5062
run: |
51-
echo "Flux Repo: ${FLUX_REPO}"
52-
echo "Flux Branch: ${FLUX_BRANCH}"
53-
if [[ "${FLUX_REPO}" == "" ]]; then
54-
FLUX_REPO="https://github.com/flux-framework/flux-core"
55-
fi
56-
if [[ "${FLUX_BRANCH}" == "" ]]; then
57-
FLUX_BRANCH="master"
63+
echo "Flux Repo: ${{ inputs.repo }}"
64+
echo "Flux Branch: ${{ inputs.branch }}"
65+
66+
# Use default values if inputs are empty
67+
FLUX_REPO_URL=${FLUX_REPO:-"https://github.com/flux-framework/flux-core"}
68+
FLUX_BRANCH_NAME=${FLUX_BRANCH:-"master"}
69+
70+
git clone -b ${FLUX_BRANCH_NAME} ${FLUX_REPO_URL} /tmp/flux-core
71+
mv /tmp/flux-core/src/bindings/python/flux ./flux
72+
python setup.py sdist
73+
74+
# I think ldconfig is Linux-specific
75+
if [[ "${{ runner.os }}" == "Linux" ]]; then
76+
sudo ldconfig
5877
fi
59-
git clone -b ${FLUX_BRANCH} ${FLUX_REPO} /tmp/flux-core
60-
mv /tmp/flux-core/src/bindings/python/flux ./flux
61-
/opt/conda/envs/build/bin/python3 setup.py sdist
62-
sudo ldconfig
6378
6479
- name: Build Python Wheels
80+
shell: bash -l {0}
6581
env:
6682
build_number: ${{ inputs.rc }}
6783
python_version: ${{ matrix.python }}
6884
run: |
69-
/bin/bash ./docker/build-wheels.sh ${build_number} ${python_version}
85+
/bin/bash ./docker/build-wheels.sh ${{ env.build_number }} ${{ env.python_version }}
7086
ls ./dist
7187
7288
- name: Upload distributions
73-
if: success()
7489
uses: actions/upload-artifact@v4
7590
with:
76-
name: ${{ matrix.python }}
91+
# The artifact name must be unique for each job in the matrix
92+
name: dist-${{ matrix.python }}-${{ matrix.os }}
7793
path: ./dist
7894

7995
upload:
8096
runs-on: ubuntu-latest
81-
needs: [build-manual]
97+
# This job now needs the entire build matrix to complete successfully
98+
needs: [build]
8299
steps:
83100
- name: Download Artifacts
84101
uses: actions/download-artifact@v4
85-
# If name unspecified, all artifacts download for run
102+
with:
103+
# Download all artifacts created by the build jobs
104+
path: artifacts
86105

87-
- name: Show Files
106+
- name: Show and Organize Files
88107
run: |
89-
ls */
90-
# We just need one of these
91-
first_dir=$(ls . | head -n 1)
108+
ls -R artifacts
92109
mkdir -p ./dist
93-
mv */*.whl ./dist/
94-
cp ${first_dir}/*.tar.gz ./dist/
110+
# This command finds all wheel and tar.gz files in all subdirectories
111+
# and moves them into a single ./dist directory for upload.
112+
find artifacts -type f \( -name "*.whl" -o -name "*.tar.gz" \) -exec mv {} ./dist/ \;
95113
echo
96114
echo "Files to distribute:"
97115
ls ./dist
98116
99117
- name: Build and publish
118+
# Ensure this only runs on pushes to a main branch, not PRs
100119
if: (github.event_name != 'pull_request')
101120
env:
102121
TWINE_USERNAME: ${{ secrets.PYPI_USER }}
103122
TWINE_PASSWORD: ${{ secrets.PYPI_PASS }}
104123
run: |
105-
ls dist/
106-
python3 -m pip install setuptools wheel
107-
python3 -m pip install twine==6.0.1
108-
python3 -m pip install --upgrade pkginfo
109-
twine upload --skip-existing dist/flux_python*
124+
python3 -m pip install --upgrade twine
125+
twine upload --skip-existing dist/*

0 commit comments

Comments
 (0)