Skip to content

Commit 00345b6

Browse files
vsochvsoch
committed
refactor: test pixi
It would be nice to have a consistent interface for Python. Signed-off-by: vsoch <vsoch@users.noreply.github.com>
1 parent 9c36063 commit 00345b6

7 files changed

Lines changed: 161 additions & 128 deletions

File tree

.github/workflows/test-build.yaml renamed to .github/deprecated/test-build.yaml

File renamed without changes.

‎.github/scripts/build-bindings.sh‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
3+
here=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
4+
FLUX_REPO=${1:-0}
5+
FLUX_BRANCH=${2:-3.11}
6+
7+
echo "Flux Repo: ${FLUX_REPO}"
8+
echo "Flux Branch: ${FLUX_BRANCH}"
9+
10+
# Use default values if inputs are empty
11+
FLUX_REPO_URL=${FLUX_REPO:-"https://github.com/flux-framework/flux-core"}
12+
FLUX_BRANCH_NAME=${FLUX_BRANCH:-"master"}
13+
14+
git clone -b ${FLUX_BRANCH_NAME} ${FLUX_REPO_URL} /tmp/flux-core
15+
mv /tmp/flux-core/src/bindings/python/flux ./flux
16+
python setup.py sdist
17+
18+
# I think ldconfig is Linux-specific
19+
sudo ldconfig || echo "Not running on Linux"

‎.github/scripts/setup.sh‎

Lines changed: 68 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -3,53 +3,14 @@
33
# Exit on error, unset variable, or pipe failure
44
set -euo pipefail
55

6-
# This will be empty for nightly test, and we will clone master branch
7-
FLUX_RELEASE_VERSION=${FLUX_RELEASE_VERSION:-0.78.0}
8-
FLUX_VERSION=${FLUX_VERSION:-0.78.0}
6+
# This must be set
7+
PYTHON_VERSION=${1:-py311}
98

10-
if [[ "$(uname)" == "Darwin" ]]; then
11-
brew install \
12-
autoconf \
13-
automake \
14-
libtool \
15-
make \
16-
pkg-config \
17-
epoll-shim \
18-
libev \
19-
zeromq \
20-
jansson \
21-
lz4 \
22-
libarchive \
23-
hwloc \
24-
sqlite \
25-
lua \
26-
luarocks \
27-
cffi \
28-
libyaml \
29-
jq
30-
31-
# Possibility - install munge with MacPorts
32-
# sudo port install munge
33-
eval "$(/opt/homebrew/bin/brew shellenv)"
34-
35-
# Set the dynamic library path variable for macOS
36-
export DYLD_LIBRARY_PATH=/usr/local/lib
37-
38-
# Ensure we activate micromamabe that has a pinned version
39-
PIP_INSTALL="/opt/conda/bin/python3 -m pip install"
40-
ln -s $(which glibtoolize) /usr/local/bin/libtoolize
41-
42-
CPPFLAGS="-I${HOMEBREW_PREFIX}/include/lua"
43-
CPPFLAGS="-I$(brew --prefix libev)/include ${CPPFLAGS}"
44-
CPPFLAGS="-I$(brew --prefix epoll-shim)/include/libepoll-shim ${CPPFLAGS}"
45-
# LDFLAGS=-L${HOMEBREW_PREFIX}/lib
46-
PKG_CONFIG_PATH=$(pkg-config --variable pc_path pkg-config)
47-
PKG_CONFIG_PATH=$(brew --prefix libarchive)/lib/pkgconfig:${PKG_CONFIG_PATH}
48-
PATH=$(brew --prefix libtool)/libexec/gnubin:$PATH
49-
export LD_LIBRARY_PATH="/opt/conda/envs/build/lib:/opt/conda/envs/build/lib64"
50-
51-
# Linux block!
52-
else
9+
# These will be empty for nightly test, and we will clone master branch
10+
FLUX_RELEASE_VERSION=${2:-0.78.0}
11+
FLUX_VERSION=${3:-0.78.0}
12+
13+
if [[ "$(uname)" != "Darwin" ]]; then
5314
export DEBIAN_FRONTEND=noninteractive
5415
export TZ=UTC
5516
sudo ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ | sudo tee /etc/timezone
@@ -76,8 +37,6 @@ else
7637
libs3-dev \
7738
libevent-dev \
7839
libarchive-dev \
79-
python3-sphinx \
80-
python3-cffi \
8140
libtool \
8241
git \
8342
build-essential \
@@ -86,53 +45,82 @@ else
8645
sudo rm -rf /var/lib/apt/lists/*
8746
sudo ldconfig
8847

89-
# Set the dynamic library path variable for Linux. The conda path is for the container.
90-
export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:/opt/conda/envs/build/lib
48+
# Set the dynamic library path variable for Linux.
49+
export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib
50+
echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"
9151
fi
9252

93-
# Ensure we find micromamba environment first
94-
# mamba init commands are intolerable so let's do basics ourselves.
95-
PIP_INSTALL="/opt/conda/envs/build/bin/python3 -m pip install"
96-
export PATH="/opt/conda/envs/build/bin:$PATH"
97-
export PYTHONPATH=$(find /opt/conda/envs/build -name site-packages)
98-
export PYTHON=/opt/conda/envs/build/bin/python3
99-
export PYTHON_PREFIX=${PYTHONPATH}
100-
export PYTHON_EXEC_PREFIX=${PYTHONPATH}
101-
# export LIBS="-L/opt/conda/envs/build/lib:/opt/conda/envs/build/lib64"
102-
10353
echo "Found Python $(which python3)"
104-
echo "PYTHONPATH: ${PATH}"
10554
echo "PATH: ${PATH}"
106-
echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"
55+
56+
MUNGE_VERSION="0.5.16"
57+
curl -L "https://github.com/dun/munge/releases/download/munge-${MUNGE_VERSION}/munge-${MUNGE_VERSION}.tar.xz" -o munge.tar.xz
58+
tar -xf munge.tar.xz
59+
cd "munge-${MUNGE_VERSION}"
60+
./configure --prefix=/usr
61+
make
62+
make install
63+
cd ..
10764

10865
# Here we can start common setup (we hope)
10966
echo "Flux Version for pypi is ${FLUX_VERSION}"
11067
if [[ "$(uname)" == "Darwin" ]]; then
68+
PIXI_ROOT=/Users/runner/work/flux-python/flux-python/.pixi
69+
PIXI_ENV=${PIXI_ROOT}/envs/${PYTHON_VERSION}
11170
sed -i '' "s/package_version = \"develop\"/package_version = \"$FLUX_VERSION\"/" setup.py
71+
echo $(find /Users/runner/ -name python3) || echo $(find /Users/runner/ -name python)
72+
73+
# Tell where to look for lua.h
74+
CPPFLAGS="-I${PIXI_ENV}/include"
75+
76+
# This makes me want to kill homebrew and MacOS with fire.
77+
# I know most of these aren't required. I can't remove them because of the ❤️‍🔥
78+
export PYTHON_SITE_PACKAGES=$(find ${PIXI_ENV} -name site-packages)
79+
export PYTHON_EXEC_PREFIX=${PYTHON_SITE_PACKAGES}
80+
export PYTHON_SITE_PKG=${PYTHON_SITE_PACKAGES}
81+
export PYTHON_PLATFORM_SITE_PKG=${PYTHON_SITE_PACKAGES}
82+
export PYTHON_SITELIB=${PYTHON_SITE_PACKAGES}
83+
export PYTHON_PREFIX=${PYTHON_SITE_PACKAGES}
84+
export PKG_CONFIG_PATH=${PIXI_ENV}/lib/pkgconfig
85+
export PYTHONDIR=${PYTHON_SITE_PACKAGES}
86+
87+
# Set these to empty.
88+
# PYTHON=${PIXI_ENV}/bin/python3
89+
# CORRECT_EXTRA_LIBS="$(${PYTHON} -c "import sysconfig; print(sysconfig.get_config_var(\"LIBS\"))")"
90+
# CORRECT_EXTRA_LDFLAGS="$(${PYTHON} -c "import sysconfig; print(sysconfig.get_config_var(\"LINKFORSHARED\"))")"
91+
export PYTHON_EXTRA_LDFLAGS=" "
92+
export PYTHON_EXTRA_LIBS=" "
93+
94+
echo "PKG_CONFIG_PATH: ${PKG_CONFIG_PATH}"
95+
echo "PYTHON_PREFIX: ${PYTHON_PREFIX}"
96+
echo "PYTHON_SITE_PACKAGES: ${PYTHON_SITE_PACKAGES}"
97+
echo "PYTHON_EXEC_PREFIX: ${PYTHON_EXEC_PREFIX}"
11298
else
99+
PIXI_ROOT=/home/runner/work/flux-python/flux-python/.pixi
100+
export PIXI_ENV=${PIXI_ROOT}/envs/${PYTHON_VERSION}
113101
sed -i "s/package_version = \"develop\"/package_version = \"$FLUX_VERSION\"/" setup.py
102+
CPPFLAGS="-I${PIXI_ENV}/include"
114103
fi
115104

105+
# Can we remove need for setuptools?
106+
export PYTHON=${PIXI_ENV}/bin/python3
107+
export PYTHON_EXECUTABLE=${PIXI_ENV}/bin/python3
108+
export PYTHON_NOVERSIONCHECK=yes
109+
export LUA_CFLAGS="-I${PIXI_ENV}/include"
110+
export LDFLAGS="-L${PIXI_ENV}/lib -L${PIXI_ENV}/lib64"
111+
echo "Found lua.h in:\n$(find ${PIXI_ROOT} -name lua.h)" || echo "Did not find lua.h"
112+
116113
# Store current directory
117114
here=$(pwd)
115+
luarocks install luaposix
118116

119-
# Install Python requirements using the OS-specific pip command - allow for ubuntu to be 24.04 or not.
120-
# I like how "--break-system-packages" is analogous with --i-acknowledge-i-am-a-terrible-person
121-
if [[ "$(uname)" == "Darwin" ]]; then
122-
echo "Installing python requirements for MacOS"
123-
${PIP_INSTALL} IPython setuptools ply sphinx cffi pyyaml
124-
${PIP_INSTALL} -r .github/scripts/requirements-dev.txt
125-
else
126-
echo "Installing python requirements for Linux"
127-
${PIP_INSTALL} IPython pyyaml || ${PIP_INSTALL} IPython --break-system-packages pyyaml
128-
${PIP_INSTALL} -r .github/scripts/requirements-dev.txt || ${PIP_INSTALL} -r .github/scripts/requirements-dev.txt --break-system-packages
129-
130-
# Flux Security --
131-
# Does not have a variant for Mac
117+
# Flux Security --
118+
# Does not have a variant for Mac
119+
if [[ "$(uname)" != "Darwin" ]]; then
132120
git clone https://github.com/flux-framework/flux-security ~/security
133121
cd ~/security
134122
./autogen.sh
135-
PYTHON_PREFIX=PYTHON_EXEC_PREFIX=${PYTHON_PREFIX} PYTHON=/opt/conda/bin/python3 ./configure --prefix=/usr/local
123+
./configure --prefix=/usr/local
136124
make
137125
sudo make install
138126
sudo ldconfig
@@ -149,11 +137,12 @@ chmod +x etc/gen-cmdhelp.py
149137
./autogen.sh || echo "No autogen here"
150138

151139
if [[ "$(uname)" == "Darwin" ]]; then
152-
CPPFLAGS="$CPPFLAGS" PKG_CONFIG_PATH=$PKG_CONFIG_PATH ./configure --prefix=/usr/local
140+
CPPFLAGS="$CPPFLAGS" CFLAGS="${CPPFLAGS}" LDFLAGS="${LDFLAGS}" PKG_CONFIG_PATH=$PKG_CONFIG_PATH ./configure --prefix=/usr/local || (cp ./config.log ../ && exit 1)
153141
ls ./scripts
154-
./scripts/check-macos.sh || echo "No check-macos.sh"
142+
./scripts/check-macos.sh || (cp ./config.log ../ && exit 1)
155143
else
156-
./configure --prefix=/usr/local
144+
./configure CPPFLAGS="$CPPFLAGS" CFLAGS="${CPPFLAGS}" LDFLAGS="${LDFLAGS}" --prefix=/usr/local
145+
cp ./config.log ../
157146
make VERBOSE=1
158147
sudo make install || true
159148
sudo make install

‎.github/workflows/build-release.yaml‎

Lines changed: 14 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -28,52 +28,29 @@ jobs:
2828
matrix:
2929
# The matrix now includes both os and python
3030
os: [ubuntu-latest, macos-latest]
31-
python: ["3.8", "3.9", "3.10", "3.11"]
31+
python: ["py38", "py39", "py310", "py311", "py312"]
3232

3333
steps:
3434
- uses: actions/checkout@v4
35-
- name: Setup Python
36-
env:
37-
python_version: ${{ matrix.python }}
38-
run: /bin/bash ./docker/install-mamba.sh ${{ matrix.python }}
35+
- uses: prefix-dev/setup-pixi@v0.9.0
36+
with:
37+
environments: ${{ matrix.python }}
3938

40-
# The login shell ensures the mamba env is active
4139
- name: Build Flux Core Branch
42-
env:
43-
FLUX_RELEASE_VERSION: ${{ inputs.release_version }}
44-
FLUX_VERSION: ${{ inputs.version }}
45-
run: /bin/bash .github/scripts/setup.sh
46-
47-
- name: Build Python Bindings
48-
shell: bash -l {0}
49-
env:
50-
FLUX_BRANCH: ${{ inputs.branch }}
51-
FLUX_REPO: ${{ inputs.repo }}
52-
run: |
53-
echo "Flux Repo: ${{ inputs.repo }}"
54-
echo "Flux Branch: ${{ inputs.branch }}"
55-
56-
# Use default values if inputs are empty
57-
FLUX_REPO_URL=${FLUX_REPO:-"https://github.com/flux-framework/flux-core"}
58-
FLUX_BRANCH_NAME=${FLUX_BRANCH:-"master"}
40+
run: pixi run -e ${{ matrix.python }} bash .github/scripts/setup.sh ${{ matrix.python }} ${{ inputs.release_version }} ${{ inputs.version }}
5941

60-
git clone -b ${FLUX_BRANCH_NAME} ${FLUX_REPO_URL} /tmp/flux-core
61-
mv /tmp/flux-core/src/bindings/python/flux ./flux
62-
python setup.py sdist
42+
- name: Upload logs on failure
43+
if: ${{ always() }}
44+
uses: actions/upload-artifact@v4
45+
with:
46+
name: config-log-${{ matrix.python }}-${{ matrix.os }}
47+
path: ./config.log
6348

64-
# I think ldconfig is Linux-specific
65-
if [[ "${{ runner.os }}" == "Linux" ]]; then
66-
sudo ldconfig
67-
fi
49+
- name: Build Python Bindings
50+
run: pixi run -e ${{ matrix.python }} bash .github/scripts/build-bindings.sh ${{ inputs.branch }} ${{ inputs.repo }}
6851

6952
- name: Build Python Wheels
70-
shell: bash -l {0}
71-
env:
72-
build_number: ${{ inputs.rc }}
73-
python_version: ${{ matrix.python }}
74-
run: |
75-
/bin/bash ./docker/build-wheels.sh ${{ env.build_number }} ${{ env.python_version }}
76-
ls ./dist
53+
run: pixi run -e ${{ matrix.python }} bash ./docker/build-wheels.sh ${{ env.build_number }} ${{ env.python_version }}
7754

7855
- name: Upload distributions
7956
uses: actions/upload-artifact@v4

‎README.md‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -121,12 +121,9 @@ $ python3 setup.py sdist
121121
$ python3 setup.py sdist bdist_wheel
122122
```
123123

124-
You can build versions of the Python wheels across 3.6 to 3.10 like:
124+
Given an environment with dependencies, you can build versions of the Python wheels across 3.6 to 3.12 like:
125125

126126
```bash
127-
# only need to run this once in the container
128-
/bin/bash ./docker/install-mamba.sh
129-
130127
# And this to install the current Flux + version in setup.py as a wheel
131128
# The number is the build number
132129
/bin/bash ./docker/build-wheels.sh 2

‎docker/build-wheels.sh‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,7 @@ version=${2:-3.11}
77
# This is intended to run in the container
88
echo "Building Python version ${version}"
99

10-
export PATH=/opt/conda/envs/build/bin:$PATH
11-
export PYTHONPATH=/opt/conda/envs/build/lib/python${version}/site-packages
12-
13-
/opt/conda/bin/mamba activate build || true
14-
1510
# Build the bindings for this python version!
16-
/opt/conda/envs/build/bin/python3 setup.py sdist
17-
/opt/conda/envs/build/bin/python3 setup.py bdist_wheel --plat-name=any --build-number=${build_number}
18-
unset PYTHONPATH
11+
python3 setup.py sdist
12+
python3 setup.py bdist_wheel --plat-name=any --build-number=${build_number}
13+
ls ./dist

‎pixi.toml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
[workspace]
2+
name = "flux-python"
3+
channels = ["conda-forge"]
4+
platforms = ["linux-64", "osx-arm64"]
5+
6+
[dependencies]
7+
python = ">=3.8"
8+
pip = "*"
9+
10+
[feature.py38.dependencies]
11+
python = "3.8.*"
12+
[feature.py39.dependencies]
13+
python = "3.9.*"
14+
[feature.py310.dependencies]
15+
python = "3.10.*"
16+
[feature.py311.dependencies]
17+
python = "3.11.*"
18+
[feature.py312.dependencies]
19+
python = "3.12.*"
20+
21+
[feature.common.dependencies]
22+
cffi = "*"
23+
pyyaml = "*"
24+
jsonschema = "*"
25+
docutils = "*"
26+
ply = "*"
27+
black = "*"
28+
ipython = "*"
29+
wheel = "*"
30+
setuptools = "*"
31+
luarocks = "*"
32+
lua = "*"
33+
lua-lpeg = "*"
34+
libev = "*"
35+
sphinx = "*"
36+
libarchive = "*"
37+
autoconf = "*"
38+
automake = "*"
39+
libtool = "*"
40+
make = "*"
41+
pkg-config = "*"
42+
zeromq = "*"
43+
jansson = "*"
44+
libhwloc = "*"
45+
munge = "*"
46+
47+
[pypi-dependencies]
48+
setuptools = "*"
49+
50+
[environments]
51+
py38 = { features = ["py38", "common"] }
52+
py39 = { features = ["py39", "common"] }
53+
py310 = { features = ["py310", "common"] }
54+
py311 = { features = ["py311", "common"] }
55+
py312 = { features = ["py312", "common"] }
56+

0 commit comments

Comments
 (0)