Skip to content

Commit 934d8bb

Browse files
nchaimovnchaimov
committed
Continuous integration with GitHub Actions
1 parent 040be0f commit 934d8bb

File tree

13 files changed

+410
-0
lines changed

13 files changed

+410
-0
lines changed

‎.github/workflows/CI.yml‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
name: CI
2+
3+
on: [push, pull_request, workflow_dispatch]
4+
5+
jobs:
6+
spindle-serial-ubuntu:
7+
name: Testsuite (Serial, Ubuntu)
8+
environment: Spindle CI
9+
runs-on: ubuntu-latest
10+
timeout-minutes: 20
11+
steps:
12+
- name: Check out Spindle
13+
uses: actions/checkout@v5
14+
with:
15+
path: 'Spindle'
16+
17+
- name: Setup Docker Compose
18+
uses: docker/setup-compose-action@v1
19+
with:
20+
version: latest
21+
22+
- name: Build spindle-serial-ubuntu image
23+
id: serial-ubuntu-build
24+
run: |
25+
cd Spindle/containers/spindle-serial-ubuntu
26+
docker compose --progress=plain build
27+
28+
- name: Bring spindle-serial-ubuntu up
29+
id: serial-ubuntu-up
30+
run: |
31+
cd Spindle/containers/spindle-serial-ubuntu
32+
docker compose up -d
33+
34+
- name: Verify munge works in spindle-serial-ubuntu
35+
id: serial-ubuntu-munge
36+
run: |
37+
docker exec spindlenode bash -c 'munge -n | unmunge'
38+
39+
- name: Run spindle-serial-ubuntu testsuite
40+
id: serial-ubuntu-testsuite
41+
run: |
42+
docker exec spindlenode bash -c 'cd Spindle-build/testsuite && ./runTests'
43+
44+
- name: Bring spindle-serial-ubuntu down
45+
id: serial-ubuntu-down
46+
if: ${{ always() }}
47+
continue-on-error: true
48+
run: |
49+
cd Spindle/containers/spindle-serial-ubuntu
50+
docker compose down
51+
52+
spindle-flux-ubuntu:
53+
name: Testsuite (Flux, Ubuntu)
54+
environment: Spindle CI
55+
runs-on: ubuntu-latest
56+
timeout-minutes: 20
57+
steps:
58+
- name: Check out Spindle
59+
uses: actions/checkout@v5
60+
with:
61+
path: 'Spindle'
62+
63+
- name: Setup Docker Compose
64+
uses: docker/setup-compose-action@v1
65+
with:
66+
version: latest
67+
68+
- name: Build spindle-flux-ubuntu image
69+
id: flux-ubuntu-build
70+
run: |
71+
cd Spindle/containers/spindle-flux-ubuntu
72+
docker compose --progress=plain build
73+
74+
- name: Bring spindle-flux-ubuntu up
75+
id: flux-ubuntu-up
76+
run: |
77+
cd Spindle/containers/spindle-flux-ubuntu
78+
docker compose up -d --wait --wait-timeout 60
79+
80+
- name: Verify munge works in spindle-flux-ubuntu
81+
id: flux-ubuntu-munge
82+
run: |
83+
docker exec node-1 bash -c 'munge -n | unmunge'
84+
85+
- name: Run spindle-flux-ubuntu testsuite
86+
id: flux-ubuntu-testsuite
87+
run: |
88+
docker exec node-1 bash -c 'cd Spindle-build/testsuite && flux alloc --nodes=${workers} ./runTests --nodes=${workers} --tasks-per-node=3'
89+
90+
- name: Bring spindle-flux-ubuntu down
91+
id: flux-ubuntu-down
92+
if: ${{ always() }}
93+
continue-on-error: true
94+
run: |
95+
cd Spindle/containers/spindle-flux-ubuntu
96+
docker compose down
97+

‎containers/spindle-flux-ubuntu/Dockerfile‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
# This is based on the Flux Container Tutorial
2+
# See https://flux-framework.readthedocs.io/en/latest/tutorials/containers
3+
ARG flux_sched_version=noble
4+
FROM fluxrm/flux-sched:${flux_sched_version} AS builder
5+
ARG replicas=4
6+
ENV workers=${replicas}
7+
USER root
8+
9+
RUN DEBIAN_FRONTEND="noninteractive" apt-get update \
10+
&& apt-get -qq install -y --no-install-recommends \
11+
autotools-dev \
12+
autoconf \
13+
automake \
14+
cmake \
15+
git \
16+
python3 \
17+
openssh-server \
18+
openssh-client \
19+
libdb-dev \
20+
apt-utils \
21+
dnsutils \
22+
iputils-ping \
23+
python3-pip \
24+
libgcrypt20 \
25+
libgcrypt20-dev \
26+
gdb \
27+
software-properties-common
28+
29+
ARG USER=fluxuser
30+
ARG CONFIG_ROOT=containers/spindle-flux-ubuntu
31+
32+
# Allow fluxuser to run as other users so it can start munged
33+
RUN sh -c "printf \"${USER} ALL=(ALL) NOPASSWD: ALL\\n\" >> /etc/sudoers"
34+
35+
# Configure flux
36+
ENV STATE_DIR=/var/lib/flux
37+
RUN mkdir -p ${STATE_DIR} /etc/flux/system /etc/flux/system/cron.d /etc/flux/config /run/flux /etc/flux/imp/conf.d
38+
COPY ${CONFIG_ROOT}/flux/imp.toml /etc/flux/imp/conf.d/
39+
COPY ${CONFIG_ROOT}/flux/broker.toml /etc/flux/config/
40+
RUN mkdir -p /etc/flux/system/cron.d && \
41+
mkdir -p /mnt/curve && \
42+
flux keygen /mnt/curve/curve.cert && \
43+
flux R encode --hosts="node-[1-${workers}]" > /etc/flux/system/R && \
44+
chown -R ${USER}:${USER} /run/flux ${STATE_DIR} /mnt/curve/curve.cert
45+
46+
# Build Spindle
47+
WORKDIR /home/${USER}
48+
# Copy the whole git repo into the container.
49+
COPY . /home/${USER}/Spindle
50+
COPY ${CONFIG_ROOT}/scripts/build_spindle.sh /home/${USER}/build_spindle.sh
51+
RUN ./build_spindle.sh
52+
53+
RUN chown -R ${USER}:${USER} /home/fluxuser && \
54+
chown -R ${USER}:${USER} /run/flux
55+
56+
USER ${USER}
57+
COPY ${CONFIG_ROOT}/scripts/flux_healthcheck.sh ./
58+
COPY ${CONFIG_ROOT}/scripts/entrypoint.sh ./
59+
ENV PATH /home/${USER}/Spindle-inst/bin:${PATH}
60+
# Make libfabric work with fork.
61+
ENV RDMAV_FORK_SAFE 1
62+
# Silence warning from hwloc about unsupported PCI device
63+
# on GitHub-hosted runners.
64+
ENV HWLOC_HIDE_ERRORS 2
65+
66+
ENTRYPOINT /bin/bash ./entrypoint.sh
67+

‎containers/spindle-flux-ubuntu/docker-compose.yml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# 4-node Flux cluster
2+
# For information on running Flux in containers, see
3+
# https://flux-framework.readthedocs.io/en/latest/tutorials/containers
4+
5+
# `replicas` must match the number of nodes defined in the services section
6+
x-shared-workers:
7+
&workers
8+
replicas: 4
9+
10+
# Ubuntu version to use (noble = 24.04)
11+
x-shared-build-args: &shared-build-args
12+
flux_sched_version: noble
13+
<<: *workers
14+
15+
# Docker prohibits copying files from outside of the build context.
16+
# In order to be able to copy the whole repo into the container,
17+
# we have to set the context to be the root of the repo.
18+
# We then have to specify the path from there to the Dockerfile.
19+
x-shared-build-context: &shared-build-context
20+
context: ../..
21+
dockerfile: containers/spindle-flux-ubuntu/Dockerfile
22+
args: *shared-build-args
23+
24+
# Name of the node that runs the Flux broker
25+
x-shared-environment: &shared-environment
26+
mainHost: node-1
27+
<<: *workers
28+
29+
networks:
30+
flux:
31+
driver: bridge
32+
33+
# Common parameters for all nodes.
34+
x-shared-node-parameters: &shared-node-parameters
35+
build: *shared-build-context
36+
networks:
37+
- flux
38+
environment: *shared-environment
39+
cap_add:
40+
- SYS_NICE # Required for libnuma
41+
42+
services:
43+
node-1:
44+
<<: *shared-node-parameters
45+
hostname: node-1
46+
container_name: node-1
47+
# Check whether all the workers have registered
48+
# with the broker on the head node.
49+
healthcheck:
50+
test: ["CMD", "./flux_healthcheck.sh"]
51+
start_period: 15s
52+
interval: 5s
53+
timeout: 10s
54+
retries: 5
55+
56+
node-2:
57+
<<: *shared-node-parameters
58+
hostname: node-2
59+
container_name: node-2
60+
61+
node-3:
62+
<<: *shared-node-parameters
63+
hostname: node-3
64+
container_name: node-3
65+
66+
node-4:
67+
<<: *shared-node-parameters
68+
hostname: node-4
69+
container_name: node-4
70+

‎containers/spindle-flux-ubuntu/flux/broker.toml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
[exec]
2+
imp = "/usr/libexec/flux/flux-imp"
3+
4+
[access]
5+
allow-guest-user = true
6+
allow-root-owner = true
7+
8+
[resource]
9+
path = "/etc/flux/system/R"
10+
11+
[bootstrap]
12+
curve_cert = "/mnt/curve/curve.cert"
13+
default_port = 8050
14+
default_bind = "tcp://eth0:%%p"
15+
default_connect = "tcp://%%h:%%p"
16+
hosts = [
17+
{ host="node-[1-4]"},
18+
]
19+

‎containers/spindle-flux-ubuntu/flux/imp.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[exec]
2+
allowed-users = [ "flux", "root" ]
3+
allowed-shells = [ "/usr/libexec/flux/flux-shell" ]

‎containers/spindle-flux-ubuntu/scripts/build_spindle.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/bin/bash
2+
3+
set -euxo pipefail
4+
5+
mkdir -p /home/${USER}/Spindle-build
6+
cd /home/${USER}/Spindle-build
7+
/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=flux --enable-flux-plugin --with-localstorage=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
8+
make -j$(nproc)
9+
make install
10+

‎containers/spindle-flux-ubuntu/scripts/entrypoint.sh‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/bin/bash
2+
3+
# Starts munged and the flux broker.
4+
#
5+
# For documentation on running Flux in containers, see
6+
# https://flux-framework.readthedocs.io/en/latest/tutorials/containers
7+
8+
brokerOptions="-Scron.directory=/etc/flux/system/cron.d \
9+
-Stbon.fanout=256 \
10+
-Srundir=/run/flux \
11+
-Sstatedir=${STATE_DIRECTORY:-/var/lib/flux} \
12+
-Slocal-uri=local:///run/flux/local \
13+
-Slog-stderr-level=6 \
14+
-Slog-stderr-mode=local"
15+
16+
# Get the hostname that will resolve for the Docker bridge network.
17+
address=$(echo $( nslookup "$( hostname -i )" | head -n 1 ))
18+
parts=(${address//=/ })
19+
hostName=${parts[2]}
20+
thisHost=(${hostName//./ })
21+
thisHost=${thisHost[0]}
22+
echo $thisHost
23+
export FLUX_FAKE_HOSTNAME=$thisHost
24+
25+
# Start munged
26+
sudo -u munge /usr/sbin/munged
27+
28+
if [ ${thisHost} != "${mainHost}" ]; then
29+
# Worker node -- wait for head node before connecting
30+
sleep 15
31+
FLUX_FAKE_HOSTNAME=$thisHost flux start -o --config /etc/flux/config ${brokerOptions} sleep inf
32+
else
33+
# Head node
34+
FLUX_FAKE_HOSTNAME=$thisHost flux start -o --config /etc/flux/config ${brokerOptions} sleep inf
35+
fi
36+

‎containers/spindle-flux-ubuntu/scripts/flux_healthcheck.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
3+
FLUX_FREE_NODES=$(flux resource list -s free -n | awk '{print $2}')
4+
5+
if [[ ${FLUX_FREE_NODES} -ne ${replicas} ]] ; then
6+
echo "FAILED: Incorrect number of Flux free nodes: expected ${replicas}, got ${FLUX_FREE_NODES}"
7+
exit 1
8+
fi
9+
10+
echo "PASSED: Found ${FLUX_FREE_NODES} nodes out of ${replicas}; all nodes registered."
11+

‎containers/spindle-serial-ubuntu/Dockerfile‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
ARG ubuntu_version=noble
2+
FROM ubuntu:${ubuntu_version}
3+
USER root
4+
5+
RUN DEBIAN_FRONTEND="noninteractive" apt-get update \
6+
# install latest pkg utils:
7+
&& apt-get -qq install -y --no-install-recommends \
8+
apt-utils
9+
10+
RUN DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \
11+
locales \
12+
ca-certificates \
13+
wget \
14+
git \
15+
ssh \
16+
sudo \
17+
build-essential \
18+
pkg-config \
19+
autotools-dev \
20+
libtool \
21+
autoconf \
22+
automake \
23+
make \
24+
gfortran-13 \
25+
gcc-13 \
26+
g++-13 \
27+
munge \
28+
libmunge-dev \
29+
libhwloc-dev \
30+
mpich \
31+
libmpich-dev
32+
33+
# Prevent hwloc from trying to use graphics cards
34+
# as this fails when X is not running.
35+
ENV HWLOC_COMPONENTS=-gl
36+
37+
# Set up munge
38+
RUN mkdir -p /run/munge && \
39+
chown munge:munge /run/munge && \
40+
chmod 0755 /run/munge
41+
42+
ARG USER=spindleuser
43+
ARG UID=1001
44+
ARG BUILD_ROOT=./containers/spindle-serial-ubuntu
45+
COPY ${BUILD_ROOT}/scripts/add_docker_user.sh /add_docker_user.sh
46+
RUN /add_docker_user.sh
47+
48+
USER ${USER}
49+
WORKDIR /home/${USER}
50+
RUN mkdir -p /home/${USER}/Spindle
51+
# Copy the Spindle repo into the container
52+
COPY . /home/${USER}/Spindle
53+
COPY ${BUILD_ROOT}/scripts/build_spindle.sh /home/${USER}/build_spindle.sh
54+
RUN ./build_spindle.sh
55+
56+
COPY ${BUILD_ROOT}/scripts/entrypoint.sh /home/${USER}/entrypoint.sh
57+
ENV PATH /home/${USER}/Spindle-inst/bin:$PATH
58+
59+
ENTRYPOINT /bin/bash ./entrypoint.sh
60+

‎containers/spindle-serial-ubuntu/docker-compose.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
services:
2+
spindlenode:
3+
# The build context is set to ../.. so that we can copy
4+
# the Spindle directory into the created image.
5+
# Because we set the context, we then have to specify
6+
# the path from the context to the Dockerfile.
7+
build:
8+
context: ../..
9+
dockerfile: containers/spindle-serial-ubuntu/Dockerfile
10+
hostname: spindlenode
11+
container_name: spindlenode
12+
tty: true
13+
# The NUMA tests require CAP_SYS_NICE.
14+
cap_add:
15+
- SYS_NICE
16+

0 commit comments

Comments
 (0)