Skip to content

Commit dcc6ec7

Browse files
jjhurseyjjhursey
authored
Merge pull request #25 from jjhursey/runtime-ci-testing
Open MPI Runtime Testing Harness
2 parents 08ab736 + f4bdde1 commit dcc6ec7

19 files changed

+1131
-0
lines changed

‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,5 @@ comm_split_type/cmsplit_type
3838
singleton/hello_c
3939
singleton/simple_spawn
4040
singleton/simple_spawn_multiple
41+
42+
.vscode

‎runtime/.ci-configure

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#
2+
# Open MPI is built with the following options:
3+
# ./configure --prefix=/opt/ci/support/exports/ompi --disable-cuda --disable-nvml --with-cuda=no --without-hcoll
4+
#
5+
# Additional options can be added by listing them.
6+
# Options can be listed as either:
7+
# - One option per line
8+
# - Multiple options per line
9+
# Note: Line continutions are not supported
10+
#
11+
# Enable Debug
12+
--enable-debug
13+
# With Python Bindings
14+
# Need to install Cython on the CI machine
15+
#--enable-python-bindings

‎runtime/.ci-tests

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Start with the basics
2+
hello_world

‎runtime/README.md

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# Test suite for Open MPI runtime
2+
3+
This test suite is meant to be able to be run stand-alone or under CI.
4+
5+
All of the tests that are intended for CI must be listed in the `.ci-tests` file.
6+
7+
If the Open MPI build needs additional `configure` options those can be added to the `.ci-configure` file.
8+
9+
## Running tests stand alone
10+
11+
1. Make sure that Open MPI and other required libraries are in your `PATH`/`LD_LIBRARY_PATH`
12+
2. Drop into a directory:
13+
- Use the `build.sh` script to build any test articles
14+
- Use the `run.sh` script to run the test program
15+
16+
17+
## CI Environment Variables
18+
19+
The CI infrastructure defines the following environment variables to be used in the test programs. These are defined during the `run.sh` phase and not the `build.sh` phase.
20+
21+
* `CI_HOSTFILE` : Absolute path to the hostfile for this run.
22+
* `CI_NUM_NODES` : Number of nodes in this cluster.
23+
* `CI_OMPI_SRC` : top level directory of the Open MPI repository checkout.
24+
* `CI_OMPI_TESTS_PUBLIC_DIR` : Top level directory of the [Open MPI Public Test](https://github.com/open-mpi/ompi-tests-public) repository checkout
25+
* `OMPI_ROOT` : Open MPI install directory.
26+
27+
28+
### Adding a new test for CI
29+
30+
1. Create a directory with your test.
31+
- **Note**: Please make your test scripts such that they can be easily run with or without the CI environment variables.
32+
2. Create a build script named `build.sh`
33+
- CI will call this exactly one time (with a timeout in case it hangs).
34+
- If the script returns `0` then it is considered successful. Otherwise it is considered failed.
35+
3. Create a run script named `run.sh`
36+
- The script is responsible for running your test including any runtime setup/shutdown and test result inspection.
37+
- CI will call this exactly one time (with a timeout in case it hangs).
38+
- If the script returns `0` then it is considered successful. Otherwise it is considered failed.
39+
4. Add your directory name to the `.ci-tests` file in this directory in the order that they should be executed.
40+
- Note that adding the directory is not sufficient to have CI run the test, it must be in the `.ci-tests` file.
41+
- Comments (starting with `#`) are allowed.

‎runtime/bin/cleanup-scrub-local.sh

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/bash
2+
3+
PROGS="prte prted prun mpirun timeout"
4+
5+
clean_files()
6+
{
7+
FILES=("pmix-*" "core*" "openmpi-sessions-*" "pmix_dstor_*" "ompi.*" "prrte.*" )
8+
9+
for fn in ${FILES[@]}; do
10+
find /tmp/ -maxdepth 1 \
11+
-user $USER -a \
12+
-name $fn \
13+
-exec rm -rf {} \;
14+
15+
if [ -n "$TMPDIR" ] ; then
16+
find $TMPDIR -maxdepth 1 \
17+
-user $USER -a \
18+
-name $fn \
19+
-exec rm -rf {} \;
20+
fi
21+
done
22+
}
23+
24+
killall -q ${PROGS} > /dev/null
25+
clean_files
26+
killall -q -9 ${PROGS} > /dev/null
27+
28+
exit 0
29+
30+

‎runtime/bin/cleanup.sh

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/bin/bash
2+
3+
clean_server()
4+
{
5+
SERVER=$1
6+
ITER=$2
7+
MAX=$3
8+
QUIET=$4
9+
10+
SCRIPTDIR=$PWD/`dirname $0`/
11+
12+
if [[ $QUIET == 0 ]] ; then
13+
echo "Cleaning server ($ITER / $MAX): $SERVER"
14+
fi
15+
ssh -oBatchMode=yes ${SERVER} ${SCRIPTDIR}/cleanup-scrub-local.sh
16+
}
17+
18+
if [[ "x" != "x$CI_HOSTFILE" && -f "$CI_HOSTFILE" ]] ; then
19+
ALLHOSTS=(`cat $CI_HOSTFILE | sort | uniq`)
20+
else
21+
ALLHOSTS=(`hostname`)
22+
fi
23+
LEN=${#ALLHOSTS[@]}
24+
25+
# Use a background mode if running at scale
26+
USE_BG=0
27+
if [ $LEN -gt 10 ] ; then
28+
USE_BG=1
29+
fi
30+
31+
for (( i=0; i<${LEN}; i++ ));
32+
do
33+
if [ $USE_BG == 1 ] ; then
34+
if [ $(($i % 100)) == 0 ] ; then
35+
echo "| $i"
36+
else
37+
if [ $(($i % 10)) == 0 ] ; then
38+
echo -n "|"
39+
else
40+
echo -n "."
41+
fi
42+
fi
43+
fi
44+
45+
if [ $USE_BG == 1 ] ; then
46+
clean_server ${ALLHOSTS[$i]} $i $LEN $USE_BG &
47+
sleep 0.25
48+
else
49+
clean_server ${ALLHOSTS[$i]} $i $LEN $USE_BG
50+
echo "-------------------------"
51+
fi
52+
done
53+
54+
if [ $USE_BG == 1 ] ; then
55+
echo ""
56+
echo "------------------------- Waiting"
57+
wait
58+
fi
59+
60+
echo "------------------------- Done"
61+
62+
exit 0

‎runtime/bin/pretty-print-hwloc/.gitignore

+72
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
# Prerequisites
2+
*.d
3+
4+
# Object files
5+
*.o
6+
*.ko
7+
*.obj
8+
*.elf
9+
10+
# Linker output
11+
*.ilk
12+
*.map
13+
*.exp
14+
15+
# Precompiled Headers
16+
*.gch
17+
*.pch
18+
19+
# Libraries
20+
*.lib
21+
*.a
22+
*.la
23+
*.lo
24+
25+
# Shared objects (inc. Windows DLLs)
26+
*.dll
27+
*.so
28+
*.so.*
29+
*.dylib
30+
31+
# Executables
32+
*.exe
33+
*.out
34+
*.app
35+
*.i*86
36+
*.x86_64
37+
*.hex
38+
39+
# Debug files
40+
*.dSYM/
41+
*.su
42+
*.idb
43+
*.pdb
44+
45+
# Kernel Module Compile Results
46+
*.mod*
47+
*.cmd
48+
.tmp_versions/
49+
modules.order
50+
Module.symvers
51+
Mkfile.old
52+
dkms.conf
53+
54+
# Autoconf/Automake leftovers
55+
autom4te.cache/
56+
compile
57+
depcomp
58+
aclocal.m4
59+
config.log
60+
config.status
61+
configure
62+
install-sh
63+
missing
64+
.deps
65+
.libs
66+
*.in
67+
Makefile
68+
src/include/autogen/config.h*
69+
src/include/autogen/stamp-h1
70+
71+
# Binary leftovers
72+
src/get-pretty-cpu

‎runtime/bin/pretty-print-hwloc/Makefile.am

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#
2+
# High level Makefile
3+
#
4+
headers =
5+
sources =
6+
nodist_headers =
7+
EXTRA_DIST =
8+
9+
SUBDIRS = . src

‎runtime/bin/pretty-print-hwloc/README.md

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
# Pretty Print HWLOC Process Binding
2+
3+
## Building
4+
5+
```shell
6+
./autogen.sh
7+
./configure --prefix=${YOUR_INSTALL_DIR} --with-hwloc=${HWLOC_INSTALL_PATH}
8+
make
9+
make install
10+
````
11+
12+
## Running
13+
14+
### Default: Print HWLOC bitmap
15+
16+
```shell
17+
shell$ get-pretty-cpu
18+
0/ 0 on c660f5n18) Process Bound : 0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff
19+
```
20+
21+
```shell
22+
shell$ hwloc-bind core:2 get-pretty-cpu
23+
0/ 0 on c660f5n18) Process Bound : 0x00ff0000
24+
```
25+
26+
```shell
27+
shell$ mpirun -np 2 get-pretty-cpu
28+
0/ 2 on c660f5n18) Process Bound : 0x000000ff
29+
1/ 2 on c660f5n18) Process Bound : 0x0000ff00
30+
```
31+
32+
### Full descriptive output
33+
34+
```shell
35+
shell$ get-pretty-cpu -b -f
36+
0/ 0 on c660f5n18) Process Bound : socket 0[core 0[hwt 0-7]],socket 0[core 1[hwt 0-7]],socket 0[core 2[hwt 0-7]],socket 0[core 3[hwt 0-7]],socket 0[core 4[hwt 0-7]],socket 0[core 5[hwt 0-7]],socket 0[core 6[hwt 0-7]],socket 0[core 7[hwt 0-7]],socket 0[core 8[hwt 0-7]],socket 0[core 9[hwt 0-7]],socket 1[core 10[hwt 0-7]],socket 1[core 11[hwt 0-7]],socket 1[core 12[hwt 0-7]],socket 1[core 13[hwt 0-7]],socket 1[core 14[hwt 0-7]],socket 1[core 15[hwt 0-7]],socket 1[core 16[hwt 0-7]],socket 1[core 17[hwt 0-7]],socket 1[core 18[hwt 0-7]],socket 1[core 19[hwt 0-7]]
37+
```
38+
39+
```shell
40+
shell$ hwloc-bind core:2 get-pretty-cpu -b -f
41+
0/ 0 on c660f5n18) Process Bound : socket 0[core 2[hwt 0-7]]
42+
```
43+
44+
```shell
45+
shell$ mpirun -np 2 get-pretty-cpu -b -f
46+
1/ 2 on c660f5n18) Process Bound : socket 0[core 1[hwt 0-7]]
47+
0/ 2 on c660f5n18) Process Bound : socket 0[core 0[hwt 0-7]]
48+
```
49+
50+
### Full descriptive bracketed output
51+
52+
```shell
53+
shell$ get-pretty-cpu -b -m
54+
0/ 0 on c660f5n18) Process Bound : [BBBBBBBB/BBBBBBBB/BBBBBBBB/BBBBBBBB/BBBBBBBB/BBBBBBBB/BBBBBBBB/BBBBBBBB/BBBBBBBB/BBBBBBBB][BBBBBBBB/BBBBBBBB/BBBBBBBB/BBBBBBBB/BBBBBBBB/BBBBBBBB/BBBBBBBB/BBBBBBBB/BBBBBBBB/BBBBBBBB]
55+
```
56+
57+
```shell
58+
shell$ hwloc-bind core:2 get-pretty-cpu -b -m
59+
0/ 0 on c660f5n18) Process Bound : [......../......../BBBBBBBB/......../......../......../......../......../......../........][......../......../......../......../......../......../......../......../......../........]
60+
```
61+
62+
```shell
63+
shell$ mpirun -np 2 get-pretty-cpu -b -m
64+
1/ 2 on c660f5n18) Process Bound : [......../BBBBBBBB/......../......../......../......../......../......../......../........][......../......../......../......../......../......../......../......../......../........]
65+
0/ 2 on c660f5n18) Process Bound : [BBBBBBBB/......../......../......../......../......../......../......../......../........][......../......../......../......../......../......../......../......../......../........]
66+
```

‎runtime/bin/pretty-print-hwloc/autogen.sh

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash -e
2+
3+
autoreconf -ivf

0 commit comments

Comments
 (0)