Implementing hierarchical distributor

.github/workflows/codecov.yml

@@ -11,7 +11,7 @@ on:
 jobs:
   codecov:
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 5
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4

.github/workflows/linux-clang.yml

@@ -13,6 +13,7 @@ env:
 jobs:
   linux-clang:
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     strategy:
       matrix:
         clang-version: [14, 15, 16, 17, 18]
@@ -75,7 +76,8 @@ jobs:
             -DCMAKE_C_COMPILER=$CC \
             -DCMAKE_CXX_COMPILER=$CXX \
             -DDYNAMPI_BUILD_TESTS=ON \
-            -DDYNAMPI_BUILD_BENCHMARKS=ON"
+            -DDYNAMPI_BUILD_BENCHMARKS=ON \
+            -DDYNAMPI_MAX_MPI_RANK=8"
 
           if [ "$MPI_TYPE" == "MPICH" ]; then
             CMAKE_ARGS="$CMAKE_ARGS -DMPI_C_COMPILER=$CC -DMPI_CXX_COMPILER=$CXX"
@@ -102,4 +104,5 @@ jobs:
           fi
           cd build
           echo "Testing with $MPI_TYPE"
-          ctest --output-on-failure --parallel -C ${{ matrix.build-type }}
+          export DYNAMPI_MAX_MPI_RANK=8
+          ctest --output-on-failure -j 1 --timeout 180 -C ${{ matrix.build-type }}

.github/workflows/linux-gcc.yml

@@ -13,6 +13,7 @@ env:
 jobs:
   linux-gcc:
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     strategy:
       matrix:
         gcc-version: [11, 12, 13, 14]
@@ -77,7 +78,8 @@ jobs:
             -DCMAKE_C_COMPILER=$CC \
             -DCMAKE_CXX_COMPILER=$CXX \
             -DDYNAMPI_BUILD_TESTS=ON \
-            -DDYNAMPI_BUILD_BENCHMARKS=ON"
+            -DDYNAMPI_BUILD_BENCHMARKS=ON \
+            -DDYNAMPI_MAX_MPI_RANK=8"
           if [ "$MPI_TYPE" == "MPICH" ]; then
             CMAKE_ARGS="$CMAKE_ARGS -DMPI_C_COMPILER=$MPI_C_COMPILER -DMPI_CXX_COMPILER=$MPI_CXX_COMPILER"
           fi
@@ -97,4 +99,5 @@ jobs:
           fi
           cd build
           echo "Testing with $MPI_TYPE"
-          ctest --output-on-failure --parallel -C ${{ matrix.build-type }}
+          export DYNAMPI_MAX_MPI_RANK=8
+          ctest --output-on-failure -j 1 --timeout 180 -C ${{ matrix.build-type }}

.github/workflows/linux-intel.yml

@@ -13,6 +13,7 @@ env:
 jobs:
   linux-intel:
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     strategy:
       matrix:
         build-type: [Debug, Release]
@@ -37,7 +38,8 @@ jobs:
             -DCMAKE_C_COMPILER=icx \
             -DCMAKE_CXX_COMPILER=icpx \
             -DDYNAMPI_BUILD_TESTS=ON \
-            -DDYNAMPI_BUILD_BENCHMARKS=ON
+            -DDYNAMPI_BUILD_BENCHMARKS=ON \
+            -DDYNAMPI_MAX_MPI_RANK=8
       - name: Build with Intel
         shell: bash
         run: |
@@ -49,4 +51,5 @@ jobs:
           source /opt/intel/oneapi/setvars.sh
           cd build
           echo "Testing with $MPI_TYPE"
-          ctest --output-on-failure --parallel -C ${{ matrix.build-type }}
+          export DYNAMPI_MAX_MPI_RANK=8
+          ctest --output-on-failure -j 1 --timeout 180 -C ${{ matrix.build-type }}

.github/workflows/macos.yml

@@ -13,6 +13,7 @@ env:
 jobs:
   macos-clang:
     runs-on: macos-latest
+    timeout-minutes: 5
     strategy:
       matrix:
         build-type: [Debug, Release]
@@ -29,7 +30,8 @@ jobs:
           cmake -B build \
             -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \
             -DDYNAMPI_BUILD_TESTS=ON \
-            -DDYNAMPI_BUILD_BENCHMARKS=ON
+            -DDYNAMPI_BUILD_BENCHMARKS=ON \
+            -DDYNAMPI_MAX_MPI_RANK=8
       - name: Build on macOS
         shell: bash
         run: cmake --build build --config ${{ matrix.build-type }} --parallel
@@ -38,4 +40,5 @@ jobs:
         run: |
           cd build
           echo "Testing with $MPI_TYPE"
+          export DYNAMPI_MAX_MPI_RANK=8
           ctest --output-on-failure --parallel -C ${{ matrix.build-type }}

.github/workflows/pre-commit.yml

@@ -10,6 +10,7 @@ on:
 jobs:
   pre-commit:
     runs-on: ubuntu-latest
+    timeout-minutes: 5
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/sanitizers.yml

@@ -13,6 +13,7 @@ env:
 jobs:
   linux-debug-sanitizers:
     runs-on: ubuntu-latest
+    timeout-minutes: 5
     strategy:
       matrix:
         sanitizer: [address, undefined]
@@ -33,6 +34,7 @@ jobs:
             -DCMAKE_CXX_COMPILER=g++-14 \
             -DDYNAMPI_BUILD_TESTS=ON \
             -DDYNAMPI_BUILD_BENCHMARKS=ON \
+            -DDYNAMPI_MAX_MPI_RANK=8 \
             -DCMAKE_CXX_FLAGS="-fsanitize=${{ matrix.sanitizer }} -fno-omit-frame-pointer" \
             -DCMAKE_C_FLAGS="-fsanitize=${{ matrix.sanitizer }} -fno-omit-frame-pointer" \
             -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=${{ matrix.sanitizer }}" \
@@ -46,4 +48,5 @@ jobs:
           cd build
           echo "Testing with $MPI_TYPE and ${{ matrix.sanitizer }} sanitizer"
           export LSAN_OPTIONS=suppressions=${{ github.workspace }}/test/lsan.supp
-          ctest --output-on-failure --parallel -C Debug
+          export DYNAMPI_MAX_MPI_RANK=8
+          ctest --output-on-failure --verbose -C Debug

.github/workflows/smpi.yml

@@ -11,7 +11,7 @@ on:
 jobs:
   smpi:
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 5
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -46,7 +46,8 @@ jobs:
             -DMPIEXEC_EXECUTABLE=/usr/bin/smpirun \
             -DMPIEXEC_PREFLAGS=-platform\;platform.xml \
             -DDYNAMPI_BUILD_TESTS=ON \
-            -DDYNAMPI_BUILD_BENCHMARKS=OFF"
+            -DDYNAMPI_BUILD_BENCHMARKS=OFF \
+            -DDYNAMPI_MAX_MPI_RANK=8"
 
           echo "CMAKE_ARGS: $CMAKE_ARGS"
 
@@ -81,7 +82,8 @@ jobs:
         run: |
           echo "Testing with $MPI_TYPE"
           cp platform.xml ${{ steps.strings.outputs.build-output-dir }}/test
+          export DYNAMPI_MAX_MPI_RANK=8
           ctest --output-on-failure --parallel
 
-          echo "Additionally running MPI tests with 100 ranks..."
-          smpirun -np 100 -platform platform.xml ./test/mpi_test
+          echo "Additionally running MPI tests with 8 ranks..."
+          smpirun -np 8 -platform platform.xml ./test/mpi_test

.github/workflows/windows.yml

@@ -19,6 +19,7 @@ jobs:
   windows-msmpi:
     name: Windows MS-MPI (MPI 3.1)
     runs-on: windows-latest
+    timeout-minutes: 10
     strategy:
       matrix:
         build-type: [Debug, Release]
@@ -34,7 +35,8 @@ jobs:
           cmake -B build `
             -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} `
             -DDYNAMPI_BUILD_TESTS=ON `
-            -DDYNAMPI_BUILD_BENCHMARKS=ON
+            -DDYNAMPI_BUILD_BENCHMARKS=ON `
+            -DDYNAMPI_MAX_MPI_RANK=8
       - name: Build (MS-MPI)
         shell: pwsh
         run: cmake --build build --config ${{ matrix.build-type }} --parallel
@@ -43,10 +45,12 @@ jobs:
         run: |
           cd build
           echo "Testing with Microsoft MPI (MPI 3.1 features)"
-          ctest --output-on-failure --parallel -C ${{ matrix.build-type }}
+          $env:DYNAMPI_MAX_MPI_RANK = "8"
+          ctest --output-on-failure -j 1 --timeout 180 -C ${{ matrix.build-type }}
   windows-mingw:
     name: Windows MinGW
     runs-on: windows-latest
+    timeout-minutes: 10
     strategy:
       matrix:
         build-type: [Debug, Release]
@@ -97,7 +101,8 @@ jobs:
             -DMPI_C_LIBRARIES=/mingw64/lib/libmsmpi.a \
             -DMPI_CXX_LIBRARIES=/mingw64/lib/libmsmpi.a \
             -DDYNAMPI_BUILD_TESTS=ON \
-            -DDYNAMPI_BUILD_BENCHMARKS=ON
+            -DDYNAMPI_BUILD_BENCHMARKS=ON \
+            -DDYNAMPI_MAX_MPI_RANK=8
       - name: Build (MinGW)
         shell: msys2 {0}
         run: cmake --build build --config ${{ matrix.build-type }} --parallel
@@ -106,10 +111,12 @@ jobs:
         run: |
           cd build
           echo "Testing with $MPI_TYPE"
-          ctest --output-on-failure --parallel -C ${{ matrix.build-type }}
+          export DYNAMPI_MAX_MPI_RANK=8
+          ctest --output-on-failure -j 1 --timeout 180 -C ${{ matrix.build-type }}
   windows-intel-mpi:
     name: Windows Intel MPI (MPI 4.0)
     runs-on: windows-latest
+    timeout-minutes: 10
     strategy:
       matrix:
         build-type: [Debug, Release]
@@ -126,7 +133,8 @@ jobs:
           cmake -B build `
             -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} `
             -DDYNAMPI_BUILD_TESTS=ON `
-            -DDYNAMPI_BUILD_BENCHMARKS=ON
+            -DDYNAMPI_BUILD_BENCHMARKS=ON `
+            -DDYNAMPI_MAX_MPI_RANK=8
       - name: Build (Intel MPI)
         shell: pwsh
         run: cmake --build build --config ${{ matrix.build-type }} --parallel
@@ -135,4 +143,5 @@ jobs:
         run: |
           cd build
           echo "Testing with Intel MPI (MPI 4.0 support)"
-          ctest --output-on-failure --parallel -C ${{ matrix.build-type }}
+          $env:DYNAMPI_MAX_MPI_RANK = "8"
+          ctest --output-on-failure -j 1 --timeout 180 -C ${{ matrix.build-type }}

.gitignore

@@ -8,3 +8,6 @@ bin
 Testing
 *.btr
 commands.txt
+core.*
+__pycache__/
+*.png

.pre-commit-config.yaml

@@ -18,6 +18,14 @@ repos:
     rev: v20.1.8
     hooks:
       - id: clang-format
+  - repo: local
+    hooks:
+      - id: cppcheck
+        name: cppcheck
+        entry: cppcheck --enable=warning --suppress=missingIncludeSystem -I include
+          --std=c++20 --inline-suppr
+        language: system
+        types_or: [c, c++]
   - repo: https://github.com/google/yamlfmt.git
     rev: v0.17.2
     hooks:

CMakeLists.txt

@@ -61,6 +61,9 @@ if(MSVC)
 else()
   add_compile_options(-Wall -Wextra -Wpedantic -Werror -fno-ms-extensions)
 endif()
+if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM")
+  add_compile_options(-diag-disable=10430 -Wno-unknown-warning-option)
+endif()
 
 set(CMAKE_CXX_FLAGS_COVERAGE "-O0 -g -coverage -fprofile-arcs -ftest-coverage \
     -fno-elide-constructors ${CMAKE_CXX_FLAGS_COVERAGE}")

REUSE.toml

@@ -0,0 +1,13 @@
+version = 1
+
+[[annotations]]
+path = "benchmark/results/**.csv"
+SPDX-FileCopyrightText = "2025 QDX Technologies"
+SPDX-License-Identifier = "Apache-2.0"
+SPDX-Comment = "Generated benchmark results."
+
+[[annotations]]
+path = "**.png"
+SPDX-FileCopyrightText = "2025 QDX Technologies"
+SPDX-License-Identifier = "Apache-2.0"
+SPDX-Comment = "Generated result plots."

benchmark/CMakeLists.txt

@@ -9,11 +9,21 @@ FetchContent_Declare(
 )
 FetchContent_MakeAvailable(cxxopts)
 
-add_executable(asymptotic_distribution_throughput
-    asymptotic_distribution_throughput.cpp
-)
-target_link_libraries(asymptotic_distribution_throughput
-    PRIVATE
-    dynampi
-    cxxopts::cxxopts
+set(benchmarks
+    asymptotic_distribution_throughput
+    strong_scaling_distribution_rate
+    pingpong
+    timer_resolution
+    naive_shutdown_time
 )
+
+foreach(benchmark IN LISTS benchmarks)
+    add_executable(${benchmark}
+        ${benchmark}.cpp
+    )
+    target_link_libraries(${benchmark}
+        PRIVATE
+        dynampi
+        cxxopts::cxxopts
+    )
+endforeach()

benchmark/asymptotic_distribution_throughput.cpp

@@ -74,11 +74,12 @@ static double run_single_benchmark(const BenchmarkOptions& opts) {
   MPI_Barrier(MPI_COMM_WORLD);
 
   using Task = size_t;
-  using Result = std::vector<std::byte>;
+  // using Result = std::vector<std::byte>;
+  using Result = size_t;
 
-  auto worker_task = [&opts](Task task) -> Result {
-    return std::vector<std::byte>(opts.message_size, std::byte(task));
-  };
+  // auto worker_task = [&opts](Task task) -> Result {
+  // return std::vector<std::byte>(opts.message_size, std::byte(task));
+  auto worker_task = [](Task task) -> Result { return task; };
 
   dynampi::Timer dynamic_timer;
   auto dynamic_communicator = make_dynamic_communicator(opts.remove_root_from_distribution);
@@ -103,8 +104,9 @@ static double run_single_benchmark(const BenchmarkOptions& opts) {
     if (work_distributer.is_root_manager()) {
       std::cout << "Dynamic task distribution completed successfully." << std::endl;
       const auto& stats = work_distributer.get_statistics();
-      for (size_t i = 0; i < stats.worker_task_counts.size(); i++) {
-        std::cout << "Rank " << i << ": " << "Tasks: " << stats.worker_task_counts[i] << std::endl;
+      for (size_t i = 0; i < stats.worker_task_counts->size(); i++) {
+        std::cout << "Rank " << i << ": " << "Tasks: " << stats.worker_task_counts->at(i)
+                  << std::endl;
       }
       std::cout << "Total messages sent: " << stats.comm_statistics.send_count << std::endl;
       std::cout << "Total messages received: " << stats.comm_statistics.recv_count << std::endl;

benchmark/aurora/aurora_compile.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+# SPDX-FileCopyrightText: 2025 QDX Technologies. Authored by Ryan Stocks <ryan.stocks00@gmail.com>
+# SPDX-License-Identifier: Apache-2.0
+
+set -euo pipefail
+
+module load cmake
+
+# Default values
+BUILD_DIR="build"
+BUILD_TYPE="Release"
+
+usage() {
+    echo "Usage: $0 [-d build_dir] [-t build_type] [--clean]"
+    echo
+    echo "Options:"
+    echo "  -d DIR    Build directory (default: build)"
+    echo "  -t TYPE   Build type: Release, Debug, RelWithDebInfo, MinSizeRel (default: Release)"
+    echo "  --clean   Remove build directory before configuring"
+    exit 1
+}
+
+# Parse args
+CLEAN=0
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        -d) BUILD_DIR="$2"; shift 2 ;;
+        -t) BUILD_TYPE="$2"; shift 2 ;;
+        --clean) CLEAN=1; shift ;;
+        -h|--help) usage ;;
+        *) echo "Unknown option: $1"; usage ;;
+    esac
+done
+
+if [[ $CLEAN -eq 1 && -d "$BUILD_DIR" ]]; then
+    echo "Cleaning $BUILD_DIR"
+    rm -rf "$BUILD_DIR"
+fi
+
+echo "Configuring with CMake..."
+cmake -DCMAKE_BUILD_TYPE="$BUILD_TYPE" -B "$BUILD_DIR"
+
+echo "Building..."
+cmake --build "$BUILD_DIR" -- -j"$(nproc)"
+
+echo "✅ Build finished in $BUILD_DIR ($BUILD_TYPE)"