Merge branch 'feature/spainhour/threaded_gwn' of https://github.com/LLNL/axom into feature/spainhour/threaded_gwn

Author: jcs15c

.github/workflows/ci-tests.yml

@@ -43,7 +43,7 @@ jobs:
         - job_name: gcc@13.3.1, shared, 32bit
           host_config: gcc@13.3.1.cmake
           compiler_image: ${{ needs.set_image_vars.outputs.gcc_docker_image }}
-          cmake_opts: '-DBUILD_SHARED_LIBS=ON -DAXOM_USE_64BIT_INDEXTYPE:BOOL=OFF'
+          cmake_opts: '-DBUILD_SHARED_LIBS=ON -DAXOM_USE_64BIT_INDEXTYPE:BOOL=OFF -DAXOM_NO_INT64_T=1'
           do_build: 'yes'
           do_benchmarks: 'no'
         - job_name: llvm@19.0.0, shared, benchmarks, quest regression

RELEASE-NOTES.md

@@ -26,12 +26,19 @@ The Axom project release numbers follow [Semantic Versioning](http://semver.org/
 
 ### Changed
 - Primal: Axom's polygon clipping was modified to handle some corner cases.
+- Bump: Several Bump algorithms were improved so they perform better with OpenMP.
+- Mir: The `EquiZAlgorithm was improved so it performs better with OpenMP.
+- Many algorithms in Bump and Mir have been enhanced with `setAllocatorID()` methods to permit use of custom allocators.
 - Uberenv's spack updated to v1.1.1
 - `radiuss-spack-configs` submodule removed.
 
 ### Fixed
 - Primal: Fixed `NURBSPatch::evaluateDerivatives` returning a matrix with empty values. The old behavior 
   can be recovered with an optional flag, which is more efficient if the previously missing values are unneeded
+- Core: the `ArrayView` class was modified so it defers initializing an internal
+  allocator id via Umpire, if present. This prevents excessive calls to Umpire,
+  which are not needed in all use cases.
+- Quest: A compilation problem with `-DAXOM_NO_INT64_T=1` was fixed.
 
 ### Removed
 

scripts/benchmarks/mir_concentric_circles.py

@@ -13,14 +13,23 @@ def lrun(np):
 
 # Parallel options use 4 ranks per node since that is how many GPUs there are.
 runs = {
-  "build-rzansel-blueos_3_ppc64le_ib_p9-clang@10.0.1.2_cuda-release" : {"policies":["seq", "omp", "cuda"], "launch":lrun},
-  "build-rzwhippet-toss_4_x86_64_ib-clang@14.0.6-release" : {"policies":["seq", "omp"], "launch":srun},
-  "build-rzwhippet-toss_4_x86_64_ib-gcc@10.3.1-release" : {"policies":["seq", "omp"], "launch":srun},
-  "build-rzwhippet-toss_4_x86_64_ib-intel@2022.1.0-release" : {"policies":["seq", "omp"], "launch":srun},
-  "build-rzvernal-toss_4_x86_64_ib_cray-rocmcc@6.2.1_hip-release" : {"policies":["seq", "hip"], "launch":srun},
-  "build-rzvernal-toss_4_x86_64_ib_cray-clang@17.0.0_hip-release" : {"policies":["seq", "hip"], "launch":srun},
-  "build-rzadams-toss_4_x86_64_ib_cray-cce@18.0.0_hip-release" :{"policies":["seq", "hip"], "launch":flux_run},
-  "build-rzadams-toss_4_x86_64_ib_cray-rocmcc@6.2.1_hip-release" :{"policies":["seq", "hip"], "launch":flux_run}
+  "build-dane-toss_4_x86_64_ib-gcc@13.3.1-release" : {"policies":["seq", "omp"], "launch":srun},
+  "build-dane-toss_4_x86_64_ib-intel-oneapi-compilers@2025.2.0-release" : {"policies":["seq", "omp"], "launch":srun},
+  "build-dane-toss_4_x86_64_ib-llvm@19.1.3-release" : {"policies":["seq", "omp"], "launch":srun},
+  "build-matrix-toss_4_x86_64_ib-gcc@13.3.1_cuda-release" : {"policies":["seq", "omp", "cuda"], "launch":srun},
+  "build-matrix-toss_4_x86_64_ib-llvm@19.1.3_cuda-release" : {"policies":["seq", "omp", "cuda"], "launch":srun},
+  "build-rzwhippet-toss_4_x86_64_ib-llvm@19.1.3-release" : {"policies":["seq", "omp"], "launch":srun},
+  "build-rzwhippet-toss_4_x86_64_ib-gcc@13.3.1-release" : {"policies":["seq", "omp"], "launch":srun},
+  "build-rzwhippet-toss_4_x86_64_ib-intel-oneapi-compilers@2025.2.0-release" : {"policies":["seq", "omp"], "launch":srun},
+  "build-rzvernal-toss_4_x86_64_ib_cray-cce@20.0.0_hip-release" : {"policies":["seq", "hip"], "launch":srun},
+  "build-rzvernal-toss_4_x86_64_ib_cray-llvm-amdgpu@6.3.1_hip-release" : {"policies":["seq", "hip"], "launch":srun},
+  "build-rzvernal-toss_4_x86_64_ib_cray-llvm-amdgpu@6.4.2_hip-release" : {"policies":["seq", "hip"], "launch":srun},
+  "build-rzadams-toss_4_x86_64_ib_cray-cce@20.0.0_hip-release" :{"policies":["seq", "hip"], "launch":flux_run},
+  "build-rzadams-toss_4_x86_64_ib_cray-llvm-amdgpu@6.3.1_hip-release" :{"policies":["seq", "hip"], "launch":flux_run},
+  "build-rzadams-toss_4_x86_64_ib_cray-llvm-amdgpu@6.4.2_hip-release" :{"policies":["seq", "hip"], "launch":flux_run},
+  "build-tioga-toss_4_x86_64_ib_cray-cce@20.0.0_hip-release" :{"policies":["seq", "hip"], "launch":flux_run},
+  "build-tioga-toss_4_x86_64_ib_cray-llvm-amdgpu@6.3.1_hip-release" :{"policies":["seq", "hip"], "launch":flux_run},
+  "build-tioga-toss_4_x86_64_ib_cray-llvm-amdgpu@6.4.2_hip-release" :{"policies":["seq", "hip"], "launch":flux_run}
 }
 
 def generate(params):
@@ -40,6 +49,10 @@ def generate(params):
     f.write("CONCENTRIC_CIRCLES=./examples/mir_concentric_circles\n")
     f.write("CONCENTRIC_CIRCLES_MPI=./examples/mir_concentric_circles_mpi\n\n")
 
+    f.write("export OMP_PLACES=cores\n")
+    f.write("export OMP_PROC_BIND=spread\n")
+    f.write("export OMP_DYNAMIC=FALSE\n")
+
     dimension = params["dimension"]
     trials = params["trials"]
     for s in params["sizes"]:
@@ -68,7 +81,10 @@ def read_timings(filename, searchKey):
   retval = "" # no data
   try:
     lines = open(filename, "rt").readlines()
-    print(f"Reading {filename}")
+    msg = ""
+    if searchKey.find("Algorithm") == -1:
+      msg = f" Getting '{searchKey}' value."
+    print(f"Reading {filename}.{msg}")
     for line in lines:
       pos = line.find(searchKey)
       if pos != -1:
@@ -113,10 +129,7 @@ def average(value, trials):
       avg = value
     return avg
 
-  # Measure just the MIR algorithm
-  searchKey = "EquizAlgorithm"
-  if params["method"] == "elvira":
-    searchKey = "ElviraAlgorithm"
+  searchKey = params["searchKey"]
 
   columns = []
   # Add NumZones column (either square or cube of s, depending on dimension)
@@ -184,9 +197,17 @@ def make_csv(params, outputfile):
     f.write(f"{line}\n")
   f.close()
 
-def seriesName(name):
+def seriesName(name, includeCompiler = False):
   policies = {"SEQ" : "Serial", "OMP": "OpenMP", "CUDA" : "CUDA", "HIP" : "HIP"}
-  newName = name[:name.find("-")]
+  if includeCompiler:
+    d0 = name.find("-")
+    d1 = name.rfind("-")
+    s0 = name.rfind(" ")
+    hostName = name[:d0]
+    compName = name[d1 + 1:s0]
+    newName = f"{hostName} {compName}"
+  else:
+    newName = name[:name.find("-")]
   for p in policies.keys():
     if name[-len(p) - 1:] == " " + p:
       newName = newName + " " + policies[p]
@@ -197,8 +218,15 @@ def lineProps(name):
   """
   Make line properties when given a series name.
   """
-  hostColor = {"rzansel" : "g", "rzwhippet" : "r", "rzvernal" : "o", "rzadams" : "b"}
-  policyStyle = {"seq" : "--", "omp": ":", "cuda" : "-", "hip" : "-"}
+  red = "#ff0000"
+  green = "#00aa00"
+  blue = "#0000ff"
+  orange = "#ff6622"
+  #magenta = "#ff00ff"
+  #purple = "#8811aa"
+  hostColor = {"rzwhippet" : red, "rzvernal" : green, "rzadams" : blue,
+               "dane" : red, "matrix" : green, "tioga": orange, "tuolumne" : blue}
+  policyStyle = {"seq" : "-", "omp": ":", "cuda" : "--", "hip" : "--"}
   policyMark = {"seq" : "o", "omp": "s", "cuda" : "^", "hip" : "^"}
   color = "b"
   style = "-"
@@ -214,6 +242,25 @@ def lineProps(name):
        break
   return color, style, mark
 
+def hex_to_rgb(hex_color: str):
+    """
+    Convert a color in "#rrggbb" (or "rrggbb") format to an (r, g, b) tuple.
+    """
+    s = hex_color.strip()
+    if s.startswith("#"):
+        s = s[1:]
+    if len(s) != 6:
+        raise ValueError(f"Expected 6 hex digits, got {len(s)}: {hex_color!r}")
+
+    try:
+        r = int(s[0:2], 16)
+        g = int(s[2:4], 16)
+        b = int(s[4:6], 16)
+    except ValueError as e:
+        raise ValueError(f"Invalid hex color: {hex_color!r}") from e
+
+    return (r, g, b)
+
 def plot(params):
   """
   Read the available timing files and plot them.
@@ -233,22 +280,78 @@ def make_series(col1, col2):
         break
     return x,y
 
+  def rgbToColor(r,g,b):
+     return "#%02x%02x%02x" % (r,g,b)
+
+  def lightenColor(r,g,b):
+     def clamp255(value):
+        if value > 255:
+          return 255
+        return int(value)
+     scale = 1.3
+     return rgbToColor(clamp255(r * scale), clamp255(g * scale), clamp255(b * scale))
+
+  def darkenColor(r,g,b):
+     scale = 0.7
+     return rgbToColor(int(r * scale), int(g * scale), int(b * scale))
+
+  def modifyColor(color, index):
+    if index == 0:
+      return color
+    r,g,b = hex_to_rgb(color)
+    if index == 1:
+      return darkenColor(r,g,b)
+    return lightenColor(r,g,b)
+
+  def valid_column(column):
+    valid = False
+    for i in range(1, len(column)):
+      if column[i] != "":
+        valid = True
+    return valid
+
+  # Make columns from the data
   columns = make_columns(params)
 
+  # Count how many plots would use the normal legend name
+  counts = {}
+  for c in range(1, len(columns)):
+    if not valid_column(columns[c]):
+      continue
+    sn = seriesName(columns[c][0])
+    if sn in counts:
+      count, _ = counts[sn]
+      counts[sn] = (count + 1, 0)
+    else:
+      counts[sn] = (1, 0)
+
   import matplotlib.pyplot as plt
+  seriesLegendNames = {}
   for c in range(1, len(columns)):
     x, y = make_series(columns[0][1:], columns[c][1:])
     if len(x) > 0:
       color, style, mark = lineProps(columns[c][0])
-      plt.plot(x, y, marker=mark, linestyle=style, color=color, linewidth=2., label=seriesName(columns[c][0]))
+      sn = seriesName(columns[c][0])
+      if counts[sn][0] > 1:
+        oldsn = sn
+        sn = seriesName(columns[c][0], True)
+        color = modifyColor(color, counts[oldsn][1])
+        counts[oldsn] = (counts[oldsn][0], counts[oldsn][1] + 1)
+      seriesLegendNames[c] = sn
+      plt.plot(x, y, marker=mark, linestyle=style, color=color, linewidth=2., label=sn)
 
   dimension = params["dimension"]
   method = params["method"]
+  searchKey = params["searchKey"]
+  if searchKey.find("Algorithm") != -1:
+    # more concise
+    searchKey = method
+  doLabels = params["labels"]
 
   # Add labels and title
   plt.xlabel('Number of Zones', fontsize=24)
   plt.ylabel('Time (s)', fontsize=24)
-  plt.title(f'{dimension}D MIR Timings ({method})', fontsize=28)
+  plt.title(f'{dimension}D MIR Timings ({searchKey})', fontsize=28)
   xlabels = make_series(columns[0][1:], columns[0][1:])[0]
   plt.xticks(ticks=xlabels, labels=xlabels, fontsize=18)
   plt.yticks(fontsize=18)
@@ -268,6 +371,65 @@ def make_series(col1, col2):
     print(f"There was an error, probably because the benchmark is still running. {columns}")
     raise
 
+  # Add labels
+  if doLabels and len(columns) >= 2:
+    def findIndex(seq, value):
+      eps = 1.e-8
+      for i in range(len(seq)):
+        if math.fabs(value - seq[i]) < eps:
+          return i
+      return -1
+
+    # Group the related series.
+    seriesGroups = {}
+    seriesSer = {}
+    for c in seriesLegendNames.keys():
+      sn = seriesLegendNames[c]
+      host = sn[:sn.rfind(" ")]
+      if host in seriesGroups:
+        seriesGroups[host].append(c)
+      else:
+        seriesGroups[host] = [c]
+      if sn.find("Serial") != -1:
+        seriesSer[host] = c
+
+    # Go through groups
+    for host in seriesGroups:
+      if not host in seriesSer:
+        continue
+
+      # The serial column
+      cser = seriesSer[host]
+
+      # Compare other columns against the serial column
+      for c in seriesGroups[host]:
+        if cser == c:
+          continue
+
+        x1, y1 = make_series(columns[0][1:], columns[cser][1:])
+        x2, y2 = make_series(columns[0][1:], columns[c][1:])
+        xa = []
+        ya = []
+        labels = []
+        idx = 0
+        for i in range(len(x1)):
+          # Where series 1 (SEQ) > series 2 (OMP)
+          j = findIndex(x2, x1[i])
+          if j != -1 and y1[i] > y2[j]:
+            xa.append(x2[j])
+            ya.append(y2[j])
+            labels.append(f"{math.trunc(100. * y1[i] / y2[i]) / 100.:.2f}x")
+            plt.annotate(
+              labels[idx],
+              (xa[idx], ya[idx]),
+              xytext=(6, -6), textcoords="offset points",  # pixel offset
+              ha="left", va="bottom",
+              fontsize=10,
+              bbox=dict(boxstyle="round,pad=0.2", fc="white", ec="none", alpha=0.8),
+              arrowprops=dict(arrowstyle="-", color="0.3", lw=0.8)
+            )
+            idx = idx + 1
+
   # Show the plot
   plt.grid(True)
   plt.show()
@@ -319,13 +481,27 @@ def get_params():
     required=False
   )
 
+  parser.add_argument(
+    "--key",
+    type=str,
+    help="Name of the caliper timings to extract from the logs",
+    required=False
+  )
+
   parser.add_argument(
     "--trials",
     type=int,
     help="Number of times to run MIR",
     required=False
     )
 
+  parser.add_argument(
+    "--labels",
+    action="store_true",
+    help="Boolean flag to indicate whether to draw speedup labels",
+    required=False
+    )
+
   args = parser.parse_args()
 
   # Convert the comma-separated string into a tuple of integers
@@ -369,6 +545,20 @@ def get_params():
   else:
     params["trials"] = 1
 
+  if args.labels is not None:
+    params["labels"] = args.labels
+  else:
+    params["labels"] = False
+
+  if args.key is not None:
+    params["searchKey"] = args.key
+  else:
+    # Default to MIR algorithm timings
+    if params["method"] == "elvira":
+      params["searchKey"] = "ElviraAlgorithm"
+    else:
+      params["searchKey"] = "EquizAlgorithm"
+
   # Generate some sizes.
   sides = (50, 100, 200, 500, 1000, 1500, 2000, 4000, 8000)
   if params["dimension"] == 3: