From 8c4210a748d87776bdba0afabc2bc177d21c121d Mon Sep 17 00:00:00 2001 From: Alex Tyler Chapman Date: Thu, 22 Aug 2024 10:11:13 -0700 Subject: [PATCH 01/22] add extra metadata tags --- src/serac/infrastructure/tests/profiling.cpp | 2 ++ src/serac/physics/benchmarks/benchmark_functional.cpp | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/serac/infrastructure/tests/profiling.cpp b/src/serac/infrastructure/tests/profiling.cpp index 8571f48b9..e38e0ef9d 100644 --- a/src/serac/infrastructure/tests/profiling.cpp +++ b/src/serac/infrastructure/tests/profiling.cpp @@ -47,6 +47,8 @@ TEST(Profiling, MeshRefinement) pmesh->UniformRefinement(); } + // Add metadata + SERAC_SET_METADATA("test", "profiling"); SERAC_SET_METADATA("mesh_file", mesh_file.c_str()); SERAC_SET_METADATA("number_mesh_elements", pmesh->GetNE()); diff --git a/src/serac/physics/benchmarks/benchmark_functional.cpp b/src/serac/physics/benchmarks/benchmark_functional.cpp index 0127e64ea..6645074b0 100644 --- a/src/serac/physics/benchmarks/benchmark_functional.cpp +++ b/src/serac/physics/benchmarks/benchmark_functional.cpp @@ -86,6 +86,9 @@ int main(int argc, char* argv[]) // Initialize profiling serac::profiling::initialize(); + // Add metadata + SERAC_SET_METADATA("test", "functional"); + SERAC_MARK_BEGIN("scalar H1"); SERAC_MARK_BEGIN("dimension 2, order 1"); From 094066f55d0147aa6d77906a7ef5d5771f117d0e Mon Sep 17 00:00:00 2001 From: Alex Tyler Chapman Date: Thu, 22 Aug 2024 17:41:28 -0700 Subject: [PATCH 02/22] Create script to run benchmarks and move caliper files to shared location --- scripts/llnl/build_devtools.py | 2 - scripts/llnl/common_build_functions.py | 5 ++ scripts/llnl/run_benchmarks.py | 92 ++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 2 deletions(-) create mode 100755 scripts/llnl/run_benchmarks.py diff --git a/scripts/llnl/build_devtools.py b/scripts/llnl/build_devtools.py index 532c486c6..9c9ceb1cd 100755 --- a/scripts/llnl/build_devtools.py +++ b/scripts/llnl/build_devtools.py @@ -18,8 +18,6 @@ from argparse import ArgumentParser -import os - def parse_args(): "Parses args from command line" diff --git a/scripts/llnl/common_build_functions.py b/scripts/llnl/common_build_functions.py index f1e9ddaa1..ce3e1c512 100755 --- a/scripts/llnl/common_build_functions.py +++ b/scripts/llnl/common_build_functions.py @@ -676,6 +676,10 @@ def get_shared_libs_dir(): return pjoin(get_shared_base_dir(), "libs", get_project_name()) +def get_shared_spot_dir(): + return pjoin(get_shared_base_dir(), "spot") + + def get_uberenv_path(): return pjoin(get_script_dir(), "../uberenv/uberenv.py") @@ -709,3 +713,4 @@ def convertSecondsToReadableTime(seconds): m, s = divmod(seconds, 60) h, m = divmod(m, 60) return "%d:%02d:%02d" % (h, m, s) + diff --git a/scripts/llnl/run_benchmarks.py b/scripts/llnl/run_benchmarks.py new file mode 100755 index 000000000..ee01ab333 --- /dev/null +++ b/scripts/llnl/run_benchmarks.py @@ -0,0 +1,92 @@ +#!/bin/sh +"exec" "python3" "-u" "-B" "$0" "$@" + +# Copyright (c) 2019-2024, Lawrence Livermore National Security, LLC and +# other Serac Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: (BSD-3-Clause) + +""" + file: run_benchmarks.py + + description: + Run benchmarks and update shared (or any desired) location with new Caliper files + +""" + +from common_build_functions import * + +from argparse import ArgumentParser + +import os + + +def parse_args(): + "Parses args from command line" + parser = ArgumentParser() + parser.add_argument("-sd", "--spot-directory", + dest="spot_dir", + default=get_shared_spot_dir(), + help="Where to put all resulting caliper files to use for SPOT analysis (defaults to a shared location)") + parser.add_argument("-hc", "--host-config", + dest="host_config", + default="", + required=True, + help="Specific host-config file to build") + parser.add_argument("-t", "--timestamp", + dest="timestamp", + default=get_timestamp(), + help="Set timestamp manually for debugging") + ############### + # parse args + ############### + args, extra_args = parser.parse_known_args() + args = vars(args) + return args + + +def main(): + args = parse_args() + + # Args + spot_dir = args["spot_dir"] + host_config = args["host_config"] + timestamp = args["timestamp"] + + # Vars + repo_dir = get_repo_dir() + test_root = get_build_and_test_root(repo_dir, timestamp) + + # Build Serac + os.chdir(repo_dir) + os.makedirs(test_root) + build_and_test_host_config(test_root, host_config, True, "-DENABLE_BENCHMARKS=ON", True, True, "") + + # Go to build location + build_dir="" + dirs = glob.glob(pjoin(test_root, "*")) + for dir in dirs: + if os.path.exists(dir) and "build-" in dir: + build_dir=dir + os.chdir(build_dir) + + # Run benchmarks + # TODO Instead of running each benchmark individually, create a custom target + # shell_exec("make benchmarks -j") + shell_exec("tests/profiling", print_output=True) + shell_exec("benchmarks/benchmark_thermal", print_output=True) + shell_exec("benchmarks/benchmark_functional", print_output=True) + + # Move resulting .cali files to specified directory + os.makedirs(spot_dir, exist_ok=True) + cali_files = glob.glob(pjoin(build_dir, "*.cali")) + print(cali_files) + for cali_file in cali_files: + if os.path.exists(cali_file): + shutil.move(cali_file, spot_dir) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 94f25911a4874ae6685da5d61ce73fc06dbf563e Mon Sep 17 00:00:00 2001 From: Alex Tyler Chapman Date: Tue, 27 Aug 2024 17:59:34 -0700 Subject: [PATCH 03/22] print spot dir, add -e option, skip building docs --- scripts/llnl/run_benchmarks.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/scripts/llnl/run_benchmarks.py b/scripts/llnl/run_benchmarks.py index ee01ab333..f6e2ad5bb 100755 --- a/scripts/llnl/run_benchmarks.py +++ b/scripts/llnl/run_benchmarks.py @@ -24,15 +24,19 @@ def parse_args(): "Parses args from command line" parser = ArgumentParser() - parser.add_argument("-sd", "--spot-directory", - dest="spot_dir", - default=get_shared_spot_dir(), - help="Where to put all resulting caliper files to use for SPOT analysis (defaults to a shared location)") + parser.add_argument("-e", "--extra-cmake-options", + dest="extra_cmake_options", + default="", + help="Extra cmake options to add to the cmake configure line. Note '-DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF' is always appended.") parser.add_argument("-hc", "--host-config", dest="host_config", default="", required=True, help="Specific host-config file to build") + parser.add_argument("-sd", "--spot-directory", + dest="spot_dir", + default=get_shared_spot_dir(), + help="Where to put all resulting caliper files to use for SPOT analysis (defaults to a shared location)") parser.add_argument("-t", "--timestamp", dest="timestamp", default=get_timestamp(), @@ -49,8 +53,9 @@ def main(): args = parse_args() # Args - spot_dir = args["spot_dir"] + cmake_options = args["extra_cmake_options"] + " -DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF" host_config = args["host_config"] + spot_dir = args["spot_dir"] timestamp = args["timestamp"] # Vars @@ -59,8 +64,8 @@ def main(): # Build Serac os.chdir(repo_dir) - os.makedirs(test_root) - build_and_test_host_config(test_root, host_config, True, "-DENABLE_BENCHMARKS=ON", True, True, "") + os.makedirs(test_root, exist_ok=True) + build_and_test_host_config(test_root, host_config, True, cmake_options, True, True, "") # Go to build location build_dir="" @@ -80,11 +85,12 @@ def main(): # Move resulting .cali files to specified directory os.makedirs(spot_dir, exist_ok=True) cali_files = glob.glob(pjoin(build_dir, "*.cali")) - print(cali_files) for cali_file in cali_files: if os.path.exists(cali_file): shutil.move(cali_file, spot_dir) + print("[View SPOT directory here: https://lc.llnl.gov/spot2/?sf={0}]".format(spot_dir)) + return 0 From 56bd86e4e10eab5475203c9a176b37e5b829fff8 Mon Sep 17 00:00:00 2001 From: Michael Tupek Date: Fri, 30 Aug 2024 08:39:39 -0700 Subject: [PATCH 04/22] Turn off petsc hijacking of signal handling. --- src/serac/infrastructure/initialize.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/serac/infrastructure/initialize.cpp b/src/serac/infrastructure/initialize.cpp index 207fd7af1..9f4622b80 100644 --- a/src/serac/infrastructure/initialize.cpp +++ b/src/serac/infrastructure/initialize.cpp @@ -63,14 +63,14 @@ std::pair initialize(int argc, char* argv[], MPI_Comm comm) #endif #ifdef SERAC_USE_PETSC - #ifdef SERAC_USE_SLEPC mfem::MFEMInitializeSlepc(&argc, &argv); #else mfem::MFEMInitializePetsc(&argc, &argv); #endif - + PetscPopSignalHandler(); #endif + // Initialize GPU (no-op if not enabled/available) // TODO for some reason this causes errors on Lassen. We need to look into this ASAP. // accelerator::initializeDevice(); From a518700982a1b08c0cd19e163d81b4b4054f7808 Mon Sep 17 00:00:00 2001 From: Michael Tupek Date: Fri, 30 Aug 2024 08:43:36 -0700 Subject: [PATCH 05/22] Fix include. --- src/serac/infrastructure/initialize.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/serac/infrastructure/initialize.cpp b/src/serac/infrastructure/initialize.cpp index 9f4622b80..4613cfea5 100644 --- a/src/serac/infrastructure/initialize.cpp +++ b/src/serac/infrastructure/initialize.cpp @@ -21,6 +21,10 @@ #include "mfem.hpp" +#ifdef SERAC_USE_PETSC +#include "petsc.h" +#endif + namespace serac { std::pair getMPIInfo(MPI_Comm comm) From 8808849c862aeb87138349f6e015b19b4a8ee0f4 Mon Sep 17 00:00:00 2001 From: Alex Tyler Chapman Date: Wed, 4 Sep 2024 17:40:28 -0700 Subject: [PATCH 06/22] Update nonlinear solve test to run specific solvers/ prec one after another --- scripts/llnl/run_benchmarks.py | 98 ------------------- src/serac/physics/benchmarks/CMakeLists.txt | 25 +++-- .../benchmark_solid_nonlinear_solve.cpp} | 55 +++++------ src/serac/physics/tests/CMakeLists.txt | 1 - 4 files changed, 42 insertions(+), 137 deletions(-) delete mode 100755 scripts/llnl/run_benchmarks.py rename src/serac/physics/{tests/solid_nonlinear_solve.cpp => benchmarks/benchmark_solid_nonlinear_solve.cpp} (87%) diff --git a/scripts/llnl/run_benchmarks.py b/scripts/llnl/run_benchmarks.py deleted file mode 100755 index f6e2ad5bb..000000000 --- a/scripts/llnl/run_benchmarks.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/bin/sh -"exec" "python3" "-u" "-B" "$0" "$@" - -# Copyright (c) 2019-2024, Lawrence Livermore National Security, LLC and -# other Serac Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: (BSD-3-Clause) - -""" - file: run_benchmarks.py - - description: - Run benchmarks and update shared (or any desired) location with new Caliper files - -""" - -from common_build_functions import * - -from argparse import ArgumentParser - -import os - - -def parse_args(): - "Parses args from command line" - parser = ArgumentParser() - parser.add_argument("-e", "--extra-cmake-options", - dest="extra_cmake_options", - default="", - help="Extra cmake options to add to the cmake configure line. Note '-DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF' is always appended.") - parser.add_argument("-hc", "--host-config", - dest="host_config", - default="", - required=True, - help="Specific host-config file to build") - parser.add_argument("-sd", "--spot-directory", - dest="spot_dir", - default=get_shared_spot_dir(), - help="Where to put all resulting caliper files to use for SPOT analysis (defaults to a shared location)") - parser.add_argument("-t", "--timestamp", - dest="timestamp", - default=get_timestamp(), - help="Set timestamp manually for debugging") - ############### - # parse args - ############### - args, extra_args = parser.parse_known_args() - args = vars(args) - return args - - -def main(): - args = parse_args() - - # Args - cmake_options = args["extra_cmake_options"] + " -DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF" - host_config = args["host_config"] - spot_dir = args["spot_dir"] - timestamp = args["timestamp"] - - # Vars - repo_dir = get_repo_dir() - test_root = get_build_and_test_root(repo_dir, timestamp) - - # Build Serac - os.chdir(repo_dir) - os.makedirs(test_root, exist_ok=True) - build_and_test_host_config(test_root, host_config, True, cmake_options, True, True, "") - - # Go to build location - build_dir="" - dirs = glob.glob(pjoin(test_root, "*")) - for dir in dirs: - if os.path.exists(dir) and "build-" in dir: - build_dir=dir - os.chdir(build_dir) - - # Run benchmarks - # TODO Instead of running each benchmark individually, create a custom target - # shell_exec("make benchmarks -j") - shell_exec("tests/profiling", print_output=True) - shell_exec("benchmarks/benchmark_thermal", print_output=True) - shell_exec("benchmarks/benchmark_functional", print_output=True) - - # Move resulting .cali files to specified directory - os.makedirs(spot_dir, exist_ok=True) - cali_files = glob.glob(pjoin(build_dir, "*.cali")) - for cali_file in cali_files: - if os.path.exists(cali_file): - shutil.move(cali_file, spot_dir) - - print("[View SPOT directory here: https://lc.llnl.gov/spot2/?sf={0}]".format(spot_dir)) - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/serac/physics/benchmarks/CMakeLists.txt b/src/serac/physics/benchmarks/CMakeLists.txt index b5c866a0b..2df52f195 100644 --- a/src/serac/physics/benchmarks/CMakeLists.txt +++ b/src/serac/physics/benchmarks/CMakeLists.txt @@ -6,14 +6,19 @@ set(benchmark_depends serac_physics) -blt_add_executable(NAME benchmark_thermal - SOURCES benchmark_thermal.cpp - DEPENDS_ON ${benchmark_depends} - OUTPUT_DIR ${PROJECT_BINARY_DIR}/benchmarks - FOLDER serac/benchmarks) +set(benchmark_targets + benchmark_functional + benchmark_solid_nonlinear_solve + benchmark_thermal + ) + +# Create executable for each benchmark +foreach(benchmark ${benchmark_targets}) + blt_add_executable(NAME ${benchmark} + SOURCES ${benchmark}.cpp + DEPENDS_ON ${benchmark_depends} + OUTPUT_DIR ${PROJECT_BINARY_DIR}/benchmarks + FOLDER serac/benchmarks + ) +endforeach() -blt_add_executable(NAME benchmark_functional - SOURCES benchmark_functional.cpp - DEPENDS_ON ${benchmark_depends} - OUTPUT_DIR ${PROJECT_BINARY_DIR}/benchmarks - FOLDER serac/benchmarks) diff --git a/src/serac/physics/tests/solid_nonlinear_solve.cpp b/src/serac/physics/benchmarks/benchmark_solid_nonlinear_solve.cpp similarity index 87% rename from src/serac/physics/tests/solid_nonlinear_solve.cpp rename to src/serac/physics/benchmarks/benchmark_solid_nonlinear_solve.cpp index 4a46f9a69..0e0b5f846 100644 --- a/src/serac/physics/tests/solid_nonlinear_solve.cpp +++ b/src/serac/physics/benchmarks/benchmark_solid_nonlinear_solve.cpp @@ -14,7 +14,6 @@ #include #include "axom/slic/core/SimpleLogger.hpp" -#include #include "mfem.hpp" #include "serac/physics/materials/liquid_crystal_elastomer.hpp" @@ -48,10 +47,7 @@ enum NonlinSolve TRUSTREGION }; -NonlinSolve nonlinSolve = NonlinSolve::TRUSTREGION; -Prec prec = Prec::JACOBI; - -auto get_opts(int max_iters, double abs_tol = 1e-9) +auto get_opts(NonlinSolve nonlinSolve, Prec prec, int max_iters, double abs_tol = 1e-9) { serac::NonlinearSolverOptions nonlinear_options{ .nonlin_solver = NonlinearSolver::TrustRegion, @@ -160,7 +156,7 @@ auto get_opts(int max_iters, double abs_tol = 1e-9) #include #include -void functional_solid_test_euler() +void functional_solid_test_euler(NonlinSolve nonlinSolve, Prec prec) { // initialize serac axom::sidre::DataStore datastore; @@ -192,7 +188,7 @@ void functional_solid_test_euler() // solid mechanics using seracSolidType = serac::SolidMechanics>; - auto [nonlinear_options, linear_options] = get_opts(3 * Nx * Ny * Nz, 1e-9); + auto [nonlinear_options, linear_options] = get_opts(nonlinSolve, prec, 3 * Nx * Ny * Nz, 1e-9); auto seracSolid = std::make_unique( nonlinear_options, linear_options, serac::solid_mechanics::default_quasistatic_options, @@ -224,7 +220,7 @@ void functional_solid_test_euler() } } -void functional_solid_test_nonlinear_buckle(double loadMagnitude) +void functional_solid_test_nonlinear_buckle(NonlinSolve nonlinSolve, Prec prec, double loadMagnitude) { // initialize serac axom::sidre::DataStore datastore; @@ -258,7 +254,7 @@ void functional_solid_test_nonlinear_buckle(double loadMagnitude) // solid mechanics using seracSolidType = serac::SolidMechanics>; - auto [nonlinear_options, linear_options] = get_opts(3 * Nx * Ny * Nz, 1e-11); + auto [nonlinear_options, linear_options] = get_opts(nonlinSolve, prec, 3 * Nx * Ny * Nz, 1e-11); auto seracSolid = std::make_unique( nonlinear_options, linear_options, serac::solid_mechanics::default_quasistatic_options, @@ -280,30 +276,33 @@ void functional_solid_test_nonlinear_buckle(double loadMagnitude) seracSolid->outputStateToDisk("paraview_buckle_easy"); } -TEST(SolidMechanics, nonlinear_solve_buckle_easy) { functional_solid_test_nonlinear_buckle(5e-10); } -// TEST(SolidMechanics, nonlinear_solve_buckle_medium) { functional_solid_test_nonlinear_buckle(4e-4); } -// TEST(SolidMechanics, nonlinear_solve_buckle_hard) { functional_solid_test_nonlinear_buckle(3e-2); } -// TEST(SolidMechanics, nonlinear_solve_euler) { functional_solid_test_euler(); } - int main(int argc, char* argv[]) { - axom::CLI::App app{"Nonlinear problems"}; - // app.add_option("-p", mesh_path, "Path to mesh files")->check(axom::CLI::ExistingDirectory); - app.add_option("--nonlinear-solver", nonlinSolve, "Nonlinear solver", true); - app.add_option("--preconditioner", prec, "Preconditioner", true); - app.set_help_flag("--help"); - app.allow_extras()->parse(argc, argv); - - ::testing::InitGoogleTest(&argc, argv); - serac::initialize(argc, argv); SERAC_SET_METADATA("test", "solid_nonlinear_solve"); - SERAC_SET_METADATA("nonlinear solver", std::to_string(nonlinSolve)); - SERAC_SET_METADATA("preconditioner", std::to_string(prec)); - int result = RUN_ALL_TESTS(); - serac::exitGracefully(result); + // TODO add commandline options. if there set, run with those options only. otherwise, run the options below: + + SERAC_MARK_BEGIN("Jacobi Preconditioner"); + functional_solid_test_nonlinear_buckle(NonlinSolve::NEWTON, Prec::JACOBI, 5e-10); + SERAC_MARK_END("Jacobi Preconditioner"); + + SERAC_MARK_BEGIN("Multigrid Preconditioner"); + functional_solid_test_nonlinear_buckle(NonlinSolve::NEWTON, Prec::MULTIGRID, 5e-10); + SERAC_MARK_END("Multigrid Preconditioner"); + + SERAC_MARK_BEGIN("Petsc Multigrid Preconditioner"); + functional_solid_test_nonlinear_buckle(NonlinSolve::NEWTON, Prec::PETSC_MULTIGRID, 5e-10); + SERAC_MARK_END("Petsc Multigrid Preconditioner"); + + // functional_solid_test_nonlinear_buckle(4e-4); + // functional_solid_test_nonlinear_buckle(3e-2); + // functional_solid_test_euler(); + + serac::profiling::finalize(); + + MPI_Finalize(); - return result; + return 0; } diff --git a/src/serac/physics/tests/CMakeLists.txt b/src/serac/physics/tests/CMakeLists.txt index 69bbdf5c3..698378cd1 100644 --- a/src/serac/physics/tests/CMakeLists.txt +++ b/src/serac/physics/tests/CMakeLists.txt @@ -39,7 +39,6 @@ set(physics_parallel_test_sources dynamic_thermal_adjoint.cpp solid_reaction_adjoint.cpp thermal_nonlinear_solve.cpp - solid_nonlinear_solve.cpp ) blt_list_append(TO physics_parallel_test_sources ELEMENTS contact_patch.cpp contact_beam.cpp IF TRIBOL_FOUND) From b5e4315701072cd38dd9838f2299b09778fe9ea2 Mon Sep 17 00:00:00 2001 From: Alex Tyler Chapman Date: Wed, 4 Sep 2024 18:07:59 -0700 Subject: [PATCH 07/22] attempt to make a benchmark custom cmake target --- src/serac/physics/benchmarks/CMakeLists.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/serac/physics/benchmarks/CMakeLists.txt b/src/serac/physics/benchmarks/CMakeLists.txt index 2df52f195..44ea42cb4 100644 --- a/src/serac/physics/benchmarks/CMakeLists.txt +++ b/src/serac/physics/benchmarks/CMakeLists.txt @@ -22,3 +22,14 @@ foreach(benchmark ${benchmark_targets}) ) endforeach() +# Create custom target to run each benchmark sequentially +add_custom_target(benchmarks DEPENDS ${benchmark_targets}) + +foreach(benchmark ${benchmark_targets}) + add_custom_command(TARGET benchmarks + COMMAND benchmarks/${benchmark} + DEPENDS ${benchmark} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + COMMENT "Running ${benchmark} benchmark" + ) +endforeach() From 65c078986473c70f814addb827b9982f60c976e1 Mon Sep 17 00:00:00 2001 From: Alex Tyler Chapman Date: Wed, 4 Sep 2024 19:22:32 -0700 Subject: [PATCH 08/22] use blt_add_benchmark, run across variable task counts --- src/serac/physics/benchmarks/CMakeLists.txt | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/serac/physics/benchmarks/CMakeLists.txt b/src/serac/physics/benchmarks/CMakeLists.txt index 44ea42cb4..f8e943734 100644 --- a/src/serac/physics/benchmarks/CMakeLists.txt +++ b/src/serac/physics/benchmarks/CMakeLists.txt @@ -20,16 +20,12 @@ foreach(benchmark ${benchmark_targets}) OUTPUT_DIR ${PROJECT_BINARY_DIR}/benchmarks FOLDER serac/benchmarks ) -endforeach() - -# Create custom target to run each benchmark sequentially -add_custom_target(benchmarks DEPENDS ${benchmark_targets}) -foreach(benchmark ${benchmark_targets}) - add_custom_command(TARGET benchmarks - COMMAND benchmarks/${benchmark} - DEPENDS ${benchmark} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR} - COMMENT "Running ${benchmark} benchmark" - ) + # Add benchmarks with various task counts + foreach(task_count 1 4 16) + blt_add_benchmark(NAME ${benchmark}_${task_count}_task_count + COMMAND ${benchmark} + NUM_MPI_TASKS ${task_count} + ) + endforeach() endforeach() From c5caac6edc08316134ab5466572f689c562dfd5c Mon Sep 17 00:00:00 2001 From: Alex Tyler Chapman Date: Mon, 9 Sep 2024 10:53:15 -0700 Subject: [PATCH 09/22] update shared file name --- scripts/llnl/common_build_functions.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/llnl/common_build_functions.py b/scripts/llnl/common_build_functions.py index ce3e1c512..927aacf17 100755 --- a/scripts/llnl/common_build_functions.py +++ b/scripts/llnl/common_build_functions.py @@ -677,7 +677,7 @@ def get_shared_libs_dir(): def get_shared_spot_dir(): - return pjoin(get_shared_base_dir(), "spot") + return pjoin(get_shared_base_dir(), "califiles") def get_uberenv_path(): @@ -713,4 +713,3 @@ def convertSecondsToReadableTime(seconds): m, s = divmod(seconds, 60) h, m = divmod(m, 60) return "%d:%02d:%02d" % (h, m, s) - From ea1f9817c3c623453676c0fdf3f8f99d6c3e11ef Mon Sep 17 00:00:00 2001 From: Alex Tyler Chapman Date: Mon, 9 Sep 2024 13:57:51 -0700 Subject: [PATCH 10/22] bring back cmd line options, optionally enter them with string --- .../benchmark_solid_nonlinear_solve.cpp | 128 ++++++++++++++---- 1 file changed, 101 insertions(+), 27 deletions(-) diff --git a/src/serac/physics/benchmarks/benchmark_solid_nonlinear_solve.cpp b/src/serac/physics/benchmarks/benchmark_solid_nonlinear_solve.cpp index 0e0b5f846..96d61146d 100644 --- a/src/serac/physics/benchmarks/benchmark_solid_nonlinear_solve.cpp +++ b/src/serac/physics/benchmarks/benchmark_solid_nonlinear_solve.cpp @@ -7,11 +7,12 @@ #include "serac/physics/solid_mechanics.hpp" #include "serac/physics/solid_mechanics_contact.hpp" -#include +#include #include +#include +#include #include #include -#include #include "axom/slic/core/SimpleLogger.hpp" #include "mfem.hpp" @@ -27,25 +28,62 @@ using namespace serac; -std::string mesh_path = "."; - -enum Prec +enum class Prec { JACOBI, STRUMPACK, CHOLESKI, LU, MULTIGRID, - PETSC_MULTIGRID + PETSC_MULTIGRID, + NONE }; -enum NonlinSolve +enum class NonlinSolve { NEWTON, LINESEARCH, CRITICALPOINT, - TRUSTREGION + TRUSTREGION, + NONE +}; + +std::string mesh_path = "."; +// string->value matching for optionally entering options as string in command line +std::map precMap = { + {"jacobi", Prec::JACOBI}, + {"strumpack", Prec::STRUMPACK}, + {"choleski", Prec::CHOLESKI}, + {"lu", Prec::LU}, + {"multigrid", Prec::MULTIGRID}, + {"petsc_multigrid", Prec::PETSC_MULTIGRID}, + {"none", Prec::NONE}, }; +std::map nonlinSolveMap = { + {"newton", NonlinSolve::NEWTON}, + {"linesearch", NonlinSolve::LINESEARCH}, + {"critialpoint", NonlinSolve::CRITICALPOINT}, + {"trustregion", NonlinSolve::TRUSTREGION}, + {"none", NonlinSolve::NONE}, +}; + +const std::string precToString(Prec prec) { + for (const auto& p : precMap) { + if (p.second == prec) { + return p.first; + } + } + return "unknown"; +} + +const std::string nonlinSolveToString(NonlinSolve nonlinSolve) { + for (const auto& n : nonlinSolveMap) { + if (n.second == nonlinSolve) { + return n.first; + } + } + return "unknown"; +} auto get_opts(NonlinSolve nonlinSolve, Prec prec, int max_iters, double abs_tol = 1e-9) { @@ -104,6 +142,10 @@ auto get_opts(NonlinSolve nonlinSolve, Prec prec, int max_iters, double abs_tol nonlinear_options.nonlin_solver = NonlinearSolver::TrustRegion; break; } + case NonlinSolve::NONE: + default: { + SLIC_ERROR_ROOT("error, invalid nonlinear solver specified"); + } } switch (prec) { @@ -145,6 +187,7 @@ auto get_opts(NonlinSolve nonlinSolve, Prec prec, int max_iters, double abs_tol linear_options.petsc_preconditioner = PetscPCType::HMG; break; } + case Prec::NONE: default: { SLIC_ERROR_ROOT("error, invalid preconditioner specified"); } @@ -153,9 +196,6 @@ auto get_opts(NonlinSolve nonlinSolve, Prec prec, int max_iters, double abs_tol return std::make_pair(nonlinear_options, linear_options); } -#include -#include - void functional_solid_test_euler(NonlinSolve nonlinSolve, Prec prec) { // initialize serac @@ -280,29 +320,63 @@ int main(int argc, char* argv[]) { serac::initialize(argc, argv); - SERAC_SET_METADATA("test", "solid_nonlinear_solve"); + SERAC_MARK_FUNCTION; - // TODO add commandline options. if there set, run with those options only. otherwise, run the options below: + NonlinSolve nonlinSolve = NonlinSolve::NONE; + Prec prec = Prec::NONE; + + axom::CLI::App app{"Solid Nonlinear Solve Benchmark"}; + //app.add_option("-m,--mesh", mesh_path, "Path to mesh files")->check(axom::CLI::ExistingDirectory); + app.add_option("-n,--nonlinear-solver", nonlinSolve, "Nonlinear solver") + ->transform(axom::CLI::CheckedTransformer(nonlinSolveMap, axom::CLI::ignore_case)); + app.add_option("-p,--preconditioner", prec, "Preconditioner") + ->transform(axom::CLI::CheckedTransformer(precMap, axom::CLI::ignore_case)); + + // Parse the arguments and check if they are good + try { + CLI11_PARSE(app, argc, argv); + } catch (const axom::CLI::ParseError& e) { + serac::logger::flush(); + if (e.get_name() == "CallForHelp") { + auto msg = app.help(); + SLIC_INFO_ROOT(msg); + serac::exitGracefully(); + } else { + auto err_msg = axom::CLI::FailureMessage::simple(&app, e); + SLIC_ERROR_ROOT(err_msg); + } + } + + SERAC_SET_METADATA("test", "solid_nonlinear_solve"); - SERAC_MARK_BEGIN("Jacobi Preconditioner"); - functional_solid_test_nonlinear_buckle(NonlinSolve::NEWTON, Prec::JACOBI, 5e-10); - SERAC_MARK_END("Jacobi Preconditioner"); + // If you do not specify preconditioner and nonlinear solver, run the following pre-selected options + if (nonlinSolve == NonlinSolve::NONE && prec == Prec::NONE) + { + SERAC_MARK_BEGIN("Jacobi Preconditioner"); + functional_solid_test_nonlinear_buckle(NonlinSolve::NEWTON, Prec::JACOBI, 5e-10); + SERAC_MARK_END("Jacobi Preconditioner"); - SERAC_MARK_BEGIN("Multigrid Preconditioner"); - functional_solid_test_nonlinear_buckle(NonlinSolve::NEWTON, Prec::MULTIGRID, 5e-10); - SERAC_MARK_END("Multigrid Preconditioner"); + SERAC_MARK_BEGIN("Multigrid Preconditioner"); + functional_solid_test_nonlinear_buckle(NonlinSolve::NEWTON, Prec::MULTIGRID, 5e-10); + SERAC_MARK_END("Multigrid Preconditioner"); - SERAC_MARK_BEGIN("Petsc Multigrid Preconditioner"); - functional_solid_test_nonlinear_buckle(NonlinSolve::NEWTON, Prec::PETSC_MULTIGRID, 5e-10); - SERAC_MARK_END("Petsc Multigrid Preconditioner"); + SERAC_MARK_BEGIN("Petsc Multigrid Preconditioner"); + functional_solid_test_nonlinear_buckle(NonlinSolve::NEWTON, Prec::PETSC_MULTIGRID, 5e-10); + SERAC_MARK_END("Petsc Multigrid Preconditioner"); + } + else + { + SERAC_SET_METADATA("nonlinear solver", nonlinSolveToString(nonlinSolve)); + SERAC_SET_METADATA("preconditioner", precToString(prec)); + + SERAC_MARK_BEGIN("Custom Preconditioner"); + functional_solid_test_nonlinear_buckle(nonlinSolve, prec, 5e-10); + SERAC_MARK_END("Custom Preconditioner"); + } // functional_solid_test_nonlinear_buckle(4e-4); // functional_solid_test_nonlinear_buckle(3e-2); // functional_solid_test_euler(); - serac::profiling::finalize(); - - MPI_Finalize(); - - return 0; + serac::exitGracefully(0); } From 232fe8890c73fe9963f249f971b8063dc5482584 Mon Sep 17 00:00:00 2001 From: Alex Tyler Chapman Date: Mon, 9 Sep 2024 14:00:51 -0700 Subject: [PATCH 11/22] update blt with benchmark fix --- cmake/blt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/blt b/cmake/blt index b7314a86e..66378271f 160000 --- a/cmake/blt +++ b/cmake/blt @@ -1 +1 @@ -Subproject commit b7314a86e9fc78baf2682ad55509bbc8acd4bce6 +Subproject commit 66378271f605de0b628ab93196fd41f5c7b59452 From 36c82a25681287c3665546afc408c3628017fe59 Mon Sep 17 00:00:00 2001 From: Alex Tyler Chapman Date: Mon, 9 Sep 2024 15:14:14 -0700 Subject: [PATCH 12/22] run_benchmarks documentation --- src/docs/sphinx/dev_guide/profiling.rst | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/docs/sphinx/dev_guide/profiling.rst b/src/docs/sphinx/dev_guide/profiling.rst index 81f5a51fa..dd0b66475 100644 --- a/src/docs/sphinx/dev_guide/profiling.rst +++ b/src/docs/sphinx/dev_guide/profiling.rst @@ -30,8 +30,8 @@ Introduction to SPOT LLNL for vizualizing performance data. SPOT is an external tool and does not need to be linked into Serac. -Build Instructions ------------------- +TPL Build Instructions +---------------------- To use Adiak and Caliper with Serac, install the ``profiling`` variant of ``serac`` with Spack, i.e., ``serac+profiling``. Note that these libraries are pre-built as @@ -109,3 +109,18 @@ of this file, use `cali-query `_), and open the directory containing one or more ``.cali`` files. For more information, watch this recorded `tutorial `_. +Benchmarking Serac +------------------ + +To run all of Serac's benchmarks in one command, first make sure you're configured Serac with benchmarking enabled. +Benchmarking is not enabled by default. Then, run the custom CMake target ``run_benchmarks``. + +.. code-block:: bash + + ./config-build.py -hc -DENABLE_BENCHMARKS=ON + cd + make -j + make run_benchmarks + +This will run all of Serac's benchmarks multiple times with varying MPI task counts, and generate a ``.cali`` file for +each benchmark run. Note that the ``-j`` option will not work. From 7417e4b1df3eaed1b5d279eb1d568cf2a190dec8 Mon Sep 17 00:00:00 2001 From: Alex Tyler Chapman Date: Mon, 9 Sep 2024 15:46:37 -0700 Subject: [PATCH 13/22] working --- src/docs/sphinx/dev_guide/profiling.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/docs/sphinx/dev_guide/profiling.rst b/src/docs/sphinx/dev_guide/profiling.rst index dd0b66475..f8ee81e76 100644 --- a/src/docs/sphinx/dev_guide/profiling.rst +++ b/src/docs/sphinx/dev_guide/profiling.rst @@ -112,7 +112,7 @@ To view this data with SPOT, open a browser, navigate to the SPOT server (e.g. ` Benchmarking Serac ------------------ -To run all of Serac's benchmarks in one command, first make sure you're configured Serac with benchmarking enabled. +To run all of Serac's benchmarks in one command, first make sure Serac is configured with benchmarking enabled. Benchmarking is not enabled by default. Then, run the custom CMake target ``run_benchmarks``. .. code-block:: bash @@ -123,4 +123,4 @@ Benchmarking is not enabled by default. Then, run the custom CMake target ``run_ make run_benchmarks This will run all of Serac's benchmarks multiple times with varying MPI task counts, and generate a ``.cali`` file for -each benchmark run. Note that the ``-j`` option will not work. +each benchmark run. From 75aa4089eeae63c0d143e7d120dfefe67969f640 Mon Sep 17 00:00:00 2001 From: Alex Tyler Chapman Date: Mon, 9 Sep 2024 17:19:37 -0700 Subject: [PATCH 14/22] style --- .../benchmark_solid_nonlinear_solve.cpp | 61 +++++++++---------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/src/serac/physics/benchmarks/benchmark_solid_nonlinear_solve.cpp b/src/serac/physics/benchmarks/benchmark_solid_nonlinear_solve.cpp index 96d61146d..0c1d63e9d 100644 --- a/src/serac/physics/benchmarks/benchmark_solid_nonlinear_solve.cpp +++ b/src/serac/physics/benchmarks/benchmark_solid_nonlinear_solve.cpp @@ -51,38 +51,36 @@ enum class NonlinSolve std::string mesh_path = "."; // string->value matching for optionally entering options as string in command line std::map precMap = { - {"jacobi", Prec::JACOBI}, - {"strumpack", Prec::STRUMPACK}, - {"choleski", Prec::CHOLESKI}, - {"lu", Prec::LU}, - {"multigrid", Prec::MULTIGRID}, - {"petsc_multigrid", Prec::PETSC_MULTIGRID}, - {"none", Prec::NONE}, + {"jacobi", Prec::JACOBI}, {"strumpack", Prec::STRUMPACK}, {"choleski", Prec::CHOLESKI}, + {"lu", Prec::LU}, {"multigrid", Prec::MULTIGRID}, {"petsc_multigrid", Prec::PETSC_MULTIGRID}, + {"none", Prec::NONE}, }; std::map nonlinSolveMap = { - {"newton", NonlinSolve::NEWTON}, - {"linesearch", NonlinSolve::LINESEARCH}, - {"critialpoint", NonlinSolve::CRITICALPOINT}, - {"trustregion", NonlinSolve::TRUSTREGION}, - {"none", NonlinSolve::NONE}, + {"newton", NonlinSolve::NEWTON}, + {"linesearch", NonlinSolve::LINESEARCH}, + {"critialpoint", NonlinSolve::CRITICALPOINT}, + {"trustregion", NonlinSolve::TRUSTREGION}, + {"none", NonlinSolve::NONE}, }; -const std::string precToString(Prec prec) { - for (const auto& p : precMap) { - if (p.second == prec) { - return p.first; - } +const std::string precToString(Prec prec) +{ + for (const auto& p : precMap) { + if (p.second == prec) { + return p.first; } - return "unknown"; + } + return "unknown"; } -const std::string nonlinSolveToString(NonlinSolve nonlinSolve) { - for (const auto& n : nonlinSolveMap) { - if (n.second == nonlinSolve) { - return n.first; - } +const std::string nonlinSolveToString(NonlinSolve nonlinSolve) +{ + for (const auto& n : nonlinSolveMap) { + if (n.second == nonlinSolve) { + return n.first; } - return "unknown"; + } + return "unknown"; } auto get_opts(NonlinSolve nonlinSolve, Prec prec, int max_iters, double abs_tol = 1e-9) @@ -323,14 +321,14 @@ int main(int argc, char* argv[]) SERAC_MARK_FUNCTION; NonlinSolve nonlinSolve = NonlinSolve::NONE; - Prec prec = Prec::NONE; + Prec prec = Prec::NONE; axom::CLI::App app{"Solid Nonlinear Solve Benchmark"}; - //app.add_option("-m,--mesh", mesh_path, "Path to mesh files")->check(axom::CLI::ExistingDirectory); + // app.add_option("-m,--mesh", mesh_path, "Path to mesh files")->check(axom::CLI::ExistingDirectory); app.add_option("-n,--nonlinear-solver", nonlinSolve, "Nonlinear solver") - ->transform(axom::CLI::CheckedTransformer(nonlinSolveMap, axom::CLI::ignore_case)); + ->transform(axom::CLI::CheckedTransformer(nonlinSolveMap, axom::CLI::ignore_case)); app.add_option("-p,--preconditioner", prec, "Preconditioner") - ->transform(axom::CLI::CheckedTransformer(precMap, axom::CLI::ignore_case)); + ->transform(axom::CLI::CheckedTransformer(precMap, axom::CLI::ignore_case)); // Parse the arguments and check if they are good try { @@ -350,8 +348,7 @@ int main(int argc, char* argv[]) SERAC_SET_METADATA("test", "solid_nonlinear_solve"); // If you do not specify preconditioner and nonlinear solver, run the following pre-selected options - if (nonlinSolve == NonlinSolve::NONE && prec == Prec::NONE) - { + if (nonlinSolve == NonlinSolve::NONE && prec == Prec::NONE) { SERAC_MARK_BEGIN("Jacobi Preconditioner"); functional_solid_test_nonlinear_buckle(NonlinSolve::NEWTON, Prec::JACOBI, 5e-10); SERAC_MARK_END("Jacobi Preconditioner"); @@ -363,9 +360,7 @@ int main(int argc, char* argv[]) SERAC_MARK_BEGIN("Petsc Multigrid Preconditioner"); functional_solid_test_nonlinear_buckle(NonlinSolve::NEWTON, Prec::PETSC_MULTIGRID, 5e-10); SERAC_MARK_END("Petsc Multigrid Preconditioner"); - } - else - { + } else { SERAC_SET_METADATA("nonlinear solver", nonlinSolveToString(nonlinSolve)); SERAC_SET_METADATA("preconditioner", precToString(prec)); From 69024a755c22d6e53d2b6875a9d598344f160ec9 Mon Sep 17 00:00:00 2001 From: Alex Tyler Chapman <100869159+chapman39@users.noreply.github.com> Date: Wed, 11 Sep 2024 12:05:33 -0700 Subject: [PATCH 15/22] Update src/docs/sphinx/dev_guide/profiling.rst Co-authored-by: Chris White --- src/docs/sphinx/dev_guide/profiling.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/docs/sphinx/dev_guide/profiling.rst b/src/docs/sphinx/dev_guide/profiling.rst index f8ee81e76..028207e38 100644 --- a/src/docs/sphinx/dev_guide/profiling.rst +++ b/src/docs/sphinx/dev_guide/profiling.rst @@ -112,8 +112,8 @@ To view this data with SPOT, open a browser, navigate to the SPOT server (e.g. ` Benchmarking Serac ------------------ -To run all of Serac's benchmarks in one command, first make sure Serac is configured with benchmarking enabled. -Benchmarking is not enabled by default. Then, run the custom CMake target ``run_benchmarks``. +To run all of Serac's benchmarks in one command, first make sure Serac is configured +with benchmarking enabled (off by default). Then, run the build target ``run_benchmarks``. .. code-block:: bash From 19ce9e806fdfc9f53c2fbb0c4c7f44f6156b3739 Mon Sep 17 00:00:00 2001 From: Alex Tyler Chapman <100869159+chapman39@users.noreply.github.com> Date: Wed, 11 Sep 2024 12:05:45 -0700 Subject: [PATCH 16/22] Update src/docs/sphinx/dev_guide/profiling.rst Co-authored-by: Chris White --- src/docs/sphinx/dev_guide/profiling.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docs/sphinx/dev_guide/profiling.rst b/src/docs/sphinx/dev_guide/profiling.rst index 028207e38..9d852e230 100644 --- a/src/docs/sphinx/dev_guide/profiling.rst +++ b/src/docs/sphinx/dev_guide/profiling.rst @@ -122,5 +122,5 @@ with benchmarking enabled (off by default). Then, run the build target ``run_ben make -j make run_benchmarks -This will run all of Serac's benchmarks multiple times with varying MPI task counts, and generate a ``.cali`` file for +This will run all of Serac's benchmarks multiple times with varying MPI task counts, and generate a Caliper file for each benchmark run. From f434964458763859edb8bc259cbb7941a00df0d1 Mon Sep 17 00:00:00 2001 From: Alex Tyler Chapman Date: Wed, 11 Sep 2024 12:16:20 -0700 Subject: [PATCH 17/22] prefix benchmarks with physics, remove manual error prefix in slic calls --- src/serac/physics/benchmarks/CMakeLists.txt | 22 +++++++++---------- ...l.cpp => physics_benchmark_functional.cpp} | 0 ...ysics_benchmark_solid_nonlinear_solve.cpp} | 4 ++-- ...rmal.cpp => physics_benchmark_thermal.cpp} | 0 4 files changed, 13 insertions(+), 13 deletions(-) rename src/serac/physics/benchmarks/{benchmark_functional.cpp => physics_benchmark_functional.cpp} (100%) rename src/serac/physics/benchmarks/{benchmark_solid_nonlinear_solve.cpp => physics_benchmark_solid_nonlinear_solve.cpp} (99%) rename src/serac/physics/benchmarks/{benchmark_thermal.cpp => physics_benchmark_thermal.cpp} (100%) diff --git a/src/serac/physics/benchmarks/CMakeLists.txt b/src/serac/physics/benchmarks/CMakeLists.txt index f8e943734..7eb946021 100644 --- a/src/serac/physics/benchmarks/CMakeLists.txt +++ b/src/serac/physics/benchmarks/CMakeLists.txt @@ -4,27 +4,27 @@ # # SPDX-License-Identifier: (BSD-3-Clause) -set(benchmark_depends serac_physics) +set(physics_benchmark_depends serac_physics) -set(benchmark_targets - benchmark_functional - benchmark_solid_nonlinear_solve - benchmark_thermal +set(physics_benchmark_targets + physics_benchmark_functional + physics_benchmark_solid_nonlinear_solve + physics_benchmark_thermal ) # Create executable for each benchmark -foreach(benchmark ${benchmark_targets}) - blt_add_executable(NAME ${benchmark} - SOURCES ${benchmark}.cpp - DEPENDS_ON ${benchmark_depends} +foreach(physics_benchmark ${physics_benchmark_targets}) + blt_add_executable(NAME ${physics_benchmark} + SOURCES ${physics_benchmark}.cpp + DEPENDS_ON ${physics_benchmark_depends} OUTPUT_DIR ${PROJECT_BINARY_DIR}/benchmarks FOLDER serac/benchmarks ) # Add benchmarks with various task counts foreach(task_count 1 4 16) - blt_add_benchmark(NAME ${benchmark}_${task_count}_task_count - COMMAND ${benchmark} + blt_add_benchmark(NAME ${physics_benchmark}_${task_count}_task_count + COMMAND ${physics_benchmark} NUM_MPI_TASKS ${task_count} ) endforeach() diff --git a/src/serac/physics/benchmarks/benchmark_functional.cpp b/src/serac/physics/benchmarks/physics_benchmark_functional.cpp similarity index 100% rename from src/serac/physics/benchmarks/benchmark_functional.cpp rename to src/serac/physics/benchmarks/physics_benchmark_functional.cpp diff --git a/src/serac/physics/benchmarks/benchmark_solid_nonlinear_solve.cpp b/src/serac/physics/benchmarks/physics_benchmark_solid_nonlinear_solve.cpp similarity index 99% rename from src/serac/physics/benchmarks/benchmark_solid_nonlinear_solve.cpp rename to src/serac/physics/benchmarks/physics_benchmark_solid_nonlinear_solve.cpp index 0c1d63e9d..23e98abb6 100644 --- a/src/serac/physics/benchmarks/benchmark_solid_nonlinear_solve.cpp +++ b/src/serac/physics/benchmarks/physics_benchmark_solid_nonlinear_solve.cpp @@ -142,7 +142,7 @@ auto get_opts(NonlinSolve nonlinSolve, Prec prec, int max_iters, double abs_tol } case NonlinSolve::NONE: default: { - SLIC_ERROR_ROOT("error, invalid nonlinear solver specified"); + SLIC_ERROR_ROOT("invalid nonlinear solver specified"); } } @@ -187,7 +187,7 @@ auto get_opts(NonlinSolve nonlinSolve, Prec prec, int max_iters, double abs_tol } case Prec::NONE: default: { - SLIC_ERROR_ROOT("error, invalid preconditioner specified"); + SLIC_ERROR_ROOT("invalid preconditioner specified"); } } diff --git a/src/serac/physics/benchmarks/benchmark_thermal.cpp b/src/serac/physics/benchmarks/physics_benchmark_thermal.cpp similarity index 100% rename from src/serac/physics/benchmarks/benchmark_thermal.cpp rename to src/serac/physics/benchmarks/physics_benchmark_thermal.cpp From 18d1d8ed1eb9678f0150ae1cf183dc13ab712277 Mon Sep 17 00:00:00 2001 From: Alex Tyler Chapman Date: Wed, 11 Sep 2024 12:26:08 -0700 Subject: [PATCH 18/22] Tie benchmark command in with spot --- src/docs/sphinx/dev_guide/profiling.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/docs/sphinx/dev_guide/profiling.rst b/src/docs/sphinx/dev_guide/profiling.rst index 9d852e230..7f5ea83a1 100644 --- a/src/docs/sphinx/dev_guide/profiling.rst +++ b/src/docs/sphinx/dev_guide/profiling.rst @@ -121,6 +121,9 @@ with benchmarking enabled (off by default). Then, run the build target ``run_ben cd make -j make run_benchmarks + find . -name "*.cali" -print0 | xargs -0 mv -t . + pwd This will run all of Serac's benchmarks multiple times with varying MPI task counts, and generate a Caliper file for -each benchmark run. +each benchmark run. The ``find`` command afterwards ensures all Caliper files are moved to the same directory. Now, you +can visualize the results with SPOT, entering the path printed from ``pwd``. From 345be2e5e2aae0a94c230a48a512e1abb450e25e Mon Sep 17 00:00:00 2001 From: Alex Tyler Chapman Date: Mon, 16 Sep 2024 10:47:12 -0700 Subject: [PATCH 19/22] Explain how to "rebaseline" and slightly update docs --- scripts/spack/specs.json | 5 +---- src/docs/sphinx/dev_guide/testing.rst | 22 +++++++++++++++++++--- src/docs/sphinx/quickstart.rst | 20 ++++++++++---------- tests | 2 +- 4 files changed, 31 insertions(+), 18 deletions(-) diff --git a/scripts/spack/specs.json b/scripts/spack/specs.json index 5a941d5d3..6f245df35 100644 --- a/scripts/spack/specs.json +++ b/scripts/spack/specs.json @@ -18,8 +18,5 @@ "clang@14.0.6+devtools+profiling" ], "blueos_3_ppc64le_ib_p9": - [ "clang@10.0.1+devtools+cuda+raja+umpire+profiling~strumpack cuda_arch=70" ], - - "darwin-x86_64": - [ "clang@9.0.0+devtools" ] + [ "clang@10.0.1+devtools+cuda+raja+umpire+profiling~strumpack cuda_arch=70" ] } diff --git a/src/docs/sphinx/dev_guide/testing.rst b/src/docs/sphinx/dev_guide/testing.rst index 1962d9603..232398588 100644 --- a/src/docs/sphinx/dev_guide/testing.rst +++ b/src/docs/sphinx/dev_guide/testing.rst @@ -11,7 +11,7 @@ Testing Serac has two levels of tests, unit and integration. Unit tests are used to test individual components of code, such as a class or function. While integration tests -are for testing the code as a whole. For example, testing the `serac` driver with +are for testing the code as a whole. For example, testing the ``serac`` driver with an input file against blessed answers. Unit Tests @@ -41,7 +41,7 @@ Requirements: >>> socket.gethostname().rstrip('1234567890') >>> exit() - Currently, there are configuration json files for Toss3 and BlueOS which can be + Currently, there are configuration json files for TOSS4 and BlueOS which can be used as reference. #. **Build the code.** @@ -54,7 +54,7 @@ Requirements: # BlueOS $ lalloc 2 ./ats.sh - # Toss3 + # TOSS4 $ salloc -N2 ./ats.sh # Personal Machine (currently runs subset of tests) @@ -72,6 +72,22 @@ Requirements: ATS also outputs both a ``.log`` and ``.log.err`` for each test and checker that is run. +#. **Rebaselining tests (as needed).** + If tolerances to tests need to be updated, first ensure you've generated new tolerances by running the integration + tests like mentioned above. Then, use the ``-b`` option. + + # Single baseline + $ ./ats.sh -b dyn_solve_serial + + # Comma-separated baselines + $ ./ats.sh -b dyn_solve_serial,dyn_solve_parallel + + # All baselines + $ ./ats.sh -b all + + This will update the json files located in the `serac_tests `_ submodule. To + avoid Caliper files from additionally being generated, configure with ``-DENABLE_BENCHMARKS=OFF``. + Installing ATS -------------- diff --git a/src/docs/sphinx/quickstart.rst b/src/docs/sphinx/quickstart.rst index 0b3ffd25a..f9fe3b4fd 100644 --- a/src/docs/sphinx/quickstart.rst +++ b/src/docs/sphinx/quickstart.rst @@ -78,7 +78,7 @@ For example on **Ubuntu 20.04**: python3 scripts/uberenv/uberenv.py --project-json=scripts/spack/devtools.json --spack-env-file=scripts/spack/configs/linux_ubuntu_20/spack.yaml --prefix=../path/to/install Unlike Serac's library dependencies, our developer tools can be built with any compiler because -they are not linked into the serac executable. We recommend GCC 8 because we have tested that they all +they are not linked into the serac executable. We recommend GCC 10 because we have tested that they all build with that compiler. Building Serac's Dependencies via Spack/uberenv @@ -97,7 +97,7 @@ This has been encapsulated using `Uberenv `_. U doing the following: * Pulls a blessed version of Spack locally -* If you are on a known operating system (like TOSS3), we have defined Spack configuration files +* If you are on a known operating system (like TOSS4), we have defined Spack configuration files to keep Spack from building the world * Installs our Spack packages into the local Spack * Simplifies whole dependency build into one command @@ -120,18 +120,18 @@ Serac. .. note:: On LC machines, it is good practice to do the build step in parallel on a compute node. - Here is an example command: ``salloc -ppdebug -N1-1 python3 scripts/uberenv/uberenv.py`` + Here is an example command: ``salloc -ppdebug -N1 python3 scripts/uberenv/uberenv.py`` Unless otherwise specified Spack will default to a compiler. This is generally not a good idea when developing large codes. To specify which compiler to use add the compiler specification to the ``--spec`` Uberenv -command line option. On TOSS3, we recommend and have tested ``--spec=%clang@10.0.0``. More compiler specs +command line option. On TOSS4, we recommend and have tested ``--spec=%clang@14.0.6``. More compiler specs can be found in the Spack compiler files in our repository: ``scripts/spack/configs//spack.yaml``. We currently regularly test the following Spack configuration files: -* Linux Ubuntu 20.04 (via Windows WSL 2) -* TOSS 3 (On Ruby at LC) +* Linux Ubuntu 22.04 (via Azure) +* TOSS4 (On Ruby at LC) * BlueOS (On Lassen at LC) To install Serac on a new platform, it is a good idea to start with a known Spack environments file, or ``spack.yaml`` file, @@ -144,7 +144,7 @@ in the `Spack docs `_ If you do not have a ``spack.yaml`` already, you can leave off that command line option from ``uberenv`` and Spack will generate a new one for you. Uberenv will copy it where you ran your uberenv command for future use. .. note:: - A newer vesion of cmake (>=3.20) and llvm (>=14) may be required. + A newer version of cmake (>=3.20) and llvm (>=14) may be required. Some helpful uberenv options include : @@ -152,11 +152,11 @@ Some helpful uberenv options include : * ``--spec=" build_type=Debug"`` (build the MFEM and Hypre libraries with debug symbols) * ``--spec=+profiling`` (build the Adiak and Caliper libraries) * ``--spec=+devtools`` (also build the devtools with one command) -* ``--spec=%clang@10.0.0`` (build with a specific compiler as defined in the ``spack.yaml`` file) +* ``--spec=%clang@14.0.6`` (build with a specific compiler as defined in the ``spack.yaml`` file) * ``--spack-env-file=`` (use specific Spack environment configuration file) * ``--prefix=`` (required, build and install the dependencies in a particular location) - this *must be outside* of your local Serac repository -The modifiers to the Spack specification ``spec`` can be chained together, e.g. ``--spec='%clang@10.0.0+devtools build_type=Debug'``. +The modifiers to the Spack specification ``spec`` can be chained together, e.g. ``--spec='%clang@14.0.6+devtools build_type=Debug'``. If you already have a Spack instance from another project that you would like to reuse, you can do so by changing the uberenv command as follows: @@ -169,7 +169,7 @@ Building Serac's Dependencies by Hand ------------------------------------- To build Serac's dependencies by hand, use of a ``host-config`` CMake configuration file is -stongly encouraged. A good place to start is by copying an existing host config in the +strongly encouraged. A good place to start is by copying an existing host config in the ``host-config`` directory and modifying it according to your system setup. .. _build-label: diff --git a/tests b/tests index 3dd50b00c..ba34e958e 160000 --- a/tests +++ b/tests @@ -1 +1 @@ -Subproject commit 3dd50b00cc8f5b5dad308c75a9fb7102272aa481 +Subproject commit ba34e958ec41f4a8d5d7197056cdb4acb38dd734 From 9d0bfe747fc4f45c6005e0d677a8e780746264ab Mon Sep 17 00:00:00 2001 From: Alex Tyler Chapman Date: Mon, 16 Sep 2024 10:58:21 -0700 Subject: [PATCH 20/22] syntax --- src/docs/sphinx/dev_guide/testing.rst | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/docs/sphinx/dev_guide/testing.rst b/src/docs/sphinx/dev_guide/testing.rst index 232398588..ebf958719 100644 --- a/src/docs/sphinx/dev_guide/testing.rst +++ b/src/docs/sphinx/dev_guide/testing.rst @@ -73,20 +73,20 @@ Requirements: ATS also outputs both a ``.log`` and ``.log.err`` for each test and checker that is run. #. **Rebaselining tests (as needed).** - If tolerances to tests need to be updated, first ensure you've generated new tolerances by running the integration - tests like mentioned above. Then, use the ``-b`` option. + If tolerances to tests need to be updated, first ensure you've generated new tolerances by running the integration + tests like mentioned above. Then, use the ``-b`` option:: - # Single baseline - $ ./ats.sh -b dyn_solve_serial + # Single baseline + $ ./ats.sh -b dyn_solve_serial - # Comma-separated baselines - $ ./ats.sh -b dyn_solve_serial,dyn_solve_parallel + # Comma-separated baselines + $ ./ats.sh -b dyn_solve_serial,dyn_solve_parallel - # All baselines - $ ./ats.sh -b all + # All baselines + $ ./ats.sh -b all - This will update the json files located in the `serac_tests `_ submodule. To - avoid Caliper files from additionally being generated, configure with ``-DENABLE_BENCHMARKS=OFF``. + This will update the json files located in the `serac_tests `_ submodule. To + avoid Caliper files from additionally being generated, configure with ``-DENABLE_BENCHMARKS=OFF``. Installing ATS From 97e39daaea326ffd37b84b836606097f13c9f77b Mon Sep 17 00:00:00 2001 From: Alex Tyler Chapman Date: Mon, 16 Sep 2024 11:05:58 -0700 Subject: [PATCH 21/22] wording --- src/docs/sphinx/dev_guide/testing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docs/sphinx/dev_guide/testing.rst b/src/docs/sphinx/dev_guide/testing.rst index ebf958719..7718c1c0a 100644 --- a/src/docs/sphinx/dev_guide/testing.rst +++ b/src/docs/sphinx/dev_guide/testing.rst @@ -72,7 +72,7 @@ Requirements: ATS also outputs both a ``.log`` and ``.log.err`` for each test and checker that is run. -#. **Rebaselining tests (as needed).** +#. **Rebaseline tests (as needed).** If tolerances to tests need to be updated, first ensure you've generated new tolerances by running the integration tests like mentioned above. Then, use the ``-b`` option:: From 1fac6576ccda915ab7246001a7a24c6c96ca6638 Mon Sep 17 00:00:00 2001 From: Chris White Date: Tue, 17 Sep 2024 11:12:18 -0700 Subject: [PATCH 22/22] change git strategy to start from a clean slate each time --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8000ea266..cd581f3bc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,6 +6,7 @@ variables: #TODO: uncomment this when everyone has service user access across all the machines #LLNL_SERVICE_USER: asmith + GIT_STRATEGY: clone GIT_SUBMODULE_STRATEGY: recursive PROJECT_ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID} BUILD_ROOT: ${CI_PROJECT_DIR}