From fb1737084fd19a31fb518f59ff5a94f4390885e5 Mon Sep 17 00:00:00 2001 From: Ben Ryan Date: Tue, 31 Oct 2023 13:19:58 -0600 Subject: [PATCH 1/8] Convert numpy arrays to ASCII strings (#956) * Convert numpy arrays to ASCII strings * format * Dont update submodule versions * Switch to fix inside parthenon * comment * upgold * hash issue * The original hash was right? --------- Co-authored-by: Jonah Miller --- CMakeLists.txt | 4 ++-- src/outputs/parthenon_hdf5.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 33c19c34256d..fe9b1f7e3d86 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,9 +62,9 @@ include(cmake/Format.cmake) include(cmake/Lint.cmake) # regression test reference data -set(REGRESSION_GOLD_STANDARD_VER 19 CACHE STRING "Version of gold standard to download and use") +set(REGRESSION_GOLD_STANDARD_VER 20 CACHE STRING "Version of gold standard to download and use") set(REGRESSION_GOLD_STANDARD_HASH - "SHA512=e1d1a06b9cf9b761d42d0b6b241056ac75658db90138b6b867b1ca7ead4308af4f980285af092b40aee1dbbfb68b4e8cb15efcc9b83d7930c18bf992ae95c729" + "SHA512=e5e421f3c0be01e4708965542bb8b1b79b5c96de97091e46972e375c7616588d026a9a8e29226d9c7ef75346bc859fd9af72acdc7e95e0d783b5ef29aa4630b1" CACHE STRING "Hash of default gold standard file to download") option(REGRESSION_GOLD_STANDARD_SYNC "Automatically sync gold standard files." ON) diff --git a/src/outputs/parthenon_hdf5.cpp b/src/outputs/parthenon_hdf5.cpp index 95b89b93b627..004054d8421a 100644 --- a/src/outputs/parthenon_hdf5.cpp +++ b/src/outputs/parthenon_hdf5.cpp @@ -820,7 +820,7 @@ HDF5GetAttributeInfo(hid_t location, const std::string &name, H5A &attr) { // template specializations for std::string and bool void HDF5WriteAttribute(const std::string &name, const std::string &value, hid_t location) { - HDF5WriteAttribute(name, value.size(), value.c_str(), location); + HDF5WriteAttribute(name, value.c_str(), location); } template <> From 285739417a0c75eb0652c235d649f6ee492974a0 Mon Sep 17 00:00:00 2001 From: Ben Ryan Date: Wed, 1 Nov 2023 10:22:42 -0600 Subject: [PATCH 2/8] Allow leading whitespace in input parameters (#965) * Fixed the bug * CHANGELOG, cleanup * Add whitespace characters to test input to confirm bug is fixed --------- Co-authored-by: Jonah Miller Co-authored-by: Philipp Grete --- CHANGELOG.md | 1 + src/parameter_input.cpp | 2 +- .../advection_performance/parthinput.advection_performance | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 978bccbf19c0..181a58b2698c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - [[PR 868]](https://github.com/parthenon-hpc-lab/parthenon/pull/868) Add block-local face, edge, and nodal fields and allow for packing ### Changed (changing behavior/API/variables/...) +- [[PR 965]](https://github.com/parthenon-hpc-lab/parthenon/pull/965) Allow leading whitespace in input parameters - [[PR 926]](https://github.com/parthenon-hpc-lab/parthenon/pull/926) Internal refinement op registration - [[PR 897]](https://github.com/parthenon-hpc-lab/parthenon/pull/897) Deflate compression filter is not called any more if compression is soft disabled - [[PR 896]](https://github.com/parthenon-hpc-lab/parthenon/pull/896) Update Kokkos integration to support installed version. Use `serial` (flat MPI) host parallelization by default (instead of OpenMP) diff --git a/src/parameter_input.cpp b/src/parameter_input.cpp index b9fd9546f2b6..45012e0912bd 100644 --- a/src/parameter_input.cpp +++ b/src/parameter_input.cpp @@ -315,7 +315,7 @@ bool ParameterInput::ParseLine(InputBlock *pib, std::string line, std::string &n name.assign(line, first_char, len); last_char = name.find_last_not_of(" "); name.erase(last_char + 1, std::string::npos); - line.erase(0, len + 1); + line.erase(0, equal_char + 1); } cont_char = line.find_first_of("&"); // find "&" continuation character diff --git a/tst/regression/test_suites/advection_performance/parthinput.advection_performance b/tst/regression/test_suites/advection_performance/parthinput.advection_performance index 4dfe731cf497..5aea8cbee069 100644 --- a/tst/regression/test_suites/advection_performance/parthinput.advection_performance +++ b/tst/regression/test_suites/advection_performance/parthinput.advection_performance @@ -49,8 +49,8 @@ nx2 = 64 nx3 = 64 -tlim = 1.0 -nlim = 20 + tlim = 1.0 # Test that leading whitespace is correctly sanitized + nlim = 20 # Test that leading tab is correctly sanitized integrator = rk2 perf_cycle_offset = 2 From 06657a4e3358b46f238d16aff9f3e6391e6586a8 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Mon, 6 Nov 2023 12:47:59 -0700 Subject: [PATCH 3/8] Add hook for UserWorkBeforeLoop --- doc/sphinx/src/interface/state.rst | 7 +++++++ example/advection/advection_package.cpp | 10 ++++++++++ example/advection/advection_package.hpp | 1 + src/application_input.hpp | 1 + src/driver/driver.cpp | 10 ++++++++++ src/interface/state_descriptor.hpp | 13 ++++++++++--- src/mesh/mesh.cpp | 2 ++ src/mesh/mesh.hpp | 1 + 8 files changed, 42 insertions(+), 3 deletions(-) diff --git a/doc/sphinx/src/interface/state.rst b/doc/sphinx/src/interface/state.rst index b2a91fa8e4cc..0841e8c5a1c7 100644 --- a/doc/sphinx/src/interface/state.rst +++ b/doc/sphinx/src/interface/state.rst @@ -112,6 +112,13 @@ several useful features and functions. deletgates to the ``std::function`` member ``PostStepDiagnosticsMesh`` if set (defaults to ``nullptr`` an therefore a no-op) to print diagnostics after the time-integration advance +- ``void UserWorkBeforeLoopMesh(Mesh *, ParameterInput *pin, SimTime + &tm)`` performs a per-package, mesh-wide calculation after the mesh + has been generated, and problem generators called, but before any + time evolution. This work is done both on first initialization and + on restart. If you would like to avoid doing the work upon restart, + you can check for the const ``is_restart`` member field of the ``Mesh`` + object. The reasoning for providing ``FillDerived*`` and ``EstimateTimestep*`` function pointers appropriate for usage with both ``MeshData`` and diff --git a/example/advection/advection_package.cpp b/example/advection/advection_package.cpp index f618de1758c3..5af3783716f8 100644 --- a/example/advection/advection_package.cpp +++ b/example/advection/advection_package.cpp @@ -13,12 +13,14 @@ #include #include +#include #include #include #include #include #include +#include #include #include "advection_package.hpp" @@ -219,6 +221,14 @@ std::shared_ptr Initialize(ParameterInput *pin) { return pkg; } +void AdvectionGreetings(Mesh *pmesh, ParameterInput *pin, SimTime &tm) { + if (GLobals::my_rank == 0) { + std::cout << "Hello from the advection package in the advection example!\n" + << "This run is a restart: " << pmesh->is_restart + << std::endl; + } +} + AmrTag CheckRefinement(MeshBlockData *rc) { // refine on advected, for example. could also be a derived quantity auto pmb = rc->GetBlockPointer(); diff --git a/example/advection/advection_package.hpp b/example/advection/advection_package.hpp index 7b529308f793..24314db0c2d8 100644 --- a/example/advection/advection_package.hpp +++ b/example/advection/advection_package.hpp @@ -21,6 +21,7 @@ namespace advection_package { using namespace parthenon::package::prelude; std::shared_ptr Initialize(ParameterInput *pin); + void AdvectionGreetings(Mesh *pmes, ParameterInput *pin, SimTime &tm); AmrTag CheckRefinement(MeshBlockData *rc); void PreFill(MeshBlockData *rc); void SquareIt(MeshBlockData *rc); diff --git a/src/application_input.hpp b/src/application_input.hpp index 543cb7a5100f..a9ec96c1551e 100644 --- a/src/application_input.hpp +++ b/src/application_input.hpp @@ -48,6 +48,7 @@ struct ApplicationInput { PostStepDiagnosticsInLoop = nullptr; std::function UserWorkAfterLoop = nullptr; + std::function UserWorkBeforeLoop = nullptr; BValFunc boundary_conditions[BOUNDARY_NFACES] = {nullptr}; SBValFunc swarm_boundary_conditions[BOUNDARY_NFACES] = {nullptr}; diff --git a/src/driver/driver.cpp b/src/driver/driver.cpp index 84371eeaeb36..8314335f3610 100644 --- a/src/driver/driver.cpp +++ b/src/driver/driver.cpp @@ -75,6 +75,16 @@ DriverStatus EvolutionDriver::Execute() { // Defaults must be set across all ranks DumpInputParameters(); + // Before loop do work + // App input version + if (app_input->UserWorkBeforeLoop != nullptr) { + app_input->UserWorkBeforeLoop(pmesh, pinput, tm); + } + // packages version + for (auto &[name, pkg] : pmesh->packages.AllPackages()) { + pkg->UserWorkBeforeLoop(pmesh, pinput, tm); + } + Kokkos::Profiling::pushRegion("Driver_Main"); while (tm.KeepGoing()) { if (Globals::my_rank == 0) OutputCycleDiagnostics(); diff --git a/src/interface/state_descriptor.hpp b/src/interface/state_descriptor.hpp index 5bb37ea83b60..b52ab744cbf6 100644 --- a/src/interface/state_descriptor.hpp +++ b/src/interface/state_descriptor.hpp @@ -217,17 +217,18 @@ class StateDescriptor { // one can pass in a reference to a SparsePool or arguments that match one of the // SparsePool constructors template - bool AddSparsePool(Args &&...args) { + bool AddSparsePool(Args &&... args) { return AddSparsePoolImpl(SparsePool(std::forward(args)...)); } template - bool AddSparsePool(const std::string &base_name, const Metadata &m_in, Args &&...args) { + bool AddSparsePool(const std::string &base_name, const Metadata &m_in, + Args &&... args) { Metadata m = m_in; // so we can modify it if (!m.IsSet(GetMetadataFlag())) m.Set(GetMetadataFlag()); return AddSparsePoolImpl(SparsePool(base_name, m, std::forward(args)...)); } template - bool AddSparsePool(const Metadata &m_in, Args &&...args) { + bool AddSparsePool(const Metadata &m_in, Args &&... args) { return AddSparsePool(T::name(), m_in, std::forward(args)...); } @@ -406,6 +407,10 @@ class StateDescriptor { if (InitNewlyAllocatedVarsBlock != nullptr) return InitNewlyAllocatedVarsBlock(rc); } + void UserWorkBeforeLoop(Mesh *pmesh, ParameterInput *pin, SimTime &tm) const { + if (UserWorkBeforeLoopMesh != nullptr) return UserWorkBeforeLoopMesh(pmesh, pin, tm); + } + std::vector> amr_criteria; std::function *rc)> PreCommFillDerivedBlock = nullptr; @@ -416,6 +421,8 @@ class StateDescriptor { std::function *rc)> PostFillDerivedMesh = nullptr; std::function *rc)> FillDerivedBlock = nullptr; std::function *rc)> FillDerivedMesh = nullptr; + std::function UserWorkBeforeLoopMesh = + nullptr; std::function *rc)> PreStepDiagnosticsMesh = nullptr; diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index ed81e3e4e219..414ea69fadf5 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -64,6 +64,7 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages, int mesh_test) : // public members: modified(true), + is_restart(false), // aggregate initialization of RegionSize struct: mesh_size({pin->GetReal("parthenon/mesh", "x1min"), pin->GetReal("parthenon/mesh", "x2min"), @@ -485,6 +486,7 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, RestartReader &rr, // aggregate initialization of RegionSize struct: // (will be overwritten by memcpy from restart file, in this case) modified(true), + is_restart(true), // aggregate initialization of RegionSize struct: mesh_size({pin->GetReal("parthenon/mesh", "x1min"), pin->GetReal("parthenon/mesh", "x2min"), diff --git a/src/mesh/mesh.hpp b/src/mesh/mesh.hpp index ad0d1a85539c..15ac8666a50c 100644 --- a/src/mesh/mesh.hpp +++ b/src/mesh/mesh.hpp @@ -94,6 +94,7 @@ class Mesh { // data bool modified; + const bool is_restart; RegionSize mesh_size; BoundaryFlag mesh_bcs[BOUNDARY_NFACES]; const int ndim; // number of dimensions From d75771263651705404b48bb7c0ff273219a160a0 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Mon, 6 Nov 2023 12:55:55 -0700 Subject: [PATCH 4/8] formatting --- example/advection/advection_package.cpp | 7 +++---- example/advection/advection_package.hpp | 2 +- src/interface/state_descriptor.hpp | 7 +++---- src/mesh/mesh.cpp | 6 ++---- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/example/advection/advection_package.cpp b/example/advection/advection_package.cpp index 5af3783716f8..1aef47b79908 100644 --- a/example/advection/advection_package.cpp +++ b/example/advection/advection_package.cpp @@ -221,11 +221,10 @@ std::shared_ptr Initialize(ParameterInput *pin) { return pkg; } -void AdvectionGreetings(Mesh *pmesh, ParameterInput *pin, SimTime &tm) { - if (GLobals::my_rank == 0) { +void AdvectionGreetings(Mesh *pmesh, ParameterInput *pin, parthenon::SimTime &tm) { + if (parthenon::Globals::my_rank == 0) { std::cout << "Hello from the advection package in the advection example!\n" - << "This run is a restart: " << pmesh->is_restart - << std::endl; + << "This run is a restart: " << pmesh->is_restart << std::endl; } } diff --git a/example/advection/advection_package.hpp b/example/advection/advection_package.hpp index 24314db0c2d8..1413aa2049fe 100644 --- a/example/advection/advection_package.hpp +++ b/example/advection/advection_package.hpp @@ -21,7 +21,7 @@ namespace advection_package { using namespace parthenon::package::prelude; std::shared_ptr Initialize(ParameterInput *pin); - void AdvectionGreetings(Mesh *pmes, ParameterInput *pin, SimTime &tm); +void AdvectionGreetings(Mesh *pmes, ParameterInput *pin, parthenon::SimTime &tm); AmrTag CheckRefinement(MeshBlockData *rc); void PreFill(MeshBlockData *rc); void SquareIt(MeshBlockData *rc); diff --git a/src/interface/state_descriptor.hpp b/src/interface/state_descriptor.hpp index b52ab744cbf6..41a5ee2d3d4e 100644 --- a/src/interface/state_descriptor.hpp +++ b/src/interface/state_descriptor.hpp @@ -217,18 +217,17 @@ class StateDescriptor { // one can pass in a reference to a SparsePool or arguments that match one of the // SparsePool constructors template - bool AddSparsePool(Args &&... args) { + bool AddSparsePool(Args &&...args) { return AddSparsePoolImpl(SparsePool(std::forward(args)...)); } template - bool AddSparsePool(const std::string &base_name, const Metadata &m_in, - Args &&... args) { + bool AddSparsePool(const std::string &base_name, const Metadata &m_in, Args &&...args) { Metadata m = m_in; // so we can modify it if (!m.IsSet(GetMetadataFlag())) m.Set(GetMetadataFlag()); return AddSparsePoolImpl(SparsePool(base_name, m, std::forward(args)...)); } template - bool AddSparsePool(const Metadata &m_in, Args &&... args) { + bool AddSparsePool(const Metadata &m_in, Args &&...args) { return AddSparsePool(T::name(), m_in, std::forward(args)...); } diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index 414ea69fadf5..958e2ffa5d77 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -63,8 +63,7 @@ namespace parthenon { Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages, int mesh_test) : // public members: - modified(true), - is_restart(false), + modified(true), is_restart(false), // aggregate initialization of RegionSize struct: mesh_size({pin->GetReal("parthenon/mesh", "x1min"), pin->GetReal("parthenon/mesh", "x2min"), @@ -485,8 +484,7 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, RestartReader &rr, : // public members: // aggregate initialization of RegionSize struct: // (will be overwritten by memcpy from restart file, in this case) - modified(true), - is_restart(true), + modified(true), is_restart(true), // aggregate initialization of RegionSize struct: mesh_size({pin->GetReal("parthenon/mesh", "x1min"), pin->GetReal("parthenon/mesh", "x2min"), From 281994a47186f2f5624e87e24b84cbfcfd937c55 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Mon, 6 Nov 2023 12:56:36 -0700 Subject: [PATCH 5/8] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 181a58b2698c..fb06582f3087 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Current develop ### Added (new features/APIs/variables/...) +- [[PR 971]](https://github.com/parthenon-hpc-lab/parthenon/pull/971) Add UserWorkBeforeLoop - [[PR 907]](https://github.com/parthenon-hpc-lab/parthenon/pull/907) PEP1: Allow subclassing StateDescriptor - [[PR 932]](https://github.com/parthenon-hpc-lab/parthenon/pull/932) Add GetOrAddFlag to metadata - [[PR 931]](https://github.com/parthenon-hpc-lab/parthenon/pull/931) Allow SparsePacks with subsets of blocks From 1387606c6eea13a8995d456ba772016903af0089 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Mon, 6 Nov 2023 13:25:22 -0700 Subject: [PATCH 6/8] oops forgot to use hook --- example/advection/advection_package.cpp | 6 ++++-- example/advection/advection_package.hpp | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/example/advection/advection_package.cpp b/example/advection/advection_package.cpp index 1aef47b79908..49200d0da87c 100644 --- a/example/advection/advection_package.cpp +++ b/example/advection/advection_package.cpp @@ -1,5 +1,5 @@ //======================================================================================== -// (C) (or copyright) 2020-2021. Triad National Security, LLC. All rights reserved. +// (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved. // // This program was produced under U.S. Government contract 89233218CNA000001 for Los // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC @@ -217,6 +217,7 @@ std::shared_ptr Initialize(ParameterInput *pin) { } pkg->CheckRefinementBlock = CheckRefinement; pkg->EstimateTimestepBlock = EstimateTimestepBlock; + pkg->UserWorkBeforeLoopMesh = AdvectionGreetings; return pkg; } @@ -224,7 +225,8 @@ std::shared_ptr Initialize(ParameterInput *pin) { void AdvectionGreetings(Mesh *pmesh, ParameterInput *pin, parthenon::SimTime &tm) { if (parthenon::Globals::my_rank == 0) { std::cout << "Hello from the advection package in the advection example!\n" - << "This run is a restart: " << pmesh->is_restart << std::endl; + << "This run is a restart: " << pmesh->is_restart << "\n" + << std::endl; } } diff --git a/example/advection/advection_package.hpp b/example/advection/advection_package.hpp index 1413aa2049fe..f1ac63471c16 100644 --- a/example/advection/advection_package.hpp +++ b/example/advection/advection_package.hpp @@ -1,5 +1,5 @@ //======================================================================================== -// (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved. +// (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved. // // This program was produced under U.S. Government contract 89233218CNA000001 for Los // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC From 1e42f76f9e3576d064168b4de9d8469b6b89e7c8 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Mon, 6 Nov 2023 14:00:51 -0700 Subject: [PATCH 7/8] typo --- example/advection/advection_package.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/advection/advection_package.hpp b/example/advection/advection_package.hpp index f1ac63471c16..85d1d6371e5a 100644 --- a/example/advection/advection_package.hpp +++ b/example/advection/advection_package.hpp @@ -21,7 +21,7 @@ namespace advection_package { using namespace parthenon::package::prelude; std::shared_ptr Initialize(ParameterInput *pin); -void AdvectionGreetings(Mesh *pmes, ParameterInput *pin, parthenon::SimTime &tm); +void AdvectionGreetings(Mesh *pmesh, ParameterInput *pin, parthenon::SimTime &tm); AmrTag CheckRefinement(MeshBlockData *rc); void PreFill(MeshBlockData *rc); void SquareIt(MeshBlockData *rc); From 291c6bbccf03d872612590b1db67e4cf4b5a5dc9 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Tue, 7 Nov 2023 09:16:55 -0700 Subject: [PATCH 8/8] add profiling --- src/driver/driver.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/driver/driver.cpp b/src/driver/driver.cpp index 8314335f3610..03acf820e536 100644 --- a/src/driver/driver.cpp +++ b/src/driver/driver.cpp @@ -77,13 +77,16 @@ DriverStatus EvolutionDriver::Execute() { // Before loop do work // App input version + Kokkos::Profiling::pushRegion("Driver_UserWorkBeforeLoop"); if (app_input->UserWorkBeforeLoop != nullptr) { app_input->UserWorkBeforeLoop(pmesh, pinput, tm); } + // packages version for (auto &[name, pkg] : pmesh->packages.AllPackages()) { pkg->UserWorkBeforeLoop(pmesh, pinput, tm); } + Kokkos::Profiling::popRegion(); // Driver_UserWorkBeforeLoop Kokkos::Profiling::pushRegion("Driver_Main"); while (tm.KeepGoing()) {