Merge branch 'develop' into lroberts36/add-multi-grid

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Added (new features/APIs/variables/...)
 - [[PR 911]](https://github.com/parthenon-hpc-lab/parthenon/pull/911) Add infrastructure for geometric multi-grid
+- [[PR 971]](https://github.com/parthenon-hpc-lab/parthenon/pull/971) Add UserWorkBeforeLoop
 - [[PR 907]](https://github.com/parthenon-hpc-lab/parthenon/pull/907) PEP1: Allow subclassing StateDescriptor
 - [[PR 932]](https://github.com/parthenon-hpc-lab/parthenon/pull/932) Add GetOrAddFlag to metadata
 - [[PR 931]](https://github.com/parthenon-hpc-lab/parthenon/pull/931) Allow SparsePacks with subsets of blocks
@@ -19,6 +20,7 @@
 - [[PR 868]](https://github.com/parthenon-hpc-lab/parthenon/pull/868) Add block-local face, edge, and nodal fields and allow for packing
 
 ### Changed (changing behavior/API/variables/...)
+- [[PR 965]](https://github.com/parthenon-hpc-lab/parthenon/pull/965) Allow leading whitespace in input parameters
 - [[PR 926]](https://github.com/parthenon-hpc-lab/parthenon/pull/926) Internal refinement op registration
 - [[PR 897]](https://github.com/parthenon-hpc-lab/parthenon/pull/897) Deflate compression filter is not called any more if compression is soft disabled
 - [[PR 896]](https://github.com/parthenon-hpc-lab/parthenon/pull/896) Update Kokkos integration to support installed version. Use `serial` (flat MPI) host parallelization by default (instead of OpenMP)

CMakeLists.txt

@@ -62,9 +62,9 @@ include(cmake/Format.cmake)
 include(cmake/Lint.cmake)
 
 # regression test reference data
-set(REGRESSION_GOLD_STANDARD_VER 19 CACHE STRING "Version of gold standard to download and use")
+set(REGRESSION_GOLD_STANDARD_VER 20 CACHE STRING "Version of gold standard to download and use")
 set(REGRESSION_GOLD_STANDARD_HASH
-  "SHA512=e1d1a06b9cf9b761d42d0b6b241056ac75658db90138b6b867b1ca7ead4308af4f980285af092b40aee1dbbfb68b4e8cb15efcc9b83d7930c18bf992ae95c729"
+  "SHA512=e5e421f3c0be01e4708965542bb8b1b79b5c96de97091e46972e375c7616588d026a9a8e29226d9c7ef75346bc859fd9af72acdc7e95e0d783b5ef29aa4630b1"
   CACHE STRING "Hash of default gold standard file to download")
 option(REGRESSION_GOLD_STANDARD_SYNC "Automatically sync gold standard files." ON)
 

doc/sphinx/src/interface/state.rst

@@ -112,6 +112,13 @@ several useful features and functions.
   deletgates to the ``std::function`` member ``PostStepDiagnosticsMesh``
   if set (defaults to ``nullptr`` an therefore a no-op) to print
   diagnostics after the time-integration advance
+- ``void UserWorkBeforeLoopMesh(Mesh *, ParameterInput *pin, SimTime
+  &tm)`` performs a per-package, mesh-wide calculation after the mesh
+  has been generated, and problem generators called, but before any
+  time evolution. This work is done both on first initialization and
+  on restart. If you would like to avoid doing the work upon restart,
+  you can check for the const ``is_restart`` member field of the ``Mesh``
+  object.
 
 The reasoning for providing ``FillDerived*`` and ``EstimateTimestep*``
 function pointers appropriate for usage with both ``MeshData`` and

example/advection/advection_package.cpp

@@ -1,5 +1,5 @@
 //========================================================================================
-// (C) (or copyright) 2020-2021. Triad National Security, LLC. All rights reserved.
+// (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001 for Los
 // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
@@ -13,12 +13,14 @@
 
 #include <algorithm>
 #include <cmath>
+#include <iostream>
 #include <limits>
 #include <memory>
 #include <string>
 #include <vector>
 
 #include <coordinates/coordinates.hpp>
+#include <globals.hpp>
 #include <parthenon/package.hpp>
 
 #include "advection_package.hpp"
@@ -215,10 +217,19 @@ std::shared_ptr<StateDescriptor> Initialize(ParameterInput *pin) {
   }
   pkg->CheckRefinementBlock = CheckRefinement;
   pkg->EstimateTimestepBlock = EstimateTimestepBlock;
+  pkg->UserWorkBeforeLoopMesh = AdvectionGreetings;
 
   return pkg;
 }
 
+void AdvectionGreetings(Mesh *pmesh, ParameterInput *pin, parthenon::SimTime &tm) {
+  if (parthenon::Globals::my_rank == 0) {
+    std::cout << "Hello from the advection package in the advection example!\n"
+              << "This run is a restart: " << pmesh->is_restart << "\n"
+              << std::endl;
+  }
+}
+
 AmrTag CheckRefinement(MeshBlockData<Real> *rc) {
   // refine on advected, for example.  could also be a derived quantity
   auto pmb = rc->GetBlockPointer();

example/advection/advection_package.hpp

@@ -1,5 +1,5 @@
 //========================================================================================
-// (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved.
+// (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001 for Los
 // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
@@ -21,6 +21,7 @@ namespace advection_package {
 using namespace parthenon::package::prelude;
 
 std::shared_ptr<StateDescriptor> Initialize(ParameterInput *pin);
+void AdvectionGreetings(Mesh *pmesh, ParameterInput *pin, parthenon::SimTime &tm);
 AmrTag CheckRefinement(MeshBlockData<Real> *rc);
 void PreFill(MeshBlockData<Real> *rc);
 void SquareIt(MeshBlockData<Real> *rc);

src/application_input.hpp

@@ -48,6 +48,7 @@ struct ApplicationInput {
       PostStepDiagnosticsInLoop = nullptr;
 
   std::function<void(Mesh *, ParameterInput *, SimTime &)> UserWorkAfterLoop = nullptr;
+  std::function<void(Mesh *, ParameterInput *, SimTime &)> UserWorkBeforeLoop = nullptr;
   BValFunc boundary_conditions[BOUNDARY_NFACES] = {nullptr};
   SBValFunc swarm_boundary_conditions[BOUNDARY_NFACES] = {nullptr};
 

src/driver/driver.cpp

@@ -75,6 +75,19 @@ DriverStatus EvolutionDriver::Execute() {
   // Defaults must be set across all ranks
   DumpInputParameters();
 
+  // Before loop do work
+  // App input version
+  Kokkos::Profiling::pushRegion("Driver_UserWorkBeforeLoop");
+  if (app_input->UserWorkBeforeLoop != nullptr) {
+    app_input->UserWorkBeforeLoop(pmesh, pinput, tm);
+  }
+
+  // packages version
+  for (auto &[name, pkg] : pmesh->packages.AllPackages()) {
+    pkg->UserWorkBeforeLoop(pmesh, pinput, tm);
+  }
+  Kokkos::Profiling::popRegion(); // Driver_UserWorkBeforeLoop
+
   Kokkos::Profiling::pushRegion("Driver_Main");
   while (tm.KeepGoing()) {
     if (Globals::my_rank == 0) OutputCycleDiagnostics();

src/interface/state_descriptor.hpp

@@ -406,6 +406,10 @@ class StateDescriptor {
     if (InitNewlyAllocatedVarsBlock != nullptr) return InitNewlyAllocatedVarsBlock(rc);
   }
 
+  void UserWorkBeforeLoop(Mesh *pmesh, ParameterInput *pin, SimTime &tm) const {
+    if (UserWorkBeforeLoopMesh != nullptr) return UserWorkBeforeLoopMesh(pmesh, pin, tm);
+  }
+
   std::vector<std::shared_ptr<AMRCriteria>> amr_criteria;
 
   std::function<void(MeshBlockData<Real> *rc)> PreCommFillDerivedBlock = nullptr;
@@ -416,6 +420,8 @@ class StateDescriptor {
   std::function<void(MeshData<Real> *rc)> PostFillDerivedMesh = nullptr;
   std::function<void(MeshBlockData<Real> *rc)> FillDerivedBlock = nullptr;
   std::function<void(MeshData<Real> *rc)> FillDerivedMesh = nullptr;
+  std::function<void(Mesh *, ParameterInput *, SimTime &)> UserWorkBeforeLoopMesh =
+      nullptr;
 
   std::function<void(SimTime const &simtime, MeshData<Real> *rc)> PreStepDiagnosticsMesh =
       nullptr;

src/mesh/mesh.cpp

@@ -63,7 +63,7 @@ namespace parthenon {
 Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages,
            int mesh_test)
     : // public members:
-      modified(true),
+      modified(true), is_restart(false),
       // aggregate initialization of RegionSize struct:
       mesh_size({pin->GetReal("parthenon/mesh", "x1min"),
                  pin->GetReal("parthenon/mesh", "x2min"),
@@ -436,7 +436,7 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, RestartReader &rr,
     : // public members:
       // aggregate initialization of RegionSize struct:
       // (will be overwritten by memcpy from restart file, in this case)
-      modified(true),
+      modified(true), is_restart(true),
       // aggregate initialization of RegionSize struct:
       mesh_size({pin->GetReal("parthenon/mesh", "x1min"),
                  pin->GetReal("parthenon/mesh", "x2min"),

src/mesh/mesh.hpp

@@ -101,6 +101,7 @@ class Mesh {
 
   // data
   bool modified;
+  const bool is_restart;
   RegionSize mesh_size;
   RegionSize base_block_size;
   BoundaryFlag mesh_bcs[BOUNDARY_NFACES];

src/outputs/parthenon_hdf5.cpp

@@ -820,7 +820,7 @@ HDF5GetAttributeInfo(hid_t location, const std::string &name, H5A &attr) {
 // template specializations for std::string and bool
 void HDF5WriteAttribute(const std::string &name, const std::string &value,
                         hid_t location) {
-  HDF5WriteAttribute(name, value.size(), value.c_str(), location);
+  HDF5WriteAttribute(name, value.c_str(), location);
 }
 
 template <>

src/parameter_input.cpp

@@ -315,7 +315,7 @@ bool ParameterInput::ParseLine(InputBlock *pib, std::string line, std::string &n
     name.assign(line, first_char, len);
     last_char = name.find_last_not_of(" ");
     name.erase(last_char + 1, std::string::npos);
-    line.erase(0, len + 1);
+    line.erase(0, equal_char + 1);
   }
 
   cont_char = line.find_first_of("&"); // find "&" continuation character

tst/regression/test_suites/advection_performance/parthinput.advection_performance

@@ -49,8 +49,8 @@ nx2 = 64
 nx3 = 64
 
 <parthenon/time>
-tlim = 1.0
-nlim = 20
+ tlim = 1.0 # Test that leading whitespace is correctly sanitized
+	nlim = 20 # Test that leading tab is correctly sanitized
 integrator = rk2
 perf_cycle_offset = 2