From 8475ac89baf019c2d54810825361dcdccc7b6383 Mon Sep 17 00:00:00 2001 From: Ben Prather Date: Fri, 10 Feb 2023 11:19:49 -0700 Subject: [PATCH 01/68] Make StateDescriptor polymorphic as needed for user package subclassing. --- src/interface/state_descriptor.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/interface/state_descriptor.hpp b/src/interface/state_descriptor.hpp index cecb6ba14602..81af5d5aeb26 100644 --- a/src/interface/state_descriptor.hpp +++ b/src/interface/state_descriptor.hpp @@ -129,6 +129,9 @@ class StateDescriptor { // Preferred constructor explicit StateDescriptor(std::string const &label) : label_(label) {} + // Virtual destructor for subclassing + virtual ~StateDescriptor() = default; + static std::shared_ptr CreateResolvedStateDescriptor(Packages_t &packages); From a1b4caebb03ffa1ca6184ea7979b3ec03aafae6c Mon Sep 17 00:00:00 2001 From: Ben Prather Date: Wed, 19 Apr 2023 10:17:31 -0600 Subject: [PATCH 02/68] PEP1: New functions for getting packages/lists by type --- src/interface/packages.hpp | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/interface/packages.hpp b/src/interface/packages.hpp index 443061d8172e..1c2ce247d34d 100644 --- a/src/interface/packages.hpp +++ b/src/interface/packages.hpp @@ -30,10 +30,43 @@ class Packages_t { return packages_.at(name); } + // Templated version for retrieving a package with a particular type + // Allows subclassing 'StateDescriptor' to add user package types to list + template + T* const &Get(const std::string &name) { + return static_cast(packages_.at(name).get()); + } + const Dictionary> &AllPackages() const { return packages_; } + // Returns a sub-Dictionary containing just pointers to packages of type T. + // Dictionary is a *new copy*, and members are bare pointers, not shared_ptr. + template + const std::vector AllPackagesOfType() const { + Dictionary sub_dict; + for (auto package : packages_) { + if (T *cast_package = dynamic_cast(package.second.get())) { + sub_dict[package.first] = cast_package; + } + } + return sub_dict; + } + + // Returns a list of pointers to packages of type T. + // List contains bare pointers, not shared_ptr objects + template + const std::vector ListPackagesOfType() const { + std::vector sub_list; + for (auto package : packages_) { + if (T *cast_package = dynamic_cast(package.second.get())) { + sub_list.append(cast_package); + } + } + return sub_list; + } + private: Dictionary> packages_; }; From 2d2ccd76e7729abb783391d4833a80ad856f4b52 Mon Sep 17 00:00:00 2001 From: Ben Prather Date: Wed, 10 May 2023 14:02:35 -0500 Subject: [PATCH 03/68] PEP1: Fix type issues --- src/interface/packages.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/interface/packages.hpp b/src/interface/packages.hpp index 1c2ce247d34d..32bbc07eab53 100644 --- a/src/interface/packages.hpp +++ b/src/interface/packages.hpp @@ -44,7 +44,7 @@ class Packages_t { // Returns a sub-Dictionary containing just pointers to packages of type T. // Dictionary is a *new copy*, and members are bare pointers, not shared_ptr. template - const std::vector AllPackagesOfType() const { + const Dictionary AllPackagesOfType() const { Dictionary sub_dict; for (auto package : packages_) { if (T *cast_package = dynamic_cast(package.second.get())) { @@ -61,7 +61,7 @@ class Packages_t { std::vector sub_list; for (auto package : packages_) { if (T *cast_package = dynamic_cast(package.second.get())) { - sub_list.append(cast_package); + sub_list.push_back(cast_package); } } return sub_list; From 89c833329e9f94b3c50e2c99383f0a52de7e199f Mon Sep 17 00:00:00 2001 From: Ben Prather Date: Wed, 10 May 2023 18:55:27 -0500 Subject: [PATCH 04/68] PEP1: Return pointer directly from Get<> as one might expect --- src/interface/packages.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/interface/packages.hpp b/src/interface/packages.hpp index 32bbc07eab53..82b15a8adbef 100644 --- a/src/interface/packages.hpp +++ b/src/interface/packages.hpp @@ -26,14 +26,14 @@ class Packages_t { Packages_t() = default; void Add(const std::shared_ptr &package); - std::shared_ptr const &Get(const std::string &name) { + std::shared_ptr const &Get(const std::string &name) const { return packages_.at(name); } // Templated version for retrieving a package with a particular type // Allows subclassing 'StateDescriptor' to add user package types to list template - T* const &Get(const std::string &name) { + T* Get(const std::string &name) const { return static_cast(packages_.at(name).get()); } From 977e60d1bd55f06e25f11a34b616acfd3b039a40 Mon Sep 17 00:00:00 2001 From: Ben Prather Date: Fri, 10 Feb 2023 11:19:49 -0700 Subject: [PATCH 05/68] Make StateDescriptor polymorphic as needed for user package subclassing. --- src/interface/state_descriptor.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/interface/state_descriptor.hpp b/src/interface/state_descriptor.hpp index ac9a93dcb58a..02f75120930c 100644 --- a/src/interface/state_descriptor.hpp +++ b/src/interface/state_descriptor.hpp @@ -102,6 +102,9 @@ class StateDescriptor { // Preferred constructor explicit StateDescriptor(std::string const &label) : label_(label) {} + // Virtual destructor for subclassing + virtual ~StateDescriptor() = default; + static std::shared_ptr CreateResolvedStateDescriptor(Packages_t &packages); From 63bd1fbf94dd77af81cd3cb0c34732b2f8896ce6 Mon Sep 17 00:00:00 2001 From: Ben Prather Date: Wed, 19 Apr 2023 10:17:31 -0600 Subject: [PATCH 06/68] PEP1: New functions for getting packages/lists by type --- src/interface/packages.hpp | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/interface/packages.hpp b/src/interface/packages.hpp index 285a815bf9ea..70966617266c 100644 --- a/src/interface/packages.hpp +++ b/src/interface/packages.hpp @@ -30,11 +30,44 @@ class Packages_t { return packages_.at(name); } + // Templated version for retrieving a package with a particular type + // Allows subclassing 'StateDescriptor' to add user package types to list + template + T* const &Get(const std::string &name) { + return static_cast(packages_.at(name).get()); + } + const Dictionary> &AllPackages() const { return packages_; } Dictionary> &AllPackages() { return packages_; } + // Returns a sub-Dictionary containing just pointers to packages of type T. + // Dictionary is a *new copy*, and members are bare pointers, not shared_ptr. + template + const std::vector AllPackagesOfType() const { + Dictionary sub_dict; + for (auto package : packages_) { + if (T *cast_package = dynamic_cast(package.second.get())) { + sub_dict[package.first] = cast_package; + } + } + return sub_dict; + } + + // Returns a list of pointers to packages of type T. + // List contains bare pointers, not shared_ptr objects + template + const std::vector ListPackagesOfType() const { + std::vector sub_list; + for (auto package : packages_) { + if (T *cast_package = dynamic_cast(package.second.get())) { + sub_list.append(cast_package); + } + } + return sub_list; + } + private: Dictionary> packages_; }; From 0e7a3ba879caa3fadbd44df864794a10a64f382e Mon Sep 17 00:00:00 2001 From: Ben Prather Date: Wed, 12 Jul 2023 16:36:39 -0600 Subject: [PATCH 07/68] Style --- src/interface/packages.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/interface/packages.hpp b/src/interface/packages.hpp index cfd5a71c95d4..92336f5617cb 100644 --- a/src/interface/packages.hpp +++ b/src/interface/packages.hpp @@ -16,6 +16,7 @@ #include #include +#include #include "basic_types.hpp" From 77b1e2ab756c9b95098f254785ba9e3116c84d68 Mon Sep 17 00:00:00 2001 From: par-hermes Date: Thu, 13 Jul 2023 00:13:00 +0000 Subject: [PATCH 08/68] cpp-py-formatter --- src/interface/packages.hpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/interface/packages.hpp b/src/interface/packages.hpp index 92336f5617cb..bd311c821ccd 100644 --- a/src/interface/packages.hpp +++ b/src/interface/packages.hpp @@ -33,9 +33,9 @@ class Packages_t { // Templated version for retrieving a package with a particular type // Allows subclassing 'StateDescriptor' to add user package types to list - template - T* const &Get(const std::string &name) const { - return static_cast(packages_.at(name).get()); + template + T *const &Get(const std::string &name) const { + return static_cast(packages_.at(name).get()); } const Dictionary> &AllPackages() const { @@ -46,10 +46,10 @@ class Packages_t { // Returns a sub-Dictionary containing just pointers to packages of type T. // Dictionary is a *new copy*, and members are bare pointers, not shared_ptr. template - const std::vector AllPackagesOfType() const { - Dictionary sub_dict; + const std::vector AllPackagesOfType() const { + Dictionary sub_dict; for (auto package : packages_) { - if (T *cast_package = dynamic_cast(package.second.get())) { + if (T *cast_package = dynamic_cast(package.second.get())) { sub_dict[package.first] = cast_package; } } @@ -59,10 +59,10 @@ class Packages_t { // Returns a list of pointers to packages of type T. // List contains bare pointers, not shared_ptr objects template - const std::vector ListPackagesOfType() const { - std::vector sub_list; + const std::vector ListPackagesOfType() const { + std::vector sub_list; for (auto package : packages_) { - if (T *cast_package = dynamic_cast(package.second.get())) { + if (T *cast_package = dynamic_cast(package.second.get())) { sub_list.append(cast_package); } } @@ -72,10 +72,10 @@ class Packages_t { // Returns a sub-Dictionary containing just pointers to packages of type T. // Dictionary is a *new copy*, and members are bare pointers, not shared_ptr. template - const Dictionary AllPackagesOfType() const { - Dictionary sub_dict; + const Dictionary AllPackagesOfType() const { + Dictionary sub_dict; for (auto package : packages_) { - if (T *cast_package = dynamic_cast(package.second.get())) { + if (T *cast_package = dynamic_cast(package.second.get())) { sub_dict[package.first] = cast_package; } } From 661ae4595981c652b73a3d9b12e6e53c50ab9991 Mon Sep 17 00:00:00 2001 From: Ben Prather Date: Wed, 9 Aug 2023 15:33:41 -0600 Subject: [PATCH 09/68] Fix the rest of merge duplication --- src/interface/packages.hpp | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/interface/packages.hpp b/src/interface/packages.hpp index bd311c821ccd..f1312d433348 100644 --- a/src/interface/packages.hpp +++ b/src/interface/packages.hpp @@ -69,19 +69,6 @@ class Packages_t { return sub_list; } - // Returns a sub-Dictionary containing just pointers to packages of type T. - // Dictionary is a *new copy*, and members are bare pointers, not shared_ptr. - template - const Dictionary AllPackagesOfType() const { - Dictionary sub_dict; - for (auto package : packages_) { - if (T *cast_package = dynamic_cast(package.second.get())) { - sub_dict[package.first] = cast_package; - } - } - return sub_dict; - } - private: Dictionary> packages_; }; From ce9786a6a6c117edba5081f98dd1ec92d2382d2d Mon Sep 17 00:00:00 2001 From: Ben Prather Date: Thu, 10 Aug 2023 15:36:44 -0600 Subject: [PATCH 10/68] Fix two issues for KHARMA that didn't make it to backport branch --- src/interface/packages.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/interface/packages.hpp b/src/interface/packages.hpp index f1312d433348..a5e12be47944 100644 --- a/src/interface/packages.hpp +++ b/src/interface/packages.hpp @@ -34,7 +34,7 @@ class Packages_t { // Templated version for retrieving a package with a particular type // Allows subclassing 'StateDescriptor' to add user package types to list template - T *const &Get(const std::string &name) const { + T *const Get(const std::string &name) const { return static_cast(packages_.at(name).get()); } @@ -46,7 +46,7 @@ class Packages_t { // Returns a sub-Dictionary containing just pointers to packages of type T. // Dictionary is a *new copy*, and members are bare pointers, not shared_ptr. template - const std::vector AllPackagesOfType() const { + const Dictionary AllPackagesOfType() const { Dictionary sub_dict; for (auto package : packages_) { if (T *cast_package = dynamic_cast(package.second.get())) { From eb4cf775e048020ecd7a9b953cd792ea91b9726a Mon Sep 17 00:00:00 2001 From: Ben Prather Date: Thu, 10 Aug 2023 16:47:11 -0600 Subject: [PATCH 11/68] PEP1: changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d841a9231061..6fccd6473415 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Current develop ### Added (new features/APIs/variables/...) +- [[PR 907]](https://github.com/parthenon-hpc-lab/parthenon/pull/907) PEP1: Allow subclassing StateDescriptor - [[PR 900]](https://github.com/parthenon-hpc-lab/parthenon/pull/900) Add Morton numbers and expand functionality of LogicalLocation - [[PR 902]](https://github.com/parthenon-hpc-lab/parthenon/pull/902) Add ability to output NaNs for de-allocated sparse fields - [[PR 887]](https://github.com/parthenon-hpc-lab/parthenon/pull/887) Add ability to dump more types of params and read them from restarts From f5ddad85540585983c93c6edaaf9ee471837bd59 Mon Sep 17 00:00:00 2001 From: Ben Prather Date: Fri, 11 Aug 2023 13:41:40 -0600 Subject: [PATCH 12/68] Remove useless const --- src/interface/packages.hpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/interface/packages.hpp b/src/interface/packages.hpp index a5e12be47944..65f1d18e5561 100644 --- a/src/interface/packages.hpp +++ b/src/interface/packages.hpp @@ -31,10 +31,9 @@ class Packages_t { return packages_.at(name); } - // Templated version for retrieving a package with a particular type - // Allows subclassing 'StateDescriptor' to add user package types to list + // Retrieve a package pointer, cast to a given type T template - T *const Get(const std::string &name) const { + T* Get(const std::string &name) const { return static_cast(packages_.at(name).get()); } From 608721e91b50872ed29df208681038e3e1447eae Mon Sep 17 00:00:00 2001 From: Ben Prather Date: Mon, 14 Aug 2023 10:04:17 -0600 Subject: [PATCH 13/68] pep1: format --- src/interface/packages.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/interface/packages.hpp b/src/interface/packages.hpp index 65f1d18e5561..1ef6db59f58f 100644 --- a/src/interface/packages.hpp +++ b/src/interface/packages.hpp @@ -33,7 +33,7 @@ class Packages_t { // Retrieve a package pointer, cast to a given type T template - T* Get(const std::string &name) const { + T *Get(const std::string &name) const { return static_cast(packages_.at(name).get()); } From 5b14ea4f24a49a5f727664be4d599e372b24d64e Mon Sep 17 00:00:00 2001 From: Ben Prather Date: Fri, 29 Sep 2023 09:07:09 -0600 Subject: [PATCH 14/68] PEP1: Make StateDescriptor members protected to allow subclasses access --- src/interface/state_descriptor.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/interface/state_descriptor.hpp b/src/interface/state_descriptor.hpp index 06537a87d88a..5bb37ea83b60 100644 --- a/src/interface/state_descriptor.hpp +++ b/src/interface/state_descriptor.hpp @@ -432,7 +432,7 @@ class StateDescriptor { friend std::ostream &operator<<(std::ostream &os, const StateDescriptor &sd); - private: + protected: void InvertControllerMap(); Params params_; From 3f3226eee642b3820d30b80593494ad1e80976e4 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 12 Oct 2023 21:39:26 +0200 Subject: [PATCH 15/68] Add output skeleton for histogram outputs --- src/CMakeLists.txt | 1 + src/outputs/outputs.cpp | 13 ++++++++++++- src/outputs/outputs.hpp | 11 +++++++++++ .../test_suites/output_hdf5/parthinput.advection | 5 +++++ 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 54a226c55383..567b3ec800f7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -172,6 +172,7 @@ add_library(parthenon mesh/meshblock.cpp outputs/ascent.cpp + outputs/histogram.cpp outputs/history.cpp outputs/io_wrapper.cpp outputs/io_wrapper.hpp diff --git a/src/outputs/outputs.cpp b/src/outputs/outputs.cpp index de54d7827c59..f1084d4a360d 100644 --- a/src/outputs/outputs.cpp +++ b/src/outputs/outputs.cpp @@ -198,7 +198,7 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { // set output variable and optional data format string used in formatted writes if ((op.file_type != "hst") && (op.file_type != "rst") && - (op.file_type != "ascent")) { + (op.file_type != "ascent") && (op.file_type != "histogram")) { op.variables = pin->GetOrAddVector(pib->block_name, "variables", std::vector()); // JMM: If the requested var isn't present for a given swarm, @@ -246,6 +246,17 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { pnew_type = new VTKOutput(op); } else if (op.file_type == "ascent") { pnew_type = new AscentOutput(op); + } else if (op.file_type == "histogram") { +#ifdef ENABLE_HDF5 + pnew_type = new HistogramOutput(op); +#else + msg << "### FATAL ERROR in Outputs constructor" << std::endl + << "Executable not configured for HDF5 outputs, but HDF5 file format " + << "is requested in output/restart block '" << op.block_name << "'. " + << "You can disable this block without deleting it by setting a dt < 0." + << std::endl; + PARTHENON_FAIL(msg); +#endif // ifdef ENABLE_HDF5 } else if (is_hdf5_output) { const bool restart = (op.file_type == "rst"); if (restart) { diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index 08c6676da8b4..66cffc56bdb3 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -214,6 +214,17 @@ class PHDF5Output : public OutputType { const IndexRange &kb, std::vector &x, std::vector &y, std::vector &z); }; + +//---------------------------------------------------------------------------------------- +//! \class HistogramOutput +// \brief derived OutputType class for histograms + +class HistogramOutput : public OutputType { + public: + explicit HistogramOutput(const OutputParameters &oparams) : OutputType(oparams) {} + void WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, + const SignalHandler::OutputSignal signal) override; +}; #endif // ifdef ENABLE_HDF5 //---------------------------------------------------------------------------------------- diff --git a/tst/regression/test_suites/output_hdf5/parthinput.advection b/tst/regression/test_suites/output_hdf5/parthinput.advection index e4ba9618c069..42472be2d012 100644 --- a/tst/regression/test_suites/output_hdf5/parthinput.advection +++ b/tst/regression/test_suites/output_hdf5/parthinput.advection @@ -69,3 +69,8 @@ variables = advected, one_minus_advected, & # comments are ok file_type = hst dt = 0.25 + + +file_type = histogram +dt = 0.25 + From 67434b3ab6482b1945833de401dc485fc282e540 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 12 Oct 2023 23:23:07 +0200 Subject: [PATCH 16/68] histogram input processing --- src/outputs/histogram.cpp | 156 ++++++++++++++++++ src/outputs/outputs.cpp | 2 +- src/outputs/outputs.hpp | 5 +- .../output_hdf5/parthinput.advection | 4 + 4 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 src/outputs/histogram.cpp diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp new file mode 100644 index 000000000000..0791f4133a2a --- /dev/null +++ b/src/outputs/histogram.cpp @@ -0,0 +1,156 @@ +//======================================================================================== +// Parthenon performance portable AMR framework +// Copyright(C) 2023 The Parthenon collaboration +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +// (C) (or copyright) 2023. Triad National Security, LLC. All rights reserved. +// +// This program was produced under U.S. Government contract 89233218CNA000001 for Los +// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +// for the U.S. Department of Energy/National Nuclear Security Administration. All rights +// in the program are reserved by Triad National Security, LLC, and the U.S. Department +// of Energy/National Nuclear Security Administration. The Government is granted for +// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +// license in this material to reproduce, prepare derivative works, distribute copies to +// the public, perform publicly and display publicly, and to permit others to do so. +//======================================================================================== +//! \file histogram.cpp +// \brief 1D and 2D histograms + +// options for building +#include "config.hpp" +#include "globals.hpp" +#include "kokkos_abstraction.hpp" +#include "parameter_input.hpp" +#include "utils/error_checking.hpp" +#include +#include +#include + +// Only proceed if HDF5 output enabled +#ifdef ENABLE_HDF5 + +#include +#include +#include +#include +#include +#include +#include + +// Parthenon headers +#include "coordinates/coordinates.hpp" +#include "defs.hpp" +#include "globals.hpp" +#include "interface/variable_state.hpp" +#include "mesh/mesh.hpp" +#include "outputs/output_utils.hpp" +#include "outputs/outputs.hpp" +#include "utils/error_checking.hpp" + +// Ascent headers +#ifdef PARTHENON_ENABLE_ASCENT +#include "ascent.hpp" +#include "conduit_blueprint.hpp" +#include "conduit_relay_io.hpp" +#include "conduit_relay_io_blueprint.hpp" +#endif // ifdef PARTHENON_ENABLE_ASCENT + +namespace parthenon { + +using namespace OutputUtils; + +namespace HistUtil { + +struct Histogram { + int ndim; // 1D or 2D histogram + std::array bin_var_names; // variable(s) for bins + std::array bin_var_components; // components of bin variables (vector) + ParArray2D bin_edges; + std::string val_var_name; // variable name of variable to be binned + int val_var_component; // component of variable to be binned + ParArray2D hist; // resulting histogram + + Histogram(ParameterInput *pin, const std::string & block_name, const std::string & prefix) { + ndim = pin->GetInteger(block_name, prefix + "ndim"); + PARTHENON_REQUIRE_THROWS(ndim == 1 || ndim == 2, "Histogram dim must be '1' or '2'"); + + const auto x_var_name = pin->GetString(block_name, prefix + "x_variable"); + const auto x_var_component = + pin->GetInteger(block_name, prefix + "x_variable_component"); + const auto x_edges = pin->GetVector(block_name, prefix + "x_edges"); + + // would add additional logic to pick it from a pack... + PARTHENON_REQUIRE_THROWS(x_var_component >= 0, + "Negative component indices are not supported"); + // required by binning index function + PARTHENON_REQUIRE_THROWS(std::is_sorted(x_edges.begin(), x_edges.end()), + "Bin edges must be in order."); + + // For 1D profile default initalize y variables + std::string y_var_name = ""; + int y_var_component = -1; + auto y_edges = std::vector(); + // and for 2D profile check if they're explicitly set (not default value) + if (ndim == 2) { + y_var_name = pin->GetString(block_name, prefix + "y_variable"); + y_var_component = pin->GetInteger(block_name, prefix + "y_variable_component"); + y_edges = pin->GetVector(block_name, prefix + "y_edges"); + + // would add additional logic to pick it from a pack... + PARTHENON_REQUIRE_THROWS(y_var_component >= 0, + "Negative component indices are not supported"); + // required by binning index function + PARTHENON_REQUIRE_THROWS(std::is_sorted(y_edges.begin(), y_edges.end()), + "Bin edges must be in order."); + } + + bin_var_names = {x_var_name, y_var_name}; + bin_var_components = {x_var_component, y_var_component}; + + bin_edges = ParArray2D(prefix + "bin_edges", 2); // TODO split these... + + + val_var_name = pin->GetString(block_name, prefix + "val_variable"); + val_var_component = + pin->GetInteger(block_name, prefix + "val_variable_component"); + // would add additional logic to pick it from a pack... + PARTHENON_REQUIRE_THROWS(val_var_component >= 0, + "Negative component indices are not supported"); + + } +}; + +} // namespace HistUtil + +//---------------------------------------------------------------------------------------- +//! \fn void HistogramOutput:::SetupHistograms(ParameterInput *pin) +// \brief Process parameter input to setup persistent histograms +HistogramOutput::HistogramOutput(const OutputParameters &op, ParameterInput *pin) + : OutputType(op) { + + num_histograms_ = pin->GetOrAddInteger(op.block_name, "num_histograms", 0); + + std::vector histograms_; // TODO make private class member + + for (int i = 0; i < num_histograms_; i++) { + const auto prefix = "hist" + std::to_string(i) + "_"; + histograms_.emplace_back(pin, op.block_name, prefix); + } +} + +//---------------------------------------------------------------------------------------- +//! \fn void HistogramOutput:::WriteOutputFile(Mesh *pm) +// \brief Calculate histograms +void HistogramOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, + const SignalHandler::OutputSignal signal) { + + // advance output parameters + output_params.file_number++; + output_params.next_time += output_params.dt; + pin->SetInteger(output_params.block_name, "file_number", output_params.file_number); + pin->SetReal(output_params.block_name, "next_time", output_params.next_time); +} + +} // namespace parthenon +#endif // ifndef PARTHENON_ENABLE_ASCENT diff --git a/src/outputs/outputs.cpp b/src/outputs/outputs.cpp index f1084d4a360d..33b35a24f284 100644 --- a/src/outputs/outputs.cpp +++ b/src/outputs/outputs.cpp @@ -248,7 +248,7 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { pnew_type = new AscentOutput(op); } else if (op.file_type == "histogram") { #ifdef ENABLE_HDF5 - pnew_type = new HistogramOutput(op); + pnew_type = new HistogramOutput(op, pin); #else msg << "### FATAL ERROR in Outputs constructor" << std::endl << "Executable not configured for HDF5 outputs, but HDF5 file format " diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index 66cffc56bdb3..fa949d2cd51a 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -221,9 +221,12 @@ class PHDF5Output : public OutputType { class HistogramOutput : public OutputType { public: - explicit HistogramOutput(const OutputParameters &oparams) : OutputType(oparams) {} + HistogramOutput(const OutputParameters &oparams, ParameterInput *pin); void WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, const SignalHandler::OutputSignal signal) override; + + private: + int num_histograms_; // number of different histograms to compute }; #endif // ifdef ENABLE_HDF5 diff --git a/tst/regression/test_suites/output_hdf5/parthinput.advection b/tst/regression/test_suites/output_hdf5/parthinput.advection index 42472be2d012..0f6161606a7f 100644 --- a/tst/regression/test_suites/output_hdf5/parthinput.advection +++ b/tst/regression/test_suites/output_hdf5/parthinput.advection @@ -74,3 +74,7 @@ dt = 0.25 file_type = histogram dt = 0.25 +num_histograms = 1 + + + From 6cfd0bbf81cd708a8fc1896a0395996eaeb4939a Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 13 Oct 2023 16:17:26 +0200 Subject: [PATCH 17/68] Add CalcHist function --- src/outputs/histogram.cpp | 165 +++++++++++++++++++++++++++++++------- 1 file changed, 137 insertions(+), 28 deletions(-) diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp index 0791f4133a2a..2183c1e68b6d 100644 --- a/src/outputs/histogram.cpp +++ b/src/outputs/histogram.cpp @@ -18,10 +18,12 @@ // \brief 1D and 2D histograms // options for building +#include "Kokkos_ScatterView.hpp" #include "config.hpp" #include "globals.hpp" #include "kokkos_abstraction.hpp" #include "parameter_input.hpp" +#include "parthenon_array_generic.hpp" #include "utils/error_checking.hpp" #include #include @@ -63,63 +65,170 @@ using namespace OutputUtils; namespace HistUtil { struct Histogram { - int ndim; // 1D or 2D histogram - std::array bin_var_names; // variable(s) for bins - std::array bin_var_components; // components of bin variables (vector) - ParArray2D bin_edges; - std::string val_var_name; // variable name of variable to be binned - int val_var_component; // component of variable to be binned - ParArray2D hist; // resulting histogram - - Histogram(ParameterInput *pin, const std::string & block_name, const std::string & prefix) { + int ndim; // 1D or 2D histogram + std::string x_var_name, y_var_name; // variable(s) for bins + int x_var_component, y_var_component; // components of bin variables (vector) + ParArray1D x_edges, y_edges; + std::string binned_var_name; // variable name of variable to be binned + int binned_var_component; // component of variable to be binned + ParArray2D result; // resulting histogram + + // temp view for histogram reduction for better performance (switches + // between atomics and data duplication depending on the platform) + Kokkos::Experimental::ScatterView scatter_result; + + Histogram(ParameterInput *pin, const std::string &block_name, + const std::string &prefix) { ndim = pin->GetInteger(block_name, prefix + "ndim"); PARTHENON_REQUIRE_THROWS(ndim == 1 || ndim == 2, "Histogram dim must be '1' or '2'"); - const auto x_var_name = pin->GetString(block_name, prefix + "x_variable"); - const auto x_var_component = - pin->GetInteger(block_name, prefix + "x_variable_component"); - const auto x_edges = pin->GetVector(block_name, prefix + "x_edges"); + x_var_name = pin->GetString(block_name, prefix + "x_variable"); + x_var_component = pin->GetInteger(block_name, prefix + "x_variable_component"); // would add additional logic to pick it from a pack... PARTHENON_REQUIRE_THROWS(x_var_component >= 0, "Negative component indices are not supported"); + + const auto x_edges_in = pin->GetVector(block_name, prefix + "x_edges"); // required by binning index function - PARTHENON_REQUIRE_THROWS(std::is_sorted(x_edges.begin(), x_edges.end()), + PARTHENON_REQUIRE_THROWS(std::is_sorted(x_edges_in.begin(), x_edges_in.end()), "Bin edges must be in order."); + PARTHENON_REQUIRE_THROWS(x_edges_in.size() >= 2, + "Need at least one bin, i.e., two edges."); + x_edges = ParArray1D(prefix + "x_edges", x_edges_in.size()); + auto x_edges_h = x_edges.GetHostMirror(); + for (int i = 0; i < x_edges_in.size(); i++) { + x_edges_h(i) = x_edges_in[i]; + } + Kokkos::deep_copy(x_edges, x_edges_h); // For 1D profile default initalize y variables std::string y_var_name = ""; int y_var_component = -1; - auto y_edges = std::vector(); // and for 2D profile check if they're explicitly set (not default value) if (ndim == 2) { y_var_name = pin->GetString(block_name, prefix + "y_variable"); - y_var_component = pin->GetInteger(block_name, prefix + "y_variable_component"); - y_edges = pin->GetVector(block_name, prefix + "y_edges"); + y_var_component = pin->GetInteger(block_name, prefix + "y_variable_component"); // would add additional logic to pick it from a pack... PARTHENON_REQUIRE_THROWS(y_var_component >= 0, "Negative component indices are not supported"); + + const auto y_edges_in = pin->GetVector(block_name, prefix + "y_edges"); // required by binning index function - PARTHENON_REQUIRE_THROWS(std::is_sorted(y_edges.begin(), y_edges.end()), + PARTHENON_REQUIRE_THROWS(std::is_sorted(y_edges_in.begin(), y_edges_in.end()), "Bin edges must be in order."); + PARTHENON_REQUIRE_THROWS(y_edges_in.size() >= 2, + "Need at least one bin, i.e., two edges."); + y_edges = ParArray1D(prefix + "y_edges", y_edges_in.size()); + auto y_edges_h = y_edges.GetHostMirror(); + for (int i = 0; i < y_edges_in.size(); i++) { + y_edges_h(i) = y_edges_in[i]; + } + Kokkos::deep_copy(y_edges, y_edges_h); + } else { + y_edges = ParArray1D(prefix + "y_edges_unused", 0); } - bin_var_names = {x_var_name, y_var_name}; - bin_var_components = {x_var_component, y_var_component}; + binned_var_name = pin->GetString(block_name, prefix + "binned_variable"); + binned_var_component = + pin->GetInteger(block_name, prefix + "binned_variable_component"); + // would add additional logic to pick it from a pack... + PARTHENON_REQUIRE_THROWS(binned_var_component >= 0, + "Negative component indices are not supported"); - bin_edges = ParArray2D(prefix + "bin_edges", 2); // TODO split these... + const auto nxbins = x_edges.extent_int(0) - 1; + const auto nybins = ndim == 2 ? y_edges.extent_int(0) - 1 : 1; + result = ParArray2D(prefix + "result", nybins, nxbins); + scatter_result = Kokkos::Experimental::ScatterView(result.KokkosView()); + } +}; - val_var_name = pin->GetString(block_name, prefix + "val_variable"); - val_var_component = - pin->GetInteger(block_name, prefix + "val_variable_component"); - // would add additional logic to pick it from a pack... - PARTHENON_REQUIRE_THROWS(val_var_component >= 0, - "Negative component indices are not supported"); +// Returns the lower bound (or the array size if value has not been found) +// Could/Should be replaced with a Kokkos std version once available (currently schedule +// for 4.2 release). +// TODO add unit test +KOKKOS_INLINE_FUNCTION int lower_bound(const ParArray1D &arr, Real val) { + int l = 0; + int r = arr.GetDim(0); + int m; + while (l < r) { + m = l + (r - l) / 2; + if (val <= arr(m)) { + r = m; + } else { + l = m + 1; + } + } + return l; +} + +// Computes a 1D or 2D histogram with inclusive lower edges (and exclusive right ones). +// Function could in principle be templated on dimension, but it's currently not expected +// to be a performance concern (because it won't be called that often). +void CalcHist(Mesh *pm, const Histogram &hist) { + const auto x_var_component = hist.x_var_component; + const auto y_var_component = hist.y_var_component; + const auto binned_var_component = hist.binned_var_component; + const auto x_edges = hist.x_edges; + const auto y_edges = hist.y_edges; + const auto hist_ndim = hist.ndim; + auto result = hist.result; + auto scatter = hist.scatter_result; + + // Reset ScatterView from previous output + scatter.reset(); + // Also reset the histogram from previous call. + // Currently still required for consistent results between host and device backends, see + // https://github.com/kokkos/kokkos/issues/6363 + result.Reset(); + const int num_partitions = pm->DefaultNumPartitions(); + + for (int p = 0; p < num_partitions; p++) { + auto &md = pm->mesh_data.GetOrAdd("base", p); + + const auto x_var = md->PackVariables(std::vector{hist.x_var_name}); + const auto y_var = md->PackVariables(std::vector{hist.y_var_name}); + const auto binned_var = + md->PackVariables(std::vector{hist.binned_var_name}); + const auto ib = md->GetBoundsI(IndexDomain::interior); + const auto jb = md->GetBoundsJ(IndexDomain::interior); + const auto kb = md->GetBoundsK(IndexDomain::interior); + + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "CalcHist", DevExecSpace(), 0, x_var.GetDim(5) - 1, kb.s, + kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto &x_val = x_var(b, x_var_component, k, j, i); + if (x_val < x_edges(0) || x_val >= x_edges(x_edges.extent_int(0))) { + return; + } + // No further check for x_bin required as the preceeding if-statement guarantees + // x_val to fall in one bin. + const auto x_bin = lower_bound(x_edges, x_val); + + int y_bin = 0; + if (hist_ndim == 2) { + const auto &y_val = y_var(b, y_var_component, k, j, i); + if (y_val < y_edges(0) || y_val >= y_edges(y_edges.extent_int(0))) { + return; + } + // No further check for y_bin required as the preceeding if-statement + // guarantees y_val to fall in one bin. + y_bin = lower_bound(y_edges, y_val); + } + auto res = scatter.access(); + res(y_bin, x_bin) += binned_var(b, binned_var_component, k, j, i); + }); + // "reduce" results from scatter view to original view. May be a no-op depending on + // backend. + Kokkos::Experimental::contribute(result.KokkosView(), scatter); } -}; + // Ensure all (implicit) reductions from contribute are done + Kokkos::fence(); // May not be required +} } // namespace HistUtil From a6d718f84954317aad03669127c132b775fa8913 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Sat, 14 Oct 2023 18:55:34 +0200 Subject: [PATCH 18/68] Actually calc histograms --- src/outputs/histogram.cpp | 176 +++++++++--------- src/outputs/outputs.hpp | 22 +++ .../output_hdf5/parthinput.advection | 7 +- 3 files changed, 121 insertions(+), 84 deletions(-) diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp index 2183c1e68b6d..ce71154f7bbe 100644 --- a/src/outputs/histogram.cpp +++ b/src/outputs/histogram.cpp @@ -19,6 +19,7 @@ // options for building #include "Kokkos_ScatterView.hpp" +#include "basic_types.hpp" #include "config.hpp" #include "globals.hpp" #include "kokkos_abstraction.hpp" @@ -64,107 +65,96 @@ using namespace OutputUtils; namespace HistUtil { -struct Histogram { - int ndim; // 1D or 2D histogram - std::string x_var_name, y_var_name; // variable(s) for bins - int x_var_component, y_var_component; // components of bin variables (vector) - ParArray1D x_edges, y_edges; - std::string binned_var_name; // variable name of variable to be binned - int binned_var_component; // component of variable to be binned - ParArray2D result; // resulting histogram - - // temp view for histogram reduction for better performance (switches - // between atomics and data duplication depending on the platform) - Kokkos::Experimental::ScatterView scatter_result; - - Histogram(ParameterInput *pin, const std::string &block_name, - const std::string &prefix) { - ndim = pin->GetInteger(block_name, prefix + "ndim"); - PARTHENON_REQUIRE_THROWS(ndim == 1 || ndim == 2, "Histogram dim must be '1' or '2'"); +Histogram::Histogram(ParameterInput *pin, const std::string &block_name, + const std::string &prefix) { + ndim = pin->GetInteger(block_name, prefix + "ndim"); + PARTHENON_REQUIRE_THROWS(ndim == 1 || ndim == 2, "Histogram dim must be '1' or '2'"); + + x_var_name = pin->GetString(block_name, prefix + "x_variable"); + + x_var_component = pin->GetInteger(block_name, prefix + "x_variable_component"); + // would add additional logic to pick it from a pack... + PARTHENON_REQUIRE_THROWS(x_var_component >= 0, + "Negative component indices are not supported"); + + const auto x_edges_in = pin->GetVector(block_name, prefix + "x_edges"); + // required by binning index function + PARTHENON_REQUIRE_THROWS(std::is_sorted(x_edges_in.begin(), x_edges_in.end()), + "Bin edges must be in order."); + PARTHENON_REQUIRE_THROWS(x_edges_in.size() >= 2, + "Need at least one bin, i.e., two edges."); + x_edges = ParArray1D(prefix + "x_edges", x_edges_in.size()); + auto x_edges_h = x_edges.GetHostMirror(); + for (int i = 0; i < x_edges_in.size(); i++) { + x_edges_h(i) = x_edges_in[i]; + } + Kokkos::deep_copy(x_edges, x_edges_h); - x_var_name = pin->GetString(block_name, prefix + "x_variable"); + // For 1D profile default initalize y variables + y_var_name = ""; + y_var_component = -1; + // and for 2D profile check if they're explicitly set (not default value) + if (ndim == 2) { + y_var_name = pin->GetString(block_name, prefix + "y_variable"); - x_var_component = pin->GetInteger(block_name, prefix + "x_variable_component"); + y_var_component = pin->GetInteger(block_name, prefix + "y_variable_component"); // would add additional logic to pick it from a pack... - PARTHENON_REQUIRE_THROWS(x_var_component >= 0, + PARTHENON_REQUIRE_THROWS(y_var_component >= 0, "Negative component indices are not supported"); - const auto x_edges_in = pin->GetVector(block_name, prefix + "x_edges"); + const auto y_edges_in = pin->GetVector(block_name, prefix + "y_edges"); // required by binning index function - PARTHENON_REQUIRE_THROWS(std::is_sorted(x_edges_in.begin(), x_edges_in.end()), + PARTHENON_REQUIRE_THROWS(std::is_sorted(y_edges_in.begin(), y_edges_in.end()), "Bin edges must be in order."); - PARTHENON_REQUIRE_THROWS(x_edges_in.size() >= 2, + PARTHENON_REQUIRE_THROWS(y_edges_in.size() >= 2, "Need at least one bin, i.e., two edges."); - x_edges = ParArray1D(prefix + "x_edges", x_edges_in.size()); - auto x_edges_h = x_edges.GetHostMirror(); - for (int i = 0; i < x_edges_in.size(); i++) { - x_edges_h(i) = x_edges_in[i]; - } - Kokkos::deep_copy(x_edges, x_edges_h); - - // For 1D profile default initalize y variables - std::string y_var_name = ""; - int y_var_component = -1; - // and for 2D profile check if they're explicitly set (not default value) - if (ndim == 2) { - y_var_name = pin->GetString(block_name, prefix + "y_variable"); - - y_var_component = pin->GetInteger(block_name, prefix + "y_variable_component"); - // would add additional logic to pick it from a pack... - PARTHENON_REQUIRE_THROWS(y_var_component >= 0, - "Negative component indices are not supported"); - - const auto y_edges_in = pin->GetVector(block_name, prefix + "y_edges"); - // required by binning index function - PARTHENON_REQUIRE_THROWS(std::is_sorted(y_edges_in.begin(), y_edges_in.end()), - "Bin edges must be in order."); - PARTHENON_REQUIRE_THROWS(y_edges_in.size() >= 2, - "Need at least one bin, i.e., two edges."); - y_edges = ParArray1D(prefix + "y_edges", y_edges_in.size()); - auto y_edges_h = y_edges.GetHostMirror(); - for (int i = 0; i < y_edges_in.size(); i++) { - y_edges_h(i) = y_edges_in[i]; - } - Kokkos::deep_copy(y_edges, y_edges_h); - } else { - y_edges = ParArray1D(prefix + "y_edges_unused", 0); + y_edges = ParArray1D(prefix + "y_edges", y_edges_in.size()); + auto y_edges_h = y_edges.GetHostMirror(); + for (int i = 0; i < y_edges_in.size(); i++) { + y_edges_h(i) = y_edges_in[i]; } + Kokkos::deep_copy(y_edges, y_edges_h); + } else { + y_edges = ParArray1D(prefix + "y_edges_unused", 0); + } - binned_var_name = pin->GetString(block_name, prefix + "binned_variable"); - binned_var_component = - pin->GetInteger(block_name, prefix + "binned_variable_component"); - // would add additional logic to pick it from a pack... - PARTHENON_REQUIRE_THROWS(binned_var_component >= 0, - "Negative component indices are not supported"); + binned_var_name = pin->GetString(block_name, prefix + "binned_variable"); + binned_var_component = + pin->GetInteger(block_name, prefix + "binned_variable_component"); + // would add additional logic to pick it from a pack... + PARTHENON_REQUIRE_THROWS(binned_var_component >= 0, + "Negative component indices are not supported"); - const auto nxbins = x_edges.extent_int(0) - 1; - const auto nybins = ndim == 2 ? y_edges.extent_int(0) - 1 : 1; + const auto nxbins = x_edges.extent_int(0) - 1; + const auto nybins = ndim == 2 ? y_edges.extent_int(0) - 1 : 1; - result = ParArray2D(prefix + "result", nybins, nxbins); - scatter_result = Kokkos::Experimental::ScatterView(result.KokkosView()); - } -}; + result = ParArray2D(prefix + "result", nybins, nxbins); + scatter_result = Kokkos::Experimental::ScatterView(result.KokkosView()); +} -// Returns the lower bound (or the array size if value has not been found) +// Returns the upper bound (or the array size if value has not been found) // Could/Should be replaced with a Kokkos std version once available (currently schedule // for 4.2 release). // TODO add unit test -KOKKOS_INLINE_FUNCTION int lower_bound(const ParArray1D &arr, Real val) { +KOKKOS_INLINE_FUNCTION int upper_bound(const ParArray1D &arr, Real val) { int l = 0; - int r = arr.GetDim(0); + int r = arr.extent_int(0); int m; while (l < r) { m = l + (r - l) / 2; - if (val <= arr(m)) { - r = m; - } else { + if (val >= arr(m)) { l = m + 1; + } else { + r = m; } } + if (l < arr.extent_int(0) && val >= arr(l)) { + l++; + } return l; } -// Computes a 1D or 2D histogram with inclusive lower edges (and exclusive right ones). +// Computes a 1D or 2D histogram with inclusive lower edges and inclusive rightmost edges. // Function could in principle be templated on dimension, but it's currently not expected // to be a performance concern (because it won't be called that often). void CalcHist(Mesh *pm, const Histogram &hist) { @@ -182,7 +172,7 @@ void CalcHist(Mesh *pm, const Histogram &hist) { // Also reset the histogram from previous call. // Currently still required for consistent results between host and device backends, see // https://github.com/kokkos/kokkos/issues/6363 - result.Reset(); + Kokkos::deep_copy(result, 0); const int num_partitions = pm->DefaultNumPartitions(); @@ -202,22 +192,22 @@ void CalcHist(Mesh *pm, const Histogram &hist) { kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { const auto &x_val = x_var(b, x_var_component, k, j, i); - if (x_val < x_edges(0) || x_val >= x_edges(x_edges.extent_int(0))) { + if (x_val < x_edges(0) || x_val > x_edges(x_edges.extent_int(0) - 1)) { return; } // No further check for x_bin required as the preceeding if-statement guarantees // x_val to fall in one bin. - const auto x_bin = lower_bound(x_edges, x_val); + const auto x_bin = upper_bound(x_edges, x_val) - 1; int y_bin = 0; if (hist_ndim == 2) { const auto &y_val = y_var(b, y_var_component, k, j, i); - if (y_val < y_edges(0) || y_val >= y_edges(y_edges.extent_int(0))) { + if (y_val < y_edges(0) || y_val > y_edges(y_edges.extent_int(0) - 1)) { return; } // No further check for y_bin required as the preceeding if-statement // guarantees y_val to fall in one bin. - y_bin = lower_bound(y_edges, y_val); + y_bin = upper_bound(y_edges, y_val) - 1; } auto res = scatter.access(); res(y_bin, x_bin) += binned_var(b, binned_var_component, k, j, i); @@ -228,6 +218,17 @@ void CalcHist(Mesh *pm, const Histogram &hist) { } // Ensure all (implicit) reductions from contribute are done Kokkos::fence(); // May not be required + + // Now reduce over ranks +#ifdef MPI_PARALLEL + if (Globals::my_rank == 0) { + PARTHENON_MPI_CHECK(MPI_Reduce(MPI_IN_PLACE, result.data(), result.size(), + MPI_PARTHENON_REAL, MPI_SUM, 0, MPI_COMM_WORLD)); + } else { + PARTHENON_MPI_CHECK(MPI_Reduce(result.data(), result.data(), result.size(), + MPI_PARTHENON_REAL, MPI_SUM, 0, MPI_COMM_WORLD)); + } +#endif } } // namespace HistUtil @@ -240,8 +241,6 @@ HistogramOutput::HistogramOutput(const OutputParameters &op, ParameterInput *pin num_histograms_ = pin->GetOrAddInteger(op.block_name, "num_histograms", 0); - std::vector histograms_; // TODO make private class member - for (int i = 0; i < num_histograms_; i++) { const auto prefix = "hist" + std::to_string(i) + "_"; histograms_.emplace_back(pin, op.block_name, prefix); @@ -253,7 +252,18 @@ HistogramOutput::HistogramOutput(const OutputParameters &op, ParameterInput *pin // \brief Calculate histograms void HistogramOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, const SignalHandler::OutputSignal signal) { - + for (auto &hist : histograms_) { + CalcHist(pm, hist); + + if (Globals::my_rank == 0) { + const auto hist_h = hist.result.GetHostMirrorAndCopy(); + std::cout << "Hist result: "; + for (int i = 0; i < hist_h.extent_int(1); i++) { + std::cout << hist_h(0, i) << " "; + } + std::cout << "\n"; + } + } // advance output parameters output_params.file_number++; output_params.next_time += output_params.dt; diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index fa949d2cd51a..cadac050da59 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -25,6 +25,8 @@ #include #include +#include "Kokkos_ScatterView.hpp" + #include "basic_types.hpp" #include "coordinates/coordinates.hpp" #include "interface/mesh_data.hpp" @@ -219,6 +221,25 @@ class PHDF5Output : public OutputType { //! \class HistogramOutput // \brief derived OutputType class for histograms +namespace HistUtil { +struct Histogram { + int ndim; // 1D or 2D histogram + std::string x_var_name, y_var_name; // variable(s) for bins + int x_var_component, y_var_component; // components of bin variables (vector) + ParArray1D x_edges, y_edges; + std::string binned_var_name; // variable name of variable to be binned + int binned_var_component; // component of variable to be binned + ParArray2D result; // resulting histogram + + // temp view for histogram reduction for better performance (switches + // between atomics and data duplication depending on the platform) + Kokkos::Experimental::ScatterView scatter_result; + Histogram(ParameterInput *pin, const std::string &block_name, + const std::string &prefix); +}; + +} // namespace HistUtil + class HistogramOutput : public OutputType { public: HistogramOutput(const OutputParameters &oparams, ParameterInput *pin); @@ -227,6 +248,7 @@ class HistogramOutput : public OutputType { private: int num_histograms_; // number of different histograms to compute + std::vector histograms_; }; #endif // ifdef ENABLE_HDF5 diff --git a/tst/regression/test_suites/output_hdf5/parthinput.advection b/tst/regression/test_suites/output_hdf5/parthinput.advection index 0f6161606a7f..bdaf6025f8f9 100644 --- a/tst/regression/test_suites/output_hdf5/parthinput.advection +++ b/tst/regression/test_suites/output_hdf5/parthinput.advection @@ -76,5 +76,10 @@ dt = 0.25 num_histograms = 1 - +hist0_ndim = 1 +hist0_x_variable = advected +hist0_x_variable_component = 0 +hist0_x_edges = 1e-9, 1e-4,1e-1, 2e-1, 5e-1 ,1.00001 +hist0_binned_variable = advected +hist0_binned_variable_component = 0 From 37eaac2154a5e093d8f00de6d0fffba984868eb0 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Sat, 14 Oct 2023 21:46:09 +0200 Subject: [PATCH 19/68] First attempt at dumping to hdf5. existing groups are a problem... --- src/outputs/histogram.cpp | 96 ++++++++++++++++++++++++++++++++------- 1 file changed, 80 insertions(+), 16 deletions(-) diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp index ce71154f7bbe..7a4cb9e8ebcf 100644 --- a/src/outputs/histogram.cpp +++ b/src/outputs/histogram.cpp @@ -18,7 +18,6 @@ // \brief 1D and 2D histograms // options for building -#include "Kokkos_ScatterView.hpp" #include "basic_types.hpp" #include "config.hpp" #include "globals.hpp" @@ -26,13 +25,12 @@ #include "parameter_input.hpp" #include "parthenon_array_generic.hpp" #include "utils/error_checking.hpp" -#include -#include -#include // Only proceed if HDF5 output enabled #ifdef ENABLE_HDF5 +#include +#include #include #include #include @@ -40,6 +38,7 @@ #include #include #include +#include // Parthenon headers #include "coordinates/coordinates.hpp" @@ -49,15 +48,14 @@ #include "mesh/mesh.hpp" #include "outputs/output_utils.hpp" #include "outputs/outputs.hpp" +#include "outputs/parthenon_hdf5.hpp" #include "utils/error_checking.hpp" -// Ascent headers -#ifdef PARTHENON_ENABLE_ASCENT -#include "ascent.hpp" -#include "conduit_blueprint.hpp" -#include "conduit_relay_io.hpp" -#include "conduit_relay_io_blueprint.hpp" -#endif // ifdef PARTHENON_ENABLE_ASCENT +// ScatterView is not part of Kokkos core interface +#include "Kokkos_ScatterView.hpp" + +#include FS_HEADER +namespace fs = FS_NAMESPACE; namespace parthenon { @@ -158,6 +156,7 @@ KOKKOS_INLINE_FUNCTION int upper_bound(const ParArray1D &arr, Real val) { // Function could in principle be templated on dimension, but it's currently not expected // to be a performance concern (because it won't be called that often). void CalcHist(Mesh *pm, const Histogram &hist) { + Kokkos::Profiling::pushRegion("Calculate single histogram"); const auto x_var_component = hist.x_var_component; const auto y_var_component = hist.y_var_component; const auto binned_var_component = hist.binned_var_component; @@ -229,6 +228,7 @@ void CalcHist(Mesh *pm, const Histogram &hist) { MPI_PARTHENON_REAL, MPI_SUM, 0, MPI_COMM_WORLD)); } #endif + Kokkos::Profiling::popRegion(); // Calculate single histogram } } // namespace HistUtil @@ -252,10 +252,66 @@ HistogramOutput::HistogramOutput(const OutputParameters &op, ParameterInput *pin // \brief Calculate histograms void HistogramOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, const SignalHandler::OutputSignal signal) { + + Kokkos::Profiling::pushRegion("Calculate all histograms"); for (auto &hist : histograms_) { CalcHist(pm, hist); + } + Kokkos::Profiling::popRegion(); // Calculate all histograms + + Kokkos::Profiling::pushRegion("Dump histograms"); + if (Globals::my_rank == 0) { + using namespace HDF5; + // create/open HDF5 file + const std::string filename = "histogram.hdf"; + H5F file; + try { + if (fs::exists(filename)) { + file = H5F::FromHIDCheck(H5Fopen(filename.c_str(), H5F_ACC_RDWR, H5P_DEFAULT)); + } else { + file = H5F::FromHIDCheck( + H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)); + } + } catch (std::exception &ex) { + std::stringstream err; + err << "### ERROR: Failed to open/create HDF5 output file '" << filename + << "' with the following error:" << std::endl + << ex.what() << std::endl; + PARTHENON_THROW(err) + } + + std::string out_label; + if (signal == SignalHandler::OutputSignal::now) { + out_label = "now"; + } else if (signal == SignalHandler::OutputSignal::final && + output_params.file_label_final) { + out_label = "final"; + // default time based data dump + } else { + std::stringstream file_number; + file_number << std::setw(output_params.file_number_width) << std::setfill('0') + << output_params.file_number; + out_label = file_number.str(); + } + + const H5G all_hist_group = MakeGroup(file, "/" + out_label); + if (tm != nullptr) { + HDF5WriteAttribute("NCycle", tm->ncycle, all_hist_group); + HDF5WriteAttribute("Time", tm->time, all_hist_group); + HDF5WriteAttribute("dt", tm->dt, all_hist_group); + } + HDF5WriteAttribute("num_histograms", num_histograms_, all_hist_group); + + for (int i = 0; i < num_histograms_; i++) { + auto &hist = histograms_[i]; + const H5G hist_group = MakeGroup(all_hist_group, "/" + std::to_string(i)); + HDF5WriteAttribute("x_var_name", hist.x_var_name, hist_group); + HDF5WriteAttribute("x_var_component", hist.x_var_component, hist_group); + HDF5WriteAttribute("y_var_name", hist.y_var_name, hist_group); + HDF5WriteAttribute("y_var_component", hist.y_var_component, hist_group); + HDF5WriteAttribute("binned_var_name", hist.binned_var_name, hist_group); + HDF5WriteAttribute("binned_var_component", hist.binned_var_component, hist_group); - if (Globals::my_rank == 0) { const auto hist_h = hist.result.GetHostMirrorAndCopy(); std::cout << "Hist result: "; for (int i = 0; i < hist_h.extent_int(1); i++) { @@ -264,11 +320,19 @@ void HistogramOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm std::cout << "\n"; } } + Kokkos::Profiling::popRegion(); // Dump histograms + // advance output parameters - output_params.file_number++; - output_params.next_time += output_params.dt; - pin->SetInteger(output_params.block_name, "file_number", output_params.file_number); - pin->SetReal(output_params.block_name, "next_time", output_params.next_time); + if (signal == SignalHandler::OutputSignal::none) { + // After file has been opened with the current number, already advance output + // parameters so that for restarts the file is not immediatly overwritten again. + // Only applies to default time-based data dumps, so that writing "now" and "final" + // outputs does not change the desired output numbering. + output_params.file_number++; + output_params.next_time += output_params.dt; + pin->SetInteger(output_params.block_name, "file_number", output_params.file_number); + pin->SetReal(output_params.block_name, "next_time", output_params.next_time); + } } } // namespace parthenon From b030779fd7c724cfc05a31340561654e6400eaa0 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Sun, 15 Oct 2023 13:46:35 +0200 Subject: [PATCH 20/68] Use separate hdf5 files for histogram outputs --- src/outputs/histogram.cpp | 70 +++++++++++++++++++----------------- src/outputs/outputs.hpp | 2 ++ src/utils/signal_handler.cpp | 7 ++-- 3 files changed, 44 insertions(+), 35 deletions(-) diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp index 7a4cb9e8ebcf..e5ad8683a278 100644 --- a/src/outputs/histogram.cpp +++ b/src/outputs/histogram.cpp @@ -54,9 +54,6 @@ // ScatterView is not part of Kokkos core interface #include "Kokkos_ScatterView.hpp" -#include FS_HEADER -namespace fs = FS_NAMESPACE; - namespace parthenon { using namespace OutputUtils; @@ -247,6 +244,31 @@ HistogramOutput::HistogramOutput(const OutputParameters &op, ParameterInput *pin } } +std::string HistogramOutput::GenerateFilename_(ParameterInput *pin, SimTime *tm, + const SignalHandler::OutputSignal signal) { + using namespace HDF5; + + auto filename = std::string(output_params.file_basename); + filename.append("."); + filename.append(output_params.file_id); + filename.append(".histograms."); + if (signal == SignalHandler::OutputSignal::now) { + filename.append("now"); + } else if (signal == SignalHandler::OutputSignal::final && + output_params.file_label_final) { + filename.append("final"); + // default time based data dump + } else { + std::stringstream file_number; + file_number << std::setw(output_params.file_number_width) << std::setfill('0') + << output_params.file_number; + filename.append(file_number.str()); + } + filename.append(".hdf"); + + return filename; +} + //---------------------------------------------------------------------------------------- //! \fn void HistogramOutput:::WriteOutputFile(Mesh *pm) // \brief Calculate histograms @@ -260,51 +282,35 @@ void HistogramOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm Kokkos::Profiling::popRegion(); // Calculate all histograms Kokkos::Profiling::pushRegion("Dump histograms"); + // Given the expect size of histograms, we'll use serial HDF if (Globals::my_rank == 0) { using namespace HDF5; // create/open HDF5 file - const std::string filename = "histogram.hdf"; + const std::string filename = GenerateFilename_(pin, tm, signal); + H5F file; try { - if (fs::exists(filename)) { - file = H5F::FromHIDCheck(H5Fopen(filename.c_str(), H5F_ACC_RDWR, H5P_DEFAULT)); - } else { - file = H5F::FromHIDCheck( - H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)); - } + file = H5F::FromHIDCheck( + H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)); } catch (std::exception &ex) { std::stringstream err; - err << "### ERROR: Failed to open/create HDF5 output file '" << filename + err << "### ERROR: Failed to create HDF5 output file '" << filename << "' with the following error:" << std::endl << ex.what() << std::endl; PARTHENON_THROW(err) } - std::string out_label; - if (signal == SignalHandler::OutputSignal::now) { - out_label = "now"; - } else if (signal == SignalHandler::OutputSignal::final && - output_params.file_label_final) { - out_label = "final"; - // default time based data dump - } else { - std::stringstream file_number; - file_number << std::setw(output_params.file_number_width) << std::setfill('0') - << output_params.file_number; - out_label = file_number.str(); - } - - const H5G all_hist_group = MakeGroup(file, "/" + out_label); + const H5G info_group = MakeGroup(file, "/Info"); if (tm != nullptr) { - HDF5WriteAttribute("NCycle", tm->ncycle, all_hist_group); - HDF5WriteAttribute("Time", tm->time, all_hist_group); - HDF5WriteAttribute("dt", tm->dt, all_hist_group); + HDF5WriteAttribute("NCycle", tm->ncycle, info_group); + HDF5WriteAttribute("Time", tm->time, info_group); + HDF5WriteAttribute("dt", tm->dt, info_group); } - HDF5WriteAttribute("num_histograms", num_histograms_, all_hist_group); + HDF5WriteAttribute("num_histograms", num_histograms_, info_group); for (int i = 0; i < num_histograms_; i++) { auto &hist = histograms_[i]; - const H5G hist_group = MakeGroup(all_hist_group, "/" + std::to_string(i)); + const H5G hist_group = MakeGroup(file, "/" + std::to_string(i)); HDF5WriteAttribute("x_var_name", hist.x_var_name, hist_group); HDF5WriteAttribute("x_var_component", hist.x_var_component, hist_group); HDF5WriteAttribute("y_var_name", hist.y_var_name, hist_group); @@ -322,7 +328,7 @@ void HistogramOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm } Kokkos::Profiling::popRegion(); // Dump histograms - // advance output parameters + // advance file ids if (signal == SignalHandler::OutputSignal::none) { // After file has been opened with the current number, already advance output // parameters so that for restarts the file is not immediatly overwritten again. diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index cadac050da59..089eff04c976 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -247,6 +247,8 @@ class HistogramOutput : public OutputType { const SignalHandler::OutputSignal signal) override; private: + std::string GenerateFilename_(ParameterInput *pin, SimTime *tm, + const SignalHandler::OutputSignal signal); int num_histograms_; // number of different histograms to compute std::vector histograms_; }; diff --git a/src/utils/signal_handler.cpp b/src/utils/signal_handler.cpp index 940721e15433..7b54540f13fc 100644 --- a/src/utils/signal_handler.cpp +++ b/src/utils/signal_handler.cpp @@ -25,6 +25,9 @@ #include #include +#include FS_HEADER +namespace fs = FS_NAMESPACE; + #include "parthenon_mpi.hpp" #include "globals.hpp" @@ -61,10 +64,8 @@ void SignalHandlerInit() { OutputSignal CheckSignalFlags() { if (Globals::my_rank == 0) { - // TODO(the person bumping std to C++17): use std::filesystem::exists - struct stat buffer; // if file "output_now" exists - if (stat("output_now", &buffer) == 0) { + if (fs::exists("output_now")) { signalflag[nsignal] = 1; } } From eacd5cb678e084adbdb67df2d66d7503bbb7673e Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Sun, 15 Oct 2023 21:02:11 +0200 Subject: [PATCH 21/68] Fix hdf5 output --- src/outputs/histogram.cpp | 57 +++++++++++++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 5 deletions(-) diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp index e5ad8683a278..8728a758ba0c 100644 --- a/src/outputs/histogram.cpp +++ b/src/outputs/histogram.cpp @@ -285,6 +285,15 @@ void HistogramOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm // Given the expect size of histograms, we'll use serial HDF if (Globals::my_rank == 0) { using namespace HDF5; + H5P const pl_xfer = H5P::FromHIDCheck(H5Pcreate(H5P_DATASET_XFER)); + + // As we're reusing the interface from the existing hdf5 output, we have to define + // everything as 7D arrays. + // Counts will be set for each histogram individually below. + const std::array local_offset({0, 0, 0, 0, 0, 0, 0}); + std::array local_count({0, 0, 0, 0, 0, 0, 0}); + std::array global_count({0, 0, 0, 0, 0, 0, 0}); + // create/open HDF5 file const std::string filename = GenerateFilename_(pin, tm, signal); @@ -308,17 +317,55 @@ void HistogramOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm } HDF5WriteAttribute("num_histograms", num_histograms_, info_group); - for (int i = 0; i < num_histograms_; i++) { - auto &hist = histograms_[i]; - const H5G hist_group = MakeGroup(file, "/" + std::to_string(i)); + for (int h = 0; h < num_histograms_; h++) { + auto &hist = histograms_[h]; + const H5G hist_group = MakeGroup(file, "/" + std::to_string(h)); + HDF5WriteAttribute("ndim", hist.ndim, hist_group); HDF5WriteAttribute("x_var_name", hist.x_var_name, hist_group); HDF5WriteAttribute("x_var_component", hist.x_var_component, hist_group); - HDF5WriteAttribute("y_var_name", hist.y_var_name, hist_group); - HDF5WriteAttribute("y_var_component", hist.y_var_component, hist_group); HDF5WriteAttribute("binned_var_name", hist.binned_var_name, hist_group); HDF5WriteAttribute("binned_var_component", hist.binned_var_component, hist_group); + const auto x_edges_h = hist.x_edges.GetHostMirrorAndCopy(); + local_count[0] = global_count[0] = x_edges_h.extent_int(0); + HDF5Write1D(hist_group, "x_edges", x_edges_h.data(), local_offset.data(), + local_count.data(), global_count.data(), pl_xfer); + + if (hist.ndim == 2) { + HDF5WriteAttribute("y_var_name", hist.y_var_name, hist_group); + HDF5WriteAttribute("y_var_component", hist.y_var_component, hist_group); + + const auto y_edges_h = hist.y_edges.GetHostMirrorAndCopy(); + local_count[0] = global_count[0] = y_edges_h.extent_int(0); + HDF5Write1D(hist_group, "y_edges", y_edges_h.data(), local_offset.data(), + local_count.data(), global_count.data(), pl_xfer); + } + const auto hist_h = hist.result.GetHostMirrorAndCopy(); + // Ensure correct output format (as the data in Parthenon may, in theory, vary by + // changing the default view layout) so that it matches the numpy output (row + // major, x first) + std::vector tmp_data(hist_h.size()); + int idx = 0; + for (int i = 0; i < hist_h.extent_int(1); ++i) { + for (int j = 0; j < hist_h.extent_int(0); ++j) { + tmp_data[idx++] = hist_h(j, i); + } + } + + local_count[0] = global_count[0] = hist_h.extent_int(1); + if (hist.ndim == 2) { + local_count[1] = global_count[1] = hist_h.extent_int(0); + + HDF5Write2D(hist_group, "data", tmp_data.data(), local_offset.data(), + local_count.data(), global_count.data(), pl_xfer); + } else { + // No y-dim for 1D histogram + local_count[1] = global_count[1] = 0; + HDF5Write2D(hist_group, "data", tmp_data.data(), local_offset.data(), + local_count.data(), global_count.data(), pl_xfer); + } + std::cout << "Hist result: "; for (int i = 0; i < hist_h.extent_int(1); i++) { std::cout << hist_h(0, i) << " "; From 47065bca5d458f8a206c831c24055c6b9bdd046e Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Sun, 15 Oct 2023 23:06:46 +0200 Subject: [PATCH 22/68] Move upper_bound to utils and add unit test --- src/outputs/histogram.cpp | 23 +------------------ src/utils/sort.hpp | 22 ++++++++++++++++++ .../output_hdf5/parthinput.advection | 12 +++++++++- tst/unit/CMakeLists.txt | 1 + 4 files changed, 35 insertions(+), 23 deletions(-) diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp index 8728a758ba0c..4d2d653c58ad 100644 --- a/src/outputs/histogram.cpp +++ b/src/outputs/histogram.cpp @@ -50,6 +50,7 @@ #include "outputs/outputs.hpp" #include "outputs/parthenon_hdf5.hpp" #include "utils/error_checking.hpp" +#include "utils/sort.hpp" // for upper_bound // ScatterView is not part of Kokkos core interface #include "Kokkos_ScatterView.hpp" @@ -127,28 +128,6 @@ Histogram::Histogram(ParameterInput *pin, const std::string &block_name, scatter_result = Kokkos::Experimental::ScatterView(result.KokkosView()); } -// Returns the upper bound (or the array size if value has not been found) -// Could/Should be replaced with a Kokkos std version once available (currently schedule -// for 4.2 release). -// TODO add unit test -KOKKOS_INLINE_FUNCTION int upper_bound(const ParArray1D &arr, Real val) { - int l = 0; - int r = arr.extent_int(0); - int m; - while (l < r) { - m = l + (r - l) / 2; - if (val >= arr(m)) { - l = m + 1; - } else { - r = m; - } - } - if (l < arr.extent_int(0) && val >= arr(l)) { - l++; - } - return l; -} - // Computes a 1D or 2D histogram with inclusive lower edges and inclusive rightmost edges. // Function could in principle be templated on dimension, but it's currently not expected // to be a performance concern (because it won't be called that often). diff --git a/src/utils/sort.hpp b/src/utils/sort.hpp index 802cd6f56f52..aed0deeff473 100644 --- a/src/utils/sort.hpp +++ b/src/utils/sort.hpp @@ -31,6 +31,28 @@ namespace parthenon { +// Returns the upper bound (or the array size if value has not been found) +// Could/Should be replaced with a Kokkos std version once available (currently schedule +// for 4.2 release). +template +KOKKOS_INLINE_FUNCTION int upper_bound(const T &arr, Real val) { + int l = 0; + int r = arr.extent_int(0); + int m; + while (l < r) { + m = l + (r - l) / 2; + if (val >= arr(m)) { + l = m + 1; + } else { + r = m; + } + } + if (l < arr.extent_int(0) && val >= arr(l)) { + l++; + } + return l; +} + template void sort(ParArray1D data, KeyComparator comparator, size_t min_idx, size_t max_idx) { diff --git a/tst/regression/test_suites/output_hdf5/parthinput.advection b/tst/regression/test_suites/output_hdf5/parthinput.advection index bdaf6025f8f9..52ad9c180752 100644 --- a/tst/regression/test_suites/output_hdf5/parthinput.advection +++ b/tst/regression/test_suites/output_hdf5/parthinput.advection @@ -74,7 +74,7 @@ dt = 0.25 file_type = histogram dt = 0.25 -num_histograms = 1 +num_histograms = 2 hist0_ndim = 1 hist0_x_variable = advected @@ -83,3 +83,13 @@ hist0_x_edges = 1e-9, 1e-4,1e-1, 2e-1, 5e-1 ,1.00001 hist0_binned_variable = advected hist0_binned_variable_component = 0 +hist1_ndim = 2 +hist1_x_variable = advected +hist1_x_variable_component = 0 +hist1_x_edges = 1e-9, 1e-4,1e-1, 2e-1, 5e-1 ,1.0000001 +hist1_y_variable = one_minus_advected_sq +hist1_y_variable_component = 0 +hist1_y_edges = 0, 0.5, 1.00001 +hist1_binned_variable = advected +hist1_binned_variable_component = 0 + diff --git a/tst/unit/CMakeLists.txt b/tst/unit/CMakeLists.txt index 9efbdff6215f..95f7ca3ecebd 100644 --- a/tst/unit/CMakeLists.txt +++ b/tst/unit/CMakeLists.txt @@ -39,6 +39,7 @@ list(APPEND unit_tests_SOURCES test_partitioning.cpp test_state_descriptor.cpp test_unit_integrators.cpp + test_upper_bound.cpp ) add_executable(unit_tests "${unit_tests_SOURCES}") From 32a25d57a008306226657f45ee61580f51c955fd Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Sun, 15 Oct 2023 23:21:16 +0200 Subject: [PATCH 23/68] Fix edge bin calc --- src/outputs/histogram.cpp | 25 ++++++++----------- .../output_hdf5/parthinput.advection | 4 +-- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp index 4d2d653c58ad..bcc41c1affb5 100644 --- a/src/outputs/histogram.cpp +++ b/src/outputs/histogram.cpp @@ -170,9 +170,11 @@ void CalcHist(Mesh *pm, const Histogram &hist) { if (x_val < x_edges(0) || x_val > x_edges(x_edges.extent_int(0) - 1)) { return; } - // No further check for x_bin required as the preceeding if-statement guarantees - // x_val to fall in one bin. - const auto x_bin = upper_bound(x_edges, x_val) - 1; + + // if we're on the rightmost edge, directly set last bin, otherwise search + const auto x_bin = x_val == x_edges(x_edges.extent_int(0) - 1) + ? x_edges.extent_int(0) - 2 + : upper_bound(x_edges, x_val) - 1; int y_bin = 0; if (hist_ndim == 2) { @@ -180,9 +182,10 @@ void CalcHist(Mesh *pm, const Histogram &hist) { if (y_val < y_edges(0) || y_val > y_edges(y_edges.extent_int(0) - 1)) { return; } - // No further check for y_bin required as the preceeding if-statement - // guarantees y_val to fall in one bin. - y_bin = upper_bound(y_edges, y_val) - 1; + // if we're on the rightmost edge, directly set last bin, otherwise search + const auto y_bin = y_val == y_edges(y_edges.extent_int(0) - 1) + ? y_edges.extent_int(0) - 2 + : upper_bound(y_edges, y_val) - 1; } auto res = scatter.access(); res(y_bin, x_bin) += binned_var(b, binned_var_component, k, j, i); @@ -339,17 +342,11 @@ void HistogramOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm HDF5Write2D(hist_group, "data", tmp_data.data(), local_offset.data(), local_count.data(), global_count.data(), pl_xfer); } else { - // No y-dim for 1D histogram + // No y-dim for 1D histogram -- though unnecessary as it's not read anyway local_count[1] = global_count[1] = 0; - HDF5Write2D(hist_group, "data", tmp_data.data(), local_offset.data(), + HDF5Write1D(hist_group, "data", tmp_data.data(), local_offset.data(), local_count.data(), global_count.data(), pl_xfer); } - - std::cout << "Hist result: "; - for (int i = 0; i < hist_h.extent_int(1); i++) { - std::cout << hist_h(0, i) << " "; - } - std::cout << "\n"; } } Kokkos::Profiling::popRegion(); // Dump histograms diff --git a/tst/regression/test_suites/output_hdf5/parthinput.advection b/tst/regression/test_suites/output_hdf5/parthinput.advection index 52ad9c180752..a9e21c6581b6 100644 --- a/tst/regression/test_suites/output_hdf5/parthinput.advection +++ b/tst/regression/test_suites/output_hdf5/parthinput.advection @@ -86,10 +86,10 @@ hist0_binned_variable_component = 0 hist1_ndim = 2 hist1_x_variable = advected hist1_x_variable_component = 0 -hist1_x_edges = 1e-9, 1e-4,1e-1, 2e-1, 5e-1 ,1.0000001 +hist1_x_edges = 1e-9, 1e-4,1e-1, 2e-1, 5e-1 ,1.00 hist1_y_variable = one_minus_advected_sq hist1_y_variable_component = 0 -hist1_y_edges = 0, 0.5, 1.00001 +hist1_y_edges = 0, 0.5, 1.0 hist1_binned_variable = advected hist1_binned_variable_component = 0 From c80c146844260c87fe499be7cb9375751c816a9f Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 16 Oct 2023 00:09:27 +0200 Subject: [PATCH 24/68] Add histogram regression test --- tst/regression/CMakeLists.txt | 1 + .../test_suites/output_hdf5/output_hdf5.py | 40 ++++++++++++++++++- .../output_hdf5/parthinput.advection | 2 +- 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/tst/regression/CMakeLists.txt b/tst/regression/CMakeLists.txt index bf902381bbe7..96ac7c583c67 100644 --- a/tst/regression/CMakeLists.txt +++ b/tst/regression/CMakeLists.txt @@ -127,6 +127,7 @@ endif() # Any external modules that are required by python can be added to REQUIRED_PYTHON_MODULES # list variable, before including TestSetup.cmake. list(APPEND REQUIRED_PYTHON_MODULES numpy) +list(APPEND REQUIRED_PYTHON_MODULES h5py) list(APPEND DESIRED_PYTHON_MODULES matplotlib) # Include test setup functions, and check for python interpreter and modules diff --git a/tst/regression/test_suites/output_hdf5/output_hdf5.py b/tst/regression/test_suites/output_hdf5/output_hdf5.py index a11d41d6a1ff..74f90e61fe3b 100644 --- a/tst/regression/test_suites/output_hdf5/output_hdf5.py +++ b/tst/regression/test_suites/output_hdf5/output_hdf5.py @@ -1,6 +1,6 @@ # ======================================================================================== # Parthenon performance portable AMR framework -# Copyright(C) 2020 The Parthenon collaboration +# Copyright(C) 2020-2023 The Parthenon collaboration # Licensed under the 3-clause BSD License, see LICENSE file for details # ======================================================================================== # (C) (or copyright) 2020-2021. Triad National Security, LLC. All rights reserved. @@ -20,6 +20,7 @@ import sys import os import utils.test_case +import h5py # To prevent littering up imported folders with .pyc files or __pycache_ folder sys.dont_write_bytecode = True @@ -92,8 +93,9 @@ def Analyse(self, parameters): try: import phdf_diff + import phdf except ModuleNotFoundError: - print("Couldn't find module to compare Parthenon hdf5 files.") + print("Couldn't find modules to read/compare Parthenon hdf5 files.") return False # TODO(pgrete) make sure this also works/doesn't fail for the user @@ -166,4 +168,38 @@ def Analyse(self, parameters): ) analyze_status = False + # Checking Parthenon histograms versus numpy ones + for dim in [2, 3]: + data = phdf.phdf(f"advection_{dim}d.out0.final.phdf") + advected = data.Get("advected") + hist_np1d = np.histogram( + advected, [1e-9, 1e-4, 1e-1, 2e-1, 5e-1, 1e0], weights=advected + ) + with h5py.File( + f"advection_{dim}d.out2.histograms.final.hdf", "r" + ) as infile: + hist_parth = infile["0/data"][:] + all_close = np.allclose(hist_parth, hist_np1d[0]) + if not all_close: + print(f"1D hist for {dim}D setup don't match") + analyze_status = False + + omadvected = data.Get("one_minus_advected_sq") + hist_np2d = np.histogram2d( + advected.flatten(), + omadvected.flatten(), + [[1e-9, 1e-4, 1e-1, 2e-1, 5e-1, 1e0], [0, 0.5, 1]], + weights=advected.flatten(), + ) + with h5py.File( + f"advection_{dim}d.out2.histograms.final.hdf", "r" + ) as infile: + hist_parth = infile["1/data"][:] + # testing slices separately to ensure matching numpy convention + all_close = np.allclose(hist_parth[:, 0], hist_np2d[0][:, 0]) + all_close &= np.allclose(hist_parth[:, 1], hist_np2d[0][:, 1]) + if not all_close: + print(f"2D hist for {dim}D setup don't match") + analyze_status = False + return analyze_status diff --git a/tst/regression/test_suites/output_hdf5/parthinput.advection b/tst/regression/test_suites/output_hdf5/parthinput.advection index a9e21c6581b6..812c4ef75605 100644 --- a/tst/regression/test_suites/output_hdf5/parthinput.advection +++ b/tst/regression/test_suites/output_hdf5/parthinput.advection @@ -79,7 +79,7 @@ num_histograms = 2 hist0_ndim = 1 hist0_x_variable = advected hist0_x_variable_component = 0 -hist0_x_edges = 1e-9, 1e-4,1e-1, 2e-1, 5e-1 ,1.00001 +hist0_x_edges = 1e-9, 1e-4,1e-1, 2e-1, 5e-1 ,1.0 hist0_binned_variable = advected hist0_binned_variable_component = 0 From e801cf48e91bc2bdea1bff82073b8894ea42b72f Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 16 Oct 2023 00:12:05 +0200 Subject: [PATCH 25/68] Add missing unit test file --- tst/unit/test_upper_bound.cpp | 100 ++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 tst/unit/test_upper_bound.cpp diff --git a/tst/unit/test_upper_bound.cpp b/tst/unit/test_upper_bound.cpp new file mode 100644 index 000000000000..dff514de8e34 --- /dev/null +++ b/tst/unit/test_upper_bound.cpp @@ -0,0 +1,100 @@ +//======================================================================================== +// Parthenon performance portable AMR framework +// Copyright(C) 2023 The Parthenon collaboration +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== + +#include +#include + +#include + +#include "Kokkos_Core.hpp" +#include "utils/sort.hpp" + +TEST_CASE("upper_bound", "[between][out of bounds][on edges]") { + GIVEN("A sorted list") { + + const std::vector data{-1, 0, 1e-2, 5, 10}; + + Kokkos::View arr("arr", data.size()); + auto arr_h = Kokkos::create_mirror_view(arr); + + for (int i = 0; i < data.size(); i++) { + arr_h(i) = data[i]; + } + + Kokkos::deep_copy(arr, arr_h); + + WHEN("a value between entries is given") { + int result; + double val = 0.001; + Kokkos::parallel_reduce( + "unit::upper_bound::between", 1, + KOKKOS_LAMBDA(int /*i*/, int &lres) { + lres = parthenon::upper_bound(arr, val); + }, + result); + THEN("then the next index is returned") { REQUIRE(result == 2); } + THEN("it matches the stl result") { + REQUIRE(result == std::upper_bound(data.begin(), data.end(), val) - data.begin()); + } + } + WHEN("a value below the lower bound is given") { + int result; + double val = -1.1; + Kokkos::parallel_reduce( + "unit::upper_bound::below", 1, + KOKKOS_LAMBDA(int /*i*/, int &lres) { + lres = parthenon::upper_bound(arr, val); + }, + result); + THEN("then the first index is returned") { REQUIRE(result == 0); } + THEN("it matches the stl result") { + REQUIRE(result == std::upper_bound(data.begin(), data.end(), val) - data.begin()); + } + } + WHEN("a value above the upper bound is given") { + int result; + double val = 10.01; + Kokkos::parallel_reduce( + "unit::upper_bound::above", 1, + KOKKOS_LAMBDA(int /*i*/, int &lres) { + lres = parthenon::upper_bound(arr, val); + }, + result); + THEN("then the length of the array is returned") { REQUIRE(result == data.size()); } + THEN("it matches the stl result") { + REQUIRE(result == std::upper_bound(data.begin(), data.end(), val) - data.begin()); + } + } + WHEN("a value on the left edge is given") { + int result; + double val = -1; + Kokkos::parallel_reduce( + "unit::upper_bound::left", 1, + KOKKOS_LAMBDA(int /*i*/, int &lres) { + lres = parthenon::upper_bound(arr, val); + }, + result); + THEN("then the second index is returned") { REQUIRE(result == 1); } + THEN("it matches the stl result") { + REQUIRE(result == std::upper_bound(data.begin(), data.end(), val) - data.begin()); + } + } + WHEN("a value on the right edge is given") { + int result; + double val = 10; + Kokkos::parallel_reduce( + "unit::upper_bound::right", 1, + KOKKOS_LAMBDA(int /*i*/, int &lres) { + lres = parthenon::upper_bound(arr, val); + }, + result); + THEN("then the length of the array is returned") { REQUIRE(result == data.size()); } + THEN("it matches the stl result") { + REQUIRE(result == std::upper_bound(data.begin(), data.end(), val) - data.begin()); + } + } + } +} From 26719323777ec6d20f873aed1410346e3754e953 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 16 Oct 2023 14:10:07 +0200 Subject: [PATCH 26/68] Allow coordinate based binning --- src/outputs/histogram.cpp | 101 ++++++++++++++---- src/outputs/outputs.hpp | 3 + src/utils/sort.hpp | 2 +- .../test_suites/output_hdf5/output_hdf5.py | 7 +- .../output_hdf5/parthinput.advection | 5 +- 5 files changed, 93 insertions(+), 25 deletions(-) diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp index bcc41c1affb5..a7217079911a 100644 --- a/src/outputs/histogram.cpp +++ b/src/outputs/histogram.cpp @@ -31,6 +31,7 @@ #include #include +#include #include #include #include @@ -67,11 +68,25 @@ Histogram::Histogram(ParameterInput *pin, const std::string &block_name, PARTHENON_REQUIRE_THROWS(ndim == 1 || ndim == 2, "Histogram dim must be '1' or '2'"); x_var_name = pin->GetString(block_name, prefix + "x_variable"); - - x_var_component = pin->GetInteger(block_name, prefix + "x_variable_component"); - // would add additional logic to pick it from a pack... - PARTHENON_REQUIRE_THROWS(x_var_component >= 0, - "Negative component indices are not supported"); + x_var_component = -1; + if (x_var_name == "COORD_X1") { + x_var_type = VarType::X1; + } else if (x_var_name == "COORD_X2") { + x_var_type = VarType::X2; + } else if (x_var_name == "COORD_X3") { + x_var_type = VarType::X3; + } else if (x_var_name == "COORD_R") { + PARTHENON_REQUIRE_THROWS( + typeid(Coordinates_t) == typeid(UniformCartesian), + "Radial coordinate currently only works for uniform Cartesian coordinates."); + x_var_type = VarType::R; + } else { + x_var_type = VarType::Var; + x_var_component = pin->GetInteger(block_name, prefix + "x_variable_component"); + // would add additional logic to pick it from a pack... + PARTHENON_REQUIRE_THROWS(x_var_component >= 0, + "Negative component indices are not supported"); + } const auto x_edges_in = pin->GetVector(block_name, prefix + "x_edges"); // required by binning index function @@ -89,14 +104,28 @@ Histogram::Histogram(ParameterInput *pin, const std::string &block_name, // For 1D profile default initalize y variables y_var_name = ""; y_var_component = -1; + y_var_type = VarType::Unused; // and for 2D profile check if they're explicitly set (not default value) if (ndim == 2) { y_var_name = pin->GetString(block_name, prefix + "y_variable"); - - y_var_component = pin->GetInteger(block_name, prefix + "y_variable_component"); - // would add additional logic to pick it from a pack... - PARTHENON_REQUIRE_THROWS(y_var_component >= 0, - "Negative component indices are not supported"); + if (y_var_name == "COORD_X1") { + y_var_type = VarType::X1; + } else if (y_var_name == "COORD_X2") { + y_var_type = VarType::X2; + } else if (y_var_name == "COORD_X3") { + y_var_type = VarType::X3; + } else if (y_var_name == "COORD_R") { + PARTHENON_REQUIRE_THROWS( + typeid(Coordinates_t) == typeid(UniformCartesian), + "Radial coordinate currently only works for uniform Cartesian coordinates."); + y_var_type = VarType::R; + } else { + y_var_type = VarType::Var; + y_var_component = pin->GetInteger(block_name, prefix + "y_variable_component"); + // would add additional logic to pick it from a pack... + PARTHENON_REQUIRE_THROWS(y_var_component >= 0, + "Negative component indices are not supported"); + } const auto y_edges_in = pin->GetVector(block_name, prefix + "y_edges"); // required by binning index function @@ -136,6 +165,8 @@ void CalcHist(Mesh *pm, const Histogram &hist) { const auto x_var_component = hist.x_var_component; const auto y_var_component = hist.y_var_component; const auto binned_var_component = hist.binned_var_component; + const auto x_var_type = hist.x_var_type; + const auto y_var_type = hist.y_var_type; const auto x_edges = hist.x_edges; const auto y_edges = hist.y_edges; const auto hist_ndim = hist.ndim; @@ -154,8 +185,14 @@ void CalcHist(Mesh *pm, const Histogram &hist) { for (int p = 0; p < num_partitions; p++) { auto &md = pm->mesh_data.GetOrAdd("base", p); - const auto x_var = md->PackVariables(std::vector{hist.x_var_name}); - const auto y_var = md->PackVariables(std::vector{hist.y_var_name}); + const auto x_var_pack_string = x_var_type == VarType::Var + ? std::vector{hist.x_var_name} + : std::vector{}; + const auto x_var = md->PackVariables(x_var_pack_string); + const auto y_var_pack_string = y_var_type == VarType::Var + ? std::vector{hist.y_var_name} + : std::vector{}; + const auto y_var = md->PackVariables(y_var_pack_string); const auto binned_var = md->PackVariables(std::vector{hist.binned_var_name}); const auto ib = md->GetBoundsI(IndexDomain::interior); @@ -163,10 +200,23 @@ void CalcHist(Mesh *pm, const Histogram &hist) { const auto kb = md->GetBoundsK(IndexDomain::interior); parthenon::par_for( - DEFAULT_LOOP_PATTERN, "CalcHist", DevExecSpace(), 0, x_var.GetDim(5) - 1, kb.s, + DEFAULT_LOOP_PATTERN, "CalcHist", DevExecSpace(), 0, md->NumBlocks() - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - const auto &x_val = x_var(b, x_var_component, k, j, i); + auto &coords = x_var.GetCoords(b); + auto x_val = std::numeric_limits::quiet_NaN(); + if (x_var_type == VarType::X1) { + x_val = coords.Xc<1>(k, j, i); + } else if (x_var_type == VarType::X2) { + x_val = coords.Xc<2>(k, j, i); + } else if (x_var_type == VarType::X3) { + x_val = coords.Xc<3>(k, j, i); + } else if (x_var_type == VarType::R) { + x_val = Kokkos::sqrt(SQR(coords.Xc<1>(k, j, i)) + SQR(coords.Xc<2>(k, j, i)) + + SQR(coords.Xc<3>(k, j, i))); + } else { + x_val = x_var(b, x_var_component, k, j, i); + } if (x_val < x_edges(0) || x_val > x_edges(x_edges.extent_int(0) - 1)) { return; } @@ -178,14 +228,28 @@ void CalcHist(Mesh *pm, const Histogram &hist) { int y_bin = 0; if (hist_ndim == 2) { - const auto &y_val = y_var(b, y_var_component, k, j, i); + auto y_val = std::numeric_limits::quiet_NaN(); + if (y_var_type == VarType::X1) { + y_val = coords.Xc<1>(k, j, i); + } else if (y_var_type == VarType::X2) { + y_val = coords.Xc<2>(k, j, i); + } else if (y_var_type == VarType::X3) { + y_val = coords.Xc<3>(k, j, i); + } else if (y_var_type == VarType::R) { + y_val = + Kokkos::sqrt(SQR(coords.Xc<1>(k, j, i)) + SQR(coords.Xc<2>(k, j, i)) + + SQR(coords.Xc<3>(k, j, i))); + } else { + y_val = y_var(b, y_var_component, k, j, i); + } + if (y_val < y_edges(0) || y_val > y_edges(y_edges.extent_int(0) - 1)) { return; } // if we're on the rightmost edge, directly set last bin, otherwise search - const auto y_bin = y_val == y_edges(y_edges.extent_int(0) - 1) - ? y_edges.extent_int(0) - 2 - : upper_bound(y_edges, y_val) - 1; + y_bin = y_val == y_edges(y_edges.extent_int(0) - 1) + ? y_edges.extent_int(0) - 2 + : upper_bound(y_edges, y_val) - 1; } auto res = scatter.access(); res(y_bin, x_bin) += binned_var(b, binned_var_component, k, j, i); @@ -256,7 +320,6 @@ std::string HistogramOutput::GenerateFilename_(ParameterInput *pin, SimTime *tm, // \brief Calculate histograms void HistogramOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, const SignalHandler::OutputSignal signal) { - Kokkos::Profiling::pushRegion("Calculate all histograms"); for (auto &hist : histograms_) { CalcHist(pm, hist); diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index 089eff04c976..2cdac27732fd 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -222,9 +222,12 @@ class PHDF5Output : public OutputType { // \brief derived OutputType class for histograms namespace HistUtil { + +enum class VarType { X1, X2, X3, R, Var, Unused }; struct Histogram { int ndim; // 1D or 2D histogram std::string x_var_name, y_var_name; // variable(s) for bins + VarType x_var_type, y_var_type; // type, e.g., coord related or actual field int x_var_component, y_var_component; // components of bin variables (vector) ParArray1D x_edges, y_edges; std::string binned_var_name; // variable name of variable to be binned diff --git a/src/utils/sort.hpp b/src/utils/sort.hpp index aed0deeff473..9662cdbc5835 100644 --- a/src/utils/sort.hpp +++ b/src/utils/sort.hpp @@ -34,7 +34,7 @@ namespace parthenon { // Returns the upper bound (or the array size if value has not been found) // Could/Should be replaced with a Kokkos std version once available (currently schedule // for 4.2 release). -template +template KOKKOS_INLINE_FUNCTION int upper_bound(const T &arr, Real val) { int l = 0; int r = arr.extent_int(0); diff --git a/tst/regression/test_suites/output_hdf5/output_hdf5.py b/tst/regression/test_suites/output_hdf5/output_hdf5.py index 74f90e61fe3b..da14f6c3e183 100644 --- a/tst/regression/test_suites/output_hdf5/output_hdf5.py +++ b/tst/regression/test_suites/output_hdf5/output_hdf5.py @@ -170,6 +170,7 @@ def Analyse(self, parameters): # Checking Parthenon histograms versus numpy ones for dim in [2, 3]: + # 1D histogram with binning of a variable with bins defined by a var data = phdf.phdf(f"advection_{dim}d.out0.final.phdf") advected = data.Get("advected") hist_np1d = np.histogram( @@ -184,11 +185,13 @@ def Analyse(self, parameters): print(f"1D hist for {dim}D setup don't match") analyze_status = False + # 2D histogram with binning of a variable with bins defined by one var and one coord omadvected = data.Get("one_minus_advected_sq") + z, y, x = data.GetVolumeLocations() hist_np2d = np.histogram2d( - advected.flatten(), + x.flatten(), omadvected.flatten(), - [[1e-9, 1e-4, 1e-1, 2e-1, 5e-1, 1e0], [0, 0.5, 1]], + [[-0.5, -0.25, 0, 0.25, 0.5], [0, 0.5, 1]], weights=advected.flatten(), ) with h5py.File( diff --git a/tst/regression/test_suites/output_hdf5/parthinput.advection b/tst/regression/test_suites/output_hdf5/parthinput.advection index 812c4ef75605..d495a94126c3 100644 --- a/tst/regression/test_suites/output_hdf5/parthinput.advection +++ b/tst/regression/test_suites/output_hdf5/parthinput.advection @@ -84,9 +84,8 @@ hist0_binned_variable = advected hist0_binned_variable_component = 0 hist1_ndim = 2 -hist1_x_variable = advected -hist1_x_variable_component = 0 -hist1_x_edges = 1e-9, 1e-4,1e-1, 2e-1, 5e-1 ,1.00 +hist1_x_variable = COORD_X1 +hist1_x_edges = -0.5, -0.25, 0.0, 0.25, 0.5 hist1_y_variable = one_minus_advected_sq hist1_y_variable_component = 0 hist1_y_edges = 0, 0.5, 1.0 From b3d144df0512313796c5e4d2e377f88dff4f4bdd Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 16 Oct 2023 14:46:12 +0200 Subject: [PATCH 27/68] Add sampling based hist --- src/outputs/histogram.cpp | 46 ++++++++++++------- src/outputs/outputs.hpp | 5 +- .../test_suites/output_hdf5/output_hdf5.py | 13 +++++- .../output_hdf5/parthinput.advection | 10 +++- 4 files changed, 52 insertions(+), 22 deletions(-) diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp index a7217079911a..42c12ac28112 100644 --- a/src/outputs/histogram.cpp +++ b/src/outputs/histogram.cpp @@ -69,13 +69,13 @@ Histogram::Histogram(ParameterInput *pin, const std::string &block_name, x_var_name = pin->GetString(block_name, prefix + "x_variable"); x_var_component = -1; - if (x_var_name == "COORD_X1") { + if (x_var_name == "HIST_COORD_X1") { x_var_type = VarType::X1; - } else if (x_var_name == "COORD_X2") { + } else if (x_var_name == "HIST_COORD_X2") { x_var_type = VarType::X2; - } else if (x_var_name == "COORD_X3") { + } else if (x_var_name == "HIST_COORD_X3") { x_var_type = VarType::X3; - } else if (x_var_name == "COORD_R") { + } else if (x_var_name == "HIST_COORD_R") { PARTHENON_REQUIRE_THROWS( typeid(Coordinates_t) == typeid(UniformCartesian), "Radial coordinate currently only works for uniform Cartesian coordinates."); @@ -108,13 +108,13 @@ Histogram::Histogram(ParameterInput *pin, const std::string &block_name, // and for 2D profile check if they're explicitly set (not default value) if (ndim == 2) { y_var_name = pin->GetString(block_name, prefix + "y_variable"); - if (y_var_name == "COORD_X1") { + if (y_var_name == "HIST_COORD_X1") { y_var_type = VarType::X1; - } else if (y_var_name == "COORD_X2") { + } else if (y_var_name == "HIST_COORD_X2") { y_var_type = VarType::X2; - } else if (y_var_name == "COORD_X3") { + } else if (y_var_name == "HIST_COORD_X3") { y_var_type = VarType::X3; - } else if (y_var_name == "COORD_R") { + } else if (y_var_name == "HIST_COORD_R") { PARTHENON_REQUIRE_THROWS( typeid(Coordinates_t) == typeid(UniformCartesian), "Radial coordinate currently only works for uniform Cartesian coordinates."); @@ -143,12 +143,16 @@ Histogram::Histogram(ParameterInput *pin, const std::string &block_name, y_edges = ParArray1D(prefix + "y_edges_unused", 0); } - binned_var_name = pin->GetString(block_name, prefix + "binned_variable"); - binned_var_component = - pin->GetInteger(block_name, prefix + "binned_variable_component"); - // would add additional logic to pick it from a pack... - PARTHENON_REQUIRE_THROWS(binned_var_component >= 0, - "Negative component indices are not supported"); + binned_var_name = + pin->GetOrAddString(block_name, prefix + "binned_variable", "HIST_ONES"); + binned_var_component = -1; // implies that we're not binning a variable but count + if (binned_var_name != "HIST_ONES") { + binned_var_component = + pin->GetInteger(block_name, prefix + "binned_variable_component"); + // would add additional logic to pick it from a pack... + PARTHENON_REQUIRE_THROWS(binned_var_component >= 0, + "Negative component indices are not supported"); + } const auto nxbins = x_edges.extent_int(0) - 1; const auto nybins = ndim == 2 ? y_edges.extent_int(0) - 1 : 1; @@ -189,12 +193,17 @@ void CalcHist(Mesh *pm, const Histogram &hist) { ? std::vector{hist.x_var_name} : std::vector{}; const auto x_var = md->PackVariables(x_var_pack_string); + const auto y_var_pack_string = y_var_type == VarType::Var ? std::vector{hist.y_var_name} : std::vector{}; const auto y_var = md->PackVariables(y_var_pack_string); - const auto binned_var = - md->PackVariables(std::vector{hist.binned_var_name}); + + const auto binned_var_pack_string = + binned_var_component == -1 ? std::vector{} + : std::vector{hist.binned_var_name}; + const auto binned_var = md->PackVariables(binned_var_pack_string); + const auto ib = md->GetBoundsI(IndexDomain::interior); const auto jb = md->GetBoundsJ(IndexDomain::interior); const auto kb = md->GetBoundsK(IndexDomain::interior); @@ -252,7 +261,10 @@ void CalcHist(Mesh *pm, const Histogram &hist) { : upper_bound(y_edges, y_val) - 1; } auto res = scatter.access(); - res(y_bin, x_bin) += binned_var(b, binned_var_component, k, j, i); + const auto to_add = binned_var_component == -1 + ? 1 + : binned_var(b, binned_var_component, k, j, i); + res(y_bin, x_bin) += to_add; }); // "reduce" results from scatter view to original view. May be a no-op depending on // backend. diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index 2cdac27732fd..64f0a96ff045 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -231,8 +231,9 @@ struct Histogram { int x_var_component, y_var_component; // components of bin variables (vector) ParArray1D x_edges, y_edges; std::string binned_var_name; // variable name of variable to be binned - int binned_var_component; // component of variable to be binned - ParArray2D result; // resulting histogram + int binned_var_component; // component of variable to be binned. If -1 means no variable + // is binned but the histgram is a sample count. + ParArray2D result; // resulting histogram // temp view for histogram reduction for better performance (switches // between atomics and data duplication depending on the platform) diff --git a/tst/regression/test_suites/output_hdf5/output_hdf5.py b/tst/regression/test_suites/output_hdf5/output_hdf5.py index da14f6c3e183..96a567377833 100644 --- a/tst/regression/test_suites/output_hdf5/output_hdf5.py +++ b/tst/regression/test_suites/output_hdf5/output_hdf5.py @@ -182,7 +182,7 @@ def Analyse(self, parameters): hist_parth = infile["0/data"][:] all_close = np.allclose(hist_parth, hist_np1d[0]) if not all_close: - print(f"1D hist for {dim}D setup don't match") + print(f"1D variable-based hist for {dim}D setup don't match") analyze_status = False # 2D histogram with binning of a variable with bins defined by one var and one coord @@ -205,4 +205,15 @@ def Analyse(self, parameters): print(f"2D hist for {dim}D setup don't match") analyze_status = False + # 1D histogram (simple sampling) with bins defined by a var + hist_np1d = np.histogram(advected, [1e-9, 1e-4, 1e-1, 2e-1, 5e-1, 1e0]) + with h5py.File( + f"advection_{dim}d.out2.histograms.final.hdf", "r" + ) as infile: + hist_parth = infile["2/data"][:] + all_close = np.allclose(hist_parth, hist_np1d[0]) + if not all_close: + print(f"1D sampling-based hist for {dim}D setup don't match") + analyze_status = False + return analyze_status diff --git a/tst/regression/test_suites/output_hdf5/parthinput.advection b/tst/regression/test_suites/output_hdf5/parthinput.advection index d495a94126c3..e31568c762a5 100644 --- a/tst/regression/test_suites/output_hdf5/parthinput.advection +++ b/tst/regression/test_suites/output_hdf5/parthinput.advection @@ -74,7 +74,7 @@ dt = 0.25 file_type = histogram dt = 0.25 -num_histograms = 2 +num_histograms = 3 hist0_ndim = 1 hist0_x_variable = advected @@ -84,7 +84,7 @@ hist0_binned_variable = advected hist0_binned_variable_component = 0 hist1_ndim = 2 -hist1_x_variable = COORD_X1 +hist1_x_variable = HIST_COORD_X1 hist1_x_edges = -0.5, -0.25, 0.0, 0.25, 0.5 hist1_y_variable = one_minus_advected_sq hist1_y_variable_component = 0 @@ -92,3 +92,9 @@ hist1_y_edges = 0, 0.5, 1.0 hist1_binned_variable = advected hist1_binned_variable_component = 0 +hist2_ndim = 1 +hist2_x_variable = advected +hist2_x_variable_component = 0 +hist2_x_edges = 1e-9, 1e-4,1e-1, 2e-1, 5e-1 ,1.0 +hist2_binned_variable = HIST_ONES + From 4a4d6ae8e66bdc69b8f20214119d66f727793557 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 16 Oct 2023 15:08:48 +0200 Subject: [PATCH 28/68] Add volume weighting --- src/outputs/histogram.cpp | 12 ++++++---- src/outputs/outputs.hpp | 8 ++++--- .../test_suites/output_hdf5/output_hdf5.py | 23 +++++++++++++++++++ .../output_hdf5/parthinput.advection | 14 ++++++++++- 4 files changed, 49 insertions(+), 8 deletions(-) diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp index 42c12ac28112..9b164a28cbf2 100644 --- a/src/outputs/histogram.cpp +++ b/src/outputs/histogram.cpp @@ -159,6 +159,8 @@ Histogram::Histogram(ParameterInput *pin, const std::string &block_name, result = ParArray2D(prefix + "result", nybins, nxbins); scatter_result = Kokkos::Experimental::ScatterView(result.KokkosView()); + + weight_by_vol = pin->GetOrAddBoolean(block_name, prefix + "weight_by_volume", false); } // Computes a 1D or 2D histogram with inclusive lower edges and inclusive rightmost edges. @@ -174,6 +176,7 @@ void CalcHist(Mesh *pm, const Histogram &hist) { const auto x_edges = hist.x_edges; const auto y_edges = hist.y_edges; const auto hist_ndim = hist.ndim; + const auto weight_by_vol = hist.weight_by_vol; auto result = hist.result; auto scatter = hist.scatter_result; @@ -261,10 +264,11 @@ void CalcHist(Mesh *pm, const Histogram &hist) { : upper_bound(y_edges, y_val) - 1; } auto res = scatter.access(); - const auto to_add = binned_var_component == -1 - ? 1 - : binned_var(b, binned_var_component, k, j, i); - res(y_bin, x_bin) += to_add; + const auto val_to_add = binned_var_component == -1 + ? 1 + : binned_var(b, binned_var_component, k, j, i); + const auto weight = weight_by_vol ? coords.CellVolume(k, j, i) : 1.0; + res(y_bin, x_bin) += val_to_add * weight; }); // "reduce" results from scatter view to original view. May be a no-op depending on // backend. diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index 64f0a96ff045..acba841dd3ff 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -231,9 +231,11 @@ struct Histogram { int x_var_component, y_var_component; // components of bin variables (vector) ParArray1D x_edges, y_edges; std::string binned_var_name; // variable name of variable to be binned - int binned_var_component; // component of variable to be binned. If -1 means no variable - // is binned but the histgram is a sample count. - ParArray2D result; // resulting histogram + // component of variable to be binned. If -1 means no variable is binned but the + // histgram is a sample count. + int binned_var_component; + bool weight_by_vol; // use volume weighting + ParArray2D result; // resulting histogram // temp view for histogram reduction for better performance (switches // between atomics and data duplication depending on the platform) diff --git a/tst/regression/test_suites/output_hdf5/output_hdf5.py b/tst/regression/test_suites/output_hdf5/output_hdf5.py index 96a567377833..857f528c578f 100644 --- a/tst/regression/test_suites/output_hdf5/output_hdf5.py +++ b/tst/regression/test_suites/output_hdf5/output_hdf5.py @@ -216,4 +216,27 @@ def Analyse(self, parameters): print(f"1D sampling-based hist for {dim}D setup don't match") analyze_status = False + # 2D histogram with volume weighted binning of a variable with bins defined by coords + vols = np.einsum( + "ai,aj,ak->aijk", np.diff(data.zf), np.diff(data.yf), np.diff(data.xf) + ) + hist_np2d = np.histogram2d( + x.flatten(), + y.flatten(), + [[-0.5, -0.25, 0, 0.25, 0.5], [-0.5, -0.1, 0, 0.1, 0.5]], + weights=advected.flatten() * vols.flatten(), + ) + with h5py.File( + f"advection_{dim}d.out2.histograms.final.hdf", "r" + ) as infile: + hist_parth = infile["3/data"][:] + # testing slices separately to ensure matching numpy convention + all_close = np.allclose(hist_parth[:, 0], hist_np2d[0][:, 0]) + all_close &= np.allclose(hist_parth[:, 1], hist_np2d[0][:, 1]) + all_close &= np.allclose(hist_parth[:, 2], hist_np2d[0][:, 2]) + all_close &= np.allclose(hist_parth[:, 3], hist_np2d[0][:, 3]) + if not all_close: + print(f"2D vol-weighted hist for {dim}D setup don't match") + analyze_status = False + return analyze_status diff --git a/tst/regression/test_suites/output_hdf5/parthinput.advection b/tst/regression/test_suites/output_hdf5/parthinput.advection index e31568c762a5..fd1270447280 100644 --- a/tst/regression/test_suites/output_hdf5/parthinput.advection +++ b/tst/regression/test_suites/output_hdf5/parthinput.advection @@ -74,8 +74,9 @@ dt = 0.25 file_type = histogram dt = 0.25 -num_histograms = 3 +num_histograms = 4 +# 1D histogram of a variable, binned by a variable hist0_ndim = 1 hist0_x_variable = advected hist0_x_variable_component = 0 @@ -83,6 +84,7 @@ hist0_x_edges = 1e-9, 1e-4,1e-1, 2e-1, 5e-1 ,1.0 hist0_binned_variable = advected hist0_binned_variable_component = 0 +# 2D histogram of a variable, binned by a coordinate and a different variable hist1_ndim = 2 hist1_x_variable = HIST_COORD_X1 hist1_x_edges = -0.5, -0.25, 0.0, 0.25, 0.5 @@ -92,9 +94,19 @@ hist1_y_edges = 0, 0.5, 1.0 hist1_binned_variable = advected hist1_binned_variable_component = 0 +# 1D histogram ("standard", i.e., counting occurance in bin) hist2_ndim = 1 hist2_x_variable = advected hist2_x_variable_component = 0 hist2_x_edges = 1e-9, 1e-4,1e-1, 2e-1, 5e-1 ,1.0 hist2_binned_variable = HIST_ONES +# 2D histogram of volume weighted variable according to two coordinates +hist3_ndim = 2 +hist3_x_variable = HIST_COORD_X1 +hist3_x_edges = -0.5, -0.25, 0.0, 0.25, 0.5 +hist3_y_variable = HIST_COORD_X2 +hist3_y_edges = -0.5, -0.1, 0.0, 0.1, 0.5 +hist3_binned_variable = advected +hist3_binned_variable_component = 0 +hist3_weight_by_volume = true From 3eb6df90067dcb5445d306a78ce9afc6a8e83df1 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 17 Oct 2023 14:23:22 +0200 Subject: [PATCH 29/68] Fix Scatterview layout --- src/outputs/histogram.cpp | 3 ++- src/outputs/outputs.hpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp index 9b164a28cbf2..1d0855a74960 100644 --- a/src/outputs/histogram.cpp +++ b/src/outputs/histogram.cpp @@ -158,7 +158,8 @@ Histogram::Histogram(ParameterInput *pin, const std::string &block_name, const auto nybins = ndim == 2 ? y_edges.extent_int(0) - 1 : 1; result = ParArray2D(prefix + "result", nybins, nxbins); - scatter_result = Kokkos::Experimental::ScatterView(result.KokkosView()); + scatter_result = + Kokkos::Experimental::ScatterView(result.KokkosView()); weight_by_vol = pin->GetOrAddBoolean(block_name, prefix + "weight_by_volume", false); } diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index acba841dd3ff..6fe08bb3af0f 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -31,6 +31,7 @@ #include "coordinates/coordinates.hpp" #include "interface/mesh_data.hpp" #include "io_wrapper.hpp" +#include "kokkos_abstraction.hpp" #include "parthenon_arrays.hpp" #include "utils/error_checking.hpp" @@ -239,7 +240,7 @@ struct Histogram { // temp view for histogram reduction for better performance (switches // between atomics and data duplication depending on the platform) - Kokkos::Experimental::ScatterView scatter_result; + Kokkos::Experimental::ScatterView scatter_result; Histogram(ParameterInput *pin, const std::string &block_name, const std::string &prefix); }; From 5961d213127ef3dbd075b089a00953bfd493fe9a Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 17 Oct 2023 15:00:38 +0200 Subject: [PATCH 30/68] Separate edge parsing --- src/outputs/histogram.cpp | 58 +++++++++++-------- .../output_hdf5/parthinput.advection | 18 ++++-- 2 files changed, 46 insertions(+), 30 deletions(-) diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp index 1d0855a74960..94158b55635e 100644 --- a/src/outputs/histogram.cpp +++ b/src/outputs/histogram.cpp @@ -62,6 +62,37 @@ using namespace OutputUtils; namespace HistUtil { +ParArray1D GetEdges(ParameterInput *pin, const std::string &block_name, + const std::string &prefix) { + + std::vector edges_in; + + const auto edge_type_str = pin->GetString(block_name, prefix + "type"); + if (edge_type_str == "lin") { + + } else if (edge_type_str == "log") { + + } else if (edge_type_str == "list") { + edges_in = pin->GetVector(block_name, prefix + "list"); + // required by binning index function + PARTHENON_REQUIRE_THROWS(std::is_sorted(edges_in.begin(), edges_in.end()), + "Bin edges must be in order."); + PARTHENON_REQUIRE_THROWS(edges_in.size() >= 2, + "Need at least one bin, i.e., two edges."); + + } else { + PARTHENON_THROW( + "Unknown edge type for histogram. Supported types are lin, log, and list.") + } + auto edges = ParArray1D(prefix, edges_in.size()); + auto edges_h = edges.GetHostMirror(); + for (int i = 0; i < edges_in.size(); i++) { + edges_h(i) = edges_in[i]; + } + Kokkos::deep_copy(edges, edges_h); + return edges; +} + Histogram::Histogram(ParameterInput *pin, const std::string &block_name, const std::string &prefix) { ndim = pin->GetInteger(block_name, prefix + "ndim"); @@ -88,18 +119,7 @@ Histogram::Histogram(ParameterInput *pin, const std::string &block_name, "Negative component indices are not supported"); } - const auto x_edges_in = pin->GetVector(block_name, prefix + "x_edges"); - // required by binning index function - PARTHENON_REQUIRE_THROWS(std::is_sorted(x_edges_in.begin(), x_edges_in.end()), - "Bin edges must be in order."); - PARTHENON_REQUIRE_THROWS(x_edges_in.size() >= 2, - "Need at least one bin, i.e., two edges."); - x_edges = ParArray1D(prefix + "x_edges", x_edges_in.size()); - auto x_edges_h = x_edges.GetHostMirror(); - for (int i = 0; i < x_edges_in.size(); i++) { - x_edges_h(i) = x_edges_in[i]; - } - Kokkos::deep_copy(x_edges, x_edges_h); + x_edges = GetEdges(pin, block_name, prefix + "x_edges_"); // For 1D profile default initalize y variables y_var_name = ""; @@ -127,18 +147,8 @@ Histogram::Histogram(ParameterInput *pin, const std::string &block_name, "Negative component indices are not supported"); } - const auto y_edges_in = pin->GetVector(block_name, prefix + "y_edges"); - // required by binning index function - PARTHENON_REQUIRE_THROWS(std::is_sorted(y_edges_in.begin(), y_edges_in.end()), - "Bin edges must be in order."); - PARTHENON_REQUIRE_THROWS(y_edges_in.size() >= 2, - "Need at least one bin, i.e., two edges."); - y_edges = ParArray1D(prefix + "y_edges", y_edges_in.size()); - auto y_edges_h = y_edges.GetHostMirror(); - for (int i = 0; i < y_edges_in.size(); i++) { - y_edges_h(i) = y_edges_in[i]; - } - Kokkos::deep_copy(y_edges, y_edges_h); + y_edges = GetEdges(pin, block_name, prefix + "y_edges_"); + } else { y_edges = ParArray1D(prefix + "y_edges_unused", 0); } diff --git a/tst/regression/test_suites/output_hdf5/parthinput.advection b/tst/regression/test_suites/output_hdf5/parthinput.advection index fd1270447280..677a2ebf2e18 100644 --- a/tst/regression/test_suites/output_hdf5/parthinput.advection +++ b/tst/regression/test_suites/output_hdf5/parthinput.advection @@ -80,17 +80,20 @@ num_histograms = 4 hist0_ndim = 1 hist0_x_variable = advected hist0_x_variable_component = 0 -hist0_x_edges = 1e-9, 1e-4,1e-1, 2e-1, 5e-1 ,1.0 +hist0_x_edges_type = list +hist0_x_edges_list = 1e-9, 1e-4,1e-1, 2e-1, 5e-1 ,1.0 hist0_binned_variable = advected hist0_binned_variable_component = 0 # 2D histogram of a variable, binned by a coordinate and a different variable hist1_ndim = 2 hist1_x_variable = HIST_COORD_X1 -hist1_x_edges = -0.5, -0.25, 0.0, 0.25, 0.5 +hist1_x_edges_type = list +hist1_x_edges_list = -0.5, -0.25, 0.0, 0.25, 0.5 hist1_y_variable = one_minus_advected_sq hist1_y_variable_component = 0 -hist1_y_edges = 0, 0.5, 1.0 +hist1_y_edges_type = list +hist1_y_edges_list = 0, 0.5, 1.0 hist1_binned_variable = advected hist1_binned_variable_component = 0 @@ -98,15 +101,18 @@ hist1_binned_variable_component = 0 hist2_ndim = 1 hist2_x_variable = advected hist2_x_variable_component = 0 -hist2_x_edges = 1e-9, 1e-4,1e-1, 2e-1, 5e-1 ,1.0 +hist2_x_edges_type = list +hist2_x_edges_list = 1e-9, 1e-4,1e-1, 2e-1, 5e-1 ,1.0 hist2_binned_variable = HIST_ONES # 2D histogram of volume weighted variable according to two coordinates hist3_ndim = 2 hist3_x_variable = HIST_COORD_X1 -hist3_x_edges = -0.5, -0.25, 0.0, 0.25, 0.5 +hist3_x_edges_type = list +hist3_x_edges_list = -0.5, -0.25, 0.0, 0.25, 0.5 hist3_y_variable = HIST_COORD_X2 -hist3_y_edges = -0.5, -0.1, 0.0, 0.1, 0.5 +hist3_y_edges_type = list +hist3_y_edges_list = -0.5, -0.1, 0.0, 0.1, 0.5 hist3_binned_variable = advected hist3_binned_variable_component = 0 hist3_weight_by_volume = true From e515f7c3dc848a88c6053aa2d97e0c2bf9e87b21 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 17 Oct 2023 15:24:26 +0200 Subject: [PATCH 31/68] Add support for calculating lin and log bin edges --- src/outputs/histogram.cpp | 31 +++++++++++++++++-- .../output_hdf5/parthinput.advection | 6 ++-- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp index 94158b55635e..046dcaec3fbc 100644 --- a/src/outputs/histogram.cpp +++ b/src/outputs/histogram.cpp @@ -64,13 +64,38 @@ namespace HistUtil { ParArray1D GetEdges(ParameterInput *pin, const std::string &block_name, const std::string &prefix) { - std::vector edges_in; const auto edge_type_str = pin->GetString(block_name, prefix + "type"); - if (edge_type_str == "lin") { + if (edge_type_str == "lin" || edge_type_str == "log") { + const auto edge_min = pin->GetReal(block_name, prefix + "min"); + const auto edge_max = pin->GetReal(block_name, prefix + "max"); + PARTHENON_REQUIRE_THROWS(edge_max > edge_min, + "Histogram max needs to be larger than min.") + + const auto edge_num_bins = pin->GetReal(block_name, prefix + "num_bins"); + PARTHENON_REQUIRE_THROWS(edge_num_bins >= 1, "Need at least one bin for histogram."); + + if (edge_type_str == "lin") { + auto dbin = (edge_max - edge_min) / (edge_num_bins); + for (int i = 0; i < edge_num_bins; i++) { + edges_in.emplace_back(edge_min + i * dbin); + } + edges_in.emplace_back(edge_max); + } else if (edge_type_str == "log") { + PARTHENON_REQUIRE_THROWS( + edge_min > 0.0 && edge_max > 0.0, + "Log binning for negative values not implemented. However, you can specify " + "arbitrary bin edges through the 'list' edge type.") - } else if (edge_type_str == "log") { + auto dbin = (std::log10(edge_max) - std::log10(edge_min)) / (edge_num_bins); + for (int i = 0; i < edge_num_bins; i++) { + edges_in.emplace_back(std::pow(10., std::log10(edge_min) + i * dbin)); + } + edges_in.emplace_back(edge_max); + } else { + PARTHENON_FAIL("Not sure how I got here...") + } } else if (edge_type_str == "list") { edges_in = pin->GetVector(block_name, prefix + "list"); diff --git a/tst/regression/test_suites/output_hdf5/parthinput.advection b/tst/regression/test_suites/output_hdf5/parthinput.advection index 677a2ebf2e18..347ff6d75644 100644 --- a/tst/regression/test_suites/output_hdf5/parthinput.advection +++ b/tst/regression/test_suites/output_hdf5/parthinput.advection @@ -88,8 +88,10 @@ hist0_binned_variable_component = 0 # 2D histogram of a variable, binned by a coordinate and a different variable hist1_ndim = 2 hist1_x_variable = HIST_COORD_X1 -hist1_x_edges_type = list -hist1_x_edges_list = -0.5, -0.25, 0.0, 0.25, 0.5 +hist1_x_edges_type = lin +hist1_x_edges_num_bins = 4 +hist1_x_edges_min = -0.5 +hist1_x_edges_max = 0.5 hist1_y_variable = one_minus_advected_sq hist1_y_variable_component = 0 hist1_y_edges_type = list From 102baa058648666255ae62cf2c0f0d31c1b900ed Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 19 Oct 2023 09:43:16 +0200 Subject: [PATCH 32/68] Write string as strings to HDF5 --- src/outputs/histogram.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp index 046dcaec3fbc..6215338ffe47 100644 --- a/src/outputs/histogram.cpp +++ b/src/outputs/histogram.cpp @@ -418,9 +418,9 @@ void HistogramOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm auto &hist = histograms_[h]; const H5G hist_group = MakeGroup(file, "/" + std::to_string(h)); HDF5WriteAttribute("ndim", hist.ndim, hist_group); - HDF5WriteAttribute("x_var_name", hist.x_var_name, hist_group); + HDF5WriteAttribute("x_var_name", hist.x_var_name.c_str(), hist_group); HDF5WriteAttribute("x_var_component", hist.x_var_component, hist_group); - HDF5WriteAttribute("binned_var_name", hist.binned_var_name, hist_group); + HDF5WriteAttribute("binned_var_name", hist.binned_var_name.c_str(), hist_group); HDF5WriteAttribute("binned_var_component", hist.binned_var_component, hist_group); const auto x_edges_h = hist.x_edges.GetHostMirrorAndCopy(); @@ -429,7 +429,7 @@ void HistogramOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm local_count.data(), global_count.data(), pl_xfer); if (hist.ndim == 2) { - HDF5WriteAttribute("y_var_name", hist.y_var_name, hist_group); + HDF5WriteAttribute("y_var_name", hist.y_var_name.c_str(), hist_group); HDF5WriteAttribute("y_var_component", hist.y_var_component, hist_group); const auto y_edges_h = hist.y_edges.GetHostMirrorAndCopy(); From 6b12bc4ceeddc909b4c5ff6bcc1b17d47bade743 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 19 Oct 2023 09:58:41 +0200 Subject: [PATCH 33/68] Add support for variable weights to histogram --- src/outputs/histogram.cpp | 22 ++++++++++++- src/outputs/outputs.hpp | 5 ++- .../test_suites/output_hdf5/output_hdf5.py | 6 ++-- .../output_hdf5/parthinput.advection | 32 ++++++++++++------- 4 files changed, 49 insertions(+), 16 deletions(-) diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp index 6215338ffe47..8b2a04bd2052 100644 --- a/src/outputs/histogram.cpp +++ b/src/outputs/histogram.cpp @@ -197,6 +197,17 @@ Histogram::Histogram(ParameterInput *pin, const std::string &block_name, Kokkos::Experimental::ScatterView(result.KokkosView()); weight_by_vol = pin->GetOrAddBoolean(block_name, prefix + "weight_by_volume", false); + + weight_var_name = + pin->GetOrAddString(block_name, prefix + "weight_variable", "HIST_ONES"); + weight_var_component = -1; // implies that weighting is not applied + if (weight_var_name != "HIST_ONES") { + weight_var_component = + pin->GetInteger(block_name, prefix + "weight_variable_component"); + // would add additional logic to pick it from a pack... + PARTHENON_REQUIRE_THROWS(weight_var_component >= 0, + "Negative component indices are not supported"); + } } // Computes a 1D or 2D histogram with inclusive lower edges and inclusive rightmost edges. @@ -207,6 +218,7 @@ void CalcHist(Mesh *pm, const Histogram &hist) { const auto x_var_component = hist.x_var_component; const auto y_var_component = hist.y_var_component; const auto binned_var_component = hist.binned_var_component; + const auto weight_var_component = hist.weight_var_component; const auto x_var_type = hist.x_var_type; const auto y_var_type = hist.y_var_type; const auto x_edges = hist.x_edges; @@ -243,6 +255,11 @@ void CalcHist(Mesh *pm, const Histogram &hist) { : std::vector{hist.binned_var_name}; const auto binned_var = md->PackVariables(binned_var_pack_string); + const auto weight_var_pack_string = + weight_var_component == -1 ? std::vector{} + : std::vector{hist.weight_var_name}; + const auto weight_var = md->PackVariables(weight_var_pack_string); + const auto ib = md->GetBoundsI(IndexDomain::interior); const auto jb = md->GetBoundsJ(IndexDomain::interior); const auto kb = md->GetBoundsK(IndexDomain::interior); @@ -303,7 +320,10 @@ void CalcHist(Mesh *pm, const Histogram &hist) { const auto val_to_add = binned_var_component == -1 ? 1 : binned_var(b, binned_var_component, k, j, i); - const auto weight = weight_by_vol ? coords.CellVolume(k, j, i) : 1.0; + auto weight = weight_by_vol ? coords.CellVolume(k, j, i) : 1.0; + weight *= weight_var_component == -1 + ? 1.0 + : weight_var(b, weight_var_component, k, j, i); res(y_bin, x_bin) += val_to_add * weight; }); // "reduce" results from scatter view to original view. May be a no-op depending on diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index 6fe08bb3af0f..00793e0f2dfe 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -235,7 +235,10 @@ struct Histogram { // component of variable to be binned. If -1 means no variable is binned but the // histgram is a sample count. int binned_var_component; - bool weight_by_vol; // use volume weighting + bool weight_by_vol; // use volume weighting + std::string weight_var_name; // variable name of variable used as weight + // component of variable to be used as weight. If -1 means no weighting + int weight_var_component; ParArray2D result; // resulting histogram // temp view for histogram reduction for better performance (switches diff --git a/tst/regression/test_suites/output_hdf5/output_hdf5.py b/tst/regression/test_suites/output_hdf5/output_hdf5.py index 857f528c578f..64e04b2c6584 100644 --- a/tst/regression/test_suites/output_hdf5/output_hdf5.py +++ b/tst/regression/test_suites/output_hdf5/output_hdf5.py @@ -224,12 +224,12 @@ def Analyse(self, parameters): x.flatten(), y.flatten(), [[-0.5, -0.25, 0, 0.25, 0.5], [-0.5, -0.1, 0, 0.1, 0.5]], - weights=advected.flatten() * vols.flatten(), + weights=advected.flatten() * vols.flatten() * omadvected.flatten(), ) with h5py.File( - f"advection_{dim}d.out2.histograms.final.hdf", "r" + f"advection_{dim}d.out3.histograms.final.hdf", "r" ) as infile: - hist_parth = infile["3/data"][:] + hist_parth = infile["0/data"][:] # testing slices separately to ensure matching numpy convention all_close = np.allclose(hist_parth[:, 0], hist_np2d[0][:, 0]) all_close &= np.allclose(hist_parth[:, 1], hist_np2d[0][:, 1]) diff --git a/tst/regression/test_suites/output_hdf5/parthinput.advection b/tst/regression/test_suites/output_hdf5/parthinput.advection index 347ff6d75644..e5981dfd77db 100644 --- a/tst/regression/test_suites/output_hdf5/parthinput.advection +++ b/tst/regression/test_suites/output_hdf5/parthinput.advection @@ -74,7 +74,7 @@ dt = 0.25 file_type = histogram dt = 0.25 -num_histograms = 4 +num_histograms = 3 # 1D histogram of a variable, binned by a variable hist0_ndim = 1 @@ -107,14 +107,24 @@ hist2_x_edges_type = list hist2_x_edges_list = 1e-9, 1e-4,1e-1, 2e-1, 5e-1 ,1.0 hist2_binned_variable = HIST_ONES +# A second output block with different dt for histograms +# to double check that writing to different files works + +file_type = histogram +dt = 0.5 + +num_histograms = 1 + # 2D histogram of volume weighted variable according to two coordinates -hist3_ndim = 2 -hist3_x_variable = HIST_COORD_X1 -hist3_x_edges_type = list -hist3_x_edges_list = -0.5, -0.25, 0.0, 0.25, 0.5 -hist3_y_variable = HIST_COORD_X2 -hist3_y_edges_type = list -hist3_y_edges_list = -0.5, -0.1, 0.0, 0.1, 0.5 -hist3_binned_variable = advected -hist3_binned_variable_component = 0 -hist3_weight_by_volume = true +hist0_ndim = 2 +hist0_x_variable = HIST_COORD_X1 +hist0_x_edges_type = list +hist0_x_edges_list = -0.5, -0.25, 0.0, 0.25, 0.5 +hist0_y_variable = HIST_COORD_X2 +hist0_y_edges_type = list +hist0_y_edges_list = -0.5, -0.1, 0.0, 0.1, 0.5 +hist0_binned_variable = advected +hist0_binned_variable_component = 0 +hist0_weight_by_volume = true +hist0_weight_variable = one_minus_advected_sq +hist0_weight_variable_component = 0 From ead3aecc63315c721ad8d3445d366b4533dd8dc2 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 19 Oct 2023 10:53:24 +0200 Subject: [PATCH 34/68] Use direct indexing for lin and log bins --- src/outputs/histogram.cpp | 85 ++++++++++++++++++++++++++++++--------- src/outputs/outputs.hpp | 7 ++++ 2 files changed, 73 insertions(+), 19 deletions(-) diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp index 8b2a04bd2052..e976884f2549 100644 --- a/src/outputs/histogram.cpp +++ b/src/outputs/histogram.cpp @@ -62,13 +62,19 @@ using namespace OutputUtils; namespace HistUtil { -ParArray1D GetEdges(ParameterInput *pin, const std::string &block_name, - const std::string &prefix) { +// Parse edges from input parameters. Returns the edges themselves (to be used as list for +// arbitrary bins) as well as min and step sizes (potentially in log space) for direct +// indexing. +std::tuple, EdgeType, Real, Real> +GetEdges(ParameterInput *pin, const std::string &block_name, const std::string &prefix) { std::vector edges_in; + auto edge_type = EdgeType::Undefined; + auto edge_min = std::numeric_limits::quiet_NaN(); + auto edge_dbin = std::numeric_limits::quiet_NaN(); const auto edge_type_str = pin->GetString(block_name, prefix + "type"); if (edge_type_str == "lin" || edge_type_str == "log") { - const auto edge_min = pin->GetReal(block_name, prefix + "min"); + edge_min = pin->GetReal(block_name, prefix + "min"); const auto edge_max = pin->GetReal(block_name, prefix + "max"); PARTHENON_REQUIRE_THROWS(edge_max > edge_min, "Histogram max needs to be larger than min.") @@ -77,20 +83,24 @@ ParArray1D GetEdges(ParameterInput *pin, const std::string &block_name, PARTHENON_REQUIRE_THROWS(edge_num_bins >= 1, "Need at least one bin for histogram."); if (edge_type_str == "lin") { - auto dbin = (edge_max - edge_min) / (edge_num_bins); + edge_type = EdgeType::Lin; + edge_dbin = (edge_max - edge_min) / (edge_num_bins); for (int i = 0; i < edge_num_bins; i++) { - edges_in.emplace_back(edge_min + i * dbin); + edges_in.emplace_back(edge_min + i * edge_dbin); } edges_in.emplace_back(edge_max); } else if (edge_type_str == "log") { + edge_type = EdgeType::Log; PARTHENON_REQUIRE_THROWS( edge_min > 0.0 && edge_max > 0.0, "Log binning for negative values not implemented. However, you can specify " "arbitrary bin edges through the 'list' edge type.") - auto dbin = (std::log10(edge_max) - std::log10(edge_min)) / (edge_num_bins); + // override start with log value for direct indexing in histogram kernel + edge_min = std::log10(edge_min); + edge_dbin = (std::log10(edge_max) - edge_min) / (edge_num_bins); for (int i = 0; i < edge_num_bins; i++) { - edges_in.emplace_back(std::pow(10., std::log10(edge_min) + i * dbin)); + edges_in.emplace_back(std::pow(10., edge_min + i * edge_dbin)); } edges_in.emplace_back(edge_max); } else { @@ -98,6 +108,7 @@ ParArray1D GetEdges(ParameterInput *pin, const std::string &block_name, } } else if (edge_type_str == "list") { + edge_type = EdgeType::List; edges_in = pin->GetVector(block_name, prefix + "list"); // required by binning index function PARTHENON_REQUIRE_THROWS(std::is_sorted(edges_in.begin(), edges_in.end()), @@ -115,7 +126,11 @@ ParArray1D GetEdges(ParameterInput *pin, const std::string &block_name, edges_h(i) = edges_in[i]; } Kokkos::deep_copy(edges, edges_h); - return edges; + + PARTHENON_REQUIRE_THROWS( + edge_type != EdgeType::Undefined, + "Edge type not set and it's unclear how this code was triggered..."); + return {edges, edge_type, edge_min, edge_dbin}; } Histogram::Histogram(ParameterInput *pin, const std::string &block_name, @@ -144,7 +159,8 @@ Histogram::Histogram(ParameterInput *pin, const std::string &block_name, "Negative component indices are not supported"); } - x_edges = GetEdges(pin, block_name, prefix + "x_edges_"); + std::tie(x_edges, x_edges_type, x_edge_min, x_edge_dbin) = + GetEdges(pin, block_name, prefix + "x_edges_"); // For 1D profile default initalize y variables y_var_name = ""; @@ -172,7 +188,8 @@ Histogram::Histogram(ParameterInput *pin, const std::string &block_name, "Negative component indices are not supported"); } - y_edges = GetEdges(pin, block_name, prefix + "y_edges_"); + std::tie(y_edges, y_edges_type, y_edge_min, y_edge_dbin) = + GetEdges(pin, block_name, prefix + "y_edges_"); } else { y_edges = ParArray1D(prefix + "y_edges_unused", 0); @@ -223,6 +240,12 @@ void CalcHist(Mesh *pm, const Histogram &hist) { const auto y_var_type = hist.y_var_type; const auto x_edges = hist.x_edges; const auto y_edges = hist.y_edges; + const auto x_edges_type = hist.x_edges_type; + const auto y_edges_type = hist.y_edges_type; + const auto x_edge_min = hist.x_edge_min; + const auto x_edge_dbin = hist.x_edge_dbin; + const auto y_edge_min = hist.y_edge_min; + const auto y_edge_dbin = hist.y_edge_dbin; const auto hist_ndim = hist.ndim; const auto weight_by_vol = hist.weight_by_vol; auto result = hist.result; @@ -286,12 +309,24 @@ void CalcHist(Mesh *pm, const Histogram &hist) { return; } - // if we're on the rightmost edge, directly set last bin, otherwise search - const auto x_bin = x_val == x_edges(x_edges.extent_int(0) - 1) - ? x_edges.extent_int(0) - 2 - : upper_bound(x_edges, x_val) - 1; + int x_bin = -1; + // if we're on the rightmost edge, directly set last bin + if (x_val == x_edges(x_edges.extent_int(0) - 1)) { + x_bin = x_edges.extent_int(0) - 2; + } else { + // for lin and log directly pick index + if (x_edges_type == EdgeType::Lin) { + x_bin = static_cast((x_val - x_edge_min) / x_edge_dbin); + } else if (x_edges_type == EdgeType::Log) { + x_bin = static_cast((Kokkos::log10(x_val) - x_edge_min) / x_edge_dbin); + // otherwise search + } else { + x_bin = upper_bound(x_edges, x_val) - 1; + } + } + PARTHENON_DEBUG_REQUIRE(x_bin >= 0, "Bin not found"); - int y_bin = 0; + int y_bin = -1; if (hist_ndim == 2) { auto y_val = std::numeric_limits::quiet_NaN(); if (y_var_type == VarType::X1) { @@ -311,10 +346,22 @@ void CalcHist(Mesh *pm, const Histogram &hist) { if (y_val < y_edges(0) || y_val > y_edges(y_edges.extent_int(0) - 1)) { return; } - // if we're on the rightmost edge, directly set last bin, otherwise search - y_bin = y_val == y_edges(y_edges.extent_int(0) - 1) - ? y_edges.extent_int(0) - 2 - : upper_bound(y_edges, y_val) - 1; + // if we're on the rightmost edge, directly set last bin + if (y_val == y_edges(y_edges.extent_int(0) - 1)) { + y_bin = y_edges.extent_int(0) - 2; + } else { + // for lin and log directly pick index + if (y_edges_type == EdgeType::Lin) { + y_bin = static_cast((y_val - y_edge_min) / y_edge_dbin); + } else if (y_edges_type == EdgeType::Log) { + y_bin = + static_cast((Kokkos::log10(y_val) - y_edge_min) / y_edge_dbin); + // otherwise search + } else { + y_bin = upper_bound(y_edges, y_val) - 1; + } + } + PARTHENON_DEBUG_REQUIRE(y_bin >= 0, "Bin not found"); } auto res = scatter.access(); const auto val_to_add = binned_var_component == -1 diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index 00793e0f2dfe..eb975ef27e51 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -225,12 +225,19 @@ class PHDF5Output : public OutputType { namespace HistUtil { enum class VarType { X1, X2, X3, R, Var, Unused }; +enum class EdgeType { Lin, Log, List, Undefined }; + struct Histogram { int ndim; // 1D or 2D histogram std::string x_var_name, y_var_name; // variable(s) for bins VarType x_var_type, y_var_type; // type, e.g., coord related or actual field int x_var_component, y_var_component; // components of bin variables (vector) ParArray1D x_edges, y_edges; + EdgeType x_edges_type, y_edges_type; + // Lowest edge and difference between edges. + // Internally used to speed up lookup for log (and lin) bins as otherwise + // two more log10 calls would be required per index. + Real x_edge_min, x_edge_dbin, y_edge_min, y_edge_dbin; std::string binned_var_name; // variable name of variable to be binned // component of variable to be binned. If -1 means no variable is binned but the // histgram is a sample count. From 2ac1c9aa98c379e087b12f5b8a5132cdc2ed15ea Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 19 Oct 2023 11:05:49 +0200 Subject: [PATCH 35/68] Fix test case for direct log indexed bins --- src/outputs/histogram.cpp | 6 +++++- tst/regression/test_suites/output_hdf5/output_hdf5.py | 2 +- tst/regression/test_suites/output_hdf5/parthinput.advection | 6 ++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp index e976884f2549..34779dbf6515 100644 --- a/src/outputs/histogram.cpp +++ b/src/outputs/histogram.cpp @@ -326,7 +326,9 @@ void CalcHist(Mesh *pm, const Histogram &hist) { } PARTHENON_DEBUG_REQUIRE(x_bin >= 0, "Bin not found"); - int y_bin = -1; + // needs to be zero as for the 1D histogram we need 0 as first index of the 2D + // result array + int y_bin = 0; if (hist_ndim == 2) { auto y_val = std::numeric_limits::quiet_NaN(); if (y_var_type == VarType::X1) { @@ -346,6 +348,8 @@ void CalcHist(Mesh *pm, const Histogram &hist) { if (y_val < y_edges(0) || y_val > y_edges(y_edges.extent_int(0) - 1)) { return; } + + y_bin = -1; // reset to impossible value // if we're on the rightmost edge, directly set last bin if (y_val == y_edges(y_edges.extent_int(0) - 1)) { y_bin = y_edges.extent_int(0) - 2; diff --git a/tst/regression/test_suites/output_hdf5/output_hdf5.py b/tst/regression/test_suites/output_hdf5/output_hdf5.py index 64e04b2c6584..1de94b678448 100644 --- a/tst/regression/test_suites/output_hdf5/output_hdf5.py +++ b/tst/regression/test_suites/output_hdf5/output_hdf5.py @@ -206,7 +206,7 @@ def Analyse(self, parameters): analyze_status = False # 1D histogram (simple sampling) with bins defined by a var - hist_np1d = np.histogram(advected, [1e-9, 1e-4, 1e-1, 2e-1, 5e-1, 1e0]) + hist_np1d = np.histogram(advected, np.logspace(-9, 0, 11, endpoint=True)) with h5py.File( f"advection_{dim}d.out2.histograms.final.hdf", "r" ) as infile: diff --git a/tst/regression/test_suites/output_hdf5/parthinput.advection b/tst/regression/test_suites/output_hdf5/parthinput.advection index e5981dfd77db..8442d082e8ec 100644 --- a/tst/regression/test_suites/output_hdf5/parthinput.advection +++ b/tst/regression/test_suites/output_hdf5/parthinput.advection @@ -103,8 +103,10 @@ hist1_binned_variable_component = 0 hist2_ndim = 1 hist2_x_variable = advected hist2_x_variable_component = 0 -hist2_x_edges_type = list -hist2_x_edges_list = 1e-9, 1e-4,1e-1, 2e-1, 5e-1 ,1.0 +hist2_x_edges_type = log +hist2_x_edges_num_bins = 10 +hist2_x_edges_min = 1e-9 +hist2_x_edges_max = 1e0 hist2_binned_variable = HIST_ONES # A second output block with different dt for histograms From bc2684f33ac09732e050a5b64f883353cda54f7c Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 19 Oct 2023 17:00:14 +0200 Subject: [PATCH 36/68] Add doc --- CHANGELOG.md | 1 + doc/sphinx/src/interface/state.rst | 2 +- doc/sphinx/src/outputs.rst | 135 ++++++++++++++++++++++++++++- 3 files changed, 135 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49d929c46ab5..0a94116ccd52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Current develop ### Added (new features/APIs/variables/...) +- [[PR 962]](https://github.com/parthenon-hpc-lab/parthenon/pull/962) Add support for in-situ histograms/profiles - [[PR 907]](https://github.com/parthenon-hpc-lab/parthenon/pull/907) PEP1: Allow subclassing StateDescriptor - [[PR 932]](https://github.com/parthenon-hpc-lab/parthenon/pull/932) Add GetOrAddFlag to metadata - [[PR 931]](https://github.com/parthenon-hpc-lab/parthenon/pull/931) Allow SparsePacks with subsets of blocks diff --git a/doc/sphinx/src/interface/state.rst b/doc/sphinx/src/interface/state.rst index b2a91fa8e4cc..ae183c1cb294 100644 --- a/doc/sphinx/src/interface/state.rst +++ b/doc/sphinx/src/interface/state.rst @@ -81,7 +81,7 @@ several useful features and functions. with specified fields to the ``DataCollection`` objects in ``Mesh`` and ``MeshBlock``. For convenience, the ``Mesh`` class also provides this function, which provides a list of variables gathered from all the package - ``StateDescriptor``s. + ``StateDescriptor``\s. - ``void FillDerivedBlock(MeshBlockData* rc)`` delgates to the ``std::function`` member ``FillDerivedBlock`` if set (defaults to ``nullptr`` and therefore a no-op) that allows an application to provide diff --git a/doc/sphinx/src/outputs.rst b/doc/sphinx/src/outputs.rst index 574d52f4a212..2e1d24fdb3d6 100644 --- a/doc/sphinx/src/outputs.rst +++ b/doc/sphinx/src/outputs.rst @@ -10,7 +10,7 @@ To disable an output block without removing it from the intput file set the block's ``dt < 0.0``. In addition to time base outputs, two additional options to trigger -outputs (applies to HDF5 and restart outputs) exist. +outputs (applies to HDF5, restart and histogram outputs) exist. - Signaling: If ``Parthenon`` catches a signal, e.g., ``SIGALRM`` which is often sent by schedulers such as Slurm to signal a job of @@ -28,7 +28,10 @@ outputs (applies to HDF5 and restart outputs) exist. Note, in both cases the original numbering of the output will be unaffected and the ``final`` and ``now`` files will be overwritten each -time without warning. ## HDF5 +time without warning. + +HDF5 +---- Parthenon allows users to select which fields are captured in the HDF5 (``.phdf``) dumps at runtime. In the input file, include a @@ -158,6 +161,134 @@ This will produce a text file (``.hst``) output file every 1 units of simulation time. The content of the file is determined by the functions enrolled by a specific package, see :ref:`state history output`. +Histograms +---------- + +Parthenon supports calculating flexible 1D and 2D histograms in-situ that +are written to disk in HDF5 format. +Currently supported are + +- 1D and 2D histograms +- binning by variable or coordinate (x1, x2, x3 and radial distance) +- counting samples and or summing a variable +- weighting by volume and/or variable + +The output format follows ``numpy`` convention, so that plotting data +with Python based machinery should be straightfoward (see example below). +In general, histograms are calculated using inclusive left bin edges and +data equal to the rightmost edge is also included in the last bin. + +A ```` block containing one simple and one complex +example might look like:: + + + file_type = histogram # required, sets the output type + dt = 1.0 # required, sets the output interval + num_histograms = 2 # required, specifies how many histograms are defined in this block + + # 1D histogram ("standard", i.e., counting occurance in bin) + hist0_ndim = 1 + hist0_x_variable = advected + hist0_x_variable_component = 0 + hist0_x_edges_type = log + hist0_x_edges_num_bins = 10 + hist0_x_edges_min = 1e-9 + hist0_x_edges_max = 1e0 + hist0_binned_variable = HIST_ONES + + # 2D histogram of volume weighted variable according to two coordinates + hist1_ndim = 2 + hist1_x_variable = HIST_COORD_X1 + hist1_x_edges_type = list + hist1_x_edges_list = -0.5, -0.25, 0.0, 0.25, 0.5 + hist1_y_variable = HIST_COORD_X2 + hist1_y_edges_type = list + hist1_y_edges_list = -0.5, -0.1, 0.0, 0.1, 0.5 + hist1_binned_variable = advected + hist1_binned_variable_component = 0 + hist1_weight_by_volume = true + hist1_weight_variable = one_minus_advected_sq + hist1_weight_variable_component = 0 + +with the following parameters + +- ``num_histograms=INT`` + The number of histograms defined in this block. + All histogram definitions need to be prefix with ``hist#_`` where ``#`` is the + histogram number starting to count from ``0``. + All histograms will be written to the same output file with the "group" in the + output corresponding to the histogram number. +- ``hist#_ndim=INT`` (either ``1`` or ``2``) + Dimensionality of the histogram. +- ``hist#_x_variable=STRING`` (variable name or special coordinate string ``HIST_COORD_X1``, ``HIST_COORD_X2``, ``HIST_COORD_X3`` or ``HIST_COORD_R``) + Variable to be used as bin. If a variable name is given a component has to be specified, too, + see next parameter. + For a scalar variable, the component needs to be specified as ``0`` anyway. + If binning should be done by coordinate the special strings allow to bin by either one + of the three dimensions or by radial distance from the origin. +- ``hist#_x_variable_component=INT`` + Component index of the binning variable. + Used/required only if a non-coordinate variable is used for binning. +- ``hist#_x_edges_type=STRING`` (``lin``, ``log``, or ``list``) + How the bin edges are defined in the first dimension. + For ``lin`` and ``log`` direct indexing is used to determine the bin, which is significantly + faster than specifying the edges via a ``list`` as the latter requires a binary search. +- ``hist#_x_edges_min=FLOAT`` + Minimum value (inclusive) of the bins in the first dim. + Used/required only for ``lin`` and ``log`` edge type. +- ``hist#_x_edges_max=FLOAT`` + Maximum value (inclusive) of the bins in the first dim. + Used/required only for ``lin`` and ``log`` edge type. +- ``hist#_x_edges_num_bins=INT`` (must be ``>=1``) + Number of equally spaced bins between min and max value in the first dim. + Used/required only for ``lin`` and ``log`` edge type. +- ``hist#_x_edges_list=FLOAT,FLOAT,FLOAT,...`` (comma separated list of increasing values) + Arbitrary definition of edge values. + Used/required only for ``list`` edge type. +- ``hist#_y_edges...`` + Same as the ``hist#_x_edges...`` parameters except for being used in the second + dimension for ``ndim=2`` histograms. +- ``hist#_binned_variable=STRING`` (variable name or ``HIST_ONES``) + Variable to be binned. If a variable name is given a component has to be specified, too, + see next parameter. + For a scalar variable, the component needs to be specified as ``0`` anyway. + If sampling (i.e., counting the number of value inside a bin) is to be used the special + string ``HIST_ONES`` can be set. +- ``hist#_binned_variable_component=INT`` + Component index of the variable to be binned. + Used/required only if a variable is binned and not ``HIST_ONES``. +- ``hist#_weight_by_volume=BOOL`` (``true`` or ``false``) + Apply volume weighting to the binned variable. Can be used simultaneously with binning + by a different variable. +- ``hist#_weight_variable=STRING`` + Variable to be used as weight. + Can be used together with volume weighting. + For a scalar variable, the component needs to be specified as ``0`` anyway. +- ``hist#_weight_variable_component=INT`` + Component index of the variable to be used as weight. + +Note, weighting by volume and variable simultaneously might seem counterintuitive, but +easily allows for, e.g., mass-weighted profiles, by enabling weighting by volume and +using a mass density field as additional weight variable. + +The following is a minimal example to plot a 1D and 2D histogram from the output file: + +.. code:: python + + with h5py.File("parthenon.out8.histograms.00040.hdf", "r") as infile: + # 1D histogram + x = infile["0/x_edges"][:] + y = infile["0/data"][:] + plt.plot(x, y) + plt.show() + + # 2D histogram + x = infile["1/x_edges"][:] + y = infile["1/y_edges"][:] + z = infile["1/data"][:].T # note the transpose here (so that the data matches the axis for the pcolormesh) + plt.pcolormesh(x,y,z,) + plt.show() + Ascent (optional) ----------------- From 081b14c6379a4adbb7beacae0cd549f78f36681b Mon Sep 17 00:00:00 2001 From: jdolence Date: Tue, 24 Oct 2023 12:10:21 -0600 Subject: [PATCH 37/68] INLINE to FORCEINLINE on all par_for_inner functions --- src/kokkos_abstraction.hpp | 54 +++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/src/kokkos_abstraction.hpp b/src/kokkos_abstraction.hpp index cabae1273b6f..f265fd22cc32 100644 --- a/src/kokkos_abstraction.hpp +++ b/src/kokkos_abstraction.hpp @@ -707,7 +707,7 @@ inline void par_for_outer(OuterLoopPatternTeams, const std::string &name, // Inner parallel loop using TeamThreadRange template -KOKKOS_INLINE_FUNCTION void +KOKKOS_FORCEINLINE_FUNCTION void par_for_inner(InnerLoopPatternTTR, team_mbr_t team_member, const int ll, const int lu, const int ml, const int mu, const int nl, const int nu, const int kl, const int ku, const int jl, const int ju, const int il, const int iu, @@ -741,7 +741,7 @@ par_for_inner(InnerLoopPatternTTR, team_mbr_t team_member, const int ll, const i }); } template -KOKKOS_INLINE_FUNCTION void +KOKKOS_FORCEINLINE_FUNCTION void par_for_inner(InnerLoopPatternTTR, team_mbr_t team_member, const int ml, const int mu, const int nl, const int nu, const int kl, const int ku, const int jl, const int ju, const int il, const int iu, const Function &function) { @@ -770,7 +770,7 @@ par_for_inner(InnerLoopPatternTTR, team_mbr_t team_member, const int ml, const i }); } template -KOKKOS_INLINE_FUNCTION void +KOKKOS_FORCEINLINE_FUNCTION void par_for_inner(InnerLoopPatternTTR, team_mbr_t team_member, const int nl, const int nu, const int kl, const int ku, const int jl, const int ju, const int il, const int iu, const Function &function) { @@ -795,10 +795,10 @@ par_for_inner(InnerLoopPatternTTR, team_mbr_t team_member, const int nl, const i }); } template -KOKKOS_INLINE_FUNCTION void par_for_inner(InnerLoopPatternTTR, team_mbr_t team_member, - const int kl, const int ku, const int jl, - const int ju, const int il, const int iu, - const Function &function) { +KOKKOS_FORCEINLINE_FUNCTION void +par_for_inner(InnerLoopPatternTTR, team_mbr_t team_member, const int kl, const int ku, + const int jl, const int ju, const int il, const int iu, + const Function &function) { const int Nk = ku - kl + 1; const int Nj = ju - jl + 1; const int Ni = iu - il + 1; @@ -815,9 +815,9 @@ KOKKOS_INLINE_FUNCTION void par_for_inner(InnerLoopPatternTTR, team_mbr_t team_m }); } template -KOKKOS_INLINE_FUNCTION void par_for_inner(InnerLoopPatternTTR, team_mbr_t team_member, - const int jl, const int ju, const int il, - const int iu, const Function &function) { +KOKKOS_FORCEINLINE_FUNCTION void +par_for_inner(InnerLoopPatternTTR, team_mbr_t team_member, const int jl, const int ju, + const int il, const int iu, const Function &function) { const int Nj = ju - jl + 1; const int Ni = iu - il + 1; const int NjNi = Nj * Ni; @@ -828,22 +828,22 @@ KOKKOS_INLINE_FUNCTION void par_for_inner(InnerLoopPatternTTR, team_mbr_t team_m }); } template -KOKKOS_INLINE_FUNCTION void par_for_inner(InnerLoopPatternTTR, team_mbr_t team_member, - const int il, const int iu, - const Function &function) { +KOKKOS_FORCEINLINE_FUNCTION void par_for_inner(InnerLoopPatternTTR, + team_mbr_t team_member, const int il, + const int iu, const Function &function) { Kokkos::parallel_for(Kokkos::TeamThreadRange(team_member, il, iu + 1), function); } // Inner parallel loop using TeamVectorRange template -KOKKOS_INLINE_FUNCTION void par_for_inner(InnerLoopPatternTVR, team_mbr_t team_member, - const int il, const int iu, - const Function &function) { +KOKKOS_FORCEINLINE_FUNCTION void par_for_inner(InnerLoopPatternTVR, + team_mbr_t team_member, const int il, + const int iu, const Function &function) { Kokkos::parallel_for(Kokkos::TeamVectorRange(team_member, il, iu + 1), function); } // Inner parallel loop using FOR SIMD template -KOKKOS_INLINE_FUNCTION void +KOKKOS_FORCEINLINE_FUNCTION void par_for_inner(InnerLoopPatternSimdFor, team_mbr_t team_member, const int nl, const int nu, const int kl, const int ku, const int jl, const int ju, const int il, const int iu, const Function &function) { @@ -859,10 +859,10 @@ par_for_inner(InnerLoopPatternSimdFor, team_mbr_t team_member, const int nl, con } } template -KOKKOS_INLINE_FUNCTION void par_for_inner(InnerLoopPatternSimdFor, team_mbr_t team_member, - const int kl, const int ku, const int jl, - const int ju, const int il, const int iu, - const Function &function) { +KOKKOS_FORCEINLINE_FUNCTION void +par_for_inner(InnerLoopPatternSimdFor, team_mbr_t team_member, const int kl, const int ku, + const int jl, const int ju, const int il, const int iu, + const Function &function) { for (int k = kl; k <= ku; ++k) { for (int j = jl; j <= ju; ++j) { #pragma omp simd @@ -873,9 +873,9 @@ KOKKOS_INLINE_FUNCTION void par_for_inner(InnerLoopPatternSimdFor, team_mbr_t te } } template -KOKKOS_INLINE_FUNCTION void par_for_inner(InnerLoopPatternSimdFor, team_mbr_t team_member, - const int jl, const int ju, const int il, - const int iu, const Function &function) { +KOKKOS_FORCEINLINE_FUNCTION void +par_for_inner(InnerLoopPatternSimdFor, team_mbr_t team_member, const int jl, const int ju, + const int il, const int iu, const Function &function) { for (int j = jl; j <= ju; ++j) { #pragma omp simd for (int i = il; i <= iu; i++) { @@ -884,9 +884,9 @@ KOKKOS_INLINE_FUNCTION void par_for_inner(InnerLoopPatternSimdFor, team_mbr_t te } } template -KOKKOS_INLINE_FUNCTION void par_for_inner(InnerLoopPatternSimdFor, team_mbr_t team_member, - const int il, const int iu, - const Function &function) { +KOKKOS_FORCEINLINE_FUNCTION void par_for_inner(InnerLoopPatternSimdFor, + team_mbr_t team_member, const int il, + const int iu, const Function &function) { #pragma omp simd for (int i = il; i <= iu; i++) { function(i); From 5145bd8887fc01f9d49562a24e2fe14d5e08d847 Mon Sep 17 00:00:00 2001 From: jdolence Date: Tue, 24 Oct 2023 12:19:41 -0600 Subject: [PATCH 38/68] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49d929c46ab5..978bccbf19c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ - [[PR 890]](https://github.com/parthenon-hpc-lab/parthenon/pull/890) Fix bugs in sparse communication and prolongation ### Infrastructure (changes irrelevant to downstream codes) +- [[PR 967]](https://github.com/parthenon-hpc-lab/parthenon/pull/967) Change INLINE to FORCEINLINE on par_for_inner overloads - [[PR 938]](https://github.com/parthenon-hpc-lab/parthenon/pull/938) Restructure buffer packing/unpacking kernel hierarchical parallelism - [[PR 944]](https://github.com/parthenon-hpc-lab/parthenon/pull/944) Move sparse pack identifier creation to descriptor - [[PR 904]](https://github.com/parthenon-hpc-lab/parthenon/pull/904) Move to prolongation/restriction in one for AMR and communicate non-cell centered fields From 790cebb6488d4fe7b24bef31c7a4de4bbb1a5a3e Mon Sep 17 00:00:00 2001 From: jdolence Date: Tue, 24 Oct 2023 12:43:20 -0600 Subject: [PATCH 39/68] update copyright --- src/kokkos_abstraction.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kokkos_abstraction.hpp b/src/kokkos_abstraction.hpp index f265fd22cc32..bfaf6a247a66 100644 --- a/src/kokkos_abstraction.hpp +++ b/src/kokkos_abstraction.hpp @@ -1,9 +1,9 @@ //======================================================================================== // Parthenon performance portable AMR framework -// Copyright(C) 2020-2022 The Parthenon collaboration +// Copyright(C) 2020-2023 The Parthenon collaboration // Licensed under the 3-clause BSD License, see LICENSE file for details //======================================================================================== -// (C) (or copyright) 2020-2022. Triad National Security, LLC. All rights reserved. +// (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved. // // This program was produced under U.S. Government contract 89233218CNA000001 // for Los Alamos National Laboratory (LANL), which is operated by Triad From 176d7c35f7549cf56e39cc1ad9255b5713390c4f Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Tue, 31 Oct 2023 11:14:35 -0600 Subject: [PATCH 40/68] Fix comment --- example/poisson_gmg/poisson_driver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/poisson_gmg/poisson_driver.cpp b/example/poisson_gmg/poisson_driver.cpp index 347b9a801bdf..2aeb1d6116cd 100644 --- a/example/poisson_gmg/poisson_driver.cpp +++ b/example/poisson_gmg/poisson_driver.cpp @@ -182,7 +182,7 @@ TaskID PoissonDriver::AddMultiGridTasksLevel(TaskRegion ®ion, TaskList &tl, bool multilevel = (level != min_level); auto &md = pmesh->gmg_mesh_data[level].GetOrAdd(level, "base", partition); - // 0. Receive residual from coarser level if there is one + // 0. Receive residual from finer level if there is one auto set_from_finer = dependency; if (level < max_level) { // Fill fields with restricted values From fb1737084fd19a31fb518f59ff5a94f4390885e5 Mon Sep 17 00:00:00 2001 From: Ben Ryan Date: Tue, 31 Oct 2023 13:19:58 -0600 Subject: [PATCH 41/68] Convert numpy arrays to ASCII strings (#956) * Convert numpy arrays to ASCII strings * format * Dont update submodule versions * Switch to fix inside parthenon * comment * upgold * hash issue * The original hash was right? --------- Co-authored-by: Jonah Miller --- CMakeLists.txt | 4 ++-- src/outputs/parthenon_hdf5.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 33c19c34256d..fe9b1f7e3d86 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,9 +62,9 @@ include(cmake/Format.cmake) include(cmake/Lint.cmake) # regression test reference data -set(REGRESSION_GOLD_STANDARD_VER 19 CACHE STRING "Version of gold standard to download and use") +set(REGRESSION_GOLD_STANDARD_VER 20 CACHE STRING "Version of gold standard to download and use") set(REGRESSION_GOLD_STANDARD_HASH - "SHA512=e1d1a06b9cf9b761d42d0b6b241056ac75658db90138b6b867b1ca7ead4308af4f980285af092b40aee1dbbfb68b4e8cb15efcc9b83d7930c18bf992ae95c729" + "SHA512=e5e421f3c0be01e4708965542bb8b1b79b5c96de97091e46972e375c7616588d026a9a8e29226d9c7ef75346bc859fd9af72acdc7e95e0d783b5ef29aa4630b1" CACHE STRING "Hash of default gold standard file to download") option(REGRESSION_GOLD_STANDARD_SYNC "Automatically sync gold standard files." ON) diff --git a/src/outputs/parthenon_hdf5.cpp b/src/outputs/parthenon_hdf5.cpp index 95b89b93b627..004054d8421a 100644 --- a/src/outputs/parthenon_hdf5.cpp +++ b/src/outputs/parthenon_hdf5.cpp @@ -820,7 +820,7 @@ HDF5GetAttributeInfo(hid_t location, const std::string &name, H5A &attr) { // template specializations for std::string and bool void HDF5WriteAttribute(const std::string &name, const std::string &value, hid_t location) { - HDF5WriteAttribute(name, value.size(), value.c_str(), location); + HDF5WriteAttribute(name, value.c_str(), location); } template <> From 285739417a0c75eb0652c235d649f6ee492974a0 Mon Sep 17 00:00:00 2001 From: Ben Ryan Date: Wed, 1 Nov 2023 10:22:42 -0600 Subject: [PATCH 42/68] Allow leading whitespace in input parameters (#965) * Fixed the bug * CHANGELOG, cleanup * Add whitespace characters to test input to confirm bug is fixed --------- Co-authored-by: Jonah Miller Co-authored-by: Philipp Grete --- CHANGELOG.md | 1 + src/parameter_input.cpp | 2 +- .../advection_performance/parthinput.advection_performance | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 978bccbf19c0..181a58b2698c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - [[PR 868]](https://github.com/parthenon-hpc-lab/parthenon/pull/868) Add block-local face, edge, and nodal fields and allow for packing ### Changed (changing behavior/API/variables/...) +- [[PR 965]](https://github.com/parthenon-hpc-lab/parthenon/pull/965) Allow leading whitespace in input parameters - [[PR 926]](https://github.com/parthenon-hpc-lab/parthenon/pull/926) Internal refinement op registration - [[PR 897]](https://github.com/parthenon-hpc-lab/parthenon/pull/897) Deflate compression filter is not called any more if compression is soft disabled - [[PR 896]](https://github.com/parthenon-hpc-lab/parthenon/pull/896) Update Kokkos integration to support installed version. Use `serial` (flat MPI) host parallelization by default (instead of OpenMP) diff --git a/src/parameter_input.cpp b/src/parameter_input.cpp index b9fd9546f2b6..45012e0912bd 100644 --- a/src/parameter_input.cpp +++ b/src/parameter_input.cpp @@ -315,7 +315,7 @@ bool ParameterInput::ParseLine(InputBlock *pib, std::string line, std::string &n name.assign(line, first_char, len); last_char = name.find_last_not_of(" "); name.erase(last_char + 1, std::string::npos); - line.erase(0, len + 1); + line.erase(0, equal_char + 1); } cont_char = line.find_first_of("&"); // find "&" continuation character diff --git a/tst/regression/test_suites/advection_performance/parthinput.advection_performance b/tst/regression/test_suites/advection_performance/parthinput.advection_performance index 4dfe731cf497..5aea8cbee069 100644 --- a/tst/regression/test_suites/advection_performance/parthinput.advection_performance +++ b/tst/regression/test_suites/advection_performance/parthinput.advection_performance @@ -49,8 +49,8 @@ nx2 = 64 nx3 = 64 -tlim = 1.0 -nlim = 20 + tlim = 1.0 # Test that leading whitespace is correctly sanitized + nlim = 20 # Test that leading tab is correctly sanitized integrator = rk2 perf_cycle_offset = 2 From 06657a4e3358b46f238d16aff9f3e6391e6586a8 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Mon, 6 Nov 2023 12:47:59 -0700 Subject: [PATCH 43/68] Add hook for UserWorkBeforeLoop --- doc/sphinx/src/interface/state.rst | 7 +++++++ example/advection/advection_package.cpp | 10 ++++++++++ example/advection/advection_package.hpp | 1 + src/application_input.hpp | 1 + src/driver/driver.cpp | 10 ++++++++++ src/interface/state_descriptor.hpp | 13 ++++++++++--- src/mesh/mesh.cpp | 2 ++ src/mesh/mesh.hpp | 1 + 8 files changed, 42 insertions(+), 3 deletions(-) diff --git a/doc/sphinx/src/interface/state.rst b/doc/sphinx/src/interface/state.rst index b2a91fa8e4cc..0841e8c5a1c7 100644 --- a/doc/sphinx/src/interface/state.rst +++ b/doc/sphinx/src/interface/state.rst @@ -112,6 +112,13 @@ several useful features and functions. deletgates to the ``std::function`` member ``PostStepDiagnosticsMesh`` if set (defaults to ``nullptr`` an therefore a no-op) to print diagnostics after the time-integration advance +- ``void UserWorkBeforeLoopMesh(Mesh *, ParameterInput *pin, SimTime + &tm)`` performs a per-package, mesh-wide calculation after the mesh + has been generated, and problem generators called, but before any + time evolution. This work is done both on first initialization and + on restart. If you would like to avoid doing the work upon restart, + you can check for the const ``is_restart`` member field of the ``Mesh`` + object. The reasoning for providing ``FillDerived*`` and ``EstimateTimestep*`` function pointers appropriate for usage with both ``MeshData`` and diff --git a/example/advection/advection_package.cpp b/example/advection/advection_package.cpp index f618de1758c3..5af3783716f8 100644 --- a/example/advection/advection_package.cpp +++ b/example/advection/advection_package.cpp @@ -13,12 +13,14 @@ #include #include +#include #include #include #include #include #include +#include #include #include "advection_package.hpp" @@ -219,6 +221,14 @@ std::shared_ptr Initialize(ParameterInput *pin) { return pkg; } +void AdvectionGreetings(Mesh *pmesh, ParameterInput *pin, SimTime &tm) { + if (GLobals::my_rank == 0) { + std::cout << "Hello from the advection package in the advection example!\n" + << "This run is a restart: " << pmesh->is_restart + << std::endl; + } +} + AmrTag CheckRefinement(MeshBlockData *rc) { // refine on advected, for example. could also be a derived quantity auto pmb = rc->GetBlockPointer(); diff --git a/example/advection/advection_package.hpp b/example/advection/advection_package.hpp index 7b529308f793..24314db0c2d8 100644 --- a/example/advection/advection_package.hpp +++ b/example/advection/advection_package.hpp @@ -21,6 +21,7 @@ namespace advection_package { using namespace parthenon::package::prelude; std::shared_ptr Initialize(ParameterInput *pin); + void AdvectionGreetings(Mesh *pmes, ParameterInput *pin, SimTime &tm); AmrTag CheckRefinement(MeshBlockData *rc); void PreFill(MeshBlockData *rc); void SquareIt(MeshBlockData *rc); diff --git a/src/application_input.hpp b/src/application_input.hpp index 543cb7a5100f..a9ec96c1551e 100644 --- a/src/application_input.hpp +++ b/src/application_input.hpp @@ -48,6 +48,7 @@ struct ApplicationInput { PostStepDiagnosticsInLoop = nullptr; std::function UserWorkAfterLoop = nullptr; + std::function UserWorkBeforeLoop = nullptr; BValFunc boundary_conditions[BOUNDARY_NFACES] = {nullptr}; SBValFunc swarm_boundary_conditions[BOUNDARY_NFACES] = {nullptr}; diff --git a/src/driver/driver.cpp b/src/driver/driver.cpp index 84371eeaeb36..8314335f3610 100644 --- a/src/driver/driver.cpp +++ b/src/driver/driver.cpp @@ -75,6 +75,16 @@ DriverStatus EvolutionDriver::Execute() { // Defaults must be set across all ranks DumpInputParameters(); + // Before loop do work + // App input version + if (app_input->UserWorkBeforeLoop != nullptr) { + app_input->UserWorkBeforeLoop(pmesh, pinput, tm); + } + // packages version + for (auto &[name, pkg] : pmesh->packages.AllPackages()) { + pkg->UserWorkBeforeLoop(pmesh, pinput, tm); + } + Kokkos::Profiling::pushRegion("Driver_Main"); while (tm.KeepGoing()) { if (Globals::my_rank == 0) OutputCycleDiagnostics(); diff --git a/src/interface/state_descriptor.hpp b/src/interface/state_descriptor.hpp index 5bb37ea83b60..b52ab744cbf6 100644 --- a/src/interface/state_descriptor.hpp +++ b/src/interface/state_descriptor.hpp @@ -217,17 +217,18 @@ class StateDescriptor { // one can pass in a reference to a SparsePool or arguments that match one of the // SparsePool constructors template - bool AddSparsePool(Args &&...args) { + bool AddSparsePool(Args &&... args) { return AddSparsePoolImpl(SparsePool(std::forward(args)...)); } template - bool AddSparsePool(const std::string &base_name, const Metadata &m_in, Args &&...args) { + bool AddSparsePool(const std::string &base_name, const Metadata &m_in, + Args &&... args) { Metadata m = m_in; // so we can modify it if (!m.IsSet(GetMetadataFlag())) m.Set(GetMetadataFlag()); return AddSparsePoolImpl(SparsePool(base_name, m, std::forward(args)...)); } template - bool AddSparsePool(const Metadata &m_in, Args &&...args) { + bool AddSparsePool(const Metadata &m_in, Args &&... args) { return AddSparsePool(T::name(), m_in, std::forward(args)...); } @@ -406,6 +407,10 @@ class StateDescriptor { if (InitNewlyAllocatedVarsBlock != nullptr) return InitNewlyAllocatedVarsBlock(rc); } + void UserWorkBeforeLoop(Mesh *pmesh, ParameterInput *pin, SimTime &tm) const { + if (UserWorkBeforeLoopMesh != nullptr) return UserWorkBeforeLoopMesh(pmesh, pin, tm); + } + std::vector> amr_criteria; std::function *rc)> PreCommFillDerivedBlock = nullptr; @@ -416,6 +421,8 @@ class StateDescriptor { std::function *rc)> PostFillDerivedMesh = nullptr; std::function *rc)> FillDerivedBlock = nullptr; std::function *rc)> FillDerivedMesh = nullptr; + std::function UserWorkBeforeLoopMesh = + nullptr; std::function *rc)> PreStepDiagnosticsMesh = nullptr; diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index ed81e3e4e219..414ea69fadf5 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -64,6 +64,7 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages, int mesh_test) : // public members: modified(true), + is_restart(false), // aggregate initialization of RegionSize struct: mesh_size({pin->GetReal("parthenon/mesh", "x1min"), pin->GetReal("parthenon/mesh", "x2min"), @@ -485,6 +486,7 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, RestartReader &rr, // aggregate initialization of RegionSize struct: // (will be overwritten by memcpy from restart file, in this case) modified(true), + is_restart(true), // aggregate initialization of RegionSize struct: mesh_size({pin->GetReal("parthenon/mesh", "x1min"), pin->GetReal("parthenon/mesh", "x2min"), diff --git a/src/mesh/mesh.hpp b/src/mesh/mesh.hpp index ad0d1a85539c..15ac8666a50c 100644 --- a/src/mesh/mesh.hpp +++ b/src/mesh/mesh.hpp @@ -94,6 +94,7 @@ class Mesh { // data bool modified; + const bool is_restart; RegionSize mesh_size; BoundaryFlag mesh_bcs[BOUNDARY_NFACES]; const int ndim; // number of dimensions From d75771263651705404b48bb7c0ff273219a160a0 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Mon, 6 Nov 2023 12:55:55 -0700 Subject: [PATCH 44/68] formatting --- example/advection/advection_package.cpp | 7 +++---- example/advection/advection_package.hpp | 2 +- src/interface/state_descriptor.hpp | 7 +++---- src/mesh/mesh.cpp | 6 ++---- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/example/advection/advection_package.cpp b/example/advection/advection_package.cpp index 5af3783716f8..1aef47b79908 100644 --- a/example/advection/advection_package.cpp +++ b/example/advection/advection_package.cpp @@ -221,11 +221,10 @@ std::shared_ptr Initialize(ParameterInput *pin) { return pkg; } -void AdvectionGreetings(Mesh *pmesh, ParameterInput *pin, SimTime &tm) { - if (GLobals::my_rank == 0) { +void AdvectionGreetings(Mesh *pmesh, ParameterInput *pin, parthenon::SimTime &tm) { + if (parthenon::Globals::my_rank == 0) { std::cout << "Hello from the advection package in the advection example!\n" - << "This run is a restart: " << pmesh->is_restart - << std::endl; + << "This run is a restart: " << pmesh->is_restart << std::endl; } } diff --git a/example/advection/advection_package.hpp b/example/advection/advection_package.hpp index 24314db0c2d8..1413aa2049fe 100644 --- a/example/advection/advection_package.hpp +++ b/example/advection/advection_package.hpp @@ -21,7 +21,7 @@ namespace advection_package { using namespace parthenon::package::prelude; std::shared_ptr Initialize(ParameterInput *pin); - void AdvectionGreetings(Mesh *pmes, ParameterInput *pin, SimTime &tm); +void AdvectionGreetings(Mesh *pmes, ParameterInput *pin, parthenon::SimTime &tm); AmrTag CheckRefinement(MeshBlockData *rc); void PreFill(MeshBlockData *rc); void SquareIt(MeshBlockData *rc); diff --git a/src/interface/state_descriptor.hpp b/src/interface/state_descriptor.hpp index b52ab744cbf6..41a5ee2d3d4e 100644 --- a/src/interface/state_descriptor.hpp +++ b/src/interface/state_descriptor.hpp @@ -217,18 +217,17 @@ class StateDescriptor { // one can pass in a reference to a SparsePool or arguments that match one of the // SparsePool constructors template - bool AddSparsePool(Args &&... args) { + bool AddSparsePool(Args &&...args) { return AddSparsePoolImpl(SparsePool(std::forward(args)...)); } template - bool AddSparsePool(const std::string &base_name, const Metadata &m_in, - Args &&... args) { + bool AddSparsePool(const std::string &base_name, const Metadata &m_in, Args &&...args) { Metadata m = m_in; // so we can modify it if (!m.IsSet(GetMetadataFlag())) m.Set(GetMetadataFlag()); return AddSparsePoolImpl(SparsePool(base_name, m, std::forward(args)...)); } template - bool AddSparsePool(const Metadata &m_in, Args &&... args) { + bool AddSparsePool(const Metadata &m_in, Args &&...args) { return AddSparsePool(T::name(), m_in, std::forward(args)...); } diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index 414ea69fadf5..958e2ffa5d77 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -63,8 +63,7 @@ namespace parthenon { Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages, int mesh_test) : // public members: - modified(true), - is_restart(false), + modified(true), is_restart(false), // aggregate initialization of RegionSize struct: mesh_size({pin->GetReal("parthenon/mesh", "x1min"), pin->GetReal("parthenon/mesh", "x2min"), @@ -485,8 +484,7 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, RestartReader &rr, : // public members: // aggregate initialization of RegionSize struct: // (will be overwritten by memcpy from restart file, in this case) - modified(true), - is_restart(true), + modified(true), is_restart(true), // aggregate initialization of RegionSize struct: mesh_size({pin->GetReal("parthenon/mesh", "x1min"), pin->GetReal("parthenon/mesh", "x2min"), From 281994a47186f2f5624e87e24b84cbfcfd937c55 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Mon, 6 Nov 2023 12:56:36 -0700 Subject: [PATCH 45/68] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 181a58b2698c..fb06582f3087 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Current develop ### Added (new features/APIs/variables/...) +- [[PR 971]](https://github.com/parthenon-hpc-lab/parthenon/pull/971) Add UserWorkBeforeLoop - [[PR 907]](https://github.com/parthenon-hpc-lab/parthenon/pull/907) PEP1: Allow subclassing StateDescriptor - [[PR 932]](https://github.com/parthenon-hpc-lab/parthenon/pull/932) Add GetOrAddFlag to metadata - [[PR 931]](https://github.com/parthenon-hpc-lab/parthenon/pull/931) Allow SparsePacks with subsets of blocks From 1387606c6eea13a8995d456ba772016903af0089 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Mon, 6 Nov 2023 13:25:22 -0700 Subject: [PATCH 46/68] oops forgot to use hook --- example/advection/advection_package.cpp | 6 ++++-- example/advection/advection_package.hpp | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/example/advection/advection_package.cpp b/example/advection/advection_package.cpp index 1aef47b79908..49200d0da87c 100644 --- a/example/advection/advection_package.cpp +++ b/example/advection/advection_package.cpp @@ -1,5 +1,5 @@ //======================================================================================== -// (C) (or copyright) 2020-2021. Triad National Security, LLC. All rights reserved. +// (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved. // // This program was produced under U.S. Government contract 89233218CNA000001 for Los // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC @@ -217,6 +217,7 @@ std::shared_ptr Initialize(ParameterInput *pin) { } pkg->CheckRefinementBlock = CheckRefinement; pkg->EstimateTimestepBlock = EstimateTimestepBlock; + pkg->UserWorkBeforeLoopMesh = AdvectionGreetings; return pkg; } @@ -224,7 +225,8 @@ std::shared_ptr Initialize(ParameterInput *pin) { void AdvectionGreetings(Mesh *pmesh, ParameterInput *pin, parthenon::SimTime &tm) { if (parthenon::Globals::my_rank == 0) { std::cout << "Hello from the advection package in the advection example!\n" - << "This run is a restart: " << pmesh->is_restart << std::endl; + << "This run is a restart: " << pmesh->is_restart << "\n" + << std::endl; } } diff --git a/example/advection/advection_package.hpp b/example/advection/advection_package.hpp index 1413aa2049fe..f1ac63471c16 100644 --- a/example/advection/advection_package.hpp +++ b/example/advection/advection_package.hpp @@ -1,5 +1,5 @@ //======================================================================================== -// (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved. +// (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved. // // This program was produced under U.S. Government contract 89233218CNA000001 for Los // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC From 1e42f76f9e3576d064168b4de9d8469b6b89e7c8 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Mon, 6 Nov 2023 14:00:51 -0700 Subject: [PATCH 47/68] typo --- example/advection/advection_package.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/advection/advection_package.hpp b/example/advection/advection_package.hpp index f1ac63471c16..85d1d6371e5a 100644 --- a/example/advection/advection_package.hpp +++ b/example/advection/advection_package.hpp @@ -21,7 +21,7 @@ namespace advection_package { using namespace parthenon::package::prelude; std::shared_ptr Initialize(ParameterInput *pin); -void AdvectionGreetings(Mesh *pmes, ParameterInput *pin, parthenon::SimTime &tm); +void AdvectionGreetings(Mesh *pmesh, ParameterInput *pin, parthenon::SimTime &tm); AmrTag CheckRefinement(MeshBlockData *rc); void PreFill(MeshBlockData *rc); void SquareIt(MeshBlockData *rc); From 6472537b2189dc2e548032c15e1f1e46f7bce90b Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 7 Nov 2023 11:05:54 +0100 Subject: [PATCH 48/68] Error check for edge case --- src/outputs/histogram.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp index 34779dbf6515..ff5462780505 100644 --- a/src/outputs/histogram.cpp +++ b/src/outputs/histogram.cpp @@ -113,8 +113,8 @@ GetEdges(ParameterInput *pin, const std::string &block_name, const std::string & // required by binning index function PARTHENON_REQUIRE_THROWS(std::is_sorted(edges_in.begin(), edges_in.end()), "Bin edges must be in order."); - PARTHENON_REQUIRE_THROWS(edges_in.size() >= 2, - "Need at least one bin, i.e., two edges."); + PARTHENON_REQUIRE_THROWS(edges_in.size() >= 2 && edges_in[1] > edges_in[0], + "Need at least one bin, i.e., two distinct edges."); } else { PARTHENON_THROW( From e84bf9ad7b01488c0aae7326b84cd391cf4d5b68 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 7 Nov 2023 11:11:45 +0100 Subject: [PATCH 49/68] Remove duplciated depdency check --- src/outputs/histogram.cpp | 2 -- tst/regression/CMakeLists.txt | 1 - tst/unit/test_upper_bound.cpp | 1 - 3 files changed, 4 deletions(-) diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp index ff5462780505..362f92533102 100644 --- a/src/outputs/histogram.cpp +++ b/src/outputs/histogram.cpp @@ -24,7 +24,6 @@ #include "kokkos_abstraction.hpp" #include "parameter_input.hpp" #include "parthenon_array_generic.hpp" -#include "utils/error_checking.hpp" // Only proceed if HDF5 output enabled #ifdef ENABLE_HDF5 @@ -44,7 +43,6 @@ // Parthenon headers #include "coordinates/coordinates.hpp" #include "defs.hpp" -#include "globals.hpp" #include "interface/variable_state.hpp" #include "mesh/mesh.hpp" #include "outputs/output_utils.hpp" diff --git a/tst/regression/CMakeLists.txt b/tst/regression/CMakeLists.txt index 96ac7c583c67..bf902381bbe7 100644 --- a/tst/regression/CMakeLists.txt +++ b/tst/regression/CMakeLists.txt @@ -127,7 +127,6 @@ endif() # Any external modules that are required by python can be added to REQUIRED_PYTHON_MODULES # list variable, before including TestSetup.cmake. list(APPEND REQUIRED_PYTHON_MODULES numpy) -list(APPEND REQUIRED_PYTHON_MODULES h5py) list(APPEND DESIRED_PYTHON_MODULES matplotlib) # Include test setup functions, and check for python interpreter and modules diff --git a/tst/unit/test_upper_bound.cpp b/tst/unit/test_upper_bound.cpp index dff514de8e34..20244ee95089 100644 --- a/tst/unit/test_upper_bound.cpp +++ b/tst/unit/test_upper_bound.cpp @@ -14,7 +14,6 @@ TEST_CASE("upper_bound", "[between][out of bounds][on edges]") { GIVEN("A sorted list") { - const std::vector data{-1, 0, 1e-2, 5, 10}; Kokkos::View arr("arr", data.size()); From adff1600badfa22d7790ded8d4ac4ebc797a6121 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 7 Nov 2023 11:52:52 +0100 Subject: [PATCH 50/68] Dedup input parsing code --- src/outputs/histogram.cpp | 79 ++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 42 deletions(-) diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp index 362f92533102..a851547c73cf 100644 --- a/src/outputs/histogram.cpp +++ b/src/outputs/histogram.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include // Parthenon headers @@ -60,11 +61,40 @@ using namespace OutputUtils; namespace HistUtil { +// Parse input for x and y vars from input +// std::tuple +auto ProcessVarInput(ParameterInput *pin, const std::string &block_name, + const std::string &prefix) { + auto var_name = pin->GetString(block_name, prefix + "variable"); + int var_component = -1; + VarType var_type; + if (var_name == "HIST_COORD_X1") { + var_type = VarType::X1; + } else if (var_name == "HIST_COORD_X2") { + var_type = VarType::X2; + } else if (var_name == "HIST_COORD_X3") { + var_type = VarType::X3; + } else if (var_name == "HIST_COORD_R") { + PARTHENON_REQUIRE_THROWS( + typeid(Coordinates_t) == typeid(UniformCartesian), + "Radial coordinate currently only works for uniform Cartesian coordinates."); + var_type = VarType::R; + } else { + var_type = VarType::Var; + var_component = pin->GetInteger(block_name, prefix + "variable_component"); + // would add additional logic to pick it from a pack... + PARTHENON_REQUIRE_THROWS(var_component >= 0, + "Negative component indices are not supported"); + } + + return std::make_tuple(var_name, var_component, var_type); +} + // Parse edges from input parameters. Returns the edges themselves (to be used as list for // arbitrary bins) as well as min and step sizes (potentially in log space) for direct // indexing. -std::tuple, EdgeType, Real, Real> -GetEdges(ParameterInput *pin, const std::string &block_name, const std::string &prefix) { +auto GetEdges(ParameterInput *pin, const std::string &block_name, + const std::string &prefix) { std::vector edges_in; auto edge_type = EdgeType::Undefined; auto edge_min = std::numeric_limits::quiet_NaN(); @@ -128,7 +158,7 @@ GetEdges(ParameterInput *pin, const std::string &block_name, const std::string & PARTHENON_REQUIRE_THROWS( edge_type != EdgeType::Undefined, "Edge type not set and it's unclear how this code was triggered..."); - return {edges, edge_type, edge_min, edge_dbin}; + return std::make_tuple(edges, edge_type, edge_min, edge_dbin); } Histogram::Histogram(ParameterInput *pin, const std::string &block_name, @@ -136,26 +166,8 @@ Histogram::Histogram(ParameterInput *pin, const std::string &block_name, ndim = pin->GetInteger(block_name, prefix + "ndim"); PARTHENON_REQUIRE_THROWS(ndim == 1 || ndim == 2, "Histogram dim must be '1' or '2'"); - x_var_name = pin->GetString(block_name, prefix + "x_variable"); - x_var_component = -1; - if (x_var_name == "HIST_COORD_X1") { - x_var_type = VarType::X1; - } else if (x_var_name == "HIST_COORD_X2") { - x_var_type = VarType::X2; - } else if (x_var_name == "HIST_COORD_X3") { - x_var_type = VarType::X3; - } else if (x_var_name == "HIST_COORD_R") { - PARTHENON_REQUIRE_THROWS( - typeid(Coordinates_t) == typeid(UniformCartesian), - "Radial coordinate currently only works for uniform Cartesian coordinates."); - x_var_type = VarType::R; - } else { - x_var_type = VarType::Var; - x_var_component = pin->GetInteger(block_name, prefix + "x_variable_component"); - // would add additional logic to pick it from a pack... - PARTHENON_REQUIRE_THROWS(x_var_component >= 0, - "Negative component indices are not supported"); - } + std::tie(x_var_name, x_var_component, x_var_type) = + ProcessVarInput(pin, block_name, prefix + "x_"); std::tie(x_edges, x_edges_type, x_edge_min, x_edge_dbin) = GetEdges(pin, block_name, prefix + "x_edges_"); @@ -166,25 +178,8 @@ Histogram::Histogram(ParameterInput *pin, const std::string &block_name, y_var_type = VarType::Unused; // and for 2D profile check if they're explicitly set (not default value) if (ndim == 2) { - y_var_name = pin->GetString(block_name, prefix + "y_variable"); - if (y_var_name == "HIST_COORD_X1") { - y_var_type = VarType::X1; - } else if (y_var_name == "HIST_COORD_X2") { - y_var_type = VarType::X2; - } else if (y_var_name == "HIST_COORD_X3") { - y_var_type = VarType::X3; - } else if (y_var_name == "HIST_COORD_R") { - PARTHENON_REQUIRE_THROWS( - typeid(Coordinates_t) == typeid(UniformCartesian), - "Radial coordinate currently only works for uniform Cartesian coordinates."); - y_var_type = VarType::R; - } else { - y_var_type = VarType::Var; - y_var_component = pin->GetInteger(block_name, prefix + "y_variable_component"); - // would add additional logic to pick it from a pack... - PARTHENON_REQUIRE_THROWS(y_var_component >= 0, - "Negative component indices are not supported"); - } + std::tie(y_var_name, y_var_component, y_var_type) = + ProcessVarInput(pin, block_name, prefix + "y_"); std::tie(y_edges, y_edges_type, y_edge_min, y_edge_dbin) = GetEdges(pin, block_name, prefix + "y_edges_"); From 58ee68981bf5f32d3a158a2478f655feaa684ad3 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 7 Nov 2023 12:17:24 +0100 Subject: [PATCH 51/68] Reserve container space --- src/outputs/histogram.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp index a851547c73cf..bbaac8c6ab15 100644 --- a/src/outputs/histogram.cpp +++ b/src/outputs/histogram.cpp @@ -107,8 +107,9 @@ auto GetEdges(ParameterInput *pin, const std::string &block_name, PARTHENON_REQUIRE_THROWS(edge_max > edge_min, "Histogram max needs to be larger than min.") - const auto edge_num_bins = pin->GetReal(block_name, prefix + "num_bins"); + const auto edge_num_bins = pin->GetInteger(block_name, prefix + "num_bins"); PARTHENON_REQUIRE_THROWS(edge_num_bins >= 1, "Need at least one bin for histogram."); + edges_in.reserve(edge_num_bins); if (edge_type_str == "lin") { edge_type = EdgeType::Lin; From 783df8856f949cc0d747c54f5696f3ce98d06b5f Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 7 Nov 2023 12:33:22 +0100 Subject: [PATCH 52/68] Make CalcHist a member function --- src/outputs/histogram.cpp | 132 +++++++++++++++++++------------------- src/outputs/outputs.hpp | 27 ++++---- 2 files changed, 80 insertions(+), 79 deletions(-) diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp index bbaac8c6ab15..e5726a0d15d5 100644 --- a/src/outputs/histogram.cpp +++ b/src/outputs/histogram.cpp @@ -164,59 +164,59 @@ auto GetEdges(ParameterInput *pin, const std::string &block_name, Histogram::Histogram(ParameterInput *pin, const std::string &block_name, const std::string &prefix) { - ndim = pin->GetInteger(block_name, prefix + "ndim"); - PARTHENON_REQUIRE_THROWS(ndim == 1 || ndim == 2, "Histogram dim must be '1' or '2'"); + ndim_ = pin->GetInteger(block_name, prefix + "ndim"); + PARTHENON_REQUIRE_THROWS(ndim_ == 1 || ndim_ == 2, "Histogram dim must be '1' or '2'"); - std::tie(x_var_name, x_var_component, x_var_type) = + std::tie(x_var_name_, x_var_component_, x_var_type_) = ProcessVarInput(pin, block_name, prefix + "x_"); - std::tie(x_edges, x_edges_type, x_edge_min, x_edge_dbin) = + std::tie(x_edges_, x_edges_type_, x_edge_min_, x_edge_dbin_) = GetEdges(pin, block_name, prefix + "x_edges_"); // For 1D profile default initalize y variables - y_var_name = ""; - y_var_component = -1; - y_var_type = VarType::Unused; + y_var_name_ = ""; + y_var_component_ = -1; + y_var_type_ = VarType::Unused; // and for 2D profile check if they're explicitly set (not default value) - if (ndim == 2) { - std::tie(y_var_name, y_var_component, y_var_type) = + if (ndim_ == 2) { + std::tie(y_var_name_, y_var_component_, y_var_type_) = ProcessVarInput(pin, block_name, prefix + "y_"); - std::tie(y_edges, y_edges_type, y_edge_min, y_edge_dbin) = + std::tie(y_edges_, y_edges_type_, y_edge_min_, y_edge_dbin_) = GetEdges(pin, block_name, prefix + "y_edges_"); } else { - y_edges = ParArray1D(prefix + "y_edges_unused", 0); + y_edges_ = ParArray1D(prefix + "y_edges_unused", 0); } - binned_var_name = + binned_var_name_ = pin->GetOrAddString(block_name, prefix + "binned_variable", "HIST_ONES"); - binned_var_component = -1; // implies that we're not binning a variable but count - if (binned_var_name != "HIST_ONES") { - binned_var_component = + binned_var_component_ = -1; // implies that we're not binning a variable but count + if (binned_var_name_ != "HIST_ONES") { + binned_var_component_ = pin->GetInteger(block_name, prefix + "binned_variable_component"); // would add additional logic to pick it from a pack... - PARTHENON_REQUIRE_THROWS(binned_var_component >= 0, + PARTHENON_REQUIRE_THROWS(binned_var_component_ >= 0, "Negative component indices are not supported"); } - const auto nxbins = x_edges.extent_int(0) - 1; - const auto nybins = ndim == 2 ? y_edges.extent_int(0) - 1 : 1; + const auto nxbins = x_edges_.extent_int(0) - 1; + const auto nybins = ndim_ == 2 ? y_edges_.extent_int(0) - 1 : 1; - result = ParArray2D(prefix + "result", nybins, nxbins); + result_ = ParArray2D(prefix + "result", nybins, nxbins); scatter_result = - Kokkos::Experimental::ScatterView(result.KokkosView()); + Kokkos::Experimental::ScatterView(result_.KokkosView()); - weight_by_vol = pin->GetOrAddBoolean(block_name, prefix + "weight_by_volume", false); + weight_by_vol_ = pin->GetOrAddBoolean(block_name, prefix + "weight_by_volume", false); - weight_var_name = + weight_var_name_ = pin->GetOrAddString(block_name, prefix + "weight_variable", "HIST_ONES"); - weight_var_component = -1; // implies that weighting is not applied - if (weight_var_name != "HIST_ONES") { - weight_var_component = + weight_var_component_ = -1; // implies that weighting is not applied + if (weight_var_name_ != "HIST_ONES") { + weight_var_component_ = pin->GetInteger(block_name, prefix + "weight_variable_component"); // would add additional logic to pick it from a pack... - PARTHENON_REQUIRE_THROWS(weight_var_component >= 0, + PARTHENON_REQUIRE_THROWS(weight_var_component_ >= 0, "Negative component indices are not supported"); } } @@ -224,26 +224,26 @@ Histogram::Histogram(ParameterInput *pin, const std::string &block_name, // Computes a 1D or 2D histogram with inclusive lower edges and inclusive rightmost edges. // Function could in principle be templated on dimension, but it's currently not expected // to be a performance concern (because it won't be called that often). -void CalcHist(Mesh *pm, const Histogram &hist) { +void Histogram::CalcHist(Mesh *pm) { Kokkos::Profiling::pushRegion("Calculate single histogram"); - const auto x_var_component = hist.x_var_component; - const auto y_var_component = hist.y_var_component; - const auto binned_var_component = hist.binned_var_component; - const auto weight_var_component = hist.weight_var_component; - const auto x_var_type = hist.x_var_type; - const auto y_var_type = hist.y_var_type; - const auto x_edges = hist.x_edges; - const auto y_edges = hist.y_edges; - const auto x_edges_type = hist.x_edges_type; - const auto y_edges_type = hist.y_edges_type; - const auto x_edge_min = hist.x_edge_min; - const auto x_edge_dbin = hist.x_edge_dbin; - const auto y_edge_min = hist.y_edge_min; - const auto y_edge_dbin = hist.y_edge_dbin; - const auto hist_ndim = hist.ndim; - const auto weight_by_vol = hist.weight_by_vol; - auto result = hist.result; - auto scatter = hist.scatter_result; + const auto x_var_component = x_var_component_; + const auto y_var_component = y_var_component_; + const auto binned_var_component = binned_var_component_; + const auto weight_var_component = weight_var_component_; + const auto x_var_type = x_var_type_; + const auto y_var_type = y_var_type_; + const auto x_edges = x_edges_; + const auto y_edges = y_edges_; + const auto x_edges_type = x_edges_type_; + const auto y_edges_type = y_edges_type_; + const auto x_edge_min = x_edge_min_; + const auto x_edge_dbin = x_edge_dbin_; + const auto y_edge_min = y_edge_min_; + const auto y_edge_dbin = y_edge_dbin_; + const auto hist_ndim = ndim_; + const auto weight_by_vol = weight_by_vol_; + auto result = result_; + auto scatter = scatter_result; // Reset ScatterView from previous output scatter.reset(); @@ -258,23 +258,23 @@ void CalcHist(Mesh *pm, const Histogram &hist) { auto &md = pm->mesh_data.GetOrAdd("base", p); const auto x_var_pack_string = x_var_type == VarType::Var - ? std::vector{hist.x_var_name} + ? std::vector{x_var_name_} : std::vector{}; const auto x_var = md->PackVariables(x_var_pack_string); const auto y_var_pack_string = y_var_type == VarType::Var - ? std::vector{hist.y_var_name} + ? std::vector{y_var_name_} : std::vector{}; const auto y_var = md->PackVariables(y_var_pack_string); - const auto binned_var_pack_string = - binned_var_component == -1 ? std::vector{} - : std::vector{hist.binned_var_name}; + const auto binned_var_pack_string = binned_var_component == -1 + ? std::vector{} + : std::vector{binned_var_name_}; const auto binned_var = md->PackVariables(binned_var_pack_string); - const auto weight_var_pack_string = - weight_var_component == -1 ? std::vector{} - : std::vector{hist.weight_var_name}; + const auto weight_var_pack_string = weight_var_component == -1 + ? std::vector{} + : std::vector{weight_var_name_}; const auto weight_var = md->PackVariables(weight_var_pack_string); const auto ib = md->GetBoundsI(IndexDomain::interior); @@ -439,7 +439,7 @@ void HistogramOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm const SignalHandler::OutputSignal signal) { Kokkos::Profiling::pushRegion("Calculate all histograms"); for (auto &hist : histograms_) { - CalcHist(pm, hist); + hist.CalcHist(pm); } Kokkos::Profiling::popRegion(); // Calculate all histograms @@ -482,28 +482,28 @@ void HistogramOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm for (int h = 0; h < num_histograms_; h++) { auto &hist = histograms_[h]; const H5G hist_group = MakeGroup(file, "/" + std::to_string(h)); - HDF5WriteAttribute("ndim", hist.ndim, hist_group); - HDF5WriteAttribute("x_var_name", hist.x_var_name.c_str(), hist_group); - HDF5WriteAttribute("x_var_component", hist.x_var_component, hist_group); - HDF5WriteAttribute("binned_var_name", hist.binned_var_name.c_str(), hist_group); - HDF5WriteAttribute("binned_var_component", hist.binned_var_component, hist_group); + HDF5WriteAttribute("ndim", hist.ndim_, hist_group); + HDF5WriteAttribute("x_var_name", hist.x_var_name_.c_str(), hist_group); + HDF5WriteAttribute("x_var_component", hist.x_var_component_, hist_group); + HDF5WriteAttribute("binned_var_name", hist.binned_var_name_.c_str(), hist_group); + HDF5WriteAttribute("binned_var_component", hist.binned_var_component_, hist_group); - const auto x_edges_h = hist.x_edges.GetHostMirrorAndCopy(); + const auto x_edges_h = hist.x_edges_.GetHostMirrorAndCopy(); local_count[0] = global_count[0] = x_edges_h.extent_int(0); HDF5Write1D(hist_group, "x_edges", x_edges_h.data(), local_offset.data(), local_count.data(), global_count.data(), pl_xfer); - if (hist.ndim == 2) { - HDF5WriteAttribute("y_var_name", hist.y_var_name.c_str(), hist_group); - HDF5WriteAttribute("y_var_component", hist.y_var_component, hist_group); + if (hist.ndim_ == 2) { + HDF5WriteAttribute("y_var_name", hist.y_var_name_.c_str(), hist_group); + HDF5WriteAttribute("y_var_component", hist.y_var_component_, hist_group); - const auto y_edges_h = hist.y_edges.GetHostMirrorAndCopy(); + const auto y_edges_h = hist.y_edges_.GetHostMirrorAndCopy(); local_count[0] = global_count[0] = y_edges_h.extent_int(0); HDF5Write1D(hist_group, "y_edges", y_edges_h.data(), local_offset.data(), local_count.data(), global_count.data(), pl_xfer); } - const auto hist_h = hist.result.GetHostMirrorAndCopy(); + const auto hist_h = hist.result_.GetHostMirrorAndCopy(); // Ensure correct output format (as the data in Parthenon may, in theory, vary by // changing the default view layout) so that it matches the numpy output (row // major, x first) @@ -516,7 +516,7 @@ void HistogramOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm } local_count[0] = global_count[0] = hist_h.extent_int(1); - if (hist.ndim == 2) { + if (hist.ndim_ == 2) { local_count[1] = global_count[1] = hist_h.extent_int(0); HDF5Write2D(hist_group, "data", tmp_data.data(), local_offset.data(), diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index eb975ef27e51..7b47480066af 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -228,31 +228,32 @@ enum class VarType { X1, X2, X3, R, Var, Unused }; enum class EdgeType { Lin, Log, List, Undefined }; struct Histogram { - int ndim; // 1D or 2D histogram - std::string x_var_name, y_var_name; // variable(s) for bins - VarType x_var_type, y_var_type; // type, e.g., coord related or actual field - int x_var_component, y_var_component; // components of bin variables (vector) - ParArray1D x_edges, y_edges; - EdgeType x_edges_type, y_edges_type; + int ndim_; // 1D or 2D histogram + std::string x_var_name_, y_var_name_; // variable(s) for bins + VarType x_var_type_, y_var_type_; // type, e.g., coord related or actual field + int x_var_component_, y_var_component_; // components of bin variables (vector) + ParArray1D x_edges_, y_edges_; + EdgeType x_edges_type_, y_edges_type_; // Lowest edge and difference between edges. // Internally used to speed up lookup for log (and lin) bins as otherwise // two more log10 calls would be required per index. - Real x_edge_min, x_edge_dbin, y_edge_min, y_edge_dbin; - std::string binned_var_name; // variable name of variable to be binned + Real x_edge_min_, x_edge_dbin_, y_edge_min_, y_edge_dbin_; + std::string binned_var_name_; // variable name of variable to be binned // component of variable to be binned. If -1 means no variable is binned but the // histgram is a sample count. - int binned_var_component; - bool weight_by_vol; // use volume weighting - std::string weight_var_name; // variable name of variable used as weight + int binned_var_component_; + bool weight_by_vol_; // use volume weighting + std::string weight_var_name_; // variable name of variable used as weight // component of variable to be used as weight. If -1 means no weighting - int weight_var_component; - ParArray2D result; // resulting histogram + int weight_var_component_; + ParArray2D result_; // resulting histogram // temp view for histogram reduction for better performance (switches // between atomics and data duplication depending on the platform) Kokkos::Experimental::ScatterView scatter_result; Histogram(ParameterInput *pin, const std::string &block_name, const std::string &prefix); + void CalcHist(Mesh *pm); }; } // namespace HistUtil From 2b33f7ad50b25a2493a0b518dab85faccdd35290 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 7 Nov 2023 17:05:33 +0100 Subject: [PATCH 53/68] Update doc --- doc/sphinx/src/outputs.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/sphinx/src/outputs.rst b/doc/sphinx/src/outputs.rst index 2e1d24fdb3d6..5aaa233bb477 100644 --- a/doc/sphinx/src/outputs.rst +++ b/doc/sphinx/src/outputs.rst @@ -175,6 +175,8 @@ Currently supported are The output format follows ``numpy`` convention, so that plotting data with Python based machinery should be straightfoward (see example below). +In other words, 2D histograms use C-ordering corresponding to ``[x,y]`` +indexing with ``y`` being the fast index. In general, histograms are calculated using inclusive left bin edges and data equal to the rightmost edge is also included in the last bin. @@ -243,7 +245,7 @@ with the following parameters Number of equally spaced bins between min and max value in the first dim. Used/required only for ``lin`` and ``log`` edge type. - ``hist#_x_edges_list=FLOAT,FLOAT,FLOAT,...`` (comma separated list of increasing values) - Arbitrary definition of edge values. + Arbitrary definition of edge values with inclusive innermost and outermost edges. Used/required only for ``list`` edge type. - ``hist#_y_edges...`` Same as the ``hist#_x_edges...`` parameters except for being used in the second @@ -259,7 +261,9 @@ with the following parameters Used/required only if a variable is binned and not ``HIST_ONES``. - ``hist#_weight_by_volume=BOOL`` (``true`` or ``false``) Apply volume weighting to the binned variable. Can be used simultaneously with binning - by a different variable. + by a different variable. Note that this does *not* include any normalization + (e.g., by total volume or the sum of the weight variable in question) and is left to + the user during post processing. - ``hist#_weight_variable=STRING`` Variable to be used as weight. Can be used together with volume weighting. From 291c6bbccf03d872612590b1db67e4cf4b5a5dc9 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Tue, 7 Nov 2023 09:16:55 -0700 Subject: [PATCH 54/68] add profiling --- src/driver/driver.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/driver/driver.cpp b/src/driver/driver.cpp index 8314335f3610..03acf820e536 100644 --- a/src/driver/driver.cpp +++ b/src/driver/driver.cpp @@ -77,13 +77,16 @@ DriverStatus EvolutionDriver::Execute() { // Before loop do work // App input version + Kokkos::Profiling::pushRegion("Driver_UserWorkBeforeLoop"); if (app_input->UserWorkBeforeLoop != nullptr) { app_input->UserWorkBeforeLoop(pmesh, pinput, tm); } + // packages version for (auto &[name, pkg] : pmesh->packages.AllPackages()) { pkg->UserWorkBeforeLoop(pmesh, pinput, tm); } + Kokkos::Profiling::popRegion(); // Driver_UserWorkBeforeLoop Kokkos::Profiling::pushRegion("Driver_Main"); while (tm.KeepGoing()) { From 7138c74b42f41dc3ceb703d310096b3471d505bf Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 7 Nov 2023 17:19:43 +0100 Subject: [PATCH 55/68] Add capability to accumulate data outside bin ranges --- doc/sphinx/src/outputs.rst | 2 + src/outputs/histogram.cpp | 43 ++++++++++++++----- src/outputs/outputs.hpp | 1 + .../output_hdf5/parthinput.advection | 8 +++- 4 files changed, 41 insertions(+), 13 deletions(-) diff --git a/doc/sphinx/src/outputs.rst b/doc/sphinx/src/outputs.rst index 5aaa233bb477..7659bf19bdde 100644 --- a/doc/sphinx/src/outputs.rst +++ b/doc/sphinx/src/outputs.rst @@ -250,6 +250,8 @@ with the following parameters - ``hist#_y_edges...`` Same as the ``hist#_x_edges...`` parameters except for being used in the second dimension for ``ndim=2`` histograms. +- ``hist#_accumulate=BOOL`` (``true`` or ``false`` default) + Accumulate data that is outside the binning range in the outermost bins. - ``hist#_binned_variable=STRING`` (variable name or ``HIST_ONES``) Variable to be binned. If a variable name is given a component has to be specified, too, see next parameter. diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp index e5726a0d15d5..9e4cf179e9b9 100644 --- a/src/outputs/histogram.cpp +++ b/src/outputs/histogram.cpp @@ -207,6 +207,7 @@ Histogram::Histogram(ParameterInput *pin, const std::string &block_name, scatter_result = Kokkos::Experimental::ScatterView(result_.KokkosView()); + accumulate_ = pin->GetOrAddBoolean(block_name, prefix + "accumulate", false); weight_by_vol_ = pin->GetOrAddBoolean(block_name, prefix + "weight_by_volume", false); weight_var_name_ = @@ -242,6 +243,7 @@ void Histogram::CalcHist(Mesh *pm) { const auto y_edge_dbin = y_edge_dbin_; const auto hist_ndim = ndim_; const auto weight_by_vol = weight_by_vol_; + const auto accumulate = accumulate_; auto result = result_; auto scatter = scatter_result; @@ -299,13 +301,23 @@ void Histogram::CalcHist(Mesh *pm) { } else { x_val = x_var(b, x_var_component, k, j, i); } - if (x_val < x_edges(0) || x_val > x_edges(x_edges.extent_int(0) - 1)) { - return; - } int x_bin = -1; - // if we're on the rightmost edge, directly set last bin - if (x_val == x_edges(x_edges.extent_int(0) - 1)) { + // First handle edge cases explicitly + if (x_val < x_edges(0)) { + if (accumulate) { + x_bin = 0; + } else { + return; + } + } else if (x_val > x_edges(x_edges.extent_int(0) - 1)) { + if (accumulate) { + x_bin = x_edges.extent_int(0) - 2; + } else { + return; + } + // if we're on the rightmost edge, directly set last bin + } else if (x_val == x_edges(x_edges.extent_int(0) - 1)) { x_bin = x_edges.extent_int(0) - 2; } else { // for lin and log directly pick index @@ -339,13 +351,22 @@ void Histogram::CalcHist(Mesh *pm) { y_val = y_var(b, y_var_component, k, j, i); } - if (y_val < y_edges(0) || y_val > y_edges(y_edges.extent_int(0) - 1)) { - return; - } - y_bin = -1; // reset to impossible value - // if we're on the rightmost edge, directly set last bin - if (y_val == y_edges(y_edges.extent_int(0) - 1)) { + // First handle edge cases explicitly + if (y_val < y_edges(0)) { + if (accumulate) { + y_bin = 0; + } else { + return; + } + } else if (y_val > y_edges(y_edges.extent_int(0) - 1)) { + if (accumulate) { + y_bin = y_edges.extent_int(0) - 2; + } else { + return; + } + // if we're on the rightmost edge, directly set last bin + } else if (y_val == y_edges(y_edges.extent_int(0) - 1)) { y_bin = y_edges.extent_int(0) - 2; } else { // for lin and log directly pick index diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index 7b47480066af..e5260a300f60 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -238,6 +238,7 @@ struct Histogram { // Internally used to speed up lookup for log (and lin) bins as otherwise // two more log10 calls would be required per index. Real x_edge_min_, x_edge_dbin_, y_edge_min_, y_edge_dbin_; + bool accumulate_; // accumulate data outside binning range in outermost bins std::string binned_var_name_; // variable name of variable to be binned // component of variable to be binned. If -1 means no variable is binned but the // histgram is a sample count. diff --git a/tst/regression/test_suites/output_hdf5/parthinput.advection b/tst/regression/test_suites/output_hdf5/parthinput.advection index 8442d082e8ec..961bc6310a91 100644 --- a/tst/regression/test_suites/output_hdf5/parthinput.advection +++ b/tst/regression/test_suites/output_hdf5/parthinput.advection @@ -121,12 +121,16 @@ num_histograms = 1 hist0_ndim = 2 hist0_x_variable = HIST_COORD_X1 hist0_x_edges_type = list -hist0_x_edges_list = -0.5, -0.25, 0.0, 0.25, 0.5 +# Note that the coordinate edges are smaller than the domain extents on purpose +# to test the accumulation feature (as the reference histogram in the test calculated +# with numpy goes all the way out to the domain edges). +hist0_x_edges_list = -0.5, -0.25, 0.0, 0.25, 0.35 hist0_y_variable = HIST_COORD_X2 hist0_y_edges_type = list -hist0_y_edges_list = -0.5, -0.1, 0.0, 0.1, 0.5 +hist0_y_edges_list = -0.25, -0.1, 0.0, 0.1, 0.5 hist0_binned_variable = advected hist0_binned_variable_component = 0 hist0_weight_by_volume = true hist0_weight_variable = one_minus_advected_sq hist0_weight_variable_component = 0 +hist0_accumulate = true From a8cea59bc544e2cf33180a29f18e627bc7af18f0 Mon Sep 17 00:00:00 2001 From: Peter Brady Date: Fri, 10 Nov 2023 09:28:22 -0600 Subject: [PATCH 56/68] Use `GNUInstallDirs` to fix installation problems on mac (#957) Co-authored-by: Jonah Miller --- CMakeLists.txt | 4 ++++ cmake/parthenonConfig.cmake.in | 10 ++++++++++ src/CMakeLists.txt | 14 ++++++-------- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fe9b1f7e3d86..0685a11d14fb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -353,6 +353,10 @@ if (PARTHENON_ENABLE_ASCENT) find_package(Ascent REQUIRED NO_DEFAULT_PATH) endif() +# Installation configuration +include(GNUInstallDirs) +set(CMAKE_INSTALL_INCLUDEDIR "${CMAKE_INSTALL_INCLUDEDIR}/parthenon") + add_subdirectory(src) add_subdirectory(example) add_subdirectory(benchmarks) diff --git a/cmake/parthenonConfig.cmake.in b/cmake/parthenonConfig.cmake.in index aaed3e28971d..e70d251505bf 100644 --- a/cmake/parthenonConfig.cmake.in +++ b/cmake/parthenonConfig.cmake.in @@ -11,6 +11,14 @@ # the public, perform publicly and display publicly, and to permit others to do so. #========================================================================================= +if(NOT PARTHENON_CMAKE) + cmake_path(SET PARTHENON_CMAKE_BASE_DIR NORMALIZE "${CMAKE_CURRENT_LIST_DIR}/..") + message(STATUS "Appending parthenon cmake module directory: " ${PARTHENON_CMAKE_BASE_DIR}) + list(APPEND CMAKE_MODULE_PATH ${PARTHENON_CMAKE_BASE_DIR}) + set(PARTHENON_CMAKE TRUE) +endif() + + # Favor using the kokkos package that was built with parthenon, if one has not been specified if (@PARTHENON_IMPORT_KOKKOS@) find_package(Kokkos 3 REQUIRED PATHS @Kokkos_DIR@ NO_DEFAULT_PATH) @@ -37,4 +45,6 @@ if(@OpenMP_FOUND@) find_package(OpenMP REQUIRED COMPONENTS CXX) endif() +find_package(Filesystem REQUIRED COMPONENTS Experimental Final) + include("${CMAKE_CURRENT_LIST_DIR}/parthenonTargets.cmake") diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 088fe6269556..512201459106 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -328,24 +328,22 @@ lint_target(parthenon) target_include_directories(parthenon PUBLIC $ $ - $ + $ ) -install(TARGETS parthenon EXPORT parthenonTargets - INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/parthenon" - LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" - ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" -) +install(TARGETS parthenon EXPORT parthenonTargets) # Maintain directory structure in installed include files -install(DIRECTORY . DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/parthenon" FILES_MATCHING PATTERN "*.hpp") +install(DIRECTORY ./ TYPE INCLUDE FILES_MATCHING PATTERN "*.hpp") # Install generated config header file install(FILES ${CMAKE_CURRENT_BINARY_DIR}/generated/config.hpp - DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/parthenon") + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") install(FILES ${PROJECT_BINARY_DIR}/cmake/parthenonConfig.cmake DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/parthenon") +install(FILES ${PROJECT_SOURCE_DIR}/cmake/FindFilesystem.cmake DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake") + install(EXPORT parthenonTargets FILE parthenonTargets.cmake NAMESPACE Parthenon:: From bd4c0b1df71b0acb0189e6acc6fcc52b759b6346 Mon Sep 17 00:00:00 2001 From: Ben Prather Date: Fri, 10 Nov 2023 11:52:46 -0500 Subject: [PATCH 57/68] Forceinline par_for_inner: add meshblock version (#972) * Forceinline par_for_inner: add meshblock version * Newlines for the formatting gods --------- Co-authored-by: Ben Prather Co-authored-by: Philipp Grete --- src/mesh/meshblock.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/mesh/meshblock.hpp b/src/mesh/meshblock.hpp index a00b2b5b0b4e..3df86d7f73e7 100644 --- a/src/mesh/meshblock.hpp +++ b/src/mesh/meshblock.hpp @@ -57,8 +57,9 @@ class StateDescriptor; // - Not defined in kokkos_abstraction.hpp because it requires the compile time option // DEFAULT_INNER_LOOP_PATTERN to be set. template -KOKKOS_INLINE_FUNCTION void par_for_inner(const team_mbr_t &team_member, const int &il, - const int &iu, const Function &function) { +KOKKOS_FORCEINLINE_FUNCTION void par_for_inner(const team_mbr_t &team_member, + const int &il, const int &iu, + const Function &function) { parthenon::par_for_inner(DEFAULT_INNER_LOOP_PATTERN, team_member, il, iu, function); } From 809decbc563eab07f05f0fbe478683390f514daa Mon Sep 17 00:00:00 2001 From: Patrick Mullen Date: Mon, 13 Nov 2023 15:28:24 -0700 Subject: [PATCH 58/68] Construct integrators via string name --- src/time_integration/butcher_integrator.cpp | 19 +++++++++++++++++-- .../low_storage_integrator.cpp | 16 ++++++++++++++-- src/time_integration/staged_integrator.hpp | 2 ++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/time_integration/butcher_integrator.cpp b/src/time_integration/butcher_integrator.cpp index d90de479ec68..40096dd1eeba 100644 --- a/src/time_integration/butcher_integrator.cpp +++ b/src/time_integration/butcher_integrator.cpp @@ -36,8 +36,12 @@ namespace parthenon { * alpha_k = c_k * c_k = b_k */ -ButcherIntegrator::ButcherIntegrator(ParameterInput *pin) - : StagedIntegrator(pin->GetOrAddString("parthenon/time", "integrator", "rk2")) { + +//---------------------------------------------------------------------------------------- +//! \class ButcherIntegrator::ButcherIntegrator(const std::string &name) +//! \brief Constructs a ButcherIntegrator instance given a string (e.g., rk2, rk3..) + +ButcherIntegrator::ButcherIntegrator(const std::string &name) : StagedIntegrator(name) { if (name_ == "rk1") { nstages = nbuffers = 1; Resize_(nstages); @@ -219,6 +223,17 @@ ButcherIntegrator::ButcherIntegrator(ParameterInput *pin) } } +//---------------------------------------------------------------------------------------- +//! \class ButcherIntegrator::ButcherIntegrator(ParameterInput *pin) +//! \brief Constructs a ButcherIntegrator instance given ParameterInput *pin + +ButcherIntegrator::ButcherIntegrator(ParameterInput *pin) + : ButcherIntegrator(pin->GetOrAddString("parthenon/time", "integrator", "rk2")) {} + +//---------------------------------------------------------------------------------------- +//! \fn void ButcherIntegrator::Resize_(int nstages) +//! \brief Resizes ButcherIntegrator registers given a supplied integer nstages + void ButcherIntegrator::Resize_(int nstages) { a.resize(nstages); for (int i = 0; i < a.size(); ++i) { diff --git a/src/time_integration/low_storage_integrator.cpp b/src/time_integration/low_storage_integrator.cpp index ab874d98e050..d833ddd8ad20 100644 --- a/src/time_integration/low_storage_integrator.cpp +++ b/src/time_integration/low_storage_integrator.cpp @@ -43,8 +43,13 @@ namespace parthenon { * Stone et al., ApJS (2020) 249:4 * See equations 11 through 15. */ -LowStorageIntegrator::LowStorageIntegrator(ParameterInput *pin) - : StagedIntegrator(pin->GetOrAddString("parthenon/time", "integrator", "rk2")) { + +//---------------------------------------------------------------------------------------- +//! \class LowStorageIntegrator::LowStorageIntegrator(const std::string &name) +//! \brief Constructs a LowStorageIntegrator instance given a string (e.g., rk2, rk3..) + +LowStorageIntegrator::LowStorageIntegrator(const std::string &name) + : StagedIntegrator(name) { if (name_ == "rk1") { nstages = 1; nbuffers = 1; @@ -155,4 +160,11 @@ LowStorageIntegrator::LowStorageIntegrator(ParameterInput *pin) MakePeriodicNames_(stage_name, nstages); } +//---------------------------------------------------------------------------------------- +//! \class LowStorageIntegrator::LowStorageIntegrator(ParameterInput *pin) +//! \brief Constructs a LowStorageIntegrator instance given ParameterInput *pin + +LowStorageIntegrator::LowStorageIntegrator(ParameterInput *pin) + : LowStorageIntegrator(pin->GetOrAddString("parthenon/time", "integrator", "rk2")) {} + } // namespace parthenon diff --git a/src/time_integration/staged_integrator.hpp b/src/time_integration/staged_integrator.hpp index 0d22d7d85600..329f69bc294b 100644 --- a/src/time_integration/staged_integrator.hpp +++ b/src/time_integration/staged_integrator.hpp @@ -49,6 +49,7 @@ class StagedIntegrator { class LowStorageIntegrator : public StagedIntegrator { public: LowStorageIntegrator() = default; + explicit LowStorageIntegrator(const std::string &name); explicit LowStorageIntegrator(ParameterInput *pin); std::vector delta; std::vector beta; @@ -60,6 +61,7 @@ class LowStorageIntegrator : public StagedIntegrator { class ButcherIntegrator : public StagedIntegrator { public: ButcherIntegrator() = default; + explicit ButcherIntegrator(const std::string &name); explicit ButcherIntegrator(ParameterInput *pin); // TODO(JMM): Should I do a flat array with indexing instead? std::vector> a; From 2f105da3426c90ccd3e9823fd4f8d8da61cafcda Mon Sep 17 00:00:00 2001 From: Patrick Mullen Date: Mon, 13 Nov 2023 16:14:13 -0700 Subject: [PATCH 59/68] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 051e8cc3fe66..915c804c52ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - [[PR 868]](https://github.com/parthenon-hpc-lab/parthenon/pull/868) Add block-local face, edge, and nodal fields and allow for packing ### Changed (changing behavior/API/variables/...) +- [[PR 975]](https://github.com/parthenon-hpc-lab/parthenon/pull/975) Construct staged integrators via arbitrary name - [[PR 965]](https://github.com/parthenon-hpc-lab/parthenon/pull/965) Allow leading whitespace in input parameters - [[PR 926]](https://github.com/parthenon-hpc-lab/parthenon/pull/926) Internal refinement op registration - [[PR 897]](https://github.com/parthenon-hpc-lab/parthenon/pull/897) Deflate compression filter is not called any more if compression is soft disabled From 81841bd6dd5b11be4e83accbc3cbfe404d347aab Mon Sep 17 00:00:00 2001 From: Jonah Maxwell Miller Date: Tue, 14 Nov 2023 14:09:15 -0700 Subject: [PATCH 60/68] move userworkbeforeloop before outputs --- src/driver/driver.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/driver/driver.cpp b/src/driver/driver.cpp index 03acf820e536..c0e21414e576 100644 --- a/src/driver/driver.cpp +++ b/src/driver/driver.cpp @@ -65,15 +65,6 @@ DriverStatus EvolutionDriver::Execute() { PreExecute(); InitializeBlockTimeStepsAndBoundaries(); SetGlobalTimeStep(); - OutputSignal signal = OutputSignal::none; - pouts->MakeOutputs(pmesh, pinput, &tm, signal); - pmesh->mbcnt = 0; - int perf_cycle_offset = - pinput->GetOrAddInteger("parthenon/time", "perf_cycle_offset", 0); - - // Output a text file of all parameters at this point - // Defaults must be set across all ranks - DumpInputParameters(); // Before loop do work // App input version @@ -81,13 +72,22 @@ DriverStatus EvolutionDriver::Execute() { if (app_input->UserWorkBeforeLoop != nullptr) { app_input->UserWorkBeforeLoop(pmesh, pinput, tm); } - // packages version for (auto &[name, pkg] : pmesh->packages.AllPackages()) { pkg->UserWorkBeforeLoop(pmesh, pinput, tm); } Kokkos::Profiling::popRegion(); // Driver_UserWorkBeforeLoop + OutputSignal signal = OutputSignal::none; + pouts->MakeOutputs(pmesh, pinput, &tm, signal); + pmesh->mbcnt = 0; + int perf_cycle_offset = + pinput->GetOrAddInteger("parthenon/time", "perf_cycle_offset", 0); + + // Output a text file of all parameters at this point + // Defaults must be set across all ranks + DumpInputParameters(); + Kokkos::Profiling::pushRegion("Driver_Main"); while (tm.KeepGoing()) { if (Globals::my_rank == 0) OutputCycleDiagnostics(); From 9c6a00aba61f49f7abe15aad489a104532f6efd2 Mon Sep 17 00:00:00 2001 From: Jonah Maxwell Miller Date: Tue, 14 Nov 2023 14:12:23 -0700 Subject: [PATCH 61/68] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 051e8cc3fe66..15c5692f356e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - [[PR 868]](https://github.com/parthenon-hpc-lab/parthenon/pull/868) Add block-local face, edge, and nodal fields and allow for packing ### Changed (changing behavior/API/variables/...) +- [[PR 976]](https://github.com/parthenon-hpc-lab/parthenon/pull/976) Move UserWorkBeforeLoop to be after first output - [[PR 965]](https://github.com/parthenon-hpc-lab/parthenon/pull/965) Allow leading whitespace in input parameters - [[PR 926]](https://github.com/parthenon-hpc-lab/parthenon/pull/926) Internal refinement op registration - [[PR 897]](https://github.com/parthenon-hpc-lab/parthenon/pull/897) Deflate compression filter is not called any more if compression is soft disabled From a9138c30ba798c61b4235caddc22ccf145858954 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 15 Nov 2023 15:06:08 +0100 Subject: [PATCH 62/68] Use names as historam ids and update doc with examples --- doc/sphinx/src/boundary_communication.rst | 5 +- .../figs/Curtis_et_al-ApJL-2023-1dhist.png | Bin 0 -> 47270 bytes doc/sphinx/src/{ => figs}/TaskDiagram.png | Bin doc/sphinx/src/mesh/mesh.rst | 6 +- doc/sphinx/src/outputs.rst | 128 +++++++++++------- doc/sphinx/src/tasks.rst | 2 +- src/outputs/histogram.cpp | 19 +-- src/outputs/outputs.hpp | 6 +- src/utils/sort.hpp | 3 + .../test_suites/output_hdf5/output_hdf5.py | 8 +- .../output_hdf5/parthinput.advection | 44 +++--- 11 files changed, 128 insertions(+), 93 deletions(-) create mode 100644 doc/sphinx/src/figs/Curtis_et_al-ApJL-2023-1dhist.png rename doc/sphinx/src/{ => figs}/TaskDiagram.png (100%) diff --git a/doc/sphinx/src/boundary_communication.rst b/doc/sphinx/src/boundary_communication.rst index 3df126de6f02..cc85fa8052c2 100644 --- a/doc/sphinx/src/boundary_communication.rst +++ b/doc/sphinx/src/boundary_communication.rst @@ -43,13 +43,14 @@ subset, and the columns point to the indices in the ``bnd_info`` array containing the subset of sub-halos you wish to operate on. To communicate across a particular boundary type, the templated -boundary communication routines (see :boundary_comm_tasks:`boundary_comm_tasks`.) +boundary communication routines (see :ref:`boundary_comm_tasks`) should be instantiated with the desired ``BoundaryType``, i.e. .. code:: cpp + SendBoundBufs(md); -The different ``BoundaryType``s are: +The different ``BoundaryType``\ s are: - ``any``: Communications are performed between all leaf blocks (i.e. the standard Parthenon grid that does not include multi-grid related blocks). diff --git a/doc/sphinx/src/figs/Curtis_et_al-ApJL-2023-1dhist.png b/doc/sphinx/src/figs/Curtis_et_al-ApJL-2023-1dhist.png new file mode 100644 index 0000000000000000000000000000000000000000..b7078512e10b5aab2b8da5065b915c75e77ad8c5 GIT binary patch literal 47270 zcmXtA1yohp*GEc9QV>x(q#Fh4lB(HlvGk0R8mSxLP8M)zJ2Gv zzFCWL2H$(!d+s@V|7sJduKMT(4h0Sh3d#)yc^OR7m z#)5x*u`D9s|8Kj=>bq$wRUvdN9z!S7jYmjl6J8) zbF*=DpwYImw?xsiw4mV;piy=7pyB1_<)h)|eaL_RA-^Dvx(bc7tTsE1Ssn@s4T^${ zq_$V)&n!=0?T=?yOMF%jzA=O_+)Os@EvNZclc!&D|M?fLmnPLOzcRndeR+cjgA}9X zE1|Y}mxqRiq%6xzy4*mFYego7P1*@@wN^j=^s#(f_*on$)8!*ptykOjc6gZONNCi% zY|PM3#OvUDR}T@!Up<1(;OhibBHC0`-ah|*4=c7^6#jM@E9B846}-*~z598l?{!_h4Zu?T7e(spT5m_2cQg8Dsr75PUTU^hUyF)E3 zEXG<~tzMjLlQu5iKJE_1HK?~H5)u+}+n$Qd7WF~ROw8TW1H?au(|Y1)rAzdxsid%e?=1|uj*9lpHrUzBRNVHQcfM{vThF4`X;h*w)QydY!*1N!oP0|#Dt6Ak+85i$ukt2t zzj+nk|7`1?VUxq>>(ssyO%4lRTixx7w7JF!chTRhFRpgM@>}M0cjr`@F&B0F|JfXu zj-?h~yzuL}@n92|tG#B*|CWhce72x-{`vWKdG*Zu7bg>GX=$ypQDiUrVkiymDvyqi z+zk!WFAsVMPA)IbPl{DCjwgzgknb%v*u9xC-1+(Q`9eoPKvU((&KEn{5dVvlFG|r| zZo`(9QH}naV|f%~N-6A-eo~im>a+D9QkOTnrWDyK8#E;=v+P-?9Tb!djW$}2lMfE` zq`n0hV(i*gx<{K7#x+)oeU96oxzJ7fQ1E%DhX@T5Z_HA|ExKGqigEVkOH}`>3(vg| z=MEcPlIOp7(?q;oQfKpPYT~BL^<0m(3&ozcB?B|)Xjg&{P zqFR$rRbR-pKMh`{bR??o9jI@9lx+c4JaEq z?+ikNjhB#=gsyL)506vxp6k`+#oCV_37MHc_vUgX(Og_z4VoM%v9Ynu9QfULW-R~g zEtvTUt>Jt>`YU+0S+L;lx-n9un)UE_a~v&#s-(_i*HBgTMB9^k)Q;iiZ*Idq--X=8Jq~E-C zE1Uxt9hE$cF;D2azP7fEhK9yh4`*j*bku7}n&`n@HHphdf8~{x@#e(HzU?h_BJt<+|T^C7-yL~pMQY7ok|Z{POLhKH3=gCtTD#5g#jC1?nX!{pV}?gaWDnb8JGpi~NYdwcU+kI~pJ8I^}!@zqN{|~d^OF@Z>=1{&At2RUC@Pcb+MHl)+3S^Z?P(Q ziXx{l!w9Vn=3iLdt%h%*QaXE3Rx6JnwF4-Yet<4xpAUw3HTVj?CchFxbpPAB$vjYKY%N<~WQ zr7-4(DXG-Kgc28z^NP&bpZ!tMKl3CCx2a9=A`k-Gkz9P`WL z08y=C<>kgXhl1wjRL72s6l@YE6sN7CHwo$KgTnjmJ^9EzkB)9+IDS}O zfAhFlr(BocZszqs;ypgwDQ4svw!c)58dzmvNSK;ZWjt_XqNfk$Flvm;%zRDw(*NR& z$8*nEdD3EUz6Cv4a(k+D_1QIc0!yN%y7u;nk`DTiDfvo?a_&4OB{b!sKubD8&A z{3MA2O0-mdLMiiHjsm?>EtA8}11Iv%`{Z=Ff)C!b=gep%8Q!UT0#otp3n;DPc=`vP+>!Sf-iYhRVG-@CI9p2x7jaVy?N}P1}*C~Y=Jtn z9!#;zQ**i39x!Mm<|Ni}Jb|E8$)pfizX;Aaz+E5C|>SjT#M;3n> z$+YU?x`Ac*jXQD<4xDgYXlZGK&i6ZV_(nxY0^whie%qqQW72m;yyATJTEb6Hy}}5n zM{M3^g%Q5|vlNgMPyfgT5Raz5et!~+*1RztwcjyOxn6Z~z}4k&rcmPiys=^9a|KvX z(HF-~Iy`Wm8SdSqPUxbWIlQ7YH^h!)^<_4r^da17pkR=)2}#Q37polKdwr9rx;||E zRyDhH>KG%Yl2Un9#T!=+yOE48>Cn7u^+nv{QndA!*7u25iGKYe*I{tXBPB=8V;6{! zqDLn;d~)+TQ5d8B9U?82CmsD7yaX5i;(b&;?4_lp@W}CcxJw>)h*cUkth67EtbR3* zEqdJH@B8cHt>c|9u||&X8w%euC-@p08>ecUrdSQZ-F|`%GZH~WumApqd-8|^yLLo7 z+p4Y>RodSR|7_nw1$-)zh|vvtPR@~q2UA&GoSZT5%=;4Fya`|0_tdJet+ZKa71-~% z5`)_>EG!IZ2mpZK1Yb$pe}W0bO8WC&l<>2+$lFEVoI!Vaa=U?jRWnlDAc9gVK@2KqYNi@v3F)PJhT^!!Ywv9B?XF2-Gfcb}FVy`VJpVGUNM&l`Q2oaH zCOW}KijhePPq;T9%Axf3^-Wek7MF{r`1p#HO~1{9qvP_7jX>;_1{P#ETQtLRD8>1E z=QUyyk{$+^N2mQGBhLU>WP1NdJKmjp@Oo2Ca2Y*NGv?NGB+lnBc{_V%DR-%O%x?X2 zELwSac_-|aitEx(pFY)h_&Ii6W_PxMjn@}}ha~KJqB|T4!st<`mieKzAM~~e{*s9?IKM{piQF(;VSc&XO?z1jOLDSUocGy(57@TU z*t4g9e>zm!pP=$}cJo?y$_Tw?${#uJi_6;|l7I#<2RU7t?G%Js_7Nr!yPALgXH&{U zsguEI`$lL+{sZ)i;T-XtL^ZK*0v4#qQ5(q;=5zWk5r$8xxHD7Pa=H@rt=@*@NvZ2C zF4JLH4P|cf*V{@|C+*WH50VuY$K~tUpK{ol7)WxCPqB{s3(v{=V0n$lQi)8780$IM zw*d&?6Gg>2wt)gBf-q0@Zcb||*s&L;UFt|~4Sx)9B^uu9x)xtzox_kG+MSXtP znD^ns<|2?7%@(~a?l01`;E|E>8?7QW86ft3``O+qleVu-j@0hkQ`mgzx}+Kd!YTOo zWU+}6RT&jl z{??k*+1=PWod@0cgnUmQHvjxq7oML_x7U88TCAF-=^aYszbFPUXXS9!_X}%ljVF1=fNcD_^%*feIT9d-5UPKfYGNf=l z9ufFrOd_|~=DGh2_TpWipYQ6Qe@j4G!<=J>D0`o#4y{|rhvfy&r6S!*LpZX^$_zX_ z@3c#7_z6fV1%RKW77qZG5jvUDvRQ2R*0&1qOrYeoz=Ga;7k0XHaM1`^o-&1>JNi( z%bK>rP($u^IlXl%Y^gX)(2R^w`_oo)7I{}%Hae{6jK%7_);ONvI;jK8;f&#ArO^Xb z=;1X|t%f@_6C!zKabuY?(D|Uk3kM8-|Btu_kQVg|18xK=m0SbLRFJjLA7=D$s<-*Qcgq()}Qb%m(uKly5uy?YK9|2L#yIfe+dHq{knd# zJ-rG)i@B)zpc~g_>4RU#GP%w}&($?-h8WkT!HVgb5wJ*0OHjbQEBJc46xS6Rm z>H|_~-bj>IP*}S&B<+3f?X3>-gpAXYBRa87#3<7*rt!|X4%8TP?wCL!J*b9*YzhV~htv4NJ(v9607spz zx9Nd(8|d%P1!g*_DjW^nJ9%x*0_DeO&M3Uld#_~!kw413Q$@UaS3gCYkL60@Q9i6c z{wghq=n$_dg;S#MJeUCLhlzhzUop#AM?1a!3#?bs8`Euk54cZ}U%e#8PF)zs+tav# z<}pHHAX>ch`Qt#rZ>^rkU$tK3S!u1hZ*3pCqF(S0WFZua9Gz`dlx`WbX;wxb0#-4J zWLJhKpGjA z+|P!|P5n-Eo{eU!0A%~t>duBNGf-#XK<>sD=YKZf8dqQsjQ3G?tD#-T@8wr{=(gE; z$FaSu!l3T-@_gTBJYP0>G&Wdq<4DZud!W&8Q>-ql9KTNhhrNLv%m<-OmFtbFcu=L? zVWh~9ci;R|8}Xj*ax=TEH#pq(rTTB*1bGfZ!>4DfW4*LR)rtXz6f`&L`PsJ&akzcu zS$Oc!)}zo0b^3V=2V*?0k~_Ka4C}2W4Gk$^O_>-OLw|hEa9#A?_yzDO0hn5c{{;%7 zvOp_!Z{eW8>*?J86E2$`-pz961o#8q+XcmZxFA++5X7SafFYnN>4+YgmDI0gLF0e+ zjLF8v28*`&XYJ6}8Z$JLRyS-dg~GRQ+tg~L<6VZ1NUC4gQ`+MxmU1V@Y}{DwlK$&d z_%x(u`zw}2daB_npR%&fiO?E<9tJbR*DbSag@*>%WtmdFBhB*xM5_%OZrtQ@nmBRtt9_}_R)dI)Ytv3B!qER5Pq_heJ`pNoWvP047y&T_!kdP2=PDM4fjqyj+ zMZh(W_ZH@M*$A-?z6TSunp;m4VzaQYFfcLc{+$oWpGfe8a&OS?_0ES7p77c?tDN=P zAx_up9af`M_;&>*dctlf?LHyiTYP6rz2hxIpfq@*tqGPq}wcRf1Z)42px)<^xI+4oe-o98SzMJl(_WGgZ6=foZ z!O4|I`3;eBMmlRna!*2TPNCN`=FuyCA&6M#*HBjv)O-c{O+1i8=rpg)$k*1^5a|PH z%pmsM=6i+*6<5wqg@*7uble~C^{h97DAW1VZASChZfc)Dd+S?obAB{@F=f+YbY(6% zZi$JCTCHl^TlxH8Y3===MS4CyXse;2p%sP=ufnPQQ-PN|ZI`~4OOAU0k0ScPqP(Ia zj#y4C;Eh@kOEr7nJFm)}9dAj)G8fxSG}!4u!M?7)_3WN<>i6yB7r%dQiQNQV*9GF~ zD17aIIJB<(3p*3=jW8mawzRY$5-SLVdME$94-UjJfCw`Z;AA-GC#w|)}9fg8!z`o_j2Tn31upOC|N zbslhqT znhv`sGgkaERUwfH)F@_lc7Pu7Vt!`_u>tx6#|7TuuPFpx!2%rwT>S|^8BQ07f(?2& zW8@f`HGePL>m{;wf7joj9V~}aBf5Haz@>M#-){r4e+SZtFbk4^a)Dg5e!CYBqCO^R zsV-O4jXeE(xb9#ig+=9etT7_AahDTt1}79+v9nEjCgs#lC9d~w{r9*g&^`r+cL*!8 z*1x%FK4xfC+k9UL-J2VAaK_Q7fsK@+K5{Boo#v&38Ri>KbgbPZ$C_K;*lzSlDBezC zCU^o+3F$>#9sed&tadlM|NQ*cbFkcVTg+FmN=O-2rq0r9VGEUvvEhU5;UpL|}PUD-JDs*Av1V zuJ%%SPqDz2m8T!BE`757&ul!~>PE%dv@iFO@BRI8$JA>X!_(lcYS!uD5SKxV3nLsE z1PlV;Q&B0hnJi{Es0{&TG1>0z3JvG3T(PXIEc@dRneb2-{8)1U_ps~N&}@4ZT*P=h z&vV3;is1E|74{RDG7lu3$IujPW9Rz5^`8@2KkEi;i~L6H+R^@yO< zjntT`bAX++qM}*ADC1L8=CdFzP5Ywk=#>7|31i8Z84n6gGa%SAYMJ(@g~Q| za0kjDSpYGUVUdL@gb38hqw&F-Ywtl(LI4juilG}R&M6ti0|5A>p=doNqnI&F%*apz z0ttO&?-1Fb-QC?a_I$j&piLEm$g@`qG&w#i>+|+>xdNO8B@%2}!5^vSU*DOBw^eEm z?vgwKzy>u#9Yi_L=dkA(MMW1*9!B2X2l1}u_pBX{3g1rw_FAk^*LFiYkKI|Fc-OtI zo_O3mW9dBUy4Dem-mF=S*Aqu2wGq$GCk*KYtSh-=Y*D1XraRf#sj!Rs7H%fR#|I@P z&`O6L0OHdyHhyhJUKRKSo?B=7j#1-tv;H_*gtkLr=QjU@x7TsyA3}J%zm&sP)%+3! zu`Ii)mM{sL&RWYMa)9x$k3f6K&dj`as=k?bkCAct&$$q=*ix~~h!T0hAt)2ftgJ4NqdRdnxRh!+Sy`V# zGu#WNnf)b0clcGr>$I&4U0A-4PMM#a<>Z(9rs}i~lcluX4La-Ct}gjmg}hVX=XKGB6d=?uJvN6t<2dQrs^x&6WTCC z)9n5DF;td$eQgbxLJvN#kkB_jLm_bK=s~ZAz2bcEUGmkdSH@dy06_sB7Ej#?nhXGO z3Epak83x@BkqTpk4qrkiO94wH?7e~@u?Ii3TcBA_*e__CdV4sCnb;*UYqpt36cgCi z{(~k&ZuQZ&dT0}4kda2JN54P1W2no^Y1N;b&#xQqaCiK){@r%iXH>UnSJdAILMGmA z-VlWF!4~B6I(XtMMm^W$Xtlri9_i~973|`qY#*JY&F> zc`gI9wU)S7r+wnqlf^dxCLmkB+Ef$;J_@vF`>(RjZeW*g z&r~YW5MpUnKA&rhCQ%jaqBJV>r^0IUdgr#1mlH2XL_l@vfwio%vq->0j4Ht9r&Ru= zJ%U>`>^V_%s&r_-KC##4hwnzoJ%L~-&!^LkS1Tr9~CWcb@6c%vk^VfBKw^8vc-$lPaCZ2!uWuuzmAdVXzjY$*m zyg0VT>&b<--G08;wx?hSodPJ_8tifgF|lk=Pw@f;>}GP4@&Ua$WB_MlfF+{T^7R$T z683xs7qAz4w%4}be0jFB@J{wlb+EN5<2dItZgTdcyYSq4r9=El8v`186WYsTo6XN0 zzL6OeWcQ;iTwU* z-VQ`i9qWW9v; znpeHxBbtvVl!X4wyLtoTZTb5n{)(frfX>(O@7Ll_4ahV1h2f*U7G95NOKU7_X z`?Tm#yV~1D{`reHBy0Qp{QoR~?KElyNvq4cqNb*%Al!`NiOw{Ms6{^^ERRqMyFoXo z`0o9?|E9+*ReNb>_{6c~O?`as><^y>8)EAaL}Qj{f!Wvd0t;tc_c!eq?jjx@?tTs& z7sZU54$ZZZj3?N3><_2>cDcibWEB&Wb$Y*u@Ah$D$P>w+D)tq~#JkHF=DWq67NTr; zR0MP56L@M@sw5GT8@7H}5~w`F)NygNRqCq>jqI&)2^!iYb}?{m<#qj#KPRX4`7RFd z%zn_)$hBOm$6iOp#%=^%r}}=jQ@s)Yh$_##F9vCIlEIizJx%;PWNC}m>L*pv@~MqC zYqiuw1-3n)O5GTu{!YWl`0;|u28&dx`9jWaGBNX=TY42+s;j>uDsQJ%N0H)?s1lp6 zHNyl>ehJ<0XBgk!J6he$czH5IU(Zlo6EP)2ZKz~~a`bJrxcFNEMXn;AuV$U49ue33 z=G!cOSO(8!DQ2_v6j^+wG>vnuXGMbQ)T*VWrY9`*&?Ag@@QCG(0V2%S1eM#Z=DkyNIn=1T@z z(}dKuz8o5ChTG^XzNf=I8v{2VKEBDMd)Ogvh99XfuEdM^a5bu4OO}r8R#KCgo44#) zDN7zVtM{WjZkx4$nCiTKJ>_iOq`Ta|x{L+rm<0SpuX$K~txGs3K4oR>(tMYC*VDi+ ztkGog*Y&PdOvdg=Dc)mpNLe>$!W%GhFfI1p&1Z*~}OW3myN($?_e<}+&E zNg0&+cN-0F_D^<}mQkpS0``^65%0Uk$PvEyh+`S-P{Qu$E9U*r@0qBrHKv)~O_77M z4h?VOb6LZS6I&FBT1W$OqV_+f1*MavuE+ZF&A2Js;NkCxc5YGuDw#$T$`_>_cldQ} z$`{U~zP_(c6+~-Gk1W#D4V)sT6_7+fpHvB{cQLfqJ2FVV%(@=xuA+>tDLk-WlqE1C z6Lp|qN7T2RN5FY!g0~jCZ`dk7M20PTxo*#{R1%AtIr=73*i0_EKtCF3SPki1iXw;1 znJzI}bXAvkAwP{Dx&FnFi(t>dvOW1s3nwQh{yAlf8nlRc99&#nfQ%q9eEFeA%nXxH_B>&T5Em(8c!9vCMSD%(#i21CLU*nca zOh}w$^Vo-9>I{oiI+XjlRE15J-E^MDj9~ILO&T8VF>(Zyu(ZK50ZYnXi#)pRU{3HQTbs ztIYflboNv+zlVU$;k<-B;qHeG$imLP20HdijPRaQ)Sxf$A%Mfd(D1)S??e}_XtSO1QLb}H(u!FM4y(ED*hlYa9qn+wlummK|`H9<)KmjeB>nskpXq5ND1 z#-Li_o;a+^iUad?Y1~joe%3Mn9j!F70qx)s12d)ZK_)DFR2&=qgtucjmCA%OE<3FT z4t9@1nj%wXCi|&M$OxxW{_4rIm6w)EC`Knx6k*7tg$7LoTm-XBktoc5WQ@WwQ!DP= z58ocoGB}!$NYb9xNYCXJQeP{uNLSP|VS4VEI40x^9&asVhY&J^&_ZYi{hvQu!VSHq z5=}>NFk+1YMUDaU=pAf*fN6?4?arsmh|UhX=520n?v#Gk?p=|~)79q-t;u%P?^Bu< zUqs<@HV1+TFaesJt&lijXnO#JVK8q{tX+Np_oD^K?x4G0IqR?FM-%Z8a$}_SY!s0` zz4p=%Z5>E#(Bn_zyCw1n#q7V7Oknw@PShu?+fh@jEAHw9l@sH73wXAx%wKC&wMiHo z(oL}1JCI$oXR2TKE7HKSj89H#bhGmrV}2Rf=;HbBW9+@~X3|+rd)i+Labijf&$OL= ztGGjDA`)t-*Aq+VEYr-ptLiRHWH_7liojxzOG){}(|0!*tVl>U!38@3`E~mDt8Wq* z69Sw-rFi(jnoB!3O4<0x)! zp>6m6eTVt}EYHOO#D)MV=@hgFMTKubFnv##!!;5GL-QXd|CyF-q%wOH-g$oz%PK$2 z!ls^)ExMNPz0DkMC@V?R^dpNqywB5)bPR%-8+ISxu#-=D_oa17C+XZYS$@$_Pyl`Y ze0(b9PVpG4Of@^VrNPnA9^Z4C#_=@fx7&ov#&)I8>2B%1`f5wuEd9e`>-sywZvQ41 zyzM~yPR?p;ze2vyq=XEI)<{OBlmpO6j=FKVoFL7Tn3fhv_!7L%KtzdwzRCl!vj@~# zCGYJ)kD7NG;%L9RJU6pHzwigUZTr#`tQf>s0)ST8o5oCCwD%i!kMnZqtqHJDO}*Di zA$5W%?P{ROrag3LHEHuG0v0&`Vy6-@ydi5*2jVb@o>?WP40%hUwZ{jZ;j7mQ8qjIk zJMhc~Mq?g?Xfm!9gm%jmaAQ48wqZz>%H!c~sK+*4;hJ3No%oBB8U8$ys`*8%n&CE2 z>B7C2({D?d91|y2-Cs9LD7Zg87bVD>cNWZ6$UB#PF%;^5|M^!!F!9Kcvkj^gxX>|B z+=Ipo9vz*o#v%(rNKLJ#tD7?YUK8ABz@GQde(x&5%UjkGl)68$g(U0QfTAM;6=B|O zQY90V00u@zou0oek}M4ax2-a}Sl(u?PBsSe4}H*ECYJoK#J)`)F>P z`FxpZYXFncon3mm5E4Hqh)X?HrW5tX^u+*&{J$Zt4sNq<^j44E!Txyqh3_4ZBl>W< zg8Sj$p%UWMt3KQ+B=2d;ZwGVB7#t%DWGO&bR)hZ($jW8bs|68&A3PJLkwP4lwgK5 zmKO=XxO(N4zDa0f6rMn;9d)zn-M<6MQq#5~dJf^pi}SH7?O}H*yK-)x7#x|_joVHp zECnAUL(1H;{tKA_6ojV`pO_e|`Rd<4-&XgXPjE+Yw^o9yulWSef7bnleLiludh{2u z^dI}3kOJE0oPNeU+i0%@>9$QsJ-+w(!&zin%BpDc*;;JzU0ht81PCFi_w6bAC_OXF z_0HZzjGt?+yoEs0XhXyw4?p%}W7ZcB4AQM765?6;D2`u@R;ttRf0tjNh z)vGpD&JkO%pA_n_o2fudce6eSeT{+yYr1|!@`~3rLwibx3t}rstGDIB9L?zIxgwUj zkVMZ-uCMT!;MC2fbi8ffhR0f;cuVe7l8jy%yIaP3D5q!RDE%Q@*o3B2}rWT84qG1gOdT$+jyzgJxCVlcle8*|Jg@~IU>9P z?c|1?hFB&5Sp)$Xf3343Kzv-trv5u$y2AYa>5iiX0GoM;!$NKb4Jq*8<9>#d1~tcC zzuAcvc{%uCIwC^L%(9E({|?aZWw~jlS$0b;i#TVV*`t6G@JLYsy5hpFh`24m6)xcu zKCiKa`W_LP%tBSwK-|;UD7kU+^_;a+g!fk|Zw}x*7!=LZmwlu*%L+K2-I)4nhh#Yu{X~LeIAmsh$k|IeY zAm?rSA2NnG+g>9p*yia7c|bP3szQhsfhP0r=|>z$(U}}Ne668{s%te>qJc;S5Ry6s z=PP`Atr8yuEw<`*MCpn`J z{Av}4gD14s?d#xRZ)%45(=&{)TDJwoIk6?p+J@h*M=x~>{5 zlRLjzZx;`ivf}Bz!LwGEic3JJyrdn^XL?!qT#fR)t+I=03gzxlWes=^B5-CkF#5*A zO+jD*qe1y=D}1-5h%C$A@x$L|4xr-J?1F^z2z)Zd=rc$$pflHflGM>Cn?62vLo6tW zBI)Ki{ZVHwVrFK30yWzWK1nge=3s;}U$ON8686qDrKD9J^>qN+z1eJ6}&#DB5FNhPEcMbWD^oxIwYTR4pI!?UNd^ z!h7x*W{}h2OABsa-^Gjy6s=4q-F>_3F7@q$#q@X9xA9oms=VwwT64V>CbuM6l!;eX z;(75Ud5Ri?S5PMS|&#cy5PmFqG(O?CYNr`IMFhhQe9zu#08oZQh z<2TZM zU!CWLShdh3tHnVTu#_W7e!Ccv%z2+|OE1KWExmQTXZZ%>%@2W3RvUe1I(|J)a+B)Z zEs@>rsW}R~b(cNgX9Lr1Q~f&ibb*-jQyHvZjf4}}+AJEhZc`B252wQ}IH+ z3Ip!fVWCxLf*Q>s{o|{Ac9wRLWEIPa54yVUtIG$$W1*cyju+Wrrq3Y|&*zcNWhl>V~WZ z76chcC5NBl>Co0%hZv3DowsZ-&=_cAR}h=MZ%Teegw8UC^I%NYx=kWDLs8c}?ecZP zPHFk5b-nW=BDH8ECF7@eaY&>U>P={*uzC)IXP7iuUki3*p~}#P21UL)+dK%tBnW?* z>glc%ao;URzxNdB1dwNd8o$@NDzjq_ZVp5)N1*R<583PLl8Ima;{zFF9W*?=IV&h_ zKj&K#K@fC;eb=%}Mn+Z(+E9{aCZzWrUOaMKxvQX_Dz)rsOJ*p!LEc^Cn~%Xv8*f;! zd)zR+g=zLx;NI#AIuoXty>~XV<@zZ1JJt_IwEMEw|57AL%851DS>_u z1mufTxy?{xMSqhceHf5P5m-0IUdy5Nc3)yZ%ZUe34&u}!gUM{a{`x|Io)?6EM3tp_ zu}P~6nmjT&fP!cyi@rxH@R_DnF-ty`qQ1EpSd@6@9OKr@4Cb0CD(by?GPFg0FO|yn zlm1nm&b={vq@1p~%7>w7_`)-fq1a7Pw)r9DAHJm80-{qQdKqiBjaP1UO-l7Rnk9I6 zlM4HKp+BTW<|=j=#tmr;eH8QMd-4_R*ETjtu+WW-q#AA;n26kbGMPpV5PB1j%~>Ci&^G>eb*2mJ~3` zSrdOyMr%L|LT(fctOW)xkn@Yd*+iN-^yBd=6JfpgEeuZ27{&jxA z@3=U>%Xip3c!(A&_Y*Qo8QndPuhXKJ&}atJN=dh5)#nB#8fY6OUN)(;C|`|I_ujVb zc>RNl`O`*SSv#{?3#MU_rz8~%x-n@4O_=l@!~BpRZw0)SGuz`&B~)+()AOYg@4OIL zo>%Q_UKB&M!j~$fGXs|2e(`$_3h?9yKcB*+=ih2WySt@g#4o)Nu^_Bs6#PqQt~~DB z+K{g!6(B~M7KX0(s*-nsGs#KD=GExt9a}$Mvrdb&WV8o4-(Pu{iRW;Nc+Fv zq^3qd4C2R+iE=$I+GDYSf~f*M%ABf}bPhLw;or1q&eq+4A#sjzVYL!emp?riStb`WJT zq3VpdS(1f<_RHo#CR0T>AxBQqaH9Ji6%SlGtx{yBbvwhxA&v#EnKal@2Y-DbJ)_y= zV1kT@iC_LA0D#m7`L8~>do9IOT+Co-4X8nZZ50(Ee(~9kfHx6W9=xa^m>=uY{zz4( z*eGS2*y6gu3vHFk_dv3K2G$a-QUIA7D9&(rI#Xfr`p$!xy@j?nW%@)9`>bgPKO#D+@r|@m8*(}RHO?O-_MBaI>lSu4gOR}RP zN1c_iA?^)PAhpo#FI*g!D3TeGng)jgIL{fR52}9Bn zI4vYR@IC*+a@#-}?`sgH`;-Pe<66ISk0#2uhlkkrG9^@1@vr0G1&;%R79E}_B87q? z(_(K|sqnwqA1Gr;{{?41cxitlKZ^6|T7NuZqw%=wfN2DrLU&N_0>&JwdQ1PPlc;N= ziQ%z~u}0Lmp0P+oNF^QQh3Nc}{d&S!`EyqC-lzA<@(hwbd|S%8jB%Q6AF(e-#!|6v zR~c%Sy?Ad~G0zjDy}K6lZZTc)M}B0dMBDudbseQ|BbXz+kCt{JKnmebpk?dOHJ%}5 z05%g8wQC2RXasSIi4cvaLxH3{Qhfm-W+9XWj+NmmJ`oBMc!$ckHB)&9U<_MPN@War zVu`06gJe-rk%qQKvMZ4JJ4mJ((SJ({VqAn_y5JcQNJ|egP<4<_E$mr!{_q%zW*dfx z_~~~v*m203tR<>FgNTa5Iry6E;QGA<9qT3;8H7VRP;|T2AQ;x_wj~8qh$ipr>W~jY z!fKG1tt!u(Ne72?3<`D0>T6$&RB-+6fR6-}7QwLT7J_8PyztIN1qtS`H`KhrZL2RW8aHLQcQo_&sQ#+cC6 zAYEQdk_-`Svc6#$rqDrX(y%o=2Uk>>1P*8rk{N@u3&Mu6zc)l!^;_LoV0Bz~9U*^! z1Rr2fk`FqpL(6)KrgtWEBQHoxBiRcfuLGK58%=HPwe3OkUrLrbHKY^XO)#j4q@^KG z!R>u$@hU0`5xf^-Jon}Y;KwMas6@X=kb>f@=JZ+M3%AV#gB^+(HKfDupnxQr4|j5U;7BIADP#e&W~)iGB6W}sus3pCklTWkCK4@nhWRR3^RLa$ z^unHd7Qim+9Tpxwz55c#X&8uLwZOO`GKkF1BBBFubp%`hxqp+;nZ=`r2}=zD{9Rvg zlOc_UT{C|K4~Pu9&Sx;Ay+DhFV^RR3;d-Mxq~U5|s0n`XBlsBaqt6daZnXjAofpuA zU}7m!FroD!Bhp8pfHYV6BV`lPo@R&^;(PQt(|yJu&@%TQbXCMs1Rby)ylkNV=z&O< z2-K1{NUFC`fn?EZHM++357i}A4O2V1a$!mp{@kIe)4#4tbltoY0*=Q&aY|J9Q#td# zCUW&2t?d(pHY(+@n9+rvILYX9CYIgJko0y^E0{^Slk4XfLYY+-imiI-F}FTR+@VC0 zL)hK$_G4dPc;z|7JyRbzQb9r&wSpo=Q*mW=_3vt|_~w^tYGg(m1fAob-x56+y~6=y zrh%RZ08AMK)vL2F9T1DjcU<~lRu&6sH|~H-U%gRC^E<-v?1pr2PvLPHP6TP0Sz26N z=X{H+0f;^Tu8_bVG$FHeCd5z$kLxW^!6H?4c0>e$fL)T|A_P`zK4kFQNd1?GS;>V` zyw}6v<_FD9&(zy;io_Kp{uY^iNRbap3x7;jiEBJ4xVg4|ou|Rf=XeSa2j}G`olz0p z&+lb%+YcYE-O?9e4>(=E&dbXSMThf$-aI%WY!FdKA_js-!+emzvHF}Y02!J(sOP#> z#sg~N7rn3&BN~q=Wr5%zv36v8f%;xy)D!~((2}*i&cru}r@$Gf=is;z2!~Ru&BGq{ z8oZDJB5_cSU<&Ui&g=0hres;=LoID@%m2>;#JE`!NjU*TY~6v6xx5zVj2ZL*RoZd0YR$`U|uUJMh>(9zn~)$;bAHvvgf}pfGaJ_ zVRukYk0;eyU|KygDyp0;o9Lkw&&}x06iyT9wp#KE3TBW(U0YvAvLCIBG?3!HpW}bV z0)2&)LfcZImY0fpdVDGGDP+S$fV4r_Z!O}@!IG~d?xgs<+|y1i4oVj8;8zcAt}%)F z5Mu{X4Ep9tD}L5^UerGAA$KA!xd%Zb<074%NzH1(lZQj9?&@hZZucBxR?m!Q#KI{k z9xwVs?OOYI>+$Q-HZFnMu`o(x76_3&p_^#}<~8-%6+ng;bo5Lx4A}vN)DOkn zKi0TCWb55aSzbBVp6=CkAH)k3R6T)ZZ4#-#n*lLzlV!e7v3>k4fo-&N8X7*lAx{;t z?nm76Q$4bcoinnSR#1B{@^1?9>;QyNz1EZUHs@RTz;FN*gOWl7fwmaL4+S|jE-lSq zIfnkaX*^oVE7l@*uWe$GW?~Z3=#=txeq9KCyLMOF(%P*EE4JQks{LI^llopMp0k{* z_-omJ2y3!y@zp+F+L6VjbhFgJiiX2uL57A6n%3R=8Ui@mHEQ9;YF z%v@uIrDaydVd0_mIn0RU;aNNNPi6@-n)*$Lkmv17eWT)FR4ARN73wKRchr8i_b9QE zty@6gBNr*%K0=Nre#4`b15pEvBt|i@Mar)bUIZ>P-5b^H_TB z_Y@f-avl6br!ni(H^C$q3i_~UP0nI&8KqNsM#9V)Jn?^9Z8>j0J*p^XA_JA~v5 zlJ$n;^J#OuAWhWgK7eNg>6FDwz9>-@d}G|w8%>z9zG&*0&yB&n)pt|3%%_s!aI|t^ zIgi+B`npuhHm(Y`vz#F4@R z8Q#{nhN=udPdP4ZR}(|G%7Rd;Mb}qae&q&Q4y#Lf4lcH$kvqBQzXq(VH$|r7@8~cz zC}>sGF$GOY8e7qX*>W2h?UyGcycj76!XFHBNEj2vUqPJ~Iqav0agzJRKGESlXhQ)5 zPd~S!d6Zpai!N;}S5&ZnFRWy9aFni$*P-L7LC2GfJz;f=AATzXU0HZa&5;UQ;`~dR zNOFBW)q>1Fta+k#4k_j!76wP?IbW+QcW?^j2w}^ zl9F!5Hpil{8^nJMr@oMUswqM-c|%5hP!Y4RzBBw&1Ybl{taOGXKLv}VPr@Iw8??0a z9jVI&Gc*^dHmu6Iy(EUYA)ok3AAP|-7#o{GUwLm>`G{39-qS)~r1K>PZZm|Idj_w_ z!-8fQBL{%8uZWwsQ$P9vf*spy0rJz?P9btx_;dS9nZYFT1^NSSRw& z(&EsshxYl3wh!}m|oDaTH|xi`Iq9|UO*CKo;ie-%co zp@7!gPQL86&x?LQd9@(r_mPBwvp%|wBwqCqIrUb}G4-eUYDq&dq=keJY&N$%6J<7|B- z1pKoRn-4Z4F&-vlehWdApLgz~#G*BJe+BZiR3P!kTnt6?+YL3(^&MEa>~oXZSxAa^ zjR&%u2F!v}f8jIZ*&D7e8h2QQ>VBW3APG`O&9fM|)2J2nV^m&sC2tlIf80~#6E$Yi zszU2GJ6W@zq%%Pl6Qv?@p-AZ#`1IQJ$riTVT^FM+`Qjq9apWo3O4jwfW8H&VCM(yU zSL0?4Nrg9)yf`h_m=P`i*{c#WnkyAp16$PO?<2c%LX7X)mA^{P`=yizgr_19s|4DH z;_{W%{_yG=bNR`Y=)TcHokbc0UYwo0e4q>sIuSZLYW`YnsLB}bpo;-|OFzE2`Wo>~ z86t~E2Ani2*KZNZ++TMgwYdH*i9LkFj36;iw9akX$3LLmaoTP4c=%X?I6<8tXKZqM zL1%h;K`rayhR~>u<7odGSk6{3%YbljWEcyh8^vezJ&H3-QEwa~5T}QD}p0oE;EY$4PKbX>uGauJQW@@>0a=>AA~`n7@Zpn6HEc;b4$pt zTzH{EpCnEuO9WMC1qM<7%9n;`yDXXvv~=w3Ro~W8ofmRh)6W{?ICOAu_`wzM7kAE~ zjpO9x#P5Xzn#jfRWPMZ9pYk4;u9p}QSa2Edi$%d!M2|DUE*z2#L6i(Y8LvJ*{n#q} z@F8LXe*XMfbngi1xz*L9>CrBSBgpE8IINjGemshRF9>nWRFm$4p(Y@zrqcpo*&D|8 zy&w=u4xvwkr)%7F_gNhE|IH@`6mrr3+&(=c1IY)}%p5{o=3p}CGU?MNIb`$#QS&ZW z#V^qUA#nNzRKnP&|3}kTheerweG~Gp(vmG>Rad5`vVZ zw2C4vp|pU6bk}>H@9%p5V6Sy{5$2gY&iT}-`V7*U|C!_gMyTFz-5)bG^Zt`UPJrEi ze^(7k-HhcXY1RPg zd5J6rsN-I_F7&t#DYNU}--Ps%KTv_qLfV7v_I$VB!H28|3$gc>5TkF~#ai1q+@(lN z3nI}uTDsTvMMG>uPEipTsg?pyhO?+d+*l%L-S8CY{RJ;i+b*NgTd-c;dv4gfK%ZF+ zb)M~$KQ6yts?3UPFT;{}kVoSUP=6ib0X&<(-ZpzMfWP^ z!#2x7of)#H4RbhgQ1QeF2nq?gf*)pQYUZMj#`|*Bux%cG$37iyAPgj-3t&%T*Icr^ zJh@)~%k3HlJjZ*8Oba|@4-VERoZh$NfcKp=i23FH#Fm4}XlR;Hy#f(*UQ|fjh2_Og z8h4%kwc~?=r`r0ftE)cdigC0eZala2QV8Hz2seW)gA9)$Wi!uFeOkY*cEsce+R+rq z%@FdnP_dzLdVi&t)?F=&#uPg1O<36q6&T2iP&aDIKlKL{T{AS(@XpvwLHQ$$c*Wj& zFWrD2MRor68FCG z-Fx?jCnvwcdeR`5r$Ajmy3})kPrWbC4{wP3c+^=!Th?RJ2Y}3Xh#2XF(*^k$e;||$ z;y`{<|GNj)eMWgvMz$r?wQ-`W#ZHn_1ecIX%GQ?ImDVu-(NL09j@D-< z$6ep-GIn)1>JN`={hVMgvUBtpua3GExrMoZdfv^bv*mKjcohD!n_Cg+&enR7;r%wxWa$bnIFWL2h4&Q&w!uN^9>BJADV>dB1g|k&o z&Iry)#7+yV716OzwB9m$9|Xq$m~K$0)}53}yU@Tclp@lXC9)_RH1NzsDp`vi#I|D94Q(&{~W=2b`r*!MRPr8|gj3pTqeAo_=o)r3Gd0F*?Md=mx&Bm#(l zYX$h0@wE#m7QGP|!Cxt>9<=KpKv4j~f-v;J;9&dN@h*}NVCqzbjX2CHEP)~JK2A(@2qRg8CN7jg|H%@pSZw$~>5dZ(zFLCPh@Y}0)Xsx~hBLK@&5Vj5j zP=bqa;5~rb)GY9PKf|R1pFCP#L-O3CqUFE@v3nt#608Xluuz*V4r|#-;ZjwM)1>w% zX23ykk)LMnME8DskkR{XUBR$C8A?Ma2;`IQ!`E&(W3$6z^6}n;dBROd|1Z=P{ z3@!aVDe(grXlAUS3o_2BI`c*jDR@~3NtA)BCIcXvphLRLw#!q=fa5+(H+ZY=@OgcJ zj1>$Q;C2Rn{rU#iwlXYn|%3J+3BgyM}F+)!n{@#4VU8;8Bc4b_A>@u&D7rsT0 z7*nb8j#=K)ilpNDWLk?nnG~$}xI~R~EpYFJq7m2*gj*~Z=QEe*%+4NAC?H(K)K108 z8gf3u*So3R)wMN*YAIdHy*K%YxgSX4(y8B|+?;@~52v*?7t1TbQorzn^{>2;3q`+5 ztfDfEFp3b3=(JIC2@;jHW1(M;XYO^A(ccMxQ>bCJ^xe^S=KxtpNA58bVu>RwgnDW{ z>b~V=3MwSkhMlk6A|S*nU5bB~!)<*KA@zeRH?h__)B6`bg8Usa@W?!S_!MOS?n5i zloD;+M}}3cZP_^Hg;!Pik=K{XRg?dl`kG+-F`LHwo}l2s0Q{ z#|uXfM31>%p04ex^>Hu_Z+~fhK`Syw;=#NLcIawozjiz6FXF-Kq|KF59sX*s&deYF z4e<{j!5clO^2(RQB07JW{4NvEA$G+*xihEVMmt7kokukhT_@aOB`vDd6-0?*7bY8C zNf!R{@<|+RAKCBEzTga3OBalUtbqz9oekbF0k3r<@bH=>3FsU8kq?!Hd7PR!YX-22 zyf`G54Sf_-{KKBT@G3A(-Ggz=9m0NYavjM&?8_v0b`_rta}U*?2q;r{aZ;4Ip;$4Z}{n4sTkbSNh{2`@x8Ham#1 zhb#NSf!B@L;hjBH4tf8<1Aw@ahk=J9WyAZe0^#rk_I(#>IiwxD{mZe=wGUYpMFkAZm4#c*~TUfEFXS_#b0|jf5)^)B z9;OvB2rRy8nd)u;}x*XyC6B%uWH{el%vk`M8gG5(Co{e>Xr{wnIh(C}J z;6C4gGpNx9&AUYHaC7%J?FQ=2hFW>_sA*YGl&dgycRK)&p z3BviWmVLYNpX!t9PGYgLSK{B`DY}Ko@+)8{`iIDqMrRirWl5l-MM;tqxw=nep6fGi zh8cAa#5T7b)K+c-g5e4c;sg3IU@HEH@`2x|+racmia2lzACa;PDGq797ZbQWdrwd0 zOCMy*T>Qb(u3Xo@!fr(_cLl!=PsfY&dfMoHEG?np=fk(0--ouc<142@Q@cBq5P&{e36#*@~j%YF#iujahkdqEWRF z7HPYo6xhV;>-ci>`(kPR{<5Wp)?iExcjJT*&XsqR?4;>pLqFPI3`$|PZp&t{yF^?` zd>?mKE%tvKpyj89FRItIyBd(n^vkm!mt~E?OG^(S>gO5p{Vgfvw#hAi`1g5bE9ak; z?@jqd%Z9%9pOLTAFa>u%n_>;`|8ma(6*_*{WSdCh8C@TV9q32&c#${%vfwjK3p6|d z#nc3ztg~IZx`n(mMVKB*;rl<1`e^n@%DFEe>uHt<8E|^055k#LH)C|d6KMuWGrT@Ao29I3D}l?!eBUU{A^zdG)Pi*9#G@ z=Qd+l6!;RsI5F8a+7zF%`6OGB+8P#Ng&Z5xIgqwIVUZbki8>`R6k^OP z-NrN|X*ZeKo1>9N_WS(-N6)H^*}D^VWEaOj0ufi?_F>y)CP1t2|23}N1~v>jn(+YR z4(JO*&On9n0G2pXmD2jGu$i zo*ixf&)tvI$YWy$&`KOlAGRvm+S+zERyNENP*YPQf)BU(f0%h%iFeaed2$|aV`g}e zn6|Ub#_Z>p3G{PpJc$)7Z$Wo_dcorQ0#hJ2jE5TcQiL*=k!zCV3(~H6XDjYhP$tAt z2?W!Fb+wpFIdMe3iheEa5ki|^j)%J+OAg7duYSGRS)_^~hn_Nd?6qJd>~Vi` zay2$i&p$7M4R1CkKAz+jZFmYE$9|(f{qLgNa%-E8rDBJD!!o7R+mB!O|5DG|)cl&o z7ghTFv0eOKuU2i6BudLp_kn@Wxm3&8@%*!D_lI>w8zJFs&V4>r;bUV%|T@?`Mk z5ilhEh>CnMK@r39eKYW&#|QOo-=R4=`8CU$X4U}&UN4l^NDYeUe~|JGRI8fU$L6r3 z)`9y&yfskZW`T?74brSb@sf(@bdjKkxz33H_0o{fwfmX*B|o?W;By^+kY129j5@OR;-7)4a$6yk zPUlif5|VG^Fa3z#NqlEo{W2QKu9%V6U%S&E&N^-*hW2H5b{3@$W1}Z}eSMqS-S039 zA6wr{t19%|YaUmA@8V);?&@^B@>Zeh#U0Hr;$&{s zh7y^3ze;80d_ZqO7zT1PAmNl?Z|DHVUPe{*8i?kKT3g>D+%B@b5Ivm!&xAy4s1Pfg zcL<;#>bv;{(aixsf_b9iliL+ZV5|t=$F`;+gM%EprGFwdPV` z;DSkg|EXZcD?RhA1}&8_Z43ggV!Q-hJ+s1f4N^tvvt-JU;1PZi3(~|{<|i02vtMzp z2$OjcwnwH3s>m$gJqg9XwHcF0M1YfVa+o61K0 z1PbbplMwqO(x*ez3*xiTfMh_BInrHh%v=x!AvSviy+fMdlarI1(IH5y4l1;fpFi#U zAZ0SVwoc~7Z9oAawb=+oSNWk|6v;9Jv7{CU`Rxa)jd4ge7c`VFA;;>r=?_?!@5id4 zwL_>S6BCo8uZFE1Vb?9j9d$`Q0kt*({+$n}=A-=P1yPREH95&;!()fvN@f&GgY?lP zdbF%p4g40@J8iqEuX3?KWpE`=C||wHW<+r$fFwxM6px_C@(M%ia|iQu*?ZPa{z*mK z=UH+NC<;>7E~es;S9r4PKS*-+XRbRA=U@|OkmM9czgKA9tZgKzs=P3?5HX`W9QqM2*FPlXT(h9DCchr=#s)tXGxzwn5#Yc=GVgw**^}?7}EpcX#)0tN98Pl6L(Gu&tRj zQN0wv385DQ#x$VL4-_Z>F=WdyOMpEadG(&NMO`N1$0{sjyE|@m73|tFrlzK;CkUiq z+Y>WaUO#OE5AVhxAHaWbfA0cJDe4GNN2FKN z?VJ0u^?Ng*zISABHQ3{oi=kbwBGD>2%`2nWqd-?$4ZG<~J{qAi9CnCFV2YW=&@<|nALAbD_BoeAFkk=9)(GMcVX-ZS4 z_J?nt^Z={@7t|F$@>Z8Q8Lx3`+`GSAWC1uxKz)aVLBzceRVNJ+m;S%Ae&Rkr8z@eKJgWZLwsKS}q&IsH;WTAzQl{KMJu zQxfe;83F1G%D_=;(wObiw3^Numa*%abj2(au3rF7%`W_Wie`?pdh?{t8VpEwfI%<8 zJ-YxEhj`oR@6Coksix22#iP0Z0>{-ge^W4Z0HD&304jius{8ti{U^RhB6)fILt7sy zOViWSjpz8+{=XIg(Ep21))90)fExCp6lta58(!Yu!9$7>nkO}=tn+pIkB}YKFdKaB z2oUWq&{g)qlU1PYMG8hFi2=Eg5Db&nV?Y|oABD3HQ$i&RT%*q4ALA=t_%R*9ZvoU%*xf&+&P~BD76pr@rpId=QsasF7`% z0t-h%^rvJg-bfY}2Pv{H`2aDSmiq$xUdj)WCcSxhN=~{H<%shH`l*=Mi}O$M4bDd+ z+UJ#Aj4VfZRj)EWcBO0%Md@tr={+w#S{`+ie-%J%-tv=`{J|;Lee?-g%_tteL90TD zdT=@qZ@$OiH~VY8=KgVi&=Oz7&Ig_9CLSWy(=Vu(5E)fXUEL6XUPuKDl!M+bPOQM$ zVc_Mx^78WE$(+zn`5gSI8FK?RYG8U=BS))vKz(j zYHDk>4Gj&eF4XT%gaWc$wB-MNF!0D~6g#o=aLQJT$;>XhGwffQpzD*{y!BV~Y*@5i zTA%ZFWhcB$3u%;B%P6=*O3pohkGUk2pV!Z4{)35}`FeB=TYG8l5fihG%j#uLCP871 z0EhnAQ8Zr24?${DTXaSz7suUX*j9~s}ZVP1R$|K#P^@}+gHMLI&4{2I$FR+Ys3-PK{XtGUD~kp<0tiXC4* z^^u)Ru(9<-c8X>xxC#$o=2D;i5RyBOx)aQ1a(6_g4{eHC___JyIT057{+_F4XbMl7 zC2yX7++JyC-i{nrN`jl_mZ1)QE+u2fJe859T#O{6qY~W<=jok*jgkY&-mh{md-VQ% z{EY~Jq1=gvqBjkxlRyH|GQS-NxIR(`BL+XACp9J(AN`C)CJy8pYZF~HesN!SkX@uw zntT*|4OQKUPE7S97K`~cr01|4n&?gy8!xAAuT6GZp+cTQl{e3xE+!iiYaIAVChD$& zAf_2ZA<6m2P_I=gF)I|!UA}x~Gon{9J{Y3k>qtL6E~@U99(co!2AD%SK3_^vXU}7m zJfEb3IEqA;2A?XYI4VajxScA?zP{g^Rei$wo3rMimP@VmVGC9vI)=eIY2G#df;VIT zqxU#VdstUC!W;y zN-XWH;o&wNOQueWvi?u%21FzzN)w;`L>uWNQD&osse3aZ_VWhH%8_aFZ6l2@(_TfN zW0ePQ+Sz^~@fYUZKi5zNvgGgH7TfsoJ?qP%f8QJ0o70k)TcNMPNz6f##DZ<(Ngk`? z7I|yjShI+d9-Ey2Tc5PbDA9`c&#k%4J92o!nr(4X-zKGP2!*2=dnQ5?3BrRljf6M! z#_|X4jU8@jTXQ^FH#yW~c``&0x~ZwuT7Bd7t7q$4HwD*FH>8{fin((sg)UUO<{nW^ zeqL_nf2$L(_fTH7tEreK%i-@U%lWWgOBUtQDR%o_t!yj5!{Q(A5s&CK*ZmXPI&%!9 zm|0xO<fEI8dmbEMRHKC5Q1^vQ&3dtm}gT+4~qF97gC5?X2ap{SzR zi)p1&*HV+kf_DS@#f*kD$>h*n`tp!Y8az|M=;MVk(z~I#d5?X)y=iw&k)?vd-rz1w5?|ZK*0bSNvLuHlF@_KheAR*g|tr zQn8vJizImGNorpOTMw?$^RK1$zL#$g_jox>RfVmGy(YDAH#K%V{zs+@Qyi0xsLaLm zz1g&+^4ip{{2oJX`{_w(JAd3_ZFE#>czbL^!{KI`irU%`aoPB6(2A}h)7^{=pUX2j zKjJSpt`q!Vtd(E9mSf}Ek201`^k9&?$bUelHyQMz#hIbkO;&wlTanfk3zyD=1UOD% zC}p${8cKKS+sRBf^{;@wC1q(m$Pkp%FAt!o>Db;I!Cknoq&yi z76*xOj$TY~o+U{~!EGrawDxM^D`Ax7I{D^*7NqJWSMjy$ek`9*K0~#$w6-fKH;bB? zE8R01A=gp~uw-g>xkavC?*8`Z$^uhH*F;yA;K{)Sm&IS}M5>>YQC&nFtwE2gx@xSe zk_Fql1Sa_e37<-JGdcX_k?_(q5lc(t=2)ePWRcF4G;pA;lOA7>euINqe$hgK@-Xg@ zeCv1f=fC$le{a~youHfiD}JGg6ekO>-+!(g&#bL#M_XYhkWk5MM7T<*5kQacLxGOexXfM8!pv}bw|rZG!{~YrW&%MeCE4POKRxkm{ zrOUs?v-$4*DAAu@rKqp=l9H4m@x`6@io>n?IM`5SwKaSuUw1Xt@$5S-{)@GdHlkiSV$(BI2EtaHEWpoX*@N#3FD9#AF=ikd+kq=lj!tC zMhL6mMEbO;C-rpD#fl(fI%bY~?qMMJAtz-tt=nftZVyFwKT!r9<>`yM>H;$?QlUpf z=3YMyuWzIWza5feA~2dc(EOuZZ6-ZDk)Bo27YYH)dpK@>P9t*Dh+KZk=>riUQ5=97 z`z?#j%z=Rfvzcxkp(+4FsF8Ul*qa_<;&IN|?!j#NcSP(kf66jxP}ef}nXzC__srVY zQM~?w`A&YC8Yh;7`f2ro0LKGY(k8V8UEB1|&|$fRun<3$p{`<@gz zoX_d^c~|_EoE*h$`Ig92ueb7d+u;XqU~H-LjhCxTxs74qVPe(oxs%n`e-(@XM&tDW7PD8u3&a zWe7?hXPRG_+VHuO`;!EDrfXV=rZo2E|hpK|vyYH#&#ycpT~vvZ%E z)b}krd$Xd~1M-=-{wma48vnc%T3QXBQ>jHZLsNSKwrX`{se%5CPcAqeA``ywjB&U06c5pqS^B(2 zZ9B&Bw0tc-QddZ^uswfHc{#RC9GTBI$Gs``NMQOz9y>bpPZ znbDEO*SIJm%H_vOl8;3Tnctk&sC?7L7BcR*3F`%E^hQQUkyv-PaqEbxQ3F!}h(tsz zC`9*ueS&UQ)!f|tJ57Z`^Pl2wzfwQ*(KD4&&WAoGmKILtcLr*y9GLm*zD-T0WG;4l zdIv0znpzHZz9o5smvLL^de!%8vEcgXmkB(qq&ymg*vnFr-aq2z->&ehV8Y^}z;|Yx zUBgKoV0%cawM#5x^){dzvcEnXCp_Y-G#wCw`KRtq0_0tx* z#6S2?s6-vv_x}YR$s!^Mz{17>xqA(QT*;&M|1c!^=b!*C|0v#Csq}Ge$0Yv&#Rakd zQKkp$F0CW+SLOQBJUcGQ^&Nj@%cS`{i^v;k@sCK+1E_^-5ejs4q+uwBw`X!uM*4E9 zat=yPS9J3{*A&CT4Sbi$%i2A=NTh?)C{GstkS+Ef=X@(AW*$@hPu_*E-Gpp%iOdCi zfK=Ik=JX~F8a>D&Q^t83W3V!1!K_U-RqAlpf~Onci9Q4r2dujvyeQwG&o$Xq@Qn^V zwbe?gc(=At8}C-G?H@OKtyXYrk5c~`ug}3@Q_h^-+;2lQ7j0!xp)p~SO?Mp+h6Khs zN>@^a((*?~duY5aS6mG;76X@Hfw4EMqBGoCC$Hst9!_&dF;Ql5X-DK5akEq{lxDRl zR8<^GaoiCiSVcD-CEdzCu2Ab2&6Kn{(4WWli2ZpU^(Z*18%^K-aD=RAUWz%x zV^)uUHG=v`S(2NQiIU`fYER)67CKhKF)`-w*RK<(NJ%wk@sm?iFj!F$j0&BLOzqyL z$t==x-t)&F8`e7teb}AaIWD(H_rIb=V_V4#Jib0sm(5D@kP8JZOgy6OgNvE(P0h5> zk7m0si2=vnhvCyYY_%j_hSe2e>c0k-!4QMeUmB}s@UTy#fl{Xm1$AMoGnGL!MsGwGx z4ljZ%CzJja(9bPgA8I-Earpj1?20SCM#qYkZIyBinpJ^BKc1>Pcbvg=Z{fG!NQ;fF zbJ|daa{W@C<)BATLFX$Dwe4-@h3Xkrf_eX8`Etvo-4JxF&+QVnqj$p6jU0rL9LxBM zH$4yOTLN^4u1bK+8nBD|2%L}5yzp)_qXNOXBM1YvywtnIfiQ2~FLy@5RFTt$uW7vl zI@X`%v%(NSjQp1>h)1>m;S0>q6_CgTw1aN*GniezAo1bgEmNoM1>!R9{*Ka8e`X&7 zXhLY|w-iH*NHn6nybU>HH+|Oy5NRm@ao0ihqc<#VQeYQ4xxdN35xVt^p5H*A-_D&# zy!$Ae$oOU#V?UXmPyPIOO6&zMbN5bLCQkhmeaQwX|7qb&uaI<-8w7Ww{KWhjcP!?# zm&;?BoHE@bN!XZvq z4{^EQf7-GgHWSGo>LDxoJ^h_noWLLrMv zP7ZnUkcs+jM_bx~+6e>Eqa%vi|9ycQ5y^T*Y^FeyBqBfxA{s_$ zY2yzMUDwL%F}E%+W-b?y88~ld&e#wo8LVy_m~e6iDhzvAh)a->Q@JWTZ8%xLjz6#R zqTySkZty=!*XE0dH13j41y(-nLH|jh1=^Kn`a18wV78Mlc^NkH%`&*~p4eR+_P={? zs_S)A|HDR;Bn}kPm5|_OR&i!E+hWx>WERl#gy-I>JKjI43CVi2@Y9(lTCn1Gs3>k> zkm&b(XG%t8v{EpB=U?D#Q)orx>Tp%-2uBN>1dyKjoDXha%^ zgahiO0Dw#%1pd$WAfP}d73_iw7O`GB%>XXth>Q@0Qn3$wRzOW+o~iXOEiK)A6E$>5 z(d#o zOY&NfZRSTXKaIX|5;o{`ltmV~K&2g`AA&XvWLeb=?V(bkudCiG|M)vy_B=!Vvc5KT ztbR$>u3FnUH!1Z#^wH{_oGH$V`dgYkWxdwcy>mZjjQCFbiJ4Jq_V;w3cGG zB9T-{8ls$QJ*%9g=0LhYp4)dNJ^jZ|pRy?@M(x+u)F1=rV4J75cpYucA+hnGbkO0U z{1DE7LI9d2*xa))?7$8Rz6qZ(6GvVQL@<7rD43mXJ~=M2zkB9r4O zH2O??AV(Cro%Fydw>YSryLv0^+4JMTn=D+EKI%!4p0A2;4q^=sSX2_*o_)DR$C=99 z8>Mh>2mEVyfs$fC0B;22hV_P|@WKgqz4#)81_J$&SvK7Zqe>ESwv&^ST+N>#CpxCK zZr&SM0TAmOI@toKZsZ6#q^|N7I&#JTB`nT&M{^%kwzl4Md?L>W--kF60(O$J>;`=` zl{DpQS9{dyIivps^ADP>m~3s22L83Zt~a9iX*H7jG$uxVf5B3um`!fpvnl6va`R1b zn`nv7b%6&WvvU4>v?7<2HP`sHvGqrOa36+AhELuhqnT_hYmL{-ye9!;! zC)U(P$f%VV0tMVc!)x_l8vS+YN|%GxQ<4wdyk;P3qTX>Nd3M7pyR z5EyghYO4@xl&1RSo?=`&$iS}YGrin7^D_MHSh_6%@f$~=)6tRc_@f<-9vB(vdT-tN z9Ast_Fq8PoEt+4C@?yD^+#rA*Tmb@blX^{i+aqac-H^RA0bCQ(Xd;6dp|dBR)x?0D z!DVmI3Ltes^+p@8;mG%a!v24OI$+t&T<6MJ^Hn>f_)Hjm5?uPQ4ujhkD?_D-D}Q%) zYcQ)lXQ)>ql(xR}9Rm}E;9_xFFLoP2S1DRR!qSPnt9r%KS-|t;&8tt@6;8H)XQ%SX zKa7p7-8pd-Bbo9V;lvh5*1^56PbxspPeq|e$o6@JRDb08cO{ipe2JmE+M!00BjZdg z^N|)KM`W)f{al3vNAq3uDD}wvwsn+08OiYqUx!F-)kUL^#j9LYy#iqLKztlPYZnMW z-Ou%OYH|`HT5do#E01&Yw!*#YwRE@lV~Efw0NgVV`;Z7BVzKr2YW>f^Np2fs4G`$^ zv$j+7I!&alMH(|?dK+-;b5K}d&w|Sq*9sn&J=pJ&VH>#x1=vA|;vJg0Sm4%?*&j&m zDWnkf!=d#JQMn-V#av}dy?BPE-tJ`o*>rxOM8G=AT;%O|mb@=Vr%d$Adc-unk>^2T z{6p1L#lJd)KH`Q z;s%eitay7RK!_LlG~t=7?A75SO?yxmfy7*(kKHf?Juo)b1FFk$APXEPkvIJVVMgKo zrc)3dPT13bQv6CsM~87Xiz_(`OgWMoE<1V}xmneF69bVZok8)s1X4kSa7OYBU>^f> z*9CP&FGR+6f_hONBrcNYe>J_4Fa;<^5ND7i=>A>$2dAg|+1@yH+-o~lFRK+VRQT#_ zh;P?q!{r=%*t2qtPne75mKiY-6XSoaw9|!A*~vGE!gAlDgIVJpmfsew5^`$RKaRxO zUL~PjC7mfpvucM}CfKnv%8dN_NLD!Ca^x94Lg?YpB3svdwxG`bJvr|ze`#2~ZoZW{ zL+>10<9MqV49mjZDcQZHnVd#0;D-V$S@O{2<$6P+0Pm+u|X$g>Pgj&Qj%TIHlc4_5Q#{LDP=Mt}Cw+ORamU*^o_ zC~L3|{7z;Q&!K*CFxD`r+uCw3adTBun^*LQ@@bOv&mU!HP1&U*>=rF~V_Go;7BA^; zII)lztFc(@T)#0uYG8g~Zp4dM!F2B?%gZB_jR768?sny__wQxZr6Z3#)5R93P!LTJ z;zvVNF`#L?BQAt3@E7O*YXS5Qclmqt9+mjY)}ouJk5h{rOU1`byUbgs-@Z*MQZ?W` zV*C`IKhmFgD6yCB9IK4>`w#~Mh~@daMHA)A87ywTPBimAd5|K1_e+r|>Y4bJW7<|@ zzV1a@N6N;8Lx^*ug1MN81Q6aSVz`1S=ebKB3gs)@teAG*2XC2{OvLDOe8ha6|GuH+ z#`$9?{n(rDw7LAjMGYn8w+0$z9*3lF()=llAFb}rYvszwQjstSdrB0WcN_D0(x{E( z_JAE1rBX1@Ioiry!X}}fwb<$7{v2eefI>z>om)Lc<{K4vYI5MZtvzP=MA9dEKynoU z{6_`%jZf6D3X=c!tCQx*6|=E$Sh=pg?`JS@Ws-QpL@5*;Nl_oQtU;^&C>irzoRaFu zM}w^e8qqhQ`MQz0M+#h$r_}?)rS5)T2Nw2@S_iJ*2uL*2Kd`NpLz~@^bdbs)Dlr~? zaFcOvg2Hb2`t1^{v{S9638w5D+Qyam|GjJ?v5t;^Tv2}K9hU366K;qcKw+SIJZI&q zxVkW_6a@b=5u2Sz3ZtS{#mxkbD2`srL~2pKV*DkvHG7*rq+<7G+Rrn^WtvwViZ&Vb z&7R87dFV{TXEH9QPE)SSm0BDnXT6sXDOC`=_Kl*Y#GFRI)8E`8VAqjkXPZKVYc^&+ zzuKR!W0Uh-`l`AgR zhxbZOpXYslM`|l#k=M#;+fEv8X&QuBr=hcPGL#?D8YwC+;u^ooZa)m6mN|UqvlEw= zs&v9tO6^R*=FpkO#rlV8#4_b5=X=2bmmVh9K8g7y*udiaJ2RD#Fqv%zd!d|v0);JuMzQ0~F8kKoZ@X|*jhppux4)!T zl~F8{4bD1_3b<+&%Qz@Z6?Wt9Z=Z#4KkpZe)u31X?Fjohe~4c&HRk(>Qgj@C#@{ER zR9n_UZPC?TM0GyTxOwZj;y&N(d~C<;^xzMv7oUXa zPyxm7>qM3$gvgY!n6&Iev<*4li$-(}yuZ*24}5S(l1ZJY6hDrISvBuz0l_1^P!=^Y z(yHvJNSUiC<5r(WU*{^Hd!pMPVYQI!4ex!S-}Fh74#}OpVXNgyrOY7`-Tt3Xq}q$X znFqY#gj@#=t(szB{?dLCS|e;!$m)W6}IbRvYgQJo|dYmDkw zLdPy;wwP`;qQ|)kN#<8|w^sd?_N4Ju?U*C$IU?<@u_;099SwUkzRKo{^cNY; z?s}hWl*yU61<7f(7x`C}To|eMSmh=Aa3uWT9lh3plDxaU-F`K}5>z4jX5FNU4Y9xAP8(WR0r~ihOT2YS# zubZK4x@e@$`>U&dK9QKkSD72+s(MtV$ihxOYuU#J-oLZm1Gy)T z&?Y12vrf5Sy7gl83#K(TGqPG-=8u{8PrYQcB#s`7J4gv0lu+1`@H%=Cz?Ql}y(s+n zVrEt;gMn>dg|Lo?qo36X7hCdWzvdn)q_nLXr4wn+u$Ph@Cipi-PhXCFl&UO^--pI} zGxMKW48>)pOT`D~##>!XMnaDgt-8EUvbn9AIfFz47i-@6aI*g+DBD>y|DKjV_EY~( z^NFB`P86m!gS6I?%?PS!_{kQjkWkOH^Wa{uQg@}1|1N|yT)9|!{=@MSW}yoR3ViWn zFq|mxYb1@RIg!{#0ikr+aEW8Dw{Zr3MWl^PPJcM%Wqhj3+;CD?-_F+YQh(kr{jfRt za4w5?baQXs+C;uLF>P%a>QxHDbIqi$BFmDH<%wb(3k^mo@z9wtMbc@Aiib5`;EV>d z5*X4&xd!|zm6W5@zsg3N6?K@J$8F%f6VAtU8PJHETA2qZ{I{wxavjZRbCSujForEs zev}5OXf9JO@%y=BzLf@Mm9gORYwnDh^dgi>ti4fTXsgV_+r2C-)YkY0A2I)6cc~nm zg3ORQ2TvpMJ&ziGF5TWj$-S6#l7y?dcTg+SJ7DDyO5#&a!*<)cxeuIW2Kr zno_+%Y{I6!=ah}v)D=a4Qh^tZqO#|uZg$@sNmXEetyjQ9Z2DVas81?AoU**}C zkuI7?E9+an-2WpsO*UrcUh2_ha3mcn67zl6{?5U^A}2+Zuq}r$T00aowC{;xq|Bhy z2nI(fD;_F!ahjT&++Zl!K8AghpHqpc>}Z&W(H}cg!i+TCI0|pW0nL#^{y`)c-DxxU zlGE@DK}mYu)|-KplCFxY>OK~2Xc9-Yb<#@5qT%O#iAm1_w>KUl>&{yiHtzc&>AXg|Pvy~cQU|HX49*ksGJ<|#@Wh$S z;N23?PT4Vvqrhd3(j_ayBdn66O3tnOQ0IG{KPL`F{qXiXqrkutL7SS;By@7aFy3@p zh2%xiijw~mQU86RBw>TG;zpT~XXz@B4pbtPlg8dk98CNBY(>6Z`pW)$Pe{6movGr0 zmGjn9ugt>X;S*nzIyPxJbPd{bmLrEPhVn!9n4*ll5CeYy5gUOkY3`2D>3Ern);7Bl z&16ptdsO>lIi}RaDi(4M)%C?D<))0p{_`!OZdm+pXuP=_mJ94HsCLvJ3?q8OD)5oj1p7cdMkzoLTA z_YB&YWiZR;16)yFH(5Ggi==2h2i$YqCYqN@z(%gFRXENYQ23RVm04BG6Vh%Q*N^|b zK3YVXr1Vh;_FKl4aCw|`d zGX>|`-cS>MblbbmeV!%^4k5(si;Sj10x7)CE;lm0{q*GPdfP@-MY$(}q%acjLjA8x zlgP_s83!{&ed<9mys7H01Rfl67=nx8ZufX?xr}*+9E~7 zw-XZ+`Z_JoPu4jzn_4dla~`xvpz@moMp!Zfb@$_Rh;R$J-4DzD39mmW(J#mk*1#iQ zrty1FA$}#a;@eB?{y{SjJSPsJ`zl1UMxmkeJIg!YdP$;uX6Yg7 z6Rre%U||t>ytqOYe030;nja^fKfBEWw`7>Ao|=**qc^8QtwaBPin)uS;6vZtK;kcx z75;piy1&F|4OjQaH@uQ{BFjg@)vIjkC#$9?*1g2;z9D+7pIt-VV#A@kTm1Bj-%Mct z*1pF1y|iZT=o`lXhuhwMjf}09!9#-AKu~=Q(GccCKVYcG)|LzO%2-k`B@e-BUIXC` z&;<^?{~ij4vLO$v1~LW#In&b7>45aIiKb__d|KW@MkUf>L&6d=@)M@BE<-@10-p3c zgK{La18^haS@SjveH!;?;xmNR*6{P0%NC)SR?14>2iW*cM5ks5~AK!G}kJ$M5%Ugi6Hmc$3#x#T|i zH=V*Lzs=A~tw>jii99BKFzGZpp^J|55jS-ZFlUX5CSTu@;NWNIGou|(5RFj38+E10 z<;ZCJ_~aegNOZZPTxh|%iMU3+2c4`wL8Nq%BE8X4uPNP$&hm7vM0YurW`A>G^-iGs zk~xa3W@-bX0C&L^|M)K?+H{rbzcbt6h?0S^kch*o#sdacgG56dte>Gs_N3<;jugz; zgV^LhAdh8&bqE38*+0cJKEQ;8A27dK_}M>NNS#IkG5G#Gf$2ODwDvPhWBEsaO;eF; zL^5gKptZtva`HuRuRg02WC&$G`d1kEr}|JzqDN}KFrSzcTSmz#*%-aTV&6Z3M-VN= zUY6cfb~5gA)b=w6kNHZ6Qa0Zw*LUi{(klr*bTZoUGQ#<8@{C%y@J4RDSH{kHMw>W6 z_iy{10WMUTw^L8uEoD(rwqy)gGDWgqZq2#vFn+2U;2Ik2U5!;~8pyd-OE$t0Gi$LS zfVCPQ<@@_T9mi{jXkZ)d^3?`<#3&Vck&+ZZX~8xH)GQp z_IScd{--#ZY!WQRVCLJQCc+~%G-++M@)>t31*ixqQvD$gc=-nm+<{DAU6?$&VGM!` zm@Mu3B`qyYQ9%KoxZuFs8KCWud8&{lf})~Y>0khC_7h}7(8FjXbaeFaGz}S9$QQeL zupOCwbdw(ryo$_whK1>fTK)Z4LtXs=a82VI2X1X8-oNsG(M$MI1tF$z_^=vlROQ*X z_%_(dLj{9sf-QQh77Y2{s0vWg)eo;f;2H7mSS1(30T;1Fvbx5yrMB#|UN@NYWtvb^C2`(eNyqbs`Fv|L9eUn4gy zdu0z#-bT@e1ZRB+b@9z)EKPcXtwYi{+Remx(*mEj=HtgD2c~dohOLCCzCsy|{pNdk`z*tZ zP1Qhyt1ToA!{~~EuiC>1r3nW$%yp~XDKp?kp-?OP8rrrQkDPxu3fcE(3TTo{PC5mI`h(J%=8H%!Zq@KL?8%Z~NE8Kd7+vBz}hNLzi#fs03-6e|=}p{SLa9 zDSS4Q9^j_nbh~P;#qUsr5_0Hv9~ro9%(T&e{CK=RSsVQ+h@rRKd~R!D<9#B?YXX$C zy{2tsS3Ki7cdA`iRg2X;A>I*rTV>`=sc-^fj0Of7^Cs4x@Z7(jA?mqmb@z=>VQNBx zG`I`Rhd|o~{wF#eKfR(tjRA!M%Yc|hLZQi>P6Y-X16v-@+>&HGGDn4GK--8npR?;asO z9u0vHZN(|w4_ZnxpYJb{7q<7KiDc%rZm))Vp%zYF6eUr2Cr#^GTwh@!AtSpgGtjV2 z_iJLWzxuts(88xph4{2ITp5xZHRJ#_j4U{w z)NxN}N-X@?)LRIS(w~Z&F57dl>rDU^C1q>TjjM3IdZy(76;XKKQnFD(V)_cK>z*oc z9e4Qhw)P=YUI`J)!@A2JsBl+X=GiaL86}1i7h6qD8OR?NLhK1X^#%FsvCkX=$+sO$ zOp0(*0a@#>JvdnEj(w`sbH`&ibNzO6Oth#gD6`uSUMLLL;n94cWZG}q;(fkc;(gV= zkZh&EgtHfTG-zp}RVKS6O^0Tr9VaZi6V2lzTEhl7=sdGIx#wqsmCK~vvv>EZ` zy$NWUmi9XnHhSJOtfrmcpHnmBa)E@?bM5pq4@dm7wPwN{&+r5 zTqtg@%unoOVQD5UDJ)hI(h}yIw7dO%>gwc(u#=aU7qNpPTIgu&UWT$D8O4{8k%3nf zagZi^QZhL51R)Xeq`V~v_{S%wGHT)zL zKh6pY{58;`e{uR=9#zGQ{^{=(jk^Q%X6f>iQx$~8CI_-S+lx-{Ph0vtw3MMbm$XZ9Zv z7r!9!9x8}JFgogPFGa^N@{v0(6t%U$Fm_L}5XZveU+vaUv%VEs(%;ET zZAy!Ab?-3M!GCVI%lUTQP?vxACcA;#<@wU3E|+t9^kch|V;)Qe&}A+uT6?))?3-~y z(z_|sWa|r8c?fIYRIbgc&9i#6=^8^3Z(-M*>*mJZ&Cd0Z2A_q*uLcaE!yWRJ(`dbrJ2=j^!Ul^~V`=&bg!p0mJ3Jhg zBa#46xQl_oqI>if1)EifnAQe0294@o7ur-_3T{g#%PXv%TyCCQEENr`=wm#_Z-u1F z$9#D=b9bn(Fl}aGfphg(Wp84B#cEL3#l5O=RvRSSE=5hBkNVUUnwS3}<*4TFlKAAR z@J)enu0xb*3Dhr|+Px1Juk>oh^`6ZVl3cl+<@X9dt^Df6mRs6w33>yoBRWxk=rogC z7RQ(Rww+A#>T`YHuX3YG=jZQ;0#=5EeASFTinSASRXhJAdx<`PO{B3%lto(t6$S{N zbvpo};oNPL#HzcukB=6Ik5&~-K#(2Ta3OKxTupWnM=VZGH9b8A#Biy|jFrfWeG*Oq zkOJQEysZwM45`{Rja@$QM20>0bZ6?|N80Aq z1a3O`svbc57m#YD+%v+8)Mk{o+c@*Q4$>=!olc6U_`Hi*XDz+Cxm=f;Hm;^k9 zz`cTTR{eI(gCmO$8$@BI4tyW}@uRK>6sD;+@A-uByXEgA(2v}Ti=BcRi)hkyN#{uP z|5KnBjvG0To2>?82so0HU|!_%J>ea64-{0Q#ZDu@>%v|kuREUI#LSG0{A}3D`kVBR zAt}utK8v?X#EZ$0IN3ILogdGJTyZD(LhD~Kk;WreYR!M1 zfQI*LeA#2|Xf!+jKoD_o9q(r;^G@Klha&uI`0@4T_-tZqK(v#`8*67&S159t(d&sn zv(a_9`hn&1HbK{+{bhgG+#^M8Gv!q=53~)Q83MuM6+4~^v$37l2-usZKE}ke)p5f14!voYWyBUGW6ZWKPlcR0<))$>- zddKYBxN)seU|X$zfY|#R=WnxWrRoMjD(aVDNr@SqL`0X~Z+OIlKm*Sg2YO7avB)Zq z1Y&`2a`%I@fZi)iMyuIow(z;MsHrgIJeW&C*0b$dGi5&{s2@Oj5)c>%PtQqHV}^p^qi}GjF!`Eq-g{U z-gv8|aEOWNMlGXE1Cs0W+0pqz!}saaJ*eq)i|jdJ9CH;*mR}a!;<_qacQn-1JnSzp@c?%yYss(qFn2|sDB*iK-Ut}MaI<2;g(U9R76--rnukvsnk9pMEdOlokde*VG z)LrzTO7vx*FoBj^7erM96jM4Zeikdj@56copB0(@JK0OdK*aCSS6vYzY?OEv{bHvk zQ+}$A8?BBQ#QYwCWaDt4=N}I0yA&}h=PqBqoY#J7$2F0Q;_t2v?hYrJ8JZb6uJdNF zBT&9{i6w+f**-$B?6obUwD}2LO+8cqZlz`Vx7IKoNpQ52lLgnz(bS7>NxfBW^Z z0+FKA2Bc2j&pi*%{&XuHk5FgW zyne$5S~|M+c3L*Xxs6h8KTj#h<8%_`bP!KQMJvPpJ`DRMCHqY?k@2OarBgLh9x1xz z^e6%7<@uywij=q&-~hA(X}^3w^oGTw{x@}7u7w6f8B%YxG&R+rNOt*jqkw&G!Q0+< zyBCfxEU^$f<#@7n`B4nR0HLuhDk`cGD!?^h(-XbgCj}*2Tetd$R-3x#D;VcZPEN`tZ zXx`h{QewLuqM_r2uHxg{WcPIA!c><(=z$MehAnXMIb&qRW_PP`VtT^~)h`_>+jKJZ zjgaAK*gh8BIEWv8jzP=l6SpEs5}EB6N8w8(8$N&jj0{CIPKeZheaOMaut|T}G}zn8 z$jWA0cJaQ2l{(BGG0#be2O`Y{JA^qXkL5s}=Fkv?)9*ieWQKct%=r42ZQD}L1+u`8 zY@gj%7NTE|Td%gu2Qe=K5vc=qB*{tI%@w}8MnlmXi`x5Lr!&$QuRV@wh0oiI#if~X6&?j&+oN+2T70&^v= zyLT@jEMcU1ABqfo*4J?Ru`bBA=-|>Tuu;BwvqU@aB1L)cD0LeQmEki~hb&)&aU5~S zsa-i&Th7u{;GOoAdzL4AEJGZ=C)Z-nxo?w>AQ)vjLS%Xnv07V%n4rZ*wjYWNAP7lg)AxVys%CGQ=DRo zIhgSfhoHOETUw_V$dnEqr*Te?3+Cuuq11XysdNK8`L;4QAoJ~kK_6<_1ILbq0k2RH zFIX7i5V!&*0pLF&Nc9qzU>GZ1zcSC;2(H2MYnY6Y)w zIuVrdw0L-VD@~4eJ-zs3>X`zJ!mf#Qq-pUnmX>x|;*#4hOlhrCPV1O-;P&%sGj|Yw z|NR(l5P4Y2k-}^uBHHP312*0HS#S;_ha2|?1o*_`^5ZR>C8dfA54tSJAPS3mvK6ig zGorF;`Uj>#4*{prY}@t@bvX`$M%Nip$;S*aHkpkeet|=V z)onwB^@q@$1OOc$R3>nM3#hO&7<6R0sIK6#GRByBsZ@ddo~BFzoyYhp-zUjJ%Z5|w zbp1&uZQC!)Dkbwy+HiVU_x(;Qmu$NwG&9L#tITw_+;KH+gJ^FoLsC&!?(c}DEyHI2 z=qN|&hQ*7tPT$;rlVf$*s@D0re?q~;Y`cr~+lPn8BDoH@{BF{5_!Y6F{4Al~a3{6C zmRg8KI*nGC6YrFNq~_9)*ON$HR@<3d&vSpd1*bJcXYxNB(tLIA)C{F+erod@sU_iK zw9Pxec0bV=uoJCuu(eecUg>ozWYA-Y+IeNt|rxw*n1;DGFsptw)` zOjtS;CJRpnhlYAz%IdaBMb3=Db<%O$<0sUdNIaYy0~;Y! z$6)|+WI+TMh8T(w<0?cTGL(n;-|2gMp1SW{Y6(kPSbXHC$u-#EJjm<*dq86i6+;kQ z=;}=>&4rN_;eAC)!UA2N?f3*iudlaff1nwq=*E*So;0TtkiL3a`$5fs}GL?hvXbbE{Fs{r>Lj5ix229N;l z`PU=h>~o!DowjP74lz%stHSCE%`p<^K&$DD>xVs1QD`ucNRxZ0vZ{&<4^!0D)m;WD*7@>c_lU&m;{w+5 zO*Eh91_qLqnUGz{jHd;PcHmt>QCige_x=2&73A^R;JKHkndVfmFE!2NP||@zhaN&H z<{rs%85c&>cr_X`u2pI~O5EbudGhe#AYxc#-nt*0lMH;t5upMU*AV7@fLAd(8d0LB z4o#HjuO-oDgp~drxW~V?9XgSakdW&jSo)Lx?DJqq#_7maHSyaqw2l7JFa4O{rk!EF zG`gnmm^AGnUK=&)fg@)iHY z3=LS!YuoZLujN36)I@hAK61(+FI5;>M9?ebZ!cPE&pmq-w z0uNP{8wbD(h3N^+)~(}k5(5p|j@H7V7|>L(6(MC$VWHR|b-&-* ze=%NN}GQvH@uMen0E8#PGiZm!%x@=_^rPqIm=- z4F&ge*RN+gDHnq%My7k>p7N_aF;T`}|M~G*`pVBvA^=r*!i;f0c{|tbj@?b%C~r)4 z^5DzBacE!fAQs4&-sRVGZ5*_880M?!zOU?@xPsI~jXgwsnUX6*}adbggO zU##Mb-dI}9x|4_V@BGQXRrnZ_1H}dJN~Wp-M&Lb3ykl|TA3S(qgz}Y=l9Je}qV1Y6 zCKS~~igM!kgnSGMVf3x8fRzS(K0nV=2rSVgVE}*ii}}}#;9a-x#LcIpqb6(U|69ga z!qPHOC1u3c^EVTL(a0=SW;xkF{lpOp)iLRQ7ut`}L6q_ekQwnOg*W(X(1Is^e-#4V zg`!g_S}Yjp)ne^hbYE2r4@IQl=Br#kU;z<Qo5&)!`?y`(|z;r-S=}E3k^CB`*#d@s=g4CSB<| z8v6PfRPUN$B?wGb0ahhC8ez=+DQ-5_!@ac4;1L8THZIUdq-zIoZvXY;5#TW5FIZ7g zaSbg}e5X0MU~rVL7_AQ>V)01*wPraxSXo3-kKU!_~CphEs3dSBGd!XfPCn#54#LmJmcZ$<#00ph%Y(#hfv= zQAo)G84Onw_9;5j!P+S;-y9l;S1lfy1Nf`Wknmh?qR(5ISD_-x`(y0yNdpu}+u6E1qpH+c$L60A^ zD|3Zx-m)bO9rkc+{HN|aeEBv}6s#7-Tc{huCZ-y*k906t!U}nkRSz=*mLbCamF|(R&WI_DDaq~O{!n+ zQ7RnO)NXRLJ|)&IO+Oa;lnU!#Bldz@QshaB7O{?`9ECn}o&%UH+%SbxWzfiC>1!xr z)_$qSNpS(Z@yh|0TR+u$vag)`hlz=A{>P8ppHsidUrGvXx6A`t)luL{lC=ADTS-O5 ziRU*8-;mpXP!Hr?ZIPT02)KVgXt#fjPcp1A>anA$jUOJc2}hL`0-HddbH6dvg7>^s z{@1`=1S_gM-a&weyy0bdlk}|(0=&GBjf{+51MeI?d2)!;8e#sxeqzQMIFpR^cdc?$^-Juobhl_A&giSVh%-SJ)z37hNg_CXs7)u4{PyA+U)Fr^G)EgIK`iFY>NyD8M)5%O}?@i&sknU z!QNE_Y$2>IAKbrxO5sJRYS-QA1vjj`JPLgb5@~n}T`p3Z&Zz8DN2Mk)6^6O})C>!UBJ?cVP3u zJan=j@$vO7&Q--HT!EQgeN)q+rNM^YZTmc$TjEY;)!d~r@6KdK_>wN#&^Wa7&-=b@ zjG7KK2L|%fK;U*8^S`;MoH`W-1`IU?ddq|^24S|}yS0;8h(`z;?@kgT{8h`xnF(tX!SUbFkcQs`_E?>C>n6++A|&<~lv# zvxtFBmM9O(~I*>OC`+gZ_*{B`$T|}z%2)X3=E$Dmd(&K z^eC=sDOsJnpB6ouE>eM*jl6B~OMM|o{g@AmDKydNv6Eu4xnY$Vr$CMep(Lz*U7?&& z*y$v`Lx*1KOC^Sa`9dcJ{-eFNv)`7g=Xl*bWqob2zap^IuMx9L)(BFP7?imOA5klv znu2RR>azwwpV8tY*H6zUiY}hbiZpzvWO)AkDJLf<@o5&vgNb^tU8c58h-7A1M|<87 zNC!cTd7Ns~;Q9RXtAn2kNamK5mz&-E5~LH^1f1zT7G9EefQ6Y^8Q2hC2O4c1N&6<` z2XJxMmHnQTP;SV&Z**L6{jA-PwDXr%ird7|acoZx_%Y1%;H=RzGEPVr3q53zUuK9; zYmZOyVMhgvgx7)k|Kat!XRJq}Q;ZndXn|zg+l#oOAO;hyTfCkmH<#c zmV0^@{pcKc|Mt!XgNE;&X0g5XeL~KTp{V-7)qse@xEpbd7S0P0rCnQE|1S*btj|z5 z6YDWlV9N>&+PkIT?b}m~R-4dgf%EYeF=M9VisjNYq*QlvJC6970vA34ZV07Ze)r76 zd42uO`f)!AJ%!kfhP;A?Z7O}fn~$Ng1egaBx4}N>7TClnGUHyHGc!xx?Ypb>Y1BM} z=T);2o1qz6%YUZExR#FgH-@M`(GW}}pwKMmA6Ydy1%+W!TqItx)30Ymp8+g6Tyri- zD4ti-)%{YbDI_EWzvRzI0g)_vGon{(C9uf54vfr2mDPDzO}3+(hKhh%?CUoM*|rRK z${&_)i;09dHmV>9fN0=#C@BR{nrLcGPvT;1fD}G}i%@Wam^b8dD%$l~&+rQcfR4`fb2KdH6a?N+eexplY#OU?-PS5VCdh z*RJ&NKhTFx4BcuK0v{<&IyMt?^VF}MFG3_e{uKU!*!d{7tM>bBe+?$?2Yg@B`2_@? zh(sx?L<-B(vHkl$MXh@ez$$>}9rB&~!W5^V!J7j~4!V1=edqc3b_Up3(gp%8+@ zz?UU=3MRrKKd=~@(zs{dMD~Wz2QD-KR|YObEgMw+`pkg-BC2qt9bn8|yLQ=uq5kw{ zlnjai3ZBN89iL9-0q~qjNn*U2x26p7LHTIbJpOR|0X&* zUGV#)_x9>l5YqM$B$!`|GwDgB1&9+-V^bi64!rkvGPwfcb^ZGFbuBFu$lQ`^%Oe7B z)N*>><1vw0NHY3}tuuCXe{$IC-@iA(AMMOCj03?69_Ury43?+X+kKS!hJ8Scq#%-$ ztB4k|fHg*6DnP-aJZ0zRrjb>vtFH&?^n^$ab6O)BF<5J0dSt=o|px@R*rV^}1}BSoyqY%lWh zRfw|RUbF0trkaP zpK;US8K4K%MZ!zNi%90?Q)N0HS4)*nSp;YZtS&HD&V!?b(@%3MkOh* zN_@t@w0M@BOg?$U{O_mjo@Ay+vhbF5jzrf=#ku$P(f-}W`f2H~BUOUuI{YIqt0I#r Iec9{(0139*i~s-t literal 0 HcmV?d00001 diff --git a/doc/sphinx/src/TaskDiagram.png b/doc/sphinx/src/figs/TaskDiagram.png similarity index 100% rename from doc/sphinx/src/TaskDiagram.png rename to doc/sphinx/src/figs/TaskDiagram.png diff --git a/doc/sphinx/src/mesh/mesh.rst b/doc/sphinx/src/mesh/mesh.rst index 1c046bc14c87..aa891633cdde 100644 --- a/doc/sphinx/src/mesh/mesh.rst +++ b/doc/sphinx/src/mesh/mesh.rst @@ -85,6 +85,7 @@ To work with these GMG levels, ``MeshData`` objects containing these blocks can be recovered from a ``Mesh`` pointer using .. code:: c++ + auto &md = pmesh->gmg_mesh_data[level].GetOrAdd(level, "base", partition_idx); This ``MeshData`` will include blocks at the current level and possibly some @@ -94,6 +95,7 @@ communication). To make packs containing only a subset of blocks from a GMG ``MeshData`` pointer ``md``, one would use .. code:: c++ + int nblocks = md->NumBlocks(); std::vector include_block(nblocks, true); for (int b = 0; b < nblocks; ++b) @@ -104,10 +106,10 @@ GMG ``MeshData`` pointer ``md``, one would use auto pack = desc.GetPack(md.get(), include_block); In addition to creating the ``LogicalLocation`` and block lists for the GMG levels, -``Mesh`` fills neigbor arrays in ``MeshBlock`` for intra- and inter-GMG block list +``Mesh`` fills neighbor arrays in ``MeshBlock`` for intra- and inter-GMG block list communication (i.e. boundary communication and internal prolongation/restriction, respectively). Communication within and between GMG levels can be done by calling boundary communication routines with the boundary tags ``gmg_same``, ``gmg_restrict_send``, ``gmg_restrict_recv``, ``gmg_prolongate_send``, -``gmg_prolongate_recv`` (see :boundary_communication:`boundary_communication`). +``gmg_prolongate_recv`` (see :ref:`boundary_comm_tasks`). diff --git a/doc/sphinx/src/outputs.rst b/doc/sphinx/src/outputs.rst index 7659bf19bdde..c01c957fdcf5 100644 --- a/doc/sphinx/src/outputs.rst +++ b/doc/sphinx/src/outputs.rst @@ -168,7 +168,7 @@ Parthenon supports calculating flexible 1D and 2D histograms in-situ that are written to disk in HDF5 format. Currently supported are -- 1D and 2D histograms +- 1D and 2D histograms (see examples below) - binning by variable or coordinate (x1, x2, x3 and radial distance) - counting samples and or summing a variable - weighting by volume and/or variable @@ -184,114 +184,142 @@ A ```` block containing one simple and one complex example might look like:: - file_type = histogram # required, sets the output type - dt = 1.0 # required, sets the output interval - num_histograms = 2 # required, specifies how many histograms are defined in this block + file_type = histogram # required, sets the output type + dt = 1.0 # required, sets the output interval + hist_names = myname, other_name # required, specifies the names of the histograms + # in this block (used a prefix below and in the output) # 1D histogram ("standard", i.e., counting occurance in bin) - hist0_ndim = 1 - hist0_x_variable = advected - hist0_x_variable_component = 0 - hist0_x_edges_type = log - hist0_x_edges_num_bins = 10 - hist0_x_edges_min = 1e-9 - hist0_x_edges_max = 1e0 - hist0_binned_variable = HIST_ONES + myname_ndim = 1 + myname_x_variable = advected + myname_x_variable_component = 0 + myname_x_edges_type = log + myname_x_edges_num_bins = 10 + myname_x_edges_min = 1e-9 + myname_x_edges_max = 1e0 + myname_binned_variable = HIST_ONES # 2D histogram of volume weighted variable according to two coordinates - hist1_ndim = 2 - hist1_x_variable = HIST_COORD_X1 - hist1_x_edges_type = list - hist1_x_edges_list = -0.5, -0.25, 0.0, 0.25, 0.5 - hist1_y_variable = HIST_COORD_X2 - hist1_y_edges_type = list - hist1_y_edges_list = -0.5, -0.1, 0.0, 0.1, 0.5 - hist1_binned_variable = advected - hist1_binned_variable_component = 0 - hist1_weight_by_volume = true - hist1_weight_variable = one_minus_advected_sq - hist1_weight_variable_component = 0 + other_name_ndim = 2 + other_name_x_variable = HIST_COORD_X1 + other_name_x_edges_type = list + other_name_x_edges_list = -0.5, -0.25, 0.0, 0.25, 0.5 + other_name_y_variable = HIST_COORD_X2 + other_name_y_edges_type = list + other_name_y_edges_list = -0.5, -0.1, 0.0, 0.1, 0.5 + other_name_binned_variable = advected + other_name_binned_variable_component = 0 + other_name_weight_by_volume = true + other_name_weight_variable = one_minus_advected_sq + other_name_weight_variable_component = 0 with the following parameters -- ``num_histograms=INT`` - The number of histograms defined in this block. - All histogram definitions need to be prefix with ``hist#_`` where ``#`` is the - histogram number starting to count from ``0``. +- ``hist_names=STRING, STRING, STRING, ...`` (comma separated names) + The names of the histograms in this block. + Will be used as preifx in the block as well as in the output file. All histograms will be written to the same output file with the "group" in the - output corresponding to the histogram number. -- ``hist#_ndim=INT`` (either ``1`` or ``2``) + output corresponding to the histogram name. +- ``NAME_ndim=INT`` (either ``1`` or ``2``) Dimensionality of the histogram. -- ``hist#_x_variable=STRING`` (variable name or special coordinate string ``HIST_COORD_X1``, ``HIST_COORD_X2``, ``HIST_COORD_X3`` or ``HIST_COORD_R``) +- ``NAME_x_variable=STRING`` (variable name or special coordinate string ``HIST_COORD_X1``, ``HIST_COORD_X2``, ``HIST_COORD_X3`` or ``HIST_COORD_R``) Variable to be used as bin. If a variable name is given a component has to be specified, too, see next parameter. For a scalar variable, the component needs to be specified as ``0`` anyway. If binning should be done by coordinate the special strings allow to bin by either one of the three dimensions or by radial distance from the origin. -- ``hist#_x_variable_component=INT`` +- ``NAME_x_variable_component=INT`` Component index of the binning variable. Used/required only if a non-coordinate variable is used for binning. -- ``hist#_x_edges_type=STRING`` (``lin``, ``log``, or ``list``) +- ``NAME_x_edges_type=STRING`` (``lin``, ``log``, or ``list``) How the bin edges are defined in the first dimension. For ``lin`` and ``log`` direct indexing is used to determine the bin, which is significantly faster than specifying the edges via a ``list`` as the latter requires a binary search. -- ``hist#_x_edges_min=FLOAT`` +- ``NAME_x_edges_min=FLOAT`` Minimum value (inclusive) of the bins in the first dim. Used/required only for ``lin`` and ``log`` edge type. -- ``hist#_x_edges_max=FLOAT`` +- ``NAME_x_edges_max=FLOAT`` Maximum value (inclusive) of the bins in the first dim. Used/required only for ``lin`` and ``log`` edge type. -- ``hist#_x_edges_num_bins=INT`` (must be ``>=1``) +- ``NAME_x_edges_num_bins=INT`` (must be ``>=1``) Number of equally spaced bins between min and max value in the first dim. Used/required only for ``lin`` and ``log`` edge type. -- ``hist#_x_edges_list=FLOAT,FLOAT,FLOAT,...`` (comma separated list of increasing values) +- ``NAME_x_edges_list=FLOAT,FLOAT,FLOAT,...`` (comma separated list of increasing values) Arbitrary definition of edge values with inclusive innermost and outermost edges. Used/required only for ``list`` edge type. -- ``hist#_y_edges...`` - Same as the ``hist#_x_edges...`` parameters except for being used in the second +- ``NAME_y_edges...`` + Same as the ``NAME_x_edges...`` parameters except for being used in the second dimension for ``ndim=2`` histograms. -- ``hist#_accumulate=BOOL`` (``true`` or ``false`` default) +- ``NAME_accumulate=BOOL`` (``true`` or ``false`` default) Accumulate data that is outside the binning range in the outermost bins. -- ``hist#_binned_variable=STRING`` (variable name or ``HIST_ONES``) +- ``NAME_binned_variable=STRING`` (variable name or ``HIST_ONES``) Variable to be binned. If a variable name is given a component has to be specified, too, see next parameter. For a scalar variable, the component needs to be specified as ``0`` anyway. If sampling (i.e., counting the number of value inside a bin) is to be used the special string ``HIST_ONES`` can be set. -- ``hist#_binned_variable_component=INT`` +- ``NAME_binned_variable_component=INT`` Component index of the variable to be binned. Used/required only if a variable is binned and not ``HIST_ONES``. -- ``hist#_weight_by_volume=BOOL`` (``true`` or ``false``) +- ``NAME_weight_by_volume=BOOL`` (``true`` or ``false``) Apply volume weighting to the binned variable. Can be used simultaneously with binning by a different variable. Note that this does *not* include any normalization (e.g., by total volume or the sum of the weight variable in question) and is left to the user during post processing. -- ``hist#_weight_variable=STRING`` +- ``NAME_weight_variable=STRING`` Variable to be used as weight. Can be used together with volume weighting. For a scalar variable, the component needs to be specified as ``0`` anyway. -- ``hist#_weight_variable_component=INT`` +- ``NAME_weight_variable_component=INT`` Component index of the variable to be used as weight. Note, weighting by volume and variable simultaneously might seem counterintuitive, but easily allows for, e.g., mass-weighted profiles, by enabling weighting by volume and using a mass density field as additional weight variable. +In practice, a 1D histogram in the astrophysical context may look like (top panel from +Fig 4 in `Curtis et al 2023 ApJL 945 L13 `_): + +.. figure:: figs/Curtis_et_al-ApJL-2023-1dhist.png + :alt: 1D histogram example from Fig 2 in Curtis et al 2023 ApJL 945 L13 + +Translating this to the notation used for Parthenon histogram outputs means specifying +for each histogram + +- the field containing the Electron fraction as ``x_variable``\ , +- the field containing the traced mass density as ``binned_variable``\ , and +- enable ``weight_by_volume`` (to get the total traced mass). + +Similarly, a 2D histogram (also referred to as phase plot) example may look like +(from the `yt Project documentation `_): + +.. figure:: figs/yt_doc-2dhist.png + :alt: 2D histogram example from the yt documentation + +Translating this to the notation used for Parthenon histogram outputs means using + +- the field containing the density as ``x_variable``\ , +- the field containing the temperature as ``y_variable``\ , +- the field containing the mass density as ``binned_variable``\ , and +- enable ``weight_by_volume`` (to get the total mass). + + + The following is a minimal example to plot a 1D and 2D histogram from the output file: .. code:: python with h5py.File("parthenon.out8.histograms.00040.hdf", "r") as infile: # 1D histogram - x = infile["0/x_edges"][:] - y = infile["0/data"][:] + x = infile["myname/x_edges"][:] + y = infile["myname/data"][:] plt.plot(x, y) plt.show() # 2D histogram - x = infile["1/x_edges"][:] - y = infile["1/y_edges"][:] - z = infile["1/data"][:].T # note the transpose here (so that the data matches the axis for the pcolormesh) + x = infile["other_name/x_edges"][:] + y = infile["other_name/y_edges"][:] + z = infile["other_name/data"][:].T # note the transpose here (so that the data matches the axis for the pcolormesh) plt.pcolormesh(x,y,z,) plt.show() diff --git a/doc/sphinx/src/tasks.rst b/doc/sphinx/src/tasks.rst index 77aa79325a97..d4c0b361b7f9 100644 --- a/doc/sphinx/src/tasks.rst +++ b/doc/sphinx/src/tasks.rst @@ -117,7 +117,7 @@ finally another round of asynchronous work. A diagram illustrating the relationship between these different classes is shown below. -.. figure:: TaskDiagram.png +.. figure:: figs/TaskDiagram.png :alt: Task Diagram ``TaskCollection`` provides two member functions, ``AddRegion`` and diff --git a/src/outputs/histogram.cpp b/src/outputs/histogram.cpp index 9e4cf179e9b9..88c6ced2ecd3 100644 --- a/src/outputs/histogram.cpp +++ b/src/outputs/histogram.cpp @@ -163,7 +163,10 @@ auto GetEdges(ParameterInput *pin, const std::string &block_name, } Histogram::Histogram(ParameterInput *pin, const std::string &block_name, - const std::string &prefix) { + const std::string &name) { + name_ = name; + const auto prefix = name + "_"; + ndim_ = pin->GetInteger(block_name, prefix + "ndim"); PARTHENON_REQUIRE_THROWS(ndim_ == 1 || ndim_ == 2, "Histogram dim must be '1' or '2'"); @@ -420,11 +423,10 @@ void Histogram::CalcHist(Mesh *pm) { HistogramOutput::HistogramOutput(const OutputParameters &op, ParameterInput *pin) : OutputType(op) { - num_histograms_ = pin->GetOrAddInteger(op.block_name, "num_histograms", 0); + hist_names_ = pin->GetVector(op.block_name, "hist_names"); - for (int i = 0; i < num_histograms_; i++) { - const auto prefix = "hist" + std::to_string(i) + "_"; - histograms_.emplace_back(pin, op.block_name, prefix); + for (auto &hist_name : hist_names_) { + histograms_.emplace_back(pin, op.block_name, hist_name); } } @@ -498,11 +500,10 @@ void HistogramOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm HDF5WriteAttribute("Time", tm->time, info_group); HDF5WriteAttribute("dt", tm->dt, info_group); } - HDF5WriteAttribute("num_histograms", num_histograms_, info_group); + HDF5WriteAttribute("hist_names", hist_names_, info_group); - for (int h = 0; h < num_histograms_; h++) { - auto &hist = histograms_[h]; - const H5G hist_group = MakeGroup(file, "/" + std::to_string(h)); + for (auto &hist : histograms_) { + const H5G hist_group = MakeGroup(file, "/" + hist.name_); HDF5WriteAttribute("ndim", hist.ndim_, hist_group); HDF5WriteAttribute("x_var_name", hist.x_var_name_.c_str(), hist_group); HDF5WriteAttribute("x_var_component", hist.x_var_component_, hist_group); diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index e5260a300f60..4fd64236a19c 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -228,6 +228,7 @@ enum class VarType { X1, X2, X3, R, Var, Unused }; enum class EdgeType { Lin, Log, List, Undefined }; struct Histogram { + std::string name_; // name (id) of histogram int ndim_; // 1D or 2D histogram std::string x_var_name_, y_var_name_; // variable(s) for bins VarType x_var_type_, y_var_type_; // type, e.g., coord related or actual field @@ -252,8 +253,7 @@ struct Histogram { // temp view for histogram reduction for better performance (switches // between atomics and data duplication depending on the platform) Kokkos::Experimental::ScatterView scatter_result; - Histogram(ParameterInput *pin, const std::string &block_name, - const std::string &prefix); + Histogram(ParameterInput *pin, const std::string &block_name, const std::string &name); void CalcHist(Mesh *pm); }; @@ -268,7 +268,7 @@ class HistogramOutput : public OutputType { private: std::string GenerateFilename_(ParameterInput *pin, SimTime *tm, const SignalHandler::OutputSignal signal); - int num_histograms_; // number of different histograms to compute + std::vector hist_names_; // names (used as id) for different histograms std::vector histograms_; }; #endif // ifdef ENABLE_HDF5 diff --git a/src/utils/sort.hpp b/src/utils/sort.hpp index 9662cdbc5835..97e9c77a88e4 100644 --- a/src/utils/sort.hpp +++ b/src/utils/sort.hpp @@ -34,6 +34,9 @@ namespace parthenon { // Returns the upper bound (or the array size if value has not been found) // Could/Should be replaced with a Kokkos std version once available (currently schedule // for 4.2 release). +// Note, the API follows the std::upper_bound with the difference of taking an +// array/view as input rather than first and last Iterators, and returning an index +// rather than an Iterator. template KOKKOS_INLINE_FUNCTION int upper_bound(const T &arr, Real val) { int l = 0; diff --git a/tst/regression/test_suites/output_hdf5/output_hdf5.py b/tst/regression/test_suites/output_hdf5/output_hdf5.py index 1de94b678448..efe7eca5b6ba 100644 --- a/tst/regression/test_suites/output_hdf5/output_hdf5.py +++ b/tst/regression/test_suites/output_hdf5/output_hdf5.py @@ -179,7 +179,7 @@ def Analyse(self, parameters): with h5py.File( f"advection_{dim}d.out2.histograms.final.hdf", "r" ) as infile: - hist_parth = infile["0/data"][:] + hist_parth = infile["hist0/data"][:] all_close = np.allclose(hist_parth, hist_np1d[0]) if not all_close: print(f"1D variable-based hist for {dim}D setup don't match") @@ -197,7 +197,7 @@ def Analyse(self, parameters): with h5py.File( f"advection_{dim}d.out2.histograms.final.hdf", "r" ) as infile: - hist_parth = infile["1/data"][:] + hist_parth = infile["name/data"][:] # testing slices separately to ensure matching numpy convention all_close = np.allclose(hist_parth[:, 0], hist_np2d[0][:, 0]) all_close &= np.allclose(hist_parth[:, 1], hist_np2d[0][:, 1]) @@ -210,7 +210,7 @@ def Analyse(self, parameters): with h5py.File( f"advection_{dim}d.out2.histograms.final.hdf", "r" ) as infile: - hist_parth = infile["2/data"][:] + hist_parth = infile["other_name/data"][:] all_close = np.allclose(hist_parth, hist_np1d[0]) if not all_close: print(f"1D sampling-based hist for {dim}D setup don't match") @@ -229,7 +229,7 @@ def Analyse(self, parameters): with h5py.File( f"advection_{dim}d.out3.histograms.final.hdf", "r" ) as infile: - hist_parth = infile["0/data"][:] + hist_parth = infile["hist0/data"][:] # testing slices separately to ensure matching numpy convention all_close = np.allclose(hist_parth[:, 0], hist_np2d[0][:, 0]) all_close &= np.allclose(hist_parth[:, 1], hist_np2d[0][:, 1]) diff --git a/tst/regression/test_suites/output_hdf5/parthinput.advection b/tst/regression/test_suites/output_hdf5/parthinput.advection index 961bc6310a91..3061d2bff961 100644 --- a/tst/regression/test_suites/output_hdf5/parthinput.advection +++ b/tst/regression/test_suites/output_hdf5/parthinput.advection @@ -74,7 +74,7 @@ dt = 0.25 file_type = histogram dt = 0.25 -num_histograms = 3 +hist_names = hist0, name, other_name # 1D histogram of a variable, binned by a variable hist0_ndim = 1 @@ -86,28 +86,28 @@ hist0_binned_variable = advected hist0_binned_variable_component = 0 # 2D histogram of a variable, binned by a coordinate and a different variable -hist1_ndim = 2 -hist1_x_variable = HIST_COORD_X1 -hist1_x_edges_type = lin -hist1_x_edges_num_bins = 4 -hist1_x_edges_min = -0.5 -hist1_x_edges_max = 0.5 -hist1_y_variable = one_minus_advected_sq -hist1_y_variable_component = 0 -hist1_y_edges_type = list -hist1_y_edges_list = 0, 0.5, 1.0 -hist1_binned_variable = advected -hist1_binned_variable_component = 0 +name_ndim = 2 +name_x_variable = HIST_COORD_X1 +name_x_edges_type = lin +name_x_edges_num_bins = 4 +name_x_edges_min = -0.5 +name_x_edges_max = 0.5 +name_y_variable = one_minus_advected_sq +name_y_variable_component = 0 +name_y_edges_type = list +name_y_edges_list = 0, 0.5, 1.0 +name_binned_variable = advected +name_binned_variable_component = 0 # 1D histogram ("standard", i.e., counting occurance in bin) -hist2_ndim = 1 -hist2_x_variable = advected -hist2_x_variable_component = 0 -hist2_x_edges_type = log -hist2_x_edges_num_bins = 10 -hist2_x_edges_min = 1e-9 -hist2_x_edges_max = 1e0 -hist2_binned_variable = HIST_ONES +other_name_ndim = 1 +other_name_x_variable = advected +other_name_x_variable_component = 0 +other_name_x_edges_type = log +other_name_x_edges_num_bins = 10 +other_name_x_edges_min = 1e-9 +other_name_x_edges_max = 1e0 +other_name_binned_variable = HIST_ONES # A second output block with different dt for histograms # to double check that writing to different files works @@ -115,7 +115,7 @@ hist2_binned_variable = HIST_ONES file_type = histogram dt = 0.5 -num_histograms = 1 +hist_names = hist0 # 2D histogram of volume weighted variable according to two coordinates hist0_ndim = 2 From 3dfb4c64c06f49835f77cc03efc3dfac174b736a Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 15 Nov 2023 15:36:17 +0100 Subject: [PATCH 63/68] Now with png --- doc/sphinx/src/figs/yt_doc-2dhist.png | Bin 0 -> 68041 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 doc/sphinx/src/figs/yt_doc-2dhist.png diff --git a/doc/sphinx/src/figs/yt_doc-2dhist.png b/doc/sphinx/src/figs/yt_doc-2dhist.png new file mode 100644 index 0000000000000000000000000000000000000000..ae9c6d1b2f73b9127755b572b2e2ea3330e71866 GIT binary patch literal 68041 zcmd>mbzIYX`!^;QprRtJQX*2)ZK4C|Mj9F2Jz^Mu1xPn2-H2m!2@29uBgQ}kB*s8u z)EIj{L(Vz(x$pD4pWo~G_u1?0C1;H7`>E@?-f?|jsw&^7I>B&)f`Wny^5CvI1;t?@ z3W`H<$Bu$mzA9E+1pg?z-P3i`aD=*fnz~p}D4DuB**Ut|S({z*uyAp;c61Qp6Xz4S ze#y$s&B;}gpWpt^H}E;SSo$iv+pBu>3i90$Td#_-q$n$pprD)+~sZ6%?4zA2Por+OZ;N6o$ z0&!;o6NIg=Kt^i35fPD*G!-&zYMd>*aKN z`dMQibbS%gMsQ~v*`k(=mecf7{hOCMVHRJMA-$1|<2^6B=(Ov5%;DEubk-N{EMyj5 zHZq5c^W@Kuhxx6(@b_%A7fHbeMpWW=8S?G#qX|P3qx=R-=2Xku5sjW910GTnRn500 zQfwD)-bed@Uv4}oE@AcGyfrsBH?oG0S8zC6CwF=Eg1Anj-r8aj;(=tPHPPBqo|meESW?SSAmew$|_HE!n)uf$HTjb zG@}<`VS~?))1G5>R4JznJvVr9o&h}14$_hT4X|qJcysmhSBcZ!w4ubn)iMZn?AV+^ zQ9S)dK?+N+x6KWF6a-2vil)6St0TURy&*Fxf$+b-{z1}_ks=^{U^kv#N-ULSJ)WMo zFJ{G)=3z(2-sz`M3;c9y@2AsR4}|vO1(W8@=o8N?Wf}tG>GytfDNi8ocO*$W(f%+^ zS6)8IS2h)_Lv(yR{oEqOeL9s{^G`G>;%&}+sd{zA)l%;#$BJ>9VY%5ubL^glrF{>n z?~AJ|8c=f~AL}pO@7PXG9rwt<9H4l2tG%sFU0XY*t-YNC;=eUbV@{Z`r|Jp%^yX?z zaRb@<%sPHTLz~jA@2CY1$qr{$uyggV?4$8 zca2EPW=BW6qVcgIgRL$9%*;aSJFj2JU;jNuFl;b5h9qahqvDP}CQWip10jbrqa1~> zEcX*yC)l$zHSI*O8ygip1z{wc&{YGPluaqbE`gs#2pg%wVv902A7u>E+AdGTSL?`r zuTjZ)zzZ(O%Io6qX=REsKCm@?+}0=HWc$OtOa9k@8xz0x67{vyKP@;9#KgZl`VXri z{mDe&;+*m&;jOJL1qejV$S4Iw6gCVq#Th5#4O1v)GNc?Ex;f}<+*O2>6*)(UGrTrU>o^BZE7bL$pHwnFM-&Unb?Wj;ep4W9(C> zvcTD{;WP?u8sB{LRAbZf!*R^_BLUh|F$Z@hPNbr;%!w9z22 zewgC4{SJx5lY~3K_YX6o3q44lH}M<=Z%u9$UGyP?{aw&-d9%~eKm8>wS3-HAalKJJ z5$A~indlf)u5q7xKf8&)5F7Kl(=D?xdH@fi|B;gyW@8>X^z^C76@s9CY;AG232ZKW z^HgW*K%2LXP|4%QlKKyhPMI$sB$TeZBUo%~S~C{!V2In>fp5Fq#r;WN3iKtqqav3) z#%CYMFMll>N4lunyUpdun}PX%kFND8AM>7J&dkhYc1j$?4p_t*W=%~!u3hbtIx~yu zdHMTr1w1=3+v0_C@*RG%8ZKzscz9+dg6JztXu8->wSVPQjcNB@mpV-}>dVaRY^}PZ z^el6=^0kpLBPRNzLi10Ay2S6Wgv|sL*ZC3Tt8B?rq`JgwTxW2IlDfZgCMAEmJ$BG* z!E@X%+q5ZYu3tk%tKOg3pRdE;886f#?Lk<_>DPFb6jjaVRn3ZqQusjc4$Tkb57cYiyb22|&Xe1oO_#(LAg&DU?VL?DzcB|$n;FbSeCWTXda#U1w=RM z`=@6}yWtyOnwxD}BRCkuo!)&!An;R>D%w*1qZQ5xlJ4`Sa!31_j|yF}n`{n&;Rkf` zsU9DGb}NKQLS4wR>y1e_c@XdX6(NwGI6-rE$fGj59x(LemY>tp*1x`gdiC;U&t#Zf z(a?}CM4hxt6oGbJR7w(8BW~mS@Po$ukOV2uqN7v{85el<`?n@<&R*;gKlSC_5-mZJUjR49F_MUsp!DRe%f`{?aNjf45TyCoh+ez6f9!d#nhQBHRw$OTOuzb<8F4U zKg?bh%ojMlHhy5ewMb=0x!H3Kt34tz`0ICWOs&>${Y*@_VKLr2!jFwpMcXQg}Qyz)#{9#*hNnxM$h1~58 zI4QZz*u~`|yLC$_bJwcfC#Hh;XPseg8iUP#DBF9-W!myuwxcTEgLZ)j-O z!$*dA{OVdUwSV>`S%-07aBO+`H4f~m6 z;``Is41@9W`Ni{Tula7orc&fEPgbW`qLz$Fx01_ouS}?do$h{6mh~>s+vpc|SbXuu zJsviH2xR#4leZ;lC@78@W=iL2Vt9i$nh8;-jcaR1db5-g^WbzBE?g54!3UV1IC+u- z0*c%sq=%a4#g3eN_p7L}`HGT)QF4t@S^@k2+(1?D-)g7ZAuR$t#<4jwhq z*H2JaSI?_kUrISio`_)EwkYm_-7Y8I_hFp5xw(>a8IkM|TU3Ey*sxq*iUq0XNj#X% z`ax5@Lk!4DYTMPHomxUN?1DUGSkiE&ny^^yPQaDx5J|zT{;Duo)y_)ZjZ>+?^*1bo zjgk>}4#07l@eg{zhzdb8oJ$Xr?Tywk(!-S{B{3BmR%?Urtor`+qz%%rp9%`?Pa_!x zTjX}S>$bMG4%Ge5XHFSE!1#&D6Z*G}b%I8R6!>|uxpMp`?ByRoATPldKNeRWc2V#) z@1wGQ5PtGI^I|hBQ@rJ$+`O57Kdfl|vdBFDQ6ZZiu>X6*u!Xyf*bn5Pzw^M2)jnc{ z@074xg8-OUP$2z1uD@TiGf_uD0?_B+wYy&VxG-{sfHHm!QahvLTg1elo$2zs$Vza zVQtUfhI(ekNcXp6)`zBd)DKeJw9kpsi9W&ma^dTU)aDJ^`tjmEWhtd`-&Li7y62$$ ze?Nk>&E6#vY(G6cG7!oNqI3+OVIC!`uPQjLRS^*pnwpv!U=dWxM_pd~*8P%^C8rx3 z`rl)`Bn?VHS;7M1?V@Whgv)w&XUn~x?K!A+ZfmBZOf4QizA7VQsHdlQ-XR# ztTsfhsHtk>m-)sh8iX;+f{x922RJA~Z(l*H!h2#bAnWH(=Uz&( zzr5|_zV{sgIgJ2fvL&1c|98{yrKT_z>sn^*H9p-YZKDw>?0G4)Gnk-w|B-<_Fm z`Jds=5XCw20Gw2#S8~#GrqU^*5BDcyaBz&`UFVdis?uU#+d~h90Ov zETW^Tg%(@WO~lIOF zV1p97x*bne66ff<2Go=lO0K ze|+>}#|txX6z-lQPhSo#K~5OM+R4C2H#($8ety|HUsW>cV`@v+L$1xq4}jN%_P8A< zRex_S`sVH?qVIYr!@uk_ay>x-29{zX@byB88mF9`F*i3i0Ac2bR4?7#OGH@nw&OPU z?XzO~k@SAX9%G`g;b_wZkmKY=>JQ8=D9P!VeUuU@DvO@dnf)0GpIKsw8~Wp0ci*Y( znqclDc882`cYD%J&EofRBlyCz}^aaM@a;19E!jo!C0rPlN-A zK-KrA4(>embNO>T9xYc}Q&e?`A|Qjc?;+HW^!h_~{G0i;qwtM}&%ePs56E9ap2B~Dyn8QEYSFLd-niIo!FekNyzw8gF$IOYOM6JJp zE$~XfvpjMrH&1xEV`_V{CpaF=2cVSzebd&~_R39%{8z#PAl53Vs0;=iI0R<%J4MIu zZKI&99O@>v_l}$K@$qdP9sR$7;e|4fSZO?~-%3CH+%L)|6z3l0;SzW^erLxwGb7_F zKYtit2Hpy%FBvD5hZuPv^-@tjF?#i@t7^zzuI~>p@OyQ7tfXslgagk9cR#&3&RC>& z+r?mLgtwD_FoTbueCqx}+kcV_e{PcKl|YG3c3%(z_jfDBzcP_v$6PYYO1jZiFkZXz zwSN>V;Ty@ z({{dM`0LxiNxylnik}EW_%9_?n&Mi*PlzBN&*!c$dlr7sP_uEFnJEwSO|1e;2d048 z$n4~$kouyp-e}?~LtqIP*o!QG#mpcum^c3(&;Q#F4|w2tM4W=mr+9dj?IHU;UXe-_ z3e5uC3euwe0=c|bm)&ZN6FA#7b?z?fKZd3Q0awr2pD932J6+LDhh$9ZSf(u3>1}u| zSl5;NxhNZYWXlV6>CM#GuceuHZm+G1Cg7!HXAMru91IBRW?g&EVfuKwFTZppe&EFR zGxxCzX>=glfkyyrGKx_d#VwXp@bT#l3+C1A2cDGpbAQnp(7h<~{oMnhTYioI%oI>C zy`S?-DRcZaluADuPE3?sP{O%p#ypgRN9yt0CjF4s`B4-CtxbbSA!OHaF(RH_PNef& zP(NSZtQcuwt86{Fq+Jw}Fe8$Obj7&eDbY*ac2h4z9(cEV;&81gL}<$z5G?RSAv}9?sK$1RQ-qZ^@aA-(P!a{hi^yz$?|baNJ#Wz4g}-~{}Hf-upm?RdKU?% zhU7%c1~yG8u1h+$+n#qsO}@q%pEqke1eG~G>0CD9I?7d)(NPd93K7O~gkVxzgxd;* zu`erqI<|u}5)$zlw>=qYNUh4p;$h2th?MbOWkTG}8>u6Y*S=ms$WFc-QRF%haHr(= zx?2xbq0bF|-qlm1YdhJtHd+$)>z}jOU%|pko4jldx3->vpmH&mp{wk)Uxg(3H8R@| zR_hWwO^RE!9*H;~Z@SnT?wlD@!K7@~>Zfxd4$s@N*$JEP5}&DDFTE~?74#>#zWkAK z0%7D)Wf+^W;#FPG?ZcRemk%s~%6Nwn5D}jZq~70QA;?-lJZi5F`%`nY>R6Ltk#V-H zH*FK$eFErfF-d$R?_B#d+* z2U@0YUOP$g{egfG+pE8dM^Gt(s}DHJ$-|=v!W{s;0qnUqW~m4uD*!r_Oc9RVAiqO0 zJU%`bBnJE8kN2#tt-}|UYkwt<@5m=hy7zv5Nt-_ac>R1b7*$FX)vxodYz|{Brd5#^ zypb%eupj9gZq+O|Gr|@tXN3KCHf`{7yUSs)^z{+%~=W7>ISJ)5aX+>m&GD~UadoB#A0W91T zM8#->;V^fZc(6nmFRZv&2Bi6vt)$w{XUUy#CMrWi=u_KI9OzK%m zrq?Tr;y+@B?7YfdTG87;s@!EYd^Xd&CtGYUW<^*QuJBr70*X zinvTYz<5oDP%>5f5j=Q}YIMWob}PJ#Fx8%l5edS%Ebg@=l9b1S>R^!}&6e}`Z~NKe zg+gUzWrC1psEdnBsoPxdGnRUnX@z9J^4#=cMoSNF+$W$P4UA}0%`pHo!};KUGc;FD^A_U zx)wylnn;^x^vB&0iA{>z$u-a)!zcmWM4<2R@iK z^;bAg<`shtZ@n_rYBO3OYujs@>sKP{`rlkQE2ID=G zOkTKyPXaRh2rj-`)3H@})T5?fUqg+^aOYjCaKQHz5KXyZ+(I#)fH_vXXm$% zS9ih`AZdD@ne(Fp4q=FpY{bISp;)!_g%&Z}3*1~cGq$Fq%c*5x0a@6+#v>n#>0q+1q=Ti<9#gk?H$s(FKd20FmQcLnt%oD6SZQh)8qY1 z7PbNhwD=VSQ)Zv&-Lbo^!ltbTFX=c;k@n3G6C98CsrTpWkdc5%`+lP7LO0h~F1?bK zY4VrR&rWef`T5XEp=V3Vi}l9c%ym zX~Q!DCe6>m-y}zbtJ;lU?%7pUiqIVWnw(|E{eW8x8**|%WFVrHw=-188riA*dU)h= zcR`D06+!ZGP_l4uNnkSd4;YiZXf^A7@1*32K8MI;=!*S@0b*>f`kWxJkeZK6>Hdgj!+3o6nQGfA#{oZo)&ws)4N^Cyc5 z*aaH-d=g`Q{=lv=;6x)b!eqB+1JLBylwKEhr4%-I1I>(@0h#U);`!x!aIZo+Q>OogS%I z&xTe`j`vvpY;z(Q|Kl~k z&0KFLrHnribbvY+`}EU0a>1e^yLC5AWzlr-88AHHFM z?=0sXcts|NJK_Y%x%X#R@*{U&8=E}#)|q_S&b25FWi<N}&24ppn~ow=oVkj&SP`Y{~Il8Uqrh zGt#`=0h33~EJ-fV-N!8VJ|gXSPXW>j)c@P-Xh#kRfDu0hWqj8A!II#|e3rC$_2}jN zYaA*jj*_)Ha%JT0U1{V!eI5fR5lyjD6im565&A1mOL8M(UZc6RB_%Bh_<#k%=L-d{ zrDtXqb-yX+UA4B!j{h7pB6Qqv?ove5hnDaM6J86M5RZ@d%<7~Ril>`>5p}hRwTiVo z%U5P|6xCnzdSIfN)X!L^=1Y$yxAofT)cB>Yulp%PEUZPE*FBqeIo2kWZVYc6sSp5v3}TBm zu{tfSk?Gq|s#(O8t|ikepseV0uD$(Su;*8}zpz-ngu<1N8{tVn==fC0b?G$sDIfuf z+K*@wt3P~D1;r&LlZLMDRfr9y3V)_NHYv%#{&&s`?T7Olla{7&H~q-7I6qcfEzJ@J zP8-m_pwv)1beU2_Kd5e-mNdmK)F_ zJGfx|KD&T|BJI%Yw>E0vXe&$3u|s+jL~Mat$(51>giTW{9eFXBxYs*?|7Jj~y{K;6 z2GV<<)z7WL9*`ya)gFbQq;UUqgzkCup80@6_}D4v^PG&w<8B=RY^z?MU!1WS{+Ri* zJJpyMb2lQz`zQU8g9k!bWU5A>?}K^%WHKdxxRMiBg9=a+xPf?d=;*0qw$B$ggj|yMven=E6~y!lK*~Y{sxM67{UnNy`zCSmz%mH%^vi+-xW}wJW%Ftujf|5zj>HAGy!zxW65K9u* z`rLoUTh3=CI`je%n;m$Pu5D&DkZ_s&brj!Byq_K2e<1SK)JOwOi=+)i`@9}i*9O~Ab4ew2W!qwi@1;vr=xqmP^9eFZB z2X7Ti7lk;@-@Np11nXWZ+tX(bP}~g(R!#1O*8l>hk3os+D*Vo4Eq3M&@KG_?Lbv$Y6AC!l^Nym}NdA)aI~=X7=JyLl$4WjG z`Iv*z2P=1pma(Eo3Osi?O;Co2xMxOE^O_MZBRX;Tp%a8<_Y(dy_SpGdt?Lrqp>Qg% zg`9zx{34>*$zmRtvSrNiz{^OCn@WV<#)wKG``iTA!ZDPw%7{gIvaQ@wm|SD8{-xfh zw~C*F*nNgf+3gqm{ftt^n>VYCwpBmfq+-hypWo*&a>VqhWmI^BQ@G%!FYCT;j_OI0hpk|TMYMLz3xW=BFmP_1eUMfLX;35$n+f8&k-|fsDW3!;O0(f1- zVt#42#<%_@yCxlL6~(c!VdVqM;_I>KzQ*K(y$+GYkr?4J0`dp^Rp&N$>dsNgCb>N3jlOL+=-wL@&4gvJgr9}UURrrGEByd>r4OCPpQ?8?FH-gRceYC{$$Yz2%SJj($Ut| z&Zs6I;CD}2fkzYQbVpBJykt#Ep&t|QFR?s28W(@&3y7-GNejFEzf-@?-;|4cRZ=%a z_6e-{5Db59b}c{g&ZnD~{%+Ge-#e>6+A)hP#u{Fa92HN}}|bmw-} z8*P9wK08!nMh;psN*Ll&>xEK2(0UqJGnSlR?2h(8evG{HLlHMvGne9pNU3Qda?fWu zWCG0}de+I!(-V8clF^k@%ysPOCCPiQ_bsdqw*qbBTUGuo zBL3UQa%$KOvcl0g9K>Ii@Ar(W>pJDL zMO`&I$9qM7MJJ?w4xCTSTj}f)J?nz|szL{kAilQ0mnKzKTnZ7Z9vj+?X0H=BK`}1n zx|L15GQmrFd?|i`(3n5@s6tM_3R&{4fV6axfV8h_@ppI~ff>BLG40V|>%ikePwod<9ydT1?9eT zmh6`J*WeEZ%%mwWt0wS9QSQGT)%oQ3>{D6MSH!`YqBb6F|UR0 zQ#Fumpc!AtmO{sQkBl!c{s=9>o?ir~TV-woD)ZJV8+lF)8OdtahJmm-f8p?86a*E% zUIdXTMmW^7EWE6!v4Bnl3CA55mmOI?Iy(5k^mQ{-<`Xr+N#Uvoyr{Gmk?eL>$0j>w z0eQzjb10)@xOo%1u-#R*ZhO(np`5pp)=v7pUp{k8;!=kY7LlD9=dP|8&~$EXk#NG& zC*aWkEOmW4B15Q~9~!H#ORt6d-T%Z8Lpbo~;e+ZJ=78r_xM>M0nV7$?X>*he@WU*IxU)w-IHBzzf-Q%`ydU`G%~hV@^gv+8bP@h7l*xB& z9!0QJ3AEIG3g0=qHqsZr1VaQOLY4>5$K!|8eX3U0yVi?uC+nbR!ZW1C?QKzTlC4x% z#`2gtq7HwNP#K+3p+4wp#+q8w;?G2D{4r98x@CwF*P=r|x$ExIOzWXe6)?Lmr_Bw0 z>24n!N;c9&uXlHRUt-by!%SQufQwfvlxcYKmwDnpl>e3+wEb*+W>QvyF5XB0q_+WZobYG{W4RtRQgoP7C+u4Y*;tnO~ zn2ZudWt!X&`a`?ssWb8$4%;-HCwXO>qQ9vPHP*V;@kDbBWxdqvwv9JIvHGyDX$C;* zuM1=4+;UHC8Sr^xOyszV!ah-Rl}wTRE#*Sxq_utKcqjsbUjNqF8*X_YWbr;Y@#uF+ z@vlV}d7Xj1;g1Y=tT73C<-hnD)XZVcu7E~fFzq5dXZW-YU~mFnFm?RLqE2cyj#0V< zDxv$a(V5dz7&~jBBk73fkj9>@RjZNiZZ2WTs@887n|86b$-wE$H#T&oA06FuRHMF&}O3Yhqb~cx*aAj!W$6rs|kCbxXmXU>#CtP*u zA=x=xZ#&J7*i|hKPMTTq`mxsy0Yw?^IkI|(n}H)M4!>s+>SCJ@{jj?eLKvCZ z{-(@ONi^n4z{gwKxr;gz>ft{f4tSXzfOk!xB-gVq;d?a%Rz*TbjvBKUX=7(Y30&t*#Vq^HYfuYYd4G_y})jn2YAazKzKBJ6d-wS>klb2 z3)p?RlANB(hce{?O^A%0$fgrOS>=c7m6!zr7+9PNFqal`qAA$6J%%?y&|IFq%CFz# zH~#*1>AHUDgOhZ81<|em>d(L8Daz4C1KohHYBnv8L;^Ht1Uv8~*X5xqoXweCI|TJV znA|E{J0$vt7lW=p>?Nn^mgHK||JDRQ^Ul8DQMq!yPB98dUs_uEIXQ3HLCcSpR2hJk zWZsB|c;M3G{i5XmK}Bqk`AAm1f>~e>5SHEA&dw#E=U^>ZZs%5nTjO!A2NAo(B{+|a zOzpfZV8&T!dOo`U(I1c#_1{T4XJE5eh}hm(Dkd{|fL{dG_r(Jnwg7Rnczr)W1oPE< zkIpbd4*{L)6dR9XCrfz1#v9D3l2QP86UB&_>w0OxK}$AcBMByIROB-E%%lSEarXkh zg_V*nbQV5=rMGf)=kj}1MMo=NE0uxR1+NO-AXpl{W}8ibbZItH*o z_jY4JwYr|ojmc3wRA#oMI3GAp>`>WMqd)B?yFAfBqs6Kra1j@&@8e`shg?!rWDncx zj_o&G41vHd8a{m-Hg!D6vFy|PQheVq%|E#72O=YbXjqFt#|IELoxw)68o1e_(v1O4 z4P+`6Q{~x{z8`z?EaF#wyLG3_8w$8rtN3J9f@pn_bRQDNL+H}xcq0P8p1!i>@@CCL&X@^Y&(rJQE>Frg=xb-O^cQ5{8+uwpz zBVEZ-`Jz)hb7L3e27sS_PvF#C>$_Y85E1c8GCIq!~`CM>B?te39I<*ht zj^gr*>+-fYRLnKuHY650pa^DW_z4!2kNa240+J!me9UW-%6)NEZzdJKdtieE_NR!~ zFB2+2%Lzd{ayl2A8dCOcJ-}B9HZhAq~)q3heUOy^#C>&xqdxg`R23I=49h<(Nj|OK^I2JM zPd`c-d>@Gji7I^533FbJf%^6GpX`LCA&!(+kJ*_H(y%eJ_n&)9A|wMUM7=H9zI51b zX-5t}{0+3N`94i&d4=DT0_ zAwQGBH^fqYsF(R<=FsLmZXiO`-k<5*_3PI)ZRVdGiV6%Q9Mu3*X=sZ}Tl{d~DmjtA zyM-kalqr7C*wAIeh{GF$K5MgQ%KRsO?b8KT-^k<~aA-p!Pz5Ei0*k*D$wm}Elk6V> z6CAh36(qsY(F*s05sytN%%8mxY6JqOCCr>3lMBS-}$rGX=K2CDhEG>3gT z8ITbMqPi_|gs9MDqV9tY9&YY%c0lf}uAgC(&#`EaNsxp4Bi(2rFhGo*J9bzYwl-^% z7?`$e_B+_09`INiD+J~Z-H6i1t!K5gwOO1V0vhsXXJYo#rw1`E?YvrZ9KihwIzwKO z`*}ce4ML8K`vT+Y#W!wTrxX68Fn!*+_GYG|Dm!RMQzv5%K%Co>gCM0=5wsM03Pj?a zyCe~^u$~f+)UNTzD4w&JHgK`&A%g$c+0c)ycpA!pnLbt`=Tbv)Ri) zT3jX&C2^&FYamnuUc4o47Kl7~aMCVFa-R-t=+(lIEWW>P`GLj=z?Q44sfGK42#*Cq zZg_D}Du0gg3uwr`RT*SmK;=MxOMUMSkU{)jqb259TKLb^tu^2h33N-1RF;IS&U6>2 zvs^aVcN+gLbXB?5;Z{{}sC>nO{;w*J+B(%+En{>6pMT@7@2ZDo7V*}|oG+QNdEmMD z2)goGalS_8E|)09o)+}4jiu!J^FS7WMsb)EvyVD$ZgZg@sS z#tBc#Q1q6S?tn^{RNz}r~6^mYb<^BXuaT#r>G?^}VVF6)xr zB-!7xjfh;ZiYgJikcjVL!$mQ)CQAzmwfi##r@zmLF_0Nhw%cwuv~L=vZIDkmK2Q4( zX4QQ_3efu>`ROEV-EW{I`6QjyOyl4S=)I6X*-u_dQ}SCq zq^27*tg{slpIw-dxi~%yl7n@f-nhSk6e7hc*z&6m>eH}1#JqaUEtoimM)7(0`O$bL zKyJUReN~3%BzWnJ$bVzR+BId)@ey?LTVscEIXLJSKS#YAl#S(ik}=6gAT;4O*|~Fv z{NDg@W~@_mqzx@8&UX~Tzgvf_K9m#Gf0jbrZGZgKVB7A|8Fc@{SpnNh{OTUI>3!%w z!UDXGd%D)YTf+k+wT#NzKwA*)OvM)#Tbj1eN#C5oJBy-ILRi5}58{OEYux0dihG$e zIs$;y4k26%%cBonvf-;+Z>7H&4~czdHjjD6LRj{07nMXG<8%|jcEYa;Vk-+HJ7CI# zk1ZMbfV&fpwkylN=!lyd`yrdv>zvBBj_-3TXI2^6O_a7|9N#Dvg0@fcbiyWO*1gr& zIVI6wZB5~<8Io6YYYz}4mVGdNrHPK^sT)d|1uN9%S%OZ>dLmPNwa8}u_^@*k-0S+9 zYv6pd&qB_qL?lM7_xE{Zaz5#w%FOdnlQ&I&FDc@@CLKqXn-3m2w7zlXlAuuf*omXh z^vk=J+xOawgjEauohxCo5-M|uls9fVs$oDkRw)4Tj#04==wbu^fgeC3318gr>7j0a zIR8>^XMzTJ=+(`B)Xim;l$2C~epj|p9&%(>U$d6=)C|S1xvaICp<|G@2 zckd(TIaCBLx;av^m;!HMf`n@pi{F~s7SW$vsMW&o>al!={blx#I5;@Cf!h$YmV$1M zK9EQER4+?QY@%bG8o5)}VN{%oN&JI00cq891(<-WKszncr4Ha6Yv3%{{{D=n)Mnrs zXpFT2EnQu!Gk-SWC@53=wRlnFoZRzp$cWf=yKUrgUq&u}1BQ8Xol#s(cj;Kz^Zuaw z9v=hwQOZ=|>?#jW5`qGfQJ3yFDqQMHcKKe|0ccsBg9=}ugw5x`yt5!K+F8K0lqZ1Q z1tg(6N8|%_|Gea@XPpc= zxS`|`yCXSWU4(b0lI9bjyy2C)xL9DZ%D4W63SLSYV{+iByrz1*x7zEj5PU zxTt!m?h(2Fmh)kp>ZD0Cxpn)0a=`jv+z02XutkyVyapAj^;?~eZaCzi4KU-_^fWG? zB)g-*%}R1%@lY4=i8jq|=(k(scHfg*g*1&1;!N_;{9*{ZL-nA4PRH~D=o502Xj~4A~tz4W02_|KXu9YSU<0&jSz#LtV8( zfF?(G4}S>o+Gex2+Hw8evoh`-3Tj(243Z;lbz}1q$t$GFkF1+Lk-WCKIxpd>oKgK0 zR$$~np|7gy)KCzlTC+Z4Tea4!Xj^^(S2*Y+jV= zX)>78ccI62#vPEo%|}$=xEfq7W7VJn zBh|h7L0R{)_14Vncy}J;;TN2E7lBD&aOeRo`5r?Xsp8INcaI9y0oP0(yxh@|{j#CkvbFiRga8UB7u08JqhOfL`T!P+5IacqMP``9M;K>?IX-21JbV%o} z0Q2*&<_l2FC zd+r4q4ekpu@Q?=5(Y81A&x>^g2aG=fr$r)x+YkWtfKMe3P_pijlWEGeN%1{7YHY-5 zet1u*b>CVsoS5HiPIi9c7lt*!JqOA_7jc*LYKH)R|1eImC z9?Jlhhum}y47ptQGRA>xB(`Nz4a6;A!bfHeq$0Y|dqXM5O2F^2;5m%l^6-CV|t z%VH^EHH%S)bZlO>11Py5DjoXuAS6AON(Auvghan!hOX%!{D$_Z+LwL{oI4)scP)CK z$ce4{Ou;{P!h8)}5iB}(q!-;vh(1GhPlcf3QKWBQ^g%PO$bR;cA!YnkrP6LK;6)Zb z`-o+vSS%B=KFOoxl%2*t!x4wyR42HmjcQicOQEBH^KlEO!rsnAoL`TY!Md&rPIrkK za9={ODhp!8#>}&4`X8@7@Uh0RrhFJ@v{5j2_THJ8dd9j&Tht4JERuKS(1D!MMG z@ z!aY32HEGV1&01*o)og?y=w}di{&CmB(o%=Y?7C5n_!XkEl2YM(({IO+qq2fNxzjVi z>LWZJB3@4s3zLXj)&M;LFksjx-=w3KxQ^cb!tei{O#Uv7)kIe$T z`Jd=3(3?%4M@|FtS`0g2AMR6*#(*1)AZ-W9cQtsse(<+ePnoyzjJ~&yo`zg8HHR|i zlT)prpde@$P)815oB%oG3Z(0E)PJQRSc?ACRIx9*_wtk+5W_Op*4(Hr3ILt@0JSPV zrI0MR#zQu|`la6pOZCOH%Hp^9YkoZib}WQ+~k@P?Munw%|=Ukeh95!Fk=DWF_U`v4c`ea)3+G$AyhA>40LgKUagx9#f?i zu?Egy9$<3!8{fY-JkhUm%YF6gl^du)u^mF){Yqk(y|#WpvAa<>lFu+`FwKv(=ER~3 zBgdD-NVATEMakqFi4f;?>U%+}%0Q*-j7>Rk1lV9CeJH1!rdPM}>~F%ZlFeeBsdDw& zguO+f!mO7Y_pEywseLOcWqr#I>eW;7RkWl|EPnnot%~OAPmj?gcKh)bY$7s~^avxP-4JjW41wbUoT$*9Rq-owfCfB@ z*!`BPVPn-4O0(8nDvJFSz^pa5jCA@F?C1>7xZFjw(E=yR_ZKEqFx-Q_kB@zX7KTV8 z-E-N#(}AJ>!^JJ@=D3+}j7};iyKJHN3%AV|jCD^Y4uxkqi+nC_jt&$(bfY|;n8Y4c zVH`GeqBFy z$WE5YGPEj$V}$G#vXgx`m6*zweHlz8>lm_(Z4A%-*7y1TzW?9#{GR`FuFiFxt7G}h z%zL@-*LELK^b4CSQ6f7lg(4wPS5m@vzZREr>#v@LYi$FL5a5-5F~nMh(MIQ)5+}B_ zDAAv_5dNjKnx5l1c_O2Af70|d{ey>Wd-IxyP_c=Q^A)h-c$^$M4Whgq9rD~=*Ec?u zZ`V3WAD%zhvLn;Op3agKiv*18OGKrKSE?JI{VT=v=toC#jEdVx&IH_z#eG}SN*o>9~E&nlFYkIB-1Vk%w);N$WBsgBBNDO3$nR!~NIkJ{uuq9{b z8iyD5MS6-oRQjTMqnGqLvT&dC`$T)vz37DVI{(&&^xll15Acuud zvX3ly;N}uM?|z%KJq6IjjpOCI>VJiH_Yd2&_>EUl8I?W(?zzX#NygC(E;VIX1rFu~s#E<6)5KLq+2O&4*pL zh>sxo$339;HX<>gn9nYDL$&k!*4yi15iWk4MQCDDIi-Z;qzhR*9^kH= z0520dvJmlg5~^x5XPWCvAW=|3EOb5l+t0dSl-(tG?AQh1Si8Xg*sxOpUlI8yb%HHs zLzz`<)%~!$m_mE9TdzXObNg}RW`jzc$s%DkWWD725Dt_i z0jw?PG(kud%?G3zdXDZqKbp|KBVv~nlSJ_Yc`W~V08tZ0H^0i2zt={!W-Mme1o-AY zrS~+f`_^=UVc9nXcRbAJ&5xd8@KiUpo0WpB$b4q&m3YUkTNbvrbB85#U2FZdwa{<0 z(lNoYQQ9WKFz@O+eU^it zauWW=S7e17lDj@=j-Aqamt>5Y)$a>ymB3X|jOcWKyMPdhvv*_S$` zpn4b`S}iw^q4}GTYqOIU`yuayCW~o%+MV#LDyL?AF*X6xPwvXsA0X24wD=d({TX@w zULl(Pp5YdF&J1ghR|RF))ctWxAcoQtwXt692oy+62R*SK_@C0(=91ROfLLyap1Ig< zR1>7)LLwrXP(n=fmlwj7_Zsf(I$|a7=Z#hN^w`om;bSlN_iNKT&iPG%AEtYT$ip-4 zwDEuM@65z<+(!!^Y*)DK;Ytue=9mt2h(D{pHGR|^o-9SHV{&Fv_I9S z$h7E>lbw>(J&!+zzmenAj13GvH_TU)Mc2F__vJ&@cx4u_B_DdIbJu_#zCi~{mS8a~uxecHC z|BsGvoM?5zgVDM6&|J{6UTA{$-7OsV*R{NS8<1{*1o$yJL(V6B{zGAOHtq!p+n=}r zQ2YMtV*|EKcWy+?qm}#n`~NdOx~aVumncplm@5=^^}Y1|x|JxraD(2%hK-}Cu86wA z1@e56%JG+qbuUZ%$Lt*no}Y$lIH>9R&Z%5d6Emdz#8OdBtmIkRPg9Vp*eDsW?eg4e ze=*(tN(g&rM$If%&o{aR$sSt%CuO7HH$7WYJkHa-m_8?I*Iuw^otT{j*>Tnb zY*PKx#aaoDrwJXQHx(EmT8-Ye5nIUTzG9~;n-w;P$v^ll%-CsT{7muRd**+KgLddb zN(FKRf-+N2N9WDu5O$PKXa~(zO65WA|AM^drK5X0^?g*6;jBj?h~5BG&4bcp0>Cca zqkl_(FMIrWD{Z`875eH>2qP3sO{y$sK$Tq^vdz^p3cdkix_=WB~#2aQ>sr$J73it?j5j7{6k`FCB zgHB$*4q0L-NDtEnA{Yq%eQVtmt5SybfCRRGrS9n7EvROZ-ozQS!+Q-B);(pK)&rx3 z*fCMj^?F4u2OArXwXbjs0Y~1rvcYajWrP*(i*zogY?kYNQ#EX8t>8CZ+I-@Q5vOw( z%Rlx?zu_l!>j8`%KIvwckOkGvW4T^)pJ9>{^st|b=W56{rKWdM zwoi7K9sQDBi8uKQw92PQonsg_*9Dx3W915mxrNeGJgb3|zh_DHql9nmcjFSo3(~A| z3qy+bEzsLa6(mVG!Tj~}zn2FxR9vkVJQ6^wyklGS!dzl^W5UfL?;QMYYIiN5K3~Vg zZ8)Nv`wr`nR8DzjYH4)q`{v^mwsN&@nSTC`Ol(dh^bZ6KdHGKVOf}OxBnm9QIFcWI}9uY7LeVNQ^>=dg#5Q%6QZ}+jX2o@|5JuA#Es`;dL}kS8tzn6*$>f zsLuYVQnCkp2Lh;7IvwH1ciQJwzeVA8kC+vo-4xxZ=cUX31I>&}@T7EUpdPpedP5@x zXWRjOA8Yy5-?gz)&WX9@BkY(qN*=-Q{aJusJ5f{~@{j`D0mtk80HHssTJ|#V2Y&Q2 z9$qJB@tqj?z2@KyNDQdGU2ZIfY}b}JmzjJ+EjvD*Tf_n z1qh@@o9RFvDQre}@DQZ~#d5X!d3?H&^?{vR=fT94XxsxrPLrPGxHxN?szBLGluIbgx-b!d(}-#iHiGxt4eBR|SQrTEc%R z$~7H5JT$0+GuR<_SfS3{{d2*T^i$)Y_q$QJy`8yKR?#mi8n>`0E%2h&@_!Lk<83T0D0ZA#&{Ctxf?~!N;9D2w-ok>UOAu9vZe#D!Xad!c8 zvY+``R{`MpFwGnO0CxwtYC)!@b>qv|C@uG0G+LGS1CIW^^A2oRZ|H=;gXe^T0%IXy z1NelOcErYb8C_@$Q0DB{NFfghY@h&bC;O&keQlAD1z1MK_}jP+z-MsK8UYX%#qBeB z7A1Y)i~Jbgm`tPUqnHFa>~$xnsr|)V`(R%v5GrH z?+bEa{)4D|05>FxUDZ-z=*3AncR`A*2jY7WTO>OIfe_t*DT~{8oyN4mPwUJvmhzbB zz@#NuiyFwXBGntBT9%fUGW+TdnP=PQmcNBf2`)K&M+%0P8lIoOP(Tde%aq(S;N0`> z7P9hn<(cO}J}9{l?l<_5Kb+r?*>{998m@j^X2SrzDEs3vi#P&oGoL)5GqU+Kr`wStHb9G*2 z97=jkIjXqF*tcU36*lVhjveT+A(+vA*d#D8u;OQsP-Z7`n+J%*|&|Xx}SXT$ORA>2N@CgM6MoZ}&#8mFQ>*)LxRVwluL2Pm?Y(wG#TWr-#e^ z^(``>a7~&{`|N!EW>+O{V6eBoVqO@kgS3R35utxZb(n8P4&J zt3#!pL6Jg{b&)AL_({$1iN(G&YeqyCvw!dVV{TPXiE@MLCKP*(~ z^zXPgN|(aD3crs z0`knN-65!a42-0^$McPlk{b9VeK-W%^{J%G01 zg69G9w;kddLdzH{BAMIIAw>a7tuj0($Kpi*Nc`oYW9U`A`T;Sw8GU=*kP2JLzhx8x&c7CH#+Pkzf6(|1fYyjw0b5K!N@0`HIN6wKJV`jc5D#M@Cl+b&u^hHX&g3R zuIvclh$wz&9vX3bNFFkR=pck9FZ)g%3E#m2p*FCI$59Y5ltAvF3%uS`c2Y#-I=mO1 z_z1=p4iIVpJKnwv*s&h!<>2(bbC#fmh&^-bBV^q80wr*30*wXObX3Q$V#ey?L!X#r zzNSP4V{}0Nb4-~@ZT;|nMGHu~VDv6teCSP^NQJi-rI#QBZHbqCLv)RLB!TX6+=@thKOpuRHt29KbK21M-vcWeg7Hhq672b|~D zm*EZUe673#g)2>T=OuYV3N_Ggc86RbeX%Rol;eXLI^520Vf{Ucj6B6-;JwH5xrqiB z6^VOL`}8T@dEJOy*dto?OXB;;)OYuRe1D3t?#GyoCUn@Q zQPuOkhC-w|%H3=beW$-z2{$y#!nT9YS>NZjFTB!Xhhtip+ z2+~CK{1rvpXQ;r0;By*y8F?!}hG$ayHf( zs}wrz-M2pxoY9m=Kn!TdXxJ^ann&F(Ye4x zPn0u0BulzeoKtQ{9(vX_Fa6dxnlDn8v^2G3QONzfH}9!X+k9Mvlq^YY&@rvD{HGmD z5cS3n^V_&lhh+ICKh3XPj$17zUk*AZl**BI1{|JSX^q4+uJrmF{&1?@PwKnp%mRJh zZ33~(ObK&k(8@g7IRt`g7uQK*h1~OHi%QazJV;>)(xX>TlOMIXBsQc@%kjj~HjCzp zQ&Kf1?Gma=cG}i8!|%Jwq+;fYy)76s-bzB^;US|iGB&hA^V}}I z7ed#TE&Un%Q)B(tsJ(MTJ`?`a(fQ=Hl+U0mSHr{FLJB1C`r86YH9;!Pll7cTM+%uf z21>+7r4Vej;tK3^E_}ln_&~lv&X|#9Qo!CLZ$dbvCfd|RJHf7z3K{>LUzD?Fp#!%A zUfg>l_1>4J51rN31X+C|7qSqZ2=PltX1yR7 z0%R`fJGk$i2N_s5U>Ow7BHNR&Hiw>{V^Vyx!dN8R1G9E(Y$M(CRf+o%@rwx}Wi=D) zk^E0q#+u#ihx-Eqm74cvr>CFUA-*u}6aM0M%kM0qZZn@8Wa}U~Rcat5KAPk(*P?_H zu3O=3d<9`sUe%p8`=j8htPR+Q?t+7&X4&v)x}a!5g9ZHHI14CV4?my}IR+MWL2Mv( zLwV_3f82iOA8B_tT=hLWGh4S>b~;UxWCzD?Ct=T?Jy~_JYegc|xbXc>C_s_CwjhNA z?*s~g0d>n;c%cSQjKv}M1z_3TTmKRi6x8ustFZzBUH-^5>0uA~&ZmeAx2@F{RQ zSiboB^$J2`^v{f$scz)MOIQ340rRzk211;g@-vqdVCIB39QQP^3#o!;AO_W@aF%7l z#p&wmie{o916HysGXzHJ35Ut_T>}0Cr9FzAZ&)M6N;qXfKR8dt0Xyst+^*bR{MZ+n&>@-XVCM!k88A!$oXTKC!xe+sw+YE6BRAfehT>A zlax;ewo0M*hv;wL689TVseQqeuyFY;OhzvRG7TyB;V&;NYG8r}7KjG7rCU5&Sv)`L z61t;fJ;HeRWu1^v2#cCaq=N(KfEr(hc7C;&#jQPZwx34tF=maa9wk3=e#92?jR5|m zS6+N8x*Pr`DnbR zUc-@{P7Pnav_kbC3BA!Ck+N6UNh$-+;C4b0OW?h0CJAW7rl39Pvsgjih$-JhGa$MJ zWDdzF>l^B%GhgoA|0xQO$305hOg#D-R4;JtUj=-}914V-lGf34E%M1QC8jl5)qr%G zm1Zzj`)dC@L=<)2t0WG!f^`K(T*RNDZTQ~*wuK&~i^$`D8e>#&J5W%FIiC9R|C&hB zlYfgU#bmD6)cEI9wZ ziZKM;rzLpC)+~>3u(45%57Pd{`cFW)cJTC#Yt$tmd{JJM_wNroJm5M7V+IDeuL>x$ z-5Z2UIrVLiwxZIIN#)k9Tl!Zulp&KU=htnYoDHUsGZ2-!6Oo59phspmts9L!UCxlT{xiI z6#jmP{b$i&?V2)&XWKLOMahP`rpY88CC;HW*O&sSF0Z)MGri7T*b)(!L4l0Vp#r~U z+t`CRp49wm>%0ufNyG0))skIO->B(a3}l!r;})zP2Www8_pce#i~D*8oQ9Sh=w6}@ z;S&y)_5yNVC&tv?x$>K_o~SfW`z0@T?6#Jhoax6Hrl0BRQ=ocV`mhrrh;+i`3EcAeCpXwXlAN=2od9WJ0j$9zRf!Wd_hJm{lb=zMF1L>)}UtmZG!BZ)JEyqKA~1$+I(F||$?zMdD7o&q6yCwdqf18r@v zmu~$G?#IY`IZlo;=H^-?DQTxykZcb>RN-gh(Y>hp%~eAkuk@j2L_T!X6thFoU|6P- zHn3beX5+tPayHO&!oYFGuXRsf(bBeUTQ;+eI)DlKC8}K^Qq{U85qDlZtH^n-1ujz< z+@SE6!UjaRUF4{jos@8A$MWRs3Ezcl`Y%`~7$hn)&wXf?qyhGJ2Q)&Z;t#!i^x+#p znbbS-X$3j5mEd zT37!zhW76>*>&}#4Tr84*rye^kMZ+Wuo15MPS<-iSA+M?C3RdSyUF79sm~BmSfEr>_tHR}u)D1_d zYf4c7F~8G=_2{wfvcIzV3pOQhD`!}3q_vK2#dESvT6S5-d{t$c8RlbUm zZX!KP;HG)|7rFj%b*~(3uatOxUTN9OEu@9R#T7-aGjsa3C+@4mb$(BN$9n1+;L}N^ zmHs^&7wpB+T>&=Zs?mkHy69gO%!9$oWP^m#r(=4~JI-CtF-nC(T|WzHf52C)d*8-#Y_R@J3SNZhqPuo42Xh?@gnqI zeuup`PToz&Uk@@_&rb~)SNkl2T#P~MMu3r?kx?czFx?|F&`P2L2EebzKNMlQRZ7g` zZm@->fBlN{Sq1&&E>wC#=BBvCr=2)BEz$JH%VFH6j?7<@G4L=;83?{^@n0)MLPw34 zH=Pe|L^PgP&bO%BjoL?cnY$@Sc?`a{eeB|DFyf{aaZVzp6)0G5;@j&-B9)T`YFMdB zFa;>%fp`l2dT8VMUFHGeYK58+A^+-lIJsXzpFV~_!Huz3Fl***027zD-dYZ)#G@l{ zH-OA7Yi_}Rb%>|023HAs?YA)fi}$(0miuE8o^>#Ygj>Efu|OJSmsb~v>jSlqDih zkq{)mKexH)lsprq)l@;iU<+&LFMe28cUDZtJ37dg2OElpl_qcEVC$KQ8lIUNFOoAR z&BTW6rmB6dvQ}w!E8og__F3dv&#e{^37s~94`)G*ka!54X= z&o$B~mef)yG*Oh?!m!uz4a6C%WHWm5aA%0wrJ@dygC^cO<*X9Vi2o#O!DsKCYci9U zyOVsENcrXQN#%-89M9xZcMcczbIZ>Yp7M5u2^tBGtTI!&Kc5*n#zS5b*aXTIIM;f< zwM144)n$(sH%d@)yEEQqEU(Yml^BZ34elozD{swGf5_`uD&HEqcv`D~;##*rkKI_%Fp}9>tHwr2<&)DQ7J6n{@Oph8Lp5jzd%M$xL?j8F z+%_DWiN!sc;asi>zvO-!oz!UTWXB}lS4})WAj;_Zo#)lJ)HTm5*RK+x+EoA}pY%w6`a*;0cREm-U?S8OQI zzrDKmE8~jzu7>E)De(xXA2y`p2#*8A=rcgK#ul@nho1N$yq7%CszStRWXkKjEVX#i=ZNAtR~E{7~QULh6R8vrq--We*#uF`q)GUUUdaO%g8ag~C^ zPoM4pwP4rfKxX;?E}U${T!1TK9A0hcZG+861?4x4dW7|#`TsOaEdRa6{$+2x#3q^(P{3dv= zU7{*YyWp4i0ZZ+F}b*1>wV2z#>`YV?`!Q`!%`h}un)-m zrp>Lb_pJ@{{5a5DGEoSi4q*y_N1$PJAA9TiU+NS?6q3Qd+iE&t+zRuHg!Ssav#5Y@ za&mg}Hp4W09VC};wh8V}~06PT~iHlPShxLvXU z%k!Sh|EB0`r-sTP=zaAK4M8cp8>LQE%{8|}C+PvpNbrO;Ss8{YgXl&5u@5^pn8^Nf zs7iwWfEN0fpLCb@!&dkBKtghJ5S!3EC|>UXf-O7p(h?b;p`1ZsgD4NV!hQVBczaqF zL}((F?(uW zRcyr|w%FF0^%?-u>!l{5PH;7w99WgLeJO`)(-bMHy&^r`jib4X-9`sHSPLZ!A*DR_On)MGcK~fhktuW-xLE**lelgw#+)%oOVZ1Aohg$?3TQFe8KtG*3B~4=L0Sc@L`hcN@PC|NS$#nDW8e^QVSJ zo{lesB463E;MR6};r-S(R{+bZ=a~P@QU5sHr9__xGT-4EXc(Qdg%TbO(LfVJ?T9Ej z1*{u^*O+;QGNWSD@MLGg<_)0j+qJ!%($}#4u1}FF6Vizx#OlE@L_coaonxHl4~BcBcPh{O)p`_`CJ3HVcd?vEFK8R& zv?YLs@#LRx-*mJJz`_5|m9KdT?K|Y?hrKUL>(K4YGgm@R7{JkH@A+}&{|naO?NQ&f zv#GIt&AMw#h>y&)Zku=~gxy#VWwJfNh9xd9SpE5r&EMV{i&`AS#MuezPu|p&iDG8rtnF(q90fj zUi^z+(gr|e{{Gv}6fgL5k}?pfsnaw6{>2tdrn@g$O~*H1PNVLpw`gG2vB=c7ln*~$ zedYYyi43{hma2ANoK%g}{Q?!Jl||Ztm6oB-Tg;#E+I|kd`ZH5q=lKx*)yJ8tr`}b3 z%W-{F1HQ*mFLm1K7geelu{#cMwd9d!Y_#?zt@>1lj<(v61td?+r~YPR0|^nU4Pi^& zJ?eDv4Fhw235hn0-Ou(1}&==4Er-|fjo&@g}O&ssSK zYrqHsPPi4k$5tqL23MaC(11rnQTmM=UG%1o04GMSL8)JN)|;a9d1R78+d$w;lLHA3><2cpzg{~y!SJ3=c2gS|44MegPy0x0n#f{_WduUdWN6S zi73dr;6nxjHBJMvNS^k!sYxF~!H@T}e}0y^?>tOL^j6co0}_b+0fXqZzHfab9`XAt z6o*w_a~(ytx){37RiNCBJ?LQV%tFp!Ao=!B+`0{w6O5#Ze+~CC>gG{$E}C_tH&6on zTIQhzm_!q#cj!+QW`Q!9FJ8RZxekdWuhCqa&QLfw65zI}?JwMT*Z6D!u17;Z;AhCA zk#3!k0!>3iT#F*zVXMzZpY=Z69xe|-At8O80@2h_bTNaIEYoo-``+y^BP&YHUrLxpYZiQ37NH}VV%*}IK6 z2#L3eB&tSna`P3g{6zjQQ|DC-e3QugwQh_&6VG;6Az*?u1kX@Hh9*bP(Aelp9WU9M zC5`HPS};MSJ=xfqK1ZpAGF}c287K8?PYf5mCj5x5{*Y3im3_;hMkkm4*i*!SBr!nj z4Hz<6tG&mWdO_}ZQKPmfQ`wO4+!jwxn+N33olRLqVPsQl)f-oIF2pbr_n#~z{gyp` z4Hr3Oe#=CT+U=fnt^BRxA-Ol|ro_tQQv4ZWgZpvg($N%L;}F)|AU|}p^kkhMk4K$r z7i&o0JuM)6x&^yy9`S}0zWr=&{)Exz%{B93oABh~HG{^2Uw+T6LM9i#?j}+W8C$EC zT$vgz&CVYPAn*Ucp8dnMj5}Eh?X>QA_<-EBihPj4|Q5EDo`OS;8 zo7_C3&z|J`D4&%U^R=|#B&bUdl7@Vzro-+Tu)6Nc9~tqgni|4|5?g!h*F_4`X5RyN z@xEqpxAXg<)tbybKOC|YC)!ejf$GdcBU|AbL4H^X=))184YPC%pal}QZR6tyO1mO! z|9<^*=eYfrZo$IR$|I~*po6zQgn9J|lJ1~Mc;~0la)wE|-6n1!HES#O!y_YL**5|5 zG*%H;fp{9g`OSbMZRc70y~7kxh43jmIFlQngICkktw}s~#$GOf zJ!~|AaSzP2IuUK|SB`ihkQQDCcm)1nFc?80y6e;LN6uNJ3UvRZ)x6i8Z<>XP?RA5x zW_Tkf&lENBd~AYQyM379beUpjFG5w%-m;;{7fubflJbQsQpAW%Z9o!U+S#p1&P{0K zhla*4SCE|~S4Tv*l2k}3izpOvRg6{75Ry|@yTxS3KMUM+dA$1o!>4KGtuK8>!nLZJ z@2lqNe^O*hdlzS9nuntq-W`13%e2ZTiP}@rHEqQMHGG62NrNucn8{NwB10?q*uH2U zJ?YUAeB0l7im4)vZ)9sM$fMT|TH|!R)C+4;i^>Z_`tjC~40~r$D7VMYJV>+vONvf; ze37mgxXn``x5JN#*(TH8wSZv;Y5W92q=e`6vR+4rA9MPh+ok)vKl|l5u-fy8VlrGk zIRZmXXN~Lo9NnK`MHlCHP+PQ3P6_!4LS+AH(B)=HNRc)jXlS5PJ`JgTl#cI~A9Jme zDw~if3h(ao5G3L_X|XyyCZj>FDe<4A6SI1BuoqRGjOjMPLJ#=9FL`nQF4q+>F(GUz z^Godh-IcprH5T*UX0f!5hBK?3-V6JcypMf4pDZNwld7mM>9#KKTvVKZ6H&cl@XgYn zjA&g!^?d(mSF_|$O3quos8>Vfc=8Nma*6wD`VSDl-XqM)wq5WIB_A<5@e6pVeLZBk zd~Ui)VCaho)+D(vO_BExPUq*fGUe~6c{(bKvIEH&}7H-8HXYt9igKjG{i_cWGhGrvol7d2xtH>*i=8kfJOA{Gw(g(_Nx8o z%-xT+Qe%(Xovx|aK3H41St{!++J?dw;HuRIU0cO3ygHJjaL?i(W4Gy31aOy*n<~g4 zCz5a-NIQ_I@XUO{T3|)YhMokHp0a0Wd87!v#rDyOek-=sU}r}7#e88BzzT*yf< znftUDq#^BMsvl?>J$aWX(*MD{w^Htql#qJ~^$OD|M~msl)9U#VXYNMy%MF$W7dlgp z*resT&-cL{&)QIHYzs@l@)Nxxin zv~*&GFVSr_a>zB0xi9b#e%?w&ps=RuGq>YlY3Euv*f~}o+~e%QDzmI(0>dg z<49fX{#ozokBjtt_(TY!g0Bj7e>l)ISpvf#-QWDYy_maFDc5&OJwHRnT0v%T&9Fln zO%~#`FEt-g+n4T=f_njSRwCPjZ-ZG|LM2Uh?V&prvapzp-ktFtuy3xGsKXw9Xl8I+ zZ_EU<|0(a7w88txgZj4SdoHLR2;loNBM*qt-4mT1zhiyWsbqoOM!lzQH@pACo=Ex_ zwJ>ckC`NaE!AHUzmUzq9I=P_Wr&=A(z?rpE_OsX%!d8AxE?(*`zSizBR*`$a?%sO- zCz%>&yNaAo&IIld-0Nat%<^Wr>a5h3H87#=s3GcgLOkpnkXL;>XgKE z9MSre6lRcm>Rh<+U^_FniT!9iDA1Z^DW?JH$n0v~paY)y!3>CRpzM1A^Epv~3L5jL zDKpH9@)w>K+ul*$4|4Zx)5?YW51uE1AfOedv=`r-`Dx_r^Zz=IXC$9g>f8;cLqJRo zI=z!Nj-P$XckdRUeS!vC!;6&#nBN1~1$!9h9zJEXQ8Gw6_DS!61Vs%EeZY?FbDH}5 zO;8OEhZ(4v$1jALm*2JOjC~6`4<+)z{H<0Pl7-Cf0!h9FeKSa;2Y0=F&c@i4`vJu1 zMSD;|qoh0pcA0_`9qsBD;C=@*RG@8OPbH9UYMPo+8%VF25EQ_}G;Zl1zd#?626Fws z@5VtN$$rlIIiKx+DEg{0LkzjmFrUvo$X|el`AEAW(PJ7o1I*fLny&7EV7-IF7tNn; zNK}5X=HBT_D%oKRb<&3B41AI>g3c83mcSUC0HVFOVBXI%E)C!@w?`EPh)*E*_Ie0$ zbzIB84Jq)_K^jMW+WD3^HzalK;>9N8t^20bd;jyzY>AejtS5fdH)JmeE|YYbBG-${|km z=YN}W*xh3mT`bJX6YYB*XTaeTJ@m`OC$QNvjI%nfo->D&o-0SecYM~yHVGBDT~1lg zcpc&+n`qrdCeOdb?T~w=JYlTN%e2Tn=T_)P#`;T9SVAj5dF5j4&6%sWT;~Y=glFGI zoqtq$y(iu++axR3!JsqgO+^mu&hj?`w&q zm}o-5@HO^;Eaw;YM3mcj4)&<0W6uphuMQyQ!5OeD&e1t)Ij3L$YT0pUPF5 zzM*T9v*i)BtD!_Q?wwQvPXiKOPNCL=Cu!0EIdI(x80I? zSRSu;u!yW`8dghNDQ|X@4tluoG%L4+|J0x*;d-v0;kdw)wQq!VmwSqo8B?LX85;Kjxn+O z?H{EiA(9W0_Tz~2fXpCeCQ>b+ZW5{trSC6azB~-I0Cy*^16rCBJX&X}AXxJ9CaI%H zYz%P;60rJ^6##9Rwv<3LS!%$gcAqR&oW6wsfn8bV<1j9bS=X5o1{?Yg+`?!`BZP9D zN%;^6#S|I}X34d1_0Iq&$9+0#^n!={PpqOg8jDwJAVz~wBJJ1k8bPL_TlMytaAU!{h1CcHh{^|M>y z4(B&#NBLsT#C&d4B)?hq|%OHMG5J+F&FE zXT$Z=Bjb0=ivV0;4|C&G-E;7C`T}G4!ReF}=fG^PND=1rUjWF(9;Tm0+Cfg03(i9w zUxc0JS^RPNa;yRO$*@S_lV`{cl&oG+F4OUKE!iKQv%g1qj1$g;Vxg76S$8SczGJY?0QQ zy8G-||I_cU8yVTfe>wQ&>BN~yNN9S(5xz6wQ$>iv9Ysa^qK7AEUkuCxGQ_bsNZeWi$M)aw;j=a>z z1d&%BEt{m0p3@{#tDn<;SC8Ddfsu8Hld&i%8YhR)x+qOydD+@K>^YR;@Ay4`y6P0l zA@6N!X{nG^tU$|pl`j+Ef}Jz5g!a)_w2WjKDjS(n4L{j{ctD%oI#8gPDvC*MjT$3~ zxe=;T2k+b3ohSIgUfdK^I`uTK^rr2g}993W~-i4z{-?v|mMae>JsR z)9q%t6)GU?O0twy4{j>mA8oh-@jbrAt~VsK%LiTN%0GT>+#fqw#%WpImu}KfZW5Jl z68CH;ncN@nrC+p^Uqn)*ZPo2WFRznhYj%k6**tFInwwL*!IC#b-QlXjO0I%{rQa7F8=dBDbVY5s?#B>SVU{iyD1t6bH0pFb`D^R zf~j$8ZSgX0zi8ePFk<^k@RO-95LXDv$QZ&vHa{62^n5RjX_MZJ%5Gl;|7IWj0)Ncb zpOmuP?y*(->tpdkOd##MCI^q+3iT+O7Ff>!J;myvCs;H$7(Rr}e*Ky`NQ!>>G98tm zNT6Q-JVQ+sqpsc#!Xo!BLz`vV!Iz-CM=u>JCO3-&gbG|ZY^t%8 z2+g)rY4L9m)`FNn5Lewc8QtJUHgzO6WQZJ0+1&T;)+4kDK_{T0%H{FmIImFN=>*v9 zG3RWqo3ObaMer>+7(k-J)K_PAzoMdY$hLlk5m)|v<~&WUTQ!7cOsJ{5k>vM=KjTDZ za`k@Vs&i=Rot%6>^}dsqI3qF-)p63)=5(}#T<47zTP_=r2*2y-yJQ<7Qor(P5f{vw z{@sWkYYRH(!>j}xZlS+PI2w2S9_x1TYL!&-;MV>iZN@j+&i~4J(qhxUF?eCi(zL8;%aQA zI+bT<`DtR==~H4Q&mOC(&j#^Dii@S$^l&FtevAb_9l`stM;+u_zhw#sWHDGu^S3~RPqAZWjBxqWH*GRR!z_! z+!v47Qd_ol&R$A^Sjv{RSvos8j%~c* zX`C2QtoorGf^Or$#+e!7Wf&7621>e@BgE(S{VU}J@ zjv437)JgbXgZ7y7K`oM{rWRAqS1sv1!nBtx4;?s(tI{7oHl7l}vyMwD)qliauCHi_ z`o!YI-HvBP`cl5y5)lc>oLOqkkzUTweoGsttzVpT+ZvQ;4cX{@JRZSHP zdKOnh>keLlIzA{kmpvh$wFi{A*hi2L}btfe3rm+5CgQwt_exam1cCaa`#YU z26yJTaGb-c-W!~w$@B791uJPO7<2bId#Kl~(5IeX%k)q?9-$@5nH zVYdR6m&Da2hj9V?FR_$cY6~gsTG`8^n9kE{V`WWwPlby=vE7<-*4A{h=LnKqpDe4T zh7F`-H_F2x&In7trh#1}t~y3{)?$QDi=|bn>D+rRW0=-BYf`>v?GEqS^9EM`dMm-+ zn2e;SW=@V*Z^3t#1=E4~o>gxoCT*gfg0=QmGa(ah#R+iI zw4orl`j9L$YuYzu=2H?95<9E^Tul%D*0Hw+-U7ldMKV2;K7UqYI~Y41xA9{-LK-MK zhiy;XIeKy8>vLenJM0MA*>(rFZG7vigH>jEXs`&(-~~^gCW3E(E)k@*Lr`hCNv@!| zg7^GRojhiyV;)uk-IPuc;&{Lbr-tWDn-C;J>VgiOb6W-^(Bgl;+4}2?Q=kgy0NXWC zeFshNe&I)onR(E2s`W)@_JY+IY4)|X{rTh+AS)SB>W@09k1ncdSf`|_hZ5%3wgTH_$VGX{+4#H_XcyYbia3F&#=J8QAoVO zQ~DDesDTqV9WDFQREJula&mwsFdUsN&l#+o@W?=u-MlmG;5{W9-HFTH#2*oNYT^AE1 zd`2~ML|j5CboHLwCrbyv=fxofJKY{zH_S)Xdk6B*7K#^i^aW_V=Fcq=iOwZi7(0gN zn5pko{>VgX3DIM<2|Za{mvNajX6QvQhk;u|@vv*p3Wu46Q?0Avn@9XIQEn=%F+^kA zK$(i7?AY%9gL4zJH_QaoUt~2LAP8b|t4GNEiI=0Eg;b9uT}Z0{)^1hqy`>lDkJgd1 zROy91Q?(x(30pqSH4!W=_Jud?m!a3cXb3mieD!dK^~-ISZryqfokhy6+5ik5tQx5= zUa$_h61mvutI(jK2|cNSdx@^@2txz>Z}arr2R>MiAa2bBuJq1yD zClR@cXh6nMAbpI<^U#(P5Zepwzs+&9wC|IP+2yY$B>7^#MDQ?s8rUHZu|0CGf#m|Y zMvC3y09fWl2&;Fw0Bl1ZWF#{I-lMyxzC~$08|u4pWb*YSAe%3gigOw3zfILlW3;Cr zx33#c*<27~=|j5*(K$I#fghUOphDhn7CQaj_C&F*ker+`iom#hIRK3L+Cyt(-?dI&c_E191w@@6scC(#nV@4IY=!HldYaS zeTrPodY3-c(2Tqnr*c3Aa|akqZxCB#ady8^*`x6L{gxFt(9m?6CP-13uCg>MoIpwB zuv`EwSfnL?kJ~O2(^~3m4kRx+_a+7^OW{yR0|mp5N@MhS-`}1h&w3FLA7R!2wkz8Z zg-=1)gKu4(zS$-ks8L_=p+FHq;hc&)_U`UQ=xvA5yC|NeYcUOB2CUZ~th66>KDP{y zLIZF`8LA2Rx^>oVUFb_+6SIuu7`>U}e^7jz5vj?U+r*OMc}s z#q((>&Fr(p^?!Xm!q8%L-QZbYCi^Pm6nFPK9d3{L|-BC?#Tff*vkD_8h zs!Ee4(nXrb0!9=RR6-GyP^1I|q*uErMS^q$6qG6u>0JehG?5lMNDI;h3?1_3ikx!4 zd+vSj`{!j0$2bQOlD*fSYp(fg*_VThns{#SmJ;XJD8qiavdcoPViHTx;&HWo+-$qw z&9a#H+M96cSHg)$vsc<)aUR+-R+47vU#i6`Hq%^cj3NKXuSsMz4xmK)55E%r__SJS zlYZ4STS0OjQNQtX*%|zR&d(X=x3Ux)c zT)uubRzRRV#HfF211^!FB?H&`89o;JXU70e>P=1^s{gxCJJ=vAs!t zC8!W&<;1-AWbF^y-_z5Byd1(lqr1#7nge;v2n-}VkBqeJh?jToFJ)3nnbCv9n`4)#QNsGdyF^7WPTrsUSakV^m~C?HS=vVY{-flN0+ zD}DUu*HcuG#z@I-I|J`8iKw%=PZ${0MmjrM-FQL(xS;n$+hCA9N-Ks}9<3HVggFNylMe1)=~~ zg`Y`1Ll6qoVU+_exR4&UYjhm_hGSkvB2K!lT}~Z6Tf&+`+>W%%02VCdJos z?dZGL!PuZUwyFkKtBjnfyw#gp!dwyfOBY#Ot$P;iQJ zey}39FPwR^CFlN|v|IgAT1;YO>lM>tBB$m}j}GIWnaqc|dUiUeK1LnJDTscE+}x7| zQIa!_gwVCR^^fOmoH}MkzVv0(gxIzF31spF?k*1>A?(Is)(H4XEAfwO*lC=0@?KWQ zh2Ydgh`i5?aypGu_^r&$@UbmZED@te<3IOjAN9A-!FjU7;_q_upSP}revj~^ps|?m zM6Pi}L1Tw#JIl5Vo=;~9q+qXJ?hp~F(*&819m-C`dDHZiuz4x#;+4Y|Y^`r=w4R5Q z59;_&1_^a69V63f3cq|4vsgJ>Yi>Ir&u9TLIJ`U`IA3d06K_&+U9qheBYhsQ zt*~JSidg5duuC2fXh6{#$#J*@Pn3}|ly{Iw#y)Wkb_;+MPeu2I##3qZ)%EhQnAjzl zZj;ey2UJvj_s=-@GS(%%eEGYY3A;nfjpM_I7YDFSW&p!WnbYYnU(?--Rs`N^(=rSF zNNIx{VutroO9>h+b2M+w2GQ+bqoKavO0u6W?A$xLz3~w=h3gCG8sA$dY@F9k+no6`Be!(q&-Z-Uos|Gazah0z&S^Ppwsj$rSj4BqPS%eu+1Sig;kxMm|K^rY~Lt ze7!NVJ3%faLrDlv4t)H0Gb&4PfcY?f?jAw`z;O=5BLNte?9=!DaEOS|w7`B4+~S zs^ckDbeIYcUu280@EG+Q;4Xco{D3U27yN0vLI4F@{ln1C{A`%9C#J2vEHalluetE* zH5;6rr|PSkv}B$$d%{WTp}c3_@Q~e7=c@V0(o+#(4T>|;V*G`t2c)FQMLjcvNy3$c zehov9RQ8a0Q*!WqZj63To_mNfLtY^zpF}7;gc(0JZMUm9)|XiP#*-$P1;7c=A%-Zb zIYNTskoWmS)mEzS^{FKH?mlpb==J7Da&y7MbA6`8;N*ceY+zPl2SQPY^I;Pgl5Xu` zV@n2#lrOui1`OW<+VSeSzn9h@Xs!lJpnM=RQGuYB2vXY*kR%H)_MaaM7mL@n>R*7O z1!h8H4bEVck^-!yCZg8O!?3i=5o;f8t%pGLe+7#Tg`NZr^2|U^@?o5nzQi}bgJ(y~ zalAt?Yibcx%=|4cd#0T{}Q8lZ42CgFa?S+@cplXqjx$z5BnQKAk_B$ej`X;k!iWMCh6ug zh<)OOA&}Pzmz3tga+d?Iv_}keXf@>vbDucYbp(b(#gi4kL@2yLIYuhj7fKr|8RE0+ zhxJZ5dk&Lts1Z!;I^M}n%P)8!yCSCag|lJ*HKXDy$BIrpByvVrTNNDmz@8z9IpRM* zoyVumSnlFcIG;C_KjyjD^$?7neQm4ja`L<>-@hJJ&3wZ%W@h0xbK$)8jTBQZLyzS{ zTdV6507f&@e{|&CS9Y232J;*u=OxQ71Hwn)of+B3wA$_10xh@-vNvB1A-U~yck*^- ze;Cp>kuRF@Gx^vTc@N9Uv}_W8YoA_GPcU^YPPnf40VSFZLwLQ7zkX?&JuBY)v<=4O zjYsZ}Ll<(d>%NcGjxQ!ZoK-m5Te%urq;b+@%|&$4-`}yO`m38!)#JiqT+W-IFEN<+ z+pY~XsbBtV(FA|4=!GY%=v!;jwjJWJiTHINHVdqNtJkxScs{swI>m2-+Zv3EQe>2f>{~S?RIeM@%q9NUgwve$P}0x;GX(W35V@;?umSO zRffkrdv@6zvxaCk9WlRQe(dJ-vFCBzT3q-PYgzNB#ID0tk>fi|FpdFn2}P9MgxpSw z-JVZj4H9~Ovc%H_y{C*mSvG3oeKhxdysCPL z!*>R|YlBW$AY)QsXMLpC(J>`ox8t=XM~M8s9b%P{IzOr|<(0d@byz;2wv#NKol_-2 zE>*4ee8{V?$rmCv@6KTau_)rt%7jvhs2HrN@^}^c2sxz2)1oMfJ5}mbKb*(p^_Su|M<+g_p z5iBU>)y}8oRZSJH{ZM8^?}-eD$@x+n4Cb0#&Oc3| zl;@N1Y8~kcTGJ$c?RJ*A>Ww&s!WQ{+x_F6M0htd`UjwW$6E~^47C1=Sv3v4Vt7xPwr`?h+8W0Y^`tK3+VON|}8p_PpOBuHv86}sLm%6as)Y;6p> zj0^|jj-nL{uaj`0>HlHgKjkto72fQu{E%ndmqT-Lzlv@LHt@WK{v(-fJhQ&~B z-FWJ9olELuH0PLRdv-zthCFwr`@;Tr?bs6cORuP*i;u$k1o=gdl9ak`2hQ8XsY$3P zovVMrT@&ETSm6HDheNT6y;}g&6CC;_;gDUNS%#%uyKDFv?e-S?v%$obx;-S;k>(B} z30pE{H9f+EF}LnEBGi46cPP3sQ$@i|;e4pq@YRFnlT`=HU2a73k~L=zYpWZD5xb5h zCld8dS`7$=!T6rwX8Gz}GCI374e_N~pM0z9ULPSv)eJM{$opwb_Z_w}@+-0rIjz(F zxXr?&hNJ2j@zbaA3rA_i&*C;$iDf_Px~bmOy&r}+oDo$v{kFqOHF0|Xw-RFO{DL7q z>{bNh=xTY9WHDG-NV&Q+6ItlT-*WBub>*wi%7^c*J%ZHHTR~6GP2Zk)#Br~aL-Hc3 z;a(dpI!r65UCN_T?6~Eg-#3tPd<$YjWdorQ$#CZryd!GGJem-g5WhcKN;dc)_#P66DzF8Uc1f zrp$*WR^?!wdOLY2-F%^%UOWOR)k{|sdCDVwcA4x~$}Zc5X=AgtQsS$HUC~?M39*qk zNXJ95U%ce;hKD>sgm>=C9gYzghKBR7s+z9<+VdE8X#86_&iWP<`Tlj-**lh7Up^H2 zRf|uYy&?W<{(676l>{u~gQkGpy=^b|bjeg74F7SXeR)jnft+#}^TE;9T@H?-k7dXc z@W^`{HH(t{y*C|4^c9M6&{Vg%igJPuF5JPMlLO+kT?P@)Yl05WbnM%-KKff z$5QMF!BAu@sCh%IOE=jUO{ZBC3){%mCW7d(T*I8u~lKvwpGy5so zBDGf<^;^?>7prjuDfR8xuwPn_+MW=3B* zQB&CEZg9tvDgE1jXfRH^N8K1egxa$VvvD4|Yj0tG#yL}Nmb)IQoSoOx>MCl-TJJaU zOOCvSze7}{3fL7I*+-S+fT2gLJ0quC5g1sh*z-=(4>cFVJVl({bwrnH;**h`VJ z`I2d7QAB-9@3qS+NzR@ab0crNEcd0B#$D>+&1H&vcRBE~j6mZw8eFzFqBbMLJInWD zZc({QpRiqFstBij?Lo>hJxrVP+*O={V#8A8ZtaE3dvExM;xpsDiTWL%*||oi@}V#N0B&u=PJ~Jk>2b&WSiXJ(Hb0N#HB)t`xV7?*8X7X5!`{!WTj|Z?EbNI_Q!1J7$RX+*V$RPy_1lo8zr2`FCa#cu zAMGDG71d&yGI6|7F||KbQ{kgLrS_6zjzn~F#df=1U!IDN`!jr4{b|E#3r^kZY8nA? zubk}GK551m*BpBHS$=Qjt8;Jhx0j8);b>vP`4q;B05?Rm2lHraxEp4SG5&t#dC40; zne-eF*guj!{Fh7CZ!2%R;b;vHy$Tr9doSp&{`COxEoSDs7x03-xj4)fC_NeCHEbWN zcW%wOW~_F`DIvU!u=r?z6#&3#(@PSAQrVFeS91z*&+|$G2g_JTG^54&123EQ7ZvzW z=Pd06ONIPfnT@PMuY08G7i9SotyI51uFY23ZXGJSvv;02_LdZ?Cu#i1J_3h%F*kvK z#4=JEZzV;pbsd$lU^}Av+h>8aG-3qi{5@VJjWKfRA_gi)@hL}DhB7+;kuoOZ=; zAT*&cDNO})si)YQy|y^bCEK#yllm!}R;d~em++4wb}X-E^NS9X`%fyT><2@N?xL-> z0|pxCb#lwE;!0nebqr2L`WK?)wPC)05ghMe@)-lO(G9<*>_0Y<*ycpv->vZ3cpXqT zgz#QX)k{R$fZ1-(X5)}+dw<~(;IukI2_xw70WAY7N7! zTe+{p8+Um+f7iC#^10t`riE#XBU@EqN+RzHPE~{$235d5LgeQwGp9`AICh3M*dlMi zrmncxI>ogS%>9rOk;V5|+M9`? zE9NjXFdS?E zW@lfT6}=z0b)u)zM+cdA0C*2EI4{8?rZJZWMt`v|;kya%0(qkyI&=sbe*lg2A~>lc z;&w87Afy{S;_{%o*@N&N*Tw9 z{JknGef+o%40)Qs{<#Sz*&_OdElfr9VdS^>AB`SH{~O2viDz_>fgP}fVK8g^bmf=% z{x_zSupSjKQfte#iRBTHm2J(f1vD}qq=O@LR3SJBs0_p57O9CL9%~uAmL(>O_LeW4 zGM$(jl6Nd@Y8=cay$;1)jE@h;9Zbf1K9>!y-&=Ehl687T({!)NxJdI zCr2ft%@QANmSSKZBD;Nh)=w^F(7dB;h9zQu@}&m>Q+e(pyfdX>yBO)V zbWOT{@>S%Wge|R5;7dlP$h~fKTTm~>@UMC;YCr(b5dn1`&=Hi37 zUA&w$GBEh!W(sXN!{3K`xXK~|=dBqunl|62xXy=a#pPj=vQH{orP=|N;jWu$vf?vk zj=@@qL8EPX6Ko`*4O3($lKjjXZy|}L;l&BsoJtG;ZsqRa5W|f@;lw;+Pj7I@SVdk< zUE(f;QVUt8^^^aHxQq*#zHB-eaKas$F06-6!0&U>yLa9J; zPGFOG8T>kKe}DOC7;yR&Vo^s&r9|(gOP8KJ5wE?ER`dzHld5X}l9{iP`pe}?rUTee z__0Nx=OHqcq*er_oHSGhIR$^}bsa;ye=ySAK z;~yHlIrW9((j<*nJ8WU*q3XRB?ifKniKPS>V(#fip~qrk#7f2a`Nh2jKmcy?$l1rm zWnlKJL7&mWo{rl+N!pY#UPO8D=+1RR_`S$XCr|H)tY0k9#M$^IQ1}$)_)H6KU8e%n z1AeAxW3>v1##So~6*tO3O=Xg&(DMS%P+5^f2RjNq7B9%m2XBXAtT@c5YDzO^X_fW1 ze*RToo#D8YlEu$3;`))O>r0I@&|w=P^fS(iTse58S$?l=c$=nnmzrz` zwgie~xy#eHl*=zV%-ho8WONh24~h6J;RREut7^{fS;S&XtRrr@x>bpFL>nXb7BAU) zW2M!*%|eZT5=vo-`Vm@0mam6sniQEkA`Cb+@{gJ(??+h2elPsF&aPcNhvO<`CJ!Ic zTYiO01#_bgb|`hPhVv(c?dohsqu$D|G zetS*)BVWX4aRo4ORzMt53#ZPVIeq$jW`o++Th>fR&YR4&K8A`UqMLu)EC>&1ITX6+ z=fL$uK3HF`(F^9>f7rpU8bgX+a3TPTh$w<&;fanDU0b&HjXAjYwo)(M=qavBE!3ZW zZ62%Qe-$d6q;*%}xgkCRvL13S()Z<3Dy}ztiI&jUAI^7w?2&))-~s4$1HmK3B?}B% z5G(ZDxhLR}tYub$Y7G}{?h*P3i2wil@Ysh(6~A#Af#WUMgA?KgszN|TpHis^t}Wqm zj;Ro!1G&ecPks`Bn~H4BO<&jt^QlFC4_QH0slf0(ci{rHY99Umn2yv3)@G%9w9NWi zbnkWs#XJThF;G+rHUR)13nm9zNf#oI(PfQLmDjZv-Qx`0pY?6hsSz>)vaGv7jXb97 zWd`X8VEXC{`ujq`MJCufAb{;6D6℘|tqL0Ti8pl2y$hk;JxOIpc)7j!rnTVTEH> zNhA|`CzVo46YTabE7}~BEwO;BycN-YcN(r;QivTmwAX?&!iy+pO4yS&;PYqR>N&{) zU=tD`A@4Fken!9)COJ2WiwYhDI=2csb&A>w1v4%H6+;Rxz6)~t{t=RB8X73V=hi}Y zsfIa?z%Hg2MOE06u@{tL$7&VE0zE}8JDQ$dZ#iCTOp46qLXI>M@6mC!>}!m##O(dUvJQ9Rwl4zOwD=n3i(3@gO-*>^*qHUxJ0O+W-q2K~vNJ$r_LR?{ri ztcUARl913u8QmyS0@NdbKVu|Js-I;}tTXW^#Uk+?%Bk41bLUG0-9a+-7AB@tFpv>a~JO8+JgR&l$K)J^()oMx+6Rr$d?(4CeoV-!bjS z15yL`z!oIIhn9o~6CY{&1Zl<~8|ECO(tEhX%?YiZFb@iVpcZnuzsCA2CV^b|N`W6A~BSXJnRiov~Da^;C|Fjg4&rGcOc&4uuv8_jt#2B`#?MyOXWM8P|8io3dpAEBbMQjl#^ggrNS1>TUY8DM!ZzA+M z=<)MB?k^Zxz4-yA*fOiWOmJ0T7*r#?lnvyS8$f+ps9rf8xO zgj(r<%(dLK2dlT0y)%7)WZVLeW-dxC+w%*&)39p-*l$o76ppxtWwFh^|wT(|BY8$ju8an&Q>e}P>e)37k8}+ZL+mCx7fEGjjpF8qUneBE zYt;eWvQd@mo5AWq_wIP6^u8qZk(YaD2K}wcizRs-qJw_k-idR?f3-GjBO%h4N>+!9 zjdy9x$}7V1!smN#cmp%Oo9s6lr-rC4Yw+ICN9Z;G{5s*aE+nF$4S8Uv10c zUP>qUN!mnDEdZZ@E#2egI9iaX)dL|cto3`Hh$YOhDfF6k@#}4|sg30}efK(pmV;sE z1-+Ci0X79|Ani=XK}y7$k$uB{?dMNob#!z-Jt=^p+MJbZd>&EJ__6HNbkCXK!dmmC z5&LPurIp=ak9Gzsw&!d>$z1BD%8I}60gZa8O#8+=WMzD@><|w?8C`pv%%(97^ zmvtUW4K82!#MIV)>tl7heo-g;2A7pgk@Ecb@JM}KHb351nLKUmLAo!#FR|d3i$?sQ zpZBDVa*t#u6xEe4&>L<$xp6Ua1kJ?5Dmf!3(OFn|Zx}ZgUz^7>mQ^+`-C9?Tomegx zjBcM>>G(j#OAR}QOHSK>AP{*d6I5)6C!_;ytHiq+_U)gj7#ZwG8PmwV5Jee1Wa82{ z(*=ee1PJjXA2}b1FFz~_61eC{8Uw@4qiR<6!B%5S)}3x0v5Fj8SnOfUluJ$Ef&bcZ z+{|6Q$@eo;QiMTsaGrq9dY}-#(#j9S4#a2E*N+0*56u950wl~q?E7~xBq zKW3~tKi2moKjd_lA11$zXv?uqgCc|>{BPtdJUQ*x;!QW+A%)Y2US7}o9SX8ssOHgr zLy+Zzm>lW~Cu3IE`^}@l#)WC3;wyu*8&23-jRI8uOppA z5)Tp-0$vGsxFx!qw4cW*RO3nlgV(X>=R2~nmhhseXbUs$imCLF0L3->Zi zB$-5bDkfQ!&alJ^r!|C}a=Dtg9AGkKxugDkYA=X)a(|uAE^bW;yYuj#)PKNLAiM6f zO`&b%`_6@)bcVx!n?~L;R;Hs4|4N|oUn;D* z{UcSia&>LWB_YkMBiTG5giu3VQJt0D$0s}kG7|B7TkFqn+2!7DK9FSA0+b6fLstV7SuyJ*(+ZgLu)zDZl zU?w=Vdw5(*@6L9KzZU9hzbIu#C+u)u|Bs_wc7}21EF#fkptImDsYAA&y+^x+*ogH% zGPSHuvJs#h(+ctDO1!eK#HDn-o!)B^pn`5%V$K-0rWK9tz;(j3#nX*IRJ2@}fHAex z%qCosWE*>t_FiMotpUEVDflwuuxLlpRJbGUn;ovm%w?+u+a;SiGM=qqzfD(?y&Z4^m{`B#=d5yc;Yo%WDx*UVJiY#$@}i{t zu`dr8%NL6FQP@VZ<2I}3R<)Hc+p+B*ar6<7k31j6Y@rZRGdEIW#s|}OrQaQ8ge1v> zD^*dJHzayrj+N)jRChS&r79f6IcCV?+~FBiqz?6Nl1OPUw3=RE;8g9TP#N)W+ZReQ z$yft7B(nv6z!$)!T+R}3n2N_={Kd%K1>+zgL_Hne?Up@1Zu$9qvM<6)EFzjaRwP<( z`pX{t%xvZ1w!s^w4Nay$iP?43K;H8Ff@*w9{5JU}tWr;GZ3_PMm=u3)Oa1w)OR-v? zW{RpEe948Oz*ZX_*lrIt7& z8Q5sXNXCB45SkgI0vpiIp-WV1kJ`V9l5i?OU4eDFxa3L$f@zJzULSzkIn zQaG=y(6di?am1x^1eVe3Qdqm$yPw?(Ssr`VC*F*+tHx!j<4&(Yw+DOy~c0icU8^}w%?uZ6S@7EmuwZ$&(Ww>C-cu`V#wT)vqi5_BC2WHkdFn!_W;eil1 zdZ{>(HC|lJ?kk9yt(8(q(s#4~-Q-BX!+m##)e#7qU&Cxw+@TPUDJHH^E^zoz0WyA< zJ?T){Q@)^`;UT{IytVx7ev9wt4JF+ZdDCDM!R2@!#6a6%YYbJuzw@yVAuY`(i9Eba z*eIXVg{`%=;w;E^Y8*DSd~tMITK9QM4I`JQdFM>ikh}YFt!pi9v#-adge3TYJ#eNV zcDaG5b9id^P?t4V|6e;MpuVds1UMZgsf|yr2@M#pxp?{B>YA#d15o4)XBnzx7rqD?o=~#&DmDuCijRblzawN6W%4k25aBk+VpP3_o_*MUiXHPjy zd6>nkYtuw7_4}B5raM*}A9)lKO6uhgq19j~lpJ4VG3uIQpyMFUR((s}>HfCerLd$L zYasRAD7x)CEGO5+!SHq^k4ZzE`vuxcQ)d6!E&a#$uDAZ^_h-TXA7l*FGz6&l0a2#E zc5IFGp9|qHgbjdASTp9Y&924~SAL9QN5uLxwzwrJ1(>UeVD?+9$p*wVno)3;Lqh6G zjb&IR&$hi5Y`%6i{*&{r5&K7+Cn==n!?i1V^TT}sr2Ql7qFb*kPzp&N66Ae;ch2#d8?_4d z|3d%&iM}Fb+aJWe$a;WU$9cW+Ax*Iq+7o01$;^NBIKx_CM zZ?|lhBAdlS6|;lIVVTG)4g9nDj)%UQ7j~(*FMbf63h$bgROX1~YiFVi71CEKT&$U1dav>`|Bd`}%&SwWIp^kO zO#WCB97u5UES7DS-RC?v} zERG54_+vMU)3kqd*{kdyq0cV;Z?X^Bv2k6qNl{;np6SggU+7h(joY=muN0KWFZVfy zbL;X76l6K;iLjA7u>gpX$`@n?UyN%;5Bo?mOCI=-OP?WTFS`J2zd`Nms#iPlzq3i; z_*a@GcaRU<`XQeTY|byaxw)kmK%w2fi#bg!K2ly@4hFnIjuX;kkajTcMO%>9p5)9RMx`swQYmZ-nmbVc^Du#Y%dsUSVMdttZmuiI4r z;zg=PX8Y{VuzbO(=ifhUtDP#|r2G#b8EFRKm9QagZG_3L&)GZ7|KTJ5BG8P6FgEYS z<|xcwennKA>be%p?WQg+`@h-Y^|ZH@g%lhkEXI8~9m9I%zr3baz;&o!3^(e3Z4#)0W+zoy7bsyAs?8{^2eEQTCK<*;OD> z*)F}n<^*oF%as6%K-z(6mm(A{hlY*0xyQD#96bTPM@jFx!E6Lb0eoN{6H@g$J^d&k zKpS3?HZkpGpu=_V5h(K?#Jrc=tYLP6dP49=jLF)4>IvC<#Un5&#o93we) zQmq_gId+{#4>n^(Gvd7NKga{BZ>9I;oH%BK0qF_}a(B_nXBX@CH~EdbB0hzLYo2hk zFV9fI=D6QBeg{W+3E8nv*1q=q)hiVxI8MHWDMHlhxm1bkhMI7qT3cBBoZD|pX&`q; zu`z(B@L*v0tpS)0crN*!TLj0EQ{exP(l77T)!f^gL!a0Jh5a-0kF@KM5SMPjSv3$1T%v_!3u3#S&kYFPA{-7$PiyUjfKY7eLuY4?6%U2CITg z=42?ioX{g|0Oe0bS-7BcZ~^~6pT+H9PYZfX0GQTjnKr^Nxtju6)F8pV0;;~II1KX( z0E)m5&WH zVml23VI~T5;HF0R(M_+k?|p{}v}VYWYmy#IPMCVgHE06a`i0P=zk|B)B^Y<#Jutfi z9HMAGsscyJrf;OB$=3M--B{7BjwyI~S&^<;zxnToX_(J!GTyv3FnqB?+08+(My}~2 z)Ol=`O(QiyOrg~YSqoSWc-W$@9C-i{lms5?a8s4%3$#yr-D?5XlEI`qNI-oT zpQ|gn!LldBfff>>lK@2}A(V+85D2s#hzY^koOg38%(iTOg(7z#3mxKy?<)JSyo3b% zmz-SNc&sE6FM%HnB&EIe@$u0~^J|Y+=R!*ngnJsNPv1v!G|+nr!v2qPnzKlv@QYy0 z))Pq!69h%HSF~7Xfml9SJWa*e^SCFmx!B=Bngam4lgkFpRfqR+l}rR@fkf{dT|W<| zg_m8s)LtAngHPo#%z_Vh>YwWvhywA`KGgaR$T!I@ zR``1b)E#m!DML9QAL0BXDMT)<$(Yr68 zud#cu$10@XVR>lD!!r5bNE;raI}=%1;r*!(zdLRP$*HbtP{-?K2O~8vRn^uuBO^Mc z7ieGdy7w6@&ophCmS@J=VW~80mmHNm8rH*aug#%FK$hV$T~4RKZ5IV(Q);8r_USwQ z9&*iLzry|n46sTG>s1kJN4?MiFJ z^WY+kK1nG@KZx!BL4*ktIS-AHCL8enFan^s!X{GC0R6WwjK+8Yaediq0XwmW6A${G z+f6_#nfOz&)Pc?2eW zL{tkD#X;`7n3<7TBI444ThX40+WPu>h|t=N+=+oBV0dEDH@{DsS&6ZHI2XF=JvoR5 zexI;ExRDTH1xKIi%=pL-)DC#snlIB{NWwT`$sv{29-y&!&GX1)}9=c%(T}h)g=c?vEO>Fh*eg83hzXuAg!_?QmR!z3Vt$vTTKOaX> z{feZo+A#BIYRnqoMQz%Kx9|9Ovo+*i`wL+Zg2Csuxqa75jg&>C%9?B%>gzK>C=6u! zxv@y}n-5_A1()8Ca92vg{QYn){M8-JYv$f*%H6?uP+w3Me<)Zjf*EJ0fxXRae(uf6 z?IYRHzvDYi#NCFk;@nr(qvKy!{^ZeU|H-C5wyVoUIe24Li$vw4nySOE==539FR}vf z71k_0t?*$@KYj$giL(SP+qKRkn{vY51Hy>D!T>z=U$pcS_X2{Klk06gc*FNJ8LsxM z+`z1`8K(8FSTSOTgP+A2S|@VO5A>~`y-u?H^6nb0GKkf_^YA`8_5s5Mx`KP1bTiyd zF~0-_kcoK>gDH;hXP|o1XCjhL0)5k9G8IXQW^zo;2Ycr)nj8jwTFKK3_p?3Zv<|c! z!fsN`07%RImVNy481Nz{Uw`n`RxmD>m9iO{@ARQ|S*M7TLzu>H;7ikXruZ?pNHUYF zPe{=PN$|P;fr#(R?%e`Od{nVIA9JFe4nP3a`{*FgoAiNfTpXN_6Yl@p9r0x+ag!2hl7UJ& zn_B4e0r+QsV{`Y~vXdAc_`gN`HAdTT0{!U5f^}!jlZXHFqn)U~ z;y9LV4Q9uDpJA(uv$1>*L^#&N<@h<@dFD9aDfy4 zueX>4@Pn>Wab?w4t@-^^$JL5*9(?`dCww?|Ff~Z+vBhh{G?j)Nk3FN~F?V|ZoYNpV zj6F0Ie?t=-;aZ;5v7b!lo&XDklB0=v>iG3?9@LI7fLyJ+8b8EOFLB!k+PAw3P!g`_ z8VRwc0%`He&Xra>1c>gm@iE<%^%b;41{_iZ9GE|T1R{NNk+ z_^+{J4PBUc!uPSsnNzDEE}%!ho}C?czeHz))uWOks*7Il|2mipZ4g(y+kXDyw!cPL z|Eh3=-yiF4#&-w*U!W1M9jDg|t*woHUrtx~{liuLLlNYAR^oT`?*98=ovx0*C98H| zko=7a@_NsmJ$n%dUi9ip=V@kUGsk^ zM5_-wEBh{0hReXxM62V3FZ%|*aH{_dR$`sQUlsM&oD|)YU-EQ#thLtEm~8tdBC=WG zuSwA_OzKBPtGZRDmFwU`Zg}GioBp7_N!b?a%O27C)&}j(<$LYtzaD_zZv^bgFF{^e zdcoawyK%o!eZ?Lwl+B%V$N8+;EOq43Q0lY5KRqiB)5XJ+PuER~y}WV5=Vh^2`Ne1! zg9PhhKQSu_+1$8ad=;58{=Y{GzIBdK|gnS?OZ{a~d7d#b%R|@R`Xh>z4ZOi}ix7B}?AIU!9 z-*9it5#CW}w^(pU&II)cOPEbVP%1W z_zW!UvhnS3@A$PCM5os>tXKE&CO{9vOXr6*)nL~Gwj&_KZ~_O3m^-)F zkum|LZ+!f?wqVrI$cTq||54PicrgeaKK%8XW9Msh=zwPW_)uJ+B(D57dz5fgT(UIv z)T(6J0jS7`qBaa_jQ6x>r*9^W9n%HAAaW63o>%_}2|wB}%(00n@Y&H@H9@qERrW=; zdiD@-w>~wNVLNt7>i#oGM~wmhUu2VoT*|+|+zH>3{<2;X5`iGZE@6e5<~f8KSeRHK zs`Q3L(q^fw{Wt~JndemlNmIb0^FKgkec1~m5KTBcbGnX-p+qH}cShc7Z~`_qHPvdf z2BI_K8mTEzFBbuq7-lsx%=;6v+Wa!$w29;bc9iNmumpx^R#7vsmH99vc~Tiq(#nHF zgsy1uoZ)pj#&2HbYhyNJ(6|Q8A|RplJSbdHq7(W(aHMiXAAn~F95$Pw)j?7eOg*rT zZ-cR50(?BwxZi7Wy?-~nyaiatjeoiZUfM(Y(>1WgPVzPoonIghDLBxm5|A|=P-;>1 z*}{)fCKOV8mVn|?Q225_c(dD=Zj)J`QMLK^EDlpShtZ2L9Y9%`*B$m9e3RXmtOt(7 zC`*9sw(gsjTj9NDnk!7-i)Oo$(SQLCU@nj~odt<4?m!FOvj}w7^!nvu;Q2P~RAGM! zXT}Y&?VAt6^@bmrjDn%zffM9%K%u@aH_> zXmhy{c;g8$?&x$Le6ePXYx{5TCe`I9@59;C-v0a7-(@;(-Sb$QBzes+zf6R8+YyHe z%*|Tw)h!lw=eRc9LKx!hex7Zy$aE8qebkRzMH6y!vGv968xgbZ1;l3G84(^%dFf9C z_qogYhfWbt$oKc}H$ektb7$!AuyAN!G8)b76&G3^&8XHF3ss;4b?J7w^UVBemvgK- z8w;K7V7c213Jd?{6n#l%n|mZi;~$2fG1UFVj1MMxG4N&@662$ifWx~n#gk$C_VXt1 z`B8}JMmT{%yu}C_w>r@I(#Ta0D7a}5O_B&^*x|5nJ!W5K)4bF`M*;yAZ;=2FWkWP3 zTmm-eqeieVTOMCGtbW*+h>XhaAXfac6&E%BppR%Es9QiC^*a)D@M2*3LB8{Gm^oAJ`xz@RVfK!rzpoBa=&tRgDVc2HGwH?Qi$Sp+JpMJ zHX`nk;^JtAV9pB$FQs&ojpn@nYn#6r3)*>_R}|3o;V_COifhqy7R7F+7R~6CWN)Q0 zeiEC}g30Swh|B12cIe^S=In;fr%hT0NNXizK48t0mPcU)rN!B{SvjZ!H(Z7jLOH( zGRrtHUtnGNESi{_%7N>W8hACq4yJ3?4hEckXKi6H+?P`f(H?RT)L{d1xDdg15J2?k z!HW0-a;?F$wlIS0c^(ur{}vCI)Q@NjNsTt!W&59<^Ix}3#oa#*hBk79q-8YcB^JTG z@0$PFpf1#e9y}F7hEWCK92KbCWdZ#VQ~mO)Kii!>=6%%^c%B`>rBpB3edn>4(#7a~ z2xHS@&aq`HuJcNR?mVm4HczmaHZm~l2`l7pnzdd*yb$gAU~0j*1}uUzMKCWbe_W;H z3D4ArmM%n82fri1cGEABi4er5iiYJL5IvsKjW!Kjid!DEQ`*a0sC#R+!oldTs{_<0gtqR7ZF&f}R z7v(;qjgl=~#A?}O-~9>_*sGJ9RyrNK--0of0H`hqmqGT1{u*|kG`oVEEUcBsbU})V z;w+)kyApW7@f2Y%Al)$jD12Y)`6P4;x*Xu?9aQ|Wv=s6i61e@2-4&{47?nw__A)@% zc29Q>zMY(_KF@U)_M>Yr+68Evem)UO{R$j@P&&8`Lta=Ts1^=k^5-DC7oJGe3+!QL zfz0^EGyASx9(360hCZ7vlrWU6M7+p-QtLoDT3)vy8#58=IJhBNy~Z~icorEN8uDP^ zCizWGPXnrd7I~MRQ&(?B-oQesxv$j5UiI8O+$XH4s1F*tI_L$EmnYKjfI4eEbH>5c zmp-Bl3D!`Cf6cOs@a08IQ4=LyTlICbmN$W_l&cA)!2wD7dnnGY4#?tCkowl?)aPq& zzHi&9Q>?fFw7o&C%}~R?yunyDcN~--La#h5!0k5#8C@r8%H7O`OFIHeD=-JU7C#-^ zV}7`=sy|IjdhPoa;NW>=!6fd<1bDB*Iv_Fb1ulJ}Z`;Z`*8Tev0X~3y;+P)uv$&KF z@J}3W+|C+x!bjB01~u1fZvpM+x4EWhCF07o%O20G`u@JY+s;Pcx(+?HqFC{c%{~&1 z{N?++?rjL%|05YGyWFCqqk-yP_(hPMu(h0Jn7ME3o%hF1xf{BzIfjMfZ3&9}7PS2e z+ldu?o5v`Tv@ylt*sY`V27UAT^8M(ay#H{w;L zpr3Yd2a9QfI$YgNTlpluIYKVOGaiv^`I(j3K0d#JG(;VNsgTCH^LuI!!~gsz>)@Yc z{BqhGk+R|fTn_w>U&6Y*qH{_2Zxh_W-?t0>>u2!d+4SQBz-N&t---|kd#W5Dx#i2C zA)?8`9Q;edrPxU5EJ!x&rciv;p!Kr@CpK9~%DOWueR4CLTf8S9oVXr|E+4~nu8_WC zY8Op4ckGhq^jDLgJdK)64L)bsLQuLU>U%21kl-0D5_p;g0f_`01A@Wh021cTY3KMn z&b|I$uN{Nn>cRIf+7cY;|2*%mqO1SEzqIDk)vIry?-pz6OfZ7vcbA?A`|hGC;+FUVh}}>eHlEfs;a4>q49|Z zYgm7xaM};r=4=Dz{e(tPULh?btD;*jxB$z?gnbk*3=2VGLZd-vBPfegQIs;o+t9N&5|q&Cdb76D%3Q3nAA2G?WP$(R za~^OQPDT!^K0ePgCwwQMxd(seSZJV!;XnfT;M=>cV>T^*iTJ+p@R?~x2qJ0{|+7GS4RHGn5{>0W4>&?wDBa|yW0S>jG*96 z=)_Dr2I0y!fjt0o3qZ%fLAVHpiARcl)-Njp>=LH$Iwpm946ywYnn$Ron1tlOGqefR z2S{Lt0&cr&LD{*4rXcTs=LRyy|EcX-qnf<3Xs2U!tSxq&Qmi~HC;|nmqC6C|DwW8R zhd^;5kP0s4p&&vS1Ov2XWEC`M1a(A2AYe#@pu7YOxNr!H2#5(35Ynnu5(E)>2q9+n zNoUQ>kNG=Y{_2V`-~H}Acb{|i*?YBkg1c;eI8P{Q@W=h^R$jlKUNVA_Pm4j!$!Ha> zmiHf<%Aa9n0V#c74T@z@eOEin@18Mb;TR}SZ`1L@iT@Nsh=gT;WG1&oX}${WG(nn3 z6ZsRv$Nm7RQ=)X*^D7Va?)D30W#dl5^ZfOL`i-n{d}auS?+fGX1e55p@Ilcycqv+v zloz4sDymbB)=#2&3}Mt`n~Z2)4zYuV|lm#KZW7g+u* zGcm{P9gR7km2xoUuOfkLESiy{FbV)2pGupMOqozFmr&6Jt_H+K`NVQ7_GG# z85uO1OC!w6M^`h+(62wTeiyaLgAx=g@DMxDn}jO3Csy9l`rY3#5r7Yr20Nvt86~(P zG4@3ne0{i-pRNW1E)3x~d}DLo4S!IAs#6r_SXMV;kEU}t=K!1lh9n%rdX0yYnI)ee z)3nh1<#a6_R?0=8qwtX8H+?TA^fU2EAtIp{1~lcbHncWNBxEW|xB_@x9T$X{co6;R z1u17tcQ|q{+9ZM#Oq<8`kxYUT9Gly;-ckp{*AC zAs6b@>3!&x<)T9eknX_2?SztK46-x)5XRY+z6uRe76XQ^s{irvHxKZ8$Z6>YnG~|f z^u^rCN!%ABb%C(D={?MG7s)#e|McU6#c^g1QR2yOwGyFUQswkWCsx1Yt+SPt@QI2R zgJQW+(%WCtg?45NkY>6OfQXqxTNx|v`1IsDjDIg>ba{I?6;UKeGmN{yV=p=dMnq-n zRxS#tl$HK#BHl-;x#+*d*!b1?s+F`q(5+Qe^#loTSX)`S!2s?G7O!4DGABcjHD{1J z(_GT%mQ~F1SeR9QJ&Hq6PE@ZWomedVN&w_1D&j-n^`l!JnH@sZBBabi$Q`cn5s2Nm zI3!mQg8lkvik{1(6uRIuDE32EcK|$u*0#25l^T>JgkUZB&NcRrl-)5EUc){krV`2n zz7`T$v;n-JT)_q!Osd(p`|=WRqx1UVNa#LcWHTX`*jb*sfEa=E{Nv){nx|#C7Gxwe04=Xrrv=W;ZI1YLL7f-?XhGRs2;jB5G7m z<%~9Jk|?hLH!C)`gFX=f0d!v_;D#TQU4#2C0lErNF}-l-Fm7%r|m=w7t6XVx<*Vi&9Cfj#Fh zQjs!CRICGY#B3O|Lw8Bnz|MS$&`puO!coga4IPN*AGDD;psSu#N(A0 zHm|N+wzIXzB@Bh3(}@6Ehv)~SZW8tEy}qY#M41ZV1}%c1*Gy(AN};IiMhB_jpx8tE zo9GtJ=5O(wHTV5l}bBLM;CymLW!b~L&wl@yUrvPA}L#H=yMW(oeG>o*$F$|cno}dEs z0D>m*VE`X3%Ni~?#1zwbDemS#5(1bHr&%0otw}uuQi5Jfl2I@tP92fM0*}9L)_rk# zO~?u2MS^fwf?<3v4^>zN6au<{Em^=&#P&%*QYB!Qwc9`a2=xYchz;xt%p3K$@>nx7 zyxsi@*c%I(!SANDC5Uu?NJh(Kzae|#VfUo_fl0=bs9T}L9*FW)$)x_Q z+G`MgiT%T3>$9n;^N=+Wtr9iUj7>%!rI~X0R`51QzCb+rSPSirq<~d23)F^Eyr)-= zQ(|>lol=le$xN!hkEX)=U6=34-%Z(Rxg9L99fwi25=~xj)Vxb9LNO=hgtkwm^A&`n zy2qM0O^JFWwq&wIbdm%*mzx#!u}zNeHYs z(!RX@|N1_JMsE~*$kC&qV0j>Xktt2FTpl>Q7bHir*Fa*4(x3&tF#xZO5}jBON)^C| z0@)e@vo|U#WFQKrv;2@_+|dS-1s5O{#iY73neL)D!X)ET%3nL3vsRz;ADI1O@;3sm z0IaJRfc#QiyoXQqtA*q1_q3x%nlKE1z@L*T6dy^6t_`3D0O5W%#`uws#NSqVCb``Q z|J7iu`c6}mRK#pd*JzMygtPCv9!vfs%MFl&uTS}C`9{11olW4aRU-b8wa~D&4Vbql zX|HZu?Bs*sNOuqASmK%IITvlixEmrLlj%DF+-x&8c%md?f_g$R9X#|*@{W)HgyLmB z3LJiyrpXrtSR1TIU%1g?92>IzzWs8jSqN}>&LX2ILljZH`U!bq(hgg&6Zm?xHguKK zm%~*iIeP@$EC{b;01vN4s1&2#0~nq~<|nx@GIo4feh188Jtx$U>kt9SJqJ;5SRgFx z0&c!ahpww(NQQkeMCeRfEgDQ1wnpKXyx~~Moygrxv{VFX2BnMORgxcv@DM71U8v9& zB7VDLCAhJu1uGHtcLScTV1T^O<&n?WlU5F`6;%xcK-3$~ z`uJcXRL&4LEU6|4JkEdQK$-U$njlaP(8;9}OM)q=6FM=vs7VC4r1)WxjVHKSC={-b z9rDDK*$|cfaLKJ;fP|P&9UfSwI3exl@{-iE+Wr?f%p_JsB5R}`RP2#g1~ewCV*=M1 zvnRZogA4K2ZEs$xOUlB+;J;7i{g2l6zgZgp8E^VO|F1b?f5S)bWCn#=hf#)$`}VS} H-~I4Uc&Dc0 literal 0 HcmV?d00001 From 4eba4d3cdd7a1757847c8c6b4c897ea14a8a1dfb Mon Sep 17 00:00:00 2001 From: Patrick Mullen Date: Wed, 15 Nov 2023 09:07:31 -0700 Subject: [PATCH 64/68] Fix bad merge conflict resolution --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5483071f9b18..7a63de5567e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,6 @@ - [[PR 868]](https://github.com/parthenon-hpc-lab/parthenon/pull/868) Add block-local face, edge, and nodal fields and allow for packing ### Changed (changing behavior/API/variables/...) -<<<<<<< HEAD - [[PR 975]](https://github.com/parthenon-hpc-lab/parthenon/pull/975) Construct staged integrators via arbitrary name - [[PR 976]](https://github.com/parthenon-hpc-lab/parthenon/pull/976) Move UserWorkBeforeLoop to be after first output - [[PR 965]](https://github.com/parthenon-hpc-lab/parthenon/pull/965) Allow leading whitespace in input parameters From 590b9c9fae5c05b955ed09b342354420eb196501 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 15 Nov 2023 09:38:35 -0700 Subject: [PATCH 65/68] Make kernel labels accurate --- src/solvers/solver_utils.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/solvers/solver_utils.hpp b/src/solvers/solver_utils.hpp index 87a2b71814c3..290ffcf60ce2 100644 --- a/src/solvers/solver_utils.hpp +++ b/src/solvers/solver_utils.hpp @@ -157,7 +157,7 @@ TaskStatus CopyData(const std::shared_ptr> &md) { auto desc = parthenon::MakePackDescriptor(md.get()); auto pack = desc.GetPack(md.get(), {}, only_fine_on_composite); parthenon::par_for( - DEFAULT_LOOP_PATTERN, "SetPotentialToZero", DevExecSpace(), 0, + DEFAULT_LOOP_PATTERN, "CopyData", DevExecSpace(), 0, pack.GetNBlocks() - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { const int nvars = pack.GetUpperBound(b, in()) - pack.GetLowerBound(b, in()) + 1; @@ -188,7 +188,7 @@ TaskStatus AddFieldsAndStoreInteriorSelect(const std::shared_ptr> auto desc = parthenon::MakePackDescriptor(md.get()); auto pack = desc.GetPack(md.get(), include_block, only_fine_on_composite); parthenon::par_for( - DEFAULT_LOOP_PATTERN, "SetPotentialToZero", DevExecSpace(), 0, + DEFAULT_LOOP_PATTERN, "AddFieldsAndStore", DevExecSpace(), 0, pack.GetNBlocks() - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { const int nvars = pack.GetUpperBound(b, a_t()) - pack.GetLowerBound(b, a_t()) + 1; @@ -219,7 +219,7 @@ TaskStatus SetToZero(const std::shared_ptr> &md) { const int scratch_level = 1; const int ng = parthenon::Globals::nghost; parthenon::par_for_outer( - DEFAULT_OUTER_LOOP_PATTERN, "Print", DevExecSpace(), scratch_size_in_bytes, + DEFAULT_OUTER_LOOP_PATTERN, "SetFieldsToZero", DevExecSpace(), scratch_size_in_bytes, scratch_level, 0, pack.GetNBlocks() - 1, KOKKOS_LAMBDA(parthenon::team_mbr_t member, const int b) { auto cb = GetIndexShape(pack(b, te, 0), ng); From aa195d45e872643be869ae671edf928c6350aa17 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 15 Nov 2023 10:51:09 -0700 Subject: [PATCH 66/68] rename and format --- src/interface/sparse_pack.hpp | 5 +++-- src/solvers/bicgstab_solver.hpp | 17 +++++++++-------- src/solvers/mg_solver.hpp | 9 +++++---- src/solvers/solver_utils.hpp | 14 +++++++------- 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/src/interface/sparse_pack.hpp b/src/interface/sparse_pack.hpp index fc30cacceed2..3ca458c2621a 100644 --- a/src/interface/sparse_pack.hpp +++ b/src/interface/sparse_pack.hpp @@ -145,8 +145,9 @@ class SparsePack : public SparsePackBase { template SparsePack GetPack(T *pmd, std::vector include_block = {}, bool only_fine_two_level_composite_blocks = true) const { - // If this is a composite grid MeshData object, only include blocks on - // the finer level + // If this is a composite grid MeshData object and if + // only_fine_two_level_composite_blocks is true, only + // include blocks on the finer level if constexpr (std::is_same>::value) { if (pmd->grid.type == GridType::two_level_composite && only_fine_two_level_composite_blocks) { diff --git a/src/solvers/bicgstab_solver.hpp b/src/solvers/bicgstab_solver.hpp index 00e071b1763f..b3af4cc6ad75 100644 --- a/src/solvers/bicgstab_solver.hpp +++ b/src/solvers/bicgstab_solver.hpp @@ -38,17 +38,18 @@ struct BiCGSTABParams { bool precondition = true; }; +// The equations type template class BiCGSTABSolver { public: - INTERNALSOLVERVARIABLE(x, rhat0); - INTERNALSOLVERVARIABLE(x, v); - INTERNALSOLVERVARIABLE(x, h); - INTERNALSOLVERVARIABLE(x, s); - INTERNALSOLVERVARIABLE(x, t); - INTERNALSOLVERVARIABLE(x, r); - INTERNALSOLVERVARIABLE(x, p); - INTERNALSOLVERVARIABLE(x, u); + PARTHENON_INTERNALSOLVERVARIABLE(x, rhat0); + PARTHENON_INTERNALSOLVERVARIABLE(x, v); + PARTHENON_INTERNALSOLVERVARIABLE(x, h); + PARTHENON_INTERNALSOLVERVARIABLE(x, s); + PARTHENON_INTERNALSOLVERVARIABLE(x, t); + PARTHENON_INTERNALSOLVERVARIABLE(x, r); + PARTHENON_INTERNALSOLVERVARIABLE(x, p); + PARTHENON_INTERNALSOLVERVARIABLE(x, u); BiCGSTABSolver(StateDescriptor *pkg, BiCGSTABParams params_in, equations eq_in = equations(), std::vector shape = {}) diff --git a/src/solvers/mg_solver.hpp b/src/solvers/mg_solver.hpp index f49e0e1ac5f4..ea56cef4e21d 100644 --- a/src/solvers/mg_solver.hpp +++ b/src/solvers/mg_solver.hpp @@ -40,10 +40,11 @@ struct MGParams { template class MGSolver { public: - INTERNALSOLVERVARIABLE(u, res_err); // residual on the way up and error on the way down - INTERNALSOLVERVARIABLE(u, temp); // Temporary storage - INTERNALSOLVERVARIABLE(u, u0); // Storage for initial solution during FAS - INTERNALSOLVERVARIABLE(u, D); // Storage for (approximate) diagonal + PARTHENON_INTERNALSOLVERVARIABLE( + u, res_err); // residual on the way up and error on the way down + PARTHENON_INTERNALSOLVERVARIABLE(u, temp); // Temporary storage + PARTHENON_INTERNALSOLVERVARIABLE(u, u0); // Storage for initial solution during FAS + PARTHENON_INTERNALSOLVERVARIABLE(u, D); // Storage for (approximate) diagonal MGSolver(StateDescriptor *pkg, MGParams params_in, equations eq_in = equations(), std::vector shape = {}) diff --git a/src/solvers/solver_utils.hpp b/src/solvers/solver_utils.hpp index 290ffcf60ce2..52c70627d5d2 100644 --- a/src/solvers/solver_utils.hpp +++ b/src/solvers/solver_utils.hpp @@ -20,7 +20,7 @@ #include "kokkos_abstraction.hpp" -#define INTERNALSOLVERVARIABLE(base, varname) \ +#define PARTHENON_INTERNALSOLVERVARIABLE(base, varname) \ struct varname : public parthenon::variable_names::base_t { \ template \ KOKKOS_INLINE_FUNCTION varname(Ts &&...args) \ @@ -157,8 +157,8 @@ TaskStatus CopyData(const std::shared_ptr> &md) { auto desc = parthenon::MakePackDescriptor(md.get()); auto pack = desc.GetPack(md.get(), {}, only_fine_on_composite); parthenon::par_for( - DEFAULT_LOOP_PATTERN, "CopyData", DevExecSpace(), 0, - pack.GetNBlocks() - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + DEFAULT_LOOP_PATTERN, "CopyData", DevExecSpace(), 0, pack.GetNBlocks() - 1, kb.s, + kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { const int nvars = pack.GetUpperBound(b, in()) - pack.GetLowerBound(b, in()) + 1; for (int c = 0; c < nvars; ++c) @@ -188,8 +188,8 @@ TaskStatus AddFieldsAndStoreInteriorSelect(const std::shared_ptr> auto desc = parthenon::MakePackDescriptor(md.get()); auto pack = desc.GetPack(md.get(), include_block, only_fine_on_composite); parthenon::par_for( - DEFAULT_LOOP_PATTERN, "AddFieldsAndStore", DevExecSpace(), 0, - pack.GetNBlocks() - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + DEFAULT_LOOP_PATTERN, "AddFieldsAndStore", DevExecSpace(), 0, pack.GetNBlocks() - 1, + kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { const int nvars = pack.GetUpperBound(b, a_t()) - pack.GetLowerBound(b, a_t()) + 1; for (int c = 0; c < nvars; ++c) { @@ -219,8 +219,8 @@ TaskStatus SetToZero(const std::shared_ptr> &md) { const int scratch_level = 1; const int ng = parthenon::Globals::nghost; parthenon::par_for_outer( - DEFAULT_OUTER_LOOP_PATTERN, "SetFieldsToZero", DevExecSpace(), scratch_size_in_bytes, - scratch_level, 0, pack.GetNBlocks() - 1, + DEFAULT_OUTER_LOOP_PATTERN, "SetFieldsToZero", DevExecSpace(), + scratch_size_in_bytes, scratch_level, 0, pack.GetNBlocks() - 1, KOKKOS_LAMBDA(parthenon::team_mbr_t member, const int b) { auto cb = GetIndexShape(pack(b, te, 0), ng); const auto &coords = pack.GetCoordinates(b); From d86cf0c5b0ac3d935af698f3b8607e1c3d4d9d2e Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 15 Nov 2023 11:05:14 -0700 Subject: [PATCH 67/68] Add comments --- src/solvers/bicgstab_solver.hpp | 8 +++++++- src/solvers/mg_solver.hpp | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/solvers/bicgstab_solver.hpp b/src/solvers/bicgstab_solver.hpp index b3af4cc6ad75..94d8479b4d64 100644 --- a/src/solvers/bicgstab_solver.hpp +++ b/src/solvers/bicgstab_solver.hpp @@ -38,7 +38,13 @@ struct BiCGSTABParams { bool precondition = true; }; -// The equations type +// The equations class must include a template method +// +// template +// TaskID Ax(TL_t &tl, TaskID depends_on, std::shared_ptr> &md) +// +// that takes a field associated with x_t and applies +// the matrix A to it and stores the result in y_t. template class BiCGSTABSolver { public: diff --git a/src/solvers/mg_solver.hpp b/src/solvers/mg_solver.hpp index ea56cef4e21d..074663983099 100644 --- a/src/solvers/mg_solver.hpp +++ b/src/solvers/mg_solver.hpp @@ -37,6 +37,20 @@ struct MGParams { std::string smoother = "SRJ2"; }; +// The equations class must include a template method +// +// template +// TaskID Ax(TL_t &tl, TaskID depends_on, std::shared_ptr> &md) +// +// that takes a field associated with x_t and applies +// the matrix A to it and stores the result in y_t. Additionally, +// it must include a template method +// +// template +// TaskStatus SetDiagonal(std::shared_ptr> &md) +// +// That stores the (possibly approximate) diagonal of matrix A in the field +// associated with the type diag_t. This is used for Jacobi iteration. template class MGSolver { public: From e20bdcacb86f58ee17aee19acd163cbc1403911a Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 15 Nov 2023 11:22:32 -0700 Subject: [PATCH 68/68] Add missing copyright in file not associated with this PR --- tst/unit/test_upper_bound.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tst/unit/test_upper_bound.cpp b/tst/unit/test_upper_bound.cpp index 20244ee95089..4bd3eef66471 100644 --- a/tst/unit/test_upper_bound.cpp +++ b/tst/unit/test_upper_bound.cpp @@ -3,6 +3,17 @@ // Copyright(C) 2023 The Parthenon collaboration // Licensed under the 3-clause BSD License, see LICENSE file for details //======================================================================================== +// (C) (or copyright) 2023. Triad National Security, LLC. All rights reserved. +// +// This program was produced under U.S. Government contract 89233218CNA000001 for Los +// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +// for the U.S. Department of Energy/National Nuclear Security Administration. All rights +// in the program are reserved by Triad National Security, LLC, and the U.S. Department +// of Energy/National Nuclear Security Administration. The Government is granted for +// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +// license in this material to reproduce, prepare derivative works, distribute copies to +// the public, perform publicly and display publicly, and to permit others to do so. +//======================================================================================== #include #include