From d7796eb022b66d56d46b352266179a0fe5a4e27c Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Thu, 3 Sep 2020 15:06:19 -0600 Subject: [PATCH 01/13] preliminary work on partitioning --- src/mesh/mesh_pack.hpp | 6 +- src/utils/partition_stl_containers.hpp | 81 +++++++++++++ tst/unit/CMakeLists.txt | 1 + tst/unit/test_partitioning.cpp | 160 +++++++++++++++++++++++++ 4 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 src/utils/partition_stl_containers.hpp create mode 100644 tst/unit/test_partitioning.cpp diff --git a/src/mesh/mesh_pack.hpp b/src/mesh/mesh_pack.hpp index e9cd335e7cea..4fd4d5da6db8 100644 --- a/src/mesh/mesh_pack.hpp +++ b/src/mesh/mesh_pack.hpp @@ -74,9 +74,13 @@ using MeshVariablePack = MeshPack>; template using MeshVariableFluxPack = MeshPack>; +namespace meshpack { +using blocks_t = std::vector; +} // namespace meshpack + // TODO(JMM): Should this be cached? namespace mesh_pack_impl { -using blocks_t = std::vector; +using meshpack::blocks_t; // TODO(JMM): blocks data type might change template diff --git a/src/utils/partition_stl_containers.hpp b/src/utils/partition_stl_containers.hpp new file mode 100644 index 000000000000..79e95e5b8472 --- /dev/null +++ b/src/utils/partition_stl_containers.hpp @@ -0,0 +1,81 @@ +//======================================================================================== +// (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved. +// +// This program was produced under U.S. Government contract 89233218CNA000001 for Los +// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +// for the U.S. Department of Energy/National Nuclear Security Administration. All rights +// in the program are reserved by Triad National Security, LLC, and the U.S. Department +// of Energy/National Nuclear Security Administration. The Government is granted for +// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +// license in this material to reproduce, prepare derivative works, distribute copies to +// the public, perform publicly and display publicly, and to permit others to do so. +//======================================================================================= +#ifndef UTILS_PARTITION_CONTAINERS_HPP_ +#define UTILS_PARTITION_CONTAINERS_HPP_ + +#include +#include + +#include "error_checking.hpp" + +namespace parthenon { +namespace Partition { +// TODO(JMM): The templated type safety with T* and T may need to be +// changed if we move to sufficiently general container objects. +template +using Partition_t = std::vector>; + +// x/y rounded up +// See discussion here for limitations and alternatives +// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c +KOKKOS_INLINE_FUNCTION +int IntCeil(int x, int y) { + PARTHENON_DEBUG_REQUIRE(x > 0 && y > 0, "ceil only works for x,y > 0"); + // avoids overflow in x+y + return 1 + ((x - 1) / y); +} + +// Takes container of elements and fills partitions +// of size N with pointers to elements. +// Assumes Container_t has STL-style iterators defined +template +void ToSizeN(Container_t &container, const int N, Partition_t &partitions) { + using std::to_string; + + int nelements = container.size(); + int npartitions = IntCeil(nelements, N); + std::string msg = ("Cannot partition " + to_string(nelements) + + " elements into partitions of size " + to_string(N) + "."); + PARTHENON_REQUIRE_THROWS(nelements >= N, msg); + + partitions.resize(npartitions); + for (auto &p : partitions) { + p.reserve(N); + p.clear(); + } + + int p = 0; + int b = 0; + for (auto &element : container) { + partitions[p].push_back(&element); + if (++b >= N) { + ++p; + b = 0; + } + } +} + +// Takes container of elements and fills N partitions +// with pointers to elements. +// Assumes Container_t has STL-style iterators defined +template +void ToNPartitions(Container_t &container, + const int N, Partition_t &partitions) { + int nelements = container.size(); + int partition_size = IntCeil(nelements, N); + ToSizeN(container, partition_size, partitions); +} +} // namespace Partition +} // namespace parthenon + +#endif // UTILS_PARTITION_CONTAINERS_HPP_ diff --git a/tst/unit/CMakeLists.txt b/tst/unit/CMakeLists.txt index 62d1cc1a9b8b..5569530abc29 100644 --- a/tst/unit/CMakeLists.txt +++ b/tst/unit/CMakeLists.txt @@ -28,6 +28,7 @@ list(APPEND unit_tests_SOURCES test_container_iterator.cpp test_required_desired.cpp test_error_checking.cpp + test_partitioning.cpp ) add_executable(unit_tests "${unit_tests_SOURCES}") diff --git a/tst/unit/test_partitioning.cpp b/tst/unit/test_partitioning.cpp new file mode 100644 index 000000000000..29f01e5f5673 --- /dev/null +++ b/tst/unit/test_partitioning.cpp @@ -0,0 +1,160 @@ +//======================================================================================== +// Athena++ astrophysical MHD code +// Copyright(C) 2014 James M. Stone and other code contributors +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +// (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved. +// +// This program was produced under U.S. Government contract 89233218CNA000001 for Los +// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +// for the U.S. Department of Energy/National Nuclear Security Administration. All rights +// in the program are reserved by Triad National Security, LLC, and the U.S. Department +// of Energy/National Nuclear Security Administration. The Government is granted for +// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +// license in this material to reproduce, prepare derivative works, distribute copies to +// the public, perform publicly and display publicly, and to permit others to do so. +//======================================================================================== + +#include +#include + +#include + +#include "utils/partition_stl_containers.hpp" + +using parthenon::Partition::Partition_t; +using parthenon::Partition::IntCeil; + +inline void check_partitions_even(Partition_t partitions, int nelements, int nparts, + int elements_per_part) { + REQUIRE(partitions.size() == nparts); + for (int p = 0; p < partitions.size(); p++) { + REQUIRE(partitions[p].size() == elements_per_part); + } + + int n_incorrect = 0; + for (int p = 0; p < partitions.size(); p++) { + for (int i = 0; i < elements_per_part; i++) { + if (*partitions[p][i] != p * elements_per_part + i) { + n_incorrect++; + } + } + } + REQUIRE(n_incorrect == 0); +} + +TEST_CASE("Check that integer division, rounded up, works", "[IntCeil]") { + GIVEN("A set of pairs of integers with a common denominator") { + constexpr int max_fac = 5; + THEN("Division works as expected") { + int n_wrong = 0; + for (int denom = 1; denom <= max_fac; denom++) { + for (int factor = 1; factor <= max_fac; factor++) { + int x = factor * denom; + int y = denom; + if (IntCeil(x, y) != factor) n_wrong++; + } + } + REQUIRE(n_wrong == 0); + } + } + GIVEN("A set of pairs of integers with non-common denominators") { + constexpr int max_fac = 5; + THEN("Division rounds up") { + int n_wrong = 0; + for (int denom = 1; denom <= max_fac; denom++) { + for (int factor = 1; factor <= max_fac; factor++) { + for (int offset = 1; offset < denom; offset++) { + int x = factor * denom + offset; + int y = denom; + if (IntCeil(x, y) != factor + 1) n_wrong++; + } + } + } + REQUIRE(n_wrong == 0); + } + } +} + +TEST_CASE("Check that partitioning a container works", "[Partition]") { + GIVEN("A list of integers of size 15") { + constexpr int nelements = 15; + constexpr int nparts = 3; + constexpr int elements_per_part = 5; + std::list l; + for (int i = 0; i < nelements; i++) { + l.push_back(i); + } + THEN("We can partition the list into 3 partitions of size 5 via Partition::ToSizeN") { + Partition_t partitions; + parthenon::Partition::ToSizeN(l, elements_per_part, partitions); + + check_partitions_even(partitions, nelements, nparts, elements_per_part); + } + THEN("We can partition the list into 3 partitions of size 5 via " + "Partition::ToNPartitions") { + Partition_t partitions; + parthenon::Partition::ToNPartitions(l, nparts, partitions); + + check_partitions_even(partitions, nelements, nparts, elements_per_part); + } + } + GIVEN("A list of size 17") { + constexpr int nelements = 17; + constexpr int nparts = 5; + constexpr int elements_per_part = 4; + constexpr int leftover = 1; + std::list l; + for (int i = 0; i < nelements; i++) { + l.push_back(i); + } + THEN("We can partition the list into 5 partitions of size 4 via Partition::ToSizeN") { + Partition_t partitions; + parthenon::Partition::ToSizeN(l, elements_per_part, partitions); + + REQUIRE(partitions.size() == nparts); + AND_THEN("The first 4 partitions are of size 4") { + for (int p = 0; p < partitions.size() - 1; p++) { + REQUIRE(partitions[p].size() == elements_per_part); + } + AND_THEN("The final partition is smaller, reflecting the mismatched sizes") { + REQUIRE(partitions.back().size() == leftover); + } + } + AND_THEN("The elements are all correct for the first 4 partitions") { + int n_incorrect = 0; + for (int p = 0; p < partitions.size() - 1; p++) { + for (int i = 0; i < elements_per_part; i++) { + if (*partitions[p][i] != p * elements_per_part + i) { + n_incorrect++; + } + } + } + REQUIRE(n_incorrect == 0); + AND_THEN("The elements are correct for the final partition") { + const int p = partitions.size() - 1; + for (int i = 0; i < leftover; i++) { + REQUIRE(*partitions[p][i] == p * elements_per_part + i); + } + } + } + AND_THEN("ToNPartitions and ToSizeN agree") { + Partition_t partitions_v2; + parthenon::Partition::ToNPartitions(l, nparts, partitions_v2); + REQUIRE(partitions.size() == partitions_v2.size()); + for (int p = 0; p < partitions.size(); p++) { + REQUIRE(partitions[p].size() == partitions_v2[p].size()); + } + int n_incorrect = 0; + for (int p = 0; p < partitions.size(); p++) { + for (int i = 0; i < partitions[p].size(); i++) { + if (*partitions[p][i] != *partitions_v2[p][i]) { + n_incorrect++; + } + } + } + REQUIRE(n_incorrect == 0); + } + } + } +} From 5a70d38faf15dab7d0a92db2aecd9e4b4d1d36b2 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Thu, 3 Sep 2020 15:33:37 -0600 Subject: [PATCH 02/13] added tests of edge cases --- src/utils/partition_stl_containers.hpp | 5 ++++- tst/unit/test_partitioning.cpp | 30 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/utils/partition_stl_containers.hpp b/src/utils/partition_stl_containers.hpp index 79e95e5b8472..2059bf117429 100644 --- a/src/utils/partition_stl_containers.hpp +++ b/src/utils/partition_stl_containers.hpp @@ -42,12 +42,15 @@ template void ToSizeN(Container_t &container, const int N, Partition_t &partitions) { using std::to_string; + PARTHENON_REQUIRE_THROWS(N > 0, "You must have at least 1 partition"); int nelements = container.size(); - int npartitions = IntCeil(nelements, N); + PARTHENON_REQUIRE_THROWS(nelements > 0, + "You must have at least 1 element to partition"); std::string msg = ("Cannot partition " + to_string(nelements) + " elements into partitions of size " + to_string(N) + "."); PARTHENON_REQUIRE_THROWS(nelements >= N, msg); + int npartitions = IntCeil(nelements, N); partitions.resize(npartitions); for (auto &p : partitions) { p.reserve(N); diff --git a/tst/unit/test_partitioning.cpp b/tst/unit/test_partitioning.cpp index 29f01e5f5673..000e1606b6f8 100644 --- a/tst/unit/test_partitioning.cpp +++ b/tst/unit/test_partitioning.cpp @@ -16,6 +16,7 @@ //======================================================================================== #include +#include #include #include @@ -77,6 +78,35 @@ TEST_CASE("Check that integer division, rounded up, works", "[IntCeil]") { } TEST_CASE("Check that partitioning a container works", "[Partition]") { + GIVEN("An attempt to partition an empty container") { + std::vector v(0); + int psize = 4; + THEN("The partition attempt throws an error") { + Partition_t partitions; + REQUIRE_THROWS_AS(parthenon::Partition::ToSizeN(v,psize,partitions), + std::runtime_error); + } + } + GIVEN("An attempt to partition into zero partitions") { + std::vector v = {1,2,3}; + int psize = 0; + THEN("The partition attempt throws an error") { + Partition_t partitions; + REQUIRE_THROWS_AS(parthenon::Partition::ToSizeN(v,psize,partitions), + std::runtime_error); + } + } + GIVEN("An attempt to partition 3 elements into partitions of size 4") { + constexpr int nelements = 3; + constexpr int psize = 4; + std::vector v = {1,2,3}; + THEN("The partition attempt throws an error") { + Partition_t partitions; + REQUIRE_THROWS_AS(parthenon::Partition::ToSizeN(v,psize,partitions), + std::runtime_error); + } + } + GIVEN("A list of integers of size 15") { constexpr int nelements = 15; constexpr int nparts = 3; From 737938940f26d8ad5546993fc981cb83cfed01ba Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Thu, 3 Sep 2020 15:35:41 -0600 Subject: [PATCH 03/13] formatting --- src/utils/partition_stl_containers.hpp | 3 +-- tst/unit/test_partitioning.cpp | 12 ++++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/utils/partition_stl_containers.hpp b/src/utils/partition_stl_containers.hpp index 2059bf117429..e49ba5588917 100644 --- a/src/utils/partition_stl_containers.hpp +++ b/src/utils/partition_stl_containers.hpp @@ -72,8 +72,7 @@ void ToSizeN(Container_t &container, const int N, Partition_t &partitions) { // with pointers to elements. // Assumes Container_t has STL-style iterators defined template -void ToNPartitions(Container_t &container, - const int N, Partition_t &partitions) { +void ToNPartitions(Container_t &container, const int N, Partition_t &partitions) { int nelements = container.size(); int partition_size = IntCeil(nelements, N); ToSizeN(container, partition_size, partitions); diff --git a/tst/unit/test_partitioning.cpp b/tst/unit/test_partitioning.cpp index 000e1606b6f8..f6de8364e12a 100644 --- a/tst/unit/test_partitioning.cpp +++ b/tst/unit/test_partitioning.cpp @@ -23,8 +23,8 @@ #include "utils/partition_stl_containers.hpp" -using parthenon::Partition::Partition_t; using parthenon::Partition::IntCeil; +using parthenon::Partition::Partition_t; inline void check_partitions_even(Partition_t partitions, int nelements, int nparts, int elements_per_part) { @@ -83,26 +83,26 @@ TEST_CASE("Check that partitioning a container works", "[Partition]") { int psize = 4; THEN("The partition attempt throws an error") { Partition_t partitions; - REQUIRE_THROWS_AS(parthenon::Partition::ToSizeN(v,psize,partitions), + REQUIRE_THROWS_AS(parthenon::Partition::ToSizeN(v, psize, partitions), std::runtime_error); } } GIVEN("An attempt to partition into zero partitions") { - std::vector v = {1,2,3}; + std::vector v = {1, 2, 3}; int psize = 0; THEN("The partition attempt throws an error") { Partition_t partitions; - REQUIRE_THROWS_AS(parthenon::Partition::ToSizeN(v,psize,partitions), + REQUIRE_THROWS_AS(parthenon::Partition::ToSizeN(v, psize, partitions), std::runtime_error); } } GIVEN("An attempt to partition 3 elements into partitions of size 4") { constexpr int nelements = 3; constexpr int psize = 4; - std::vector v = {1,2,3}; + std::vector v = {1, 2, 3}; THEN("The partition attempt throws an error") { Partition_t partitions; - REQUIRE_THROWS_AS(parthenon::Partition::ToSizeN(v,psize,partitions), + REQUIRE_THROWS_AS(parthenon::Partition::ToSizeN(v, psize, partitions), std::runtime_error); } } From 88950762f6d1007f25be99e82b2786c8a356fd07 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Wed, 9 Sep 2020 12:21:39 -0600 Subject: [PATCH 04/13] pi example works with variable mesh pack size --- example/calculate_pi/calculate_pi.cpp | 81 ++++++++----------------- example/calculate_pi/calculate_pi.hpp | 21 ++++--- example/calculate_pi/parthinput.example | 6 +- example/calculate_pi/pi_driver.cpp | 64 +++++++++---------- example/calculate_pi/pi_driver.hpp | 9 +-- src/driver/driver.hpp | 11 +--- 6 files changed, 75 insertions(+), 117 deletions(-) diff --git a/example/calculate_pi/calculate_pi.cpp b/example/calculate_pi/calculate_pi.cpp index 75d5bcc7e75c..fd7bef33e305 100644 --- a/example/calculate_pi/calculate_pi.cpp +++ b/example/calculate_pi/calculate_pi.cpp @@ -11,9 +11,6 @@ // the public, perform publicly and display publicly, and to permit others to do so. //======================================================================================== -// Self Include -#include "calculate_pi.hpp" - // Standard Includes #include #include @@ -24,9 +21,11 @@ // Parthenon Includes #include #include -#include #include +// Local Includes +#include "calculate_pi.hpp" + using namespace parthenon::package::prelude; // This defines a "physics" package @@ -81,62 +80,11 @@ std::shared_ptr Initialize(ParameterInput *pin) { return package; } -TaskStatus ComputeArea(MeshBlock *pmb) { - // compute 1/r0^2 \int d^2x in_or_out(x,y) over the block's domain - auto &rc = pmb->real_containers.Get(); - const IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - const IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - const IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); - auto &coords = pmb->coords; - - ParArrayND &v = rc->Get("in_or_out").data; - Real area; - Kokkos::parallel_reduce( - "calculate_pi compute area", - Kokkos::MDRangePolicy>(pmb->exec_space, {kb.s, jb.s, ib.s}, - {kb.e + 1, jb.e + 1, ib.e + 1}, - {1, 1, ib.e + 1 - ib.s}), - KOKKOS_LAMBDA(int k, int j, int i, Real &larea) { - larea += v(k, j, i) * coords.Area(parthenon::X3DIR, k, j, i); - }, - area); - Kokkos::deep_copy(pmb->exec_space, v.Get(0, 0, 0, 0, 0, 0), area); - - return TaskStatus::complete; -} - -TaskStatus RetrieveAreas(std::vector &blocks, - parthenon::Packages_t &packages) { - const auto &radius = packages["calculate_pi"]->Param("radius"); - - Real area = 0.0; - for (auto pmb : blocks) { - auto &rc = pmb->real_containers.Get(); - ParArrayND v = rc->Get("in_or_out").data; - // extract area from device memory - Real block_area; - Kokkos::deep_copy(pmb->exec_space, block_area, v.Get(0, 0, 0, 0, 0, 0)); - pmb->exec_space.fence(); // as the deep copy may be async - // area must be reduced by r^2 to get the block's contribution to PI - block_area /= (radius * radius); - // accumulate - area += block_area; - } - - packages["calculate_pi"]->AddParam("area", area); - return TaskStatus::complete; -} - -TaskStatus ComputeAreaOnMesh(std::vector &blocks, - parthenon::Packages_t &packages) { - auto pack = parthenon::PackVariablesOnMesh(blocks, "base", - std::vector{"in_or_out"}); +TaskStatus ComputeArea(Pack_t pack, ParArrayHost areas, int i) { const IndexRange ib = pack.cellbounds.GetBoundsI(IndexDomain::interior); const IndexRange jb = pack.cellbounds.GetBoundsJ(IndexDomain::interior); const IndexRange kb = pack.cellbounds.GetBoundsK(IndexDomain::interior); - const auto &radius = packages["calculate_pi"]->Param("radius"); - Real area = 0.0; using policy = Kokkos::MDRangePolicy>; Kokkos::parallel_reduce( @@ -148,9 +96,28 @@ TaskStatus ComputeAreaOnMesh(std::vector &blocks, larea += pack(b, v, k, j, i) * pack.coords(b).Area(parthenon::X3DIR, k, j, i); }, area); + + areas(i) = area; + return TaskStatus::complete; +} + +TaskStatus AccumulateAreas(ParArrayHost areas, Packages_t &packages) { + const auto &radius = packages["calculate_pi"]->Param("radius"); + + Real area = 0.0; + for (int i = 0; i < areas.GetSize(); i++) { + area += areas(i); + } area /= (radius * radius); - packages["calculate_pi"]->AddParam("area", area); +#ifdef MPI_PARALLEL + Real pi_val; + MPI_Reduce(&area, &pi_val, 1, MPI_PARTHENON_REAL, MPI_SUM, 0, MPI_COMM_WORLD); +#else + Real pi_val = area; +#endif + + packages["calculate_pi"]->AddParam("pi_val", pi_val); return TaskStatus::complete; } diff --git a/example/calculate_pi/calculate_pi.hpp b/example/calculate_pi/calculate_pi.hpp index fce149f8a3d4..38bea1976fa6 100644 --- a/example/calculate_pi/calculate_pi.hpp +++ b/example/calculate_pi/calculate_pi.hpp @@ -18,25 +18,28 @@ #include // Parthenon Includes +#include +#include #include namespace calculate_pi { using namespace parthenon::package::prelude; +using parthenon::Packages_t; +using parthenon::ParArrayHost; +using Pack_t = parthenon::MeshPack>; // Package Callbacks void SetInOrOut(std::shared_ptr> &rc); std::shared_ptr Initialize(ParameterInput *pin); // Task Implementations -// task per meshblock -parthenon::TaskStatus ComputeArea(parthenon::MeshBlock *pmb); -// Task over whole mesh, no packs -parthenon::TaskStatus RetrieveAreas(std::vector &blocks, - parthenon::Packages_t &packages); - -// Run task on the entire mesh at once -parthenon::TaskStatus ComputeAreaOnMesh(std::vector &blocks, - parthenon::Packages_t &packages); +// Note pass by value here. Required for capture. +// All objects here have reference semantics, so capture by value is ok. +// TODO(JMM) A std::shared_ptr might be better. +// Computes area on a given meshpack +parthenon::TaskStatus ComputeArea(Pack_t pack, ParArrayHost areas, int i); +// Sums up areas accross packs. +parthenon::TaskStatus AccumulateAreas(ParArrayHost areas, Packages_t &packages); } // namespace calculate_pi #endif // EXAMPLE_CALCULATE_PI_CALCULATE_PI_HPP_ diff --git a/example/calculate_pi/parthinput.example b/example/calculate_pi/parthinput.example index 1c3c93d3758b..d763484550dd 100644 --- a/example/calculate_pi/parthinput.example +++ b/example/calculate_pi/parthinput.example @@ -52,10 +52,8 @@ max_level = 2 # if set, limits refinement level from this crit radius = 1.5 -# Whether or not to use mesh-wide kernel calls -# A value of 0 is default task-based mechanism, where kernels are per meshblock -# A value of 1 makes a single global kernel call over all meshblocks on a rank -use_mesh_pack = 0 +# How many meshblocks to use in a kernel. A value of -1 means use the whole mesh. +pack_size = 4 file_type = hdf5 diff --git a/example/calculate_pi/pi_driver.cpp b/example/calculate_pi/pi_driver.cpp index 87c6b046c7ba..568a45b54681 100644 --- a/example/calculate_pi/pi_driver.cpp +++ b/example/calculate_pi/pi_driver.cpp @@ -19,6 +19,7 @@ // Parthenon Includes #include +#include // Local Includes #include "calculate_pi.hpp" @@ -93,19 +94,11 @@ parthenon::DriverStatus PiDriver::Execute() { pouts->MakeOutputs(pmesh, pinput); - // The task lists constructed depends on whether we're doing local tasking - // or a global meshpack. + // The tasks compute pi and store it in the param "pi_val" ConstructAndExecuteTaskLists<>(this); - // Retrive and MPI reduce the area from mesh params - auto &area = pmesh->packages["calculate_pi"]->Param("area"); - -#ifdef MPI_PARALLEL - Real pi_val; - MPI_Reduce(&area, &pi_val, 1, MPI_PARTHENON_REAL, MPI_SUM, 0, MPI_COMM_WORLD); -#else - Real pi_val = area; -#endif + // retrieve "pi_val" and post execute. + auto &pi_val = pmesh->packages["calculate_pi"]->Param("pi_val"); pmesh->mbcnt = pmesh->nbtotal; // this is how many blocks were processed PostExecute(pi_val); return DriverStatus::complete; @@ -128,33 +121,36 @@ void PiDriver::PostExecute(Real pi_val) { Driver::PostExecute(); } -TaskCollection PiDriver::MakeTasks(std::vector blocks) { +template +TaskCollection PiDriver::MakeTasks(T &blocks) { + using calculate_pi::AccumulateAreas; using calculate_pi::ComputeArea; - using calculate_pi::ComputeAreaOnMesh; - using calculate_pi::RetrieveAreas; TaskCollection tc; - if (pinput->GetOrAddBoolean("Pi", "use_mesh_pack", false)) { - TaskRegion &tr = tc.AddRegion(1); - { - // tasks should be local per region. Be sure to scope them appropriately. - TaskID none(0); - auto get_area = tr[0].AddTask(ComputeAreaOnMesh, none, blocks, pmesh->packages); - } - } else { - // asynchronous region where area is computed per block - TaskRegion &async_region = tc.AddRegion(blocks.size()); - for (int i = 0; i < blocks.size(); i++) { - TaskID none(0); - auto get_area = async_region[i].AddTask(ComputeArea, none, blocks[i]); - } - // synchronous region where the area is retrieved and accumulated - // and stored in params - TaskRegion &sync_region = tc.AddRegion(1); - { + + // 1 means pack is 1 meshblock, <1 means use entire mesh + int pack_size = pinput->GetOrAddInteger("Pi", "pack_size", 1); + if (pack_size < 1) pack_size = blocks.size(); + + parthenon::Partition::Partition_t partitions; + parthenon::Partition::ToSizeN(blocks, pack_size, partitions); + parthenon::ParArrayHost areas("areas", partitions.size()); + + TaskRegion &async_region = tc.AddRegion(partitions.size()); + { + // asynchronous region where area is computed per mesh pack + for (int i = 0; i < partitions.size(); i++) { TaskID none(0); - auto get_area = - sync_region[0].AddTask(RetrieveAreas, none, blocks, pmesh->packages); + auto pack = parthenon::PackVariablesOnMesh(partitions[i], "base", + std::vector{"in_or_out"}); + auto get_area = async_region[i].AddTask(ComputeArea, none, pack, areas, i); } } + TaskRegion &sync_region = tc.AddRegion(1); + { + TaskID none(0); + auto accumulate_areas = + sync_region[0].AddTask(AccumulateAreas, none, areas, pmesh->packages); + } + return tc; } diff --git a/example/calculate_pi/pi_driver.hpp b/example/calculate_pi/pi_driver.hpp index c550ef5b5cc0..b352680e92e8 100644 --- a/example/calculate_pi/pi_driver.hpp +++ b/example/calculate_pi/pi_driver.hpp @@ -32,10 +32,11 @@ class PiDriver : public Driver { pin->CheckDesired("Pi", "radius"); } - /// MakeTaskList isn't a virtual routine on `Driver`, but each driver is expected to - /// implement it. - TaskList MakeTaskList(MeshBlock *pmb); - TaskCollection MakeTasks(std::vector blocks); + /// MakeTaskList and MakeTasks aren't virtual routines on `Driver`, + // but each driver is expected to implement at least one of them. + /// TaskList MakeTaskList(MeshBlock *pmb); + template + TaskCollection MakeTasks(T &blocks); /// `Execute` cylces until simulation completion. DriverStatus Execute() override; diff --git a/src/driver/driver.hpp b/src/driver/driver.hpp index fe3961c92544..f26c70dddc66 100644 --- a/src/driver/driver.hpp +++ b/src/driver/driver.hpp @@ -103,15 +103,8 @@ TaskListStatus ConstructAndExecuteBlockTasks(T *driver, Args... args) { template TaskListStatus ConstructAndExecuteTaskLists(T *driver, Args... args) { - int nmb = driver->pmesh->GetNumMeshBlocksThisRank(Globals::my_rank); - std::vector blocks(nmb); - - int i = 0; - for (auto &mb : driver->pmesh->block_list) { - blocks[i++] = &mb; - } - - TaskCollection tc = driver->MakeTasks(blocks, std::forward(args)...); + TaskCollection tc = + driver->MakeTasks(driver->pmesh->block_list, std::forward(args)...); TaskListStatus status = tc.Execute(); return status; } From 99393d878a2e771320e930fa113c08bdbc46d735 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Wed, 9 Sep 2020 12:26:44 -0600 Subject: [PATCH 05/13] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1254efb8b325..fd928a4abbe0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - [[PR 267]](https://github.com/lanl/parthenon/pull/267) Introduced TaskRegions and TaskCollections to allow for task launches on multiple blocks. - [[PR 287]](https://github.com/lanl/parthenon/pull/287) Added machine configuration file for compile options, see [documentation](https://github.com/lanl/parthenon/blob/develop/docs/building.md#default-machine-configurations) - [[PR 290]](https://github.com/lanl/parthenon/pull/290) Added per cycle performance output diagnostic. +- [[PR 298]](https://github.com/lanl/parthenon/pull/298) Introduced Partition, a tiny utility for partitioning STL containers. Used for MeshPacks, to enable packing over a fraction of the mesh. ### Changed (changing behavior/API/variables/...) - [\#68](https://github.com/lanl/parthenon/issues/68) Moved default `par_for` wrappers to `MeshBlock` From bfdf449e75c1df0c27c3c155a6b005aaaef4c2fe Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Wed, 9 Sep 2020 12:37:38 -0600 Subject: [PATCH 06/13] added documentation --- docs/mesh/packing.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/mesh/packing.md b/docs/mesh/packing.md index 9eef3c85a94b..16459a4550b2 100644 --- a/docs/mesh/packing.md +++ b/docs/mesh/packing.md @@ -27,6 +27,35 @@ can pass in a metadata vector, a vector of names, and optionally IDs and a map from names to indices. See [here](../interface/containers.md) for more details. +## Packing over a piece of the mesh + +Instead of packing over the whole mesh, you can pack over only a piece +of it by using the `Partition` machinery found in +`utils/partition_stl_containers.hpp`. For example, to break the mesh +into four evenly spaced meshpacks, do +```C++ +using parthenon::MeshBlock; +using parthenon::Partition::Partition_t; +Partition_t partitions; +parthenon::Partition::ToNPartitions(mesh->block, 4, partitions); +MeshPack> packs[4]; +for (int i = 0; i < partitions.size() { + packs[i] = PackVariablesOnMesh(partitions[i], "base"); +} +``` + +There are two partitioning functions: +```C++ +// Splits container into N equally sized partitions +template +void ToNPartitions(Container_t &container, const int N, Partition_t &partitions); + +// Splits container into partitions of size N +template +void ToSizeN(Container_t &container, const int N, Partition_t &partitions); +``` +Both functions live within the namespace `parthenon::Partition`. + ## Data Layout The Mesh Pack is indexable as a five-dimensional `Kokkos` view. The From 60f0e26becbac07596de7ee62f9ff7c277d48977 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Wed, 9 Sep 2020 15:58:23 -0600 Subject: [PATCH 07/13] renamed MeshPack to MeshBlockPack --- CHANGELOG.md | 6 ++--- docs/mesh/packing.md | 17 ++++++++------ example/calculate_pi/calculate_pi.hpp | 4 ++-- .../{mesh_pack.hpp => meshblock_pack.hpp} | 23 ++++++++++--------- src/utils/partition_stl_containers.hpp | 6 ++--- 5 files changed, 30 insertions(+), 26 deletions(-) rename src/mesh/{mesh_pack.hpp => meshblock_pack.hpp} (89%) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd928a4abbe0..700ee73cdb3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,11 @@ ### Added (new features/APIs/variables/...) - [[PR 250]](https://github.com/lanl/parthenon/pull/250) Feature::Restart. If output file format 'rst' is specified restart files are written using independent variables and those marked with Restart metadata flag. Simulations can be restarted with a '-r \' argument to the code. -- [[PR 263]](https://github.com/lanl/parthenon/pull/263) Added MeshPack, a mechanism for looping over the whole mesh at once within a `Kokkos` kernel. See [documentation](docs/mesh/packing.md) +- [[PR 263]](https://github.com/lanl/parthenon/pull/263) Added MeshBlockPack, a mechanism for looping over the whole mesh at once within a `Kokkos` kernel. See [documentation](docs/mesh/packing.md) - [[PR 267]](https://github.com/lanl/parthenon/pull/267) Introduced TaskRegions and TaskCollections to allow for task launches on multiple blocks. - [[PR 287]](https://github.com/lanl/parthenon/pull/287) Added machine configuration file for compile options, see [documentation](https://github.com/lanl/parthenon/blob/develop/docs/building.md#default-machine-configurations) - [[PR 290]](https://github.com/lanl/parthenon/pull/290) Added per cycle performance output diagnostic. -- [[PR 298]](https://github.com/lanl/parthenon/pull/298) Introduced Partition, a tiny utility for partitioning STL containers. Used for MeshPacks, to enable packing over a fraction of the mesh. +- [[PR 298]](https://github.com/lanl/parthenon/pull/298) Introduced Partition, a tiny utility for partitioning STL containers. Used for MeshBlockPacks, to enable packing over a fraction of the mesh. ### Changed (changing behavior/API/variables/...) - [\#68](https://github.com/lanl/parthenon/issues/68) Moved default `par_for` wrappers to `MeshBlock` @@ -20,7 +20,7 @@ - [[PR 262]](https://github.com/lanl/parthenon/pull/262) Fix setting of "coverage" label in testing. Automatically applies coverage tag to all tests not containing "performance" label. - [[PR 276]](https://github.com/lanl/parthenon/pull/276) Decrease required Python version from 3.6 to 3.5. - [[PR 283]](https://github.com/lanl/parthenon/pull/283) Change CI to extended nightly develop tests and short push tests. -- [[PR 282]](https://github.com/lanl/parthenon/pull/282) Integrated meshpack and tasking in pi example +- [[PR 282]](https://github.com/lanl/parthenon/pull/282) Integrated MeshBlockPack and tasking in pi example - [[PR 294]](https://github.com/lanl/parthenon/pull/294) Fix `IndexShape::GetTotal(IndexDomain)` - previously was returning opposite of expected domain result. ### Removed diff --git a/docs/mesh/packing.md b/docs/mesh/packing.md index 16459a4550b2..a28ce5dcc9ef 100644 --- a/docs/mesh/packing.md +++ b/docs/mesh/packing.md @@ -4,23 +4,26 @@ kernels that perform little work, this can be a perforamnce bottleneck when each kernel is launched per meshblock. Parthenon therefore provides the capability to combine variables into a single data -structure that spans all meshblocks, the `MeshPack`. +structure that spans some number of meshblocks, the `MeshBlockPack`. -## Creating a Mesh Pack +## Creating a MeshBlockPack There are two methods for creating mesh packs, which are analogous to the `VariablePack` and `VariableFluxPack` available in [containers](../interface/containers.md). ```C++ -template -auto PackVariablesOnMesh(Mesh *pmesh, const std::string &container_name, +template +auto PackVariablesOnMesh(T &blocks, const std::string &container_name, Args &&... args) ``` and ```C++ -auto PackVariablesAndFluxesOnMesh(Mesh *pmesh, const std::string &container_name, +template +auto PackVariablesAndFluxesOnMesh(T &blocks, const std::string &container_name, Args &&... args) ``` The former packs only the variables, the latter packs in the variables and associated flux vectors. +Here `T` can be the mesh or any standard template container that contains meshblocks. + The variatic arguments take exactly the same arguments as `container.PackVariables` and `container.PackVariablesAndFluxes`. You can pass in a metadata vector, a vector of names, and optionally IDs @@ -38,7 +41,7 @@ using parthenon::MeshBlock; using parthenon::Partition::Partition_t; Partition_t partitions; parthenon::Partition::ToNPartitions(mesh->block, 4, partitions); -MeshPack> packs[4]; +MeshBlockPack> packs[4]; for (int i = 0; i < partitions.size() { packs[i] = PackVariablesOnMesh(partitions[i], "base"); } @@ -73,7 +76,7 @@ auto var = meshpack(m,l); // Indexes into the k'th variable on the m'th MB Real r = meshpack(m,l,k,j,i); ``` -For convenience, `MeshPack` also includes the following methods and fields: +For convenience, `MeshBlockPack` also includes the following methods and fields: ```C++ // the IndexShape object describing the bounds of all blocks in the pack IndexShape bounds = meshpack.cellbounds; diff --git a/example/calculate_pi/calculate_pi.hpp b/example/calculate_pi/calculate_pi.hpp index 38bea1976fa6..e69dc13e1e40 100644 --- a/example/calculate_pi/calculate_pi.hpp +++ b/example/calculate_pi/calculate_pi.hpp @@ -19,14 +19,14 @@ // Parthenon Includes #include -#include +#include #include namespace calculate_pi { using namespace parthenon::package::prelude; using parthenon::Packages_t; using parthenon::ParArrayHost; -using Pack_t = parthenon::MeshPack>; +using Pack_t = parthenon::MeshBlockPack>; // Package Callbacks void SetInOrOut(std::shared_ptr> &rc); diff --git a/src/mesh/mesh_pack.hpp b/src/mesh/meshblock_pack.hpp similarity index 89% rename from src/mesh/mesh_pack.hpp rename to src/mesh/meshblock_pack.hpp index 4fd4d5da6db8..2229f73e629f 100644 --- a/src/mesh/mesh_pack.hpp +++ b/src/mesh/meshblock_pack.hpp @@ -10,8 +10,8 @@ // license in this material to reproduce, prepare derivative works, distribute copies to // the public, perform publicly and display publicly, and to permit others to do so. //======================================================================================= -#ifndef MESH_MESH_PACK_HPP_ -#define MESH_MESH_PACK_HPP_ +#ifndef MESH_MESHBLOCK_PACK_HPP_ +#define MESH_MESHBLOCK_PACK_HPP_ #include #include @@ -31,11 +31,12 @@ namespace parthenon { // TODO(JMM): Using one IndexShape because its the same for all // meshblocks. This needs careful thought before sticking with it. template -class MeshPack { +class MeshBlockPack { public: - MeshPack() = default; - MeshPack(const ParArray1D view, const IndexShape shape, - const ParArray1D coordinates, const std::array dims) + MeshBlockPack() = default; + MeshBlockPack(const ParArray1D view, const IndexShape shape, + const ParArray1D coordinates, + const std::array dims) : v_(view), cellbounds(shape), coords(coordinates), dims_(dims) {} KOKKOS_FORCEINLINE_FUNCTION auto &operator()(const int block) const { return v_(block); } @@ -70,9 +71,9 @@ using ViewOfPacks = ParArray1D>; template using ViewOfFluxPacks = ParArray1D>; template -using MeshVariablePack = MeshPack>; +using MeshVariablePack = MeshBlockPack>; template -using MeshVariableFluxPack = MeshPack>; +using MeshVariableFluxPack = MeshBlockPack>; namespace meshpack { using blocks_t = std::vector; @@ -88,7 +89,7 @@ auto PackMesh(blocks_t &blocks, F &packing_function) { int nblocks = blocks.size(); ParArray1D packs("MakeMeshVariablePack::view", nblocks); auto packs_host = Kokkos::create_mirror_view(packs); - ParArray1D coords("MakeMeshPackVariable::coords", nblocks); + ParArray1D coords("MakeMeshBlockPackVariable::coords", nblocks); auto coords_host = Kokkos::create_mirror_view(coords); int b = 0; @@ -109,7 +110,7 @@ auto PackMesh(blocks_t &blocks, F &packing_function) { auto cellbounds = blocks[0]->cellbounds; - return MeshPack(packs, cellbounds, coords, dims); + return MeshBlockPack(packs, cellbounds, coords, dims); } // TODO(JMM): Should we merge block_list and the vector of meshblock pointers @@ -150,4 +151,4 @@ auto PackVariablesAndFluxesOnMesh(T &blocks, const std::string &container_name, } // namespace parthenon -#endif // MESH_MESH_PACK_HPP_ +#endif // MESH_MESHBLOCK_PACK_HPP_ diff --git a/src/utils/partition_stl_containers.hpp b/src/utils/partition_stl_containers.hpp index e49ba5588917..45fd66d8cb32 100644 --- a/src/utils/partition_stl_containers.hpp +++ b/src/utils/partition_stl_containers.hpp @@ -10,8 +10,8 @@ // license in this material to reproduce, prepare derivative works, distribute copies to // the public, perform publicly and display publicly, and to permit others to do so. //======================================================================================= -#ifndef UTILS_PARTITION_CONTAINERS_HPP_ -#define UTILS_PARTITION_CONTAINERS_HPP_ +#ifndef UTILS_PARTITION_STL_CONTAINERS_HPP_ +#define UTILS_PARTITION_STL_CONTAINERS_HPP_ #include #include @@ -80,4 +80,4 @@ void ToNPartitions(Container_t &container, const int N, Partition_t &partitio } // namespace Partition } // namespace parthenon -#endif // UTILS_PARTITION_CONTAINERS_HPP_ +#endif // UTILS_PARTITION_STL_CONTAINERS_HPP_ From b8ae7835d9a8e61b56c8563ee9960229094359d9 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Wed, 9 Sep 2020 16:13:34 -0600 Subject: [PATCH 08/13] clean up types a little --- docs/mesh/packing.md | 13 +++++++++++++ example/calculate_pi/calculate_pi.hpp | 2 +- src/mesh/meshblock_pack.hpp | 5 ++++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/docs/mesh/packing.md b/docs/mesh/packing.md index a28ce5dcc9ef..3a20a973d913 100644 --- a/docs/mesh/packing.md +++ b/docs/mesh/packing.md @@ -95,3 +95,16 @@ int sparse = meshpack.GetSparse(n); // The size of the n'th dimension of the pack int dim = meshpack.GetDim(n); ``` + +## Type + +The types for packs are: +```C++ +MeshBlockVarPack +``` +and +```C++ +MeshBlockVarFluxPack +``` +which correspond to packs over meshblocks that contain just variables +or contain variables and fluxes. diff --git a/example/calculate_pi/calculate_pi.hpp b/example/calculate_pi/calculate_pi.hpp index e69dc13e1e40..182df5277925 100644 --- a/example/calculate_pi/calculate_pi.hpp +++ b/example/calculate_pi/calculate_pi.hpp @@ -26,7 +26,7 @@ namespace calculate_pi { using namespace parthenon::package::prelude; using parthenon::Packages_t; using parthenon::ParArrayHost; -using Pack_t = parthenon::MeshBlockPack>; +using Pack_t = parthenon::MeshBlockVarPack; // Package Callbacks void SetInOrOut(std::shared_ptr> &rc); diff --git a/src/mesh/meshblock_pack.hpp b/src/mesh/meshblock_pack.hpp index 2229f73e629f..8a77e7a54e84 100644 --- a/src/mesh/meshblock_pack.hpp +++ b/src/mesh/meshblock_pack.hpp @@ -112,6 +112,10 @@ auto PackMesh(blocks_t &blocks, F &packing_function) { return MeshBlockPack(packs, cellbounds, coords, dims); } +template +using MeshBlockVarPack = MeshBlockPack>; +template +using MeshBlockVarFluxPack = MeshBlockPack>; // TODO(JMM): Should we merge block_list and the vector of meshblock pointers // in some way? What's the right thing to do here? @@ -148,7 +152,6 @@ auto PackVariablesAndFluxesOnMesh(T &blocks, const std::string &container_name, }; return PackMesh>(blocks, pack_function); } - } // namespace parthenon #endif // MESH_MESHBLOCK_PACK_HPP_ From 20364e4bbd6204ff2c7c19c475b5740f0ce8776f Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Wed, 9 Sep 2020 16:56:29 -0600 Subject: [PATCH 09/13] added meshpack to appropriate preludes --- example/calculate_pi/calculate_pi.hpp | 1 - example/calculate_pi/pi_driver.cpp | 11 +++++------ src/mesh/meshblock_pack.hpp | 9 +++------ src/parthenon/driver.hpp | 13 +++++++++++++ src/parthenon/package.hpp | 4 ++++ src/parthenon/prelude.hpp | 1 + 6 files changed, 26 insertions(+), 13 deletions(-) diff --git a/example/calculate_pi/calculate_pi.hpp b/example/calculate_pi/calculate_pi.hpp index 182df5277925..bb9bfebe12bc 100644 --- a/example/calculate_pi/calculate_pi.hpp +++ b/example/calculate_pi/calculate_pi.hpp @@ -19,7 +19,6 @@ // Parthenon Includes #include -#include #include namespace calculate_pi { diff --git a/example/calculate_pi/pi_driver.cpp b/example/calculate_pi/pi_driver.cpp index 568a45b54681..4287b88ff431 100644 --- a/example/calculate_pi/pi_driver.cpp +++ b/example/calculate_pi/pi_driver.cpp @@ -19,7 +19,6 @@ // Parthenon Includes #include -#include // Local Includes #include "calculate_pi.hpp" @@ -131,17 +130,17 @@ TaskCollection PiDriver::MakeTasks(T &blocks) { int pack_size = pinput->GetOrAddInteger("Pi", "pack_size", 1); if (pack_size < 1) pack_size = blocks.size(); - parthenon::Partition::Partition_t partitions; - parthenon::Partition::ToSizeN(blocks, pack_size, partitions); - parthenon::ParArrayHost areas("areas", partitions.size()); + Partition::Partition_t partitions; + Partition::ToSizeN(blocks, pack_size, partitions); + ParArrayHost areas("areas", partitions.size()); TaskRegion &async_region = tc.AddRegion(partitions.size()); { // asynchronous region where area is computed per mesh pack for (int i = 0; i < partitions.size(); i++) { TaskID none(0); - auto pack = parthenon::PackVariablesOnMesh(partitions[i], "base", - std::vector{"in_or_out"}); + auto pack = PackVariablesOnMesh(partitions[i], "base", + std::vector{"in_or_out"}); auto get_area = async_region[i].AddTask(ComputeArea, none, pack, areas, i); } } diff --git a/src/mesh/meshblock_pack.hpp b/src/mesh/meshblock_pack.hpp index 8a77e7a54e84..0cee0074bd7a 100644 --- a/src/mesh/meshblock_pack.hpp +++ b/src/mesh/meshblock_pack.hpp @@ -70,10 +70,11 @@ template using ViewOfPacks = ParArray1D>; template using ViewOfFluxPacks = ParArray1D>; + template -using MeshVariablePack = MeshBlockPack>; +using MeshBlockVarPack = MeshBlockPack>; template -using MeshVariableFluxPack = MeshBlockPack>; +using MeshBlockVarFluxPack = MeshBlockPack>; namespace meshpack { using blocks_t = std::vector; @@ -112,10 +113,6 @@ auto PackMesh(blocks_t &blocks, F &packing_function) { return MeshBlockPack(packs, cellbounds, coords, dims); } -template -using MeshBlockVarPack = MeshBlockPack>; -template -using MeshBlockVarFluxPack = MeshBlockPack>; // TODO(JMM): Should we merge block_list and the vector of meshblock pointers // in some way? What's the right thing to do here? diff --git a/src/parthenon/driver.hpp b/src/parthenon/driver.hpp index 2e60e7f7443e..c54fb8d6e01b 100644 --- a/src/parthenon/driver.hpp +++ b/src/parthenon/driver.hpp @@ -21,11 +21,13 @@ #include #include #include +#include #include #include #include #include #include +#include // Local Includes #include "prelude.hpp" @@ -42,9 +44,14 @@ using ::parthenon::DriverStatus; using ::parthenon::Integrator; using ::parthenon::Mesh; using ::parthenon::MeshBlock; +using ::parthenon::MeshBlockPack; +using ::parthenon::MeshBlockVarFluxPack; +using ::parthenon::MeshBlockVarPack; using ::parthenon::MultiStageBlockTaskDriver; using ::parthenon::Outputs; using ::parthenon::Packages_t; +using ::parthenon::PackVariablesAndFluxesOnMesh; +using ::parthenon::PackVariablesOnMesh; using ::parthenon::ParameterInput; using ::parthenon::ParthenonManager; using ::parthenon::Task; @@ -55,6 +62,12 @@ using ::parthenon::TaskRegion; using ::parthenon::TaskStatus; using ::parthenon::DriverUtils::ConstructAndExecuteBlockTasks; using ::parthenon::DriverUtils::ConstructAndExecuteTaskLists; + +namespace Partition { +using ::parthenon::Partition::Partition_t; +using ::parthenon::Partition::ToNPartitions; +using ::parthenon::Partition::ToSizeN; +} // namespace Partition } // namespace prelude } // namespace driver } // namespace parthenon diff --git a/src/parthenon/package.hpp b/src/parthenon/package.hpp index 48b2526afd15..53766bb0afa5 100644 --- a/src/parthenon/package.hpp +++ b/src/parthenon/package.hpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -43,6 +44,9 @@ using ::parthenon::Coordinates; using ::parthenon::DerivedOwnership; using ::parthenon::DevExecSpace; using ::parthenon::MeshBlock; +using ::parthenon::MeshBlockPack; +using ::parthenon::MeshBlockVarFluxPack; +using ::parthenon::MeshBlockVarPack; using ::parthenon::Metadata; using ::parthenon::PackIndexMap; using ::parthenon::par_for; diff --git a/src/parthenon/prelude.hpp b/src/parthenon/prelude.hpp index 221e3f1e9869..169488a04dc4 100644 --- a/src/parthenon/prelude.hpp +++ b/src/parthenon/prelude.hpp @@ -34,6 +34,7 @@ using ::parthenon::Container; using ::parthenon::IndexDomain; using ::parthenon::IndexRange; using ::parthenon::MeshBlock; +using ::parthenon::ParArrayHost; using ::parthenon::ParArrayND; using ::parthenon::ParthenonStatus; using ::parthenon::Real; From c89fa2c3574d14158a8b18cb0309f7c8a662e9b4 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Thu, 10 Sep 2020 15:20:45 -0600 Subject: [PATCH 10/13] Update docs/mesh/packing.md Co-authored-by: Andrew Gaspar --- docs/mesh/packing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/mesh/packing.md b/docs/mesh/packing.md index 3a20a973d913..132041618520 100644 --- a/docs/mesh/packing.md +++ b/docs/mesh/packing.md @@ -35,7 +35,7 @@ and a map from names to indices. See Instead of packing over the whole mesh, you can pack over only a piece of it by using the `Partition` machinery found in `utils/partition_stl_containers.hpp`. For example, to break the mesh -into four evenly spaced meshpacks, do +into four evenly sized meshpacks, do ```C++ using parthenon::MeshBlock; using parthenon::Partition::Partition_t; From ca7a4c1b02a5bdec66142d28e8b7934c799dfa17 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Thu, 10 Sep 2020 15:21:08 -0600 Subject: [PATCH 11/13] Update docs/mesh/packing.md Co-authored-by: Andrew Gaspar --- docs/mesh/packing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/mesh/packing.md b/docs/mesh/packing.md index 132041618520..2b7c188260ae 100644 --- a/docs/mesh/packing.md +++ b/docs/mesh/packing.md @@ -40,7 +40,7 @@ into four evenly sized meshpacks, do using parthenon::MeshBlock; using parthenon::Partition::Partition_t; Partition_t partitions; -parthenon::Partition::ToNPartitions(mesh->block, 4, partitions); +parthenon::Partition::ToNPartitions(mesh->block_list, 4, partitions); MeshBlockPack> packs[4]; for (int i = 0; i < partitions.size() { packs[i] = PackVariablesOnMesh(partitions[i], "base"); From 983c55bbced31d93bedb82e08a8901e5185b2b3f Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Thu, 10 Sep 2020 16:58:45 -0600 Subject: [PATCH 12/13] address Andrew Gaspar's comments --- docs/mesh/packing.md | 6 ++++++ example/calculate_pi/pi_driver.cpp | 4 ++-- src/parthenon/driver.hpp | 10 +++++----- src/utils/partition_stl_containers.hpp | 21 ++++++++++---------- tst/unit/test_partitioning.cpp | 27 +++++++++++++------------- 5 files changed, 38 insertions(+), 30 deletions(-) diff --git a/docs/mesh/packing.md b/docs/mesh/packing.md index 2b7c188260ae..84bd019e337f 100644 --- a/docs/mesh/packing.md +++ b/docs/mesh/packing.md @@ -46,6 +46,12 @@ for (int i = 0; i < partitions.size() { packs[i] = PackVariablesOnMesh(partitions[i], "base"); } ``` +To pack only the variables "var1" and "var2" in the container named "mycontainer", do: +```C++ +for (int i = 0; i < partitions.size() { + packs[i] = PackVariablesOnMesh(partitions[i], "myContainer", {"var1","var2"}); +} +``` There are two partitioning functions: ```C++ diff --git a/example/calculate_pi/pi_driver.cpp b/example/calculate_pi/pi_driver.cpp index 4287b88ff431..3b0ba18fdc9e 100644 --- a/example/calculate_pi/pi_driver.cpp +++ b/example/calculate_pi/pi_driver.cpp @@ -130,8 +130,8 @@ TaskCollection PiDriver::MakeTasks(T &blocks) { int pack_size = pinput->GetOrAddInteger("Pi", "pack_size", 1); if (pack_size < 1) pack_size = blocks.size(); - Partition::Partition_t partitions; - Partition::ToSizeN(blocks, pack_size, partitions); + partition::Partition_t partitions; + partition::ToSizeN(blocks, pack_size, partitions); ParArrayHost areas("areas", partitions.size()); TaskRegion &async_region = tc.AddRegion(partitions.size()); diff --git a/src/parthenon/driver.hpp b/src/parthenon/driver.hpp index c54fb8d6e01b..f36c16907dc2 100644 --- a/src/parthenon/driver.hpp +++ b/src/parthenon/driver.hpp @@ -63,11 +63,11 @@ using ::parthenon::TaskStatus; using ::parthenon::DriverUtils::ConstructAndExecuteBlockTasks; using ::parthenon::DriverUtils::ConstructAndExecuteTaskLists; -namespace Partition { -using ::parthenon::Partition::Partition_t; -using ::parthenon::Partition::ToNPartitions; -using ::parthenon::Partition::ToSizeN; -} // namespace Partition +namespace partition { +using ::parthenon::partition::Partition_t; +using ::parthenon::partition::ToNPartitions; +using ::parthenon::partition::ToSizeN; +} // namespace partition } // namespace prelude } // namespace driver } // namespace parthenon diff --git a/src/utils/partition_stl_containers.hpp b/src/utils/partition_stl_containers.hpp index 45fd66d8cb32..b254a1f995ae 100644 --- a/src/utils/partition_stl_containers.hpp +++ b/src/utils/partition_stl_containers.hpp @@ -19,21 +19,24 @@ #include "error_checking.hpp" namespace parthenon { -namespace Partition { +namespace partition { // TODO(JMM): The templated type safety with T* and T may need to be // changed if we move to sufficiently general container objects. template using Partition_t = std::vector>; +namespace partition_impl { // x/y rounded up // See discussion here for limitations and alternatives // https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c KOKKOS_INLINE_FUNCTION int IntCeil(int x, int y) { - PARTHENON_DEBUG_REQUIRE(x > 0 && y > 0, "ceil only works for x,y > 0"); + PARTHENON_DEBUG_REQUIRE(x >= 0, "ceil only works for x >= 0"); + PARTHENON_DEBUG_REQUIRE(y > 0, "ceil only works for y > 0"); // avoids overflow in x+y - return 1 + ((x - 1) / y); + return x > 0 ? 1 + ((x - 1) / y) : 0; } +} // namespace partition_impl // Takes container of elements and fills partitions // of size N with pointers to elements. @@ -41,16 +44,13 @@ int IntCeil(int x, int y) { template void ToSizeN(Container_t &container, const int N, Partition_t &partitions) { using std::to_string; + using namespace partition_impl; PARTHENON_REQUIRE_THROWS(N > 0, "You must have at least 1 partition"); - int nelements = container.size(); - PARTHENON_REQUIRE_THROWS(nelements > 0, - "You must have at least 1 element to partition"); - std::string msg = ("Cannot partition " + to_string(nelements) + - " elements into partitions of size " + to_string(N) + "."); - PARTHENON_REQUIRE_THROWS(nelements >= N, msg); + int nelements = container.size(); int npartitions = IntCeil(nelements, N); + partitions.resize(npartitions); for (auto &p : partitions) { p.reserve(N); @@ -73,11 +73,12 @@ void ToSizeN(Container_t &container, const int N, Partition_t &partitions) { // Assumes Container_t has STL-style iterators defined template void ToNPartitions(Container_t &container, const int N, Partition_t &partitions) { + using namespace partition_impl; int nelements = container.size(); int partition_size = IntCeil(nelements, N); ToSizeN(container, partition_size, partitions); } -} // namespace Partition +} // namespace partition } // namespace parthenon #endif // UTILS_PARTITION_STL_CONTAINERS_HPP_ diff --git a/tst/unit/test_partitioning.cpp b/tst/unit/test_partitioning.cpp index f6de8364e12a..63183bbfeefe 100644 --- a/tst/unit/test_partitioning.cpp +++ b/tst/unit/test_partitioning.cpp @@ -23,8 +23,8 @@ #include "utils/partition_stl_containers.hpp" -using parthenon::Partition::IntCeil; -using parthenon::Partition::Partition_t; +using parthenon::partition::Partition_t; +using parthenon::partition::partition_impl::IntCeil; inline void check_partitions_even(Partition_t partitions, int nelements, int nparts, int elements_per_part) { @@ -81,10 +81,10 @@ TEST_CASE("Check that partitioning a container works", "[Partition]") { GIVEN("An attempt to partition an empty container") { std::vector v(0); int psize = 4; - THEN("The partition attempt throws an error") { + THEN("We get zero partitions") { Partition_t partitions; - REQUIRE_THROWS_AS(parthenon::Partition::ToSizeN(v, psize, partitions), - std::runtime_error); + parthenon::partition::ToSizeN(v, psize, partitions); + REQUIRE(partitions.size() == 0); } } GIVEN("An attempt to partition into zero partitions") { @@ -92,7 +92,7 @@ TEST_CASE("Check that partitioning a container works", "[Partition]") { int psize = 0; THEN("The partition attempt throws an error") { Partition_t partitions; - REQUIRE_THROWS_AS(parthenon::Partition::ToSizeN(v, psize, partitions), + REQUIRE_THROWS_AS(parthenon::partition::ToSizeN(v, psize, partitions), std::runtime_error); } } @@ -100,10 +100,11 @@ TEST_CASE("Check that partitioning a container works", "[Partition]") { constexpr int nelements = 3; constexpr int psize = 4; std::vector v = {1, 2, 3}; - THEN("The partition attempt throws an error") { + THEN("We get a single partition of size 3") { Partition_t partitions; - REQUIRE_THROWS_AS(parthenon::Partition::ToSizeN(v, psize, partitions), - std::runtime_error); + parthenon::partition::ToSizeN(v, psize, partitions); + REQUIRE(partitions.size() == 1); + REQUIRE(partitions[0].size() == 3); } } @@ -117,14 +118,14 @@ TEST_CASE("Check that partitioning a container works", "[Partition]") { } THEN("We can partition the list into 3 partitions of size 5 via Partition::ToSizeN") { Partition_t partitions; - parthenon::Partition::ToSizeN(l, elements_per_part, partitions); + parthenon::partition::ToSizeN(l, elements_per_part, partitions); check_partitions_even(partitions, nelements, nparts, elements_per_part); } THEN("We can partition the list into 3 partitions of size 5 via " "Partition::ToNPartitions") { Partition_t partitions; - parthenon::Partition::ToNPartitions(l, nparts, partitions); + parthenon::partition::ToNPartitions(l, nparts, partitions); check_partitions_even(partitions, nelements, nparts, elements_per_part); } @@ -140,7 +141,7 @@ TEST_CASE("Check that partitioning a container works", "[Partition]") { } THEN("We can partition the list into 5 partitions of size 4 via Partition::ToSizeN") { Partition_t partitions; - parthenon::Partition::ToSizeN(l, elements_per_part, partitions); + parthenon::partition::ToSizeN(l, elements_per_part, partitions); REQUIRE(partitions.size() == nparts); AND_THEN("The first 4 partitions are of size 4") { @@ -170,7 +171,7 @@ TEST_CASE("Check that partitioning a container works", "[Partition]") { } AND_THEN("ToNPartitions and ToSizeN agree") { Partition_t partitions_v2; - parthenon::Partition::ToNPartitions(l, nparts, partitions_v2); + parthenon::partition::ToNPartitions(l, nparts, partitions_v2); REQUIRE(partitions.size() == partitions_v2.size()); for (int p = 0; p < partitions.size(); p++) { REQUIRE(partitions[p].size() == partitions_v2[p].size()); From 9a80a76fb2e52e7afa5dff8f0035fb7729248312 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Fri, 11 Sep 2020 14:04:49 -0600 Subject: [PATCH 13/13] Fix type inference in documentation Thanks Philipp! --- docs/mesh/packing.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/mesh/packing.md b/docs/mesh/packing.md index 84bd019e337f..76f6c2ebe7d0 100644 --- a/docs/mesh/packing.md +++ b/docs/mesh/packing.md @@ -48,8 +48,9 @@ for (int i = 0; i < partitions.size() { ``` To pack only the variables "var1" and "var2" in the container named "mycontainer", do: ```C++ +std::vector> vars = {"var1", "var2"}; for (int i = 0; i < partitions.size() { - packs[i] = PackVariablesOnMesh(partitions[i], "myContainer", {"var1","var2"}); + packs[i] = PackVariablesOnMesh(partitions[i], "myContainer", vars); } ```