split things up

src/CMakeLists.txt

@@ -101,6 +101,8 @@ add_library(parthenon
   bvals/comms/bvals_in_one.hpp
   bvals/comms/bvals_utils.hpp
   bvals/comms/build_boundary_buffers.cpp
+  bvals/comms/bnd_id.cpp
+  bvals/comms/bnd_id.hpp
   bvals/comms/bnd_info.cpp
   bvals/comms/bnd_info.hpp
   bvals/comms/boundary_communication.cpp

src/bvals/comms/bnd_id.cpp

@@ -0,0 +1,69 @@
+//========================================================================================
+// Parthenon performance portable AMR framework
+// Copyright(C) 2024 The Parthenon collaboration
+// Licensed under the 3-clause BSD License, see LICENSE file for details
+//========================================================================================
+// (C) (or copyright) 2020-2024. Triad National Security, LLC. All rights reserved.
+//
+// This program was produced under U.S. Government contract 89233218CNA000001 for Los
+// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
+// for the U.S. Department of Energy/National Nuclear Security Administration. All rights
+// in the program are reserved by Triad National Security, LLC, and the U.S. Department
+// of Energy/National Nuclear Security Administration. The Government is granted for
+// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide
+// license in this material to reproduce, prepare derivative works, distribute copies to
+// the public, perform publicly and display publicly, and to permit others to do so.
+//========================================================================================
+
+#include <algorithm>
+#include <cstdio>
+#include <iostream> // debug
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "basic_types.hpp"
+#include "bvals/comms/bnd_id.hpp"
+#include "bvals/comms/bvals_utils.hpp"
+#include "bvals/neighbor_block.hpp"
+#include "config.hpp"
+#include "globals.hpp"
+#include "interface/state_descriptor.hpp"
+#include "interface/variable.hpp"
+#include "kokkos_abstraction.hpp"
+#include "mesh/domain.hpp"
+#include "mesh/mesh.hpp"
+#include "mesh/mesh_refinement.hpp"
+#include "mesh/meshblock.hpp"
+#include "prolong_restrict/prolong_restrict.hpp"
+#include "utils/error_checking.hpp"
+
+namespace parthenon {
+
+BndId BndId::GetSend(MeshBlock *pmb, const NeighborBlock &nb,
+                     std::shared_ptr<Variable<Real>> v, BoundaryType b_type,
+                     int partition, int start_idx) {
+  auto [send_gid, recv_gid, vlabel, loc, extra_id] = SendKey(pmb, nb, v, b_type);
+  BndId out;
+  out.send_gid() = send_gid;
+  out.recv_gid() = recv_gid;
+  out.loc_idx() = loc;
+  out.var_id() = v->GetUniqueID();
+  out.extra_id() = extra_id;
+  out.rank_send() = Globals::my_rank;
+  out.rank_recv() = nb.rank;
+  out.partition() = partition;
+  out.size() = BndInfo::GetSendBndInfo(pmb, nb, v, nullptr).size();
+  out.start_idx() = start_idx;
+  return out;
+}
+
+void BndId::PrintInfo(const std::string &start) {
+  printf("%s var %s (%i -> %i) starting at %i with size %i (Total combined buffer size = "
+         "%i, buffer size = %i, buf_allocated = %i) [rank = %i]\n",
+         start.c_str(), Variable<Real>::GetLabel(var_id()).c_str(), send_gid(),
+         recv_gid(), start_idx(), size(), coalesced_buf.size(), buf.size(), buf_allocated,
+         Globals::my_rank);
+}
+
+} // namespace parthenon

src/bvals/comms/bnd_id.hpp

@@ -0,0 +1,111 @@
+//========================================================================================
+// Parthenon performance portable AMR framework
+// Copyright(C) 2020 The Parthenon collaboration
+// Licensed under the 3-clause BSD License, see LICENSE file for details
+//========================================================================================
+// (C) (or copyright) 2020-2024. Triad National Security, LLC. All rights reserved.
+//
+// This program was produced under U.S. Government contract 89233218CNA000001 for Los
+// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
+// for the U.S. Department of Energy/National Nuclear Security Administration. All rights
+// in the program are reserved by Triad National Security, LLC, and the U.S. Department
+// of Energy/National Nuclear Security Administration. The Government is granted for
+// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide
+// license in this material to reproduce, prepare derivative works, distribute copies to
+// the public, perform publicly and display publicly, and to permit others to do so.
+//========================================================================================
+
+#ifndef BVALS_COMMS_BND_ID_HPP_
+#define BVALS_COMMS_BND_ID_HPP_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "basic_types.hpp"
+#include "bvals/neighbor_block.hpp"
+#include "coordinates/coordinates.hpp"
+#include "interface/variable_state.hpp"
+#include "mesh/domain.hpp"
+#include "mesh/forest/logical_coordinate_transformation.hpp"
+#include "utils/communication_buffer.hpp"
+#include "utils/indexer.hpp"
+#include "utils/object_pool.hpp"
+
+namespace parthenon {
+
+template <typename T>
+class Variable;
+
+// Provides the information necessary for identifying a unique variable-boundary
+// buffer, identifying the coalesced buffer it is associated with, and its
+// position within the coalesced buffer.
+struct BndId {
+  constexpr static std::size_t NDAT = 10;
+  int data[NDAT];
+
+  // Information for identifying the buffer with a communication
+  // channel, variable, and the ranks it is communicated across
+  KOKKOS_FORCEINLINE_FUNCTION
+  int &send_gid() { return data[0]; }
+  KOKKOS_FORCEINLINE_FUNCTION
+  int &recv_gid() { return data[1]; }
+  KOKKOS_FORCEINLINE_FUNCTION
+  int &loc_idx() { return data[2]; }
+  KOKKOS_FORCEINLINE_FUNCTION
+  int &var_id() { return data[3]; }
+  KOKKOS_FORCEINLINE_FUNCTION
+  int &extra_id() { return data[4]; }
+  KOKKOS_FORCEINLINE_FUNCTION
+  int &rank_send() { return data[5]; }
+  KOKKOS_FORCEINLINE_FUNCTION
+  int &rank_recv() { return data[6]; }
+  BoundaryType bound_type;
+
+  // MeshData partition id of the *sender*
+  // not set by constructors and only necessary for coalesced comms
+  KOKKOS_FORCEINLINE_FUNCTION
+  int &partition() { return data[7]; }
+  KOKKOS_FORCEINLINE_FUNCTION
+  int &size() { return data[8]; }
+  KOKKOS_FORCEINLINE_FUNCTION
+  int &start_idx() { return data[9]; }
+
+  bool buf_allocated;
+  buf_pool_t<Real>::weak_t buf;   // comm buffer from pool
+  BufArray1D<Real> coalesced_buf; // Combined buffer
+
+  void PrintInfo(const std::string &start);
+
+  KOKKOS_DEFAULTED_FUNCTION
+  BndId() = default;
+  KOKKOS_DEFAULTED_FUNCTION
+  BndId(const BndId &) = default;
+
+  explicit BndId(const int *const data_in) {
+    for (int i = 0; i < NDAT; ++i) {
+      data[i] = data_in[i];
+    }
+  }
+
+  void Serialize(int *data_out) {
+    for (int i = 0; i < NDAT; ++i) {
+      data_out[i] = data[i];
+    }
+  }
+
+  bool SameBVChannel(const BndId &other) {
+    // Don't want to compare start_idx, so -1
+    for (int i = 0; i < NDAT - 1; ++i) {
+      if (data[i] != other.data[i]) return false;
+    }
+    return true;
+  }
+
+  static BndId GetSend(MeshBlock *pmb, const NeighborBlock &nb,
+                       std::shared_ptr<Variable<Real>> v, BoundaryType b_type,
+                       int partition, int start_idx);
+};
+} // namespace parthenon
+
+#endif // BVALS_COMMS_BND_ID_HPP_

src/bvals/comms/bnd_info.cpp

@@ -306,32 +306,6 @@ BndInfo::BndInfo(MeshBlock *pmb, const NeighborBlock &nb,
   }
 }
 
-BndId BndId::GetSend(MeshBlock *pmb, const NeighborBlock &nb,
-                     std::shared_ptr<Variable<Real>> v, BoundaryType b_type,
-                     int partition, int start_idx) {
-  auto [send_gid, recv_gid, vlabel, loc, extra_id] = SendKey(pmb, nb, v, b_type);
-  BndId out;
-  out.send_gid() = send_gid;
-  out.recv_gid() = recv_gid;
-  out.loc_idx() = loc;
-  out.var_id() = v->GetUniqueID();
-  out.extra_id() = extra_id;
-  out.rank_send() = Globals::my_rank;
-  out.rank_recv() = nb.rank;
-  out.partition() = partition;
-  out.size() = BndInfo::GetSendBndInfo(pmb, nb, v, nullptr).size();
-  out.start_idx() = start_idx;
-  return out;
-}
-
-void BndId::PrintInfo(const std::string &start) {
-  printf("%s var %s (%i -> %i) starting at %i with size %i (Total combined buffer size = "
-         "%i, buffer size = %i, buf_allocated = %i) [rank = %i]\n",
-         start.c_str(), Variable<Real>::GetLabel(var_id()).c_str(), send_gid(),
-         recv_gid(), start_idx(), size(), coalesced_buf.size(), buf.size(), buf_allocated,
-         Globals::my_rank);
-}
-
 BndInfo BndInfo::GetSendBndInfo(MeshBlock *pmb, const NeighborBlock &nb,
                                 std::shared_ptr<Variable<Real>> v,
                                 CommBuffer<buf_pool_t<Real>::owner_t> *buf) {

src/bvals/comms/bnd_info.hpp

@@ -48,73 +48,6 @@ enum class IndexRangeType {
   InteriorRecv
 };
 
-struct BndId {
-  constexpr static std::size_t NDAT = 10;
-  int data[NDAT];
-
-  // Information for identifying the buffer with a communication
-  // channel, variable, and the ranks it is communicated across
-  KOKKOS_FORCEINLINE_FUNCTION
-  int &send_gid() { return data[0]; }
-  KOKKOS_FORCEINLINE_FUNCTION
-  int &recv_gid() { return data[1]; }
-  KOKKOS_FORCEINLINE_FUNCTION
-  int &loc_idx() { return data[2]; }
-  KOKKOS_FORCEINLINE_FUNCTION
-  int &var_id() { return data[3]; }
-  KOKKOS_FORCEINLINE_FUNCTION
-  int &extra_id() { return data[4]; }
-  KOKKOS_FORCEINLINE_FUNCTION
-  int &rank_send() { return data[5]; }
-  KOKKOS_FORCEINLINE_FUNCTION
-  int &rank_recv() { return data[6]; }
-  BoundaryType bound_type;
-
-  // MeshData partition id of the *sender*
-  // not set by constructors and only necessary for coalesced comms
-  KOKKOS_FORCEINLINE_FUNCTION
-  int &partition() { return data[7]; }
-  KOKKOS_FORCEINLINE_FUNCTION
-  int &size() { return data[8]; }
-  KOKKOS_FORCEINLINE_FUNCTION
-  int &start_idx() { return data[9]; }
-
-  bool buf_allocated;
-  buf_pool_t<Real>::weak_t buf;   // comm buffer from pool
-  BufArray1D<Real> coalesced_buf; // Combined buffer
-
-  void PrintInfo(const std::string &start);
-
-  KOKKOS_DEFAULTED_FUNCTION
-  BndId() = default;
-  KOKKOS_DEFAULTED_FUNCTION
-  BndId(const BndId &) = default;
-
-  explicit BndId(const int *const data_in) {
-    for (int i = 0; i < NDAT; ++i) {
-      data[i] = data_in[i];
-    }
-  }
-
-  void Serialize(int *data_out) {
-    for (int i = 0; i < NDAT; ++i) {
-      data_out[i] = data[i];
-    }
-  }
-
-  bool SameBVChannel(const BndId &other) {
-    // Don't want to compare start_idx, so -1
-    for (int i = 0; i < NDAT - 1; ++i) {
-      if (data[i] != other.data[i]) return false;
-    }
-    return true;
-  }
-
-  static BndId GetSend(MeshBlock *pmb, const NeighborBlock &nb,
-                       std::shared_ptr<Variable<Real>> v, BoundaryType b_type,
-                       int partition, int start_idx);
-};
-
 struct BndInfo {
   int ntopological_elements = 1;
   using TE = TopologicalElement;

src/bvals/comms/bvals_utils.hpp

@@ -25,6 +25,7 @@
 #include <utility>
 #include <vector>
 
+#include "bvals/comms/bnd_id.hpp"
 #include "bvals/comms/bnd_info.hpp"
 #include "bvals/comms/bvals_in_one.hpp"
 #include "interface/variable.hpp"

src/bvals/comms/coalesced_buffers.cpp

@@ -19,6 +19,7 @@
 #include <vector>
 
 #include "basic_types.hpp"
+#include "bvals/comms/bnd_id.hpp"
 #include "bvals/comms/bvals_utils.hpp"
 #include "bvals/comms/coalesced_buffers.hpp"
 #include "bvals/neighbor_block.hpp"

src/bvals/comms/coalesced_buffers.hpp

@@ -22,6 +22,7 @@
 #include <vector>
 
 #include "basic_types.hpp"
+#include "bvals/comms/bnd_id.hpp"
 #include "bvals/comms/bvals_utils.hpp"
 #include "bvals/neighbor_block.hpp"
 #include "coordinates/coordinates.hpp"