act on a bunch of small comments

src/bvals/comms/bnd_id.cpp

@@ -60,7 +60,7 @@ BndId BndId::GetSend(MeshBlock *pmb, const NeighborBlock &nb,
 
 void BndId::PrintInfo(const std::string &start) {
   printf("%s var %s (%i -> %i) starting at %i with size %i (Total combined buffer size = "
-         "%i, buffer size = %i, buf_allocated = %i) [rank = %i]\n",
+         "%li, buffer size = %li, buf_allocated = %i) [rank = %i]\n",
          start.c_str(), Variable<Real>::GetLabel(var_id()).c_str(), send_gid(),
          recv_gid(), start_idx(), size(), coalesced_buf.size(), buf.size(), buf_allocated,
          Globals::my_rank);

src/bvals/comms/bnd_info.cpp

@@ -278,6 +278,10 @@ BndInfo::BndInfo(MeshBlock *pmb, const NeighborBlock &nb,
   allocated = v->IsAllocated();
   alloc_status = v->GetAllocationStatus();
 
+  // Sometimes we may build a BndInfo object just to get the
+  // size of the index space associated with the boundary. In
+  // that case an associated communication buffer may not exist
+  // and a nullptr will be passed instead.
   if (combuf != nullptr) buf = combuf->buffer();
   same_to_same = pmb->gid == nb.gid && nb.offsets.IsCell();
   lcoord_trans = nb.lcoord_trans;

src/bvals/comms/bnd_info.hpp

@@ -58,8 +58,8 @@ struct BndInfo {
 
   // Number of points contained in this boundary region
   KOKKOS_FORCEINLINE_FUNCTION
-  int size() const {
-    int s = 0;
+  std::size_t size() const {
+    std::size_t s = 0;
     for (int n = 0; n < ntopological_elements; ++n) {
       s += idxer[n].size();
     }

src/bvals/comms/boundary_communication.cpp

@@ -209,8 +209,6 @@ template <BoundaryType bound_type>
 TaskStatus ReceiveBoundBufs(std::shared_ptr<MeshData<Real>> &md) {
   PARTHENON_INSTRUMENT
 
-  static int ntotal_prints{0};
-
   Mesh *pmesh = md->GetMeshPointer();
   auto &cache = md->GetBvarsCache().GetSubCache(bound_type, false);
   if (cache.buf_vec.size() == 0)
@@ -219,9 +217,7 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr<MeshData<Real>> &md) {
 
   bool all_received = true;
   if (pmesh->do_coalesced_comms) {
-    // Receive any messages that are around
-    bool all_coalesced_received =
-        pmesh->pcoalesced_comms->TryReceiveAny(md.get(), bound_type);
+    pmesh->pcoalesced_comms->TryReceiveAny(md.get(), bound_type);
   }
   const bool coal_comm = pmesh->do_coalesced_comms;
   std::for_each(std::begin(cache.buf_vec), std::end(cache.buf_vec),

src/bvals/comms/coalesced_buffers.cpp

@@ -37,17 +37,17 @@ void CoalescedBuffer::AllocateCoalescedBuffer() {
   int send_rank = sender ? Globals::my_rank : other_rank;
   int recv_rank = sender ? other_rank : Globals::my_rank;
   coalesced_comm_buffer = CommBuffer<buf_t>(
-      2 * partition, send_rank, recv_rank, comm_,
+      2 * partition, send_rank, recv_rank, comm,
       [](int size) { return buf_t("Combined Buffer", 2 * size); }, true);
 
   sparse_status_buffer =
-      CommBuffer<std::vector<int>>(2 * partition + 1, send_rank, recv_rank, comm_);
+      CommBuffer<std::vector<int>>(2 * partition + 1, send_rank, recv_rank, comm);
   sparse_status_buffer.ConstructBuffer(current_size + 1);
 }
 
 //----------------------------------------------------------------------------------------
 ParArray1DRaw<BndId> &CoalescedBuffer::GetBndIdsOnDevice(const std::set<Uid_t> &vars,
-                                                      int *pcomb_size) {
+                                                         int *pcomb_size) {
   const auto &var_set = vars.size() == 0 ? all_vars : vars;
   auto &bnd_ids_device = bnd_ids_device_map[var_set];
   auto &bnd_ids_host = bnd_ids_host_map[var_set];
@@ -258,16 +258,16 @@ void CoalescedBuffer::AddVarBoundary(MeshBlock *pmb, const NeighborBlock &nb,
 //----------------------------------------------------------------------------------------
 CoalescedBuffersRank::CoalescedBuffersRank(int o_rank, BoundaryType b_type, bool send,
                                            mpi_comm_t comm, Mesh *pmesh)
-    : other_rank(o_rank), b_type(b_type), sender(send), buffers_built(false), comm_(comm),
+    : other_rank(o_rank), b_type(b_type), sender(send), buffers_built(false), comm(comm),
       pmesh(pmesh) {
 
-  int tag = 1234 + static_cast<int>(GetAssociatedSender(b_type));
+  int tag = static_cast<int>(GetAssociatedSender(b_type));
   if (sender) {
-    message = com_buf_t(tag, Globals::my_rank, other_rank, comm_,
+    message = com_buf_t(tag, Globals::my_rank, other_rank, comm,
                         [](int size) { return std::vector<int>(size); });
   } else {
     message = com_buf_t(
-        tag, other_rank, Globals::my_rank, comm_,
+        tag, other_rank, Globals::my_rank, comm,
         [](int size) { return std::vector<int>(size); }, true);
   }
   PARTHENON_REQUIRE(other_rank != Globals::my_rank, "Should only build for other ranks.");
@@ -280,7 +280,7 @@ void CoalescedBuffersRank::AddSendBuffer(int partition, MeshBlock *pmb,
   if (coalesced_bufs.count(partition) == 0)
     coalesced_bufs.emplace(
         std::make_pair(partition, CoalescedBuffer(true, partition, other_rank, b_type,
-                                                  comm_, pmb->pmy_mesh)));
+                                                  comm, pmb->pmy_mesh)));
 
   auto &coal_buf = coalesced_bufs.at(partition);
   coal_buf.AddVarBoundary(pmb, nb, var);
@@ -304,9 +304,8 @@ bool CoalescedBuffersRank::TryReceiveBufInfo() {
       const int total_size = mess_buf[idx++];
 
       // Create the new partition
-      coalesced_bufs.emplace(
-          std::make_pair(partition, CoalescedBuffer(false, partition, other_rank, b_type,
-                                                    comm_, pmesh)));
+      coalesced_bufs.emplace(std::make_pair(
+          partition, CoalescedBuffer(false, partition, other_rank, b_type, comm, pmesh)));
       auto &coal_buf = coalesced_bufs.at(partition);
 
       for (int b = 0; b < nbuf; ++b) {
@@ -401,7 +400,7 @@ void CoalescedComms::AddSendBuffer(int partition, MeshBlock *pmb, const Neighbor
     coalesced_send_buffers.emplace(
         std::make_pair(std::make_pair(nb.rank, b_type),
                        CoalescedBuffersRank(nb.rank, b_type, true,
-                                            comms_[GetAssociatedSender(b_type)], pmesh)));
+                                            comms[GetAssociatedSender(b_type)], pmesh)));
   coalesced_send_buffers.at({nb.rank, b_type}).AddSendBuffer(partition, pmb, nb, var);
 }
 
@@ -416,7 +415,7 @@ void CoalescedComms::AddRecvBuffer(MeshBlock *pmb, const NeighborBlock &nb,
     coalesced_recv_buffers.emplace(
         std::make_pair(std::make_pair(nb.rank, b_type),
                        CoalescedBuffersRank(nb.rank, b_type, false,
-                                            comms_[GetAssociatedSender(b_type)], pmesh)));
+                                            comms[GetAssociatedSender(b_type)], pmesh)));
 }
 
 //----------------------------------------------------------------------------------------

src/bvals/comms/coalesced_buffers.hpp

@@ -41,6 +41,7 @@ struct uid_set_hash {
     std::size_t lhs{0};
     for (const auto &uid : in) {
       std::size_t rhs = std::hash<Uid_t>()(uid);
+      // Use the Boost hash function for lack of a better idea
       lhs ^= rhs + 0x9e3779b9 + (lhs << 6) + (lhs >> 2);
     }
     return lhs;
@@ -54,15 +55,17 @@ struct CoalescedBuffer {
   BoundaryType b_type;
   int other_rank;
   int partition;
-  mpi_comm_t comm_;
+  mpi_comm_t comm;
   Mesh *pmesh;
   bool sender;
 
   using var_buf_t = CommBuffer<buf_pool_t<Real>::owner_t>;
   std::map<Uid_t, std::vector<std::pair<BndId, var_buf_t *>>> coalesced_info_buf;
   std::set<Uid_t> all_vars;
-  std::unordered_map<std::set<Uid_t>, ParArray1DRaw<BndId>, uid_set_hash> bnd_ids_device_map;
-  std::unordered_map<std::set<Uid_t>, ParArray1DRaw<BndId>::host_mirror_type, uid_set_hash>
+  std::unordered_map<std::set<Uid_t>, ParArray1DRaw<BndId>, uid_set_hash>
+      bnd_ids_device_map;
+  std::unordered_map<std::set<Uid_t>, ParArray1DRaw<BndId>::host_mirror_type,
+                     uid_set_hash>
       bnd_ids_host_map;
   CommBuffer<buf_t> coalesced_comm_buffer;
   CommBuffer<std::vector<int>> sparse_status_buffer;
@@ -71,7 +74,7 @@ struct CoalescedBuffer {
   CoalescedBuffer(bool sender, int partition, int other_rank, BoundaryType b_type,
                   mpi_comm_t comm, Mesh *pmesh)
       : sender(sender), partition(partition), other_rank(other_rank), b_type(b_type),
-        comm_(comm), pmesh(pmesh), current_size(0) {}
+        comm(comm), pmesh(pmesh), current_size(0) {}
 
   int TotalBuffers() const {
     int total_buffers{0};
@@ -93,7 +96,7 @@ struct CoalescedBuffer {
   }
 
   ParArray1DRaw<BndId> &GetBndIdsOnDevice(const std::set<Uid_t> &vars,
-                                       int *pcomb_size = nullptr);
+                                          int *pcomb_size = nullptr);
 
   void PackAndSend(const std::set<Uid_t> &vars);
 
@@ -120,7 +123,7 @@ struct CoalescedBuffersRank {
   using com_buf_t = CommBuffer<std::vector<int>>;
   com_buf_t message;
 
-  mpi_comm_t comm_;
+  mpi_comm_t comm;
   Mesh *pmesh;
   bool sender{true};
 
@@ -146,7 +149,7 @@ struct CoalescedComms {
   std::map<std::pair<int, BoundaryType>, CoalescedBuffersRank> coalesced_send_buffers;
   std::map<std::pair<int, BoundaryType>, CoalescedBuffersRank> coalesced_recv_buffers;
 
-  std::map<BoundaryType, mpi_comm_t> comms_;
+  std::map<BoundaryType, mpi_comm_t> comms;
 
   Mesh *pmesh;
 
@@ -155,7 +158,7 @@ struct CoalescedComms {
     for (auto b_type :
          {BoundaryType::any, BoundaryType::flxcor_send, BoundaryType::gmg_same,
           BoundaryType::gmg_restrict_send, BoundaryType::gmg_prolongate_send}) {
-      auto &comm = comms_[b_type];
+      auto &comm = comms[b_type];
 #ifdef MPI_PARALLEL
       PARTHENON_MPI_CHECK(MPI_Comm_dup(MPI_COMM_WORLD, &comm));
 #else
@@ -166,7 +169,7 @@ struct CoalescedComms {
 
   ~CoalescedComms() {
 #ifdef MPI_PARALLEL
-    for (auto &[b_type, comm] : comms_)
+    for (auto &[b_type, comm] : comms)
       PARTHENON_MPI_CHECK(MPI_Comm_free(&comm));
 #endif
   }

src/mesh/mesh.hpp

@@ -131,7 +131,7 @@ class Mesh {
   int step_since_lb;
   int gflag;
 
-  bool do_coalesced_comms;
+  const bool do_coalesced_comms;
 
   BlockList_t block_list;
   Packages_t packages;