Skip to content

Commit 43c4efa

Browse files
lroberts36lroberts36
committed
split things up
1 parent 451b246 commit 43c4efa

File tree

8 files changed

+185
-93
lines changed

8 files changed

+185
-93
lines changed

‎src/CMakeLists.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,8 @@ add_library(parthenon
101101
bvals/comms/bvals_in_one.hpp
102102
bvals/comms/bvals_utils.hpp
103103
bvals/comms/build_boundary_buffers.cpp
104+
bvals/comms/bnd_id.cpp
105+
bvals/comms/bnd_id.hpp
104106
bvals/comms/bnd_info.cpp
105107
bvals/comms/bnd_info.hpp
106108
bvals/comms/boundary_communication.cpp

‎src/bvals/comms/bnd_id.cpp

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
//========================================================================================
2+
// Parthenon performance portable AMR framework
3+
// Copyright(C) 2024 The Parthenon collaboration
4+
// Licensed under the 3-clause BSD License, see LICENSE file for details
5+
//========================================================================================
6+
// (C) (or copyright) 2020-2024. Triad National Security, LLC. All rights reserved.
7+
//
8+
// This program was produced under U.S. Government contract 89233218CNA000001 for Los
9+
// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
10+
// for the U.S. Department of Energy/National Nuclear Security Administration. All rights
11+
// in the program are reserved by Triad National Security, LLC, and the U.S. Department
12+
// of Energy/National Nuclear Security Administration. The Government is granted for
13+
// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide
14+
// license in this material to reproduce, prepare derivative works, distribute copies to
15+
// the public, perform publicly and display publicly, and to permit others to do so.
16+
//========================================================================================
17+
18+
#include <algorithm>
19+
#include <cstdio>
20+
#include <iostream> // debug
21+
#include <memory>
22+
#include <string>
23+
#include <vector>
24+
25+
#include "basic_types.hpp"
26+
#include "bvals/comms/bnd_id.hpp"
27+
#include "bvals/comms/bvals_utils.hpp"
28+
#include "bvals/neighbor_block.hpp"
29+
#include "config.hpp"
30+
#include "globals.hpp"
31+
#include "interface/state_descriptor.hpp"
32+
#include "interface/variable.hpp"
33+
#include "kokkos_abstraction.hpp"
34+
#include "mesh/domain.hpp"
35+
#include "mesh/mesh.hpp"
36+
#include "mesh/mesh_refinement.hpp"
37+
#include "mesh/meshblock.hpp"
38+
#include "prolong_restrict/prolong_restrict.hpp"
39+
#include "utils/error_checking.hpp"
40+
41+
namespace parthenon {
42+
43+
BndId BndId::GetSend(MeshBlock *pmb, const NeighborBlock &nb,
44+
std::shared_ptr<Variable<Real>> v, BoundaryType b_type,
45+
int partition, int start_idx) {
46+
auto [send_gid, recv_gid, vlabel, loc, extra_id] = SendKey(pmb, nb, v, b_type);
47+
BndId out;
48+
out.send_gid() = send_gid;
49+
out.recv_gid() = recv_gid;
50+
out.loc_idx() = loc;
51+
out.var_id() = v->GetUniqueID();
52+
out.extra_id() = extra_id;
53+
out.rank_send() = Globals::my_rank;
54+
out.rank_recv() = nb.rank;
55+
out.partition() = partition;
56+
out.size() = BndInfo::GetSendBndInfo(pmb, nb, v, nullptr).size();
57+
out.start_idx() = start_idx;
58+
return out;
59+
}
60+
61+
void BndId::PrintInfo(const std::string &start) {
62+
printf("%s var %s (%i -> %i) starting at %i with size %i (Total combined buffer size = "
63+
"%i, buffer size = %i, buf_allocated = %i) [rank = %i]\n",
64+
start.c_str(), Variable<Real>::GetLabel(var_id()).c_str(), send_gid(),
65+
recv_gid(), start_idx(), size(), coalesced_buf.size(), buf.size(), buf_allocated,
66+
Globals::my_rank);
67+
}
68+
69+
} // namespace parthenon

‎src/bvals/comms/bnd_id.hpp

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
//========================================================================================
2+
// Parthenon performance portable AMR framework
3+
// Copyright(C) 2020 The Parthenon collaboration
4+
// Licensed under the 3-clause BSD License, see LICENSE file for details
5+
//========================================================================================
6+
// (C) (or copyright) 2020-2024. Triad National Security, LLC. All rights reserved.
7+
//
8+
// This program was produced under U.S. Government contract 89233218CNA000001 for Los
9+
// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
10+
// for the U.S. Department of Energy/National Nuclear Security Administration. All rights
11+
// in the program are reserved by Triad National Security, LLC, and the U.S. Department
12+
// of Energy/National Nuclear Security Administration. The Government is granted for
13+
// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide
14+
// license in this material to reproduce, prepare derivative works, distribute copies to
15+
// the public, perform publicly and display publicly, and to permit others to do so.
16+
//========================================================================================
17+
18+
#ifndef BVALS_COMMS_BND_ID_HPP_
19+
#define BVALS_COMMS_BND_ID_HPP_
20+
21+
#include <memory>
22+
#include <string>
23+
#include <vector>
24+
25+
#include "basic_types.hpp"
26+
#include "bvals/neighbor_block.hpp"
27+
#include "coordinates/coordinates.hpp"
28+
#include "interface/variable_state.hpp"
29+
#include "mesh/domain.hpp"
30+
#include "mesh/forest/logical_coordinate_transformation.hpp"
31+
#include "utils/communication_buffer.hpp"
32+
#include "utils/indexer.hpp"
33+
#include "utils/object_pool.hpp"
34+
35+
namespace parthenon {
36+
37+
template <typename T>
38+
class Variable;
39+
40+
// Provides the information necessary for identifying a unique variable-boundary
41+
// buffer, identifying the coalesced buffer it is associated with, and its
42+
// position within the coalesced buffer.
43+
struct BndId {
44+
constexpr static std::size_t NDAT = 10;
45+
int data[NDAT];
46+
47+
// Information for identifying the buffer with a communication
48+
// channel, variable, and the ranks it is communicated across
49+
KOKKOS_FORCEINLINE_FUNCTION
50+
int &send_gid() { return data[0]; }
51+
KOKKOS_FORCEINLINE_FUNCTION
52+
int &recv_gid() { return data[1]; }
53+
KOKKOS_FORCEINLINE_FUNCTION
54+
int &loc_idx() { return data[2]; }
55+
KOKKOS_FORCEINLINE_FUNCTION
56+
int &var_id() { return data[3]; }
57+
KOKKOS_FORCEINLINE_FUNCTION
58+
int &extra_id() { return data[4]; }
59+
KOKKOS_FORCEINLINE_FUNCTION
60+
int &rank_send() { return data[5]; }
61+
KOKKOS_FORCEINLINE_FUNCTION
62+
int &rank_recv() { return data[6]; }
63+
BoundaryType bound_type;
64+
65+
// MeshData partition id of the *sender*
66+
// not set by constructors and only necessary for coalesced comms
67+
KOKKOS_FORCEINLINE_FUNCTION
68+
int &partition() { return data[7]; }
69+
KOKKOS_FORCEINLINE_FUNCTION
70+
int &size() { return data[8]; }
71+
KOKKOS_FORCEINLINE_FUNCTION
72+
int &start_idx() { return data[9]; }
73+
74+
bool buf_allocated;
75+
buf_pool_t<Real>::weak_t buf; // comm buffer from pool
76+
BufArray1D<Real> coalesced_buf; // Combined buffer
77+
78+
void PrintInfo(const std::string &start);
79+
80+
KOKKOS_DEFAULTED_FUNCTION
81+
BndId() = default;
82+
KOKKOS_DEFAULTED_FUNCTION
83+
BndId(const BndId &) = default;
84+
85+
explicit BndId(const int *const data_in) {
86+
for (int i = 0; i < NDAT; ++i) {
87+
data[i] = data_in[i];
88+
}
89+
}
90+
91+
void Serialize(int *data_out) {
92+
for (int i = 0; i < NDAT; ++i) {
93+
data_out[i] = data[i];
94+
}
95+
}
96+
97+
bool SameBVChannel(const BndId &other) {
98+
// Don't want to compare start_idx, so -1
99+
for (int i = 0; i < NDAT - 1; ++i) {
100+
if (data[i] != other.data[i]) return false;
101+
}
102+
return true;
103+
}
104+
105+
static BndId GetSend(MeshBlock *pmb, const NeighborBlock &nb,
106+
std::shared_ptr<Variable<Real>> v, BoundaryType b_type,
107+
int partition, int start_idx);
108+
};
109+
} // namespace parthenon
110+
111+
#endif // BVALS_COMMS_BND_ID_HPP_

‎src/bvals/comms/bnd_info.cpp

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -306,32 +306,6 @@ BndInfo::BndInfo(MeshBlock *pmb, const NeighborBlock &nb,
306306
}
307307
}
308308

309-
BndId BndId::GetSend(MeshBlock *pmb, const NeighborBlock &nb,
310-
std::shared_ptr<Variable<Real>> v, BoundaryType b_type,
311-
int partition, int start_idx) {
312-
auto [send_gid, recv_gid, vlabel, loc, extra_id] = SendKey(pmb, nb, v, b_type);
313-
BndId out;
314-
out.send_gid() = send_gid;
315-
out.recv_gid() = recv_gid;
316-
out.loc_idx() = loc;
317-
out.var_id() = v->GetUniqueID();
318-
out.extra_id() = extra_id;
319-
out.rank_send() = Globals::my_rank;
320-
out.rank_recv() = nb.rank;
321-
out.partition() = partition;
322-
out.size() = BndInfo::GetSendBndInfo(pmb, nb, v, nullptr).size();
323-
out.start_idx() = start_idx;
324-
return out;
325-
}
326-
327-
void BndId::PrintInfo(const std::string &start) {
328-
printf("%s var %s (%i -> %i) starting at %i with size %i (Total combined buffer size = "
329-
"%i, buffer size = %i, buf_allocated = %i) [rank = %i]\n",
330-
start.c_str(), Variable<Real>::GetLabel(var_id()).c_str(), send_gid(),
331-
recv_gid(), start_idx(), size(), coalesced_buf.size(), buf.size(), buf_allocated,
332-
Globals::my_rank);
333-
}
334-
335309
BndInfo BndInfo::GetSendBndInfo(MeshBlock *pmb, const NeighborBlock &nb,
336310
std::shared_ptr<Variable<Real>> v,
337311
CommBuffer<buf_pool_t<Real>::owner_t> *buf) {

‎src/bvals/comms/bnd_info.hpp

Lines changed: 0 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -48,73 +48,6 @@ enum class IndexRangeType {
4848
InteriorRecv
4949
};
5050

51-
struct BndId {
52-
constexpr static std::size_t NDAT = 10;
53-
int data[NDAT];
54-
55-
// Information for identifying the buffer with a communication
56-
// channel, variable, and the ranks it is communicated across
57-
KOKKOS_FORCEINLINE_FUNCTION
58-
int &send_gid() { return data[0]; }
59-
KOKKOS_FORCEINLINE_FUNCTION
60-
int &recv_gid() { return data[1]; }
61-
KOKKOS_FORCEINLINE_FUNCTION
62-
int &loc_idx() { return data[2]; }
63-
KOKKOS_FORCEINLINE_FUNCTION
64-
int &var_id() { return data[3]; }
65-
KOKKOS_FORCEINLINE_FUNCTION
66-
int &extra_id() { return data[4]; }
67-
KOKKOS_FORCEINLINE_FUNCTION
68-
int &rank_send() { return data[5]; }
69-
KOKKOS_FORCEINLINE_FUNCTION
70-
int &rank_recv() { return data[6]; }
71-
BoundaryType bound_type;
72-
73-
// MeshData partition id of the *sender*
74-
// not set by constructors and only necessary for coalesced comms
75-
KOKKOS_FORCEINLINE_FUNCTION
76-
int &partition() { return data[7]; }
77-
KOKKOS_FORCEINLINE_FUNCTION
78-
int &size() { return data[8]; }
79-
KOKKOS_FORCEINLINE_FUNCTION
80-
int &start_idx() { return data[9]; }
81-
82-
bool buf_allocated;
83-
buf_pool_t<Real>::weak_t buf; // comm buffer from pool
84-
BufArray1D<Real> coalesced_buf; // Combined buffer
85-
86-
void PrintInfo(const std::string &start);
87-
88-
KOKKOS_DEFAULTED_FUNCTION
89-
BndId() = default;
90-
KOKKOS_DEFAULTED_FUNCTION
91-
BndId(const BndId &) = default;
92-
93-
explicit BndId(const int *const data_in) {
94-
for (int i = 0; i < NDAT; ++i) {
95-
data[i] = data_in[i];
96-
}
97-
}
98-
99-
void Serialize(int *data_out) {
100-
for (int i = 0; i < NDAT; ++i) {
101-
data_out[i] = data[i];
102-
}
103-
}
104-
105-
bool SameBVChannel(const BndId &other) {
106-
// Don't want to compare start_idx, so -1
107-
for (int i = 0; i < NDAT - 1; ++i) {
108-
if (data[i] != other.data[i]) return false;
109-
}
110-
return true;
111-
}
112-
113-
static BndId GetSend(MeshBlock *pmb, const NeighborBlock &nb,
114-
std::shared_ptr<Variable<Real>> v, BoundaryType b_type,
115-
int partition, int start_idx);
116-
};
117-
11851
struct BndInfo {
11952
int ntopological_elements = 1;
12053
using TE = TopologicalElement;

‎src/bvals/comms/bvals_utils.hpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
#include <utility>
2626
#include <vector>
2727

28+
#include "bvals/comms/bnd_id.hpp"
2829
#include "bvals/comms/bnd_info.hpp"
2930
#include "bvals/comms/bvals_in_one.hpp"
3031
#include "interface/variable.hpp"

‎src/bvals/comms/coalesced_buffers.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
#include <vector>
2020

2121
#include "basic_types.hpp"
22+
#include "bvals/comms/bnd_id.hpp"
2223
#include "bvals/comms/bvals_utils.hpp"
2324
#include "bvals/comms/coalesced_buffers.hpp"
2425
#include "bvals/neighbor_block.hpp"

‎src/bvals/comms/coalesced_buffers.hpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include <vector>
2323

2424
#include "basic_types.hpp"
25+
#include "bvals/comms/bnd_id.hpp"
2526
#include "bvals/comms/bvals_utils.hpp"
2627
#include "bvals/neighbor_block.hpp"
2728
#include "coordinates/coordinates.hpp"

0 commit comments

Comments
 (0)