Skip to content

Commit 2d50266

Browse files
yfguoyfguo
committed
ch4/shm: add fast path for host buffer
Check if source buffer is on host and choose typerep fast path for H2H.
1 parent 174033a commit 2d50266

File tree

6 files changed

+47
-12
lines changed

6 files changed

+47
-12
lines changed

‎src/mpid/ch4/shm/posix/eager/iqueue/iqueue_send.h

+13-8
Original file line numberDiff line numberDiff line change
@@ -95,9 +95,11 @@ MPIDI_POSIX_eager_send(int grank, MPIDI_POSIX_am_header_t * msg_hdr, const void
9595
cell->type = MPIDI_POSIX_EAGER_IQUEUE_CELL_TYPE_HDR;
9696
/* send am_hdr if this is the first segment */
9797
if (is_topo_local) {
98-
MPIR_Typerep_copy(payload, am_hdr, am_hdr_sz, MPIR_TYPEREP_FLAG_NONE);
98+
MPIR_Typerep_copy(payload, am_hdr, am_hdr_sz,
99+
MPIR_TYPEREP_FLAG_H2H | MPIR_TYPEREP_FLAG_NONE);
99100
} else {
100-
MPIR_Typerep_copy(payload, am_hdr, am_hdr_sz, MPIR_TYPEREP_FLAG_STREAM);
101+
MPIR_Typerep_copy(payload, am_hdr, am_hdr_sz,
102+
MPIR_TYPEREP_FLAG_H2H | MPIR_TYPEREP_FLAG_STREAM);
101103
}
102104
/* make sure the data region starts at the boundary of MAX_ALIGNMENT */
103105
payload = payload + resized_am_hdr_sz;
@@ -114,13 +116,16 @@ MPIDI_POSIX_eager_send(int grank, MPIDI_POSIX_am_header_t * msg_hdr, const void
114116
* not reliable because the derived datatype could have zero block size which contains no
115117
* data. */
116118
if (bytes_sent) {
117-
if (is_topo_local) {
118-
MPIR_Typerep_pack(buf, count, datatype, offset, payload, available, &packed_size,
119-
MPIR_TYPEREP_FLAG_NONE);
120-
} else {
121-
MPIR_Typerep_pack(buf, count, datatype, offset, payload, available, &packed_size,
122-
MPIR_TYPEREP_FLAG_STREAM);
119+
int typerep_flags = MPIR_TYPEREP_NONE;
120+
if (msg_hdr == MPIDI_POSIX_AM_TYPE__SHORT_HOST
121+
|| msg_hdr == MPIDI_POSIX_AM_TYPE__PIPELINE_HOST) {
122+
typerep_flags |= MPIR_TYPEREP_FLAG_H2H;
123+
}
124+
if (!is_topo_local) {
125+
typerep_flags |= MPIR_TYPEREP_FLAG_STREAM;
123126
}
127+
MPIR_Typerep_pack(buf, count, datatype, offset, payload, available, &packed_size,
128+
typerep_flags);
124129
cell->payload_size += packed_size;
125130
*bytes_sent = packed_size;
126131
}

‎src/mpid/ch4/shm/posix/posix_am.h

+14-2
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@
1111
#include "posix_eager.h"
1212
#include "mpidu_genq.h"
1313

14+
#undef IS_HOST
15+
#define IS_HOST(attr) \
16+
((attr).type & (MPL_GPU_POINTER_UNREGISTERED_HOST | MPL_GPU_POINTER_REGISTERED_HOST))
17+
1418
MPL_STATIC_INLINE_PREFIX MPI_Aint MPIDI_POSIX_am_eager_limit(void)
1519
{
1620
return MPIDI_POSIX_eager_payload_limit() - MAX_ALIGNMENT;
@@ -284,9 +288,17 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_do_am_isend(int grank,
284288

285289
msg_hdr_p = msg_hdr;
286290
if (data_sz + am_hdr_sz <= MPIDI_POSIX_am_eager_limit()) {
287-
msg_hdr_p->am_type = MPIDI_POSIX_AM_TYPE__SHORT;
291+
if (IS_HOST(MPIDIG_REQUEST(sreq, buf_attr))) {
292+
msg_hdr_p->am_type = MPIDI_POSIX_AM_TYPE__SHORT_HOST;
293+
} else {
294+
msg_hdr_p->am_type = MPIDI_POSIX_AM_TYPE__SHORT;
295+
}
288296
} else {
289-
msg_hdr_p->am_type = MPIDI_POSIX_AM_TYPE__PIPELINE;
297+
if (IS_HOST(MPIDIG_REQUEST(sreq, buf_attr))) {
298+
msg_hdr_p->am_type = MPIDI_POSIX_AM_TYPE__PIPELINE_HOST;
299+
} else {
300+
msg_hdr_p->am_type = MPIDI_POSIX_AM_TYPE__PIPELINE;
301+
}
290302
}
291303

292304
MPIDIG_am_send_async_init(sreq, datatype, data_sz);

‎src/mpid/ch4/shm/posix/posix_pre.h

+3-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,9 @@ typedef enum {
2222
typedef enum {
2323
MPIDI_POSIX_AM_TYPE__HDR,
2424
MPIDI_POSIX_AM_TYPE__SHORT,
25-
MPIDI_POSIX_AM_TYPE__PIPELINE
25+
MPIDI_POSIX_AM_TYPE__SHORT_HOST,
26+
MPIDI_POSIX_AM_TYPE__PIPELINE,
27+
MPIDI_POSIX_AM_TYPE__PIPELINE_HOST
2628
} MPIDI_POSIX_am_type_t;
2729

2830
struct MPIR_Request;

‎src/mpid/ch4/shm/posix/posix_progress.h

+2
Original file line numberDiff line numberDiff line change
@@ -57,11 +57,13 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_progress_recv(int vci, int *made_progre
5757
switch (msg_hdr->am_type) {
5858
case MPIDI_POSIX_AM_TYPE__HDR:
5959
case MPIDI_POSIX_AM_TYPE__SHORT:
60+
case MPIDI_POSIX_AM_TYPE__SHORT_HOST:
6061
MPIDIG_global.target_msg_cbs[msg_hdr->handler_id] (am_hdr, payload, payload_left,
6162
attr, NULL);
6263
MPIDI_POSIX_eager_recv_commit(&transaction);
6364
goto fn_exit;
6465
case MPIDI_POSIX_AM_TYPE__PIPELINE:
66+
case MPIDI_POSIX_AM_TYPE__PIPELINE_HOST:
6567
MPIDIG_global.target_msg_cbs[msg_hdr->handler_id] (am_hdr, NULL, payload_left,
6668
attr | MPIDIG_AM_ATTR__IS_ASYNC,
6769
&rreq);

‎src/mpid/ch4/shm/posix/posix_send.h

+11-1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,10 @@
1616

1717
#include "posix_impl.h"
1818

19+
#undef IS_HOST
20+
#define IS_HOST(attr) \
21+
((attr).type & (MPL_GPU_POINTER_UNREGISTERED_HOST | MPL_GPU_POINTER_REGISTERED_HOST))
22+
1923
#define MPIDI_POSIX_SEND_VSIS(vci_src_, vci_dst_) \
2024
do { \
2125
MPIDI_EXPLICIT_VCIS(comm, attr, comm->rank, rank, vci_src_, vci_dst_); \
@@ -52,7 +56,13 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_isend(const void *buf, MPI_Aint cou
5256
MPIDI_POSIX_am_header_t msg_hdr;
5357
msg_hdr.handler_id = MPIDIG_SEND;
5458
msg_hdr.am_hdr_sz = sizeof(MPIDIG_hdr_t);
55-
msg_hdr.am_type = MPIDI_POSIX_AM_TYPE__SHORT;
59+
MPL_pointer_attr_t attr;
60+
MPIR_GPU_query_pointer_attr(buf, &attr);
61+
if (IS_HOST(attr)) {
62+
msg_hdr.am_type = MPIDI_POSIX_AM_TYPE__SHORT_HOST;
63+
} else {
64+
msg_hdr.am_type = MPIDI_POSIX_AM_TYPE__SHORT;
65+
}
5666

5767
MPIDIG_hdr_t am_hdr;
5868
am_hdr.src_rank = comm->rank;

‎src/mpid/ch4/src/mpidig_send_utils.h

+4
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@
66
#ifndef MPIDIG_SEND_UTILS_H_INCLUDED
77
#define MPIDIG_SEND_UTILS_H_INCLUDED
88

9+
#undef
10+
#define IS_HOST(attr) \
11+
((attr).type & (MPL_GPU_POINTER_UNREGISTERED_HOST | MPL_GPU_POINTER_REGISTERED_HOST))
12+
913
/* This file is for supporting routines used for pipelined data send. These routines mainly is for
1014
* managing the send request counters, completion counters and DT refcount */
1115

0 commit comments

Comments
 (0)