ch4: replace MPIDIU_get_av

We no longer expose avtid. Replace MPIDIU_get_av with MPIDIU_lpid_to_av.

Also remote unused GPID macros.

Author: hzhou

src/mpid/ch4/include/mpidpre.h

@@ -637,17 +637,6 @@ typedef struct MPIDI_av_entry {
 
 #define HAVE_DEV_COMM_HOOK
 
-/*
- * operation for (avtid, lpid) to/from gpid
- */
-#define MPIDIU_LPID_BITS                     32
-#define MPIDIU_LPID_MASK                     0xFFFFFFFFU
-#define MPIDIU_GPID_CREATE(avtid, lpid)      (((uint64_t) (avtid) << MPIDIU_LPID_BITS) | (lpid))
-#define MPIDIU_GPID_GET_AVTID(gpid)          ((gpid) >> MPIDIU_LPID_BITS)
-#define MPIDIU_GPID_GET_LPID(gpid)           ((gpid) & MPIDIU_LPID_MASK)
-
-#define MPIDI_DYNPROC_MASK                 (0x80000000U)
-
 int MPIDI_check_for_failed_procs(void);
 
 #ifdef HAVE_SIGNAL

src/mpid/ch4/netmod/ofi/init_addrxchg.c

@@ -133,7 +133,7 @@ int MPIDI_OFI_addr_exchange_root_ctx(void)
 
         for (int i = 0; i < num_nodes; i++) {
             MPIR_Assert(mapped_table[i] != FI_ADDR_NOTAVAIL);
-            MPIDI_OFI_AV(&MPIDIU_get_av(0, node_roots[i])).dest[0][0] = mapped_table[i];
+            MPIDI_OFI_AV(MPIDIU_lpid_to_av(node_roots[i])).dest[0][0] = mapped_table[i];
         }
         MPL_free(mapped_table);
         /* Then, allgather all address names using init_comm */
@@ -149,7 +149,7 @@ int MPIDI_OFI_addr_exchange_root_ctx(void)
                 char *addrname = (char *) table + recv_bc_len * rank_map[i];
                 MPIDI_OFI_CALL(fi_av_insert(MPIDI_OFI_global.ctx[0].av,
                                             addrname, 1, &addr, 0ULL, NULL), avmap);
-                MPIDI_OFI_AV(&MPIDIU_get_av(0, i)).dest[0][0] = addr;
+                MPIDI_OFI_AV(MPIDIU_lpid_to_av(i)).dest[0][0] = addr;
             }
         }
         mpi_errno = MPIDU_bc_table_destroy();
@@ -163,7 +163,7 @@ int MPIDI_OFI_addr_exchange_root_ctx(void)
 
         for (int i = 0; i < size; i++) {
             MPIR_Assert(mapped_table[i] != FI_ADDR_NOTAVAIL);
-            MPIDI_OFI_AV(&MPIDIU_get_av(0, i)).dest[0][0] = mapped_table[i];
+            MPIDI_OFI_AV(MPIDIU_lpid_to_av(i)).dest[0][0] = mapped_table[i];
         }
         MPL_free(mapped_table);
         mpi_errno = MPIDU_bc_table_destroy();
@@ -173,7 +173,7 @@ int MPIDI_OFI_addr_exchange_root_ctx(void)
     /* check */
     if (MPIDI_OFI_ENABLE_AV_TABLE) {
         for (int r = 0; r < size; r++) {
-            MPIDI_OFI_addr_t *av ATTRIBUTE((unused)) = &MPIDI_OFI_AV(&MPIDIU_get_av(0, r));
+            MPIDI_OFI_addr_t *av ATTRIBUTE((unused)) = &MPIDI_OFI_AV(MPIDIU_lpid_to_av(r));
             MPIR_Assert(av->dest[0][0] == get_root_av_table_index(r));
         }
     }
@@ -192,7 +192,7 @@ int MPIDI_OFI_addr_exchange_root_ctx(void)
 /* Macros to reduce clutter, so we can focus on the ordering logics.
  * Note: they are not perfectly wrapped, but tolerable since only used here. */
 #define GET_AV_AND_ADDRNAMES(rank) \
-    MPIDI_OFI_addr_t *av ATTRIBUTE((unused)) = &MPIDI_OFI_AV(&MPIDIU_get_av(0, rank)); \
+    MPIDI_OFI_addr_t *av ATTRIBUTE((unused)) = &MPIDI_OFI_AV(MPIDIU_lpid_to_av(rank)); \
     char *r_names = all_names + rank * max_vcis * num_nics * name_len;
 
 #define DO_AV_INSERT(ctx_idx, nic, vci) \
@@ -346,7 +346,7 @@ int MPIDI_OFI_addr_exchange_all_ctx(void)
 #if MPIDI_CH4_MAX_VCIS > 1
     if (MPIDI_OFI_ENABLE_AV_TABLE) {
         for (int r = 0; r < size; r++) {
-            MPIDI_OFI_addr_t *av ATTRIBUTE((unused)) = &MPIDI_OFI_AV(&MPIDIU_get_av(0, r));
+            MPIDI_OFI_addr_t *av ATTRIBUTE((unused)) = &MPIDI_OFI_AV(MPIDIU_lpid_to_av(r));
             for (int nic = 0; nic < num_nics; nic++) {
                 for (int vci = 0; vci < NUM_VCIS_FOR_RANK(r); vci++) {
                     MPIR_Assert(av->dest[nic][vci] == get_av_table_index(r, nic, vci,

src/mpid/ch4/netmod/ofi/ofi_impl.h

@@ -31,9 +31,6 @@ ATTRIBUTE((unused));
 #define MPIDI_OFI_DT(dt)         ((dt)->dev.netmod.ofi)
 #define MPIDI_OFI_OP(op)         ((op)->dev.netmod.ofi)
 #define MPIDI_OFI_COMM(comm)     ((comm)->dev.ch4.netmod.ofi)
-#define MPIDI_OFI_TO_PHYS(avtid, lpid, _nic) \
-    MPIDI_OFI_AV(&MPIDIU_get_av((avtid), (lpid))).dest[_nic][0]
-
 #define MPIDI_OFI_WIN(win)     ((win)->dev.netmod.ofi)
 
 #define MPIDI_OFI_NIC_NAME(nic)    (MPIDI_OFI_global.prov_use[nic] ? \

src/mpid/ch4/netmod/ofi/ofi_init.c

@@ -943,7 +943,7 @@ static int flush_send(int dst, int nic, int vci, MPIDI_OFI_dynamic_process_reque
 {
     int mpi_errno = MPI_SUCCESS;
 
-    fi_addr_t addr = MPIDI_OFI_av_to_phys(&MPIDIU_get_av(0, dst), nic, vci);
+    fi_addr_t addr = MPIDI_OFI_av_to_phys(MPIDIU_lpid_to_av(dst), nic, vci);
     static int data = 0;
     uint64_t match_bits =
         MPIDI_OFI_init_sendtag(MPIDI_OFI_FLUSH_CONTEXT_ID, 0, MPIDI_OFI_FLUSH_TAG);
@@ -974,7 +974,7 @@ static int flush_recv(int src, int nic, int vci, MPIDI_OFI_dynamic_process_reque
 {
     int mpi_errno = MPI_SUCCESS;
 
-    fi_addr_t addr = MPIDI_OFI_av_to_phys(&MPIDIU_get_av(0, src), nic, vci);
+    fi_addr_t addr = MPIDI_OFI_av_to_phys(MPIDIU_lpid_to_av(src), nic, vci);
     uint64_t mask_bits = 0;
     uint64_t match_bits =
         MPIDI_OFI_init_sendtag(MPIDI_OFI_FLUSH_CONTEXT_ID, 0, MPIDI_OFI_FLUSH_TAG);
@@ -1555,10 +1555,10 @@ static int try_open_shared_av(struct fid_domain *domain, struct fid_av **p_av, i
         /* directly references the mapped fi_addr_t array instead               */
         fi_addr_t *mapped_table = (fi_addr_t *) av_attr.map_addr;
         for (int i = 0; i < MPIR_Process.size; i++) {
-            MPIDI_OFI_AV(&MPIDIU_get_av(0, i)).dest[nic][0] = mapped_table[i];
+            MPIDI_OFI_AV(MPIDIU_lpid_to_av(i)).dest[nic][0] = mapped_table[i];
             MPL_DBG_MSG_FMT(MPIDI_CH4_DBG_MAP, VERBOSE,
                             (MPL_DBG_FDEST, " grank mapped to: rank=%d, av=%p, dest=%" PRIu64,
-                             i, (void *) &MPIDIU_get_av(0, i), mapped_table[i]));
+                             i, (void *) MPIDIU_lpid_to_av(i), mapped_table[i]));
         }
         ret = 1;
     }

src/mpid/ch4/netmod/ofi/ofi_spawn.c

@@ -16,9 +16,7 @@ int MPIDI_OFI_dynamic_send(MPIR_Lpid remote_lpid, int tag, const void *buf, int
     int nic = 0;                /* dynamic process only use nic 0 */
     int vci = 0;                /* dynamic process only use vci 0 */
     int ctx_idx = 0;
-    int avtid = MPIDIU_GPID_GET_AVTID(remote_lpid);
-    int lpid = MPIDIU_GPID_GET_LPID(remote_lpid);
-    fi_addr_t remote_addr = MPIDI_OFI_av_to_phys(&MPIDIU_get_av(avtid, lpid), nic, vci);
+    fi_addr_t remote_addr = MPIDI_OFI_av_to_phys(MPIDIU_lpid_to_av_slow(remote_lpid), nic, vci);
 
     MPID_THREAD_CS_ENTER(VCI, MPIDI_VCI(vci).lock);
 

src/mpid/ch4/netmod/ucx/ucx_init.c

@@ -102,7 +102,7 @@ static int initial_address_exchange(void)
             ep_params.address = (ucp_address_t *) ((char *) table + i * recv_bc_len);
             ucx_status =
                 ucp_ep_create(MPIDI_UCX_global.ctx[0].worker, &ep_params,
-                              &MPIDI_UCX_AV(&MPIDIU_get_av(0, node_roots[i])).dest[0][0]);
+                              &MPIDI_UCX_AV(MPIDIU_lpid_to_av(node_roots[i])).dest[0][0]);
             MPIDI_UCX_CHK_STATUS(ucx_status);
             MPIDIU_upidhash_add(ep_params.address, recv_bc_len, node_roots[i]);
         }
@@ -117,7 +117,7 @@ static int initial_address_exchange(void)
                 ep_params.field_mask = UCP_EP_PARAM_FIELD_REMOTE_ADDRESS;
                 ep_params.address = (ucp_address_t *) ((char *) table + rank_map[i] * recv_bc_len);
                 ucx_status = ucp_ep_create(MPIDI_UCX_global.ctx[0].worker, &ep_params,
-                                           &MPIDI_UCX_AV(&MPIDIU_get_av(0, i)).dest[0][0]);
+                                           &MPIDI_UCX_AV(MPIDIU_lpid_to_av(i)).dest[0][0]);
                 MPIDI_UCX_CHK_STATUS(ucx_status);
                 MPIDIU_upidhash_add(ep_params.address, recv_bc_len, i);
             }
@@ -130,7 +130,7 @@ static int initial_address_exchange(void)
             ep_params.address = (ucp_address_t *) ((char *) table + i * recv_bc_len);
             ucx_status =
                 ucp_ep_create(MPIDI_UCX_global.ctx[0].worker, &ep_params,
-                              &MPIDI_UCX_AV(&MPIDIU_get_av(0, i)).dest[0][0]);
+                              &MPIDI_UCX_AV(MPIDIU_lpid_to_av(i)).dest[0][0]);
             MPIDI_UCX_CHK_STATUS(ucx_status);
             MPIDIU_upidhash_add(ep_params.address, recv_bc_len, i);
         }
@@ -180,7 +180,7 @@ static int all_vcis_address_exchange(void)
     ucp_ep_params_t ep_params;
     for (int vci_local = 0; vci_local < num_vcis; vci_local++) {
         for (int r = 0; r < size; r++) {
-            MPIDI_UCX_addr_t *av = &MPIDI_UCX_AV(&MPIDIU_get_av(0, r));
+            MPIDI_UCX_addr_t *av = &MPIDI_UCX_AV(MPIDIU_lpid_to_av(r));
             for (int vci_remote = 0; vci_remote < num_vcis; vci_remote++) {
                 if (vci_local == 0 && vci_remote == 0) {
                     /* don't overwrite existing addr, or bad things will happen */
@@ -369,7 +369,7 @@ int MPIDI_UCX_mpi_finalize_hook(void)
 
     int p = 0;
     for (int i = 0; i < MPIR_Process.size; i++) {
-        MPIDI_UCX_addr_t *av = &MPIDI_UCX_AV(&MPIDIU_get_av(0, i));
+        MPIDI_UCX_addr_t *av = &MPIDI_UCX_AV(MPIDIU_lpid_to_av(i));
         for (int vci_local = 0; vci_local < MPIDI_UCX_global.num_vcis; vci_local++) {
             for (int vci_remote = 0; vci_remote < MPIDI_UCX_global.num_vcis; vci_remote++) {
                 ucp_request = ucp_disconnect_nb(av->dest[vci_local][vci_remote]);

src/mpid/ch4/netmod/ucx/ucx_spawn.c

@@ -29,9 +29,7 @@ int MPIDI_UCX_dynamic_send(MPIR_Lpid remote_lpid, int tag, const void *buf, int
 
     MPID_THREAD_CS_ENTER(VCI, MPIDI_VCI(vci).lock);
 
-    int avtid = MPIDIU_GPID_GET_AVTID(remote_lpid);
-    int lpid = MPIDIU_GPID_GET_LPID(remote_lpid);
-    ucp_ep_h ep = MPIDI_UCX_AV_TO_EP(&MPIDIU_get_av(avtid, lpid), vci, vci);
+    ucp_ep_h ep = MPIDI_UCX_AV_TO_EP(MPIDIU_lpid_to_av(remote_lpid), vci, vci);
 
     bool done = false;
     ucp_request_param_t param = {

src/mpid/ch4/src/ch4_types.h

@@ -214,9 +214,6 @@ typedef struct {
     MPIDI_dyn_av_table_t dynamic_av_table;
 } MPIDIU_avt_manager;
 
-#define MPIDIU_get_av_table(avtid) (MPIDI_global.avt_mgr.av_tables[(avtid)])
-#define MPIDIU_get_av(avtid, lpid) (MPIDI_global.avt_mgr.av_tables[(avtid)]->table[(lpid)])
-
 typedef struct {
     uint64_t key;
     void *value;