ch4: separate multi-vci/nic initialization

src/include/mpir_comm.h

@@ -212,11 +212,8 @@ struct MPIR_Comm {
      * because context_id is non-sequential and can't be used to identify user-level
      * communicators (due to sub-comms). */
     int seq;
-    /* Certain comm and its offsprings should be restricted to sequence 0 due to
-     * various restrictions. E.g. multiple-vci doesn't support dynamic process,
-     * nor intercomms (even after its merge).
-     */
-    int tainted;
+    /* Whether multiple-vci is enabled. This is ONLY inherited in Comm_dup and Comm_split */
+    bool vcis_enabled;
 
 
     int hints[MPIR_COMM_HINT_MAX];      /* Hints to the communicator

src/include/mpir_objects.h

@@ -526,12 +526,6 @@ typedef struct MPIR_Object_alloc_t {
     void *direct;               /* Pointer to direct block, used
                                  * for allocation */
     int direct_size;            /* Size of direct block */
-    void *lock;                 /* lower-layer may register a lock to use. This is
-                                 * mostly for multipool requests. For other objects
-                                 * or not per-vci thread granularity, this lock
-                                 * pointer is ignored. Ref. mpir_request.h.
-                                 * NOTE: it is `void *` because mutex type not defined yet.
-                                 */
     /* The following padding is to avoid cache line sharing with other MPIR_Object_alloc_t.  This
      * padding is particularly important for an array of per-vci MPI_Request pools. */
     char pad[MPL_CACHELINE_SIZE];

src/include/mpir_request.h

@@ -372,15 +372,6 @@ extern MPIR_Request MPIR_Request_direct[MPIR_REQUEST_PREALLOC];
 
 void MPII_init_request(void);
 
-/* To get the benefit of multiple request pool, device layer need register their per-vci lock
- * with each pool that they are going to use, typically a 1-1 vci-pool mapping.
- * NOTE: currently, only per-vci thread granularity utilizes multiple request pool.
- */
-static inline void MPIR_Request_register_pool_lock(int pool, MPID_Thread_mutex_t * lock)
-{
-    MPIR_Request_mem[pool].lock = lock;
-}
-
 static inline int MPIR_Request_is_persistent(MPIR_Request * req_ptr)
 {
     return (req_ptr->kind == MPIR_REQUEST_KIND__PREQUEST_SEND ||
@@ -445,7 +436,7 @@ static inline MPIR_Request *MPIR_Request_create_from_pool(MPIR_Request_kind_t ki
     MPIR_Request *req;
 
 #ifdef MPICH_DEBUG_MUTEX
-    MPID_THREAD_ASSERT_IN_CS(VCI, (*(MPID_Thread_mutex_t *) MPIR_Request_mem[pool].lock));
+    MPID_THREAD_ASSERT_IN_CS(VCI, MPIR_THREAD_VCI_REQUEST_POOL_MUTEXES[pool]);
 #endif
     int max_blocks = (pool == 0) ? REQUEST_NUM_BLOCKS0 : REQUEST_NUM_BLOCKS;
     req = MPIR_Handle_obj_alloc_unsafe(&MPIR_Request_mem[pool], max_blocks, REQUEST_NUM_INDICES);
@@ -520,19 +511,19 @@ static inline MPIR_Request *MPIR_Request_create_from_pool_safe(MPIR_Request_kind
 {
     MPIR_Request *req;
 
-    MPID_THREAD_CS_ENTER(VCI, (*(MPID_Thread_mutex_t *) MPIR_Request_mem[pool].lock));
+    MPID_THREAD_CS_ENTER(VCI, MPIR_THREAD_VCI_REQUEST_POOL_MUTEXES[pool]);
     req = MPIR_Request_create_from_pool(kind, pool, ref_count);
-    MPID_THREAD_CS_EXIT(VCI, (*(MPID_Thread_mutex_t *) MPIR_Request_mem[pool].lock));
+    MPID_THREAD_CS_EXIT(VCI, MPIR_THREAD_VCI_REQUEST_POOL_MUTEXES[pool]);
     return req;
 }
 
 /* NOTE: safe under per-vci, per-obj, or global thread granularity */
 static inline MPIR_Request *MPIR_Request_create(MPIR_Request_kind_t kind)
 {
     MPIR_Request *req;
-    MPID_THREAD_CS_ENTER(VCI, (*(MPID_Thread_mutex_t *) MPIR_Request_mem[0].lock));
+    MPID_THREAD_CS_ENTER(VCI, MPIR_THREAD_VCI_REQUEST_POOL_MUTEXES[0]);
     req = MPIR_Request_create_from_pool(kind, 0, 1);
-    MPID_THREAD_CS_EXIT(VCI, (*(MPID_Thread_mutex_t *) MPIR_Request_mem[0].lock));
+    MPID_THREAD_CS_EXIT(VCI, MPIR_THREAD_VCI_REQUEST_POOL_MUTEXES[0]);
     return req;
 }
 
@@ -572,10 +563,10 @@ static inline void MPIR_Request_free_with_safety(MPIR_Request * req,
     }
 
     if (need_safety) {
-        MPID_THREAD_CS_ENTER(VCI, (*(MPID_Thread_mutex_t *) MPIR_Request_mem[pool].lock));
+        MPID_THREAD_CS_ENTER(VCI, MPIR_THREAD_VCI_REQUEST_POOL_MUTEXES[pool]);
     }
 #ifdef MPICH_DEBUG_MUTEX
-    MPID_THREAD_ASSERT_IN_CS(VCI, (*(MPID_Thread_mutex_t *) MPIR_Request_mem[pool].lock));
+    MPID_THREAD_ASSERT_IN_CS(VCI, MPIR_THREAD_VCI_REQUEST_POOL_MUTEXES[pool]);
 #endif
     /* inform the device that we are decrementing the ref-count on
      * this request */
@@ -654,7 +645,7 @@ static inline void MPIR_Request_free_with_safety(MPIR_Request * req,
         MPIR_Handle_obj_free_unsafe(&MPIR_Request_mem[pool], req, /* not info */ FALSE);
     }
     if (need_safety) {
-        MPID_THREAD_CS_EXIT(VCI, (*(MPID_Thread_mutex_t *) MPIR_Request_mem[pool].lock));
+        MPID_THREAD_CS_EXIT(VCI, MPIR_THREAD_VCI_REQUEST_POOL_MUTEXES[pool]);
     }
 }
 

src/include/mpir_thread.h

@@ -84,6 +84,7 @@ extern MPID_Thread_mutex_t MPIR_THREAD_VCI_HANDLE_MUTEX;
 extern MPID_Thread_mutex_t MPIR_THREAD_VCI_CTX_MUTEX;
 extern MPID_Thread_mutex_t MPIR_THREAD_VCI_PMI_MUTEX;
 extern MPID_Thread_mutex_t MPIR_THREAD_VCI_BSEND_MUTEX;
+extern MPID_Thread_mutex_t MPIR_THREAD_VCI_REQUEST_POOL_MUTEXES[];
 #endif /* MPICH_THREAD_GRANULARITY */
 #endif /* MPICH_IS_THREADED */
 

src/mpi/attr/attrutil.c

@@ -25,7 +25,7 @@ MPIR_Object_alloc_t MPII_Keyval_mem = { 0, 0, 0, 0, 0, 0, 0, MPIR_KEYVAL,
     sizeof(MPII_Keyval),
     MPII_Keyval_direct,
     MPID_KEYVAL_PREALLOC,
-    NULL, {0}
+    {0}
 };
 
 /* Preallocated keyval objects */
@@ -35,7 +35,7 @@ MPIR_Object_alloc_t MPID_Attr_mem = { 0, 0, 0, 0, 0, 0, 0, MPIR_ATTR,
     sizeof(MPIR_Attribute),
     MPID_Attr_direct,
     MPIR_ATTR_PREALLOC,
-    NULL, {0}
+    {0}
 };
 
 /* Provides a way to trap all attribute allocations when debugging leaks. */

src/mpi/coll/op/op_impl.c

@@ -13,7 +13,7 @@ MPIR_Object_alloc_t MPIR_Op_mem = { 0, 0, 0, 0, 0, 0, 0, MPIR_OP,
     sizeof(MPIR_Op),
     MPIR_Op_direct,
     MPIR_OP_PREALLOC,
-    NULL, {0}
+    {0}
 };
 
 int MPIR_Op_create_impl(MPI_User_function * user_fn, int commute, MPIR_Op ** p_op_ptr)

src/mpi/comm/comm_impl.c

@@ -346,7 +346,7 @@ int MPIR_Comm_create_intra(MPIR_Comm * comm_ptr, MPIR_Group * group_ptr, MPIR_Co
         mpi_errno = MPII_Comm_create_map(n, 0, mapping, NULL, mapping_comm, *newcomm_ptr);
         MPIR_ERR_CHECK(mpi_errno);
 
-        (*newcomm_ptr)->tainted = comm_ptr->tainted;
+        (*newcomm_ptr)->vcis_enabled = comm_ptr->vcis_enabled;
         mpi_errno = MPIR_Comm_commit(*newcomm_ptr);
         MPIR_ERR_CHECK(mpi_errno);
     } else {
@@ -518,7 +518,7 @@ int MPIR_Comm_create_inter(MPIR_Comm * comm_ptr, MPIR_Group * group_ptr, MPIR_Co
                                           &remote_group);
         (*newcomm_ptr)->remote_group = remote_group;
 
-        (*newcomm_ptr)->tainted = comm_ptr->tainted;
+        (*newcomm_ptr)->vcis_enabled = comm_ptr->vcis_enabled;
         mpi_errno = MPIR_Comm_commit(*newcomm_ptr);
         MPIR_ERR_CHECK(mpi_errno);
 
@@ -629,7 +629,7 @@ int MPIR_Comm_create_group_impl(MPIR_Comm * comm_ptr, MPIR_Group * group_ptr, in
         mpi_errno = MPII_Comm_create_map(n, 0, mapping, NULL, mapping_comm, *newcomm_ptr);
         MPIR_ERR_CHECK(mpi_errno);
 
-        (*newcomm_ptr)->tainted = comm_ptr->tainted;
+        (*newcomm_ptr)->vcis_enabled = comm_ptr->vcis_enabled;
         mpi_errno = MPIR_Comm_commit(*newcomm_ptr);
         MPIR_ERR_CHECK(mpi_errno);
     } else {
@@ -1082,7 +1082,6 @@ int MPIR_Intercomm_create_impl(MPIR_Comm * local_comm_ptr, int local_leader,
     }
     MPID_THREAD_CS_EXIT(VCI, local_comm_ptr->mutex);
 
-    (*new_intercomm_ptr)->tainted = 1;
     mpi_errno = MPIR_Comm_commit(*new_intercomm_ptr);
     MPIR_ERR_CHECK(mpi_errno);
 
@@ -1137,7 +1136,6 @@ int MPIR_peer_intercomm_create(int context_id, int recvcontext_id,
                                          &(*newcomm)->remote_group);
     MPIR_ERR_CHECK(mpi_errno);
 
-    (*newcomm)->tainted = 1;
     mpi_errno = MPIR_Comm_commit(*newcomm);
     MPIR_ERR_CHECK(mpi_errno);
 
@@ -1295,7 +1293,6 @@ int MPIR_Intercomm_merge_impl(MPIR_Comm * comm_ptr, int high, MPIR_Comm ** new_i
      * operations within the context id algorithm, since we already
      * have a valid (almost - see comm_create_hook) communicator.
      */
-    (*new_intracomm_ptr)->tainted = 1;
     mpi_errno = MPIR_Comm_commit((*new_intracomm_ptr));
     MPIR_ERR_CHECK(mpi_errno);
 
@@ -1328,7 +1325,6 @@ int MPIR_Intercomm_merge_impl(MPIR_Comm * comm_ptr, int high, MPIR_Comm ** new_i
     mpi_errno = create_and_map(comm_ptr, local_high, (*new_intracomm_ptr));
     MPIR_ERR_CHECK(mpi_errno);
 
-    (*new_intracomm_ptr)->tainted = 1;
     mpi_errno = MPIR_Comm_commit((*new_intracomm_ptr));
     MPIR_ERR_CHECK(mpi_errno);
 

src/mpi/comm/comm_split.c

@@ -348,7 +348,7 @@ int MPIR_Comm_split_impl(MPIR_Comm * comm_ptr, int color, int key, MPIR_Comm **
         }
         MPID_THREAD_CS_EXIT(VCI, comm_ptr->mutex);
 
-        (*newcomm_ptr)->tainted = comm_ptr->tainted;
+        (*newcomm_ptr)->vcis_enabled = comm_ptr->vcis_enabled;
         mpi_errno = MPIR_Comm_commit(*newcomm_ptr);
         MPIR_ERR_CHECK(mpi_errno);
     }

src/mpi/comm/commutil.c

@@ -22,7 +22,7 @@ MPIR_Object_alloc_t MPIR_Comm_mem = { 0, 0, 0, 0, 0, 0, 0, MPIR_COMM,
     sizeof(MPIR_Comm),
     MPIR_Comm_direct,
     MPIR_COMM_PREALLOC,
-    NULL, {0}
+    {0}
 };
 
 /* Communicator creation functions */
@@ -289,7 +289,7 @@ int MPII_Comm_init(MPIR_Comm * comm_p)
     comm_p->bsendbuffer = NULL;
     comm_p->name[0] = '\0';
     comm_p->seq = 0;    /* default to 0, to be updated at Comm_commit */
-    comm_p->tainted = 0;
+    comm_p->vcis_enabled = false;
     memset(comm_p->hints, 0, sizeof(comm_p->hints));
     for (int i = 0; i < next_comm_hint_index; i++) {
         if (MPIR_comm_hint_list[i].key) {
@@ -416,9 +416,6 @@ int MPII_Setup_intercomm_localcomm(MPIR_Comm * intercomm_ptr)
     intercomm_ptr->local_comm = localcomm_ptr;
 
     /* sets up the SMP-aware sub-communicators and tables */
-    /* This routine maybe used inside MPI_Comm_idup, so we can't synchronize
-     * seq using blocking collectives, thus mark as tainted. */
-    localcomm_ptr->tainted = 1;
     mpi_errno = MPIR_Comm_commit(localcomm_ptr);
     MPIR_ERR_CHECK(mpi_errno);
 
@@ -597,13 +594,13 @@ static void propagate_hints_to_subcomm(MPIR_Comm * comm, MPIR_Comm * subcomm)
     subcomm->hints[MPIR_COMM_HINT_VCI] = comm->hints[MPIR_COMM_HINT_VCI];
 }
 
-static void propagate_tainted_to_subcomms(MPIR_Comm * comm)
+static void propagate_vcis_enabled(MPIR_Comm * comm)
 {
     if (comm->node_comm != NULL)
-        comm->node_comm->tainted = comm->tainted;
+        comm->node_comm->vcis_enabled = comm->vcis_enabled;
 
     if (comm->node_roots_comm != NULL)
-        comm->node_roots_comm->tainted = comm->tainted;
+        comm->node_roots_comm->vcis_enabled = comm->vcis_enabled;
 }
 
 int MPIR_Comm_create_subcomms(MPIR_Comm * comm)
@@ -734,7 +731,7 @@ int MPIR_Comm_create_subcomms(MPIR_Comm * comm)
         MPIR_ERR_CHECK(mpi_errno);
     }
 
-    propagate_tainted_to_subcomms(comm);
+    propagate_vcis_enabled(comm);
 
     comm->hierarchy_kind = MPIR_COMM_HIERARCHY_KIND__PARENT;
 
@@ -840,7 +837,7 @@ int MPIR_Comm_commit(MPIR_Comm * comm)
         MPIR_ERR_CHECK(mpi_errno);
     }
 
-    if (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM && !comm->tainted) {
+    if (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM && comm->vcis_enabled) {
         mpi_errno = init_comm_seq(comm);
         MPIR_ERR_CHECK(mpi_errno);
     }
@@ -1038,7 +1035,7 @@ int MPII_Comm_copy(MPIR_Comm * comm_ptr, int size, MPIR_Info * info, MPIR_Comm *
         MPII_Comm_set_hints(newcomm_ptr, info, true);
     }
 
-    newcomm_ptr->tainted = comm_ptr->tainted;
+    newcomm_ptr->vcis_enabled = comm_ptr->vcis_enabled;
     mpi_errno = MPIR_Comm_commit(newcomm_ptr);
     MPIR_ERR_CHECK(mpi_errno);
 
@@ -1121,9 +1118,8 @@ int MPII_Comm_copy_data(MPIR_Comm * comm_ptr, MPIR_Info * info, MPIR_Comm ** out
     newcomm_ptr->attributes = 0;
     *outcomm_ptr = newcomm_ptr;
 
-    /* inherit tainted flag */
-    newcomm_ptr->tainted = comm_ptr->tainted;
-    propagate_tainted_to_subcomms(newcomm_ptr);
+    newcomm_ptr->vcis_enabled = comm_ptr->vcis_enabled;
+    propagate_vcis_enabled(newcomm_ptr);
 
   fn_fail:
     MPIR_FUNC_EXIT;

src/mpi/datatype/typeutil.c

@@ -16,7 +16,7 @@ MPIR_Datatype MPIR_Datatype_direct[MPIR_DATATYPE_PREALLOC];
 MPIR_Object_alloc_t MPIR_Datatype_mem = { 0, 0, 0, 0, 0, 0, 0, MPIR_DATATYPE,
     sizeof(MPIR_Datatype), MPIR_Datatype_direct,
     MPIR_DATATYPE_PREALLOC,
-    NULL, {0}
+    {0}
 };
 
 static int datatype_attr_finalize_cb(void *dummy);

src/mpi/errhan/errutil.c

@@ -120,7 +120,7 @@ MPIR_Object_alloc_t MPIR_Errhandler_mem = { 0, 0, 0, 0, 0, 0, 0, MPIR_ERRHANDLER
     sizeof(MPIR_Errhandler),
     MPIR_Errhandler_direct,
     MPIR_ERRHANDLER_PREALLOC,
-    NULL, {0}
+    {0}
 };
 
 static void init_builtins(void)

src/mpi/group/grouputil.c

@@ -39,7 +39,7 @@ MPIR_Group MPIR_Group_direct[MPIR_GROUP_PREALLOC];
 MPIR_Object_alloc_t MPIR_Group_mem = { 0, 0, 0, 0, 0, 0, 0, MPIR_GROUP,
     sizeof(MPIR_Group), MPIR_Group_direct,
     MPIR_GROUP_PREALLOC,
-    NULL, {0}
+    {0}
 };
 
 MPIR_Group *const MPIR_Group_empty = &MPIR_Group_builtin[0];

src/mpi/info/infoutil.c

@@ -17,7 +17,7 @@ MPIR_Info MPIR_Info_direct[MPIR_INFO_PREALLOC];
 MPIR_Object_alloc_t MPIR_Info_mem = { 0, 0, 0, 0, 0, 0, 0, MPIR_INFO,
     sizeof(MPIR_Info), MPIR_Info_direct,
     MPIR_INFO_PREALLOC,
-    NULL, {0}
+    {0}
 };
 
 /* Free an info structure.  In the multithreaded case, this routine

src/mpi/init/mutex.c

@@ -18,6 +18,7 @@ MPID_Thread_mutex_t MPIR_THREAD_VCI_HANDLE_MUTEX;
 MPID_Thread_mutex_t MPIR_THREAD_VCI_CTX_MUTEX;
 MPID_Thread_mutex_t MPIR_THREAD_VCI_PMI_MUTEX;
 MPID_Thread_mutex_t MPIR_THREAD_VCI_BSEND_MUTEX;
+MPID_Thread_mutex_t MPIR_THREAD_VCI_REQUEST_POOL_MUTEXES[MPIR_REQUEST_NUM_POOLS];
 #endif /* MPICH_THREAD_GRANULARITY */
 
 /* called the first thing in init so it can enter critical section immediately */
@@ -42,6 +43,11 @@ void MPII_thread_mutex_create(void)
     MPID_Thread_mutex_create(&MPIR_THREAD_VCI_BSEND_MUTEX, &err);
     MPIR_Assert(err == 0);
 
+    for (int i = 0; i < MPIR_REQUEST_NUM_POOLS; i++) {
+        MPID_Thread_mutex_create(&MPIR_THREAD_VCI_REQUEST_POOL_MUTEXES[i], &err);
+        MPIR_Assert(err == 0);
+    }
+
 #elif MPICH_THREAD_GRANULARITY == MPICH_THREAD_GRANULARITY__LOCKFREE
     /* Updates to shared data and access to shared services is handled
      * without locks where ever possible. */
@@ -79,6 +85,11 @@ void MPII_thread_mutex_destroy(void)
     MPID_Thread_mutex_destroy(&MPIR_THREAD_VCI_BSEND_MUTEX, &err);
     MPIR_Assert(err == 0);
 
+    for (int i = 0; i < MPIR_REQUEST_NUM_POOLS; i++) {
+        MPID_Thread_mutex_destroy(&MPIR_THREAD_VCI_REQUEST_POOL_MUTEXES[i], &err);
+        MPIR_Assert(err == 0);
+    }
+
 #elif MPICH_THREAD_GRANULARITY == MPICH_THREAD_GRANULARITY__LOCKFREE
     /* Updates to shared data and access to shared services is handled
      * without locks where ever possible. */

src/mpi/request/mpir_greq.c

@@ -12,7 +12,7 @@ MPIR_Object_alloc_t MPIR_Grequest_class_mem = { 0, 0, 0, 0, 0, 0, 0, MPIR_GREQ_C
     sizeof(MPIR_Grequest_class),
     MPIR_Grequest_class_direct,
     MPIR_GREQ_CLASS_PREALLOC,
-    NULL, {0}
+    {0}
 };
 
 /* We jump through some minor hoops to manage the list of classes ourselves and

src/mpi/request/mpir_request.c

@@ -31,15 +31,10 @@ static void init_builtin_request(MPIR_Request * req, int handle, MPIR_Request_ki
 
 void MPII_init_request(void)
 {
-    MPID_Thread_mutex_t *lock_ptr = NULL;
-#if MPICH_THREAD_GRANULARITY == MPICH_THREAD_GRANULARITY__VCI
-    lock_ptr = &MPIR_THREAD_VCI_HANDLE_MUTEX;
-#endif
-
     /* *INDENT-OFF* */
-    MPIR_Request_mem[0] = (MPIR_Object_alloc_t) { 0, 0, 0, 0, 0, 0, 0, MPIR_REQUEST, sizeof(MPIR_Request), MPIR_Request_direct, MPIR_REQUEST_PREALLOC, lock_ptr, {0}};
+    MPIR_Request_mem[0] = (MPIR_Object_alloc_t) { 0, 0, 0, 0, 0, 0, 0, MPIR_REQUEST, sizeof(MPIR_Request), MPIR_Request_direct, MPIR_REQUEST_PREALLOC, {0}};
     for (int i = 1; i < MPIR_REQUEST_NUM_POOLS; i++) {
-        MPIR_Request_mem[i] = (MPIR_Object_alloc_t) { 0, 0, 0, 0, 0, 0, 0, MPIR_REQUEST, sizeof(MPIR_Request), NULL, 0, lock_ptr, {0}};
+        MPIR_Request_mem[i] = (MPIR_Object_alloc_t) { 0, 0, 0, 0, 0, 0, 0, MPIR_REQUEST, sizeof(MPIR_Request), NULL, 0, {0}};
     }
     /* *INDENT-ON* */
 

src/mpi/rma/winutil.c

@@ -14,5 +14,5 @@ MPIR_Win MPIR_Win_direct[MPIR_WIN_PREALLOC];
 MPIR_Object_alloc_t MPIR_Win_mem = { 0, 0, 0, 0, 0, 0, 0, MPIR_WIN,
     sizeof(MPIR_Win), MPIR_Win_direct,
     MPIR_WIN_PREALLOC,
-    NULL, {0}
+    {0}
 };

src/mpi/session/session_util.c

@@ -16,7 +16,7 @@ MPIR_Session MPIR_Session_direct[MPIR_SESSION_PREALLOC];
 MPIR_Object_alloc_t MPIR_Session_mem = { 0, 0, 0, 0, 0, 0, 0,
     MPIR_SESSION, sizeof(MPIR_Session),
     MPIR_Session_direct, MPIR_SESSION_PREALLOC,
-    NULL, {0}
+    {0}
 };
 
 int MPIR_Session_create(MPIR_Session ** p_session_ptr, int thread_level)