RFC: Provide equivalence of MPICH_ASYNC_PROGRESS

config/opal_configure_options.m4

@@ -544,9 +544,21 @@ fi
 AC_DEFINE_UNQUOTED([OPAL_ENABLE_GETPWUID], [$opal_want_getpwuid],
                    [Disable getpwuid support (default: enabled)])
 
-dnl We no longer support the old OPAL_ENABLE_PROGRESS_THREADS.  At
-dnl some point, this should die.
-AC_DEFINE([OPAL_ENABLE_PROGRESS_THREADS],
-          [0],
-          [Whether we want BTL progress threads enabled])
+#
+# Disable progress threads
+#
+AC_MSG_CHECKING([if want asynchronous progress threads])
+AC_ARG_ENABLE([progress_threads],
+    [AS_HELP_STRING([--disable-progress-threads],
+                   [Disable asynchronous progress threads. Note that when enabled, for performance-related reasons, the progress thread is still not spawned by default. User must enable MCA variables 'opal_async_progress' or 'mpi_async_progress' to have the progress thread spawned at runtime. (default: enabled)])])
+if test "$enable_progress_threads" = "no"; then
+    AC_MSG_RESULT([no])
+    opal_want_progress_threads=0
+else
+    AC_MSG_RESULT([yes])
+    opal_want_progress_threads=1
+fi
+AC_DEFINE_UNQUOTED([OPAL_ENABLE_PROGRESS_THREADS], [$opal_want_progress_threads],
+    [Disable BTL asynchronous progress threads (default: enabled)])
+
 ])dnl

docs/installing-open-mpi/configure-cli-options/misc.rst

@@ -34,6 +34,19 @@ above categories that can be used with ``configure``:
   .. danger:: The heterogeneous functionality is currently broken |mdash|
               do not use.
 
+* ``--enable-progress-threads``
+* ``--disable-progress-threads``:
+  Enable or disable (default = ``enabled``) support of software-based progress
+  thread for each MPI process to execute the internal communication progression
+  engine. Note that even when the support is built, the progress thread is not
+  spawned by default at runtime. This behavior is controlled by the associated
+  runtime MCA variable ``opal_async_progress`` or ``mpi_async_progress``
+  (default = false).
+
+  .. warning:: Be aware of performance degradation. Please read
+               :ref:`this section <async-progress-thread-label>` for
+               more documentation.
+
 .. _install-wrapper-flags-label:
 
 * ``--with-wrapper-cflags=CFLAGS``

docs/launching-apps/index.rst

@@ -39,6 +39,7 @@ same command).
    prerequisites
    pmix-and-prrte
    scheduling
+   progress_thread
 
    localhost
    ssh

docs/launching-apps/progress_thread.rst

@@ -0,0 +1,75 @@
+.. _async-progress-thread-label:
+
+Asynchronous progress thread
+============================
+
+Open MPI provides an experimental support of software-based asynchronous
+progress thread. This progress thread is in charge of running internal
+progression engine in the background to advance non-blocking overlapping
+communication.
+
+Enabling progress thread at configuration time
+----------------------------------------------
+
+The feature is can be enabled or disabled at configuration by passing
+``--enable-progress-threads`` or ``--disable-progress-threads`` to
+``configure``. The default state is enabled.
+
+Enabling progress thread at runtime
+-----------------------------------
+
+When Open MPI was configured and built with ``--enable-progress-threads``, the
+progress thread is still deactivated at runtime by default.
+
+The progress thread can be activated by setting one of the following
+MCA boolean variables in the launching command:
+
+.. code-block:: sh
+
+   shell$ mpirun --mca opal_async_progress 1 ...
+   shell$ mpirun --mca mpi_async_progress 1 ...
+   shell$ OMPI_MCA_opal_async_progress=1 mpirun ...
+   shell$ OMPI_MCA_mpi_async_progress=1 mpirun ...
+
+Note that ``mpi_async_progress`` is a synonym of ``opal_async_progress``.
+
+.. warning:: Progress threads are a somewhat complicated issue. Activating them
+             at run time may improve overlap of communication and computation in
+             your application (particularly those with non-blocking communication)
+             which will improve overall performance. But there may be unintended
+             consequences which may degrade overall application performance.
+             Users are advised to experiment and see what works best for their
+             applications.
+
+Rationale
+---------
+
+A possible beneficial usecase of software progress thread is *intra-node
+shared-memory non-blocking* communication, running on some high core-count CPUs,
+on which application may not use all the available cores, or the CPU has some
+reserved cores dedicated to communication tasks. In such configurations, the
+latency of some non-blocking collective operations (e.g. ``MPI_Ireduce()``)
+can be improved thanks to arithmetic operations being performed in the
+background by the progress thread, instead of deferring the computations to
+being executed by the main thread during ``MPI_Wait()``.
+
+Alternatively, on systems where *inter-node communications* are already
+offloaded to dedicated hardware, enabling the software-based progress threads
+could degrade performance, since the additional thread will force progress up
+through the CPU and potentially away from more optimized hardware functionality.
+
+For these performance reasons, the progress thread is not activated (spawned)
+by default at runtime. It is upon developers to decide to switch on the
+progress thread, depending on their application and system setup.
+
+Limitations
+-----------
+
+#. The current implementation does not support (yet) binding the progress
+   thread to a specific core (or set of cores).
+
+#. There are still some hard-coded constant parameters in the code that
+   would require further tuning.
+
+#. It was observed that some multi-threading overhead may impact performance
+   on small buffers.

ompi/instance/instance.c

@@ -512,7 +512,7 @@ static int ompi_mpi_instance_init_common (int argc, char **argv)
        ddt_init, but before mca_coll_base_open, since some collective
        modules (e.g., the hierarchical coll component) may need ops in
        their query function. */
-    if (OMPI_SUCCESS != (ret = ompi_op_base_find_available (OPAL_ENABLE_PROGRESS_THREADS, ompi_mpi_thread_multiple))) {
+    if (OMPI_SUCCESS != (ret = ompi_op_base_find_available (opal_async_progress_thread_spawned, ompi_mpi_thread_multiple))) {
         return ompi_instance_print_error ("ompi_op_base_find_available() failed", ret);
     }
 
@@ -532,7 +532,7 @@ static int ompi_mpi_instance_init_common (int argc, char **argv)
         return ompi_instance_print_error ("mca_smsc_base_select() failed", ret);
     }
 
-    if (OMPI_SUCCESS != (ret = mca_pml_base_select (OPAL_ENABLE_PROGRESS_THREADS, ompi_mpi_thread_multiple))) {
+    if (OMPI_SUCCESS != (ret = mca_pml_base_select (opal_async_progress_thread_spawned, ompi_mpi_thread_multiple))) {
         return ompi_instance_print_error ("mca_pml_base_select() failed", ret);
     }
 
@@ -617,7 +617,7 @@ static int ompi_mpi_instance_init_common (int argc, char **argv)
         return ompi_instance_print_error ("mca_pml_base_bsend_init() failed", ret);
     }
 
-    if (OMPI_SUCCESS != (ret = mca_coll_base_find_available (OPAL_ENABLE_PROGRESS_THREADS, ompi_mpi_thread_multiple))) {
+    if (OMPI_SUCCESS != (ret = mca_coll_base_find_available (opal_async_progress_thread_spawned, ompi_mpi_thread_multiple))) {
         return ompi_instance_print_error ("mca_coll_base_find_available() failed", ret);
     }
 

ompi/mpi/c/request_get_status.c.in

@@ -38,9 +38,7 @@
 PROTOTYPE ERROR_CLASS request_get_status(REQUEST request, INT_OUT flag,
                                          STATUS_OUT status)
 {
-#if OPAL_ENABLE_PROGRESS_THREADS == 0
     int do_it_once = 0;
-#endif
 
     MEMCHECKER(
         memchecker_request(&request);
@@ -56,9 +54,7 @@ PROTOTYPE ERROR_CLASS request_get_status(REQUEST request, INT_OUT flag,
         }
     }
 
-#if OPAL_ENABLE_PROGRESS_THREADS == 0
  recheck_request_status:
-#endif
     opal_atomic_mb();
     if( (request == MPI_REQUEST_NULL) || (request->req_state == OMPI_REQUEST_INACTIVE) ) {
         *flag = true;
@@ -80,16 +76,16 @@ PROTOTYPE ERROR_CLASS request_get_status(REQUEST request, INT_OUT flag,
         }
         return MPI_SUCCESS;
     }
-#if OPAL_ENABLE_PROGRESS_THREADS == 0
-    if( 0 == do_it_once ) {
+
+    if( 0 == do_it_once && !opal_async_progress_thread_spawned ) {
         /* If we run the opal_progress then check the status of the
            request before leaving. We will call the opal_progress only
            once per call. */
         opal_progress();
         do_it_once++;
         goto recheck_request_status;
     }
-#endif
+
     *flag = false;
     return MPI_SUCCESS;
 }

ompi/request/req_test.c

@@ -32,11 +32,9 @@ int ompi_request_default_test(ompi_request_t ** rptr,
 {
     ompi_request_t *request = *rptr;
 
-#if OPAL_ENABLE_PROGRESS_THREADS == 0
     int do_it_once = 0;
 
 recheck_request_status:
-#endif
     if( request->req_state == OMPI_REQUEST_INACTIVE ) {
         *completed = true;
         if (MPI_STATUS_IGNORE != status) {
@@ -81,8 +79,8 @@ int ompi_request_default_test(ompi_request_t ** rptr,
         return MPI_ERR_PROC_FAILED_PENDING;
     }
 #endif
-#if OPAL_ENABLE_PROGRESS_THREADS == 0
-    if( 0 == do_it_once ) {
+
+    if( 0 == do_it_once && !opal_async_progress_thread_spawned ) {
         /**
          * If we run the opal_progress then check the status of the request before
          * leaving. We will call the opal_progress only once per call.
@@ -92,7 +90,7 @@ int ompi_request_default_test(ompi_request_t ** rptr,
             goto recheck_request_status;
         }
     }
-#endif
+
     *completed = false;
     return OMPI_SUCCESS;
 }
@@ -163,9 +161,9 @@ int ompi_request_default_test_any(
     *index = MPI_UNDEFINED;
     if(num_requests_null_inactive != count) {
         *completed = false;
-#if OPAL_ENABLE_PROGRESS_THREADS == 0
-        opal_progress();
-#endif
+        if (!opal_async_progress_thread_spawned) {
+            opal_progress();
+        }
     } else {
         *completed = true;
         if (MPI_STATUS_IGNORE != status) {
@@ -208,16 +206,16 @@ int ompi_request_default_test_all(
             return MPI_ERR_PROC_FAILED_PENDING;
         }
 #endif /* OPAL_ENABLE_FT_MPI */
-#if OPAL_ENABLE_PROGRESS_THREADS == 0
-        if (0 == do_it_once) {
+
+        if (0 == do_it_once && !opal_async_progress_thread_spawned) {
             ++do_it_once;
             if (0 != opal_progress()) {
                 /* continue walking the list, retest the current request */
                 --i;
                 continue;
             }
         }
-#endif /* OPAL_ENABLE_PROGRESS_THREADS */
+
         /* short-circuit */
         break;
     }
@@ -353,9 +351,9 @@ int ompi_request_default_test_some(
     *outcount = num_requests_done;
 
     if (num_requests_done == 0) {
-#if OPAL_ENABLE_PROGRESS_THREADS == 0
-        opal_progress();
-#endif
+        if (!opal_async_progress_thread_spawned) {
+            opal_progress();
+        }
         return OMPI_SUCCESS;
     }
 

ompi/runtime/ompi_mpi_finalize.c

@@ -193,9 +193,10 @@ int ompi_mpi_finalize(void)
     opal_atomic_swap_32(&ompi_mpi_state,
                         OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT);
 
-#if OPAL_ENABLE_PROGRESS_THREADS == 0
-    opal_progress_set_event_flag(OPAL_EVLOOP_ONCE | OPAL_EVLOOP_NONBLOCK);
-#endif
+    /* shutdown async progress thread before tearing down further services */
+    if (opal_async_progress_thread_spawned) {
+        opal_progress_shutdown_async_progress_thread();
+    }
 
     /* NOTE: MPI-2.1 requires that MPI_FINALIZE is "collective" across
        *all* connected processes.  This only means that all processes

ompi/runtime/ompi_mpi_init.c

@@ -559,16 +559,16 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided,
        time if so, then start the clock again */
     OMPI_TIMING_NEXT("barrier");
 
-#if OPAL_ENABLE_PROGRESS_THREADS == 0
     /* Start setting up the event engine for MPI operations.  Don't
        block in the event library, so that communications don't take
        forever between procs in the dynamic code.  This will increase
        CPU utilization for the remainder of MPI_INIT when we are
        blocking on RTE-level events, but may greatly reduce non-TCP
        latency. */
-    int old_event_flags = opal_progress_set_event_flag(0);
-    opal_progress_set_event_flag(old_event_flags | OPAL_EVLOOP_NONBLOCK);
-#endif
+    if (!opal_async_progress_thread_spawned) {
+        int old_event_flags = opal_progress_set_event_flag(0);
+        opal_progress_set_event_flag(old_event_flags | OPAL_EVLOOP_NONBLOCK);
+    }
 
     /* wire up the mpi interface, if requested.  Do this after the
        non-block switch for non-TCP performance.  Do before the

ompi/runtime/ompi_mpi_params.c

@@ -351,6 +351,13 @@ int ompi_mpi_register_params(void)
                                  MCA_BASE_VAR_SCOPE_READONLY,
                                  &ompi_async_mpi_finalize);
 
+#if OPAL_ENABLE_PROGRESS_THREADS == 1
+    value = mca_base_var_find ("opal", "opal", NULL, "async_progress");
+    if (0 <= value) {
+        (void) mca_base_var_register_synonym(value, "ompi", "mpi", NULL, "async_progress", 0);
+    }
+#endif
+
     value = mca_base_var_find ("opal", "opal", NULL, "abort_delay");
     if (0 <= value) {
         (void) mca_base_var_register_synonym(value, "ompi", "mpi", NULL, "abort_delay",

opal/mca/btl/smcuda/btl_smcuda.c

@@ -487,7 +487,7 @@ static struct mca_btl_base_endpoint_t *create_sm_endpoint(int local_proc, struct
     OBJ_CONSTRUCT(&ep->endpoint_lock, opal_mutex_t);
 #if OPAL_ENABLE_PROGRESS_THREADS == 1
     sprintf(path, "%s" OPAL_PATH_SEP "sm_fifo.%lu", opal_process_info.job_session_dir,
-            (unsigned long) proc->proc_name);
+            (unsigned long) proc->proc_name.vpid);
     ep->fifo_fd = open(path, O_WRONLY);
     if (ep->fifo_fd < 0) {
         opal_output(0, "mca_btl_smcuda_add_procs: open(%s) failed with errno=%d\n", path, errno);

opal/mca/btl/smcuda/btl_smcuda_component.c

@@ -860,7 +860,7 @@ mca_btl_smcuda_component_init(int *num_btls, bool enable_progress_threads, bool
 #if OPAL_ENABLE_PROGRESS_THREADS == 1
     /* create a named pipe to receive events  */
     sprintf(mca_btl_smcuda_component.sm_fifo_path, "%s" OPAL_PATH_SEP "sm_fifo.%lu",
-            opal_process_info.job_session_dir, (unsigned long) OPAL_PROC_MY_NAME->vpid);
+            opal_process_info.job_session_dir, (unsigned long) OPAL_PROC_MY_NAME.vpid);
     if (mkfifo(mca_btl_smcuda_component.sm_fifo_path, 0660) < 0) {
         opal_output(0, "mca_btl_smcuda_component_init: mkfifo failed with errno=%d\n", errno);
         return NULL;

opal/runtime/opal_params_core.c

@@ -90,6 +90,10 @@ int opal_abort_delay = 0;
 
 int opal_max_thread_in_progress = 1;
 
+#if OPAL_ENABLE_PROGRESS_THREADS == 1
+bool opal_async_progress = false;
+#endif
+
 static bool opal_register_util_done = false;
 
 static char *opal_var_dump_color_string = NULL;
@@ -416,6 +420,14 @@ int opal_register_util_params(void)
                                  MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_8,
                                  MCA_BASE_VAR_SCOPE_READONLY, &opal_max_thread_in_progress);
 
+#if OPAL_ENABLE_PROGRESS_THREADS == 1
+    /* Spawn a dedicated software progress-thread to execute opal_progress() */
+    (void) mca_base_var_register("opal", "opal", NULL, "async_progress",
+                                 "Spawn a dedicated software progress-thread. Default: false",
+                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                 MCA_BASE_VAR_SCOPE_READONLY, &opal_async_progress);
+#endif
+
     /* Use sync_memops functionality with accelerator codes or deploy
        alternative path using IPC events to ensure consistency */
     opal_accelerator_use_sync_memops = true;

opal/runtime/opal_params_core.h

@@ -60,6 +60,16 @@ OPAL_DECLSPEC extern bool opal_built_with_cuda_support;
 OPAL_DECLSPEC extern bool opal_built_with_rocm_support;
 OPAL_DECLSPEC extern bool opal_built_with_ze_support;
 
+#    if OPAL_ENABLE_PROGRESS_THREADS == 1
+OPAL_DECLSPEC extern bool opal_async_progress_thread_spawned;
+#    else
+/* When disabled at configure, using 'static const' will allow compilers to
+ * not only evaluate the boolean at compile time, but also to remove its
+ * storage within any object file which is including this header.
+ * See https://godbolt.org/z/hxc8EjPa4  */
+OPAL_DECLSPEC static const bool opal_async_progress_thread_spawned = false;
+#    endif
+
 /**
  *  * Whether we want to enable CUDA GPU buffer send and receive support.
  *   */