ofi: Reuse MTL's domain and fabric in BTL

jiaxiyan · jiaxiyan · commit e1e6f0be896f · 2025-10-07T23:52:01.000Z
Share the domain among BTL and MTL to reduce the number of domains needed.
This prevents hitting the resource limit on systems with high core counts.

Signed-off-by: Jessie Yang &lt;jiaxiyan@amazon.com&gt;
diff --git a/ompi/mca/mtl/ofi/mtl_ofi_component.c b/ompi/mca/mtl/ofi/mtl_ofi_component.c
@@ -972,14 +972,11 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
      * instantiate the virtual or physical network. This opens a "fabric
      * provider". See man fi_fabric for details.
      */
-    ret = fi_fabric(prov->fabric_attr,    /* In:  Fabric attributes             */
+    ret = opal_common_ofi_fi_fabric(prov->fabric_attr,    /* In:  Fabric attributes             */
                     &ompi_mtl_ofi.fabric, /* Out: Fabric handle                 */
                     NULL);                /* Optional context for fabric events */
     if (0 != ret) {
-        opal_show_help("help-mtl-ofi.txt", "OFI call fail", true,
-                       "fi_fabric",
-                       ompi_process_info.nodename, __FILE__, __LINE__,
-                       fi_strerror(-ret), -ret);
+        MTL_OFI_LOG_FI_ERR(ret, "opal_common_ofi_fi_fabric failed");
         goto error;
     }
 
@@ -988,15 +985,12 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
      * hardware port/collection of ports.  Returns a domain object that can be
      * used to create endpoints.  See man fi_domain for details.
      */
-    ret = fi_domain(ompi_mtl_ofi.fabric,  /* In:  Fabric object                 */
+    ret = opal_common_ofi_fi_domain(ompi_mtl_ofi.fabric,  /* In:  Fabric object                 */
                     prov,                 /* In:  Provider                      */
                     &ompi_mtl_ofi.domain, /* Out: Domain object                 */
                     NULL);                /* Optional context for domain events */
     if (0 != ret) {
-        opal_show_help("help-mtl-ofi.txt", "OFI call fail", true,
-                       "fi_domain",
-                       ompi_process_info.nodename, __FILE__, __LINE__,
-                       fi_strerror(-ret), -ret);
+        MTL_OFI_LOG_FI_ERR(ret, "opal_common_ofi_fi_domain failed");
         goto error;
     }
 
@@ -1155,10 +1149,10 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
         (void) fi_close((fid_t)ompi_mtl_ofi.ofi_ctxt[0].cq);
     }
     if (ompi_mtl_ofi.domain) {
-        (void) fi_close((fid_t)ompi_mtl_ofi.domain);
+        (void) opal_common_ofi_domain_release(ompi_mtl_ofi.domain);
     }
     if (ompi_mtl_ofi.fabric) {
-        (void) fi_close((fid_t)ompi_mtl_ofi.fabric);
+        (void) opal_common_ofi_fabric_release(ompi_mtl_ofi.fabric);
     }
     if (ompi_mtl_ofi.comm_to_context) {
         free(ompi_mtl_ofi.comm_to_context);
@@ -1206,11 +1200,11 @@ ompi_mtl_ofi_finalize(struct mca_mtl_base_module_t *mtl)
         }
     }
 
-    if ((ret = fi_close((fid_t)ompi_mtl_ofi.domain))) {
+    if ((ret = opal_common_ofi_domain_release(ompi_mtl_ofi.domain))) {
         goto finalize_err;
     }
 
-    if ((ret = fi_close((fid_t)ompi_mtl_ofi.fabric))) {
+    if ((ret = opal_common_ofi_fabric_release(ompi_mtl_ofi.fabric))) {
         goto finalize_err;
     }
 
diff --git a/opal/mca/btl/ofi/btl_ofi_component.c b/opal/mca/btl/ofi/btl_ofi_component.c
@@ -372,6 +372,13 @@ static mca_btl_base_module_t **mca_btl_ofi_component_init(int *num_btl_modules,
 no_hmem:
 #endif
 
+    if (opal_common_ofi.fabric) {
+        hints.fabric_attr->fabric = opal_common_ofi.fabric;
+    }
+    if (opal_common_ofi.domain) {
+        hints.domain_attr->domain = opal_common_ofi.domain;
+    }
+
     /* Do the query. The earliest version that supports FI_HMEM hints is 1.9.
      * The earliest version the explictly allow provider to call CUDA API is 1.18  */
     rc = fi_getinfo(FI_VERSION(1, 18), NULL, NULL, 0, &hints, &info_list);
@@ -553,16 +560,16 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
         ("initializing dev:%s provider:%s", linux_device_name, info->fabric_attr->prov_name));
 
     /* fabric */
-    rc = fi_fabric(ofi_info->fabric_attr, &fabric, NULL);
+    rc = opal_common_ofi_fi_fabric(ofi_info->fabric_attr, &fabric, NULL);
     if (0 != rc) {
-        BTL_VERBOSE(("%s failed fi_fabric with err=%s", linux_device_name, fi_strerror(-rc)));
+        BTL_VERBOSE(("%s failed opal_common_ofi_fi_fabric with err=%d", linux_device_name, rc));
         goto fail;
     }
 
     /* domain */
-    rc = fi_domain(fabric, ofi_info, &domain, NULL);
+    rc = opal_common_ofi_fi_domain(fabric, ofi_info, &domain, NULL);
     if (0 != rc) {
-        BTL_VERBOSE(("%s failed fi_domain with err=%s", linux_device_name, fi_strerror(-rc)));
+        BTL_VERBOSE(("%s failed opal_common_ofi_fi_domain with err=%d", linux_device_name, rc));
         goto fail;
     }
 
@@ -743,11 +750,11 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
     }
 
     if (NULL != domain) {
-        fi_close(&domain->fid);
+        opal_common_ofi_domain_release(domain);
     }
 
     if (NULL != fabric) {
-        fi_close(&fabric->fid);
+        opal_common_ofi_fabric_release(fabric);
     }
     free(module);
 
diff --git a/opal/mca/btl/ofi/btl_ofi_module.c b/opal/mca/btl/ofi/btl_ofi_module.c
@@ -385,11 +385,11 @@ int mca_btl_ofi_finalize(mca_btl_base_module_t *btl)
     }
 
     if (NULL != ofi_btl->domain) {
-        fi_close(&ofi_btl->domain->fid);
+        opal_common_ofi_domain_release(ofi_btl->domain);
     }
 
     if (NULL != ofi_btl->fabric) {
-        fi_close(&ofi_btl->fabric->fid);
+        opal_common_ofi_fabric_release(ofi_btl->fabric);
     }
 
     if (NULL != ofi_btl->fabric_info) {
diff --git a/opal/mca/common/ofi/common_ofi.c b/opal/mca/common/ofi/common_ofi.c
@@ -6,7 +6,7 @@
  *                         reserved.
  * Copyright (c) 2020-2021 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2021-2023 Nanook Consulting.  All rights reserved.
- * Copyright (c) 2021      Amazon.com, Inc. or its affiliates. All rights
+ * Copyright (c) 2021-2025 Amazon.com, Inc. or its affiliates. All rights
  *                         reserved.
  * Copyright (c) 2023      UT-Battelle, LLC.  All rights reserved.
  * $COPYRIGHT$
@@ -42,7 +42,11 @@
 extern opal_accelerator_base_module_t opal_accelerator;
 opal_common_ofi_module_t opal_common_ofi = {.prov_include = NULL,
                                             .prov_exclude = NULL,
-                                            .output = -1};
+                                            .output = -1,
+                                            .fabric = NULL,
+                                            .domain = NULL,
+                                            .fabric_ref_count = 0,
+                                            .domain_ref_count = 0};
 static const char default_prov_exclude_list[] = "shm,sockets,tcp,udp,rstream,usnic,net";
 static opal_mutex_t opal_common_ofi_mutex = OPAL_MUTEX_STATIC_INIT;
 static int opal_common_ofi_verbose_level = 0;
@@ -1257,3 +1261,175 @@ OPAL_DECLSPEC int opal_common_ofi_fi_getname(fid_t fid, void **addr, size_t *add
     }
     return ret;
 }
+
+/**
+ * Get or create fabric object
+ *
+ * Reuses existing fabric from fabric_attr->fabric if available,
+ * otherwise creates new fabric using fi_fabric().
+ *
+ * @param fabric_attr (IN) Fabric attributes
+ * @param fabric (OUT)     Fabric object (new or existing)
+ * @param context (IN)     Optional context
+ *
+ * @return                 OPAL_SUCCESS or OPAL error code
+ */
+int opal_common_ofi_fi_fabric(struct fi_fabric_attr *fabric_attr,
+                              struct fid_fabric **fabric,
+                              void *context)
+{
+    int ret;
+    
+    if (!fabric_attr || !fabric) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    OPAL_THREAD_LOCK(&opal_common_ofi_mutex);
+    
+    if (fabric_attr->fabric) {
+        *fabric = fabric_attr->fabric;
+        opal_common_ofi.fabric_ref_count++;
+        opal_output_verbose(1, opal_common_ofi.output, "Reusing existing fabric: %s",
+                            fabric_attr->name);
+    } else {
+        ret = fi_fabric(fabric_attr, fabric, context);
+        if (0 != ret) {
+            opal_show_help("help-mtl-ofi.txt", "OFI call fail", true,
+                       "fi_fabric",
+                       opal_process_info.nodename, __FILE__, __LINE__,
+                       fi_strerror(-ret), -ret);
+            OPAL_THREAD_UNLOCK(&opal_common_ofi_mutex);
+            return OPAL_ERROR;
+        }
+        opal_common_ofi.fabric = *fabric;
+        opal_common_ofi.fabric_ref_count = 1;
+    }
+
+    OPAL_THREAD_UNLOCK(&opal_common_ofi_mutex);
+    return OPAL_SUCCESS;
+}
+
+/**
+ * Get or create domain object
+ *
+ * Reuses existing domain from info->domain_attr->domain if available,
+ * otherwise creates new domain using fi_domain().
+ *
+ * @param fabric (IN)      Fabric object
+ * @param info (IN)        Provider info
+ * @param domain (OUT)     Domain object (new or existing)
+ * @param context (IN)     Optional context
+ *
+ * @return                 OPAL_SUCCESS or OPAL error code
+ */
+int opal_common_ofi_fi_domain(struct fid_fabric *fabric, struct fi_info *info,
+                              struct fid_domain **domain, void *context)
+{
+    int ret;
+    
+    if (!info || !fabric || !domain || !info->domain_attr) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    OPAL_THREAD_LOCK(&opal_common_ofi_mutex);
+
+    if (info->domain_attr->domain) {
+        *domain = info->domain_attr->domain;
+        opal_common_ofi.domain_ref_count++;
+        opal_output_verbose(1, opal_common_ofi.output, "Reusing existing domain: %s",
+                            info->domain_attr->name);
+    } else {
+        ret = fi_domain(fabric, info, domain, context);
+        if (0 != ret) {
+            opal_show_help("help-mtl-ofi.txt", "OFI call fail", true,
+                       "fi_domain",
+                       opal_process_info.nodename, __FILE__, __LINE__,
+                       fi_strerror(-ret), -ret);
+            OPAL_THREAD_UNLOCK(&opal_common_ofi_mutex);
+            return OPAL_ERROR;
+        }
+        opal_common_ofi.domain = *domain;
+        opal_common_ofi.domain_ref_count = 1;
+    }
+
+    OPAL_THREAD_UNLOCK(&opal_common_ofi_mutex);
+    return OPAL_SUCCESS;
+}
+
+/**
+ * Release fabric reference
+ *
+ * Decrements fabric reference count and closes fabric if count reaches zero.
+ *
+ * @param fabric (IN)      Fabric object to release
+ *
+ * @return                 OPAL_SUCCESS or OPAL error code
+ */
+int opal_common_ofi_fabric_release(struct fid_fabric *fabric)
+{
+    int ret = OPAL_SUCCESS;
+    
+    if (!fabric) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    OPAL_THREAD_LOCK(&opal_common_ofi_mutex);
+    
+    if (fabric == opal_common_ofi.fabric && opal_common_ofi.fabric_ref_count > 0) {
+        opal_common_ofi.fabric_ref_count--;
+        if (opal_common_ofi.fabric_ref_count == 0) {
+            ret = fi_close(&fabric->fid);
+            if (0 != ret) {
+                ret = OPAL_ERROR;
+            }
+            opal_common_ofi.fabric = NULL;
+        }
+    } else {
+        ret = fi_close(&fabric->fid);
+        if (0 != ret) {
+            ret = OPAL_ERROR;
+        }
+    }
+    
+    OPAL_THREAD_UNLOCK(&opal_common_ofi_mutex);
+    return ret;
+}
+
+/**
+ * Release domain reference
+ *
+ * Decrements domain reference count and closes domain if count reaches zero.
+ *
+ * @param domain (IN)      Domain object to release
+ *
+ * @return                 OPAL_SUCCESS or OPAL error code
+ */
+int opal_common_ofi_domain_release(struct fid_domain *domain)
+{
+    int ret = OPAL_SUCCESS;
+    
+    if (!domain) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    OPAL_THREAD_LOCK(&opal_common_ofi_mutex);
+    
+    if (domain == opal_common_ofi.domain && opal_common_ofi.domain_ref_count > 0) {
+        opal_common_ofi.domain_ref_count--;
+        if (opal_common_ofi.domain_ref_count == 0) {
+            ret = fi_close(&domain->fid);
+            if (0 != ret) {
+                ret = OPAL_ERROR;
+            }
+            opal_common_ofi.domain = NULL;
+        }
+    } else {
+        ret = fi_close(&domain->fid);
+        if (0 != ret) {
+            ret = OPAL_ERROR;
+        }
+    }
+    
+    OPAL_THREAD_UNLOCK(&opal_common_ofi_mutex);
+    return ret;
+}
diff --git a/opal/mca/common/ofi/common_ofi.h b/opal/mca/common/ofi/common_ofi.h

Original file line number	Diff line number	Diff line change
`@@ -385,11 +385,11 @@ int mca_btl_ofi_finalize(mca_btl_base_module_t *btl)`
`385`	`385`	`}`
`386`	`386`
`387`	`387`	`if (NULL != ofi_btl->domain) {`
`388`		`- fi_close(&ofi_btl->domain->fid);`
	`388`	`+ opal_common_ofi_domain_release(ofi_btl->domain);`
`389`	`389`	`}`
`390`	`390`
`391`	`391`	`if (NULL != ofi_btl->fabric) {`
`392`		`- fi_close(&ofi_btl->fabric->fid);`
	`392`	`+ opal_common_ofi_fabric_release(ofi_btl->fabric);`
`393`	`393`	`}`
`394`	`394`
`395`	`395`	`if (NULL != ofi_btl->fabric_info) {`