DAOS-18981 mercury: Fix handling of connection reset.

Test-provider: ucx+dc_x
Test-provider-hw-large: ucx+dc_x
Test-provider-hw-medium: ucx+dc_x
Test-provider-ucx: ucx+dc_x
Skip-func-hw-test-medium-verbs-provider: true
Skip-func-test-vm: true
Allow-unstable-test: true

Signed-off-by: Joseph Moore <joseph.moore@hpe.com>

Author: jgmoore-or

deps/patches/mercury/0004_ucx_connection_reset.patch

@@ -0,0 +1,24 @@
+diff --git a/src/na/na_ucx.c b/src/na/na_ucx.c
+index 96501b2..f97ce69 100644
+--- a/src/na/na_ucx.c
++++ b/src/na/na_ucx.c
+@@ -3251,16 +3251,10 @@ na_ucx_rma(struct na_ucx_class NA_UNUSED *na_ucx_class, na_context_t *context,
+ 
+     /* There is no need to have a fully resolved address to start an RMA.
+      * This is only necessary for two-sided communication. */
+-    /* The above assumption is now in question, so the following will resolve
+-     * the address if required. */
++    NA_CHECK_SUBSYS_WARNING(rma,
++        (!(hg_atomic_get32(&na_ucx_addr->status) & NA_UCX_ADDR_RESOLVED)),
++        "Connection is not resolved for rma operation");
+ 
+-    /* Check addr to ensure the EP for that addr is still valid */
+-    if (!(hg_atomic_get32(&na_ucx_addr->status) & NA_UCX_ADDR_RESOLVED)) {
+-        ret = na_ucx_addr_map_update(
+-            na_ucx_class, &na_ucx_class->addr_map, na_ucx_addr);
+-        NA_CHECK_SUBSYS_NA_ERROR(
+-            addr, error, ret, "Could not update NA UCX address");
+-    }
+     NA_CHECK_SUBSYS_ERROR(rma, na_ucx_addr->ucp_ep == NULL, error, ret,
+         NA_ADDRNOTAVAIL, "UCP endpoint is NULL for that address");
+ 

src/tests/ftest/util/general_utils.py

@@ -545,7 +545,7 @@ def get_errors_count(log, hostlist, file_glob):
     """
     # Get the Client side Error from client_log file.
     cmd = "cat {} | sed -n -E -e ".format(get_log_file(file_glob))
-    cmd += r"'/^.+[[:space:]]ERR[[:space:]].+[[:space:]]DER_[^(]+\([^)]+\).+$/"
+    cmd += r"'/^(ERR|.+[[:space:]]ERR)[[:space:]].+[[:space:]]DER_[^(]+\([^)]+\).+$/"
     cmd += r"s/^.+[[:space:]]DER_[^(]+\((-[[:digit:]]+)\).+$/\1/p'"
     result = run_remote(log, hostlist, cmd, verbose=False)
     errors_count = {}

utils/build.config

@@ -27,6 +27,6 @@ ucx=https://github.com/openucx/ucx.git
 
 [patch_versions]
 spdk=0001_external_isal_v26.01.patch
-mercury=0001_dep_versions.patch,0002_ofi_counters.patch,0003_ofi_auth_key.patch
+mercury=0001_dep_versions.patch,0002_ofi_counters.patch,0003_ofi_auth_key.patch,0004_ucx_connection_reset.patch 
 pmdk=https://github.com/daos-stack/pmdk/commit/bb048d67ccd07609f86a5e8b3c6ad54414d593ee.diff,https://github.com/daos-stack/pmdk/commit/69925cf455ef672c4cbdbdb13bef7ae581e67045.diff,https://github.com/daos-stack/pmdk/commit/6805ed4f8d1a4e4c6070bf8b68f0dffef08b9c99.diff
 argobots=0001_411e5b344642ebc82190fd8b125db512e5b449d1.diff,0002_bb0c908abfac4bfe37852eee621930634183c6aa.diff

utils/rpms/package_info.sh

@@ -45,7 +45,7 @@ export libfabric_version="1.22.0"
 export libfabric_release="5${distro_name}"
 export libfabric_full="${libfabric_version}-${libfabric_release}"
 export mercury_version="2.4.1"
-export mercury_release="2${distro_name}"
+export mercury_release="3${distro_name}"
 export mercury_full="${mercury_version}-${mercury_release}"
 export argobots_version="1.2"
 export argobots_release="4${distro_name}"