diff --git a/makefile b/makefile index 4fb4ec0..b445c94 100644 --- a/makefile +++ b/makefile @@ -10,5 +10,5 @@ all: $(PROGS) clean: rm -f $(PROGS) -nsdperf: nsdperf.C +$(PROGS): nsdperf.C $(CXX) $(CFLAGS) -o $@ nsdperf.C $(LIBS) diff --git a/makefile.rdma b/makefile.rdma new file mode 100644 index 0000000..3339fb1 --- /dev/null +++ b/makefile.rdma @@ -0,0 +1,14 @@ +# Makefile for net programs +CXX = g++ +LIBS = -lpthread -lrt -libverbs -lrdmacm +CFLAGS = -O2 -DRDMA + +PROGS = nsdperf_rdma + +all: $(PROGS) + +clean: + rm -f $(PROGS) + +$(PROGS): nsdperf.C + $(CXX) $(CFLAGS) -o $@ nsdperf.C $(LIBS) diff --git a/nsdperf.C b/nsdperf.C index 7b2dbc1..47d0ec6 100755 --- a/nsdperf.C +++ b/nsdperf.C @@ -2937,7 +2937,7 @@ static string rdmaStart() for (rdi = rdmaDevTab.begin(); rdi != rdmaDevTab.end(); ++rdi) { int j; - UInt64 portIf; + set portIfSet; IpAddr portAddr; string ifName; RdmaDevice *rdevP = *rdi; @@ -2963,7 +2963,8 @@ static string rdmaStart() } // Fetch the interface ID portion of the GID for the port - portIf = 0; + portIfSet.clear(); + UInt64 portIf = 0; for (j = 0; j < portAttr.gid_tbl_len; j++) { union ibv_gid gid; @@ -2976,6 +2977,8 @@ static string rdmaStart() portIf = gid.global.interface_id; if (portIf != 0) + portIfSet.insert(portIf); + else break; } @@ -2988,56 +2991,68 @@ static string rdmaStart() ifName.clear(); if (useCM) { - for (ni = netInterfaces.begin(); ni != netInterfaces.end(); ++ni) + bool stop = false; + for (auto it = portIfSet.begin(); it != portIfSet.end(); ++it) { - if (!ni->addr.isLinkLocal()) - continue; - if (rdevP->ibContext->device->transport_type == IBV_TRANSPORT_IWARP) + portIf = *it; + for (ni = netInterfaces.begin(); ni != netInterfaces.end(); ++ni) { - struct ifreq mac_req; - - if (ioctl_sock < 0) - ioctl_sock = socket(AF_INET, SOCK_DGRAM, 0); - if (ioctl_sock < 0) - break; - - memset(&mac_req, 0, sizeof mac_req); - strcpy(mac_req.ifr_name, ni->ifName.c_str()); - - if (ioctl(ioctl_sock, SIOCGIFHWADDR, &mac_req) < 0) + if (!ni->addr.isLinkLocal()) + continue; + if (rdevP->ibContext->device->transport_type == IBV_TRANSPORT_IWARP) { - Logm("Cannot get HW address"); - continue; + struct ifreq mac_req; + + if (ioctl_sock < 0) + ioctl_sock = socket(AF_INET, SOCK_DGRAM, 0); + if (ioctl_sock < 0) + { + stop = true; + break; + } + + memset(&mac_req, 0, sizeof mac_req); + strcpy(mac_req.ifr_name, ni->ifName.c_str()); + + if (ioctl(ioctl_sock, SIOCGIFHWADDR, &mac_req) < 0) + { + Logm("Cannot get HW address"); + continue; + } + + if (!(memcmp(&portIf, mac_req.ifr_hwaddr.sa_data, 6))) + { + ifName = ni->ifName; + Logt(3, "Found iWarp RDMA port " << portIfToString(portIf) << " " << ni->ifName); + stop = true; + break; + } } - - if (!(memcmp(&portIf, mac_req.ifr_hwaddr.sa_data, 6))) + else { - ifName = ni->ifName; - Logt(3, "Found iWarp RDMA port " << portIfToString(portIf) << " " << ni->ifName); - break; - } - } - else - { -#ifdef IPV6_SUPPORT - struct sockaddr_storage saddr; - struct sockaddr_in6 *sip6 = - reinterpret_cast(ni->addr.toSockaddr(0, &saddr)); - - // 7 bytes of the port interface ID will match 7 bytes in the - // IPv6 address. The first byte won't match because the - // universal/local bit is flipped in the IPv6 address (see - // RFC 4291). - char *a = reinterpret_cast(&portIf); - char *b = reinterpret_cast(&sip6->sin6_addr.s6_addr); - if (!memcmp(a+1, b+9, 7)) - { - ifName = ni->ifName; - break; + #ifdef IPV6_SUPPORT + struct sockaddr_storage saddr; + struct sockaddr_in6 *sip6 = + reinterpret_cast(ni->addr.toSockaddr(0, &saddr)); + + // 7 bytes of the port interface ID will match 7 bytes in the + // IPv6 address. The first byte won't match because the + // universal/local bit is flipped in the IPv6 address (see + // RFC 4291). + char *a = reinterpret_cast(&portIf); + char *b = reinterpret_cast(&sip6->sin6_addr.s6_addr); + if (!memcmp(a+1, b+9, 7)) + { + ifName = ni->ifName; + stop = true; + break; + } + #endif } -#endif } + if (stop) + break; } // Skip this port if no link-local address was found