Merge pull request #105 from DUNE-DAQ/roland-sipos/require-pci-addr

Require PCI addr to configure available interface

include/dpdklibs/EALSetup.hpp

@@ -200,27 +200,21 @@ iface_init(uint16_t iface, uint16_t rx_rings, uint16_t tx_rings,
     return retval;
   }
 
-
-
   // Get interface info
   retval = rte_eth_dev_info_get(iface, &dev_info);
   if (retval != 0) {
     TLOG() << "Error during getting device (iface " << iface << ") retval: " << retval;
     return retval;
   }
 
-
-  TLOG() << "Iface " << iface << " Rx Ring info :" 
-    << "min " << dev_info.rx_desc_lim.nb_min 
-    << "max " << dev_info.rx_desc_lim.nb_max 
-    << "align " << dev_info.rx_desc_lim.nb_align
-  ;
-
-  TLOG() << "Iface " << iface << " Tx Ring info :" 
-    << "min " << dev_info.rx_desc_lim.nb_min 
-    << "max " << dev_info.rx_desc_lim.nb_max 
-    << "align " << dev_info.rx_desc_lim.nb_align
-  ;
+  TLOG() << "Iface[" << iface << "] Rx Ring info:"
+    << " min=" << dev_info.rx_desc_lim.nb_min 
+    << " max=" << dev_info.rx_desc_lim.nb_max 
+    << " align=" << dev_info.rx_desc_lim.nb_align;
+  TLOG() << "Iface[" << iface << "] Tx Ring info:" 
+    << " min=" << dev_info.rx_desc_lim.nb_min 
+    << " max=" << dev_info.rx_desc_lim.nb_max 
+    << " align=" << dev_info.rx_desc_lim.nb_align;
 
   for (size_t j = 0; j < dev_info.nb_rx_queues; j++) {
 
@@ -232,16 +226,13 @@ iface_init(uint16_t iface, uint16_t rx_rings, uint16_t tx_rings,
       break;
 
     count = rte_eth_rx_queue_count(iface, j);
-    TLOG() << "rx " << j << " descriptors  " << count << "/" << queue_info.nb_desc;
-    TLOG() << "rx " << j << " scattered  " << (queue_info.scattered_rx ? "  yes" : "no");
-    TLOG() << "rx " << j << " conf.drop_en  " << (queue_info.conf.rx_drop_en ? "  yes" : "no");
-    TLOG() << "rx " << j << " conf.rx_deferred_start  " << (queue_info.conf.rx_deferred_start ? "  yes" : "no");
-    TLOG() << "rx " << j << " rx_buf_size  " << queue_info.rx_buf_size;
-
-
+    TLOG() << "rx[" << j << "] descriptors=" << count << "/" << queue_info.nb_desc
+           << " scattered=" << (queue_info.scattered_rx ? "yes" : "no")
+           << " conf.drop_en=" << (queue_info.conf.rx_drop_en ? "yes" : "no")
+           << " conf.rx_deferred_start=" << (queue_info.conf.rx_deferred_start ? "yes" : "no")
+           << " rx_buf_size=" << queue_info.rx_buf_size;
   }
 
-
   return 0;
 }
 
@@ -266,17 +257,16 @@ get_mempool(const std::string& pool_name,
   return std::unique_ptr<rte_mempool>(mbuf_pool);
 }
 
-std::vector<char*>
-string_to_eal_args(const std::string& params)
-{
-  auto parts = boost::program_options::split_unix(params);
-  std::vector<char*> cstrings;
-  for(auto& str : parts){
-    cstrings.push_back(const_cast<char*> (str.c_str()));
+std::vector<char*> 
+construct_eal_argv(std::vector<std::string> &std_argv){
+  std::vector<char*> vec_argv;
+  for (int i=0; i < std_argv.size() ; i++){
+      vec_argv.insert(vec_argv.end(), std_argv[i].data());
   }
-  return cstrings;
+  return vec_argv;
 }
 
+
 void
 init_eal(int argc, char* argv[]) {
 

include/dpdklibs/RTEIfaceSetup.hpp

@@ -126,6 +126,27 @@ get_iface_mac_str(uint16_t iface)
   }
 }
 
+inline std::string
+get_iface_pci_str(uint16_t iface)
+{
+  std::string iface_pci_addr_str;
+  int retval = -1;
+  struct rte_eth_dev_info dev_info;
+  // Get interface info
+  retval = rte_eth_dev_info_get(iface, &dev_info);
+  if (retval != 0) {
+    TLOG() << "Error during getting device (iface " << iface << ") retval: " << retval;
+  } else if (dev_info.device) {
+    auto dev_name = rte_dev_name(dev_info.device);
+    iface_pci_addr_str = dev_name;
+    //TLOG() << "Dev name: " << dev_name;
+    //const auto bus = rte_dev_bus(dev_info.device);
+    //const auto bus_info = rte_dev_bus_info(dev_info.device);
+    //const auto dev_driver = rte_dev_driver(dev_info.device);
+  }
+  return iface_pci_addr_str;
+}
+
 inline int
 iface_reset(uint16_t iface)
 {

plugins/NICReceiver.cpp

@@ -117,141 +117,82 @@ NICReceiver::do_configure(const data_t& args)
 {
   TLOG() << get_name() << ": Entering do_conf() method";
   m_cfg = args.get<module_conf_t>();
+  auto ifaces_cfg = m_cfg.ifaces;
 
   // EAL setup
   TLOG() << "Setting up EAL with params from config.";
-  std::vector<char*> eal_params = ealutils::string_to_eal_args(m_cfg.eal_arg_list);
-  ealutils::init_eal(eal_params.size(), eal_params.data());
+  std::vector<std::string> eal_args;
+
+  eal_args.push_back("eal_cmdline");
+
+  // Enforce the process type to primary  
+  eal_args.push_back("--proc-type=primary");
+
+  // Construct the pcie devices allowed mask
+  for (const auto& iface_cfg : ifaces_cfg) {
+     eal_args.push_back("-a");
+     eal_args.push_back(iface_cfg.pci_addr);
+  }
+
+  // Use the first pcie device id as file prefix
+  // FIXME: Review this strategy - should work in most of cases, but it could be 
+  // confusing in configs with multiple interfaces
+  eal_args.push_back(fmt::format("--file-prefix={}", ifaces_cfg.front().pci_addr));
+
+  // Append the remaining dpdk parameters
+  std::istringstream iss(m_cfg.eal_arg_list);
+  std::string arg_from_str;
+  while (iss >> arg_from_str) {
+    if (!arg_from_str.empty()) {
+      eal_args.push_back(arg_from_str);
+    }
+  }
+
+  std::stringstream ss;
+  for( const auto& arg : eal_args) {
+    ss << arg << " ";
+  }
+  TLOG() << "EAL Init arguments: " << ss.str();
+
+  std::vector<char*> eal_argv = ealutils::construct_eal_argv(eal_args);
+  char** constructed_eal_argv = eal_argv.data();
+  int constructed_eal_argc = eal_args.size();
+  ealutils::init_eal(constructed_eal_argc, constructed_eal_argv);
 
   // Get available Interfaces from EAL
   auto available_ifaces = ifaceutils::get_num_available_ifaces();
   TLOG() << "Number of available interfaces: " << available_ifaces;
   for (unsigned int ifc_id=0; ifc_id<available_ifaces; ++ifc_id) {
     std::string mac_addr_str = ifaceutils::get_iface_mac_str(ifc_id);
+    std::string pci_addr_str = ifaceutils::get_iface_pci_str(ifc_id);
     m_mac_to_id_map[mac_addr_str] = ifc_id;
-    TLOG() << "Available iface with MAC=" << mac_addr_str << " logical ID=" << ifc_id;
+    m_pci_to_id_map[pci_addr_str] = ifc_id;
+    TLOG() << "Available iface with MAC=" << mac_addr_str << " PCIe=" <<  pci_addr_str << " logical ID=" << ifc_id;
   }
 
   // Configure expected (and available!) interfaces
-  auto ifaces_cfg = m_cfg.ifaces;
   for (const auto& iface_cfg : ifaces_cfg) {
      auto iface_mac_addr = iface_cfg.mac_addr;
-     if (m_mac_to_id_map.count(iface_mac_addr) != 0) {
+     auto iface_pci_addr = iface_cfg.pci_addr;
+     if ((m_mac_to_id_map.count(iface_mac_addr) != 0) && 
+         (m_pci_to_id_map.count(iface_pci_addr) != 0)) {
        auto iface_id = m_mac_to_id_map[iface_mac_addr];
-       TLOG() << "Configuring expected interface with MAC=" << iface_mac_addr << " Logical ID=" << iface_id;
+       TLOG() << "Configuring expected interface with MAC=" << iface_mac_addr 
+              << " PCIe=" << iface_pci_addr << " Logical ID=" << iface_id;
        m_ifaces[iface_id] = std::make_unique<IfaceWrapper>(iface_id, m_sources, m_run_marker);
        m_ifaces[iface_id]->conf(iface_cfg);
        m_ifaces[iface_id]->allocate_mbufs();
        m_ifaces[iface_id]->setup_interface();
        m_ifaces[iface_id]->setup_flow_steering();
        m_ifaces[iface_id]->setup_xstats();
      } else {
-       TLOG() << "No available interface with MAC=" << iface_mac_addr;
+       TLOG() << "No available interface with MAC=" << iface_mac_addr << " PCI=" << iface_pci_addr;
        ers::fatal(dunedaq::readoutlibs::InitializationError(
           ERS_HERE, "NICReceiver configuration failed due expected but unavailable interface!"));
      }
   }
 
   return;
-
-// #warning RS FIXME -> Removed for conf overhaul
-//     auto ip_sources = nullptr;
-//     auto rx_cores = nullptr;
-// //  m_iface_id = (uint16_t)m_cfg.card_id;
-// //  m_dest_ip = m_cfg.dest_ip;
-// //  auto ip_sources = m_cfg.ip_sources;
-// //  auto rx_cores = m_cfg.rx_cores; 
-// //  m_num_ip_sources = ip_sources.size();
-// //  m_num_rx_cores = rx_cores.size();
-// /*
-//   // Initialize RX core map
-//   for (auto rxc : rx_cores) {
-//     for (auto qid : rxc.rx_qs) {
-//       m_rx_core_map[rxc.lcore_id][qid] = "";
-//     }
-//   }
-
-//   // Setup expected IP sources
-//   for (auto src : ip_sources) {
-//     TLOG() << "IP source to register: ID=" << src.id << " IP=" << src.ip << " RX_Q=" << src.rx_q << " LC=" << src.lcore;
-//     // Extend mapping
-//     m_rx_core_map[src.lcore][src.rx_q] = src.ip;    
-//     m_rx_qs.insert(src.rx_q);
-//     // Create frame counter metric
-//     m_num_frames[src.id] = { 0 };
-//     m_num_bytes[src.id] = { 0 };
-//   }
-// */
-
-//   // Setup SourceConcepts
-//   // m_sources[]->configure if needed!?
-
-//   // Allocate pools and mbufs per queue
-//   TLOG() << "Allocating pools and mbufs.";
-//   for (size_t i=0; i<m_rx_qs.size(); ++i) {
-//     std::stringstream ss;
-//     ss << "MBP-" << i;
-//     TLOG() << "Pool acquire: " << ss.str(); 
-//     m_mbuf_pools[i] = ealutils::get_mempool(ss.str());
-//     m_bufs[i] = (rte_mbuf**) malloc(sizeof(struct rte_mbuf*) * m_burst_size);
-//     rte_pktmbuf_alloc_bulk(m_mbuf_pools[i].get(), m_bufs[i], m_burst_size);
-//   }
-
-//   // Setting up interface
-//   TLOG() << "Initialize interface " << m_iface_id;
-//   bool with_reset = true, with_mq_mode = true; // go to config
-//   ealutils::iface_init(m_iface_id, m_rx_qs.size(), 0, m_mbuf_pools, with_reset, with_mq_mode); // 0 = no tx queues
-//   // Promiscuous mode
-//   ealutils::iface_promiscuous_mode(m_iface_id, false); // should come from config
-
-//   // Flow steering setup
-//   TLOG() << "Configuring Flow steering rules.";
-//   struct rte_flow_error error;
-//   struct rte_flow *flow;
-//   TLOG() << "Attempt to flush previous flow rules...";
-//   rte_flow_flush(m_iface_id, &error);
-// #warning RS: FIXME -> Check for flow flush return!
-//   for (auto const& [lcoreid, rxqs] : m_rx_core_map) {
-//     for (auto const& [rxqid, srcip] : rxqs) {
-
-//       // Put the IP numbers temporarily in a vector, so they can be converted easily to uint32_t
-//       TLOG() << "Current ip is " << srcip;
-//       size_t ind = 0, current_ind = 0;
-//       std::vector<uint8_t> v;
-//       for (int i = 0; i < 4; ++i) {
-//         v.push_back(std::stoi(srcip.substr(current_ind, srcip.size() - current_ind), &ind));
-//         current_ind += ind + 1;
-//       }
-
-//       flow = generate_ipv4_flow(m_iface_id, rxqid, 
-// 		    RTE_IPV4(v[0], v[1], v[2], v[3]), 0xffffffff, 0, 0, &error);
-
-//       if (not flow) { // ers::fatal
-//         TLOG() << "Flow can't be created for " << rxqid
-// 	       << " Error type: " << (unsigned)error.type
-// 	       << " Message: " << error.message;
-//         ers::fatal(dunedaq::readoutlibs::InitializationError(
-//           ERS_HERE, "Couldn't create Flow API rules!"));
-//     	  rte_exit(EXIT_FAILURE, "error in creating flow");
-//       }
-//     }
-//   }
-
-// #warning RS FIXME -> Removed for conf overhaul
-// //  if (m_cfg.with_drop_flow) {
-//     // Adding drop flow
-//     TLOG() << "Adding Drop Flow.";
-//     flow = generate_drop_flow(m_iface_id, &error);
-//     if (not flow) { // ers::fatal
-//       TLOG() << "Drop flow can't be created for interface!"
-//              << " Error type: " << (unsigned)error.type
-//              << " Message: " << error.message;
-//       rte_exit(EXIT_FAILURE, "error in creating flow");
-//     }
-// //  }
-
-//   TLOG() << "DPDK EAL & RTE configured.";
-
 }
 
 void
@@ -334,6 +275,4 @@ NICReceiver::set_running(bool should_run)
 } // namespace dpdklibs
 } // namespace dunedaq
 
-// #include "detail/NICReceiver.hxx"
-
 DEFINE_DUNE_DAQ_MODULE(dunedaq::dpdklibs::NICReceiver)

plugins/NICReceiver.hpp

@@ -106,6 +106,7 @@ class NICReceiver : public dunedaq::appfwk::DAQModule
 
   // Interfaces (logical ID, MAC) -> IfaceWrapper
   std::map<std::string, uint16_t> m_mac_to_id_map;
+  std::map<std::string, uint16_t> m_pci_to_id_map;
   std::map<uint16_t, std::unique_ptr<IfaceWrapper>> m_ifaces;
 
   // Sinks (SourceConcepts)

schema/dpdklibs/nicreader.jsonnet

@@ -31,6 +31,8 @@ local nicreader = {
 
     ipv4:   s.string("ipv4", pattern=moo.re.ipv4, doc="ipv4 string"),
 
+    pci:    s.string("pci", doc="PCIe address string"),
+
     mac:    s.string("mac", pattern="^[a-fA-F0-9]{2}(:[a-fA-F0-9]{2}){5}$", doc="mac string"),
 
     float : s.number("Float", "f4", doc="A float number"),
@@ -67,7 +69,8 @@ local nicreader = {
     ], doc="Source field"),
 
     iface : s.record("Interface", [
-        s.field("mac_addr", self.mac, "AA:BB:CC:DD:EE:FF", doc="Logical Interface ID"),
+        s.field("pci_addr", self.pci, "0000:00:00.0", doc="PCIe address of the interface"),
+        s.field("mac_addr", self.mac, "AA:BB:CC:DD:EE:FF", doc="MAC address of the interface"),
         s.field("ip_addr", self.ipv4, "192.168.0.1", doc="IP address of interface"),
         s.field("with_flow_control", self.choice, true, doc="FlowAPI enabled"),
         s.field("promiscuous_mode", self.choice, false, doc="Promiscuous mode enabled"),
@@ -88,7 +91,7 @@ local nicreader = {
         s.field("ifaces", self.ifaces,
                 doc="List of interfaces to configure"),
 
-        s.field("eal_arg_list", self.string, "",
+        s.field("eal_arg_list", self.string, "daq_application",
                 doc="A string with EAL arguments"),
     ], doc="Generic UIO reader DAQ Module Configuration"),
 

test/apps/test_dpdk_stats.cxx

@@ -1,6 +1,7 @@
 /* Application will run until quit or killed. */
 
 #include "dpdklibs/EALSetup.hpp"
+#include "dpdklibs/RTEIfaceSetup.hpp"
 #include "logging/Logging.hpp"
 #include "dpdklibs/udp/PacketCtor.hpp"
 #include "dpdklibs/udp/Utils.hpp"
@@ -198,6 +199,9 @@ main(int argc, char* argv[])
   auto nb_ifaces = rte_eth_dev_count_avail();
   TLOG() << "# of available interfaces: " << nb_ifaces;
   TLOG() << "Initialize interface " << iface_id;
+  TLOG() << "  -> Iface MAC: " << ifaceutils::get_iface_mac_str(iface_id);
+  TLOG() << "  -> Iface PCI: " << ifaceutils::get_iface_pci_str(iface_id);
+
   ealutils::iface_init(iface_id, rx_qs, tx_qs, rx_ring_size, tx_ring_size, mbuf_pools, false, false);
   ealutils::iface_promiscuous_mode(iface_id, false); // should come from config
 

test/apps/test_multi_proc.cxx

@@ -362,6 +362,7 @@ int main(int argc, char** argv){
     app.add_option("-i", iface, "Interface to init");
     app.add_option("-t", time_per_report, "Time Per Report");
     app.add_option("-a", allow_dev, "device to allow");
+
     app.add_flag("--check-time", check_timestamp, "Report back differences in timestamp");
     app.add_flag("-p", per_stream_reports, "Detailed per stream reports");
     CLI11_PARSE(app, argc, argv);
@@ -372,9 +373,11 @@ int main(int argc, char** argv){
     std::vector<std::string> eal_args;
     eal_args.push_back("dpdklibds_test_frame_receiver");
     eal_args.push_back("--proc-type=primary");
-    // eal_args.push_back(fmt::format("--file-prexix={}"));
+
     if (!allow_dev.empty()) {
         eal_args.push_back(fmt::format("--allow={}",allow_dev));
+        // Use the device id as file prefix to 
+        eal_args.push_back(fmt::format("--file-prefix={}", allow_dev));
     }
 
     TLOG() << "Status of RTE_ETH_DEV_FLOW_OPS_THREAD_SAFE=" << RTE_ETH_DEV_FLOW_OPS_THREAD_SAFE;