Skip to content

Commit

Permalink
Merge pull request #106 from DUNE-DAQ/feature/dpdk_multiproc
Browse files Browse the repository at this point in the history 
Moved EALSetup code to source file
  • Loading branch information
@roland-sipos
roland-sipos authored Jan 18, 2024
2 parents eb148c0 + a40a4f2 commit 37745fb
Show file tree
Hide file tree
Showing 3 changed files with 335 additions and 270 deletions.
273 changes: 13 additions & 260 deletions include/dpdklibs/EALSetup.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,288 +19,41 @@ namespace dunedaq {
namespace dpdklibs {
namespace ealutils {

#define RX_RING_SIZE 1024
#define TX_RING_SIZE 1024

#define NUM_MBUFS 8191
#define MBUF_CACHE_SIZE 250

#define PG_JUMBO_FRAME_LEN (9600 + RTE_ETHER_CRC_LEN + RTE_ETHER_HDR_LEN)
#ifndef RTE_JUMBO_ETHER_MTU
#define RTE_JUMBO_ETHER_MTU (PG_JUMBO_FRAME_LEN - RTE_ETHER_HDR_LEN - RTE_ETHER_CRC_LEN) /*< Ethernet MTU. */
#endif

static volatile uint8_t dpdk_quit_signal;

static const struct rte_eth_conf iface_conf_default = {
.rxmode = {
.mtu = 9000,
.max_lro_pkt_size = 9000,
//.split_hdr_size = 0, // deprecated in [email protected]
.offloads = (RTE_ETH_RX_OFFLOAD_TIMESTAMP
| RTE_ETH_RX_OFFLOAD_IPV4_CKSUM
| RTE_ETH_RX_OFFLOAD_UDP_CKSUM),
},

.txmode = {
.offloads = (RTE_ETH_TX_OFFLOAD_MULTI_SEGS),
},
};


std::string get_mac_addr_str(const rte_ether_addr& addr) {
std::stringstream macstr;
macstr << std::hex << static_cast<int>(addr.addr_bytes[0]) << ":" << static_cast<int>(addr.addr_bytes[1]) << ":" << static_cast<int>(addr.addr_bytes[2]) << ":" << static_cast<int>(addr.addr_bytes[3]) << ":" << static_cast<int>(addr.addr_bytes[4]) << ":" << static_cast<int>(addr.addr_bytes[5]) << std::dec;
return macstr.str();
}

std::string get_mac_addr_str(const rte_ether_addr& addr);

// Modifies Ethernet device configuration to multi-queue RSS with offload
inline void
iface_conf_rss_mode(struct rte_eth_conf& iface_conf, bool mode = false, bool offload = false)
{
if (mode) {
iface_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_RSS;
if (offload) {
iface_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_RSS_HASH;
}
} else {
iface_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE;
}
}
void iface_conf_rss_mode(struct rte_eth_conf& iface_conf, bool mode = false, bool offload = false);

// Enables RX in promiscuous mode for the Ethernet device.
inline int
iface_promiscuous_mode(std::uint16_t iface, bool mode = false)
{
int retval = -1;
retval = rte_eth_promiscuous_get(iface);
TLOG() << "Before modification attempt, promiscuous mode is: " << retval;
if (mode) {
retval = rte_eth_promiscuous_enable(iface);
} else {
retval = rte_eth_promiscuous_disable(iface);
}
if (retval != 0) {
TLOG() << "Couldn't modify promiscuous mode of iface[" << iface << "]! Error code: " << retval;
}
retval = rte_eth_promiscuous_get(iface);
TLOG() << "New promiscuous mode of iface[" << iface << "] is: " << retval;
return retval;
}

int iface_promiscuous_mode(std::uint16_t iface, bool mode = false);


inline int
iface_init(uint16_t iface, uint16_t rx_rings, uint16_t tx_rings,
int iface_init(uint16_t iface, uint16_t rx_rings, uint16_t tx_rings,
uint16_t rx_ring_size, uint16_t tx_ring_size,
std::map<int, std::unique_ptr<rte_mempool>>& mbuf_pool,
bool with_reset=false, bool with_mq_rss=false)
{
struct rte_eth_conf iface_conf = iface_conf_default;
// uint16_t nb_rxd = RX_RING_SIZE;
// uint16_t nb_txd = TX_RING_SIZE;
uint16_t nb_rxd = rx_ring_size;
uint16_t nb_txd = tx_ring_size;
int retval = -1;
uint16_t q;
struct rte_eth_dev_info dev_info;
struct rte_eth_txconf txconf;

// Get interface validity
if (!rte_eth_dev_is_valid_port(iface)) {
TLOG() << "Specified interface " << iface << " is not valid in EAL!";
return retval;
}

// Get interface info
retval = rte_eth_dev_info_get(iface, &dev_info);
if (retval != 0) {
TLOG() << "Error during getting device (iface " << iface << ") retval: " << retval;
return retval;
}

TLOG() << "Iface " << iface << " RX Ring info :"
<< " min " << dev_info.rx_desc_lim.nb_min
<< " max " << dev_info.rx_desc_lim.nb_max
<< " align " << dev_info.rx_desc_lim.nb_align
;

// Carry out a reset of the interface
if (with_reset) {
retval = rte_eth_dev_reset(iface);
if (retval != 0) {
TLOG() << "Resetting device (iface " << iface << ") failed. Retval: " << retval;
}
}

// Should we configure MQ RSS and offload?
if (with_mq_rss) {
iface_conf_rss_mode(iface_conf, true, true); // with_rss, with_offload
// RSS
if ((iface_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG) != 0) {
TLOG() << "Ethdev port config prepared with RX RSS mq_mode!";
if ((iface_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_RSS_HASH) != 0) {
TLOG() << "Ethdev port config prepared with RX RSS mq_mode with offloading is requested!";
}
}
}

// Configure the Ethernet interface
retval = rte_eth_dev_configure(iface, rx_rings, tx_rings, &iface_conf);
if (retval != 0)
return retval;

// Set MTU of interface
rte_eth_dev_set_mtu(iface, RTE_JUMBO_ETHER_MTU);
{
uint16_t mtu;
rte_eth_dev_get_mtu(iface, &mtu);
TLOG() << "Interface: " << iface << " MTU: " << mtu;
}

// Adjust RX/TX ring sizes
retval = rte_eth_dev_adjust_nb_rx_tx_desc(iface, &nb_rxd, &nb_txd);
if (retval != 0)
return retval;
bool with_reset=false, bool with_mq_rss=false);

// Allocate and set up RX queues for interface.
for (q = 0; q < rx_rings; q++) {
retval = rte_eth_rx_queue_setup(iface, q, nb_rxd, rte_eth_dev_socket_id(iface), NULL, mbuf_pool[q].get());
if (retval < 0)
return retval;
}

txconf = dev_info.default_txconf;
txconf.offloads = iface_conf.txmode.offloads;

// These values influenced by Sec. 8.4.4 of https://doc.dpdk.org/guides-1.8/prog_guide/poll_mode_drv.html
txconf.tx_rs_thresh = 32;
txconf.tx_free_thresh = 32;
txconf.tx_thresh.wthresh = 0;

// Allocate and set up TX queues for interface.
for (q = 0; q < tx_rings; q++) {
retval = rte_eth_tx_queue_setup(iface, q, nb_txd, rte_eth_dev_socket_id(iface), &txconf);
if (retval < 0)
return retval;
}

// Start the Ethernet interface.
retval = rte_eth_dev_start(iface);
if (retval < 0)
return retval;

// Display the interface MAC address.
struct rte_ether_addr addr;
retval = rte_eth_macaddr_get(iface, &addr);
if (retval == 0) {
TLOG() << "MAC address: " << get_mac_addr_str(addr);
} else {
return retval;
}

// Get interface info
retval = rte_eth_dev_info_get(iface, &dev_info);
if (retval != 0) {
TLOG() << "Error during getting device (iface " << iface << ") retval: " << retval;
return retval;
}

TLOG() << "Iface[" << iface << "] Rx Ring info:"
<< " min=" << dev_info.rx_desc_lim.nb_min
<< " max=" << dev_info.rx_desc_lim.nb_max
<< " align=" << dev_info.rx_desc_lim.nb_align;
TLOG() << "Iface[" << iface << "] Tx Ring info:"
<< " min=" << dev_info.rx_desc_lim.nb_min
<< " max=" << dev_info.rx_desc_lim.nb_max
<< " align=" << dev_info.rx_desc_lim.nb_align;

for (size_t j = 0; j < dev_info.nb_rx_queues; j++) {

struct rte_eth_rxq_info queue_info;
int count;

retval = rte_eth_rx_queue_info_get(iface, j, &queue_info);
if (retval != 0)
break;

count = rte_eth_rx_queue_count(iface, j);
TLOG() << "rx[" << j << "] descriptors=" << count << "/" << queue_info.nb_desc
<< " scattered=" << (queue_info.scattered_rx ? "yes" : "no")
<< " conf.drop_en=" << (queue_info.conf.rx_drop_en ? "yes" : "no")
<< " conf.rx_deferred_start=" << (queue_info.conf.rx_deferred_start ? "yes" : "no")
<< " rx_buf_size=" << queue_info.rx_buf_size;
}

return 0;
}

std::unique_ptr<rte_mempool>
get_mempool(const std::string& pool_name,
std::unique_ptr<rte_mempool> get_mempool(const std::string& pool_name,
int num_mbufs=NUM_MBUFS, int mbuf_cache_size=MBUF_CACHE_SIZE,
int data_room_size=9800, int socket_id=0) {
TLOG() << "get_mempool with: NUM_MBUFS = " << num_mbufs
<< " | MBUF_CACHE_SIZE = " << mbuf_cache_size
<< " | data_room_size = " << data_room_size
<< " | SOCKET_ID = " << socket_id;

struct rte_mempool *mbuf_pool;
mbuf_pool = rte_pktmbuf_pool_create(pool_name.c_str(), num_mbufs,
mbuf_cache_size, 0, data_room_size,
socket_id);

if (mbuf_pool == NULL) {
// ers fatal
rte_exit(EXIT_FAILURE, "ERROR: Cannot create rte_mempool!\n");
}
return std::unique_ptr<rte_mempool>(mbuf_pool);
}
int data_room_size=9800, int socket_id=0);

std::vector<char*>
construct_eal_argv(std::vector<std::string> &std_argv){
std::vector<char*> vec_argv;
for (int i=0; i < std_argv.size() ; i++){
vec_argv.insert(vec_argv.end(), std_argv[i].data());
}
return vec_argv;
}
std::vector<const char*> construct_eal_argv(const std::vector<std::string> &std_argv);

void init_eal(int argc, const char* argv[]);

void
init_eal(int argc, char* argv[]) {
void init_eal( const std::vector<std::string>& args );

// Init EAL
int ret = rte_eal_init(argc, argv);
if (ret < 0) {
rte_exit(EXIT_FAILURE, "ERROR: EAL initialization failed.\n");
}
TLOG() << "EAL initialized with provided parameters.";
}
int get_available_ifaces();

int
get_available_ifaces() {
// Check that there is an even number of interfaces to send/receive on
unsigned nb_ifaces;
nb_ifaces = rte_eth_dev_count_avail();
TLOG() << "Available interfaces: " << nb_ifaces;
return nb_ifaces;
}
int wait_for_lcores();

int
wait_for_lcores() {
int lcore_id;
int ret = 0;
RTE_LCORE_FOREACH_WORKER(lcore_id) {
//TLOG() << "Waiting for lcore[" << lcore_id << "] to finish packet processing.";
ret = rte_eal_wait_lcore(lcore_id);
}
return ret;
}
void finish_eal();

void finish_eal() {
rte_eal_cleanup();
}

} // namespace ealutils
} // namespace dpdklibs
Expand Down
11 changes: 1 addition & 10 deletions plugins/NICReceiver.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,16 +148,7 @@ NICReceiver::do_configure(const data_t& args)
}
}

std::stringstream ss;
for( const auto& arg : eal_args) {
ss << arg << " ";
}
TLOG() << "EAL Init arguments: " << ss.str();

std::vector<char*> eal_argv = ealutils::construct_eal_argv(eal_args);
char** constructed_eal_argv = eal_argv.data();
int constructed_eal_argc = eal_args.size();
ealutils::init_eal(constructed_eal_argc, constructed_eal_argv);
ealutils::init_eal(eal_args);

// Get available Interfaces from EAL
auto available_ifaces = ifaceutils::get_num_available_ifaces();
Expand Down
Loading

0 comments on commit 37745fb

Please sign in to comment.