Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Moved EALSetup code to source file #106

Merged
merged 1 commit into from
Jan 18, 2024
Merged

Moved EALSetup code to source file #106

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
273 changes: 13 additions & 260 deletions include/dpdklibs/EALSetup.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,288 +19,41 @@ namespace dunedaq {
namespace dpdklibs {
namespace ealutils {

#define RX_RING_SIZE 1024
#define TX_RING_SIZE 1024

#define NUM_MBUFS 8191
#define MBUF_CACHE_SIZE 250

#define PG_JUMBO_FRAME_LEN (9600 + RTE_ETHER_CRC_LEN + RTE_ETHER_HDR_LEN)
#ifndef RTE_JUMBO_ETHER_MTU
#define RTE_JUMBO_ETHER_MTU (PG_JUMBO_FRAME_LEN - RTE_ETHER_HDR_LEN - RTE_ETHER_CRC_LEN) /*< Ethernet MTU. */
#endif

static volatile uint8_t dpdk_quit_signal;

static const struct rte_eth_conf iface_conf_default = {
.rxmode = {
.mtu = 9000,
.max_lro_pkt_size = 9000,
//.split_hdr_size = 0, // deprecated in [email protected]
.offloads = (RTE_ETH_RX_OFFLOAD_TIMESTAMP
| RTE_ETH_RX_OFFLOAD_IPV4_CKSUM
| RTE_ETH_RX_OFFLOAD_UDP_CKSUM),
},

.txmode = {
.offloads = (RTE_ETH_TX_OFFLOAD_MULTI_SEGS),
},
};


std::string get_mac_addr_str(const rte_ether_addr& addr) {
std::stringstream macstr;
macstr << std::hex << static_cast<int>(addr.addr_bytes[0]) << ":" << static_cast<int>(addr.addr_bytes[1]) << ":" << static_cast<int>(addr.addr_bytes[2]) << ":" << static_cast<int>(addr.addr_bytes[3]) << ":" << static_cast<int>(addr.addr_bytes[4]) << ":" << static_cast<int>(addr.addr_bytes[5]) << std::dec;
return macstr.str();
}

std::string get_mac_addr_str(const rte_ether_addr& addr);

// Modifies Ethernet device configuration to multi-queue RSS with offload
inline void
iface_conf_rss_mode(struct rte_eth_conf& iface_conf, bool mode = false, bool offload = false)
{
if (mode) {
iface_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_RSS;
if (offload) {
iface_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_RSS_HASH;
}
} else {
iface_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE;
}
}
void iface_conf_rss_mode(struct rte_eth_conf& iface_conf, bool mode = false, bool offload = false);

// Enables RX in promiscuous mode for the Ethernet device.
inline int
iface_promiscuous_mode(std::uint16_t iface, bool mode = false)
{
int retval = -1;
retval = rte_eth_promiscuous_get(iface);
TLOG() << "Before modification attempt, promiscuous mode is: " << retval;
if (mode) {
retval = rte_eth_promiscuous_enable(iface);
} else {
retval = rte_eth_promiscuous_disable(iface);
}
if (retval != 0) {
TLOG() << "Couldn't modify promiscuous mode of iface[" << iface << "]! Error code: " << retval;
}
retval = rte_eth_promiscuous_get(iface);
TLOG() << "New promiscuous mode of iface[" << iface << "] is: " << retval;
return retval;
}

int iface_promiscuous_mode(std::uint16_t iface, bool mode = false);


inline int
iface_init(uint16_t iface, uint16_t rx_rings, uint16_t tx_rings,
int iface_init(uint16_t iface, uint16_t rx_rings, uint16_t tx_rings,
uint16_t rx_ring_size, uint16_t tx_ring_size,
std::map<int, std::unique_ptr<rte_mempool>>& mbuf_pool,
bool with_reset=false, bool with_mq_rss=false)
{
struct rte_eth_conf iface_conf = iface_conf_default;
// uint16_t nb_rxd = RX_RING_SIZE;
// uint16_t nb_txd = TX_RING_SIZE;
uint16_t nb_rxd = rx_ring_size;
uint16_t nb_txd = tx_ring_size;
int retval = -1;
uint16_t q;
struct rte_eth_dev_info dev_info;
struct rte_eth_txconf txconf;

// Get interface validity
if (!rte_eth_dev_is_valid_port(iface)) {
TLOG() << "Specified interface " << iface << " is not valid in EAL!";
return retval;
}

// Get interface info
retval = rte_eth_dev_info_get(iface, &dev_info);
if (retval != 0) {
TLOG() << "Error during getting device (iface " << iface << ") retval: " << retval;
return retval;
}

TLOG() << "Iface " << iface << " RX Ring info :"
<< " min " << dev_info.rx_desc_lim.nb_min
<< " max " << dev_info.rx_desc_lim.nb_max
<< " align " << dev_info.rx_desc_lim.nb_align
;

// Carry out a reset of the interface
if (with_reset) {
retval = rte_eth_dev_reset(iface);
if (retval != 0) {
TLOG() << "Resetting device (iface " << iface << ") failed. Retval: " << retval;
}
}

// Should we configure MQ RSS and offload?
if (with_mq_rss) {
iface_conf_rss_mode(iface_conf, true, true); // with_rss, with_offload
// RSS
if ((iface_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG) != 0) {
TLOG() << "Ethdev port config prepared with RX RSS mq_mode!";
if ((iface_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_RSS_HASH) != 0) {
TLOG() << "Ethdev port config prepared with RX RSS mq_mode with offloading is requested!";
}
}
}

// Configure the Ethernet interface
retval = rte_eth_dev_configure(iface, rx_rings, tx_rings, &iface_conf);
if (retval != 0)
return retval;

// Set MTU of interface
rte_eth_dev_set_mtu(iface, RTE_JUMBO_ETHER_MTU);
{
uint16_t mtu;
rte_eth_dev_get_mtu(iface, &mtu);
TLOG() << "Interface: " << iface << " MTU: " << mtu;
}

// Adjust RX/TX ring sizes
retval = rte_eth_dev_adjust_nb_rx_tx_desc(iface, &nb_rxd, &nb_txd);
if (retval != 0)
return retval;
bool with_reset=false, bool with_mq_rss=false);

// Allocate and set up RX queues for interface.
for (q = 0; q < rx_rings; q++) {
retval = rte_eth_rx_queue_setup(iface, q, nb_rxd, rte_eth_dev_socket_id(iface), NULL, mbuf_pool[q].get());
if (retval < 0)
return retval;
}

txconf = dev_info.default_txconf;
txconf.offloads = iface_conf.txmode.offloads;

// These values influenced by Sec. 8.4.4 of https://doc.dpdk.org/guides-1.8/prog_guide/poll_mode_drv.html
txconf.tx_rs_thresh = 32;
txconf.tx_free_thresh = 32;
txconf.tx_thresh.wthresh = 0;

// Allocate and set up TX queues for interface.
for (q = 0; q < tx_rings; q++) {
retval = rte_eth_tx_queue_setup(iface, q, nb_txd, rte_eth_dev_socket_id(iface), &txconf);
if (retval < 0)
return retval;
}

// Start the Ethernet interface.
retval = rte_eth_dev_start(iface);
if (retval < 0)
return retval;

// Display the interface MAC address.
struct rte_ether_addr addr;
retval = rte_eth_macaddr_get(iface, &addr);
if (retval == 0) {
TLOG() << "MAC address: " << get_mac_addr_str(addr);
} else {
return retval;
}

// Get interface info
retval = rte_eth_dev_info_get(iface, &dev_info);
if (retval != 0) {
TLOG() << "Error during getting device (iface " << iface << ") retval: " << retval;
return retval;
}

TLOG() << "Iface[" << iface << "] Rx Ring info:"
<< " min=" << dev_info.rx_desc_lim.nb_min
<< " max=" << dev_info.rx_desc_lim.nb_max
<< " align=" << dev_info.rx_desc_lim.nb_align;
TLOG() << "Iface[" << iface << "] Tx Ring info:"
<< " min=" << dev_info.rx_desc_lim.nb_min
<< " max=" << dev_info.rx_desc_lim.nb_max
<< " align=" << dev_info.rx_desc_lim.nb_align;

for (size_t j = 0; j < dev_info.nb_rx_queues; j++) {

struct rte_eth_rxq_info queue_info;
int count;

retval = rte_eth_rx_queue_info_get(iface, j, &queue_info);
if (retval != 0)
break;

count = rte_eth_rx_queue_count(iface, j);
TLOG() << "rx[" << j << "] descriptors=" << count << "/" << queue_info.nb_desc
<< " scattered=" << (queue_info.scattered_rx ? "yes" : "no")
<< " conf.drop_en=" << (queue_info.conf.rx_drop_en ? "yes" : "no")
<< " conf.rx_deferred_start=" << (queue_info.conf.rx_deferred_start ? "yes" : "no")
<< " rx_buf_size=" << queue_info.rx_buf_size;
}

return 0;
}

std::unique_ptr<rte_mempool>
get_mempool(const std::string& pool_name,
std::unique_ptr<rte_mempool> get_mempool(const std::string& pool_name,
int num_mbufs=NUM_MBUFS, int mbuf_cache_size=MBUF_CACHE_SIZE,
int data_room_size=9800, int socket_id=0) {
TLOG() << "get_mempool with: NUM_MBUFS = " << num_mbufs
<< " | MBUF_CACHE_SIZE = " << mbuf_cache_size
<< " | data_room_size = " << data_room_size
<< " | SOCKET_ID = " << socket_id;

struct rte_mempool *mbuf_pool;
mbuf_pool = rte_pktmbuf_pool_create(pool_name.c_str(), num_mbufs,
mbuf_cache_size, 0, data_room_size,
socket_id);

if (mbuf_pool == NULL) {
// ers fatal
rte_exit(EXIT_FAILURE, "ERROR: Cannot create rte_mempool!\n");
}
return std::unique_ptr<rte_mempool>(mbuf_pool);
}
int data_room_size=9800, int socket_id=0);

std::vector<char*>
construct_eal_argv(std::vector<std::string> &std_argv){
std::vector<char*> vec_argv;
for (int i=0; i < std_argv.size() ; i++){
vec_argv.insert(vec_argv.end(), std_argv[i].data());
}
return vec_argv;
}
std::vector<const char*> construct_eal_argv(const std::vector<std::string> &std_argv);

void init_eal(int argc, const char* argv[]);

void
init_eal(int argc, char* argv[]) {
void init_eal( const std::vector<std::string>& args );

// Init EAL
int ret = rte_eal_init(argc, argv);
if (ret < 0) {
rte_exit(EXIT_FAILURE, "ERROR: EAL initialization failed.\n");
}
TLOG() << "EAL initialized with provided parameters.";
}
int get_available_ifaces();

int
get_available_ifaces() {
// Check that there is an even number of interfaces to send/receive on
unsigned nb_ifaces;
nb_ifaces = rte_eth_dev_count_avail();
TLOG() << "Available interfaces: " << nb_ifaces;
return nb_ifaces;
}
int wait_for_lcores();

int
wait_for_lcores() {
int lcore_id;
int ret = 0;
RTE_LCORE_FOREACH_WORKER(lcore_id) {
//TLOG() << "Waiting for lcore[" << lcore_id << "] to finish packet processing.";
ret = rte_eal_wait_lcore(lcore_id);
}
return ret;
}
void finish_eal();

void finish_eal() {
rte_eal_cleanup();
}

} // namespace ealutils
} // namespace dpdklibs
Expand Down
11 changes: 1 addition & 10 deletions plugins/NICReceiver.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,16 +148,7 @@ NICReceiver::do_configure(const data_t& args)
}
}

std::stringstream ss;
for( const auto& arg : eal_args) {
ss << arg << " ";
}
TLOG() << "EAL Init arguments: " << ss.str();

std::vector<char*> eal_argv = ealutils::construct_eal_argv(eal_args);
char** constructed_eal_argv = eal_argv.data();
int constructed_eal_argc = eal_args.size();
ealutils::init_eal(constructed_eal_argc, constructed_eal_argv);
ealutils::init_eal(eal_args);

// Get available Interfaces from EAL
auto available_ifaces = ifaceutils::get_num_available_ifaces();
Expand Down
Loading