From 8b2b0899905821dd94aab57b36ed6995096b36aa Mon Sep 17 00:00:00 2001 From: koichiimai Date: Thu, 2 Jan 2025 15:50:53 +0900 Subject: [PATCH 01/14] Copyless net driver Signed-off-by: koichiimai --- Makefile | 2 +- applications/tests/test_network/src/lib.rs | 54 ++--- .../src/pcie/intel/e1000e_example.rs | 2 +- awkernel_drivers/src/pcie/intel/igb.rs | 226 ++++++++++------- awkernel_drivers/src/pcie/intel/ixgbe.rs | 229 ++++++++++++------ awkernel_lib/src/dma_pool.rs | 22 ++ awkernel_lib/src/net.rs | 2 +- awkernel_lib/src/net/if_net.rs | 67 ++--- awkernel_lib/src/net/net_device.rs | 21 +- kernel/src/arch/x86_64/config.rs | 2 +- kernel/src/arch/x86_64/kernel_main.rs | 2 +- scripts/udp.py | 4 +- userland/Cargo.toml | 2 +- 13 files changed, 399 insertions(+), 236 deletions(-) diff --git a/Makefile b/Makefile index 14be06395..aa95fab58 100644 --- a/Makefile +++ b/Makefile @@ -56,7 +56,7 @@ X86_64_LD=$(LINKERDIR)/x86_64-link.lds RV32_LD=$(LINKERDIR)/rv32-link.lds RV64_LD=$(LINKERDIR)/rv64-link.lds -RUSTV=nightly-2024-12-02 +RUSTV=nightly-2024-12-30 all: aarch64 x86_64 riscv32 riscv64 std diff --git a/applications/tests/test_network/src/lib.rs b/applications/tests/test_network/src/lib.rs index 18949840a..9df6a3a15 100644 --- a/applications/tests/test_network/src/lib.rs +++ b/applications/tests/test_network/src/lib.rs @@ -31,33 +31,33 @@ pub async fn run() { ) .await; - awkernel_async_lib::spawn( - "test tcp listen".into(), - tcp_listen_test(), - awkernel_async_lib::scheduler::SchedulerType::FIFO, - ) - .await; - - awkernel_async_lib::spawn( - "test tcp connect".into(), - tcp_connect_test(), - awkernel_async_lib::scheduler::SchedulerType::FIFO, - ) - .await; - - awkernel_async_lib::spawn( - "test IPv4 multicast recv".into(), - ipv4_multicast_recv_test(), - awkernel_async_lib::scheduler::SchedulerType::FIFO, - ) - .await; - - awkernel_async_lib::spawn( - "test IPv4 multicast send".into(), - ipv4_multicast_send_test(), - awkernel_async_lib::scheduler::SchedulerType::FIFO, - ) - .await; + //awkernel_async_lib::spawn( + //"test tcp listen".into(), + //tcp_listen_test(), + //awkernel_async_lib::scheduler::SchedulerType::FIFO, + //) + //.await; + + //awkernel_async_lib::spawn( + //"test tcp connect".into(), + //tcp_connect_test(), + //awkernel_async_lib::scheduler::SchedulerType::FIFO, + //) + //.await; + + //awkernel_async_lib::spawn( + //"test IPv4 multicast recv".into(), + //ipv4_multicast_recv_test(), + //awkernel_async_lib::scheduler::SchedulerType::FIFO, + //) + //.await; + + //awkernel_async_lib::spawn( + //"test IPv4 multicast send".into(), + //ipv4_multicast_send_test(), + //awkernel_async_lib::scheduler::SchedulerType::FIFO, + //) + //.await; } async fn ipv4_multicast_send_test() { diff --git a/awkernel_drivers/src/pcie/intel/e1000e_example.rs b/awkernel_drivers/src/pcie/intel/e1000e_example.rs index 60672e516..cfc716b8b 100644 --- a/awkernel_drivers/src/pcie/intel/e1000e_example.rs +++ b/awkernel_drivers/src/pcie/intel/e1000e_example.rs @@ -127,7 +127,7 @@ impl NetDevice for E1000eExample { fn send( &self, - _data: net_device::EtherFrameRef, + _data: net_device::EtherFrameBuf, _que_id: usize, ) -> Result<(), net_device::NetDevError> { todo!("send"); diff --git a/awkernel_drivers/src/pcie/intel/igb.rs b/awkernel_drivers/src/pcie/intel/igb.rs index 8113114e4..b0691b84f 100644 --- a/awkernel_drivers/src/pcie/intel/igb.rs +++ b/awkernel_drivers/src/pcie/intel/igb.rs @@ -20,8 +20,8 @@ use awkernel_lib::{ ipv6::Ip6Hdr, multicast::MulticastAddrs, net_device::{ - EtherFrameBuf, EtherFrameRef, LinkStatus, NetCapabilities, NetDevError, NetDevice, - NetFlags, PacketHeaderFlags, + EtherFrameBuf, LinkStatus, NetCapabilities, NetDevError, NetDevice, NetFlags, + PacketHeaderFlags, }, tcp::TCPHdr, udp::UDPHdr, @@ -128,9 +128,8 @@ struct Rx { rx_desc_tail: usize, rx_desc_ring: DMAPool, - read_buf: Option>, - read_queue: RingQ, + dma_info: Vec<(usize, usize, usize)>, // Statistics dropped_pkts: u64, @@ -151,8 +150,7 @@ struct Tx { txd_cmd: u32, active_checksum_context: ActiveChecksumContext, - - write_buf: Option>, + dma_info: Vec<(bool, usize, usize, usize)>, } struct Queue { @@ -421,7 +419,7 @@ impl IgbInner { let mut que = Vec::new(); for i in 0..que_num { - que.push(allocate_desc_rings(&info, i)?); + que.push(allocate_desc_rings(&info, que_num, i)?); } let is_poll_mode; @@ -504,6 +502,9 @@ impl IgbInner { hw.set_get_link_status(true); + let mut dma_info = Vec::with_capacity(MAX_RXD * que_num); + dma_info.resize(MAX_RXD * que_num, (0, 0, 0)); + let igb = Self { info, hw, @@ -906,54 +907,50 @@ impl IgbInner { tx.tx_desc_head = 0; let tx_desc_ring = tx.tx_desc_ring.as_mut(); - - let tx_buffer_size = TXBUFFER_2048 as usize * MAX_TXD; - let write_buf = DMAPool::new( - self.info.get_segment_group() as usize, - tx_buffer_size / PAGESIZE, - ) - .ok_or(IgbDriverErr::DMAPool)?; - - let buf_phy_addr = write_buf.get_phy_addr().as_usize(); - - for (i, desc) in tx_desc_ring.iter_mut().enumerate() { - desc.legacy.buf = (buf_phy_addr + i * TXBUFFER_2048 as usize) as u64; + for desc in tx_desc_ring.iter_mut() { + desc.legacy.buf = 0; desc.legacy.lower.raw = 0; desc.legacy.upper.raw = 0; } - - tx.write_buf = Some(write_buf); } Ok(()) } fn setup_receive_structures(&mut self, que: &[Queue]) -> Result<(), IgbDriverErr> { - for que in que.iter() { + for (i, que) in que.iter().enumerate() { let mut node = MCSNode::new(); let mut rx = que.rx.lock(&mut node); rx.rx_desc_tail = 0; - rx.rx_desc_head = rx.rx_desc_ring.as_ref().len() as u32 - 1; + let rx_desc_ring_len = rx.rx_desc_ring.as_ref().len(); + rx.rx_desc_head = rx_desc_ring_len as u32 - 1; let rx_desc_ring = rx.rx_desc_ring.as_mut(); + let mut dma_info_temp = Vec::new(); - let rx_buffer_size = RXBUFFER_2048 as usize * MAX_RXD; - let read_buf = DMAPool::new( - self.info.get_segment_group() as usize, - rx_buffer_size / PAGESIZE, - ) - .ok_or(IgbDriverErr::DMAPool)?; + for desc in rx_desc_ring { + let read_buf: DMAPool<[u8; PAGESIZE]> = + DMAPool::new(self.info.get_segment_group() as usize, 1) + .ok_or(IgbDriverErr::DMAPool)?; + let buf_phy_addr = read_buf.get_phy_addr().as_usize(); + desc.data = [0; 2]; - let buf_phy_addr = read_buf.get_phy_addr().as_usize(); + desc.desc.buf = buf_phy_addr as u64; - for (i, desc) in rx_desc_ring.iter_mut().enumerate() { - desc.data = [0; 2]; - desc.desc.buf = (buf_phy_addr + i * RXBUFFER_2048 as usize) as u64; - desc.desc.len = RXBUFFER_2048 as u16; + desc.desc.len = RXBUFFER_2048 as u16; // TODO: Need to check if we really need this. + + dma_info_temp.push(( + read_buf.get_virt_addr().as_usize(), + buf_phy_addr, + self.info.get_segment_group() as usize, + )); + read_buf.leak(); } - rx.read_buf = Some(read_buf); + for (j, info) in dma_info_temp.iter().enumerate() { + rx.dma_info[rx_desc_ring_len * i + j] = info.clone(); + } } Ok(()) @@ -974,15 +971,7 @@ impl IgbInner { self.hw.reset_hw(&self.info)?; } - for que in que.iter() { - let mut node = MCSNode::new(); - let mut tx = que.tx.lock(&mut node); - tx.write_buf = None; - - let mut node = MCSNode::new(); - let mut rx = que.rx.lock(&mut node); - rx.read_buf = None; - } + // TODO: Need to think about the deallocation of DMAPool Ok(()) } @@ -1186,6 +1175,21 @@ impl Igb { Ok(()) } + // TODO: Not sure if we really need this + unsafe fn invalidate_cache(&self, addr: *const u8) -> Result<(), IgbDriverErr> { + #[cfg(target_arch = "x86_64")] + { + core::arch::x86_64::_mm_clflush(addr); + Ok(()) + } + + #[cfg(not(any(target_arch = "x86_64")))] + { + let _ = addr; + Err(IxgbeDriverErr::NotSupported) + } + } + fn rx_recv(&self, que_id: usize) -> Result<(), IgbDriverErr> { let mac_type = self.inner.read().hw.get_mac_type(); @@ -1193,14 +1197,11 @@ impl Igb { { let inner = self.inner.read(); + let numa_id = inner.info.get_segment_group() as usize; let mut node = MCSNode::new(); let mut rx = que.rx.lock(&mut node); - if rx.read_buf.is_none() { - return Ok(()); - } - let mut i = rx.rx_desc_tail; loop { @@ -1212,10 +1213,11 @@ impl Igb { break; } - let rx_desc_ring = rx.rx_desc_ring.as_ref(); + let mut dropped_pkts = rx.dropped_pkts; + let rx_desc_ring = rx.rx_desc_ring.as_mut(); let rx_desc_ring_len = rx_desc_ring.len(); - let desc = unsafe { &rx_desc_ring[i].desc }; + let desc = unsafe { &mut rx_desc_ring.as_mut()[i].desc }; let status = desc.status; let errors = desc.error; @@ -1251,7 +1253,7 @@ impl Igb { len -= 1; true } else { - rx.dropped_pkts += 1; + dropped_pkts += 1; false } } else { @@ -1259,13 +1261,46 @@ impl Igb { }; if is_accept { - let read_buf = rx.read_buf.as_ref().unwrap(); - let src = &read_buf.as_ref()[i]; - let data = src[0..len].to_vec(); + let read_buf: DMAPool<[u8; PAGESIZE]> = + DMAPool::new(numa_id, 1).ok_or(IgbDriverErr::DMAPool)?; + let buf_phy_addr = read_buf.get_phy_addr().as_usize(); + desc.buf = buf_phy_addr as u64; + + let index = (rx_desc_ring_len * que_id + i) as usize; + let (virt_addr, phy_addr, numa_id) = rx.dma_info[index]; + + let ptr = virt_addr as *mut [u8; PAGESIZE]; + let data; + unsafe { + data = DMAPool::<[u8; PAGESIZE]>::from_raw_parts( + ptr, phy_addr, + PAGESIZE, // RECONSIDER: Not using "len" here, might be better to give the "len" information somehow to protocol stack. + numa_id, + ) + .unwrap(); + } + + rx.dma_info[index] = + (read_buf.get_virt_addr().as_usize(), buf_phy_addr, numa_id); + + // Need clflush + unsafe { + self.invalidate_cache(read_buf.get_virt_addr().as_usize() as *const u8)?; + } - rx.read_queue.push(EtherFrameBuf { data, vlan }).unwrap(); + read_buf.leak(); + rx.read_queue + .push(EtherFrameBuf { + data, + len: len as usize, + vlan, + csum_flags: PacketHeaderFlags::EMPTY, + }) + .unwrap(); }; + rx.dropped_pkts = dropped_pkts; + i += 1; if i == rx_desc_ring_len { i = 0; @@ -1312,13 +1347,15 @@ impl Igb { fn transmit_checksum_setup( &self, tx: &mut Tx, - ether_frame: &EtherFrameRef, + ether_frame: &EtherFrameBuf, head: usize, ) -> Result<(usize, u32, u32, u16, Option), IgbDriverErr> { let txd_upper; let txd_lower; - let ext = extract_headers(ether_frame.data).or(Err(IgbDriverErr::InvalidPacket))?; + let ptr = ether_frame.data.get_virt_addr().as_mut_ptr(); + let slice = unsafe { core::slice::from_raw_parts(ptr as *mut u8, ether_frame.len) }; + let ext = extract_headers(slice).or(Err(IgbDriverErr::InvalidPacket))?; if !(matches!(ext.network, NetworkHdr::Ipv4(_)) && matches!(ext.transport, TransportHdr::Tcp(_) | TransportHdr::Udp(_))) { @@ -1398,7 +1435,7 @@ impl Igb { mac_type: MacType, me: usize, tx: &mut Tx, - ether_frame: &EtherFrameRef, + ether_frame: &EtherFrameBuf, head: usize, ) -> Result<(usize, u32, u32, u16, Option), IgbDriverErr> { let mut olinfo_status = 0; @@ -1414,7 +1451,9 @@ impl Igb { off = true; } - let ext = extract_headers(ether_frame.data).or(Err(IgbDriverErr::InvalidPacket))?; + let ptr = ether_frame.data.get_virt_addr().as_mut_ptr(); + let slice = unsafe { core::slice::from_raw_parts(ptr as *mut u8, ether_frame.len) }; + let ext = extract_headers(slice).or(Err(IgbDriverErr::InvalidPacket))?; vlan_macip_lens |= (core::mem::size_of::() as u32) << ADVTXD_MACLEN_SHIFT; @@ -1443,7 +1482,7 @@ impl Igb { let mut mss_l4len_idx = 0; - let mut paylen = ether_frame.data.len() as u32; + let mut paylen = ether_frame.len as u32; match ext.transport { TransportHdr::Tcp(tcp) => { type_tucmd_mlhl |= ADVTXD_TUCMD_L4T_TCP; @@ -1457,7 +1496,7 @@ impl Igb { cmd_type_len |= ADVTXD_DCMD_TSE; off = true; - paylen = ether_frame.data.len() as u32 - header_len; + paylen = ether_frame.len as u32 - header_len; } } TransportHdr::Udp(_udp) => { @@ -1471,13 +1510,13 @@ impl Igb { mss_l4len_idx |= (mem::size_of::() as u32) << 8; off = true; - paylen = ether_frame.data.len() as u32 - header_len; + paylen = ether_frame.len as u32 - header_len; } } _ => (), } - olinfo_status |= (ether_frame.data.len() as u32) << ADVTXD_PAYLEN_SHIFT; + olinfo_status |= (ether_frame.len as u32) << ADVTXD_PAYLEN_SHIFT; if !off { return Ok((0, cmd_type_len, olinfo_status, 0, None)); @@ -1588,9 +1627,9 @@ impl Igb { mac_type: MacType, me: usize, tx: &mut Tx, - ether_frame: &EtherFrameRef, + ether_frame: &EtherFrameBuf, ) -> Result { - let len = ether_frame.data.len(); + let len = ether_frame.len; if len > TXBUFFER_2048 as usize { return Err(IgbDriverErr::InvalidPacket); } @@ -1615,20 +1654,27 @@ impl Igb { head -= tx_slots; } - let addr = unsafe { - let write_buf = tx.write_buf.as_mut().unwrap(); - let dst = &mut write_buf.as_mut()[head]; - core::ptr::copy_nonoverlapping(ether_frame.data.as_ptr(), dst.as_mut_ptr(), len); - if let Some(cksum_offset) = cksum_offset { - core::ptr::write( - dst.as_mut_ptr().add(cksum_offset as usize) as *mut u16, - cksum_pseudo.to_be(), - ); - } - (write_buf.get_phy_addr().as_usize() + head * TXBUFFER_2048 as usize) as u64 - }; + // TODO: Checksum Offloading + let addr = ether_frame.data.get_phy_addr().as_usize() as u64; let desc = &mut tx.tx_desc_ring.as_mut()[head]; + let (prev_used, virt_addr, phy_addr, numa_id) = tx.dma_info[head]; + if prev_used { + let ptr = virt_addr as *mut [u8; PAGESIZE]; + let data; + unsafe { + data = DMAPool::<[u8; PAGESIZE]>::from_raw_parts(ptr, phy_addr, PAGESIZE, numa_id) + .unwrap(); + } + drop(data); + } + + tx.dma_info[head] = ( + true, + ether_frame.data.get_virt_addr().as_usize(), + ether_frame.data.get_phy_addr().as_usize(), + ether_frame.data.get_numa_id(), + ); desc.legacy.buf = u64::to_le(addr); desc.legacy.lower.raw = u32::to_le(tx.txd_cmd | txd_lower | (len & 0xffff) as u32); @@ -1661,7 +1707,7 @@ impl Igb { Ok(used) } - fn send(&self, que_id: usize, ether_frames: &[EtherFrameRef]) -> Result<(), IgbDriverErr> { + fn send(&self, que_id: usize, ether_frames: &[EtherFrameBuf]) -> Result<(), IgbDriverErr> { let inner = self.inner.read(); if !inner.link_active { @@ -1877,7 +1923,11 @@ fn allocate_msi(info: &mut PCIeInfo) -> Result { } } -fn allocate_desc_rings(info: &PCIeInfo, que_id: usize) -> Result { +fn allocate_desc_rings( + info: &PCIeInfo, + que_num: usize, + que_id: usize, +) -> Result { let tx_size = core::mem::size_of::() * MAX_TXD; assert_eq!(tx_size & (PAGESIZE - 1), 0); @@ -1889,22 +1939,27 @@ fn allocate_desc_rings(info: &PCIeInfo, que_id: usize) -> Result Result<(), NetDevError> { + fn send(&self, data: EtherFrameBuf, que_id: usize) -> Result<(), NetDevError> { let frames = [data]; - self.send(que_id, &frames).or(Err(NetDevError::DeviceError)) + self.send(que_id, &frames) + .or(Err(NetDevError::DeviceError))?; + for data in frames { + data.data.leak(); + } + Ok(()) } fn up(&self) -> Result<(), NetDevError> { diff --git a/awkernel_drivers/src/pcie/intel/ixgbe.rs b/awkernel_drivers/src/pcie/intel/ixgbe.rs index 34502fa97..9ee6550a7 100644 --- a/awkernel_drivers/src/pcie/intel/ixgbe.rs +++ b/awkernel_drivers/src/pcie/intel/ixgbe.rs @@ -27,8 +27,8 @@ use awkernel_lib::{ ip::Ip, ipv6::Ip6Hdr, net_device::{ - EtherFrameBuf, EtherFrameRef, LinkStatus, NetCapabilities, NetDevError, NetDevice, - NetFlags, PacketHeaderFlags, + EtherFrameBuf, LinkStatus, NetCapabilities, NetDevError, NetDevice, NetFlags, + PacketHeaderFlags, }, tcp::TCPHdr, udp::UDPHdr, @@ -83,15 +83,15 @@ pub struct Tx { tx_desc_tail: usize, tx_desc_ring: DMAPool, txd_cmd: u32, - write_buf: Option>, + dma_info: Vec<(bool, usize, usize, usize)>, } pub struct Rx { rx_desc_head: u32, rx_desc_tail: usize, rx_desc_ring: DMAPool, - read_buf: Option>, read_queue: RingQ, + dma_info: Vec<(usize, usize, usize)>, } pub struct Queue { @@ -306,7 +306,7 @@ impl IxgbeInner { let mut que = Vec::new(); let que_num = get_num_queues(&hw.mac.mac_type); for i in 0..que_num { - que.push(allocate_queue(&info, i)?); + que.push(allocate_queue(&info, que_num, i)?); } // ixgbe_identify_hardware() @@ -389,6 +389,9 @@ impl IxgbeInner { ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; ixgbe_hw::write_reg(&info, IXGBE_CTRL_EXT, ctrl_ext)?; + let mut dma_info = Vec::with_capacity(DEFAULT_RXD * que_num); + dma_info.resize(DEFAULT_RXD * que_num, (0, 0, 0)); + let ixgbe = Self { info, hw, @@ -601,15 +604,7 @@ impl IxgbeInner { // ctrl_ext &= !IXGBE_CTRL_EXT_DRV_LOAD; // ixgbe_hw::write_reg(&self.info, IXGBE_CTRL_EXT, ctrl_ext); - for que in que.iter() { - let mut node = MCSNode::new(); - let mut tx = que.tx.lock(&mut node); - tx.write_buf = None; - - let mut node = MCSNode::new(); - let mut rx = que.rx.lock(&mut node); - rx.read_buf = None; - } + // TODO: Need to think about the deallocation of DMAPool self.update_link_status()?; @@ -661,23 +656,11 @@ impl IxgbeInner { tx.tx_desc_head = 0; let tx_desc_ring = tx.tx_desc_ring.as_mut(); - - let tx_buffer_size = MCLBYTES as usize * DEFAULT_TXD; - let write_buf = DMAPool::new( - self.info.get_segment_group() as usize, - tx_buffer_size / PAGESIZE, - ) - .ok_or(IxgbeDriverErr::DMAPool)?; - - let buf_phy_addr = write_buf.get_phy_addr().as_usize(); - - for (i, desc) in tx_desc_ring.iter_mut().enumerate() { - desc.adv_tx.buffer_addr = (buf_phy_addr + i * MCLBYTES as usize) as u64; + for desc in tx_desc_ring.iter_mut() { + desc.adv_tx.buffer_addr = 0; desc.adv_tx.cmd_type_len = 0; desc.adv_tx.olinfo_status = 0; } - - tx.write_buf = Some(write_buf); } Ok(()) @@ -751,30 +734,36 @@ impl IxgbeInner { } fn setup_receive_structures(&mut self, que: &[Queue]) -> Result<(), IxgbeDriverErr> { - for que in que.iter() { + for (i, que) in que.iter().enumerate() { let mut node = MCSNode::new(); let mut rx = que.rx.lock(&mut node); rx.rx_desc_tail = 0; - rx.rx_desc_head = rx.rx_desc_ring.as_ref().len() as u32 - 1; + let rx_desc_ring_len = rx.rx_desc_ring.as_ref().len(); + rx.rx_desc_head = rx_desc_ring_len as u32 - 1; let rx_desc_ring = rx.rx_desc_ring.as_mut(); + let mut dma_info_temp = Vec::new(); - let rx_buffer_size = MCLBYTES as usize * DEFAULT_RXD; - let read_buf = DMAPool::new( - self.info.get_segment_group() as usize, - rx_buffer_size / PAGESIZE, - ) - .ok_or(IxgbeDriverErr::DMAPool)?; - - let buf_phy_addr = read_buf.get_phy_addr().as_usize(); + for desc in rx_desc_ring { + let read_buf: DMAPool<[u8; PAGESIZE]> = + DMAPool::new(self.info.get_segment_group() as usize, 1) + .ok_or(IxgbeDriverErr::DMAPool)?; + let buf_phy_addr = read_buf.get_phy_addr().as_usize(); - for (i, desc) in rx_desc_ring.iter_mut().enumerate() { desc.data = [0; 2]; - desc.read.pkt_addr = (buf_phy_addr + i * MCLBYTES as usize) as u64; + desc.read.pkt_addr = buf_phy_addr as u64; + dma_info_temp.push(( + read_buf.get_virt_addr().as_usize(), + buf_phy_addr, + self.info.get_segment_group() as usize, + )); + read_buf.leak(); } - rx.read_buf = Some(read_buf); + for (j, info) in dma_info_temp.iter().enumerate() { + rx.dma_info[rx_desc_ring_len * i + j] = info.clone(); + } } Ok(()) @@ -1552,19 +1541,32 @@ impl Ixgbe { Ok(()) } + // TODO: Not sure if we really need this + unsafe fn invalidate_cache(&self, addr: *const u8) -> Result<(), IxgbeDriverErr> { + #[cfg(target_arch = "x86_64")] + { + core::arch::x86_64::_mm_clflush(addr); + Ok(()) + } + + #[cfg(not(any(target_arch = "x86_64")))] + { + let _ = addr; + Err(IxgbeDriverErr::NotSupported) + } + } + fn rx_recv(&self, que_id: usize) -> Result<(), IxgbeDriverErr> { let que = &self.que[que_id]; { let inner = self.inner.read(); + let numa_id = inner.info.get_segment_group() as usize; + drop(inner); let mut node = MCSNode::new(); let mut rx = que.rx.lock(&mut node); - if rx.read_buf.is_none() { - return Ok(()); - } - let mut i = rx.rx_desc_tail; loop { @@ -1576,9 +1578,9 @@ impl Ixgbe { break; } - let rx_desc_ring = rx.rx_desc_ring.as_ref(); + let rx_desc_ring = rx.rx_desc_ring.as_mut(); let rx_desc_ring_len = rx_desc_ring.len(); - let desc = &rx_desc_ring[i]; + let desc = &mut rx.rx_desc_ring.as_mut()[i]; let staterr; unsafe { @@ -1588,7 +1590,7 @@ impl Ixgbe { break; } - let len; + let len: u16; let vtag; let eop; unsafe { @@ -1613,13 +1615,44 @@ impl Ixgbe { if !eop { drop(rx); - drop(inner); return self.recv_jumbo(que_id); } else { - let read_buf = rx.read_buf.as_ref().unwrap(); - let src = &read_buf.as_ref()[i]; - let data = src[0..len as usize].to_vec(); - rx.read_queue.push(EtherFrameBuf { data, vlan }).unwrap(); + let read_buf: DMAPool<[u8; PAGESIZE]> = + DMAPool::new(numa_id, 1).ok_or(IxgbeDriverErr::DMAPool)?; + let buf_phy_addr = read_buf.get_phy_addr().as_usize(); + desc.read.pkt_addr = buf_phy_addr as u64; + + let index = (rx_desc_ring_len * que_id + i) as usize; + let (virt_addr, phy_addr, numa_id) = rx.dma_info[index]; + + let ptr = virt_addr as *mut [u8; PAGESIZE]; + let data; + unsafe { + data = DMAPool::<[u8; PAGESIZE]>::from_raw_parts( + ptr, phy_addr, + PAGESIZE, // RECONSIDER: Not using "len" here, might be better to give the "len" information somehow to protocol stack. + numa_id, + ) + .unwrap(); + } + + rx.dma_info[index] = + (read_buf.get_virt_addr().as_usize(), buf_phy_addr, numa_id); + + // Need clflush + unsafe { + self.invalidate_cache(read_buf.get_virt_addr().as_usize() as *const u8)?; + } + + read_buf.leak(); + rx.read_queue + .push(EtherFrameBuf { + data, + len: len as usize, + vlan, + csum_flags: PacketHeaderFlags::EMPTY, + }) + .unwrap(); } i += 1; @@ -1685,17 +1718,19 @@ impl Ixgbe { #[allow(clippy::type_complexity)] fn tx_offload( &self, - ether_frame: &EtherFrameRef, + ether_frame: &EtherFrameBuf, ) -> Result<(bool, u32, u32, u32, u16, Option), IxgbeDriverErr> { let mut vlan_macip_lens = 0; let mut olinfo_status = 0; let mut offload = false; - let ext = extract_headers(ether_frame.data).or(Err(IxgbeDriverErr::InvalidPacket))?; + let ptr = ether_frame.data.get_virt_addr().as_mut_ptr(); + let slice = unsafe { core::slice::from_raw_parts(ptr as *mut u8, ether_frame.len) }; + let ext = extract_headers(slice).or(Err(IxgbeDriverErr::InvalidPacket))?; // Depend on whether doing TSO or not // Indicate the whole packet as payload when not doing TSO - olinfo_status |= (ether_frame.data.len() as u32) << IXGBE_ADVTXD_PAYLEN_SHIFT; + olinfo_status |= (ether_frame.len as u32) << IXGBE_ADVTXD_PAYLEN_SHIFT; vlan_macip_lens |= (core::mem::size_of::() as u32) << IXGBE_ADVTXD_MACLEN_SHIFT; @@ -1774,7 +1809,7 @@ impl Ixgbe { fn tx_ctx_setup( &self, tx: &mut Tx, - ether_frame: &EtherFrameRef, + ether_frame: &EtherFrameBuf, head: usize, ) -> Result<(u32, u32, u32, u16, Option), IxgbeDriverErr> { let mut cmd_type_len = 0; @@ -1802,6 +1837,18 @@ impl Ixgbe { type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; let desc = &mut tx.tx_desc_ring.as_mut()[head]; + let (used, virt_addr, phy_addr, numa_id) = tx.dma_info[head]; + if used { + let ptr = virt_addr as *mut [u8; PAGESIZE]; + let data; + unsafe { + data = DMAPool::<[u8; PAGESIZE]>::from_raw_parts(ptr, phy_addr, PAGESIZE, numa_id) + .unwrap(); + } + drop(data); + } + + tx.dma_info[head] = (false, 0, 0, 0); // Now copy bits into descriptor desc.adv_ctx.vlan_macip_lens = u32::to_le(vlan_macip_lens); desc.adv_ctx.type_tucmd_mlhl = u32::to_le(type_tucmd_mlhl); @@ -1813,8 +1860,8 @@ impl Ixgbe { /// This routine maps the mbufs to tx descriptors, allowing the /// TX engine to transmit the packets. - fn encap(&self, tx: &mut Tx, ether_frame: &EtherFrameRef) -> Result { - let len = ether_frame.data.len(); + fn encap(&self, tx: &mut Tx, ether_frame: &EtherFrameBuf) -> Result { + let len = ether_frame.len; if len > MCLBYTES as usize { return Err(IxgbeDriverErr::InvalidPacket); } @@ -1833,22 +1880,30 @@ impl Ixgbe { if head == tx_slots { head = 0; } - let addr = unsafe { - let write_buf = tx.write_buf.as_mut().unwrap(); - let dst = &mut write_buf.as_mut()[head]; - core::ptr::copy_nonoverlapping(ether_frame.data.as_ptr(), dst.as_mut_ptr(), len); - // TODO: cksum offloading - //if let Some(cksum_offset) = cksum_offset { - //log::info!("cksum: {}", cksum_pseudo); - //core::ptr::write( - //dst.as_mut_ptr().add(cksum_offset as usize) as *mut u16, - //cksum_pseudo.to_be(), - //); - //} - (write_buf.get_phy_addr().as_usize() + head * MCLBYTES as usize) as u64 - }; + + // TODO: Checksum Offloading + + let addr = ether_frame.data.get_phy_addr().as_usize() as u64; let desc = &mut tx.tx_desc_ring.as_mut()[head]; + let (used, virt_addr, phy_addr, numa_id) = tx.dma_info[head]; + if used { + let ptr = virt_addr as *mut [u8; PAGESIZE]; + let data; + unsafe { + data = DMAPool::<[u8; PAGESIZE]>::from_raw_parts(ptr, phy_addr, PAGESIZE, numa_id) + .unwrap(); + } + drop(data); + } + + tx.dma_info[head] = ( + true, + ether_frame.data.get_virt_addr().as_usize(), + ether_frame.data.get_phy_addr().as_usize(), + ether_frame.data.get_numa_id(), + ); + desc.adv_tx.buffer_addr = u64::to_le(addr); desc.adv_tx.cmd_type_len = u32::to_le( tx.txd_cmd | cmd_type_len | IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS | len as u32, @@ -1865,13 +1920,16 @@ impl Ixgbe { Ok(ntxc as usize + 1) } - fn send(&self, que_id: usize, ether_frames: &[EtherFrameRef]) -> Result<(), IxgbeDriverErr> { + fn send(&self, que_id: usize, ether_frames: &[EtherFrameBuf]) -> Result<(), IxgbeDriverErr> { let inner = self.inner.read(); if !inner.link_active { return Ok(()); } + let num_segs = inner.num_segs; + drop(inner); + let mut node = MCSNode::new(); let mut tx = self.que[que_id].tx.lock(&mut node); @@ -1888,7 +1946,7 @@ impl Ixgbe { for ether_frame in ether_frames.iter() { // Check that we have the minimal number of TX descriptors. // use 2 because cksum setup can use an extra slot - if free <= inner.num_segs as usize + 2 { + if free <= num_segs as usize + 2 { break; } @@ -1899,6 +1957,7 @@ impl Ixgbe { post = true; } + let inner = self.inner.read(); if post { ixgbe_hw::write_reg(&inner.info, IXGBE_TDT(que_id), tx.tx_desc_head as u32)?; } @@ -2003,7 +2062,7 @@ fn allocate_msi(info: &mut PCIeInfo) -> Result { /// Allocate memory for the transmit and receive rings, and then /// the descriptors associated with each, called only once at attach. -fn allocate_queue(info: &PCIeInfo, que_id: usize) -> Result { +fn allocate_queue(info: &PCIeInfo, que_num: usize, que_id: usize) -> Result { let tx_size = core::mem::size_of::() * DEFAULT_TXD; assert_eq!(tx_size & (PAGESIZE - 1), 0); @@ -2015,20 +2074,25 @@ fn allocate_queue(info: &PCIeInfo, que_id: usize) -> Result Result<(), NetDevError> { + fn send(&self, data: EtherFrameBuf, que_id: usize) -> Result<(), NetDevError> { let frames = [data]; - self.send(que_id, &frames).or(Err(NetDevError::DeviceError)) + self.send(que_id, &frames) + .or(Err(NetDevError::DeviceError))?; + for data in frames { + data.data.leak(); + } + Ok(()) } fn up(&self) -> Result<(), NetDevError> { diff --git a/awkernel_lib/src/dma_pool.rs b/awkernel_lib/src/dma_pool.rs index aae52405b..12e1583f0 100644 --- a/awkernel_lib/src/dma_pool.rs +++ b/awkernel_lib/src/dma_pool.rs @@ -89,6 +89,28 @@ impl DMAPool { ptr } + pub unsafe fn from_raw_parts( + ptr: *mut T, + phy_addr: usize, + size: usize, + numa_id: usize, + ) -> Option { + assert!(numa_id < NUMA_NUM_MAX); + + let virt_addr = VirtAddr::new(ptr as usize); + let phy_addr = PhyAddr::new(phy_addr); + let ptr = NonNull::new(ptr)?; + //let ptr = NonNull::new_unchecked(core::ptr::slice_from_raw_parts_mut(ptr, size)); + + Some(Self { + virt_addr, + phy_addr, + size, + numa_id, + ptr, + }) + } + #[inline(always)] pub fn get_virt_addr(&self) -> VirtAddr { self.virt_addr diff --git a/awkernel_lib/src/net.rs b/awkernel_lib/src/net.rs index be4e05114..70da46d79 100644 --- a/awkernel_lib/src/net.rs +++ b/awkernel_lib/src/net.rs @@ -22,7 +22,7 @@ use crate::sync::rwlock::RwLock; pub mod ether; pub mod ethertypes; -mod if_net; +pub mod if_net; pub mod in_cksum; pub mod ip; pub mod ip_addr; diff --git a/awkernel_lib/src/net/if_net.rs b/awkernel_lib/src/net/if_net.rs index b80abfaf9..b4cb507d2 100644 --- a/awkernel_lib/src/net/if_net.rs +++ b/awkernel_lib/src/net/if_net.rs @@ -27,12 +27,17 @@ use smoltcp::{ wire::HardwareAddress, }; -use crate::sync::{mcs::MCSNode, mutex::Mutex}; +use crate::{ + addr::Addr, + dma_pool::DMAPool, + paging::PAGESIZE, + sync::{mcs::MCSNode, mutex::Mutex}, +}; use super::{ ether::{extract_headers, NetworkHdr, TransportHdr, ETHER_ADDR_LEN}, multicast::ipv4_addr_to_mac_addr, - net_device::{EtherFrameBuf, EtherFrameRef, NetCapabilities, NetDevice, PacketHeaderFlags}, + net_device::{EtherFrameBuf, NetCapabilities, NetDevice, PacketHeaderFlags}, NetManagerError, }; @@ -50,7 +55,7 @@ struct NetDriverRef<'a> { inner: &'a Arc, rx_ringq: Option<&'a mut RingQ>, - tx_ringq: &'a mut RingQ>, + tx_ringq: &'a mut RingQ<(DMAPool<[u8; PAGESIZE]>, usize)>, } impl NetDriverRef<'_> { @@ -136,6 +141,7 @@ impl Device for NetDriverRef<'_> { NRxToken { data }, NTxToken { tx_ring: self.tx_ringq, + driver_ref_inner: self.inner, }, )); } @@ -156,6 +162,7 @@ impl Device for NetDriverRef<'_> { Some(NTxToken { tx_ring: self.tx_ringq, + driver_ref_inner: self.inner, }) } } @@ -164,7 +171,7 @@ pub(super) struct IfNet { vlan: Option, pub(super) inner: Mutex, rx_irq_to_driver: BTreeMap, - tx_only_ringq: Vec>>>, + tx_only_ringq: Vec, usize)>>>, pub(super) net_device: Arc, pub(super) is_poll_mode: bool, poll_driver: Option, @@ -494,11 +501,14 @@ impl IfNet { // send packets from the queue. while !device_ref.tx_ringq.is_empty() { - if let Some(data) = device_ref.tx_ringq.pop() { - let tx_packet_header_flags = device_ref.tx_packet_header_flags(&data); - - let data = EtherFrameRef { - data: &data, + if let Some((data, len)) = device_ref.tx_ringq.pop() { + let ptr = data.get_virt_addr().as_mut_ptr(); + let slice = unsafe { core::slice::from_raw_parts_mut(ptr as *mut u8, len) }; + let tx_packet_header_flags = device_ref.tx_packet_header_flags(slice); + + let data = EtherFrameBuf { + data, + len, vlan: self.vlan, csum_flags: tx_packet_header_flags, }; @@ -516,10 +526,6 @@ impl IfNet { fn poll_rx(&self, ref_net_driver: &NetDriver) -> bool { let que_id = ref_net_driver.rx_que_id; - let Some(tx_ringq) = self.tx_only_ringq.get(que_id) else { - return false; - }; - let mut node = MCSNode::new(); let mut rx_ringq = ref_net_driver.rx_ringq.lock(&mut node); @@ -532,6 +538,9 @@ impl IfNet { } } + let Some(tx_ringq) = self.tx_only_ringq.get(que_id) else { + return false; + }; let mut node = MCSNode::new(); let mut tx_ringq = tx_ringq.lock(&mut node); @@ -554,15 +563,17 @@ impl IfNet { // send packets from the queue. while !device_ref.tx_ringq.is_empty() { - if let Some(data) = device_ref.tx_ringq.pop() { - let tx_packet_header_flags = device_ref.tx_packet_header_flags(&data); - - let data = EtherFrameRef { - data: &data, + if let Some((data, len)) = device_ref.tx_ringq.pop() { + let ptr = data.get_virt_addr().as_mut_ptr(); + let slice = unsafe { core::slice::from_raw_parts_mut(ptr as *mut u8, len) }; + let tx_packet_header_flags = device_ref.tx_packet_header_flags(slice); + + let data = EtherFrameBuf { + data, + len, vlan: self.vlan, csum_flags: tx_packet_header_flags, }; - let _ = self.net_device.send(data, ref_net_driver.rx_que_id); } else { break; @@ -624,12 +635,13 @@ impl phy::RxToken for NRxToken { where F: FnOnce(&mut [u8]) -> R, { - f(&mut self.data.data) + f(self.data.data.as_mut()) } } pub struct NTxToken<'a> { - tx_ring: &'a mut RingQ>, + tx_ring: &'a mut RingQ<(DMAPool<[u8; PAGESIZE]>, usize)>, + driver_ref_inner: &'a Arc, } impl phy::TxToken for NTxToken<'_> { @@ -637,16 +649,15 @@ impl phy::TxToken for NTxToken<'_> { where F: FnOnce(&mut [u8]) -> R, { - let mut buf = Vec::with_capacity(len); + let segment_group = self.driver_ref_inner.get_segment_group().unwrap_or(0); + let buf: DMAPool<[u8; PAGESIZE]> = DMAPool::new(segment_group as usize, 1).unwrap(); // RECONSIDER: Not sure this unwrap is acceptable - #[allow(clippy::uninit_vec)] - unsafe { - buf.set_len(len); - }; + let ptr = buf.get_virt_addr().as_mut_ptr(); + let slice = unsafe { core::slice::from_raw_parts_mut(ptr as *mut u8, len) }; - let result = f(&mut buf[..len]); + let result: R = f(slice); - let _ = self.tx_ring.push(buf); + let _ = self.tx_ring.push((buf, len)); result } diff --git a/awkernel_lib/src/net/net_device.rs b/awkernel_lib/src/net/net_device.rs index b6eee8011..0bd77021b 100644 --- a/awkernel_lib/src/net/net_device.rs +++ b/awkernel_lib/src/net/net_device.rs @@ -1,3 +1,4 @@ +use crate::{dma_pool::DMAPool, paging::PAGESIZE}; use alloc::borrow::Cow; use bitflags::bitflags; use core::fmt::Display; @@ -8,6 +9,7 @@ use alloc::{string::String, vec::Vec}; bitflags! { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct PacketHeaderFlags: u16 { + const EMPTY = 0x0000; // Empty const IPV4_CSUM_OUT = 0x0001; // IPv4 checksum needed const TCP_CSUM_OUT = 0x0002; // TCP checksum needed const UDP_CSUM_OUT = 0x0004; // UDP checksum needed @@ -132,17 +134,12 @@ pub enum NetDevError { MulticastAddrError, } -#[derive(Debug, Clone)] -pub struct EtherFrameRef<'a> { - pub data: &'a [u8], - pub vlan: Option, - pub csum_flags: PacketHeaderFlags, -} - -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct EtherFrameBuf { - pub data: Vec, + pub data: DMAPool<[u8; PAGESIZE]>, + pub len: usize, pub vlan: Option, + pub csum_flags: PacketHeaderFlags, } /// Because the network will have multiple queues @@ -150,7 +147,7 @@ pub struct EtherFrameBuf { /// the network device must be thread-safe. pub trait NetDevice { fn recv(&self, que_id: usize) -> Result, NetDevError>; - fn send(&self, data: EtherFrameRef, que_id: usize) -> Result<(), NetDevError>; + fn send(&self, data: EtherFrameBuf, que_id: usize) -> Result<(), NetDevError>; fn flags(&self) -> NetFlags; fn capabilities(&self) -> NetCapabilities; @@ -214,6 +211,10 @@ pub trait NetDevice { fn add_multicast_addr(&self, addr: &[u8; 6]) -> Result<(), NetDevError>; fn remove_multicast_addr(&self, addr: &[u8; 6]) -> Result<(), NetDevError>; + + fn get_segment_group(&self) -> Option { + None + } } impl Display for LinkStatus { diff --git a/kernel/src/arch/x86_64/config.rs b/kernel/src/arch/x86_64/config.rs index d14a54f56..7e0f39987 100644 --- a/kernel/src/arch/x86_64/config.rs +++ b/kernel/src/arch/x86_64/config.rs @@ -7,4 +7,4 @@ pub const HEAP_START: usize = 0x41000000000; pub const PREEMPT_IRQ: u16 = 255; -pub const DMA_SIZE: usize = 64 * 1024 * 1024; // 64MiB +pub const DMA_SIZE: usize = 512 * 1024 * 1024; // 512MiB diff --git a/kernel/src/arch/x86_64/kernel_main.rs b/kernel/src/arch/x86_64/kernel_main.rs index 5ad004d15..75e847657 100644 --- a/kernel/src/arch/x86_64/kernel_main.rs +++ b/kernel/src/arch/x86_64/kernel_main.rs @@ -814,7 +814,7 @@ fn init_dma( let dma_start = DMA_START + numa_id as usize * DMA_SIZE; let flags = PageTableFlags::PRESENT | PageTableFlags::WRITABLE - // | PageTableFlags::NO_CACHE + | PageTableFlags::NO_CACHE | PageTableFlags::NO_EXECUTE | PageTableFlags::GLOBAL; diff --git a/scripts/udp.py b/scripts/udp.py index 28482af36..7b2c7eb6b 100644 --- a/scripts/udp.py +++ b/scripts/udp.py @@ -3,8 +3,8 @@ sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) -addr = ('localhost', 26099) -#addr = ('10.0.2.2', 26099) +#addr = ('localhost', 26099) +addr = ('10.0.2.2', 26099) print('listening on %s port %s' % addr, file=sys.stderr) sock.bind(addr) diff --git a/userland/Cargo.toml b/userland/Cargo.toml index f36e9fd55..7d1c6cb30 100644 --- a/userland/Cargo.toml +++ b/userland/Cargo.toml @@ -67,7 +67,7 @@ path = "../applications/tests/test_sched_preempt" optional = true [features] -default = ["test_sched_preempt"] +default = ["test_network"] perf = ["awkernel_services/perf"] # Test applications From 1c7c6aa813f97ccfe5fb78430b5277582186f1f7 Mon Sep 17 00:00:00 2001 From: koichiimai Date: Thu, 2 Jan 2025 20:56:39 +0900 Subject: [PATCH 02/14] Checksum offload for igb Signed-off-by: koichiimai --- Makefile | 2 +- applications/tests/test_network/src/lib.rs | 54 +++++++++++----------- awkernel_drivers/src/pcie/intel/igb.rs | 29 ++++++++---- awkernel_drivers/src/pcie/intel/ixgbe.rs | 25 +++++----- awkernel_lib/src/net.rs | 2 +- awkernel_lib/src/net/if_net.rs | 2 +- userland/Cargo.toml | 2 +- 7 files changed, 61 insertions(+), 55 deletions(-) diff --git a/Makefile b/Makefile index aa95fab58..14be06395 100644 --- a/Makefile +++ b/Makefile @@ -56,7 +56,7 @@ X86_64_LD=$(LINKERDIR)/x86_64-link.lds RV32_LD=$(LINKERDIR)/rv32-link.lds RV64_LD=$(LINKERDIR)/rv64-link.lds -RUSTV=nightly-2024-12-30 +RUSTV=nightly-2024-12-02 all: aarch64 x86_64 riscv32 riscv64 std diff --git a/applications/tests/test_network/src/lib.rs b/applications/tests/test_network/src/lib.rs index 9df6a3a15..18949840a 100644 --- a/applications/tests/test_network/src/lib.rs +++ b/applications/tests/test_network/src/lib.rs @@ -31,33 +31,33 @@ pub async fn run() { ) .await; - //awkernel_async_lib::spawn( - //"test tcp listen".into(), - //tcp_listen_test(), - //awkernel_async_lib::scheduler::SchedulerType::FIFO, - //) - //.await; - - //awkernel_async_lib::spawn( - //"test tcp connect".into(), - //tcp_connect_test(), - //awkernel_async_lib::scheduler::SchedulerType::FIFO, - //) - //.await; - - //awkernel_async_lib::spawn( - //"test IPv4 multicast recv".into(), - //ipv4_multicast_recv_test(), - //awkernel_async_lib::scheduler::SchedulerType::FIFO, - //) - //.await; - - //awkernel_async_lib::spawn( - //"test IPv4 multicast send".into(), - //ipv4_multicast_send_test(), - //awkernel_async_lib::scheduler::SchedulerType::FIFO, - //) - //.await; + awkernel_async_lib::spawn( + "test tcp listen".into(), + tcp_listen_test(), + awkernel_async_lib::scheduler::SchedulerType::FIFO, + ) + .await; + + awkernel_async_lib::spawn( + "test tcp connect".into(), + tcp_connect_test(), + awkernel_async_lib::scheduler::SchedulerType::FIFO, + ) + .await; + + awkernel_async_lib::spawn( + "test IPv4 multicast recv".into(), + ipv4_multicast_recv_test(), + awkernel_async_lib::scheduler::SchedulerType::FIFO, + ) + .await; + + awkernel_async_lib::spawn( + "test IPv4 multicast send".into(), + ipv4_multicast_send_test(), + awkernel_async_lib::scheduler::SchedulerType::FIFO, + ) + .await; } async fn ipv4_multicast_send_test() { diff --git a/awkernel_drivers/src/pcie/intel/igb.rs b/awkernel_drivers/src/pcie/intel/igb.rs index b0691b84f..ce2b98c03 100644 --- a/awkernel_drivers/src/pcie/intel/igb.rs +++ b/awkernel_drivers/src/pcie/intel/igb.rs @@ -118,10 +118,8 @@ const MAX_INTS_PER_SEC: u32 = 8000; const DEFAULT_ITR: u32 = 1000000000 / (MAX_INTS_PER_SEC * 256); type RxRing = [RxDescriptor; MAX_RXD]; -type RxBuffer = [[u8; RXBUFFER_2048 as usize]; MAX_RXD]; type TxRing = [TxDescriptor; MAX_TXD]; -type TxBuffer = [[u8; TXBUFFER_2048 as usize]; MAX_TXD]; struct Rx { rx_desc_head: u32, @@ -949,7 +947,7 @@ impl IgbInner { } for (j, info) in dma_info_temp.iter().enumerate() { - rx.dma_info[rx_desc_ring_len * i + j] = info.clone(); + rx.dma_info[rx_desc_ring_len * i + j] = *info; } } @@ -1266,7 +1264,7 @@ impl Igb { let buf_phy_addr = read_buf.get_phy_addr().as_usize(); desc.buf = buf_phy_addr as u64; - let index = (rx_desc_ring_len * que_id + i) as usize; + let index = rx_desc_ring_len * que_id + i; let (virt_addr, phy_addr, numa_id) = rx.dma_info[index]; let ptr = virt_addr as *mut [u8; PAGESIZE]; @@ -1292,7 +1290,7 @@ impl Igb { rx.read_queue .push(EtherFrameBuf { data, - len: len as usize, + len, vlan, csum_flags: PacketHeaderFlags::EMPTY, }) @@ -1354,7 +1352,7 @@ impl Igb { let txd_lower; let ptr = ether_frame.data.get_virt_addr().as_mut_ptr(); - let slice = unsafe { core::slice::from_raw_parts(ptr as *mut u8, ether_frame.len) }; + let slice = unsafe { core::slice::from_raw_parts(ptr, ether_frame.len) }; let ext = extract_headers(slice).or(Err(IgbDriverErr::InvalidPacket))?; if !(matches!(ext.network, NetworkHdr::Ipv4(_)) && matches!(ext.transport, TransportHdr::Tcp(_) | TransportHdr::Udp(_))) @@ -1452,7 +1450,7 @@ impl Igb { } let ptr = ether_frame.data.get_virt_addr().as_mut_ptr(); - let slice = unsafe { core::slice::from_raw_parts(ptr as *mut u8, ether_frame.len) }; + let slice = unsafe { core::slice::from_raw_parts(ptr, ether_frame.len) }; let ext = extract_headers(slice).or(Err(IgbDriverErr::InvalidPacket))?; vlan_macip_lens |= (core::mem::size_of::() as u32) << ADVTXD_MACLEN_SHIFT; @@ -1654,8 +1652,19 @@ impl Igb { head -= tx_slots; } - // TODO: Checksum Offloading let addr = ether_frame.data.get_phy_addr().as_usize() as u64; + if let Some(cksum_offset) = cksum_offset { + unsafe { + core::ptr::write( + ether_frame + .data + .get_virt_addr() + .as_mut_ptr::() + .add(cksum_offset as usize) as *mut u16, + cksum_pseudo.to_be(), + ); + } + } let desc = &mut tx.tx_desc_ring.as_mut()[head]; let (prev_used, virt_addr, phy_addr, numa_id) = tx.dma_info[head]; @@ -1951,8 +1960,8 @@ fn allocate_desc_rings( dma_info: dma_info_tx, }; - let mut dma_info_rx = Vec::with_capacity(MAX_RXD * que_num as usize); - dma_info_rx.resize(MAX_RXD * que_num as usize, (0, 0, 0)); + let mut dma_info_rx = Vec::with_capacity(MAX_RXD * que_num); + dma_info_rx.resize(MAX_RXD * que_num, (0, 0, 0)); let rx = Rx { rx_desc_head: 0, rx_desc_tail: 0, diff --git a/awkernel_drivers/src/pcie/intel/ixgbe.rs b/awkernel_drivers/src/pcie/intel/ixgbe.rs index 9ee6550a7..1528af378 100644 --- a/awkernel_drivers/src/pcie/intel/ixgbe.rs +++ b/awkernel_drivers/src/pcie/intel/ixgbe.rs @@ -73,10 +73,8 @@ const MCLBYTES: u32 = 1 << MCLSHIFT; const MAXMCLBYTES: u32 = 64 * 1024; type TxRing = [TxDescriptor; DEFAULT_TXD]; -type TxBuffer = [[u8; MCLBYTES as usize]; DEFAULT_TXD]; type RxRing = [AdvRxDesc; DEFAULT_RXD]; -type RxBuffer = [[u8; MCLBYTES as usize]; DEFAULT_RXD]; pub struct Tx { tx_desc_head: usize, @@ -415,7 +413,7 @@ impl IxgbeInner { fn init(&mut self, que: &[Queue]) -> Result<(), IxgbeDriverErr> { use ixgbe_hw::MacType::*; - self.stop(que)?; + self.stop()?; // reprogram the RAR[0] in case user changed it. self.ops.mac_set_rar( @@ -430,7 +428,7 @@ impl IxgbeInner { if let Err(e) = self.setup_transmit_structures(que) { log::error!("Could not setup transmit structures"); - self.stop(que)?; + self.stop()?; return Err(e); } @@ -440,7 +438,7 @@ impl IxgbeInner { // Prepare receive descriptors and buffers if let Err(e) = self.setup_receive_structures(que) { log::error!("Could not setup receieve structures"); - self.stop(que)?; + self.stop()?; return Err(e); } @@ -571,7 +569,7 @@ impl IxgbeInner { Ok(()) } - fn stop(&mut self, que: &[Queue]) -> Result<(), IxgbeDriverErr> { + fn stop(&mut self) -> Result<(), IxgbeDriverErr> { self.flags.remove(NetFlags::RUNNING); self.disable_intr()?; @@ -750,7 +748,6 @@ impl IxgbeInner { DMAPool::new(self.info.get_segment_group() as usize, 1) .ok_or(IxgbeDriverErr::DMAPool)?; let buf_phy_addr = read_buf.get_phy_addr().as_usize(); - desc.data = [0; 2]; desc.read.pkt_addr = buf_phy_addr as u64; dma_info_temp.push(( @@ -762,7 +759,7 @@ impl IxgbeInner { } for (j, info) in dma_info_temp.iter().enumerate() { - rx.dma_info[rx_desc_ring_len * i + j] = info.clone(); + rx.dma_info[rx_desc_ring_len * i + j] = *info; } } @@ -1622,7 +1619,7 @@ impl Ixgbe { let buf_phy_addr = read_buf.get_phy_addr().as_usize(); desc.read.pkt_addr = buf_phy_addr as u64; - let index = (rx_desc_ring_len * que_id + i) as usize; + let index = rx_desc_ring_len * que_id + i; let (virt_addr, phy_addr, numa_id) = rx.dma_info[index]; let ptr = virt_addr as *mut [u8; PAGESIZE]; @@ -1725,7 +1722,7 @@ impl Ixgbe { let mut offload = false; let ptr = ether_frame.data.get_virt_addr().as_mut_ptr(); - let slice = unsafe { core::slice::from_raw_parts(ptr as *mut u8, ether_frame.len) }; + let slice = unsafe { core::slice::from_raw_parts(ptr, ether_frame.len) }; let ext = extract_headers(slice).or(Err(IxgbeDriverErr::InvalidPacket))?; // Depend on whether doing TSO or not @@ -2085,8 +2082,8 @@ fn allocate_queue(info: &PCIeInfo, que_num: usize, que_id: usize) -> Result { let ptr = buf.get_virt_addr().as_mut_ptr(); let slice = unsafe { core::slice::from_raw_parts_mut(ptr as *mut u8, len) }; - let result: R = f(slice); + let result = f(slice); let _ = self.tx_ring.push((buf, len)); diff --git a/userland/Cargo.toml b/userland/Cargo.toml index 7d1c6cb30..f36e9fd55 100644 --- a/userland/Cargo.toml +++ b/userland/Cargo.toml @@ -67,7 +67,7 @@ path = "../applications/tests/test_sched_preempt" optional = true [features] -default = ["test_network"] +default = ["test_sched_preempt"] perf = ["awkernel_services/perf"] # Test applications From 7694ae85b0569757f944174e307fc9a4c7b88d28 Mon Sep 17 00:00:00 2001 From: koichiimai Date: Thu, 2 Jan 2025 21:20:46 +0900 Subject: [PATCH 03/14] Applied clippy Signed-off-by: koichiimai --- awkernel_drivers/src/pcie/intel/igb.rs | 22 +++++++----------- awkernel_drivers/src/pcie/intel/ixgbe.rs | 29 ++++++++---------------- awkernel_lib/src/dma_pool.rs | 3 +-- awkernel_lib/src/net/if_net.rs | 16 +++++++------ 4 files changed, 28 insertions(+), 42 deletions(-) diff --git a/awkernel_drivers/src/pcie/intel/igb.rs b/awkernel_drivers/src/pcie/intel/igb.rs index ce2b98c03..1c7351b5c 100644 --- a/awkernel_drivers/src/pcie/intel/igb.rs +++ b/awkernel_drivers/src/pcie/intel/igb.rs @@ -1268,15 +1268,12 @@ impl Igb { let (virt_addr, phy_addr, numa_id) = rx.dma_info[index]; let ptr = virt_addr as *mut [u8; PAGESIZE]; - let data; - unsafe { - data = DMAPool::<[u8; PAGESIZE]>::from_raw_parts( - ptr, phy_addr, - PAGESIZE, // RECONSIDER: Not using "len" here, might be better to give the "len" information somehow to protocol stack. - numa_id, - ) - .unwrap(); - } + let data = DMAPool::<[u8; PAGESIZE]>::from_raw_parts( + ptr, phy_addr, + PAGESIZE, // RECONSIDER: Not using "len" here, might be better to give the "len" information somehow to protocol stack. + numa_id, + ) + .unwrap(); rx.dma_info[index] = (read_buf.get_virt_addr().as_usize(), buf_phy_addr, numa_id); @@ -1670,11 +1667,8 @@ impl Igb { let (prev_used, virt_addr, phy_addr, numa_id) = tx.dma_info[head]; if prev_used { let ptr = virt_addr as *mut [u8; PAGESIZE]; - let data; - unsafe { - data = DMAPool::<[u8; PAGESIZE]>::from_raw_parts(ptr, phy_addr, PAGESIZE, numa_id) - .unwrap(); - } + let data = DMAPool::<[u8; PAGESIZE]>::from_raw_parts(ptr, phy_addr, PAGESIZE, numa_id) + .unwrap(); drop(data); } diff --git a/awkernel_drivers/src/pcie/intel/ixgbe.rs b/awkernel_drivers/src/pcie/intel/ixgbe.rs index 1528af378..b5227b4cf 100644 --- a/awkernel_drivers/src/pcie/intel/ixgbe.rs +++ b/awkernel_drivers/src/pcie/intel/ixgbe.rs @@ -1623,15 +1623,12 @@ impl Ixgbe { let (virt_addr, phy_addr, numa_id) = rx.dma_info[index]; let ptr = virt_addr as *mut [u8; PAGESIZE]; - let data; - unsafe { - data = DMAPool::<[u8; PAGESIZE]>::from_raw_parts( - ptr, phy_addr, - PAGESIZE, // RECONSIDER: Not using "len" here, might be better to give the "len" information somehow to protocol stack. - numa_id, - ) - .unwrap(); - } + let data = DMAPool::<[u8; PAGESIZE]>::from_raw_parts( + ptr, phy_addr, + PAGESIZE, // RECONSIDER: Not using "len" here, might be better to give the "len" information somehow to protocol stack. + numa_id, + ) + .unwrap(); rx.dma_info[index] = (read_buf.get_virt_addr().as_usize(), buf_phy_addr, numa_id); @@ -1837,11 +1834,8 @@ impl Ixgbe { let (used, virt_addr, phy_addr, numa_id) = tx.dma_info[head]; if used { let ptr = virt_addr as *mut [u8; PAGESIZE]; - let data; - unsafe { - data = DMAPool::<[u8; PAGESIZE]>::from_raw_parts(ptr, phy_addr, PAGESIZE, numa_id) - .unwrap(); - } + let data = DMAPool::<[u8; PAGESIZE]>::from_raw_parts(ptr, phy_addr, PAGESIZE, numa_id) + .unwrap(); drop(data); } @@ -1886,11 +1880,8 @@ impl Ixgbe { let (used, virt_addr, phy_addr, numa_id) = tx.dma_info[head]; if used { let ptr = virt_addr as *mut [u8; PAGESIZE]; - let data; - unsafe { - data = DMAPool::<[u8; PAGESIZE]>::from_raw_parts(ptr, phy_addr, PAGESIZE, numa_id) - .unwrap(); - } + let data = DMAPool::<[u8; PAGESIZE]>::from_raw_parts(ptr, phy_addr, PAGESIZE, numa_id) + .unwrap(); drop(data); } diff --git a/awkernel_lib/src/dma_pool.rs b/awkernel_lib/src/dma_pool.rs index 12e1583f0..c1f05b6e3 100644 --- a/awkernel_lib/src/dma_pool.rs +++ b/awkernel_lib/src/dma_pool.rs @@ -89,7 +89,7 @@ impl DMAPool { ptr } - pub unsafe fn from_raw_parts( + pub fn from_raw_parts( ptr: *mut T, phy_addr: usize, size: usize, @@ -100,7 +100,6 @@ impl DMAPool { let virt_addr = VirtAddr::new(ptr as usize); let phy_addr = PhyAddr::new(phy_addr); let ptr = NonNull::new(ptr)?; - //let ptr = NonNull::new_unchecked(core::ptr::slice_from_raw_parts_mut(ptr, size)); Some(Self { virt_addr, diff --git a/awkernel_lib/src/net/if_net.rs b/awkernel_lib/src/net/if_net.rs index 8f0fb7c81..54e827d85 100644 --- a/awkernel_lib/src/net/if_net.rs +++ b/awkernel_lib/src/net/if_net.rs @@ -44,6 +44,8 @@ use super::{ #[cfg(not(feature = "std"))] use alloc::{vec, vec::Vec}; +type DMABuffer = DMAPool<[u8; PAGESIZE]>; + struct NetDriver { inner: Arc, rx_que_id: usize, @@ -55,7 +57,7 @@ struct NetDriverRef<'a> { inner: &'a Arc, rx_ringq: Option<&'a mut RingQ>, - tx_ringq: &'a mut RingQ<(DMAPool<[u8; PAGESIZE]>, usize)>, + tx_ringq: &'a mut RingQ<(DMABuffer, usize)>, } impl NetDriverRef<'_> { @@ -171,7 +173,7 @@ pub(super) struct IfNet { vlan: Option, pub(super) inner: Mutex, rx_irq_to_driver: BTreeMap, - tx_only_ringq: Vec, usize)>>>, + tx_only_ringq: Vec>>, pub(super) net_device: Arc, pub(super) is_poll_mode: bool, poll_driver: Option, @@ -503,7 +505,7 @@ impl IfNet { while !device_ref.tx_ringq.is_empty() { if let Some((data, len)) = device_ref.tx_ringq.pop() { let ptr = data.get_virt_addr().as_mut_ptr(); - let slice = unsafe { core::slice::from_raw_parts_mut(ptr as *mut u8, len) }; + let slice = unsafe { core::slice::from_raw_parts_mut(ptr, len) }; let tx_packet_header_flags = device_ref.tx_packet_header_flags(slice); let data = EtherFrameBuf { @@ -565,7 +567,7 @@ impl IfNet { while !device_ref.tx_ringq.is_empty() { if let Some((data, len)) = device_ref.tx_ringq.pop() { let ptr = data.get_virt_addr().as_mut_ptr(); - let slice = unsafe { core::slice::from_raw_parts_mut(ptr as *mut u8, len) }; + let slice = unsafe { core::slice::from_raw_parts_mut(ptr, len) }; let tx_packet_header_flags = device_ref.tx_packet_header_flags(slice); let data = EtherFrameBuf { @@ -640,7 +642,7 @@ impl phy::RxToken for NRxToken { } pub struct NTxToken<'a> { - tx_ring: &'a mut RingQ<(DMAPool<[u8; PAGESIZE]>, usize)>, + tx_ring: &'a mut RingQ<(DMABuffer, usize)>, driver_ref_inner: &'a Arc, } @@ -650,10 +652,10 @@ impl phy::TxToken for NTxToken<'_> { F: FnOnce(&mut [u8]) -> R, { let segment_group = self.driver_ref_inner.get_segment_group().unwrap_or(0); - let buf: DMAPool<[u8; PAGESIZE]> = DMAPool::new(segment_group as usize, 1).unwrap(); // RECONSIDER: Not sure this unwrap is acceptable + let buf: DMABuffer = DMAPool::new(segment_group as usize, 1).unwrap(); // RECONSIDER: Not sure this unwrap is acceptable let ptr = buf.get_virt_addr().as_mut_ptr(); - let slice = unsafe { core::slice::from_raw_parts_mut(ptr as *mut u8, len) }; + let slice = unsafe { core::slice::from_raw_parts_mut(ptr, len) }; let result = f(slice); From 2fc18d23d03ebf2801d80888eb5668b04e0e780d Mon Sep 17 00:00:00 2001 From: koichiimai Date: Thu, 2 Jan 2025 21:21:48 +0900 Subject: [PATCH 04/14] fix Signed-off-by: koichiimai --- scripts/udp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/udp.py b/scripts/udp.py index 7b2c7eb6b..28482af36 100644 --- a/scripts/udp.py +++ b/scripts/udp.py @@ -3,8 +3,8 @@ sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) -#addr = ('localhost', 26099) -addr = ('10.0.2.2', 26099) +addr = ('localhost', 26099) +#addr = ('10.0.2.2', 26099) print('listening on %s port %s' % addr, file=sys.stderr) sock.bind(addr) From 9235d72f58e45acfaacf276d9a3e518cc5e7dbf6 Mon Sep 17 00:00:00 2001 From: koichiimai Date: Mon, 6 Jan 2025 10:59:06 +0900 Subject: [PATCH 05/14] fix Signed-off-by: koichiimai --- awkernel_drivers/src/pcie/intel/igb.rs | 2 +- awkernel_drivers/src/pcie/intel/ixgbe.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/awkernel_drivers/src/pcie/intel/igb.rs b/awkernel_drivers/src/pcie/intel/igb.rs index 1c7351b5c..0cf996e23 100644 --- a/awkernel_drivers/src/pcie/intel/igb.rs +++ b/awkernel_drivers/src/pcie/intel/igb.rs @@ -1184,7 +1184,7 @@ impl Igb { #[cfg(not(any(target_arch = "x86_64")))] { let _ = addr; - Err(IxgbeDriverErr::NotSupported) + Err(IgbDriverErr::NotSupported) } } diff --git a/awkernel_drivers/src/pcie/intel/ixgbe.rs b/awkernel_drivers/src/pcie/intel/ixgbe.rs index b5227b4cf..0161305c5 100644 --- a/awkernel_drivers/src/pcie/intel/ixgbe.rs +++ b/awkernel_drivers/src/pcie/intel/ixgbe.rs @@ -1549,7 +1549,7 @@ impl Ixgbe { #[cfg(not(any(target_arch = "x86_64")))] { let _ = addr; - Err(IxgbeDriverErr::NotSupported) + Err(IxgbeDriverErr::NotImplemented) } } From 80cc06974f408aa8f40495e53338fad248f747d4 Mon Sep 17 00:00:00 2001 From: Koichi Imai Date: Thu, 9 Jan 2025 18:34:09 +0900 Subject: [PATCH 06/14] just made it build successfully for genet: including some unnecessary copies Signed-off-by: Koichi Imai --- awkernel_drivers/src/ic/genet.rs | 36 ++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/awkernel_drivers/src/ic/genet.rs b/awkernel_drivers/src/ic/genet.rs index 09bd4541d..3a4134bad 100644 --- a/awkernel_drivers/src/ic/genet.rs +++ b/awkernel_drivers/src/ic/genet.rs @@ -6,8 +6,8 @@ use awkernel_lib::{ ether::{ETHER_ADDR_LEN, ETHER_BROADCAST_ADDR}, multicast::MulticastAddrs, net_device::{ - self, EtherFrameBuf, EtherFrameRef, LinkStatus, NetCapabilities, NetDevError, - NetDevice, NetFlags, + self, EtherFrameBuf, LinkStatus, NetCapabilities, NetDevError, NetDevice, NetFlags, + PacketHeaderFlags, }, }, paging::PAGESIZE, @@ -335,7 +335,7 @@ impl NetDevice for Genet { Ok(()) } - fn send(&self, data: EtherFrameRef, _que_id: usize) -> Result<(), NetDevError> { + fn send(&self, data: EtherFrameBuf, _que_id: usize) -> Result<(), NetDevError> { let inner = self.inner.read(); let frames = [data]; inner.send(&frames); @@ -746,7 +746,7 @@ impl GenetInner { registers::TBUF_CTRL.write(buf_ctrl, base); } - fn send(&self, ether_frames: &[EtherFrameRef]) { + fn send(&self, ether_frames: &[EtherFrameBuf]) { if !self.if_flags.contains(NetFlags::RUNNING | NetFlags::UP) { return; } @@ -776,7 +776,7 @@ impl GenetInner { break; } - let size = frame.data.len(); + let size = frame.data.get_size(); let mut length_status = registers::TX_DESC_STATUS_QTAG_MASK | registers::TX_DESC_STATUS_SOP @@ -786,7 +786,16 @@ impl GenetInner { length_status |= (size as u32) << registers::TX_DESC_STATUS_BUFLEN_SHIFT; let buf = tx.buf.as_mut().get_mut(index).unwrap(); - unsafe { core::ptr::copy_nonoverlapping(frame.data.as_ptr(), buf.as_mut_ptr(), size) }; + let mut temp = [0u8; 4096]; + let ptr = temp.as_mut_ptr() as *mut [u8; 4096]; + + unsafe { + core::ptr::copy_nonoverlapping(frame.data.as_ref(), ptr, 4096); + core::ptr::copy_nonoverlapping(temp.as_ptr(), buf.as_mut_ptr(), size); + } + + //let ptr = buf as *mut [u8; 4096]; + //unsafe { core::ptr::copy_nonoverlapping(frame.data.as_ref(), buf.as_mut_ptr(), size) }; let addr = tx.buf.get_phy_addr() + index * TX_BUF_SIZE; let addr = addr.as_usize(); @@ -1092,9 +1101,18 @@ impl GenetInner { { // error } else if let Some(buf) = rx.buf.as_ref().get(index) { - let data = buf[2..len as usize].to_vec(); - - let frame = EtherFrameBuf { data, vlan: None }; + // TODO: This implementation has unnecessary copy + let mut vector = buf[2..len as usize].to_vec(); + vector.resize(4096, 0); + let ptr = vector.as_mut_ptr() as *mut [u8; 4096]; + let data = DMAPool::<[u8; PAGESIZE]>::from_raw_parts(ptr, 0, PAGESIZE, 0).unwrap(); + + let frame = EtherFrameBuf { + data, + len: (len - 2) as usize, // Not sure + vlan: None, + csum_flags: PacketHeaderFlags::EMPTY, + }; let _ = rx.read_queue.push(frame); } From 7d6f6849ce741c55dda6ce289408215798f7ee1d Mon Sep 17 00:00:00 2001 From: koichiimai Date: Fri, 10 Jan 2025 15:44:22 +0900 Subject: [PATCH 07/14] fix genet.rs Signed-off-by: koichiimai --- awkernel_drivers/src/ic/genet.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/awkernel_drivers/src/ic/genet.rs b/awkernel_drivers/src/ic/genet.rs index 3a4134bad..5a07fe8fa 100644 --- a/awkernel_drivers/src/ic/genet.rs +++ b/awkernel_drivers/src/ic/genet.rs @@ -786,17 +786,15 @@ impl GenetInner { length_status |= (size as u32) << registers::TX_DESC_STATUS_BUFLEN_SHIFT; let buf = tx.buf.as_mut().get_mut(index).unwrap(); - let mut temp = [0u8; 4096]; - let ptr = temp.as_mut_ptr() as *mut [u8; 4096]; unsafe { - core::ptr::copy_nonoverlapping(frame.data.as_ref(), ptr, 4096); - core::ptr::copy_nonoverlapping(temp.as_ptr(), buf.as_mut_ptr(), size); + core::ptr::copy_nonoverlapping( + frame.data.as_ref().as_ptr(), + buf.as_mut_ptr(), + buf.len(), + ); } - //let ptr = buf as *mut [u8; 4096]; - //unsafe { core::ptr::copy_nonoverlapping(frame.data.as_ref(), buf.as_mut_ptr(), size) }; - let addr = tx.buf.get_phy_addr() + index * TX_BUF_SIZE; let addr = addr.as_usize(); @@ -1102,10 +1100,12 @@ impl GenetInner { // error } else if let Some(buf) = rx.buf.as_ref().get(index) { // TODO: This implementation has unnecessary copy - let mut vector = buf[2..len as usize].to_vec(); - vector.resize(4096, 0); - let ptr = vector.as_mut_ptr() as *mut [u8; 4096]; - let data = DMAPool::<[u8; PAGESIZE]>::from_raw_parts(ptr, 0, PAGESIZE, 0).unwrap(); + let mut data = DMAPool::<[u8; PAGESIZE]>::new(0, 1).unwrap(); + let dst_ptr = data.as_mut() as *mut u8; + unsafe { + let buf_ptr = buf.as_ptr().add(2); + core::ptr::copy_nonoverlapping(buf_ptr, dst_ptr, len as usize - 2); + } let frame = EtherFrameBuf { data, From 405fb49e8867e54be35d3574e19583e30d803661 Mon Sep 17 00:00:00 2001 From: koichiimai Date: Fri, 10 Jan 2025 15:47:10 +0900 Subject: [PATCH 08/14] add comment for genet.rs Signed-off-by: koichiimai --- awkernel_drivers/src/ic/genet.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/awkernel_drivers/src/ic/genet.rs b/awkernel_drivers/src/ic/genet.rs index 5a07fe8fa..89bbab213 100644 --- a/awkernel_drivers/src/ic/genet.rs +++ b/awkernel_drivers/src/ic/genet.rs @@ -787,6 +787,7 @@ impl GenetInner { let buf = tx.buf.as_mut().get_mut(index).unwrap(); + // TODO: This implementation has an unnecessary copy unsafe { core::ptr::copy_nonoverlapping( frame.data.as_ref().as_ptr(), @@ -1099,7 +1100,7 @@ impl GenetInner { { // error } else if let Some(buf) = rx.buf.as_ref().get(index) { - // TODO: This implementation has unnecessary copy + // TODO: This implementation has an unnecessary copy let mut data = DMAPool::<[u8; PAGESIZE]>::new(0, 1).unwrap(); let dst_ptr = data.as_mut() as *mut u8; unsafe { From 268b3efffe505a1f15b296c84fffa8da2bd86288 Mon Sep 17 00:00:00 2001 From: Koichi Imai Date: Fri, 10 Jan 2025 18:51:38 +0900 Subject: [PATCH 09/14] implement pseudo dma_pool for std Signed-off-by: Koichi Imai --- awkernel_lib/src/arch/std_common.rs | 2 ++ awkernel_lib/src/lib.rs | 1 - awkernel_lib/src/paging.rs | 2 -- kernel/src/arch/std_common/config.rs | 2 ++ kernel/src/arch/std_common/kernel_main.rs | 17 +++++++++++++++-- 5 files changed, 19 insertions(+), 5 deletions(-) diff --git a/awkernel_lib/src/arch/std_common.rs b/awkernel_lib/src/arch/std_common.rs index f012ca042..7c895e4c0 100644 --- a/awkernel_lib/src/arch/std_common.rs +++ b/awkernel_lib/src/arch/std_common.rs @@ -3,6 +3,8 @@ pub(super) mod delay; pub(super) mod dvfs; pub(super) mod interrupt; +pub(super) mod paging; + pub fn init() { delay::init(); } diff --git a/awkernel_lib/src/lib.rs b/awkernel_lib/src/lib.rs index 51c3ae8a6..fa80b86ab 100644 --- a/awkernel_lib/src/lib.rs +++ b/awkernel_lib/src/lib.rs @@ -30,7 +30,6 @@ pub mod unwind; #[cfg(not(feature = "std"))] pub mod heap; -#[cfg(not(feature = "std"))] pub mod dma_pool; #[cfg(not(feature = "std"))] diff --git a/awkernel_lib/src/paging.rs b/awkernel_lib/src/paging.rs index bf8382508..9d40226dc 100644 --- a/awkernel_lib/src/paging.rs +++ b/awkernel_lib/src/paging.rs @@ -1,6 +1,5 @@ use crate::addr::{phy_addr::PhyAddr, virt_addr::VirtAddr}; -#[cfg(not(feature = "std"))] use crate::arch::ArchImpl; pub trait Frame { @@ -89,7 +88,6 @@ pub trait Mapper { } /// Return the physical address of `vm_addr`. -#[cfg(not(feature = "std"))] #[inline(always)] pub fn vm_to_phy(vm_addr: VirtAddr) -> Option { ArchImpl::vm_to_phy(vm_addr) diff --git a/kernel/src/arch/std_common/config.rs b/kernel/src/arch/std_common/config.rs index fc975ad06..60ebc0f1c 100644 --- a/kernel/src/arch/std_common/config.rs +++ b/kernel/src/arch/std_common/config.rs @@ -1,2 +1,4 @@ #[allow(dead_code)] pub const PREEMPT_IRQ: u16 = 0; // unused + +pub const DMA_SIZE: usize = 512 * 1024 * 1024; // 512MiB diff --git a/kernel/src/arch/std_common/kernel_main.rs b/kernel/src/arch/std_common/kernel_main.rs index 680d9c6f2..50e7615f9 100644 --- a/kernel/src/arch/std_common/kernel_main.rs +++ b/kernel/src/arch/std_common/kernel_main.rs @@ -1,6 +1,9 @@ -use crate::{arch::std_common::console, kernel_info::KernelInfo}; +use crate::{ + arch::std_common::{config::DMA_SIZE, console}, + kernel_info::KernelInfo, +}; use core::{mem::MaybeUninit, ptr::null_mut}; -use libc::c_void; +use libc::{c_void, malloc, size_t}; // #[cfg(target_os = "linux")] // use core::mem::size_of; @@ -21,6 +24,16 @@ pub extern "C" fn main(_argc: isize, _argv: *const *const u8) -> isize { } } + // Init dma pool. This is actually just a heap memory in the host system. + unsafe { + let ptr = malloc(DMA_SIZE as size_t) as *mut u8; + awkernel_lib::dma_pool::init_dma_pool( + 0, + awkernel_lib::addr::virt_addr::VirtAddr::new(ptr as usize), + DMA_SIZE, + ) + }; + // Create worker threads. let mut threads = Vec::new(); for i in 1..nprocs() { From 96390b767f35e745f98347f8930fcc2c47081e83 Mon Sep 17 00:00:00 2001 From: Koichi Imai Date: Fri, 10 Jan 2025 18:54:25 +0900 Subject: [PATCH 10/14] apply clippy Signed-off-by: Koichi Imai --- awkernel_lib/src/arch/std_common/paging.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 awkernel_lib/src/arch/std_common/paging.rs diff --git a/awkernel_lib/src/arch/std_common/paging.rs b/awkernel_lib/src/arch/std_common/paging.rs new file mode 100644 index 000000000..5eed6bad8 --- /dev/null +++ b/awkernel_lib/src/arch/std_common/paging.rs @@ -0,0 +1,22 @@ +use crate::{ + addr::{phy_addr, virt_addr}, + paging::MapError, +}; + +impl crate::paging::Mapper for super::StdCommon { + unsafe fn map( + _vm_addr: virt_addr::VirtAddr, + _phy_addr: phy_addr::PhyAddr, + _flags: crate::paging::Flags, + ) -> Result<(), MapError> { + Err(MapError::InvalidPageTable) + } + + unsafe fn unmap(_vm_addr: virt_addr::VirtAddr) { + {} + } + + fn vm_to_phy(_vm_addr: virt_addr::VirtAddr) -> Option { + None + } +} From a98eb7030c6875ca42e3c7e183b5453ee377f137 Mon Sep 17 00:00:00 2001 From: Koichi Imai Date: Tue, 14 Jan 2025 11:06:30 +0900 Subject: [PATCH 11/14] fix PacketHeaderFlags empty and delete unnecessary comment Signed-off-by: Koichi Imai --- awkernel_drivers/src/pcie/intel/igb.rs | 13 ++++--------- awkernel_drivers/src/pcie/intel/ixgbe.rs | 13 ++++--------- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/awkernel_drivers/src/pcie/intel/igb.rs b/awkernel_drivers/src/pcie/intel/igb.rs index 0cf996e23..ae919d679 100644 --- a/awkernel_drivers/src/pcie/intel/igb.rs +++ b/awkernel_drivers/src/pcie/intel/igb.rs @@ -969,8 +969,6 @@ impl IgbInner { self.hw.reset_hw(&self.info)?; } - // TODO: Need to think about the deallocation of DMAPool - Ok(()) } @@ -1268,12 +1266,9 @@ impl Igb { let (virt_addr, phy_addr, numa_id) = rx.dma_info[index]; let ptr = virt_addr as *mut [u8; PAGESIZE]; - let data = DMAPool::<[u8; PAGESIZE]>::from_raw_parts( - ptr, phy_addr, - PAGESIZE, // RECONSIDER: Not using "len" here, might be better to give the "len" information somehow to protocol stack. - numa_id, - ) - .unwrap(); + let data = + DMAPool::<[u8; PAGESIZE]>::from_raw_parts(ptr, phy_addr, PAGESIZE, numa_id) + .unwrap(); rx.dma_info[index] = (read_buf.get_virt_addr().as_usize(), buf_phy_addr, numa_id); @@ -1289,7 +1284,7 @@ impl Igb { data, len, vlan, - csum_flags: PacketHeaderFlags::EMPTY, + csum_flags: PacketHeaderFlags::empty(), }) .unwrap(); }; diff --git a/awkernel_drivers/src/pcie/intel/ixgbe.rs b/awkernel_drivers/src/pcie/intel/ixgbe.rs index 0161305c5..06f35b27e 100644 --- a/awkernel_drivers/src/pcie/intel/ixgbe.rs +++ b/awkernel_drivers/src/pcie/intel/ixgbe.rs @@ -602,8 +602,6 @@ impl IxgbeInner { // ctrl_ext &= !IXGBE_CTRL_EXT_DRV_LOAD; // ixgbe_hw::write_reg(&self.info, IXGBE_CTRL_EXT, ctrl_ext); - // TODO: Need to think about the deallocation of DMAPool - self.update_link_status()?; Ok(()) @@ -1623,12 +1621,9 @@ impl Ixgbe { let (virt_addr, phy_addr, numa_id) = rx.dma_info[index]; let ptr = virt_addr as *mut [u8; PAGESIZE]; - let data = DMAPool::<[u8; PAGESIZE]>::from_raw_parts( - ptr, phy_addr, - PAGESIZE, // RECONSIDER: Not using "len" here, might be better to give the "len" information somehow to protocol stack. - numa_id, - ) - .unwrap(); + let data = + DMAPool::<[u8; PAGESIZE]>::from_raw_parts(ptr, phy_addr, PAGESIZE, numa_id) + .unwrap(); rx.dma_info[index] = (read_buf.get_virt_addr().as_usize(), buf_phy_addr, numa_id); @@ -1644,7 +1639,7 @@ impl Ixgbe { data, len: len as usize, vlan, - csum_flags: PacketHeaderFlags::EMPTY, + csum_flags: PacketHeaderFlags::empty(), }) .unwrap(); } From 368b8e1c89e418d50a9195e8c1edd29e3a52228b Mon Sep 17 00:00:00 2001 From: Koichi Imai Date: Tue, 14 Jan 2025 11:07:40 +0900 Subject: [PATCH 12/14] delete PacketHeaderFlags EMPTY Signed-off-by: Koichi Imai --- awkernel_lib/src/net/net_device.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/awkernel_lib/src/net/net_device.rs b/awkernel_lib/src/net/net_device.rs index 0bd77021b..cedc7d2a1 100644 --- a/awkernel_lib/src/net/net_device.rs +++ b/awkernel_lib/src/net/net_device.rs @@ -9,7 +9,6 @@ use alloc::{string::String, vec::Vec}; bitflags! { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct PacketHeaderFlags: u16 { - const EMPTY = 0x0000; // Empty const IPV4_CSUM_OUT = 0x0001; // IPv4 checksum needed const TCP_CSUM_OUT = 0x0002; // TCP checksum needed const UDP_CSUM_OUT = 0x0004; // UDP checksum needed From 5d7dbaf1d7b8505621755daaf5b530870c549f43 Mon Sep 17 00:00:00 2001 From: Koichi Imai Date: Tue, 14 Jan 2025 11:19:48 +0900 Subject: [PATCH 13/14] fix genet.rs Signed-off-by: Koichi Imai --- awkernel_drivers/src/ic/genet.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/awkernel_drivers/src/ic/genet.rs b/awkernel_drivers/src/ic/genet.rs index 89bbab213..b01b09e81 100644 --- a/awkernel_drivers/src/ic/genet.rs +++ b/awkernel_drivers/src/ic/genet.rs @@ -1110,9 +1110,9 @@ impl GenetInner { let frame = EtherFrameBuf { data, - len: (len - 2) as usize, // Not sure + len: (len - 2) as usize, vlan: None, - csum_flags: PacketHeaderFlags::EMPTY, + csum_flags: PacketHeaderFlags::empty(), }; let _ = rx.read_queue.push(frame); } From 758c7c66832b1bc7b63a6ed91471e7e38bef4c1c Mon Sep 17 00:00:00 2001 From: Koichi Imai Date: Tue, 14 Jan 2025 12:32:53 +0900 Subject: [PATCH 14/14] delete initializing len in setup_receive_structures() Signed-off-by: Koichi Imai --- awkernel_drivers/src/pcie/intel/igb.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/awkernel_drivers/src/pcie/intel/igb.rs b/awkernel_drivers/src/pcie/intel/igb.rs index ae919d679..f422e07dd 100644 --- a/awkernel_drivers/src/pcie/intel/igb.rs +++ b/awkernel_drivers/src/pcie/intel/igb.rs @@ -936,8 +936,6 @@ impl IgbInner { desc.desc.buf = buf_phy_addr as u64; - desc.desc.len = RXBUFFER_2048 as u16; // TODO: Need to check if we really need this. - dma_info_temp.push(( read_buf.get_virt_addr().as_usize(), buf_phy_addr,