diff --git a/awkernel_drivers/src/ic/genet.rs b/awkernel_drivers/src/ic/genet.rs index 09bd4541d..b01b09e81 100644 --- a/awkernel_drivers/src/ic/genet.rs +++ b/awkernel_drivers/src/ic/genet.rs @@ -6,8 +6,8 @@ use awkernel_lib::{ ether::{ETHER_ADDR_LEN, ETHER_BROADCAST_ADDR}, multicast::MulticastAddrs, net_device::{ - self, EtherFrameBuf, EtherFrameRef, LinkStatus, NetCapabilities, NetDevError, - NetDevice, NetFlags, + self, EtherFrameBuf, LinkStatus, NetCapabilities, NetDevError, NetDevice, NetFlags, + PacketHeaderFlags, }, }, paging::PAGESIZE, @@ -335,7 +335,7 @@ impl NetDevice for Genet { Ok(()) } - fn send(&self, data: EtherFrameRef, _que_id: usize) -> Result<(), NetDevError> { + fn send(&self, data: EtherFrameBuf, _que_id: usize) -> Result<(), NetDevError> { let inner = self.inner.read(); let frames = [data]; inner.send(&frames); @@ -746,7 +746,7 @@ impl GenetInner { registers::TBUF_CTRL.write(buf_ctrl, base); } - fn send(&self, ether_frames: &[EtherFrameRef]) { + fn send(&self, ether_frames: &[EtherFrameBuf]) { if !self.if_flags.contains(NetFlags::RUNNING | NetFlags::UP) { return; } @@ -776,7 +776,7 @@ impl GenetInner { break; } - let size = frame.data.len(); + let size = frame.data.get_size(); let mut length_status = registers::TX_DESC_STATUS_QTAG_MASK | registers::TX_DESC_STATUS_SOP @@ -786,7 +786,15 @@ impl GenetInner { length_status |= (size as u32) << registers::TX_DESC_STATUS_BUFLEN_SHIFT; let buf = tx.buf.as_mut().get_mut(index).unwrap(); - unsafe { core::ptr::copy_nonoverlapping(frame.data.as_ptr(), buf.as_mut_ptr(), size) }; + + // TODO: This implementation has an unnecessary copy + unsafe { + core::ptr::copy_nonoverlapping( + frame.data.as_ref().as_ptr(), + buf.as_mut_ptr(), + buf.len(), + ); + } let addr = tx.buf.get_phy_addr() + index * TX_BUF_SIZE; let addr = addr.as_usize(); @@ -1092,9 +1100,20 @@ impl GenetInner { { // error } else if let Some(buf) = rx.buf.as_ref().get(index) { - let data = buf[2..len as usize].to_vec(); + // TODO: This implementation has an unnecessary copy + let mut data = DMAPool::<[u8; PAGESIZE]>::new(0, 1).unwrap(); + let dst_ptr = data.as_mut() as *mut u8; + unsafe { + let buf_ptr = buf.as_ptr().add(2); + core::ptr::copy_nonoverlapping(buf_ptr, dst_ptr, len as usize - 2); + } - let frame = EtherFrameBuf { data, vlan: None }; + let frame = EtherFrameBuf { + data, + len: (len - 2) as usize, + vlan: None, + csum_flags: PacketHeaderFlags::empty(), + }; let _ = rx.read_queue.push(frame); } diff --git a/awkernel_drivers/src/pcie/intel/e1000e_example.rs b/awkernel_drivers/src/pcie/intel/e1000e_example.rs index 60672e516..cfc716b8b 100644 --- a/awkernel_drivers/src/pcie/intel/e1000e_example.rs +++ b/awkernel_drivers/src/pcie/intel/e1000e_example.rs @@ -127,7 +127,7 @@ impl NetDevice for E1000eExample { fn send( &self, - _data: net_device::EtherFrameRef, + _data: net_device::EtherFrameBuf, _que_id: usize, ) -> Result<(), net_device::NetDevError> { todo!("send"); diff --git a/awkernel_drivers/src/pcie/intel/igb.rs b/awkernel_drivers/src/pcie/intel/igb.rs index 8113114e4..f422e07dd 100644 --- a/awkernel_drivers/src/pcie/intel/igb.rs +++ b/awkernel_drivers/src/pcie/intel/igb.rs @@ -20,8 +20,8 @@ use awkernel_lib::{ ipv6::Ip6Hdr, multicast::MulticastAddrs, net_device::{ - EtherFrameBuf, EtherFrameRef, LinkStatus, NetCapabilities, NetDevError, NetDevice, - NetFlags, PacketHeaderFlags, + EtherFrameBuf, LinkStatus, NetCapabilities, NetDevError, NetDevice, NetFlags, + PacketHeaderFlags, }, tcp::TCPHdr, udp::UDPHdr, @@ -118,19 +118,16 @@ const MAX_INTS_PER_SEC: u32 = 8000; const DEFAULT_ITR: u32 = 1000000000 / (MAX_INTS_PER_SEC * 256); type RxRing = [RxDescriptor; MAX_RXD]; -type RxBuffer = [[u8; RXBUFFER_2048 as usize]; MAX_RXD]; type TxRing = [TxDescriptor; MAX_TXD]; -type TxBuffer = [[u8; TXBUFFER_2048 as usize]; MAX_TXD]; struct Rx { rx_desc_head: u32, rx_desc_tail: usize, rx_desc_ring: DMAPool, - read_buf: Option>, - read_queue: RingQ, + dma_info: Vec<(usize, usize, usize)>, // Statistics dropped_pkts: u64, @@ -151,8 +148,7 @@ struct Tx { txd_cmd: u32, active_checksum_context: ActiveChecksumContext, - - write_buf: Option>, + dma_info: Vec<(bool, usize, usize, usize)>, } struct Queue { @@ -421,7 +417,7 @@ impl IgbInner { let mut que = Vec::new(); for i in 0..que_num { - que.push(allocate_desc_rings(&info, i)?); + que.push(allocate_desc_rings(&info, que_num, i)?); } let is_poll_mode; @@ -504,6 +500,9 @@ impl IgbInner { hw.set_get_link_status(true); + let mut dma_info = Vec::with_capacity(MAX_RXD * que_num); + dma_info.resize(MAX_RXD * que_num, (0, 0, 0)); + let igb = Self { info, hw, @@ -906,54 +905,48 @@ impl IgbInner { tx.tx_desc_head = 0; let tx_desc_ring = tx.tx_desc_ring.as_mut(); - - let tx_buffer_size = TXBUFFER_2048 as usize * MAX_TXD; - let write_buf = DMAPool::new( - self.info.get_segment_group() as usize, - tx_buffer_size / PAGESIZE, - ) - .ok_or(IgbDriverErr::DMAPool)?; - - let buf_phy_addr = write_buf.get_phy_addr().as_usize(); - - for (i, desc) in tx_desc_ring.iter_mut().enumerate() { - desc.legacy.buf = (buf_phy_addr + i * TXBUFFER_2048 as usize) as u64; + for desc in tx_desc_ring.iter_mut() { + desc.legacy.buf = 0; desc.legacy.lower.raw = 0; desc.legacy.upper.raw = 0; } - - tx.write_buf = Some(write_buf); } Ok(()) } fn setup_receive_structures(&mut self, que: &[Queue]) -> Result<(), IgbDriverErr> { - for que in que.iter() { + for (i, que) in que.iter().enumerate() { let mut node = MCSNode::new(); let mut rx = que.rx.lock(&mut node); rx.rx_desc_tail = 0; - rx.rx_desc_head = rx.rx_desc_ring.as_ref().len() as u32 - 1; + let rx_desc_ring_len = rx.rx_desc_ring.as_ref().len(); + rx.rx_desc_head = rx_desc_ring_len as u32 - 1; let rx_desc_ring = rx.rx_desc_ring.as_mut(); + let mut dma_info_temp = Vec::new(); - let rx_buffer_size = RXBUFFER_2048 as usize * MAX_RXD; - let read_buf = DMAPool::new( - self.info.get_segment_group() as usize, - rx_buffer_size / PAGESIZE, - ) - .ok_or(IgbDriverErr::DMAPool)?; + for desc in rx_desc_ring { + let read_buf: DMAPool<[u8; PAGESIZE]> = + DMAPool::new(self.info.get_segment_group() as usize, 1) + .ok_or(IgbDriverErr::DMAPool)?; + let buf_phy_addr = read_buf.get_phy_addr().as_usize(); + desc.data = [0; 2]; - let buf_phy_addr = read_buf.get_phy_addr().as_usize(); + desc.desc.buf = buf_phy_addr as u64; - for (i, desc) in rx_desc_ring.iter_mut().enumerate() { - desc.data = [0; 2]; - desc.desc.buf = (buf_phy_addr + i * RXBUFFER_2048 as usize) as u64; - desc.desc.len = RXBUFFER_2048 as u16; + dma_info_temp.push(( + read_buf.get_virt_addr().as_usize(), + buf_phy_addr, + self.info.get_segment_group() as usize, + )); + read_buf.leak(); } - rx.read_buf = Some(read_buf); + for (j, info) in dma_info_temp.iter().enumerate() { + rx.dma_info[rx_desc_ring_len * i + j] = *info; + } } Ok(()) @@ -974,16 +967,6 @@ impl IgbInner { self.hw.reset_hw(&self.info)?; } - for que in que.iter() { - let mut node = MCSNode::new(); - let mut tx = que.tx.lock(&mut node); - tx.write_buf = None; - - let mut node = MCSNode::new(); - let mut rx = que.rx.lock(&mut node); - rx.read_buf = None; - } - Ok(()) } @@ -1186,6 +1169,21 @@ impl Igb { Ok(()) } + // TODO: Not sure if we really need this + unsafe fn invalidate_cache(&self, addr: *const u8) -> Result<(), IgbDriverErr> { + #[cfg(target_arch = "x86_64")] + { + core::arch::x86_64::_mm_clflush(addr); + Ok(()) + } + + #[cfg(not(any(target_arch = "x86_64")))] + { + let _ = addr; + Err(IgbDriverErr::NotSupported) + } + } + fn rx_recv(&self, que_id: usize) -> Result<(), IgbDriverErr> { let mac_type = self.inner.read().hw.get_mac_type(); @@ -1193,14 +1191,11 @@ impl Igb { { let inner = self.inner.read(); + let numa_id = inner.info.get_segment_group() as usize; let mut node = MCSNode::new(); let mut rx = que.rx.lock(&mut node); - if rx.read_buf.is_none() { - return Ok(()); - } - let mut i = rx.rx_desc_tail; loop { @@ -1212,10 +1207,11 @@ impl Igb { break; } - let rx_desc_ring = rx.rx_desc_ring.as_ref(); + let mut dropped_pkts = rx.dropped_pkts; + let rx_desc_ring = rx.rx_desc_ring.as_mut(); let rx_desc_ring_len = rx_desc_ring.len(); - let desc = unsafe { &rx_desc_ring[i].desc }; + let desc = unsafe { &mut rx_desc_ring.as_mut()[i].desc }; let status = desc.status; let errors = desc.error; @@ -1251,7 +1247,7 @@ impl Igb { len -= 1; true } else { - rx.dropped_pkts += 1; + dropped_pkts += 1; false } } else { @@ -1259,13 +1255,40 @@ impl Igb { }; if is_accept { - let read_buf = rx.read_buf.as_ref().unwrap(); - let src = &read_buf.as_ref()[i]; - let data = src[0..len].to_vec(); + let read_buf: DMAPool<[u8; PAGESIZE]> = + DMAPool::new(numa_id, 1).ok_or(IgbDriverErr::DMAPool)?; + let buf_phy_addr = read_buf.get_phy_addr().as_usize(); + desc.buf = buf_phy_addr as u64; + + let index = rx_desc_ring_len * que_id + i; + let (virt_addr, phy_addr, numa_id) = rx.dma_info[index]; + + let ptr = virt_addr as *mut [u8; PAGESIZE]; + let data = + DMAPool::<[u8; PAGESIZE]>::from_raw_parts(ptr, phy_addr, PAGESIZE, numa_id) + .unwrap(); - rx.read_queue.push(EtherFrameBuf { data, vlan }).unwrap(); + rx.dma_info[index] = + (read_buf.get_virt_addr().as_usize(), buf_phy_addr, numa_id); + + // Need clflush + unsafe { + self.invalidate_cache(read_buf.get_virt_addr().as_usize() as *const u8)?; + } + + read_buf.leak(); + rx.read_queue + .push(EtherFrameBuf { + data, + len, + vlan, + csum_flags: PacketHeaderFlags::empty(), + }) + .unwrap(); }; + rx.dropped_pkts = dropped_pkts; + i += 1; if i == rx_desc_ring_len { i = 0; @@ -1312,13 +1335,15 @@ impl Igb { fn transmit_checksum_setup( &self, tx: &mut Tx, - ether_frame: &EtherFrameRef, + ether_frame: &EtherFrameBuf, head: usize, ) -> Result<(usize, u32, u32, u16, Option), IgbDriverErr> { let txd_upper; let txd_lower; - let ext = extract_headers(ether_frame.data).or(Err(IgbDriverErr::InvalidPacket))?; + let ptr = ether_frame.data.get_virt_addr().as_mut_ptr(); + let slice = unsafe { core::slice::from_raw_parts(ptr, ether_frame.len) }; + let ext = extract_headers(slice).or(Err(IgbDriverErr::InvalidPacket))?; if !(matches!(ext.network, NetworkHdr::Ipv4(_)) && matches!(ext.transport, TransportHdr::Tcp(_) | TransportHdr::Udp(_))) { @@ -1398,7 +1423,7 @@ impl Igb { mac_type: MacType, me: usize, tx: &mut Tx, - ether_frame: &EtherFrameRef, + ether_frame: &EtherFrameBuf, head: usize, ) -> Result<(usize, u32, u32, u16, Option), IgbDriverErr> { let mut olinfo_status = 0; @@ -1414,7 +1439,9 @@ impl Igb { off = true; } - let ext = extract_headers(ether_frame.data).or(Err(IgbDriverErr::InvalidPacket))?; + let ptr = ether_frame.data.get_virt_addr().as_mut_ptr(); + let slice = unsafe { core::slice::from_raw_parts(ptr, ether_frame.len) }; + let ext = extract_headers(slice).or(Err(IgbDriverErr::InvalidPacket))?; vlan_macip_lens |= (core::mem::size_of::() as u32) << ADVTXD_MACLEN_SHIFT; @@ -1443,7 +1470,7 @@ impl Igb { let mut mss_l4len_idx = 0; - let mut paylen = ether_frame.data.len() as u32; + let mut paylen = ether_frame.len as u32; match ext.transport { TransportHdr::Tcp(tcp) => { type_tucmd_mlhl |= ADVTXD_TUCMD_L4T_TCP; @@ -1457,7 +1484,7 @@ impl Igb { cmd_type_len |= ADVTXD_DCMD_TSE; off = true; - paylen = ether_frame.data.len() as u32 - header_len; + paylen = ether_frame.len as u32 - header_len; } } TransportHdr::Udp(_udp) => { @@ -1471,13 +1498,13 @@ impl Igb { mss_l4len_idx |= (mem::size_of::() as u32) << 8; off = true; - paylen = ether_frame.data.len() as u32 - header_len; + paylen = ether_frame.len as u32 - header_len; } } _ => (), } - olinfo_status |= (ether_frame.data.len() as u32) << ADVTXD_PAYLEN_SHIFT; + olinfo_status |= (ether_frame.len as u32) << ADVTXD_PAYLEN_SHIFT; if !off { return Ok((0, cmd_type_len, olinfo_status, 0, None)); @@ -1588,9 +1615,9 @@ impl Igb { mac_type: MacType, me: usize, tx: &mut Tx, - ether_frame: &EtherFrameRef, + ether_frame: &EtherFrameBuf, ) -> Result { - let len = ether_frame.data.len(); + let len = ether_frame.len; if len > TXBUFFER_2048 as usize { return Err(IgbDriverErr::InvalidPacket); } @@ -1615,20 +1642,35 @@ impl Igb { head -= tx_slots; } - let addr = unsafe { - let write_buf = tx.write_buf.as_mut().unwrap(); - let dst = &mut write_buf.as_mut()[head]; - core::ptr::copy_nonoverlapping(ether_frame.data.as_ptr(), dst.as_mut_ptr(), len); - if let Some(cksum_offset) = cksum_offset { + let addr = ether_frame.data.get_phy_addr().as_usize() as u64; + if let Some(cksum_offset) = cksum_offset { + unsafe { core::ptr::write( - dst.as_mut_ptr().add(cksum_offset as usize) as *mut u16, + ether_frame + .data + .get_virt_addr() + .as_mut_ptr::() + .add(cksum_offset as usize) as *mut u16, cksum_pseudo.to_be(), ); } - (write_buf.get_phy_addr().as_usize() + head * TXBUFFER_2048 as usize) as u64 - }; + } let desc = &mut tx.tx_desc_ring.as_mut()[head]; + let (prev_used, virt_addr, phy_addr, numa_id) = tx.dma_info[head]; + if prev_used { + let ptr = virt_addr as *mut [u8; PAGESIZE]; + let data = DMAPool::<[u8; PAGESIZE]>::from_raw_parts(ptr, phy_addr, PAGESIZE, numa_id) + .unwrap(); + drop(data); + } + + tx.dma_info[head] = ( + true, + ether_frame.data.get_virt_addr().as_usize(), + ether_frame.data.get_phy_addr().as_usize(), + ether_frame.data.get_numa_id(), + ); desc.legacy.buf = u64::to_le(addr); desc.legacy.lower.raw = u32::to_le(tx.txd_cmd | txd_lower | (len & 0xffff) as u32); @@ -1661,7 +1703,7 @@ impl Igb { Ok(used) } - fn send(&self, que_id: usize, ether_frames: &[EtherFrameRef]) -> Result<(), IgbDriverErr> { + fn send(&self, que_id: usize, ether_frames: &[EtherFrameBuf]) -> Result<(), IgbDriverErr> { let inner = self.inner.read(); if !inner.link_active { @@ -1877,7 +1919,11 @@ fn allocate_msi(info: &mut PCIeInfo) -> Result { } } -fn allocate_desc_rings(info: &PCIeInfo, que_id: usize) -> Result { +fn allocate_desc_rings( + info: &PCIeInfo, + que_num: usize, + que_id: usize, +) -> Result { let tx_size = core::mem::size_of::() * MAX_TXD; assert_eq!(tx_size & (PAGESIZE - 1), 0); @@ -1889,22 +1935,27 @@ fn allocate_desc_rings(info: &PCIeInfo, que_id: usize) -> Result Result<(), NetDevError> { + fn send(&self, data: EtherFrameBuf, que_id: usize) -> Result<(), NetDevError> { let frames = [data]; - self.send(que_id, &frames).or(Err(NetDevError::DeviceError)) + self.send(que_id, &frames) + .or(Err(NetDevError::DeviceError))?; + for data in frames { + data.data.leak(); + } + Ok(()) } fn up(&self) -> Result<(), NetDevError> { diff --git a/awkernel_drivers/src/pcie/intel/ixgbe.rs b/awkernel_drivers/src/pcie/intel/ixgbe.rs index 34502fa97..06f35b27e 100644 --- a/awkernel_drivers/src/pcie/intel/ixgbe.rs +++ b/awkernel_drivers/src/pcie/intel/ixgbe.rs @@ -27,8 +27,8 @@ use awkernel_lib::{ ip::Ip, ipv6::Ip6Hdr, net_device::{ - EtherFrameBuf, EtherFrameRef, LinkStatus, NetCapabilities, NetDevError, NetDevice, - NetFlags, PacketHeaderFlags, + EtherFrameBuf, LinkStatus, NetCapabilities, NetDevError, NetDevice, NetFlags, + PacketHeaderFlags, }, tcp::TCPHdr, udp::UDPHdr, @@ -73,25 +73,23 @@ const MCLBYTES: u32 = 1 << MCLSHIFT; const MAXMCLBYTES: u32 = 64 * 1024; type TxRing = [TxDescriptor; DEFAULT_TXD]; -type TxBuffer = [[u8; MCLBYTES as usize]; DEFAULT_TXD]; type RxRing = [AdvRxDesc; DEFAULT_RXD]; -type RxBuffer = [[u8; MCLBYTES as usize]; DEFAULT_RXD]; pub struct Tx { tx_desc_head: usize, tx_desc_tail: usize, tx_desc_ring: DMAPool, txd_cmd: u32, - write_buf: Option>, + dma_info: Vec<(bool, usize, usize, usize)>, } pub struct Rx { rx_desc_head: u32, rx_desc_tail: usize, rx_desc_ring: DMAPool, - read_buf: Option>, read_queue: RingQ, + dma_info: Vec<(usize, usize, usize)>, } pub struct Queue { @@ -306,7 +304,7 @@ impl IxgbeInner { let mut que = Vec::new(); let que_num = get_num_queues(&hw.mac.mac_type); for i in 0..que_num { - que.push(allocate_queue(&info, i)?); + que.push(allocate_queue(&info, que_num, i)?); } // ixgbe_identify_hardware() @@ -389,6 +387,9 @@ impl IxgbeInner { ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; ixgbe_hw::write_reg(&info, IXGBE_CTRL_EXT, ctrl_ext)?; + let mut dma_info = Vec::with_capacity(DEFAULT_RXD * que_num); + dma_info.resize(DEFAULT_RXD * que_num, (0, 0, 0)); + let ixgbe = Self { info, hw, @@ -412,7 +413,7 @@ impl IxgbeInner { fn init(&mut self, que: &[Queue]) -> Result<(), IxgbeDriverErr> { use ixgbe_hw::MacType::*; - self.stop(que)?; + self.stop()?; // reprogram the RAR[0] in case user changed it. self.ops.mac_set_rar( @@ -427,7 +428,7 @@ impl IxgbeInner { if let Err(e) = self.setup_transmit_structures(que) { log::error!("Could not setup transmit structures"); - self.stop(que)?; + self.stop()?; return Err(e); } @@ -437,7 +438,7 @@ impl IxgbeInner { // Prepare receive descriptors and buffers if let Err(e) = self.setup_receive_structures(que) { log::error!("Could not setup receieve structures"); - self.stop(que)?; + self.stop()?; return Err(e); } @@ -568,7 +569,7 @@ impl IxgbeInner { Ok(()) } - fn stop(&mut self, que: &[Queue]) -> Result<(), IxgbeDriverErr> { + fn stop(&mut self) -> Result<(), IxgbeDriverErr> { self.flags.remove(NetFlags::RUNNING); self.disable_intr()?; @@ -601,16 +602,6 @@ impl IxgbeInner { // ctrl_ext &= !IXGBE_CTRL_EXT_DRV_LOAD; // ixgbe_hw::write_reg(&self.info, IXGBE_CTRL_EXT, ctrl_ext); - for que in que.iter() { - let mut node = MCSNode::new(); - let mut tx = que.tx.lock(&mut node); - tx.write_buf = None; - - let mut node = MCSNode::new(); - let mut rx = que.rx.lock(&mut node); - rx.read_buf = None; - } - self.update_link_status()?; Ok(()) @@ -661,23 +652,11 @@ impl IxgbeInner { tx.tx_desc_head = 0; let tx_desc_ring = tx.tx_desc_ring.as_mut(); - - let tx_buffer_size = MCLBYTES as usize * DEFAULT_TXD; - let write_buf = DMAPool::new( - self.info.get_segment_group() as usize, - tx_buffer_size / PAGESIZE, - ) - .ok_or(IxgbeDriverErr::DMAPool)?; - - let buf_phy_addr = write_buf.get_phy_addr().as_usize(); - - for (i, desc) in tx_desc_ring.iter_mut().enumerate() { - desc.adv_tx.buffer_addr = (buf_phy_addr + i * MCLBYTES as usize) as u64; + for desc in tx_desc_ring.iter_mut() { + desc.adv_tx.buffer_addr = 0; desc.adv_tx.cmd_type_len = 0; desc.adv_tx.olinfo_status = 0; } - - tx.write_buf = Some(write_buf); } Ok(()) @@ -751,30 +730,35 @@ impl IxgbeInner { } fn setup_receive_structures(&mut self, que: &[Queue]) -> Result<(), IxgbeDriverErr> { - for que in que.iter() { + for (i, que) in que.iter().enumerate() { let mut node = MCSNode::new(); let mut rx = que.rx.lock(&mut node); rx.rx_desc_tail = 0; - rx.rx_desc_head = rx.rx_desc_ring.as_ref().len() as u32 - 1; + let rx_desc_ring_len = rx.rx_desc_ring.as_ref().len(); + rx.rx_desc_head = rx_desc_ring_len as u32 - 1; let rx_desc_ring = rx.rx_desc_ring.as_mut(); + let mut dma_info_temp = Vec::new(); - let rx_buffer_size = MCLBYTES as usize * DEFAULT_RXD; - let read_buf = DMAPool::new( - self.info.get_segment_group() as usize, - rx_buffer_size / PAGESIZE, - ) - .ok_or(IxgbeDriverErr::DMAPool)?; - - let buf_phy_addr = read_buf.get_phy_addr().as_usize(); - - for (i, desc) in rx_desc_ring.iter_mut().enumerate() { + for desc in rx_desc_ring { + let read_buf: DMAPool<[u8; PAGESIZE]> = + DMAPool::new(self.info.get_segment_group() as usize, 1) + .ok_or(IxgbeDriverErr::DMAPool)?; + let buf_phy_addr = read_buf.get_phy_addr().as_usize(); desc.data = [0; 2]; - desc.read.pkt_addr = (buf_phy_addr + i * MCLBYTES as usize) as u64; + desc.read.pkt_addr = buf_phy_addr as u64; + dma_info_temp.push(( + read_buf.get_virt_addr().as_usize(), + buf_phy_addr, + self.info.get_segment_group() as usize, + )); + read_buf.leak(); } - rx.read_buf = Some(read_buf); + for (j, info) in dma_info_temp.iter().enumerate() { + rx.dma_info[rx_desc_ring_len * i + j] = *info; + } } Ok(()) @@ -1552,19 +1536,32 @@ impl Ixgbe { Ok(()) } + // TODO: Not sure if we really need this + unsafe fn invalidate_cache(&self, addr: *const u8) -> Result<(), IxgbeDriverErr> { + #[cfg(target_arch = "x86_64")] + { + core::arch::x86_64::_mm_clflush(addr); + Ok(()) + } + + #[cfg(not(any(target_arch = "x86_64")))] + { + let _ = addr; + Err(IxgbeDriverErr::NotImplemented) + } + } + fn rx_recv(&self, que_id: usize) -> Result<(), IxgbeDriverErr> { let que = &self.que[que_id]; { let inner = self.inner.read(); + let numa_id = inner.info.get_segment_group() as usize; + drop(inner); let mut node = MCSNode::new(); let mut rx = que.rx.lock(&mut node); - if rx.read_buf.is_none() { - return Ok(()); - } - let mut i = rx.rx_desc_tail; loop { @@ -1576,9 +1573,9 @@ impl Ixgbe { break; } - let rx_desc_ring = rx.rx_desc_ring.as_ref(); + let rx_desc_ring = rx.rx_desc_ring.as_mut(); let rx_desc_ring_len = rx_desc_ring.len(); - let desc = &rx_desc_ring[i]; + let desc = &mut rx.rx_desc_ring.as_mut()[i]; let staterr; unsafe { @@ -1588,7 +1585,7 @@ impl Ixgbe { break; } - let len; + let len: u16; let vtag; let eop; unsafe { @@ -1613,13 +1610,38 @@ impl Ixgbe { if !eop { drop(rx); - drop(inner); return self.recv_jumbo(que_id); } else { - let read_buf = rx.read_buf.as_ref().unwrap(); - let src = &read_buf.as_ref()[i]; - let data = src[0..len as usize].to_vec(); - rx.read_queue.push(EtherFrameBuf { data, vlan }).unwrap(); + let read_buf: DMAPool<[u8; PAGESIZE]> = + DMAPool::new(numa_id, 1).ok_or(IxgbeDriverErr::DMAPool)?; + let buf_phy_addr = read_buf.get_phy_addr().as_usize(); + desc.read.pkt_addr = buf_phy_addr as u64; + + let index = rx_desc_ring_len * que_id + i; + let (virt_addr, phy_addr, numa_id) = rx.dma_info[index]; + + let ptr = virt_addr as *mut [u8; PAGESIZE]; + let data = + DMAPool::<[u8; PAGESIZE]>::from_raw_parts(ptr, phy_addr, PAGESIZE, numa_id) + .unwrap(); + + rx.dma_info[index] = + (read_buf.get_virt_addr().as_usize(), buf_phy_addr, numa_id); + + // Need clflush + unsafe { + self.invalidate_cache(read_buf.get_virt_addr().as_usize() as *const u8)?; + } + + read_buf.leak(); + rx.read_queue + .push(EtherFrameBuf { + data, + len: len as usize, + vlan, + csum_flags: PacketHeaderFlags::empty(), + }) + .unwrap(); } i += 1; @@ -1685,17 +1707,19 @@ impl Ixgbe { #[allow(clippy::type_complexity)] fn tx_offload( &self, - ether_frame: &EtherFrameRef, + ether_frame: &EtherFrameBuf, ) -> Result<(bool, u32, u32, u32, u16, Option), IxgbeDriverErr> { let mut vlan_macip_lens = 0; let mut olinfo_status = 0; let mut offload = false; - let ext = extract_headers(ether_frame.data).or(Err(IxgbeDriverErr::InvalidPacket))?; + let ptr = ether_frame.data.get_virt_addr().as_mut_ptr(); + let slice = unsafe { core::slice::from_raw_parts(ptr, ether_frame.len) }; + let ext = extract_headers(slice).or(Err(IxgbeDriverErr::InvalidPacket))?; // Depend on whether doing TSO or not // Indicate the whole packet as payload when not doing TSO - olinfo_status |= (ether_frame.data.len() as u32) << IXGBE_ADVTXD_PAYLEN_SHIFT; + olinfo_status |= (ether_frame.len as u32) << IXGBE_ADVTXD_PAYLEN_SHIFT; vlan_macip_lens |= (core::mem::size_of::() as u32) << IXGBE_ADVTXD_MACLEN_SHIFT; @@ -1774,7 +1798,7 @@ impl Ixgbe { fn tx_ctx_setup( &self, tx: &mut Tx, - ether_frame: &EtherFrameRef, + ether_frame: &EtherFrameBuf, head: usize, ) -> Result<(u32, u32, u32, u16, Option), IxgbeDriverErr> { let mut cmd_type_len = 0; @@ -1802,6 +1826,15 @@ impl Ixgbe { type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; let desc = &mut tx.tx_desc_ring.as_mut()[head]; + let (used, virt_addr, phy_addr, numa_id) = tx.dma_info[head]; + if used { + let ptr = virt_addr as *mut [u8; PAGESIZE]; + let data = DMAPool::<[u8; PAGESIZE]>::from_raw_parts(ptr, phy_addr, PAGESIZE, numa_id) + .unwrap(); + drop(data); + } + + tx.dma_info[head] = (false, 0, 0, 0); // Now copy bits into descriptor desc.adv_ctx.vlan_macip_lens = u32::to_le(vlan_macip_lens); desc.adv_ctx.type_tucmd_mlhl = u32::to_le(type_tucmd_mlhl); @@ -1813,8 +1846,8 @@ impl Ixgbe { /// This routine maps the mbufs to tx descriptors, allowing the /// TX engine to transmit the packets. - fn encap(&self, tx: &mut Tx, ether_frame: &EtherFrameRef) -> Result { - let len = ether_frame.data.len(); + fn encap(&self, tx: &mut Tx, ether_frame: &EtherFrameBuf) -> Result { + let len = ether_frame.len; if len > MCLBYTES as usize { return Err(IxgbeDriverErr::InvalidPacket); } @@ -1833,22 +1866,27 @@ impl Ixgbe { if head == tx_slots { head = 0; } - let addr = unsafe { - let write_buf = tx.write_buf.as_mut().unwrap(); - let dst = &mut write_buf.as_mut()[head]; - core::ptr::copy_nonoverlapping(ether_frame.data.as_ptr(), dst.as_mut_ptr(), len); - // TODO: cksum offloading - //if let Some(cksum_offset) = cksum_offset { - //log::info!("cksum: {}", cksum_pseudo); - //core::ptr::write( - //dst.as_mut_ptr().add(cksum_offset as usize) as *mut u16, - //cksum_pseudo.to_be(), - //); - //} - (write_buf.get_phy_addr().as_usize() + head * MCLBYTES as usize) as u64 - }; + + // TODO: Checksum Offloading + + let addr = ether_frame.data.get_phy_addr().as_usize() as u64; let desc = &mut tx.tx_desc_ring.as_mut()[head]; + let (used, virt_addr, phy_addr, numa_id) = tx.dma_info[head]; + if used { + let ptr = virt_addr as *mut [u8; PAGESIZE]; + let data = DMAPool::<[u8; PAGESIZE]>::from_raw_parts(ptr, phy_addr, PAGESIZE, numa_id) + .unwrap(); + drop(data); + } + + tx.dma_info[head] = ( + true, + ether_frame.data.get_virt_addr().as_usize(), + ether_frame.data.get_phy_addr().as_usize(), + ether_frame.data.get_numa_id(), + ); + desc.adv_tx.buffer_addr = u64::to_le(addr); desc.adv_tx.cmd_type_len = u32::to_le( tx.txd_cmd | cmd_type_len | IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS | len as u32, @@ -1865,13 +1903,16 @@ impl Ixgbe { Ok(ntxc as usize + 1) } - fn send(&self, que_id: usize, ether_frames: &[EtherFrameRef]) -> Result<(), IxgbeDriverErr> { + fn send(&self, que_id: usize, ether_frames: &[EtherFrameBuf]) -> Result<(), IxgbeDriverErr> { let inner = self.inner.read(); if !inner.link_active { return Ok(()); } + let num_segs = inner.num_segs; + drop(inner); + let mut node = MCSNode::new(); let mut tx = self.que[que_id].tx.lock(&mut node); @@ -1888,7 +1929,7 @@ impl Ixgbe { for ether_frame in ether_frames.iter() { // Check that we have the minimal number of TX descriptors. // use 2 because cksum setup can use an extra slot - if free <= inner.num_segs as usize + 2 { + if free <= num_segs as usize + 2 { break; } @@ -1899,6 +1940,7 @@ impl Ixgbe { post = true; } + let inner = self.inner.read(); if post { ixgbe_hw::write_reg(&inner.info, IXGBE_TDT(que_id), tx.tx_desc_head as u32)?; } @@ -2003,7 +2045,7 @@ fn allocate_msi(info: &mut PCIeInfo) -> Result { /// Allocate memory for the transmit and receive rings, and then /// the descriptors associated with each, called only once at attach. -fn allocate_queue(info: &PCIeInfo, que_id: usize) -> Result { +fn allocate_queue(info: &PCIeInfo, que_num: usize, que_id: usize) -> Result { let tx_size = core::mem::size_of::() * DEFAULT_TXD; assert_eq!(tx_size & (PAGESIZE - 1), 0); @@ -2015,20 +2057,25 @@ fn allocate_queue(info: &PCIeInfo, que_id: usize) -> Result Result<(), NetDevError> { + fn send(&self, data: EtherFrameBuf, que_id: usize) -> Result<(), NetDevError> { let frames = [data]; - self.send(que_id, &frames).or(Err(NetDevError::DeviceError)) + self.send(que_id, &frames) + .or(Err(NetDevError::DeviceError))?; + for data in frames { + data.data.leak(); + } + Ok(()) } fn up(&self) -> Result<(), NetDevError> { @@ -2152,7 +2204,7 @@ impl NetDevice for Ixgbe { if !inner.flags.contains(NetFlags::UP) { if let Err(err_init) = inner.init(&self.que) { - if let Err(err_stop) = inner.stop(&self.que) { + if let Err(err_stop) = inner.stop() { log::error!("ixgbe: stop failed: {:?}", err_stop); } @@ -2171,7 +2223,7 @@ impl NetDevice for Ixgbe { let mut inner = self.inner.write(); if inner.flags.contains(NetFlags::UP) { - if let Err(e) = inner.stop(&self.que) { + if let Err(e) = inner.stop() { log::error!("ixgbe: stop failed: {:?}", e); Err(NetDevError::DeviceError) } else { diff --git a/awkernel_lib/src/arch/std_common.rs b/awkernel_lib/src/arch/std_common.rs index f012ca042..7c895e4c0 100644 --- a/awkernel_lib/src/arch/std_common.rs +++ b/awkernel_lib/src/arch/std_common.rs @@ -3,6 +3,8 @@ pub(super) mod delay; pub(super) mod dvfs; pub(super) mod interrupt; +pub(super) mod paging; + pub fn init() { delay::init(); } diff --git a/awkernel_lib/src/arch/std_common/paging.rs b/awkernel_lib/src/arch/std_common/paging.rs new file mode 100644 index 000000000..5eed6bad8 --- /dev/null +++ b/awkernel_lib/src/arch/std_common/paging.rs @@ -0,0 +1,22 @@ +use crate::{ + addr::{phy_addr, virt_addr}, + paging::MapError, +}; + +impl crate::paging::Mapper for super::StdCommon { + unsafe fn map( + _vm_addr: virt_addr::VirtAddr, + _phy_addr: phy_addr::PhyAddr, + _flags: crate::paging::Flags, + ) -> Result<(), MapError> { + Err(MapError::InvalidPageTable) + } + + unsafe fn unmap(_vm_addr: virt_addr::VirtAddr) { + {} + } + + fn vm_to_phy(_vm_addr: virt_addr::VirtAddr) -> Option { + None + } +} diff --git a/awkernel_lib/src/dma_pool.rs b/awkernel_lib/src/dma_pool.rs index aae52405b..c1f05b6e3 100644 --- a/awkernel_lib/src/dma_pool.rs +++ b/awkernel_lib/src/dma_pool.rs @@ -89,6 +89,27 @@ impl DMAPool { ptr } + pub fn from_raw_parts( + ptr: *mut T, + phy_addr: usize, + size: usize, + numa_id: usize, + ) -> Option { + assert!(numa_id < NUMA_NUM_MAX); + + let virt_addr = VirtAddr::new(ptr as usize); + let phy_addr = PhyAddr::new(phy_addr); + let ptr = NonNull::new(ptr)?; + + Some(Self { + virt_addr, + phy_addr, + size, + numa_id, + ptr, + }) + } + #[inline(always)] pub fn get_virt_addr(&self) -> VirtAddr { self.virt_addr diff --git a/awkernel_lib/src/lib.rs b/awkernel_lib/src/lib.rs index 51c3ae8a6..fa80b86ab 100644 --- a/awkernel_lib/src/lib.rs +++ b/awkernel_lib/src/lib.rs @@ -30,7 +30,6 @@ pub mod unwind; #[cfg(not(feature = "std"))] pub mod heap; -#[cfg(not(feature = "std"))] pub mod dma_pool; #[cfg(not(feature = "std"))] diff --git a/awkernel_lib/src/net/if_net.rs b/awkernel_lib/src/net/if_net.rs index c16f54840..a0feccec2 100644 --- a/awkernel_lib/src/net/if_net.rs +++ b/awkernel_lib/src/net/if_net.rs @@ -27,18 +27,25 @@ use smoltcp::{ wire::HardwareAddress, }; -use crate::sync::{mcs::MCSNode, mutex::Mutex, rwlock::RwLock}; +use crate::{ + addr::Addr, + dma_pool::DMAPool, + paging::PAGESIZE, + sync::{mcs::MCSNode, mutex::Mutex, rwlock::RwLock}, +}; use super::{ ether::{extract_headers, NetworkHdr, TransportHdr, ETHER_ADDR_LEN}, multicast::ipv4_addr_to_mac_addr, - net_device::{EtherFrameBuf, EtherFrameRef, NetCapabilities, NetDevice, PacketHeaderFlags}, + net_device::{EtherFrameBuf, NetCapabilities, NetDevice, PacketHeaderFlags}, NetManagerError, }; #[cfg(not(feature = "std"))] use alloc::{vec, vec::Vec}; +type DMABuffer = DMAPool<[u8; PAGESIZE]>; + struct NetDriver { inner: Arc, rx_que_id: usize, @@ -50,7 +57,7 @@ struct NetDriverRef<'a> { inner: &'a Arc, rx_ringq: Option<&'a mut RingQ>, - tx_ringq: &'a mut RingQ>, + tx_ringq: &'a mut RingQ<(DMABuffer, usize)>, } impl NetDriverRef<'_> { @@ -136,6 +143,7 @@ impl Device for NetDriverRef<'_> { NRxToken { data }, NTxToken { tx_ring: self.tx_ringq, + driver_ref_inner: self.inner, }, )); } @@ -156,6 +164,7 @@ impl Device for NetDriverRef<'_> { Some(NTxToken { tx_ring: self.tx_ringq, + driver_ref_inner: self.inner, }) } } @@ -165,7 +174,7 @@ pub(super) struct IfNet { pub(super) inner: Mutex, pub(super) socket_set: RwLock>, rx_irq_to_driver: BTreeMap, - tx_only_ringq: Vec>>>, + tx_only_ringq: Vec>>, pub(super) net_device: Arc, pub(super) is_poll_mode: bool, poll_driver: Option, @@ -494,11 +503,14 @@ impl IfNet { // send packets from the queue. while !device_ref.tx_ringq.is_empty() { - if let Some(data) = device_ref.tx_ringq.pop() { - let tx_packet_header_flags = device_ref.tx_packet_header_flags(&data); - - let data = EtherFrameRef { - data: &data, + if let Some((data, len)) = device_ref.tx_ringq.pop() { + let ptr = data.get_virt_addr().as_mut_ptr(); + let slice = unsafe { core::slice::from_raw_parts_mut(ptr, len) }; + let tx_packet_header_flags = device_ref.tx_packet_header_flags(slice); + + let data = EtherFrameBuf { + data, + len, vlan: self.vlan, csum_flags: tx_packet_header_flags, }; @@ -516,10 +528,6 @@ impl IfNet { fn poll_rx(&self, ref_net_driver: &NetDriver) -> bool { let que_id = ref_net_driver.rx_que_id; - let Some(tx_ringq) = self.tx_only_ringq.get(que_id) else { - return false; - }; - let mut node = MCSNode::new(); let mut rx_ringq = ref_net_driver.rx_ringq.lock(&mut node); @@ -532,6 +540,9 @@ impl IfNet { } } + let Some(tx_ringq) = self.tx_only_ringq.get(que_id) else { + return false; + }; let mut node = MCSNode::new(); let mut tx_ringq = tx_ringq.lock(&mut node); @@ -553,15 +564,17 @@ impl IfNet { // send packets from the queue. while !device_ref.tx_ringq.is_empty() { - if let Some(data) = device_ref.tx_ringq.pop() { - let tx_packet_header_flags = device_ref.tx_packet_header_flags(&data); - - let data = EtherFrameRef { - data: &data, + if let Some((data, len)) = device_ref.tx_ringq.pop() { + let ptr = data.get_virt_addr().as_mut_ptr(); + let slice = unsafe { core::slice::from_raw_parts_mut(ptr, len) }; + let tx_packet_header_flags = device_ref.tx_packet_header_flags(slice); + + let data = EtherFrameBuf { + data, + len, vlan: self.vlan, csum_flags: tx_packet_header_flags, }; - let _ = self.net_device.send(data, ref_net_driver.rx_que_id); } else { break; @@ -623,12 +636,13 @@ impl phy::RxToken for NRxToken { where F: FnOnce(&mut [u8]) -> R, { - f(&mut self.data.data) + f(self.data.data.as_mut()) } } pub struct NTxToken<'a> { - tx_ring: &'a mut RingQ>, + tx_ring: &'a mut RingQ<(DMABuffer, usize)>, + driver_ref_inner: &'a Arc, } impl phy::TxToken for NTxToken<'_> { @@ -636,16 +650,15 @@ impl phy::TxToken for NTxToken<'_> { where F: FnOnce(&mut [u8]) -> R, { - let mut buf = Vec::with_capacity(len); + let segment_group = self.driver_ref_inner.get_segment_group().unwrap_or(0); + let buf: DMABuffer = DMAPool::new(segment_group as usize, 1).unwrap(); // RECONSIDER: Not sure this unwrap is acceptable - #[allow(clippy::uninit_vec)] - unsafe { - buf.set_len(len); - }; + let ptr = buf.get_virt_addr().as_mut_ptr(); + let slice = unsafe { core::slice::from_raw_parts_mut(ptr, len) }; - let result = f(&mut buf[..len]); + let result = f(slice); - let _ = self.tx_ring.push(buf); + let _ = self.tx_ring.push((buf, len)); result } diff --git a/awkernel_lib/src/net/net_device.rs b/awkernel_lib/src/net/net_device.rs index b6eee8011..cedc7d2a1 100644 --- a/awkernel_lib/src/net/net_device.rs +++ b/awkernel_lib/src/net/net_device.rs @@ -1,3 +1,4 @@ +use crate::{dma_pool::DMAPool, paging::PAGESIZE}; use alloc::borrow::Cow; use bitflags::bitflags; use core::fmt::Display; @@ -132,17 +133,12 @@ pub enum NetDevError { MulticastAddrError, } -#[derive(Debug, Clone)] -pub struct EtherFrameRef<'a> { - pub data: &'a [u8], - pub vlan: Option, - pub csum_flags: PacketHeaderFlags, -} - -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct EtherFrameBuf { - pub data: Vec, + pub data: DMAPool<[u8; PAGESIZE]>, + pub len: usize, pub vlan: Option, + pub csum_flags: PacketHeaderFlags, } /// Because the network will have multiple queues @@ -150,7 +146,7 @@ pub struct EtherFrameBuf { /// the network device must be thread-safe. pub trait NetDevice { fn recv(&self, que_id: usize) -> Result, NetDevError>; - fn send(&self, data: EtherFrameRef, que_id: usize) -> Result<(), NetDevError>; + fn send(&self, data: EtherFrameBuf, que_id: usize) -> Result<(), NetDevError>; fn flags(&self) -> NetFlags; fn capabilities(&self) -> NetCapabilities; @@ -214,6 +210,10 @@ pub trait NetDevice { fn add_multicast_addr(&self, addr: &[u8; 6]) -> Result<(), NetDevError>; fn remove_multicast_addr(&self, addr: &[u8; 6]) -> Result<(), NetDevError>; + + fn get_segment_group(&self) -> Option { + None + } } impl Display for LinkStatus { diff --git a/awkernel_lib/src/paging.rs b/awkernel_lib/src/paging.rs index bf8382508..9d40226dc 100644 --- a/awkernel_lib/src/paging.rs +++ b/awkernel_lib/src/paging.rs @@ -1,6 +1,5 @@ use crate::addr::{phy_addr::PhyAddr, virt_addr::VirtAddr}; -#[cfg(not(feature = "std"))] use crate::arch::ArchImpl; pub trait Frame { @@ -89,7 +88,6 @@ pub trait Mapper { } /// Return the physical address of `vm_addr`. -#[cfg(not(feature = "std"))] #[inline(always)] pub fn vm_to_phy(vm_addr: VirtAddr) -> Option { ArchImpl::vm_to_phy(vm_addr) diff --git a/kernel/src/arch/std_common/config.rs b/kernel/src/arch/std_common/config.rs index fc975ad06..60ebc0f1c 100644 --- a/kernel/src/arch/std_common/config.rs +++ b/kernel/src/arch/std_common/config.rs @@ -1,2 +1,4 @@ #[allow(dead_code)] pub const PREEMPT_IRQ: u16 = 0; // unused + +pub const DMA_SIZE: usize = 512 * 1024 * 1024; // 512MiB diff --git a/kernel/src/arch/std_common/kernel_main.rs b/kernel/src/arch/std_common/kernel_main.rs index 680d9c6f2..50e7615f9 100644 --- a/kernel/src/arch/std_common/kernel_main.rs +++ b/kernel/src/arch/std_common/kernel_main.rs @@ -1,6 +1,9 @@ -use crate::{arch::std_common::console, kernel_info::KernelInfo}; +use crate::{ + arch::std_common::{config::DMA_SIZE, console}, + kernel_info::KernelInfo, +}; use core::{mem::MaybeUninit, ptr::null_mut}; -use libc::c_void; +use libc::{c_void, malloc, size_t}; // #[cfg(target_os = "linux")] // use core::mem::size_of; @@ -21,6 +24,16 @@ pub extern "C" fn main(_argc: isize, _argv: *const *const u8) -> isize { } } + // Init dma pool. This is actually just a heap memory in the host system. + unsafe { + let ptr = malloc(DMA_SIZE as size_t) as *mut u8; + awkernel_lib::dma_pool::init_dma_pool( + 0, + awkernel_lib::addr::virt_addr::VirtAddr::new(ptr as usize), + DMA_SIZE, + ) + }; + // Create worker threads. let mut threads = Vec::new(); for i in 1..nprocs() { diff --git a/kernel/src/arch/x86_64/config.rs b/kernel/src/arch/x86_64/config.rs index d14a54f56..7e0f39987 100644 --- a/kernel/src/arch/x86_64/config.rs +++ b/kernel/src/arch/x86_64/config.rs @@ -7,4 +7,4 @@ pub const HEAP_START: usize = 0x41000000000; pub const PREEMPT_IRQ: u16 = 255; -pub const DMA_SIZE: usize = 64 * 1024 * 1024; // 64MiB +pub const DMA_SIZE: usize = 512 * 1024 * 1024; // 512MiB diff --git a/kernel/src/arch/x86_64/kernel_main.rs b/kernel/src/arch/x86_64/kernel_main.rs index 5ad004d15..75e847657 100644 --- a/kernel/src/arch/x86_64/kernel_main.rs +++ b/kernel/src/arch/x86_64/kernel_main.rs @@ -814,7 +814,7 @@ fn init_dma( let dma_start = DMA_START + numa_id as usize * DMA_SIZE; let flags = PageTableFlags::PRESENT | PageTableFlags::WRITABLE - // | PageTableFlags::NO_CACHE + | PageTableFlags::NO_CACHE | PageTableFlags::NO_EXECUTE | PageTableFlags::GLOBAL;