FPGA/corundum
1
0
Fork 0
mirror of https://github.com/corundum/corundum.git synced 2025-02-06 08:38:23 +08:00
Code Issues Projects Releases Wiki Activity

Advance TX/RX queue pointers based on completion records instead of MMIO reads

Browse Source 
Signed-off-by: Alex Forencich <alex@alexforencich.com>
This commit is contained in:
Alex Forencich 2023-04-04 22:12:32 -07:00
parent 147f09e62e
commit a8feaf2383
4 changed files with 48 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
fpga/common/tb/mqnic.py
View File

@ -663,7 +663,6 @@ class TxRing:
self.head_ptr = 0 self.head_ptr = 0
self.tail_ptr = 0 self.tail_ptr = 0
self.clean_tail_ptr = 0
self.clean_event = Event() self.clean_event = Event()
@ -748,10 +747,10 @@ class TxRing:
self.active = False self.active = False
def empty(self): def empty(self):
return self.head_ptr == self.clean_tail_ptr return self.head_ptr == self.tail_ptr
def full(self): def full(self):
return self.head_ptr - self.clean_tail_ptr >= self.full_size return self.head_ptr - self.tail_ptr >= self.full_size
async def read_tail_ptr(self): async def read_tail_ptr(self):
val = await self.hw_regs.read_dword(MQNIC_QUEUE_TAIL_PTR_REG) val = await self.hw_regs.read_dword(MQNIC_QUEUE_TAIL_PTR_REG)
@ -767,9 +766,9 @@ class TxRing:
def free_buf(self): def free_buf(self):
while not self.empty(): while not self.empty():
index = self.clean_tail_ptr & self.size_mask index = self.tail_ptr & self.size_mask
self.free_desc(index) self.free_desc(index)
self.clean_tail_ptr += 1 self.tail_ptr += 1
@staticmethod @staticmethod
async def process_tx_cq(cq): async def process_tx_cq(cq):
@ -805,19 +804,17 @@ class TxRing:
await cq.write_tail_ptr() await cq.write_tail_ptr()
# process ring # process ring
await ring.read_tail_ptr() ring_tail_ptr = ring.tail_ptr
ring_index = ring_tail_ptr & ring.size_mask
ring_clean_tail_ptr = ring.clean_tail_ptr while (ring_tail_ptr != ring.head_ptr):
ring_index = ring_clean_tail_ptr & ring.size_mask
while (ring_clean_tail_ptr != ring.tail_ptr):
if ring.tx_info[ring_index]: if ring.tx_info[ring_index]:
break break
ring_clean_tail_ptr += 1 ring_tail_ptr += 1
ring_index = ring_clean_tail_ptr & ring.size_mask ring_index = ring_tail_ptr & ring.size_mask
ring.clean_tail_ptr = ring_clean_tail_ptr ring.tail_ptr = ring_tail_ptr
ring.clean_event.set() ring.clean_event.set()
@ -846,7 +843,6 @@ class RxRing:
self.head_ptr = 0 self.head_ptr = 0
self.tail_ptr = 0 self.tail_ptr = 0
self.clean_tail_ptr = 0
self.packets = 0 self.packets = 0
self.bytes = 0 self.bytes = 0
@ -931,10 +927,10 @@ class RxRing:
self.active = False self.active = False
def empty(self): def empty(self):
return self.head_ptr == self.clean_tail_ptr return self.head_ptr == self.tail_ptr
def full(self): def full(self):
return self.head_ptr - self.clean_tail_ptr >= self.full_size return self.head_ptr - self.tail_ptr >= self.full_size
async def read_tail_ptr(self): async def read_tail_ptr(self):
val = await self.hw_regs.read_dword(MQNIC_QUEUE_TAIL_PTR_REG) val = await self.hw_regs.read_dword(MQNIC_QUEUE_TAIL_PTR_REG)
@ -950,9 +946,9 @@ class RxRing:
def free_buf(self): def free_buf(self):
while not self.empty(): while not self.empty():
index = self.clean_tail_ptr & self.size_mask index = self.tail_ptr & self.size_mask
self.free_desc(index) self.free_desc(index)
self.clean_tail_ptr += 1 self.tail_ptr += 1
def prepare_desc(self, index): def prepare_desc(self, index):
pkt = self.driver.alloc_pkt() pkt = self.driver.alloc_pkt()
@ -969,7 +965,7 @@ class RxRing:
offset += seg offset += seg
async def refill_buffers(self): async def refill_buffers(self):
missing = self.size - (self.head_ptr - self.clean_tail_ptr) missing = self.size - (self.head_ptr - self.tail_ptr)
if missing < 8: if missing < 8:
return return
@ -1029,19 +1025,17 @@ class RxRing:
await cq.write_tail_ptr() await cq.write_tail_ptr()
# process ring # process ring
await ring.read_tail_ptr() ring_tail_ptr = ring.tail_ptr
ring_index = ring_tail_ptr & ring.size_mask
ring_clean_tail_ptr = ring.clean_tail_ptr while (ring_tail_ptr != ring.head_ptr):
ring_index = ring_clean_tail_ptr & ring.size_mask
while (ring_clean_tail_ptr != ring.tail_ptr):
if ring.rx_info[ring_index]: if ring.rx_info[ring_index]:
break break
ring_clean_tail_ptr += 1 ring_tail_ptr += 1
ring_index = ring_clean_tail_ptr & ring.size_mask ring_index = ring_tail_ptr & ring.size_mask
ring.clean_tail_ptr = ring_clean_tail_ptr ring.tail_ptr = ring_tail_ptr
# replenish buffers # replenish buffers
await ring.refill_buffers() await ring.refill_buffers()
@ -1414,7 +1408,7 @@ class Interface:
while True: while True:
# check for space in ring # check for space in ring
if ring.head_ptr - ring.clean_tail_ptr < ring.full_size: if ring.head_ptr - ring.tail_ptr < ring.full_size:
break break
# wait for space # wait for space

1
modules/mqnic/mqnic.h
View File

@ -251,7 +251,6 @@ struct mqnic_ring {
// written from completion // written from completion
u32 tail_ptr ____cacheline_aligned_in_smp; u32 tail_ptr ____cacheline_aligned_in_smp;
u32 clean_tail_ptr;
u64 ts_s; u64 ts_s;
u8 ts_valid; u8 ts_valid;

33
modules/mqnic/mqnic_rx.c
View File

@ -59,7 +59,6 @@ int mqnic_create_rx_ring(struct mqnic_if *interface, struct mqnic_ring **ring_pt
ring->head_ptr = 0; ring->head_ptr = 0;
ring->tail_ptr = 0; ring->tail_ptr = 0;
ring->clean_tail_ptr = 0;
// deactivate queue // deactivate queue
iowrite32(0, ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG); iowrite32(0, ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
@ -105,7 +104,6 @@ int mqnic_alloc_rx_ring(struct mqnic_ring *ring, int size, int stride)
ring->head_ptr = 0; ring->head_ptr = 0;
ring->tail_ptr = 0; ring->tail_ptr = 0;
ring->clean_tail_ptr = 0;
// deactivate queue // deactivate queue
iowrite32(0, ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG); iowrite32(0, ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
@ -199,12 +197,12 @@ void mqnic_deactivate_rx_ring(struct mqnic_ring *ring)
bool mqnic_is_rx_ring_empty(const struct mqnic_ring *ring) bool mqnic_is_rx_ring_empty(const struct mqnic_ring *ring)
{ {
return ring->head_ptr == ring->clean_tail_ptr; return ring->head_ptr == ring->tail_ptr;
} }
bool mqnic_is_rx_ring_full(const struct mqnic_ring *ring) bool mqnic_is_rx_ring_full(const struct mqnic_ring *ring)
{ {
return ring->head_ptr - ring->clean_tail_ptr >= ring->size; return ring->head_ptr - ring->tail_ptr >= ring->size;
} }
void mqnic_rx_read_tail_ptr(struct mqnic_ring *ring) void mqnic_rx_read_tail_ptr(struct mqnic_ring *ring)
@ -235,16 +233,12 @@ int mqnic_free_rx_buf(struct mqnic_ring *ring)
int cnt = 0; int cnt = 0;
while (!mqnic_is_rx_ring_empty(ring)) { while (!mqnic_is_rx_ring_empty(ring)) {
index = ring->clean_tail_ptr & ring->size_mask; index = ring->tail_ptr & ring->size_mask;
mqnic_free_rx_desc(ring, index); mqnic_free_rx_desc(ring, index);
ring->clean_tail_ptr++; ring->tail_ptr++;
cnt++; cnt++;
} }
ring->head_ptr = 0;
ring->tail_ptr = 0;
ring->clean_tail_ptr = 0;
return cnt; return cnt;
} }
@ -296,7 +290,7 @@ int mqnic_prepare_rx_desc(struct mqnic_ring *ring, int index)
void mqnic_refill_rx_buffers(struct mqnic_ring *ring) void mqnic_refill_rx_buffers(struct mqnic_ring *ring)
{ {
u32 missing = ring->size - (ring->head_ptr - ring->clean_tail_ptr); u32 missing = ring->size - (ring->head_ptr - ring->tail_ptr);
if (missing < 8) if (missing < 8)
return; return;
@ -325,7 +319,7 @@ int mqnic_process_rx_cq(struct mqnic_cq_ring *cq_ring, int napi_budget)
u32 cq_index; u32 cq_index;
u32 cq_tail_ptr; u32 cq_tail_ptr;
u32 ring_index; u32 ring_index;
u32 ring_clean_tail_ptr; u32 ring_tail_ptr;
int done = 0; int done = 0;
int budget = napi_budget; int budget = napi_budget;
u32 len; u32 len;
@ -410,24 +404,21 @@ int mqnic_process_rx_cq(struct mqnic_cq_ring *cq_ring, int napi_budget)
mqnic_cq_write_tail_ptr(cq_ring); mqnic_cq_write_tail_ptr(cq_ring);
// process ring // process ring
// read tail pointer from NIC ring_tail_ptr = READ_ONCE(rx_ring->tail_ptr);
mqnic_rx_read_tail_ptr(rx_ring); ring_index = ring_tail_ptr & rx_ring->size_mask;
ring_clean_tail_ptr = READ_ONCE(rx_ring->clean_tail_ptr); while (ring_tail_ptr != rx_ring->head_ptr) {
ring_index = ring_clean_tail_ptr & rx_ring->size_mask;
while (ring_clean_tail_ptr != rx_ring->tail_ptr) {
rx_info = &rx_ring->rx_info[ring_index]; rx_info = &rx_ring->rx_info[ring_index];
if (rx_info->page) if (rx_info->page)
break; break;
ring_clean_tail_ptr++; ring_tail_ptr++;
ring_index = ring_clean_tail_ptr & rx_ring->size_mask; ring_index = ring_tail_ptr & rx_ring->size_mask;
} }
// update ring tail // update ring tail
WRITE_ONCE(rx_ring->clean_tail_ptr, ring_clean_tail_ptr); WRITE_ONCE(rx_ring->tail_ptr, ring_tail_ptr);
// replenish buffers // replenish buffers
mqnic_refill_rx_buffers(rx_ring); mqnic_refill_rx_buffers(rx_ring);

37
modules/mqnic/mqnic_tx.c
View File

@ -60,7 +60,6 @@ int mqnic_create_tx_ring(struct mqnic_if *interface, struct mqnic_ring **ring_pt
ring->head_ptr = 0; ring->head_ptr = 0;
ring->tail_ptr = 0; ring->tail_ptr = 0;
ring->clean_tail_ptr = 0;
// deactivate queue // deactivate queue
iowrite32(0, ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG); iowrite32(0, ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
@ -107,7 +106,6 @@ int mqnic_alloc_tx_ring(struct mqnic_ring *ring, int size, int stride)
ring->head_ptr = 0; ring->head_ptr = 0;
ring->tail_ptr = 0; ring->tail_ptr = 0;
ring->clean_tail_ptr = 0;
// deactivate queue // deactivate queue
iowrite32(0, ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG); iowrite32(0, ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
@ -199,12 +197,12 @@ void mqnic_deactivate_tx_ring(struct mqnic_ring *ring)
bool mqnic_is_tx_ring_empty(const struct mqnic_ring *ring) bool mqnic_is_tx_ring_empty(const struct mqnic_ring *ring)
{ {
return ring->head_ptr == ring->clean_tail_ptr; return ring->head_ptr == ring->tail_ptr;
} }
bool mqnic_is_tx_ring_full(const struct mqnic_ring *ring) bool mqnic_is_tx_ring_full(const struct mqnic_ring *ring)
{ {
return ring->head_ptr - ring->clean_tail_ptr >= ring->full_size; return ring->head_ptr - ring->tail_ptr >= ring->full_size;
} }
void mqnic_tx_read_tail_ptr(struct mqnic_ring *ring) void mqnic_tx_read_tail_ptr(struct mqnic_ring *ring)
@ -244,16 +242,12 @@ int mqnic_free_tx_buf(struct mqnic_ring *ring)
int cnt = 0; int cnt = 0;
while (!mqnic_is_tx_ring_empty(ring)) { while (!mqnic_is_tx_ring_empty(ring)) {
index = ring->clean_tail_ptr & ring->size_mask; index = ring->tail_ptr & ring->size_mask;
mqnic_free_tx_desc(ring, index, 0); mqnic_free_tx_desc(ring, index, 0);
ring->clean_tail_ptr++; ring->tail_ptr++;
cnt++; cnt++;
} }
ring->head_ptr = 0;
ring->tail_ptr = 0;
ring->clean_tail_ptr = 0;
return cnt; return cnt;
} }
@ -268,7 +262,7 @@ int mqnic_process_tx_cq(struct mqnic_cq_ring *cq_ring, int napi_budget)
u32 cq_index; u32 cq_index;
u32 cq_tail_ptr; u32 cq_tail_ptr;
u32 ring_index; u32 ring_index;
u32 ring_clean_tail_ptr; u32 ring_tail_ptr;
u32 packets = 0; u32 packets = 0;
u32 bytes = 0; u32 bytes = 0;
int done = 0; int done = 0;
@ -315,24 +309,21 @@ int mqnic_process_tx_cq(struct mqnic_cq_ring *cq_ring, int napi_budget)
mqnic_cq_write_tail_ptr(cq_ring); mqnic_cq_write_tail_ptr(cq_ring);
// process ring // process ring
// read tail pointer from NIC ring_tail_ptr = READ_ONCE(tx_ring->tail_ptr);
mqnic_tx_read_tail_ptr(tx_ring); ring_index = ring_tail_ptr & tx_ring->size_mask;
ring_clean_tail_ptr = READ_ONCE(tx_ring->clean_tail_ptr); while (ring_tail_ptr != tx_ring->head_ptr) {
ring_index = ring_clean_tail_ptr & tx_ring->size_mask;
while (ring_clean_tail_ptr != tx_ring->tail_ptr) {
tx_info = &tx_ring->tx_info[ring_index]; tx_info = &tx_ring->tx_info[ring_index];
if (tx_info->skb) if (tx_info->skb)
break; break;
ring_clean_tail_ptr++; ring_tail_ptr++;
ring_index = ring_clean_tail_ptr & tx_ring->size_mask; ring_index = ring_tail_ptr & tx_ring->size_mask;
} }
// update ring tail // update ring tail
WRITE_ONCE(tx_ring->clean_tail_ptr, ring_clean_tail_ptr); WRITE_ONCE(tx_ring->tail_ptr, ring_tail_ptr);
// BQL // BQL
//netdev_tx_completed_queue(tx_ring->tx_queue, packets, bytes); //netdev_tx_completed_queue(tx_ring->tx_queue, packets, bytes);
@ -445,7 +436,7 @@ netdev_tx_t mqnic_start_xmit(struct sk_buff *skb, struct net_device *ndev)
int ring_index; int ring_index;
u32 index; u32 index;
bool stop_queue; bool stop_queue;
u32 clean_tail_ptr; u32 tail_ptr;
if (unlikely(!priv->port_up)) if (unlikely(!priv->port_up))
goto tx_drop; goto tx_drop;
@ -458,7 +449,7 @@ netdev_tx_t mqnic_start_xmit(struct sk_buff *skb, struct net_device *ndev)
ring = priv->tx_ring[ring_index]; ring = priv->tx_ring[ring_index];
clean_tail_ptr = READ_ONCE(ring->clean_tail_ptr); tail_ptr = READ_ONCE(ring->tail_ptr);
// prefetch for BQL // prefetch for BQL
netdev_txq_bql_enqueue_prefetchw(ring->tx_queue); netdev_txq_bql_enqueue_prefetchw(ring->tx_queue);
@ -544,7 +535,7 @@ netdev_tx_t mqnic_start_xmit(struct sk_buff *skb, struct net_device *ndev)
if (unlikely(stop_queue)) { if (unlikely(stop_queue)) {
smp_rmb(); smp_rmb();
clean_tail_ptr = READ_ONCE(ring->clean_tail_ptr); tail_ptr = READ_ONCE(ring->tail_ptr);
if (unlikely(!mqnic_is_tx_ring_full(ring))) if (unlikely(!mqnic_is_tx_ring_full(ring)))
netif_tx_wake_queue(ring->tx_queue); netif_tx_wake_queue(ring->tx_queue);