FPGA/corundum
1
0
Fork 0
mirror of https://github.com/corundum/corundum.git synced 2025-01-16 08:12:53 +08:00
Code Issues Projects Releases Wiki Activity

Implement dynamic queue allocation in testbench and driver

Browse Source 
Signed-off-by: Alex Forencich <alex@alexforencich.com>
This commit is contained in:
Alex Forencich 2023-05-02 21:23:30 -07:00
parent d82f37b3b4
commit 3c995dc8e0
10 changed files with 851 additions and 767 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

446
fpga/common/tb/mqnic.py
View File

@ -319,6 +319,14 @@ class Resource:
self.stride = stride
self.windows = {}
self.free_list = list(range(count))
def alloc(self):
return self.free_list.pop(0)
def free(self, index):
self.free_list.append(index)
self.free_list.sort()
def get_count(self):
return self.count
@ -401,7 +409,7 @@ class Packet:
class Eq:
def __init__(self, interface, eqn, hw_regs):
def __init__(self, interface):
self.interface = interface
self.log = interface.log
self.driver = interface.driver
@ -409,8 +417,8 @@ class Eq:
self.size = 0
self.size_mask = 0
self.stride = 0
self.eqn = eqn
self.active = False
self.eqn = None
self.enabled = False
self.buf_size = 0
self.buf_region = None
@ -419,80 +427,82 @@ class Eq:
self.irq = None
self.cq_table = {}
self.head_ptr = 0
self.tail_ptr = 0
self.hw_ptr_mask = 0xffff
self.hw_regs = hw_regs
self.hw_regs = None
async def init(self):
self.log.info("Init EQ %d (interface %d)", self.eqn, self.interface.index)
async def open(self, irq, size):
if self.hw_regs:
raise Exception("Already open")
await self.hw_regs.write_dword(MQNIC_EQ_ACTIVE_LOG_SIZE_REG, 0) # active, log size
self.eqn = self.interface.eq_res.alloc()
async def alloc(self, size, stride):
if self.active:
raise Exception("Cannot allocate active ring")
if self.buf:
raise Exception("Already allocated")
self.log.info("Open EQ %d (interface %d)", self.eqn, self.interface.index)
self.log_size = size.bit_length() - 1
self.size = 2**self.log_size
self.size_mask = self.size-1
self.stride = stride
self.stride = MQNIC_EVENT_SIZE
self.buf_size = self.size*self.stride
self.buf_region = self.driver.pool.alloc_region(self.buf_size)
self.buf_dma = self.buf_region.get_absolute_address(0)
self.buf = self.buf_region.mem
self.buf[0:self.buf_size] = b'\x00'*self.buf_size
self.head_ptr = 0
self.tail_ptr = 0
await self.hw_regs.write_dword(MQNIC_EQ_ACTIVE_LOG_SIZE_REG, 0) # active, log size
await self.hw_regs.write_dword(MQNIC_EQ_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address
await self.hw_regs.write_dword(MQNIC_EQ_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address
await self.hw_regs.write_dword(MQNIC_EQ_INTERRUPT_INDEX_REG, 0) # interrupt index
await self.hw_regs.write_dword(MQNIC_EQ_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer
await self.hw_regs.write_dword(MQNIC_EQ_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer
await self.hw_regs.write_dword(MQNIC_EQ_ACTIVE_LOG_SIZE_REG, self.log_size) # active, log size
async def free(self):
await self.deactivate()
if self.buf:
# TODO
pass
async def activate(self, irq):
self.log.info("Activate Eq %d (interface %d)", self.eqn, self.interface.index)
await self.deactivate()
self.irq = irq
self.head_ptr = 0
self.tail_ptr = 0
self.cq_table = {}
self.buf[0:self.buf_size] = b'\x00'*self.buf_size
self.hw_regs = self.interface.eq_res.get_window(self.eqn)
await self.hw_regs.write_dword(MQNIC_EQ_ACTIVE_LOG_SIZE_REG, 0) # active, log size
await self.hw_regs.write_dword(MQNIC_EQ_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address
await self.hw_regs.write_dword(MQNIC_EQ_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address
await self.hw_regs.write_dword(MQNIC_EQ_INTERRUPT_INDEX_REG, irq) # interrupt index
await self.hw_regs.write_dword(MQNIC_EQ_INTERRUPT_INDEX_REG, self.irq) # interrupt index
await self.hw_regs.write_dword(MQNIC_EQ_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer
await self.hw_regs.write_dword(MQNIC_EQ_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer
await self.hw_regs.write_dword(MQNIC_EQ_ACTIVE_LOG_SIZE_REG, self.log_size | MQNIC_EQ_ACTIVE_MASK) # active, log size
self.active = True
self.enabled = True
async def close(self):
if not self.hw_regs:
return
async def deactivate(self):
await self.hw_regs.write_dword(MQNIC_EQ_ACTIVE_LOG_SIZE_REG, self.log_size) # active, log size
await self.hw_regs.write_dword(MQNIC_EQ_INTERRUPT_INDEX_REG, 0) # interrupt index
# TODO free buffer
self.irq = None
self.active = False
self.enabled = False
self.hw_regs = None
self.interface.eq_res.free(self.eqn)
self.eqn = None
def attach_cq(self, cq):
if cq.is_txcq:
self.cq_table[cq.cqn | 0x80000000] = cq
else:
self.cq_table[cq.cqn] = cq
def detach_cq(self, cq):
if cq.is_txcq:
del self.cq_table[cq.cqn | 0x80000000]
else:
del self.cq_table[cq.cqn]
async def read_head_ptr(self):
val = await self.hw_regs.read_dword(MQNIC_EQ_HEAD_PTR_REG)
@ -502,7 +512,7 @@ class Eq:
await self.hw_regs.write_dword(MQNIC_EQ_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask)
async def arm(self):
if not self.active:
if not self.hw_regs:
return
await self.hw_regs.write_dword(MQNIC_EQ_INTERRUPT_INDEX_REG, self.irq | MQNIC_EQ_ARM_MASK) # interrupt index
@ -527,12 +537,12 @@ class Eq:
if event_data[0] == 0:
# transmit completion
cq = self.interface.tx_cq[event_data[1]]
cq = self.cq_table[event_data[1] | 0x80000000]
await cq.handler(cq)
await cq.arm()
elif event_data[0] == 1:
# receive completion
cq = self.interface.rx_cq[event_data[1]]
cq = self.cq_table[event_data[1]]
await cq.handler(cq)
await cq.arm()
@ -544,7 +554,7 @@ class Eq:
class Cq:
def __init__(self, interface, cqn, hw_regs):
def __init__(self, interface):
self.interface = interface
self.log = interface.log
self.driver = interface.driver
@ -552,8 +562,8 @@ class Cq:
self.size = 0
self.size_mask = 0
self.stride = 0
self.cqn = cqn
self.active = False
self.cqn = None
self.enabled = False
self.buf_size = 0
self.buf_region = None
@ -569,76 +579,74 @@ class Cq:
self.tail_ptr = 0
self.hw_ptr_mask = 0xffff
self.hw_regs = hw_regs
self.hw_regs = None
async def init(self):
self.log.info("Init CQ %d (interface %d)", self.cqn, self.interface.index)
async def open(self, eq, size, is_txcq=True):
if self.hw_regs:
raise Exception("Already open")
await self.hw_regs.write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, 0) # active, log size
self.is_txcq = is_txcq
if is_txcq:
self.cqn = self.interface.tx_cq_res.alloc()
else:
self.cqn = self.interface.rx_cq_res.alloc()
async def alloc(self, size, stride):
if self.active:
raise Exception("Cannot allocate active ring")
if self.buf:
raise Exception("Already allocated")
self.log.info("Open %s CQ %d (interface %d)", "TX" if is_txcq else "RX", self.cqn, self.interface.index)
self.log_size = size.bit_length() - 1
self.size = 2**self.log_size
self.size_mask = self.size-1
self.stride = stride
self.stride = MQNIC_EVENT_SIZE
self.buf_size = self.size*self.stride
self.buf_region = self.driver.pool.alloc_region(self.buf_size)
self.buf_dma = self.buf_region.get_absolute_address(0)
self.buf = self.buf_region.mem
self.head_ptr = 0
self.tail_ptr = 0
await self.hw_regs.write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, 0) # active, log size
await self.hw_regs.write_dword(MQNIC_CQ_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address
await self.hw_regs.write_dword(MQNIC_CQ_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address
await self.hw_regs.write_dword(MQNIC_CQ_INTERRUPT_INDEX_REG, 0) # event index
await self.hw_regs.write_dword(MQNIC_CQ_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer
await self.hw_regs.write_dword(MQNIC_CQ_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer
await self.hw_regs.write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, self.log_size) # active, log size
async def free(self):
await self.deactivate()
if self.buf:
# TODO
pass
async def activate(self, eq):
self.log.info("Activate CQ %d (interface %d)", self.cqn, self.interface.index)
await self.deactivate()
self.eq = eq
self.head_ptr = 0
self.tail_ptr = 0
self.buf[0:self.buf_size] = b'\x00'*self.buf_size
self.head_ptr = 0
self.tail_ptr = 0
eq.attach_cq(self)
self.eq = eq
if is_txcq:
self.hw_regs = self.interface.tx_cq_res.get_window(self.cqn)
else:
self.hw_regs = self.interface.rx_cq_res.get_window(self.cqn)
await self.hw_regs.write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, 0) # active, log size
await self.hw_regs.write_dword(MQNIC_CQ_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address
await self.hw_regs.write_dword(MQNIC_CQ_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address
await self.hw_regs.write_dword(MQNIC_CQ_INTERRUPT_INDEX_REG, eq.eqn) # event index
await self.hw_regs.write_dword(MQNIC_CQ_INTERRUPT_INDEX_REG, self.eq.eqn) # event index
await self.hw_regs.write_dword(MQNIC_CQ_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer
await self.hw_regs.write_dword(MQNIC_CQ_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer
await self.hw_regs.write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, self.log_size | MQNIC_CQ_ACTIVE_MASK) # active, log size
self.active = True
self.enabled = True
async def close(self):
if not self.hw_regs:
return
async def deactivate(self):
await self.hw_regs.write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, self.log_size) # active, log size
await self.hw_regs.write_dword(MQNIC_CQ_INTERRUPT_INDEX_REG, 0) # event index
# TODO free buffer
self.eq.detach_cq(self)
self.eq = None
self.active = False
self.enabled = False
self.hw_regs = None
if self.is_txcq:
self.interface.tx_cq_res.free(self.cqn)
else:
self.interface.rx_cq_res.free(self.cqn)
self.cqn = None
async def read_head_ptr(self):
val = await self.hw_regs.read_dword(MQNIC_CQ_HEAD_PTR_REG)
@ -648,14 +656,14 @@ class Cq:
await self.hw_regs.write_dword(MQNIC_CQ_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask)
async def arm(self):
if not self.active:
if not self.hw_regs:
return
await self.hw_regs.write_dword(MQNIC_CQ_INTERRUPT_INDEX_REG, self.eq.eqn | MQNIC_CQ_ARM_MASK) # event index
class Txq:
def __init__(self, interface, index, hw_regs):
def __init__(self, interface):
self.interface = interface
self.log = interface.log
self.driver = interface.driver
@ -666,8 +674,8 @@ class Txq:
self.size_mask = 0
self.full_size = 0
self.stride = 0
self.index = index
self.active = False
self.index = None
self.enabled = False
self.buf_size = 0
self.buf_region = None
@ -685,26 +693,23 @@ class Txq:
self.bytes = 0
self.hw_ptr_mask = 0xffff
self.hw_regs = hw_regs
self.hw_regs = None
async def init(self):
self.log.info("Init TXQ %d (interface %d)", self.index, self.interface.index)
async def open(self, cq, size, desc_block_size):
if self.hw_regs:
raise Exception("Already open")
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0) # active, log size
self.index = self.interface.txq_res.alloc()
async def alloc(self, size, stride):
if self.active:
raise Exception("Cannot allocate active ring")
if self.buf:
raise Exception("Already allocated")
self.log.info("Open TXQ %d (interface %d)", self.index, self.interface.index)
self.log_queue_size = size.bit_length() - 1
self.log_desc_block_size = int(stride/MQNIC_DESC_SIZE).bit_length() - 1
self.log_desc_block_size = desc_block_size.bit_length() - 1
self.desc_block_size = 2**self.log_desc_block_size
self.size = 2**self.log_queue_size
self.size_mask = self.size-1
self.full_size = self.size >> 1
self.stride = stride
self.stride = MQNIC_DESC_SIZE*self.desc_block_size
self.tx_info = [None]*self.size
@ -716,53 +721,54 @@ class Txq:
self.head_ptr = 0
self.tail_ptr = 0
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0) # active, log size
await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address
await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address
await self.hw_regs.write_dword(MQNIC_QUEUE_CPL_QUEUE_INDEX_REG, 0) # completion queue index
await self.hw_regs.write_dword(MQNIC_QUEUE_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer
await self.hw_regs.write_dword(MQNIC_QUEUE_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8)) # active, log desc block size, log queue size
async def free(self):
await self.deactivate()
if self.buf:
# TODO
pass
async def activate(self, cq):
self.log.info("Activate TXQ %d (interface %d)", self.index, self.interface.index)
await self.deactivate()
self.cq = cq
self.cq.src_ring = self
self.cq.handler = Txq.process_tx_cq
self.head_ptr = 0
self.tail_ptr = 0
self.hw_regs = self.interface.txq_res.get_window(self.index)
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0) # active, log size
await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address
await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address
await self.hw_regs.write_dword(MQNIC_QUEUE_CPL_QUEUE_INDEX_REG, cq.cqn) # completion queue index
await self.hw_regs.write_dword(MQNIC_QUEUE_CPL_QUEUE_INDEX_REG, self.cq.cqn) # completion queue index
await self.hw_regs.write_dword(MQNIC_QUEUE_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer
await self.hw_regs.write_dword(MQNIC_QUEUE_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8) | MQNIC_QUEUE_ACTIVE_MASK) # active, log desc block size, log queue size
self.active = True
async def deactivate(self):
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8)) # active, log desc block size, log queue size
async def close(self):
if not self.hw_regs:
return
await self.disable()
# TODO free buffer
if self.cq:
self.cq.src_ring = None
self.cq.handler = None
self.cq = None
self.active = False
self.hw_regs = None
self.interface.txq_res.free(self.index)
self.index = None
async def enable(self):
if not self.hw_regs:
raise Exception("Not open")
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8) | MQNIC_QUEUE_ACTIVE_MASK) # active, log desc block size, log queue size
self.enabled = True
async def disable(self):
if not self.hw_regs:
return
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8)) # active, log desc block size, log queue size
self.enabled = False
def empty(self):
return self.head_ptr == self.tail_ptr
@ -840,7 +846,7 @@ class Txq:
class Rxq:
def __init__(self, interface, index, hw_regs):
def __init__(self, interface):
self.interface = interface
self.log = interface.log
self.driver = interface.driver
@ -851,8 +857,8 @@ class Rxq:
self.size_mask = 0
self.full_size = 0
self.stride = 0
self.index = index
self.active = False
self.index = None
self.enabled = False
self.buf_size = 0
self.buf_region = None
@ -868,26 +874,23 @@ class Rxq:
self.bytes = 0
self.hw_ptr_mask = 0xffff
self.hw_regs = hw_regs
self.hw_regs = None
async def init(self):
self.log.info("Init RXQ %d (interface %d)", self.index, self.interface.index)
async def open(self, cq, size, desc_block_size):
if self.hw_regs:
raise Exception("Already open")
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0) # active, log size
self.index = self.interface.rxq_res.alloc()
async def alloc(self, size, stride):
if self.active:
raise Exception("Cannot allocate active ring")
if self.buf:
raise Exception("Already allocated")
self.log.info("Open RXQ %d (interface %d)", self.index, self.interface.index)
self.log_queue_size = size.bit_length() - 1
self.log_desc_block_size = int(stride/MQNIC_DESC_SIZE).bit_length() - 1
self.log_desc_block_size = desc_block_size.bit_length() - 1
self.desc_block_size = 2**self.log_desc_block_size
self.size = 2**self.log_queue_size
self.size_mask = self.size-1
self.full_size = self.size >> 1
self.stride = stride
self.stride = MQNIC_DESC_SIZE*self.desc_block_size
self.rx_info = [None]*self.size
@ -899,47 +902,29 @@ class Rxq:
self.head_ptr = 0
self.tail_ptr = 0
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0) # active, log size
await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address
await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address
await self.hw_regs.write_dword(MQNIC_QUEUE_CPL_QUEUE_INDEX_REG, 0) # completion queue index
await self.hw_regs.write_dword(MQNIC_QUEUE_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer
await self.hw_regs.write_dword(MQNIC_QUEUE_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8)) # active, log desc block size, log queue size
async def free(self):
await self.deactivate()
if self.buf:
# TODO
pass
async def activate(self, cq):
self.log.info("Activate RXQ %d (interface %d)", self.index, self.interface.index)
await self.deactivate()
self.cq = cq
self.cq.src_ring = self
self.cq.handler = Rxq.process_rx_cq
self.head_ptr = 0
self.tail_ptr = 0
self.hw_regs = self.interface.rxq_res.get_window(self.index)
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0) # active, log size
await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address
await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address
await self.hw_regs.write_dword(MQNIC_QUEUE_CPL_QUEUE_INDEX_REG, cq.cqn) # completion queue index
await self.hw_regs.write_dword(MQNIC_QUEUE_CPL_QUEUE_INDEX_REG, self.cq.cqn) # completion queue index
await self.hw_regs.write_dword(MQNIC_QUEUE_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer
await self.hw_regs.write_dword(MQNIC_QUEUE_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8) | MQNIC_QUEUE_ACTIVE_MASK) # active, log desc block size, log queue size
self.active = True
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8)) # active, log desc block size, log queue size
await self.refill_buffers()
async def deactivate(self):
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8)) # active, log desc block size, log queue size
async def close(self):
if not self.hw_regs:
return
await self.disable()
# TODO free buffer
if self.cq:
self.cq.src_ring = None
@ -947,7 +932,26 @@ class Rxq:
self.cq = None
self.active = False
self.hw_regs = None
self.interface.rxq_res.free(self.index)
self.index = None
async def enable(self):
if not self.hw_regs:
raise Exception("Not open")
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8) | MQNIC_QUEUE_ACTIVE_MASK) # active, log desc block size, log queue size
self.enabled = True
async def disable(self):
if not self.hw_regs:
return
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8)) # active, log desc block size, log queue size
self.enabled = False
def empty(self):
return self.head_ptr == self.tail_ptr
@ -1326,40 +1330,24 @@ class Interface:
await self.set_rx_queue_map_app_mask(k, 0)
await self.set_rx_queue_map_indir_table(k, 0, 0)
self.eq = []
self.txq = []
self.tx_cq = []
self.rxq = []
self.rx_cq = []
self.ports = []
self.sched_blocks = []
# ensure all queues are disabled
for k in range(self.eq_res.get_count()):
eq = Eq(self, k, self.eq_res.get_window(k))
await eq.init()
self.eq.append(eq)
await self.eq_res.get_window(k).write_dword(MQNIC_EQ_ACTIVE_LOG_SIZE_REG, 0)
for k in range(self.txq_res.get_count()):
txq = Txq(self, k, self.txq_res.get_window(k))
await txq.init()
self.txq.append(txq)
await self.txq_res.get_window(k).write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0)
for k in range(self.tx_cq_res.get_count()):
cq = Cq(self, k, self.tx_cq_res.get_window(k))
await cq.init()
self.tx_cq.append(cq)
await self.tx_cq_res.get_window(k).write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, 0)
for k in range(self.rxq_res.get_count()):
rxq = Rxq(self, k, self.rxq_res.get_window(k))
await rxq.init()
self.rxq.append(rxq)
await self.rxq_res.get_window(k).write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0)
for k in range(self.rx_cq_res.get_count()):
cq = Cq(self, k, self.rx_cq_res.get_window(k))
await cq.init()
self.rx_cq.append(cq)
await self.rx_cq_res.get_window(k).write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, 0)
# create ports
self.ports = []
for k in range(self.port_count):
rb = self.reg_blocks.find(MQNIC_RB_PORT_TYPE, MQNIC_RB_PORT_VER, index=k)
@ -1367,6 +1355,8 @@ class Interface:
await p.init()
self.ports.append(p)
# create schedulers
self.sched_blocks = []
for k in range(self.sched_block_count):
rb = self.reg_blocks.find(MQNIC_RB_SCHED_BLOCK_TYPE, MQNIC_RB_SCHED_BLOCK_VER, index=k)
@ -1376,30 +1366,42 @@ class Interface:
assert self.sched_block_count == len(self.sched_blocks)
for eq in self.eq:
await eq.alloc(1024, MQNIC_EVENT_SIZE)
await eq.activate(self.index) # TODO?
# create EQs
self.eq = []
for k in range(self.eq_res.get_count()):
eq = Eq(self)
await eq.open(self.index, 1024)
self.eq.append(eq)
await eq.arm()
self.txq = []
self.tx_cq = []
self.rxq = []
self.rx_cq = []
# wait for all writes to complete
await self.hw_regs.read_dword(0)
async def open(self):
for rxq in self.rxq:
cq = self.rx_cq[rxq.index]
await cq.alloc(1024, MQNIC_CPL_SIZE)
await cq.activate(self.eq[cq.cqn % len(self.eq)])
for k in range(self.rxq_res.get_count()):
cq = Cq(self)
await cq.open(self.eq[k % len(self.eq)], 1024, is_txcq=False)
self.rx_cq.append(cq)
await cq.arm()
await rxq.alloc(1024, MQNIC_DESC_SIZE*4)
await rxq.activate(cq)
rxq = Rxq(self)
await rxq.open(cq, 1024, 4)
await rxq.enable()
self.rxq.append(rxq)
for txq in self.txq:
cq = self.tx_cq[txq.index]
await cq.alloc(1024, MQNIC_CPL_SIZE)
await cq.activate(self.eq[cq.cqn % len(self.eq)])
for k in range(self.txq_res.get_count()):
cq = Cq(self)
await cq.open(self.eq[k % len(self.eq)], 1024, is_txcq=True)
self.tx_cq.append(cq)
await cq.arm()
await txq.alloc(1024, MQNIC_DESC_SIZE*4)
await txq.activate(cq)
txq = Txq(self)
await txq.open(cq, 1024, 4)
await txq.enable()
self.txq.append(txq)
# wait for all writes to complete
await self.hw_regs.read_dword(0)
@ -1409,22 +1411,26 @@ class Interface:
async def close(self):
self.port_up = False
for txq in self.txq:
await txq.deactivate()
await txq.cq.deactivate()
for q in self.txq:
q.disable()
for rxq in self.rxq:
await rxq.deactivate()
await rxq.cq.deactivate()
for q in self.rxq:
q.disable()
# wait for all writes to complete
await self.hw_regs.read_dword(0)
for q in self.txq:
cq = q.cq
await q.free_buf()
await q.close()
await cq.close()
for q in self.rxq:
cq = q.cq
await q.free_buf()
await q.close()
await cq.close()
async def start_xmit(self, skb, tx_ring=None, csum_start=None, csum_offset=None):
if not self.port_up:

91
modules/mqnic/mqnic.h
View File

@ -83,6 +83,9 @@ struct mqnic_res {
unsigned int count;
u8 __iomem *base;
unsigned int stride;
spinlock_t lock;
unsigned long *bmap;
};
struct mqnic_reg_block {
@ -288,7 +291,7 @@ struct mqnic_ring {
struct mqnic_priv *priv;
int index;
struct mqnic_cq *cq;
int active;
int enabled;
u32 hw_ptr_mask;
u8 __iomem *hw_addr;
@ -313,9 +316,10 @@ struct mqnic_cq {
struct mqnic_if *interface;
struct napi_struct napi;
int cqn;
int is_txcq;
struct mqnic_eq *eq;
struct mqnic_ring *src_ring;
int active;
int enabled;
void (*handler)(struct mqnic_cq *cq);
@ -342,12 +346,15 @@ struct mqnic_eq {
struct mqnic_if *interface;
int eqn;
struct mqnic_irq *irq;
int active;
int enabled;
struct notifier_block irq_nb;
void (*handler)(struct mqnic_eq *eq);
spinlock_t table_lock;
struct radix_tree_root cq_table;
u32 hw_ptr_mask;
u8 __iomem *hw_addr;
u8 __iomem *hw_head_ptr;
@ -393,8 +400,6 @@ struct mqnic_sched_block {
int index;
u32 txq_count;
u32 sched_count;
struct mqnic_sched *sched[MQNIC_MAX_PORTS];
};
@ -432,18 +437,6 @@ struct mqnic_if {
u32 eq_count;
struct mqnic_eq *eq[MQNIC_MAX_EQ];
u32 txq_count;
struct mqnic_ring *txq[MQNIC_MAX_TXQ];
u32 tx_cq_count;
struct mqnic_cq *tx_cq[MQNIC_MAX_TX_CQ];
u32 rxq_count;
struct mqnic_ring *rxq[MQNIC_MAX_RXQ];
u32 rx_cq_count;
struct mqnic_cq *rx_cq[MQNIC_MAX_RX_CQ];
u32 port_count;
struct mqnic_port *port[MQNIC_MAX_PORTS];
@ -482,20 +475,14 @@ struct mqnic_priv {
unsigned int link_status;
struct timer_list link_status_timer;
u32 eq_count;
struct mqnic_eq *eq[MQNIC_MAX_EQ];
u32 txq_count;
struct mqnic_ring *txq[MQNIC_MAX_TXQ];
u32 tx_cq_count;
struct mqnic_cq *tx_cq[MQNIC_MAX_TX_CQ];
u32 rxq_count;
struct mqnic_ring *rxq[MQNIC_MAX_RXQ];
u32 rx_cq_count;
struct mqnic_cq *rx_cq[MQNIC_MAX_RX_CQ];
struct rw_semaphore txq_table_sem;
struct radix_tree_root txq_table;
struct rw_semaphore rxq_table_sem;
struct radix_tree_root rxq_table;
u32 sched_block_count;
struct mqnic_sched_block *sched_block[MQNIC_MAX_PORTS];
@ -512,6 +499,8 @@ struct mqnic_priv {
// mqnic_res.c
struct mqnic_res *mqnic_create_res(unsigned int count, u8 __iomem *base, unsigned int stride);
void mqnic_destroy_res(struct mqnic_res *res);
int mqnic_res_alloc(struct mqnic_res *res);
void mqnic_res_free(struct mqnic_res *res, int index);
unsigned int mqnic_res_get_count(struct mqnic_res *res);
u8 __iomem *mqnic_res_get_addr(struct mqnic_res *res, int index);
@ -602,39 +591,34 @@ void mqnic_stats_init(struct mqnic_dev *mdev);
u64 mqnic_stats_read(struct mqnic_dev *mdev, int index);
// mqnic_eq.c
struct mqnic_eq *mqnic_create_eq(struct mqnic_if *interface,
int eqn, u8 __iomem *hw_addr);
struct mqnic_eq *mqnic_create_eq(struct mqnic_if *interface);
void mqnic_destroy_eq(struct mqnic_eq *eq);
int mqnic_alloc_eq(struct mqnic_eq *eq, int size, int stride);
void mqnic_free_eq(struct mqnic_eq *eq);
int mqnic_activate_eq(struct mqnic_eq *eq, struct mqnic_irq *irq);
void mqnic_deactivate_eq(struct mqnic_eq *eq);
int mqnic_open_eq(struct mqnic_eq *eq, struct mqnic_irq *irq, int size);
void mqnic_close_eq(struct mqnic_eq *eq);
int mqnic_eq_attach_cq(struct mqnic_eq *eq, struct mqnic_cq *cq);
void mqnic_eq_detach_cq(struct mqnic_eq *eq, struct mqnic_cq *cq);
void mqnic_eq_read_head_ptr(struct mqnic_eq *eq);
void mqnic_eq_write_tail_ptr(struct mqnic_eq *eq);
void mqnic_arm_eq(struct mqnic_eq *eq);
void mqnic_process_eq(struct mqnic_eq *eq);
// mqnic_cq.c
struct mqnic_cq *mqnic_create_cq(struct mqnic_if *interface,
int cqn, u8 __iomem *hw_addr);
struct mqnic_cq *mqnic_create_cq(struct mqnic_if *interface);
void mqnic_destroy_cq(struct mqnic_cq *cq);
int mqnic_alloc_cq(struct mqnic_cq *cq, int size, int stride);
void mqnic_free_cq(struct mqnic_cq *cq);
int mqnic_activate_cq(struct mqnic_cq *cq, struct mqnic_eq *eq);
void mqnic_deactivate_cq(struct mqnic_cq *cq);
int mqnic_open_cq(struct mqnic_cq *cq, struct mqnic_eq *eq, int size, int is_txcq);
void mqnic_close_cq(struct mqnic_cq *cq);
void mqnic_cq_read_head_ptr(struct mqnic_cq *cq);
void mqnic_cq_write_tail_ptr(struct mqnic_cq *cq);
void mqnic_arm_cq(struct mqnic_cq *cq);
// mqnic_tx.c
struct mqnic_ring *mqnic_create_tx_ring(struct mqnic_if *interface,
int index, u8 __iomem *hw_addr);
struct mqnic_ring *mqnic_create_tx_ring(struct mqnic_if *interface);
void mqnic_destroy_tx_ring(struct mqnic_ring *ring);
int mqnic_alloc_tx_ring(struct mqnic_ring *ring, int size, int stride);
void mqnic_free_tx_ring(struct mqnic_ring *ring);
int mqnic_activate_tx_ring(struct mqnic_ring *ring, struct mqnic_priv *priv,
struct mqnic_cq *cq);
void mqnic_deactivate_tx_ring(struct mqnic_ring *ring);
int mqnic_open_tx_ring(struct mqnic_ring *ring, struct mqnic_priv *priv,
struct mqnic_cq *cq, int size, int desc_block_size);
void mqnic_close_tx_ring(struct mqnic_ring *ring);
int mqnic_enable_tx_ring(struct mqnic_ring *ring);
void mqnic_disable_tx_ring(struct mqnic_ring *ring);
bool mqnic_is_tx_ring_empty(const struct mqnic_ring *ring);
bool mqnic_is_tx_ring_full(const struct mqnic_ring *ring);
void mqnic_tx_read_tail_ptr(struct mqnic_ring *ring);
@ -647,14 +631,13 @@ int mqnic_poll_tx_cq(struct napi_struct *napi, int budget);
netdev_tx_t mqnic_start_xmit(struct sk_buff *skb, struct net_device *dev);
// mqnic_rx.c
struct mqnic_ring *mqnic_create_rx_ring(struct mqnic_if *interface,
int index, u8 __iomem *hw_addr);
struct mqnic_ring *mqnic_create_rx_ring(struct mqnic_if *interface);
void mqnic_destroy_rx_ring(struct mqnic_ring *ring);
int mqnic_alloc_rx_ring(struct mqnic_ring *ring, int size, int stride);
void mqnic_free_rx_ring(struct mqnic_ring *ring);
int mqnic_activate_rx_ring(struct mqnic_ring *ring, struct mqnic_priv *priv,
struct mqnic_cq *cq);
void mqnic_deactivate_rx_ring(struct mqnic_ring *ring);
int mqnic_open_rx_ring(struct mqnic_ring *ring, struct mqnic_priv *priv,
struct mqnic_cq *cq, int size, int desc_block_size);
void mqnic_close_rx_ring(struct mqnic_ring *ring);
int mqnic_enable_rx_ring(struct mqnic_ring *ring);
void mqnic_disable_rx_ring(struct mqnic_ring *ring);
bool mqnic_is_rx_ring_empty(const struct mqnic_ring *ring);
bool mqnic_is_rx_ring_full(const struct mqnic_ring *ring);
void mqnic_rx_read_tail_ptr(struct mqnic_ring *ring);

135
modules/mqnic/mqnic_cq.c
View File

@ -35,8 +35,7 @@
#include "mqnic.h"
struct mqnic_cq *mqnic_create_cq(struct mqnic_if *interface,
int cqn, u8 __iomem *hw_addr)
struct mqnic_cq *mqnic_create_cq(struct mqnic_if *interface)
{
struct mqnic_cq *cq;
@ -47,82 +46,63 @@ struct mqnic_cq *mqnic_create_cq(struct mqnic_if *interface,
cq->dev = interface->dev;
cq->interface = interface;
cq->cqn = cqn;
cq->active = 0;
cq->cqn = -1;
cq->enabled = 0;
cq->hw_addr = hw_addr;
cq->hw_addr = NULL;
cq->hw_ptr_mask = 0xffff;
cq->hw_head_ptr = hw_addr + MQNIC_CQ_HEAD_PTR_REG;
cq->hw_tail_ptr = hw_addr + MQNIC_CQ_TAIL_PTR_REG;
cq->hw_head_ptr = NULL;
cq->hw_tail_ptr = NULL;
cq->head_ptr = 0;
cq->tail_ptr = 0;
// deactivate queue
iowrite32(0, cq->hw_addr + MQNIC_CQ_ACTIVE_LOG_SIZE_REG);
return cq;
}
void mqnic_destroy_cq(struct mqnic_cq *cq)
{
mqnic_free_cq(cq);
mqnic_close_cq(cq);
kfree(cq);
}
int mqnic_alloc_cq(struct mqnic_cq *cq, int size, int stride)
int mqnic_open_cq(struct mqnic_cq *cq, struct mqnic_eq *eq, int size, int is_txcq)
{
if (cq->active || cq->buf)
int ret;
if (cq->enabled || cq->hw_addr || cq->buf || !eq)
return -EINVAL;
cq->is_txcq = is_txcq;
if (is_txcq) {
cq->cqn = mqnic_res_alloc(cq->interface->tx_cq_res);
} else {
cq->cqn = mqnic_res_alloc(cq->interface->rx_cq_res);
}
if (cq->cqn < 0)
return -ENOMEM;
cq->size = roundup_pow_of_two(size);
cq->size_mask = cq->size - 1;
cq->stride = roundup_pow_of_two(stride);
cq->stride = roundup_pow_of_two(MQNIC_CPL_SIZE);
cq->buf_size = cq->size * cq->stride;
cq->buf = dma_alloc_coherent(cq->dev, cq->buf_size, &cq->buf_dma_addr, GFP_KERNEL);
if (!cq->buf)
return -ENOMEM;
cq->head_ptr = 0;
cq->tail_ptr = 0;
// deactivate queue
iowrite32(0, cq->hw_addr + MQNIC_CQ_ACTIVE_LOG_SIZE_REG);
// set base address
iowrite32(cq->buf_dma_addr, cq->hw_addr + MQNIC_CQ_BASE_ADDR_REG + 0);
iowrite32(cq->buf_dma_addr >> 32, cq->hw_addr + MQNIC_CQ_BASE_ADDR_REG + 4);
// set interrupt index
iowrite32(0, cq->hw_addr + MQNIC_CQ_INTERRUPT_INDEX_REG);
// set pointers
iowrite32(cq->head_ptr & cq->hw_ptr_mask, cq->hw_addr + MQNIC_CQ_HEAD_PTR_REG);
iowrite32(cq->tail_ptr & cq->hw_ptr_mask, cq->hw_addr + MQNIC_CQ_TAIL_PTR_REG);
// set size
iowrite32(ilog2(cq->size), cq->hw_addr + MQNIC_CQ_ACTIVE_LOG_SIZE_REG);
return 0;
}
void mqnic_free_cq(struct mqnic_cq *cq)
{
mqnic_deactivate_cq(cq);
if (cq->buf) {
dma_free_coherent(cq->dev, cq->buf_size, cq->buf, cq->buf_dma_addr);
cq->buf = NULL;
cq->buf_dma_addr = 0;
if (!cq->buf) {
ret = -ENOMEM;
goto fail;
}
}
int mqnic_activate_cq(struct mqnic_cq *cq, struct mqnic_eq *eq)
{
mqnic_deactivate_cq(cq);
if (!cq->buf || !eq)
return -EINVAL;
cq->eq = eq;
mqnic_eq_attach_cq(eq, cq);
if (is_txcq)
cq->hw_addr = mqnic_res_get_addr(cq->interface->tx_cq_res, cq->cqn);
else
cq->hw_addr = mqnic_res_get_addr(cq->interface->rx_cq_res, cq->cqn);
cq->hw_head_ptr = cq->hw_addr + MQNIC_CQ_HEAD_PTR_REG;
cq->hw_tail_ptr = cq->hw_addr + MQNIC_CQ_TAIL_PTR_REG;
cq->head_ptr = 0;
cq->tail_ptr = 0;
@ -134,30 +114,55 @@ int mqnic_activate_cq(struct mqnic_cq *cq, struct mqnic_eq *eq)
// set base address
iowrite32(cq->buf_dma_addr, cq->hw_addr + MQNIC_CQ_BASE_ADDR_REG + 0);
iowrite32(cq->buf_dma_addr >> 32, cq->hw_addr + MQNIC_CQ_BASE_ADDR_REG + 4);
// set interrupt index
// set EQN
iowrite32(cq->eq->eqn, cq->hw_addr + MQNIC_CQ_INTERRUPT_INDEX_REG);
// set pointers
iowrite32(cq->head_ptr & cq->hw_ptr_mask, cq->hw_addr + MQNIC_CQ_HEAD_PTR_REG);
iowrite32(cq->tail_ptr & cq->hw_ptr_mask, cq->hw_addr + MQNIC_CQ_TAIL_PTR_REG);
// set size and activate queue
iowrite32(ilog2(cq->size) | MQNIC_CQ_ACTIVE_MASK,
cq->hw_addr + MQNIC_CQ_ACTIVE_LOG_SIZE_REG);
// set size
iowrite32(ilog2(cq->size) | MQNIC_CQ_ACTIVE_MASK, cq->hw_addr + MQNIC_CQ_ACTIVE_LOG_SIZE_REG);
cq->active = 1;
cq->enabled = 1;
return 0;
fail:
mqnic_close_eq(eq);
return ret;
}
void mqnic_deactivate_cq(struct mqnic_cq *cq)
void mqnic_close_cq(struct mqnic_cq *cq)
{
// deactivate queue
iowrite32(ilog2(cq->size), cq->hw_addr + MQNIC_CQ_ACTIVE_LOG_SIZE_REG);
// disarm queue
iowrite32(0, cq->hw_addr + MQNIC_CQ_INTERRUPT_INDEX_REG);
if (cq->hw_addr) {
// deactivate queue
iowrite32(ilog2(cq->size), cq->hw_addr + MQNIC_CQ_ACTIVE_LOG_SIZE_REG);
// disarm queue
iowrite32(0, cq->hw_addr + MQNIC_CQ_INTERRUPT_INDEX_REG);
}
cq->eq = NULL;
if (cq->eq) {
mqnic_eq_detach_cq(cq->eq, cq);
cq->eq = NULL;
}
cq->active = 0;
cq->hw_addr = NULL;
cq->hw_head_ptr = NULL;
cq->hw_tail_ptr = NULL;
if (cq->buf) {
dma_free_coherent(cq->dev, cq->buf_size, cq->buf, cq->buf_dma_addr);
cq->buf = NULL;
cq->buf_dma_addr = 0;
}
if (cq->is_txcq) {
mqnic_res_free(cq->interface->tx_cq_res, cq->cqn);
} else {
mqnic_res_free(cq->interface->rx_cq_res, cq->cqn);
}
cq->cqn = -1;
cq->enabled = 0;
}
void mqnic_cq_read_head_ptr(struct mqnic_cq *cq)
@ -172,7 +177,7 @@ void mqnic_cq_write_tail_ptr(struct mqnic_cq *cq)
void mqnic_arm_cq(struct mqnic_cq *cq)
{
if (!cq->active)
if (!cq->enabled)
return;
iowrite32(cq->eq->eqn | MQNIC_CQ_ARM_MASK,

211
modules/mqnic/mqnic_eq.c
View File

@ -45,8 +45,7 @@ static int mqnic_eq_int(struct notifier_block *nb, unsigned long action, void *d
return NOTIFY_DONE;
}
struct mqnic_eq *mqnic_create_eq(struct mqnic_if *interface,
int eqn, u8 __iomem *hw_addr)
struct mqnic_eq *mqnic_create_eq(struct mqnic_if *interface)
{
struct mqnic_eq *eq;
@ -57,92 +56,66 @@ struct mqnic_eq *mqnic_create_eq(struct mqnic_if *interface,
eq->dev = interface->dev;
eq->interface = interface;
eq->eqn = eqn;
eq->active = 0;
eq->eqn = -1;
eq->enabled = 0;
eq->irq_nb.notifier_call = mqnic_eq_int;
eq->hw_addr = hw_addr;
eq->hw_addr = NULL;
eq->hw_ptr_mask = 0xffff;
eq->hw_head_ptr = hw_addr + MQNIC_EQ_HEAD_PTR_REG;
eq->hw_tail_ptr = hw_addr + MQNIC_EQ_TAIL_PTR_REG;
eq->hw_head_ptr = NULL;
eq->hw_tail_ptr = NULL;
eq->head_ptr = 0;
eq->tail_ptr = 0;
// deactivate queue
iowrite32(0, eq->hw_addr + MQNIC_EQ_ACTIVE_LOG_SIZE_REG);
spin_lock_init(&eq->table_lock);
INIT_RADIX_TREE(&eq->cq_table, GFP_KERNEL);
return eq;
}
void mqnic_destroy_eq(struct mqnic_eq *eq)
{
mqnic_free_eq(eq);
mqnic_close_eq(eq);
kfree(eq);
}
int mqnic_alloc_eq(struct mqnic_eq *eq, int size, int stride)
int mqnic_open_eq(struct mqnic_eq *eq, struct mqnic_irq *irq, int size)
{
if (eq->active || eq->buf)
int ret;
if (eq->enabled || eq->hw_addr || eq->buf || !irq)
return -EINVAL;
eq->eqn = mqnic_res_alloc(eq->interface->eq_res);
if (eq->eqn < 0)
return -ENOMEM;
eq->size = roundup_pow_of_two(size);
eq->size_mask = eq->size - 1;
eq->stride = roundup_pow_of_two(stride);
eq->stride = roundup_pow_of_two(MQNIC_EVENT_SIZE);
eq->buf_size = eq->size * eq->stride;
eq->buf = dma_alloc_coherent(eq->dev, eq->buf_size, &eq->buf_dma_addr, GFP_KERNEL);
if (!eq->buf)
return -ENOMEM;
eq->head_ptr = 0;
eq->tail_ptr = 0;
// deactivate queue
iowrite32(0, eq->hw_addr + MQNIC_EQ_ACTIVE_LOG_SIZE_REG);
// set base address
iowrite32(eq->buf_dma_addr, eq->hw_addr + MQNIC_EQ_BASE_ADDR_REG + 0);
iowrite32(eq->buf_dma_addr >> 32, eq->hw_addr + MQNIC_EQ_BASE_ADDR_REG + 4);
// set interrupt index
iowrite32(0, eq->hw_addr + MQNIC_EQ_INTERRUPT_INDEX_REG);
// set pointers
iowrite32(eq->head_ptr & eq->hw_ptr_mask, eq->hw_addr + MQNIC_EQ_HEAD_PTR_REG);
iowrite32(eq->tail_ptr & eq->hw_ptr_mask, eq->hw_addr + MQNIC_EQ_TAIL_PTR_REG);
// set size
iowrite32(ilog2(eq->size), eq->hw_addr + MQNIC_EQ_ACTIVE_LOG_SIZE_REG);
return 0;
}
void mqnic_free_eq(struct mqnic_eq *eq)
{
mqnic_deactivate_eq(eq);
if (eq->buf) {
dma_free_coherent(eq->dev, eq->buf_size, eq->buf, eq->buf_dma_addr);
eq->buf = NULL;
eq->buf_dma_addr = 0;
if (!eq->buf) {
ret = -ENOMEM;
goto fail;
}
}
int mqnic_activate_eq(struct mqnic_eq *eq, struct mqnic_irq *irq)
{
int ret = 0;
mqnic_deactivate_eq(eq);
if (!eq->buf || !irq)
return -EINVAL;
// register interrupt
ret = atomic_notifier_chain_register(&irq->nh, &eq->irq_nb);
if (ret)
return ret;
goto fail;
eq->irq = irq;
eq->hw_addr = mqnic_res_get_addr(eq->interface->eq_res, eq->eqn);
eq->hw_head_ptr = eq->hw_addr + MQNIC_EQ_HEAD_PTR_REG;
eq->hw_tail_ptr = eq->hw_addr + MQNIC_EQ_TAIL_PTR_REG;
eq->head_ptr = 0;
eq->tail_ptr = 0;
@ -158,23 +131,30 @@ int mqnic_activate_eq(struct mqnic_eq *eq, struct mqnic_irq *irq)
// set pointers
iowrite32(eq->head_ptr & eq->hw_ptr_mask, eq->hw_addr + MQNIC_EQ_HEAD_PTR_REG);
iowrite32(eq->tail_ptr & eq->hw_ptr_mask, eq->hw_addr + MQNIC_EQ_TAIL_PTR_REG);
// set size
iowrite32(ilog2(eq->size), eq->hw_addr + MQNIC_EQ_ACTIVE_LOG_SIZE_REG);
// set size and activate queue
iowrite32(ilog2(eq->size) | MQNIC_EQ_ACTIVE_MASK,
eq->hw_addr + MQNIC_EQ_ACTIVE_LOG_SIZE_REG);
iowrite32(ilog2(eq->size) | MQNIC_EQ_ACTIVE_MASK, eq->hw_addr + MQNIC_EQ_ACTIVE_LOG_SIZE_REG);
eq->active = 1;
eq->enabled = 1;
return 0;
fail:
mqnic_close_eq(eq);
return ret;
}
void mqnic_deactivate_eq(struct mqnic_eq *eq)
void mqnic_close_eq(struct mqnic_eq *eq)
{
int ret = 0;
int ret;
// deactivate queue
iowrite32(ilog2(eq->size), eq->hw_addr + MQNIC_EQ_ACTIVE_LOG_SIZE_REG);
// disarm queue
iowrite32(0, eq->hw_addr + MQNIC_EQ_INTERRUPT_INDEX_REG);
if (eq->hw_addr) {
// deactivate queue
iowrite32(ilog2(eq->size), eq->hw_addr + MQNIC_EQ_ACTIVE_LOG_SIZE_REG);
// disarm queue
iowrite32(0, eq->hw_addr + MQNIC_EQ_INTERRUPT_INDEX_REG);
}
// unregister interrupt
if (eq->irq)
@ -182,7 +162,56 @@ void mqnic_deactivate_eq(struct mqnic_eq *eq)
eq->irq = NULL;
eq->active = 0;
eq->hw_addr = NULL;
eq->hw_head_ptr = NULL;
eq->hw_tail_ptr = NULL;
if (eq->buf) {
dma_free_coherent(eq->dev, eq->buf_size, eq->buf, eq->buf_dma_addr);
eq->buf = NULL;
eq->buf_dma_addr = 0;
}
mqnic_res_free(eq->interface->eq_res, eq->eqn);
eq->eqn = -1;
eq->enabled = 0;
}
int mqnic_eq_attach_cq(struct mqnic_eq *eq, struct mqnic_cq *cq)
{
int ret;
int cqn = cq->cqn;
if (cq->is_txcq)
cqn |= 0x80000000;
spin_lock_irq(&eq->table_lock);
ret = radix_tree_insert(&eq->cq_table, cqn, cq);
spin_unlock_irq(&eq->table_lock);
return ret;
}
void mqnic_eq_detach_cq(struct mqnic_eq *eq, struct mqnic_cq *cq)
{
struct mqnic_cq *item;
int cqn = cq->cqn;
if (cq->is_txcq)
cqn |= 0x80000000;
spin_lock_irq(&eq->table_lock);
item = radix_tree_delete(&eq->cq_table, cqn);
spin_unlock_irq(&eq->table_lock);
if (IS_ERR(item)) {
dev_err(eq->dev, "%s on IF %d EQ %d: radix_tree_delete failed: %ld",
__func__, eq->interface->index, eq->eqn, PTR_ERR(item));
} else if (!item) {
dev_err(eq->dev, "%s on IF %d EQ %d: CQ %d not in table",
__func__, eq->interface->index, eq->eqn, cqn);
} else if (item != cq) {
dev_err(eq->dev, "%s on IF %d EQ %d: entry mismatch when removing CQ %d",
__func__, eq->interface->index, eq->eqn, cqn);
}
}
void mqnic_eq_read_head_ptr(struct mqnic_eq *eq)
@ -197,11 +226,10 @@ void mqnic_eq_write_tail_ptr(struct mqnic_eq *eq)
void mqnic_arm_eq(struct mqnic_eq *eq)
{
if (!eq->active)
if (!eq->enabled)
return;
iowrite32(eq->irq->index | MQNIC_EQ_ARM_MASK,
eq->hw_addr + MQNIC_EQ_INTERRUPT_INDEX_REG);
iowrite32(eq->irq->index | MQNIC_EQ_ARM_MASK, eq->hw_addr + MQNIC_EQ_INTERRUPT_INDEX_REG);
}
void mqnic_process_eq(struct mqnic_eq *eq)
@ -212,6 +240,7 @@ void mqnic_process_eq(struct mqnic_eq *eq)
u32 eq_index;
u32 eq_tail_ptr;
int done = 0;
int cqn;
// read head pointer from NIC
eq_tail_ptr = eq->tail_ptr;
@ -227,34 +256,44 @@ void mqnic_process_eq(struct mqnic_eq *eq)
if (event->type == MQNIC_EVENT_TYPE_TX_CPL) {
// transmit completion event
if (unlikely(le16_to_cpu(event->source) > interface->tx_cq_count)) {
dev_err(eq->dev, "%s on port %d: unknown event source %d (index %d, type %d)",
__func__, interface->index, le16_to_cpu(event->source), eq_index,
le16_to_cpu(event->type));
cqn = le16_to_cpu(event->source) | 0x80000000;
rcu_read_lock();
cq = radix_tree_lookup(&eq->cq_table, cqn);
rcu_read_unlock();
if (likely(cq)) {
if (likely(cq->handler))
cq->handler(cq);
} else {
dev_err(eq->dev, "%s on IF %d EQ %d: unknown event source %d (index %d, type %d)",
__func__, interface->index, eq->eqn, le16_to_cpu(event->source),
eq_index, le16_to_cpu(event->type));
print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 1,
event, MQNIC_EVENT_SIZE, true);
} else {
cq = interface->tx_cq[le16_to_cpu(event->source)];
if (likely(cq && cq->handler))
cq->handler(cq);
}
} else if (le16_to_cpu(event->type) == MQNIC_EVENT_TYPE_RX_CPL) {
// receive completion event
if (unlikely(le16_to_cpu(event->source) > interface->rx_cq_count)) {
dev_err(eq->dev, "%s on port %d: unknown event source %d (index %d, type %d)",
__func__, interface->index, le16_to_cpu(event->source), eq_index,
le16_to_cpu(event->type));
cqn = le16_to_cpu(event->source);
rcu_read_lock();
cq = radix_tree_lookup(&eq->cq_table, cqn);
rcu_read_unlock();
if (likely(cq)) {
if (likely(cq->handler))
cq->handler(cq);
} else {
dev_err(eq->dev, "%s on IF %d EQ %d: unknown event source %d (index %d, type %d)",
__func__, interface->index, eq->eqn, le16_to_cpu(event->source),
eq_index, le16_to_cpu(event->type));
print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 1,
event, MQNIC_EVENT_SIZE, true);
} else {
cq = interface->rx_cq[le16_to_cpu(event->source)];
if (likely(cq && cq->handler))
cq->handler(cq);
}
} else {
dev_err(eq->dev, "%s on port %d: unknown event type %d (index %d, source %d)",
__func__, interface->index, le16_to_cpu(event->type), eq_index,
le16_to_cpu(event->source));
dev_err(eq->dev, "%s on IF %d EQ %d: unknown event type %d (index %d, source %d)",
__func__, interface->index, eq->eqn, le16_to_cpu(event->type),
eq_index, le16_to_cpu(event->source));
print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 1,
event, MQNIC_EVENT_SIZE, true);
}

119
modules/mqnic/mqnic_if.c
View File

@ -250,70 +250,21 @@ struct mqnic_if *mqnic_create_interface(struct mqnic_dev *mdev, int index, u8 __
desc_block_size = min_t(u32, interface->max_desc_block_size, 4);
// create rings
interface->eq_count = mqnic_res_get_count(interface->eq_res);
for (k = 0; k < interface->eq_count; k++) {
struct mqnic_eq *eq = mqnic_create_eq(interface, k,
mqnic_res_get_addr(interface->eq_res, k));
if (IS_ERR_OR_NULL(eq)) {
ret = PTR_ERR(eq);
goto fail;
}
// disable queues
for (k = 0; k < mqnic_res_get_count(interface->eq_res); k++)
iowrite32(0, mqnic_res_get_addr(interface->eq_res, k) + MQNIC_EQ_ACTIVE_LOG_SIZE_REG);
interface->eq[k] = eq;
for (k = 0; k < mqnic_res_get_count(interface->txq_res); k++)
iowrite32(0, mqnic_res_get_addr(interface->txq_res, k) + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
ret = mqnic_alloc_eq(eq, mqnic_num_eq_entries,
MQNIC_EVENT_SIZE);
if (ret)
goto fail;
for (k = 0; k < mqnic_res_get_count(interface->tx_cq_res); k++)
iowrite32(0, mqnic_res_get_addr(interface->tx_cq_res, k) + MQNIC_CQ_ACTIVE_LOG_SIZE_REG);
mqnic_activate_eq(eq, mdev->irq[k % mdev->irq_count]);
mqnic_arm_eq(eq);
}
for (k = 0; k < mqnic_res_get_count(interface->rxq_res); k++)
iowrite32(0, mqnic_res_get_addr(interface->rxq_res, k) + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
interface->txq_count = mqnic_res_get_count(interface->txq_res);
for (k = 0; k < interface->txq_count; k++) {
struct mqnic_ring *txq = mqnic_create_tx_ring(interface, k,
mqnic_res_get_addr(interface->txq_res, k));
if (IS_ERR_OR_NULL(txq)) {
ret = PTR_ERR(txq);
goto fail;
}
interface->txq[k] = txq;
}
interface->tx_cq_count = mqnic_res_get_count(interface->tx_cq_res);
for (k = 0; k < interface->tx_cq_count; k++) {
struct mqnic_cq *cq = mqnic_create_cq(interface, k,
mqnic_res_get_addr(interface->tx_cq_res, k));
if (IS_ERR_OR_NULL(cq)) {
ret = PTR_ERR(cq);
goto fail;
}
interface->tx_cq[k] = cq;
}
interface->rxq_count = mqnic_res_get_count(interface->rxq_res);
for (k = 0; k < interface->rxq_count; k++) {
struct mqnic_ring *rxq = mqnic_create_rx_ring(interface, k,
mqnic_res_get_addr(interface->rxq_res, k));
if (IS_ERR_OR_NULL(rxq)) {
ret = PTR_ERR(rxq);
goto fail;
}
interface->rxq[k] = rxq;
}
interface->rx_cq_count = mqnic_res_get_count(interface->rx_cq_res);
for (k = 0; k < interface->rx_cq_count; k++) {
struct mqnic_cq *cq = mqnic_create_cq(interface, k,
mqnic_res_get_addr(interface->rx_cq_res, k));
if (IS_ERR_OR_NULL(cq)) {
ret = PTR_ERR(cq);
goto fail;
}
interface->rx_cq[k] = cq;
}
for (k = 0; k < mqnic_res_get_count(interface->rx_cq_res); k++)
iowrite32(0, mqnic_res_get_addr(interface->rx_cq_res, k) + MQNIC_CQ_ACTIVE_LOG_SIZE_REG);
// create ports
for (k = 0; k < interface->port_count; k++) {
@ -353,6 +304,24 @@ struct mqnic_if *mqnic_create_interface(struct mqnic_dev *mdev, int index, u8 __
interface->sched_block[k] = sched_block;
}
// create EQs
interface->eq_count = mqnic_res_get_count(interface->eq_res);
for (k = 0; k < interface->eq_count; k++) {
struct mqnic_eq *eq = mqnic_create_eq(interface);
if (IS_ERR_OR_NULL(eq)) {
ret = PTR_ERR(eq);
goto fail;
}
interface->eq[k] = eq;
ret = mqnic_open_eq(eq, mdev->irq[k % mdev->irq_count], mqnic_num_eq_entries);
if (ret)
goto fail;
mqnic_arm_eq(eq);
}
// create net_devices
interface->dev_port_base = mdev->dev_port_max;
interface->dev_port_max = mdev->dev_port_max;
@ -386,7 +355,7 @@ void mqnic_destroy_interface(struct mqnic_if *interface)
}
}
// free rings
// free EQs
for (k = 0; k < ARRAY_SIZE(interface->eq); k++) {
if (interface->eq[k]) {
mqnic_destroy_eq(interface->eq[k]);
@ -394,34 +363,6 @@ void mqnic_destroy_interface(struct mqnic_if *interface)
}
}
for (k = 0; k < ARRAY_SIZE(interface->txq); k++) {
if (interface->txq[k]) {
mqnic_destroy_tx_ring(interface->txq[k]);
interface->txq[k] = NULL;
}
}
for (k = 0; k < ARRAY_SIZE(interface->tx_cq); k++) {
if (interface->tx_cq[k]) {
mqnic_destroy_cq(interface->tx_cq[k]);
interface->tx_cq[k] = NULL;
}
}
for (k = 0; k < ARRAY_SIZE(interface->rxq); k++) {
if (interface->rxq[k]) {
mqnic_destroy_rx_ring(interface->rxq[k]);
interface->rxq[k] = NULL;
}
}
for (k = 0; k < ARRAY_SIZE(interface->rx_cq); k++) {
if (interface->rx_cq[k]) {
mqnic_destroy_cq(interface->rx_cq[k]);
interface->rx_cq[k] = NULL;
}
}
// free schedulers
for (k = 0; k < ARRAY_SIZE(interface->sched_block); k++) {
if (interface->sched_block[k]) {

317
modules/mqnic/mqnic_netdev.c
View File

@ -37,74 +37,169 @@
#include <linux/version.h>
static int mqnic_stop_port(struct net_device *ndev);
static int mqnic_start_port(struct net_device *ndev)
{
struct mqnic_priv *priv = netdev_priv(ndev);
struct mqnic_dev *mdev = priv->mdev;
struct mqnic_if *iface = priv->interface;
struct mqnic_ring *q;
struct mqnic_cq *cq;
struct radix_tree_iter iter;
void **slot;
int k;
int ret;
u32 desc_block_size;
dev_info(mdev->dev, "%s on interface %d netdev %d", __func__,
priv->interface->index, priv->index);
desc_block_size = min_t(u32, priv->interface->max_desc_block_size, 4);
// set up RX queues
for (k = 0; k < min(priv->rxq_count, priv->rx_cq_count); k++) {
// set up CQ
mqnic_activate_cq(priv->rx_cq[k], priv->eq[k % priv->eq_count]);
for (k = 0; k < priv->rxq_count; k++) {
// create CQ
cq = mqnic_create_cq(iface);
if (IS_ERR_OR_NULL(cq)) {
ret = PTR_ERR(cq);
goto fail;
}
ret = mqnic_open_cq(cq, iface->eq[k % iface->eq_count], mqnic_num_rxq_entries, 0);
if (ret) {
mqnic_destroy_cq(cq);
goto fail;
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0)
netif_napi_add(ndev, &priv->rx_cq[k]->napi, mqnic_poll_rx_cq);
netif_napi_add(ndev, &cq->napi, mqnic_poll_rx_cq);
#else
netif_napi_add(ndev, &priv->rx_cq[k]->napi, mqnic_poll_rx_cq, NAPI_POLL_WEIGHT);
netif_napi_add(ndev, &cq->napi, mqnic_poll_rx_cq, NAPI_POLL_WEIGHT);
#endif
napi_enable(&priv->rx_cq[k]->napi);
napi_enable(&cq->napi);
mqnic_arm_cq(priv->rx_cq[k]);
mqnic_arm_cq(cq);
// set up queue
priv->rxq[k]->mtu = ndev->mtu;
// create RX queue
q = mqnic_create_rx_ring(iface);
if (IS_ERR_OR_NULL(q)) {
ret = PTR_ERR(q);
mqnic_destroy_cq(cq);
goto fail;
}
q->mtu = ndev->mtu;
if (ndev->mtu + ETH_HLEN <= PAGE_SIZE)
priv->rxq[k]->page_order = 0;
q->page_order = 0;
else
priv->rxq[k]->page_order = ilog2((ndev->mtu + ETH_HLEN + PAGE_SIZE - 1) / PAGE_SIZE - 1) + 1;
mqnic_activate_rx_ring(priv->rxq[k], priv, priv->rx_cq[k]);
q->page_order = ilog2((ndev->mtu + ETH_HLEN + PAGE_SIZE - 1) / PAGE_SIZE - 1) + 1;
ret = mqnic_open_rx_ring(q, priv, cq, mqnic_num_rxq_entries, 1);
if (ret) {
mqnic_destroy_rx_ring(q);
mqnic_destroy_cq(cq);
goto fail;
}
down_write(&priv->rxq_table_sem);
ret = radix_tree_insert(&priv->rxq_table, k, q);
up_write(&priv->rxq_table_sem);
if (ret) {
mqnic_destroy_rx_ring(q);
mqnic_destroy_cq(cq);
goto fail;
}
}
// set up TX queues
for (k = 0; k < min(priv->txq_count, priv->tx_cq_count); k++) {
// set up CQ
mqnic_activate_cq(priv->tx_cq[k], priv->eq[k % priv->eq_count]);
for (k = 0; k < priv->txq_count; k++) {
// create CQ
cq = mqnic_create_cq(iface);
if (IS_ERR_OR_NULL(cq)) {
ret = PTR_ERR(cq);
goto fail;
}
ret = mqnic_open_cq(cq, iface->eq[k % iface->eq_count], mqnic_num_txq_entries, 1);
if (ret) {
mqnic_destroy_cq(cq);
goto fail;
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0)
netif_napi_add_tx(ndev, &priv->tx_cq[k]->napi, mqnic_poll_tx_cq);
netif_napi_add_tx(ndev, &cq->napi, mqnic_poll_tx_cq);
#else
netif_tx_napi_add(ndev, &priv->tx_cq[k]->napi, mqnic_poll_tx_cq, NAPI_POLL_WEIGHT);
netif_tx_napi_add(ndev, &cq->napi, mqnic_poll_tx_cq, NAPI_POLL_WEIGHT);
#endif
napi_enable(&priv->tx_cq[k]->napi);
napi_enable(&cq->napi);
mqnic_arm_cq(priv->tx_cq[k]);
mqnic_arm_cq(cq);
// set up queue
priv->txq[k]->tx_queue = netdev_get_tx_queue(ndev, k);
mqnic_activate_tx_ring(priv->txq[k], priv, priv->tx_cq[k]);
// create TX queue
q = mqnic_create_tx_ring(iface);
if (IS_ERR_OR_NULL(q)) {
ret = PTR_ERR(q);
mqnic_destroy_cq(cq);
goto fail;
}
q->tx_queue = netdev_get_tx_queue(ndev, k);
ret = mqnic_open_tx_ring(q, priv, cq, mqnic_num_txq_entries, desc_block_size);
if (ret) {
mqnic_destroy_tx_ring(q);
mqnic_destroy_cq(cq);
goto fail;
}
down_write(&priv->txq_table_sem);
ret = radix_tree_insert(&priv->txq_table, k, q);
up_write(&priv->txq_table_sem);
if (ret) {
mqnic_destroy_tx_ring(q);
mqnic_destroy_cq(cq);
goto fail;
}
}
// set MTU
mqnic_interface_set_tx_mtu(priv->interface, ndev->mtu + ETH_HLEN);
mqnic_interface_set_rx_mtu(priv->interface, ndev->mtu + ETH_HLEN);
mqnic_interface_set_tx_mtu(iface, ndev->mtu + ETH_HLEN);
mqnic_interface_set_rx_mtu(iface, ndev->mtu + ETH_HLEN);
// configure RX indirection and RSS
mqnic_interface_set_rx_queue_map_rss_mask(priv->interface, 0, 0xffffffff);
mqnic_interface_set_rx_queue_map_app_mask(priv->interface, 0, 0);
mqnic_interface_set_rx_queue_map_rss_mask(iface, 0, 0xffffffff);
mqnic_interface_set_rx_queue_map_app_mask(iface, 0, 0);
for (k = 0; k < priv->interface->rx_queue_map_indir_table_size; k++) {
mqnic_interface_set_rx_queue_map_indir_table(priv->interface, 0, k, priv->rxq[k % priv->rxq_count]->index);
for (k = 0; k < iface->rx_queue_map_indir_table_size; k++) {
rcu_read_lock();
q = radix_tree_lookup(&priv->rxq_table, k % priv->rxq_count);
rcu_read_unlock();
mqnic_interface_set_rx_queue_map_indir_table(iface, 0, k, q->index);
}
priv->port_up = true;
// enable TX and RX queues
down_read(&priv->txq_table_sem);
radix_tree_for_each_slot(slot, &priv->txq_table, &iter, 0) {
struct mqnic_ring *q = (struct mqnic_ring *)*slot;
mqnic_enable_tx_ring(q);
}
up_read(&priv->txq_table_sem);
down_read(&priv->rxq_table_sem);
radix_tree_for_each_slot(slot, &priv->rxq_table, &iter, 0) {
struct mqnic_ring *q = (struct mqnic_ring *)*slot;
mqnic_enable_rx_ring(q);
}
up_read(&priv->rxq_table_sem);
// enable first scheduler
mqnic_activate_sched_block(priv->sched_block[0]);
priv->port_up = true;
netif_tx_start_all_queues(ndev);
netif_device_attach(ndev);
@ -117,12 +212,19 @@ static int mqnic_start_port(struct net_device *ndev)
}
return 0;
fail:
mqnic_stop_port(ndev);
return ret;
}
static int mqnic_stop_port(struct net_device *ndev)
{
struct mqnic_priv *priv = netdev_priv(ndev);
struct mqnic_dev *mdev = priv->mdev;
struct mqnic_cq *cq;
struct radix_tree_iter iter;
void **slot;
int k;
dev_info(mdev->dev, "%s on interface %d netdev %d", __func__,
@ -142,7 +244,6 @@ static int mqnic_stop_port(struct net_device *ndev)
spin_lock_bh(&priv->stats_lock);
mqnic_update_stats(ndev);
priv->port_up = false;
spin_unlock_bh(&priv->stats_lock);
// disable schedulers
@ -150,28 +251,56 @@ static int mqnic_stop_port(struct net_device *ndev)
mqnic_deactivate_sched_block(priv->sched_block[k]);
// disable TX and RX queues
for (k = 0; k < min(priv->txq_count, priv->tx_cq_count); k++)
mqnic_deactivate_tx_ring(priv->txq[k]);
down_read(&priv->txq_table_sem);
radix_tree_for_each_slot(slot, &priv->txq_table, &iter, 0) {
struct mqnic_ring *q = (struct mqnic_ring *)*slot;
for (k = 0; k < min(priv->rxq_count, priv->rx_cq_count); k++)
mqnic_deactivate_rx_ring(priv->rxq[k]);
mqnic_disable_tx_ring(q);
}
up_read(&priv->txq_table_sem);
down_read(&priv->rxq_table_sem);
radix_tree_for_each_slot(slot, &priv->rxq_table, &iter, 0) {
struct mqnic_ring *q = (struct mqnic_ring *)*slot;
mqnic_disable_rx_ring(q);
}
up_read(&priv->rxq_table_sem);
msleep(20);
// shut down NAPI and clean queues
for (k = 0; k < priv->txq_count; k++) {
napi_disable(&priv->tx_cq[k]->napi);
mqnic_deactivate_cq(priv->tx_cq[k]);
netif_napi_del(&priv->tx_cq[k]->napi);
mqnic_free_tx_buf(priv->txq[k]);
}
priv->port_up = false;
for (k = 0; k < priv->rxq_count; k++) {
napi_disable(&priv->rx_cq[k]->napi);
mqnic_deactivate_cq(priv->rx_cq[k]);
netif_napi_del(&priv->rx_cq[k]->napi);
mqnic_free_rx_buf(priv->rxq[k]);
// shut down NAPI and clean queues
down_write(&priv->txq_table_sem);
radix_tree_for_each_slot(slot, &priv->txq_table, &iter, 0) {
struct mqnic_ring *q = (struct mqnic_ring *)*slot;
cq = q->cq;
napi_disable(&cq->napi);
netif_napi_del(&cq->napi);
mqnic_close_tx_ring(q);
mqnic_destroy_tx_ring(q);
radix_tree_delete(&priv->txq_table, iter.index);
mqnic_close_cq(cq);
mqnic_destroy_cq(cq);
}
up_write(&priv->txq_table_sem);
down_write(&priv->rxq_table_sem);
radix_tree_for_each_slot(slot, &priv->rxq_table, &iter, 0) {
struct mqnic_ring *q = (struct mqnic_ring *)*slot;
cq = q->cq;
napi_disable(&cq->napi);
netif_napi_del(&cq->napi);
mqnic_close_rx_ring(q);
mqnic_destroy_rx_ring(q);
radix_tree_delete(&priv->rxq_table, iter.index);
mqnic_close_cq(cq);
mqnic_destroy_cq(cq);
}
up_write(&priv->rxq_table_sem);
return 0;
}
@ -215,31 +344,36 @@ static int mqnic_close(struct net_device *ndev)
void mqnic_update_stats(struct net_device *ndev)
{
struct mqnic_priv *priv = netdev_priv(ndev);
struct radix_tree_iter iter;
void **slot;
unsigned long packets, bytes;
int k;
if (unlikely(!priv->port_up))
return;
packets = 0;
bytes = 0;
for (k = 0; k < priv->rxq_count; k++) {
const struct mqnic_ring *ring = priv->rxq[k];
down_read(&priv->rxq_table_sem);
radix_tree_for_each_slot(slot, &priv->rxq_table, &iter, 0) {
const struct mqnic_ring *q = (struct mqnic_ring *)*slot;
packets += READ_ONCE(ring->packets);
bytes += READ_ONCE(ring->bytes);
packets += READ_ONCE(q->packets);
bytes += READ_ONCE(q->bytes);
}
up_read(&priv->rxq_table_sem);
ndev->stats.rx_packets = packets;
ndev->stats.rx_bytes = bytes;
packets = 0;
bytes = 0;
for (k = 0; k < priv->txq_count; k++) {
const struct mqnic_ring *ring = priv->txq[k];
down_read(&priv->txq_table_sem);
radix_tree_for_each_slot(slot, &priv->txq_table, &iter, 0) {
const struct mqnic_ring *q = (struct mqnic_ring *)*slot;
packets += READ_ONCE(ring->packets);
bytes += READ_ONCE(ring->bytes);
packets += READ_ONCE(q->packets);
bytes += READ_ONCE(q->bytes);
}
up_read(&priv->txq_table_sem);
ndev->stats.tx_packets = packets;
ndev->stats.tx_bytes = bytes;
}
@ -437,25 +571,14 @@ struct net_device *mqnic_create_netdev(struct mqnic_if *interface, int index, in
// associate interface resources
priv->if_features = interface->if_features;
priv->eq_count = interface->eq_count;
for (k = 0; k < interface->eq_count; k++)
priv->eq[k] = interface->eq[k];
priv->txq_count = mqnic_res_get_count(interface->txq_res);
priv->rxq_count = mqnic_res_get_count(interface->rxq_res);
priv->txq_count = interface->txq_count;
for (k = 0; k < interface->txq_count; k++)
priv->txq[k] = interface->txq[k];
init_rwsem(&priv->txq_table_sem);
INIT_RADIX_TREE(&priv->txq_table, GFP_KERNEL);
priv->tx_cq_count = interface->tx_cq_count;
for (k = 0; k < interface->tx_cq_count; k++)
priv->tx_cq[k] = interface->tx_cq[k];
priv->rxq_count = interface->rxq_count;
for (k = 0; k < interface->rxq_count; k++)
priv->rxq[k] = interface->rxq[k];
priv->rx_cq_count = interface->rx_cq_count;
for (k = 0; k < interface->rx_cq_count; k++)
priv->rx_cq[k] = interface->rx_cq[k];
init_rwsem(&priv->rxq_table_sem);
INIT_RADIX_TREE(&priv->rxq_table, GFP_KERNEL);
priv->sched_block_count = interface->sched_block_count;
for (k = 0; k < interface->sched_block_count; k++)
@ -489,36 +612,6 @@ struct net_device *mqnic_create_netdev(struct mqnic_if *interface, int index, in
desc_block_size = min_t(u32, interface->max_desc_block_size, 4);
// allocate ring buffers
for (k = 0; k < priv->txq_count; k++) {
ret = mqnic_alloc_tx_ring(priv->txq[k], mqnic_num_txq_entries,
MQNIC_DESC_SIZE * desc_block_size);
if (ret)
goto fail;
}
for (k = 0; k < priv->tx_cq_count; k++) {
ret = mqnic_alloc_cq(priv->tx_cq[k], mqnic_num_txq_entries,
MQNIC_CPL_SIZE);
if (ret)
goto fail;
}
for (k = 0; k < priv->rxq_count; k++) {
ret = mqnic_alloc_rx_ring(priv->rxq[k], mqnic_num_rxq_entries,
MQNIC_DESC_SIZE);
if (ret)
goto fail;
}
for (k = 0; k < priv->rx_cq_count; k++) {
ret = mqnic_alloc_cq(priv->rx_cq[k], mqnic_num_rxq_entries,
MQNIC_CPL_SIZE);
if (ret)
goto fail;
}
// entry points
ndev->netdev_ops = &mqnic_netdev_ops;
ndev->ethtool_ops = &mqnic_ethtool_ops;
@ -564,27 +657,9 @@ fail:
void mqnic_destroy_netdev(struct net_device *ndev)
{
struct mqnic_priv *priv = netdev_priv(ndev);
int k;
if (priv->registered)
unregister_netdev(ndev);
// free rings
for (k = 0; k < ARRAY_SIZE(priv->txq); k++)
if (priv->txq[k])
mqnic_free_tx_ring(priv->txq[k]);
for (k = 0; k < ARRAY_SIZE(priv->tx_cq); k++)
if (priv->tx_cq[k])
mqnic_free_cq(priv->tx_cq[k]);
for (k = 0; k < ARRAY_SIZE(priv->rxq); k++)
if (priv->rxq[k])
mqnic_free_rx_ring(priv->rxq[k]);
for (k = 0; k < ARRAY_SIZE(priv->rx_cq); k++)
if (priv->rx_cq[k])
mqnic_free_cq(priv->rx_cq[k]);
free_netdev(ndev);
}

36
modules/mqnic/mqnic_res.c
View File

@ -38,6 +38,7 @@
struct mqnic_res *mqnic_create_res(unsigned int count, u8 __iomem *base, unsigned int stride)
{
struct mqnic_res *res;
int ret;
res = kzalloc(sizeof(*res), GFP_KERNEL);
if (!res)
@ -47,7 +48,19 @@ struct mqnic_res *mqnic_create_res(unsigned int count, u8 __iomem *base, unsigne
res->base = base;
res->stride = stride;
spin_lock_init(&res->lock);
res->bmap = bitmap_zalloc(count, GFP_KERNEL);
if (!res) {
ret = -ENOMEM;
goto fail;
}
return res;
fail:
mqnic_destroy_res(res);
return ERR_PTR(ret);
}
void mqnic_destroy_res(struct mqnic_res *res)
@ -55,9 +68,32 @@ void mqnic_destroy_res(struct mqnic_res *res)
if (!res)
return;
bitmap_free(res->bmap);
kfree(res);
}
int mqnic_res_alloc(struct mqnic_res *res)
{
int index;
spin_lock(&res->lock);
index = bitmap_find_free_region(res->bmap, res->count, 0);
spin_unlock(&res->lock);
return index;
}
void mqnic_res_free(struct mqnic_res *res, int index)
{
if (index < 0 || index >= res->count)
return;
spin_lock(&res->lock);
bitmap_clear(res->bmap, index, 1);
spin_unlock(&res->lock);
}
unsigned int mqnic_res_get_count(struct mqnic_res *res)
{
return res->count;

128
modules/mqnic/mqnic_rx.c
View File

@ -35,8 +35,7 @@
#include "mqnic.h"
struct mqnic_ring *mqnic_create_rx_ring(struct mqnic_if *interface,
int index, u8 __iomem *hw_addr)
struct mqnic_ring *mqnic_create_rx_ring(struct mqnic_if *interface)
{
struct mqnic_ring *ring;
@ -47,48 +46,52 @@ struct mqnic_ring *mqnic_create_rx_ring(struct mqnic_if *interface,
ring->dev = interface->dev;
ring->interface = interface;
ring->index = index;
ring->active = 0;
ring->index = -1;
ring->enabled = 0;
ring->hw_addr = hw_addr;
ring->hw_addr = NULL;
ring->hw_ptr_mask = 0xffff;
ring->hw_head_ptr = hw_addr + MQNIC_QUEUE_HEAD_PTR_REG;
ring->hw_tail_ptr = hw_addr + MQNIC_QUEUE_TAIL_PTR_REG;
ring->hw_head_ptr = NULL;
ring->hw_tail_ptr = NULL;
ring->head_ptr = 0;
ring->tail_ptr = 0;
// deactivate queue
iowrite32(0, ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
return ring;
}
void mqnic_destroy_rx_ring(struct mqnic_ring *ring)
{
mqnic_free_rx_ring(ring);
mqnic_close_rx_ring(ring);
kfree(ring);
}
int mqnic_alloc_rx_ring(struct mqnic_ring *ring, int size, int stride)
int mqnic_open_rx_ring(struct mqnic_ring *ring, struct mqnic_priv *priv,
struct mqnic_cq *cq, int size, int desc_block_size)
{
int ret = 0;
if (ring->active || ring->buf)
if (ring->enabled || ring->hw_addr || ring->buf || !priv || !cq)
return -EINVAL;
ring->size = roundup_pow_of_two(size);
ring->size_mask = ring->size - 1;
ring->stride = roundup_pow_of_two(stride);
ring->index = mqnic_res_alloc(ring->interface->rxq_res);
if (ring->index < 0)
return -ENOMEM;
ring->desc_block_size = ring->stride / MQNIC_DESC_SIZE;
ring->log_desc_block_size = ring->desc_block_size < 2 ? 0 : ilog2(ring->desc_block_size - 1) + 1;
ring->log_desc_block_size = desc_block_size < 2 ? 0 : ilog2(desc_block_size - 1) + 1;
ring->desc_block_size = 1 << ring->log_desc_block_size;
ring->size = roundup_pow_of_two(size);
ring->full_size = ring->size >> 1;
ring->size_mask = ring->size - 1;
ring->stride = roundup_pow_of_two(MQNIC_DESC_SIZE * ring->desc_block_size);
ring->rx_info = kvzalloc(sizeof(*ring->rx_info) * ring->size, GFP_KERNEL);
if (!ring->rx_info)
return -ENOMEM;
if (!ring->rx_info) {
ret = -ENOMEM;
goto fail;
}
ring->buf_size = ring->size * ring->stride;
ring->buf = dma_alloc_coherent(ring->dev, ring->buf_size, &ring->buf_dma_addr, GFP_KERNEL);
@ -97,6 +100,15 @@ int mqnic_alloc_rx_ring(struct mqnic_ring *ring, int size, int stride)
goto fail;
}
ring->priv = priv;
ring->cq = cq;
cq->src_ring = ring;
cq->handler = mqnic_rx_irq;
ring->hw_addr = mqnic_res_get_addr(ring->interface->rxq_res, ring->index);
ring->hw_head_ptr = ring->hw_addr + MQNIC_QUEUE_HEAD_PTR_REG;
ring->hw_tail_ptr = ring->hw_addr + MQNIC_QUEUE_TAIL_PTR_REG;
ring->head_ptr = 0;
ring->tail_ptr = 0;
@ -105,8 +117,8 @@ int mqnic_alloc_rx_ring(struct mqnic_ring *ring, int size, int stride)
// set base address
iowrite32(ring->buf_dma_addr, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 0);
iowrite32(ring->buf_dma_addr >> 32, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 4);
// set completion queue index
iowrite32(0, ring->hw_addr + MQNIC_QUEUE_CPL_QUEUE_INDEX_REG);
// set CQN
iowrite32(ring->cq->cqn, ring->hw_addr + MQNIC_QUEUE_CPL_QUEUE_INDEX_REG);
// set pointers
iowrite32(ring->head_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_HEAD_PTR_REG);
iowrite32(ring->tail_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_TAIL_PTR_REG);
@ -114,16 +126,30 @@ int mqnic_alloc_rx_ring(struct mqnic_ring *ring, int size, int stride)
iowrite32(ilog2(ring->size) | (ring->log_desc_block_size << 8),
ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
mqnic_refill_rx_buffers(ring);
return 0;
fail:
mqnic_free_rx_ring(ring);
mqnic_close_rx_ring(ring);
return ret;
}
void mqnic_free_rx_ring(struct mqnic_ring *ring)
void mqnic_close_rx_ring(struct mqnic_ring *ring)
{
mqnic_deactivate_rx_ring(ring);
mqnic_disable_rx_ring(ring);
if (ring->cq) {
ring->cq->src_ring = NULL;
ring->cq->handler = NULL;
}
ring->priv = NULL;
ring->cq = NULL;
ring->hw_addr = NULL;
ring->hw_head_ptr = NULL;
ring->hw_tail_ptr = NULL;
if (ring->buf) {
mqnic_free_rx_buf(ring);
@ -137,60 +163,34 @@ void mqnic_free_rx_ring(struct mqnic_ring *ring)
kvfree(ring->rx_info);
ring->rx_info = NULL;
}
mqnic_res_free(ring->interface->rxq_res, ring->index);
ring->index = -1;
}
int mqnic_activate_rx_ring(struct mqnic_ring *ring, struct mqnic_priv *priv,
struct mqnic_cq *cq)
int mqnic_enable_rx_ring(struct mqnic_ring *ring)
{
mqnic_deactivate_rx_ring(ring);
if (!ring->buf || !priv || !cq || cq->handler || cq->src_ring)
if (!ring->hw_addr)
return -EINVAL;
ring->priv = priv;
ring->cq = cq;
cq->src_ring = ring;
cq->handler = mqnic_rx_irq;
ring->head_ptr = 0;
ring->tail_ptr = 0;
// deactivate queue
iowrite32(0, ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
// set base address
iowrite32(ring->buf_dma_addr, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 0);
iowrite32(ring->buf_dma_addr >> 32, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 4);
// set CQN
iowrite32(cq->cqn, ring->hw_addr + MQNIC_QUEUE_CPL_QUEUE_INDEX_REG);
// set pointers
iowrite32(ring->head_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_HEAD_PTR_REG);
iowrite32(ring->tail_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_TAIL_PTR_REG);
// set size and activate queue
// enable queue
iowrite32(ilog2(ring->size) | (ring->log_desc_block_size << 8) | MQNIC_QUEUE_ACTIVE_MASK,
ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
ring->active = 1;
mqnic_refill_rx_buffers(ring);
ring->enabled = 1;
return 0;
}
void mqnic_deactivate_rx_ring(struct mqnic_ring *ring)
void mqnic_disable_rx_ring(struct mqnic_ring *ring)
{
// deactivate queue
iowrite32(ilog2(ring->size) | (ring->log_desc_block_size << 8),
ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
if (ring->cq) {
ring->cq->src_ring = NULL;
ring->cq->handler = NULL;
// disable queue
if (ring->hw_addr) {
iowrite32(ilog2(ring->size) | (ring->log_desc_block_size << 8),
ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
}
ring->priv = NULL;
ring->cq = NULL;
ring->active = 0;
ring->enabled = 0;
}
bool mqnic_is_rx_ring_empty(const struct mqnic_ring *ring)

2
modules/mqnic/mqnic_sched_block.c
View File

@ -53,8 +53,6 @@ struct mqnic_sched_block *mqnic_create_sched_block(struct mqnic_if *interface,
block->index = index;
block->txq_count = interface->txq_count;
block->block_rb = block_rb;
offset = ioread32(block_rb->regs + MQNIC_RB_SCHED_BLOCK_REG_OFFSET);

133
modules/mqnic/mqnic_tx.c
View File

@ -36,8 +36,7 @@
#include <linux/version.h>
#include "mqnic.h"
struct mqnic_ring *mqnic_create_tx_ring(struct mqnic_if *interface,
int index, u8 __iomem *hw_addr)
struct mqnic_ring *mqnic_create_tx_ring(struct mqnic_if *interface)
{
struct mqnic_ring *ring;
@ -48,49 +47,52 @@ struct mqnic_ring *mqnic_create_tx_ring(struct mqnic_if *interface,
ring->dev = interface->dev;
ring->interface = interface;
ring->index = index;
ring->active = 0;
ring->index = -1;
ring->enabled = 0;
ring->hw_addr = hw_addr;
ring->hw_addr = NULL;
ring->hw_ptr_mask = 0xffff;
ring->hw_head_ptr = hw_addr + MQNIC_QUEUE_HEAD_PTR_REG;
ring->hw_tail_ptr = hw_addr + MQNIC_QUEUE_TAIL_PTR_REG;
ring->hw_head_ptr = NULL;
ring->hw_tail_ptr = NULL;
ring->head_ptr = 0;
ring->tail_ptr = 0;
// deactivate queue
iowrite32(0, ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
return ring;
}
void mqnic_destroy_tx_ring(struct mqnic_ring *ring)
{
mqnic_free_tx_ring(ring);
mqnic_close_tx_ring(ring);
kfree(ring);
}
int mqnic_alloc_tx_ring(struct mqnic_ring *ring, int size, int stride)
int mqnic_open_tx_ring(struct mqnic_ring *ring, struct mqnic_priv *priv,
struct mqnic_cq *cq, int size, int desc_block_size)
{
int ret = 0;
if (ring->active || ring->buf)
if (ring->enabled || ring->hw_addr || ring->buf || !priv || !cq)
return -EINVAL;
ring->index = mqnic_res_alloc(ring->interface->txq_res);
if (ring->index < 0)
return -ENOMEM;
ring->log_desc_block_size = desc_block_size < 2 ? 0 : ilog2(desc_block_size - 1) + 1;
ring->desc_block_size = 1 << ring->log_desc_block_size;
ring->size = roundup_pow_of_two(size);
ring->full_size = ring->size >> 1;
ring->size_mask = ring->size - 1;
ring->stride = roundup_pow_of_two(stride);
ring->desc_block_size = ring->stride / MQNIC_DESC_SIZE;
ring->log_desc_block_size = ring->desc_block_size < 2 ? 0 : ilog2(ring->desc_block_size - 1) + 1;
ring->desc_block_size = 1 << ring->log_desc_block_size;
ring->stride = roundup_pow_of_two(MQNIC_DESC_SIZE * ring->desc_block_size);
ring->tx_info = kvzalloc(sizeof(*ring->tx_info) * ring->size, GFP_KERNEL);
if (!ring->tx_info)
return -ENOMEM;
if (!ring->tx_info) {
ret = -ENOMEM;
goto fail;
}
ring->buf_size = ring->size * ring->stride;
ring->buf = dma_alloc_coherent(ring->dev, ring->buf_size, &ring->buf_dma_addr, GFP_KERNEL);
@ -99,6 +101,15 @@ int mqnic_alloc_tx_ring(struct mqnic_ring *ring, int size, int stride)
goto fail;
}
ring->priv = priv;
ring->cq = cq;
cq->src_ring = ring;
cq->handler = mqnic_tx_irq;
ring->hw_addr = mqnic_res_get_addr(ring->interface->txq_res, ring->index);
ring->hw_head_ptr = ring->hw_addr + MQNIC_QUEUE_HEAD_PTR_REG;
ring->hw_tail_ptr = ring->hw_addr + MQNIC_QUEUE_TAIL_PTR_REG;
ring->head_ptr = 0;
ring->tail_ptr = 0;
@ -107,8 +118,8 @@ int mqnic_alloc_tx_ring(struct mqnic_ring *ring, int size, int stride)
// set base address
iowrite32(ring->buf_dma_addr, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 0);
iowrite32(ring->buf_dma_addr >> 32, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 4);
// set completion queue index
iowrite32(0, ring->hw_addr + MQNIC_QUEUE_CPL_QUEUE_INDEX_REG);
// set CQN
iowrite32(ring->cq->cqn, ring->hw_addr + MQNIC_QUEUE_CPL_QUEUE_INDEX_REG);
// set pointers
iowrite32(ring->head_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_HEAD_PTR_REG);
iowrite32(ring->tail_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_TAIL_PTR_REG);
@ -119,13 +130,25 @@ int mqnic_alloc_tx_ring(struct mqnic_ring *ring, int size, int stride)
return 0;
fail:
mqnic_free_tx_ring(ring);
mqnic_close_tx_ring(ring);
return ret;
}
void mqnic_free_tx_ring(struct mqnic_ring *ring)
void mqnic_close_tx_ring(struct mqnic_ring *ring)
{
mqnic_deactivate_tx_ring(ring);
mqnic_disable_tx_ring(ring);
if (ring->cq) {
ring->cq->src_ring = NULL;
ring->cq->handler = NULL;
}
ring->priv = NULL;
ring->cq = NULL;
ring->hw_addr = NULL;
ring->hw_head_ptr = NULL;
ring->hw_tail_ptr = NULL;
if (ring->buf) {
mqnic_free_tx_buf(ring);
@ -139,58 +162,34 @@ void mqnic_free_tx_ring(struct mqnic_ring *ring)
kvfree(ring->tx_info);
ring->tx_info = NULL;
}
mqnic_res_free(ring->interface->txq_res, ring->index);
ring->index = -1;
}
int mqnic_activate_tx_ring(struct mqnic_ring *ring, struct mqnic_priv *priv,
struct mqnic_cq *cq)
int mqnic_enable_tx_ring(struct mqnic_ring *ring)
{
mqnic_deactivate_tx_ring(ring);
if (!ring->buf || !priv || !cq || cq->handler || cq->src_ring)
if (!ring->hw_addr)
return -EINVAL;
ring->priv = priv;
ring->cq = cq;
cq->src_ring = ring;
cq->handler = mqnic_tx_irq;
ring->head_ptr = 0;
ring->tail_ptr = 0;
// deactivate queue
iowrite32(0, ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
// set base address
iowrite32(ring->buf_dma_addr, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 0);
iowrite32(ring->buf_dma_addr >> 32, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 4);
// set CQN
iowrite32(cq->cqn, ring->hw_addr + MQNIC_QUEUE_CPL_QUEUE_INDEX_REG);
// set pointers
iowrite32(ring->head_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_HEAD_PTR_REG);
iowrite32(ring->tail_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_TAIL_PTR_REG);
// set size and activate queue
// enable queue
iowrite32(ilog2(ring->size) | (ring->log_desc_block_size << 8) | MQNIC_QUEUE_ACTIVE_MASK,
ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
ring->active = 1;
ring->enabled = 1;
return 0;
}
void mqnic_deactivate_tx_ring(struct mqnic_ring *ring)
void mqnic_disable_tx_ring(struct mqnic_ring *ring)
{
// deactivate queue
iowrite32(ilog2(ring->size) | (ring->log_desc_block_size << 8),
ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
if (ring->cq) {
ring->cq->src_ring = NULL;
ring->cq->handler = NULL;
// disable queue
if (ring->hw_addr) {
iowrite32(ilog2(ring->size) | (ring->log_desc_block_size << 8),
ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
}
ring->priv = NULL;
ring->cq = NULL;
ring->active = 0;
ring->enabled = 0;
}
bool mqnic_is_tx_ring_empty(const struct mqnic_ring *ring)
@ -444,11 +443,13 @@ netdev_tx_t mqnic_start_xmit(struct sk_buff *skb, struct net_device *ndev)
ring_index = skb_get_queue_mapping(skb);
if (unlikely(ring_index >= priv->txq_count))
// queue mapping out of range
goto tx_drop;
rcu_read_lock();
ring = radix_tree_lookup(&priv->txq_table, ring_index);
rcu_read_unlock();
ring = priv->txq[ring_index];
if (unlikely(!ring))
// unknown TX queue
goto tx_drop;
tail_ptr = READ_ONCE(ring->tail_ptr);