FPGA/corundum
1
0
Fork 0
mirror of https://github.com/corundum/corundum.git synced 2025-02-06 08:38:23 +08:00
Code Issues Projects Releases Wiki Activity

Implement dynamic queue allocation in testbench and driver

Browse Source 
Signed-off-by: Alex Forencich <alex@alexforencich.com>
This commit is contained in:
Alex Forencich 2023-05-02 21:23:30 -07:00
parent d82f37b3b4
commit 3c995dc8e0
10 changed files with 851 additions and 767 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

446
fpga/common/tb/mqnic.py
View File

@ -319,6 +319,14 @@ class Resource:
self.stride = stride self.stride = stride
self.windows = {} self.windows = {}
self.free_list = list(range(count))
def alloc(self):
return self.free_list.pop(0)
def free(self, index):
self.free_list.append(index)
self.free_list.sort()
def get_count(self): def get_count(self):
return self.count return self.count
@ -401,7 +409,7 @@ class Packet:
class Eq: class Eq:
def __init__(self, interface, eqn, hw_regs): def __init__(self, interface):
self.interface = interface self.interface = interface
self.log = interface.log self.log = interface.log
self.driver = interface.driver self.driver = interface.driver
@ -409,8 +417,8 @@ class Eq:
self.size = 0 self.size = 0
self.size_mask = 0 self.size_mask = 0
self.stride = 0 self.stride = 0
self.eqn = eqn self.eqn = None
self.active = False self.enabled = False
self.buf_size = 0 self.buf_size = 0
self.buf_region = None self.buf_region = None
@ -419,80 +427,82 @@ class Eq:
self.irq = None self.irq = None
self.cq_table = {}
self.head_ptr = 0 self.head_ptr = 0
self.tail_ptr = 0 self.tail_ptr = 0
self.hw_ptr_mask = 0xffff self.hw_ptr_mask = 0xffff
self.hw_regs = hw_regs self.hw_regs = None
async def init(self): async def open(self, irq, size):
self.log.info("Init EQ %d (interface %d)", self.eqn, self.interface.index) if self.hw_regs:
raise Exception("Already open")
await self.hw_regs.write_dword(MQNIC_EQ_ACTIVE_LOG_SIZE_REG, 0) # active, log size self.eqn = self.interface.eq_res.alloc()
async def alloc(self, size, stride): self.log.info("Open EQ %d (interface %d)", self.eqn, self.interface.index)
if self.active:
raise Exception("Cannot allocate active ring")
if self.buf:
raise Exception("Already allocated")
self.log_size = size.bit_length() - 1 self.log_size = size.bit_length() - 1
self.size = 2**self.log_size self.size = 2**self.log_size
self.size_mask = self.size-1 self.size_mask = self.size-1
self.stride = stride self.stride = MQNIC_EVENT_SIZE
self.buf_size = self.size*self.stride self.buf_size = self.size*self.stride
self.buf_region = self.driver.pool.alloc_region(self.buf_size) self.buf_region = self.driver.pool.alloc_region(self.buf_size)
self.buf_dma = self.buf_region.get_absolute_address(0) self.buf_dma = self.buf_region.get_absolute_address(0)
self.buf = self.buf_region.mem self.buf = self.buf_region.mem
self.buf[0:self.buf_size] = b'\x00'*self.buf_size
self.head_ptr = 0 self.head_ptr = 0
self.tail_ptr = 0 self.tail_ptr = 0
await self.hw_regs.write_dword(MQNIC_EQ_ACTIVE_LOG_SIZE_REG, 0) # active, log size
await self.hw_regs.write_dword(MQNIC_EQ_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address
await self.hw_regs.write_dword(MQNIC_EQ_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address
await self.hw_regs.write_dword(MQNIC_EQ_INTERRUPT_INDEX_REG, 0) # interrupt index
await self.hw_regs.write_dword(MQNIC_EQ_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer
await self.hw_regs.write_dword(MQNIC_EQ_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer
await self.hw_regs.write_dword(MQNIC_EQ_ACTIVE_LOG_SIZE_REG, self.log_size) # active, log size
async def free(self):
await self.deactivate()
if self.buf:
# TODO
pass
async def activate(self, irq):
self.log.info("Activate Eq %d (interface %d)", self.eqn, self.interface.index)
await self.deactivate()
self.irq = irq self.irq = irq
self.head_ptr = 0 self.cq_table = {}
self.tail_ptr = 0
self.buf[0:self.buf_size] = b'\x00'*self.buf_size self.hw_regs = self.interface.eq_res.get_window(self.eqn)
await self.hw_regs.write_dword(MQNIC_EQ_ACTIVE_LOG_SIZE_REG, 0) # active, log size await self.hw_regs.write_dword(MQNIC_EQ_ACTIVE_LOG_SIZE_REG, 0) # active, log size
await self.hw_regs.write_dword(MQNIC_EQ_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address await self.hw_regs.write_dword(MQNIC_EQ_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address
await self.hw_regs.write_dword(MQNIC_EQ_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address await self.hw_regs.write_dword(MQNIC_EQ_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address
await self.hw_regs.write_dword(MQNIC_EQ_INTERRUPT_INDEX_REG, irq) # interrupt index await self.hw_regs.write_dword(MQNIC_EQ_INTERRUPT_INDEX_REG, self.irq) # interrupt index
await self.hw_regs.write_dword(MQNIC_EQ_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer await self.hw_regs.write_dword(MQNIC_EQ_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer
await self.hw_regs.write_dword(MQNIC_EQ_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer await self.hw_regs.write_dword(MQNIC_EQ_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer
await self.hw_regs.write_dword(MQNIC_EQ_ACTIVE_LOG_SIZE_REG, self.log_size | MQNIC_EQ_ACTIVE_MASK) # active, log size await self.hw_regs.write_dword(MQNIC_EQ_ACTIVE_LOG_SIZE_REG, self.log_size | MQNIC_EQ_ACTIVE_MASK) # active, log size
self.active = True self.enabled = True
async def close(self):
if not self.hw_regs:
return
async def deactivate(self):
await self.hw_regs.write_dword(MQNIC_EQ_ACTIVE_LOG_SIZE_REG, self.log_size) # active, log size await self.hw_regs.write_dword(MQNIC_EQ_ACTIVE_LOG_SIZE_REG, self.log_size) # active, log size
await self.hw_regs.write_dword(MQNIC_EQ_INTERRUPT_INDEX_REG, 0) # interrupt index await self.hw_regs.write_dword(MQNIC_EQ_INTERRUPT_INDEX_REG, 0) # interrupt index
# TODO free buffer
self.irq = None self.irq = None
self.active = False self.enabled = False
self.hw_regs = None
self.interface.eq_res.free(self.eqn)
self.eqn = None
def attach_cq(self, cq):
if cq.is_txcq:
self.cq_table[cq.cqn | 0x80000000] = cq
else:
self.cq_table[cq.cqn] = cq
def detach_cq(self, cq):
if cq.is_txcq:
del self.cq_table[cq.cqn | 0x80000000]
else:
del self.cq_table[cq.cqn]
async def read_head_ptr(self): async def read_head_ptr(self):
val = await self.hw_regs.read_dword(MQNIC_EQ_HEAD_PTR_REG) val = await self.hw_regs.read_dword(MQNIC_EQ_HEAD_PTR_REG)
@ -502,7 +512,7 @@ class Eq:
await self.hw_regs.write_dword(MQNIC_EQ_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) await self.hw_regs.write_dword(MQNIC_EQ_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask)
async def arm(self): async def arm(self):
if not self.active: if not self.hw_regs:
return return
await self.hw_regs.write_dword(MQNIC_EQ_INTERRUPT_INDEX_REG, self.irq | MQNIC_EQ_ARM_MASK) # interrupt index await self.hw_regs.write_dword(MQNIC_EQ_INTERRUPT_INDEX_REG, self.irq | MQNIC_EQ_ARM_MASK) # interrupt index
@ -527,12 +537,12 @@ class Eq:
if event_data[0] == 0: if event_data[0] == 0:
# transmit completion # transmit completion
cq = self.interface.tx_cq[event_data[1]] cq = self.cq_table[event_data[1] | 0x80000000]
await cq.handler(cq) await cq.handler(cq)
await cq.arm() await cq.arm()
elif event_data[0] == 1: elif event_data[0] == 1:
# receive completion # receive completion
cq = self.interface.rx_cq[event_data[1]] cq = self.cq_table[event_data[1]]
await cq.handler(cq) await cq.handler(cq)
await cq.arm() await cq.arm()
@ -544,7 +554,7 @@ class Eq:
class Cq: class Cq:
def __init__(self, interface, cqn, hw_regs): def __init__(self, interface):
self.interface = interface self.interface = interface
self.log = interface.log self.log = interface.log
self.driver = interface.driver self.driver = interface.driver
@ -552,8 +562,8 @@ class Cq:
self.size = 0 self.size = 0
self.size_mask = 0 self.size_mask = 0
self.stride = 0 self.stride = 0
self.cqn = cqn self.cqn = None
self.active = False self.enabled = False
self.buf_size = 0 self.buf_size = 0
self.buf_region = None self.buf_region = None
@ -569,76 +579,74 @@ class Cq:
self.tail_ptr = 0 self.tail_ptr = 0
self.hw_ptr_mask = 0xffff self.hw_ptr_mask = 0xffff
self.hw_regs = hw_regs self.hw_regs = None
async def init(self): async def open(self, eq, size, is_txcq=True):
self.log.info("Init CQ %d (interface %d)", self.cqn, self.interface.index) if self.hw_regs:
raise Exception("Already open")
await self.hw_regs.write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, 0) # active, log size self.is_txcq = is_txcq
if is_txcq:
self.cqn = self.interface.tx_cq_res.alloc()
else:
self.cqn = self.interface.rx_cq_res.alloc()
async def alloc(self, size, stride): self.log.info("Open %s CQ %d (interface %d)", "TX" if is_txcq else "RX", self.cqn, self.interface.index)
if self.active:
raise Exception("Cannot allocate active ring")
if self.buf:
raise Exception("Already allocated")
self.log_size = size.bit_length() - 1 self.log_size = size.bit_length() - 1
self.size = 2**self.log_size self.size = 2**self.log_size
self.size_mask = self.size-1 self.size_mask = self.size-1
self.stride = stride self.stride = MQNIC_EVENT_SIZE
self.buf_size = self.size*self.stride self.buf_size = self.size*self.stride
self.buf_region = self.driver.pool.alloc_region(self.buf_size) self.buf_region = self.driver.pool.alloc_region(self.buf_size)
self.buf_dma = self.buf_region.get_absolute_address(0) self.buf_dma = self.buf_region.get_absolute_address(0)
self.buf = self.buf_region.mem self.buf = self.buf_region.mem
self.head_ptr = 0
self.tail_ptr = 0
await self.hw_regs.write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, 0) # active, log size
await self.hw_regs.write_dword(MQNIC_CQ_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address
await self.hw_regs.write_dword(MQNIC_CQ_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address
await self.hw_regs.write_dword(MQNIC_CQ_INTERRUPT_INDEX_REG, 0) # event index
await self.hw_regs.write_dword(MQNIC_CQ_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer
await self.hw_regs.write_dword(MQNIC_CQ_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer
await self.hw_regs.write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, self.log_size) # active, log size
async def free(self):
await self.deactivate()
if self.buf:
# TODO
pass
async def activate(self, eq):
self.log.info("Activate CQ %d (interface %d)", self.cqn, self.interface.index)
await self.deactivate()
self.eq = eq
self.head_ptr = 0
self.tail_ptr = 0
self.buf[0:self.buf_size] = b'\x00'*self.buf_size self.buf[0:self.buf_size] = b'\x00'*self.buf_size
self.head_ptr = 0
self.tail_ptr = 0
eq.attach_cq(self)
self.eq = eq
if is_txcq:
self.hw_regs = self.interface.tx_cq_res.get_window(self.cqn)
else:
self.hw_regs = self.interface.rx_cq_res.get_window(self.cqn)
await self.hw_regs.write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, 0) # active, log size await self.hw_regs.write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, 0) # active, log size
await self.hw_regs.write_dword(MQNIC_CQ_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address await self.hw_regs.write_dword(MQNIC_CQ_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address
await self.hw_regs.write_dword(MQNIC_CQ_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address await self.hw_regs.write_dword(MQNIC_CQ_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address
await self.hw_regs.write_dword(MQNIC_CQ_INTERRUPT_INDEX_REG, eq.eqn) # event index await self.hw_regs.write_dword(MQNIC_CQ_INTERRUPT_INDEX_REG, self.eq.eqn) # event index
await self.hw_regs.write_dword(MQNIC_CQ_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer await self.hw_regs.write_dword(MQNIC_CQ_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer
await self.hw_regs.write_dword(MQNIC_CQ_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer await self.hw_regs.write_dword(MQNIC_CQ_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer
await self.hw_regs.write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, self.log_size | MQNIC_CQ_ACTIVE_MASK) # active, log size await self.hw_regs.write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, self.log_size | MQNIC_CQ_ACTIVE_MASK) # active, log size
self.active = True self.enabled = True
async def close(self):
if not self.hw_regs:
return
async def deactivate(self):
await self.hw_regs.write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, self.log_size) # active, log size await self.hw_regs.write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, self.log_size) # active, log size
await self.hw_regs.write_dword(MQNIC_CQ_INTERRUPT_INDEX_REG, 0) # event index await self.hw_regs.write_dword(MQNIC_CQ_INTERRUPT_INDEX_REG, 0) # event index
# TODO free buffer
self.eq.detach_cq(self)
self.eq = None self.eq = None
self.active = False self.enabled = False
self.hw_regs = None
if self.is_txcq:
self.interface.tx_cq_res.free(self.cqn)
else:
self.interface.rx_cq_res.free(self.cqn)
self.cqn = None
async def read_head_ptr(self): async def read_head_ptr(self):
val = await self.hw_regs.read_dword(MQNIC_CQ_HEAD_PTR_REG) val = await self.hw_regs.read_dword(MQNIC_CQ_HEAD_PTR_REG)
@ -648,14 +656,14 @@ class Cq:
await self.hw_regs.write_dword(MQNIC_CQ_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) await self.hw_regs.write_dword(MQNIC_CQ_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask)
async def arm(self): async def arm(self):
if not self.active: if not self.hw_regs:
return return
await self.hw_regs.write_dword(MQNIC_CQ_INTERRUPT_INDEX_REG, self.eq.eqn | MQNIC_CQ_ARM_MASK) # event index await self.hw_regs.write_dword(MQNIC_CQ_INTERRUPT_INDEX_REG, self.eq.eqn | MQNIC_CQ_ARM_MASK) # event index
class Txq: class Txq:
def __init__(self, interface, index, hw_regs): def __init__(self, interface):
self.interface = interface self.interface = interface
self.log = interface.log self.log = interface.log
self.driver = interface.driver self.driver = interface.driver
@ -666,8 +674,8 @@ class Txq:
self.size_mask = 0 self.size_mask = 0
self.full_size = 0 self.full_size = 0
self.stride = 0 self.stride = 0
self.index = index self.index = None
self.active = False self.enabled = False
self.buf_size = 0 self.buf_size = 0
self.buf_region = None self.buf_region = None
@ -685,26 +693,23 @@ class Txq:
self.bytes = 0 self.bytes = 0
self.hw_ptr_mask = 0xffff self.hw_ptr_mask = 0xffff
self.hw_regs = hw_regs self.hw_regs = None
async def init(self): async def open(self, cq, size, desc_block_size):
self.log.info("Init TXQ %d (interface %d)", self.index, self.interface.index) if self.hw_regs:
raise Exception("Already open")
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0) # active, log size self.index = self.interface.txq_res.alloc()
async def alloc(self, size, stride): self.log.info("Open TXQ %d (interface %d)", self.index, self.interface.index)
if self.active:
raise Exception("Cannot allocate active ring")
if self.buf:
raise Exception("Already allocated")
self.log_queue_size = size.bit_length() - 1 self.log_queue_size = size.bit_length() - 1
self.log_desc_block_size = int(stride/MQNIC_DESC_SIZE).bit_length() - 1 self.log_desc_block_size = desc_block_size.bit_length() - 1
self.desc_block_size = 2**self.log_desc_block_size self.desc_block_size = 2**self.log_desc_block_size
self.size = 2**self.log_queue_size self.size = 2**self.log_queue_size
self.size_mask = self.size-1 self.size_mask = self.size-1
self.full_size = self.size >> 1 self.full_size = self.size >> 1
self.stride = stride self.stride = MQNIC_DESC_SIZE*self.desc_block_size
self.tx_info = [None]*self.size self.tx_info = [None]*self.size
@ -716,53 +721,54 @@ class Txq:
self.head_ptr = 0 self.head_ptr = 0
self.tail_ptr = 0 self.tail_ptr = 0
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0) # active, log size
await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address
await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address
await self.hw_regs.write_dword(MQNIC_QUEUE_CPL_QUEUE_INDEX_REG, 0) # completion queue index
await self.hw_regs.write_dword(MQNIC_QUEUE_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer
await self.hw_regs.write_dword(MQNIC_QUEUE_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8)) # active, log desc block size, log queue size
async def free(self):
await self.deactivate()
if self.buf:
# TODO
pass
async def activate(self, cq):
self.log.info("Activate TXQ %d (interface %d)", self.index, self.interface.index)
await self.deactivate()
self.cq = cq self.cq = cq
self.cq.src_ring = self self.cq.src_ring = self
self.cq.handler = Txq.process_tx_cq self.cq.handler = Txq.process_tx_cq
self.head_ptr = 0 self.hw_regs = self.interface.txq_res.get_window(self.index)
self.tail_ptr = 0
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0) # active, log size await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0) # active, log size
await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address
await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address
await self.hw_regs.write_dword(MQNIC_QUEUE_CPL_QUEUE_INDEX_REG, cq.cqn) # completion queue index await self.hw_regs.write_dword(MQNIC_QUEUE_CPL_QUEUE_INDEX_REG, self.cq.cqn) # completion queue index
await self.hw_regs.write_dword(MQNIC_QUEUE_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer await self.hw_regs.write_dword(MQNIC_QUEUE_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer
await self.hw_regs.write_dword(MQNIC_QUEUE_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer await self.hw_regs.write_dword(MQNIC_QUEUE_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8) | MQNIC_QUEUE_ACTIVE_MASK) # active, log desc block size, log queue size
self.active = True
async def deactivate(self):
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8)) # active, log desc block size, log queue size await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8)) # active, log desc block size, log queue size
async def close(self):
if not self.hw_regs:
return
await self.disable()
# TODO free buffer
if self.cq: if self.cq:
self.cq.src_ring = None self.cq.src_ring = None
self.cq.handler = None self.cq.handler = None
self.cq = None self.cq = None
self.active = False self.hw_regs = None
self.interface.txq_res.free(self.index)
self.index = None
async def enable(self):
if not self.hw_regs:
raise Exception("Not open")
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8) | MQNIC_QUEUE_ACTIVE_MASK) # active, log desc block size, log queue size
self.enabled = True
async def disable(self):
if not self.hw_regs:
return
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8)) # active, log desc block size, log queue size
self.enabled = False
def empty(self): def empty(self):
return self.head_ptr == self.tail_ptr return self.head_ptr == self.tail_ptr
@ -840,7 +846,7 @@ class Txq:
class Rxq: class Rxq:
def __init__(self, interface, index, hw_regs): def __init__(self, interface):
self.interface = interface self.interface = interface
self.log = interface.log self.log = interface.log
self.driver = interface.driver self.driver = interface.driver
@ -851,8 +857,8 @@ class Rxq:
self.size_mask = 0 self.size_mask = 0
self.full_size = 0 self.full_size = 0
self.stride = 0 self.stride = 0
self.index = index self.index = None
self.active = False self.enabled = False
self.buf_size = 0 self.buf_size = 0
self.buf_region = None self.buf_region = None
@ -868,26 +874,23 @@ class Rxq:
self.bytes = 0 self.bytes = 0
self.hw_ptr_mask = 0xffff self.hw_ptr_mask = 0xffff
self.hw_regs = hw_regs self.hw_regs = None
async def init(self): async def open(self, cq, size, desc_block_size):
self.log.info("Init RXQ %d (interface %d)", self.index, self.interface.index) if self.hw_regs:
raise Exception("Already open")
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0) # active, log size self.index = self.interface.rxq_res.alloc()
async def alloc(self, size, stride): self.log.info("Open RXQ %d (interface %d)", self.index, self.interface.index)
if self.active:
raise Exception("Cannot allocate active ring")
if self.buf:
raise Exception("Already allocated")
self.log_queue_size = size.bit_length() - 1 self.log_queue_size = size.bit_length() - 1
self.log_desc_block_size = int(stride/MQNIC_DESC_SIZE).bit_length() - 1 self.log_desc_block_size = desc_block_size.bit_length() - 1
self.desc_block_size = 2**self.log_desc_block_size self.desc_block_size = 2**self.log_desc_block_size
self.size = 2**self.log_queue_size self.size = 2**self.log_queue_size
self.size_mask = self.size-1 self.size_mask = self.size-1
self.full_size = self.size >> 1 self.full_size = self.size >> 1
self.stride = stride self.stride = MQNIC_DESC_SIZE*self.desc_block_size
self.rx_info = [None]*self.size self.rx_info = [None]*self.size
@ -899,47 +902,29 @@ class Rxq:
self.head_ptr = 0 self.head_ptr = 0
self.tail_ptr = 0 self.tail_ptr = 0
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0) # active, log size
await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address
await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address
await self.hw_regs.write_dword(MQNIC_QUEUE_CPL_QUEUE_INDEX_REG, 0) # completion queue index
await self.hw_regs.write_dword(MQNIC_QUEUE_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer
await self.hw_regs.write_dword(MQNIC_QUEUE_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8)) # active, log desc block size, log queue size
async def free(self):
await self.deactivate()
if self.buf:
# TODO
pass
async def activate(self, cq):
self.log.info("Activate RXQ %d (interface %d)", self.index, self.interface.index)
await self.deactivate()
self.cq = cq self.cq = cq
self.cq.src_ring = self self.cq.src_ring = self
self.cq.handler = Rxq.process_rx_cq self.cq.handler = Rxq.process_rx_cq
self.head_ptr = 0 self.hw_regs = self.interface.rxq_res.get_window(self.index)
self.tail_ptr = 0
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0) # active, log size await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0) # active, log size
await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address
await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address
await self.hw_regs.write_dword(MQNIC_QUEUE_CPL_QUEUE_INDEX_REG, cq.cqn) # completion queue index await self.hw_regs.write_dword(MQNIC_QUEUE_CPL_QUEUE_INDEX_REG, self.cq.cqn) # completion queue index
await self.hw_regs.write_dword(MQNIC_QUEUE_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer await self.hw_regs.write_dword(MQNIC_QUEUE_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer
await self.hw_regs.write_dword(MQNIC_QUEUE_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer await self.hw_regs.write_dword(MQNIC_QUEUE_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8) | MQNIC_QUEUE_ACTIVE_MASK) # active, log desc block size, log queue size await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8)) # active, log desc block size, log queue size
self.active = True
await self.refill_buffers() await self.refill_buffers()
async def deactivate(self): async def close(self):
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8)) # active, log desc block size, log queue size if not self.hw_regs:
return
await self.disable()
# TODO free buffer
if self.cq: if self.cq:
self.cq.src_ring = None self.cq.src_ring = None
@ -947,7 +932,26 @@ class Rxq:
self.cq = None self.cq = None
self.active = False self.hw_regs = None
self.interface.rxq_res.free(self.index)
self.index = None
async def enable(self):
if not self.hw_regs:
raise Exception("Not open")
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8) | MQNIC_QUEUE_ACTIVE_MASK) # active, log desc block size, log queue size
self.enabled = True
async def disable(self):
if not self.hw_regs:
return
await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8)) # active, log desc block size, log queue size
self.enabled = False
def empty(self): def empty(self):
return self.head_ptr == self.tail_ptr return self.head_ptr == self.tail_ptr
@ -1326,40 +1330,24 @@ class Interface:
await self.set_rx_queue_map_app_mask(k, 0) await self.set_rx_queue_map_app_mask(k, 0)
await self.set_rx_queue_map_indir_table(k, 0, 0) await self.set_rx_queue_map_indir_table(k, 0, 0)
self.eq = [] # ensure all queues are disabled
self.txq = []
self.tx_cq = []
self.rxq = []
self.rx_cq = []
self.ports = []
self.sched_blocks = []
for k in range(self.eq_res.get_count()): for k in range(self.eq_res.get_count()):
eq = Eq(self, k, self.eq_res.get_window(k)) await self.eq_res.get_window(k).write_dword(MQNIC_EQ_ACTIVE_LOG_SIZE_REG, 0)
await eq.init()
self.eq.append(eq)
for k in range(self.txq_res.get_count()): for k in range(self.txq_res.get_count()):
txq = Txq(self, k, self.txq_res.get_window(k)) await self.txq_res.get_window(k).write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0)
await txq.init()
self.txq.append(txq)
for k in range(self.tx_cq_res.get_count()): for k in range(self.tx_cq_res.get_count()):
cq = Cq(self, k, self.tx_cq_res.get_window(k)) await self.tx_cq_res.get_window(k).write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, 0)
await cq.init()
self.tx_cq.append(cq)
for k in range(self.rxq_res.get_count()): for k in range(self.rxq_res.get_count()):
rxq = Rxq(self, k, self.rxq_res.get_window(k)) await self.rxq_res.get_window(k).write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0)
await rxq.init()
self.rxq.append(rxq)
for k in range(self.rx_cq_res.get_count()): for k in range(self.rx_cq_res.get_count()):
cq = Cq(self, k, self.rx_cq_res.get_window(k)) await self.rx_cq_res.get_window(k).write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, 0)
await cq.init()
self.rx_cq.append(cq)
# create ports
self.ports = []
for k in range(self.port_count): for k in range(self.port_count):
rb = self.reg_blocks.find(MQNIC_RB_PORT_TYPE, MQNIC_RB_PORT_VER, index=k) rb = self.reg_blocks.find(MQNIC_RB_PORT_TYPE, MQNIC_RB_PORT_VER, index=k)
@ -1367,6 +1355,8 @@ class Interface:
await p.init() await p.init()
self.ports.append(p) self.ports.append(p)
# create schedulers
self.sched_blocks = []
for k in range(self.sched_block_count): for k in range(self.sched_block_count):
rb = self.reg_blocks.find(MQNIC_RB_SCHED_BLOCK_TYPE, MQNIC_RB_SCHED_BLOCK_VER, index=k) rb = self.reg_blocks.find(MQNIC_RB_SCHED_BLOCK_TYPE, MQNIC_RB_SCHED_BLOCK_VER, index=k)
@ -1376,30 +1366,42 @@ class Interface:
assert self.sched_block_count == len(self.sched_blocks) assert self.sched_block_count == len(self.sched_blocks)
for eq in self.eq: # create EQs
await eq.alloc(1024, MQNIC_EVENT_SIZE) self.eq = []
await eq.activate(self.index) # TODO? for k in range(self.eq_res.get_count()):
eq = Eq(self)
await eq.open(self.index, 1024)
self.eq.append(eq)
await eq.arm() await eq.arm()
self.txq = []
self.tx_cq = []
self.rxq = []
self.rx_cq = []
# wait for all writes to complete # wait for all writes to complete
await self.hw_regs.read_dword(0) await self.hw_regs.read_dword(0)
async def open(self): async def open(self):
for rxq in self.rxq: for k in range(self.rxq_res.get_count()):
cq = self.rx_cq[rxq.index] cq = Cq(self)
await cq.alloc(1024, MQNIC_CPL_SIZE) await cq.open(self.eq[k % len(self.eq)], 1024, is_txcq=False)
await cq.activate(self.eq[cq.cqn % len(self.eq)]) self.rx_cq.append(cq)
await cq.arm() await cq.arm()
await rxq.alloc(1024, MQNIC_DESC_SIZE*4) rxq = Rxq(self)
await rxq.activate(cq) await rxq.open(cq, 1024, 4)
await rxq.enable()
self.rxq.append(rxq)
for txq in self.txq: for k in range(self.txq_res.get_count()):
cq = self.tx_cq[txq.index] cq = Cq(self)
await cq.alloc(1024, MQNIC_CPL_SIZE) await cq.open(self.eq[k % len(self.eq)], 1024, is_txcq=True)
await cq.activate(self.eq[cq.cqn % len(self.eq)]) self.tx_cq.append(cq)
await cq.arm() await cq.arm()
await txq.alloc(1024, MQNIC_DESC_SIZE*4) txq = Txq(self)
await txq.activate(cq) await txq.open(cq, 1024, 4)
await txq.enable()
self.txq.append(txq)
# wait for all writes to complete # wait for all writes to complete
await self.hw_regs.read_dword(0) await self.hw_regs.read_dword(0)
@ -1409,22 +1411,26 @@ class Interface:
async def close(self): async def close(self):
self.port_up = False self.port_up = False
for txq in self.txq: for q in self.txq:
await txq.deactivate() q.disable()
await txq.cq.deactivate()
for rxq in self.rxq: for q in self.rxq:
await rxq.deactivate() q.disable()
await rxq.cq.deactivate()
# wait for all writes to complete # wait for all writes to complete
await self.hw_regs.read_dword(0) await self.hw_regs.read_dword(0)
for q in self.txq: for q in self.txq:
cq = q.cq
await q.free_buf() await q.free_buf()
await q.close()
await cq.close()
for q in self.rxq: for q in self.rxq:
cq = q.cq
await q.free_buf() await q.free_buf()
await q.close()
await cq.close()
async def start_xmit(self, skb, tx_ring=None, csum_start=None, csum_offset=None): async def start_xmit(self, skb, tx_ring=None, csum_start=None, csum_offset=None):
if not self.port_up: if not self.port_up:

91
modules/mqnic/mqnic.h
View File

@ -83,6 +83,9 @@ struct mqnic_res {
unsigned int count; unsigned int count;
u8 __iomem *base; u8 __iomem *base;
unsigned int stride; unsigned int stride;
spinlock_t lock;
unsigned long *bmap;
}; };
struct mqnic_reg_block { struct mqnic_reg_block {
@ -288,7 +291,7 @@ struct mqnic_ring {
struct mqnic_priv *priv; struct mqnic_priv *priv;
int index; int index;
struct mqnic_cq *cq; struct mqnic_cq *cq;
int active; int enabled;
u32 hw_ptr_mask; u32 hw_ptr_mask;
u8 __iomem *hw_addr; u8 __iomem *hw_addr;
@ -313,9 +316,10 @@ struct mqnic_cq {
struct mqnic_if *interface; struct mqnic_if *interface;
struct napi_struct napi; struct napi_struct napi;
int cqn; int cqn;
int is_txcq;
struct mqnic_eq *eq; struct mqnic_eq *eq;
struct mqnic_ring *src_ring; struct mqnic_ring *src_ring;
int active; int enabled;
void (*handler)(struct mqnic_cq *cq); void (*handler)(struct mqnic_cq *cq);
@ -342,12 +346,15 @@ struct mqnic_eq {
struct mqnic_if *interface; struct mqnic_if *interface;
int eqn; int eqn;
struct mqnic_irq *irq; struct mqnic_irq *irq;
int active; int enabled;
struct notifier_block irq_nb; struct notifier_block irq_nb;
void (*handler)(struct mqnic_eq *eq); void (*handler)(struct mqnic_eq *eq);
spinlock_t table_lock;
struct radix_tree_root cq_table;
u32 hw_ptr_mask; u32 hw_ptr_mask;
u8 __iomem *hw_addr; u8 __iomem *hw_addr;
u8 __iomem *hw_head_ptr; u8 __iomem *hw_head_ptr;
@ -393,8 +400,6 @@ struct mqnic_sched_block {
int index; int index;
u32 txq_count;
u32 sched_count; u32 sched_count;
struct mqnic_sched *sched[MQNIC_MAX_PORTS]; struct mqnic_sched *sched[MQNIC_MAX_PORTS];
}; };
@ -432,18 +437,6 @@ struct mqnic_if {
u32 eq_count; u32 eq_count;
struct mqnic_eq *eq[MQNIC_MAX_EQ]; struct mqnic_eq *eq[MQNIC_MAX_EQ];
u32 txq_count;
struct mqnic_ring *txq[MQNIC_MAX_TXQ];
u32 tx_cq_count;
struct mqnic_cq *tx_cq[MQNIC_MAX_TX_CQ];
u32 rxq_count;
struct mqnic_ring *rxq[MQNIC_MAX_RXQ];
u32 rx_cq_count;
struct mqnic_cq *rx_cq[MQNIC_MAX_RX_CQ];
u32 port_count; u32 port_count;
struct mqnic_port *port[MQNIC_MAX_PORTS]; struct mqnic_port *port[MQNIC_MAX_PORTS];
@ -482,20 +475,14 @@ struct mqnic_priv {
unsigned int link_status; unsigned int link_status;
struct timer_list link_status_timer; struct timer_list link_status_timer;
u32 eq_count;
struct mqnic_eq *eq[MQNIC_MAX_EQ];
u32 txq_count; u32 txq_count;
struct mqnic_ring *txq[MQNIC_MAX_TXQ];
u32 tx_cq_count;
struct mqnic_cq *tx_cq[MQNIC_MAX_TX_CQ];
u32 rxq_count; u32 rxq_count;
struct mqnic_ring *rxq[MQNIC_MAX_RXQ];
u32 rx_cq_count; struct rw_semaphore txq_table_sem;
struct mqnic_cq *rx_cq[MQNIC_MAX_RX_CQ]; struct radix_tree_root txq_table;
struct rw_semaphore rxq_table_sem;
struct radix_tree_root rxq_table;
u32 sched_block_count; u32 sched_block_count;
struct mqnic_sched_block *sched_block[MQNIC_MAX_PORTS]; struct mqnic_sched_block *sched_block[MQNIC_MAX_PORTS];
@ -512,6 +499,8 @@ struct mqnic_priv {
// mqnic_res.c // mqnic_res.c
struct mqnic_res *mqnic_create_res(unsigned int count, u8 __iomem *base, unsigned int stride); struct mqnic_res *mqnic_create_res(unsigned int count, u8 __iomem *base, unsigned int stride);
void mqnic_destroy_res(struct mqnic_res *res); void mqnic_destroy_res(struct mqnic_res *res);
int mqnic_res_alloc(struct mqnic_res *res);
void mqnic_res_free(struct mqnic_res *res, int index);
unsigned int mqnic_res_get_count(struct mqnic_res *res); unsigned int mqnic_res_get_count(struct mqnic_res *res);
u8 __iomem *mqnic_res_get_addr(struct mqnic_res *res, int index); u8 __iomem *mqnic_res_get_addr(struct mqnic_res *res, int index);
@ -602,39 +591,34 @@ void mqnic_stats_init(struct mqnic_dev *mdev);
u64 mqnic_stats_read(struct mqnic_dev *mdev, int index); u64 mqnic_stats_read(struct mqnic_dev *mdev, int index);
// mqnic_eq.c // mqnic_eq.c
struct mqnic_eq *mqnic_create_eq(struct mqnic_if *interface, struct mqnic_eq *mqnic_create_eq(struct mqnic_if *interface);
int eqn, u8 __iomem *hw_addr);
void mqnic_destroy_eq(struct mqnic_eq *eq); void mqnic_destroy_eq(struct mqnic_eq *eq);
int mqnic_alloc_eq(struct mqnic_eq *eq, int size, int stride); int mqnic_open_eq(struct mqnic_eq *eq, struct mqnic_irq *irq, int size);
void mqnic_free_eq(struct mqnic_eq *eq); void mqnic_close_eq(struct mqnic_eq *eq);
int mqnic_activate_eq(struct mqnic_eq *eq, struct mqnic_irq *irq); int mqnic_eq_attach_cq(struct mqnic_eq *eq, struct mqnic_cq *cq);
void mqnic_deactivate_eq(struct mqnic_eq *eq); void mqnic_eq_detach_cq(struct mqnic_eq *eq, struct mqnic_cq *cq);
void mqnic_eq_read_head_ptr(struct mqnic_eq *eq); void mqnic_eq_read_head_ptr(struct mqnic_eq *eq);
void mqnic_eq_write_tail_ptr(struct mqnic_eq *eq); void mqnic_eq_write_tail_ptr(struct mqnic_eq *eq);
void mqnic_arm_eq(struct mqnic_eq *eq); void mqnic_arm_eq(struct mqnic_eq *eq);
void mqnic_process_eq(struct mqnic_eq *eq); void mqnic_process_eq(struct mqnic_eq *eq);
// mqnic_cq.c // mqnic_cq.c
struct mqnic_cq *mqnic_create_cq(struct mqnic_if *interface, struct mqnic_cq *mqnic_create_cq(struct mqnic_if *interface);
int cqn, u8 __iomem *hw_addr);
void mqnic_destroy_cq(struct mqnic_cq *cq); void mqnic_destroy_cq(struct mqnic_cq *cq);
int mqnic_alloc_cq(struct mqnic_cq *cq, int size, int stride); int mqnic_open_cq(struct mqnic_cq *cq, struct mqnic_eq *eq, int size, int is_txcq);
void mqnic_free_cq(struct mqnic_cq *cq); void mqnic_close_cq(struct mqnic_cq *cq);
int mqnic_activate_cq(struct mqnic_cq *cq, struct mqnic_eq *eq);
void mqnic_deactivate_cq(struct mqnic_cq *cq);
void mqnic_cq_read_head_ptr(struct mqnic_cq *cq); void mqnic_cq_read_head_ptr(struct mqnic_cq *cq);
void mqnic_cq_write_tail_ptr(struct mqnic_cq *cq); void mqnic_cq_write_tail_ptr(struct mqnic_cq *cq);
void mqnic_arm_cq(struct mqnic_cq *cq); void mqnic_arm_cq(struct mqnic_cq *cq);
// mqnic_tx.c // mqnic_tx.c
struct mqnic_ring *mqnic_create_tx_ring(struct mqnic_if *interface, struct mqnic_ring *mqnic_create_tx_ring(struct mqnic_if *interface);
int index, u8 __iomem *hw_addr);
void mqnic_destroy_tx_ring(struct mqnic_ring *ring); void mqnic_destroy_tx_ring(struct mqnic_ring *ring);
int mqnic_alloc_tx_ring(struct mqnic_ring *ring, int size, int stride); int mqnic_open_tx_ring(struct mqnic_ring *ring, struct mqnic_priv *priv,
void mqnic_free_tx_ring(struct mqnic_ring *ring); struct mqnic_cq *cq, int size, int desc_block_size);
int mqnic_activate_tx_ring(struct mqnic_ring *ring, struct mqnic_priv *priv, void mqnic_close_tx_ring(struct mqnic_ring *ring);
struct mqnic_cq *cq); int mqnic_enable_tx_ring(struct mqnic_ring *ring);
void mqnic_deactivate_tx_ring(struct mqnic_ring *ring); void mqnic_disable_tx_ring(struct mqnic_ring *ring);
bool mqnic_is_tx_ring_empty(const struct mqnic_ring *ring); bool mqnic_is_tx_ring_empty(const struct mqnic_ring *ring);
bool mqnic_is_tx_ring_full(const struct mqnic_ring *ring); bool mqnic_is_tx_ring_full(const struct mqnic_ring *ring);
void mqnic_tx_read_tail_ptr(struct mqnic_ring *ring); void mqnic_tx_read_tail_ptr(struct mqnic_ring *ring);
@ -647,14 +631,13 @@ int mqnic_poll_tx_cq(struct napi_struct *napi, int budget);
netdev_tx_t mqnic_start_xmit(struct sk_buff *skb, struct net_device *dev); netdev_tx_t mqnic_start_xmit(struct sk_buff *skb, struct net_device *dev);
// mqnic_rx.c // mqnic_rx.c
struct mqnic_ring *mqnic_create_rx_ring(struct mqnic_if *interface, struct mqnic_ring *mqnic_create_rx_ring(struct mqnic_if *interface);
int index, u8 __iomem *hw_addr);
void mqnic_destroy_rx_ring(struct mqnic_ring *ring); void mqnic_destroy_rx_ring(struct mqnic_ring *ring);
int mqnic_alloc_rx_ring(struct mqnic_ring *ring, int size, int stride); int mqnic_open_rx_ring(struct mqnic_ring *ring, struct mqnic_priv *priv,
void mqnic_free_rx_ring(struct mqnic_ring *ring); struct mqnic_cq *cq, int size, int desc_block_size);
int mqnic_activate_rx_ring(struct mqnic_ring *ring, struct mqnic_priv *priv, void mqnic_close_rx_ring(struct mqnic_ring *ring);
struct mqnic_cq *cq); int mqnic_enable_rx_ring(struct mqnic_ring *ring);
void mqnic_deactivate_rx_ring(struct mqnic_ring *ring); void mqnic_disable_rx_ring(struct mqnic_ring *ring);
bool mqnic_is_rx_ring_empty(const struct mqnic_ring *ring); bool mqnic_is_rx_ring_empty(const struct mqnic_ring *ring);
bool mqnic_is_rx_ring_full(const struct mqnic_ring *ring); bool mqnic_is_rx_ring_full(const struct mqnic_ring *ring);
void mqnic_rx_read_tail_ptr(struct mqnic_ring *ring); void mqnic_rx_read_tail_ptr(struct mqnic_ring *ring);

125
modules/mqnic/mqnic_cq.c
View File

@ -35,8 +35,7 @@
#include "mqnic.h" #include "mqnic.h"
struct mqnic_cq *mqnic_create_cq(struct mqnic_if *interface, struct mqnic_cq *mqnic_create_cq(struct mqnic_if *interface)
int cqn, u8 __iomem *hw_addr)
{ {
struct mqnic_cq *cq; struct mqnic_cq *cq;
@ -47,82 +46,63 @@ struct mqnic_cq *mqnic_create_cq(struct mqnic_if *interface,
cq->dev = interface->dev; cq->dev = interface->dev;
cq->interface = interface; cq->interface = interface;
cq->cqn = cqn; cq->cqn = -1;
cq->active = 0; cq->enabled = 0;
cq->hw_addr = hw_addr; cq->hw_addr = NULL;
cq->hw_ptr_mask = 0xffff; cq->hw_ptr_mask = 0xffff;
cq->hw_head_ptr = hw_addr + MQNIC_CQ_HEAD_PTR_REG; cq->hw_head_ptr = NULL;
cq->hw_tail_ptr = hw_addr + MQNIC_CQ_TAIL_PTR_REG; cq->hw_tail_ptr = NULL;
cq->head_ptr = 0; cq->head_ptr = 0;
cq->tail_ptr = 0; cq->tail_ptr = 0;
// deactivate queue
iowrite32(0, cq->hw_addr + MQNIC_CQ_ACTIVE_LOG_SIZE_REG);
return cq; return cq;
} }
void mqnic_destroy_cq(struct mqnic_cq *cq) void mqnic_destroy_cq(struct mqnic_cq *cq)
{ {
mqnic_free_cq(cq); mqnic_close_cq(cq);
kfree(cq); kfree(cq);
} }
int mqnic_alloc_cq(struct mqnic_cq *cq, int size, int stride) int mqnic_open_cq(struct mqnic_cq *cq, struct mqnic_eq *eq, int size, int is_txcq)
{ {
if (cq->active || cq->buf) int ret;
if (cq->enabled || cq->hw_addr || cq->buf || !eq)
return -EINVAL; return -EINVAL;
cq->is_txcq = is_txcq;
if (is_txcq) {
cq->cqn = mqnic_res_alloc(cq->interface->tx_cq_res);
} else {
cq->cqn = mqnic_res_alloc(cq->interface->rx_cq_res);
}
if (cq->cqn < 0)
return -ENOMEM;
cq->size = roundup_pow_of_two(size); cq->size = roundup_pow_of_two(size);
cq->size_mask = cq->size - 1; cq->size_mask = cq->size - 1;
cq->stride = roundup_pow_of_two(stride); cq->stride = roundup_pow_of_two(MQNIC_CPL_SIZE);
cq->buf_size = cq->size * cq->stride; cq->buf_size = cq->size * cq->stride;
cq->buf = dma_alloc_coherent(cq->dev, cq->buf_size, &cq->buf_dma_addr, GFP_KERNEL); cq->buf = dma_alloc_coherent(cq->dev, cq->buf_size, &cq->buf_dma_addr, GFP_KERNEL);
if (!cq->buf) if (!cq->buf) {
return -ENOMEM; ret = -ENOMEM;
goto fail;
cq->head_ptr = 0;
cq->tail_ptr = 0;
// deactivate queue
iowrite32(0, cq->hw_addr + MQNIC_CQ_ACTIVE_LOG_SIZE_REG);
// set base address
iowrite32(cq->buf_dma_addr, cq->hw_addr + MQNIC_CQ_BASE_ADDR_REG + 0);
iowrite32(cq->buf_dma_addr >> 32, cq->hw_addr + MQNIC_CQ_BASE_ADDR_REG + 4);
// set interrupt index
iowrite32(0, cq->hw_addr + MQNIC_CQ_INTERRUPT_INDEX_REG);
// set pointers
iowrite32(cq->head_ptr & cq->hw_ptr_mask, cq->hw_addr + MQNIC_CQ_HEAD_PTR_REG);
iowrite32(cq->tail_ptr & cq->hw_ptr_mask, cq->hw_addr + MQNIC_CQ_TAIL_PTR_REG);
// set size
iowrite32(ilog2(cq->size), cq->hw_addr + MQNIC_CQ_ACTIVE_LOG_SIZE_REG);
return 0;
} }
void mqnic_free_cq(struct mqnic_cq *cq)
{
mqnic_deactivate_cq(cq);
if (cq->buf) {
dma_free_coherent(cq->dev, cq->buf_size, cq->buf, cq->buf_dma_addr);
cq->buf = NULL;
cq->buf_dma_addr = 0;
}
}
int mqnic_activate_cq(struct mqnic_cq *cq, struct mqnic_eq *eq)
{
mqnic_deactivate_cq(cq);
if (!cq->buf || !eq)
return -EINVAL;
cq->eq = eq; cq->eq = eq;
mqnic_eq_attach_cq(eq, cq);
if (is_txcq)
cq->hw_addr = mqnic_res_get_addr(cq->interface->tx_cq_res, cq->cqn);
else
cq->hw_addr = mqnic_res_get_addr(cq->interface->rx_cq_res, cq->cqn);
cq->hw_head_ptr = cq->hw_addr + MQNIC_CQ_HEAD_PTR_REG;
cq->hw_tail_ptr = cq->hw_addr + MQNIC_CQ_TAIL_PTR_REG;
cq->head_ptr = 0; cq->head_ptr = 0;
cq->tail_ptr = 0; cq->tail_ptr = 0;
@ -134,30 +114,55 @@ int mqnic_activate_cq(struct mqnic_cq *cq, struct mqnic_eq *eq)
// set base address // set base address
iowrite32(cq->buf_dma_addr, cq->hw_addr + MQNIC_CQ_BASE_ADDR_REG + 0); iowrite32(cq->buf_dma_addr, cq->hw_addr + MQNIC_CQ_BASE_ADDR_REG + 0);
iowrite32(cq->buf_dma_addr >> 32, cq->hw_addr + MQNIC_CQ_BASE_ADDR_REG + 4); iowrite32(cq->buf_dma_addr >> 32, cq->hw_addr + MQNIC_CQ_BASE_ADDR_REG + 4);
// set interrupt index // set EQN
iowrite32(cq->eq->eqn, cq->hw_addr + MQNIC_CQ_INTERRUPT_INDEX_REG); iowrite32(cq->eq->eqn, cq->hw_addr + MQNIC_CQ_INTERRUPT_INDEX_REG);
// set pointers // set pointers
iowrite32(cq->head_ptr & cq->hw_ptr_mask, cq->hw_addr + MQNIC_CQ_HEAD_PTR_REG); iowrite32(cq->head_ptr & cq->hw_ptr_mask, cq->hw_addr + MQNIC_CQ_HEAD_PTR_REG);
iowrite32(cq->tail_ptr & cq->hw_ptr_mask, cq->hw_addr + MQNIC_CQ_TAIL_PTR_REG); iowrite32(cq->tail_ptr & cq->hw_ptr_mask, cq->hw_addr + MQNIC_CQ_TAIL_PTR_REG);
// set size and activate queue // set size
iowrite32(ilog2(cq->size) | MQNIC_CQ_ACTIVE_MASK, iowrite32(ilog2(cq->size) | MQNIC_CQ_ACTIVE_MASK, cq->hw_addr + MQNIC_CQ_ACTIVE_LOG_SIZE_REG);
cq->hw_addr + MQNIC_CQ_ACTIVE_LOG_SIZE_REG);
cq->active = 1; cq->enabled = 1;
return 0; return 0;
fail:
mqnic_close_eq(eq);
return ret;
} }
void mqnic_deactivate_cq(struct mqnic_cq *cq) void mqnic_close_cq(struct mqnic_cq *cq)
{ {
if (cq->hw_addr) {
// deactivate queue // deactivate queue
iowrite32(ilog2(cq->size), cq->hw_addr + MQNIC_CQ_ACTIVE_LOG_SIZE_REG); iowrite32(ilog2(cq->size), cq->hw_addr + MQNIC_CQ_ACTIVE_LOG_SIZE_REG);
// disarm queue // disarm queue
iowrite32(0, cq->hw_addr + MQNIC_CQ_INTERRUPT_INDEX_REG); iowrite32(0, cq->hw_addr + MQNIC_CQ_INTERRUPT_INDEX_REG);
}
if (cq->eq) {
mqnic_eq_detach_cq(cq->eq, cq);
cq->eq = NULL; cq->eq = NULL;
}
cq->active = 0; cq->hw_addr = NULL;
cq->hw_head_ptr = NULL;
cq->hw_tail_ptr = NULL;
if (cq->buf) {
dma_free_coherent(cq->dev, cq->buf_size, cq->buf, cq->buf_dma_addr);
cq->buf = NULL;
cq->buf_dma_addr = 0;
}
if (cq->is_txcq) {
mqnic_res_free(cq->interface->tx_cq_res, cq->cqn);
} else {
mqnic_res_free(cq->interface->rx_cq_res, cq->cqn);
}
cq->cqn = -1;
cq->enabled = 0;
} }
void mqnic_cq_read_head_ptr(struct mqnic_cq *cq) void mqnic_cq_read_head_ptr(struct mqnic_cq *cq)
@ -172,7 +177,7 @@ void mqnic_cq_write_tail_ptr(struct mqnic_cq *cq)
void mqnic_arm_cq(struct mqnic_cq *cq) void mqnic_arm_cq(struct mqnic_cq *cq)
{ {
if (!cq->active) if (!cq->enabled)
return; return;
iowrite32(cq->eq->eqn | MQNIC_CQ_ARM_MASK, iowrite32(cq->eq->eqn | MQNIC_CQ_ARM_MASK,

203
modules/mqnic/mqnic_eq.c
View File

@ -45,8 +45,7 @@ static int mqnic_eq_int(struct notifier_block *nb, unsigned long action, void *d
return NOTIFY_DONE; return NOTIFY_DONE;
} }
struct mqnic_eq *mqnic_create_eq(struct mqnic_if *interface, struct mqnic_eq *mqnic_create_eq(struct mqnic_if *interface)
int eqn, u8 __iomem *hw_addr)
{ {
struct mqnic_eq *eq; struct mqnic_eq *eq;
@ -57,92 +56,66 @@ struct mqnic_eq *mqnic_create_eq(struct mqnic_if *interface,
eq->dev = interface->dev; eq->dev = interface->dev;
eq->interface = interface; eq->interface = interface;
eq->eqn = eqn; eq->eqn = -1;
eq->active = 0; eq->enabled = 0;
eq->irq_nb.notifier_call = mqnic_eq_int; eq->irq_nb.notifier_call = mqnic_eq_int;
eq->hw_addr = hw_addr; eq->hw_addr = NULL;
eq->hw_ptr_mask = 0xffff; eq->hw_ptr_mask = 0xffff;
eq->hw_head_ptr = hw_addr + MQNIC_EQ_HEAD_PTR_REG; eq->hw_head_ptr = NULL;
eq->hw_tail_ptr = hw_addr + MQNIC_EQ_TAIL_PTR_REG; eq->hw_tail_ptr = NULL;
eq->head_ptr = 0; eq->head_ptr = 0;
eq->tail_ptr = 0; eq->tail_ptr = 0;
// deactivate queue spin_lock_init(&eq->table_lock);
iowrite32(0, eq->hw_addr + MQNIC_EQ_ACTIVE_LOG_SIZE_REG);
INIT_RADIX_TREE(&eq->cq_table, GFP_KERNEL);
return eq; return eq;
} }
void mqnic_destroy_eq(struct mqnic_eq *eq) void mqnic_destroy_eq(struct mqnic_eq *eq)
{ {
mqnic_free_eq(eq); mqnic_close_eq(eq);
kfree(eq); kfree(eq);
} }
int mqnic_alloc_eq(struct mqnic_eq *eq, int size, int stride) int mqnic_open_eq(struct mqnic_eq *eq, struct mqnic_irq *irq, int size)
{ {
if (eq->active || eq->buf) int ret;
if (eq->enabled || eq->hw_addr || eq->buf || !irq)
return -EINVAL; return -EINVAL;
eq->eqn = mqnic_res_alloc(eq->interface->eq_res);
if (eq->eqn < 0)
return -ENOMEM;
eq->size = roundup_pow_of_two(size); eq->size = roundup_pow_of_two(size);
eq->size_mask = eq->size - 1; eq->size_mask = eq->size - 1;
eq->stride = roundup_pow_of_two(stride); eq->stride = roundup_pow_of_two(MQNIC_EVENT_SIZE);
eq->buf_size = eq->size * eq->stride; eq->buf_size = eq->size * eq->stride;
eq->buf = dma_alloc_coherent(eq->dev, eq->buf_size, &eq->buf_dma_addr, GFP_KERNEL); eq->buf = dma_alloc_coherent(eq->dev, eq->buf_size, &eq->buf_dma_addr, GFP_KERNEL);
if (!eq->buf) if (!eq->buf) {
return -ENOMEM; ret = -ENOMEM;
goto fail;
eq->head_ptr = 0;
eq->tail_ptr = 0;
// deactivate queue
iowrite32(0, eq->hw_addr + MQNIC_EQ_ACTIVE_LOG_SIZE_REG);
// set base address
iowrite32(eq->buf_dma_addr, eq->hw_addr + MQNIC_EQ_BASE_ADDR_REG + 0);
iowrite32(eq->buf_dma_addr >> 32, eq->hw_addr + MQNIC_EQ_BASE_ADDR_REG + 4);
// set interrupt index
iowrite32(0, eq->hw_addr + MQNIC_EQ_INTERRUPT_INDEX_REG);
// set pointers
iowrite32(eq->head_ptr & eq->hw_ptr_mask, eq->hw_addr + MQNIC_EQ_HEAD_PTR_REG);
iowrite32(eq->tail_ptr & eq->hw_ptr_mask, eq->hw_addr + MQNIC_EQ_TAIL_PTR_REG);
// set size
iowrite32(ilog2(eq->size), eq->hw_addr + MQNIC_EQ_ACTIVE_LOG_SIZE_REG);
return 0;
} }
void mqnic_free_eq(struct mqnic_eq *eq)
{
mqnic_deactivate_eq(eq);
if (eq->buf) {
dma_free_coherent(eq->dev, eq->buf_size, eq->buf, eq->buf_dma_addr);
eq->buf = NULL;
eq->buf_dma_addr = 0;
}
}
int mqnic_activate_eq(struct mqnic_eq *eq, struct mqnic_irq *irq)
{
int ret = 0;
mqnic_deactivate_eq(eq);
if (!eq->buf || !irq)
return -EINVAL;
// register interrupt // register interrupt
ret = atomic_notifier_chain_register(&irq->nh, &eq->irq_nb); ret = atomic_notifier_chain_register(&irq->nh, &eq->irq_nb);
if (ret) if (ret)
return ret; goto fail;
eq->irq = irq; eq->irq = irq;
eq->hw_addr = mqnic_res_get_addr(eq->interface->eq_res, eq->eqn);
eq->hw_head_ptr = eq->hw_addr + MQNIC_EQ_HEAD_PTR_REG;
eq->hw_tail_ptr = eq->hw_addr + MQNIC_EQ_TAIL_PTR_REG;
eq->head_ptr = 0; eq->head_ptr = 0;
eq->tail_ptr = 0; eq->tail_ptr = 0;
@ -158,23 +131,30 @@ int mqnic_activate_eq(struct mqnic_eq *eq, struct mqnic_irq *irq)
// set pointers // set pointers
iowrite32(eq->head_ptr & eq->hw_ptr_mask, eq->hw_addr + MQNIC_EQ_HEAD_PTR_REG); iowrite32(eq->head_ptr & eq->hw_ptr_mask, eq->hw_addr + MQNIC_EQ_HEAD_PTR_REG);
iowrite32(eq->tail_ptr & eq->hw_ptr_mask, eq->hw_addr + MQNIC_EQ_TAIL_PTR_REG); iowrite32(eq->tail_ptr & eq->hw_ptr_mask, eq->hw_addr + MQNIC_EQ_TAIL_PTR_REG);
// set size
iowrite32(ilog2(eq->size), eq->hw_addr + MQNIC_EQ_ACTIVE_LOG_SIZE_REG);
// set size and activate queue // set size and activate queue
iowrite32(ilog2(eq->size) | MQNIC_EQ_ACTIVE_MASK, iowrite32(ilog2(eq->size) | MQNIC_EQ_ACTIVE_MASK, eq->hw_addr + MQNIC_EQ_ACTIVE_LOG_SIZE_REG);
eq->hw_addr + MQNIC_EQ_ACTIVE_LOG_SIZE_REG);
eq->active = 1; eq->enabled = 1;
return 0; return 0;
fail:
mqnic_close_eq(eq);
return ret;
} }
void mqnic_deactivate_eq(struct mqnic_eq *eq) void mqnic_close_eq(struct mqnic_eq *eq)
{ {
int ret = 0; int ret;
if (eq->hw_addr) {
// deactivate queue // deactivate queue
iowrite32(ilog2(eq->size), eq->hw_addr + MQNIC_EQ_ACTIVE_LOG_SIZE_REG); iowrite32(ilog2(eq->size), eq->hw_addr + MQNIC_EQ_ACTIVE_LOG_SIZE_REG);
// disarm queue // disarm queue
iowrite32(0, eq->hw_addr + MQNIC_EQ_INTERRUPT_INDEX_REG); iowrite32(0, eq->hw_addr + MQNIC_EQ_INTERRUPT_INDEX_REG);
}
// unregister interrupt // unregister interrupt
if (eq->irq) if (eq->irq)
@ -182,7 +162,56 @@ void mqnic_deactivate_eq(struct mqnic_eq *eq)
eq->irq = NULL; eq->irq = NULL;
eq->active = 0; eq->hw_addr = NULL;
eq->hw_head_ptr = NULL;
eq->hw_tail_ptr = NULL;
if (eq->buf) {
dma_free_coherent(eq->dev, eq->buf_size, eq->buf, eq->buf_dma_addr);
eq->buf = NULL;
eq->buf_dma_addr = 0;
}
mqnic_res_free(eq->interface->eq_res, eq->eqn);
eq->eqn = -1;
eq->enabled = 0;
}
int mqnic_eq_attach_cq(struct mqnic_eq *eq, struct mqnic_cq *cq)
{
int ret;
int cqn = cq->cqn;
if (cq->is_txcq)
cqn |= 0x80000000;
spin_lock_irq(&eq->table_lock);
ret = radix_tree_insert(&eq->cq_table, cqn, cq);
spin_unlock_irq(&eq->table_lock);
return ret;
}
void mqnic_eq_detach_cq(struct mqnic_eq *eq, struct mqnic_cq *cq)
{
struct mqnic_cq *item;
int cqn = cq->cqn;
if (cq->is_txcq)
cqn |= 0x80000000;
spin_lock_irq(&eq->table_lock);
item = radix_tree_delete(&eq->cq_table, cqn);
spin_unlock_irq(&eq->table_lock);
if (IS_ERR(item)) {
dev_err(eq->dev, "%s on IF %d EQ %d: radix_tree_delete failed: %ld",
__func__, eq->interface->index, eq->eqn, PTR_ERR(item));
} else if (!item) {
dev_err(eq->dev, "%s on IF %d EQ %d: CQ %d not in table",
__func__, eq->interface->index, eq->eqn, cqn);
} else if (item != cq) {
dev_err(eq->dev, "%s on IF %d EQ %d: entry mismatch when removing CQ %d",
__func__, eq->interface->index, eq->eqn, cqn);
}
} }
void mqnic_eq_read_head_ptr(struct mqnic_eq *eq) void mqnic_eq_read_head_ptr(struct mqnic_eq *eq)
@ -197,11 +226,10 @@ void mqnic_eq_write_tail_ptr(struct mqnic_eq *eq)
void mqnic_arm_eq(struct mqnic_eq *eq) void mqnic_arm_eq(struct mqnic_eq *eq)
{ {
if (!eq->active) if (!eq->enabled)
return; return;
iowrite32(eq->irq->index | MQNIC_EQ_ARM_MASK, iowrite32(eq->irq->index | MQNIC_EQ_ARM_MASK, eq->hw_addr + MQNIC_EQ_INTERRUPT_INDEX_REG);
eq->hw_addr + MQNIC_EQ_INTERRUPT_INDEX_REG);
} }
void mqnic_process_eq(struct mqnic_eq *eq) void mqnic_process_eq(struct mqnic_eq *eq)
@ -212,6 +240,7 @@ void mqnic_process_eq(struct mqnic_eq *eq)
u32 eq_index; u32 eq_index;
u32 eq_tail_ptr; u32 eq_tail_ptr;
int done = 0; int done = 0;
int cqn;
// read head pointer from NIC // read head pointer from NIC
eq_tail_ptr = eq->tail_ptr; eq_tail_ptr = eq->tail_ptr;
@ -227,34 +256,44 @@ void mqnic_process_eq(struct mqnic_eq *eq)
if (event->type == MQNIC_EVENT_TYPE_TX_CPL) { if (event->type == MQNIC_EVENT_TYPE_TX_CPL) {
// transmit completion event // transmit completion event
if (unlikely(le16_to_cpu(event->source) > interface->tx_cq_count)) { cqn = le16_to_cpu(event->source) | 0x80000000;
dev_err(eq->dev, "%s on port %d: unknown event source %d (index %d, type %d)",
__func__, interface->index, le16_to_cpu(event->source), eq_index, rcu_read_lock();
le16_to_cpu(event->type)); cq = radix_tree_lookup(&eq->cq_table, cqn);
rcu_read_unlock();
if (likely(cq)) {
if (likely(cq->handler))
cq->handler(cq);
} else {
dev_err(eq->dev, "%s on IF %d EQ %d: unknown event source %d (index %d, type %d)",
__func__, interface->index, eq->eqn, le16_to_cpu(event->source),
eq_index, le16_to_cpu(event->type));
print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 1, print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 1,
event, MQNIC_EVENT_SIZE, true); event, MQNIC_EVENT_SIZE, true);
} else {
cq = interface->tx_cq[le16_to_cpu(event->source)];
if (likely(cq && cq->handler))
cq->handler(cq);
} }
} else if (le16_to_cpu(event->type) == MQNIC_EVENT_TYPE_RX_CPL) { } else if (le16_to_cpu(event->type) == MQNIC_EVENT_TYPE_RX_CPL) {
// receive completion event // receive completion event
if (unlikely(le16_to_cpu(event->source) > interface->rx_cq_count)) { cqn = le16_to_cpu(event->source);
dev_err(eq->dev, "%s on port %d: unknown event source %d (index %d, type %d)",
__func__, interface->index, le16_to_cpu(event->source), eq_index, rcu_read_lock();
le16_to_cpu(event->type)); cq = radix_tree_lookup(&eq->cq_table, cqn);
rcu_read_unlock();
if (likely(cq)) {
if (likely(cq->handler))
cq->handler(cq);
} else {
dev_err(eq->dev, "%s on IF %d EQ %d: unknown event source %d (index %d, type %d)",
__func__, interface->index, eq->eqn, le16_to_cpu(event->source),
eq_index, le16_to_cpu(event->type));
print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 1, print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 1,
event, MQNIC_EVENT_SIZE, true); event, MQNIC_EVENT_SIZE, true);
} else {
cq = interface->rx_cq[le16_to_cpu(event->source)];
if (likely(cq && cq->handler))
cq->handler(cq);
} }
} else { } else {
dev_err(eq->dev, "%s on port %d: unknown event type %d (index %d, source %d)", dev_err(eq->dev, "%s on IF %d EQ %d: unknown event type %d (index %d, source %d)",
__func__, interface->index, le16_to_cpu(event->type), eq_index, __func__, interface->index, eq->eqn, le16_to_cpu(event->type),
le16_to_cpu(event->source)); eq_index, le16_to_cpu(event->source));
print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 1, print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 1,
event, MQNIC_EVENT_SIZE, true); event, MQNIC_EVENT_SIZE, true);
} }

119
modules/mqnic/mqnic_if.c
View File

@ -250,70 +250,21 @@ struct mqnic_if *mqnic_create_interface(struct mqnic_dev *mdev, int index, u8 __
desc_block_size = min_t(u32, interface->max_desc_block_size, 4); desc_block_size = min_t(u32, interface->max_desc_block_size, 4);
// create rings // disable queues
interface->eq_count = mqnic_res_get_count(interface->eq_res); for (k = 0; k < mqnic_res_get_count(interface->eq_res); k++)
for (k = 0; k < interface->eq_count; k++) { iowrite32(0, mqnic_res_get_addr(interface->eq_res, k) + MQNIC_EQ_ACTIVE_LOG_SIZE_REG);
struct mqnic_eq *eq = mqnic_create_eq(interface, k,
mqnic_res_get_addr(interface->eq_res, k));
if (IS_ERR_OR_NULL(eq)) {
ret = PTR_ERR(eq);
goto fail;
}
interface->eq[k] = eq; for (k = 0; k < mqnic_res_get_count(interface->txq_res); k++)
iowrite32(0, mqnic_res_get_addr(interface->txq_res, k) + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
ret = mqnic_alloc_eq(eq, mqnic_num_eq_entries, for (k = 0; k < mqnic_res_get_count(interface->tx_cq_res); k++)
MQNIC_EVENT_SIZE); iowrite32(0, mqnic_res_get_addr(interface->tx_cq_res, k) + MQNIC_CQ_ACTIVE_LOG_SIZE_REG);
if (ret)
goto fail;
mqnic_activate_eq(eq, mdev->irq[k % mdev->irq_count]); for (k = 0; k < mqnic_res_get_count(interface->rxq_res); k++)
mqnic_arm_eq(eq); iowrite32(0, mqnic_res_get_addr(interface->rxq_res, k) + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
}
interface->txq_count = mqnic_res_get_count(interface->txq_res); for (k = 0; k < mqnic_res_get_count(interface->rx_cq_res); k++)
for (k = 0; k < interface->txq_count; k++) { iowrite32(0, mqnic_res_get_addr(interface->rx_cq_res, k) + MQNIC_CQ_ACTIVE_LOG_SIZE_REG);
struct mqnic_ring *txq = mqnic_create_tx_ring(interface, k,
mqnic_res_get_addr(interface->txq_res, k));
if (IS_ERR_OR_NULL(txq)) {
ret = PTR_ERR(txq);
goto fail;
}
interface->txq[k] = txq;
}
interface->tx_cq_count = mqnic_res_get_count(interface->tx_cq_res);
for (k = 0; k < interface->tx_cq_count; k++) {
struct mqnic_cq *cq = mqnic_create_cq(interface, k,
mqnic_res_get_addr(interface->tx_cq_res, k));
if (IS_ERR_OR_NULL(cq)) {
ret = PTR_ERR(cq);
goto fail;
}
interface->tx_cq[k] = cq;
}
interface->rxq_count = mqnic_res_get_count(interface->rxq_res);
for (k = 0; k < interface->rxq_count; k++) {
struct mqnic_ring *rxq = mqnic_create_rx_ring(interface, k,
mqnic_res_get_addr(interface->rxq_res, k));
if (IS_ERR_OR_NULL(rxq)) {
ret = PTR_ERR(rxq);
goto fail;
}
interface->rxq[k] = rxq;
}
interface->rx_cq_count = mqnic_res_get_count(interface->rx_cq_res);
for (k = 0; k < interface->rx_cq_count; k++) {
struct mqnic_cq *cq = mqnic_create_cq(interface, k,
mqnic_res_get_addr(interface->rx_cq_res, k));
if (IS_ERR_OR_NULL(cq)) {
ret = PTR_ERR(cq);
goto fail;
}
interface->rx_cq[k] = cq;
}
// create ports // create ports
for (k = 0; k < interface->port_count; k++) { for (k = 0; k < interface->port_count; k++) {
@ -353,6 +304,24 @@ struct mqnic_if *mqnic_create_interface(struct mqnic_dev *mdev, int index, u8 __
interface->sched_block[k] = sched_block; interface->sched_block[k] = sched_block;
} }
// create EQs
interface->eq_count = mqnic_res_get_count(interface->eq_res);
for (k = 0; k < interface->eq_count; k++) {
struct mqnic_eq *eq = mqnic_create_eq(interface);
if (IS_ERR_OR_NULL(eq)) {
ret = PTR_ERR(eq);
goto fail;
}
interface->eq[k] = eq;
ret = mqnic_open_eq(eq, mdev->irq[k % mdev->irq_count], mqnic_num_eq_entries);
if (ret)
goto fail;
mqnic_arm_eq(eq);
}
// create net_devices // create net_devices
interface->dev_port_base = mdev->dev_port_max; interface->dev_port_base = mdev->dev_port_max;
interface->dev_port_max = mdev->dev_port_max; interface->dev_port_max = mdev->dev_port_max;
@ -386,7 +355,7 @@ void mqnic_destroy_interface(struct mqnic_if *interface)
} }
} }
// free rings // free EQs
for (k = 0; k < ARRAY_SIZE(interface->eq); k++) { for (k = 0; k < ARRAY_SIZE(interface->eq); k++) {
if (interface->eq[k]) { if (interface->eq[k]) {
mqnic_destroy_eq(interface->eq[k]); mqnic_destroy_eq(interface->eq[k]);
@ -394,34 +363,6 @@ void mqnic_destroy_interface(struct mqnic_if *interface)
} }
} }
for (k = 0; k < ARRAY_SIZE(interface->txq); k++) {
if (interface->txq[k]) {
mqnic_destroy_tx_ring(interface->txq[k]);
interface->txq[k] = NULL;
}
}
for (k = 0; k < ARRAY_SIZE(interface->tx_cq); k++) {
if (interface->tx_cq[k]) {
mqnic_destroy_cq(interface->tx_cq[k]);
interface->tx_cq[k] = NULL;
}
}
for (k = 0; k < ARRAY_SIZE(interface->rxq); k++) {
if (interface->rxq[k]) {
mqnic_destroy_rx_ring(interface->rxq[k]);
interface->rxq[k] = NULL;
}
}
for (k = 0; k < ARRAY_SIZE(interface->rx_cq); k++) {
if (interface->rx_cq[k]) {
mqnic_destroy_cq(interface->rx_cq[k]);
interface->rx_cq[k] = NULL;
}
}
// free schedulers // free schedulers
for (k = 0; k < ARRAY_SIZE(interface->sched_block); k++) { for (k = 0; k < ARRAY_SIZE(interface->sched_block); k++) {
if (interface->sched_block[k]) { if (interface->sched_block[k]) {

317
modules/mqnic/mqnic_netdev.c
View File

@ -37,74 +37,169 @@
#include <linux/version.h> #include <linux/version.h>
static int mqnic_stop_port(struct net_device *ndev);
static int mqnic_start_port(struct net_device *ndev) static int mqnic_start_port(struct net_device *ndev)
{ {
struct mqnic_priv *priv = netdev_priv(ndev); struct mqnic_priv *priv = netdev_priv(ndev);
struct mqnic_dev *mdev = priv->mdev; struct mqnic_dev *mdev = priv->mdev;
struct mqnic_if *iface = priv->interface;
struct mqnic_ring *q;
struct mqnic_cq *cq;
struct radix_tree_iter iter;
void **slot;
int k; int k;
int ret;
u32 desc_block_size;
dev_info(mdev->dev, "%s on interface %d netdev %d", __func__, dev_info(mdev->dev, "%s on interface %d netdev %d", __func__,
priv->interface->index, priv->index); priv->interface->index, priv->index);
desc_block_size = min_t(u32, priv->interface->max_desc_block_size, 4);
// set up RX queues // set up RX queues
for (k = 0; k < min(priv->rxq_count, priv->rx_cq_count); k++) { for (k = 0; k < priv->rxq_count; k++) {
// set up CQ // create CQ
mqnic_activate_cq(priv->rx_cq[k], priv->eq[k % priv->eq_count]); cq = mqnic_create_cq(iface);
if (IS_ERR_OR_NULL(cq)) {
ret = PTR_ERR(cq);
goto fail;
}
ret = mqnic_open_cq(cq, iface->eq[k % iface->eq_count], mqnic_num_rxq_entries, 0);
if (ret) {
mqnic_destroy_cq(cq);
goto fail;
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0) #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0)
netif_napi_add(ndev, &priv->rx_cq[k]->napi, mqnic_poll_rx_cq); netif_napi_add(ndev, &cq->napi, mqnic_poll_rx_cq);
#else #else
netif_napi_add(ndev, &priv->rx_cq[k]->napi, mqnic_poll_rx_cq, NAPI_POLL_WEIGHT); netif_napi_add(ndev, &cq->napi, mqnic_poll_rx_cq, NAPI_POLL_WEIGHT);
#endif #endif
napi_enable(&priv->rx_cq[k]->napi); napi_enable(&cq->napi);
mqnic_arm_cq(priv->rx_cq[k]); mqnic_arm_cq(cq);
// set up queue // create RX queue
priv->rxq[k]->mtu = ndev->mtu; q = mqnic_create_rx_ring(iface);
if (IS_ERR_OR_NULL(q)) {
ret = PTR_ERR(q);
mqnic_destroy_cq(cq);
goto fail;
}
q->mtu = ndev->mtu;
if (ndev->mtu + ETH_HLEN <= PAGE_SIZE) if (ndev->mtu + ETH_HLEN <= PAGE_SIZE)
priv->rxq[k]->page_order = 0; q->page_order = 0;
else else
priv->rxq[k]->page_order = ilog2((ndev->mtu + ETH_HLEN + PAGE_SIZE - 1) / PAGE_SIZE - 1) + 1; q->page_order = ilog2((ndev->mtu + ETH_HLEN + PAGE_SIZE - 1) / PAGE_SIZE - 1) + 1;
mqnic_activate_rx_ring(priv->rxq[k], priv, priv->rx_cq[k]);
ret = mqnic_open_rx_ring(q, priv, cq, mqnic_num_rxq_entries, 1);
if (ret) {
mqnic_destroy_rx_ring(q);
mqnic_destroy_cq(cq);
goto fail;
}
down_write(&priv->rxq_table_sem);
ret = radix_tree_insert(&priv->rxq_table, k, q);
up_write(&priv->rxq_table_sem);
if (ret) {
mqnic_destroy_rx_ring(q);
mqnic_destroy_cq(cq);
goto fail;
}
} }
// set up TX queues // set up TX queues
for (k = 0; k < min(priv->txq_count, priv->tx_cq_count); k++) { for (k = 0; k < priv->txq_count; k++) {
// set up CQ // create CQ
mqnic_activate_cq(priv->tx_cq[k], priv->eq[k % priv->eq_count]); cq = mqnic_create_cq(iface);
if (IS_ERR_OR_NULL(cq)) {
ret = PTR_ERR(cq);
goto fail;
}
ret = mqnic_open_cq(cq, iface->eq[k % iface->eq_count], mqnic_num_txq_entries, 1);
if (ret) {
mqnic_destroy_cq(cq);
goto fail;
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0) #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0)
netif_napi_add_tx(ndev, &priv->tx_cq[k]->napi, mqnic_poll_tx_cq); netif_napi_add_tx(ndev, &cq->napi, mqnic_poll_tx_cq);
#else #else
netif_tx_napi_add(ndev, &priv->tx_cq[k]->napi, mqnic_poll_tx_cq, NAPI_POLL_WEIGHT); netif_tx_napi_add(ndev, &cq->napi, mqnic_poll_tx_cq, NAPI_POLL_WEIGHT);
#endif #endif
napi_enable(&priv->tx_cq[k]->napi); napi_enable(&cq->napi);
mqnic_arm_cq(priv->tx_cq[k]); mqnic_arm_cq(cq);
// set up queue // create TX queue
priv->txq[k]->tx_queue = netdev_get_tx_queue(ndev, k); q = mqnic_create_tx_ring(iface);
mqnic_activate_tx_ring(priv->txq[k], priv, priv->tx_cq[k]); if (IS_ERR_OR_NULL(q)) {
ret = PTR_ERR(q);
mqnic_destroy_cq(cq);
goto fail;
}
q->tx_queue = netdev_get_tx_queue(ndev, k);
ret = mqnic_open_tx_ring(q, priv, cq, mqnic_num_txq_entries, desc_block_size);
if (ret) {
mqnic_destroy_tx_ring(q);
mqnic_destroy_cq(cq);
goto fail;
}
down_write(&priv->txq_table_sem);
ret = radix_tree_insert(&priv->txq_table, k, q);
up_write(&priv->txq_table_sem);
if (ret) {
mqnic_destroy_tx_ring(q);
mqnic_destroy_cq(cq);
goto fail;
}
} }
// set MTU // set MTU
mqnic_interface_set_tx_mtu(priv->interface, ndev->mtu + ETH_HLEN); mqnic_interface_set_tx_mtu(iface, ndev->mtu + ETH_HLEN);
mqnic_interface_set_rx_mtu(priv->interface, ndev->mtu + ETH_HLEN); mqnic_interface_set_rx_mtu(iface, ndev->mtu + ETH_HLEN);
// configure RX indirection and RSS // configure RX indirection and RSS
mqnic_interface_set_rx_queue_map_rss_mask(priv->interface, 0, 0xffffffff); mqnic_interface_set_rx_queue_map_rss_mask(iface, 0, 0xffffffff);
mqnic_interface_set_rx_queue_map_app_mask(priv->interface, 0, 0); mqnic_interface_set_rx_queue_map_app_mask(iface, 0, 0);
for (k = 0; k < priv->interface->rx_queue_map_indir_table_size; k++) { for (k = 0; k < iface->rx_queue_map_indir_table_size; k++) {
mqnic_interface_set_rx_queue_map_indir_table(priv->interface, 0, k, priv->rxq[k % priv->rxq_count]->index); rcu_read_lock();
q = radix_tree_lookup(&priv->rxq_table, k % priv->rxq_count);
rcu_read_unlock();
mqnic_interface_set_rx_queue_map_indir_table(iface, 0, k, q->index);
} }
priv->port_up = true;
// enable TX and RX queues
down_read(&priv->txq_table_sem);
radix_tree_for_each_slot(slot, &priv->txq_table, &iter, 0) {
struct mqnic_ring *q = (struct mqnic_ring *)*slot;
mqnic_enable_tx_ring(q);
}
up_read(&priv->txq_table_sem);
down_read(&priv->rxq_table_sem);
radix_tree_for_each_slot(slot, &priv->rxq_table, &iter, 0) {
struct mqnic_ring *q = (struct mqnic_ring *)*slot;
mqnic_enable_rx_ring(q);
}
up_read(&priv->rxq_table_sem);
// enable first scheduler // enable first scheduler
mqnic_activate_sched_block(priv->sched_block[0]); mqnic_activate_sched_block(priv->sched_block[0]);
priv->port_up = true;
netif_tx_start_all_queues(ndev); netif_tx_start_all_queues(ndev);
netif_device_attach(ndev); netif_device_attach(ndev);
@ -117,12 +212,19 @@ static int mqnic_start_port(struct net_device *ndev)
} }
return 0; return 0;
fail:
mqnic_stop_port(ndev);
return ret;
} }
static int mqnic_stop_port(struct net_device *ndev) static int mqnic_stop_port(struct net_device *ndev)
{ {
struct mqnic_priv *priv = netdev_priv(ndev); struct mqnic_priv *priv = netdev_priv(ndev);
struct mqnic_dev *mdev = priv->mdev; struct mqnic_dev *mdev = priv->mdev;
struct mqnic_cq *cq;
struct radix_tree_iter iter;
void **slot;
int k; int k;
dev_info(mdev->dev, "%s on interface %d netdev %d", __func__, dev_info(mdev->dev, "%s on interface %d netdev %d", __func__,
@ -142,7 +244,6 @@ static int mqnic_stop_port(struct net_device *ndev)
spin_lock_bh(&priv->stats_lock); spin_lock_bh(&priv->stats_lock);
mqnic_update_stats(ndev); mqnic_update_stats(ndev);
priv->port_up = false;
spin_unlock_bh(&priv->stats_lock); spin_unlock_bh(&priv->stats_lock);
// disable schedulers // disable schedulers
@ -150,28 +251,56 @@ static int mqnic_stop_port(struct net_device *ndev)
mqnic_deactivate_sched_block(priv->sched_block[k]); mqnic_deactivate_sched_block(priv->sched_block[k]);
// disable TX and RX queues // disable TX and RX queues
for (k = 0; k < min(priv->txq_count, priv->tx_cq_count); k++) down_read(&priv->txq_table_sem);
mqnic_deactivate_tx_ring(priv->txq[k]); radix_tree_for_each_slot(slot, &priv->txq_table, &iter, 0) {
struct mqnic_ring *q = (struct mqnic_ring *)*slot;
for (k = 0; k < min(priv->rxq_count, priv->rx_cq_count); k++) mqnic_disable_tx_ring(q);
mqnic_deactivate_rx_ring(priv->rxq[k]); }
up_read(&priv->txq_table_sem);
down_read(&priv->rxq_table_sem);
radix_tree_for_each_slot(slot, &priv->rxq_table, &iter, 0) {
struct mqnic_ring *q = (struct mqnic_ring *)*slot;
mqnic_disable_rx_ring(q);
}
up_read(&priv->rxq_table_sem);
msleep(20); msleep(20);
// shut down NAPI and clean queues priv->port_up = false;
for (k = 0; k < priv->txq_count; k++) {
napi_disable(&priv->tx_cq[k]->napi);
mqnic_deactivate_cq(priv->tx_cq[k]);
netif_napi_del(&priv->tx_cq[k]->napi);
mqnic_free_tx_buf(priv->txq[k]);
}
for (k = 0; k < priv->rxq_count; k++) { // shut down NAPI and clean queues
napi_disable(&priv->rx_cq[k]->napi); down_write(&priv->txq_table_sem);
mqnic_deactivate_cq(priv->rx_cq[k]); radix_tree_for_each_slot(slot, &priv->txq_table, &iter, 0) {
netif_napi_del(&priv->rx_cq[k]->napi); struct mqnic_ring *q = (struct mqnic_ring *)*slot;
mqnic_free_rx_buf(priv->rxq[k]);
cq = q->cq;
napi_disable(&cq->napi);
netif_napi_del(&cq->napi);
mqnic_close_tx_ring(q);
mqnic_destroy_tx_ring(q);
radix_tree_delete(&priv->txq_table, iter.index);
mqnic_close_cq(cq);
mqnic_destroy_cq(cq);
} }
up_write(&priv->txq_table_sem);
down_write(&priv->rxq_table_sem);
radix_tree_for_each_slot(slot, &priv->rxq_table, &iter, 0) {
struct mqnic_ring *q = (struct mqnic_ring *)*slot;
cq = q->cq;
napi_disable(&cq->napi);
netif_napi_del(&cq->napi);
mqnic_close_rx_ring(q);
mqnic_destroy_rx_ring(q);
radix_tree_delete(&priv->rxq_table, iter.index);
mqnic_close_cq(cq);
mqnic_destroy_cq(cq);
}
up_write(&priv->rxq_table_sem);
return 0; return 0;
} }
@ -215,31 +344,36 @@ static int mqnic_close(struct net_device *ndev)
void mqnic_update_stats(struct net_device *ndev) void mqnic_update_stats(struct net_device *ndev)
{ {
struct mqnic_priv *priv = netdev_priv(ndev); struct mqnic_priv *priv = netdev_priv(ndev);
struct radix_tree_iter iter;
void **slot;
unsigned long packets, bytes; unsigned long packets, bytes;
int k;
if (unlikely(!priv->port_up)) if (unlikely(!priv->port_up))
return; return;
packets = 0; packets = 0;
bytes = 0; bytes = 0;
for (k = 0; k < priv->rxq_count; k++) { down_read(&priv->rxq_table_sem);
const struct mqnic_ring *ring = priv->rxq[k]; radix_tree_for_each_slot(slot, &priv->rxq_table, &iter, 0) {
const struct mqnic_ring *q = (struct mqnic_ring *)*slot;
packets += READ_ONCE(ring->packets); packets += READ_ONCE(q->packets);
bytes += READ_ONCE(ring->bytes); bytes += READ_ONCE(q->bytes);
} }
up_read(&priv->rxq_table_sem);
ndev->stats.rx_packets = packets; ndev->stats.rx_packets = packets;
ndev->stats.rx_bytes = bytes; ndev->stats.rx_bytes = bytes;
packets = 0; packets = 0;
bytes = 0; bytes = 0;
for (k = 0; k < priv->txq_count; k++) { down_read(&priv->txq_table_sem);
const struct mqnic_ring *ring = priv->txq[k]; radix_tree_for_each_slot(slot, &priv->txq_table, &iter, 0) {
const struct mqnic_ring *q = (struct mqnic_ring *)*slot;
packets += READ_ONCE(ring->packets); packets += READ_ONCE(q->packets);
bytes += READ_ONCE(ring->bytes); bytes += READ_ONCE(q->bytes);
} }
up_read(&priv->txq_table_sem);
ndev->stats.tx_packets = packets; ndev->stats.tx_packets = packets;
ndev->stats.tx_bytes = bytes; ndev->stats.tx_bytes = bytes;
} }
@ -437,25 +571,14 @@ struct net_device *mqnic_create_netdev(struct mqnic_if *interface, int index, in
// associate interface resources // associate interface resources
priv->if_features = interface->if_features; priv->if_features = interface->if_features;
priv->eq_count = interface->eq_count; priv->txq_count = mqnic_res_get_count(interface->txq_res);
for (k = 0; k < interface->eq_count; k++) priv->rxq_count = mqnic_res_get_count(interface->rxq_res);
priv->eq[k] = interface->eq[k];
priv->txq_count = interface->txq_count; init_rwsem(&priv->txq_table_sem);
for (k = 0; k < interface->txq_count; k++) INIT_RADIX_TREE(&priv->txq_table, GFP_KERNEL);
priv->txq[k] = interface->txq[k];
priv->tx_cq_count = interface->tx_cq_count; init_rwsem(&priv->rxq_table_sem);
for (k = 0; k < interface->tx_cq_count; k++) INIT_RADIX_TREE(&priv->rxq_table, GFP_KERNEL);
priv->tx_cq[k] = interface->tx_cq[k];
priv->rxq_count = interface->rxq_count;
for (k = 0; k < interface->rxq_count; k++)
priv->rxq[k] = interface->rxq[k];
priv->rx_cq_count = interface->rx_cq_count;
for (k = 0; k < interface->rx_cq_count; k++)
priv->rx_cq[k] = interface->rx_cq[k];
priv->sched_block_count = interface->sched_block_count; priv->sched_block_count = interface->sched_block_count;
for (k = 0; k < interface->sched_block_count; k++) for (k = 0; k < interface->sched_block_count; k++)
@ -489,36 +612,6 @@ struct net_device *mqnic_create_netdev(struct mqnic_if *interface, int index, in
desc_block_size = min_t(u32, interface->max_desc_block_size, 4); desc_block_size = min_t(u32, interface->max_desc_block_size, 4);
// allocate ring buffers
for (k = 0; k < priv->txq_count; k++) {
ret = mqnic_alloc_tx_ring(priv->txq[k], mqnic_num_txq_entries,
MQNIC_DESC_SIZE * desc_block_size);
if (ret)
goto fail;
}
for (k = 0; k < priv->tx_cq_count; k++) {
ret = mqnic_alloc_cq(priv->tx_cq[k], mqnic_num_txq_entries,
MQNIC_CPL_SIZE);
if (ret)
goto fail;
}
for (k = 0; k < priv->rxq_count; k++) {
ret = mqnic_alloc_rx_ring(priv->rxq[k], mqnic_num_rxq_entries,
MQNIC_DESC_SIZE);
if (ret)
goto fail;
}
for (k = 0; k < priv->rx_cq_count; k++) {
ret = mqnic_alloc_cq(priv->rx_cq[k], mqnic_num_rxq_entries,
MQNIC_CPL_SIZE);
if (ret)
goto fail;
}
// entry points // entry points
ndev->netdev_ops = &mqnic_netdev_ops; ndev->netdev_ops = &mqnic_netdev_ops;
ndev->ethtool_ops = &mqnic_ethtool_ops; ndev->ethtool_ops = &mqnic_ethtool_ops;
@ -564,27 +657,9 @@ fail:
void mqnic_destroy_netdev(struct net_device *ndev) void mqnic_destroy_netdev(struct net_device *ndev)
{ {
struct mqnic_priv *priv = netdev_priv(ndev); struct mqnic_priv *priv = netdev_priv(ndev);
int k;
if (priv->registered) if (priv->registered)
unregister_netdev(ndev); unregister_netdev(ndev);
// free rings
for (k = 0; k < ARRAY_SIZE(priv->txq); k++)
if (priv->txq[k])
mqnic_free_tx_ring(priv->txq[k]);
for (k = 0; k < ARRAY_SIZE(priv->tx_cq); k++)
if (priv->tx_cq[k])
mqnic_free_cq(priv->tx_cq[k]);
for (k = 0; k < ARRAY_SIZE(priv->rxq); k++)
if (priv->rxq[k])
mqnic_free_rx_ring(priv->rxq[k]);
for (k = 0; k < ARRAY_SIZE(priv->rx_cq); k++)
if (priv->rx_cq[k])
mqnic_free_cq(priv->rx_cq[k]);
free_netdev(ndev); free_netdev(ndev);
} }

36
modules/mqnic/mqnic_res.c
View File

@ -38,6 +38,7 @@
struct mqnic_res *mqnic_create_res(unsigned int count, u8 __iomem *base, unsigned int stride) struct mqnic_res *mqnic_create_res(unsigned int count, u8 __iomem *base, unsigned int stride)
{ {
struct mqnic_res *res; struct mqnic_res *res;
int ret;
res = kzalloc(sizeof(*res), GFP_KERNEL); res = kzalloc(sizeof(*res), GFP_KERNEL);
if (!res) if (!res)
@ -47,7 +48,19 @@ struct mqnic_res *mqnic_create_res(unsigned int count, u8 __iomem *base, unsigne
res->base = base; res->base = base;
res->stride = stride; res->stride = stride;
spin_lock_init(&res->lock);
res->bmap = bitmap_zalloc(count, GFP_KERNEL);
if (!res) {
ret = -ENOMEM;
goto fail;
}
return res; return res;
fail:
mqnic_destroy_res(res);
return ERR_PTR(ret);
} }
void mqnic_destroy_res(struct mqnic_res *res) void mqnic_destroy_res(struct mqnic_res *res)
@ -55,9 +68,32 @@ void mqnic_destroy_res(struct mqnic_res *res)
if (!res) if (!res)
return; return;
bitmap_free(res->bmap);
kfree(res); kfree(res);
} }
int mqnic_res_alloc(struct mqnic_res *res)
{
int index;
spin_lock(&res->lock);
index = bitmap_find_free_region(res->bmap, res->count, 0);
spin_unlock(&res->lock);
return index;
}
void mqnic_res_free(struct mqnic_res *res, int index)
{
if (index < 0 || index >= res->count)
return;
spin_lock(&res->lock);
bitmap_clear(res->bmap, index, 1);
spin_unlock(&res->lock);
}
unsigned int mqnic_res_get_count(struct mqnic_res *res) unsigned int mqnic_res_get_count(struct mqnic_res *res)
{ {
return res->count; return res->count;

124
modules/mqnic/mqnic_rx.c
View File

@ -35,8 +35,7 @@
#include "mqnic.h" #include "mqnic.h"
struct mqnic_ring *mqnic_create_rx_ring(struct mqnic_if *interface, struct mqnic_ring *mqnic_create_rx_ring(struct mqnic_if *interface)
int index, u8 __iomem *hw_addr)
{ {
struct mqnic_ring *ring; struct mqnic_ring *ring;
@ -47,48 +46,52 @@ struct mqnic_ring *mqnic_create_rx_ring(struct mqnic_if *interface,
ring->dev = interface->dev; ring->dev = interface->dev;
ring->interface = interface; ring->interface = interface;
ring->index = index; ring->index = -1;
ring->active = 0; ring->enabled = 0;
ring->hw_addr = hw_addr; ring->hw_addr = NULL;
ring->hw_ptr_mask = 0xffff; ring->hw_ptr_mask = 0xffff;
ring->hw_head_ptr = hw_addr + MQNIC_QUEUE_HEAD_PTR_REG; ring->hw_head_ptr = NULL;
ring->hw_tail_ptr = hw_addr + MQNIC_QUEUE_TAIL_PTR_REG; ring->hw_tail_ptr = NULL;
ring->head_ptr = 0; ring->head_ptr = 0;
ring->tail_ptr = 0; ring->tail_ptr = 0;
// deactivate queue
iowrite32(0, ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
return ring; return ring;
} }
void mqnic_destroy_rx_ring(struct mqnic_ring *ring) void mqnic_destroy_rx_ring(struct mqnic_ring *ring)
{ {
mqnic_free_rx_ring(ring); mqnic_close_rx_ring(ring);
kfree(ring); kfree(ring);
} }
int mqnic_alloc_rx_ring(struct mqnic_ring *ring, int size, int stride) int mqnic_open_rx_ring(struct mqnic_ring *ring, struct mqnic_priv *priv,
struct mqnic_cq *cq, int size, int desc_block_size)
{ {
int ret = 0; int ret = 0;
if (ring->active || ring->buf) if (ring->enabled || ring->hw_addr || ring->buf || !priv || !cq)
return -EINVAL; return -EINVAL;
ring->size = roundup_pow_of_two(size); ring->index = mqnic_res_alloc(ring->interface->rxq_res);
ring->size_mask = ring->size - 1; if (ring->index < 0)
ring->stride = roundup_pow_of_two(stride); return -ENOMEM;
ring->desc_block_size = ring->stride / MQNIC_DESC_SIZE; ring->log_desc_block_size = desc_block_size < 2 ? 0 : ilog2(desc_block_size - 1) + 1;
ring->log_desc_block_size = ring->desc_block_size < 2 ? 0 : ilog2(ring->desc_block_size - 1) + 1;
ring->desc_block_size = 1 << ring->log_desc_block_size; ring->desc_block_size = 1 << ring->log_desc_block_size;
ring->size = roundup_pow_of_two(size);
ring->full_size = ring->size >> 1;
ring->size_mask = ring->size - 1;
ring->stride = roundup_pow_of_two(MQNIC_DESC_SIZE * ring->desc_block_size);
ring->rx_info = kvzalloc(sizeof(*ring->rx_info) * ring->size, GFP_KERNEL); ring->rx_info = kvzalloc(sizeof(*ring->rx_info) * ring->size, GFP_KERNEL);
if (!ring->rx_info) if (!ring->rx_info) {
return -ENOMEM; ret = -ENOMEM;
goto fail;
}
ring->buf_size = ring->size * ring->stride; ring->buf_size = ring->size * ring->stride;
ring->buf = dma_alloc_coherent(ring->dev, ring->buf_size, &ring->buf_dma_addr, GFP_KERNEL); ring->buf = dma_alloc_coherent(ring->dev, ring->buf_size, &ring->buf_dma_addr, GFP_KERNEL);
@ -97,6 +100,15 @@ int mqnic_alloc_rx_ring(struct mqnic_ring *ring, int size, int stride)
goto fail; goto fail;
} }
ring->priv = priv;
ring->cq = cq;
cq->src_ring = ring;
cq->handler = mqnic_rx_irq;
ring->hw_addr = mqnic_res_get_addr(ring->interface->rxq_res, ring->index);
ring->hw_head_ptr = ring->hw_addr + MQNIC_QUEUE_HEAD_PTR_REG;
ring->hw_tail_ptr = ring->hw_addr + MQNIC_QUEUE_TAIL_PTR_REG;
ring->head_ptr = 0; ring->head_ptr = 0;
ring->tail_ptr = 0; ring->tail_ptr = 0;
@ -105,8 +117,8 @@ int mqnic_alloc_rx_ring(struct mqnic_ring *ring, int size, int stride)
// set base address // set base address
iowrite32(ring->buf_dma_addr, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 0); iowrite32(ring->buf_dma_addr, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 0);
iowrite32(ring->buf_dma_addr >> 32, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 4); iowrite32(ring->buf_dma_addr >> 32, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 4);
// set completion queue index // set CQN
iowrite32(0, ring->hw_addr + MQNIC_QUEUE_CPL_QUEUE_INDEX_REG); iowrite32(ring->cq->cqn, ring->hw_addr + MQNIC_QUEUE_CPL_QUEUE_INDEX_REG);
// set pointers // set pointers
iowrite32(ring->head_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_HEAD_PTR_REG); iowrite32(ring->head_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_HEAD_PTR_REG);
iowrite32(ring->tail_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_TAIL_PTR_REG); iowrite32(ring->tail_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_TAIL_PTR_REG);
@ -114,16 +126,30 @@ int mqnic_alloc_rx_ring(struct mqnic_ring *ring, int size, int stride)
iowrite32(ilog2(ring->size) | (ring->log_desc_block_size << 8), iowrite32(ilog2(ring->size) | (ring->log_desc_block_size << 8),
ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG); ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
mqnic_refill_rx_buffers(ring);
return 0; return 0;
fail: fail:
mqnic_free_rx_ring(ring); mqnic_close_rx_ring(ring);
return ret; return ret;
} }
void mqnic_free_rx_ring(struct mqnic_ring *ring) void mqnic_close_rx_ring(struct mqnic_ring *ring)
{ {
mqnic_deactivate_rx_ring(ring); mqnic_disable_rx_ring(ring);
if (ring->cq) {
ring->cq->src_ring = NULL;
ring->cq->handler = NULL;
}
ring->priv = NULL;
ring->cq = NULL;
ring->hw_addr = NULL;
ring->hw_head_ptr = NULL;
ring->hw_tail_ptr = NULL;
if (ring->buf) { if (ring->buf) {
mqnic_free_rx_buf(ring); mqnic_free_rx_buf(ring);
@ -137,60 +163,34 @@ void mqnic_free_rx_ring(struct mqnic_ring *ring)
kvfree(ring->rx_info); kvfree(ring->rx_info);
ring->rx_info = NULL; ring->rx_info = NULL;
} }
mqnic_res_free(ring->interface->rxq_res, ring->index);
ring->index = -1;
} }
int mqnic_activate_rx_ring(struct mqnic_ring *ring, struct mqnic_priv *priv, int mqnic_enable_rx_ring(struct mqnic_ring *ring)
struct mqnic_cq *cq)
{ {
mqnic_deactivate_rx_ring(ring); if (!ring->hw_addr)
if (!ring->buf || !priv || !cq || cq->handler || cq->src_ring)
return -EINVAL; return -EINVAL;
ring->priv = priv; // enable queue
ring->cq = cq;
cq->src_ring = ring;
cq->handler = mqnic_rx_irq;
ring->head_ptr = 0;
ring->tail_ptr = 0;
// deactivate queue
iowrite32(0, ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
// set base address
iowrite32(ring->buf_dma_addr, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 0);
iowrite32(ring->buf_dma_addr >> 32, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 4);
// set CQN
iowrite32(cq->cqn, ring->hw_addr + MQNIC_QUEUE_CPL_QUEUE_INDEX_REG);
// set pointers
iowrite32(ring->head_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_HEAD_PTR_REG);
iowrite32(ring->tail_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_TAIL_PTR_REG);
// set size and activate queue
iowrite32(ilog2(ring->size) | (ring->log_desc_block_size << 8) | MQNIC_QUEUE_ACTIVE_MASK, iowrite32(ilog2(ring->size) | (ring->log_desc_block_size << 8) | MQNIC_QUEUE_ACTIVE_MASK,
ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG); ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
ring->active = 1; ring->enabled = 1;
mqnic_refill_rx_buffers(ring);
return 0; return 0;
} }
void mqnic_deactivate_rx_ring(struct mqnic_ring *ring) void mqnic_disable_rx_ring(struct mqnic_ring *ring)
{ {
// deactivate queue // disable queue
if (ring->hw_addr) {
iowrite32(ilog2(ring->size) | (ring->log_desc_block_size << 8), iowrite32(ilog2(ring->size) | (ring->log_desc_block_size << 8),
ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG); ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
if (ring->cq) {
ring->cq->src_ring = NULL;
ring->cq->handler = NULL;
} }
ring->priv = NULL; ring->enabled = 0;
ring->cq = NULL;
ring->active = 0;
} }
bool mqnic_is_rx_ring_empty(const struct mqnic_ring *ring) bool mqnic_is_rx_ring_empty(const struct mqnic_ring *ring)

2
modules/mqnic/mqnic_sched_block.c
View File

@ -53,8 +53,6 @@ struct mqnic_sched_block *mqnic_create_sched_block(struct mqnic_if *interface,
block->index = index; block->index = index;
block->txq_count = interface->txq_count;
block->block_rb = block_rb; block->block_rb = block_rb;
offset = ioread32(block_rb->regs + MQNIC_RB_SCHED_BLOCK_REG_OFFSET); offset = ioread32(block_rb->regs + MQNIC_RB_SCHED_BLOCK_REG_OFFSET);

129
modules/mqnic/mqnic_tx.c
View File

@ -36,8 +36,7 @@
#include <linux/version.h> #include <linux/version.h>
#include "mqnic.h" #include "mqnic.h"
struct mqnic_ring *mqnic_create_tx_ring(struct mqnic_if *interface, struct mqnic_ring *mqnic_create_tx_ring(struct mqnic_if *interface)
int index, u8 __iomem *hw_addr)
{ {
struct mqnic_ring *ring; struct mqnic_ring *ring;
@ -48,49 +47,52 @@ struct mqnic_ring *mqnic_create_tx_ring(struct mqnic_if *interface,
ring->dev = interface->dev; ring->dev = interface->dev;
ring->interface = interface; ring->interface = interface;
ring->index = index; ring->index = -1;
ring->active = 0; ring->enabled = 0;
ring->hw_addr = hw_addr; ring->hw_addr = NULL;
ring->hw_ptr_mask = 0xffff; ring->hw_ptr_mask = 0xffff;
ring->hw_head_ptr = hw_addr + MQNIC_QUEUE_HEAD_PTR_REG; ring->hw_head_ptr = NULL;
ring->hw_tail_ptr = hw_addr + MQNIC_QUEUE_TAIL_PTR_REG; ring->hw_tail_ptr = NULL;
ring->head_ptr = 0; ring->head_ptr = 0;
ring->tail_ptr = 0; ring->tail_ptr = 0;
// deactivate queue
iowrite32(0, ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
return ring; return ring;
} }
void mqnic_destroy_tx_ring(struct mqnic_ring *ring) void mqnic_destroy_tx_ring(struct mqnic_ring *ring)
{ {
mqnic_free_tx_ring(ring); mqnic_close_tx_ring(ring);
kfree(ring); kfree(ring);
} }
int mqnic_alloc_tx_ring(struct mqnic_ring *ring, int size, int stride) int mqnic_open_tx_ring(struct mqnic_ring *ring, struct mqnic_priv *priv,
struct mqnic_cq *cq, int size, int desc_block_size)
{ {
int ret = 0; int ret = 0;
if (ring->active || ring->buf) if (ring->enabled || ring->hw_addr || ring->buf || !priv || !cq)
return -EINVAL; return -EINVAL;
ring->index = mqnic_res_alloc(ring->interface->txq_res);
if (ring->index < 0)
return -ENOMEM;
ring->log_desc_block_size = desc_block_size < 2 ? 0 : ilog2(desc_block_size - 1) + 1;
ring->desc_block_size = 1 << ring->log_desc_block_size;
ring->size = roundup_pow_of_two(size); ring->size = roundup_pow_of_two(size);
ring->full_size = ring->size >> 1; ring->full_size = ring->size >> 1;
ring->size_mask = ring->size - 1; ring->size_mask = ring->size - 1;
ring->stride = roundup_pow_of_two(stride); ring->stride = roundup_pow_of_two(MQNIC_DESC_SIZE * ring->desc_block_size);
ring->desc_block_size = ring->stride / MQNIC_DESC_SIZE;
ring->log_desc_block_size = ring->desc_block_size < 2 ? 0 : ilog2(ring->desc_block_size - 1) + 1;
ring->desc_block_size = 1 << ring->log_desc_block_size;
ring->tx_info = kvzalloc(sizeof(*ring->tx_info) * ring->size, GFP_KERNEL); ring->tx_info = kvzalloc(sizeof(*ring->tx_info) * ring->size, GFP_KERNEL);
if (!ring->tx_info) if (!ring->tx_info) {
return -ENOMEM; ret = -ENOMEM;
goto fail;
}
ring->buf_size = ring->size * ring->stride; ring->buf_size = ring->size * ring->stride;
ring->buf = dma_alloc_coherent(ring->dev, ring->buf_size, &ring->buf_dma_addr, GFP_KERNEL); ring->buf = dma_alloc_coherent(ring->dev, ring->buf_size, &ring->buf_dma_addr, GFP_KERNEL);
@ -99,6 +101,15 @@ int mqnic_alloc_tx_ring(struct mqnic_ring *ring, int size, int stride)
goto fail; goto fail;
} }
ring->priv = priv;
ring->cq = cq;
cq->src_ring = ring;
cq->handler = mqnic_tx_irq;
ring->hw_addr = mqnic_res_get_addr(ring->interface->txq_res, ring->index);
ring->hw_head_ptr = ring->hw_addr + MQNIC_QUEUE_HEAD_PTR_REG;
ring->hw_tail_ptr = ring->hw_addr + MQNIC_QUEUE_TAIL_PTR_REG;
ring->head_ptr = 0; ring->head_ptr = 0;
ring->tail_ptr = 0; ring->tail_ptr = 0;
@ -107,8 +118,8 @@ int mqnic_alloc_tx_ring(struct mqnic_ring *ring, int size, int stride)
// set base address // set base address
iowrite32(ring->buf_dma_addr, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 0); iowrite32(ring->buf_dma_addr, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 0);
iowrite32(ring->buf_dma_addr >> 32, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 4); iowrite32(ring->buf_dma_addr >> 32, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 4);
// set completion queue index // set CQN
iowrite32(0, ring->hw_addr + MQNIC_QUEUE_CPL_QUEUE_INDEX_REG); iowrite32(ring->cq->cqn, ring->hw_addr + MQNIC_QUEUE_CPL_QUEUE_INDEX_REG);
// set pointers // set pointers
iowrite32(ring->head_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_HEAD_PTR_REG); iowrite32(ring->head_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_HEAD_PTR_REG);
iowrite32(ring->tail_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_TAIL_PTR_REG); iowrite32(ring->tail_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_TAIL_PTR_REG);
@ -119,13 +130,25 @@ int mqnic_alloc_tx_ring(struct mqnic_ring *ring, int size, int stride)
return 0; return 0;
fail: fail:
mqnic_free_tx_ring(ring); mqnic_close_tx_ring(ring);
return ret; return ret;
} }
void mqnic_free_tx_ring(struct mqnic_ring *ring) void mqnic_close_tx_ring(struct mqnic_ring *ring)
{ {
mqnic_deactivate_tx_ring(ring); mqnic_disable_tx_ring(ring);
if (ring->cq) {
ring->cq->src_ring = NULL;
ring->cq->handler = NULL;
}
ring->priv = NULL;
ring->cq = NULL;
ring->hw_addr = NULL;
ring->hw_head_ptr = NULL;
ring->hw_tail_ptr = NULL;
if (ring->buf) { if (ring->buf) {
mqnic_free_tx_buf(ring); mqnic_free_tx_buf(ring);
@ -139,58 +162,34 @@ void mqnic_free_tx_ring(struct mqnic_ring *ring)
kvfree(ring->tx_info); kvfree(ring->tx_info);
ring->tx_info = NULL; ring->tx_info = NULL;
} }
mqnic_res_free(ring->interface->txq_res, ring->index);
ring->index = -1;
} }
int mqnic_activate_tx_ring(struct mqnic_ring *ring, struct mqnic_priv *priv, int mqnic_enable_tx_ring(struct mqnic_ring *ring)
struct mqnic_cq *cq)
{ {
mqnic_deactivate_tx_ring(ring); if (!ring->hw_addr)
if (!ring->buf || !priv || !cq || cq->handler || cq->src_ring)
return -EINVAL; return -EINVAL;
ring->priv = priv; // enable queue
ring->cq = cq;
cq->src_ring = ring;
cq->handler = mqnic_tx_irq;
ring->head_ptr = 0;
ring->tail_ptr = 0;
// deactivate queue
iowrite32(0, ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
// set base address
iowrite32(ring->buf_dma_addr, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 0);
iowrite32(ring->buf_dma_addr >> 32, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 4);
// set CQN
iowrite32(cq->cqn, ring->hw_addr + MQNIC_QUEUE_CPL_QUEUE_INDEX_REG);
// set pointers
iowrite32(ring->head_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_HEAD_PTR_REG);
iowrite32(ring->tail_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_TAIL_PTR_REG);
// set size and activate queue
iowrite32(ilog2(ring->size) | (ring->log_desc_block_size << 8) | MQNIC_QUEUE_ACTIVE_MASK, iowrite32(ilog2(ring->size) | (ring->log_desc_block_size << 8) | MQNIC_QUEUE_ACTIVE_MASK,
ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG); ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
ring->active = 1; ring->enabled = 1;
return 0; return 0;
} }
void mqnic_deactivate_tx_ring(struct mqnic_ring *ring) void mqnic_disable_tx_ring(struct mqnic_ring *ring)
{ {
// deactivate queue // disable queue
if (ring->hw_addr) {
iowrite32(ilog2(ring->size) | (ring->log_desc_block_size << 8), iowrite32(ilog2(ring->size) | (ring->log_desc_block_size << 8),
ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG); ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
if (ring->cq) {
ring->cq->src_ring = NULL;
ring->cq->handler = NULL;
} }
ring->priv = NULL; ring->enabled = 0;
ring->cq = NULL;
ring->active = 0;
} }
bool mqnic_is_tx_ring_empty(const struct mqnic_ring *ring) bool mqnic_is_tx_ring_empty(const struct mqnic_ring *ring)
@ -444,11 +443,13 @@ netdev_tx_t mqnic_start_xmit(struct sk_buff *skb, struct net_device *ndev)
ring_index = skb_get_queue_mapping(skb); ring_index = skb_get_queue_mapping(skb);
if (unlikely(ring_index >= priv->txq_count)) rcu_read_lock();
// queue mapping out of range ring = radix_tree_lookup(&priv->txq_table, ring_index);
goto tx_drop; rcu_read_unlock();
ring = priv->txq[ring_index]; if (unlikely(!ring))
// unknown TX queue
goto tx_drop;
tail_ptr = READ_ONCE(ring->tail_ptr); tail_ptr = READ_ONCE(ring->tail_ptr);