diff --git a/fpga/common/tb/mqnic.py b/fpga/common/tb/mqnic.py index 8737e194d..7703e3c88 100644 --- a/fpga/common/tb/mqnic.py +++ b/fpga/common/tb/mqnic.py @@ -319,6 +319,14 @@ class Resource: self.stride = stride self.windows = {} + self.free_list = list(range(count)) + + def alloc(self): + return self.free_list.pop(0) + + def free(self, index): + self.free_list.append(index) + self.free_list.sort() def get_count(self): return self.count @@ -401,7 +409,7 @@ class Packet: class Eq: - def __init__(self, interface, eqn, hw_regs): + def __init__(self, interface): self.interface = interface self.log = interface.log self.driver = interface.driver @@ -409,8 +417,8 @@ class Eq: self.size = 0 self.size_mask = 0 self.stride = 0 - self.eqn = eqn - self.active = False + self.eqn = None + self.enabled = False self.buf_size = 0 self.buf_region = None @@ -419,80 +427,82 @@ class Eq: self.irq = None + self.cq_table = {} + self.head_ptr = 0 self.tail_ptr = 0 self.hw_ptr_mask = 0xffff - self.hw_regs = hw_regs + self.hw_regs = None - async def init(self): - self.log.info("Init EQ %d (interface %d)", self.eqn, self.interface.index) + async def open(self, irq, size): + if self.hw_regs: + raise Exception("Already open") - await self.hw_regs.write_dword(MQNIC_EQ_ACTIVE_LOG_SIZE_REG, 0) # active, log size + self.eqn = self.interface.eq_res.alloc() - async def alloc(self, size, stride): - if self.active: - raise Exception("Cannot allocate active ring") - if self.buf: - raise Exception("Already allocated") + self.log.info("Open EQ %d (interface %d)", self.eqn, self.interface.index) self.log_size = size.bit_length() - 1 self.size = 2**self.log_size self.size_mask = self.size-1 - self.stride = stride + self.stride = MQNIC_EVENT_SIZE self.buf_size = self.size*self.stride self.buf_region = self.driver.pool.alloc_region(self.buf_size) self.buf_dma = self.buf_region.get_absolute_address(0) self.buf = self.buf_region.mem + self.buf[0:self.buf_size] = b'\x00'*self.buf_size + self.head_ptr = 0 self.tail_ptr = 0 - await self.hw_regs.write_dword(MQNIC_EQ_ACTIVE_LOG_SIZE_REG, 0) # active, log size - await self.hw_regs.write_dword(MQNIC_EQ_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address - await self.hw_regs.write_dword(MQNIC_EQ_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address - await self.hw_regs.write_dword(MQNIC_EQ_INTERRUPT_INDEX_REG, 0) # interrupt index - await self.hw_regs.write_dword(MQNIC_EQ_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer - await self.hw_regs.write_dword(MQNIC_EQ_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer - await self.hw_regs.write_dword(MQNIC_EQ_ACTIVE_LOG_SIZE_REG, self.log_size) # active, log size - - async def free(self): - await self.deactivate() - - if self.buf: - # TODO - pass - - async def activate(self, irq): - self.log.info("Activate Eq %d (interface %d)", self.eqn, self.interface.index) - - await self.deactivate() - self.irq = irq - self.head_ptr = 0 - self.tail_ptr = 0 + self.cq_table = {} - self.buf[0:self.buf_size] = b'\x00'*self.buf_size + self.hw_regs = self.interface.eq_res.get_window(self.eqn) await self.hw_regs.write_dword(MQNIC_EQ_ACTIVE_LOG_SIZE_REG, 0) # active, log size await self.hw_regs.write_dword(MQNIC_EQ_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address await self.hw_regs.write_dword(MQNIC_EQ_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address - await self.hw_regs.write_dword(MQNIC_EQ_INTERRUPT_INDEX_REG, irq) # interrupt index + await self.hw_regs.write_dword(MQNIC_EQ_INTERRUPT_INDEX_REG, self.irq) # interrupt index await self.hw_regs.write_dword(MQNIC_EQ_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer await self.hw_regs.write_dword(MQNIC_EQ_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer await self.hw_regs.write_dword(MQNIC_EQ_ACTIVE_LOG_SIZE_REG, self.log_size | MQNIC_EQ_ACTIVE_MASK) # active, log size - self.active = True + self.enabled = True + + async def close(self): + if not self.hw_regs: + return - async def deactivate(self): await self.hw_regs.write_dword(MQNIC_EQ_ACTIVE_LOG_SIZE_REG, self.log_size) # active, log size await self.hw_regs.write_dword(MQNIC_EQ_INTERRUPT_INDEX_REG, 0) # interrupt index + # TODO free buffer + self.irq = None - self.active = False + self.enabled = False + + self.hw_regs = None + + self.interface.eq_res.free(self.eqn) + self.eqn = None + + def attach_cq(self, cq): + if cq.is_txcq: + self.cq_table[cq.cqn | 0x80000000] = cq + else: + self.cq_table[cq.cqn] = cq + + def detach_cq(self, cq): + if cq.is_txcq: + del self.cq_table[cq.cqn | 0x80000000] + else: + del self.cq_table[cq.cqn] async def read_head_ptr(self): val = await self.hw_regs.read_dword(MQNIC_EQ_HEAD_PTR_REG) @@ -502,7 +512,7 @@ class Eq: await self.hw_regs.write_dword(MQNIC_EQ_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) async def arm(self): - if not self.active: + if not self.hw_regs: return await self.hw_regs.write_dword(MQNIC_EQ_INTERRUPT_INDEX_REG, self.irq | MQNIC_EQ_ARM_MASK) # interrupt index @@ -527,12 +537,12 @@ class Eq: if event_data[0] == 0: # transmit completion - cq = self.interface.tx_cq[event_data[1]] + cq = self.cq_table[event_data[1] | 0x80000000] await cq.handler(cq) await cq.arm() elif event_data[0] == 1: # receive completion - cq = self.interface.rx_cq[event_data[1]] + cq = self.cq_table[event_data[1]] await cq.handler(cq) await cq.arm() @@ -544,7 +554,7 @@ class Eq: class Cq: - def __init__(self, interface, cqn, hw_regs): + def __init__(self, interface): self.interface = interface self.log = interface.log self.driver = interface.driver @@ -552,8 +562,8 @@ class Cq: self.size = 0 self.size_mask = 0 self.stride = 0 - self.cqn = cqn - self.active = False + self.cqn = None + self.enabled = False self.buf_size = 0 self.buf_region = None @@ -569,76 +579,74 @@ class Cq: self.tail_ptr = 0 self.hw_ptr_mask = 0xffff - self.hw_regs = hw_regs + self.hw_regs = None - async def init(self): - self.log.info("Init CQ %d (interface %d)", self.cqn, self.interface.index) + async def open(self, eq, size, is_txcq=True): + if self.hw_regs: + raise Exception("Already open") - await self.hw_regs.write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, 0) # active, log size + self.is_txcq = is_txcq + if is_txcq: + self.cqn = self.interface.tx_cq_res.alloc() + else: + self.cqn = self.interface.rx_cq_res.alloc() - async def alloc(self, size, stride): - if self.active: - raise Exception("Cannot allocate active ring") - if self.buf: - raise Exception("Already allocated") + self.log.info("Open %s CQ %d (interface %d)", "TX" if is_txcq else "RX", self.cqn, self.interface.index) self.log_size = size.bit_length() - 1 self.size = 2**self.log_size self.size_mask = self.size-1 - self.stride = stride + self.stride = MQNIC_EVENT_SIZE self.buf_size = self.size*self.stride self.buf_region = self.driver.pool.alloc_region(self.buf_size) self.buf_dma = self.buf_region.get_absolute_address(0) self.buf = self.buf_region.mem - self.head_ptr = 0 - self.tail_ptr = 0 - - await self.hw_regs.write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, 0) # active, log size - await self.hw_regs.write_dword(MQNIC_CQ_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address - await self.hw_regs.write_dword(MQNIC_CQ_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address - await self.hw_regs.write_dword(MQNIC_CQ_INTERRUPT_INDEX_REG, 0) # event index - await self.hw_regs.write_dword(MQNIC_CQ_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer - await self.hw_regs.write_dword(MQNIC_CQ_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer - await self.hw_regs.write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, self.log_size) # active, log size - - async def free(self): - await self.deactivate() - - if self.buf: - # TODO - pass - - async def activate(self, eq): - self.log.info("Activate CQ %d (interface %d)", self.cqn, self.interface.index) - - await self.deactivate() - - self.eq = eq - - self.head_ptr = 0 - self.tail_ptr = 0 - self.buf[0:self.buf_size] = b'\x00'*self.buf_size + self.head_ptr = 0 + self.tail_ptr = 0 + + eq.attach_cq(self) + self.eq = eq + + if is_txcq: + self.hw_regs = self.interface.tx_cq_res.get_window(self.cqn) + else: + self.hw_regs = self.interface.rx_cq_res.get_window(self.cqn) + await self.hw_regs.write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, 0) # active, log size await self.hw_regs.write_dword(MQNIC_CQ_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address await self.hw_regs.write_dword(MQNIC_CQ_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address - await self.hw_regs.write_dword(MQNIC_CQ_INTERRUPT_INDEX_REG, eq.eqn) # event index + await self.hw_regs.write_dword(MQNIC_CQ_INTERRUPT_INDEX_REG, self.eq.eqn) # event index await self.hw_regs.write_dword(MQNIC_CQ_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer await self.hw_regs.write_dword(MQNIC_CQ_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer await self.hw_regs.write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, self.log_size | MQNIC_CQ_ACTIVE_MASK) # active, log size - self.active = True + self.enabled = True + + async def close(self): + if not self.hw_regs: + return - async def deactivate(self): await self.hw_regs.write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, self.log_size) # active, log size await self.hw_regs.write_dword(MQNIC_CQ_INTERRUPT_INDEX_REG, 0) # event index + # TODO free buffer + + self.eq.detach_cq(self) self.eq = None - self.active = False + self.enabled = False + + self.hw_regs = None + + if self.is_txcq: + self.interface.tx_cq_res.free(self.cqn) + else: + self.interface.rx_cq_res.free(self.cqn) + self.cqn = None async def read_head_ptr(self): val = await self.hw_regs.read_dword(MQNIC_CQ_HEAD_PTR_REG) @@ -648,14 +656,14 @@ class Cq: await self.hw_regs.write_dword(MQNIC_CQ_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) async def arm(self): - if not self.active: + if not self.hw_regs: return await self.hw_regs.write_dword(MQNIC_CQ_INTERRUPT_INDEX_REG, self.eq.eqn | MQNIC_CQ_ARM_MASK) # event index class Txq: - def __init__(self, interface, index, hw_regs): + def __init__(self, interface): self.interface = interface self.log = interface.log self.driver = interface.driver @@ -666,8 +674,8 @@ class Txq: self.size_mask = 0 self.full_size = 0 self.stride = 0 - self.index = index - self.active = False + self.index = None + self.enabled = False self.buf_size = 0 self.buf_region = None @@ -685,26 +693,23 @@ class Txq: self.bytes = 0 self.hw_ptr_mask = 0xffff - self.hw_regs = hw_regs + self.hw_regs = None - async def init(self): - self.log.info("Init TXQ %d (interface %d)", self.index, self.interface.index) + async def open(self, cq, size, desc_block_size): + if self.hw_regs: + raise Exception("Already open") - await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0) # active, log size + self.index = self.interface.txq_res.alloc() - async def alloc(self, size, stride): - if self.active: - raise Exception("Cannot allocate active ring") - if self.buf: - raise Exception("Already allocated") + self.log.info("Open TXQ %d (interface %d)", self.index, self.interface.index) self.log_queue_size = size.bit_length() - 1 - self.log_desc_block_size = int(stride/MQNIC_DESC_SIZE).bit_length() - 1 + self.log_desc_block_size = desc_block_size.bit_length() - 1 self.desc_block_size = 2**self.log_desc_block_size self.size = 2**self.log_queue_size self.size_mask = self.size-1 self.full_size = self.size >> 1 - self.stride = stride + self.stride = MQNIC_DESC_SIZE*self.desc_block_size self.tx_info = [None]*self.size @@ -716,53 +721,54 @@ class Txq: self.head_ptr = 0 self.tail_ptr = 0 - await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0) # active, log size - await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address - await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address - await self.hw_regs.write_dword(MQNIC_QUEUE_CPL_QUEUE_INDEX_REG, 0) # completion queue index - await self.hw_regs.write_dword(MQNIC_QUEUE_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer - await self.hw_regs.write_dword(MQNIC_QUEUE_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer - await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8)) # active, log desc block size, log queue size - - async def free(self): - await self.deactivate() - - if self.buf: - # TODO - pass - - async def activate(self, cq): - self.log.info("Activate TXQ %d (interface %d)", self.index, self.interface.index) - - await self.deactivate() - self.cq = cq self.cq.src_ring = self self.cq.handler = Txq.process_tx_cq - self.head_ptr = 0 - self.tail_ptr = 0 + self.hw_regs = self.interface.txq_res.get_window(self.index) await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0) # active, log size await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address - await self.hw_regs.write_dword(MQNIC_QUEUE_CPL_QUEUE_INDEX_REG, cq.cqn) # completion queue index + await self.hw_regs.write_dword(MQNIC_QUEUE_CPL_QUEUE_INDEX_REG, self.cq.cqn) # completion queue index await self.hw_regs.write_dword(MQNIC_QUEUE_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer await self.hw_regs.write_dword(MQNIC_QUEUE_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer - await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8) | MQNIC_QUEUE_ACTIVE_MASK) # active, log desc block size, log queue size - - self.active = True - - async def deactivate(self): await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8)) # active, log desc block size, log queue size + async def close(self): + if not self.hw_regs: + return + + await self.disable() + + # TODO free buffer + if self.cq: self.cq.src_ring = None self.cq.handler = None self.cq = None - self.active = False + self.hw_regs = None + + self.interface.txq_res.free(self.index) + self.index = None + + async def enable(self): + if not self.hw_regs: + raise Exception("Not open") + + await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8) | MQNIC_QUEUE_ACTIVE_MASK) # active, log desc block size, log queue size + + self.enabled = True + + async def disable(self): + if not self.hw_regs: + return + + await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8)) # active, log desc block size, log queue size + + self.enabled = False def empty(self): return self.head_ptr == self.tail_ptr @@ -840,7 +846,7 @@ class Txq: class Rxq: - def __init__(self, interface, index, hw_regs): + def __init__(self, interface): self.interface = interface self.log = interface.log self.driver = interface.driver @@ -851,8 +857,8 @@ class Rxq: self.size_mask = 0 self.full_size = 0 self.stride = 0 - self.index = index - self.active = False + self.index = None + self.enabled = False self.buf_size = 0 self.buf_region = None @@ -868,26 +874,23 @@ class Rxq: self.bytes = 0 self.hw_ptr_mask = 0xffff - self.hw_regs = hw_regs + self.hw_regs = None - async def init(self): - self.log.info("Init RXQ %d (interface %d)", self.index, self.interface.index) + async def open(self, cq, size, desc_block_size): + if self.hw_regs: + raise Exception("Already open") - await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0) # active, log size + self.index = self.interface.rxq_res.alloc() - async def alloc(self, size, stride): - if self.active: - raise Exception("Cannot allocate active ring") - if self.buf: - raise Exception("Already allocated") + self.log.info("Open RXQ %d (interface %d)", self.index, self.interface.index) self.log_queue_size = size.bit_length() - 1 - self.log_desc_block_size = int(stride/MQNIC_DESC_SIZE).bit_length() - 1 + self.log_desc_block_size = desc_block_size.bit_length() - 1 self.desc_block_size = 2**self.log_desc_block_size self.size = 2**self.log_queue_size self.size_mask = self.size-1 self.full_size = self.size >> 1 - self.stride = stride + self.stride = MQNIC_DESC_SIZE*self.desc_block_size self.rx_info = [None]*self.size @@ -899,47 +902,29 @@ class Rxq: self.head_ptr = 0 self.tail_ptr = 0 - await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0) # active, log size - await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address - await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address - await self.hw_regs.write_dword(MQNIC_QUEUE_CPL_QUEUE_INDEX_REG, 0) # completion queue index - await self.hw_regs.write_dword(MQNIC_QUEUE_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer - await self.hw_regs.write_dword(MQNIC_QUEUE_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer - await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8)) # active, log desc block size, log queue size - - async def free(self): - await self.deactivate() - - if self.buf: - # TODO - pass - - async def activate(self, cq): - self.log.info("Activate RXQ %d (interface %d)", self.index, self.interface.index) - - await self.deactivate() - self.cq = cq self.cq.src_ring = self self.cq.handler = Rxq.process_rx_cq - self.head_ptr = 0 - self.tail_ptr = 0 + self.hw_regs = self.interface.rxq_res.get_window(self.index) await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0) # active, log size await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG, self.buf_dma & 0xffffffff) # base address await self.hw_regs.write_dword(MQNIC_QUEUE_BASE_ADDR_REG+4, self.buf_dma >> 32) # base address - await self.hw_regs.write_dword(MQNIC_QUEUE_CPL_QUEUE_INDEX_REG, cq.cqn) # completion queue index + await self.hw_regs.write_dword(MQNIC_QUEUE_CPL_QUEUE_INDEX_REG, self.cq.cqn) # completion queue index await self.hw_regs.write_dword(MQNIC_QUEUE_HEAD_PTR_REG, self.head_ptr & self.hw_ptr_mask) # head pointer await self.hw_regs.write_dword(MQNIC_QUEUE_TAIL_PTR_REG, self.tail_ptr & self.hw_ptr_mask) # tail pointer - await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8) | MQNIC_QUEUE_ACTIVE_MASK) # active, log desc block size, log queue size - - self.active = True + await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8)) # active, log desc block size, log queue size await self.refill_buffers() - async def deactivate(self): - await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8)) # active, log desc block size, log queue size + async def close(self): + if not self.hw_regs: + return + + await self.disable() + + # TODO free buffer if self.cq: self.cq.src_ring = None @@ -947,7 +932,26 @@ class Rxq: self.cq = None - self.active = False + self.hw_regs = None + + self.interface.rxq_res.free(self.index) + self.index = None + + async def enable(self): + if not self.hw_regs: + raise Exception("Not open") + + await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8) | MQNIC_QUEUE_ACTIVE_MASK) # active, log desc block size, log queue size + + self.enabled = True + + async def disable(self): + if not self.hw_regs: + return + + await self.hw_regs.write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, self.log_queue_size | (self.log_desc_block_size << 8)) # active, log desc block size, log queue size + + self.enabled = False def empty(self): return self.head_ptr == self.tail_ptr @@ -1326,40 +1330,24 @@ class Interface: await self.set_rx_queue_map_app_mask(k, 0) await self.set_rx_queue_map_indir_table(k, 0, 0) - self.eq = [] - - self.txq = [] - self.tx_cq = [] - self.rxq = [] - self.rx_cq = [] - self.ports = [] - self.sched_blocks = [] - + # ensure all queues are disabled for k in range(self.eq_res.get_count()): - eq = Eq(self, k, self.eq_res.get_window(k)) - await eq.init() - self.eq.append(eq) + await self.eq_res.get_window(k).write_dword(MQNIC_EQ_ACTIVE_LOG_SIZE_REG, 0) for k in range(self.txq_res.get_count()): - txq = Txq(self, k, self.txq_res.get_window(k)) - await txq.init() - self.txq.append(txq) + await self.txq_res.get_window(k).write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0) for k in range(self.tx_cq_res.get_count()): - cq = Cq(self, k, self.tx_cq_res.get_window(k)) - await cq.init() - self.tx_cq.append(cq) + await self.tx_cq_res.get_window(k).write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, 0) for k in range(self.rxq_res.get_count()): - rxq = Rxq(self, k, self.rxq_res.get_window(k)) - await rxq.init() - self.rxq.append(rxq) + await self.rxq_res.get_window(k).write_dword(MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG, 0) for k in range(self.rx_cq_res.get_count()): - cq = Cq(self, k, self.rx_cq_res.get_window(k)) - await cq.init() - self.rx_cq.append(cq) + await self.rx_cq_res.get_window(k).write_dword(MQNIC_CQ_ACTIVE_LOG_SIZE_REG, 0) + # create ports + self.ports = [] for k in range(self.port_count): rb = self.reg_blocks.find(MQNIC_RB_PORT_TYPE, MQNIC_RB_PORT_VER, index=k) @@ -1367,6 +1355,8 @@ class Interface: await p.init() self.ports.append(p) + # create schedulers + self.sched_blocks = [] for k in range(self.sched_block_count): rb = self.reg_blocks.find(MQNIC_RB_SCHED_BLOCK_TYPE, MQNIC_RB_SCHED_BLOCK_VER, index=k) @@ -1376,30 +1366,42 @@ class Interface: assert self.sched_block_count == len(self.sched_blocks) - for eq in self.eq: - await eq.alloc(1024, MQNIC_EVENT_SIZE) - await eq.activate(self.index) # TODO? + # create EQs + self.eq = [] + for k in range(self.eq_res.get_count()): + eq = Eq(self) + await eq.open(self.index, 1024) + self.eq.append(eq) await eq.arm() + self.txq = [] + self.tx_cq = [] + self.rxq = [] + self.rx_cq = [] + # wait for all writes to complete await self.hw_regs.read_dword(0) async def open(self): - for rxq in self.rxq: - cq = self.rx_cq[rxq.index] - await cq.alloc(1024, MQNIC_CPL_SIZE) - await cq.activate(self.eq[cq.cqn % len(self.eq)]) + for k in range(self.rxq_res.get_count()): + cq = Cq(self) + await cq.open(self.eq[k % len(self.eq)], 1024, is_txcq=False) + self.rx_cq.append(cq) await cq.arm() - await rxq.alloc(1024, MQNIC_DESC_SIZE*4) - await rxq.activate(cq) + rxq = Rxq(self) + await rxq.open(cq, 1024, 4) + await rxq.enable() + self.rxq.append(rxq) - for txq in self.txq: - cq = self.tx_cq[txq.index] - await cq.alloc(1024, MQNIC_CPL_SIZE) - await cq.activate(self.eq[cq.cqn % len(self.eq)]) + for k in range(self.txq_res.get_count()): + cq = Cq(self) + await cq.open(self.eq[k % len(self.eq)], 1024, is_txcq=True) + self.tx_cq.append(cq) await cq.arm() - await txq.alloc(1024, MQNIC_DESC_SIZE*4) - await txq.activate(cq) + txq = Txq(self) + await txq.open(cq, 1024, 4) + await txq.enable() + self.txq.append(txq) # wait for all writes to complete await self.hw_regs.read_dword(0) @@ -1409,22 +1411,26 @@ class Interface: async def close(self): self.port_up = False - for txq in self.txq: - await txq.deactivate() - await txq.cq.deactivate() + for q in self.txq: + q.disable() - for rxq in self.rxq: - await rxq.deactivate() - await rxq.cq.deactivate() + for q in self.rxq: + q.disable() # wait for all writes to complete await self.hw_regs.read_dword(0) for q in self.txq: + cq = q.cq await q.free_buf() + await q.close() + await cq.close() for q in self.rxq: + cq = q.cq await q.free_buf() + await q.close() + await cq.close() async def start_xmit(self, skb, tx_ring=None, csum_start=None, csum_offset=None): if not self.port_up: diff --git a/modules/mqnic/mqnic.h b/modules/mqnic/mqnic.h index ffd6cfa07..0164d4a0c 100644 --- a/modules/mqnic/mqnic.h +++ b/modules/mqnic/mqnic.h @@ -83,6 +83,9 @@ struct mqnic_res { unsigned int count; u8 __iomem *base; unsigned int stride; + + spinlock_t lock; + unsigned long *bmap; }; struct mqnic_reg_block { @@ -288,7 +291,7 @@ struct mqnic_ring { struct mqnic_priv *priv; int index; struct mqnic_cq *cq; - int active; + int enabled; u32 hw_ptr_mask; u8 __iomem *hw_addr; @@ -313,9 +316,10 @@ struct mqnic_cq { struct mqnic_if *interface; struct napi_struct napi; int cqn; + int is_txcq; struct mqnic_eq *eq; struct mqnic_ring *src_ring; - int active; + int enabled; void (*handler)(struct mqnic_cq *cq); @@ -342,12 +346,15 @@ struct mqnic_eq { struct mqnic_if *interface; int eqn; struct mqnic_irq *irq; - int active; + int enabled; struct notifier_block irq_nb; void (*handler)(struct mqnic_eq *eq); + spinlock_t table_lock; + struct radix_tree_root cq_table; + u32 hw_ptr_mask; u8 __iomem *hw_addr; u8 __iomem *hw_head_ptr; @@ -393,8 +400,6 @@ struct mqnic_sched_block { int index; - u32 txq_count; - u32 sched_count; struct mqnic_sched *sched[MQNIC_MAX_PORTS]; }; @@ -432,18 +437,6 @@ struct mqnic_if { u32 eq_count; struct mqnic_eq *eq[MQNIC_MAX_EQ]; - u32 txq_count; - struct mqnic_ring *txq[MQNIC_MAX_TXQ]; - - u32 tx_cq_count; - struct mqnic_cq *tx_cq[MQNIC_MAX_TX_CQ]; - - u32 rxq_count; - struct mqnic_ring *rxq[MQNIC_MAX_RXQ]; - - u32 rx_cq_count; - struct mqnic_cq *rx_cq[MQNIC_MAX_RX_CQ]; - u32 port_count; struct mqnic_port *port[MQNIC_MAX_PORTS]; @@ -482,20 +475,14 @@ struct mqnic_priv { unsigned int link_status; struct timer_list link_status_timer; - u32 eq_count; - struct mqnic_eq *eq[MQNIC_MAX_EQ]; - u32 txq_count; - struct mqnic_ring *txq[MQNIC_MAX_TXQ]; - - u32 tx_cq_count; - struct mqnic_cq *tx_cq[MQNIC_MAX_TX_CQ]; - u32 rxq_count; - struct mqnic_ring *rxq[MQNIC_MAX_RXQ]; - u32 rx_cq_count; - struct mqnic_cq *rx_cq[MQNIC_MAX_RX_CQ]; + struct rw_semaphore txq_table_sem; + struct radix_tree_root txq_table; + + struct rw_semaphore rxq_table_sem; + struct radix_tree_root rxq_table; u32 sched_block_count; struct mqnic_sched_block *sched_block[MQNIC_MAX_PORTS]; @@ -512,6 +499,8 @@ struct mqnic_priv { // mqnic_res.c struct mqnic_res *mqnic_create_res(unsigned int count, u8 __iomem *base, unsigned int stride); void mqnic_destroy_res(struct mqnic_res *res); +int mqnic_res_alloc(struct mqnic_res *res); +void mqnic_res_free(struct mqnic_res *res, int index); unsigned int mqnic_res_get_count(struct mqnic_res *res); u8 __iomem *mqnic_res_get_addr(struct mqnic_res *res, int index); @@ -602,39 +591,34 @@ void mqnic_stats_init(struct mqnic_dev *mdev); u64 mqnic_stats_read(struct mqnic_dev *mdev, int index); // mqnic_eq.c -struct mqnic_eq *mqnic_create_eq(struct mqnic_if *interface, - int eqn, u8 __iomem *hw_addr); +struct mqnic_eq *mqnic_create_eq(struct mqnic_if *interface); void mqnic_destroy_eq(struct mqnic_eq *eq); -int mqnic_alloc_eq(struct mqnic_eq *eq, int size, int stride); -void mqnic_free_eq(struct mqnic_eq *eq); -int mqnic_activate_eq(struct mqnic_eq *eq, struct mqnic_irq *irq); -void mqnic_deactivate_eq(struct mqnic_eq *eq); +int mqnic_open_eq(struct mqnic_eq *eq, struct mqnic_irq *irq, int size); +void mqnic_close_eq(struct mqnic_eq *eq); +int mqnic_eq_attach_cq(struct mqnic_eq *eq, struct mqnic_cq *cq); +void mqnic_eq_detach_cq(struct mqnic_eq *eq, struct mqnic_cq *cq); void mqnic_eq_read_head_ptr(struct mqnic_eq *eq); void mqnic_eq_write_tail_ptr(struct mqnic_eq *eq); void mqnic_arm_eq(struct mqnic_eq *eq); void mqnic_process_eq(struct mqnic_eq *eq); // mqnic_cq.c -struct mqnic_cq *mqnic_create_cq(struct mqnic_if *interface, - int cqn, u8 __iomem *hw_addr); +struct mqnic_cq *mqnic_create_cq(struct mqnic_if *interface); void mqnic_destroy_cq(struct mqnic_cq *cq); -int mqnic_alloc_cq(struct mqnic_cq *cq, int size, int stride); -void mqnic_free_cq(struct mqnic_cq *cq); -int mqnic_activate_cq(struct mqnic_cq *cq, struct mqnic_eq *eq); -void mqnic_deactivate_cq(struct mqnic_cq *cq); +int mqnic_open_cq(struct mqnic_cq *cq, struct mqnic_eq *eq, int size, int is_txcq); +void mqnic_close_cq(struct mqnic_cq *cq); void mqnic_cq_read_head_ptr(struct mqnic_cq *cq); void mqnic_cq_write_tail_ptr(struct mqnic_cq *cq); void mqnic_arm_cq(struct mqnic_cq *cq); // mqnic_tx.c -struct mqnic_ring *mqnic_create_tx_ring(struct mqnic_if *interface, - int index, u8 __iomem *hw_addr); +struct mqnic_ring *mqnic_create_tx_ring(struct mqnic_if *interface); void mqnic_destroy_tx_ring(struct mqnic_ring *ring); -int mqnic_alloc_tx_ring(struct mqnic_ring *ring, int size, int stride); -void mqnic_free_tx_ring(struct mqnic_ring *ring); -int mqnic_activate_tx_ring(struct mqnic_ring *ring, struct mqnic_priv *priv, - struct mqnic_cq *cq); -void mqnic_deactivate_tx_ring(struct mqnic_ring *ring); +int mqnic_open_tx_ring(struct mqnic_ring *ring, struct mqnic_priv *priv, + struct mqnic_cq *cq, int size, int desc_block_size); +void mqnic_close_tx_ring(struct mqnic_ring *ring); +int mqnic_enable_tx_ring(struct mqnic_ring *ring); +void mqnic_disable_tx_ring(struct mqnic_ring *ring); bool mqnic_is_tx_ring_empty(const struct mqnic_ring *ring); bool mqnic_is_tx_ring_full(const struct mqnic_ring *ring); void mqnic_tx_read_tail_ptr(struct mqnic_ring *ring); @@ -647,14 +631,13 @@ int mqnic_poll_tx_cq(struct napi_struct *napi, int budget); netdev_tx_t mqnic_start_xmit(struct sk_buff *skb, struct net_device *dev); // mqnic_rx.c -struct mqnic_ring *mqnic_create_rx_ring(struct mqnic_if *interface, - int index, u8 __iomem *hw_addr); +struct mqnic_ring *mqnic_create_rx_ring(struct mqnic_if *interface); void mqnic_destroy_rx_ring(struct mqnic_ring *ring); -int mqnic_alloc_rx_ring(struct mqnic_ring *ring, int size, int stride); -void mqnic_free_rx_ring(struct mqnic_ring *ring); -int mqnic_activate_rx_ring(struct mqnic_ring *ring, struct mqnic_priv *priv, - struct mqnic_cq *cq); -void mqnic_deactivate_rx_ring(struct mqnic_ring *ring); +int mqnic_open_rx_ring(struct mqnic_ring *ring, struct mqnic_priv *priv, + struct mqnic_cq *cq, int size, int desc_block_size); +void mqnic_close_rx_ring(struct mqnic_ring *ring); +int mqnic_enable_rx_ring(struct mqnic_ring *ring); +void mqnic_disable_rx_ring(struct mqnic_ring *ring); bool mqnic_is_rx_ring_empty(const struct mqnic_ring *ring); bool mqnic_is_rx_ring_full(const struct mqnic_ring *ring); void mqnic_rx_read_tail_ptr(struct mqnic_ring *ring); diff --git a/modules/mqnic/mqnic_cq.c b/modules/mqnic/mqnic_cq.c index 093a9e9a2..198b34ef1 100644 --- a/modules/mqnic/mqnic_cq.c +++ b/modules/mqnic/mqnic_cq.c @@ -35,8 +35,7 @@ #include "mqnic.h" -struct mqnic_cq *mqnic_create_cq(struct mqnic_if *interface, - int cqn, u8 __iomem *hw_addr) +struct mqnic_cq *mqnic_create_cq(struct mqnic_if *interface) { struct mqnic_cq *cq; @@ -47,82 +46,63 @@ struct mqnic_cq *mqnic_create_cq(struct mqnic_if *interface, cq->dev = interface->dev; cq->interface = interface; - cq->cqn = cqn; - cq->active = 0; + cq->cqn = -1; + cq->enabled = 0; - cq->hw_addr = hw_addr; + cq->hw_addr = NULL; cq->hw_ptr_mask = 0xffff; - cq->hw_head_ptr = hw_addr + MQNIC_CQ_HEAD_PTR_REG; - cq->hw_tail_ptr = hw_addr + MQNIC_CQ_TAIL_PTR_REG; + cq->hw_head_ptr = NULL; + cq->hw_tail_ptr = NULL; cq->head_ptr = 0; cq->tail_ptr = 0; - // deactivate queue - iowrite32(0, cq->hw_addr + MQNIC_CQ_ACTIVE_LOG_SIZE_REG); - return cq; } void mqnic_destroy_cq(struct mqnic_cq *cq) { - mqnic_free_cq(cq); + mqnic_close_cq(cq); kfree(cq); } -int mqnic_alloc_cq(struct mqnic_cq *cq, int size, int stride) +int mqnic_open_cq(struct mqnic_cq *cq, struct mqnic_eq *eq, int size, int is_txcq) { - if (cq->active || cq->buf) + int ret; + + if (cq->enabled || cq->hw_addr || cq->buf || !eq) return -EINVAL; + cq->is_txcq = is_txcq; + + if (is_txcq) { + cq->cqn = mqnic_res_alloc(cq->interface->tx_cq_res); + } else { + cq->cqn = mqnic_res_alloc(cq->interface->rx_cq_res); + } + if (cq->cqn < 0) + return -ENOMEM; + cq->size = roundup_pow_of_two(size); cq->size_mask = cq->size - 1; - cq->stride = roundup_pow_of_two(stride); + cq->stride = roundup_pow_of_two(MQNIC_CPL_SIZE); cq->buf_size = cq->size * cq->stride; cq->buf = dma_alloc_coherent(cq->dev, cq->buf_size, &cq->buf_dma_addr, GFP_KERNEL); - if (!cq->buf) - return -ENOMEM; - - cq->head_ptr = 0; - cq->tail_ptr = 0; - - // deactivate queue - iowrite32(0, cq->hw_addr + MQNIC_CQ_ACTIVE_LOG_SIZE_REG); - // set base address - iowrite32(cq->buf_dma_addr, cq->hw_addr + MQNIC_CQ_BASE_ADDR_REG + 0); - iowrite32(cq->buf_dma_addr >> 32, cq->hw_addr + MQNIC_CQ_BASE_ADDR_REG + 4); - // set interrupt index - iowrite32(0, cq->hw_addr + MQNIC_CQ_INTERRUPT_INDEX_REG); - // set pointers - iowrite32(cq->head_ptr & cq->hw_ptr_mask, cq->hw_addr + MQNIC_CQ_HEAD_PTR_REG); - iowrite32(cq->tail_ptr & cq->hw_ptr_mask, cq->hw_addr + MQNIC_CQ_TAIL_PTR_REG); - // set size - iowrite32(ilog2(cq->size), cq->hw_addr + MQNIC_CQ_ACTIVE_LOG_SIZE_REG); - - return 0; -} - -void mqnic_free_cq(struct mqnic_cq *cq) -{ - mqnic_deactivate_cq(cq); - - if (cq->buf) { - dma_free_coherent(cq->dev, cq->buf_size, cq->buf, cq->buf_dma_addr); - cq->buf = NULL; - cq->buf_dma_addr = 0; + if (!cq->buf) { + ret = -ENOMEM; + goto fail; } -} - -int mqnic_activate_cq(struct mqnic_cq *cq, struct mqnic_eq *eq) -{ - mqnic_deactivate_cq(cq); - - if (!cq->buf || !eq) - return -EINVAL; cq->eq = eq; + mqnic_eq_attach_cq(eq, cq); + if (is_txcq) + cq->hw_addr = mqnic_res_get_addr(cq->interface->tx_cq_res, cq->cqn); + else + cq->hw_addr = mqnic_res_get_addr(cq->interface->rx_cq_res, cq->cqn); + cq->hw_head_ptr = cq->hw_addr + MQNIC_CQ_HEAD_PTR_REG; + cq->hw_tail_ptr = cq->hw_addr + MQNIC_CQ_TAIL_PTR_REG; cq->head_ptr = 0; cq->tail_ptr = 0; @@ -134,30 +114,55 @@ int mqnic_activate_cq(struct mqnic_cq *cq, struct mqnic_eq *eq) // set base address iowrite32(cq->buf_dma_addr, cq->hw_addr + MQNIC_CQ_BASE_ADDR_REG + 0); iowrite32(cq->buf_dma_addr >> 32, cq->hw_addr + MQNIC_CQ_BASE_ADDR_REG + 4); - // set interrupt index + // set EQN iowrite32(cq->eq->eqn, cq->hw_addr + MQNIC_CQ_INTERRUPT_INDEX_REG); // set pointers iowrite32(cq->head_ptr & cq->hw_ptr_mask, cq->hw_addr + MQNIC_CQ_HEAD_PTR_REG); iowrite32(cq->tail_ptr & cq->hw_ptr_mask, cq->hw_addr + MQNIC_CQ_TAIL_PTR_REG); - // set size and activate queue - iowrite32(ilog2(cq->size) | MQNIC_CQ_ACTIVE_MASK, - cq->hw_addr + MQNIC_CQ_ACTIVE_LOG_SIZE_REG); + // set size + iowrite32(ilog2(cq->size) | MQNIC_CQ_ACTIVE_MASK, cq->hw_addr + MQNIC_CQ_ACTIVE_LOG_SIZE_REG); - cq->active = 1; + cq->enabled = 1; return 0; + +fail: + mqnic_close_eq(eq); + return ret; } -void mqnic_deactivate_cq(struct mqnic_cq *cq) +void mqnic_close_cq(struct mqnic_cq *cq) { - // deactivate queue - iowrite32(ilog2(cq->size), cq->hw_addr + MQNIC_CQ_ACTIVE_LOG_SIZE_REG); - // disarm queue - iowrite32(0, cq->hw_addr + MQNIC_CQ_INTERRUPT_INDEX_REG); + if (cq->hw_addr) { + // deactivate queue + iowrite32(ilog2(cq->size), cq->hw_addr + MQNIC_CQ_ACTIVE_LOG_SIZE_REG); + // disarm queue + iowrite32(0, cq->hw_addr + MQNIC_CQ_INTERRUPT_INDEX_REG); + } - cq->eq = NULL; + if (cq->eq) { + mqnic_eq_detach_cq(cq->eq, cq); + cq->eq = NULL; + } - cq->active = 0; + cq->hw_addr = NULL; + cq->hw_head_ptr = NULL; + cq->hw_tail_ptr = NULL; + + if (cq->buf) { + dma_free_coherent(cq->dev, cq->buf_size, cq->buf, cq->buf_dma_addr); + cq->buf = NULL; + cq->buf_dma_addr = 0; + } + + if (cq->is_txcq) { + mqnic_res_free(cq->interface->tx_cq_res, cq->cqn); + } else { + mqnic_res_free(cq->interface->rx_cq_res, cq->cqn); + } + cq->cqn = -1; + + cq->enabled = 0; } void mqnic_cq_read_head_ptr(struct mqnic_cq *cq) @@ -172,7 +177,7 @@ void mqnic_cq_write_tail_ptr(struct mqnic_cq *cq) void mqnic_arm_cq(struct mqnic_cq *cq) { - if (!cq->active) + if (!cq->enabled) return; iowrite32(cq->eq->eqn | MQNIC_CQ_ARM_MASK, diff --git a/modules/mqnic/mqnic_eq.c b/modules/mqnic/mqnic_eq.c index 187ef4b81..de2f049ed 100644 --- a/modules/mqnic/mqnic_eq.c +++ b/modules/mqnic/mqnic_eq.c @@ -45,8 +45,7 @@ static int mqnic_eq_int(struct notifier_block *nb, unsigned long action, void *d return NOTIFY_DONE; } -struct mqnic_eq *mqnic_create_eq(struct mqnic_if *interface, - int eqn, u8 __iomem *hw_addr) +struct mqnic_eq *mqnic_create_eq(struct mqnic_if *interface) { struct mqnic_eq *eq; @@ -57,92 +56,66 @@ struct mqnic_eq *mqnic_create_eq(struct mqnic_if *interface, eq->dev = interface->dev; eq->interface = interface; - eq->eqn = eqn; - eq->active = 0; + eq->eqn = -1; + eq->enabled = 0; eq->irq_nb.notifier_call = mqnic_eq_int; - eq->hw_addr = hw_addr; + eq->hw_addr = NULL; eq->hw_ptr_mask = 0xffff; - eq->hw_head_ptr = hw_addr + MQNIC_EQ_HEAD_PTR_REG; - eq->hw_tail_ptr = hw_addr + MQNIC_EQ_TAIL_PTR_REG; + eq->hw_head_ptr = NULL; + eq->hw_tail_ptr = NULL; eq->head_ptr = 0; eq->tail_ptr = 0; - // deactivate queue - iowrite32(0, eq->hw_addr + MQNIC_EQ_ACTIVE_LOG_SIZE_REG); + spin_lock_init(&eq->table_lock); + + INIT_RADIX_TREE(&eq->cq_table, GFP_KERNEL); return eq; } void mqnic_destroy_eq(struct mqnic_eq *eq) { - mqnic_free_eq(eq); + mqnic_close_eq(eq); kfree(eq); } -int mqnic_alloc_eq(struct mqnic_eq *eq, int size, int stride) +int mqnic_open_eq(struct mqnic_eq *eq, struct mqnic_irq *irq, int size) { - if (eq->active || eq->buf) + int ret; + + if (eq->enabled || eq->hw_addr || eq->buf || !irq) return -EINVAL; + eq->eqn = mqnic_res_alloc(eq->interface->eq_res); + if (eq->eqn < 0) + return -ENOMEM; + eq->size = roundup_pow_of_two(size); eq->size_mask = eq->size - 1; - eq->stride = roundup_pow_of_two(stride); + eq->stride = roundup_pow_of_two(MQNIC_EVENT_SIZE); eq->buf_size = eq->size * eq->stride; eq->buf = dma_alloc_coherent(eq->dev, eq->buf_size, &eq->buf_dma_addr, GFP_KERNEL); - if (!eq->buf) - return -ENOMEM; - - eq->head_ptr = 0; - eq->tail_ptr = 0; - - // deactivate queue - iowrite32(0, eq->hw_addr + MQNIC_EQ_ACTIVE_LOG_SIZE_REG); - // set base address - iowrite32(eq->buf_dma_addr, eq->hw_addr + MQNIC_EQ_BASE_ADDR_REG + 0); - iowrite32(eq->buf_dma_addr >> 32, eq->hw_addr + MQNIC_EQ_BASE_ADDR_REG + 4); - // set interrupt index - iowrite32(0, eq->hw_addr + MQNIC_EQ_INTERRUPT_INDEX_REG); - // set pointers - iowrite32(eq->head_ptr & eq->hw_ptr_mask, eq->hw_addr + MQNIC_EQ_HEAD_PTR_REG); - iowrite32(eq->tail_ptr & eq->hw_ptr_mask, eq->hw_addr + MQNIC_EQ_TAIL_PTR_REG); - // set size - iowrite32(ilog2(eq->size), eq->hw_addr + MQNIC_EQ_ACTIVE_LOG_SIZE_REG); - - return 0; -} - -void mqnic_free_eq(struct mqnic_eq *eq) -{ - mqnic_deactivate_eq(eq); - - if (eq->buf) { - dma_free_coherent(eq->dev, eq->buf_size, eq->buf, eq->buf_dma_addr); - eq->buf = NULL; - eq->buf_dma_addr = 0; + if (!eq->buf) { + ret = -ENOMEM; + goto fail; } -} - -int mqnic_activate_eq(struct mqnic_eq *eq, struct mqnic_irq *irq) -{ - int ret = 0; - - mqnic_deactivate_eq(eq); - - if (!eq->buf || !irq) - return -EINVAL; // register interrupt ret = atomic_notifier_chain_register(&irq->nh, &eq->irq_nb); if (ret) - return ret; + goto fail; eq->irq = irq; + eq->hw_addr = mqnic_res_get_addr(eq->interface->eq_res, eq->eqn); + eq->hw_head_ptr = eq->hw_addr + MQNIC_EQ_HEAD_PTR_REG; + eq->hw_tail_ptr = eq->hw_addr + MQNIC_EQ_TAIL_PTR_REG; + eq->head_ptr = 0; eq->tail_ptr = 0; @@ -158,23 +131,30 @@ int mqnic_activate_eq(struct mqnic_eq *eq, struct mqnic_irq *irq) // set pointers iowrite32(eq->head_ptr & eq->hw_ptr_mask, eq->hw_addr + MQNIC_EQ_HEAD_PTR_REG); iowrite32(eq->tail_ptr & eq->hw_ptr_mask, eq->hw_addr + MQNIC_EQ_TAIL_PTR_REG); + // set size + iowrite32(ilog2(eq->size), eq->hw_addr + MQNIC_EQ_ACTIVE_LOG_SIZE_REG); // set size and activate queue - iowrite32(ilog2(eq->size) | MQNIC_EQ_ACTIVE_MASK, - eq->hw_addr + MQNIC_EQ_ACTIVE_LOG_SIZE_REG); + iowrite32(ilog2(eq->size) | MQNIC_EQ_ACTIVE_MASK, eq->hw_addr + MQNIC_EQ_ACTIVE_LOG_SIZE_REG); - eq->active = 1; + eq->enabled = 1; return 0; + +fail: + mqnic_close_eq(eq); + return ret; } -void mqnic_deactivate_eq(struct mqnic_eq *eq) +void mqnic_close_eq(struct mqnic_eq *eq) { - int ret = 0; + int ret; - // deactivate queue - iowrite32(ilog2(eq->size), eq->hw_addr + MQNIC_EQ_ACTIVE_LOG_SIZE_REG); - // disarm queue - iowrite32(0, eq->hw_addr + MQNIC_EQ_INTERRUPT_INDEX_REG); + if (eq->hw_addr) { + // deactivate queue + iowrite32(ilog2(eq->size), eq->hw_addr + MQNIC_EQ_ACTIVE_LOG_SIZE_REG); + // disarm queue + iowrite32(0, eq->hw_addr + MQNIC_EQ_INTERRUPT_INDEX_REG); + } // unregister interrupt if (eq->irq) @@ -182,7 +162,56 @@ void mqnic_deactivate_eq(struct mqnic_eq *eq) eq->irq = NULL; - eq->active = 0; + eq->hw_addr = NULL; + eq->hw_head_ptr = NULL; + eq->hw_tail_ptr = NULL; + + if (eq->buf) { + dma_free_coherent(eq->dev, eq->buf_size, eq->buf, eq->buf_dma_addr); + eq->buf = NULL; + eq->buf_dma_addr = 0; + } + + mqnic_res_free(eq->interface->eq_res, eq->eqn); + eq->eqn = -1; + + eq->enabled = 0; +} + +int mqnic_eq_attach_cq(struct mqnic_eq *eq, struct mqnic_cq *cq) +{ + int ret; + int cqn = cq->cqn; + if (cq->is_txcq) + cqn |= 0x80000000; + + spin_lock_irq(&eq->table_lock); + ret = radix_tree_insert(&eq->cq_table, cqn, cq); + spin_unlock_irq(&eq->table_lock); + return ret; +} + +void mqnic_eq_detach_cq(struct mqnic_eq *eq, struct mqnic_cq *cq) +{ + struct mqnic_cq *item; + int cqn = cq->cqn; + if (cq->is_txcq) + cqn |= 0x80000000; + + spin_lock_irq(&eq->table_lock); + item = radix_tree_delete(&eq->cq_table, cqn); + spin_unlock_irq(&eq->table_lock); + + if (IS_ERR(item)) { + dev_err(eq->dev, "%s on IF %d EQ %d: radix_tree_delete failed: %ld", + __func__, eq->interface->index, eq->eqn, PTR_ERR(item)); + } else if (!item) { + dev_err(eq->dev, "%s on IF %d EQ %d: CQ %d not in table", + __func__, eq->interface->index, eq->eqn, cqn); + } else if (item != cq) { + dev_err(eq->dev, "%s on IF %d EQ %d: entry mismatch when removing CQ %d", + __func__, eq->interface->index, eq->eqn, cqn); + } } void mqnic_eq_read_head_ptr(struct mqnic_eq *eq) @@ -197,11 +226,10 @@ void mqnic_eq_write_tail_ptr(struct mqnic_eq *eq) void mqnic_arm_eq(struct mqnic_eq *eq) { - if (!eq->active) + if (!eq->enabled) return; - iowrite32(eq->irq->index | MQNIC_EQ_ARM_MASK, - eq->hw_addr + MQNIC_EQ_INTERRUPT_INDEX_REG); + iowrite32(eq->irq->index | MQNIC_EQ_ARM_MASK, eq->hw_addr + MQNIC_EQ_INTERRUPT_INDEX_REG); } void mqnic_process_eq(struct mqnic_eq *eq) @@ -212,6 +240,7 @@ void mqnic_process_eq(struct mqnic_eq *eq) u32 eq_index; u32 eq_tail_ptr; int done = 0; + int cqn; // read head pointer from NIC eq_tail_ptr = eq->tail_ptr; @@ -227,34 +256,44 @@ void mqnic_process_eq(struct mqnic_eq *eq) if (event->type == MQNIC_EVENT_TYPE_TX_CPL) { // transmit completion event - if (unlikely(le16_to_cpu(event->source) > interface->tx_cq_count)) { - dev_err(eq->dev, "%s on port %d: unknown event source %d (index %d, type %d)", - __func__, interface->index, le16_to_cpu(event->source), eq_index, - le16_to_cpu(event->type)); + cqn = le16_to_cpu(event->source) | 0x80000000; + + rcu_read_lock(); + cq = radix_tree_lookup(&eq->cq_table, cqn); + rcu_read_unlock(); + + if (likely(cq)) { + if (likely(cq->handler)) + cq->handler(cq); + } else { + dev_err(eq->dev, "%s on IF %d EQ %d: unknown event source %d (index %d, type %d)", + __func__, interface->index, eq->eqn, le16_to_cpu(event->source), + eq_index, le16_to_cpu(event->type)); print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 1, event, MQNIC_EVENT_SIZE, true); - } else { - cq = interface->tx_cq[le16_to_cpu(event->source)]; - if (likely(cq && cq->handler)) - cq->handler(cq); } } else if (le16_to_cpu(event->type) == MQNIC_EVENT_TYPE_RX_CPL) { // receive completion event - if (unlikely(le16_to_cpu(event->source) > interface->rx_cq_count)) { - dev_err(eq->dev, "%s on port %d: unknown event source %d (index %d, type %d)", - __func__, interface->index, le16_to_cpu(event->source), eq_index, - le16_to_cpu(event->type)); + cqn = le16_to_cpu(event->source); + + rcu_read_lock(); + cq = radix_tree_lookup(&eq->cq_table, cqn); + rcu_read_unlock(); + + if (likely(cq)) { + if (likely(cq->handler)) + cq->handler(cq); + } else { + dev_err(eq->dev, "%s on IF %d EQ %d: unknown event source %d (index %d, type %d)", + __func__, interface->index, eq->eqn, le16_to_cpu(event->source), + eq_index, le16_to_cpu(event->type)); print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 1, event, MQNIC_EVENT_SIZE, true); - } else { - cq = interface->rx_cq[le16_to_cpu(event->source)]; - if (likely(cq && cq->handler)) - cq->handler(cq); } } else { - dev_err(eq->dev, "%s on port %d: unknown event type %d (index %d, source %d)", - __func__, interface->index, le16_to_cpu(event->type), eq_index, - le16_to_cpu(event->source)); + dev_err(eq->dev, "%s on IF %d EQ %d: unknown event type %d (index %d, source %d)", + __func__, interface->index, eq->eqn, le16_to_cpu(event->type), + eq_index, le16_to_cpu(event->source)); print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 1, event, MQNIC_EVENT_SIZE, true); } diff --git a/modules/mqnic/mqnic_if.c b/modules/mqnic/mqnic_if.c index ba530c401..846cba681 100644 --- a/modules/mqnic/mqnic_if.c +++ b/modules/mqnic/mqnic_if.c @@ -250,70 +250,21 @@ struct mqnic_if *mqnic_create_interface(struct mqnic_dev *mdev, int index, u8 __ desc_block_size = min_t(u32, interface->max_desc_block_size, 4); - // create rings - interface->eq_count = mqnic_res_get_count(interface->eq_res); - for (k = 0; k < interface->eq_count; k++) { - struct mqnic_eq *eq = mqnic_create_eq(interface, k, - mqnic_res_get_addr(interface->eq_res, k)); - if (IS_ERR_OR_NULL(eq)) { - ret = PTR_ERR(eq); - goto fail; - } + // disable queues + for (k = 0; k < mqnic_res_get_count(interface->eq_res); k++) + iowrite32(0, mqnic_res_get_addr(interface->eq_res, k) + MQNIC_EQ_ACTIVE_LOG_SIZE_REG); - interface->eq[k] = eq; + for (k = 0; k < mqnic_res_get_count(interface->txq_res); k++) + iowrite32(0, mqnic_res_get_addr(interface->txq_res, k) + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG); - ret = mqnic_alloc_eq(eq, mqnic_num_eq_entries, - MQNIC_EVENT_SIZE); - if (ret) - goto fail; + for (k = 0; k < mqnic_res_get_count(interface->tx_cq_res); k++) + iowrite32(0, mqnic_res_get_addr(interface->tx_cq_res, k) + MQNIC_CQ_ACTIVE_LOG_SIZE_REG); - mqnic_activate_eq(eq, mdev->irq[k % mdev->irq_count]); - mqnic_arm_eq(eq); - } + for (k = 0; k < mqnic_res_get_count(interface->rxq_res); k++) + iowrite32(0, mqnic_res_get_addr(interface->rxq_res, k) + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG); - interface->txq_count = mqnic_res_get_count(interface->txq_res); - for (k = 0; k < interface->txq_count; k++) { - struct mqnic_ring *txq = mqnic_create_tx_ring(interface, k, - mqnic_res_get_addr(interface->txq_res, k)); - if (IS_ERR_OR_NULL(txq)) { - ret = PTR_ERR(txq); - goto fail; - } - interface->txq[k] = txq; - } - - interface->tx_cq_count = mqnic_res_get_count(interface->tx_cq_res); - for (k = 0; k < interface->tx_cq_count; k++) { - struct mqnic_cq *cq = mqnic_create_cq(interface, k, - mqnic_res_get_addr(interface->tx_cq_res, k)); - if (IS_ERR_OR_NULL(cq)) { - ret = PTR_ERR(cq); - goto fail; - } - interface->tx_cq[k] = cq; - } - - interface->rxq_count = mqnic_res_get_count(interface->rxq_res); - for (k = 0; k < interface->rxq_count; k++) { - struct mqnic_ring *rxq = mqnic_create_rx_ring(interface, k, - mqnic_res_get_addr(interface->rxq_res, k)); - if (IS_ERR_OR_NULL(rxq)) { - ret = PTR_ERR(rxq); - goto fail; - } - interface->rxq[k] = rxq; - } - - interface->rx_cq_count = mqnic_res_get_count(interface->rx_cq_res); - for (k = 0; k < interface->rx_cq_count; k++) { - struct mqnic_cq *cq = mqnic_create_cq(interface, k, - mqnic_res_get_addr(interface->rx_cq_res, k)); - if (IS_ERR_OR_NULL(cq)) { - ret = PTR_ERR(cq); - goto fail; - } - interface->rx_cq[k] = cq; - } + for (k = 0; k < mqnic_res_get_count(interface->rx_cq_res); k++) + iowrite32(0, mqnic_res_get_addr(interface->rx_cq_res, k) + MQNIC_CQ_ACTIVE_LOG_SIZE_REG); // create ports for (k = 0; k < interface->port_count; k++) { @@ -353,6 +304,24 @@ struct mqnic_if *mqnic_create_interface(struct mqnic_dev *mdev, int index, u8 __ interface->sched_block[k] = sched_block; } + // create EQs + interface->eq_count = mqnic_res_get_count(interface->eq_res); + for (k = 0; k < interface->eq_count; k++) { + struct mqnic_eq *eq = mqnic_create_eq(interface); + if (IS_ERR_OR_NULL(eq)) { + ret = PTR_ERR(eq); + goto fail; + } + + interface->eq[k] = eq; + + ret = mqnic_open_eq(eq, mdev->irq[k % mdev->irq_count], mqnic_num_eq_entries); + if (ret) + goto fail; + + mqnic_arm_eq(eq); + } + // create net_devices interface->dev_port_base = mdev->dev_port_max; interface->dev_port_max = mdev->dev_port_max; @@ -386,7 +355,7 @@ void mqnic_destroy_interface(struct mqnic_if *interface) } } - // free rings + // free EQs for (k = 0; k < ARRAY_SIZE(interface->eq); k++) { if (interface->eq[k]) { mqnic_destroy_eq(interface->eq[k]); @@ -394,34 +363,6 @@ void mqnic_destroy_interface(struct mqnic_if *interface) } } - for (k = 0; k < ARRAY_SIZE(interface->txq); k++) { - if (interface->txq[k]) { - mqnic_destroy_tx_ring(interface->txq[k]); - interface->txq[k] = NULL; - } - } - - for (k = 0; k < ARRAY_SIZE(interface->tx_cq); k++) { - if (interface->tx_cq[k]) { - mqnic_destroy_cq(interface->tx_cq[k]); - interface->tx_cq[k] = NULL; - } - } - - for (k = 0; k < ARRAY_SIZE(interface->rxq); k++) { - if (interface->rxq[k]) { - mqnic_destroy_rx_ring(interface->rxq[k]); - interface->rxq[k] = NULL; - } - } - - for (k = 0; k < ARRAY_SIZE(interface->rx_cq); k++) { - if (interface->rx_cq[k]) { - mqnic_destroy_cq(interface->rx_cq[k]); - interface->rx_cq[k] = NULL; - } - } - // free schedulers for (k = 0; k < ARRAY_SIZE(interface->sched_block); k++) { if (interface->sched_block[k]) { diff --git a/modules/mqnic/mqnic_netdev.c b/modules/mqnic/mqnic_netdev.c index fe2048f46..7ee0ee058 100644 --- a/modules/mqnic/mqnic_netdev.c +++ b/modules/mqnic/mqnic_netdev.c @@ -37,74 +37,169 @@ #include +static int mqnic_stop_port(struct net_device *ndev); + static int mqnic_start_port(struct net_device *ndev) { struct mqnic_priv *priv = netdev_priv(ndev); struct mqnic_dev *mdev = priv->mdev; + struct mqnic_if *iface = priv->interface; + struct mqnic_ring *q; + struct mqnic_cq *cq; + struct radix_tree_iter iter; + void **slot; int k; + int ret; + u32 desc_block_size; dev_info(mdev->dev, "%s on interface %d netdev %d", __func__, priv->interface->index, priv->index); + desc_block_size = min_t(u32, priv->interface->max_desc_block_size, 4); + // set up RX queues - for (k = 0; k < min(priv->rxq_count, priv->rx_cq_count); k++) { - // set up CQ - mqnic_activate_cq(priv->rx_cq[k], priv->eq[k % priv->eq_count]); + for (k = 0; k < priv->rxq_count; k++) { + // create CQ + cq = mqnic_create_cq(iface); + if (IS_ERR_OR_NULL(cq)) { + ret = PTR_ERR(cq); + goto fail; + } + + ret = mqnic_open_cq(cq, iface->eq[k % iface->eq_count], mqnic_num_rxq_entries, 0); + if (ret) { + mqnic_destroy_cq(cq); + goto fail; + } #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0) - netif_napi_add(ndev, &priv->rx_cq[k]->napi, mqnic_poll_rx_cq); + netif_napi_add(ndev, &cq->napi, mqnic_poll_rx_cq); #else - netif_napi_add(ndev, &priv->rx_cq[k]->napi, mqnic_poll_rx_cq, NAPI_POLL_WEIGHT); + netif_napi_add(ndev, &cq->napi, mqnic_poll_rx_cq, NAPI_POLL_WEIGHT); #endif - napi_enable(&priv->rx_cq[k]->napi); + napi_enable(&cq->napi); - mqnic_arm_cq(priv->rx_cq[k]); + mqnic_arm_cq(cq); - // set up queue - priv->rxq[k]->mtu = ndev->mtu; + // create RX queue + q = mqnic_create_rx_ring(iface); + if (IS_ERR_OR_NULL(q)) { + ret = PTR_ERR(q); + mqnic_destroy_cq(cq); + goto fail; + } + + q->mtu = ndev->mtu; if (ndev->mtu + ETH_HLEN <= PAGE_SIZE) - priv->rxq[k]->page_order = 0; + q->page_order = 0; else - priv->rxq[k]->page_order = ilog2((ndev->mtu + ETH_HLEN + PAGE_SIZE - 1) / PAGE_SIZE - 1) + 1; - mqnic_activate_rx_ring(priv->rxq[k], priv, priv->rx_cq[k]); + q->page_order = ilog2((ndev->mtu + ETH_HLEN + PAGE_SIZE - 1) / PAGE_SIZE - 1) + 1; + + ret = mqnic_open_rx_ring(q, priv, cq, mqnic_num_rxq_entries, 1); + if (ret) { + mqnic_destroy_rx_ring(q); + mqnic_destroy_cq(cq); + goto fail; + } + + down_write(&priv->rxq_table_sem); + ret = radix_tree_insert(&priv->rxq_table, k, q); + up_write(&priv->rxq_table_sem); + if (ret) { + mqnic_destroy_rx_ring(q); + mqnic_destroy_cq(cq); + goto fail; + } } // set up TX queues - for (k = 0; k < min(priv->txq_count, priv->tx_cq_count); k++) { - // set up CQ - mqnic_activate_cq(priv->tx_cq[k], priv->eq[k % priv->eq_count]); + for (k = 0; k < priv->txq_count; k++) { + // create CQ + cq = mqnic_create_cq(iface); + if (IS_ERR_OR_NULL(cq)) { + ret = PTR_ERR(cq); + goto fail; + } + + ret = mqnic_open_cq(cq, iface->eq[k % iface->eq_count], mqnic_num_txq_entries, 1); + if (ret) { + mqnic_destroy_cq(cq); + goto fail; + } #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0) - netif_napi_add_tx(ndev, &priv->tx_cq[k]->napi, mqnic_poll_tx_cq); + netif_napi_add_tx(ndev, &cq->napi, mqnic_poll_tx_cq); #else - netif_tx_napi_add(ndev, &priv->tx_cq[k]->napi, mqnic_poll_tx_cq, NAPI_POLL_WEIGHT); + netif_tx_napi_add(ndev, &cq->napi, mqnic_poll_tx_cq, NAPI_POLL_WEIGHT); #endif - napi_enable(&priv->tx_cq[k]->napi); + napi_enable(&cq->napi); - mqnic_arm_cq(priv->tx_cq[k]); + mqnic_arm_cq(cq); - // set up queue - priv->txq[k]->tx_queue = netdev_get_tx_queue(ndev, k); - mqnic_activate_tx_ring(priv->txq[k], priv, priv->tx_cq[k]); + // create TX queue + q = mqnic_create_tx_ring(iface); + if (IS_ERR_OR_NULL(q)) { + ret = PTR_ERR(q); + mqnic_destroy_cq(cq); + goto fail; + } + + q->tx_queue = netdev_get_tx_queue(ndev, k); + + ret = mqnic_open_tx_ring(q, priv, cq, mqnic_num_txq_entries, desc_block_size); + if (ret) { + mqnic_destroy_tx_ring(q); + mqnic_destroy_cq(cq); + goto fail; + } + + down_write(&priv->txq_table_sem); + ret = radix_tree_insert(&priv->txq_table, k, q); + up_write(&priv->txq_table_sem); + if (ret) { + mqnic_destroy_tx_ring(q); + mqnic_destroy_cq(cq); + goto fail; + } } // set MTU - mqnic_interface_set_tx_mtu(priv->interface, ndev->mtu + ETH_HLEN); - mqnic_interface_set_rx_mtu(priv->interface, ndev->mtu + ETH_HLEN); + mqnic_interface_set_tx_mtu(iface, ndev->mtu + ETH_HLEN); + mqnic_interface_set_rx_mtu(iface, ndev->mtu + ETH_HLEN); // configure RX indirection and RSS - mqnic_interface_set_rx_queue_map_rss_mask(priv->interface, 0, 0xffffffff); - mqnic_interface_set_rx_queue_map_app_mask(priv->interface, 0, 0); + mqnic_interface_set_rx_queue_map_rss_mask(iface, 0, 0xffffffff); + mqnic_interface_set_rx_queue_map_app_mask(iface, 0, 0); - for (k = 0; k < priv->interface->rx_queue_map_indir_table_size; k++) { - mqnic_interface_set_rx_queue_map_indir_table(priv->interface, 0, k, priv->rxq[k % priv->rxq_count]->index); + for (k = 0; k < iface->rx_queue_map_indir_table_size; k++) { + rcu_read_lock(); + q = radix_tree_lookup(&priv->rxq_table, k % priv->rxq_count); + rcu_read_unlock(); + mqnic_interface_set_rx_queue_map_indir_table(iface, 0, k, q->index); } + priv->port_up = true; + + // enable TX and RX queues + down_read(&priv->txq_table_sem); + radix_tree_for_each_slot(slot, &priv->txq_table, &iter, 0) { + struct mqnic_ring *q = (struct mqnic_ring *)*slot; + + mqnic_enable_tx_ring(q); + } + up_read(&priv->txq_table_sem); + + down_read(&priv->rxq_table_sem); + radix_tree_for_each_slot(slot, &priv->rxq_table, &iter, 0) { + struct mqnic_ring *q = (struct mqnic_ring *)*slot; + + mqnic_enable_rx_ring(q); + } + up_read(&priv->rxq_table_sem); + // enable first scheduler mqnic_activate_sched_block(priv->sched_block[0]); - priv->port_up = true; - netif_tx_start_all_queues(ndev); netif_device_attach(ndev); @@ -117,12 +212,19 @@ static int mqnic_start_port(struct net_device *ndev) } return 0; + +fail: + mqnic_stop_port(ndev); + return ret; } static int mqnic_stop_port(struct net_device *ndev) { struct mqnic_priv *priv = netdev_priv(ndev); struct mqnic_dev *mdev = priv->mdev; + struct mqnic_cq *cq; + struct radix_tree_iter iter; + void **slot; int k; dev_info(mdev->dev, "%s on interface %d netdev %d", __func__, @@ -142,7 +244,6 @@ static int mqnic_stop_port(struct net_device *ndev) spin_lock_bh(&priv->stats_lock); mqnic_update_stats(ndev); - priv->port_up = false; spin_unlock_bh(&priv->stats_lock); // disable schedulers @@ -150,28 +251,56 @@ static int mqnic_stop_port(struct net_device *ndev) mqnic_deactivate_sched_block(priv->sched_block[k]); // disable TX and RX queues - for (k = 0; k < min(priv->txq_count, priv->tx_cq_count); k++) - mqnic_deactivate_tx_ring(priv->txq[k]); + down_read(&priv->txq_table_sem); + radix_tree_for_each_slot(slot, &priv->txq_table, &iter, 0) { + struct mqnic_ring *q = (struct mqnic_ring *)*slot; - for (k = 0; k < min(priv->rxq_count, priv->rx_cq_count); k++) - mqnic_deactivate_rx_ring(priv->rxq[k]); + mqnic_disable_tx_ring(q); + } + up_read(&priv->txq_table_sem); + + down_read(&priv->rxq_table_sem); + radix_tree_for_each_slot(slot, &priv->rxq_table, &iter, 0) { + struct mqnic_ring *q = (struct mqnic_ring *)*slot; + + mqnic_disable_rx_ring(q); + } + up_read(&priv->rxq_table_sem); msleep(20); - // shut down NAPI and clean queues - for (k = 0; k < priv->txq_count; k++) { - napi_disable(&priv->tx_cq[k]->napi); - mqnic_deactivate_cq(priv->tx_cq[k]); - netif_napi_del(&priv->tx_cq[k]->napi); - mqnic_free_tx_buf(priv->txq[k]); - } + priv->port_up = false; - for (k = 0; k < priv->rxq_count; k++) { - napi_disable(&priv->rx_cq[k]->napi); - mqnic_deactivate_cq(priv->rx_cq[k]); - netif_napi_del(&priv->rx_cq[k]->napi); - mqnic_free_rx_buf(priv->rxq[k]); + // shut down NAPI and clean queues + down_write(&priv->txq_table_sem); + radix_tree_for_each_slot(slot, &priv->txq_table, &iter, 0) { + struct mqnic_ring *q = (struct mqnic_ring *)*slot; + + cq = q->cq; + napi_disable(&cq->napi); + netif_napi_del(&cq->napi); + mqnic_close_tx_ring(q); + mqnic_destroy_tx_ring(q); + radix_tree_delete(&priv->txq_table, iter.index); + mqnic_close_cq(cq); + mqnic_destroy_cq(cq); } + up_write(&priv->txq_table_sem); + + down_write(&priv->rxq_table_sem); + radix_tree_for_each_slot(slot, &priv->rxq_table, &iter, 0) { + struct mqnic_ring *q = (struct mqnic_ring *)*slot; + + cq = q->cq; + napi_disable(&cq->napi); + netif_napi_del(&cq->napi); + mqnic_close_rx_ring(q); + mqnic_destroy_rx_ring(q); + radix_tree_delete(&priv->rxq_table, iter.index); + mqnic_close_cq(cq); + mqnic_destroy_cq(cq); + } + up_write(&priv->rxq_table_sem); return 0; } @@ -215,31 +344,36 @@ static int mqnic_close(struct net_device *ndev) void mqnic_update_stats(struct net_device *ndev) { struct mqnic_priv *priv = netdev_priv(ndev); + struct radix_tree_iter iter; + void **slot; unsigned long packets, bytes; - int k; if (unlikely(!priv->port_up)) return; packets = 0; bytes = 0; - for (k = 0; k < priv->rxq_count; k++) { - const struct mqnic_ring *ring = priv->rxq[k]; + down_read(&priv->rxq_table_sem); + radix_tree_for_each_slot(slot, &priv->rxq_table, &iter, 0) { + const struct mqnic_ring *q = (struct mqnic_ring *)*slot; - packets += READ_ONCE(ring->packets); - bytes += READ_ONCE(ring->bytes); + packets += READ_ONCE(q->packets); + bytes += READ_ONCE(q->bytes); } + up_read(&priv->rxq_table_sem); ndev->stats.rx_packets = packets; ndev->stats.rx_bytes = bytes; packets = 0; bytes = 0; - for (k = 0; k < priv->txq_count; k++) { - const struct mqnic_ring *ring = priv->txq[k]; + down_read(&priv->txq_table_sem); + radix_tree_for_each_slot(slot, &priv->txq_table, &iter, 0) { + const struct mqnic_ring *q = (struct mqnic_ring *)*slot; - packets += READ_ONCE(ring->packets); - bytes += READ_ONCE(ring->bytes); + packets += READ_ONCE(q->packets); + bytes += READ_ONCE(q->bytes); } + up_read(&priv->txq_table_sem); ndev->stats.tx_packets = packets; ndev->stats.tx_bytes = bytes; } @@ -437,25 +571,14 @@ struct net_device *mqnic_create_netdev(struct mqnic_if *interface, int index, in // associate interface resources priv->if_features = interface->if_features; - priv->eq_count = interface->eq_count; - for (k = 0; k < interface->eq_count; k++) - priv->eq[k] = interface->eq[k]; + priv->txq_count = mqnic_res_get_count(interface->txq_res); + priv->rxq_count = mqnic_res_get_count(interface->rxq_res); - priv->txq_count = interface->txq_count; - for (k = 0; k < interface->txq_count; k++) - priv->txq[k] = interface->txq[k]; + init_rwsem(&priv->txq_table_sem); + INIT_RADIX_TREE(&priv->txq_table, GFP_KERNEL); - priv->tx_cq_count = interface->tx_cq_count; - for (k = 0; k < interface->tx_cq_count; k++) - priv->tx_cq[k] = interface->tx_cq[k]; - - priv->rxq_count = interface->rxq_count; - for (k = 0; k < interface->rxq_count; k++) - priv->rxq[k] = interface->rxq[k]; - - priv->rx_cq_count = interface->rx_cq_count; - for (k = 0; k < interface->rx_cq_count; k++) - priv->rx_cq[k] = interface->rx_cq[k]; + init_rwsem(&priv->rxq_table_sem); + INIT_RADIX_TREE(&priv->rxq_table, GFP_KERNEL); priv->sched_block_count = interface->sched_block_count; for (k = 0; k < interface->sched_block_count; k++) @@ -489,36 +612,6 @@ struct net_device *mqnic_create_netdev(struct mqnic_if *interface, int index, in desc_block_size = min_t(u32, interface->max_desc_block_size, 4); - // allocate ring buffers - for (k = 0; k < priv->txq_count; k++) { - ret = mqnic_alloc_tx_ring(priv->txq[k], mqnic_num_txq_entries, - MQNIC_DESC_SIZE * desc_block_size); - if (ret) - goto fail; - } - - for (k = 0; k < priv->tx_cq_count; k++) { - ret = mqnic_alloc_cq(priv->tx_cq[k], mqnic_num_txq_entries, - MQNIC_CPL_SIZE); - if (ret) - goto fail; - } - - for (k = 0; k < priv->rxq_count; k++) { - ret = mqnic_alloc_rx_ring(priv->rxq[k], mqnic_num_rxq_entries, - MQNIC_DESC_SIZE); - if (ret) - goto fail; - } - - for (k = 0; k < priv->rx_cq_count; k++) { - ret = mqnic_alloc_cq(priv->rx_cq[k], mqnic_num_rxq_entries, - MQNIC_CPL_SIZE); - - if (ret) - goto fail; - } - // entry points ndev->netdev_ops = &mqnic_netdev_ops; ndev->ethtool_ops = &mqnic_ethtool_ops; @@ -564,27 +657,9 @@ fail: void mqnic_destroy_netdev(struct net_device *ndev) { struct mqnic_priv *priv = netdev_priv(ndev); - int k; if (priv->registered) unregister_netdev(ndev); - // free rings - for (k = 0; k < ARRAY_SIZE(priv->txq); k++) - if (priv->txq[k]) - mqnic_free_tx_ring(priv->txq[k]); - - for (k = 0; k < ARRAY_SIZE(priv->tx_cq); k++) - if (priv->tx_cq[k]) - mqnic_free_cq(priv->tx_cq[k]); - - for (k = 0; k < ARRAY_SIZE(priv->rxq); k++) - if (priv->rxq[k]) - mqnic_free_rx_ring(priv->rxq[k]); - - for (k = 0; k < ARRAY_SIZE(priv->rx_cq); k++) - if (priv->rx_cq[k]) - mqnic_free_cq(priv->rx_cq[k]); - free_netdev(ndev); } diff --git a/modules/mqnic/mqnic_res.c b/modules/mqnic/mqnic_res.c index 0985b90f2..5f2c408bd 100644 --- a/modules/mqnic/mqnic_res.c +++ b/modules/mqnic/mqnic_res.c @@ -38,6 +38,7 @@ struct mqnic_res *mqnic_create_res(unsigned int count, u8 __iomem *base, unsigned int stride) { struct mqnic_res *res; + int ret; res = kzalloc(sizeof(*res), GFP_KERNEL); if (!res) @@ -47,7 +48,19 @@ struct mqnic_res *mqnic_create_res(unsigned int count, u8 __iomem *base, unsigne res->base = base; res->stride = stride; + spin_lock_init(&res->lock); + + res->bmap = bitmap_zalloc(count, GFP_KERNEL); + if (!res) { + ret = -ENOMEM; + goto fail; + } + return res; + +fail: + mqnic_destroy_res(res); + return ERR_PTR(ret); } void mqnic_destroy_res(struct mqnic_res *res) @@ -55,9 +68,32 @@ void mqnic_destroy_res(struct mqnic_res *res) if (!res) return; + bitmap_free(res->bmap); + kfree(res); } +int mqnic_res_alloc(struct mqnic_res *res) +{ + int index; + + spin_lock(&res->lock); + index = bitmap_find_free_region(res->bmap, res->count, 0); + spin_unlock(&res->lock); + + return index; +} + +void mqnic_res_free(struct mqnic_res *res, int index) +{ + if (index < 0 || index >= res->count) + return; + + spin_lock(&res->lock); + bitmap_clear(res->bmap, index, 1); + spin_unlock(&res->lock); +} + unsigned int mqnic_res_get_count(struct mqnic_res *res) { return res->count; diff --git a/modules/mqnic/mqnic_rx.c b/modules/mqnic/mqnic_rx.c index e5bcda8a9..94409a781 100644 --- a/modules/mqnic/mqnic_rx.c +++ b/modules/mqnic/mqnic_rx.c @@ -35,8 +35,7 @@ #include "mqnic.h" -struct mqnic_ring *mqnic_create_rx_ring(struct mqnic_if *interface, - int index, u8 __iomem *hw_addr) +struct mqnic_ring *mqnic_create_rx_ring(struct mqnic_if *interface) { struct mqnic_ring *ring; @@ -47,48 +46,52 @@ struct mqnic_ring *mqnic_create_rx_ring(struct mqnic_if *interface, ring->dev = interface->dev; ring->interface = interface; - ring->index = index; - ring->active = 0; + ring->index = -1; + ring->enabled = 0; - ring->hw_addr = hw_addr; + ring->hw_addr = NULL; ring->hw_ptr_mask = 0xffff; - ring->hw_head_ptr = hw_addr + MQNIC_QUEUE_HEAD_PTR_REG; - ring->hw_tail_ptr = hw_addr + MQNIC_QUEUE_TAIL_PTR_REG; + ring->hw_head_ptr = NULL; + ring->hw_tail_ptr = NULL; ring->head_ptr = 0; ring->tail_ptr = 0; - // deactivate queue - iowrite32(0, ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG); - return ring; } void mqnic_destroy_rx_ring(struct mqnic_ring *ring) { - mqnic_free_rx_ring(ring); + mqnic_close_rx_ring(ring); kfree(ring); } -int mqnic_alloc_rx_ring(struct mqnic_ring *ring, int size, int stride) +int mqnic_open_rx_ring(struct mqnic_ring *ring, struct mqnic_priv *priv, + struct mqnic_cq *cq, int size, int desc_block_size) { int ret = 0; - if (ring->active || ring->buf) + if (ring->enabled || ring->hw_addr || ring->buf || !priv || !cq) return -EINVAL; - ring->size = roundup_pow_of_two(size); - ring->size_mask = ring->size - 1; - ring->stride = roundup_pow_of_two(stride); + ring->index = mqnic_res_alloc(ring->interface->rxq_res); + if (ring->index < 0) + return -ENOMEM; - ring->desc_block_size = ring->stride / MQNIC_DESC_SIZE; - ring->log_desc_block_size = ring->desc_block_size < 2 ? 0 : ilog2(ring->desc_block_size - 1) + 1; + ring->log_desc_block_size = desc_block_size < 2 ? 0 : ilog2(desc_block_size - 1) + 1; ring->desc_block_size = 1 << ring->log_desc_block_size; + ring->size = roundup_pow_of_two(size); + ring->full_size = ring->size >> 1; + ring->size_mask = ring->size - 1; + ring->stride = roundup_pow_of_two(MQNIC_DESC_SIZE * ring->desc_block_size); + ring->rx_info = kvzalloc(sizeof(*ring->rx_info) * ring->size, GFP_KERNEL); - if (!ring->rx_info) - return -ENOMEM; + if (!ring->rx_info) { + ret = -ENOMEM; + goto fail; + } ring->buf_size = ring->size * ring->stride; ring->buf = dma_alloc_coherent(ring->dev, ring->buf_size, &ring->buf_dma_addr, GFP_KERNEL); @@ -97,6 +100,15 @@ int mqnic_alloc_rx_ring(struct mqnic_ring *ring, int size, int stride) goto fail; } + ring->priv = priv; + ring->cq = cq; + cq->src_ring = ring; + cq->handler = mqnic_rx_irq; + + ring->hw_addr = mqnic_res_get_addr(ring->interface->rxq_res, ring->index); + ring->hw_head_ptr = ring->hw_addr + MQNIC_QUEUE_HEAD_PTR_REG; + ring->hw_tail_ptr = ring->hw_addr + MQNIC_QUEUE_TAIL_PTR_REG; + ring->head_ptr = 0; ring->tail_ptr = 0; @@ -105,8 +117,8 @@ int mqnic_alloc_rx_ring(struct mqnic_ring *ring, int size, int stride) // set base address iowrite32(ring->buf_dma_addr, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 0); iowrite32(ring->buf_dma_addr >> 32, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 4); - // set completion queue index - iowrite32(0, ring->hw_addr + MQNIC_QUEUE_CPL_QUEUE_INDEX_REG); + // set CQN + iowrite32(ring->cq->cqn, ring->hw_addr + MQNIC_QUEUE_CPL_QUEUE_INDEX_REG); // set pointers iowrite32(ring->head_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_HEAD_PTR_REG); iowrite32(ring->tail_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_TAIL_PTR_REG); @@ -114,16 +126,30 @@ int mqnic_alloc_rx_ring(struct mqnic_ring *ring, int size, int stride) iowrite32(ilog2(ring->size) | (ring->log_desc_block_size << 8), ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG); + mqnic_refill_rx_buffers(ring); + return 0; fail: - mqnic_free_rx_ring(ring); + mqnic_close_rx_ring(ring); return ret; } -void mqnic_free_rx_ring(struct mqnic_ring *ring) +void mqnic_close_rx_ring(struct mqnic_ring *ring) { - mqnic_deactivate_rx_ring(ring); + mqnic_disable_rx_ring(ring); + + if (ring->cq) { + ring->cq->src_ring = NULL; + ring->cq->handler = NULL; + } + + ring->priv = NULL; + ring->cq = NULL; + + ring->hw_addr = NULL; + ring->hw_head_ptr = NULL; + ring->hw_tail_ptr = NULL; if (ring->buf) { mqnic_free_rx_buf(ring); @@ -137,60 +163,34 @@ void mqnic_free_rx_ring(struct mqnic_ring *ring) kvfree(ring->rx_info); ring->rx_info = NULL; } + + mqnic_res_free(ring->interface->rxq_res, ring->index); + ring->index = -1; } -int mqnic_activate_rx_ring(struct mqnic_ring *ring, struct mqnic_priv *priv, - struct mqnic_cq *cq) +int mqnic_enable_rx_ring(struct mqnic_ring *ring) { - mqnic_deactivate_rx_ring(ring); - - if (!ring->buf || !priv || !cq || cq->handler || cq->src_ring) + if (!ring->hw_addr) return -EINVAL; - ring->priv = priv; - ring->cq = cq; - cq->src_ring = ring; - cq->handler = mqnic_rx_irq; - - ring->head_ptr = 0; - ring->tail_ptr = 0; - - // deactivate queue - iowrite32(0, ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG); - // set base address - iowrite32(ring->buf_dma_addr, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 0); - iowrite32(ring->buf_dma_addr >> 32, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 4); - // set CQN - iowrite32(cq->cqn, ring->hw_addr + MQNIC_QUEUE_CPL_QUEUE_INDEX_REG); - // set pointers - iowrite32(ring->head_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_HEAD_PTR_REG); - iowrite32(ring->tail_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_TAIL_PTR_REG); - // set size and activate queue + // enable queue iowrite32(ilog2(ring->size) | (ring->log_desc_block_size << 8) | MQNIC_QUEUE_ACTIVE_MASK, ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG); - ring->active = 1; - - mqnic_refill_rx_buffers(ring); + ring->enabled = 1; return 0; } -void mqnic_deactivate_rx_ring(struct mqnic_ring *ring) +void mqnic_disable_rx_ring(struct mqnic_ring *ring) { - // deactivate queue - iowrite32(ilog2(ring->size) | (ring->log_desc_block_size << 8), - ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG); - - if (ring->cq) { - ring->cq->src_ring = NULL; - ring->cq->handler = NULL; + // disable queue + if (ring->hw_addr) { + iowrite32(ilog2(ring->size) | (ring->log_desc_block_size << 8), + ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG); } - ring->priv = NULL; - ring->cq = NULL; - - ring->active = 0; + ring->enabled = 0; } bool mqnic_is_rx_ring_empty(const struct mqnic_ring *ring) diff --git a/modules/mqnic/mqnic_sched_block.c b/modules/mqnic/mqnic_sched_block.c index 582aa98af..aa09220ed 100644 --- a/modules/mqnic/mqnic_sched_block.c +++ b/modules/mqnic/mqnic_sched_block.c @@ -53,8 +53,6 @@ struct mqnic_sched_block *mqnic_create_sched_block(struct mqnic_if *interface, block->index = index; - block->txq_count = interface->txq_count; - block->block_rb = block_rb; offset = ioread32(block_rb->regs + MQNIC_RB_SCHED_BLOCK_REG_OFFSET); diff --git a/modules/mqnic/mqnic_tx.c b/modules/mqnic/mqnic_tx.c index 9da6c21c1..6808e6058 100644 --- a/modules/mqnic/mqnic_tx.c +++ b/modules/mqnic/mqnic_tx.c @@ -36,8 +36,7 @@ #include #include "mqnic.h" -struct mqnic_ring *mqnic_create_tx_ring(struct mqnic_if *interface, - int index, u8 __iomem *hw_addr) +struct mqnic_ring *mqnic_create_tx_ring(struct mqnic_if *interface) { struct mqnic_ring *ring; @@ -48,49 +47,52 @@ struct mqnic_ring *mqnic_create_tx_ring(struct mqnic_if *interface, ring->dev = interface->dev; ring->interface = interface; - ring->index = index; - ring->active = 0; + ring->index = -1; + ring->enabled = 0; - ring->hw_addr = hw_addr; + ring->hw_addr = NULL; ring->hw_ptr_mask = 0xffff; - ring->hw_head_ptr = hw_addr + MQNIC_QUEUE_HEAD_PTR_REG; - ring->hw_tail_ptr = hw_addr + MQNIC_QUEUE_TAIL_PTR_REG; + ring->hw_head_ptr = NULL; + ring->hw_tail_ptr = NULL; ring->head_ptr = 0; ring->tail_ptr = 0; - // deactivate queue - iowrite32(0, ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG); - return ring; } void mqnic_destroy_tx_ring(struct mqnic_ring *ring) { - mqnic_free_tx_ring(ring); + mqnic_close_tx_ring(ring); kfree(ring); } -int mqnic_alloc_tx_ring(struct mqnic_ring *ring, int size, int stride) +int mqnic_open_tx_ring(struct mqnic_ring *ring, struct mqnic_priv *priv, + struct mqnic_cq *cq, int size, int desc_block_size) { int ret = 0; - if (ring->active || ring->buf) + if (ring->enabled || ring->hw_addr || ring->buf || !priv || !cq) return -EINVAL; + ring->index = mqnic_res_alloc(ring->interface->txq_res); + if (ring->index < 0) + return -ENOMEM; + + ring->log_desc_block_size = desc_block_size < 2 ? 0 : ilog2(desc_block_size - 1) + 1; + ring->desc_block_size = 1 << ring->log_desc_block_size; + ring->size = roundup_pow_of_two(size); ring->full_size = ring->size >> 1; ring->size_mask = ring->size - 1; - ring->stride = roundup_pow_of_two(stride); - - ring->desc_block_size = ring->stride / MQNIC_DESC_SIZE; - ring->log_desc_block_size = ring->desc_block_size < 2 ? 0 : ilog2(ring->desc_block_size - 1) + 1; - ring->desc_block_size = 1 << ring->log_desc_block_size; + ring->stride = roundup_pow_of_two(MQNIC_DESC_SIZE * ring->desc_block_size); ring->tx_info = kvzalloc(sizeof(*ring->tx_info) * ring->size, GFP_KERNEL); - if (!ring->tx_info) - return -ENOMEM; + if (!ring->tx_info) { + ret = -ENOMEM; + goto fail; + } ring->buf_size = ring->size * ring->stride; ring->buf = dma_alloc_coherent(ring->dev, ring->buf_size, &ring->buf_dma_addr, GFP_KERNEL); @@ -99,6 +101,15 @@ int mqnic_alloc_tx_ring(struct mqnic_ring *ring, int size, int stride) goto fail; } + ring->priv = priv; + ring->cq = cq; + cq->src_ring = ring; + cq->handler = mqnic_tx_irq; + + ring->hw_addr = mqnic_res_get_addr(ring->interface->txq_res, ring->index); + ring->hw_head_ptr = ring->hw_addr + MQNIC_QUEUE_HEAD_PTR_REG; + ring->hw_tail_ptr = ring->hw_addr + MQNIC_QUEUE_TAIL_PTR_REG; + ring->head_ptr = 0; ring->tail_ptr = 0; @@ -107,8 +118,8 @@ int mqnic_alloc_tx_ring(struct mqnic_ring *ring, int size, int stride) // set base address iowrite32(ring->buf_dma_addr, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 0); iowrite32(ring->buf_dma_addr >> 32, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 4); - // set completion queue index - iowrite32(0, ring->hw_addr + MQNIC_QUEUE_CPL_QUEUE_INDEX_REG); + // set CQN + iowrite32(ring->cq->cqn, ring->hw_addr + MQNIC_QUEUE_CPL_QUEUE_INDEX_REG); // set pointers iowrite32(ring->head_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_HEAD_PTR_REG); iowrite32(ring->tail_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_TAIL_PTR_REG); @@ -119,13 +130,25 @@ int mqnic_alloc_tx_ring(struct mqnic_ring *ring, int size, int stride) return 0; fail: - mqnic_free_tx_ring(ring); + mqnic_close_tx_ring(ring); return ret; } -void mqnic_free_tx_ring(struct mqnic_ring *ring) +void mqnic_close_tx_ring(struct mqnic_ring *ring) { - mqnic_deactivate_tx_ring(ring); + mqnic_disable_tx_ring(ring); + + if (ring->cq) { + ring->cq->src_ring = NULL; + ring->cq->handler = NULL; + } + + ring->priv = NULL; + ring->cq = NULL; + + ring->hw_addr = NULL; + ring->hw_head_ptr = NULL; + ring->hw_tail_ptr = NULL; if (ring->buf) { mqnic_free_tx_buf(ring); @@ -139,58 +162,34 @@ void mqnic_free_tx_ring(struct mqnic_ring *ring) kvfree(ring->tx_info); ring->tx_info = NULL; } + + mqnic_res_free(ring->interface->txq_res, ring->index); + ring->index = -1; } -int mqnic_activate_tx_ring(struct mqnic_ring *ring, struct mqnic_priv *priv, - struct mqnic_cq *cq) +int mqnic_enable_tx_ring(struct mqnic_ring *ring) { - mqnic_deactivate_tx_ring(ring); - - if (!ring->buf || !priv || !cq || cq->handler || cq->src_ring) + if (!ring->hw_addr) return -EINVAL; - ring->priv = priv; - ring->cq = cq; - cq->src_ring = ring; - cq->handler = mqnic_tx_irq; - - ring->head_ptr = 0; - ring->tail_ptr = 0; - - // deactivate queue - iowrite32(0, ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG); - // set base address - iowrite32(ring->buf_dma_addr, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 0); - iowrite32(ring->buf_dma_addr >> 32, ring->hw_addr + MQNIC_QUEUE_BASE_ADDR_REG + 4); - // set CQN - iowrite32(cq->cqn, ring->hw_addr + MQNIC_QUEUE_CPL_QUEUE_INDEX_REG); - // set pointers - iowrite32(ring->head_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_HEAD_PTR_REG); - iowrite32(ring->tail_ptr & ring->hw_ptr_mask, ring->hw_addr + MQNIC_QUEUE_TAIL_PTR_REG); - // set size and activate queue + // enable queue iowrite32(ilog2(ring->size) | (ring->log_desc_block_size << 8) | MQNIC_QUEUE_ACTIVE_MASK, ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG); - ring->active = 1; + ring->enabled = 1; return 0; } -void mqnic_deactivate_tx_ring(struct mqnic_ring *ring) +void mqnic_disable_tx_ring(struct mqnic_ring *ring) { - // deactivate queue - iowrite32(ilog2(ring->size) | (ring->log_desc_block_size << 8), - ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG); - - if (ring->cq) { - ring->cq->src_ring = NULL; - ring->cq->handler = NULL; + // disable queue + if (ring->hw_addr) { + iowrite32(ilog2(ring->size) | (ring->log_desc_block_size << 8), + ring->hw_addr + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG); } - ring->priv = NULL; - ring->cq = NULL; - - ring->active = 0; + ring->enabled = 0; } bool mqnic_is_tx_ring_empty(const struct mqnic_ring *ring) @@ -444,11 +443,13 @@ netdev_tx_t mqnic_start_xmit(struct sk_buff *skb, struct net_device *ndev) ring_index = skb_get_queue_mapping(skb); - if (unlikely(ring_index >= priv->txq_count)) - // queue mapping out of range - goto tx_drop; + rcu_read_lock(); + ring = radix_tree_lookup(&priv->txq_table, ring_index); + rcu_read_unlock(); - ring = priv->txq[ring_index]; + if (unlikely(!ring)) + // unknown TX queue + goto tx_drop; tail_ptr = READ_ONCE(ring->tail_ptr);