corundum/modules/mqnic/mqnic_netdev.c

/*

Copyright 2019-2021, The Regents of the University of California.
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

   1. Redistributions of source code must retain the above copyright notice,
      this list of conditions and the following disclaimer.

   2. Redistributions in binary form must reproduce the above copyright notice,
      this list of conditions and the following disclaimer in the documentation
      and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE REGENTS OF THE UNIVERSITY OF CALIFORNIA ''AS
IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE REGENTS OF THE UNIVERSITY OF CALIFORNIA OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
OF SUCH DAMAGE.

The views and conclusions contained in the software and documentation are those
of the authors and should not be interpreted as representing official policies,
either expressed or implied, of The Regents of the University of California.

*/

#include "mqnic.h"

static int mqnic_start_port(struct net_device *ndev)
{
	struct mqnic_priv *priv = netdev_priv(ndev);
	struct mqnic_dev *mdev = priv->mdev;
	int k;

	dev_info(mdev->dev, "mqnic_start_port on port %d", priv->port);

	// set up event queues
	for (k = 0; k < priv->event_queue_count; k++) {
		priv->event_ring[k]->irq = mdev->irq_map[k % mdev->irq_count];
		mqnic_activate_eq_ring(priv, priv->event_ring[k], k % mdev->irq_count);
		mqnic_arm_eq(priv->event_ring[k]);
	}

	// set up RX completion queues
	for (k = 0; k < priv->rx_cpl_queue_count; k++) {
		mqnic_activate_cq_ring(priv, priv->rx_cpl_ring[k], k % priv->event_queue_count);
		priv->rx_cpl_ring[k]->ring_index = k;
		priv->rx_cpl_ring[k]->handler = mqnic_rx_irq;

		netif_napi_add(ndev, &priv->rx_cpl_ring[k]->napi,
				mqnic_poll_rx_cq, NAPI_POLL_WEIGHT);
		napi_enable(&priv->rx_cpl_ring[k]->napi);

		mqnic_arm_cq(priv->rx_cpl_ring[k]);
	}

	// set up RX queues
	for (k = 0; k < priv->rx_queue_count; k++) {
		priv->rx_ring[k]->mtu = ndev->mtu;
		if (ndev->mtu + ETH_HLEN <= PAGE_SIZE)
			priv->rx_ring[k]->page_order = 0;
		else
			priv->rx_ring[k]->page_order = ilog2((ndev->mtu + ETH_HLEN + PAGE_SIZE - 1) / PAGE_SIZE - 1) + 1;
		mqnic_activate_rx_ring(priv, priv->rx_ring[k], k);
	}

	// set up TX completion queues
	for (k = 0; k < priv->tx_cpl_queue_count; k++) {
		mqnic_activate_cq_ring(priv, priv->tx_cpl_ring[k], k % priv->event_queue_count);
		priv->tx_cpl_ring[k]->ring_index = k;
		priv->tx_cpl_ring[k]->handler = mqnic_tx_irq;

		netif_tx_napi_add(ndev, &priv->tx_cpl_ring[k]->napi,
				mqnic_poll_tx_cq, NAPI_POLL_WEIGHT);
		napi_enable(&priv->tx_cpl_ring[k]->napi);

		mqnic_arm_cq(priv->tx_cpl_ring[k]);
	}

	// set up TX queues
	for (k = 0; k < priv->tx_queue_count; k++) {
		mqnic_activate_tx_ring(priv, priv->tx_ring[k], k);
		priv->tx_ring[k]->tx_queue = netdev_get_tx_queue(ndev, k);
	}

	// configure ports
	for (k = 0; k < priv->port_count; k++) {
		// set port MTU
		mqnic_port_set_tx_mtu(priv->ports[k], ndev->mtu + ETH_HLEN);
		mqnic_port_set_rx_mtu(priv->ports[k], ndev->mtu + ETH_HLEN);
	}

	// enable first port
	mqnic_activate_port(priv->ports[0]);

	priv->port_up = true;

	netif_tx_start_all_queues(ndev);
	netif_device_attach(ndev);

	//netif_carrier_off(ndev);
	netif_carrier_on(ndev); // TODO link status monitoring

	return 0;
}

static int mqnic_stop_port(struct net_device *ndev)
{
	struct mqnic_priv *priv = netdev_priv(ndev);
	struct mqnic_dev *mdev = priv->mdev;
	int k;

	dev_info(mdev->dev, "mqnic_stop_port on port %d", priv->port);

	netif_tx_lock_bh(ndev);
//	if (detach)
//		netif_device_detach(ndev);
	netif_tx_stop_all_queues(ndev);
	netif_tx_unlock_bh(ndev);

	netif_tx_disable(ndev);

	spin_lock_bh(&priv->stats_lock);
	mqnic_update_stats(ndev);
	priv->port_up = false;
	spin_unlock_bh(&priv->stats_lock);

	// disable ports
	for (k = 0; k < priv->port_count; k++)
		mqnic_deactivate_port(priv->ports[k]);

	// deactivate TX queues
	for (k = 0; k < priv->tx_queue_count; k++)
		mqnic_deactivate_tx_ring(priv, priv->tx_ring[k]);

	// deactivate TX completion queues
	for (k = 0; k < priv->tx_cpl_queue_count; k++) {
		mqnic_deactivate_cq_ring(priv, priv->tx_cpl_ring[k]);

		napi_disable(&priv->tx_cpl_ring[k]->napi);
		netif_napi_del(&priv->tx_cpl_ring[k]->napi);
	}

	// deactivate RX queues
	for (k = 0; k < priv->rx_queue_count; k++)
		mqnic_deactivate_rx_ring(priv, priv->rx_ring[k]);

	// deactivate RX completion queues
	for (k = 0; k < priv->rx_cpl_queue_count; k++) {
		mqnic_deactivate_cq_ring(priv, priv->rx_cpl_ring[k]);

		napi_disable(&priv->rx_cpl_ring[k]->napi);
		netif_napi_del(&priv->rx_cpl_ring[k]->napi);
	}

	// deactivate event queues
	for (k = 0; k < priv->event_queue_count; k++)
		mqnic_deactivate_eq_ring(priv, priv->event_ring[k]);

	msleep(20);

	// free descriptors in TX queues
	for (k = 0; k < priv->tx_queue_count; k++)
		mqnic_free_tx_buf(priv, priv->tx_ring[k]);

	// free descriptors in RX queues
	for (k = 0; k < priv->rx_queue_count; k++)
		mqnic_free_rx_buf(priv, priv->rx_ring[k]);

	netif_carrier_off(ndev);
	return 0;
}

static int mqnic_open(struct net_device *ndev)
{
	struct mqnic_priv *priv = netdev_priv(ndev);
	struct mqnic_dev *mdev = priv->mdev;
	int ret = 0;

	mutex_lock(&mdev->state_lock);

	ret = mqnic_start_port(ndev);

	if (ret)
		dev_err(mdev->dev, "Failed to start port: %d", priv->port);

	mutex_unlock(&mdev->state_lock);
	return ret;
}

static int mqnic_close(struct net_device *ndev)
{
	struct mqnic_priv *priv = netdev_priv(ndev);
	struct mqnic_dev *mdev = priv->mdev;
	int ret = 0;

	mutex_lock(&mdev->state_lock);

	ret = mqnic_stop_port(ndev);

	if (ret)
		dev_err(mdev->dev, "Failed to stop port: %d", priv->port);

	mutex_unlock(&mdev->state_lock);
	return ret;
}

void mqnic_update_stats(struct net_device *ndev)
{
	struct mqnic_priv *priv = netdev_priv(ndev);
	unsigned long packets, bytes;
	int k;

	if (unlikely(!priv->port_up))
		return;

	packets = 0;
	bytes = 0;
	for (k = 0; k < priv->rx_queue_count; k++) {
		const struct mqnic_ring *ring = priv->rx_ring[k];

		packets += READ_ONCE(ring->packets);
		bytes += READ_ONCE(ring->bytes);
	}
	ndev->stats.rx_packets = packets;
	ndev->stats.rx_bytes = bytes;

	packets = 0;
	bytes = 0;
	for (k = 0; k < priv->tx_queue_count; k++) {
		const struct mqnic_ring *ring = priv->tx_ring[k];

		packets += READ_ONCE(ring->packets);
		bytes += READ_ONCE(ring->bytes);
	}
	ndev->stats.tx_packets = packets;
	ndev->stats.tx_bytes = bytes;
}

static void mqnic_get_stats64(struct net_device *ndev,
		struct rtnl_link_stats64 *stats)
{
	struct mqnic_priv *priv = netdev_priv(ndev);

	spin_lock_bh(&priv->stats_lock);
	mqnic_update_stats(ndev);
	netdev_stats_to_stats64(stats, &ndev->stats);
	spin_unlock_bh(&priv->stats_lock);
}

static int mqnic_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr)
{
	struct mqnic_priv *priv = netdev_priv(ndev);
	struct hwtstamp_config hwts_config;

	if (copy_from_user(&hwts_config, ifr->ifr_data, sizeof(hwts_config)))
		return -EFAULT;

	if (hwts_config.flags)
		return -EINVAL;

	switch (hwts_config.tx_type) {
	case HWTSTAMP_TX_OFF:
	case HWTSTAMP_TX_ON:
		break;
	default:
		return -ERANGE;
	}

	switch (hwts_config.rx_filter) {
	case HWTSTAMP_FILTER_NONE:
		break;
	case HWTSTAMP_FILTER_ALL:
	case HWTSTAMP_FILTER_SOME:
	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
	case HWTSTAMP_FILTER_PTP_V2_EVENT:
	case HWTSTAMP_FILTER_PTP_V2_SYNC:
	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
	case HWTSTAMP_FILTER_NTP_ALL:
		hwts_config.rx_filter = HWTSTAMP_FILTER_ALL;
		break;
	default:
		return -ERANGE;
	}

	memcpy(&priv->hwts_config, &hwts_config, sizeof(hwts_config));

	if (copy_to_user(ifr->ifr_data, &hwts_config, sizeof(hwts_config)))
		return -EFAULT;

	return 0;
}

static int mqnic_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr)
{
	struct mqnic_priv *priv = netdev_priv(ndev);

	if (copy_to_user(ifr->ifr_data, &priv->hwts_config, sizeof(priv->hwts_config)))
		return -EFAULT;

	return 0;
}

static int mqnic_change_mtu(struct net_device *ndev, int new_mtu)
{
	struct mqnic_priv *priv = netdev_priv(ndev);
	struct mqnic_dev *mdev = priv->mdev;

	if (new_mtu < ndev->min_mtu || new_mtu > ndev->max_mtu) {
		dev_err(mdev->dev, "Bad MTU: %d", new_mtu);
		return -EPERM;
	}

	dev_info(mdev->dev, "New MTU: %d", new_mtu);

	ndev->mtu = new_mtu;

	if (netif_running(ndev)) {
		mutex_lock(&mdev->state_lock);

		mqnic_stop_port(ndev);
		mqnic_start_port(ndev);

		mutex_unlock(&mdev->state_lock);
	}

	return 0;
}

static int mqnic_ioctl(struct net_device *ndev, struct ifreq *ifr, int cmd)
{
	switch (cmd) {
	case SIOCSHWTSTAMP:
		return mqnic_hwtstamp_set(ndev, ifr);
	case SIOCGHWTSTAMP:
		return mqnic_hwtstamp_get(ndev, ifr);
	default:
		return -EOPNOTSUPP;
	}
}

static const struct net_device_ops mqnic_netdev_ops = {
	.ndo_open = mqnic_open,
	.ndo_stop = mqnic_close,
	.ndo_start_xmit = mqnic_start_xmit,
	.ndo_get_stats64 = mqnic_get_stats64,
	.ndo_validate_addr = eth_validate_addr,
	.ndo_change_mtu = mqnic_change_mtu,
	.ndo_do_ioctl = mqnic_ioctl,
};

int mqnic_init_netdev(struct mqnic_dev *mdev, int port, u8 __iomem *hw_addr)
{
	struct device *dev = mdev->dev;
	struct net_device *ndev;
	struct mqnic_priv *priv;
	int ret = 0;
	int k;
	u32 desc_block_size;

	ndev = alloc_etherdev_mqs(sizeof(*priv), MQNIC_MAX_TX_RINGS, MQNIC_MAX_RX_RINGS);
	if (!ndev) {
		dev_err(dev, "Failed to allocate memory");
		return -ENOMEM;
	}

	SET_NETDEV_DEV(ndev, dev);
	ndev->dev_port = port;

	// init private data
	priv = netdev_priv(ndev);
	memset(priv, 0, sizeof(struct mqnic_priv));

	spin_lock_init(&priv->stats_lock);

	priv->ndev = ndev;
	priv->mdev = mdev;
	priv->dev = dev;
	priv->port = port;
	priv->port_up = false;

	priv->hw_addr = hw_addr;
	priv->csr_hw_addr = hw_addr + mdev->if_csr_offset;

	// read ID registers
	priv->if_id = ioread32(priv->csr_hw_addr + MQNIC_IF_REG_IF_ID);
	dev_info(dev, "IF ID: 0x%08x", priv->if_id);
	priv->if_features = ioread32(priv->csr_hw_addr + MQNIC_IF_REG_IF_FEATURES);
	dev_info(dev, "IF features: 0x%08x", priv->if_features);

	priv->event_queue_count = ioread32(priv->csr_hw_addr + MQNIC_IF_REG_EVENT_QUEUE_COUNT);
	dev_info(dev, "Event queue count: %d", priv->event_queue_count);
	priv->event_queue_offset = ioread32(priv->csr_hw_addr + MQNIC_IF_REG_EVENT_QUEUE_OFFSET);
	dev_info(dev, "Event queue offset: 0x%08x", priv->event_queue_offset);
	priv->tx_queue_count = ioread32(priv->csr_hw_addr + MQNIC_IF_REG_TX_QUEUE_COUNT);
	dev_info(dev, "TX queue count: %d", priv->tx_queue_count);
	priv->tx_queue_offset = ioread32(priv->csr_hw_addr + MQNIC_IF_REG_TX_QUEUE_OFFSET);
	dev_info(dev, "TX queue offset: 0x%08x", priv->tx_queue_offset);
	priv->tx_cpl_queue_count = ioread32(priv->csr_hw_addr + MQNIC_IF_REG_TX_CPL_QUEUE_COUNT);
	dev_info(dev, "TX completion queue count: %d", priv->tx_cpl_queue_count);
	priv->tx_cpl_queue_offset = ioread32(priv->csr_hw_addr + MQNIC_IF_REG_TX_CPL_QUEUE_OFFSET);
	dev_info(dev, "TX completion queue offset: 0x%08x", priv->tx_cpl_queue_offset);
	priv->rx_queue_count = ioread32(priv->csr_hw_addr + MQNIC_IF_REG_RX_QUEUE_COUNT);
	dev_info(dev, "RX queue count: %d", priv->rx_queue_count);
	priv->rx_queue_offset = ioread32(priv->csr_hw_addr + MQNIC_IF_REG_RX_QUEUE_OFFSET);
	dev_info(dev, "RX queue offset: 0x%08x", priv->rx_queue_offset);
	priv->rx_cpl_queue_count = ioread32(priv->csr_hw_addr + MQNIC_IF_REG_RX_CPL_QUEUE_COUNT);
	dev_info(dev, "RX completion queue count: %d", priv->rx_cpl_queue_count);
	priv->rx_cpl_queue_offset = ioread32(priv->csr_hw_addr + MQNIC_IF_REG_RX_CPL_QUEUE_OFFSET);
	dev_info(dev, "RX completion queue offset: 0x%08x", priv->rx_cpl_queue_offset);
	priv->port_count = ioread32(priv->csr_hw_addr + MQNIC_IF_REG_PORT_COUNT);
	dev_info(dev, "Port count: %d", priv->port_count);
	priv->port_offset = ioread32(priv->csr_hw_addr + MQNIC_IF_REG_PORT_OFFSET);
	dev_info(dev, "Port offset: 0x%08x", priv->port_offset);
	priv->port_stride = ioread32(priv->csr_hw_addr + MQNIC_IF_REG_PORT_STRIDE);
	dev_info(dev, "Port stride: 0x%08x", priv->port_stride);

	if (priv->event_queue_count > MQNIC_MAX_EVENT_RINGS)
		priv->event_queue_count = MQNIC_MAX_EVENT_RINGS;
	if (priv->tx_queue_count > MQNIC_MAX_TX_RINGS)
		priv->tx_queue_count = MQNIC_MAX_TX_RINGS;
	if (priv->tx_cpl_queue_count > MQNIC_MAX_TX_CPL_RINGS)
		priv->tx_cpl_queue_count = MQNIC_MAX_TX_CPL_RINGS;
	if (priv->rx_queue_count > MQNIC_MAX_RX_RINGS)
		priv->rx_queue_count = MQNIC_MAX_RX_RINGS;
	if (priv->rx_cpl_queue_count > MQNIC_MAX_RX_CPL_RINGS)
		priv->rx_cpl_queue_count = MQNIC_MAX_RX_CPL_RINGS;

	if (priv->port_count > MQNIC_MAX_PORTS)
		priv->port_count = MQNIC_MAX_PORTS;

	netif_set_real_num_tx_queues(ndev, priv->tx_queue_count);
	netif_set_real_num_rx_queues(ndev, priv->rx_queue_count);

	// set MAC
	ndev->addr_len = ETH_ALEN;

	if (port >= mdev->mac_count) {
		dev_warn(dev, "Exhausted permanent MAC addresses; using random MAC");
		eth_hw_addr_random(ndev);
	} else {
		memcpy(ndev->dev_addr, mdev->mac_list[port], ETH_ALEN);

		if (!is_valid_ether_addr(ndev->dev_addr)) {
			dev_warn(dev, "Invalid MAC address in list; using random MAC");
			eth_hw_addr_random(ndev);
		}
	}

	priv->hwts_config.flags = 0;
	priv->hwts_config.tx_type = HWTSTAMP_TX_OFF;
	priv->hwts_config.rx_filter = HWTSTAMP_FILTER_NONE;

	// determine desc block size
	iowrite32(0xf << 8, hw_addr + priv->tx_queue_offset + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);
	priv->max_desc_block_size = 1 << ((ioread32(hw_addr + priv->tx_queue_offset + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG) >> 8) & 0xf);
	iowrite32(0, hw_addr + priv->tx_queue_offset + MQNIC_QUEUE_ACTIVE_LOG_SIZE_REG);

	dev_info(dev, "Max desc block size: %d", priv->max_desc_block_size);

	priv->max_desc_block_size = priv->max_desc_block_size < MQNIC_MAX_FRAGS ? priv->max_desc_block_size : MQNIC_MAX_FRAGS;

	desc_block_size = priv->max_desc_block_size < 4 ? priv->max_desc_block_size : 4;

	// allocate rings
	for (k = 0; k < priv->event_queue_count; k++) {
		ret = mqnic_create_eq_ring(priv, &priv->event_ring[k], 1024, MQNIC_EVENT_SIZE, k,
				hw_addr + priv->event_queue_offset + k * MQNIC_EVENT_QUEUE_STRIDE); // TODO configure/constant
		if (ret)
			goto fail;
	}

	for (k = 0; k < priv->tx_queue_count; k++) {
		ret = mqnic_create_tx_ring(priv, &priv->tx_ring[k], 1024, MQNIC_DESC_SIZE * desc_block_size, k,
				hw_addr + priv->tx_queue_offset + k * MQNIC_QUEUE_STRIDE); // TODO configure/constant
		if (ret)
			goto fail;
	}

	for (k = 0; k < priv->tx_cpl_queue_count; k++) {
		ret = mqnic_create_cq_ring(priv, &priv->tx_cpl_ring[k], 1024, MQNIC_CPL_SIZE, k,
				hw_addr + priv->tx_cpl_queue_offset + k * MQNIC_CPL_QUEUE_STRIDE); // TODO configure/constant
		if (ret)
			goto fail;
	}

	for (k = 0; k < priv->rx_queue_count; k++) {
		ret = mqnic_create_rx_ring(priv, &priv->rx_ring[k], 1024, MQNIC_DESC_SIZE, k,
				hw_addr + priv->rx_queue_offset + k * MQNIC_QUEUE_STRIDE); // TODO configure/constant
		if (ret)
			goto fail;
	}

	for (k = 0; k < priv->rx_cpl_queue_count; k++) {
		ret = mqnic_create_cq_ring(priv, &priv->rx_cpl_ring[k], 1024, MQNIC_CPL_SIZE, k,
				hw_addr + priv->rx_cpl_queue_offset + k * MQNIC_CPL_QUEUE_STRIDE); // TODO configure/constant
		if (ret)
			goto fail;
	}

	for (k = 0; k < priv->port_count; k++) {
		ret = mqnic_create_port(priv, &priv->ports[k], k,
				hw_addr + priv->port_offset + k * priv->port_stride);
		if (ret)
			goto fail;

		mqnic_port_set_rss_mask(priv->ports[k], 0xffffffff);
	}

	// entry points
	ndev->netdev_ops = &mqnic_netdev_ops;
	ndev->ethtool_ops = &mqnic_ethtool_ops;

	// set up features
	ndev->hw_features = NETIF_F_SG;

	if (priv->if_features & MQNIC_IF_FEATURE_RX_CSUM)
		ndev->hw_features |= NETIF_F_RXCSUM;

	if (priv->if_features & MQNIC_IF_FEATURE_TX_CSUM)
		ndev->hw_features |= NETIF_F_HW_CSUM;

	ndev->features = ndev->hw_features | NETIF_F_HIGHDMA;
	ndev->hw_features |= 0;

	ndev->min_mtu = ETH_MIN_MTU;
	ndev->max_mtu = 1500;

	if (priv->ports[0] && priv->ports[0]->port_mtu)
		ndev->max_mtu = priv->ports[0]->port_mtu - ETH_HLEN;

	netif_carrier_off(ndev);

	ret = register_netdev(ndev);
	if (ret) {
		dev_err(dev, "netdev registration failed on port %d", port);
		goto fail;
	}

	priv->registered = 1;

	mdev->ndev[port] = ndev;

	return 0;

fail:
	mqnic_destroy_netdev(ndev);
	return ret;
}

void mqnic_destroy_netdev(struct net_device *ndev)
{
	struct mqnic_priv *priv = netdev_priv(ndev);
	struct mqnic_dev *mdev = priv->mdev;
	int k;

	if (priv->registered)
		unregister_netdev(ndev);

	mdev->ndev[priv->port] = NULL;

	// free rings
	for (k = 0; k < ARRAY_SIZE(priv->event_ring); k++)
		if (priv->event_ring[k])
			mqnic_destroy_eq_ring(priv, &priv->event_ring[k]);

	for (k = 0; k < ARRAY_SIZE(priv->tx_ring); k++)
		if (priv->tx_ring[k])
			mqnic_destroy_tx_ring(priv, &priv->tx_ring[k]);

	for (k = 0; k < ARRAY_SIZE(priv->tx_cpl_ring); k++)
		if (priv->tx_cpl_ring[k])
			mqnic_destroy_cq_ring(priv, &priv->tx_cpl_ring[k]);

	for (k = 0; k < ARRAY_SIZE(priv->rx_ring); k++)
		if (priv->rx_ring[k])
			mqnic_destroy_rx_ring(priv, &priv->rx_ring[k]);

	for (k = 0; k < ARRAY_SIZE(priv->rx_cpl_ring); k++)
		if (priv->rx_cpl_ring[k])
			mqnic_destroy_cq_ring(priv, &priv->rx_cpl_ring[k]);

	for (k = 0; k < ARRAY_SIZE(priv->ports); k++)
		if (priv->ports[k])
			mqnic_destroy_port(priv, &priv->ports[k]);

	free_netdev(ndev);
}