From bf51c8b7bb0900a295e5d612c34d77f59eb61d42 Mon Sep 17 00:00:00 2001 From: Alex Forencich Date: Tue, 13 Jun 2023 15:38:59 -0700 Subject: [PATCH] Connect DMA engine busy status outputs Signed-off-by: Alex Forencich --- .../common/driver/example/example_driver.c | 10 ++++++++ example/common/rtl/example_core.v | 13 +++++++++- example/common/rtl/example_core_pcie.v | 22 +++++++++++++---- .../test_example_core_pcie.py | 24 +++++++++++++++---- .../test_example_core_pcie_ptile.py | 24 +++++++++++++++---- .../test_example_core_pcie_s10.py | 24 +++++++++++++++---- .../test_example_core_pcie_us.py | 24 +++++++++++++++---- 7 files changed, 118 insertions(+), 23 deletions(-) diff --git a/example/common/driver/example/example_driver.c b/example/common/driver/example/example_driver.c index 8dcd3b40c..3cd3a6c97 100644 --- a/example/common/driver/example/example_driver.c +++ b/example/common/driver/example/example_driver.c @@ -103,6 +103,8 @@ static void dma_block_read(struct example_dev *edev, if ((ioread32(edev->bar[0] + 0x001000) & 1) != 0) dev_warn(edev->dev, "%s: operation timed out", __func__); + if ((ioread32(edev->bar[0] + 0x000000) & 0x300) != 0) + dev_warn(edev->dev, "%s: DMA engine busy", __func__); } static void dma_block_write(struct example_dev *edev, @@ -157,6 +159,8 @@ static void dma_block_write(struct example_dev *edev, if ((ioread32(edev->bar[0] + 0x001100) & 1) != 0) dev_warn(edev->dev, "%s: operation timed out", __func__); + if ((ioread32(edev->bar[0] + 0x000000) & 0x300) != 0) + dev_warn(edev->dev, "%s: DMA engine busy", __func__); } static void dma_block_read_bench(struct example_dev *edev, @@ -365,6 +369,7 @@ static int edev_probe(struct pci_dev *pdev, const struct pci_device_id *ent) msleep(1); dev_info(dev, "Read status"); + dev_info(dev, "%08x", ioread32(edev->bar[0] + 0x000000)); dev_info(dev, "%08x", ioread32(edev->bar[0] + 0x000118)); dev_info(dev, "start copy to host"); @@ -378,6 +383,7 @@ static int edev_probe(struct pci_dev *pdev, const struct pci_device_id *ent) msleep(1); dev_info(dev, "Read status"); + dev_info(dev, "%08x", ioread32(edev->bar[0] + 0x000000)); dev_info(dev, "%08x", ioread32(edev->bar[0] + 0x000218)); dev_info(dev, "read test data"); @@ -402,6 +408,7 @@ static int edev_probe(struct pci_dev *pdev, const struct pci_device_id *ent) msleep(1); dev_info(dev, "Read status"); + dev_info(dev, "%08x", ioread32(edev->bar[0] + 0x000000)); dev_info(dev, "%08x", ioread32(edev->bar[0] + 0x000218)); dev_info(dev, "read data"); @@ -436,6 +443,9 @@ static int edev_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } } + dev_info(dev, "Read status"); + dev_info(dev, "%08x", ioread32(edev->bar[0] + 0x000000)); + // probe complete return 0; diff --git a/example/common/rtl/example_core.v b/example/common/rtl/example_core.v index e0ae1578c..038e37925 100644 --- a/example/common/rtl/example_core.v +++ b/example/common/rtl/example_core.v @@ -152,7 +152,14 @@ module example_core # */ output wire [IRQ_INDEX_WIDTH-1:0] irq_index, output wire irq_valid, - input wire irq_ready + input wire irq_ready, + + /* + * Control and status + */ + output wire dma_enable, + input wire dma_rd_busy, + input wire dma_wr_busy ); localparam RAM_ADDR_IMM_WIDTH = (DMA_IMM_ENABLE && (DMA_IMM_WIDTH > RAM_ADDR_WIDTH)) ? DMA_IMM_WIDTH : RAM_ADDR_WIDTH; @@ -284,6 +291,8 @@ assign m_axis_dma_write_desc_valid = dma_write_desc_valid_reg; assign irq_index = 0; assign irq_valid = irq_valid_reg; +assign dma_enable = dma_enable_reg; + always @* begin axil_ctrl_awready_next = 1'b0; axil_ctrl_wready_next = 1'b0; @@ -437,6 +446,8 @@ always @* begin // control 16'h0000: begin axil_ctrl_rdata_next[0] = dma_enable_reg; + axil_ctrl_rdata_next[8] = dma_wr_busy; + axil_ctrl_rdata_next[9] = dma_rd_busy; end 16'h0008: begin axil_ctrl_rdata_next[0] = dma_rd_int_en_reg; diff --git a/example/common/rtl/example_core_pcie.v b/example/common/rtl/example_core_pcie.v index db710450c..83b24d419 100644 --- a/example/common/rtl/example_core_pcie.v +++ b/example/common/rtl/example_core_pcie.v @@ -345,6 +345,11 @@ wire [IRQ_INDEX_WIDTH-1:0] irq_index; wire irq_valid; wire irq_ready; +// Control and status +wire dma_enable; +wire dma_rd_busy; +wire dma_wr_busy; + pcie_tlp_demux_bar #( .PORTS(3), .TLP_DATA_WIDTH(TLP_DATA_WIDTH), @@ -900,8 +905,8 @@ dma_if_pcie_inst ( /* * Configuration */ - .read_enable(1'b1), - .write_enable(1'b1), + .read_enable(dma_enable), + .write_enable(dma_enable), .ext_tag_enable(ext_tag_enable), .rcb_128b(rcb_128b), .requester_id({bus_num, 5'd0, 3'd0}), @@ -911,8 +916,8 @@ dma_if_pcie_inst ( /* * Status */ - .status_rd_busy(), - .status_wr_busy(), + .status_rd_busy(dma_rd_busy), + .status_wr_busy(dma_wr_busy), .status_error_cor(status_error_cor_int[3]), .status_error_uncor(status_error_uncor_int[3]) ); @@ -1109,7 +1114,14 @@ core_inst ( */ .irq_index(irq_index), .irq_valid(irq_valid), - .irq_ready(irq_ready) + .irq_ready(irq_ready), + + /* + * Control and status + */ + .dma_enable(dma_enable), + .dma_rd_busy(dma_rd_busy), + .dma_wr_busy(dma_wr_busy) ); endmodule diff --git a/example/common/tb/example_core_pcie/test_example_core_pcie.py b/example/common/tb/example_core_pcie/test_example_core_pcie.py index 70f1b9813..2d4634c54 100644 --- a/example/common/tb/example_core_pcie/test_example_core_pcie.py +++ b/example/common/tb/example_core_pcie/test_example_core_pcie.py @@ -224,6 +224,8 @@ async def run_test(dut): await Timer(2000, 'ns') # read status + status = await dev_pf0_bar0.read_dword(0x000000) + tb.log.info("DMA Status: 0x%x", status) val = await dev_pf0_bar0.read_dword(0x000118) tb.log.info("Status: 0x%x", val) assert val == 0x800000AA @@ -238,6 +240,8 @@ async def run_test(dut): await Timer(2000, 'ns') # read status + status = await dev_pf0_bar0.read_dword(0x000000) + tb.log.info("DMA Status: 0x%x", status) val = await dev_pf0_bar0.read_dword(0x000218) tb.log.info("Status: 0x%x", val) assert val == 0x80000055 @@ -258,6 +262,8 @@ async def run_test(dut): await Timer(2000, 'ns') # read status + status = await dev_pf0_bar0.read_dword(0x000000) + tb.log.info("DMA Status: 0x%x", status) val = await dev_pf0_bar0.read_dword(0x000218) tb.log.info("Status: 0x%x", val) assert val == 0x800000AA @@ -321,11 +327,15 @@ async def run_test(dut): await dev_pf0_bar0.write_dword(0x001000, 1) for k in range(10): - cnt = await dev_pf0_bar0.read_dword(0x001018) await Timer(1000, 'ns') - if cnt == 0: + run = await dev_pf0_bar0.read_dword(0x001000) + if run == 0: break + # read status + status = await dev_pf0_bar0.read_dword(0x000000) + tb.log.info("DMA Status: 0x%x", status) + # configure operation (write) # DMA base address await dev_pf0_bar0.write_dword(0x001180, (mem_base+dest_offset) & 0xffffffff) @@ -363,11 +373,17 @@ async def run_test(dut): await dev_pf0_bar0.write_dword(0x001100, 1) for k in range(10): - cnt = await dev_pf0_bar0.read_dword(0x001118) await Timer(1000, 'ns') - if cnt == 0: + run = await dev_pf0_bar0.read_dword(0x001100) + if run == 0: break + # read status + status = await dev_pf0_bar0.read_dword(0x000000) + tb.log.info("DMA Status: 0x%x", status) + + assert status & 0x300 == 0 + tb.log.info("%s", mem.hexdump_str(dest_offset, region_len)) assert mem[src_offset:src_offset+region_len] == mem[dest_offset:dest_offset+region_len] diff --git a/example/common/tb/example_core_pcie_ptile/test_example_core_pcie_ptile.py b/example/common/tb/example_core_pcie_ptile/test_example_core_pcie_ptile.py index b82024cc6..fddffe5ee 100644 --- a/example/common/tb/example_core_pcie_ptile/test_example_core_pcie_ptile.py +++ b/example/common/tb/example_core_pcie_ptile/test_example_core_pcie_ptile.py @@ -309,6 +309,8 @@ async def run_test(dut): await Timer(2000, 'ns') # read status + status = await dev_pf0_bar0.read_dword(0x000000) + tb.log.info("DMA Status: 0x%x", status) val = await dev_pf0_bar0.read_dword(0x000118) tb.log.info("Status: 0x%x", val) assert val == 0x800000AA @@ -323,6 +325,8 @@ async def run_test(dut): await Timer(2000, 'ns') # read status + status = await dev_pf0_bar0.read_dword(0x000000) + tb.log.info("DMA Status: 0x%x", status) val = await dev_pf0_bar0.read_dword(0x000218) tb.log.info("Status: 0x%x", val) assert val == 0x80000055 @@ -343,6 +347,8 @@ async def run_test(dut): await Timer(2000, 'ns') # read status + status = await dev_pf0_bar0.read_dword(0x000000) + tb.log.info("DMA Status: 0x%x", status) val = await dev_pf0_bar0.read_dword(0x000218) tb.log.info("Status: 0x%x", val) assert val == 0x800000AA @@ -406,11 +412,15 @@ async def run_test(dut): await dev_pf0_bar0.write_dword(0x001000, 1) for k in range(10): - cnt = await dev_pf0_bar0.read_dword(0x001018) await Timer(1000, 'ns') - if cnt == 0: + run = await dev_pf0_bar0.read_dword(0x001000) + if run == 0: break + # read status + status = await dev_pf0_bar0.read_dword(0x000000) + tb.log.info("DMA Status: 0x%x", status) + # configure operation (write) # DMA base address await dev_pf0_bar0.write_dword(0x001180, (mem_base+dest_offset) & 0xffffffff) @@ -448,11 +458,17 @@ async def run_test(dut): await dev_pf0_bar0.write_dword(0x001100, 1) for k in range(10): - cnt = await dev_pf0_bar0.read_dword(0x001118) await Timer(1000, 'ns') - if cnt == 0: + run = await dev_pf0_bar0.read_dword(0x001100) + if run == 0: break + # read status + status = await dev_pf0_bar0.read_dword(0x000000) + tb.log.info("DMA Status: 0x%x", status) + + assert status & 0x300 == 0 + tb.log.info("%s", mem.hexdump_str(dest_offset, region_len)) assert mem[src_offset:src_offset+region_len] == mem[dest_offset:dest_offset+region_len] diff --git a/example/common/tb/example_core_pcie_s10/test_example_core_pcie_s10.py b/example/common/tb/example_core_pcie_s10/test_example_core_pcie_s10.py index b74f4a58e..59929ddbf 100644 --- a/example/common/tb/example_core_pcie_s10/test_example_core_pcie_s10.py +++ b/example/common/tb/example_core_pcie_s10/test_example_core_pcie_s10.py @@ -257,6 +257,8 @@ async def run_test(dut): await Timer(2000, 'ns') # read status + status = await dev_pf0_bar0.read_dword(0x000000) + tb.log.info("DMA Status: 0x%x", status) val = await dev_pf0_bar0.read_dword(0x000118) tb.log.info("Status: 0x%x", val) assert val == 0x800000AA @@ -271,6 +273,8 @@ async def run_test(dut): await Timer(2000, 'ns') # read status + status = await dev_pf0_bar0.read_dword(0x000000) + tb.log.info("DMA Status: 0x%x", status) val = await dev_pf0_bar0.read_dword(0x000218) tb.log.info("Status: 0x%x", val) assert val == 0x80000055 @@ -291,6 +295,8 @@ async def run_test(dut): await Timer(2000, 'ns') # read status + status = await dev_pf0_bar0.read_dword(0x000000) + tb.log.info("DMA Status: 0x%x", status) val = await dev_pf0_bar0.read_dword(0x000218) tb.log.info("Status: 0x%x", val) assert val == 0x800000AA @@ -354,11 +360,15 @@ async def run_test(dut): await dev_pf0_bar0.write_dword(0x001000, 1) for k in range(10): - cnt = await dev_pf0_bar0.read_dword(0x001018) await Timer(1000, 'ns') - if cnt == 0: + run = await dev_pf0_bar0.read_dword(0x001000) + if run == 0: break + # read status + status = await dev_pf0_bar0.read_dword(0x000000) + tb.log.info("DMA Status: 0x%x", status) + # configure operation (write) # DMA base address await dev_pf0_bar0.write_dword(0x001180, (mem_base+dest_offset) & 0xffffffff) @@ -396,11 +406,17 @@ async def run_test(dut): await dev_pf0_bar0.write_dword(0x001100, 1) for k in range(10): - cnt = await dev_pf0_bar0.read_dword(0x001118) await Timer(1000, 'ns') - if cnt == 0: + run = await dev_pf0_bar0.read_dword(0x001100) + if run == 0: break + # read status + status = await dev_pf0_bar0.read_dword(0x000000) + tb.log.info("DMA Status: 0x%x", status) + + assert status & 0x300 == 0 + tb.log.info("%s", mem.hexdump_str(dest_offset, region_len)) assert mem[src_offset:src_offset+region_len] == mem[dest_offset:dest_offset+region_len] diff --git a/example/common/tb/example_core_pcie_us/test_example_core_pcie_us.py b/example/common/tb/example_core_pcie_us/test_example_core_pcie_us.py index e728be19c..a57d9993e 100644 --- a/example/common/tb/example_core_pcie_us/test_example_core_pcie_us.py +++ b/example/common/tb/example_core_pcie_us/test_example_core_pcie_us.py @@ -350,6 +350,8 @@ async def run_test(dut): await Timer(2000, 'ns') # read status + status = await dev_pf0_bar0.read_dword(0x000000) + tb.log.info("DMA Status: 0x%x", status) val = await dev_pf0_bar0.read_dword(0x000118) tb.log.info("Status: 0x%x", val) assert val == 0x800000AA @@ -364,6 +366,8 @@ async def run_test(dut): await Timer(2000, 'ns') # read status + status = await dev_pf0_bar0.read_dword(0x000000) + tb.log.info("DMA Status: 0x%x", status) val = await dev_pf0_bar0.read_dword(0x000218) tb.log.info("Status: 0x%x", val) assert val == 0x80000055 @@ -384,6 +388,8 @@ async def run_test(dut): await Timer(2000, 'ns') # read status + status = await dev_pf0_bar0.read_dword(0x000000) + tb.log.info("DMA Status: 0x%x", status) val = await dev_pf0_bar0.read_dword(0x000218) tb.log.info("Status: 0x%x", val) assert val == 0x800000AA @@ -447,11 +453,15 @@ async def run_test(dut): await dev_pf0_bar0.write_dword(0x001000, 1) for k in range(10): - cnt = await dev_pf0_bar0.read_dword(0x001018) await Timer(1000, 'ns') - if cnt == 0: + run = await dev_pf0_bar0.read_dword(0x001000) + if run == 0: break + # read status + status = await dev_pf0_bar0.read_dword(0x000000) + tb.log.info("DMA Status: 0x%x", status) + # configure operation (write) # DMA base address await dev_pf0_bar0.write_dword(0x001180, (mem_base+dest_offset) & 0xffffffff) @@ -489,12 +499,16 @@ async def run_test(dut): await dev_pf0_bar0.write_dword(0x001100, 1) for k in range(10): - cnt = await dev_pf0_bar0.read_dword(0x001118) await Timer(1000, 'ns') - if cnt == 0: + run = await dev_pf0_bar0.read_dword(0x001100) + if run == 0: break - await Timer(2000, 'ns') + # read status + status = await dev_pf0_bar0.read_dword(0x000000) + tb.log.info("DMA Status: 0x%x", status) + + assert status & 0x300 == 0 tb.log.info("%s", mem.hexdump_str(dest_offset, region_len))