From 9297c518f13df23f90dd3acdd30fa67a45a627bd Mon Sep 17 00:00:00 2001 From: Alex Forencich Date: Wed, 3 Nov 2021 18:10:17 -0700 Subject: [PATCH] Add example design for ExaNIC X10 --- example/ExaNIC_X10/fpga/README.md | 19 + example/ExaNIC_X10/fpga/common/vivado.mk | 126 +++++ example/ExaNIC_X10/fpga/driver | 1 + example/ExaNIC_X10/fpga/fpga.xdc | 183 +++++++ example/ExaNIC_X10/fpga/fpga/Makefile | 94 ++++ .../ExaNIC_X10/fpga/ip/pcie3_ultrascale_0.tcl | 31 ++ example/ExaNIC_X10/fpga/lib/pcie | 1 + example/ExaNIC_X10/fpga/rtl/common | 1 + example/ExaNIC_X10/fpga/rtl/fpga.v | 449 ++++++++++++++++++ example/ExaNIC_X10/fpga/rtl/fpga_core.v | 274 +++++++++++ example/ExaNIC_X10/fpga/rtl/sync_reset.v | 56 +++ example/ExaNIC_X10/fpga/rtl/sync_signal.v | 62 +++ example/ExaNIC_X10/fpga/tb/fpga_core/Makefile | 123 +++++ .../fpga/tb/fpga_core/test_fpga_core.py | 381 +++++++++++++++ 14 files changed, 1801 insertions(+) create mode 100644 example/ExaNIC_X10/fpga/README.md create mode 100644 example/ExaNIC_X10/fpga/common/vivado.mk create mode 120000 example/ExaNIC_X10/fpga/driver create mode 100644 example/ExaNIC_X10/fpga/fpga.xdc create mode 100644 example/ExaNIC_X10/fpga/fpga/Makefile create mode 100644 example/ExaNIC_X10/fpga/ip/pcie3_ultrascale_0.tcl create mode 120000 example/ExaNIC_X10/fpga/lib/pcie create mode 120000 example/ExaNIC_X10/fpga/rtl/common create mode 100644 example/ExaNIC_X10/fpga/rtl/fpga.v create mode 100644 example/ExaNIC_X10/fpga/rtl/fpga_core.v create mode 100644 example/ExaNIC_X10/fpga/rtl/sync_reset.v create mode 100644 example/ExaNIC_X10/fpga/rtl/sync_signal.v create mode 100644 example/ExaNIC_X10/fpga/tb/fpga_core/Makefile create mode 100644 example/ExaNIC_X10/fpga/tb/fpga_core/test_fpga_core.py diff --git a/example/ExaNIC_X10/fpga/README.md b/example/ExaNIC_X10/fpga/README.md new file mode 100644 index 000000000..d6597ec86 --- /dev/null +++ b/example/ExaNIC_X10/fpga/README.md @@ -0,0 +1,19 @@ +# Verilog PCIe ExaNIC X10 Example Design + +## Introduction + +This example design targets the Exablaze ExaNIC X10 FPGA board. + +The design implements the PCIe AXI lite master module, the PCIe AXI master module, and the PCIe DMA module. A very simple Linux driver is included to test the FPGA design. + +* FPGA: xcku035-fbva676-2-c + +## How to build + +Run `make` to build. Ensure that the Xilinx Vivado toolchain components are in PATH. + +Run `make` to build the driver. Ensure the headers for the running kernel are installed, otherwise the driver cannot be compiled. + +## How to test + +Run `make program` to program the ExaNIC X10 board with Vivado. Then load the driver with `insmod example.ko`. Check dmesg for the output. diff --git a/example/ExaNIC_X10/fpga/common/vivado.mk b/example/ExaNIC_X10/fpga/common/vivado.mk new file mode 100644 index 000000000..d47a43947 --- /dev/null +++ b/example/ExaNIC_X10/fpga/common/vivado.mk @@ -0,0 +1,126 @@ +################################################################### +# +# Xilinx Vivado FPGA Makefile +# +# Copyright (c) 2016 Alex Forencich +# +################################################################### +# +# Parameters: +# FPGA_TOP - Top module name +# FPGA_FAMILY - FPGA family (e.g. VirtexUltrascale) +# FPGA_DEVICE - FPGA device (e.g. xcvu095-ffva2104-2-e) +# SYN_FILES - space-separated list of source files +# INC_FILES - space-separated list of include files +# XDC_FILES - space-separated list of timing constraint files +# XCI_FILES - space-separated list of IP XCI files +# +# Example: +# +# FPGA_TOP = fpga +# FPGA_FAMILY = VirtexUltrascale +# FPGA_DEVICE = xcvu095-ffva2104-2-e +# SYN_FILES = rtl/fpga.v +# XDC_FILES = fpga.xdc +# XCI_FILES = ip/pcspma.xci +# include ../common/vivado.mk +# +################################################################### + +# phony targets +.PHONY: fpga vivado tmpclean clean distclean + +# prevent make from deleting intermediate files and reports +.PRECIOUS: %.xpr %.bit %.mcs %.prm +.SECONDARY: + +CONFIG ?= config.mk +-include ../$(CONFIG) + +SYN_FILES_REL = $(patsubst %, ../%, $(filter-out /% ./%,$(SYN_FILES))) $(filter /% ./%,$(SYN_FILES)) +INC_FILES_REL = $(patsubst %, ../%, $(filter-out /% ./%,$(INC_FILES))) $(filter /% ./%,$(INC_FILES)) +XCI_FILES_REL = $(patsubst %, ../%, $(filter-out /% ./%,$(XCI_FILES))) $(filter /% ./%,$(XCI_FILES)) +IP_TCL_FILES_REL = $(patsubst %, ../%, $(filter-out /% ./%,$(IP_TCL_FILES))) $(filter /% ./%,$(IP_TCL_FILES)) +CONFIG_TCL_FILES_REL = $(patsubst %, ../%, $(filter-out /% ./%,$(CONFIG_TCL_FILES))) $(filter /% ./%,$(CONFIG_TCL_FILES)) + +ifdef XDC_FILES + XDC_FILES_REL = $(patsubst %, ../%, $(filter-out /% ./%,$(XDC_FILES))) $(filter /% ./%,$(XDC_FILES)) +else + XDC_FILES_REL = $(FPGA_TOP).xdc +endif + +################################################################### +# Main Targets +# +# all: build everything +# clean: remove output files and project files +################################################################### + +all: fpga + +fpga: $(FPGA_TOP).bit + +vivado: $(FPGA_TOP).xpr + vivado $(FPGA_TOP).xpr + +tmpclean:: + -rm -rf *.log *.jou *.cache *.gen *.hbs *.hw *.ip_user_files *.runs *.xpr *.html *.xml *.sim *.srcs *.str .Xil defines.v + -rm -rf create_project.tcl update_config.tcl run_synth.tcl run_impl.tcl generate_bit.tcl + +clean:: tmpclean + -rm -rf *.bit program.tcl generate_mcs.tcl *.mcs *.prm flash.tcl + +distclean:: clean + -rm -rf rev + +################################################################### +# Target implementations +################################################################### + +# Vivado project file +create_project.tcl: Makefile $(XCI_FILES_REL) $(IP_TCL_FILES_REL) + rm -rf defines.v + touch defines.v + for x in $(DEFS); do echo '`define' $$x >> defines.v; done + echo "create_project -force -part $(FPGA_PART) $(FPGA_TOP)" > $@ + echo "add_files -fileset sources_1 defines.v $(SYN_FILES_REL)" >> $@ + echo "add_files -fileset constrs_1 $(XDC_FILES_REL)" >> $@ + for x in $(XCI_FILES_REL); do echo "import_ip $$x" >> $@; done + for x in $(IP_TCL_FILES_REL); do echo "source $$x" >> $@; done + for x in $(CONFIG_TCL_FILES_REL); do echo "source $$x" >> $@; done + +update_config.tcl: $(CONFIG_TCL_FILES_REL) + echo "open_project -quiet $(FPGA_TOP).xpr" > $@ + for x in $(CONFIG_TCL_FILES_REL); do echo "source $$x" >> $@; done + +$(FPGA_TOP).xpr: create_project.tcl update_config.tcl + vivado -nojournal -nolog -mode batch $(foreach x,$?,-source $x) + +# synthesis run +%.runs/synth_1/%.dcp: %.xpr $(SYN_FILES_REL) $(INC_FILES_REL) $(XDC_FILES_REL) $(CONFIG_TCL_FILES_REL) + echo "open_project $*.xpr" > run_synth.tcl + echo "reset_run synth_1" >> run_synth.tcl + echo "launch_runs -jobs 4 synth_1" >> run_synth.tcl + echo "wait_on_run synth_1" >> run_synth.tcl + vivado -nojournal -nolog -mode batch -source run_synth.tcl + +# implementation run +%.runs/impl_1/%_routed.dcp: %.runs/synth_1/%.dcp + echo "open_project $*.xpr" > run_impl.tcl + echo "reset_run impl_1" >> run_impl.tcl + echo "launch_runs -jobs 4 impl_1" >> run_impl.tcl + echo "wait_on_run impl_1" >> run_impl.tcl + vivado -nojournal -nolog -mode batch -source run_impl.tcl + +# bit file +%.bit: %.runs/impl_1/%_routed.dcp + echo "open_project $*.xpr" > generate_bit.tcl + echo "open_run impl_1" >> generate_bit.tcl + echo "write_bitstream -force $*.bit" >> generate_bit.tcl + vivado -nojournal -nolog -mode batch -source generate_bit.tcl + mkdir -p rev + EXT=bit; COUNT=100; \ + while [ -e rev/$*_rev$$COUNT.$$EXT ]; \ + do COUNT=$$((COUNT+1)); done; \ + cp $@ rev/$*_rev$$COUNT.$$EXT; \ + echo "Output: rev/$*_rev$$COUNT.$$EXT"; diff --git a/example/ExaNIC_X10/fpga/driver b/example/ExaNIC_X10/fpga/driver new file mode 120000 index 000000000..4a54c76b5 --- /dev/null +++ b/example/ExaNIC_X10/fpga/driver @@ -0,0 +1 @@ +../../common/driver/example/ \ No newline at end of file diff --git a/example/ExaNIC_X10/fpga/fpga.xdc b/example/ExaNIC_X10/fpga/fpga.xdc new file mode 100644 index 000000000..ccd6a97e3 --- /dev/null +++ b/example/ExaNIC_X10/fpga/fpga.xdc @@ -0,0 +1,183 @@ +# XDC constraints for the ExaNIC X10 +# part: xcku035-fbva676-2-e + +# General configuration +set_property CFGBVS GND [current_design] +set_property CONFIG_VOLTAGE 1.8 [current_design] +set_property BITSTREAM.GENERAL.COMPRESS true [current_design] +set_property BITSTREAM.CONFIG.UNUSEDPIN Pullup [current_design] +set_property BITSTREAM.CONFIG.CONFIGRATE 50 [current_design] +set_property BITSTREAM.CONFIG.BPI_SYNC_MODE Type2 [current_design] +set_property CONFIG_MODE BPI16 [current_design] + +# 100 MHz system clock +#set_property -dict {LOC D18 IOSTANDARD LVDS} [get_ports clk_100mhz_p] +#set_property -dict {LOC C18 IOSTANDARD LVDS} [get_ports clk_100mhz_n] +#create_clock -period 10 -name clk_100mhz [get_ports clk_100mhz_p] + +# LEDs +set_property -dict {LOC A25 IOSTANDARD LVCMOS18 SLEW SLOW DRIVE 12} [get_ports {sfp_1_led[0]}] +set_property -dict {LOC A24 IOSTANDARD LVCMOS18 SLEW SLOW DRIVE 12} [get_ports {sfp_1_led[1]}] +set_property -dict {LOC E23 IOSTANDARD LVCMOS18 SLEW SLOW DRIVE 12} [get_ports {sfp_2_led[0]}] +set_property -dict {LOC D26 IOSTANDARD LVCMOS18 SLEW SLOW DRIVE 12} [get_ports {sfp_2_led[1]}] +set_property -dict {LOC C23 IOSTANDARD LVCMOS18 SLEW SLOW DRIVE 12} [get_ports {sma_led[0]}] +set_property -dict {LOC D23 IOSTANDARD LVCMOS18 SLEW SLOW DRIVE 12} [get_ports {sma_led[1]}] + +set_false_path -to [get_ports {sfp_1_led[*] sfp_2_led[*] sma_led[*]}] +set_output_delay 0 [get_ports {sfp_1_led[*] sfp_2_led[*] sma_led[*]}] + +# GPIO +#set_property -dict {LOC W26 IOSTANDARD LVCMOS18} [get_ports gpio[0]] +#set_property -dict {LOC Y26 IOSTANDARD LVCMOS18} [get_ports gpio[1]] +#set_property -dict {LOC AB26 IOSTANDARD LVCMOS18} [get_ports gpio[2]] +#set_property -dict {LOC AC26 IOSTANDARD LVCMOS18} [get_ports gpio[3]] + +# SMA +#set_property -dict {LOC B17 IOSTANDARD LVCMOS18} [get_ports sma_in] +#set_property -dict {LOC B16 IOSTANDARD LVCMOS18 SLEW FAST DRIVE 12} [get_ports sma_out] +#set_property -dict {LOC B19 IOSTANDARD LVCMOS18 SLEW SLOW DRIVE 12} [get_ports sma_out_en] +#set_property -dict {LOC C16 IOSTANDARD LVCMOS18 SLEW SLOW DRIVE 12} [get_ports sma_term_en] + +#set_false_path -to [get_ports {sma_out sma_term sma_term_en}] +#set_output_delay 0 [get_ports {sma_out sma_term sma_term_en}] +#set_false_path -from [get_ports {sma_in}] +#set_input_delay 0 [get_ports {sma_in}] + +# SFP+ Interfaces +#set_property -dict {LOC D2 } [get_ports sfp_1_rx_p] ;# MGTHRXP0_227 GTHE3_CHANNEL_X0Y12 / GTHE3_COMMON_X0Y3 +#set_property -dict {LOC D1 } [get_ports sfp_1_rx_n] ;# MGTHRXN0_227 GTHE3_CHANNEL_X0Y12 / GTHE3_COMMON_X0Y3 +#set_property -dict {LOC E4 } [get_ports sfp_1_tx_p] ;# MGTHTXP0_227 GTHE3_CHANNEL_X0Y12 / GTHE3_COMMON_X0Y3 +#set_property -dict {LOC E3 } [get_ports sfp_1_tx_n] ;# MGTHTXN0_227 GTHE3_CHANNEL_X0Y12 / GTHE3_COMMON_X0Y3 +#set_property -dict {LOC C4 } [get_ports sfp_2_rx_p] ;# MGTHRXP1_227 GTHE3_CHANNEL_X0Y13 / GTHE3_COMMON_X0Y3 +#set_property -dict {LOC C3 } [get_ports sfp_2_rx_n] ;# MGTHRXN1_227 GTHE3_CHANNEL_X0Y13 / GTHE3_COMMON_X0Y3 +#set_property -dict {LOC D6 } [get_ports sfp_2_tx_p] ;# MGTHTXP1_227 GTHE3_CHANNEL_X0Y13 / GTHE3_COMMON_X0Y3 +#set_property -dict {LOC D5 } [get_ports sfp_2_tx_n] ;# MGTHTXN1_227 GTHE3_CHANNEL_X0Y13 / GTHE3_COMMON_X0Y3 +#set_property -dict {LOC H6 } [get_ports sfp_mgt_refclk_p] ;# MGTREFCLK0P_227 from X2 +#set_property -dict {LOC H5 } [get_ports sfp_mgt_refclk_n] ;# MGTREFCLK0N_227 from X2 +#set_property -dict {LOC AA12 IOSTANDARD LVCMOS18 SLEW SLOW DRIVE 12} [get_ports sfp_1_tx_disable] +#set_property -dict {LOC W14 IOSTANDARD LVCMOS18 SLEW SLOW DRIVE 12} [get_ports sfp_2_tx_disable] +#set_property -dict {LOC C24 IOSTANDARD LVCMOS18 PULLUP true} [get_ports sfp_1_npres] +#set_property -dict {LOC D24 IOSTANDARD LVCMOS18 PULLUP true} [get_ports sfp_2_npres] +#set_property -dict {LOC W13 IOSTANDARD LVCMOS18 PULLUP true} [get_ports sfp_1_los] +#set_property -dict {LOC AB12 IOSTANDARD LVCMOS18 PULLUP true} [get_ports sfp_2_los] +#set_property -dict {LOC B25 IOSTANDARD LVCMOS18 SLEW SLOW DRIVE 12} [get_ports sfp_1_rs] +#set_property -dict {LOC D25 IOSTANDARD LVCMOS18 SLEW SLOW DRIVE 12} [get_ports sfp_2_rs] +#set_property -dict {LOC W11 IOSTANDARD LVCMOS18 SLEW SLOW DRIVE 12 PULLUP true} [get_ports sfp_i2c_scl] +#set_property -dict {LOC Y11 IOSTANDARD LVCMOS18 SLEW SLOW DRIVE 12 PULLUP true} [get_ports sfp_1_i2c_sda] +#set_property -dict {LOC Y13 IOSTANDARD LVCMOS18 SLEW SLOW DRIVE 12 PULLUP true} [get_ports sfp_2_i2c_sda] + +# 161.1328125 MHz MGT reference clock +#create_clock -period 6.206 -name sfp_mgt_refclk [get_ports sfp_mgt_refclk_p] + +#set_false_path -to [get_ports {sfp_1_tx_disable sfp_2_tx_disable sfp_1_rs sfp_2_rs}] +#set_output_delay 0 [get_ports {sfp_1_tx_disable sfp_2_tx_disable sfp_1_rs sfp_2_rs}] +#set_false_path -from [get_ports {sfp_1_npres sfp_2_npres sfp_1_los sfp_2_los}] +#set_input_delay 0 [get_ports {sfp_1_npres sfp_2_npres sfp_1_los sfp_2_los}] + +#set_false_path -to [get_ports {sfp_1_i2c_sda sfp_2_i2c_sda sfp_i2c_scl}] +#set_output_delay 0 [get_ports {sfp_1_i2c_sda sfp_2_i2c_sda sfp_i2c_scl}] +#set_false_path -from [get_ports {sfp_1_i2c_sda sfp_2_i2c_sda sfp_i2c_scl}] +#set_input_delay 0 [get_ports {sfp_1_i2c_sda sfp_2_i2c_sda sfp_i2c_scl}] + +# I2C interface +#set_property -dict {LOC B26 IOSTANDARD LVCMOS18 SLEW SLOW DRIVE 12 PULLUP true} [get_ports eeprom_i2c_scl] +#set_property -dict {LOC C26 IOSTANDARD LVCMOS18 SLEW SLOW DRIVE 12 PULLUP true} [get_ports eeprom_i2c_sda] + +#set_false_path -to [get_ports {eeprom_i2c_sda eeprom_i2c_scl}] +#set_output_delay 0 [get_ports {eeprom_i2c_sda eeprom_i2c_scl}] +#set_false_path -from [get_ports {eeprom_i2c_sda eeprom_i2c_scl}] +#set_input_delay 0 [get_ports {eeprom_i2c_sda eeprom_i2c_scl}] + +# PCIe Interface +set_property -dict {LOC P2 } [get_ports {pcie_rx_p[0]}] ;# MGTHRXP3_225 GTHE3_CHANNEL_X0Y7 / GTHE3_COMMON_X0Y1 +set_property -dict {LOC P1 } [get_ports {pcie_rx_n[0]}] ;# MGTHRXN3_225 GTHE3_CHANNEL_X0Y7 / GTHE3_COMMON_X0Y1 +set_property -dict {LOC R4 } [get_ports {pcie_tx_p[0]}] ;# MGTHTXP3_225 GTHE3_CHANNEL_X0Y7 / GTHE3_COMMON_X0Y1 +set_property -dict {LOC R3 } [get_ports {pcie_tx_n[0]}] ;# MGTHTXN3_225 GTHE3_CHANNEL_X0Y7 / GTHE3_COMMON_X0Y1 +set_property -dict {LOC T2 } [get_ports {pcie_rx_p[1]}] ;# MGTHRXP2_225 GTHE3_CHANNEL_X0Y6 / GTHE3_COMMON_X0Y1 +set_property -dict {LOC T1 } [get_ports {pcie_rx_n[1]}] ;# MGTHRXN2_225 GTHE3_CHANNEL_X0Y6 / GTHE3_COMMON_X0Y1 +set_property -dict {LOC U4 } [get_ports {pcie_tx_p[1]}] ;# MGTHTXP2_225 GTHE3_CHANNEL_X0Y6 / GTHE3_COMMON_X0Y1 +set_property -dict {LOC U3 } [get_ports {pcie_tx_n[1]}] ;# MGTHTXN2_225 GTHE3_CHANNEL_X0Y6 / GTHE3_COMMON_X0Y1 +set_property -dict {LOC V2 } [get_ports {pcie_rx_p[2]}] ;# MGTHRXP1_225 GTHE3_CHANNEL_X0Y5 / GTHE3_COMMON_X0Y1 +set_property -dict {LOC V1 } [get_ports {pcie_rx_n[2]}] ;# MGTHRXN1_225 GTHE3_CHANNEL_X0Y5 / GTHE3_COMMON_X0Y1 +set_property -dict {LOC W4 } [get_ports {pcie_tx_p[2]}] ;# MGTHTXP1_225 GTHE3_CHANNEL_X0Y5 / GTHE3_COMMON_X0Y1 +set_property -dict {LOC W3 } [get_ports {pcie_tx_n[2]}] ;# MGTHTXN1_225 GTHE3_CHANNEL_X0Y5 / GTHE3_COMMON_X0Y1 +set_property -dict {LOC Y2 } [get_ports {pcie_rx_p[3]}] ;# MGTHRXP0_225 GTHE3_CHANNEL_X0Y4 / GTHE3_COMMON_X0Y1 +set_property -dict {LOC Y1 } [get_ports {pcie_rx_n[3]}] ;# MGTHRXN0_225 GTHE3_CHANNEL_X0Y4 / GTHE3_COMMON_X0Y1 +set_property -dict {LOC AA4 } [get_ports {pcie_tx_p[3]}] ;# MGTHTXP0_225 GTHE3_CHANNEL_X0Y4 / GTHE3_COMMON_X0Y1 +set_property -dict {LOC AA3 } [get_ports {pcie_tx_n[3]}] ;# MGTHTXN0_225 GTHE3_CHANNEL_X0Y4 / GTHE3_COMMON_X0Y1 +set_property -dict {LOC AB2 } [get_ports {pcie_rx_p[4]}] ;# MGTHRXP3_224 GTHE3_CHANNEL_X0Y3 / GTHE3_COMMON_X0Y0 +set_property -dict {LOC AB1 } [get_ports {pcie_rx_n[4]}] ;# MGTHRXN3_224 GTHE3_CHANNEL_X0Y3 / GTHE3_COMMON_X0Y0 +set_property -dict {LOC AB6 } [get_ports {pcie_tx_p[4]}] ;# MGTHTXP3_224 GTHE3_CHANNEL_X0Y3 / GTHE3_COMMON_X0Y0 +set_property -dict {LOC AB5 } [get_ports {pcie_tx_n[4]}] ;# MGTHTXN3_224 GTHE3_CHANNEL_X0Y3 / GTHE3_COMMON_X0Y0 +set_property -dict {LOC AD2 } [get_ports {pcie_rx_p[5]}] ;# MGTHRXP2_224 GTHE3_CHANNEL_X0Y2 / GTHE3_COMMON_X0Y0 +set_property -dict {LOC AD1 } [get_ports {pcie_rx_n[5]}] ;# MGTHRXN2_224 GTHE3_CHANNEL_X0Y2 / GTHE3_COMMON_X0Y0 +set_property -dict {LOC AC4 } [get_ports {pcie_tx_p[5]}] ;# MGTHTXP2_224 GTHE3_CHANNEL_X0Y2 / GTHE3_COMMON_X0Y0 +set_property -dict {LOC AC3 } [get_ports {pcie_tx_n[5]}] ;# MGTHTXN2_224 GTHE3_CHANNEL_X0Y2 / GTHE3_COMMON_X0Y0 +set_property -dict {LOC AE4 } [get_ports {pcie_rx_p[6]}] ;# MGTHRXP1_224 GTHE3_CHANNEL_X0Y1 / GTHE3_COMMON_X0Y0 +set_property -dict {LOC AE3 } [get_ports {pcie_rx_n[6]}] ;# MGTHRXN1_224 GTHE3_CHANNEL_X0Y1 / GTHE3_COMMON_X0Y0 +set_property -dict {LOC AD6 } [get_ports {pcie_tx_p[6]}] ;# MGTHTXP1_224 GTHE3_CHANNEL_X0Y1 / GTHE3_COMMON_X0Y0 +set_property -dict {LOC AD5 } [get_ports {pcie_tx_n[6]}] ;# MGTHTXN1_224 GTHE3_CHANNEL_X0Y1 / GTHE3_COMMON_X0Y0 +set_property -dict {LOC AF2 } [get_ports {pcie_rx_p[7]}] ;# MGTHRXP0_224 GTHE3_CHANNEL_X0Y0 / GTHE3_COMMON_X0Y0 +set_property -dict {LOC AF1 } [get_ports {pcie_rx_n[7]}] ;# MGTHRXN0_224 GTHE3_CHANNEL_X0Y0 / GTHE3_COMMON_X0Y0 +set_property -dict {LOC AF6 } [get_ports {pcie_tx_p[7]}] ;# MGTHTXP0_224 GTHE3_CHANNEL_X0Y0 / GTHE3_COMMON_X0Y0 +set_property -dict {LOC AF5 } [get_ports {pcie_tx_n[7]}] ;# MGTHTXN0_224 GTHE3_CHANNEL_X0Y0 / GTHE3_COMMON_X0Y0 +set_property -dict {LOC T6 } [get_ports pcie_mgt_refclk_p] ;# MGTREFCLK0P_225 +set_property -dict {LOC T5 } [get_ports pcie_mgt_refclk_n] ;# MGTREFCLK0N_225 +set_property -dict {LOC AC22 IOSTANDARD LVCMOS18 PULLUP true} [get_ports pcie_reset_n] + +# 100 MHz MGT reference clock +create_clock -period 10 -name pcie_mgt_refclk [get_ports pcie_mgt_refclk_p] + +set_false_path -from [get_ports {pcie_reset_n}] +set_input_delay 0 [get_ports {pcie_reset_n}] + +# BPI flash +#set_property -dict {LOC AE10 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_dq[0]}] +#set_property -dict {LOC AC8 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_dq[1]}] +#set_property -dict {LOC AD10 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_dq[2]}] +#set_property -dict {LOC AD9 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_dq[3]}] +#set_property -dict {LOC AC11 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_dq[4]}] +#set_property -dict {LOC AF10 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_dq[5]}] +#set_property -dict {LOC AF14 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_dq[6]}] +#set_property -dict {LOC AE12 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_dq[7]}] +#set_property -dict {LOC AD14 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_dq[8]}] +#set_property -dict {LOC AF13 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_dq[9]}] +#set_property -dict {LOC AE13 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_dq[10]}] +#set_property -dict {LOC AD8 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_dq[11]}] +#set_property -dict {LOC AC13 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_dq[12]}] +#set_property -dict {LOC AD13 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_dq[13]}] +#set_property -dict {LOC AA14 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_dq[14]}] +#set_property -dict {LOC AB15 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_dq[15]}] +#set_property -dict {LOC AD11 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_addr[0]}] +#set_property -dict {LOC AE11 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_addr[1]}] +#set_property -dict {LOC AF12 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_addr[2]}] +#set_property -dict {LOC AB11 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_addr[3]}] +#set_property -dict {LOC AB9 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_addr[4]}] +#set_property -dict {LOC AB14 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_addr[5]}] +#set_property -dict {LOC AA10 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_addr[6]}] +#set_property -dict {LOC AA9 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_addr[7]}] +#set_property -dict {LOC W10 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_addr[8]}] +#set_property -dict {LOC AA13 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_addr[9]}] +#set_property -dict {LOC Y15 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_addr[10]}] +#set_property -dict {LOC AC12 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_addr[11]}] +#set_property -dict {LOC V12 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_addr[12]}] +#set_property -dict {LOC V11 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_addr[13]}] +#set_property -dict {LOC Y12 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_addr[14]}] +#set_property -dict {LOC W9 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_addr[15]}] +#set_property -dict {LOC Y8 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_addr[16]}] +#set_property -dict {LOC W8 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_addr[17]}] +#set_property -dict {LOC W15 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_addr[18]}] +#set_property -dict {LOC AA15 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_addr[19]}] +#set_property -dict {LOC AE16 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_addr[20]}] +#set_property -dict {LOC AF15 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_addr[21]}] +#set_property -dict {LOC AE15 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_addr[22]}] +#set_property -dict {LOC AD15 IOSTANDARD LVCMOS18 PULLUP true} [get_ports {flash_region}] +#set_property -dict {LOC AC9 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_ce_n}] +#set_property -dict {LOC AC14 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_oe_n}] +#set_property -dict {LOC AB10 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_we_n}] +#set_property -dict {LOC Y10 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {flash_adv_n}] + +#set_false_path -to [get_ports {flash_dq[*] flash_addr[*] flash_region flash_ce_n flash_oe_n flash_we_n flash_adv_n}] +#set_output_delay 0 [get_ports {flash_dq[*] flash_addr[*] flash_region flash_ce_n flash_oe_n flash_we_n flash_adv_n}] +#set_false_path -from [get_ports {flash_dq[*]}] +#set_input_delay 0 [get_ports {flash_dq[*]}] diff --git a/example/ExaNIC_X10/fpga/fpga/Makefile b/example/ExaNIC_X10/fpga/fpga/Makefile new file mode 100644 index 000000000..ee821b8fb --- /dev/null +++ b/example/ExaNIC_X10/fpga/fpga/Makefile @@ -0,0 +1,94 @@ + +# FPGA settings +FPGA_PART = xcku035-fbva676-2-e +FPGA_TOP = fpga +FPGA_ARCH = kintexu + +# Files for synthesis +SYN_FILES = rtl/fpga.v +SYN_FILES += rtl/fpga_core.v +SYN_FILES += rtl/sync_reset.v +SYN_FILES += rtl/sync_signal.v +SYN_FILES += rtl/common/example_core_pcie_us.v +SYN_FILES += rtl/common/example_core_pcie.v +SYN_FILES += rtl/common/example_core.v +SYN_FILES += rtl/common/axi_ram.v +SYN_FILES += lib/pcie/rtl/pcie_us_if.v +SYN_FILES += lib/pcie/rtl/pcie_us_if_rc.v +SYN_FILES += lib/pcie/rtl/pcie_us_if_rq.v +SYN_FILES += lib/pcie/rtl/pcie_us_if_cq.v +SYN_FILES += lib/pcie/rtl/pcie_us_if_cc.v +SYN_FILES += lib/pcie/rtl/pcie_us_cfg.v +SYN_FILES += lib/pcie/rtl/pcie_us_msi.v +SYN_FILES += lib/pcie/rtl/pcie_axil_master.v +SYN_FILES += lib/pcie/rtl/pcie_axi_master.v +SYN_FILES += lib/pcie/rtl/pcie_axi_master_rd.v +SYN_FILES += lib/pcie/rtl/pcie_axi_master_wr.v +SYN_FILES += lib/pcie/rtl/pcie_tlp_demux_bar.v +SYN_FILES += lib/pcie/rtl/pcie_tlp_demux.v +SYN_FILES += lib/pcie/rtl/pcie_tlp_mux.v +SYN_FILES += lib/pcie/rtl/dma_if_pcie.v +SYN_FILES += lib/pcie/rtl/dma_if_pcie_rd.v +SYN_FILES += lib/pcie/rtl/dma_if_pcie_wr.v +SYN_FILES += lib/pcie/rtl/dma_psdpram.v +SYN_FILES += lib/pcie/rtl/arbiter.v +SYN_FILES += lib/pcie/rtl/priority_encoder.v +SYN_FILES += lib/pcie/rtl/pulse_merge.v + +# XDC files +XDC_FILES = fpga.xdc + +# IP +IP_TCL_FILES = ip/pcie3_ultrascale_0.tcl + +include ../common/vivado.mk + +program: $(FPGA_TOP).bit + echo "open_hw" > program.tcl + echo "connect_hw_server" >> program.tcl + echo "open_hw_target" >> program.tcl + echo "current_hw_device [lindex [get_hw_devices] 0]" >> program.tcl + echo "refresh_hw_device -update_hw_probes false [current_hw_device]" >> program.tcl + echo "set_property PROGRAM.FILE {$(FPGA_TOP).bit} [current_hw_device]" >> program.tcl + echo "program_hw_devices [current_hw_device]" >> program.tcl + echo "exit" >> program.tcl + vivado -nojournal -nolog -mode batch -source program.tcl + +%.mcs %.prm: %.bit + echo "write_cfgmem -force -format mcs -size 32 -interface BPIx16 -loadbit {up 0x00800000 $*.bit} -checksum -file $*.mcs" > generate_mcs.tcl + echo "exit" >> generate_mcs.tcl + vivado -nojournal -nolog -mode batch -source generate_mcs.tcl + mkdir -p rev + COUNT=100; \ + while [ -e rev/$*_rev$$COUNT.bit ]; \ + do COUNT=$$((COUNT+1)); done; \ + COUNT=$$((COUNT-1)); \ + for x in .mcs .prm; \ + do cp $*$$x rev/$*_rev$$COUNT$$x; \ + echo "Output: rev/$*_rev$$COUNT$$x"; done; + +flash: $(FPGA_TOP).mcs $(FPGA_TOP).prm + echo "open_hw" > flash.tcl + echo "connect_hw_server" >> flash.tcl + echo "open_hw_target" >> flash.tcl + echo "current_hw_device [lindex [get_hw_devices] 0]" >> flash.tcl + echo "refresh_hw_device -update_hw_probes false [current_hw_device]" >> flash.tcl + echo "create_hw_cfgmem -hw_device [current_hw_device] [lindex [get_cfgmem_parts {s29gl256p-bpi-x16}] 0]" >> flash.tcl + echo "current_hw_cfgmem -hw_device [current_hw_device] [get_property PROGRAM.HW_CFGMEM [current_hw_device]]" >> flash.tcl + echo "set_property PROGRAM.FILES [list \"$(FPGA_TOP).mcs\"] [current_hw_cfgmem]" >> flash.tcl + echo "set_property PROGRAM.PRM_FILES [list \"$(FPGA_TOP).prm\"] [current_hw_cfgmem]" >> flash.tcl + echo "set_property PROGRAM.ERASE 1 [current_hw_cfgmem]" >> flash.tcl + echo "set_property PROGRAM.CFG_PROGRAM 1 [current_hw_cfgmem]" >> flash.tcl + echo "set_property PROGRAM.VERIFY 1 [current_hw_cfgmem]" >> flash.tcl + echo "set_property PROGRAM.CHECKSUM 0 [current_hw_cfgmem]" >> flash.tcl + echo "set_property PROGRAM.ADDRESS_RANGE {use_file} [current_hw_cfgmem]" >> flash.tcl + echo "set_property PROGRAM.BPI_RS_PINS {24:23} [current_hw_cfgmem]" >> flash.tcl + echo "set_property PROGRAM.UNUSED_PIN_TERMINATION {pull-none} [current_hw_cfgmem]" >> flash.tcl + echo "create_hw_bitstream -hw_device [current_hw_device] [get_property PROGRAM.HW_CFGMEM_BITFILE [current_hw_device]]" >> flash.tcl + echo "program_hw_devices [current_hw_device]" >> flash.tcl + echo "refresh_hw_device [current_hw_device]" >> flash.tcl + echo "program_hw_cfgmem -hw_cfgmem [current_hw_cfgmem]" >> flash.tcl + echo "boot_hw_device [current_hw_device]" >> flash.tcl + echo "exit" >> flash.tcl + vivado -nojournal -nolog -mode batch -source flash.tcl + diff --git a/example/ExaNIC_X10/fpga/ip/pcie3_ultrascale_0.tcl b/example/ExaNIC_X10/fpga/ip/pcie3_ultrascale_0.tcl new file mode 100644 index 000000000..d39fa31a0 --- /dev/null +++ b/example/ExaNIC_X10/fpga/ip/pcie3_ultrascale_0.tcl @@ -0,0 +1,31 @@ + +create_ip -name pcie3_ultrascale -vendor xilinx.com -library ip -module_name pcie3_ultrascale_0 + +set_property -dict [list \ + CONFIG.PL_LINK_CAP_MAX_LINK_SPEED {8.0_GT/s} \ + CONFIG.PL_LINK_CAP_MAX_LINK_WIDTH {X8} \ + CONFIG.AXISTEN_IF_RC_STRADDLE {false} \ + CONFIG.axisten_if_width {256_bit} \ + CONFIG.extended_tag_field {true} \ + CONFIG.axisten_freq {250} \ + CONFIG.PF0_CLASS_CODE {058000} \ + CONFIG.PF0_DEVICE_ID {0001} \ + CONFIG.PF0_MSI_CAP_MULTIMSGCAP {32_vectors} \ + CONFIG.PF0_SUBSYSTEM_ID {0003} \ + CONFIG.PF0_SUBSYSTEM_VENDOR_ID {1ce4} \ + CONFIG.pf0_bar0_64bit {true} \ + CONFIG.pf0_bar0_prefetchable {true} \ + CONFIG.pf0_bar0_scale {Megabytes} \ + CONFIG.pf0_bar0_size {16} \ + CONFIG.pf0_bar2_64bit {true} \ + CONFIG.pf0_bar2_prefetchable {true} \ + CONFIG.pf0_bar2_enabled {true} \ + CONFIG.pf0_bar2_type {Memory} \ + CONFIG.pf0_bar2_scale {Megabytes} \ + CONFIG.pf0_bar2_size {16} \ + CONFIG.PF0_INTERRUPT_PIN {NONE} \ + CONFIG.PF0_MSIX_CAP_TABLE_BIR {BAR_0} \ + CONFIG.PF0_MSIX_CAP_PBA_BIR {BAR_0} \ + CONFIG.vendor_id {1234} \ + CONFIG.en_msi_per_vec_masking {true} \ +] [get_ips pcie3_ultrascale_0] diff --git a/example/ExaNIC_X10/fpga/lib/pcie b/example/ExaNIC_X10/fpga/lib/pcie new file mode 120000 index 000000000..11a54ed36 --- /dev/null +++ b/example/ExaNIC_X10/fpga/lib/pcie @@ -0,0 +1 @@ +../../../../ \ No newline at end of file diff --git a/example/ExaNIC_X10/fpga/rtl/common b/example/ExaNIC_X10/fpga/rtl/common new file mode 120000 index 000000000..e60fada70 --- /dev/null +++ b/example/ExaNIC_X10/fpga/rtl/common @@ -0,0 +1 @@ +../../../common/rtl/ \ No newline at end of file diff --git a/example/ExaNIC_X10/fpga/rtl/fpga.v b/example/ExaNIC_X10/fpga/rtl/fpga.v new file mode 100644 index 000000000..209e1325f --- /dev/null +++ b/example/ExaNIC_X10/fpga/rtl/fpga.v @@ -0,0 +1,449 @@ +/* + +Copyright (c) 2018 Alex Forencich + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +*/ + +// Language: Verilog 2001 + +`resetall +`timescale 1ns / 1ps +`default_nettype none + +/* + * FPGA top-level module + */ +module fpga ( + /* + * GPIO + */ + output wire [1:0] sfp_1_led, + output wire [1:0] sfp_2_led, + output wire [1:0] sma_led, + + /* + * PCI express + */ + input wire [7:0] pcie_rx_p, + input wire [7:0] pcie_rx_n, + output wire [7:0] pcie_tx_p, + output wire [7:0] pcie_tx_n, + input wire pcie_mgt_refclk_p, + input wire pcie_mgt_refclk_n, + input wire pcie_reset_n +); + +parameter AXIS_PCIE_DATA_WIDTH = 256; +parameter AXIS_PCIE_KEEP_WIDTH = (AXIS_PCIE_DATA_WIDTH/32); +parameter AXIS_PCIE_RC_USER_WIDTH = 75; +parameter AXIS_PCIE_RQ_USER_WIDTH = 60; +parameter AXIS_PCIE_CQ_USER_WIDTH = 85; +parameter AXIS_PCIE_CC_USER_WIDTH = 33; + +parameter RQ_SEQ_NUM_WIDTH = AXIS_PCIE_RQ_USER_WIDTH == 60 ? 4 : 6; +parameter RQ_SEQ_NUM_ENABLE = 1; + +parameter PCIE_TAG_COUNT = 64; +parameter BAR0_APERTURE = 24; +parameter BAR2_APERTURE = 24; + +// PCIe +wire pcie_user_clk; +wire pcie_user_reset; + +wire pcie_sys_clk; +wire pcie_sys_clk_gt; + +IBUFDS_GTE3 #( + .REFCLK_HROW_CK_SEL(2'b00) +) +ibufds_gte3_pcie_mgt_refclk_inst ( + .I (pcie_mgt_refclk_p), + .IB (pcie_mgt_refclk_n), + .CEB (1'b0), + .O (pcie_sys_clk_gt), + .ODIV2 (pcie_sys_clk) +); + +wire [AXIS_PCIE_DATA_WIDTH-1:0] axis_rq_tdata; +wire [AXIS_PCIE_KEEP_WIDTH-1:0] axis_rq_tkeep; +wire axis_rq_tlast; +wire axis_rq_tready; +wire [AXIS_PCIE_RQ_USER_WIDTH-1:0] axis_rq_tuser; +wire axis_rq_tvalid; + +wire [AXIS_PCIE_DATA_WIDTH-1:0] axis_rc_tdata; +wire [AXIS_PCIE_KEEP_WIDTH-1:0] axis_rc_tkeep; +wire axis_rc_tlast; +wire axis_rc_tready; +wire [AXIS_PCIE_RC_USER_WIDTH-1:0] axis_rc_tuser; +wire axis_rc_tvalid; + +wire [AXIS_PCIE_DATA_WIDTH-1:0] axis_cq_tdata; +wire [AXIS_PCIE_KEEP_WIDTH-1:0] axis_cq_tkeep; +wire axis_cq_tlast; +wire axis_cq_tready; +wire [AXIS_PCIE_CQ_USER_WIDTH-1:0] axis_cq_tuser; +wire axis_cq_tvalid; + +wire [AXIS_PCIE_DATA_WIDTH-1:0] axis_cc_tdata; +wire [AXIS_PCIE_KEEP_WIDTH-1:0] axis_cc_tkeep; +wire axis_cc_tlast; +wire axis_cc_tready; +wire [AXIS_PCIE_CC_USER_WIDTH-1:0] axis_cc_tuser; +wire axis_cc_tvalid; + +wire [RQ_SEQ_NUM_WIDTH-1:0] pcie_rq_seq_num; +wire pcie_rq_seq_num_vld; + +// ila_0 rq_ila ( +// .clk(pcie_user_clk), +// .probe0(axis_rq_tdata), +// .probe1(axis_rq_tkeep), +// .probe2(axis_rq_tlast), +// .probe3(axis_rq_tready), +// .probe4(axis_rq_tuser), +// .probe5(axis_rq_tvalid) +// ); + +// ila_0 rc_ila ( +// .clk(pcie_user_clk), +// .probe0(axis_rc_tdata), +// .probe1(axis_rc_tkeep), +// .probe2(axis_rc_tlast), +// .probe3(axis_rc_tready), +// .probe4(axis_rc_tuser), +// .probe5(axis_rc_tvalid) +// ); + +wire [2:0] cfg_max_payload; +wire [2:0] cfg_max_read_req; + +wire [18:0] cfg_mgmt_addr; +wire cfg_mgmt_write; +wire [31:0] cfg_mgmt_write_data; +wire [3:0] cfg_mgmt_byte_enable; +wire cfg_mgmt_read; +wire [31:0] cfg_mgmt_read_data; +wire cfg_mgmt_read_write_done; + +wire [7:0] cfg_fc_ph; +wire [11:0] cfg_fc_pd; +wire [7:0] cfg_fc_nph; +wire [11:0] cfg_fc_npd; +wire [7:0] cfg_fc_cplh; +wire [11:0] cfg_fc_cpld; +wire [2:0] cfg_fc_sel; + +wire [3:0] cfg_interrupt_msi_enable; +wire [7:0] cfg_interrupt_msi_vf_enable; +wire [11:0] cfg_interrupt_msi_mmenable; +wire cfg_interrupt_msi_mask_update; +wire [31:0] cfg_interrupt_msi_data; +wire [3:0] cfg_interrupt_msi_select; +wire [31:0] cfg_interrupt_msi_int; +wire [31:0] cfg_interrupt_msi_pending_status; +wire cfg_interrupt_msi_pending_status_data_enable; +wire [3:0] cfg_interrupt_msi_pending_status_function_num; +wire cfg_interrupt_msi_sent; +wire cfg_interrupt_msi_fail; +wire [2:0] cfg_interrupt_msi_attr; +wire cfg_interrupt_msi_tph_present; +wire [1:0] cfg_interrupt_msi_tph_type; +wire [8:0] cfg_interrupt_msi_tph_st_tag; +wire [3:0] cfg_interrupt_msi_function_number; + +wire status_error_cor; +wire status_error_uncor; + +pcie3_ultrascale_0 +pcie3_ultrascale_inst ( + .pci_exp_txn(pcie_tx_n), + .pci_exp_txp(pcie_tx_p), + .pci_exp_rxn(pcie_rx_n), + .pci_exp_rxp(pcie_rx_p), + .user_clk(pcie_user_clk), + .user_reset(pcie_user_reset), + .user_lnk_up(), + + .s_axis_rq_tdata(axis_rq_tdata), + .s_axis_rq_tkeep(axis_rq_tkeep), + .s_axis_rq_tlast(axis_rq_tlast), + .s_axis_rq_tready(axis_rq_tready), + .s_axis_rq_tuser(axis_rq_tuser), + .s_axis_rq_tvalid(axis_rq_tvalid), + + .m_axis_rc_tdata(axis_rc_tdata), + .m_axis_rc_tkeep(axis_rc_tkeep), + .m_axis_rc_tlast(axis_rc_tlast), + .m_axis_rc_tready(axis_rc_tready), + .m_axis_rc_tuser(axis_rc_tuser), + .m_axis_rc_tvalid(axis_rc_tvalid), + + .m_axis_cq_tdata(axis_cq_tdata), + .m_axis_cq_tkeep(axis_cq_tkeep), + .m_axis_cq_tlast(axis_cq_tlast), + .m_axis_cq_tready(axis_cq_tready), + .m_axis_cq_tuser(axis_cq_tuser), + .m_axis_cq_tvalid(axis_cq_tvalid), + + .s_axis_cc_tdata(axis_cc_tdata), + .s_axis_cc_tkeep(axis_cc_tkeep), + .s_axis_cc_tlast(axis_cc_tlast), + .s_axis_cc_tready(axis_cc_tready), + .s_axis_cc_tuser(axis_cc_tuser), + .s_axis_cc_tvalid(axis_cc_tvalid), + + .pcie_rq_seq_num(pcie_rq_seq_num), + .pcie_rq_seq_num_vld(pcie_rq_seq_num_vld), + .pcie_rq_tag(), + .pcie_rq_tag_av(), + .pcie_rq_tag_vld(), + + .pcie_tfc_nph_av(), + .pcie_tfc_npd_av(), + + .pcie_cq_np_req(1'b1), + .pcie_cq_np_req_count(), + + .cfg_phy_link_down(), + .cfg_phy_link_status(), + .cfg_negotiated_width(), + .cfg_current_speed(), + .cfg_max_payload(cfg_max_payload), + .cfg_max_read_req(cfg_max_read_req), + .cfg_function_status(), + .cfg_function_power_state(), + .cfg_vf_status(), + .cfg_vf_power_state(), + .cfg_link_power_state(), + + .cfg_mgmt_addr(cfg_mgmt_addr), + .cfg_mgmt_write(cfg_mgmt_write), + .cfg_mgmt_write_data(cfg_mgmt_write_data), + .cfg_mgmt_byte_enable(cfg_mgmt_byte_enable), + .cfg_mgmt_read(cfg_mgmt_read), + .cfg_mgmt_read_data(cfg_mgmt_read_data), + .cfg_mgmt_read_write_done(cfg_mgmt_read_write_done), + .cfg_mgmt_type1_cfg_reg_access(1'b0), + + .cfg_err_cor_out(), + .cfg_err_nonfatal_out(), + .cfg_err_fatal_out(), + .cfg_local_error(), + .cfg_ltr_enable(), + .cfg_ltssm_state(), + .cfg_rcb_status(), + .cfg_dpa_substate_change(), + .cfg_obff_enable(), + .cfg_pl_status_change(), + .cfg_tph_requester_enable(), + .cfg_tph_st_mode(), + .cfg_vf_tph_requester_enable(), + .cfg_vf_tph_st_mode(), + + .cfg_msg_received(), + .cfg_msg_received_data(), + .cfg_msg_received_type(), + .cfg_msg_transmit(1'b0), + .cfg_msg_transmit_type(3'd0), + .cfg_msg_transmit_data(32'd0), + .cfg_msg_transmit_done(), + + .cfg_fc_ph(cfg_fc_ph), + .cfg_fc_pd(cfg_fc_pd), + .cfg_fc_nph(cfg_fc_nph), + .cfg_fc_npd(cfg_fc_npd), + .cfg_fc_cplh(cfg_fc_cplh), + .cfg_fc_cpld(cfg_fc_cpld), + .cfg_fc_sel(cfg_fc_sel), + + .cfg_per_func_status_control(3'd0), + .cfg_per_func_status_data(), + .cfg_per_function_number(4'd0), + .cfg_per_function_output_request(1'b0), + .cfg_per_function_update_done(), + + .cfg_dsn(64'd0), + + .cfg_power_state_change_ack(1'b1), + .cfg_power_state_change_interrupt(), + + .cfg_err_cor_in(status_error_cor), + .cfg_err_uncor_in(status_error_uncor), + .cfg_flr_in_process(), + .cfg_flr_done(4'd0), + .cfg_vf_flr_in_process(), + .cfg_vf_flr_done(8'd0), + + .cfg_link_training_enable(1'b1), + + .cfg_interrupt_int(4'd0), + .cfg_interrupt_pending(4'd0), + .cfg_interrupt_sent(), + .cfg_interrupt_msi_enable(cfg_interrupt_msi_enable), + .cfg_interrupt_msi_vf_enable(cfg_interrupt_msi_vf_enable), + .cfg_interrupt_msi_mmenable(cfg_interrupt_msi_mmenable), + .cfg_interrupt_msi_mask_update(cfg_interrupt_msi_mask_update), + .cfg_interrupt_msi_data(cfg_interrupt_msi_data), + .cfg_interrupt_msi_select(cfg_interrupt_msi_select), + .cfg_interrupt_msi_int(cfg_interrupt_msi_int), + .cfg_interrupt_msi_pending_status(cfg_interrupt_msi_pending_status), + .cfg_interrupt_msi_pending_status_data_enable(cfg_interrupt_msi_pending_status_data_enable), + .cfg_interrupt_msi_pending_status_function_num(cfg_interrupt_msi_pending_status_function_num), + .cfg_interrupt_msi_sent(cfg_interrupt_msi_sent), + .cfg_interrupt_msi_fail(cfg_interrupt_msi_fail), + .cfg_interrupt_msi_attr(cfg_interrupt_msi_attr), + .cfg_interrupt_msi_tph_present(cfg_interrupt_msi_tph_present), + .cfg_interrupt_msi_tph_type(cfg_interrupt_msi_tph_type), + .cfg_interrupt_msi_tph_st_tag(cfg_interrupt_msi_tph_st_tag), + .cfg_interrupt_msi_function_number(cfg_interrupt_msi_function_number), + + .cfg_hot_reset_out(), + + .cfg_config_space_enable(1'b1), + .cfg_req_pm_transition_l23_ready(1'b0), + .cfg_hot_reset_in(1'b0), + + .cfg_ds_port_number(8'd0), + .cfg_ds_bus_number(8'd0), + .cfg_ds_device_number(5'd0), + .cfg_ds_function_number(3'd0), + + .cfg_subsys_vend_id(16'h1234), + + .sys_clk(pcie_sys_clk), + .sys_clk_gt(pcie_sys_clk_gt), + .sys_reset(pcie_reset_n), + .pcie_perstn1_in(1'b0), + .pcie_perstn0_out(), + .pcie_perstn1_out(), + + .int_qpll1lock_out(), + .int_qpll1outrefclk_out(), + .int_qpll1outclk_out(), + .phy_rdy_out() +); + +fpga_core #( + .AXIS_PCIE_DATA_WIDTH(AXIS_PCIE_DATA_WIDTH), + .AXIS_PCIE_KEEP_WIDTH(AXIS_PCIE_KEEP_WIDTH), + .AXIS_PCIE_RC_USER_WIDTH(AXIS_PCIE_RC_USER_WIDTH), + .AXIS_PCIE_RQ_USER_WIDTH(AXIS_PCIE_RQ_USER_WIDTH), + .AXIS_PCIE_CQ_USER_WIDTH(AXIS_PCIE_CQ_USER_WIDTH), + .AXIS_PCIE_CC_USER_WIDTH(AXIS_PCIE_CC_USER_WIDTH), + .RQ_SEQ_NUM_WIDTH(RQ_SEQ_NUM_WIDTH), + .RQ_SEQ_NUM_ENABLE(RQ_SEQ_NUM_ENABLE), + .PCIE_TAG_COUNT(PCIE_TAG_COUNT), + .BAR0_APERTURE(BAR0_APERTURE), + .BAR2_APERTURE(BAR2_APERTURE) +) +core_inst ( + /* + * Clock: 250 MHz + * Synchronous reset + */ + .clk(pcie_user_clk), + .rst(pcie_user_reset), + /* + * GPIO + */ + .sfp_1_led(sfp_1_led), + .sfp_2_led(sfp_2_led), + .sma_led(sma_led), + /* + * PCIe + */ + .m_axis_rq_tdata(axis_rq_tdata), + .m_axis_rq_tkeep(axis_rq_tkeep), + .m_axis_rq_tlast(axis_rq_tlast), + .m_axis_rq_tready(axis_rq_tready), + .m_axis_rq_tuser(axis_rq_tuser), + .m_axis_rq_tvalid(axis_rq_tvalid), + + .s_axis_rc_tdata(axis_rc_tdata), + .s_axis_rc_tkeep(axis_rc_tkeep), + .s_axis_rc_tlast(axis_rc_tlast), + .s_axis_rc_tready(axis_rc_tready), + .s_axis_rc_tuser(axis_rc_tuser), + .s_axis_rc_tvalid(axis_rc_tvalid), + + .s_axis_cq_tdata(axis_cq_tdata), + .s_axis_cq_tkeep(axis_cq_tkeep), + .s_axis_cq_tlast(axis_cq_tlast), + .s_axis_cq_tready(axis_cq_tready), + .s_axis_cq_tuser(axis_cq_tuser), + .s_axis_cq_tvalid(axis_cq_tvalid), + + .m_axis_cc_tdata(axis_cc_tdata), + .m_axis_cc_tkeep(axis_cc_tkeep), + .m_axis_cc_tlast(axis_cc_tlast), + .m_axis_cc_tready(axis_cc_tready), + .m_axis_cc_tuser(axis_cc_tuser), + .m_axis_cc_tvalid(axis_cc_tvalid), + + .s_axis_rq_seq_num(pcie_rq_seq_num), + .s_axis_rq_seq_num_valid(pcie_rq_seq_num_vld), + + .cfg_max_payload(cfg_max_payload), + .cfg_max_read_req(cfg_max_read_req), + + .cfg_mgmt_addr(cfg_mgmt_addr), + .cfg_mgmt_write(cfg_mgmt_write), + .cfg_mgmt_write_data(cfg_mgmt_write_data), + .cfg_mgmt_byte_enable(cfg_mgmt_byte_enable), + .cfg_mgmt_read(cfg_mgmt_read), + .cfg_mgmt_read_data(cfg_mgmt_read_data), + .cfg_mgmt_read_write_done(cfg_mgmt_read_write_done), + + .cfg_fc_ph(cfg_fc_ph), + .cfg_fc_pd(cfg_fc_pd), + .cfg_fc_nph(cfg_fc_nph), + .cfg_fc_npd(cfg_fc_npd), + .cfg_fc_cplh(cfg_fc_cplh), + .cfg_fc_cpld(cfg_fc_cpld), + .cfg_fc_sel(cfg_fc_sel), + + .cfg_interrupt_msi_enable(cfg_interrupt_msi_enable), + .cfg_interrupt_msi_vf_enable(cfg_interrupt_msi_vf_enable), + .cfg_interrupt_msi_mmenable(cfg_interrupt_msi_mmenable), + .cfg_interrupt_msi_mask_update(cfg_interrupt_msi_mask_update), + .cfg_interrupt_msi_data(cfg_interrupt_msi_data), + .cfg_interrupt_msi_select(cfg_interrupt_msi_select), + .cfg_interrupt_msi_int(cfg_interrupt_msi_int), + .cfg_interrupt_msi_pending_status(cfg_interrupt_msi_pending_status), + .cfg_interrupt_msi_pending_status_data_enable(cfg_interrupt_msi_pending_status_data_enable), + .cfg_interrupt_msi_pending_status_function_num(cfg_interrupt_msi_pending_status_function_num), + .cfg_interrupt_msi_sent(cfg_interrupt_msi_sent), + .cfg_interrupt_msi_fail(cfg_interrupt_msi_fail), + .cfg_interrupt_msi_attr(cfg_interrupt_msi_attr), + .cfg_interrupt_msi_tph_present(cfg_interrupt_msi_tph_present), + .cfg_interrupt_msi_tph_type(cfg_interrupt_msi_tph_type), + .cfg_interrupt_msi_tph_st_tag(cfg_interrupt_msi_tph_st_tag), + .cfg_interrupt_msi_function_number(cfg_interrupt_msi_function_number), + + .status_error_cor(status_error_cor), + .status_error_uncor(status_error_uncor) +); + +endmodule + +`resetall diff --git a/example/ExaNIC_X10/fpga/rtl/fpga_core.v b/example/ExaNIC_X10/fpga/rtl/fpga_core.v new file mode 100644 index 000000000..de673de2c --- /dev/null +++ b/example/ExaNIC_X10/fpga/rtl/fpga_core.v @@ -0,0 +1,274 @@ +/* + +Copyright (c) 2018 Alex Forencich + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +*/ + +// Language: Verilog 2001 + +`resetall +`timescale 1ns / 1ps +`default_nettype none + +/* + * FPGA core logic + */ +module fpga_core # +( + parameter AXIS_PCIE_DATA_WIDTH = 256, + parameter AXIS_PCIE_KEEP_WIDTH = (AXIS_PCIE_DATA_WIDTH/32), + parameter AXIS_PCIE_RC_USER_WIDTH = 75, + parameter AXIS_PCIE_RQ_USER_WIDTH = 60, + parameter AXIS_PCIE_CQ_USER_WIDTH = 85, + parameter AXIS_PCIE_CC_USER_WIDTH = 33, + parameter RQ_SEQ_NUM_WIDTH = AXIS_PCIE_RQ_USER_WIDTH == 60 ? 4 : 6, + parameter RQ_SEQ_NUM_ENABLE = 1, + parameter PCIE_TAG_COUNT = 64, + parameter BAR0_APERTURE = 24, + parameter BAR2_APERTURE = 24 +) +( + /* + * Clock: 250 MHz + * Synchronous reset + */ + input wire clk, + input wire rst, + + /* + * GPIO + */ + output wire [1:0] sfp_1_led, + output wire [1:0] sfp_2_led, + output wire [1:0] sma_led, + + /* + * PCIe + */ + output wire [AXIS_PCIE_DATA_WIDTH-1:0] m_axis_rq_tdata, + output wire [AXIS_PCIE_KEEP_WIDTH-1:0] m_axis_rq_tkeep, + output wire m_axis_rq_tlast, + input wire m_axis_rq_tready, + output wire [AXIS_PCIE_RQ_USER_WIDTH-1:0] m_axis_rq_tuser, + output wire m_axis_rq_tvalid, + + input wire [AXIS_PCIE_DATA_WIDTH-1:0] s_axis_rc_tdata, + input wire [AXIS_PCIE_KEEP_WIDTH-1:0] s_axis_rc_tkeep, + input wire s_axis_rc_tlast, + output wire s_axis_rc_tready, + input wire [AXIS_PCIE_RC_USER_WIDTH-1:0] s_axis_rc_tuser, + input wire s_axis_rc_tvalid, + + input wire [AXIS_PCIE_DATA_WIDTH-1:0] s_axis_cq_tdata, + input wire [AXIS_PCIE_KEEP_WIDTH-1:0] s_axis_cq_tkeep, + input wire s_axis_cq_tlast, + output wire s_axis_cq_tready, + input wire [AXIS_PCIE_CQ_USER_WIDTH-1:0] s_axis_cq_tuser, + input wire s_axis_cq_tvalid, + + output wire [AXIS_PCIE_DATA_WIDTH-1:0] m_axis_cc_tdata, + output wire [AXIS_PCIE_KEEP_WIDTH-1:0] m_axis_cc_tkeep, + output wire m_axis_cc_tlast, + input wire m_axis_cc_tready, + output wire [AXIS_PCIE_CC_USER_WIDTH-1:0] m_axis_cc_tuser, + output wire m_axis_cc_tvalid, + + input wire [RQ_SEQ_NUM_WIDTH-1:0] s_axis_rq_seq_num, + input wire s_axis_rq_seq_num_valid, + + input wire [2:0] cfg_max_payload, + input wire [2:0] cfg_max_read_req, + + output wire [18:0] cfg_mgmt_addr, + output wire cfg_mgmt_write, + output wire [31:0] cfg_mgmt_write_data, + output wire [3:0] cfg_mgmt_byte_enable, + output wire cfg_mgmt_read, + input wire [31:0] cfg_mgmt_read_data, + input wire cfg_mgmt_read_write_done, + + input wire [7:0] cfg_fc_ph, + input wire [11:0] cfg_fc_pd, + input wire [7:0] cfg_fc_nph, + input wire [11:0] cfg_fc_npd, + input wire [7:0] cfg_fc_cplh, + input wire [11:0] cfg_fc_cpld, + output wire [2:0] cfg_fc_sel, + + input wire [3:0] cfg_interrupt_msi_enable, + input wire [7:0] cfg_interrupt_msi_vf_enable, + input wire [11:0] cfg_interrupt_msi_mmenable, + input wire cfg_interrupt_msi_mask_update, + input wire [31:0] cfg_interrupt_msi_data, + output wire [3:0] cfg_interrupt_msi_select, + output wire [31:0] cfg_interrupt_msi_int, + output wire [31:0] cfg_interrupt_msi_pending_status, + output wire cfg_interrupt_msi_pending_status_data_enable, + output wire [3:0] cfg_interrupt_msi_pending_status_function_num, + input wire cfg_interrupt_msi_sent, + input wire cfg_interrupt_msi_fail, + output wire [2:0] cfg_interrupt_msi_attr, + output wire cfg_interrupt_msi_tph_present, + output wire [1:0] cfg_interrupt_msi_tph_type, + output wire [8:0] cfg_interrupt_msi_tph_st_tag, + output wire [3:0] cfg_interrupt_msi_function_number, + + output wire status_error_cor, + output wire status_error_uncor +); + +assign sfp_1_led = 2'b00; +assign sfp_2_led = 2'b00; +assign sma_led = 2'b00; + +example_core_pcie_us #( + .AXIS_PCIE_DATA_WIDTH(AXIS_PCIE_DATA_WIDTH), + .AXIS_PCIE_KEEP_WIDTH(AXIS_PCIE_KEEP_WIDTH), + .AXIS_PCIE_RC_USER_WIDTH(AXIS_PCIE_RC_USER_WIDTH), + .AXIS_PCIE_RQ_USER_WIDTH(AXIS_PCIE_RQ_USER_WIDTH), + .AXIS_PCIE_CQ_USER_WIDTH(AXIS_PCIE_CQ_USER_WIDTH), + .AXIS_PCIE_CC_USER_WIDTH(AXIS_PCIE_CC_USER_WIDTH), + .RQ_SEQ_NUM_WIDTH(RQ_SEQ_NUM_WIDTH), + .RQ_SEQ_NUM_ENABLE(RQ_SEQ_NUM_ENABLE), + .PCIE_TAG_COUNT(PCIE_TAG_COUNT), + .READ_OP_TABLE_SIZE(PCIE_TAG_COUNT), + .READ_TX_LIMIT(2**(RQ_SEQ_NUM_WIDTH-1)), + .READ_TX_FC_ENABLE(1), + .WRITE_OP_TABLE_SIZE(2**(RQ_SEQ_NUM_WIDTH-1)), + .WRITE_TX_LIMIT(2**(RQ_SEQ_NUM_WIDTH-1)), + .WRITE_TX_FC_ENABLE(1), + .BAR0_APERTURE(BAR0_APERTURE), + .BAR2_APERTURE(BAR2_APERTURE) +) +example_core_pcie_us_inst ( + .clk(clk), + .rst(rst), + + /* + * AXI input (RC) + */ + .s_axis_rc_tdata(s_axis_rc_tdata), + .s_axis_rc_tkeep(s_axis_rc_tkeep), + .s_axis_rc_tvalid(s_axis_rc_tvalid), + .s_axis_rc_tready(s_axis_rc_tready), + .s_axis_rc_tlast(s_axis_rc_tlast), + .s_axis_rc_tuser(s_axis_rc_tuser), + + /* + * AXI output (RQ) + */ + .m_axis_rq_tdata(m_axis_rq_tdata), + .m_axis_rq_tkeep(m_axis_rq_tkeep), + .m_axis_rq_tvalid(m_axis_rq_tvalid), + .m_axis_rq_tready(m_axis_rq_tready), + .m_axis_rq_tlast(m_axis_rq_tlast), + .m_axis_rq_tuser(m_axis_rq_tuser), + + /* + * AXI input (CQ) + */ + .s_axis_cq_tdata(s_axis_cq_tdata), + .s_axis_cq_tkeep(s_axis_cq_tkeep), + .s_axis_cq_tvalid(s_axis_cq_tvalid), + .s_axis_cq_tready(s_axis_cq_tready), + .s_axis_cq_tlast(s_axis_cq_tlast), + .s_axis_cq_tuser(s_axis_cq_tuser), + + /* + * AXI output (CC) + */ + .m_axis_cc_tdata(m_axis_cc_tdata), + .m_axis_cc_tkeep(m_axis_cc_tkeep), + .m_axis_cc_tvalid(m_axis_cc_tvalid), + .m_axis_cc_tready(m_axis_cc_tready), + .m_axis_cc_tlast(m_axis_cc_tlast), + .m_axis_cc_tuser(m_axis_cc_tuser), + + /* + * Transmit sequence number input + */ + .s_axis_rq_seq_num_0(s_axis_rq_seq_num), + .s_axis_rq_seq_num_valid_0(s_axis_rq_seq_num_valid), + .s_axis_rq_seq_num_1(0), + .s_axis_rq_seq_num_valid_1(0), + + /* + * Flow control + */ + .cfg_fc_ph(cfg_fc_ph), + .cfg_fc_pd(cfg_fc_pd), + .cfg_fc_nph(cfg_fc_nph), + .cfg_fc_npd(cfg_fc_npd), + .cfg_fc_cplh(cfg_fc_cplh), + .cfg_fc_cpld(cfg_fc_cpld), + .cfg_fc_sel(cfg_fc_sel), + + /* + * Configuration interface + */ + .cfg_mgmt_addr(cfg_mgmt_addr[9:0]), + .cfg_mgmt_function_number(cfg_mgmt_addr[17:10]), + .cfg_mgmt_write(cfg_mgmt_write), + .cfg_mgmt_write_data(cfg_mgmt_write_data), + .cfg_mgmt_byte_enable(cfg_mgmt_byte_enable), + .cfg_mgmt_read(cfg_mgmt_read), + .cfg_mgmt_read_data(cfg_mgmt_read_data), + .cfg_mgmt_read_write_done(cfg_mgmt_read_write_done), + + /* + * Interrupt interface + */ + .cfg_interrupt_msi_enable(cfg_interrupt_msi_enable), + .cfg_interrupt_msi_vf_enable(cfg_interrupt_msi_vf_enable), + .cfg_interrupt_msi_mmenable(cfg_interrupt_msi_mmenable), + .cfg_interrupt_msi_mask_update(cfg_interrupt_msi_mask_update), + .cfg_interrupt_msi_data(cfg_interrupt_msi_data), + .cfg_interrupt_msi_select(cfg_interrupt_msi_select), + .cfg_interrupt_msi_int(cfg_interrupt_msi_int), + .cfg_interrupt_msi_pending_status(cfg_interrupt_msi_pending_status), + .cfg_interrupt_msi_pending_status_data_enable(cfg_interrupt_msi_pending_status_data_enable), + .cfg_interrupt_msi_pending_status_function_num(cfg_interrupt_msi_pending_status_function_num), + .cfg_interrupt_msi_sent(cfg_interrupt_msi_sent), + .cfg_interrupt_msi_fail(cfg_interrupt_msi_fail), + .cfg_interrupt_msi_attr(cfg_interrupt_msi_attr), + .cfg_interrupt_msi_tph_present(cfg_interrupt_msi_tph_present), + .cfg_interrupt_msi_tph_type(cfg_interrupt_msi_tph_type), + .cfg_interrupt_msi_tph_st_tag(cfg_interrupt_msi_tph_st_tag), + .cfg_interrupt_msi_function_number(cfg_interrupt_msi_function_number), + + /* + * Configuration + */ + .cfg_max_read_req(cfg_max_read_req), + .cfg_max_payload(cfg_max_payload), + + /* + * Status + */ + .status_error_cor(status_error_cor), + .status_error_uncor(status_error_uncor) +); + +assign cfg_mgmt_addr[18] = 1'b0; + +endmodule + +`resetall diff --git a/example/ExaNIC_X10/fpga/rtl/sync_reset.v b/example/ExaNIC_X10/fpga/rtl/sync_reset.v new file mode 100644 index 000000000..1fd24d361 --- /dev/null +++ b/example/ExaNIC_X10/fpga/rtl/sync_reset.v @@ -0,0 +1,56 @@ +/* + +Copyright (c) 2014-2018 Alex Forencich + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +*/ + +// Language: Verilog-2001 + +`resetall +`timescale 1 ns / 1 ps +`default_nettype none + +/* + * Synchronizes an active-high asynchronous reset signal to a given clock by + * using a pipeline of N registers. + */ +module sync_reset #( + parameter N=2 // depth of synchronizer +)( + input wire clk, + input wire rst, + output wire sync_reset_out +); + +reg [N-1:0] sync_reg = {N{1'b1}}; + +assign sync_reset_out = sync_reg[N-1]; + +always @(posedge clk or posedge rst) begin + if (rst) + sync_reg <= {N{1'b1}}; + else + sync_reg <= {sync_reg[N-2:0], 1'b0}; +end + +endmodule + +`resetall diff --git a/example/ExaNIC_X10/fpga/rtl/sync_signal.v b/example/ExaNIC_X10/fpga/rtl/sync_signal.v new file mode 100644 index 000000000..74b855fa1 --- /dev/null +++ b/example/ExaNIC_X10/fpga/rtl/sync_signal.v @@ -0,0 +1,62 @@ +/* + +Copyright (c) 2014-2018 Alex Forencich + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +*/ + +// Language: Verilog-2001 + +`resetall +`timescale 1 ns / 1 ps +`default_nettype none + +/* + * Synchronizes an asyncronous signal to a given clock by using a pipeline of + * two registers. + */ +module sync_signal #( + parameter WIDTH=1, // width of the input and output signals + parameter N=2 // depth of synchronizer +)( + input wire clk, + input wire [WIDTH-1:0] in, + output wire [WIDTH-1:0] out +); + +reg [WIDTH-1:0] sync_reg[N-1:0]; + +/* + * The synchronized output is the last register in the pipeline. + */ +assign out = sync_reg[N-1]; + +integer k; + +always @(posedge clk) begin + sync_reg[0] <= in; + for (k = 1; k < N; k = k + 1) begin + sync_reg[k] <= sync_reg[k-1]; + end +end + +endmodule + +`resetall diff --git a/example/ExaNIC_X10/fpga/tb/fpga_core/Makefile b/example/ExaNIC_X10/fpga/tb/fpga_core/Makefile new file mode 100644 index 000000000..2cd4c6b21 --- /dev/null +++ b/example/ExaNIC_X10/fpga/tb/fpga_core/Makefile @@ -0,0 +1,123 @@ +# Copyright (c) 2020 Alex Forencich +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +TOPLEVEL_LANG = verilog + +SIM ?= icarus +WAVES ?= 0 + +COCOTB_HDL_TIMEUNIT = 1ns +COCOTB_HDL_TIMEPRECISION = 1ps + +DUT = fpga_core +TOPLEVEL = $(DUT) +MODULE = test_$(DUT) +VERILOG_SOURCES += ../../rtl/$(DUT).v +VERILOG_SOURCES += ../../rtl/common/example_core_pcie_us.v +VERILOG_SOURCES += ../../rtl/common/example_core_pcie.v +VERILOG_SOURCES += ../../rtl/common/example_core.v +VERILOG_SOURCES += ../../rtl/common/axi_ram.v +VERILOG_SOURCES += ../../lib/pcie/rtl/pcie_us_if.v +VERILOG_SOURCES += ../../lib/pcie/rtl/pcie_us_if_rc.v +VERILOG_SOURCES += ../../lib/pcie/rtl/pcie_us_if_rq.v +VERILOG_SOURCES += ../../lib/pcie/rtl/pcie_us_if_cq.v +VERILOG_SOURCES += ../../lib/pcie/rtl/pcie_us_if_cc.v +VERILOG_SOURCES += ../../lib/pcie/rtl/pcie_us_cfg.v +VERILOG_SOURCES += ../../lib/pcie/rtl/pcie_us_msi.v +VERILOG_SOURCES += ../../lib/pcie/rtl/pcie_axil_master.v +VERILOG_SOURCES += ../../lib/pcie/rtl/pcie_axi_master.v +VERILOG_SOURCES += ../../lib/pcie/rtl/pcie_axi_master_rd.v +VERILOG_SOURCES += ../../lib/pcie/rtl/pcie_axi_master_wr.v +VERILOG_SOURCES += ../../lib/pcie/rtl/pcie_tlp_demux_bar.v +VERILOG_SOURCES += ../../lib/pcie/rtl/pcie_tlp_demux.v +VERILOG_SOURCES += ../../lib/pcie/rtl/pcie_tlp_mux.v +VERILOG_SOURCES += ../../lib/pcie/rtl/dma_if_pcie.v +VERILOG_SOURCES += ../../lib/pcie/rtl/dma_if_pcie_rd.v +VERILOG_SOURCES += ../../lib/pcie/rtl/dma_if_pcie_wr.v +VERILOG_SOURCES += ../../lib/pcie/rtl/dma_psdpram.v +VERILOG_SOURCES += ../../lib/pcie/rtl/arbiter.v +VERILOG_SOURCES += ../../lib/pcie/rtl/priority_encoder.v +VERILOG_SOURCES += ../../lib/pcie/rtl/pulse_merge.v + +# module parameters +export PARAM_AXIS_PCIE_DATA_WIDTH ?= 256 +export PARAM_AXIS_PCIE_KEEP_WIDTH ?= $(shell expr $(PARAM_AXIS_PCIE_DATA_WIDTH) / 32 ) +export PARAM_AXIS_PCIE_RQ_USER_WIDTH ?= 60 +export PARAM_AXIS_PCIE_RC_USER_WIDTH ?= 75 +export PARAM_AXIS_PCIE_CQ_USER_WIDTH ?= 85 +export PARAM_AXIS_PCIE_CC_USER_WIDTH ?= 33 +export PARAM_RQ_SEQ_NUM_WIDTH ?= 4 +export PARAM_RQ_SEQ_NUM_ENABLE ?= 1 +export PARAM_PCIE_TAG_COUNT ?= 64 +export PARAM_BAR0_APERTURE ?= 24 +export PARAM_BAR2_APERTURE ?= 24 + +ifeq ($(SIM), icarus) + PLUSARGS += -fst + + COMPILE_ARGS += -P $(TOPLEVEL).AXIS_PCIE_DATA_WIDTH=$(PARAM_AXIS_PCIE_DATA_WIDTH) + COMPILE_ARGS += -P $(TOPLEVEL).AXIS_PCIE_KEEP_WIDTH=$(PARAM_AXIS_PCIE_KEEP_WIDTH) + COMPILE_ARGS += -P $(TOPLEVEL).AXIS_PCIE_RQ_USER_WIDTH=$(PARAM_AXIS_PCIE_RQ_USER_WIDTH) + COMPILE_ARGS += -P $(TOPLEVEL).AXIS_PCIE_RC_USER_WIDTH=$(PARAM_AXIS_PCIE_RC_USER_WIDTH) + COMPILE_ARGS += -P $(TOPLEVEL).AXIS_PCIE_CQ_USER_WIDTH=$(PARAM_AXIS_PCIE_CQ_USER_WIDTH) + COMPILE_ARGS += -P $(TOPLEVEL).AXIS_PCIE_CC_USER_WIDTH=$(PARAM_AXIS_PCIE_CC_USER_WIDTH) + COMPILE_ARGS += -P $(TOPLEVEL).RQ_SEQ_NUM_WIDTH=$(PARAM_RQ_SEQ_NUM_WIDTH) + COMPILE_ARGS += -P $(TOPLEVEL).RQ_SEQ_NUM_ENABLE=$(PARAM_RQ_SEQ_NUM_ENABLE) + COMPILE_ARGS += -P $(TOPLEVEL).PCIE_TAG_COUNT=$(PARAM_PCIE_TAG_COUNT) + COMPILE_ARGS += -P $(TOPLEVEL).BAR0_APERTURE=$(PARAM_BAR0_APERTURE) + COMPILE_ARGS += -P $(TOPLEVEL).BAR2_APERTURE=$(PARAM_BAR2_APERTURE) + + ifeq ($(WAVES), 1) + VERILOG_SOURCES += iverilog_dump.v + COMPILE_ARGS += -s iverilog_dump + endif +else ifeq ($(SIM), verilator) + COMPILE_ARGS += -Wno-SELRANGE -Wno-WIDTH + + COMPILE_ARGS += -GAXIS_PCIE_DATA_WIDTH=$(PARAM_AXIS_PCIE_DATA_WIDTH) + COMPILE_ARGS += -GAXIS_PCIE_KEEP_WIDTH=$(PARAM_AXIS_PCIE_KEEP_WIDTH) + COMPILE_ARGS += -GAXIS_PCIE_RQ_USER_WIDTH=$(PARAM_AXIS_PCIE_RQ_USER_WIDTH) + COMPILE_ARGS += -GAXIS_PCIE_RC_USER_WIDTH=$(PARAM_AXIS_PCIE_RC_USER_WIDTH) + COMPILE_ARGS += -GAXIS_PCIE_CQ_USER_WIDTH=$(PARAM_AXIS_PCIE_CQ_USER_WIDTH) + COMPILE_ARGS += -GAXIS_PCIE_CC_USER_WIDTH=$(PARAM_AXIS_PCIE_CC_USER_WIDTH) + COMPILE_ARGS += -GRQ_SEQ_NUM_WIDTH=$(PARAM_RQ_SEQ_NUM_WIDTH) + COMPILE_ARGS += -GRQ_SEQ_NUM_ENABLE=$(PARAM_RQ_SEQ_NUM_ENABLE) + COMPILE_ARGS += -GPCIE_TAG_COUNT=$(PARAM_PCIE_TAG_COUNT) + COMPILE_ARGS += -GBAR0_APERTURE=$(PARAM_BAR0_APERTURE) + COMPILE_ARGS += -GBAR2_APERTURE=$(PARAM_BAR2_APERTURE) + + ifeq ($(WAVES), 1) + COMPILE_ARGS += --trace-fst + endif +endif + +include $(shell cocotb-config --makefiles)/Makefile.sim + +iverilog_dump.v: + echo 'module iverilog_dump();' > $@ + echo 'initial begin' >> $@ + echo ' $$dumpfile("$(TOPLEVEL).fst");' >> $@ + echo ' $$dumpvars(0, $(TOPLEVEL));' >> $@ + echo 'end' >> $@ + echo 'endmodule' >> $@ + +clean:: + @rm -rf iverilog_dump.v + @rm -rf dump.fst $(TOPLEVEL).fst diff --git a/example/ExaNIC_X10/fpga/tb/fpga_core/test_fpga_core.py b/example/ExaNIC_X10/fpga/tb/fpga_core/test_fpga_core.py new file mode 100644 index 000000000..40aca6918 --- /dev/null +++ b/example/ExaNIC_X10/fpga/tb/fpga_core/test_fpga_core.py @@ -0,0 +1,381 @@ +""" + +Copyright (c) 2020 Alex Forencich + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +""" + +import logging +import os + +import cocotb_test.simulator + +import cocotb +from cocotb.log import SimLog +from cocotb.triggers import RisingEdge, FallingEdge, Timer + +from cocotbext.axi import AxiStreamBus +from cocotbext.pcie.core import RootComplex +from cocotbext.pcie.xilinx.us import UltraScalePcieDevice +from cocotbext.axi.utils import hexdump_str + + +class TB(object): + def __init__(self, dut): + self.dut = dut + + self.log = SimLog("cocotb.tb") + self.log.setLevel(logging.DEBUG) + + # PCIe + self.rc = RootComplex() + + self.dev = UltraScalePcieDevice( + # configuration options + pcie_generation=3, + pcie_link_width=8, + user_clk_frequency=250e6, + alignment="dword", + straddle=False, + enable_pf1=False, + enable_client_tag=True, + enable_extended_tag=True, + enable_parity=False, + enable_rx_msg_interface=False, + enable_sriov=False, + enable_extended_configuration=False, + + enable_pf0_msi=True, + enable_pf1_msi=False, + + # signals + # Clock and Reset Interface + user_clk=dut.clk, + user_reset=dut.rst, + # user_lnk_up + # sys_clk + # sys_clk_gt + # sys_reset + # phy_rdy_out + + # Requester reQuest Interface + rq_bus=AxiStreamBus.from_prefix(dut, "m_axis_rq"), + pcie_rq_seq_num=dut.s_axis_rq_seq_num, + pcie_rq_seq_num_vld=dut.s_axis_rq_seq_num_valid, + # pcie_rq_tag + # pcie_rq_tag_av + # pcie_rq_tag_vld + + # Requester Completion Interface + rc_bus=AxiStreamBus.from_prefix(dut, "s_axis_rc"), + + # Completer reQuest Interface + cq_bus=AxiStreamBus.from_prefix(dut, "s_axis_cq"), + # pcie_cq_np_req + # pcie_cq_np_req_count + + # Completer Completion Interface + cc_bus=AxiStreamBus.from_prefix(dut, "m_axis_cc"), + + # Transmit Flow Control Interface + # pcie_tfc_nph_av=dut.pcie_tfc_nph_av, + # pcie_tfc_npd_av=dut.pcie_tfc_npd_av, + + # Configuration Management Interface + cfg_mgmt_addr=dut.cfg_mgmt_addr, + cfg_mgmt_write=dut.cfg_mgmt_write, + cfg_mgmt_write_data=dut.cfg_mgmt_write_data, + cfg_mgmt_byte_enable=dut.cfg_mgmt_byte_enable, + cfg_mgmt_read=dut.cfg_mgmt_read, + cfg_mgmt_read_data=dut.cfg_mgmt_read_data, + cfg_mgmt_read_write_done=dut.cfg_mgmt_read_write_done, + # cfg_mgmt_debug_access + + # Configuration Status Interface + # cfg_phy_link_down + # cfg_phy_link_status + # cfg_negotiated_width + # cfg_current_speed + cfg_max_payload=dut.cfg_max_payload, + cfg_max_read_req=dut.cfg_max_read_req, + # cfg_function_status + # cfg_vf_status + # cfg_function_power_state + # cfg_vf_power_state + # cfg_link_power_state + # cfg_err_cor_out + # cfg_err_nonfatal_out + # cfg_err_fatal_out + # cfg_local_error_out + # cfg_local_error_valid + # cfg_rx_pm_state + # cfg_tx_pm_state + # cfg_ltssm_state + # cfg_rcb_status + # cfg_obff_enable + # cfg_pl_status_change + # cfg_tph_requester_enable + # cfg_tph_st_mode + # cfg_vf_tph_requester_enable + # cfg_vf_tph_st_mode + + # Configuration Received Message Interface + # cfg_msg_received + # cfg_msg_received_data + # cfg_msg_received_type + + # Configuration Transmit Message Interface + # cfg_msg_transmit + # cfg_msg_transmit_type + # cfg_msg_transmit_data + # cfg_msg_transmit_done + + # Configuration Flow Control Interface + cfg_fc_ph=dut.cfg_fc_ph, + cfg_fc_pd=dut.cfg_fc_pd, + cfg_fc_nph=dut.cfg_fc_nph, + cfg_fc_npd=dut.cfg_fc_npd, + cfg_fc_cplh=dut.cfg_fc_cplh, + cfg_fc_cpld=dut.cfg_fc_cpld, + cfg_fc_sel=dut.cfg_fc_sel, + + # Configuration Control Interface + # cfg_hot_reset_in + # cfg_hot_reset_out + # cfg_config_space_enable + # cfg_dsn + # cfg_bus_number + # cfg_ds_port_number + # cfg_ds_bus_number + # cfg_ds_device_number + # cfg_ds_function_number + # cfg_power_state_change_ack + # cfg_power_state_change_interrupt + cfg_err_cor_in=dut.status_error_cor, + cfg_err_uncor_in=dut.status_error_uncor, + # cfg_flr_in_process + # cfg_flr_done + # cfg_vf_flr_in_process + # cfg_vf_flr_func_num + # cfg_vf_flr_done + # cfg_pm_aspm_l1_entry_reject + # cfg_pm_aspm_tx_l0s_entry_disable + # cfg_req_pm_transition_l23_ready + # cfg_link_training_enable + + # Configuration Interrupt Controller Interface + # cfg_interrupt_int + # cfg_interrupt_sent + # cfg_interrupt_pending + cfg_interrupt_msi_enable=dut.cfg_interrupt_msi_enable, + cfg_interrupt_msi_vf_enable=dut.cfg_interrupt_msi_vf_enable, + cfg_interrupt_msi_mmenable=dut.cfg_interrupt_msi_mmenable, + cfg_interrupt_msi_mask_update=dut.cfg_interrupt_msi_mask_update, + cfg_interrupt_msi_data=dut.cfg_interrupt_msi_data, + cfg_interrupt_msi_select=dut.cfg_interrupt_msi_select, + cfg_interrupt_msi_int=dut.cfg_interrupt_msi_int, + cfg_interrupt_msi_pending_status=dut.cfg_interrupt_msi_pending_status, + cfg_interrupt_msi_pending_status_data_enable=dut.cfg_interrupt_msi_pending_status_data_enable, + cfg_interrupt_msi_pending_status_function_num=dut.cfg_interrupt_msi_pending_status_function_num, + cfg_interrupt_msi_sent=dut.cfg_interrupt_msi_sent, + cfg_interrupt_msi_fail=dut.cfg_interrupt_msi_fail, + # cfg_interrupt_msix_enable + # cfg_interrupt_msix_mask + # cfg_interrupt_msix_vf_enable + # cfg_interrupt_msix_vf_mask + # cfg_interrupt_msix_address + # cfg_interrupt_msix_data + # cfg_interrupt_msix_int + # cfg_interrupt_msix_vec_pending + # cfg_interrupt_msix_vec_pending_status + cfg_interrupt_msi_attr=dut.cfg_interrupt_msi_attr, + cfg_interrupt_msi_tph_present=dut.cfg_interrupt_msi_tph_present, + cfg_interrupt_msi_tph_type=dut.cfg_interrupt_msi_tph_type, + # cfg_interrupt_msi_tph_st_tag=dut.cfg_interrupt_msi_tph_st_tag, + # cfg_interrupt_msi_function_number=dut.cfg_interrupt_msi_function_number, + + # Configuration Extend Interface + # cfg_ext_read_received + # cfg_ext_write_received + # cfg_ext_register_number + # cfg_ext_function_number + # cfg_ext_write_data + # cfg_ext_write_byte_enable + # cfg_ext_read_data + # cfg_ext_read_data_valid + ) + + # self.dev.log.setLevel(logging.DEBUG) + + self.rc.make_port().connect(self.dev) + + self.dev.functions[0].msi_multiple_message_capable = 5 + + self.dev.functions[0].configure_bar(0, 2**24) + self.dev.functions[0].configure_bar(2, 2**24) + + async def init(self): + + await FallingEdge(self.dut.rst) + await Timer(100, 'ns') + + await self.rc.enumerate(enable_bus_mastering=True, configure_msi=True) + + +@cocotb.test() +async def run_test(dut): + + tb = TB(dut) + + await tb.init() + + mem_base, mem_data = tb.rc.alloc_region(16*1024*1024) + + dev_pf0_bar0 = tb.rc.tree[0][0].bar_addr[0] + dev_pf0_bar2 = tb.rc.tree[0][0].bar_addr[2] + + tb.log.info("Test memory write to BAR 2") + + await tb.rc.mem_write(dev_pf0_bar2, b'\x11\x22\x33\x44') + + await Timer(100, 'ns') + + tb.log.info("Test memory read from BAR 2") + + val = await tb.rc.mem_read(dev_pf0_bar2, 4, 1000) + tb.log.info("Read data: %s", val) + assert val == b'\x11\x22\x33\x44' + + tb.log.info("Test DMA") + + # write packet data + mem_data[0:1024] = bytearray([x % 256 for x in range(1024)]) + + # enable DMA + await tb.rc.mem_write_dword(dev_pf0_bar0+0x000000, 1) + + # write pcie read descriptor + await tb.rc.mem_write_dword(dev_pf0_bar0+0x000100, (mem_base+0x0000) & 0xffffffff) + await tb.rc.mem_write_dword(dev_pf0_bar0+0x000104, (mem_base+0x0000 >> 32) & 0xffffffff) + await tb.rc.mem_write_dword(dev_pf0_bar0+0x000108, (0x100) & 0xffffffff) + await tb.rc.mem_write_dword(dev_pf0_bar0+0x00010C, (0x100 >> 32) & 0xffffffff) + await tb.rc.mem_write_dword(dev_pf0_bar0+0x000110, 0x400) + await tb.rc.mem_write_dword(dev_pf0_bar0+0x000114, 0xAA) + + await Timer(2000, 'ns') + + # read status + val = await tb.rc.mem_read_dword(dev_pf0_bar0+0x000118) + tb.log.info("Status: 0x%x", val) + assert val == 0x800000AA + + # write pcie write descriptor + await tb.rc.mem_write_dword(dev_pf0_bar0+0x000200, (mem_base+0x1000) & 0xffffffff) + await tb.rc.mem_write_dword(dev_pf0_bar0+0x000204, (mem_base+0x1000 >> 32) & 0xffffffff) + await tb.rc.mem_write_dword(dev_pf0_bar0+0x000208, (0x100) & 0xffffffff) + await tb.rc.mem_write_dword(dev_pf0_bar0+0x00020C, (0x100 >> 32) & 0xffffffff) + await tb.rc.mem_write_dword(dev_pf0_bar0+0x000210, 0x400) + await tb.rc.mem_write_dword(dev_pf0_bar0+0x000214, 0x55) + + await Timer(2000, 'ns') + + # read status + val = await tb.rc.mem_read_dword(dev_pf0_bar0+0x000218) + tb.log.info("Status: 0x%x", val) + assert val == 0x80000055 + + tb.log.info("%s", hexdump_str(mem_data, 0x1000, 64)) + + assert mem_data[0:1024] == mem_data[0x1000:0x1000+1024] + + await RisingEdge(dut.clk) + await RisingEdge(dut.clk) + + +# cocotb-test + +tests_dir = os.path.dirname(__file__) +rtl_dir = os.path.abspath(os.path.join(tests_dir, '..', '..', 'rtl')) +lib_dir = os.path.abspath(os.path.join(rtl_dir, '..', 'lib')) +pcie_rtl_dir = os.path.abspath(os.path.join(lib_dir, 'pcie', 'rtl')) + + +def test_fpga_core(request): + dut = "fpga_core" + module = os.path.splitext(os.path.basename(__file__))[0] + toplevel = dut + + verilog_sources = [ + os.path.join(rtl_dir, f"{dut}.v"), + os.path.join(rtl_dir, "common", "example_core_pcie_us.v"), + os.path.join(rtl_dir, "common", "example_core_pcie.v"), + os.path.join(rtl_dir, "common", "example_core.v"), + os.path.join(rtl_dir, "common", "axi_ram.v"), + os.path.join(pcie_rtl_dir, "pcie_us_if.v"), + os.path.join(pcie_rtl_dir, "pcie_us_if_rc.v"), + os.path.join(pcie_rtl_dir, "pcie_us_if_rq.v"), + os.path.join(pcie_rtl_dir, "pcie_us_if_cq.v"), + os.path.join(pcie_rtl_dir, "pcie_us_if_cc.v"), + os.path.join(pcie_rtl_dir, "pcie_us_cfg.v"), + os.path.join(pcie_rtl_dir, "pcie_us_msi.v"), + os.path.join(pcie_rtl_dir, "pcie_axil_master.v"), + os.path.join(pcie_rtl_dir, "pcie_axi_master.v"), + os.path.join(pcie_rtl_dir, "pcie_axi_master_rd.v"), + os.path.join(pcie_rtl_dir, "pcie_axi_master_wr.v"), + os.path.join(pcie_rtl_dir, "pcie_tlp_demux_bar.v"), + os.path.join(pcie_rtl_dir, "pcie_tlp_demux.v"), + os.path.join(pcie_rtl_dir, "pcie_tlp_mux.v"), + os.path.join(pcie_rtl_dir, "dma_if_pcie.v"), + os.path.join(pcie_rtl_dir, "dma_if_pcie_rd.v"), + os.path.join(pcie_rtl_dir, "dma_if_pcie_wr.v"), + os.path.join(pcie_rtl_dir, "dma_psdpram.v"), + os.path.join(pcie_rtl_dir, "arbiter.v"), + os.path.join(pcie_rtl_dir, "priority_encoder.v"), + os.path.join(pcie_rtl_dir, "pulse_merge.v"), + ] + + parameters = {} + + parameters['AXIS_PCIE_DATA_WIDTH'] = 256 + parameters['AXIS_PCIE_KEEP_WIDTH'] = parameters['AXIS_PCIE_DATA_WIDTH'] // 32 + parameters['AXIS_PCIE_RQ_USER_WIDTH'] = 60 + parameters['AXIS_PCIE_RC_USER_WIDTH'] = 75 + parameters['AXIS_PCIE_CQ_USER_WIDTH'] = 85 + parameters['AXIS_PCIE_CC_USER_WIDTH'] = 33 + parameters['RQ_SEQ_NUM_WIDTH'] = 4 + parameters['RQ_SEQ_NUM_ENABLE'] = 1 + parameters['PCIE_TAG_COUNT'] = 64 + parameters['BAR0_APERTURE'] = 24 + parameters['BAR2_APERTURE'] = 24 + + extra_env = {f'PARAM_{k}': str(v) for k, v in parameters.items()} + + sim_build = os.path.join(tests_dir, "sim_build", + request.node.name.replace('[', '-').replace(']', '')) + + cocotb_test.simulator.run( + python_search=[tests_dir], + verilog_sources=verilog_sources, + toplevel=toplevel, + module=module, + parameters=parameters, + sim_build=sim_build, + extra_env=extra_env, + )