From beaf1c6fbf2b1d772301a7e8095fb140ea76f83f Mon Sep 17 00:00:00 2001 From: Alex Forencich Date: Fri, 2 Jun 2023 01:53:24 -0700 Subject: [PATCH] fpga/mqnic/ZCU106/fpga_zynqmp: Add support for Ubuntu for ZCU106 MPSoC Signed-off-by: Alex Forencich --- fpga/mqnic/ZCU106/fpga_zynqmp/README.md | 42 +++++-- .../mqnic/ZCU106/fpga_zynqmp/common/vivado.mk | 10 +- fpga/mqnic/ZCU106/fpga_zynqmp/fpga/Makefile | 20 ++++ .../fpga_zynqmp/fpga_app_dma_bench/Makefile | 20 ++++ fpga/mqnic/ZCU106/fpga_zynqmp/ps/overlay.dtsi | 112 ++++++++++++++++++ 5 files changed, 191 insertions(+), 13 deletions(-) create mode 100644 fpga/mqnic/ZCU106/fpga_zynqmp/ps/overlay.dtsi diff --git a/fpga/mqnic/ZCU106/fpga_zynqmp/README.md b/fpga/mqnic/ZCU106/fpga_zynqmp/README.md index f3641abbd..428f8f851 100644 --- a/fpga/mqnic/ZCU106/fpga_zynqmp/README.md +++ b/fpga/mqnic/ZCU106/fpga_zynqmp/README.md @@ -1,4 +1,4 @@ -# Corundum mqnic for ZCU106 using ZynqMP PS as host system +# Corundum mqnic for ZCU106 (Zynq MPSoC PS host) ## Introduction @@ -9,20 +9,44 @@ the Zynq US+ MPSoC. * PHY: 10G BASE-R PHY IP core and internal GTH transceiver * RAM: 2 GB DDR4 2400 (256M x64) -## How to build +## Quick start for Ubuntu -Run make in this directory to build the bitstream and the .xsa -file. Ensure that the Xilinx Vivado toolchain components are in PATH. +### Build FPGA bitstream -Then change into sub-directory ps/petalinux/ and build the PetaLinux project. -Ensure that the Xilinx PetaLinux toolchain components are in PATH. +Run `make app` in the `fpga` subdirectory to build the bitstream, `.xsa` file, and device tree overlay. Ensure that the Xilinx Vivado toolchain components are in PATH (source `settings64.sh` in Vivado installation directory). + +### Installation + +Download an Ubuntu image for the ZCU106 here: https://ubuntu.com/download/amd-xilinx. Write the image to an SD card with `dd`, for example: + + xzcat ubuntu.img.xz | dd of=/dev/sdX + +Copy files in `fpga/app` to `/lib/firmware/xilinx/mqnic` on the ZCU106. Also make a copy of the source repo on the ZCU106 from which the kernel module and userspace tools can be built. + +### Build driver and userspace tools + +On the ZCU106, run `make` in `modules/mqnic` to build the driver. Ensure the headers for the running kernel are installed, otherwise the driver cannot be compiled. Then run `make` in `utils` to build the userspace tools. + +### Testing + +On the ZCU106, run `sudo xmutil unloadapp` to unload the FPGA, then `sudo xmutil loadapp mqnic` to load the configuration. Then, build the kernel module and userspace tools by running `make` in `modules/mqnic` and `utils`. Finally, load the kernel module with `insmod mqnic.ko`. Check `dmesg` for output from driver initialization. Run `mqnic-dump -d /dev/mqnic0` to dump the internal state. + +## Quick start for PetaLinux + +### Build FPGA bitstream + +Run `make` in the `fpga` subdirectory to build the bitstream and the `.xsa` +file. Ensure that the Xilinx Vivado toolchain components are in PATH (source `settings64.sh` in Vivado installation directory). + +### Build PetaLinux image + +Then change into sub-directory `ps/petalinux/` and build the PetaLinux project. Ensure that the Xilinx PetaLinux toolchain components are in PATH. make -C ps/petalinux/ build-boot -## How to test +### Testing -Copy the following, resulting files of building the PetaLinux project onto an -SDcard suitable for then booting the ZCU106 in SDcard boot mode. +Copy the following, resulting files of building the PetaLinux project onto an SDcard suitable for then booting the ZCU106 in SDcard boot mode. ps/petalinux/images/linux/: BOOT.BIN diff --git a/fpga/mqnic/ZCU106/fpga_zynqmp/common/vivado.mk b/fpga/mqnic/ZCU106/fpga_zynqmp/common/vivado.mk index a337628f3..302e74568 100644 --- a/fpga/mqnic/ZCU106/fpga_zynqmp/common/vivado.mk +++ b/fpga/mqnic/ZCU106/fpga_zynqmp/common/vivado.mk @@ -31,7 +31,7 @@ .PHONY: fpga vivado tmpclean clean distclean # prevent make from deleting intermediate files and reports -.PRECIOUS: %.xpr %.bit %.mcs %.prm +.PRECIOUS: %.xpr %.bit %.bin %.mcs %.prm .SECONDARY: CONFIG ?= config.mk @@ -71,7 +71,7 @@ tmpclean:: -rm -rf create_project.tcl update_config.tcl run_synth.tcl run_impl.tcl generate_bit.tcl clean:: tmpclean - -rm -rf *.bit *.ltx *.xsa program.tcl generate_mcs.tcl *.mcs *.prm flash.tcl + -rm -rf *.bit *.bin *.ltx *.xsa program.tcl generate_mcs.tcl *.mcs *.prm flash.tcl -rm -rf *_utilization.rpt *_utilization_hierarchical.rpt distclean:: clean @@ -121,19 +121,21 @@ $(PROJECT).runs/impl_1/$(PROJECT)_routed.dcp: $(PROJECT).runs/synth_1/$(PROJECT) vivado -nojournal -nolog -mode batch -source run_impl.tcl # bit file -$(PROJECT).bit $(PROJECT).ltx $(PROJECT).xsa: $(PROJECT).runs/impl_1/$(PROJECT)_routed.dcp +$(PROJECT).bit $(PROJECT).bin $(PROJECT).ltx $(PROJECT).xsa: $(PROJECT).runs/impl_1/$(PROJECT)_routed.dcp echo "open_project $(PROJECT).xpr" > generate_bit.tcl echo "open_run impl_1" >> generate_bit.tcl - echo "write_bitstream -force $(PROJECT).runs/impl_1/$(PROJECT).bit" >> generate_bit.tcl + echo "write_bitstream -force -bin_file $(PROJECT).runs/impl_1/$(PROJECT).bit" >> generate_bit.tcl echo "write_debug_probes -force $(PROJECT).runs/impl_1/$(PROJECT).ltx" >> generate_bit.tcl echo "write_hw_platform -fixed -force -include_bit $(PROJECT).xsa" >> generate_bit.tcl vivado -nojournal -nolog -mode batch -source generate_bit.tcl ln -f -s $(PROJECT).runs/impl_1/$(PROJECT).bit . + ln -f -s $(PROJECT).runs/impl_1/$(PROJECT).bin . if [ -e $(PROJECT).runs/impl_1/$(PROJECT).ltx ]; then ln -f -s $(PROJECT).runs/impl_1/$(PROJECT).ltx .; fi mkdir -p rev COUNT=100; \ while [ -e rev/$(PROJECT)_rev$$COUNT.bit ]; \ do COUNT=$$((COUNT+1)); done; \ cp -pv $(PROJECT).runs/impl_1/$(PROJECT).bit rev/$(PROJECT)_rev$$COUNT.bit; \ + cp -pv $(PROJECT).runs/impl_1/$(PROJECT).bin rev/$(PROJECT)_rev$$COUNT.bin; \ if [ -e $(PROJECT).runs/impl_1/$(PROJECT).ltx ]; then cp -pv $(PROJECT).runs/impl_1/$(PROJECT).ltx rev/$(PROJECT)_rev$$COUNT.ltx; fi; \ cp -pv $(PROJECT).xsa rev/$(PROJECT)_rev$$COUNT.xsa; diff --git a/fpga/mqnic/ZCU106/fpga_zynqmp/fpga/Makefile b/fpga/mqnic/ZCU106/fpga_zynqmp/fpga/Makefile index a2584c1b1..16664177e 100644 --- a/fpga/mqnic/ZCU106/fpga_zynqmp/fpga/Makefile +++ b/fpga/mqnic/ZCU106/fpga_zynqmp/fpga/Makefile @@ -139,3 +139,23 @@ program: $(FPGA_TOP).bit echo "program_hw_devices [current_hw_device]" >> program.tcl echo "exit" >> program.tcl vivado -nojournal -nolog -mode batch -source program.tcl + +APP_DIR = app + +$(APP_DIR)/shell.json: + @mkdir -p $(@D) + echo '{"shell_type": "XRT_FLAT", "num_slots": "1"}' > $@ + +$(APP_DIR)/$(FPGA_TOP).bin: $(FPGA_TOP).bin + @mkdir -p $(@D) + cp $< $@ + +$(APP_DIR)/overlay.dtbo: ../ps/overlay.dtsi + @mkdir -p $(@D) + dtc -@ -O dtb -o $@ $^ + +.PHONY: app +app: $(APP_DIR)/$(FPGA_TOP).bin $(APP_DIR)/shell.json $(APP_DIR)/overlay.dtbo + +clean:: + -rm -rf $(APP_DIR) diff --git a/fpga/mqnic/ZCU106/fpga_zynqmp/fpga_app_dma_bench/Makefile b/fpga/mqnic/ZCU106/fpga_zynqmp/fpga_app_dma_bench/Makefile index f6e5b628f..3fbc7665d 100644 --- a/fpga/mqnic/ZCU106/fpga_zynqmp/fpga_app_dma_bench/Makefile +++ b/fpga/mqnic/ZCU106/fpga_zynqmp/fpga_app_dma_bench/Makefile @@ -149,3 +149,23 @@ program: $(FPGA_TOP).bit echo "program_hw_devices [current_hw_device]" >> program.tcl echo "exit" >> program.tcl vivado -nojournal -nolog -mode batch -source program.tcl + +APP_DIR = app + +$(APP_DIR)/shell.json: + @mkdir -p $(@D) + echo '{"shell_type": "XRT_FLAT", "num_slots": "1"}' > $@ + +$(APP_DIR)/$(FPGA_TOP).bin: $(FPGA_TOP).bin + @mkdir -p $(@D) + cp $< $@ + +$(APP_DIR)/overlay.dtbo: ../ps/overlay.dtsi + @mkdir -p $(@D) + dtc -@ -O dtb -o $@ $^ + +.PHONY: app +app: $(APP_DIR)/$(FPGA_TOP).bin $(APP_DIR)/shell.json $(APP_DIR)/overlay.dtbo + +clean:: + -rm -rf $(APP_DIR) diff --git a/fpga/mqnic/ZCU106/fpga_zynqmp/ps/overlay.dtsi b/fpga/mqnic/ZCU106/fpga_zynqmp/ps/overlay.dtsi new file mode 100644 index 000000000..8372a3a4b --- /dev/null +++ b/fpga/mqnic/ZCU106/fpga_zynqmp/ps/overlay.dtsi @@ -0,0 +1,112 @@ +/dts-v1/; +/plugin/; +/ { + fragment@0 { + target = <&fpga_full>; + __overlay__ { + #address-cells = <2>; + #size-cells = <2>; + firmware-name = "fpga.bin"; + }; + }; + fragment@1 { + target = <&amba>; + __overlay__ { + #address-cells = <2>; + #size-cells = <2>; + + afi0: afi0 { + compatible = "xlnx,afi-fpga"; + config-afi = + // PL-to-PS AXI + // 0: 128-bit, 1: 64-bit, 2: 32-bit + <0 0>, // AFIFM0_RDCTRL [1:0] S_AXI_HPC0_FPD + <1 0>, // AFIFM0_WRCTRL [1:0] S_AXI_HPC0_FPD + <2 0>, // AFIFM1_RDCTRL [1:0] S_AXI_HPC1_FPD + <3 0>, // AFIFM1_WRCTRL [1:0] S_AXI_HPC1_FPD + <4 0>, // AFIFM2_RDCTRL [1:0] S_AXI_HP0_FPD + <5 0>, // AFIFM2_WRCTRL [1:0] S_AXI_HP0_FPD + <6 0>, // AFIFM3_RDCTRL [1:0] S_AXI_HP1_FPD + <7 0>, // AFIFM3_WRCTRL [1:0] S_AXI_HP1_FPD + <8 0>, // AFIFM4_RDCTRL [1:0] S_AXI_HP2_FPD + <9 0>, // AFIFM4_WRCTRL [1:0] S_AXI_HP2_FPD + <10 0>, // AFIFM5_RDCTRL [1:0] S_AXI_HP3_FPD + <11 0>, // AFIFM5_WRCTRL [1:0] S_AXI_HP3_FPD + <12 0>, // AFIFM6_RDCTRL [1:0] S_AXI_LPD + <13 0>, // AFIFM6_WRCTRL [1:0] S_AXI_LPD + // PS-to-PL AXI + // 0: 32-bit, 1: 64-bit, 2: 128-bit + <14 0x000>, // FPD_SLCR [9:8] AFIFS0 M_AXI_HPM0_FPD [11:10] AFIFS1 M_AXI_HPM1_FPD + <15 0x000>; // LPD_SLCR [9:8] AFIFS2 M_AXI_HPM0_LPD + }; + + mqnic0: ethernet@a0000000 { + compatible = "corundum,mqnic"; + reg = <0x0 0xa0000000 0x0 0x1000000>, + <0x0 0xa8000000 0x0 0x1000000>; + reg-names = "csr", "app"; + interrupt-parent = <&gic>; + interrupts = <0x0 0x59 0x1>, <0x0 0x5a 0x1>, <0x0 0x5b 0x1>, + <0x0 0x5c 0x1>; + assigned-clocks = <&zynqmp_clk 71>, // PL0_REF + <&si570_2 0>; // MGT SI570 (U56) + assigned-clock-rates = <300000000>, + <156250000>; + resets = <&zynqmp_reset 116>; // ZYNQMP_RESET_PS_PL0 + reset-names = "reset"; + + nvmem-cells = <&macaddress>; + nvmem-cell-names = "mac-address"; + + /* NOTE: The nvmem-cells property provides us with a base MAC + * address. We increment its last byte (default) by 0x1. And we + * mark the derived address as "locally administrated". The + * result is used to derive MAC addresses for mqnic interfaces. + */ + mac-address-increment = <0x1>; + mac-address-local; + + module-eeproms = <&module_eeprom_sfp0>, <&module_eeprom_sfp1>; + }; + }; + }; + fragment@2 { + target = <&eeprom>; + __overlay__ { + #address-cells = <1>; + #size-cells = <1>; + + macaddress: macaddress@20 { + /* NOTE: On Xilinx Zynq boards there usually is an + * EEPROM with a MAC address for one of the PS GEMs at + * offset 0x20. So we take that address as our base + * address. + */ + reg = <0x20 0x06>; + }; + }; + }; + fragment@3 { + target = <&i2c1>; + __overlay__ { + i2c-mux@75 { + i2c@6 { + #address-cells = <1>; + #size-cells = <0>; + module_eeprom_sfp1: eeprom@50 { + compatible = "atmel,24c02"; + reg = <0x50>; + }; + }; + i2c@7 { + #address-cells = <1>; + #size-cells = <0>; + module_eeprom_sfp0: eeprom@50 { + compatible = "atmel,24c02"; + reg = <0x50>; + }; + }; + }; + }; + }; +};