Add QSPI flash access and IPROG for VCU118

fpga/mqnic/VCU118/fpga_100g/boot.xdc

@@ -0,0 +1,4 @@
+# Timing constraints for FPGA boot logic
+
+set_property ASYNC_REG TRUE [get_cells "fpga_boot_sync_reg_0_reg fpga_boot_sync_reg_1_reg"]
+set_false_path -to [get_pins "fpga_boot_sync_reg_0_reg/D"]

fpga/mqnic/VCU118/fpga_100g/fpga.xdc

@@ -243,4 +243,11 @@ set_property -dict {LOC AM17 IOSTANDARD LVCMOS18 PULLUP true} [get_ports pcie_re
 #create_clock -period 10 -name pcie_mgt_refclk_1 [get_ports pcie_refclk_1_p]
 create_clock -period 10 -name pcie_mgt_refclk_2 [get_ports pcie_refclk_2_p]
 
+# Flash
+set_property -dict {LOC AM19 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {qspi_1_dq[0]}]
+set_property -dict {LOC AM18 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {qspi_1_dq[1]}]
+set_property -dict {LOC AN20 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {qspi_1_dq[2]}]
+set_property -dict {LOC AP20 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {qspi_1_dq[3]}]
+set_property -dict {LOC BF16 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {qspi_1_cs}]
+
 

fpga/mqnic/VCU118/fpga_100g/fpga/Makefile

@@ -55,6 +55,7 @@ SYN_FILES += lib/pcie/rtl/pulse_merge.v
 
 # XDC files
 XDC_FILES = fpga.xdc
+XDC_FILES += boot.xdc
 XDC_FILES += lib/axis/syn/axis_async_fifo.tcl
 XDC_FILES += lib/axis/syn/sync_reset.tcl
 XDC_FILES += lib/eth/syn/ptp_clock_cdc.tcl

fpga/mqnic/VCU118/fpga_100g/rtl/fpga.v

@@ -134,7 +134,10 @@ module fpga (
     output wire         qsfp2_resetl,
     input  wire         qsfp2_modprsl,
     input  wire         qsfp2_intl,
-    output wire         qsfp2_lpmode
+    output wire         qsfp2_lpmode,
+
+    inout  wire [3:0]   qspi_1_dq,
+    output wire         qspi_1_cs
 );
 
 parameter AXIS_PCIE_DATA_WIDTH = 512;
@@ -306,6 +309,170 @@ sync_signal_inst (
 assign i2c_scl = i2c_scl_t ? 1'bz : i2c_scl_o;
 assign i2c_sda = i2c_sda_t ? 1'bz : i2c_sda_o;
 
+// Flash
+wire qspi_clk_int;
+wire [3:0] qspi_0_dq_int;
+wire [3:0] qspi_0_dq_i_int;
+wire [3:0] qspi_0_dq_o_int;
+wire [3:0] qspi_0_dq_oe_int;
+wire qspi_0_cs_int;
+wire [3:0] qspi_1_dq_i_int;
+wire [3:0] qspi_1_dq_o_int;
+wire [3:0] qspi_1_dq_oe_int;
+wire qspi_1_cs_int;
+
+assign qspi_1_dq[0] = qspi_1_dq_oe_int[0] ? qspi_1_dq_o_int[0] : 1'bz;
+assign qspi_1_dq[1] = qspi_1_dq_oe_int[1] ? qspi_1_dq_o_int[1] : 1'bz;
+assign qspi_1_dq[2] = qspi_1_dq_oe_int[2] ? qspi_1_dq_o_int[2] : 1'bz;
+assign qspi_1_dq[3] = qspi_1_dq_oe_int[3] ? qspi_1_dq_o_int[3] : 1'bz;
+assign qspi_1_cs = qspi_1_cs_int;
+
+sync_signal #(
+    .WIDTH(8),
+    .N(2)
+)
+flash_sync_signal_inst (
+    .clk(pcie_user_clk),
+    .in({qspi_1_dq, qspi_0_dq_int}),
+    .out({qspi_1_dq_i_int, qspi_0_dq_i_int})
+);
+
+STARTUPE3
+startupe3_inst (
+    .CFGCLK(),
+    .CFGMCLK(),
+    .DI(qspi_0_dq_int),
+    .DO(qspi_0_dq_o_int),
+    .DTS(~qspi_0_dq_oe_int),
+    .EOS(),
+    .FCSBO(qspi_0_cs_int),
+    .FCSBTS(1'b0),
+    .GSR(1'b0),
+    .GTS(1'b0),
+    .KEYCLEARB(1'b1),
+    .PACK(1'b0),
+    .PREQ(),
+    .USRCCLKO(qspi_clk_int),
+    .USRCCLKTS(1'b0),
+    .USRDONEO(1'b0),
+    .USRDONETS(1'b1)
+);
+
+// FPGA boot
+wire fpga_boot;
+
+reg fpga_boot_sync_reg_0 = 1'b0;
+reg fpga_boot_sync_reg_1 = 1'b0;
+reg fpga_boot_sync_reg_2 = 1'b0;
+
+wire icap_avail;
+reg [2:0] icap_state = 0;
+reg icap_csib_reg = 1'b1;
+reg icap_rdwrb_reg = 1'b0;
+reg [31:0] icap_di_reg = 32'hffffffff;
+
+wire [31:0] icap_di_rev;
+
+assign icap_di_rev[ 7] = icap_di_reg[ 0];
+assign icap_di_rev[ 6] = icap_di_reg[ 1];
+assign icap_di_rev[ 5] = icap_di_reg[ 2];
+assign icap_di_rev[ 4] = icap_di_reg[ 3];
+assign icap_di_rev[ 3] = icap_di_reg[ 4];
+assign icap_di_rev[ 2] = icap_di_reg[ 5];
+assign icap_di_rev[ 1] = icap_di_reg[ 6];
+assign icap_di_rev[ 0] = icap_di_reg[ 7];
+
+assign icap_di_rev[15] = icap_di_reg[ 8];
+assign icap_di_rev[14] = icap_di_reg[ 9];
+assign icap_di_rev[13] = icap_di_reg[10];
+assign icap_di_rev[12] = icap_di_reg[11];
+assign icap_di_rev[11] = icap_di_reg[12];
+assign icap_di_rev[10] = icap_di_reg[13];
+assign icap_di_rev[ 9] = icap_di_reg[14];
+assign icap_di_rev[ 8] = icap_di_reg[15];
+
+assign icap_di_rev[23] = icap_di_reg[16];
+assign icap_di_rev[22] = icap_di_reg[17];
+assign icap_di_rev[21] = icap_di_reg[18];
+assign icap_di_rev[20] = icap_di_reg[19];
+assign icap_di_rev[19] = icap_di_reg[20];
+assign icap_di_rev[18] = icap_di_reg[21];
+assign icap_di_rev[17] = icap_di_reg[22];
+assign icap_di_rev[16] = icap_di_reg[23];
+
+assign icap_di_rev[31] = icap_di_reg[24];
+assign icap_di_rev[30] = icap_di_reg[25];
+assign icap_di_rev[29] = icap_di_reg[26];
+assign icap_di_rev[28] = icap_di_reg[27];
+assign icap_di_rev[27] = icap_di_reg[28];
+assign icap_di_rev[26] = icap_di_reg[29];
+assign icap_di_rev[25] = icap_di_reg[30];
+assign icap_di_rev[24] = icap_di_reg[31];
+
+always @(posedge clk_125mhz_int) begin
+    case (icap_state)
+        0: begin
+            icap_state <= 0;
+            icap_csib_reg <= 1'b1;
+            icap_rdwrb_reg <= 1'b0;
+            icap_di_reg <= 32'hffffffff; // dummy word
+
+            if (fpga_boot_sync_reg_2 && icap_avail) begin
+                icap_state <= 1;
+                icap_csib_reg <= 1'b0;
+                icap_rdwrb_reg <= 1'b0;
+                icap_di_reg <= 32'hffffffff; // dummy word
+            end
+        end
+        1: begin
+            icap_state <= 2;
+            icap_csib_reg <= 1'b0;
+            icap_rdwrb_reg <= 1'b0;
+            icap_di_reg <= 32'hAA995566; // sync word
+        end
+        2: begin
+            icap_state <= 3;
+            icap_csib_reg <= 1'b0;
+            icap_rdwrb_reg <= 1'b0;
+            icap_di_reg <= 32'h20000000; // type 1 noop
+        end
+        3: begin
+            icap_state <= 4;
+            icap_csib_reg <= 1'b0;
+            icap_rdwrb_reg <= 1'b0;
+            icap_di_reg <= 32'h30008001; // write 1 word to CMD
+        end
+        4: begin
+            icap_state <= 5;
+            icap_csib_reg <= 1'b0;
+            icap_rdwrb_reg <= 1'b0;
+            icap_di_reg <= 32'h0000000F; // IPROG
+        end
+        5: begin
+            icap_state <= 0;
+            icap_csib_reg <= 1'b0;
+            icap_rdwrb_reg <= 1'b0;
+            icap_di_reg <= 32'h20000000; // type 1 noop
+        end
+    endcase
+
+    fpga_boot_sync_reg_0 <= fpga_boot;
+    fpga_boot_sync_reg_1 <= fpga_boot_sync_reg_0;
+    fpga_boot_sync_reg_2 <= fpga_boot_sync_reg_1;
+end
+
+ICAPE3
+icape3_inst (
+    .AVAIL(icap_avail),
+    .CLK(clk_125mhz_int),
+    .CSIB(icap_csib_reg),
+    .I(icap_di_rev),
+    .O(),
+    .PRDONE(),
+    .PRERROR(),
+    .RDWRB(icap_rdwrb_reg)
+);
+
 // PCIe
 wire pcie_sys_clk;
 wire pcie_sys_clk_gt;
@@ -1363,7 +1530,21 @@ core_inst (
     .qsfp2_modsell(qsfp2_modsell),
     .qsfp2_resetl(qsfp2_resetl),
     .qsfp2_intl(qsfp2_intl_int),
-    .qsfp2_lpmode(qsfp2_lpmode_int)
+    .qsfp2_lpmode(qsfp2_lpmode_int),
+
+    /*
+     * QSPI flash
+     */
+    .fpga_boot(fpga_boot),
+    .qspi_clk(qspi_clk_int),
+    .qspi_0_dq_i(qspi_0_dq_i_int),
+    .qspi_0_dq_o(qspi_0_dq_o_int),
+    .qspi_0_dq_oe(qspi_0_dq_oe_int),
+    .qspi_0_cs(qspi_0_cs_int),
+    .qspi_1_dq_i(qspi_1_dq_i_int),
+    .qspi_1_dq_o(qspi_1_dq_o_int),
+    .qspi_1_dq_oe(qspi_1_dq_oe_int),
+    .qspi_1_cs(qspi_1_cs_int)
 );
 
 endmodule

fpga/mqnic/VCU118/fpga_100g/rtl/fpga_core.v

@@ -213,7 +213,21 @@ module fpga_core #
     output wire                               qsfp2_resetl,
     input  wire                               qsfp2_modprsl,
     input  wire                               qsfp2_intl,
-    output wire                               qsfp2_lpmode
+    output wire                               qsfp2_lpmode,
+
+    /*
+     * QSPI flash
+     */
+    output wire                               fpga_boot,
+    output wire                               qspi_clk,
+    input  wire [3:0]                         qspi_0_dq_i,
+    output wire [3:0]                         qspi_0_dq_o,
+    output wire [3:0]                         qspi_0_dq_oe,
+    output wire                               qspi_0_cs,
+    input  wire [3:0]                         qspi_1_dq_i,
+    output wire [3:0]                         qspi_1_dq_o,
+    output wire [3:0]                         qspi_1_dq_oe,
+    output wire                               qspi_1_cs
 );
 
 // PHC parameters
@@ -436,6 +450,16 @@ reg qsfp2_lpmode_reg = 1'b0;
 reg i2c_scl_o_reg = 1'b1;
 reg i2c_sda_o_reg = 1'b1;
 
+reg fpga_boot_reg = 1'b0;
+
+reg qspi_clk_reg = 1'b0;
+reg qspi_0_cs_reg = 1'b1;
+reg [3:0] qspi_0_dq_o_reg = 4'd0;
+reg [3:0] qspi_0_dq_oe_reg = 4'd0;
+reg qspi_1_cs_reg = 1'b1;
+reg [3:0] qspi_1_dq_o_reg = 4'd0;
+reg [3:0] qspi_1_dq_oe_reg = 4'd0;
+
 reg pcie_dma_enable_reg = 0;
 
 reg [95:0] get_ptp_ts_96_reg = 0;
@@ -480,6 +504,16 @@ assign i2c_scl_t = i2c_scl_o_reg;
 assign i2c_sda_o = i2c_sda_o_reg;
 assign i2c_sda_t = i2c_sda_o_reg;
 
+assign fpga_boot = fpga_boot_reg;
+
+assign qspi_clk = qspi_clk_reg;
+assign qspi_0_cs = qspi_0_cs_reg;
+assign qspi_0_dq_o = qspi_0_dq_o_reg;
+assign qspi_0_dq_oe = qspi_0_dq_oe_reg;
+assign qspi_1_cs = qspi_1_cs_reg;
+assign qspi_1_dq_o = qspi_1_dq_o_reg;
+assign qspi_1_dq_oe = qspi_1_dq_oe_reg;
+
 //assign pcie_dma_enable = pcie_dma_enable_reg;
 
 always @(posedge clk_250mhz) begin
@@ -506,6 +540,10 @@ always @(posedge clk_250mhz) begin
         axil_csr_bvalid_reg <= 1'b1;
 
         case ({axil_csr_awaddr[15:2], 2'b00})
+            16'h0040: begin
+                // FPGA ID
+                fpga_boot_reg <= axil_csr_wdata == 32'hFEE1DEAD;
+            end
             // GPIO
             16'h0110: begin
                 // GPIO I2C 0
@@ -527,6 +565,33 @@ always @(posedge clk_250mhz) begin
                     qsfp2_lpmode_reg <= axil_csr_wdata[13];
                 end
             end
+            // Flash
+            16'h0144: begin
+                // QSPI 0 control
+                if (axil_csr_wstrb[0]) begin
+                    qspi_0_dq_o_reg <= axil_csr_wdata[3:0];
+                end
+                if (axil_csr_wstrb[1]) begin
+                    qspi_0_dq_oe_reg <= axil_csr_wdata[11:8];
+                end
+                if (axil_csr_wstrb[2]) begin
+                    qspi_clk_reg <= axil_csr_wdata[16];
+                    qspi_0_cs_reg <= axil_csr_wdata[17];
+                end
+            end
+            16'h0148: begin
+                // QSPI 1 control
+                if (axil_csr_wstrb[0]) begin
+                    qspi_1_dq_o_reg <= axil_csr_wdata[3:0];
+                end
+                if (axil_csr_wstrb[1]) begin
+                    qspi_1_dq_oe_reg <= axil_csr_wdata[11:8];
+                end
+                if (axil_csr_wstrb[2]) begin
+                    qspi_clk_reg <= axil_csr_wdata[16];
+                    qspi_1_cs_reg <= axil_csr_wdata[17];
+                end
+            end
             // PHC
             16'h0230: set_ptp_ts_96_reg[15:0] <= axil_csr_wdata;  // PTP set fns
             16'h0234: set_ptp_ts_96_reg[45:16] <= axil_csr_wdata; // PTP set ns
@@ -617,6 +682,22 @@ always @(posedge clk_250mhz) begin
                 axil_csr_rdata_reg[12] <= qsfp2_reset_reg;
                 axil_csr_rdata_reg[13] <= qsfp2_lpmode_reg;
             end
+            // Flash
+            16'h0140: axil_csr_rdata_reg <= {8'd2, 8'd8, 8'd2, 8'd0}; // Flash ID
+            16'h0144: begin
+                // QSPI 0 control
+                axil_csr_rdata_reg[3:0] <= qspi_0_dq_i;
+                axil_csr_rdata_reg[11:8] <= qspi_0_dq_oe;
+                axil_csr_rdata_reg[16] <= qspi_clk;
+                axil_csr_rdata_reg[17] <= qspi_0_cs;
+            end
+            16'h0148: begin
+                // QSPI 1 control
+                axil_csr_rdata_reg[3:0] <= qspi_1_dq_i;
+                axil_csr_rdata_reg[11:8] <= qspi_1_dq_oe;
+                axil_csr_rdata_reg[16] <= qspi_clk;
+                axil_csr_rdata_reg[17] <= qspi_1_cs;
+            end
             // PHC
             16'h0200: axil_csr_rdata_reg <= {8'd0, 8'd0, 8'd0, 8'd1};  // PHC features
             16'h0210: axil_csr_rdata_reg <= ptp_ts_96[15:0];  // PTP cur fns
@@ -683,6 +764,16 @@ always @(posedge clk_250mhz) begin
         i2c_scl_o_reg <= 1'b1;
         i2c_sda_o_reg <= 1'b1;
 
+        fpga_boot_reg <= 1'b0;
+
+        qspi_clk_reg <= 1'b0;
+        qspi_0_cs_reg <= 1'b1;
+        qspi_0_dq_o_reg <= 4'd0;
+        qspi_0_dq_oe_reg <= 4'd0;
+        qspi_1_cs_reg <= 1'b1;
+        qspi_1_dq_o_reg <= 4'd0;
+        qspi_1_dq_oe_reg <= 4'd0;
+
         pcie_dma_enable_reg <= 1'b0;
 
         ptp_perout_enable_reg <= 1'b0;

fpga/mqnic/VCU118/fpga_100g/tb/test_fpga_core.py

@@ -204,6 +204,8 @@ def bench():
     qsfp2_rx_axis_tuser = Signal(bool(0))
     qsfp2_modprsl = Signal(bool(1))
     qsfp2_intl = Signal(bool(1))
+    qspi_0_dq_i = Signal(intbv(0)[4:])
+    qspi_1_dq_i = Signal(intbv(0)[4:])
 
     # Outputs
     led = Signal(intbv(0)[8:])
@@ -258,6 +260,14 @@ def bench():
     qsfp2_modsell = Signal(bool(0))
     qsfp2_resetl = Signal(bool(0))
     qsfp2_lpmode = Signal(bool(0))
+    fpga_boot = Signal(bool(0))
+    qspi_clk = Signal(bool(0))
+    qspi_0_dq_o = Signal(intbv(0)[4:])
+    qspi_0_dq_oe = Signal(intbv(0)[4:])
+    qspi_0_cs = Signal(bool(1))
+    qspi_1_dq_o = Signal(intbv(0)[4:])
+    qspi_1_dq_oe = Signal(intbv(0)[4:])
+    qspi_1_cs = Signal(bool(1))
 
     # sources and sinks
     qsfp1_source = axis_ep.AXIStreamSource()
@@ -663,7 +673,17 @@ def bench():
         qsfp2_modsell=qsfp2_modsell,
         qsfp2_resetl=qsfp2_resetl,
         qsfp2_intl=qsfp2_intl,
-        qsfp2_lpmode=qsfp2_lpmode
+        qsfp2_lpmode=qsfp2_lpmode,
+        fpga_boot=fpga_boot,
+        qspi_clk=qspi_clk,
+        qspi_0_dq_i=qspi_0_dq_i,
+        qspi_0_dq_o=qspi_0_dq_o,
+        qspi_0_dq_oe=qspi_0_dq_oe,
+        qspi_0_cs=qspi_0_cs,
+        qspi_1_dq_i=qspi_1_dq_i,
+        qspi_1_dq_o=qspi_1_dq_o,
+        qspi_1_dq_oe=qspi_1_dq_oe,
+        qspi_1_cs=qspi_1_cs
     )
 
     @always(delay(5))

fpga/mqnic/VCU118/fpga_100g/tb/test_fpga_core.v

@@ -125,6 +125,8 @@ reg qsfp2_rx_axis_tlast = 0;
 reg qsfp2_rx_axis_tuser = 0;
 reg qsfp2_modprsl = 1;
 reg qsfp2_intl = 1;
+reg [3:0] qspi_0_dq_i = 0;
+reg [3:0] qspi_1_dq_i = 0;
 
 // Outputs
 wire [7:0] led;
@@ -179,6 +181,14 @@ wire qsfp2_tx_axis_tuser;
 wire qsfp2_modsell;
 wire qsfp2_resetl;
 wire qsfp2_lpmode;
+wire fpga_boot;
+wire qspi_clk;
+wire [3:0] qspi_0_dq_o;
+wire [3:0] qspi_0_dq_oe;
+wire qspi_0_cs;
+wire [3:0] qspi_1_dq_o;
+wire [3:0] qspi_1_dq_oe;
+wire qspi_1_cs;
 
 initial begin
     // myhdl integration
@@ -251,7 +261,9 @@ initial begin
         qsfp2_rx_axis_tlast,
         qsfp2_rx_axis_tuser,
         qsfp2_modprsl,
-        qsfp2_intl
+        qsfp2_intl,
+        qspi_0_dq_i,
+        qspi_1_dq_i
     );
     $to_myhdl(
         led,
@@ -305,7 +317,15 @@ initial begin
         qsfp2_tx_axis_tuser,
         qsfp2_modsell,
         qsfp2_resetl,
-        qsfp2_lpmode
+        qsfp2_lpmode,
+        fpga_boot,
+        qspi_clk,
+        qspi_0_dq_o,
+        qspi_0_dq_oe,
+        qspi_0_cs,
+        qspi_1_dq_o,
+        qspi_1_dq_oe,
+        qspi_1_cs
     );
 
     // dump file
@@ -445,7 +465,17 @@ UUT (
     .qsfp2_modsell(qsfp2_modsell),
     .qsfp2_resetl(qsfp2_resetl),
     .qsfp2_intl(qsfp2_intl_int),
-    .qsfp2_lpmode(qsfp2_lpmode_int)
+    .qsfp2_lpmode(qsfp2_lpmode_int),
+    .fpga_boot(fpga_boot),
+    .qspi_clk(qspi_clk),
+    .qspi_0_dq_i(qspi_0_dq_i),
+    .qspi_0_dq_o(qspi_0_dq_o),
+    .qspi_0_dq_oe(qspi_0_dq_oe),
+    .qspi_0_cs(qspi_0_cs),
+    .qspi_1_dq_i(qspi_1_dq_i),
+    .qspi_1_dq_o(qspi_1_dq_o),
+    .qspi_1_dq_oe(qspi_1_dq_oe),
+    .qspi_1_cs(qspi_1_cs)
 );
 
 endmodule

fpga/mqnic/VCU118/fpga_10g/boot.xdc

@@ -0,0 +1,4 @@
+# Timing constraints for FPGA boot logic
+
+set_property ASYNC_REG TRUE [get_cells "fpga_boot_sync_reg_0_reg fpga_boot_sync_reg_1_reg"]
+set_false_path -to [get_pins "fpga_boot_sync_reg_0_reg/D"]

fpga/mqnic/VCU118/fpga_10g/fpga.xdc

@@ -243,4 +243,11 @@ set_property -dict {LOC AM17 IOSTANDARD LVCMOS18 PULLUP true} [get_ports pcie_re
 #create_clock -period 10 -name pcie_mgt_refclk_1 [get_ports pcie_refclk_1_p]
 create_clock -period 10 -name pcie_mgt_refclk_2 [get_ports pcie_refclk_2_p]
 
+# Flash
+set_property -dict {LOC AM19 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {qspi_1_dq[0]}]
+set_property -dict {LOC AM18 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {qspi_1_dq[1]}]
+set_property -dict {LOC AN20 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {qspi_1_dq[2]}]
+set_property -dict {LOC AP20 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {qspi_1_dq[3]}]
+set_property -dict {LOC BF16 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {qspi_1_cs}]
+
 

fpga/mqnic/VCU118/fpga_10g/fpga/Makefile

@@ -72,6 +72,7 @@ SYN_FILES += lib/pcie/rtl/pulse_merge.v
 
 # XDC files
 XDC_FILES = fpga.xdc
+XDC_FILES += boot.xdc
 XDC_FILES += lib/axis/syn/axis_async_fifo.tcl
 XDC_FILES += lib/axis/syn/sync_reset.tcl
 XDC_FILES += lib/eth/syn/ptp_clock_cdc.tcl

fpga/mqnic/VCU118/fpga_10g/rtl/fpga.v

@@ -134,7 +134,10 @@ module fpga (
     output wire         qsfp2_resetl,
     input  wire         qsfp2_modprsl,
     input  wire         qsfp2_intl,
-    output wire         qsfp2_lpmode
+    output wire         qsfp2_lpmode,
+
+    inout  wire [3:0]   qspi_1_dq,
+    output wire         qspi_1_cs
 );
 
 parameter AXIS_PCIE_DATA_WIDTH = 512;
@@ -303,6 +306,170 @@ sync_signal_inst (
 assign i2c_scl = i2c_scl_t ? 1'bz : i2c_scl_o;
 assign i2c_sda = i2c_sda_t ? 1'bz : i2c_sda_o;
 
+// Flash
+wire qspi_clk_int;
+wire [3:0] qspi_0_dq_int;
+wire [3:0] qspi_0_dq_i_int;
+wire [3:0] qspi_0_dq_o_int;
+wire [3:0] qspi_0_dq_oe_int;
+wire qspi_0_cs_int;
+wire [3:0] qspi_1_dq_i_int;
+wire [3:0] qspi_1_dq_o_int;
+wire [3:0] qspi_1_dq_oe_int;
+wire qspi_1_cs_int;
+
+assign qspi_1_dq[0] = qspi_1_dq_oe_int[0] ? qspi_1_dq_o_int[0] : 1'bz;
+assign qspi_1_dq[1] = qspi_1_dq_oe_int[1] ? qspi_1_dq_o_int[1] : 1'bz;
+assign qspi_1_dq[2] = qspi_1_dq_oe_int[2] ? qspi_1_dq_o_int[2] : 1'bz;
+assign qspi_1_dq[3] = qspi_1_dq_oe_int[3] ? qspi_1_dq_o_int[3] : 1'bz;
+assign qspi_1_cs = qspi_1_cs_int;
+
+sync_signal #(
+    .WIDTH(8),
+    .N(2)
+)
+flash_sync_signal_inst (
+    .clk(pcie_user_clk),
+    .in({qspi_1_dq, qspi_0_dq_int}),
+    .out({qspi_1_dq_i_int, qspi_0_dq_i_int})
+);
+
+STARTUPE3
+startupe3_inst (
+    .CFGCLK(),
+    .CFGMCLK(),
+    .DI(qspi_0_dq_int),
+    .DO(qspi_0_dq_o_int),
+    .DTS(~qspi_0_dq_oe_int),
+    .EOS(),
+    .FCSBO(qspi_0_cs_int),
+    .FCSBTS(1'b0),
+    .GSR(1'b0),
+    .GTS(1'b0),
+    .KEYCLEARB(1'b1),
+    .PACK(1'b0),
+    .PREQ(),
+    .USRCCLKO(qspi_clk_int),
+    .USRCCLKTS(1'b0),
+    .USRDONEO(1'b0),
+    .USRDONETS(1'b1)
+);
+
+// FPGA boot
+wire fpga_boot;
+
+reg fpga_boot_sync_reg_0 = 1'b0;
+reg fpga_boot_sync_reg_1 = 1'b0;
+reg fpga_boot_sync_reg_2 = 1'b0;
+
+wire icap_avail;
+reg [2:0] icap_state = 0;
+reg icap_csib_reg = 1'b1;
+reg icap_rdwrb_reg = 1'b0;
+reg [31:0] icap_di_reg = 32'hffffffff;
+
+wire [31:0] icap_di_rev;
+
+assign icap_di_rev[ 7] = icap_di_reg[ 0];
+assign icap_di_rev[ 6] = icap_di_reg[ 1];
+assign icap_di_rev[ 5] = icap_di_reg[ 2];
+assign icap_di_rev[ 4] = icap_di_reg[ 3];
+assign icap_di_rev[ 3] = icap_di_reg[ 4];
+assign icap_di_rev[ 2] = icap_di_reg[ 5];
+assign icap_di_rev[ 1] = icap_di_reg[ 6];
+assign icap_di_rev[ 0] = icap_di_reg[ 7];
+
+assign icap_di_rev[15] = icap_di_reg[ 8];
+assign icap_di_rev[14] = icap_di_reg[ 9];
+assign icap_di_rev[13] = icap_di_reg[10];
+assign icap_di_rev[12] = icap_di_reg[11];
+assign icap_di_rev[11] = icap_di_reg[12];
+assign icap_di_rev[10] = icap_di_reg[13];
+assign icap_di_rev[ 9] = icap_di_reg[14];
+assign icap_di_rev[ 8] = icap_di_reg[15];
+
+assign icap_di_rev[23] = icap_di_reg[16];
+assign icap_di_rev[22] = icap_di_reg[17];
+assign icap_di_rev[21] = icap_di_reg[18];
+assign icap_di_rev[20] = icap_di_reg[19];
+assign icap_di_rev[19] = icap_di_reg[20];
+assign icap_di_rev[18] = icap_di_reg[21];
+assign icap_di_rev[17] = icap_di_reg[22];
+assign icap_di_rev[16] = icap_di_reg[23];
+
+assign icap_di_rev[31] = icap_di_reg[24];
+assign icap_di_rev[30] = icap_di_reg[25];
+assign icap_di_rev[29] = icap_di_reg[26];
+assign icap_di_rev[28] = icap_di_reg[27];
+assign icap_di_rev[27] = icap_di_reg[28];
+assign icap_di_rev[26] = icap_di_reg[29];
+assign icap_di_rev[25] = icap_di_reg[30];
+assign icap_di_rev[24] = icap_di_reg[31];
+
+always @(posedge clk_125mhz_int) begin
+    case (icap_state)
+        0: begin
+            icap_state <= 0;
+            icap_csib_reg <= 1'b1;
+            icap_rdwrb_reg <= 1'b0;
+            icap_di_reg <= 32'hffffffff; // dummy word
+
+            if (fpga_boot_sync_reg_2 && icap_avail) begin
+                icap_state <= 1;
+                icap_csib_reg <= 1'b0;
+                icap_rdwrb_reg <= 1'b0;
+                icap_di_reg <= 32'hffffffff; // dummy word
+            end
+        end
+        1: begin
+            icap_state <= 2;
+            icap_csib_reg <= 1'b0;
+            icap_rdwrb_reg <= 1'b0;
+            icap_di_reg <= 32'hAA995566; // sync word
+        end
+        2: begin
+            icap_state <= 3;
+            icap_csib_reg <= 1'b0;
+            icap_rdwrb_reg <= 1'b0;
+            icap_di_reg <= 32'h20000000; // type 1 noop
+        end
+        3: begin
+            icap_state <= 4;
+            icap_csib_reg <= 1'b0;
+            icap_rdwrb_reg <= 1'b0;
+            icap_di_reg <= 32'h30008001; // write 1 word to CMD
+        end
+        4: begin
+            icap_state <= 5;
+            icap_csib_reg <= 1'b0;
+            icap_rdwrb_reg <= 1'b0;
+            icap_di_reg <= 32'h0000000F; // IPROG
+        end
+        5: begin
+            icap_state <= 0;
+            icap_csib_reg <= 1'b0;
+            icap_rdwrb_reg <= 1'b0;
+            icap_di_reg <= 32'h20000000; // type 1 noop
+        end
+    endcase
+
+    fpga_boot_sync_reg_0 <= fpga_boot;
+    fpga_boot_sync_reg_1 <= fpga_boot_sync_reg_0;
+    fpga_boot_sync_reg_2 <= fpga_boot_sync_reg_1;
+end
+
+ICAPE3
+icape3_inst (
+    .AVAIL(icap_avail),
+    .CLK(clk_125mhz_int),
+    .CSIB(icap_csib_reg),
+    .I(icap_di_rev),
+    .O(),
+    .PRDONE(),
+    .PRERROR(),
+    .RDWRB(icap_rdwrb_reg)
+);
+
 // PCIe
 wire pcie_sys_clk;
 wire pcie_sys_clk_gt;
@@ -1400,7 +1567,21 @@ core_inst (
     .qsfp2_modsell(qsfp2_modsell),
     .qsfp2_resetl(qsfp2_resetl),
     .qsfp2_intl(qsfp2_intl_int),
-    .qsfp2_lpmode(qsfp2_lpmode_int)
+    .qsfp2_lpmode(qsfp2_lpmode_int),
+
+    /*
+     * QSPI flash
+     */
+    .fpga_boot(fpga_boot),
+    .qspi_clk(qspi_clk_int),
+    .qspi_0_dq_i(qspi_0_dq_i_int),
+    .qspi_0_dq_o(qspi_0_dq_o_int),
+    .qspi_0_dq_oe(qspi_0_dq_oe_int),
+    .qspi_0_cs(qspi_0_cs_int),
+    .qspi_1_dq_i(qspi_1_dq_i_int),
+    .qspi_1_dq_o(qspi_1_dq_o_int),
+    .qspi_1_dq_oe(qspi_1_dq_oe_int),
+    .qspi_1_cs(qspi_1_cs_int)
 );
 
 endmodule

fpga/mqnic/VCU118/fpga_10g/rtl/fpga_core.v

@@ -263,7 +263,21 @@ module fpga_core #
     output wire                               qsfp2_resetl,
     input  wire                               qsfp2_modprsl,
     input  wire                               qsfp2_intl,
-    output wire                               qsfp2_lpmode
+    output wire                               qsfp2_lpmode,
+
+    /*
+     * QSPI flash
+     */
+    output wire                               fpga_boot,
+    output wire                               qspi_clk,
+    input  wire [3:0]                         qspi_0_dq_i,
+    output wire [3:0]                         qspi_0_dq_o,
+    output wire [3:0]                         qspi_0_dq_oe,
+    output wire                               qspi_0_cs,
+    input  wire [3:0]                         qspi_1_dq_i,
+    output wire [3:0]                         qspi_1_dq_o,
+    output wire [3:0]                         qspi_1_dq_oe,
+    output wire                               qspi_1_cs
 );
 
 // PHC parameters
@@ -506,6 +520,16 @@ reg qsfp2_lpmode_reg = 1'b0;
 reg i2c_scl_o_reg = 1'b1;
 reg i2c_sda_o_reg = 1'b1;
 
+reg fpga_boot_reg = 1'b0;
+
+reg qspi_clk_reg = 1'b0;
+reg qspi_0_cs_reg = 1'b1;
+reg [3:0] qspi_0_dq_o_reg = 4'd0;
+reg [3:0] qspi_0_dq_oe_reg = 4'd0;
+reg qspi_1_cs_reg = 1'b1;
+reg [3:0] qspi_1_dq_o_reg = 4'd0;
+reg [3:0] qspi_1_dq_oe_reg = 4'd0;
+
 reg pcie_dma_enable_reg = 0;
 
 reg [95:0] get_ptp_ts_96_reg = 0;
@@ -550,6 +574,16 @@ assign i2c_scl_t = i2c_scl_o_reg;
 assign i2c_sda_o = i2c_sda_o_reg;
 assign i2c_sda_t = i2c_sda_o_reg;
 
+assign fpga_boot = fpga_boot_reg;
+
+assign qspi_clk = qspi_clk_reg;
+assign qspi_0_cs = qspi_0_cs_reg;
+assign qspi_0_dq_o = qspi_0_dq_o_reg;
+assign qspi_0_dq_oe = qspi_0_dq_oe_reg;
+assign qspi_1_cs = qspi_1_cs_reg;
+assign qspi_1_dq_o = qspi_1_dq_o_reg;
+assign qspi_1_dq_oe = qspi_1_dq_oe_reg;
+
 //assign pcie_dma_enable = pcie_dma_enable_reg;
 
 always @(posedge clk_250mhz) begin
@@ -576,6 +610,10 @@ always @(posedge clk_250mhz) begin
         axil_csr_bvalid_reg <= 1'b1;
 
         case ({axil_csr_awaddr[15:2], 2'b00})
+            16'h0040: begin
+                // FPGA ID
+                fpga_boot_reg <= axil_csr_wdata == 32'hFEE1DEAD;
+            end
             // GPIO
             16'h0110: begin
                 // GPIO I2C 0
@@ -597,6 +635,33 @@ always @(posedge clk_250mhz) begin
                     qsfp2_lpmode_reg <= axil_csr_wdata[13];
                 end
             end
+            // Flash
+            16'h0144: begin
+                // QSPI 0 control
+                if (axil_csr_wstrb[0]) begin
+                    qspi_0_dq_o_reg <= axil_csr_wdata[3:0];
+                end
+                if (axil_csr_wstrb[1]) begin
+                    qspi_0_dq_oe_reg <= axil_csr_wdata[11:8];
+                end
+                if (axil_csr_wstrb[2]) begin
+                    qspi_clk_reg <= axil_csr_wdata[16];
+                    qspi_0_cs_reg <= axil_csr_wdata[17];
+                end
+            end
+            16'h0148: begin
+                // QSPI 1 control
+                if (axil_csr_wstrb[0]) begin
+                    qspi_1_dq_o_reg <= axil_csr_wdata[3:0];
+                end
+                if (axil_csr_wstrb[1]) begin
+                    qspi_1_dq_oe_reg <= axil_csr_wdata[11:8];
+                end
+                if (axil_csr_wstrb[2]) begin
+                    qspi_clk_reg <= axil_csr_wdata[16];
+                    qspi_1_cs_reg <= axil_csr_wdata[17];
+                end
+            end
             // PHC
             16'h0230: set_ptp_ts_96_reg[15:0] <= axil_csr_wdata;  // PTP set fns
             16'h0234: set_ptp_ts_96_reg[45:16] <= axil_csr_wdata; // PTP set ns
@@ -687,6 +752,22 @@ always @(posedge clk_250mhz) begin
                 axil_csr_rdata_reg[12] <= qsfp2_reset_reg;
                 axil_csr_rdata_reg[13] <= qsfp2_lpmode_reg;
             end
+            // Flash
+            16'h0140: axil_csr_rdata_reg <= {8'd2, 8'd8, 8'd2, 8'd0}; // Flash ID
+            16'h0144: begin
+                // QSPI 0 control
+                axil_csr_rdata_reg[3:0] <= qspi_0_dq_i;
+                axil_csr_rdata_reg[11:8] <= qspi_0_dq_oe;
+                axil_csr_rdata_reg[16] <= qspi_clk;
+                axil_csr_rdata_reg[17] <= qspi_0_cs;
+            end
+            16'h0148: begin
+                // QSPI 1 control
+                axil_csr_rdata_reg[3:0] <= qspi_1_dq_i;
+                axil_csr_rdata_reg[11:8] <= qspi_1_dq_oe;
+                axil_csr_rdata_reg[16] <= qspi_clk;
+                axil_csr_rdata_reg[17] <= qspi_1_cs;
+            end
             // PHC
             16'h0200: axil_csr_rdata_reg <= {8'd0, 8'd0, 8'd0, 8'd1};  // PHC features
             16'h0210: axil_csr_rdata_reg <= ptp_ts_96[15:0];  // PTP cur fns
@@ -753,6 +834,16 @@ always @(posedge clk_250mhz) begin
         i2c_scl_o_reg <= 1'b1;
         i2c_sda_o_reg <= 1'b1;
 
+        fpga_boot_reg <= 1'b0;
+
+        qspi_clk_reg <= 1'b0;
+        qspi_0_cs_reg <= 1'b1;
+        qspi_0_dq_o_reg <= 4'd0;
+        qspi_0_dq_oe_reg <= 4'd0;
+        qspi_1_cs_reg <= 1'b1;
+        qspi_1_dq_o_reg <= 4'd0;
+        qspi_1_dq_oe_reg <= 4'd0;
+
         pcie_dma_enable_reg <= 1'b0;
 
         ptp_perout_enable_reg <= 1'b0;

fpga/mqnic/VCU118/fpga_10g/tb/test_fpga_core.py

@@ -239,6 +239,8 @@ def bench():
     qsfp2_rxc_4 = Signal(intbv(0)[8:])
     qsfp2_modprsl = Signal(bool(1))
     qsfp2_intl = Signal(bool(1))
+    qspi_0_dq_i = Signal(intbv(0)[4:])
+    qspi_1_dq_i = Signal(intbv(0)[4:])
 
     # Outputs
     led = Signal(intbv(0)[8:])
@@ -299,6 +301,14 @@ def bench():
     qsfp2_modsell = Signal(bool(0))
     qsfp2_resetl = Signal(bool(0))
     qsfp2_lpmode = Signal(bool(0))
+    fpga_boot = Signal(bool(0))
+    qspi_clk = Signal(bool(0))
+    qspi_0_dq_o = Signal(intbv(0)[4:])
+    qspi_0_dq_oe = Signal(intbv(0)[4:])
+    qspi_0_cs = Signal(bool(1))
+    qspi_1_dq_o = Signal(intbv(0)[4:])
+    qspi_1_dq_oe = Signal(intbv(0)[4:])
+    qspi_1_cs = Signal(bool(1))
 
     # sources and sinks
     qsfp1_1_source = xgmii_ep.XGMIISource()
@@ -724,7 +734,17 @@ def bench():
         qsfp2_modsell=qsfp2_modsell,
         qsfp2_resetl=qsfp2_resetl,
         qsfp2_intl=qsfp2_intl,
-        qsfp2_lpmode=qsfp2_lpmode
+        qsfp2_lpmode=qsfp2_lpmode,
+        fpga_boot=fpga_boot,
+        qspi_clk=qspi_clk,
+        qspi_0_dq_i=qspi_0_dq_i,
+        qspi_0_dq_o=qspi_0_dq_o,
+        qspi_0_dq_oe=qspi_0_dq_oe,
+        qspi_0_cs=qspi_0_cs,
+        qspi_1_dq_i=qspi_1_dq_i,
+        qspi_1_dq_o=qspi_1_dq_o,
+        qspi_1_dq_oe=qspi_1_dq_oe,
+        qspi_1_cs=qspi_1_cs
     )
 
     @always(delay(5))

fpga/mqnic/VCU118/fpga_10g/tb/test_fpga_core.v

@@ -151,6 +151,8 @@ reg [63:0] qsfp2_rxd_4 = 0;
 reg [7:0] qsfp2_rxc_4 = 0;
 reg qsfp2_modprsl = 1;
 reg qsfp2_intl = 1;
+reg [3:0] qspi_0_dq_i = 0;
+reg [3:0] qspi_1_dq_i = 0;
 
 // Outputs
 wire [7:0] led;
@@ -211,6 +213,14 @@ wire [7:0] qsfp2_txc_4;
 wire qsfp2_modsell;
 wire qsfp2_resetl;
 wire qsfp2_lpmode;
+wire fpga_boot;
+wire qspi_clk;
+wire [3:0] qspi_0_dq_o;
+wire [3:0] qspi_0_dq_oe;
+wire qspi_0_cs;
+wire [3:0] qspi_1_dq_o;
+wire [3:0] qspi_1_dq_oe;
+wire qspi_1_cs;
 
 initial begin
     // myhdl integration
@@ -311,7 +321,9 @@ initial begin
         qsfp2_rxd_4,
         qsfp2_rxc_4,
         qsfp2_modprsl,
-        qsfp2_intl
+        qsfp2_intl,
+        qspi_0_dq_i,
+        qspi_1_dq_i
     );
     $to_myhdl(
         led,
@@ -371,7 +383,15 @@ initial begin
         qsfp2_txc_4,
         qsfp2_modsell,
         qsfp2_resetl,
-        qsfp2_lpmode
+        qsfp2_lpmode,
+        fpga_boot,
+        qspi_clk,
+        qspi_0_dq_o,
+        qspi_0_dq_oe,
+        qspi_0_cs,
+        qspi_1_dq_o,
+        qspi_1_dq_oe,
+        qspi_1_cs
     );
 
     // dump file
@@ -543,7 +563,17 @@ UUT (
     .qsfp2_modsell(qsfp2_modsell),
     .qsfp2_resetl(qsfp2_resetl),
     .qsfp2_intl(qsfp2_intl_int),
-    .qsfp2_lpmode(qsfp2_lpmode_int)
+    .qsfp2_lpmode(qsfp2_lpmode_int),
+    .fpga_boot(fpga_boot),
+    .qspi_clk(qspi_clk),
+    .qspi_0_dq_i(qspi_0_dq_i),
+    .qspi_0_dq_o(qspi_0_dq_o),
+    .qspi_0_dq_oe(qspi_0_dq_oe),
+    .qspi_0_cs(qspi_0_cs),
+    .qspi_1_dq_i(qspi_1_dq_i),
+    .qspi_1_dq_o(qspi_1_dq_o),
+    .qspi_1_dq_oe(qspi_1_dq_oe),
+    .qspi_1_cs(qspi_1_cs)
 );
 
 endmodule

fpga/mqnic_tdma/VCU118/fpga_10g/boot.xdc

@@ -0,0 +1,4 @@
+# Timing constraints for FPGA boot logic
+
+set_property ASYNC_REG TRUE [get_cells "fpga_boot_sync_reg_0_reg fpga_boot_sync_reg_1_reg"]
+set_false_path -to [get_pins "fpga_boot_sync_reg_0_reg/D"]

fpga/mqnic_tdma/VCU118/fpga_10g/fpga.xdc

@@ -243,4 +243,11 @@ set_property -dict {LOC AM17 IOSTANDARD LVCMOS18 PULLUP true} [get_ports pcie_re
 #create_clock -period 10 -name pcie_mgt_refclk_1 [get_ports pcie_refclk_1_p]
 create_clock -period 10 -name pcie_mgt_refclk_2 [get_ports pcie_refclk_2_p]
 
+# Flash
+set_property -dict {LOC AM19 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {qspi_1_dq[0]}]
+set_property -dict {LOC AM18 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {qspi_1_dq[1]}]
+set_property -dict {LOC AN20 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {qspi_1_dq[2]}]
+set_property -dict {LOC AP20 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {qspi_1_dq[3]}]
+set_property -dict {LOC BF16 IOSTANDARD LVCMOS18 DRIVE 16} [get_ports {qspi_1_cs}]
+
 

fpga/mqnic_tdma/VCU118/fpga_10g/fpga/Makefile

@@ -73,6 +73,7 @@ SYN_FILES += lib/pcie/rtl/pulse_merge.v
 
 # XDC files
 XDC_FILES = fpga.xdc
+XDC_FILES += boot.xdc
 XDC_FILES += lib/axis/syn/axis_async_fifo.tcl
 XDC_FILES += lib/axis/syn/sync_reset.tcl
 XDC_FILES += lib/eth/syn/ptp_clock_cdc.tcl

fpga/mqnic_tdma/VCU118/fpga_10g/rtl/fpga.v

@@ -134,7 +134,10 @@ module fpga (
     output wire         qsfp2_resetl,
     input  wire         qsfp2_modprsl,
     input  wire         qsfp2_intl,
-    output wire         qsfp2_lpmode
+    output wire         qsfp2_lpmode,
+
+    inout  wire [3:0]   qspi_1_dq,
+    output wire         qspi_1_cs
 );
 
 parameter AXIS_PCIE_DATA_WIDTH = 512;
@@ -303,6 +306,170 @@ sync_signal_inst (
 assign i2c_scl = i2c_scl_t ? 1'bz : i2c_scl_o;
 assign i2c_sda = i2c_sda_t ? 1'bz : i2c_sda_o;
 
+// Flash
+wire qspi_clk_int;
+wire [3:0] qspi_0_dq_int;
+wire [3:0] qspi_0_dq_i_int;
+wire [3:0] qspi_0_dq_o_int;
+wire [3:0] qspi_0_dq_oe_int;
+wire qspi_0_cs_int;
+wire [3:0] qspi_1_dq_i_int;
+wire [3:0] qspi_1_dq_o_int;
+wire [3:0] qspi_1_dq_oe_int;
+wire qspi_1_cs_int;
+
+assign qspi_1_dq[0] = qspi_1_dq_oe_int[0] ? qspi_1_dq_o_int[0] : 1'bz;
+assign qspi_1_dq[1] = qspi_1_dq_oe_int[1] ? qspi_1_dq_o_int[1] : 1'bz;
+assign qspi_1_dq[2] = qspi_1_dq_oe_int[2] ? qspi_1_dq_o_int[2] : 1'bz;
+assign qspi_1_dq[3] = qspi_1_dq_oe_int[3] ? qspi_1_dq_o_int[3] : 1'bz;
+assign qspi_1_cs = qspi_1_cs_int;
+
+sync_signal #(
+    .WIDTH(8),
+    .N(2)
+)
+flash_sync_signal_inst (
+    .clk(pcie_user_clk),
+    .in({qspi_1_dq, qspi_0_dq_int}),
+    .out({qspi_1_dq_i_int, qspi_0_dq_i_int})
+);
+
+STARTUPE3
+startupe3_inst (
+    .CFGCLK(),
+    .CFGMCLK(),
+    .DI(qspi_0_dq_int),
+    .DO(qspi_0_dq_o_int),
+    .DTS(~qspi_0_dq_oe_int),
+    .EOS(),
+    .FCSBO(qspi_0_cs_int),
+    .FCSBTS(1'b0),
+    .GSR(1'b0),
+    .GTS(1'b0),
+    .KEYCLEARB(1'b1),
+    .PACK(1'b0),
+    .PREQ(),
+    .USRCCLKO(qspi_clk_int),
+    .USRCCLKTS(1'b0),
+    .USRDONEO(1'b0),
+    .USRDONETS(1'b1)
+);
+
+// FPGA boot
+wire fpga_boot;
+
+reg fpga_boot_sync_reg_0 = 1'b0;
+reg fpga_boot_sync_reg_1 = 1'b0;
+reg fpga_boot_sync_reg_2 = 1'b0;
+
+wire icap_avail;
+reg [2:0] icap_state = 0;
+reg icap_csib_reg = 1'b1;
+reg icap_rdwrb_reg = 1'b0;
+reg [31:0] icap_di_reg = 32'hffffffff;
+
+wire [31:0] icap_di_rev;
+
+assign icap_di_rev[ 7] = icap_di_reg[ 0];
+assign icap_di_rev[ 6] = icap_di_reg[ 1];
+assign icap_di_rev[ 5] = icap_di_reg[ 2];
+assign icap_di_rev[ 4] = icap_di_reg[ 3];
+assign icap_di_rev[ 3] = icap_di_reg[ 4];
+assign icap_di_rev[ 2] = icap_di_reg[ 5];
+assign icap_di_rev[ 1] = icap_di_reg[ 6];
+assign icap_di_rev[ 0] = icap_di_reg[ 7];
+
+assign icap_di_rev[15] = icap_di_reg[ 8];
+assign icap_di_rev[14] = icap_di_reg[ 9];
+assign icap_di_rev[13] = icap_di_reg[10];
+assign icap_di_rev[12] = icap_di_reg[11];
+assign icap_di_rev[11] = icap_di_reg[12];
+assign icap_di_rev[10] = icap_di_reg[13];
+assign icap_di_rev[ 9] = icap_di_reg[14];
+assign icap_di_rev[ 8] = icap_di_reg[15];
+
+assign icap_di_rev[23] = icap_di_reg[16];
+assign icap_di_rev[22] = icap_di_reg[17];
+assign icap_di_rev[21] = icap_di_reg[18];
+assign icap_di_rev[20] = icap_di_reg[19];
+assign icap_di_rev[19] = icap_di_reg[20];
+assign icap_di_rev[18] = icap_di_reg[21];
+assign icap_di_rev[17] = icap_di_reg[22];
+assign icap_di_rev[16] = icap_di_reg[23];
+
+assign icap_di_rev[31] = icap_di_reg[24];
+assign icap_di_rev[30] = icap_di_reg[25];
+assign icap_di_rev[29] = icap_di_reg[26];
+assign icap_di_rev[28] = icap_di_reg[27];
+assign icap_di_rev[27] = icap_di_reg[28];
+assign icap_di_rev[26] = icap_di_reg[29];
+assign icap_di_rev[25] = icap_di_reg[30];
+assign icap_di_rev[24] = icap_di_reg[31];
+
+always @(posedge clk_125mhz_int) begin
+    case (icap_state)
+        0: begin
+            icap_state <= 0;
+            icap_csib_reg <= 1'b1;
+            icap_rdwrb_reg <= 1'b0;
+            icap_di_reg <= 32'hffffffff; // dummy word
+
+            if (fpga_boot_sync_reg_2 && icap_avail) begin
+                icap_state <= 1;
+                icap_csib_reg <= 1'b0;
+                icap_rdwrb_reg <= 1'b0;
+                icap_di_reg <= 32'hffffffff; // dummy word
+            end
+        end
+        1: begin
+            icap_state <= 2;
+            icap_csib_reg <= 1'b0;
+            icap_rdwrb_reg <= 1'b0;
+            icap_di_reg <= 32'hAA995566; // sync word
+        end
+        2: begin
+            icap_state <= 3;
+            icap_csib_reg <= 1'b0;
+            icap_rdwrb_reg <= 1'b0;
+            icap_di_reg <= 32'h20000000; // type 1 noop
+        end
+        3: begin
+            icap_state <= 4;
+            icap_csib_reg <= 1'b0;
+            icap_rdwrb_reg <= 1'b0;
+            icap_di_reg <= 32'h30008001; // write 1 word to CMD
+        end
+        4: begin
+            icap_state <= 5;
+            icap_csib_reg <= 1'b0;
+            icap_rdwrb_reg <= 1'b0;
+            icap_di_reg <= 32'h0000000F; // IPROG
+        end
+        5: begin
+            icap_state <= 0;
+            icap_csib_reg <= 1'b0;
+            icap_rdwrb_reg <= 1'b0;
+            icap_di_reg <= 32'h20000000; // type 1 noop
+        end
+    endcase
+
+    fpga_boot_sync_reg_0 <= fpga_boot;
+    fpga_boot_sync_reg_1 <= fpga_boot_sync_reg_0;
+    fpga_boot_sync_reg_2 <= fpga_boot_sync_reg_1;
+end
+
+ICAPE3
+icape3_inst (
+    .AVAIL(icap_avail),
+    .CLK(clk_125mhz_int),
+    .CSIB(icap_csib_reg),
+    .I(icap_di_rev),
+    .O(),
+    .PRDONE(),
+    .PRERROR(),
+    .RDWRB(icap_rdwrb_reg)
+);
+
 // PCIe
 wire pcie_sys_clk;
 wire pcie_sys_clk_gt;
@@ -1400,7 +1567,21 @@ core_inst (
     .qsfp2_modsell(qsfp2_modsell),
     .qsfp2_resetl(qsfp2_resetl),
     .qsfp2_intl(qsfp2_intl_int),
-    .qsfp2_lpmode(qsfp2_lpmode_int)
+    .qsfp2_lpmode(qsfp2_lpmode_int),
+
+    /*
+     * QSPI flash
+     */
+    .fpga_boot(fpga_boot),
+    .qspi_clk(qspi_clk_int),
+    .qspi_0_dq_i(qspi_0_dq_i_int),
+    .qspi_0_dq_o(qspi_0_dq_o_int),
+    .qspi_0_dq_oe(qspi_0_dq_oe_int),
+    .qspi_0_cs(qspi_0_cs_int),
+    .qspi_1_dq_i(qspi_1_dq_i_int),
+    .qspi_1_dq_o(qspi_1_dq_o_int),
+    .qspi_1_dq_oe(qspi_1_dq_oe_int),
+    .qspi_1_cs(qspi_1_cs_int)
 );
 
 endmodule

fpga/mqnic_tdma/VCU118/fpga_10g/rtl/fpga_core.v

@@ -263,7 +263,21 @@ module fpga_core #
     output wire                               qsfp2_resetl,
     input  wire                               qsfp2_modprsl,
     input  wire                               qsfp2_intl,
-    output wire                               qsfp2_lpmode
+    output wire                               qsfp2_lpmode,
+
+    /*
+     * QSPI flash
+     */
+    output wire                               fpga_boot,
+    output wire                               qspi_clk,
+    input  wire [3:0]                         qspi_0_dq_i,
+    output wire [3:0]                         qspi_0_dq_o,
+    output wire [3:0]                         qspi_0_dq_oe,
+    output wire                               qspi_0_cs,
+    input  wire [3:0]                         qspi_1_dq_i,
+    output wire [3:0]                         qspi_1_dq_o,
+    output wire [3:0]                         qspi_1_dq_oe,
+    output wire                               qspi_1_cs
 );
 
 // PHC parameters
@@ -506,6 +520,16 @@ reg qsfp2_lpmode_reg = 1'b0;
 reg i2c_scl_o_reg = 1'b1;
 reg i2c_sda_o_reg = 1'b1;
 
+reg fpga_boot_reg = 1'b0;
+
+reg qspi_clk_reg = 1'b0;
+reg qspi_0_cs_reg = 1'b1;
+reg [3:0] qspi_0_dq_o_reg = 4'd0;
+reg [3:0] qspi_0_dq_oe_reg = 4'd0;
+reg qspi_1_cs_reg = 1'b1;
+reg [3:0] qspi_1_dq_o_reg = 4'd0;
+reg [3:0] qspi_1_dq_oe_reg = 4'd0;
+
 reg pcie_dma_enable_reg = 0;
 
 reg [95:0] get_ptp_ts_96_reg = 0;
@@ -550,6 +574,16 @@ assign i2c_scl_t = i2c_scl_o_reg;
 assign i2c_sda_o = i2c_sda_o_reg;
 assign i2c_sda_t = i2c_sda_o_reg;
 
+assign fpga_boot = fpga_boot_reg;
+
+assign qspi_clk = qspi_clk_reg;
+assign qspi_0_cs = qspi_0_cs_reg;
+assign qspi_0_dq_o = qspi_0_dq_o_reg;
+assign qspi_0_dq_oe = qspi_0_dq_oe_reg;
+assign qspi_1_cs = qspi_1_cs_reg;
+assign qspi_1_dq_o = qspi_1_dq_o_reg;
+assign qspi_1_dq_oe = qspi_1_dq_oe_reg;
+
 //assign pcie_dma_enable = pcie_dma_enable_reg;
 
 always @(posedge clk_250mhz) begin
@@ -576,6 +610,10 @@ always @(posedge clk_250mhz) begin
         axil_csr_bvalid_reg <= 1'b1;
 
         case ({axil_csr_awaddr[15:2], 2'b00})
+            16'h0040: begin
+                // FPGA ID
+                fpga_boot_reg <= axil_csr_wdata == 32'hFEE1DEAD;
+            end
             // GPIO
             16'h0110: begin
                 // GPIO I2C 0
@@ -597,6 +635,33 @@ always @(posedge clk_250mhz) begin
                     qsfp2_lpmode_reg <= axil_csr_wdata[13];
                 end
             end
+            // Flash
+            16'h0144: begin
+                // QSPI 0 control
+                if (axil_csr_wstrb[0]) begin
+                    qspi_0_dq_o_reg <= axil_csr_wdata[3:0];
+                end
+                if (axil_csr_wstrb[1]) begin
+                    qspi_0_dq_oe_reg <= axil_csr_wdata[11:8];
+                end
+                if (axil_csr_wstrb[2]) begin
+                    qspi_clk_reg <= axil_csr_wdata[16];
+                    qspi_0_cs_reg <= axil_csr_wdata[17];
+                end
+            end
+            16'h0148: begin
+                // QSPI 1 control
+                if (axil_csr_wstrb[0]) begin
+                    qspi_1_dq_o_reg <= axil_csr_wdata[3:0];
+                end
+                if (axil_csr_wstrb[1]) begin
+                    qspi_1_dq_oe_reg <= axil_csr_wdata[11:8];
+                end
+                if (axil_csr_wstrb[2]) begin
+                    qspi_clk_reg <= axil_csr_wdata[16];
+                    qspi_1_cs_reg <= axil_csr_wdata[17];
+                end
+            end
             // PHC
             16'h0230: set_ptp_ts_96_reg[15:0] <= axil_csr_wdata;  // PTP set fns
             16'h0234: set_ptp_ts_96_reg[45:16] <= axil_csr_wdata; // PTP set ns
@@ -687,6 +752,22 @@ always @(posedge clk_250mhz) begin
                 axil_csr_rdata_reg[12] <= qsfp2_reset_reg;
                 axil_csr_rdata_reg[13] <= qsfp2_lpmode_reg;
             end
+            // Flash
+            16'h0140: axil_csr_rdata_reg <= {8'd2, 8'd8, 8'd2, 8'd0}; // Flash ID
+            16'h0144: begin
+                // QSPI 0 control
+                axil_csr_rdata_reg[3:0] <= qspi_0_dq_i;
+                axil_csr_rdata_reg[11:8] <= qspi_0_dq_oe;
+                axil_csr_rdata_reg[16] <= qspi_clk;
+                axil_csr_rdata_reg[17] <= qspi_0_cs;
+            end
+            16'h0148: begin
+                // QSPI 1 control
+                axil_csr_rdata_reg[3:0] <= qspi_1_dq_i;
+                axil_csr_rdata_reg[11:8] <= qspi_1_dq_oe;
+                axil_csr_rdata_reg[16] <= qspi_clk;
+                axil_csr_rdata_reg[17] <= qspi_1_cs;
+            end
             // PHC
             16'h0200: axil_csr_rdata_reg <= {8'd0, 8'd0, 8'd0, 8'd1};  // PHC features
             16'h0210: axil_csr_rdata_reg <= ptp_ts_96[15:0];  // PTP cur fns
@@ -753,6 +834,16 @@ always @(posedge clk_250mhz) begin
         i2c_scl_o_reg <= 1'b1;
         i2c_sda_o_reg <= 1'b1;
 
+        fpga_boot_reg <= 1'b0;
+
+        qspi_clk_reg <= 1'b0;
+        qspi_0_cs_reg <= 1'b1;
+        qspi_0_dq_o_reg <= 4'd0;
+        qspi_0_dq_oe_reg <= 4'd0;
+        qspi_1_cs_reg <= 1'b1;
+        qspi_1_dq_o_reg <= 4'd0;
+        qspi_1_dq_oe_reg <= 4'd0;
+
         pcie_dma_enable_reg <= 1'b0;
 
         ptp_perout_enable_reg <= 1'b0;

fpga/mqnic_tdma/VCU118/fpga_10g/tb/test_fpga_core.py

@@ -240,6 +240,8 @@ def bench():
     qsfp2_rxc_4 = Signal(intbv(0)[8:])
     qsfp2_modprsl = Signal(bool(1))
     qsfp2_intl = Signal(bool(1))
+    qspi_0_dq_i = Signal(intbv(0)[4:])
+    qspi_1_dq_i = Signal(intbv(0)[4:])
 
     # Outputs
     led = Signal(intbv(0)[8:])
@@ -300,6 +302,14 @@ def bench():
     qsfp2_modsell = Signal(bool(0))
     qsfp2_resetl = Signal(bool(0))
     qsfp2_lpmode = Signal(bool(0))
+    fpga_boot = Signal(bool(0))
+    qspi_clk = Signal(bool(0))
+    qspi_0_dq_o = Signal(intbv(0)[4:])
+    qspi_0_dq_oe = Signal(intbv(0)[4:])
+    qspi_0_cs = Signal(bool(1))
+    qspi_1_dq_o = Signal(intbv(0)[4:])
+    qspi_1_dq_oe = Signal(intbv(0)[4:])
+    qspi_1_cs = Signal(bool(1))
 
     # sources and sinks
     qsfp1_1_source = xgmii_ep.XGMIISource()
@@ -725,7 +735,17 @@ def bench():
         qsfp2_modsell=qsfp2_modsell,
         qsfp2_resetl=qsfp2_resetl,
         qsfp2_intl=qsfp2_intl,
-        qsfp2_lpmode=qsfp2_lpmode
+        qsfp2_lpmode=qsfp2_lpmode,
+        fpga_boot=fpga_boot,
+        qspi_clk=qspi_clk,
+        qspi_0_dq_i=qspi_0_dq_i,
+        qspi_0_dq_o=qspi_0_dq_o,
+        qspi_0_dq_oe=qspi_0_dq_oe,
+        qspi_0_cs=qspi_0_cs,
+        qspi_1_dq_i=qspi_1_dq_i,
+        qspi_1_dq_o=qspi_1_dq_o,
+        qspi_1_dq_oe=qspi_1_dq_oe,
+        qspi_1_cs=qspi_1_cs
     )
 
     @always(delay(5))

fpga/mqnic_tdma/VCU118/fpga_10g/tb/test_fpga_core.v

@@ -151,6 +151,8 @@ reg [63:0] qsfp2_rxd_4 = 0;
 reg [7:0] qsfp2_rxc_4 = 0;
 reg qsfp2_modprsl = 1;
 reg qsfp2_intl = 1;
+reg [3:0] qspi_0_dq_i = 0;
+reg [3:0] qspi_1_dq_i = 0;
 
 // Outputs
 wire [7:0] led;
@@ -211,6 +213,14 @@ wire [7:0] qsfp2_txc_4;
 wire qsfp2_modsell;
 wire qsfp2_resetl;
 wire qsfp2_lpmode;
+wire fpga_boot;
+wire qspi_clk;
+wire [3:0] qspi_0_dq_o;
+wire [3:0] qspi_0_dq_oe;
+wire qspi_0_cs;
+wire [3:0] qspi_1_dq_o;
+wire [3:0] qspi_1_dq_oe;
+wire qspi_1_cs;
 
 initial begin
     // myhdl integration
@@ -311,7 +321,9 @@ initial begin
         qsfp2_rxd_4,
         qsfp2_rxc_4,
         qsfp2_modprsl,
-        qsfp2_intl
+        qsfp2_intl,
+        qspi_0_dq_i,
+        qspi_1_dq_i
     );
     $to_myhdl(
         led,
@@ -371,7 +383,15 @@ initial begin
         qsfp2_txc_4,
         qsfp2_modsell,
         qsfp2_resetl,
-        qsfp2_lpmode
+        qsfp2_lpmode,
+        fpga_boot,
+        qspi_clk,
+        qspi_0_dq_o,
+        qspi_0_dq_oe,
+        qspi_0_cs,
+        qspi_1_dq_o,
+        qspi_1_dq_oe,
+        qspi_1_cs
     );
 
     // dump file
@@ -543,7 +563,17 @@ UUT (
     .qsfp2_modsell(qsfp2_modsell),
     .qsfp2_resetl(qsfp2_resetl),
     .qsfp2_intl(qsfp2_intl_int),
-    .qsfp2_lpmode(qsfp2_lpmode_int)
+    .qsfp2_lpmode(qsfp2_lpmode_int),
+    .fpga_boot(fpga_boot),
+    .qspi_clk(qspi_clk),
+    .qspi_0_dq_i(qspi_0_dq_i),
+    .qspi_0_dq_o(qspi_0_dq_o),
+    .qspi_0_dq_oe(qspi_0_dq_oe),
+    .qspi_0_cs(qspi_0_cs),
+    .qspi_1_dq_i(qspi_1_dq_i),
+    .qspi_1_dq_o(qspi_1_dq_o),
+    .qspi_1_dq_oe(qspi_1_dq_oe),
+    .qspi_1_cs(qspi_1_cs)
 );
 
 endmodule