treewide: Add narrow multicast support

Author: Lore0599

hw/reqrsp_interface/include/reqrsp_interface/typedef.svh

@@ -11,6 +11,7 @@
 `define REQRSP_TYPEDEF_REQ_CHAN_T(__req_chan_t, __addr_t, __data_t, __strb_t) \
   typedef struct packed { \
     __addr_t             addr;  \
+    __addr_t             mask;  \
     logic                write; \
     reqrsp_pkg::amo_op_e amo;   \
     __data_t             data;  \

hw/reqrsp_interface/src/axi_to_reqrsp.sv

@@ -303,7 +303,8 @@ module axi_to_reqrsp #(
     // Silence those channels in case of a read.
     data: data & {DataWidth{meta.write}},
     strb: axi_req_i.w.strb & {StrbWidth{meta.write}},
-    size: meta.size
+    size: meta.size,
+    default: '0
   };
 
   always_comb begin

hw/snitch/src/riscv_instr.sv

@@ -329,7 +329,6 @@ package riscv_instr;
   localparam logic [31:0] DMREP              = 32'b000011100000?????000000000101011;
   localparam logic [31:0] DMMCAST            = 32'b000100000000?????000000000101011;
   localparam logic [31:0] FREP_O             = 32'b????????????????????????10001011;
-  localparam logic [31:0] FREP_I             = 32'b????????????????????????00001011;
   localparam logic [31:0] IREP               = 32'b?????????????????????????0111111;
   localparam logic [31:0] SCFGRI             = 32'b????????????00000001?????0101011;
   localparam logic [31:0] SCFGWI             = 32'b?????????????????010000000101011;
@@ -1141,6 +1140,7 @@ package riscv_instr;
   localparam logic [11:0] CSR_FPMODE = 12'h7c1;
   localparam logic [11:0] CSR_BARRIER = 12'h7c2;
   localparam logic [11:0] CSR_SC = 12'h7c3;
+  localparam logic [11:0] CSR_MCAST = 12'h7c4;
   localparam logic [11:0] CSR_HTIMEDELTAH = 12'h615;
   localparam logic [11:0] CSR_CYCLEH = 12'hc80;
   localparam logic [11:0] CSR_TIMEH = 12'hc81;

hw/snitch/src/snitch.sv

@@ -282,6 +282,9 @@ module snitch import snitch_pkg::*; import riscv_instr::*; #(
   logic [31:0] dpc_d, dpc_q;
   logic [31:0] dscratch_d, dscratch_q;
   logic debug_d, debug_q;
+  
+  // Multicast mask
+  logic [31:0] csr_mcast_d, csr_mcast_q;
 
   `FFAR(scratch_q, scratch_d, '0, clk_i, rst_i)
   `FFAR(tvec_q, tvec_d, '0, clk_i, rst_i)
@@ -317,6 +320,7 @@ module snitch import snitch_pkg::*; import riscv_instr::*; #(
   end
 
   `FFAR(csr_stall_q, csr_stall_d, '0, clk_i, rst_i)
+  `FFAR(csr_mcast_q, csr_mcast_d, '0, clk_i, rst_i)
 
   typedef struct packed {
     fpnew_pkg::fmt_mode_t  fmode;
@@ -2353,6 +2357,7 @@ module snitch import snitch_pkg::*; import riscv_instr::*; #(
     dcsr_d = dcsr_q;
     dpc_d = dpc_q;
     dscratch_d = dscratch_q;
+    csr_mcast_d = csr_mcast_q;
 
     csr_stall_d = csr_stall_q;
 
@@ -2578,6 +2583,11 @@ module snitch import snitch_pkg::*; import riscv_instr::*; #(
             barrier_o = 1'b1;
             csr_stall_d = 1'b1;
           end
+          // Multicast mask
+          CSR_MCAST: begin
+            csr_rvalue = csr_mcast_q;
+            csr_mcast_d = alu_result[31:0];
+          end
           default: begin
             csr_rvalue = '0;
             csr_dump = 1'b1;
@@ -2898,6 +2908,7 @@ module snitch import snitch_pkg::*; import riscv_instr::*; #(
     .lsu_qsize_i (ls_size),
     .lsu_qamo_i (ls_amo),
     .lsu_qrepd_i (1'b0),
+    .lsu_qmcast_i (addr_t'(csr_mcast_q)),
     .lsu_qvalid_i (lsu_qvalid),
     .lsu_qready_o (lsu_qready),
     .lsu_pdata_o (ld_result),

hw/snitch/src/snitch_lsu.sv

@@ -52,6 +52,7 @@ module snitch_lsu #(
   input  logic [1:0]           lsu_qsize_i,
   input  reqrsp_pkg::amo_op_e  lsu_qamo_i,
   input  logic                 lsu_qrepd_i,  // Whether this is a sequencer repetition
+  input  addr_t                lsu_qmcast_i,  // Multicast mask
   input  logic                 lsu_qvalid_i,
   output logic                 lsu_qready_o,
   // response channel
@@ -250,6 +251,7 @@ module snitch_lsu #(
   assign data_req_o.q_valid = lsu_postcaq_qvalid & (lsu_qwrite_i | ~laq_full) & ~mem_full;
   assign data_req_o.q.write = lsu_qwrite_i;
   assign data_req_o.q.addr = lsu_qaddr_i;
+  assign data_req_o.q.mask = lsu_qmcast_i;
   assign data_req_o.q.amo  = lsu_qamo_i;
   assign data_req_o.q.size = lsu_qsize_i;
 

hw/snitch_cluster/src/snitch_cluster.sv

@@ -38,6 +38,8 @@ module snitch_cluster
   parameter int unsigned NarrowUserWidth    = 1,
   /// AXI: dma user width.
   parameter int unsigned WideUserWidth      = 1,
+  /// Width of the atomic ID to be used in a system.
+  parameter int unsigned AtomicIdWidth      = 1,
   /// Boot Address from which to fetch the first instructions.
   /// Used if `AliasRegionEnable` or `IntBootromEnable` is not set.
   parameter logic [31:0] BootAddr           = 32'h0,
@@ -1149,7 +1151,7 @@ module snitch_cluster
   user_t cluster_user;
   // Atomic ID, needs to be unique ID of cluster
   // cluster_id + HartIdOffset + 1 (because 0 is for non-atomic masters)
-  assign cluster_user = (hart_base_id_i / NrCores) +  (hart_base_id_i % NrCores) + 1'b1;
+  assign cluster_user = (core_to_axi_req.q.mask << AtomicIdWidth) | ((hart_base_id_i / NrCores) +  (hart_base_id_i % NrCores) + 1'b1);
 
   reqrsp_mux #(
     .NrPorts (NrCores),

hw/snitch_cluster/src/snitch_cluster_pkg.sv.tpl

@@ -58,6 +58,7 @@ package ${cfg['cluster']['name']}_pkg;
 
   localparam int unsigned NarrowUserWidth = ${cfg['cluster']['user_width']};
   localparam int unsigned WideUserWidth = ${cfg['cluster']['dma_user_width']};
+  localparam int unsigned AtomicIdWidth = ${cfg['cluster']['atomic_id_width']};
 
   localparam int unsigned ICacheLineWidth [NrHives] = '{${icache_cfg('cacheline')}};
   localparam int unsigned ICacheLineCount [NrHives] = '{${icache_cfg('depth')}};

hw/snitch_cluster/src/snitch_cluster_wrapper.sv.tpl

@@ -68,6 +68,7 @@ module ${cfg['cluster']['name']}_wrapper (
     .WideIdWidthIn (${cfg['cluster']['name']}_pkg::WideIdWidthIn),
     .NarrowUserWidth (${cfg['cluster']['name']}_pkg::NarrowUserWidth),
     .WideUserWidth (${cfg['cluster']['name']}_pkg::WideUserWidth),
+    .AtomicIdWidth (${cfg['cluster']['name']}_pkg::AtomicIdWidth),
     .BootAddr (${to_sv_hex(cfg['cluster']['boot_addr'], 32)}),
     .IntBootromEnable (${int(cfg['cluster']['int_bootrom_enable'])}),
     .narrow_in_req_t (${cfg['cluster']['name']}_pkg::narrow_in_req_t),

hw/snitch_cluster/src/snitch_fp_ss.sv

@@ -2709,6 +2709,7 @@ module snitch_fp_ss import snitch_pkg::*; #(
     .lsu_qsize_i (ls_size),
     .lsu_qamo_i (reqrsp_pkg::AMONone),
     .lsu_qrepd_i (acc_req_repd_q),
+    .lsu_qmcast_i ('0),
     .lsu_qvalid_i (lsu_qvalid),
     .lsu_qready_o (lsu_qready),
     .lsu_pdata_o (ld_result),

sw/deps/riscv-opcodes

@@ -30,3 +30,7 @@ inline void snrt_global_barrier();
 inline uint32_t snrt_global_all_to_all_reduction(uint32_t value);
 
 inline void snrt_wait_writeback(uint32_t val);
+
+inline void snrt_enable_multicast(uint32_t mask);
+
+inline void snrt_disable_multicast();

sw/snRuntime/api/sync_decls.h

@@ -98,6 +98,11 @@ inline uint32_t snrt_dma_start_1d_mcast_wideptr(uint64_t dst, uint64_t src,
                                      10, OP_CUSTOM1)),
                    "r"(reg_size));
 
+    // Reset dmmcast or next transfers will inherit this setting
+    asm volatile(".word %0\n" ::"i"(R_TYPE_ENCODE(DMMCAST_FUNCT7, 0b00000, 0,
+                                                  XDMA_FUNCT3, 0, OP_CUSTOM1)),
+                 "r"(reg_mcast));
+
     return reg_txid;
 }
 
@@ -472,6 +477,22 @@ inline snrt_dma_txid_t snrt_dma_load_1d_tile(void *dst, void *src,
     return snrt_dma_start_1d(dst, src + tile_idx * tile_nbytes, tile_nbytes);
 }
 
+/**
+ * @brief Load a tile of a 1D array.
+ * @param dst Pointer to the tile destination.
+ * @param src Pointer to the source array.
+ * @param tile_idx Index of the tile in the 1D array.
+ * @param tile_size Number of elements within a tile of the 1D array.
+ * @param prec Number of bytes of each element in the 1D array.
+ * @param mcast Multicast mask applied on the destination address.
+ */
+inline snrt_dma_txid_t snrt_dma_mcast_load_1d_tile(void *dst, void *src,
+                                             size_t tile_idx, size_t tile_size,
+                                             uint32_t prec, uint32_t mcast) {
+    size_t tile_nbytes = tile_size * prec;
+    return snrt_dma_start_1d_mcast(dst, src + tile_idx * tile_nbytes, tile_nbytes, mcast);
+}
+
 /**
  * @brief Store a tile to a 1D array.
  * @param dst Pointer to the destination array.

sw/snRuntime/src/dma.h

@@ -34,3 +34,7 @@ extern void snrt_global_reduction_dma(double *dst_buffer, double *src_buffer,
 extern uint32_t snrt_global_all_to_all_reduction(uint32_t value);
 
 extern void snrt_wait_writeback(uint32_t val);
+
+extern void snrt_enable_multicast(uint32_t mask);
+
+extern void snrt_disable_multicast();

sw/snRuntime/src/sync.c

@@ -12,6 +12,8 @@
 
 #pragma once
 
+#include "../../deps/riscv-opcodes/encoding.h"
+
 #include <math.h>
 
 //================================================================================
@@ -89,6 +91,35 @@ inline void snrt_cluster_hw_barrier() {
  *       will stall indefinitely.
  */
 inline void snrt_inter_cluster_barrier() {
+#ifdef SUPPORTS_MULTICAST
+    // Everyone increments a shared counter
+    uint32_t cnt =
+        __atomic_add_fetch(&(_snrt_barrier.cnt), 1, __ATOMIC_RELAXED);
+
+    // All but the last cluster enter WFI, while the last cluster resets the
+    // counter for the next barrier and multicasts an interrupt to wake up the
+    // other clusters.
+    if (cnt == snrt_cluster_num()) {
+        _snrt_barrier.cnt = 0;
+
+        // Multicast cluster interrupt to every other cluster's core
+        // Note: we need to address another cluster's address space
+        //       because the cluster XBAR has not been extended to support
+        //       multicast yet. We address the second cluster, if we are the
+        //       first cluster, and the second otherwise.
+        uintptr_t addr = (uintptr_t)snrt_cluster_clint_set_ptr() - SNRT_CLUSTER_OFFSET * snrt_cluster_idx();
+        if (snrt_cluster_idx() == 0) addr += SNRT_CLUSTER_OFFSET;
+        snrt_enable_multicast(BCAST_MASK_ALL);
+        *((uint32_t *)addr) = 1 << snrt_cluster_core_idx();
+        snrt_disable_multicast();
+        // Clear interrupt for next barrier
+        snrt_int_clr_mcip();
+    } else {
+        snrt_wfi();
+        // Clear interrupt for next barrier
+        snrt_int_clr_mcip();
+    }
+#else
     // Remember previous iteration
     uint32_t prev_barrier_iteration = _snrt_barrier.iteration;
     uint32_t cnt =
@@ -102,6 +133,7 @@ inline void snrt_inter_cluster_barrier() {
         while (prev_barrier_iteration == _snrt_barrier.iteration)
             ;
     }
+#endif
 }
 
 /**
@@ -119,6 +151,7 @@ inline void snrt_global_barrier() {
     // Synchronize all DM cores in software
     if (snrt_is_dm_core()) {
         snrt_inter_cluster_barrier();
+
     }
     // Synchronize cores in a cluster with the HW barrier
     snrt_cluster_hw_barrier();
@@ -264,3 +297,21 @@ inline void snrt_global_reduction_dma(double *dst_buffer, double *src_buffer,
 inline void snrt_wait_writeback(uint32_t val) {
     asm volatile("mv %0, %0" : "+r"(val)::);
 }
+
+//================================================================================
+// Multicast functions
+//================================================================================
+
+/**
+ * @brief Enable LSU multicast
+ * @details All stores performed after this call will be multicast to all
+ *          addresses specified by the address and mask pair.
+ *
+ * @param mask Multicast mask value
+ */
+inline void snrt_enable_multicast(uint32_t mask) { write_csr(0x7c3, mask); }
+
+/**
+ * @brief Disable LSU multicast
+ */
+inline void snrt_disable_multicast() { write_csr(0x7c3, 0); }

sw/snRuntime/src/sync.h

@@ -10,7 +10,8 @@
         cluster_base_hartid: 0,
         addr_width: 48,
         data_width: 64,
-        user_width: 5, // clog2(total number of clusters)
+        atomic_id_width: 5, // clog2(total number of clusters)
+        user_width: 53, // addr_width + atomic_id_width
         tcdm: {
             size: 128,
             banks: 32,

target/snitch_cluster/cfg/mcast.hjson

@@ -153,6 +153,11 @@
                     "description": "User width of the wide AXI plug into the cluster.",
                     "default": 1
                 },
+                "atomic_id_width": {
+                    "type": "number",
+                    "description": "Width of the cluster's atomics ID.",
+                    "default": 1
+                },  
                 "enable_multicast": {
                     "type": "boolean",
                     "description": "Whether to enable the mutlicast feature into the cluster",