Drain outstanding I$ traffic before completing a flush

A flush could invalidate the L0 while pre-flush refill or prefetch
responses were still in flight. Those old responses could then be
accepted after the flush and repopulate the cache with stale
pre-flush state.

Fix this by turning the shared I$ flush into a two-phase operation.
Mark each L0 as flush-pending immediately so it stops accepting hits
and issuing new refill or prefetch requests, then wait for the shared
handler and all L0-side state to drain before firing the actual
invalidate and completing the flush handshake.

Relax the stalled-fetch assertions so a pending flush may legally
withdraw a blocked fetch request.

Author: Scheremo

src/snitch_icache.sv

@@ -228,6 +228,16 @@ module snitch_icache
   logic [NR_FETCH_PORTS-1:0] in_cache_ready, in_bypass_ready;
   logic [NR_FETCH_PORTS-1:0][FETCH_DW-1:0] in_cache_data, in_bypass_data;
   logic [NR_FETCH_PORTS-1:0] in_cache_error, in_bypass_error;
+  logic [NR_FETCH_PORTS-1:0] l0_flush_ready;
+  logic                      handler_idle;
+
+  logic flush_req;
+  logic flush_busy_d, flush_busy_q;
+  logic flush_shared_done_d, flush_shared_done_q;
+  logic flush_wait_drop_d, flush_wait_drop_q;
+  logic flush_l0_fire, flush_l0_pending;
+  logic flush_req_valid_lookup, flush_req_ready_lookup, flush_fire_lookup;
+  logic l0_flush_ready_all, shared_cache_idle;
 
   for (genvar i = 0; i < NR_FETCH_PORTS; i++) begin : gen_prefetcher
     prefetch_req_t                     local_prefetch_req;
@@ -258,7 +268,9 @@ module snitch_icache
     ) i_snitch_icache_l0 (
       .clk_i          (clk_d2_i),
       .rst_ni,
-      .flush_valid_i  (flush_valid_i[i]),
+      .flush_pending_i(flush_l0_pending),
+      .flush_valid_i  (flush_l0_fire),
+      .flush_ready_o  (l0_flush_ready[i]),
       .enable_prefetching_i,
       .enable_branch_pred_i,
       .icache_events_o(icache_l0_events_o[i]),
@@ -516,30 +528,62 @@ module snitch_icache
   logic                       write_valid;
   logic                       write_ready;
 
-  logic flush_valid, flush_ready;
-  logic flush_valid_lookup, flush_ready_lookup;
+  logic flush_ready_lookup_unused;
+
+  assign flush_req          = |flush_valid_i;
+  assign l0_flush_ready_all = &l0_flush_ready;
+  // Hold the L0s in a "flush pending" state immediately so they stop creating new work while
+  // the shared cache and any already-issued responses drain.
+  assign flush_l0_pending   = flush_busy_q | (flush_req & ~flush_wait_drop_q);
+  // Only invalidate the L0s after the shared cache has flushed and every port has drained its
+  // old responses, so no pre-flush traffic can repopulate an L0 afterwards.
+  assign flush_l0_fire      = flush_busy_q & flush_shared_done_q & l0_flush_ready_all;
+  assign flush_ready_o      = {CFG.NR_FETCH_PORTS{flush_l0_fire}};
+  assign shared_cache_idle  = ~(|prefetch_req_valid) & ~prefetch_lookup_req_valid &
+                              ~lookup_valid & ~handler_req_valid & handler_idle;
+  assign flush_fire_lookup  = flush_req_valid_lookup & shared_cache_idle;
+
+  always_comb begin
+    flush_busy_d        = flush_busy_q;
+    flush_shared_done_d = flush_shared_done_q;
+    flush_wait_drop_d   = flush_wait_drop_q;
+
+    if (flush_wait_drop_q) begin
+      if (!flush_req) flush_wait_drop_d = 1'b0;
+    end else if (!flush_busy_q) begin
+      if (flush_req) begin
+        flush_busy_d        = 1'b1;
+        flush_shared_done_d = 1'b0;
+      end
+    end else begin
+      if (!flush_shared_done_q && flush_req_ready_lookup) flush_shared_done_d = 1'b1;
+      if (flush_l0_fire) begin
+        flush_busy_d      = 1'b0;
+        flush_wait_drop_d = 1'b1;
+      end
+    end
+  end
 
-  assign flush_ready_o = {CFG.NR_FETCH_PORTS{flush_ready}};
-  assign flush_valid   = |flush_valid_i;
+  `FF(flush_busy_q, flush_busy_d, '0, clk_d2_i, rst_ni)
+  `FF(flush_shared_done_q, flush_shared_done_d, '0, clk_d2_i, rst_ni)
+  `FF(flush_wait_drop_q, flush_wait_drop_d, '0, clk_d2_i, rst_ni)
 
-  // We need to propagate the handshake into the other
+  // We need to propagate the shared-cache flush request into the other
   // clock domain in case we operate w/ different clocks.
   if (ISO_CROSSING) begin : gen_flush_crossing
-    isochronous_spill_register i_isochronous_4phase_handshake (
+    isochronous_4phase_handshake i_isochronous_4phase_handshake (
       .src_clk_i  (clk_d2_i),
       .src_rst_ni (rst_ni),
-      .src_valid_i(flush_valid),
-      .src_ready_o(flush_ready),
-      .src_data_i ('0),
+      .src_valid_i(flush_busy_q & ~flush_shared_done_q),
+      .src_ready_o(flush_req_ready_lookup),
       .dst_clk_i  (clk_i),
       .dst_rst_ni (rst_ni),
-      .dst_valid_o(flush_valid_lookup),
-      .dst_ready_i(flush_ready_lookup),
-      .dst_data_o (  /* Unused */)
+      .dst_valid_o(flush_req_valid_lookup),
+      .dst_ready_i(shared_cache_idle)
     );
   end else begin : gen_no_flush_crossing
-    assign flush_valid_lookup = flush_valid;
-    assign flush_ready        = flush_ready_lookup;
+    assign flush_req_valid_lookup = flush_busy_q & ~flush_shared_done_q;
+    assign flush_req_ready_lookup = shared_cache_idle;
   end
 
   if (SERIAL_LOOKUP) begin : gen_serial_lookup
@@ -553,8 +597,8 @@ module snitch_icache
       .clk_i,
       .rst_ni,
 
-      .flush_valid_i  (flush_valid_lookup),
-      .flush_ready_o  (flush_ready_lookup),
+      .flush_valid_i  (flush_fire_lookup),
+      .flush_ready_o  (flush_ready_lookup_unused),
       .icache_events_o(icache_l1_events_o),
 
       .in_addr_i (prefetch_lookup_req.addr),
@@ -596,8 +640,8 @@ module snitch_icache
       .clk_i,
       .rst_ni,
 
-      .flush_valid_i  (flush_valid_lookup),
-      .flush_ready_o  (flush_ready_lookup),
+      .flush_valid_i  (flush_fire_lookup),
+      .flush_ready_o  (flush_ready_lookup_unused),
       .icache_events_o(icache_l1_events_o),
 
       .in_addr_i (prefetch_lookup_req.addr),
@@ -651,6 +695,7 @@ module snitch_icache
     .in_rsp_id_o   (prefetch_lookup_rsp.id),
     .in_rsp_valid_o(prefetch_lookup_rsp_valid),
     .in_rsp_ready_i(prefetch_lookup_rsp_ready),
+    .idle_o        (handler_idle),
 
     .write_addr_o (write_addr),
     .write_way_o  (write_way),

src/snitch_icache_handler.sv

@@ -29,6 +29,8 @@ module snitch_icache_handler
   output logic                      in_rsp_valid_o,
   input  logic                      in_rsp_ready_i,
 
+  output logic                      idle_o,
+
   output logic [CFG.COUNT_ALIGN-1:0] write_addr_o,
   output logic [  CFG.WAY_ALIGN-1:0] write_way_o,
   output logic [ CFG.LINE_WIDTH-1:0] write_data_o,
@@ -350,4 +352,11 @@ module snitch_icache_handler
     end
   end
 
+  always_comb begin
+    idle_o = ~arb_q.lock & ~write_served_q & ~in_rsp_served_q;
+    for (int i = 0; i < CFG.PENDING_COUNT; i++) begin
+      idle_o &= ~pending_q[i].valid;
+    end
+  end
+
 endmodule

src/snitch_icache_l0.sv

@@ -17,7 +17,9 @@ module snitch_icache_l0
 ) (
   input logic clk_i,
   input logic rst_ni,
+  input logic flush_pending_i,
   input logic flush_valid_i,
+  output logic flush_ready_o,
 
   input logic enable_prefetching_i,
   input logic enable_branch_pred_i,
@@ -104,6 +106,9 @@ module snitch_icache_l0
   logic refill_ready, prefetch_ready;
   req_t out_req;
 
+  // The local flush can fire once no queued request or in-flight L0 response remains.
+  assign flush_ready_o = ~pending_refill_q & ~pending_prefetch_q & ~prefetch_req_vld_q;
+
   assign evict_because_miss = miss & ~last_cycle_was_miss_q;
   assign evict_because_prefetch = latch_prefetch & ~last_cycle_was_prefetch_q;
   assign incoming_rsp_is_prefetch = (out_rsp_id_i == (1'b1 << {L0_ID, 1'b1}));
@@ -191,7 +196,7 @@ module snitch_icache_l0
   // HIT
   // ----
   // we hit in the cache and there was a unique hit.
-  assign in_ready_o = hit_any;
+  assign in_ready_o = hit_any & ~flush_pending_i;
 
   logic [minwidth(CFG.LINE_WIDTH,32)-1:0] ins_data;
   always_comb begin : data_muxer
@@ -258,7 +263,7 @@ module snitch_icache_l0
   // -------------
   assign refill.addr        = addr_tag << CFG.LINE_ALIGN;
   assign refill.is_prefetch = 1'b0;
-  assign refill_valid       = miss & ~pending_prefetch_q;
+  assign refill_valid       = miss & ~pending_prefetch_q & ~flush_pending_i;
 
   `FFL(pending_line_refill_q, evict_strb, evict_because_miss, '0, clk_i, rst_ni)
   `FFL(pending_line_prefetch_q, evict_strb, evict_because_prefetch, '0, clk_i, rst_ni)
@@ -324,7 +329,7 @@ module snitch_icache_l0
   // -------------
   // Generate a prefetch request if the cache hits and we haven't
   // pre-fetched the line yet and there is no other refill in progress.
-  assign prefetcher_out.vld = enable_prefetching_i &
+  assign prefetcher_out.vld = enable_prefetching_i & ~flush_pending_i &
                               hit_any & ~hit_prefetch_any &
                               hit_early_is_onehot & ~pending_prefetch_q;
 
@@ -431,18 +436,22 @@ module snitch_icache_l0
     prefetch_req_vld_d  = prefetch_req_vld_q;
     prefetch_req_addr_d = prefetch_req_addr_q;
 
-    if (prefetch_ready) prefetch_req_vld_d = 1'b0;
+    if (flush_pending_i) begin
+      prefetch_req_vld_d = 1'b0;
+    end else begin
+      if (prefetch_ready) prefetch_req_vld_d = 1'b0;
 
-    if (latch_prefetch) begin
-      prefetch_req_vld_d  = 1'b1;
-      prefetch_req_addr_d = prefetcher_out.addr;
+      if (latch_prefetch) begin
+        prefetch_req_vld_d  = 1'b1;
+        prefetch_req_addr_d = prefetcher_out.addr;
+      end
     end
   end
 
   assign addr_tag_prefetch_req = CFG.L0_TAG_WIDTH'(prefetch_req_addr_q >> CFG.LINE_ALIGN);
   assign prefetch.is_prefetch  = 1'b1;
   assign prefetch.addr         = prefetch_req_addr_q;
-  assign prefetch_valid        = prefetch_req_vld_q;
+  assign prefetch_valid        = prefetch_req_vld_q & ~flush_pending_i;
 
   `FF(prefetch_req_vld_q, prefetch_req_vld_d, '0)
   `FF(prefetch_req_addr_q, prefetch_req_addr_d, '0)
@@ -466,8 +475,8 @@ module snitch_icache_l0
   // make sure only one signal is high and the conditions are mutual exclusive
   `ASSERT(ExclusiveEvict, $onehot0({evict_because_miss, evict_because_prefetch}))
   // request must be stable
-  `ASSERT(InstReqStable, in_valid_i && !in_ready_o |=> in_valid_i)
-  `ASSERT(InstReqDataStable, in_valid_i && !in_ready_o |=> $stable(in_addr_i))
+  `ASSERT(InstReqStable, in_valid_i && !in_ready_o |=> (in_valid_i || flush_pending_i))
+  `ASSERT(InstReqDataStable, in_valid_i && !in_ready_o |=> ($stable(in_addr_i) || flush_pending_i))
 
   `ASSERT(RefillReqStable, out_req_valid_o && !out_req_ready_i |=> out_req_valid_o)
   `ASSERT(RefillReqDataStable, out_req_valid_o && !out_req_ready_i |=> $stable(out_req_addr_o)

src/snitch_read_only_cache.sv

@@ -393,6 +393,7 @@ module snitch_read_only_cache
     .in_rsp_id_o   (in_rsp_id),
     .in_rsp_valid_o(in_rsp_valid),
     .in_rsp_ready_i(in_rsp_ready),
+    .idle_o        (),
 
     .write_addr_o (write_addr),
     .write_way_o  (write_way),

test/snitch_icache_l0_tb.sv

@@ -151,6 +151,8 @@ module snitch_icache_l0_tb #(
   ) dut (
     .clk_i               (clk),
     .rst_ni              (~rst),
+    .flush_pending_i     (dut_flush_valid),
+    .flush_ready_o       (),
     .enable_prefetching_i(1'b1),
     .enable_branch_pred_i(1'b1),
     .icache_events_o     (),