P4 cleanup and a return to 18 stages

Author: zeeshanlakhani

.github/buildomat/common.sh

@@ -1,14 +1,11 @@
 #!/bin/bash
 
-# The tofino2 has 20 stages, and the current sidecar.p4 needs all 20 of them.
-# Specifying the number of stages isn't strictly necessary, but it allows us to
-# track when we exceed the current ceiling. The underlying intention is to grow
-# deliberately and thoughtfully, given the limited space on the ASIC.
-#
-# Note: this now seems silly since we have maxed out the number of stages, but
-# we want to leave this check and note in place should we ever find a way to
-# reduce our footprint below 20 stages.
-TOFINO_STAGES=20
+# The tofino2 has 20 stages. With multicast as a default feature, sidecar.p4
+# needs 18 stages. Specifying the number of stages isn't strictly necessary,
+# but it allows us to track when we exceed the current ceiling. The underlying
+# intention is to grow deliberately and thoughtfully, given the limited space
+# on the ASIC.
+TOFINO_STAGES=18
 
 # These describe which version of the SDE to download and where to find it
 SDE_COMMIT=53519b8cf74fe832cc7838ea92683564ce4026f2

.github/buildomat/packet-test-common.sh

@@ -3,16 +3,16 @@ export RUST_BACKTRACE=1
 source .github/buildomat/common.sh
 source .github/buildomat/linux.sh
 
-wd=`pwd`
+wd=$(pwd)
 export WS=$wd
 MODEL_STARTUP_TIMEOUT=${MODEL_STARTUP_TIMEOUT:=5}
 STARTUP_TIMEOUT=${STARTUP_TIMEOUT:=120}
 
 BUILD_FEATURES=tofino_asic
 
 CODEGEN_FEATURES=--multicast
-SWADM_FEATURES="--features=multicast"
 
+TOFINO_STAGES=18
 function cleanup {
     set +o errexit
     set +o pipefail
@@ -62,29 +62,33 @@ fi
 
 banner "Test"
 sudo -E ./tools/veth_setup.sh
-id=`id -un`
-gr=`id -gn`
+id=$(id -un)
+gr=$(id -gn)
 sudo -E mkdir -p /work
 sudo -E chown $id:$gr /work
-sudo -E ./tools/run_tofino_model.sh &> /work/simulator.log &
+sudo -E ./tools/run_tofino_model.sh &>/work/simulator.log </dev/null &
+stty sane 2>/dev/null || true
 sleep $MODEL_STARTUP_TIMEOUT
-sudo -E ./tools/run_dpd.sh -m 127.0.0.1 &> /work/dpd.log &
+sudo -E ./tools/run_dpd.sh -m 127.0.0.1 &>/work/dpd.log </dev/null &
+stty sane 2>/dev/null || true
 echo "waiting for dpd to come online"
 set +o errexit
 
 SLEEP_TIME=5
-iters=$(( $STARTUP_TIMEOUT / $SLEEP_TIME ))
-while [ 1 ] ; do
-	./target/debug/swadm --host '[::1]' build-info 2> /dev/null
-	if [ $? == 0 ]; then
-		break
-	fi
-	iters=$(($iters - 1))
-	if [ $iters = 0 ]; then
-		echo "dpd failed to come online in $STARTUP_TIMEOUT seconds"
-		exit 1
-	fi
-	sleep $SLEEP_TIME
+iters=$(($STARTUP_TIMEOUT / $SLEEP_TIME))
+while [ 1 ]; do
+    ./target/debug/swadm --host '[::1]' build-info 2>/dev/null
+    rc=$?
+    stty sane 2>/dev/null || true
+    if [ $rc == 0 ]; then
+        break
+    fi
+    iters=$(($iters - 1))
+    if [ $iters = 0 ]; then
+        echo "dpd failed to come online in $STARTUP_TIMEOUT seconds"
+        exit 1
+    fi
+    sleep $SLEEP_TIME
 done
 set -o errexit
 
@@ -100,7 +104,6 @@ DENDRITE_TEST_HOST='[::1]' \
     DENDRITE_TEST_VERBOSITY=3 \
     cargo test \
     --no-fail-fast \
-    $SWADM_FEATURES \
     --test \
     counters \
     -- \

asic/src/chaos/table.rs

@@ -20,7 +20,7 @@ pub const ROUTE_IPV4: &str = "pipe.Ingress.l3_router.routes_ipv4";
 pub const ROUTE_IPV6: &str = "pipe.Ingress.l3_router.routes_ipv6";
 pub const ARP_IPV4: &str = "pipe.Ingress.l3_router.arp_ipv4";
 pub const NEIGHBOR_IPV6: &str = "pipe.Ingress.l3_router.neighbor_ipv6";
-pub const MAC_REWRITE: &str = "pipe.Ingress.mac_rewrite.mac_rewrite";
+pub const MAC_REWRITE: &str = "pipe.Egress.unicast_mac_rewrite.mac_rewrite";
 pub const SWITCH_IPV4_ADDR: &str = "pipe.Ingress.filter.switch_ipv4_addr";
 pub const SWITCH_IPV6_ADDR: &str = "pipe.Ingress.filter.switch_ipv6_addr";
 pub const NAT_INGRESS_IPV4: &str = "pipe.Ingress.nat_ingress.ingress_ipv4";
@@ -42,7 +42,7 @@ pub(crate) const MCAST_ROUTE_IPV4: &str =
 pub(crate) const MCAST_ROUTE_IPV6: &str =
     "pipe.Ingress.l3_router.MulticastRouter6.tbl";
 pub(crate) const MCAST_MAC_REWRITE: &str =
-    "pipe.Egress.mac_rewrite.mac_rewrite";
+    "pipe.Egress.mcast_mac_rewrite.mac_rewrite";
 pub(crate) const MCAST_DECAP_PORTS: &str =
     "pipe.Egress.mcast_egress.tbl_decap_ports";
 pub(crate) const MCAST_PORT_ID_MAPPING: &str =

asic/src/softnpu/table.rs

@@ -23,6 +23,12 @@ pub struct Table {
 }
 
 // soft-npu table names
+// Route tables are idx-only in sidecar-lite; route_ttl_is_1 is ignored here.
+// TODO: remove compat once sidecar-lite updates route keys/actions:
+// - p4/sidecar-lite.p4: add route_ttl_is_1 to route table keys and add
+//   ttl_exceeded actions; set ingress.route_ttl_is_1 in router.
+// - p4/softnpu.p4: add route_ttl_is_1 to ingress metadata.
+// - scadm + softnpu tests: encode/decode idx + ttl in route keys.
 const ROUTER_V4_RT: &str = "ingress.router.v4_route.rtr";
 const ROUTER_V4_IDX: &str = "ingress.router.v4_idx.rtr";
 const ROUTER_V6_RT: &str = "ingress.router.v6_route.rtr";
@@ -44,16 +50,19 @@ const _PROXY_ARP: &str = "ingress.pxarp.proxy_arp";
 const SWITCH_ADDR4: &str = "pipe.Ingress.filter.switch_ipv4_addr";
 const SWITCH_ADDR6: &str = "pipe.Ingress.filter.switch_ipv6_addr";
 const ROUTER4_LOOKUP_RT: &str =
-    "pipe.Ingress.l3_router.Router4.lookup_idx.route";
+    "pipe.Ingress.l3_router.router4.lookup_idx.route";
 const ROUTER4_LOOKUP_IDX: &str =
-    "pipe.Ingress.l3_router.Router4.lookup_idx.lookup";
+    "pipe.Ingress.l3_router.router4.lookup_idx.lookup";
 const ROUTER6_LOOKUP_RT: &str =
-    "pipe.Ingress.l3_router.Router6.lookup_idx.route";
+    "pipe.Ingress.l3_router.router6.lookup_idx.route";
 const ROUTER6_LOOKUP_IDX: &str =
-    "pipe.Ingress.l3_router.Router6.lookup_idx.lookup";
+    "pipe.Ingress.l3_router.router6.lookup_idx.lookup";
 const NDP: &str = "pipe.Ingress.l3_router.Ndp.tbl";
 const ARP: &str = "pipe.Ingress.l3_router.Arp.tbl";
-const DPD_MAC_REWRITE: &str = "pipe.Ingress.mac_rewrite.mac_rewrite";
+const DPD_UNICAST_MAC_REWRITE: &str =
+    "pipe.Egress.unicast_mac_rewrite.mac_rewrite";
+#[cfg(feature = "multicast")]
+const DPD_MCAST_MAC_REWRITE: &str = "pipe.Egress.mcast_mac_rewrite.mac_rewrite";
 const NAT_INGRESS4: &str = "pipe.Ingress.nat_ingress.ingress_ipv4";
 const NAT_INGRESS6: &str = "pipe.Ingress.nat_ingress.ingress_ipv6";
 const ATTACHED_SUBNET_INGRESS4: &str =
@@ -85,8 +94,12 @@ impl TableOps<Handle> for Table {
             SWITCH_ADDR6 => (Some(LOCAL_V6.into()), Some(SWITCH_ADDR6.into())),
             NDP => (Some(RESOLVER_V6.into()), Some(NDP.into())),
             ARP => (Some(RESOLVER_V4.into()), Some(ARP.into())),
-            DPD_MAC_REWRITE => {
-                (Some(MAC_REWRITE.into()), Some(DPD_MAC_REWRITE.into()))
+            DPD_UNICAST_MAC_REWRITE => {
+                (Some(MAC_REWRITE.into()), Some(DPD_UNICAST_MAC_REWRITE.into()))
+            }
+            #[cfg(feature = "multicast")]
+            DPD_MCAST_MAC_REWRITE => {
+                (Some(MAC_REWRITE.into()), Some(DPD_MCAST_MAC_REWRITE.into()))
             }
             NAT_INGRESS4 => (Some(NAT_V4.into()), Some(NAT_INGRESS4.into())),
             NAT_INGRESS6 => (Some(NAT_V6.into()), Some(NAT_INGRESS6.into())),
@@ -135,9 +148,22 @@ impl TableOps<Handle> for Table {
         let action_data = data.action_to_ir().unwrap();
 
         trace!(hdl.log, "entry_add called");
-        trace!(hdl.log, "table: {}", table);
-        trace!(hdl.log, "match_data:\n{:#?}", match_data);
-        trace!(hdl.log, "action_data:\n{:#?}", action_data);
+        trace!(hdl.log, "table: {table}");
+        trace!(hdl.log, "match_data:\n{match_data:#?}");
+        trace!(hdl.log, "action_data:\n{action_data:#?}");
+
+        let is_route_table =
+            matches!(dpd_table.as_str(), ROUTER4_LOOKUP_RT | ROUTER6_LOOKUP_RT);
+        if is_route_table {
+            if route_ttl_is_1(&match_data.fields) {
+                trace!(hdl.log, "skipping ttl==1 route entry for {dpd_table}");
+                return Ok(());
+            }
+            if action_data.action == "ttl_exceeded" {
+                trace!(hdl.log, "skipping ttl_exceeded action for {dpd_table}");
+                return Ok(());
+            }
+        }
 
         let keyset_data = keyset_data(match_data.fields, &table);
 
@@ -440,7 +466,23 @@ impl TableOps<Handle> for Table {
                 }
                 ("rewrite_dst", params)
             }
-            (DPD_MAC_REWRITE, "rewrite") => {
+            (DPD_UNICAST_MAC_REWRITE, "rewrite") => {
+                let mut params = Vec::new();
+                for arg in action_data.args {
+                    match arg.value {
+                        ValueTypes::U64(v) => {
+                            let mac = v.to_le_bytes();
+                            params.extend_from_slice(&mac[0..6]);
+                        }
+                        ValueTypes::Ptr(v) => {
+                            params.extend_from_slice(v.as_slice());
+                        }
+                    }
+                }
+                ("rewrite", params)
+            }
+            #[cfg(feature = "multicast")]
+            (DPD_MCAST_MAC_REWRITE, "rewrite") => {
                 let mut params = Vec::new();
                 for arg in action_data.args {
                     match arg.value {
@@ -545,10 +587,10 @@ impl TableOps<Handle> for Table {
         };
         let action = action.to_string();
         trace!(hdl.log, "sending request to softnpu");
-        trace!(hdl.log, "table: {}", table);
-        trace!(hdl.log, "action: {:#?}", action);
-        trace!(hdl.log, "keyset_data:\n{:#?}", keyset_data);
-        trace!(hdl.log, "parameter_data:\n{:#?}", parameter_data);
+        trace!(hdl.log, "table: {table}");
+        trace!(hdl.log, "action: {action:#?}");
+        trace!(hdl.log, "keyset_data:\n{keyset_data:#?}");
+        trace!(hdl.log, "parameter_data:\n{parameter_data:#?}");
 
         let msg = ManagementRequest::TableAdd(TableAdd {
             table,
@@ -576,9 +618,9 @@ impl TableOps<Handle> for Table {
         let action_data = data.action_to_ir().unwrap();
 
         trace!(hdl.log, "entry_update called");
-        trace!(hdl.log, "table: {}", table);
-        trace!(hdl.log, "match_data:\n{:#?}", match_data);
-        trace!(hdl.log, "action_data:\n{:#?}", action_data);
+        trace!(hdl.log, "table: {table}");
+        trace!(hdl.log, "match_data:\n{match_data:#?}");
+        trace!(hdl.log, "action_data:\n{action_data:#?}");
 
         //TODO implement in softnpu
         Ok(())
@@ -593,17 +635,31 @@ impl TableOps<Handle> for Table {
             None => return Ok(()),
             Some(id) => id.clone(),
         };
+        let dpd_table = match &self.dpd_id {
+            None => return Ok(()),
+            Some(id) => id.clone(),
+        };
         let match_data = key.key_to_ir().unwrap();
 
         trace!(hdl.log, "entry_del called");
-        trace!(hdl.log, "table: {}", table);
-        trace!(hdl.log, "match_data:\n{:#?}", match_data);
+        trace!(hdl.log, "table: {table}");
+        trace!(hdl.log, "match_data:\n{match_data:#?}");
+
+        let is_route_table =
+            matches!(dpd_table.as_str(), ROUTER4_LOOKUP_RT | ROUTER6_LOOKUP_RT);
+        if is_route_table && route_ttl_is_1(&match_data.fields) {
+            trace!(
+                hdl.log,
+                "skipping ttl==1 route entry delete for {dpd_table}"
+            );
+            return Ok(());
+        }
 
         let keyset_data = keyset_data(match_data.fields, &table);
 
         trace!(hdl.log, "sending request to softnpu");
-        trace!(hdl.log, "table: {}", table);
-        trace!(hdl.log, "keyset_data:\n{:#?}", keyset_data);
+        trace!(hdl.log, "table: {table}");
+        trace!(hdl.log, "keyset_data:\n{keyset_data:#?}");
 
         let msg =
             ManagementRequest::TableRemove(TableRemove { keyset_data, table });
@@ -641,12 +697,12 @@ fn keyset_data(match_data: Vec<MatchEntryField>, table: &str) -> Vec<u8> {
                 let mut data: Vec<u8> = Vec::new();
                 match table {
                     RESOLVER_V4 => {
-                        // "nexthop_ipv4" => bit<32>
+                        // "nexthop" => bit<32>
                         serialize_value_type(&x, &mut data);
                         keyset_data.extend_from_slice(&data[..4]);
                     }
                     RESOLVER_V6 => {
-                        // "nexthop_ipv4" => bit<128>
+                        // "nexthop" => bit<128>
                         let mut buf = Vec::new();
                         serialize_value_type(&x, &mut buf);
                         buf.reverse();
@@ -656,10 +712,12 @@ fn keyset_data(match_data: Vec<MatchEntryField>, table: &str) -> Vec<u8> {
                         serialize_value_type(&x, &mut data);
                         keyset_data.extend_from_slice(&data[..2]);
                     }
-                    ROUTER_V4_RT => {
-                        // "idx" => exact => bit<16>
-                        serialize_value_type(&x, &mut data);
-                        keyset_data.extend_from_slice(&data[..2]);
+                    ROUTER_V4_RT | ROUTER_V6_RT => {
+                        // sidecar-lite route keys are idx-only.
+                        if m.name == "idx" {
+                            serialize_value_type(&x, &mut data);
+                            keyset_data.extend_from_slice(&data[..2]);
+                        }
                     }
                     NAT_V4 => {
                         // "dst_addr" => hdr.ipv4.dst: exact => bit<32>
@@ -749,3 +807,18 @@ fn serialize_value_type_be(x: &ValueTypes, data: &mut Vec<u8>) {
         }
     }
 }
+
+fn route_ttl_is_1(fields: &[MatchEntryField]) -> bool {
+    fields.iter().any(|field| {
+        if field.name != "route_ttl_is_1" {
+            return false;
+        }
+        match &field.value {
+            MatchEntryValue::Value(ValueTypes::U64(v)) => *v != 0,
+            MatchEntryValue::Value(ValueTypes::Ptr(v)) => {
+                v.first().is_some_and(|b| *b != 0)
+            }
+            _ => false,
+        }
+    })
+}

common/src/illumos.rs

@@ -2,7 +2,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/
 //
-// Copyright 2025 Oxide Computer Company
+// Copyright 2026 Oxide Computer Company
 
 //! Illumos-specific common modules and operations.
 

dpd-client/tests/integration_tests/mcast.rs

@@ -1710,14 +1710,6 @@ async fn test_ipv6_multicast_invalid_destination_mac() -> TestResult {
     let ctr_baseline =
         switch.get_counter("multicast_invalid_mac", None).await.unwrap();
 
-    let port_label_ingress = switch.port_label(ingress).unwrap();
-
-    // Check the Multicast_Drop counter baseline for the ingress port
-    let drop_mcast_baseline = switch
-        .get_counter(&port_label_ingress, Some("multicast_drop"))
-        .await
-        .unwrap();
-
     switch.packet_test(vec![test_pkt], expected_pkts).unwrap();
 
     check_counter_incremented(
@@ -1730,17 +1722,6 @@ async fn test_ipv6_multicast_invalid_destination_mac() -> TestResult {
     .await
     .unwrap();
 
-    // Verify that the Multicast_Drop counter also incremented
-    check_counter_incremented(
-        switch,
-        &port_label_ingress,
-        drop_mcast_baseline,
-        1,
-        Some("multicast_drop"),
-    )
-    .await
-    .unwrap();
-
     cleanup_test_group(switch, get_group_ip(&created_group), TEST_TAG).await
 }
 

dpd-client/tests/integration_tests/table_tests.rs

@@ -42,17 +42,7 @@ use crate::integration_tests::common::prelude::*;
 // investigating. If it only changes by an entry or two, it's fine to just
 // adjust the constant below to match the observed result.
 //
-// TODO: Multicast drops IPv4 LPM capacity to 7164 (from 8187) due to
-// ingress TCAM pressure. Investigate moving MulticastRouter4/6 into the
-// egress pipeline to reclaim capacity.
-#[cfg(feature = "multicast")]
-const IPV4_LPM_SIZE: usize = 7164; // ipv4 forwarding table
-#[cfg(not(feature = "multicast"))]
-const IPV4_LPM_SIZE: usize = 8187; // ipv4 forwarding table
-
-#[cfg(feature = "multicast")]
-const IPV6_LPM_SIZE: usize = 1023; // ipv6 forwarding table
-#[cfg(not(feature = "multicast"))]
+const IPV4_LPM_SIZE: usize = 8191; // ipv4 forwarding table
 const IPV6_LPM_SIZE: usize = 1023; // ipv6 forwarding table
 
 const SWITCH_IPV4_ADDRS_SIZE: usize = 511; // ipv4 addrs assigned to our ports