[multicast, refactor]: add ddm-peers omdb view, shift member writes to mg-lower

Under RFD 488 (incoming changes), and akin to unicast tunnel routes, mg-lower
and ddmd derive underlay members directly from DDM peer subscriptions, so Nexus
no longer resolves live sled-to-switch-port from inventory or DDM peers. The DDM
client, instead, becomes a read-only peer view, and the reconciler's
member->"Joined" path is a pure DB CAS plus best-effort sled-agent propagation,
decoupled from MGS, the switch zone, and DPD member updates.

Includes:

- Adds the `omdb nexus multicast ddm-peers` view backed by the read-only DDM
  client, with a new internal-dns resolver helper (`lookup_all_socket_v6_by_target`)
  so per-switch clients correlate sockets back to their SRV targets
- Adds reconciler config knobs to `MulticastGroupReconcilerConfig`:
  `group_concurrency_limit`, `member_concurrency_limit`, and `orphan_grace_secs`
  (grace before an orphaned empty "Creating" group is reaped), wired through the
  example, SMF, and test configs
- Shifts underlay member writes to mg-lower, trimming the reconciler's
  member state machine and switch-zone coordination accordingly
- Splits the multicast dataplane client into focused modules: `mrib.rs` for
  per-pass MRIB route reconciliation and `switch_zone.rs` for the MGD/DDM
  switch-zone clients, leaving `dataplane.rs` to DPD group operations
- Adds schema migration `multicast-member-origin` (schema version 267): a new
  `multicast_group_member_origin` enum (`static` / `igmp_snooped`) and a
  `membership_origin` column on multicast group members defaulting to `static`.
  Groundwork for the RFD 488 static and IGMP/MLD paths, distinguishing which
  rows a future soft-state reaper may expire
- Aligns the simulated sled-agent's `set_mcast_fwd` with real OPTE's additive
  next-hop semantics, and reads back through the `list_mcast_fwd` API in the
  multi-sled test
  - Also, the sled-agent multicast membership API was renamed
  `instance_join/leave_multicast_group` → `vmm_join/leave_multicast_group`
  (for sim + client/HTTP surface).
- Corrects stale reconciler, switch-zone, and API-version docs throughout
- Documents the test boundary: coverage stops at the Omicron-owned edge, while
  DDM exchange and DPD member programming are maghemite/dendrite's domain
- Cleans up the now-dead inventory-based test scaffolding

Author: zeeshanlakhani

Cargo.lock

@@ -615,9 +615,10 @@ ntp-admin-client = { path = "clients/ntp-admin-client" }
 ntp-admin-v1-client = { path = "clients/ntp-admin-v1-client" }
 ntp-admin-types = { path = "ntp-admin/types" }
 ntp-admin-types-versions = { path = "ntp-admin/types/versions" }
-mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "5aaa7c9bc2d0e541d29604453dd98cb32676b347" }
-mg-api-types = { git = "https://github.com/oxidecomputer/maghemite", rev = "5aaa7c9bc2d0e541d29604453dd98cb32676b347" }
-ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "5aaa7c9bc2d0e541d29604453dd98cb32676b347" }
+mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "4cc569836dc1c5d13a5f2396de310220eafee2fe" }
+mg-api-types = { git = "https://github.com/oxidecomputer/maghemite", rev = "4cc569836dc1c5d13a5f2396de310220eafee2fe" }
+ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "4cc569836dc1c5d13a5f2396de310220eafee2fe" }
+ddm-api-types-versions = { git = "https://github.com/oxidecomputer/maghemite", rev = "4cc569836dc1c5d13a5f2396de310220eafee2fe" }
 multimap = "0.10.1"
 nexus-auth = { path = "nexus/auth" }
 nexus-background-task-interface = { path = "nexus/background-task-interface" }

Cargo.toml

@@ -20,5 +20,6 @@ omicron-common.workspace = true
 sled-hardware-types.workspace = true
 omicron-workspace-hack.workspace = true
 ddm-admin-client.workspace = true
+ddm-api-types-versions.workspace = true
 uuid.workspace = true
 slog-error-chain.workspace = true

clients/ddm-admin-client/Cargo.toml

@@ -2,14 +2,18 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-// Copyright 2023 Oxide Computer Company
+// Copyright 2026 Oxide Computer Company
 
 #![allow(clippy::redundant_closure_call)]
 #![allow(clippy::needless_lifetimes)]
 #![allow(clippy::match_single_binding)]
 #![allow(clippy::clone_on_copy)]
 pub use ddm_admin_client::Error;
 pub use ddm_admin_client::types;
+pub use ddm_api_types_versions::latest::db::{
+    MulticastRoute, PeerInfo, PeerStatus,
+};
+pub use ddm_api_types_versions::latest::net::MulticastOrigin;
 
 use ddm_admin_client::Client as InnerClient;
 use either::Either;
@@ -105,6 +109,38 @@ impl Client {
         self.inner.enable_stats(request).await.map(|resp| resp.into_inner())
     }
 
+    /// Returns DDM peer information including interface names.
+    ///
+    /// The `if_name` field on each peer provides a live sled-to-port
+    /// mapping, identifying which switch port a peer sled is connected
+    /// through (e.g., `"tfportrear0_0"`).
+    pub async fn get_peers(
+        &self,
+    ) -> Result<std::collections::HashMap<String, PeerInfo>, Error<types::Error>>
+    {
+        self.inner.get_peers().await.map(|resp| resp.into_inner())
+    }
+
+    /// Returns multicast routes learned from DDM peers.
+    ///
+    /// Each route includes the origin (overlay/underlay mapping),
+    /// the nexthop peer that advertised it, and the path vector.
+    pub async fn get_multicast_groups(
+        &self,
+    ) -> Result<Vec<MulticastRoute>, Error<types::Error>> {
+        self.inner.get_multicast_groups().await.map(|resp| resp.into_inner())
+    }
+
+    /// Returns multicast origins that this DDM instance is advertising.
+    pub async fn get_originated_multicast_groups(
+        &self,
+    ) -> Result<Vec<MulticastOrigin>, Error<types::Error>> {
+        self.inner
+            .get_originated_multicast_groups()
+            .await
+            .map(|resp| resp.into_inner())
+    }
+
     /// Returns the addresses of connected sleds.
     ///
     /// Note: These sleds have not yet been verified.

clients/ddm-admin-client/src/lib.rs

@@ -2538,6 +2538,8 @@ impl Vni {
     ///
     /// This is a low-numbered VNI to avoid colliding with user VNIs.
     /// However, it is not in the Oxide-reserved range yet.
+    ///
+    /// Should match `oxide_vpc::api::DEFAULT_MULTICAST_VNI`.
     pub const DEFAULT_MULTICAST_VNI: Self = Self(77);
 
     /// Oxide reserves a slice of initial VNIs for its own use.

common/src/api/external/mod.rs

@@ -29,6 +29,7 @@ Crucible Pantry (client: crucible-pantry-client)
 Maghemite DDM Admin (client: ddm-admin-client)
     consumed by: installinator (omicron/installinator) via 1 path
     consumed by: mgd (maghemite/mgd) via 1 path
+    consumed by: omicron-nexus (omicron/nexus) via 1 path
     consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path
     consumed by: sled-agent-rack-setup (omicron/sled-agent/rack-setup) via 1 path [embedded in omicron-sled-agent; rack-init only]
     consumed by: wicketd (omicron/wicketd) via 1 path

dev-tools/ls-apis/tests/api_dependencies.out

@@ -167,6 +167,8 @@ enum NexusCommands {
     FetchOmdb(FetchOmdbArgs),
     /// print information about pending MGS updates
     MgsUpdates,
+    /// inspect multicast state
+    Multicast(MulticastArgs),
     /// interact with oximeter read policy
     OximeterReadPolicy(OximeterReadPolicyArgs),
     /// view or modify the quiesce status
@@ -450,6 +452,26 @@ struct FetchOmdbArgs {
     output: Utf8PathBuf,
 }
 
+#[derive(Debug, Args)]
+struct MulticastArgs {
+    #[command(subcommand)]
+    command: MulticastCommands,
+}
+
+#[derive(Debug, Subcommand)]
+enum MulticastCommands {
+    /// List DDM underlay peers observed across switch zones
+    DdmPeers(MulticastDdmPeersArgs),
+}
+
+#[derive(Debug, Args)]
+struct MulticastDdmPeersArgs {
+    /// Show only peers that become multicast underlay members
+    /// (DDM session in `Exchange` with a switch rear-port interface).
+    #[arg(long)]
+    mcast: bool,
+}
+
 #[derive(Debug, Args)]
 struct OximeterReadPolicyArgs {
     #[command(subcommand)]
@@ -825,6 +847,14 @@ impl NexusArgs {
 
             NexusCommands::MgsUpdates => cmd_nexus_mgs_updates(&client).await,
 
+            NexusCommands::Multicast(MulticastArgs { command }) => {
+                match command {
+                    MulticastCommands::DdmPeers(args) => {
+                        cmd_nexus_multicast_ddm_peers(&client, args).await
+                    }
+                }
+            }
+
             NexusCommands::OximeterReadPolicy(OximeterReadPolicyArgs {
                 command,
             }) => match command {
@@ -4713,6 +4743,68 @@ async fn cmd_nexus_mgs_updates(
     Ok(())
 }
 
+async fn cmd_nexus_multicast_ddm_peers(
+    client: &nexus_lockstep_client::Client,
+    args: &MulticastDdmPeersArgs,
+) -> Result<(), anyhow::Error> {
+    let view = client
+        .multicast_ddm_peers()
+        .await
+        .context("fetching multicast DDM peers")?
+        .into_inner();
+
+    let mut peers = view.peers;
+    if args.mcast {
+        // Members are derived only from peers in DDM `Exchange` on a switch
+        // rear-port interface (see the multicast members background task).
+        peers.retain(|peer| {
+            peer.status
+                == nexus_lockstep_client::types::MulticastDdmPeerStatus::Exchange
+                && peer.if_name.is_some()
+        });
+    }
+
+    if peers.is_empty() {
+        println!("no DDM peers reported by any switch zone");
+        return Ok(());
+    }
+
+    #[derive(Tabled)]
+    #[tabled(rename_all = "SCREAMING_SNAKE_CASE")]
+    struct PeerRow {
+        switch: String,
+        addr: String,
+        host: String,
+        status: String,
+        duration: String,
+        if_name: String,
+    }
+
+    peers.sort_by(|a, b| (&a.switch, &a.addr).cmp(&(&b.switch, &b.addr)));
+    let rows = peers.into_iter().map(|peer| PeerRow {
+        switch: peer.switch,
+        addr: peer.addr.to_string(),
+        host: peer.host,
+        status: peer.status.to_string(),
+        duration: format!(
+            "{:?}",
+            std::time::Duration::new(
+                peer.status_duration.secs,
+                peer.status_duration.nanos,
+            )
+        ),
+        if_name: peer.if_name.unwrap_or_else(|| "-".to_string()),
+    });
+
+    let table = tabled::Table::new(rows)
+        .with(tabled::settings::Style::empty())
+        .with(tabled::settings::Padding::new(0, 1, 0, 0))
+        .to_string();
+    println!("{table}");
+
+    Ok(())
+}
+
 async fn cmd_nexus_clickhouse_policy_set(
     client: &nexus_lockstep_client::Client,
     args: &ClickhousePolicySetArgs,

dev-tools/omdb/src/bin/omdb/nexus.rs

@@ -22,8 +22,9 @@ use nexus_test_utils::resource_helpers::{
     create_project, link_ip_pool, object_put_upsert, objects_list_page_authz,
 };
 use nexus_test_utils_macros::nexus_test;
-use nexus_types::external_api::instance::Instance;
-use nexus_types::external_api::instance::InstanceNetworkInterfaceAttachment;
+use nexus_types::external_api::instance::{
+    Instance, InstanceNetworkInterfaceAttachment,
+};
 use nexus_types::external_api::multicast::{
     InstanceMulticastGroupJoin, MulticastGroup, MulticastGroupMember,
 };

dev-tools/omdb/tests/test_multicast.rs

@@ -1031,6 +1031,7 @@ Commands:
   clickhouse-policy      interact with clickhouse policy
   fetch-omdb             fetch an omdb binary associated with an active Nexus
   mgs-updates            print information about pending MGS updates
+  multicast              inspect multicast state
   oximeter-read-policy   interact with oximeter read policy
   quiesce                view or modify the quiesce status
   reconfigurator-config  interact with reconfigurator config

dev-tools/omdb/tests/usage_errors.out

@@ -76,6 +76,12 @@ pub enum Error {
         "address {0} is not within the underlay multicast subnet (ff04::/16)"
     )]
     InvalidMcastUnderlay(Ipv6Addr),
+
+    #[error(
+        "failed to install NIC multicast MAC filter for underlay {0}, \
+         caller should retry"
+    )]
+    UnderlayMcastJoinFailed(Ipv6Addr),
 }
 
 /// Delete all xde devices on the system.