From 183f02d221c4ec9ae0e18c0fabfcdaea561fd36c Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Wed, 10 Jun 2026 14:14:20 +0000 Subject: [PATCH] [multicast, refactor]: add ddm-peers omdb view, shift member writes to mg-lower MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Under RFD 488 (incoming changes), and akin to unicast tunnel routes, mg-lower and ddmd derive underlay members directly from DDM peer subscriptions, so Nexus no longer resolves live sled-to-switch-port from inventory or DDM peers. The DDM client, instead, becomes a read-only peer view, and the reconciler's member->"Joined" path is a pure DB CAS plus best-effort sled-agent propagation, decoupled from MGS, the switch zone, and DPD member updates. Includes: - Adds the `omdb nexus multicast ddm-peers` view backed by the read-only DDM client, with a new internal-dns resolver helper (`lookup_all_socket_v6_by_target`) so per-switch clients correlate sockets back to their SRV targets - Adds reconciler config knobs to `MulticastGroupReconcilerConfig`: `group_concurrency_limit`, `member_concurrency_limit`, and `orphan_grace_secs` (grace before an orphaned empty "Creating" group is reaped), wired through the example, SMF, and test configs - Shifts underlay member writes to mg-lower, trimming the reconciler's member state machine and switch-zone coordination accordingly - Splits the multicast dataplane client into focused modules: `mrib.rs` for per-pass MRIB route reconciliation and `switch_zone.rs` for the MGD/DDM switch-zone clients, leaving `dataplane.rs` to DPD group operations - Adds schema migration `multicast-member-origin` (schema version 267): a new `multicast_group_member_origin` enum (`static` / `igmp_snooped`) and a `membership_origin` column on multicast group members defaulting to `static`. Groundwork for the RFD 488 static and IGMP/MLD paths, distinguishing which rows a future soft-state reaper may expire - Aligns the simulated sled-agent's `set_mcast_fwd` with real OPTE's additive next-hop semantics, and reads back through the `list_mcast_fwd` API in the multi-sled test - Also, the sled-agent multicast membership API was renamed `instance_join/leave_multicast_group` → `vmm_join/leave_multicast_group` (for sim + client/HTTP surface). - Corrects stale reconciler, switch-zone, and API-version docs throughout - Documents the test boundary: coverage stops at the Omicron-owned edge, while DDM exchange and DPD member programming are maghemite/dendrite's domain - Cleans up the now-dead inventory-based test scaffolding --- Cargo.lock | 18 +- Cargo.toml | 7 +- clients/ddm-admin-client/Cargo.toml | 1 + clients/ddm-admin-client/src/lib.rs | 38 +- common/src/api/external/mod.rs | 2 + dev-tools/ls-apis/tests/api_dependencies.out | 1 + dev-tools/omdb/src/bin/omdb/nexus.rs | 92 + dev-tools/omdb/tests/test_multicast.rs | 5 +- dev-tools/omdb/tests/usage_errors.out | 1 + illumos-utils/src/opte/illumos.rs | 6 + illumos-utils/src/opte/mod.rs | 11 + illumos-utils/src/opte/non_illumos.rs | 48 +- illumos-utils/src/opte/port_manager.rs | 838 ++++++++- internal-dns/resolver/src/resolver.rs | 70 + nexus-config/src/nexus_config.rs | 67 +- nexus/Cargo.toml | 1 + nexus/db-model/src/multicast_group.rs | 64 +- nexus/db-model/src/schema_versions.rs | 3 +- .../src/db/datastore/multicast/groups.rs | 2 +- .../src/db/datastore/multicast/members.rs | 8 +- .../datastore/multicast/ops/member_attach.rs | 14 +- nexus/db-schema/src/enums.rs | 1 + nexus/db-schema/src/schema.rs | 1 + nexus/examples/config-second.toml | 6 - nexus/examples/config.toml | 6 - nexus/lockstep-api/src/lib.rs | 12 + nexus/src/app/background/init.rs | 13 +- .../app/background/tasks/multicast/groups.rs | 345 ++-- .../app/background/tasks/multicast/members.rs | 1629 +++-------------- .../src/app/background/tasks/multicast/mod.rs | 431 ++--- .../app/background/tasks/multicast/mrib.rs | 186 ++ .../tasks/sync_switch_configuration.rs | 80 +- nexus/src/app/bgp.rs | 14 +- nexus/src/app/instance.rs | 21 +- nexus/src/app/mod.rs | 133 +- nexus/src/app/multicast/dataplane.rs | 542 +----- nexus/src/app/multicast/mod.rs | 163 +- nexus/src/app/multicast/sled.rs | 285 ++- nexus/src/app/multicast/switch_zone.rs | 457 +++++ .../app/sagas/multicast_group_dpd_ensure.rs | 55 +- nexus/src/external_api/http_entrypoints.rs | 6 +- nexus/src/lockstep_api/http_entrypoints.rs | 17 + nexus/test-utils/src/lib.rs | 74 +- nexus/test-utils/src/starter.rs | 48 +- nexus/tests/config.test.toml | 3 - .../tests/integration_tests/initialization.rs | 18 +- .../tests/integration_tests/multicast/api.rs | 2 - .../multicast/authorization.rs | 2 +- .../multicast/cache_invalidation.rs | 645 ------- .../integration_tests/multicast/failures.rs | 116 +- .../integration_tests/multicast/groups.rs | 427 ++++- .../integration_tests/multicast/instances.rs | 572 ++++-- .../tests/integration_tests/multicast/mod.rs | 989 ++++++---- .../multicast/networking_integration.rs | 198 +- nexus/types/src/internal_api/background.rs | 5 + nexus/types/src/internal_api/views.rs | 58 + openapi/nexus-lockstep.json | 96 + package-manifest.toml | 12 +- schema/crdb/dbinit.sql | 19 +- schema/crdb/multicast-member-origin/up01.sql | 4 + schema/crdb/multicast-member-origin/up02.sql | 2 + sled-agent/early-networking/src/lib.rs | 66 +- sled-agent/src/http_entrypoints.rs | 58 +- sled-agent/src/instance.rs | 1 + sled-agent/src/instance_manager.rs | 60 +- sled-agent/src/sim/http_entrypoints.rs | 4 +- sled-agent/src/sim/sled_agent.rs | 56 +- sled-agent/src/sled_agent.rs | 18 +- .../versions/src/mcast_m2p_forwarding/mod.rs | 6 +- .../src/mcast_m2p_forwarding/multicast.rs | 2 - smf/nexus/multi-sled/config-partial.toml | 6 - smf/nexus/single-sled/config-partial.toml | 6 - test-utils/Cargo.toml | 2 + test-utils/src/dev/maghemite.rs | 124 +- tools/maghemite_ddm_openapi_version | 2 +- tools/maghemite_mg_openapi_version | 2 +- tools/maghemite_mgd_checksums | 8 +- 77 files changed, 5321 insertions(+), 4090 deletions(-) create mode 100644 nexus/src/app/background/tasks/multicast/mrib.rs create mode 100644 nexus/src/app/multicast/switch_zone.rs delete mode 100644 nexus/tests/integration_tests/multicast/cache_invalidation.rs create mode 100644 schema/crdb/multicast-member-origin/up01.sql create mode 100644 schema/crdb/multicast-member-origin/up02.sql diff --git a/Cargo.lock b/Cargo.lock index 7be0aeced2c..cd0a8f43829 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1607,7 +1607,7 @@ dependencies = [ [[package]] name = "client-common" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=5aaa7c9bc2d0e541d29604453dd98cb32676b347#5aaa7c9bc2d0e541d29604453dd98cb32676b347" +source = "git+https://github.com/oxidecomputer/maghemite?rev=4cc569836dc1c5d13a5f2396de310220eafee2fe#4cc569836dc1c5d13a5f2396de310220eafee2fe" dependencies = [ "oxnet", "schemars 0.8.22", @@ -2572,7 +2572,7 @@ dependencies = [ [[package]] name = "ddm-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=5aaa7c9bc2d0e541d29604453dd98cb32676b347#5aaa7c9bc2d0e541d29604453dd98cb32676b347" +source = "git+https://github.com/oxidecomputer/maghemite?rev=4cc569836dc1c5d13a5f2396de310220eafee2fe#4cc569836dc1c5d13a5f2396de310220eafee2fe" dependencies = [ "ddm-api-types-versions", "oxnet", @@ -2586,7 +2586,7 @@ dependencies = [ [[package]] name = "ddm-api-types-versions" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=5aaa7c9bc2d0e541d29604453dd98cb32676b347#5aaa7c9bc2d0e541d29604453dd98cb32676b347" +source = "git+https://github.com/oxidecomputer/maghemite?rev=4cc569836dc1c5d13a5f2396de310220eafee2fe#4cc569836dc1c5d13a5f2396de310220eafee2fe" dependencies = [ "client-common", "ddm-protocol", @@ -2602,7 +2602,7 @@ dependencies = [ [[package]] name = "ddm-protocol" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=5aaa7c9bc2d0e541d29604453dd98cb32676b347#5aaa7c9bc2d0e541d29604453dd98cb32676b347" +source = "git+https://github.com/oxidecomputer/maghemite?rev=4cc569836dc1c5d13a5f2396de310220eafee2fe#4cc569836dc1c5d13a5f2396de310220eafee2fe" dependencies = [ "oxnet", "schemars 0.8.22", @@ -6572,12 +6572,13 @@ dependencies = [ [[package]] name = "mg-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=5aaa7c9bc2d0e541d29604453dd98cb32676b347#5aaa7c9bc2d0e541d29604453dd98cb32676b347" +source = "git+https://github.com/oxidecomputer/maghemite?rev=4cc569836dc1c5d13a5f2396de310220eafee2fe#4cc569836dc1c5d13a5f2396de310220eafee2fe" dependencies = [ "chrono", "client-common", "colored 3.1.1", "mg-api-types-versions", + "oxnet", "progenitor 0.14.0", "reqwest 0.13.2", "schemars 0.8.22", @@ -6591,7 +6592,7 @@ dependencies = [ [[package]] name = "mg-api-types" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=5aaa7c9bc2d0e541d29604453dd98cb32676b347#5aaa7c9bc2d0e541d29604453dd98cb32676b347" +source = "git+https://github.com/oxidecomputer/maghemite?rev=4cc569836dc1c5d13a5f2396de310220eafee2fe#4cc569836dc1c5d13a5f2396de310220eafee2fe" dependencies = [ "mg-api-types-versions", ] @@ -6599,7 +6600,7 @@ dependencies = [ [[package]] name = "mg-api-types-versions" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=5aaa7c9bc2d0e541d29604453dd98cb32676b347#5aaa7c9bc2d0e541d29604453dd98cb32676b347" +source = "git+https://github.com/oxidecomputer/maghemite?rev=4cc569836dc1c5d13a5f2396de310220eafee2fe#4cc569836dc1c5d13a5f2396de310220eafee2fe" dependencies = [ "chrono", "client-common", @@ -8491,6 +8492,7 @@ name = "omicron-ddm-admin-client" version = "0.1.0" dependencies = [ "ddm-admin-client", + "ddm-api-types-versions", "either", "omicron-common", "omicron-workspace-hack", @@ -8820,6 +8822,7 @@ dependencies = [ "num-integer", "omicron-cockroach-metrics", "omicron-common", + "omicron-ddm-admin-client", "omicron-passwords", "omicron-rpaths", "omicron-sled-agent", @@ -9395,6 +9398,7 @@ dependencies = [ "reqwest 0.13.2", "ring", "rustls 0.22.4", + "schemars 0.8.22", "serde", "sha2", "slog", diff --git a/Cargo.toml b/Cargo.toml index 9370d6365fa..39daa415a0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -615,9 +615,10 @@ ntp-admin-client = { path = "clients/ntp-admin-client" } ntp-admin-v1-client = { path = "clients/ntp-admin-v1-client" } ntp-admin-types = { path = "ntp-admin/types" } ntp-admin-types-versions = { path = "ntp-admin/types/versions" } -mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "5aaa7c9bc2d0e541d29604453dd98cb32676b347" } -mg-api-types = { git = "https://github.com/oxidecomputer/maghemite", rev = "5aaa7c9bc2d0e541d29604453dd98cb32676b347" } -ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "5aaa7c9bc2d0e541d29604453dd98cb32676b347" } +mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "4cc569836dc1c5d13a5f2396de310220eafee2fe" } +mg-api-types = { git = "https://github.com/oxidecomputer/maghemite", rev = "4cc569836dc1c5d13a5f2396de310220eafee2fe" } +ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "4cc569836dc1c5d13a5f2396de310220eafee2fe" } +ddm-api-types-versions = { git = "https://github.com/oxidecomputer/maghemite", rev = "4cc569836dc1c5d13a5f2396de310220eafee2fe" } multimap = "0.10.1" nexus-auth = { path = "nexus/auth" } nexus-background-task-interface = { path = "nexus/background-task-interface" } diff --git a/clients/ddm-admin-client/Cargo.toml b/clients/ddm-admin-client/Cargo.toml index 56e3c9d2328..83d581d641a 100644 --- a/clients/ddm-admin-client/Cargo.toml +++ b/clients/ddm-admin-client/Cargo.toml @@ -20,5 +20,6 @@ omicron-common.workspace = true sled-hardware-types.workspace = true omicron-workspace-hack.workspace = true ddm-admin-client.workspace = true +ddm-api-types-versions.workspace = true uuid.workspace = true slog-error-chain.workspace = true diff --git a/clients/ddm-admin-client/src/lib.rs b/clients/ddm-admin-client/src/lib.rs index 466a8883918..df30cb3d982 100644 --- a/clients/ddm-admin-client/src/lib.rs +++ b/clients/ddm-admin-client/src/lib.rs @@ -2,7 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -// Copyright 2023 Oxide Computer Company +// Copyright 2026 Oxide Computer Company #![allow(clippy::redundant_closure_call)] #![allow(clippy::needless_lifetimes)] @@ -10,6 +10,10 @@ #![allow(clippy::clone_on_copy)] pub use ddm_admin_client::Error; pub use ddm_admin_client::types; +pub use ddm_api_types_versions::latest::db::{ + MulticastRoute, PeerInfo, PeerStatus, +}; +pub use ddm_api_types_versions::latest::net::MulticastOrigin; use ddm_admin_client::Client as InnerClient; use either::Either; @@ -105,6 +109,38 @@ impl Client { self.inner.enable_stats(request).await.map(|resp| resp.into_inner()) } + /// Returns DDM peer information including interface names. + /// + /// The `if_name` field on each peer provides a live sled-to-port + /// mapping, identifying which switch port a peer sled is connected + /// through (e.g., `"tfportrear0_0"`). + pub async fn get_peers( + &self, + ) -> Result, Error> + { + self.inner.get_peers().await.map(|resp| resp.into_inner()) + } + + /// Returns multicast routes learned from DDM peers. + /// + /// Each route includes the origin (overlay/underlay mapping), + /// the nexthop peer that advertised it, and the path vector. + pub async fn get_multicast_groups( + &self, + ) -> Result, Error> { + self.inner.get_multicast_groups().await.map(|resp| resp.into_inner()) + } + + /// Returns multicast origins that this DDM instance is advertising. + pub async fn get_originated_multicast_groups( + &self, + ) -> Result, Error> { + self.inner + .get_originated_multicast_groups() + .await + .map(|resp| resp.into_inner()) + } + /// Returns the addresses of connected sleds. /// /// Note: These sleds have not yet been verified. diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 10206ade61d..e9ee0e2c24f 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -2538,6 +2538,8 @@ impl Vni { /// /// This is a low-numbered VNI to avoid colliding with user VNIs. /// However, it is not in the Oxide-reserved range yet. + /// + /// Should match `oxide_vpc::api::DEFAULT_MULTICAST_VNI`. pub const DEFAULT_MULTICAST_VNI: Self = Self(77); /// Oxide reserves a slice of initial VNIs for its own use. diff --git a/dev-tools/ls-apis/tests/api_dependencies.out b/dev-tools/ls-apis/tests/api_dependencies.out index 8b6aad31d25..f5da994a163 100644 --- a/dev-tools/ls-apis/tests/api_dependencies.out +++ b/dev-tools/ls-apis/tests/api_dependencies.out @@ -29,6 +29,7 @@ Crucible Pantry (client: crucible-pantry-client) Maghemite DDM Admin (client: ddm-admin-client) consumed by: installinator (omicron/installinator) via 1 path consumed by: mgd (maghemite/mgd) via 1 path + consumed by: omicron-nexus (omicron/nexus) via 1 path consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path consumed by: sled-agent-rack-setup (omicron/sled-agent/rack-setup) via 1 path [embedded in omicron-sled-agent; rack-init only] consumed by: wicketd (omicron/wicketd) via 1 path diff --git a/dev-tools/omdb/src/bin/omdb/nexus.rs b/dev-tools/omdb/src/bin/omdb/nexus.rs index 0bd2f1d4fcc..b96e5a78976 100644 --- a/dev-tools/omdb/src/bin/omdb/nexus.rs +++ b/dev-tools/omdb/src/bin/omdb/nexus.rs @@ -167,6 +167,8 @@ enum NexusCommands { FetchOmdb(FetchOmdbArgs), /// print information about pending MGS updates MgsUpdates, + /// inspect multicast state + Multicast(MulticastArgs), /// interact with oximeter read policy OximeterReadPolicy(OximeterReadPolicyArgs), /// view or modify the quiesce status @@ -450,6 +452,26 @@ struct FetchOmdbArgs { output: Utf8PathBuf, } +#[derive(Debug, Args)] +struct MulticastArgs { + #[command(subcommand)] + command: MulticastCommands, +} + +#[derive(Debug, Subcommand)] +enum MulticastCommands { + /// List DDM underlay peers observed across switch zones + DdmPeers(MulticastDdmPeersArgs), +} + +#[derive(Debug, Args)] +struct MulticastDdmPeersArgs { + /// Show only peers that become multicast underlay members + /// (DDM session in `Exchange` with a switch rear-port interface). + #[arg(long)] + mcast: bool, +} + #[derive(Debug, Args)] struct OximeterReadPolicyArgs { #[command(subcommand)] @@ -825,6 +847,14 @@ impl NexusArgs { NexusCommands::MgsUpdates => cmd_nexus_mgs_updates(&client).await, + NexusCommands::Multicast(MulticastArgs { command }) => { + match command { + MulticastCommands::DdmPeers(args) => { + cmd_nexus_multicast_ddm_peers(&client, args).await + } + } + } + NexusCommands::OximeterReadPolicy(OximeterReadPolicyArgs { command, }) => match command { @@ -4713,6 +4743,68 @@ async fn cmd_nexus_mgs_updates( Ok(()) } +async fn cmd_nexus_multicast_ddm_peers( + client: &nexus_lockstep_client::Client, + args: &MulticastDdmPeersArgs, +) -> Result<(), anyhow::Error> { + let view = client + .multicast_ddm_peers() + .await + .context("fetching multicast DDM peers")? + .into_inner(); + + let mut peers = view.peers; + if args.mcast { + // Members are derived only from peers in DDM `Exchange` on a switch + // rear-port interface (see the multicast members background task). + peers.retain(|peer| { + peer.status + == nexus_lockstep_client::types::MulticastDdmPeerStatus::Exchange + && peer.if_name.is_some() + }); + } + + if peers.is_empty() { + println!("no DDM peers reported by any switch zone"); + return Ok(()); + } + + #[derive(Tabled)] + #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] + struct PeerRow { + switch: String, + addr: String, + host: String, + status: String, + duration: String, + if_name: String, + } + + peers.sort_by(|a, b| (&a.switch, &a.addr).cmp(&(&b.switch, &b.addr))); + let rows = peers.into_iter().map(|peer| PeerRow { + switch: peer.switch, + addr: peer.addr.to_string(), + host: peer.host, + status: peer.status.to_string(), + duration: format!( + "{:?}", + std::time::Duration::new( + peer.status_duration.secs, + peer.status_duration.nanos, + ) + ), + if_name: peer.if_name.unwrap_or_else(|| "-".to_string()), + }); + + let table = tabled::Table::new(rows) + .with(tabled::settings::Style::empty()) + .with(tabled::settings::Padding::new(0, 1, 0, 0)) + .to_string(); + println!("{table}"); + + Ok(()) +} + async fn cmd_nexus_clickhouse_policy_set( client: &nexus_lockstep_client::Client, args: &ClickhousePolicySetArgs, diff --git a/dev-tools/omdb/tests/test_multicast.rs b/dev-tools/omdb/tests/test_multicast.rs index 6e5d5ed4956..3bb984421a3 100644 --- a/dev-tools/omdb/tests/test_multicast.rs +++ b/dev-tools/omdb/tests/test_multicast.rs @@ -22,8 +22,9 @@ use nexus_test_utils::resource_helpers::{ create_project, link_ip_pool, object_put_upsert, objects_list_page_authz, }; use nexus_test_utils_macros::nexus_test; -use nexus_types::external_api::instance::Instance; -use nexus_types::external_api::instance::InstanceNetworkInterfaceAttachment; +use nexus_types::external_api::instance::{ + Instance, InstanceNetworkInterfaceAttachment, +}; use nexus_types::external_api::multicast::{ InstanceMulticastGroupJoin, MulticastGroup, MulticastGroupMember, }; diff --git a/dev-tools/omdb/tests/usage_errors.out b/dev-tools/omdb/tests/usage_errors.out index cbd6ed7f6bc..46e299d0129 100644 --- a/dev-tools/omdb/tests/usage_errors.out +++ b/dev-tools/omdb/tests/usage_errors.out @@ -1031,6 +1031,7 @@ Commands: clickhouse-policy interact with clickhouse policy fetch-omdb fetch an omdb binary associated with an active Nexus mgs-updates print information about pending MGS updates + multicast inspect multicast state oximeter-read-policy interact with oximeter read policy quiesce view or modify the quiesce status reconfigurator-config interact with reconfigurator config diff --git a/illumos-utils/src/opte/illumos.rs b/illumos-utils/src/opte/illumos.rs index 3dcdd8cfdcd..b0c908c392c 100644 --- a/illumos-utils/src/opte/illumos.rs +++ b/illumos-utils/src/opte/illumos.rs @@ -76,6 +76,12 @@ pub enum Error { "address {0} is not within the underlay multicast subnet (ff04::/16)" )] InvalidMcastUnderlay(Ipv6Addr), + + #[error( + "failed to install NIC multicast MAC filter for underlay {0}, \ + caller should retry" + )] + UnderlayMcastJoinFailed(Ipv6Addr), } /// Delete all xde devices on the system. diff --git a/illumos-utils/src/opte/mod.rs b/illumos-utils/src/opte/mod.rs index 4903e61db8b..1592a9af337 100644 --- a/illumos-utils/src/opte/mod.rs +++ b/illumos-utils/src/opte/mod.rs @@ -41,6 +41,17 @@ use std::net::IpAddr; use std::net::Ipv4Addr; use std::net::Ipv6Addr; +// `oxide_vpc::api::DEFAULT_MULTICAST_VNI` and +// `omicron_common::api::external::Vni::DEFAULT_MULTICAST_VNI` live in sibling +// crates that cannot reference each other's constant. They must stay +// numerically equal: the MRIB, M2P mappings, and OPTE all route on this +// value, so any divergence would silently drop multicast traffic. +const _: () = assert!( + oxide_vpc::api::DEFAULT_MULTICAST_VNI + == omicron_common::api::external::Vni::DEFAULT_MULTICAST_VNI.as_u32(), + "oxide_vpc::api::DEFAULT_MULTICAST_VNI must equal omicron_common Vni::DEFAULT_MULTICAST_VNI", +); + /// Information about the gateway for an OPTE port #[derive(Debug, Clone, Copy)] #[allow(dead_code)] diff --git a/illumos-utils/src/opte/non_illumos.rs b/illumos-utils/src/opte/non_illumos.rs index 5a1c46b33e1..bfe3e41c46a 100644 --- a/illumos-utils/src/opte/non_illumos.rs +++ b/illumos-utils/src/opte/non_illumos.rs @@ -4,7 +4,7 @@ //! Mock / dummy versions of the OPTE module, for non-illumos platforms. //! -//! Most methods are either `unimplemented!()` or silent no-ops. +//! Most methods are either `unimplemented!()` or silent noops. //! Multicast subscribe/unsubscribe is an exception, as it maintains real //! in-memory state because port manager tests assert on subscription contents. @@ -23,6 +23,7 @@ use oxide_vpc::api::IpCidr; use oxide_vpc::api::ListPortsResp; use oxide_vpc::api::McastSubscribeReq; use oxide_vpc::api::McastUnsubscribeReq; +use oxide_vpc::api::MulticastUnderlay; use oxide_vpc::api::NoResp; use oxide_vpc::api::PortInfo; use oxide_vpc::api::RouterClass; @@ -36,6 +37,7 @@ use oxide_vpc::api::SourceFilter; use oxide_vpc::api::VpcCfg; use sled_agent_types::inventory::NetworkInterfaceKind; use slog::Logger; +use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::hash_map::Entry; use std::net::IpAddr; @@ -93,6 +95,12 @@ pub enum Error { "address {0} is not within the underlay multicast subnet (ff04::/16)" )] InvalidMcastUnderlay(std::net::Ipv6Addr), + + #[error( + "failed to install NIC multicast MAC filter for underlay {0}, \ + caller should retry" + )] + UnderlayMcastJoinFailed(std::net::Ipv6Addr), } pub fn initialize_xde_driver( @@ -197,6 +205,12 @@ pub(crate) struct PortData { pub(crate) struct State { pub ports: HashMap, pub underlay_initialized: bool, + /// Multicast-to-physical mappings, keyed by group. + /// + /// Persisted across [`Handle`] lifetimes to simulate xde kernel state + /// surviving sled-agent restarts. Mirrors the upsert-by-group + /// semantics of xde's `Mcast2Phys`. + pub m2p: BTreeMap, } const NO_RESPONSE: NoResp = NoResp { unused: 99 }; @@ -204,7 +218,11 @@ static OPTE_STATE: OnceLock> = OnceLock::new(); fn opte_state() -> &'static Mutex { OPTE_STATE.get_or_init(|| { - Mutex::new(State { ports: HashMap::new(), underlay_initialized: false }) + Mutex::new(State { + ports: HashMap::new(), + underlay_initialized: false, + m2p: BTreeMap::new(), + }) }) } @@ -379,15 +397,22 @@ impl Handle { } /// Set a multicast-to-physical mapping. - pub fn set_m2p(&self, _: &SetMcast2PhysReq) -> Result { + pub fn set_m2p(&self, req: &SetMcast2PhysReq) -> Result { + let mut state = opte_state().lock().unwrap(); + // Upsert: replaces any existing entry for the same group. + state.m2p.insert(req.group, req.underlay); Ok(NO_RESPONSE) } /// Clear a multicast-to-physical mapping. pub fn clear_m2p( &self, - _: &ClearMcast2PhysReq, + req: &ClearMcast2PhysReq, ) -> Result { + let mut state = opte_state().lock().unwrap(); + if state.m2p.get(&req.group) == Some(&req.underlay) { + state.m2p.remove(&req.group); + } Ok(NO_RESPONSE) } @@ -409,7 +434,20 @@ impl Handle { /// Dump all multicast-to-physical mappings. pub fn dump_m2p(&self) -> Result { - Ok(DumpMcast2PhysResp { ip4: Vec::new(), ip6: Vec::new() }) + let state = opte_state().lock().unwrap(); + let mut ip4 = Vec::new(); + let mut ip6 = Vec::new(); + for (group, underlay) in &state.m2p { + match group { + oxide_vpc::api::IpAddr::Ip4(v4) => { + ip4.push((*v4, *underlay)); + } + oxide_vpc::api::IpAddr::Ip6(v6) => { + ip6.push((*v6, *underlay)); + } + } + } + Ok(DumpMcast2PhysResp { ip4, ip6 }) } /// Dump all multicast forwarding entries. diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index 796ae8f6cc5..75713da200f 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -4,6 +4,7 @@ //! Manager for all OPTE ports on a Helios system +use crate::addrobj::AddrObject; use crate::dladm::OPTE_LINK_PREFIX; use crate::opte::AttachedSubnet; use crate::opte::EnsureAttachedSubnetResult; @@ -89,6 +90,7 @@ use std::collections::HashSet; use std::net::IpAddr; use std::net::Ipv4Addr; use std::net::Ipv6Addr; +use std::net::UdpSocket; use std::sync::Arc; use std::sync::Mutex; use std::sync::atomic::AtomicU64; @@ -119,6 +121,151 @@ impl PortState { } } +/// This owns the UDP sockets that hold open NIC multicast MAC filters for +/// underlay multicast addresses (see [opte#908] for context). +/// +/// Encapsulated so the leaf-lock invariant is structural rather than +/// manually enforced: methods on this type take only what they need by +/// parameter (`underlay_nics`) and have no back-ref to `PortManager` or +/// `PortManagerInner`. Code inside the locked region cannot reach the other +/// inner locks because it does not have access to them. +/// +/// [opte#908]: https://github.com/oxidecomputer/opte/issues/908 +#[derive(Debug)] +struct MulticastFilterMap { + sockets: Mutex>, +} + +impl MulticastFilterMap { + fn new() -> Self { + Self { sockets: Mutex::new(HashMap::new()) } + } + + /// Join the underlay multicast group `addr` on every NIC in + /// `underlay_nics`, holding a UDP socket open per address to keep + /// the NIC MAC filter installed. + /// + /// # Returns + /// + /// `true` if a socket already exists or was successfully created on + /// at least one NIC; `false` otherwise. + fn join( + &self, + log: &Logger, + addr: Ipv6Addr, + underlay_nics: &[AddrObject], + ) -> bool { + if underlay_nics.is_empty() { + return false; + } + + let mut sockets = self.sockets.lock().unwrap(); + if sockets.contains_key(&addr) { + return true; + } + + let sock = match UdpSocket::bind((Ipv6Addr::UNSPECIFIED, 0)) { + Ok(s) => s, + Err(e) => { + warn!( + log, + "Failed to bind UDP socket for underlay multicast filter"; + "addr" => %addr, + "error" => %e, + ); + return false; + } + }; + + // Minimize the receive buffer. + // + // This socket exists solely to trigger MAC filter programming. xde + // intercepts packets before they reach the socket. The small buffer + // limits resource waste if that invariant is ever violated. + if let Err(e) = sock.set_nonblocking(true) { + warn!( + log, + "Failed to set underlay multicast socket non-blocking"; + "addr" => %addr, + "error" => %e, + ); + } + + // The kernel may round up from 1 to its own minimum. + let _ = nix::sys::socket::setsockopt( + &sock, + nix::sys::socket::sockopt::RcvBuf, + &1, + ); + + let joined_any = underlay_nics + .iter() + .filter_map(|nic| { + let nic_name = nic.interface(); + let if_index = nix::net::if_::if_nametoindex(nic_name) + .map_err(|e| { + warn!( + log, + "Failed to resolve underlay NIC index"; + "nic" => nic_name, + "error" => %e, + ); + }) + .ok()?; + + sock.join_multicast_v6(&addr, if_index) + .map_err(|e| { + warn!( + log, + "Failed to join underlay multicast group on NIC"; + "addr" => %addr, + "nic" => nic_name, + "if_index" => if_index, + "error" => %e, + ); + }) + .ok()?; + + debug!( + log, + "Joined underlay multicast group on NIC"; + "addr" => %addr, + "nic" => nic_name, + "if_index" => if_index, + ); + Some(()) + }) + .count() + > 0; + + if joined_any { + sockets.insert(addr, sock); + true + } else { + warn!( + log, + "no NIC joins succeeded for underlay multicast group, \ + will retry on next call"; + "addr" => %addr, + ); + false + } + } + + /// Drop the UDP socket for an underlay multicast address, removing + /// the NIC MAC filter entries. + fn leave(&self, log: &Logger, addr: Ipv6Addr) { + let mut sockets = self.sockets.lock().unwrap(); + if sockets.remove(&addr).is_some() { + debug!( + log, + "Removed underlay multicast filter socket"; + "addr" => %addr, + ); + } + } +} + /// Convert a `MulticastGroupCfg` into OPTE's `SourceFilter`. /// /// Empty sources maps to ASM (EXCLUDE with no entries, accepting all @@ -160,6 +307,28 @@ struct PortManagerInner { /// /// IGW IDs are specific to the VPC of each NIC. eip_gateways: Mutex>>>, + + /// Underlay NIC address objects (e.g., for "cxgbe0", "cxgbe1"). + /// + /// This is used to program NIC multicast MAC filters via + /// [`UdpSocket::join_multicast_v6`]. + // Note: Empty in tests where no real underlay NICs exist. + underlay_nics: Vec, + + /// UDP sockets held open to maintain NIC multicast MAC filters. + /// + /// On Chelsio T6 hardware the NIC will not deliver multicast frames to + /// xde unless the corresponding multicast MAC filter is programmed. + /// Joining an IPv6 multicast group on a UDP socket causes the + /// kernel to call `mac_multicast_add` on the interface, which + /// programs the filter. The socket receives no data (xde's + /// siphon/flow hook intercepts first) and exists solely to hold + /// the filter entry. + /// + /// Dropping the socket removes the filter. + /// + /// See . + mcast_underlay_sockets: MulticastFilterMap, } impl PortManagerInner { @@ -378,8 +547,22 @@ pub struct PortManager { } impl PortManager { - /// Create a new manager, for creating OPTE ports - pub fn new(log: Logger, underlay_ip: Ipv6Addr) -> Self { + /// Create a new manager, for creating OPTE ports. + /// + /// # Arguments + /// + /// * `log`: Logger inherited by the manager and its ports. + /// * `underlay_ip`: This sled's underlay IPv6 address. + /// * `underlay_nics`: Underlay NIC interfaces for multicast MAC + /// filter rehydration. When non-empty, the constructor performs + /// kernel I/O: one ioctl to list existing multicast-to-physical + /// (M2P) mappings, then one `setsockopt(IPV6_JOIN_GROUP)` per + /// mapping per NIC. + pub fn new( + log: Logger, + underlay_ip: Ipv6Addr, + underlay_nics: &[AddrObject], + ) -> Self { let inner = Arc::new(PortManagerInner { log, next_port_id: AtomicU64::new(0), @@ -387,9 +570,110 @@ impl PortManager { ports: Mutex::new(BTreeMap::new()), routes: Mutex::new(Default::default()), eip_gateways: Mutex::new(Default::default()), + underlay_nics: underlay_nics.to_vec(), + mcast_underlay_sockets: MulticastFilterMap::new(), }); - Self { inner } + let mgr = Self { inner }; + + // Rehydrate MAC filter sockets for any M2P mappings that + // survived in the xde kernel module across a sled-agent + // restart. Without this, the NIC's multicast MAC filters + // are lost when the old process exits. + // + // Eager rehydration occurs here, not a lazy approach: the Nexus + // convergence loop's `converge_m2p` treats an M2P present on both + // DB and sled as already converged and never reapplies `set_mcast_m2p`, + // so a missing MAC filter would never be healed by convergence alone. + // + // Cost accrued: one `dump_m2p` ioctl + one `setsockopt(IPV6_JOIN_GROUP)` + // per remaining group per underlay NIC. This is bounded by active + // groups on this sled and runs only at sled-agent startup. + mgr.rehydrate_underlay_multicast_filters(); + + mgr + } + + /// Re-open underlay multicast filter sockets for M2P mappings + /// that already exist in the xde kernel module. + /// + /// Called at startup to cover the sled-agent restart case where + /// OPTE kernel state persists but userspace socket state is lost. + /// + /// On a cold boot (no prior xde state), `list_mcast_m2p` returns + /// an error or an empty list. + fn rehydrate_underlay_multicast_filters(&self) { + if self.inner.underlay_nics.is_empty() { + return; + } + + let mappings = match self.list_mcast_m2p() { + Ok(m) => m, + Err(e) => { + // Expected on cold boot when xde has no prior state. + debug!( + self.inner.log, + "No M2P mappings to rehydrate"; + "error" => InlineErrorChain::new(&e), + ); + return; + } + }; + + let mut failed: Vec = Vec::new(); + for mapping in &mappings { + if self.inner.mcast_underlay_sockets.join( + &self.inner.log, + mapping.underlay, + &self.inner.underlay_nics, + ) { + continue; + } + // Clear the surviving xde M2P entry so `converge_m2p` sees + // the gap on its next pass and re-issues `set_mcast_m2p`, + // which retries the underlay join. Without this, the entry + // stays in xde and convergence treats it as already + // converged, silently dropping traffic for the group until + // it is cycled inactive→active. + let clear_req = ClearMcast2Phys { + group: mapping.group, + underlay: mapping.underlay, + }; + if let Err(e) = self.clear_mcast_m2p(&clear_req) { + warn!( + self.inner.log, + "Failed to clear M2P after rehydration join failure, \ + group will silently drop traffic until convergence \ + retries"; + "group" => %mapping.group, + "underlay" => %mapping.underlay, + "error" => InlineErrorChain::new(&e), + ); + } + failed.push(mapping.underlay.to_string()); + } + + let total = mappings.len(); + let succeeded = total - failed.len(); + if !mappings.is_empty() { + info!( + self.inner.log, + "Rehydrated underlay multicast filter sockets"; + "succeeded" => succeeded, + "total" => total, + ); + } + if !failed.is_empty() { + warn!( + self.inner.log, + "Some underlay multicast filter sockets failed to \ + rehydrate; M2P entries cleared so convergence will \ + reissue on the next pass"; + "failed_count" => failed.len(), + "total" => total, + "failed_underlay_addrs" => ?failed, + ); + } } pub fn underlay_ip(&self) -> &Ipv6Addr { @@ -931,6 +1215,16 @@ impl PortManager { } /// Install a multicast overlay-to-underlay (M2P) mapping in OPTE. + /// + /// Also installs the corresponding multicast MAC filter on each underlay + /// NIC by joining the underlay IPv6 multicast group on a UDP socket, so + /// the NIC delivers frames to xde. See `mcast_underlay_sockets` docs. + /// + /// Out-of-band state (e.g., set via `opteadm`) is not consulted on this + /// path: the xde `set_m2p` ioctl is upsert by group, so any prior entry + /// for `req.group` is overwritten. The convergence loop reads xde via + /// `list_mcast_m2p` and reconciles against desired state on each pass, + /// so even unmanaged entries do not wedge convergence. pub fn set_mcast_m2p(&self, req: &Mcast2PhysMapping) -> Result<(), Error> { let addr: Ipv6Addr = req.underlay; @@ -945,10 +1239,55 @@ impl PortManager { .map_err(|_| Error::InvalidMcastUnderlay(addr))?; let hdl = Handle::new()?; hdl.set_m2p(&SetMcast2PhysReq { group: req.group.into(), underlay })?; + + // In legit scenarios, install the NIC multicast MAC filter via a + // UDP socket join. If no NIC accepts the join, roll back the + // xde M2P entry so the convergence loop (discovery via `list_mcast_m2p`) + // sees the gap on the next pass and retries `set_mcast_m2p`. Without + // this rollback mechanism the xde entry would remain and convergence + // would treat the mapping as already applied, silently dropping traffic + // for this group until the agent restarts or the mapping is cycled. + // + // In tests / sim mode, `underlay_nics` is empty and there is no + // MAC filter to install, so this whole step is skipped. + if !self.inner.underlay_nics.is_empty() { + let joined = self.inner.mcast_underlay_sockets.join( + &self.inner.log, + addr, + &self.inner.underlay_nics, + ); + if !joined { + warn!( + self.inner.log, + "underlay NIC join failed, rolling back xde M2P \ + entry to force convergence retry"; + "group" => %req.group, + "underlay" => %addr, + ); + if let Err(e) = hdl.clear_m2p(&ClearMcast2PhysReq { + group: req.group.into(), + underlay, + }) { + warn!( + self.inner.log, + "failed to roll back xde M2P entry after NIC \ + join failure, convergence may still drift"; + "group" => %req.group, + "underlay" => %addr, + "error" => %e, + ); + } + return Err(Error::UnderlayMcastJoinFailed(addr)); + } + } + Ok(()) } /// Remove a multicast overlay-to-underlay (M2P) mapping from OPTE. + /// + /// Drops the corresponding underlay MAC filter socket, removing the + /// NIC multicast MAC filter entry. pub fn clear_mcast_m2p(&self, req: &ClearMcast2Phys) -> Result<(), Error> { let addr: Ipv6Addr = req.underlay; @@ -966,6 +1305,9 @@ impl PortManager { group: req.group.into(), underlay, })?; + + self.inner.mcast_underlay_sockets.leave(&self.inner.log, addr); + Ok(()) } @@ -1540,6 +1882,8 @@ impl Drop for PortTicket { mod tests { use super::PortCreateParams; use super::PortManager; + #[cfg(target_os = "illumos")] + use crate::addrobj::AddrObject; use crate::opte::Error; use crate::opte::Handle; use macaddr::MacAddr6; @@ -1574,17 +1918,70 @@ mod tests { use std::net::IpAddr; use std::net::Ipv4Addr; use std::net::Ipv6Addr; + #[cfg(target_os = "illumos")] + use std::time::Duration; + #[cfg(target_os = "illumos")] + use std::time::Instant; use uuid::Uuid; // Maximum ephemeral port number for source NAT (14-bit range). const MAX_PORT: u16 = (1 << 14) - 1; + /// Loopback interface name on illumos. Tests that verify kernel + /// IPv6 multicast membership are illumos-only because they shell + /// out to illumos's `netstat -g -f inet6`. + #[cfg(target_os = "illumos")] + const LOOPBACK_IF: &str = "lo0"; + + /// Returns `true` iff `netstat -g -f inet6` reports `group` as a + /// membership on `interface`. + /// + /// Used to verify that `join_multicast_v6`/leave on the filter + /// socket actually reached the kernel's IP layer for the named + /// underlay NIC, rather than just updating the in-process + /// `mcast_underlay_sockets` map. + #[cfg(target_os = "illumos")] + fn netstat_v6_has_membership(interface: &str, group: &Ipv6Addr) -> bool { + let out = std::process::Command::new("netstat") + .args(["-g", "-n", "-f", "inet6"]) + .output() + .expect("netstat -g invocation failed"); + let group_str = group.to_string(); + String::from_utf8_lossy(&out.stdout).lines().any(|line| { + let mut fields = line.split_whitespace(); + if let (Some(iface), Some(grp)) = (fields.next(), fields.next()) { + iface == interface && grp == group_str + } else { + false + } + }) + } + + /// Poll `netstat -g` until membership matches `expected`, panicking + /// on timeout. The kernel should update synchronously on the join + /// or leave syscall, but polling tolerates any transient delay. + #[cfg(target_os = "illumos")] + fn poll_v6_membership(interface: &str, group: &Ipv6Addr, expected: bool) { + let deadline = Instant::now() + Duration::from_secs(5); + while Instant::now() < deadline { + if netstat_v6_has_membership(interface, group) == expected { + return; + } + std::thread::sleep(Duration::from_millis(100)); + } + panic!( + "timeout: membership for {group} on {interface} expected {}", + if expected { "present" } else { "absent" } + ); + } + // Regression for https://github.com/oxidecomputer/omicron/issues/7541. #[test] fn multiple_ports_does_not_destroy_default_route() { let logctx = test_setup_log("multiple_ports_does_not_destroy_default_route"); - let manager = PortManager::new(logctx.log.clone(), Ipv6Addr::LOCALHOST); + let manager = + PortManager::new(logctx.log.clone(), Ipv6Addr::LOCALHOST, &[]); let default_ipv4_route = IpNet::V4(Ipv4Net::new(Ipv4Addr::UNSPECIFIED, 0).unwrap()); let default_ipv6_route = @@ -2319,7 +2716,8 @@ mod tests { #[test] fn multicast_groups_ensure_diffing() { let logctx = test_setup_log("multicast_groups_ensure_diffing"); - let manager = PortManager::new(logctx.log.clone(), Ipv6Addr::LOCALHOST); + let manager = + PortManager::new(logctx.log.clone(), Ipv6Addr::LOCALHOST, &[]); let handle = Handle::new().unwrap(); handle.set_xde_underlay("underlay0", "underlay1").unwrap(); @@ -2462,7 +2860,8 @@ mod tests { #[test] fn multicast_port_deletion_cleanup() { let logctx = test_setup_log("multicast_port_deletion_cleanup"); - let manager = PortManager::new(logctx.log.clone(), Ipv6Addr::LOCALHOST); + let manager = + PortManager::new(logctx.log.clone(), Ipv6Addr::LOCALHOST, &[]); let handle = Handle::new().unwrap(); handle.set_xde_underlay("underlay0", "underlay1").unwrap(); @@ -2557,7 +2956,8 @@ mod tests { #[test] fn multicast_ensure_missing_port_error() { let logctx = test_setup_log("multicast_ensure_missing_port_error"); - let manager = PortManager::new(logctx.log.clone(), Ipv6Addr::LOCALHOST); + let manager = + PortManager::new(logctx.log.clone(), Ipv6Addr::LOCALHOST, &[]); let nic_id = Uuid::new_v4(); let nic_kind = NetworkInterfaceKind::Instance { id: Uuid::new_v4() }; @@ -2581,4 +2981,428 @@ mod tests { logctx.cleanup_successful(); } + + /// Verify that `set_mcast_m2p` installs the multicast MAC filter on + /// each underlay NIC via UDP socket join and that `clear_mcast_m2p` + /// removes them. + /// + /// Asserts both the in-process `mcast_underlay_sockets` bookkeeping + /// and kernel-level IPv6 group membership on the underlay interface + /// (observable via `netstat -g -f inet6`). Kernel-level verification + /// is what ensures `join_multicast_v6` actually reached IP and, on + /// actual hardware, would drive `mac_multicast_add` to program the + /// NIC filter. + #[cfg(target_os = "illumos")] + #[test] + fn underlay_multicast_mac_filter_lifecycle() { + let logctx = test_setup_log("underlay_multicast_mac_filter_lifecycle"); + let nics = vec![AddrObject::new_control(LOOPBACK_IF).unwrap()]; + let manager = + PortManager::new(logctx.log.clone(), Ipv6Addr::LOCALHOST, &nics); + + let handle = Handle::new().unwrap(); + handle.set_xde_underlay("underlay0", "underlay1").unwrap(); + + // ff04::1 is within the underlay multicast subnet. + let underlay: Ipv6Addr = "ff04::1".parse().unwrap(); + let group: IpAddr = "239.10.10.1".parse().unwrap(); + + let req = + sled_agent_types::multicast::Mcast2PhysMapping { group, underlay }; + + // Prefligt: the group must not already be joined on the + // underlay interface. + assert!( + !netstat_v6_has_membership(LOOPBACK_IF, &underlay), + "unexpected pre-existing membership {underlay} on {LOOPBACK_IF}", + ); + + // Set M2P -> socket should be created and kernel should show join. + manager.set_mcast_m2p(&req).unwrap(); + { + let sockets = + manager.inner.mcast_underlay_sockets.sockets.lock().unwrap(); + assert!( + sockets.contains_key(&underlay), + "Socket should exist after set_mcast_m2p" + ); + } + poll_v6_membership(LOOPBACK_IF, &underlay, true); + + // Setting the same M2P again should be idempotent. + manager.set_mcast_m2p(&req).unwrap(); + { + let sockets = + manager.inner.mcast_underlay_sockets.sockets.lock().unwrap(); + assert_eq!( + sockets.len(), + 1, + "Duplicate set_mcast_m2p should not create extra sockets" + ); + } + assert!( + netstat_v6_has_membership(LOOPBACK_IF, &underlay), + "membership should still be present after idempotent re-set" + ); + + // Clear M2P -> socket should be removed and kernel membership gone. + let clear_req = + sled_agent_types::multicast::ClearMcast2Phys { group, underlay }; + manager.clear_mcast_m2p(&clear_req).unwrap(); + { + let sockets = + manager.inner.mcast_underlay_sockets.sockets.lock().unwrap(); + assert!( + !sockets.contains_key(&underlay), + "Socket should be removed after clear_mcast_m2p" + ); + } + poll_v6_membership(LOOPBACK_IF, &underlay, false); + + logctx.cleanup_successful(); + } + + /// Verify that rehydration at startup reopens filter sockets for + /// M2P mappings that survived in mock xde state across a + /// PortManager drop (which simulates sled-agent restart). + #[cfg(target_os = "illumos")] + #[test] + fn underlay_multicast_mac_filter_rehydration() { + let logctx = + test_setup_log("underlay_multicast_mac_filter_rehydration"); + let nics = vec![AddrObject::new_control(LOOPBACK_IF).unwrap()]; + + let handle = Handle::new().unwrap(); + handle.set_xde_underlay("underlay0", "underlay1").unwrap(); + + // Use a distinct underlay address to avoid collisions with + // other tests sharing the static OPTE_STATE. + let underlay: Ipv6Addr = "ff04::99".parse().unwrap(); + let group: IpAddr = "239.10.10.99".parse().unwrap(); + + let req = + sled_agent_types::multicast::Mcast2PhysMapping { group, underlay }; + + // First phase: first PortManager sets M2P (populates mock xde state). + { + let mgr1 = PortManager::new( + logctx.log.clone(), + Ipv6Addr::LOCALHOST, + &nics, + ); + mgr1.set_mcast_m2p(&req).unwrap(); + { + let sockets = + mgr1.inner.mcast_underlay_sockets.sockets.lock().unwrap(); + assert!(sockets.contains_key(&underlay)); + } + poll_v6_membership(LOOPBACK_IF, &underlay, true); + } + + // mgr1 dropped: socket closed, kernel membership removed. + poll_v6_membership(LOOPBACK_IF, &underlay, false); + + // Mock xde state (static) still has the M2P entry, simulating + // xde kernel state surviving a sled-agent restart. + { + let hdl = Handle::new().unwrap(); + let dump = hdl.dump_m2p().unwrap(); + assert!( + !dump.ip4.is_empty() || !dump.ip6.is_empty(), + "Mock xde should still hold the M2P mapping after drop" + ); + } + + // Second phase: new PortManager rehydrates from surviving xde state. + let mgr2 = + PortManager::new(logctx.log.clone(), Ipv6Addr::LOCALHOST, &nics); + { + let sockets = + mgr2.inner.mcast_underlay_sockets.sockets.lock().unwrap(); + assert!( + sockets.contains_key(&underlay), + "Rehydration should reopen socket for surviving M2P" + ); + } + poll_v6_membership(LOOPBACK_IF, &underlay, true); + + // Cleanup and clear the M2P. + let clear_req = + sled_agent_types::multicast::ClearMcast2Phys { group, underlay }; + mgr2.clear_mcast_m2p(&clear_req).unwrap(); + poll_v6_membership(LOOPBACK_IF, &underlay, false); + + logctx.cleanup_successful(); + } + + /// Verify that no sockets are created when no underlay NICs are + /// configured (test/sim mode). + #[test] + fn underlay_multicast_mac_filter_no_nics() { + let logctx = test_setup_log("underlay_multicast_mac_filter_no_nics"); + let manager = + PortManager::new(logctx.log.clone(), Ipv6Addr::LOCALHOST, &[]); + + let handle = Handle::new().unwrap(); + handle.set_xde_underlay("underlay0", "underlay1").unwrap(); + + let underlay: Ipv6Addr = "ff04::2".parse().unwrap(); + let group: IpAddr = "239.10.10.2".parse().unwrap(); + + let req = + sled_agent_types::multicast::Mcast2PhysMapping { group, underlay }; + + manager.set_mcast_m2p(&req).unwrap(); + { + let sockets = + manager.inner.mcast_underlay_sockets.sockets.lock().unwrap(); + assert!( + sockets.is_empty(), + "No sockets should be created without underlay NICs" + ); + } + + logctx.cleanup_successful(); + } + + /// Verify that setting the same group again with a different underlay + /// replaces the prior mapping rather than accumulating entries. + /// + /// This pins the upsert-by-group semantics of the mock's `m2p` map, + /// matching xde's `Mcast2Phys` shape. A regression here would surface + /// as duplicate entries in `list_mcast_m2p` and prevent the reconciler + /// from converging. + #[test] + fn multicast_m2p_set_replaces_underlay_for_same_group() { + let logctx = test_setup_log( + "multicast_m2p_set_replaces_underlay_for_same_group", + ); + let manager = + PortManager::new(logctx.log.clone(), Ipv6Addr::LOCALHOST, &[]); + + let handle = Handle::new().unwrap(); + handle.set_xde_underlay("underlay0", "underlay1").unwrap(); + + let group: IpAddr = "239.10.10.41".parse().unwrap(); + let underlay_x: Ipv6Addr = "ff04::a1".parse().unwrap(); + let underlay_y: Ipv6Addr = "ff04::a2".parse().unwrap(); + + manager + .set_mcast_m2p(&sled_agent_types::multicast::Mcast2PhysMapping { + group, + underlay: underlay_x, + }) + .unwrap(); + manager + .set_mcast_m2p(&sled_agent_types::multicast::Mcast2PhysMapping { + group, + underlay: underlay_y, + }) + .unwrap(); + + let mappings: Vec<_> = manager + .list_mcast_m2p() + .unwrap() + .into_iter() + .filter(|m| m.group == group) + .collect(); + assert_eq!( + mappings.len(), + 1, + "expected a single mapping for group after setting it again, got {mappings:?}", + ); + assert_eq!(mappings[0].underlay, underlay_y); + + // Cleanup so other tests sharing the static OPTE_STATE start clean. + manager + .clear_mcast_m2p(&sled_agent_types::multicast::ClearMcast2Phys { + group, + underlay: underlay_y, + }) + .unwrap(); + + logctx.cleanup_successful(); + } + + /// Verify that `clear_mcast_m2p` with a mismatched underlay is a + /// noop, preserving the prior mapping. This pins the requirement + /// that `clear_mcast_m2p` only removes the entry when both `group` + /// and `underlay` match. + #[test] + fn multicast_m2p_clear_mismatched_underlay_is_noop() { + let logctx = + test_setup_log("multicast_m2p_clear_mismatched_underlay_is_noop"); + let manager = + PortManager::new(logctx.log.clone(), Ipv6Addr::LOCALHOST, &[]); + + let handle = Handle::new().unwrap(); + handle.set_xde_underlay("underlay0", "underlay1").unwrap(); + + let group: IpAddr = "239.10.10.42".parse().unwrap(); + let underlay_x: Ipv6Addr = "ff04::b1".parse().unwrap(); + let underlay_z: Ipv6Addr = "ff04::b2".parse().unwrap(); + + manager + .set_mcast_m2p(&sled_agent_types::multicast::Mcast2PhysMapping { + group, + underlay: underlay_x, + }) + .unwrap(); + + // Clear with a different underlay than was set (not removing the + // original mapping). + manager + .clear_mcast_m2p(&sled_agent_types::multicast::ClearMcast2Phys { + group, + underlay: underlay_z, + }) + .unwrap(); + + let mappings: Vec<_> = manager + .list_mcast_m2p() + .unwrap() + .into_iter() + .filter(|m| m.group == group) + .collect(); + assert_eq!( + mappings.len(), + 1, + "mismatched clear should be a noop, expected mapping to remain", + ); + assert_eq!(mappings[0].underlay, underlay_x); + + // Cleanup for shared static OPTE_STATE. + manager + .clear_mcast_m2p(&sled_agent_types::multicast::ClearMcast2Phys { + group, + underlay: underlay_x, + }) + .unwrap(); + + logctx.cleanup_successful(); + } + + /// Verify that `set_mcast_m2p` rejects an underlay address outside the + /// admin-local multicast subnet (ff04::/16) with the proper error. + #[test] + fn multicast_m2p_set_rejects_non_admin_local_underlay() { + let logctx = test_setup_log( + "multicast_m2p_set_rejects_non_admin_local_underlay", + ); + let manager = + PortManager::new(logctx.log.clone(), Ipv6Addr::LOCALHOST, &[]); + + let handle = Handle::new().unwrap(); + handle.set_xde_underlay("underlay0", "underlay1").unwrap(); + + let group: IpAddr = "239.10.10.51".parse().unwrap(); + let bad_underlay: Ipv6Addr = "fd00::1".parse().unwrap(); + + let err = manager + .set_mcast_m2p(&sled_agent_types::multicast::Mcast2PhysMapping { + group, + underlay: bad_underlay, + }) + .expect_err("non-admin-local underlay must be rejected"); + assert!( + matches!(err, Error::InvalidMcastUnderlay(addr) if addr == bad_underlay), + "expected InvalidMcastUnderlay({bad_underlay}), got {err:?}", + ); + + logctx.cleanup_successful(); + } + + /// Verify that `clear_mcast_m2p` rejects an underlay address outside + /// the admin-local multicast subnet (ff04::/16) with the proper error. + #[test] + fn multicast_m2p_clear_rejects_non_admin_local_underlay() { + let logctx = test_setup_log( + "multicast_m2p_clear_rejects_non_admin_local_underlay", + ); + let manager = + PortManager::new(logctx.log.clone(), Ipv6Addr::LOCALHOST, &[]); + + let handle = Handle::new().unwrap(); + handle.set_xde_underlay("underlay0", "underlay1").unwrap(); + + let group: IpAddr = "239.10.10.52".parse().unwrap(); + let bad_underlay: Ipv6Addr = "fd00::2".parse().unwrap(); + + let err = manager + .clear_mcast_m2p(&sled_agent_types::multicast::ClearMcast2Phys { + group, + underlay: bad_underlay, + }) + .expect_err("non-admin-local underlay must be rejected"); + assert!( + matches!(err, Error::InvalidMcastUnderlay(addr) if addr == bad_underlay), + "expected InvalidMcastUnderlay({bad_underlay}), got {err:?}", + ); + + logctx.cleanup_successful(); + } + + /// Verify that when the NIC multicast MAC filter join fails (no NIC + /// accepts the join), `set_mcast_m2p` rolls back the xde M2P entry + /// and returns the proper error. + /// + /// Without rollback the xde entry would stay present and the + /// `list_mcast_m2p`-checked convergence loop would treat the mapping + /// as already applied, silently dropping traffic for the group. + /// + /// We force the failure by handing PortManager an `AddrObject` + /// whose interface name does not exist on the host, so every + /// per-NIC `nix::if_nametoindex` lookup fails and the join returns + /// false. + #[cfg(target_os = "illumos")] + #[test] + fn multicast_m2p_set_rolls_back_on_nic_join_failure() { + let logctx = + test_setup_log("multicast_m2p_set_rolls_back_on_nic_join_failure"); + // Fake NIC name that `if_nametoindex` will reject. `AddrObject` + // only validates that the name contains no slashes, so this + // constructs fine. + let nics = vec![ + AddrObject::new_control("nonexistent_xyz_nic_for_test").unwrap(), + ]; + let manager = + PortManager::new(logctx.log.clone(), Ipv6Addr::LOCALHOST, &nics); + + let handle = Handle::new().unwrap(); + handle.set_xde_underlay("underlay0", "underlay1").unwrap(); + + let group: IpAddr = "239.10.10.61".parse().unwrap(); + let underlay: Ipv6Addr = "ff04::c1".parse().unwrap(); + let req = + sled_agent_types::multicast::Mcast2PhysMapping { group, underlay }; + + let err = manager + .set_mcast_m2p(&req) + .expect_err("set_mcast_m2p must fail when every NIC join fails"); + assert!( + matches!(err, Error::UnderlayMcastJoinFailed(addr) if addr == underlay), + "expected UnderlayMcastJoinFailed({underlay}), got {err:?}", + ); + + // The xde M2P entry must have been rolled back so the + // convergence loop sees the gap and retries on the next pass. + let listed = + manager.list_mcast_m2p().expect("list_mcast_m2p succeeds in mock"); + assert!( + !listed.iter().any(|m| m.group == group), + "xde M2P entry must be rolled back after join failure, found {listed:?}", + ); + + // No socket should be present either. + { + let sockets = + manager.inner.mcast_underlay_sockets.sockets.lock().unwrap(); + assert!( + !sockets.contains_key(&underlay), + "no filter socket should exist after join failure", + ); + } + + logctx.cleanup_successful(); + } } diff --git a/internal-dns/resolver/src/resolver.rs b/internal-dns/resolver/src/resolver.rs index 4c046ef5bff..d6a66094364 100644 --- a/internal-dns/resolver/src/resolver.rs +++ b/internal-dns/resolver/src/resolver.rs @@ -345,6 +345,76 @@ impl Resolver { } } + /// Returns the SRV targets paired with their resolved IPv6 sockets. + /// + /// Like [`Resolver::lookup_all_socket_v6`], but preserves the SRV + /// target name so callers can correlate sockets back to their + /// source. Per-target IPv6 lookups are best-effort: failures for a + /// given target are logged and the entry is dropped from the result. + pub async fn lookup_all_socket_v6_by_target( + &self, + service: ServiceName, + ) -> Result, ResolveError> { + let name = service.srv_name(); + trace!(self.log, "lookup_all_socket_v6_by_target srv"; "dns_name" => &name); + let response = self.resolver.srv_lookup(&name).await?; + debug!( + self.log, + "lookup_all_socket_v6_by_target srv"; + "dns_name" => &name, + "response" => ?response + ); + + let futures = + std::iter::repeat((self.log.clone(), self.resolver.clone())) + .zip(response.into_iter()) + .map(|((log, resolver), srv)| async move { + let target = srv.target().to_string(); + let port = srv.port(); + trace!( + log, + "lookup_all_socket_v6_by_target: looking up SRV target"; + "name" => &target, + ); + resolver + .ipv6_lookup(target.clone()) + .await + .map(|ips| (target.clone(), ips, port)) + .map_err(|err| (target, err)) + }); + + let log = self.log.clone(); + let pairs: Vec<(String, SocketAddrV6)> = futures::future::join_all( + futures, + ) + .await + .into_iter() + .flat_map(move |res| match res { + Ok((target, ips, port)) => ips + .into_iter() + .map(|ip| { + (target.clone(), SocketAddrV6::new(ip.into(), port, 0, 0)) + }) + .collect::>(), + Err((target, err)) => { + error!( + log, + "lookup_all_socket_v6_by_target: failed looking up target"; + "name" => %target, + "error" => ?err, + ); + Vec::new() + } + }) + .collect(); + + if pairs.is_empty() { + Err(ResolveError::NotFound(service)) + } else { + Ok(pairs) + } + } + // Returns an iterator of SocketAddrs for the specified SRV name. // // Acts on a raw string for compatibility with the reqwest::dns::Resolve diff --git a/nexus-config/src/nexus_config.rs b/nexus-config/src/nexus_config.rs index 56a10a87773..e4531fb00db 100644 --- a/nexus-config/src/nexus_config.rs +++ b/nexus-config/src/nexus_config.rs @@ -934,38 +934,59 @@ pub struct MulticastGroupReconcilerConfig { #[serde_as(as = "DurationSeconds")] pub period_secs: Duration, - /// TTL (in seconds) for the sled-to-switch-port mapping cache. + /// Maximum number of groups to process concurrently per reconciler pass. /// - /// This cache maps sled IDs to their physical switch ports. It changes when - /// sleds are added/removed or inventory is updated. + /// Each slot may hold an in-flight `dpd_ensure` saga that the reconciler + /// awaits in the same pass. A wedged dependency holds a slot until the + /// saga unwinds, so this cap bounds the worst-case in-flight saga count. + #[serde( + default = "MulticastGroupReconcilerConfig::default_group_concurrency_limit" + )] + pub group_concurrency_limit: usize, + + /// Maximum number of members to process concurrently per group. /// - /// Default: 3600 seconds (1 hour) + /// Members do not start sagas. Per-member work involves HTTP calls to + /// DPD and sled-agent. The default accounts for the outer + /// `group_concurrency_limit` multiplier, since peak in-flight + /// per-member futures is the product of the two limits. #[serde( - default = "MulticastGroupReconcilerConfig::default_sled_cache_ttl_secs" + default = "MulticastGroupReconcilerConfig::default_member_concurrency_limit" )] - #[serde_as(as = "DurationSeconds")] - pub sled_cache_ttl_secs: Duration, + pub member_concurrency_limit: usize, - /// TTL (in seconds) for the backplane hardware topology cache. + /// Grace period before an orphaned "Creating" group with no members is + /// reaped by the emptiness sweep. /// - /// This cache stores the hardware platform's port mapping. It effectively - /// never changes during normal operation. + /// Implicit group creation and first-member attach are not atomic, and even + /// static membership must propagate through the ddm/MRIB exchange before a + /// group is live. This grace gates reaping on the group's creation time so a + /// group whose first attach is still in flight is not collected prematurely. /// - /// Default: 86400 seconds (24 hours) with smart invalidation + /// The value should bound that worst-case attach-and-propagate latency with + /// margin. A future IGMP/MLD hold-down for emptied "Active" groups (RFD + /// 0488) is a sibling timer, not a reuse of this knob: snooped membership + /// expiry must derive from the protocol's Group Membership Interval + /// (robustness variable x query interval + max response time, ~260s with + /// RFC 3376 defaults), which is far longer than this grace. + #[serde_as(as = "DurationSeconds")] #[serde( - default = "MulticastGroupReconcilerConfig::default_backplane_cache_ttl_secs" + default = "MulticastGroupReconcilerConfig::default_orphan_grace_secs" )] - #[serde_as(as = "DurationSeconds")] - pub backplane_cache_ttl_secs: Duration, + pub orphan_grace_secs: Duration, } impl MulticastGroupReconcilerConfig { - const fn default_sled_cache_ttl_secs() -> Duration { - Duration::from_secs(3600) // 1 hour + const fn default_group_concurrency_limit() -> usize { + 16 + } + + const fn default_member_concurrency_limit() -> usize { + 32 } - const fn default_backplane_cache_ttl_secs() -> Duration { - Duration::from_secs(86400) // 24 hours + const fn default_orphan_grace_secs() -> Duration { + Duration::from_secs(60) } } @@ -973,8 +994,9 @@ impl Default for MulticastGroupReconcilerConfig { fn default() -> Self { Self { period_secs: Duration::from_secs(60), - sled_cache_ttl_secs: Self::default_sled_cache_ttl_secs(), - backplane_cache_ttl_secs: Self::default_backplane_cache_ttl_secs(), + group_concurrency_limit: Self::default_group_concurrency_limit(), + member_concurrency_limit: Self::default_member_concurrency_limit(), + orphan_grace_secs: Self::default_orphan_grace_secs(), } } } @@ -1604,8 +1626,9 @@ mod test { }, multicast_reconciler: MulticastGroupReconcilerConfig { period_secs: Duration::from_secs(60), - sled_cache_ttl_secs: MulticastGroupReconcilerConfig::default_sled_cache_ttl_secs(), - backplane_cache_ttl_secs: MulticastGroupReconcilerConfig::default_backplane_cache_ttl_secs(), + group_concurrency_limit: MulticastGroupReconcilerConfig::default_group_concurrency_limit(), + member_concurrency_limit: MulticastGroupReconcilerConfig::default_member_concurrency_limit(), + orphan_grace_secs: MulticastGroupReconcilerConfig::default_orphan_grace_secs(), }, trust_quorum: TrustQuorumConfig { period_secs: Duration::from_secs(60), diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index d5995aed929..6a93cb459d8 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -151,6 +151,7 @@ nexus-reconfigurator-rendezvous.workspace = true nexus-types.workspace = true nexus-types-versions.workspace = true omicron-common.workspace = true +omicron-ddm-admin-client.workspace = true omicron-passwords.workspace = true oxide-tokio-rt.workspace = true oximeter.workspace = true diff --git a/nexus/db-model/src/multicast_group.rs b/nexus/db-model/src/multicast_group.rs index ed23b3a650b..9b8801c9342 100644 --- a/nexus/db-model/src/multicast_group.rs +++ b/nexus/db-model/src/multicast_group.rs @@ -56,25 +56,29 @@ //! //! ## Member Lifecycle (handled by RPW) //! -//! Multicast group members follow a 3-state lifecycle managed by the -//! Reliable Persistent Workflow (RPW) reconciler: +//! Multicast group members follow a 3-state lifecycle managed by the Reliable +//! Persistent Workflow (RPW) reconciler. Nexus owns the member database +//! lifecycle, per-sled forwarding propagation, and OPTE subscriptions. +//! Rear-port underlay membership is not tracked per member. `ddmd` derives it +//! from DDM peer subscriptions and programs it into the switch dataplane via +//! mg-lower/DDM. +//! //! - ["Joining"](MulticastGroupMemberState::Joining): Member created, awaiting -//! dataplane configuration (via DPD) -//! - ["Joined"](MulticastGroupMemberState::Joined): Member configuration applied -//! in the dataplane, ready to receive multicast traffic -//! - ["Left"](MulticastGroupMemberState::Left): Member configuration removed from -//! the dataplane (e.g., instance stopping/stopped, explicit detach, delete) +//! group activation and sled assignment +//! - ["Joined"](MulticastGroupMemberState::Joined): Member is associated with +//! a live VMM. Nexus has propagated sled forwarding state and subscribed +//! the VMM's OPTE port to the group +//! - ["Left"](MulticastGroupMemberState::Left): Member should not receive +//! traffic once cleanup converges (e.g., instance stopping/stopped, explicit +//! detach, delete) //! -//! Migration note: during instance migration, membership is reconfigured in -//! place—the reconciler removes configuration from the old sled and applies it -//! on the new sled without transitioning the member to "Left". In other words, -//! migration is not considered leaving; the member generally remains "Joined" -//! while its `sled_id` and dataplane configuration are updated. -//! - If an instance is deleted, the member will be marked for removal with a -//! deleted timestamp, and the reconciler will remove it from the dataplane +//! Migration and deletion are handled specially: //! -//! The RPW ensures eventual consistency between database state and dataplane -//! configuration (applied via DPD to switches). +//! - Migration is not a leave: the reconciler reconfigures the member in place, +//! updating the recorded `sled_id`, re-propagating sled forwarding state, and +//! subscribing the VMM on the new sled while the member stays "Joined". +//! - Deletion marks the member for removal with a deleted timestamp, after +//! which the reconciler finishes cleanup. //! //! [`UNDERLAY_MULTICAST_SUBNET`]: omicron_common::address::UNDERLAY_MULTICAST_SUBNET @@ -121,6 +125,16 @@ impl_enum_type!( Left => b"left" ); +impl_enum_type!( + MulticastGroupMemberOriginEnum: + + #[derive(Clone, Copy, Debug, PartialEq, Eq, AsExpression, FromSqlRow, Serialize, Deserialize, JsonSchema)] + pub enum MulticastGroupMemberOrigin; + + Static => b"static" + IgmpSnooped => b"igmp_snooped" +); + impl std::fmt::Display for MulticastGroupState { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(match self { @@ -142,6 +156,15 @@ impl std::fmt::Display for MulticastGroupMemberState { } } +impl std::fmt::Display for MulticastGroupMemberOrigin { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + MulticastGroupMemberOrigin::Static => "Static", + MulticastGroupMemberOrigin::IgmpSnooped => "IgmpSnooped", + }) + } +} + /// Type alias for lookup resource naming convention. /// /// This alias maps the generic name [MulticastGroup] to [ExternalMulticastGroup], @@ -272,6 +295,14 @@ pub struct MulticastGroupMember { pub multicast_ip: IpNetwork, /// Source IPs for source-filtered multicast (optional for ASM, required for SSM). pub source_ips: Vec, + /// Origin of this membership. + /// + /// [`Static`](MulticastGroupMemberOrigin::Static) for administratively + /// configured (API) membership; + /// [`IgmpSnooped`](MulticastGroupMemberOrigin::IgmpSnooped) for dynamic + /// soft-state learned from snooped IGMP/MLD reports (RFD 488). Only snooped + /// rows are subject to soft-state expiry. + pub membership_origin: MulticastGroupMemberOrigin, } // Conversions to external API views @@ -369,6 +400,7 @@ impl MulticastGroupMember { sled_id, state: MulticastGroupMemberState::Joining, source_ips, + membership_origin: MulticastGroupMemberOrigin::Static, // Placeholder - will be overwritten by database sequence on insert version_added: Generation::new(), version_removed: None, diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index e0b188f44a6..b201f326b63 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -16,7 +16,7 @@ use std::{collections::BTreeMap, sync::LazyLock}; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: Version = Version::new(266, 0, 0); +pub const SCHEMA_VERSION: Version = Version::new(267, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -28,6 +28,7 @@ pub static KNOWN_VERSIONS: LazyLock> = LazyLock::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(267, "multicast-member-origin"), KnownVersion::new(266, "alert-version"), KnownVersion::new(265, "sled-resource-vmm-state"), KnownVersion::new(264, "instance-cpu-type-turin-v2"), diff --git a/nexus/db-queries/src/db/datastore/multicast/groups.rs b/nexus/db-queries/src/db/datastore/multicast/groups.rs index 41f19abcb3e..7cb13a5023e 100644 --- a/nexus/db-queries/src/db/datastore/multicast/groups.rs +++ b/nexus/db-queries/src/db/datastore/multicast/groups.rs @@ -408,7 +408,7 @@ impl DataStore { use nexus_db_schema::schema::multicast_group_member; let now = Utc::now(); - // Atomic: only mark `Deleting` if no active members exist. + // Atomically mark "Deleting" only if no active members exist. let rows = diesel::update(multicast_group::table) .filter(multicast_group::id.eq(group_id.into_untyped_uuid())) .filter( diff --git a/nexus/db-queries/src/db/datastore/multicast/members.rs b/nexus/db-queries/src/db/datastore/multicast/members.rs index c9f2cd712b9..f65c40cc753 100644 --- a/nexus/db-queries/src/db/datastore/multicast/members.rs +++ b/nexus/db-queries/src/db/datastore/multicast/members.rs @@ -2289,7 +2289,7 @@ mod tests { assert_eq!(unchanged_member.state, MulticastGroupMemberState::Joined); assert_eq!(unchanged_member.time_modified, before_modification); - // Test starting instance that has no multicast memberships (should be no-op) + // Test starting instance that has no multicast memberships (should be noop) let non_member_instance = InstanceUuid::new_v4(); datastore .multicast_group_member_set_instance_sled( @@ -2454,7 +2454,7 @@ mod tests { .await .expect("Should handle duplicate mark for removal"); - // Test marking instance with no memberships (should be no-op) + // Test marking instance with no memberships (should be noop) let non_member_instance = InstanceUuid::new_v4(); datastore .multicast_group_members_mark_for_removal( @@ -2672,7 +2672,7 @@ mod tests { .expect("Should list group2 members"); assert_eq!(group2_members.len(), 2); - // Test deleting from group with no members (should be no-op) + // Test deleting from group with no members (should be noop) datastore .multicast_group_members_delete_by_group( &opctx, @@ -2681,7 +2681,7 @@ mod tests { .await .expect("Should handle deleting from empty group"); - // Test deleting from nonexistent group (should be no-op) + // Test deleting from nonexistent group (should be noop) let fake_group_id = Uuid::new_v4(); datastore .multicast_group_members_delete_by_group( diff --git a/nexus/db-queries/src/db/datastore/multicast/ops/member_attach.rs b/nexus/db-queries/src/db/datastore/multicast/ops/member_attach.rs index 6c5b43cfc85..fc6189e8c1a 100644 --- a/nexus/db-queries/src/db/datastore/multicast/ops/member_attach.rs +++ b/nexus/db-queries/src/db/datastore/multicast/ops/member_attach.rs @@ -158,7 +158,7 @@ impl From for external::Error { /// - **Reactivate**: Member in "Left" (time_deleted=NULL) → transition to /// "Joining", update `sled_id` /// - **Insert new**: Member in "Left" (time_deleted set) → create new row -/// - **Idempotent**: Member already "Joining" or "Joined" → no-op +/// - **Idempotent**: Member already "Joining" or "Joined" → noop /// /// Atomically validates group and instance exist, retrieves instance's current /// sled_id, and performs member upsert. Returns member ID. @@ -450,11 +450,15 @@ impl AttachMemberToGroupStatement { // Return all columns so caller gets full member record // Column order must match schema: id, time_created, time_modified, time_deleted, // external_group_id, parent_id, sled_id, state, version_added, version_removed, - // multicast_ip, source_ips + // multicast_ip, source_ips, membership_origin + // + // membership_origin is not written by the INSERT (its column DEFAULT + // 'static' applies) nor by the ON CONFLICT path (a reactivated member + // preserves its original origin), so it is returned but never set here. out.push_sql( " RETURNING id, time_created, time_modified, time_deleted, \ external_group_id, parent_id, sled_id, state, version_added, \ - version_removed, multicast_ip, source_ips", + version_removed, multicast_ip, source_ips, membership_origin", ); Ok(()) } @@ -469,11 +473,11 @@ impl AttachMemberToGroupStatement { ) -> QueryResult<()> { // Column order must match schema: id, time_created, time_modified, time_deleted, // external_group_id, parent_id, sled_id, state, version_added, version_removed, - // multicast_ip, source_ips + // multicast_ip, source_ips, membership_origin out.push_sql( "SELECT id, time_created, time_modified, time_deleted, \ external_group_id, parent_id, sled_id, state, version_added, \ - version_removed, multicast_ip, source_ips \ + version_removed, multicast_ip, source_ips, membership_origin \ FROM upserted_member", ); Ok(()) diff --git a/nexus/db-schema/src/enums.rs b/nexus/db-schema/src/enums.rs index ee1e07fe7b4..0216e1e865c 100644 --- a/nexus/db-schema/src/enums.rs +++ b/nexus/db-schema/src/enums.rs @@ -81,6 +81,7 @@ define_enums! { MigrationStateEnum => "migration_state", MulticastGroupStateEnum => "multicast_group_state", MulticastGroupMemberStateEnum => "multicast_group_member_state", + MulticastGroupMemberOriginEnum => "multicast_group_member_origin", NetworkInterfaceKindEnum => "network_interface_kind", OximeterReadModeEnum => "oximeter_read_mode", PhysicalDiskKindEnum => "physical_disk_kind", diff --git a/nexus/db-schema/src/schema.rs b/nexus/db-schema/src/schema.rs index fe59b15cb95..14139d7bae2 100644 --- a/nexus/db-schema/src/schema.rs +++ b/nexus/db-schema/src/schema.rs @@ -3024,6 +3024,7 @@ table! { version_removed -> Nullable, multicast_ip -> Inet, source_ips -> Array, + membership_origin -> crate::enums::MulticastGroupMemberOriginEnum, } } diff --git a/nexus/examples/config-second.toml b/nexus/examples/config-second.toml index 3f255af6287..0b24550a160 100644 --- a/nexus/examples/config-second.toml +++ b/nexus/examples/config-second.toml @@ -189,12 +189,6 @@ fm.sitrep_gc_period_secs = 600 fm.rendezvous_period_secs = 300 probe_distributor.period_secs = 60 multicast_reconciler.period_secs = 60 -# TTL for sled-to-backplane-port mapping cache -# Default: 3600 seconds (1 hour) - detects new sleds and inventory changes -# multicast_reconciler.sled_cache_ttl_secs = 3600 -# TTL for backplane topology cache (static platform configuration) -# Default: 86400 seconds (24 hours) - refreshed on-demand when validation fails -# multicast_reconciler.backplane_cache_ttl_secs = 86400 trust_quorum.period_secs = 60 attached_subnet_manager.period_secs = 60 session_cleanup.period_secs = 300 diff --git a/nexus/examples/config.toml b/nexus/examples/config.toml index 942e639d7b8..4a012ed7623 100644 --- a/nexus/examples/config.toml +++ b/nexus/examples/config.toml @@ -173,12 +173,6 @@ fm.sitrep_gc_period_secs = 600 fm.rendezvous_period_secs = 300 probe_distributor.period_secs = 60 multicast_reconciler.period_secs = 60 -# TTL for sled-to-backplane-port mapping cache -# Default: 3600 seconds (1 hour) - detects new sleds and inventory changes -# multicast_reconciler.sled_cache_ttl_secs = 3600 -# TTL for backplane topology cache (static platform configuration) -# Default: 86400 seconds (24 hours) - refreshed on-demand when validation fails -# multicast_reconciler.backplane_cache_ttl_secs = 86400 trust_quorum.period_secs = 60 attached_subnet_manager.period_secs = 60 session_cleanup.period_secs = 300 diff --git a/nexus/lockstep-api/src/lib.rs b/nexus/lockstep-api/src/lib.rs index d5d918cd570..a7970f485e4 100644 --- a/nexus/lockstep-api/src/lib.rs +++ b/nexus/lockstep-api/src/lib.rs @@ -36,6 +36,7 @@ use nexus_types::internal_api::params::RackInitializationRequest; use nexus_types::internal_api::views::BackgroundTask; use nexus_types::internal_api::views::DemoSaga; use nexus_types::internal_api::views::MgsUpdateDriverStatus; +use nexus_types::internal_api::views::MulticastDdmPeersView; use nexus_types::internal_api::views::QuiesceStatus; use nexus_types::internal_api::views::Saga; use nexus_types::internal_api::views::SupportBundleInfo; @@ -207,6 +208,17 @@ pub trait NexusLockstepApi { rqctx: RequestContext, ) -> Result, HttpError>; + // Debug interface for DDM underlay peers + + /// Read-only view of DDM underlay peers across switch zones + #[endpoint { + method = GET, + path = "/multicast/ddm-peers", + }] + async fn multicast_ddm_peers( + rqctx: RequestContext, + ) -> Result, HttpError>; + // APIs for managing blueprints // // These are not (yet) intended for use by any other programs. Eventually, we diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index cab3275dc77..37a3d8c0862 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -1100,12 +1100,19 @@ impl BackgroundTasksInitializer { datastore.clone(), resolver.clone(), sagas.clone(), - inventory_load_watcher.clone(), args.multicast_enabled, - config.multicast_reconciler.sled_cache_ttl_secs, - config.multicast_reconciler.backplane_cache_ttl_secs, + config.multicast_reconciler.group_concurrency_limit, + config.multicast_reconciler.member_concurrency_limit, + chrono::TimeDelta::from_std( + config.multicast_reconciler.orphan_grace_secs, + ) + .unwrap_or_else(|_| chrono::TimeDelta::seconds(60)), )), opctx: opctx.child(BTreeMap::new()), + // Wake the reconciler whenever the inventory loader publishes a + // fresh collection so newly-discovered sleds become resolvable + // (DDM-peer fallback / inventory mapping) within the same tick + // instead of waiting for the periodic timer. watchers: vec![Box::new(inventory_load_watcher.clone())], activator: task_multicast_reconciler, }); diff --git a/nexus/src/app/background/tasks/multicast/groups.rs b/nexus/src/app/background/tasks/multicast/groups.rs index 0db31b033dc..db0054a7838 100644 --- a/nexus/src/app/background/tasks/multicast/groups.rs +++ b/nexus/src/app/background/tasks/multicast/groups.rs @@ -19,7 +19,10 @@ //! ## Operations Handled //! - **"Creating" state**: Initiate DPD "ensure" to apply configuration //! - **"Active" state**: Detect DPD drift and sync directly -//! - **"Deleting" state**: Switch cleanup and database removal +//! - **MRIB programming**: For Active groups, reconcile switch MRIB +//! routes against a per-pass snapshot (see [`super::mrib`]) +//! - **"Deleting" state**: Switch cleanup, MRIB route withdrawal, and +//! database removal //! - **M2P/forwarding propagation**: Convergent per-sled propagation of //! M2P mappings and forwarding entries via sled-agent after member //! state changes @@ -77,10 +80,11 @@ //! - **Partial cleanup**: "Deleting" state preserved until complete cleanup use anyhow::Context; -use chrono::Utc; +use futures::future::try_join_all; use futures::stream::{self, StreamExt}; use slog::{debug, error, info, trace, warn}; +use dpd_client::types::IpSrc; use nexus_db_model::{MulticastGroup, MulticastGroupState, SqlU8}; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::datastore::multicast::EnsureUnderlayResult; @@ -90,22 +94,18 @@ use omicron_common::address::is_ssm_address; use omicron_common::api::external::{self, DataPageParams}; use omicron_uuid_kinds::{GenericUuid, MulticastGroupUuid}; -use super::{ - MulticastGroupReconciler, StateTransition, map_external_to_underlay_ip, -}; +use super::{MulticastGroupReconciler, StateTransition}; use crate::app::multicast::dataplane::{ GroupUpdateParams, MulticastDataplaneClient, }; +use crate::app::multicast::map_external_to_underlay_ip; use crate::app::multicast::sled::MulticastSledClient; +use crate::app::multicast::switch_zone::{ + MribRouteIndex, MulticastSwitchZoneClient, +}; use crate::app::saga::create_saga_dag; use crate::app::sagas; -/// Minimum age before an orphaned group in "Creating" state can be cleaned up. -/// -/// This grace period avoids racing with in-progress member attachment operations -/// that occur immediately after group creation. -const ORPHAN_GROUP_MIN_AGE: chrono::TimeDelta = chrono::TimeDelta::seconds(10); - /// Check if DPD tag matches the database group's tag. /// /// Tags use format `{uuid}:{ip}` to prevent collision when group names are reused. @@ -142,7 +142,7 @@ fn dpd_state_matches_sources( let mut dpd_ips: Vec<_> = dpd_srcs .into_iter() .filter_map(|src| match src { - dpd_client::types::IpSrc::Exact(ip) => Some(ip), + IpSrc::Exact(ip) => Some(ip), _ => None, }) .collect(); @@ -164,7 +164,7 @@ fn dpd_state_matches_sources( let mut dpd_ips: Vec<_> = dpd_srcs .into_iter() .filter_map(|src| match src { - dpd_client::types::IpSrc::Exact(ip) => Some(ip), + IpSrc::Exact(ip) => Some(ip), _ => None, }) .collect(); @@ -180,6 +180,13 @@ fn dpd_state_matches_sources( } } +/// Switch-side clients threaded through group state processors. +struct GroupReconcileClients<'a> { + dataplane: &'a MulticastDataplaneClient, + sled: &'a MulticastSledClient, + switch_zone: &'a MulticastSwitchZoneClient, +} + /// Trait for processing different types of multicast groups trait GroupStateProcessor { /// Process a group in "Creating" state. @@ -196,8 +203,7 @@ trait GroupStateProcessor { reconciler: &MulticastGroupReconciler, opctx: &OpContext, group: &MulticastGroup, - dataplane_client: &MulticastDataplaneClient, - sled_client: &MulticastSledClient, + clients: &GroupReconcileClients<'_>, ) -> Result; /// Process a group in "Active" state (check DPD sync status). @@ -206,8 +212,8 @@ trait GroupStateProcessor { reconciler: &MulticastGroupReconciler, opctx: &OpContext, group: &MulticastGroup, - dataplane_client: &MulticastDataplaneClient, - sled_client: &MulticastSledClient, + clients: &GroupReconcileClients<'_>, + mrib_route_index: Option<&MribRouteIndex>, ) -> Result; } @@ -231,34 +237,35 @@ impl GroupStateProcessor for ExternalGroupProcessor { reconciler: &MulticastGroupReconciler, opctx: &OpContext, group: &MulticastGroup, - dataplane_client: &MulticastDataplaneClient, - sled_client: &MulticastSledClient, + clients: &GroupReconcileClients<'_>, ) -> Result { reconciler .handle_deleting_external_group( opctx, group, - dataplane_client, - sled_client, + clients.dataplane, + clients.sled, + clients.switch_zone, ) .await } - /// Handle groups in "Active" state (check DPD sync status). async fn process_active( &self, reconciler: &MulticastGroupReconciler, opctx: &OpContext, group: &MulticastGroup, - dataplane_client: &MulticastDataplaneClient, - sled_client: &MulticastSledClient, + clients: &GroupReconcileClients<'_>, + mrib_route_index: Option<&MribRouteIndex>, ) -> Result { reconciler .handle_active_external_group( opctx, group, - dataplane_client, - sled_client, + clients.dataplane, + clients.sled, + clients.switch_zone, + mrib_route_index, ) .await } @@ -368,6 +375,7 @@ impl MulticastGroupReconciler { state: MulticastGroupState, dataplane_client: Option<&MulticastDataplaneClient>, sled_client: Option<&MulticastSledClient>, + switch_zone_client: Option<&MulticastSwitchZoneClient>, ) -> Result { trace!(opctx.log, "searching for multicast groups"; "state" => %state); @@ -391,8 +399,24 @@ impl MulticastGroupReconciler { trace!(opctx.log, "found multicast groups"; "count" => groups.len(), "state" => %state); + let mrib_route_index = match (state, switch_zone_client) { + (MulticastGroupState::Active, Some(client)) => client + .list_routes_indexed() + .await + .inspect_err(|e| { + warn!( + opctx.log, + "failed to build per-pass MRIB route snapshot"; + "error" => %e, + ) + }) + .ok(), + _ => None, + }; + let mrib_route_index = mrib_route_index.as_ref(); + // Process groups concurrently with configurable parallelism - let results = stream::iter(groups) + let group_outcomes = stream::iter(groups) .map(|group| async move { let result = self .process_group_state( @@ -400,6 +424,8 @@ impl MulticastGroupReconciler { &group, dataplane_client, sled_client, + switch_zone_client, + mrib_route_index, ) .await; (group, result) @@ -410,8 +436,8 @@ impl MulticastGroupReconciler { // Handle results with state-appropriate logging and counting let mut processed = 0; - let total_results = results.len(); - for (group, result) in results { + let total = group_outcomes.len(); + for (group, result) in group_outcomes { match result { Ok(transition) => { // Count successful transitions based on state expectations @@ -461,13 +487,13 @@ impl MulticastGroupReconciler { } } - if total_results > 0 { + if total > 0 { debug!( opctx.log, "group reconciliation completed"; "state" => %state, "processed" => processed, - "total" => total_results + "total" => total ); } @@ -475,7 +501,7 @@ impl MulticastGroupReconciler { } /// Process multicast groups that are in "Creating" state. - pub async fn reconcile_creating_groups( + pub(super) async fn reconcile_creating_groups( &self, opctx: &OpContext, ) -> Result { @@ -484,38 +510,43 @@ impl MulticastGroupReconciler { MulticastGroupState::Creating, None, None, + None, ) .await } /// Process multicast groups that are in "Deleting" state. - pub async fn reconcile_deleting_groups( + pub(super) async fn reconcile_deleting_groups( &self, opctx: &OpContext, dataplane_client: &MulticastDataplaneClient, sled_client: &MulticastSledClient, + switch_zone_client: &MulticastSwitchZoneClient, ) -> Result { self.reconcile_groups_by_state( opctx, MulticastGroupState::Deleting, Some(dataplane_client), Some(sled_client), + Some(switch_zone_client), ) .await } /// Reconcile active multicast groups with DPD (drift detection and correction). - pub async fn reconcile_active_groups( + pub(super) async fn reconcile_active_groups( &self, opctx: &OpContext, dataplane_client: &MulticastDataplaneClient, sled_client: &MulticastSledClient, + switch_zone_client: &MulticastSwitchZoneClient, ) -> Result { self.reconcile_groups_by_state( opctx, MulticastGroupState::Active, Some(dataplane_client), Some(sled_client), + Some(switch_zone_client), ) .await } @@ -528,6 +559,8 @@ impl MulticastGroupReconciler { group: &MulticastGroup, dataplane_client: Option<&MulticastDataplaneClient>, sled_client: Option<&MulticastSledClient>, + switch_zone_client: Option<&MulticastSwitchZoneClient>, + mrib_route_index: Option<&MribRouteIndex>, ) -> Result { // Future: Match on group type to select different processors if // we add more nuanced group types @@ -538,32 +571,36 @@ impl MulticastGroupReconciler { processor.process_creating(self, opctx, group).await } MulticastGroupState::Deleting => { - let dataplane_client = dataplane_client - .context("dataplane client required for deleting state")?; - let sled_client = sled_client - .context("sled client required for deleting state")?; - processor - .process_deleting( - self, - opctx, - group, - dataplane_client, - sled_client, - ) - .await + let clients = GroupReconcileClients { + dataplane: dataplane_client.context( + "dataplane client required for deleting state", + )?, + sled: sled_client + .context("sled client required for deleting state")?, + switch_zone: switch_zone_client.context( + "switch zone client required for deleting state", + )?, + }; + processor.process_deleting(self, opctx, group, &clients).await } MulticastGroupState::Active => { - let dataplane_client = dataplane_client - .context("dataplane client required for active state")?; - let sled_client = sled_client - .context("sled client required for active state")?; + let clients = GroupReconcileClients { + dataplane: dataplane_client.context( + "dataplane client required for active state", + )?, + sled: sled_client + .context("sled client required for active state")?, + switch_zone: switch_zone_client.context( + "switch zone client required for active state", + )?, + }; processor .process_active( self, opctx, group, - dataplane_client, - sled_client, + &clients, + mrib_route_index, ) .await } @@ -616,63 +653,24 @@ impl MulticastGroupReconciler { "underlay_linked" => group.underlay_group_id.is_some() ); - // Clean up orphaned groups stuck in "Creating" state with no members. - // This handles cases where implicit group creation succeeded but member - // attachment failed (e.g., SSM validation error, transient failures). - // - // We only clean up groups that have been in "Creating" for at least - // ORPHAN_GROUP_MIN_AGE to avoid racing with in-progress member - // attachment operations. - let age = Utc::now() - group.time_created(); - if age > ORPHAN_GROUP_MIN_AGE { - let group_id = MulticastGroupUuid::from_untyped_uuid(group.id()); - match self - .datastore - .mark_multicast_group_for_removal_if_no_members(opctx, group_id) - .await - { - Ok(true) => { - info!( - opctx.log, - "cleaned up orphaned multicast group in \"Creating\" state with no members"; - "group_id" => %group.id(), - "group_name" => group.name().as_str(), - "age_seconds" => age.num_seconds(), - ); - return Ok(StateTransition::NeedsCleanup); - } - Ok(false) => { - // Group has members, continue with normal processing - } - Err(e) => { - warn!( - opctx.log, - "failed to check/cleanup orphaned group"; - "group_id" => %group.id(), - "error" => ?e, - ); - } - } - } + // Orphaned "Creating" groups with no members are reaped by the unified + // emptiness sweep (`cleanup_empty_groups`), not here; this handler only + // drives the "Creating" → "Active" programming. - // TODO: Add front port selection for egress traffic (instances → - // external). When transitioning groups to Active, we need to identify - // and validate front ports against DPD's QSFP topology (similar to - // `backplane_map` validation for rear ports). These uplink members use - // `Direction::External` and follow a different lifecycle - added when - // first instance joins, removed when last instance leaves. - // Should integrate with `switch_ports_with_uplinks()` or - // equivalent front port discovery mechanism, which would be - // configurable, and later learned (i.e., via `mcastd`/IGMP). + // TODO: Front port selection for egress (instance → external) is + // not yet implemented. See RFD 488 (§sect-external-mcast) for + // the design (mcastd / PIM-SM). // Handle underlay group creation/linking (same logic as before) if !self.process_creating_group_inner(opctx, group).await? { return Ok(StateTransition::EntityGone); } - // Successfully started saga - the saga will handle state transition to "Active". - // We return NoChange because the reconciler shouldn't change the state; - // the saga applies external + underlay configuration via DPD. + // The blocking await above ensures the saga has finished + // applying external + underlay DPD configuration before we + // return. The reconciler does not write "Active" itself; the + // saga is the writer. `NoChange` here means this pass made no + // further state change beyond what the saga already committed. Ok(StateTransition::NoChange) } @@ -683,6 +681,7 @@ impl MulticastGroupReconciler { group: &MulticastGroup, dataplane_client: &MulticastDataplaneClient, sled_client: &MulticastSledClient, + switch_zone_client: &MulticastSwitchZoneClient, ) -> Result { debug!( opctx.log, @@ -695,6 +694,46 @@ impl MulticastGroupReconciler { "dpd_cleanup_required" => true ); + // Remove MRIB routes so `mg-lower` withdraws DDM advertisements + // before cleaning up DPD and DB state. Return early on failure so + // the next pass can retry. Proceeding would delete DB rows and + // leave stale DDM advertisements. + let group_ip = group.multicast_ip.ip(); + let group_id = MulticastGroupUuid::from_untyped_uuid(group.id()); + + // Remove (*,G) route. + switch_zone_client + .remove_route(group_ip, None) + .await + .context("failed to remove MRIB (*,G) route for deleting group")?; + + // Remove (S,G) routes for any sources. Return early on failure + // to preserve DB state for retry on the next pass. + let source_filter = self + .datastore + .multicast_groups_source_filter_state(opctx, &[group_id]) + .await + .context( + "failed to load source filter for MRIB cleanup; \ + returning early to preserve DB state for retry", + )?; + + if let Some(filter) = source_filter.get(&group.id()) { + // Per-source removals target distinct (S,G) keys. We fan out so + // a group with N sources doesn't pay N round-trips serially. + try_join_all(filter.specific_sources.iter().map( + |source| async move { + switch_zone_client + .remove_route(group_ip, Some(*source)) + .await + .with_context(|| format!( + "failed to remove MRIB (S,G) route for source {source}" + )) + }), + ) + .await?; + } + self.process_deleting_group_inner( opctx, group, @@ -715,6 +754,8 @@ impl MulticastGroupReconciler { group: &MulticastGroup, dataplane_client: &MulticastDataplaneClient, sled_client: &MulticastSledClient, + switch_zone_client: &MulticastSwitchZoneClient, + mrib_route_index: Option<&MribRouteIndex>, ) -> Result { let underlay_group_id = group .underlay_group_id @@ -778,7 +819,14 @@ impl MulticastGroupReconciler { } }; - if needs_update { + // Track whether DPD is in the desired state for this group, either + // because we just synced it or because it was already in sync. MRIB + // advertisement below is gated on this. Advertising via MGD/DDM + // while the switch dataplane state is stale would cause peer sleds + // to forward traffic to a switch that cannot deliver it. + let mut dpd_synced = false; + + let res = if needs_update { debug!( opctx.log, "updating active multicast group in DPD"; @@ -814,6 +862,8 @@ impl MulticastGroupReconciler { "multicast_ip" => %group.multicast_ip ); + dpd_synced = true; + // Propagate M2P/forwarding to member sleds after DPD // sync to ensure OPTE state is also consistent. if let Err(e) = sled_client @@ -843,6 +893,9 @@ impl MulticastGroupReconciler { } } } else { + // DPD was already in sync (no update needed). + dpd_synced = true; + // Even when DPD is in sync, propagate M2P/forwarding to // member sleds to correct any sled-level drift. if let Err(e) = @@ -857,7 +910,38 @@ impl MulticastGroupReconciler { } Ok(StateTransition::NoChange) + }; + + // Reconcile MRIB routes based on whether the group has active + // ("Joined") members. If all members are "Left", withdraw the DDM + // advertisement so peer sleds stop sending traffic. + // + // Gate on `dpd_synced` so we never advertise via MGD/DDM while + // the switch DPD state is stale. Advertising peer sleds toward + // a switch that cannot deliver the traffic would just drop + // packets at the switch. We retry on the next reconciler pass + // once DPD is back in sync. + if dpd_synced { + super::mrib::reconcile_group( + opctx, + &self.datastore, + switch_zone_client, + mrib_route_index, + group, + &source_filter, + underlay_group_id, + ) + .await; + } else { + debug!( + opctx.log, + "skipping MRIB reconcile: DPD update failed for this group"; + "group_id" => %group.id(), + "multicast_ip" => %group.multicast_ip, + ); } + + res } /// Process a single multicast group in "Creating" state. @@ -898,7 +982,7 @@ impl MulticastGroupReconciler { "creating new underlay group"; "group" => ?group ); - match self.ensure_underlay_for_external(opctx, &group).await? { + match self.ensure_underlay_for_external(opctx, group).await? { Some(underlay) => underlay, None => return Ok(false), // Group deleted during processing } @@ -930,9 +1014,9 @@ impl MulticastGroupReconciler { >(saga_params) .context("failed to create multicast group transaction saga")?; - let saga_id = self + let (saga_id, completion) = self .sagas - .saga_start(dag) + .saga_run(dag) .await .context("failed to start multicast group transaction saga")?; @@ -946,6 +1030,11 @@ impl MulticastGroupReconciler { "expected_outcome" => "Creating → Active" ); + // Block this pass on saga completion so subsequent reconciler + // steps observe "Active" within the same pass. See mod-level + // "RPW Saga Coordination" for rationale. + completion.await.context("multicast group transaction saga failed")?; + Ok(true) } @@ -1033,7 +1122,7 @@ mod tests { use omicron_common::api::external::IdentityMetadataCreateParams; fn create_dpd_group( - sources: Option>, + sources: Option>, ) -> dpd_client::types::MulticastGroupExternalResponse { dpd_client::types::MulticastGroupExternalResponse { group_ip: "232.1.1.1".parse().unwrap(), @@ -1086,15 +1175,15 @@ mod tests { // DPD has matching sources let dpd_group = create_dpd_group(Some(vec![ - dpd_client::types::IpSrc::Exact("10.0.0.1".parse().unwrap()), - dpd_client::types::IpSrc::Exact("10.0.0.2".parse().unwrap()), + IpSrc::Exact("10.0.0.1".parse().unwrap()), + IpSrc::Exact("10.0.0.2".parse().unwrap()), ])); assert!(dpd_state_matches_sources(&dpd_group, &source_filter, &group)); // DPD has sources in different order (should still match) let dpd_group = create_dpd_group(Some(vec![ - dpd_client::types::IpSrc::Exact("10.0.0.2".parse().unwrap()), - dpd_client::types::IpSrc::Exact("10.0.0.1".parse().unwrap()), + IpSrc::Exact("10.0.0.2".parse().unwrap()), + IpSrc::Exact("10.0.0.1".parse().unwrap()), ])); assert!(dpd_state_matches_sources(&dpd_group, &source_filter, &group)); @@ -1104,8 +1193,8 @@ mod tests { // DPD has wrong sources (mismatch) let dpd_group = create_dpd_group(Some(vec![ - dpd_client::types::IpSrc::Exact("10.0.0.1".parse().unwrap()), - dpd_client::types::IpSrc::Exact("10.0.0.3".parse().unwrap()), // wrong + IpSrc::Exact("10.0.0.1".parse().unwrap()), + IpSrc::Exact("10.0.0.3".parse().unwrap()), // wrong ])); assert!(!dpd_state_matches_sources(&dpd_group, &source_filter, &group)); } @@ -1128,8 +1217,8 @@ mod tests { // DPD should have specific sources (RFC 4607 compliance) let dpd_group = create_dpd_group(Some(vec![ - dpd_client::types::IpSrc::Exact("10.0.0.1".parse().unwrap()), - dpd_client::types::IpSrc::Exact("10.0.0.2".parse().unwrap()), + IpSrc::Exact("10.0.0.1".parse().unwrap()), + IpSrc::Exact("10.0.0.2".parse().unwrap()), ])); assert!(dpd_state_matches_sources(&dpd_group, &source_filter, &group)); @@ -1151,10 +1240,9 @@ mod tests { let group = create_group("224.1.1.1"); // ASM address // DPD has matching specific sources - let dpd_group = - create_dpd_group(Some(vec![dpd_client::types::IpSrc::Exact( - "10.0.0.1".parse().unwrap(), - )])); + let dpd_group = create_dpd_group(Some(vec![IpSrc::Exact( + "10.0.0.1".parse().unwrap(), + )])); assert!(dpd_state_matches_sources(&dpd_group, &source_filter, &group)); // DPD has None (mismatch: should have specific sources) @@ -1162,8 +1250,7 @@ mod tests { assert!(!dpd_state_matches_sources(&dpd_group, &source_filter, &group)); // DPD has IpSrc::Any (mismatch: should have specific sources) - let dpd_group = - create_dpd_group(Some(vec![dpd_client::types::IpSrc::Any])); + let dpd_group = create_dpd_group(Some(vec![IpSrc::Any])); assert!(!dpd_state_matches_sources(&dpd_group, &source_filter, &group)); } @@ -1183,10 +1270,9 @@ mod tests { assert!(dpd_state_matches_sources(&dpd_group, &source_filter, &group)); // DPD has specific sources (mismatch) - let dpd_group = - create_dpd_group(Some(vec![dpd_client::types::IpSrc::Exact( - "10.0.0.1".parse().unwrap(), - )])); + let dpd_group = create_dpd_group(Some(vec![IpSrc::Exact( + "10.0.0.1".parse().unwrap(), + )])); assert!(!dpd_state_matches_sources(&dpd_group, &source_filter, &group)); } @@ -1205,10 +1291,9 @@ mod tests { assert!(dpd_state_matches_sources(&dpd_group, &source_filter, &group)); // DPD has sources (mismatch) - let dpd_group = - create_dpd_group(Some(vec![dpd_client::types::IpSrc::Exact( - "10.0.0.1".parse().unwrap(), - )])); + let dpd_group = create_dpd_group(Some(vec![IpSrc::Exact( + "10.0.0.1".parse().unwrap(), + )])); assert!(!dpd_state_matches_sources(&dpd_group, &source_filter, &group)); } } diff --git a/nexus/src/app/background/tasks/multicast/members.rs b/nexus/src/app/background/tasks/multicast/members.rs index 294afe76831..72074c610f8 100644 --- a/nexus/src/app/background/tasks/multicast/members.rs +++ b/nexus/src/app/background/tasks/multicast/members.rs @@ -12,9 +12,16 @@ //! # RPW Member Processing Model //! //! Member management is more complex than group management because members have -//! dynamic lifecycle tied to instance state (start/stop/migrate) and require -//! dataplane updates. The RPW maintains eventual consistency between intended -//! membership (database) and actual forwarding (dataplane configuration). +//! a dynamic lifecycle tied to instance state (start/stop/migrate). The RPW +//! maintains eventual consistency between intended membership (database) and the +//! per-sled state that lets a member's VMM receive traffic. +//! +//! Members do not program switch dataplane membership. Rear-port underlay +//! membership is owned by `ddmd`, which derives it from DDM peer subscriptions +//! and programs it into the switch dataplane via mg-lower/DDM. Group-level +//! switch effects (group creation, source filters) are applied elsewhere by +//! the group reconciler. This module's effects are limited to database state +//! and sled state reached through sled-agent. //! //! ## 3-State Member Lifecycle //! @@ -24,7 +31,7 @@ //! - RPW transitions to "Joined" when ready //! //! - **Joined**: Member actively receiving multicast traffic -//! - Dataplane configured via DPD client(s) +//! - VMM's OPTE port subscribed and sled forwarding state propagated //! - Instance is running and reachable on assigned sled //! - RPW responds to sled migrations //! @@ -33,39 +40,35 @@ //! - time_deleted=NULL: temporary (can rejoin) //! - time_deleted=SET: permanent deletion pending //! -//! Migration note: migration is not treated as leaving. The reconciler removes -//! dataplane membership from the old sled and applies it on the new sled while -//! keeping the member in "Joined" (reconfigures in place). -//! //! ## Operations Handled //! //! - **State transitions**: "Joining" → "Joined" → "Left" with reactivation -//! - **Dataplane updates**: Applying and removing configuration via DPD -//! client(s) on switches +//! - **OPTE subscriptions**: The active VMM's OPTE port is subscribed and +//! unsubscribed via sled-agent on the hosting sled (keyed by the active +//! VMM's propolis ID) //! - **M2P/forwarding propagation**: After join, leave, or migration, M2P -//! mappings and forwarding entries are propagated to all sleds via -//! sled-agent inline (not deferred to the next reconciliation pass) -//! - **OPTE subscriptions**: Per-VMM multicast group filters managed via -//! sled-agent on the hosting sled -//! - **Sled migration**: Detecting moves and updating dataplane configuration +//! mappings and forwarding entries are propagated to sleds via sled-agent +//! inline (not deferred to the next reconciliation pass) +//! - **Sled migration**: Detecting moves and re-subscribing on the new sled //! (no transition to "Left") -//! - **Cleanup**: Removing orphaned switch state for deleted members +//! - **Cleanup**: Unsubscribing deleted members from their VMM //! - **Extensible processing**: Support for different member types (designed for //! future extension) //! -//! ## Separation of Concerns: RPW +/- Sagas +//! ## Separation of Concerns: RPW and Sagas //! //! **Sagas:** //! - Instance create/start → member "Joining" state //! - Instance stop/delete → member "Left" state + time_deleted //! - Sled assignment updates during instance operations -//! - Database state changes only (no switch operations) +//! - Database state changes only (no sled or switch operations) //! //! **RPW (background):** -//! - Determining switch ports and updating dataplane switches when members join +//! - Subscribing/unsubscribing the active VMM's OPTE port via sled-agent +//! - Propagating M2P/forwarding state to sleds //! - Handling sled migrations //! - Instance state monitoring and member state transitions -//! - Cleanup of deleted members from switch state +//! - Cleanup of deleted members //! //! # Member State Transition Matrix //! @@ -84,27 +87,26 @@ //! | 1 | "Creating" | Any | Any | Wait for activation | "Joining" | //! | 2 | "Active" | Invalid | Any | Clear sled_id → "Left" | "Left" | //! | 3 | "Active" | Valid | No | Wait for sled assignment | "Joining" | -//! | 4 | "Active" | Valid | Yes | Add to DPD → "Joined" | "Joined" | +//! | 4 | "Active" | Valid | Yes | Subscribe VMM + propagate → "Joined" | "Joined" | //! //! ### JOINED State Transitions //! | # | Instance Valid | Sled Changed | Has sled_id | Action | Next State | //! |---|----------------|--------------|-------------|---------|------------| -//! | 1 | Invalid | Any | Any | Remove DPD + clear sled_id → "Left" | "Left" | -//! | 2 | Valid | Yes | Yes | Remove old + update sled_id + add new | "Joined" | -//! | 3 | Valid | No | Yes | Verify DPD config (idempotent) | "Joined" | -//! | 4 | Valid | N/A | No | Remove DPD → "Left" (edge case) | "Left" | +//! | 1 | Invalid | Any | Any | Unsubscribe VMM + clear sled_id → "Left" | "Left" | +//! | 2 | Valid | Yes | Yes | Unsubscribe old + update sled_id + subscribe new | "Joined" | +//! | 3 | Valid | No | Yes | Verify config, no changes needed | "Joined" | +//! | 4 | Valid | N/A | No | DB state only → "Left" (edge case) | "Left" | //! //! ### LEFT State Transitions //! | # | time_deleted | Instance Valid | Group State | Action | Next State | //! |---|--------------|----------------|-------------|---------|------------| -//! | 1 | Set | Any | Any | Cleanup DPD config | NeedsCleanup | +//! | 1 | Set | Any | Any | Unsubscribe VMM | NeedsCleanup | //! | 2 | None | Invalid | Any | No action (stay stopped) | "Left" | //! | 3 | None | Valid | "Creating" | Wait for activation | "Left" | //! | 4 | None | Valid | "Active" | Reactivate member | "Joining" | -use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::collections::HashMap; use std::sync::Arc; -use std::time::Instant; use anyhow::{Context, Result}; use futures::stream::{self, StreamExt}; @@ -113,24 +115,34 @@ use uuid::Uuid; use nexus_db_model::{ DbTypedUuid, MulticastGroup, MulticastGroupMember, - MulticastGroupMemberState, MulticastGroupState, Sled, + MulticastGroupMemberState, MulticastGroupState, }; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::datastore::multicast::ops::member_reconcile::{ ReconcileAction, ReconcileJoiningResult, }; -use nexus_types::deployment::SledFilter; -use nexus_types::identity::{Asset, Resource}; +use nexus_types::identity::Resource; use omicron_common::api::external::{DataPageParams, InstanceState}; use omicron_uuid_kinds::{ GenericUuid, InstanceUuid, MulticastGroupUuid, PropolisUuid, SledKind, SledUuid, }; -use super::{MulticastGroupReconciler, StateTransition, SwitchBackplanePort}; -use crate::app::multicast::dataplane::MulticastDataplaneClient; +use super::{MulticastGroupReconciler, StateTransition}; use crate::app::multicast::sled::MulticastSledClient; +/// Whether at least `min_age` has elapsed since `since`. +/// +/// Gates implicit group deletion behind a grace period. Orphaned "Creating" +/// groups are gated on their creation time so a group whose first member attach +/// is still in flight is not reaped before that member reaches "Joined". +fn grace_period_elapsed( + since: chrono::DateTime, + min_age: chrono::TimeDelta, +) -> bool { + chrono::Utc::now() - since >= min_age +} + /// Pre-fetched instance state for multicast reconciliation. #[derive(Clone, Copy, Debug, Default)] struct InstanceMulticastState { @@ -138,8 +150,6 @@ struct InstanceMulticastState { valid: bool, /// Current sled hosting the VMM, if any. sled_id: Option, - /// Current propolis VMM identifier, if any. - propolis_id: Option, } /// Context shared across member reconciliation operations. @@ -148,37 +158,19 @@ struct MemberReconcileCtx<'a> { group: &'a MulticastGroup, member: &'a MulticastGroupMember, instance_states: &'a InstanceStateMap, - dataplane_client: &'a MulticastDataplaneClient, sled_client: &'a MulticastSledClient, } /// Maps instance_id to pre-fetched multicast-relevant state. type InstanceStateMap = HashMap; -/// Backplane port mapping from DPD-client. -/// Maps switch port ID to backplane link configuration. -type BackplaneMap = - BTreeMap; - -/// Result of computing the union of member ports across a group. -/// -/// Indicates whether all "Joined" members were successfully resolved when -/// computing the port union. Callers should only prune stale ports when -/// the union is `Complete` to avoid disrupting members that failed resolution. -enum MemberPortUnion { - /// Union is complete: all "Joined" members were successfully resolved. - Complete(BTreeSet), - /// Union is partial: some "Joined" members failed to resolve. - /// The port set may be incomplete. - Partial(BTreeSet), -} - -/// Check if a DPD member is a rear/underlay port (instance member). -fn is_rear_underlay_member( - member: &dpd_client::types::MulticastGroupMember, -) -> bool { - matches!(member.port_id, dpd_client::types::PortId::Rear(_)) - && member.direction == dpd_client::types::Direction::Underlay +/// Outcome of a single [`MulticastGroupReconciler::reconcile_member_states`] +/// pass. +#[derive(Clone, Copy, Debug, Default)] +pub(super) struct MemberReconcileCounts { + /// Members whose state advanced this pass (e.g., "Joining" → "Joined", + /// "Joining" → "Left"). + pub(super) processed: usize, } /// Represents a sled_id update for a multicast group member. @@ -250,12 +242,11 @@ impl MulticastGroupReconciler { ]; /// Process member state changes ("Joining"→"Joined"→"Left"). - pub async fn reconcile_member_states( + pub(super) async fn reconcile_member_states( &self, opctx: &OpContext, - dataplane_client: &MulticastDataplaneClient, sled_client: &MulticastSledClient, - ) -> Result { + ) -> Result { trace!(opctx.log, "reconciling member state changes"); let mut processed = 0; @@ -265,12 +256,7 @@ impl MulticastGroupReconciler { for group in groups { match self - .process_group_member_states( - opctx, - &group, - dataplane_client, - sled_client, - ) + .process_group_member_states(opctx, &group, sled_client) .await { Ok(count) => { @@ -298,10 +284,10 @@ impl MulticastGroupReconciler { debug!( opctx.log, "member state reconciliation completed"; - "members_processed" => processed + "members_processed" => processed, ); - Ok(processed) + Ok(MemberReconcileCounts { processed }) } /// Process member state changes for a single group. @@ -309,7 +295,6 @@ impl MulticastGroupReconciler { &self, opctx: &OpContext, group: &MulticastGroup, - dataplane_client: &MulticastDataplaneClient, sled_client: &MulticastSledClient, ) -> Result { let mut processed = 0; @@ -322,20 +307,19 @@ impl MulticastGroupReconciler { Arc::new(self.batch_fetch_instance_states(opctx, &members).await?); // Process members concurrently with configurable parallelism - let results = stream::iter(members) + let member_outcomes = stream::iter(members) .map(|member| { let instance_states = Arc::clone(&instance_states); async move { - let res = self - .process_member_state( - opctx, - group, - &member, - &instance_states, - dataplane_client, - sled_client, - ) - .await; + let ctx = MemberReconcileCtx { + opctx, + group, + member: &member, + instance_states: &instance_states, + sled_client, + }; + + let res = self.process_member_state(&ctx).await; (member, res) } }) @@ -344,7 +328,7 @@ impl MulticastGroupReconciler { .await; // Process results and update counters - for (member, result) in results { + for (member, result) in member_outcomes { match result { Ok(transition) => match transition { StateTransition::StateChanged @@ -396,13 +380,10 @@ impl MulticastGroupReconciler { /// Routes to the appropriate handler based on member state. async fn process_member_state( &self, - opctx: &OpContext, - group: &MulticastGroup, - member: &MulticastGroupMember, - instance_states: &InstanceStateMap, - dataplane_client: &MulticastDataplaneClient, - sled_client: &MulticastSledClient, + ctx: &MemberReconcileCtx<'_>, ) -> Result { + let MemberReconcileCtx { opctx, group, member, .. } = *ctx; + // Check if the parent group has been deleted or is being deleted. // If so, delete the member so cleanup can proceed. // @@ -431,24 +412,16 @@ impl MulticastGroupReconciler { // For now, all members are instance-based, but this is where we'd // dispatch to different processors for different member types let processor = InstanceMemberProcessor; - let ctx = MemberReconcileCtx { - opctx, - group, - member, - instance_states, - dataplane_client, - sled_client, - }; match member.state { MulticastGroupMemberState::Joining => { - processor.process_joining(self, &ctx).await + processor.process_joining(self, ctx).await } MulticastGroupMemberState::Joined => { - processor.process_joined(self, &ctx).await + processor.process_joined(self, ctx).await } MulticastGroupMemberState::Left => { - processor.process_left(self, &ctx).await + processor.process_left(self, ctx).await } } } @@ -652,13 +625,7 @@ impl MulticastGroupReconciler { instance_state: InstanceMulticastState, ) -> Result { if self.is_ready_to_join(ctx.group, instance_state.valid) { - let joined = self - .complete_instance_member_join( - ctx, - None, - instance_state.propolis_id, - ) - .await?; + let joined = self.complete_instance_member_join(ctx, None).await?; if joined { Ok(StateTransition::StateChanged) } else { @@ -695,16 +662,10 @@ impl MulticastGroupReconciler { (true, Some(sled_id)) if ctx.member.sled_id != Some(sled_id.into()) => { - self.handle_sled_migration( - ctx, - sled_id, - instance_state.propolis_id, - ) - .await + self.handle_sled_migration(ctx, sled_id).await } (true, Some(_)) => { - self.verify_members(ctx).await?; trace!( ctx.opctx.log, "member configuration verified, no changes needed"; @@ -724,28 +685,17 @@ impl MulticastGroupReconciler { ctx: &MemberReconcileCtx<'_>, ) -> Result { let MemberReconcileCtx { opctx, group, member, sled_client, .. } = ctx; - // Remove from dataplane first - if let Err(e) = self.remove_member_from_dataplane(ctx).await { - warn!( - opctx.log, - "failed to remove member from dataplane, will retry"; - "member_id" => %member.id, - "error" => ?e - ); - return Err(e); - } - - // Unsubscribe the VMM from the multicast group before the CAS + // Unsubscribe the instance from the multicast group before the CAS // clears the sled ID. Best-effort since the VMM may already be torn // down. if let Some(sled_id) = member.sled_id { if let Err(e) = sled_client - .unsubscribe_vmm(opctx, group, member, sled_id.into(), None) + .unsubscribe_instance(opctx, group, member, sled_id.into()) .await { warn!( opctx.log, - "failed to unsubscribe VMM during instance invalidation"; + "failed to unsubscribe instance during instance invalidation"; "member_id" => %member.id, "sled_id" => %sled_id, "error" => %e @@ -810,7 +760,6 @@ impl MulticastGroupReconciler { &self, ctx: &MemberReconcileCtx<'_>, new_sled_id: SledUuid, - cached_propolis_id: Option, ) -> Result { info!( ctx.opctx.log, @@ -824,18 +773,6 @@ impl MulticastGroupReconciler { "new_sled_id" => %new_sled_id ); - // Remove from old sled's dataplane first - if let Err(e) = self.remove_member_from_dataplane(ctx).await { - warn!( - ctx.opctx.log, - "failed to remove member from old sled, will retry"; - "member_id" => %ctx.member.id, - "old_sled_id" => ?ctx.member.sled_id, - "error" => ?e - ); - return Err(e); - } - // Source-sled OPTE cleanup (M2P, forwarding, port subscription) // is handled by VMM teardown: remove_propolis_zone -> // release_opte_ports -> PortTicket::release_inner, which @@ -874,14 +811,7 @@ impl MulticastGroupReconciler { // Re-apply configuration on new sled. Pass `new_sled_id` explicitly // because the in-memory member struct still has the old sled_id. - match self - .complete_instance_member_join( - ctx, - Some(new_sled_id), - cached_propolis_id, - ) - .await - { + match self.complete_instance_member_join(ctx, Some(new_sled_id)).await { Ok(joined) => { info!( ctx.opctx.log, @@ -913,26 +843,6 @@ impl MulticastGroupReconciler { "error" => %e ); - // TODO: Use DDM as the primary source of truth for sled→port - // mapping, with inventory as cross-validation. - // - // Currently we trust inventory (MGS/SP topology) for sled→port - // mapping. DDM (maghemite/ddmd) on switches has authoritative - // knowledge of which sleds are reachable on which ports. - // - // Future approach: - // 1. Query DDM for operational sled→port mapping - // // TODO: Add GET /peers endpoint to ddm-admin-client - // // returning Map where PeerInfo - // // includes port/interface field (requires maghemite change) - // 2. Use DDM mapping as primary source for multicast routing - // 3. Cross-validate against inventory to detect mismatches - // 4. On mismatch: invalidate cache, log warning, potentially - // trigger inventory reconciliation - // - // This catches cases where inventory is stale or a sled moved - // but inventory hasn't updated yet. - let updated = self .datastore .multicast_group_member_set_state_if_current( @@ -977,17 +887,9 @@ impl MulticastGroupReconciler { "parent_id" => %member.parent_id ); - // Remove from dataplane and transition to "Left" - if let Err(e) = self.remove_member_from_dataplane(ctx).await { - warn!( - opctx.log, - "failed to remove member with no sled_id from dataplane"; - "member_id" => %member.id, - "error" => ?e - ); - return Err(e); - } - + // Member has no `sled_id`, so there is no VMM OPTE port to unsubscribe + // and no underlay member to remove (those are owned by `ddmd`). Only the + // DB state moves to "Left". let updated = self .datastore .multicast_group_member_set_state_if_current( @@ -1047,38 +949,23 @@ impl MulticastGroupReconciler { return Ok(StateTransition::NeedsCleanup); } - // Always clean DPD for "Left" members before any other action. - // This ensures stale DPD state is removed before reactivation attempts. - // The cleanup is idempotent and handles cases where: - // - sled_id is None (uses fallback path) - // - member was already removed from DPD - if let Err(e) = self.remove_member_from_dataplane(ctx).await { - warn!( - ctx.opctx.log, - "failed to clean up DPD state for 'Left' member (will retry)"; - "member_id" => %ctx.member.id, - "error" => ?e - ); - } - - // Unsubscribe the VMM's OPTE port from this multicast group. - // Best-effort since if the VMM is already gone, there's nothing to - // unsubscribe (the OPTE port was destroyed with the VMM). + // Unsubscribe the instance's active VMM OPTE port from this multicast + // group. Best-effort since if the VMM is already gone, there's + // nothing to unsubscribe (the OPTE port was destroyed with the VMM). if let Some(sled_id) = ctx.member.sled_id { if let Err(e) = ctx .sled_client - .unsubscribe_vmm( + .unsubscribe_instance( ctx.opctx, ctx.group, ctx.member, sled_id.into(), - None, ) .await { warn!( ctx.opctx.log, - "failed to unsubscribe VMM from multicast group"; + "failed to unsubscribe instance from multicast group"; "member_id" => %ctx.member.id, "sled_id" => %sled_id, "error" => %e @@ -1206,11 +1093,7 @@ impl MulticastGroupReconciler { SledUuid::from_untyped_uuid(vmm.sled_id.into_untyped_uuid()) }); - let propolis_id = vmm_opt - .as_ref() - .map(|vmm| PropolisUuid::from_untyped_uuid(vmm.id)); - - InstanceMulticastState { valid, sled_id, propolis_id } + InstanceMulticastState { valid, sled_id } } else { InstanceMulticastState::default() }; @@ -1323,8 +1206,8 @@ impl MulticastGroupReconciler { } } - /// Complete a member join by configuring the dataplane and subscribing - /// the VMM. + /// Complete a member join by propagating sled forwarding state and + /// subscribing the VMM's OPTE port. /// /// When `sled_id_override` is provided (e.g., during migration), it /// is used instead of the potentially stale `member.sled_id`. @@ -1332,12 +1215,11 @@ impl MulticastGroupReconciler { /// # Returns /// /// `Ok(true)` when the join completed successfully. `Ok(false)` when no - /// sled was available and the operation was a no-op. + /// sled was available and the operation was a noop. async fn complete_instance_member_join( &self, ctx: &MemberReconcileCtx<'_>, sled_id_override: Option, - cached_propolis_id: Option, ) -> Result { debug!( ctx.opctx.log, @@ -1360,8 +1242,6 @@ impl MulticastGroupReconciler { return Ok(false); }; - self.add_member_to_dataplane(ctx, sled_id).await?; - // If the member is already in a "Joined" state (migration path), skip // the state transition but still propagate and subscribe. During // migration the caller updates the sled ID without changing state, @@ -1396,11 +1276,11 @@ impl MulticastGroupReconciler { // Propagate M2P mappings and forwarding entries to all sleds. // - // Athis point, the member is now "Joined" in the database, so propagate + // At this point, the member is now "Joined" in the database, so propagate // includes this sled in forwarding next-hops. If propagation or // subscribe fails below, the member remains "Joined" with incomplete - // sled state. The reconciler's next pass converges via - // `handle_instance_joined` -> `verify_members`. + // sled state. The reconciler's next pass re-converges sled state. DPD + // members are programmed by `ddmd` from DDM peer subscriptions. // // Propagation failures are best-effort since the reconciler will // re-converge all sleds on the next cycle. Subscribe failures @@ -1420,23 +1300,17 @@ impl MulticastGroupReconciler { ); } - // Subscribe the VMM's OPTE port last. Propagation above is - // best-effort, and any sleds that failed will be converged by the - // reconciler on the next cycle. + // Subscribe the instance's active VMM OPTE port last. Propagation + // above is best-effort, and any sleds that failed will be converged + // by the reconciler on the next cycle. if let Err(e) = ctx .sled_client - .subscribe_vmm( - ctx.opctx, - ctx.group, - ctx.member, - sled_id, - cached_propolis_id, - ) + .subscribe_instance(ctx.opctx, ctx.group, ctx.member, sled_id) .await { warn!( ctx.opctx.log, - "failed to subscribe VMM to multicast group via sled-agent \ + "failed to subscribe instance to multicast group via sled-agent \ (will retry next cycle)"; "member_id" => %ctx.member.id, "group_id" => %ctx.group.id(), @@ -1457,1229 +1331,170 @@ impl MulticastGroupReconciler { Ok(true) } - /// Apply member dataplane configuration (via DPD-client). - async fn add_member_to_dataplane( + /// Cleanup members that are "Left" and time_deleted. + /// This permanently removes member records that are no longer needed. + pub(super) async fn cleanup_deleted_members( &self, - ctx: &MemberReconcileCtx<'_>, - sled_id: SledUuid, - ) -> Result<(), anyhow::Error> { - let MemberReconcileCtx { - opctx, group, member, dataplane_client, .. - } = ctx; - let underlay_group_id = group.underlay_group_id.with_context(|| { - format!("no underlay group for external group {}", group.id()) - })?; - - let underlay_group = self - .datastore - .underlay_multicast_group_fetch(opctx, underlay_group_id) - .await - .context( - "failed to fetch underlay group for member configuration", - )?; + opctx: &OpContext, + ) -> Result { + trace!(opctx.log, "cleaning up deleted multicast members"); - // Resolve sled to switch port configurations - let port_configs = self - .resolve_sled_to_switch_ports(opctx, sled_id, dataplane_client) + let deleted_count = self + .datastore + .multicast_group_members_complete_delete(opctx) .await - .context("failed to resolve sled to switch ports")?; - - for port_config in &port_configs { - let dataplane_member = dpd_client::types::MulticastGroupMember { - port_id: port_config.port_id.clone(), - link_id: port_config.link_id, - direction: port_config.direction, - }; - - dataplane_client - .add_member(&underlay_group, dataplane_member) - .await - .context("failed to apply member configuration via DPD")?; + .context("failed to cleanup deleted members")?; - debug!( + if deleted_count > 0 { + info!( opctx.log, - "member added to DPD"; - "member_id" => %member.id, - "sled_id" => %sled_id, - "port_id" => %port_config.port_id + "cleaned up deleted multicast members"; + "members_deleted" => deleted_count ); } - // TODO: Add uplink (front port) members for egress traffic through to - // Dendrite. - // - // When this is the first instance joining the group, we should also add - // uplink members with `Direction::External` for multicast egress - // traffic out of the rack. - // These uplink members follow a different lifecycle: - // - Added when first instance joins (check group member count) - // - Removed when last instance leaves (would be handled in - // `remove_member_from_dataplane`) - // - // Uplink ports are probably going to be a group-level configuration - // added by external params. + Ok(deleted_count) + } - info!( + /// Implicitly delete empty multicast groups by marking them "Deleting". + /// + /// Group lifecycle is membership-driven, so a group is reaped once it has no + /// member rows with `time_deleted IS NULL`. Emptiness is defined by member + /// row absence rather than by "no Joined members", so a group whose members + /// are all "Left" (e.g. instances stopped) but whose rows persist is not + /// empty and stays alive. Membership origin does not matter here, as static + /// API joins today and IGMP/MLD snooping in the future (RFD 0488) are + /// treated alike. + /// + /// This sweep is the authority for the emptiness decision. The synchronous + /// mark on explicit member leave is only a best-effort latency + /// optimization; correctness does not depend on it firing. + /// + /// Both "Creating" and "Active" groups are swept: + /// - "Active" groups go empty when their last member leaves or their + /// instances are deleted (member rows soft-deleted via + /// `multicast_group_members_mark_for_removal`, then hard-deleted by + /// `cleanup_deleted_members`, which must run before this sweep). + /// - "Creating" groups go empty when implicit creation succeeded but the + /// first member attach failed (orphans). These are gated by the + /// configured `orphan_grace_secs` so an in-flight first attach is not + /// raced. + /// + /// The underlying datastore method uses an atomic NOT EXISTS guard so a + /// concurrent join that adds a member between the emptiness check and the + /// mark-for-removal cannot be lost. + pub(super) async fn cleanup_empty_groups( + &self, + opctx: &OpContext, + ) -> Result { + trace!( opctx.log, - "multicast member configuration applied to switch forwarding tables"; - "member_id" => %member.id, - "instance_id" => %member.parent_id, - "sled_id" => %sled_id, - "port_count" => port_configs.len(), - "dpd_operation" => "add_member_to_underlay_multicast_group" + "checking for empty multicast groups to implicitly delete" ); - Ok(()) - } + let groups = self + .datastore + .multicast_groups_list_by_states( + opctx, + &[MulticastGroupState::Creating, MulticastGroupState::Active], + &DataPageParams::max_page(), + ) + .await + .context("failed to list creating/active groups")?; - /// Remove member from known port configurations. - async fn remove_from_known_ports( - &self, - opctx: &OpContext, - member: &MulticastGroupMember, - sled_id: DbTypedUuid, - port_configs: &[SwitchBackplanePort], - underlay_group: &nexus_db_model::UnderlayMulticastGroup, - dataplane_client: &MulticastDataplaneClient, - ) -> Result<(), anyhow::Error> { - // Remove member from DPD for each port on the sled - for port_config in port_configs { - let dataplane_member = dpd_client::types::MulticastGroupMember { - port_id: port_config.port_id.clone(), - link_id: port_config.link_id, - direction: port_config.direction, - }; + let mut groups_marked = 0; + + for group in groups { + // "Creating" orphans wait out the grace period; "Active" groups + // are reaped as soon as they are observed empty. + if group.state == MulticastGroupState::Creating + && !grace_period_elapsed( + group.time_created(), + self.orphan_grace_period, + ) + { + continue; + } - dataplane_client - .remove_member(underlay_group, dataplane_member) + // Atomically mark for deletion only if no members exist. + // This is race-safe: the NOT EXISTS guard in the datastore method + // ensures we don't delete a group that just gained a member. + let marked = self + .datastore + .mark_multicast_group_for_removal_if_no_members( + opctx, + MulticastGroupUuid::from_untyped_uuid(group.id()), + ) .await - .context("failed to remove member configuration via DPD")?; + .context("failed to check/mark empty group for removal")?; - debug!( + if marked { + info!( + opctx.log, + "auto-deleting empty multicast group"; + "group_id" => %group.id(), + "group_name" => %group.name(), + "group_state" => ?group.state, + ); + groups_marked += 1; + } + } + + if groups_marked > 0 { + info!( opctx.log, - "member removed from DPD"; - "port_id" => %port_config.port_id, - "sled_id" => %sled_id + "marked empty multicast groups for deletion"; + "groups_marked" => groups_marked ); } - info!( - opctx.log, - "multicast member configuration removed from switch forwarding tables"; - "member_id" => %member.id, - "instance_id" => %member.parent_id, - "sled_id" => %sled_id, - "port_count" => port_configs.len(), - "dpd_operation" => "remove_member_from_underlay_multicast_group", - "reason" => "instance_state_change_or_migration" - ); - Ok(()) + Ok(groups_marked) } - /// Compute union of active rear/underlay port IDs across all "Joined" - /// members in a group. Excludes a specific member ID if provided - /// (useful when removing a member). - /// - /// Returns `MemberPortUnion::Complete` if all "Joined" members were - /// successfully resolved, or `MemberPortUnion::Partial` if some members - /// failed to resolve. - async fn compute_active_member_ports( + /// Get all members for a group. + async fn get_group_members( &self, opctx: &OpContext, group_id: Uuid, - dataplane_client: &MulticastDataplaneClient, - exclude_member_id: Option, - ) -> Result { - let group_members = self - .get_group_members(opctx, group_id) - .await - .context("failed to fetch group members for expected port union")?; - - // Filter to joined members, excluding specified member if provided - let joined_members = group_members - .into_iter() - .filter(|mem| { - exclude_member_id - .map_or(true, |id| mem.id.into_untyped_uuid() != id) - }) - .filter(|mem| mem.state == MulticastGroupMemberState::Joined) - .collect::>(); - - // Resolve all members to ports, tracking successes and failures - let member_ports = stream::iter(joined_members) - .then(|mem| async move { - // Check for missing sled_id - let Some(mem_sled_id) = mem.sled_id else { - warn!( - opctx.log, - "joined member missing sled_id: marking union incomplete"; - "member_id" => %mem.id, - "group_id" => %group_id - ); - return None; - }; - - // Attempt to resolve sled to switch ports - match self - .resolve_sled_to_switch_ports( - opctx, - mem_sled_id.into(), - dataplane_client, - ) - .await - { - Ok(ports) => Some((mem, ports)), - Err(e) => { - warn!( - opctx.log, - "failed to resolve member ports for union computation"; - "member_id" => %mem.id, - "sled_id" => %mem_sled_id, - "error" => %e - ); - None - } - } - }) - .collect::>() - .await; - - // Separate successful resolutions from failures - let (resolved, failures): (Vec<_>, Vec<_>) = - member_ports.into_iter().partition(Option::is_some); - let resolved: Vec<_> = resolved.into_iter().flatten().collect(); - let failure_cnt = failures.len(); - - // Extract rear/underlay ports from all successfully resolved members - let active_member_ports = resolved - .into_iter() - .flat_map(|(_, ports)| ports) - .filter_map(|cfg| { - let member = dpd_client::types::MulticastGroupMember { - port_id: cfg.port_id.clone(), - link_id: cfg.link_id, - direction: cfg.direction, - }; - is_rear_underlay_member(&member).then(|| cfg.port_id) - }) - .collect::>(); - - // Return `Complete` or `Partial` based on whether all members resolved - if failure_cnt == 0 { - Ok(MemberPortUnion::Complete(active_member_ports)) - } else { - Ok(MemberPortUnion::Partial(active_member_ports)) - } - } - - /// Remove member by querying DPD directly when sled info is unavailable. - /// (Used when `sled_id` unavailable or resolution fails). - async fn remove_member_fallback( - &self, - opctx: &OpContext, - member: &MulticastGroupMember, - underlay_group: &nexus_db_model::UnderlayMulticastGroup, - dataplane_client: &MulticastDataplaneClient, - ) -> Result<(), anyhow::Error> { - // Sled resolution failed or no sled_id available (e.g., removed - // from inventory, or member.sled_id=NULL). - // - // We only remove rear/underlay ports to avoid interfering with - // other member types (i.e., uplink/external members). - info!( - opctx.log, - "using fallback path: querying DPD directly for member removal"; - "member_id" => %member.id, - "member_sled_id" => ?member.sled_id, - "reason" => "sled_id_unavailable_or_resolution_failed" - ); - - let current_members = dataplane_client - .fetch_underlay_members(underlay_group.multicast_ip.ip()) - .await - .context("failed to fetch DPD state for member removal")?; - - // Compute union of active member ports across all currently - // "Joined" members for this group. We will only remove ports that are - // not required by any active member. - // - // We exclude the current member from the union since we're removing it. - let active_member_ports = match self - .compute_active_member_ports( + ) -> Result, anyhow::Error> { + self.datastore + .multicast_group_members_list_by_id( opctx, - member.external_group_id, - dataplane_client, - Some(member.id.into_untyped_uuid()), + MulticastGroupUuid::from_untyped_uuid(group_id), + &DataPageParams::max_page(), ) .await - { - Ok(MemberPortUnion::Complete(ports)) => ports, - Ok(MemberPortUnion::Partial(_ports)) => { - // Union is partial (some members failed resolution) - // Skip pruning to avoid removing ports that may still be needed - info!( - opctx.log, - "union incomplete: skipping stale port removal to avoid disrupting unresolved members"; - "member_id" => %member.id, - "reason" => "some_joined_members_failed_port_resolution" - ); - return Ok(()); - } - Err(e) => { - // Failed to compute union (avoid removing anything) - info!( - opctx.log, - "failed to compute active member ports for fallback removal: skipping cleanup"; - "member_id" => %member.id, - "error" => %e - ); - return Ok(()); - } - }; - - if let Some(members) = current_members { - for current_member in &members { - // Only consider rear/underlay ports (instance members) - if !is_rear_underlay_member(current_member) { - continue; - } - - // Remove only if not in union of active member ports - if !active_member_ports.contains(¤t_member.port_id) { - dataplane_client - .remove_member(underlay_group, current_member.clone()) - .await - .context( - "failed to remove member from DPD (fallback)", - )?; - - info!( - opctx.log, - "removed stale rear/underlay member via fallback"; - "member_id" => %member.id, - "port_id" => %current_member.port_id - ); - } - } - } - Ok(()) + .context("failed to list group members") } - /// Remove member dataplane configuration (via DPD-client). - async fn remove_member_from_dataplane( + /// Cleanup a member that is marked for deletion (time_deleted set). + /// + /// This unsubscribes a member from its VMM. DPD member removal is + /// handled by `ddmd` based on DDM peer subscriptions. + async fn cleanup_deleted_member( &self, ctx: &MemberReconcileCtx<'_>, ) -> Result<(), anyhow::Error> { - let MemberReconcileCtx { - opctx, group, member, dataplane_client, .. - } = ctx; - - let underlay_group_id = group.underlay_group_id.with_context(|| { - format!( - "no underlay group for external group {}", - member.external_group_id - ) - })?; - - let underlay_group = self - .datastore - .underlay_multicast_group_fetch(opctx, underlay_group_id) - .await - .context("failed to fetch underlay group for member removal")?; - - // Try to remove via known ports if we have a `sled_id` and can resolve it + let MemberReconcileCtx { opctx, group, member, sled_client, .. } = ctx; + // Unsubscribe from sled-agent (best-effort, VMM may be gone). if let Some(sled_id) = member.sled_id { - if let Ok(port_configs) = self - .resolve_sled_to_switch_ports( - opctx, - sled_id.into(), - dataplane_client, - ) + if let Err(e) = sled_client + .unsubscribe_instance(opctx, group, member, sled_id.into()) .await { - self.remove_from_known_ports( - opctx, - member, - sled_id, - &port_configs, - &underlay_group, - dataplane_client, - ) - .await?; - return Ok(()); + debug!( + opctx.log, + "failed to unsubscribe instance during member cleanup"; + "member_id" => %member.id, + "sled_id" => %sled_id, + "error" => %e + ); } } - // Fallback: query DPD directly when `sled_id` unavailable or - // resolution fails - self.remove_member_fallback( - opctx, - member, - &underlay_group, - dataplane_client, - ) - .await?; - - Ok(()) - } - - /// Clean up member dataplane configuration with strict error handling. - /// Ensures dataplane consistency by failing if removal operations fail. - async fn cleanup_member_from_dataplane( - &self, - ctx: &MemberReconcileCtx<'_>, - ) -> Result<(), anyhow::Error> { - let MemberReconcileCtx { opctx, group, member, .. } = ctx; - debug!( - opctx.log, - "cleaning up member from dataplane"; - "member_id" => %member.id, - "group_id" => %group.id(), - "group_name" => group.name().as_str(), - "parent_id" => %member.parent_id, - "time_deleted" => ?member.time_deleted - ); - - // Strict removal from dataplane (fail on errors) - self.remove_member_from_dataplane(ctx).await.context( - "failed to remove member configuration via DPD during cleanup", - )?; - - info!( - opctx.log, - "member cleaned up from dataplane"; - "member_id" => %member.id, - "group_id" => %group.id(), - "group_name" => group.name().as_str() - ); - Ok(()) - } - - /// Verify that a "Joined" member is consistent with dataplane configuration. - /// - /// This function ensures the member is on the correct switch ports by: - /// - Fetching current DPD state to see what ports the member is actually on - /// - Computing expected ports from a refreshed cache - /// - Removing the member from any unexpected/stale rear ports - /// - Adding the member to expected ports - /// - /// If the sled cannot be resolved (e.g., decommissioned), the member - /// is transitioned to "Left" and M2P/forwarding is propagated inline - /// to remove stale entries. - /// - /// This handles cases like `sp_slot` changes where the sled's physical - /// location changed but the `sled_id` stayed the same. - async fn verify_members( - &self, - ctx: &MemberReconcileCtx<'_>, - ) -> Result<(), anyhow::Error> { - let MemberReconcileCtx { - opctx, - group, - member, - dataplane_client, - sled_client, - .. - } = ctx; - debug!( - opctx.log, - "verifying joined member consistency"; - "member_id" => %member.id, - "group_id" => %group.id(), - "group_name" => group.name().as_str() - ); - - // Get sled_id from member - let sled_id = match member.sled_id { - Some(id) => id, - None => { - debug!(opctx.log, - "member has no sled_id, skipping verification"; - "member_id" => %member.id - ); - return Ok(()); - } - }; - - // Get underlay group - let underlay_group_id = group.underlay_group_id.with_context(|| { - format!("no underlay group for external group {}", group.id()) - })?; - - let underlay_group = self - .datastore - .underlay_multicast_group_fetch(opctx, underlay_group_id) - .await - .context("failed to fetch underlay group")?; - - // Resolve expected member configurations (may refresh cache if TTL expired) - let expected_port_configs = match self - .resolve_sled_to_switch_ports( - opctx, - sled_id.into(), - dataplane_client, - ) - .await - { - Ok(configs) => configs, - Err(e) => { - // If we can't resolve the sled anymore (e.g., removed from inventory), - // remove from dataplane and transition to "Left" - warn!( - opctx.log, - "failed to resolve sled to switch ports: removing from dataplane"; - "member_id" => %member.id, - "sled_id" => %sled_id, - "error" => %e - ); - - // Best effort removal on verification - let _ = self.remove_member_from_dataplane(ctx).await; - - // Unsubscribe the VMM before the CAS clears sled_id; - // otherwise, the OPTE subscription is stranded with no - // way to identify the sled on later passes. Best-effort - // since the VMM may already be torn down. - if let Err(e) = sled_client - .unsubscribe_vmm(opctx, group, member, sled_id.into(), None) - .await - { - warn!( - opctx.log, - "failed to unsubscribe VMM during port resolution failure"; - "member_id" => %member.id, - "sled_id" => %sled_id, - "error" => %e - ); - } - - let updated = self - .datastore - .multicast_group_member_to_left_if_current( - opctx, - MulticastGroupUuid::from_untyped_uuid(group.id()), - InstanceUuid::from_untyped_uuid(member.parent_id), - MulticastGroupMemberState::Joined, - ) - .await - .context("failed to transition member to 'Left' after port resolution failure")?; - - if updated { - // Propagate updated M2P/forwarding to remove - // stale entries for this now-Left member. - if let Err(e) = sled_client - .propagate_m2p_and_forwarding(opctx, group) - .await - { - warn!( - opctx.log, - "failed to propagate M2P/forwarding after \ - member left due to unresolvable sled"; - "member_id" => %member.id, - "group_id" => %group.id(), - "error" => %e - ); - } - info!( - opctx.log, - "member transitioned to 'Left': sled no longer resolvable"; - "member_id" => %member.id, - "group_id" => %group.id() - ); - } - return Ok(()); - } - }; - - // Fetch current DPD state to identify stale ports - // We fetch from one switch since all should be consistent - let current_dpd_members = dataplane_client - .fetch_underlay_members(underlay_group.multicast_ip.ip()) - .await - .context( - "failed to fetch current underlay group members from DPD", - )?; - - // Build union of active member ports across all currently - // joined members for this group. This avoids removing ports needed by - // other members while verifying a single member. - let active_member_ports = match self - .compute_active_member_ports( - opctx, - group.id(), - dataplane_client, - None, // Don't exclude any member - ) - .await - { - Ok(MemberPortUnion::Complete(ports)) => Some(ports), - Ok(MemberPortUnion::Partial(_ports)) => { - // Union is partial (skip stale port removal) - info!( - opctx.log, - "union incomplete: skipping stale port removal to avoid disrupting unresolved members"; - "member_id" => %member.id, - "group_id" => %group.id(), - "reason" => "some_joined_members_failed_port_resolution" - ); - None - } - Err(e) => { - // Failed to compute union (skip stale port removal) - info!( - opctx.log, - "failed to compute active member ports for verification: skipping stale port removal"; - "member_id" => %member.id, - "group_id" => %group.id(), - "error" => %e - ); - None - } - }; - - // Only prune stale ports if we successfully resolved All "Joined" members. - // If we could not compute active member ports or if some members failed - // to resolve, avoid removing anything to prevent disrupting other members. - // We'll still proceed to ensure adding expected ports for this member. - let mut stale_ports = Vec::new(); - if let Some(ref active_ports) = active_member_ports { - if let Some(current_members) = ¤t_dpd_members { - for current_member in current_members { - // Only consider rear ports with underlay direction - if !is_rear_underlay_member(current_member) { - continue; - } - - // If this port is not in our active member set, it's stale - if !active_ports.contains(¤t_member.port_id) { - stale_ports.push(current_member.clone()); - } - } - } - } - - // Remove stale ports first - if !stale_ports.is_empty() { - info!( - opctx.log, - "detected member on stale ports: removing before verifying expected ports"; - "member_id" => %member.id, - "sled_id" => %sled_id, - "group_id" => %group.id(), - "stale_port_count" => stale_ports.len(), - "reason" => "sled_physical_location_changed_or_cache_refresh" - ); - - for stale_member in &stale_ports { - match dataplane_client - .remove_member(&underlay_group, stale_member.clone()) - .await - { - Ok(()) => { - debug!( - opctx.log, - "removed member from stale port"; - "member_id" => %member.id, - "old_port_id" => %stale_member.port_id, - "sled_id" => %sled_id - ); - } - Err(e) => { - // Continue as the port might have been removed already - warn!( - opctx.log, - "failed to remove member from stale port (may already be gone)"; - "member_id" => %member.id, - "port_id" => %stale_member.port_id, - "error" => %e - ); - } - } - } - } - - // Add member to all expected ports - for port_config in &expected_port_configs { - let expected_member = dpd_client::types::MulticastGroupMember { - port_id: port_config.port_id.clone(), - link_id: port_config.link_id, - direction: port_config.direction, - }; - - match dataplane_client - .add_member(&underlay_group, expected_member) - .await - { - Ok(()) => { - debug!( - opctx.log, - "member verified/added to expected port"; - "member_id" => %member.id, - "sled_id" => %sled_id, - "port_id" => %port_config.port_id - ); - } - Err(e) => { - // Log as warning since we expect this to succeed - warn!( - opctx.log, - "failed to add member to expected port"; - "member_id" => %member.id, - "port_id" => %port_config.port_id, - "error" => %e - ); - return Err(e.into()); - } - } - } - - // Ensure the VMM subscription is in place for the current propolis_id. - // This is idempotent and covers cases where the propolis_id changed - // (e.g., after live migration) but the sled_id stayed the same. - if let Err(e) = sled_client - .subscribe_vmm(opctx, group, member, sled_id.into(), None) - .await - { - warn!( - opctx.log, - "failed to verify VMM subscription during member verification"; - "member_id" => %member.id, - "sled_id" => %sled_id, - "error" => %e - ); - return Err(e); - } - - info!( - opctx.log, - "member verification completed"; - "member_id" => %member.id, - "sled_id" => %sled_id, - "expected_port_count" => expected_port_configs.len(), - "stale_ports_removed" => stale_ports.len() - ); - - Ok(()) - } - - /// Cleanup members that are "Left" and time_deleted. - /// This permanently removes member records that are no longer needed. - pub async fn cleanup_deleted_members( - &self, - opctx: &OpContext, - ) -> Result { - trace!(opctx.log, "cleaning up deleted multicast members"); - - let deleted_count = self - .datastore - .multicast_group_members_complete_delete(opctx) - .await - .context("failed to cleanup deleted members")?; - - if deleted_count > 0 { - info!( - opctx.log, - "cleaned up deleted multicast members"; - "members_deleted" => deleted_count - ); - } - - Ok(deleted_count) - } - - /// Check for and implicitly delete empty groups. - /// - /// With implicit deletion, all multicast groups are deleted when all members - /// are removed. This function checks "Active" groups for any that have no - /// active members and marks them for deletion. - /// - /// This handles the case where instance deletion causes members to be - /// soft-deleted (via `multicast_group_members_mark_for_removal`), and after - /// the member cleanup removes those records, the group becomes empty. - /// - /// The underlying datastore method uses an atomic NOT EXISTS guard to - /// prevent race conditions where a concurrent join could create a member - /// between the emptiness check and the mark-for-removal. - pub async fn cleanup_empty_groups( - &self, - opctx: &OpContext, - ) -> Result { - trace!( - opctx.log, - "checking for empty multicast groups to implicitly delete" - ); - - // List all Active groups - let active_groups = self - .datastore - .multicast_groups_list_by_state( - opctx, - MulticastGroupState::Active, - &DataPageParams::max_page(), - ) - .await - .context("failed to list active groups")?; - - let mut groups_marked = 0; - - for group in active_groups { - // Atomically mark for deletion only if no members exist. - // This is race-safe: the NOT EXISTS guard in the datastore method - // ensures we don't delete a group that just gained a member. - let marked = self - .datastore - .mark_multicast_group_for_removal_if_no_members( - opctx, - MulticastGroupUuid::from_untyped_uuid(group.id()), - ) - .await - .context("failed to check/mark empty group for removal")?; - - if marked { - info!( - opctx.log, - "auto-deleting empty multicast group"; - "group_id" => %group.id(), - "group_name" => %group.name() - ); - groups_marked += 1; - } - } - - if groups_marked > 0 { - info!( - opctx.log, - "marked empty multicast groups for deletion"; - "groups_marked" => groups_marked - ); - } - - Ok(groups_marked) - } - - /// Get all members for a group. - async fn get_group_members( - &self, - opctx: &OpContext, - group_id: Uuid, - ) -> Result, anyhow::Error> { - self.datastore - .multicast_group_members_list_by_id( - opctx, - MulticastGroupUuid::from_untyped_uuid(group_id), - &DataPageParams::max_page(), - ) - .await - .context("failed to list group members") - } - - /// Check cache for a sled mapping. - async fn check_sled_cache( - &self, - cache_key: SledUuid, - ) -> Option> { - let cache = self.sled_mapping_cache.read().await; - let (cached_at, mappings) = &*cache; - let elapsed = cached_at.elapsed(); - - if elapsed < self.sled_cache_ttl { - mappings.get(&cache_key).cloned() - } else { - None - } - } - - /// Detect backplane topology change and invalidate sled cache if needed. - /// - /// Compares the full (PortId, BackplaneLink) pairs to detect changes in: - /// - Port count (sleds added/removed) - /// - Port IDs (different physical slots) - /// - Link attributes (speed, lanes, connector type changes) - async fn handle_backplane_topology_change( - &self, - opctx: &OpContext, - previous_map: &Option, - new_map: &BackplaneMap, - ) { - if let Some(prev_map) = previous_map { - // Compare full maps (keys + values) to detect any topology changes - if prev_map != new_map { - info!( - opctx.log, - "backplane map topology change detected"; - "previous_port_count" => prev_map.len(), - "new_port_count" => new_map.len() - ); - info!( - opctx.log, - "invalidating sled mapping cache due to backplane topology change" - ); - self.invalidate_sled_mapping_cache().await; - } - } - } - - /// Fetch the backplane map from DPD-client with caching. - /// - /// The client responds with the entire mapping of all cubbies in a rack. - /// - /// The backplane map should remain consistent same across all switches, - /// so we query one switch and cache the result. - async fn fetch_backplane_map( - &self, - opctx: &OpContext, - dataplane_client: &MulticastDataplaneClient, - ) -> Result { - // Check cache first - let previous_map = { - let cache = self.backplane_map_cache.read().await; - if let Some((cached_at, ref map)) = *cache { - let elapsed = cached_at.elapsed(); - - if elapsed < self.backplane_cache_ttl { - trace!( - opctx.log, - "backplane map cache hit"; - "port_count" => map.len() - ); - return Ok(map.clone()); - } - // Cache expired but keep reference to previous map for comparison - Some(map.clone()) - } else { - None - } - }; - - // Fetch from DPD via dataplane client on cache miss - debug!( - opctx.log, - "fetching backplane map from DPD (cache miss or stale)" - ); - - let backplane_map = - dataplane_client.fetch_backplane_map().await.context( - "failed to query backplane_map from DPD via dataplane client", - )?; - - // Detect topology change and invalidate sled cache if needed - self.handle_backplane_topology_change( - opctx, - &previous_map, - &backplane_map, - ) - .await; - - info!( - opctx.log, - "fetched backplane map from DPD"; - "port_count" => backplane_map.len() - ); - - // Update cache - let mut cache = self.backplane_map_cache.write().await; - *cache = Some((Instant::now(), backplane_map.clone())); - - Ok(backplane_map) - } - - /// Resolve a sled ID to switch ports for multicast traffic. - pub async fn resolve_sled_to_switch_ports( - &self, - opctx: &OpContext, - sled_id: SledUuid, - dataplane_client: &MulticastDataplaneClient, - ) -> Result, anyhow::Error> { - // Check cache first - if let Some(port_configs) = self.check_sled_cache(sled_id).await { - return Ok(port_configs); - } - - // Refresh cache if stale or missing entry - if let Err(e) = - self.refresh_sled_mapping_cache(opctx, dataplane_client).await - { - warn!( - opctx.log, - "failed to refresh sled mapping cache, using stale data"; - "sled_id" => %sled_id, - "error" => %e - ); - // Try cache again even with stale data - if let Some(port_configs) = self.check_sled_cache(sled_id).await { - return Ok(port_configs); - } - // If cache refresh failed and no stale data, propagate error - return Err(e.context("failed to refresh sled mapping cache and no cached data available")); - } - - // Try cache again after successful refresh - if let Some(port_configs) = self.check_sled_cache(sled_id).await { - return Ok(port_configs); - } - - // Sled not found after successful cache refresh. We treat this as an error - // so callers can surface this condition rather than silently applying - // no changes. - Err(anyhow::Error::msg(format!( - "failed to resolve sled to switch ports: \ - sled {sled_id} not found in mapping cache (not a scrimlet or removed)" - ))) - } - - /// Find SP in inventory for a given sled's baseboard. - /// Tries exact match (serial + part), then falls back to serial-only. - fn find_sp_for_sled<'a>( - &self, - inventory: &'a nexus_types::inventory::Collection, - sled: &Sled, - ) -> Option<&'a nexus_types::inventory::ServiceProcessor> { - // Try exact match first (serial + part) - if let Some((_, sp)) = inventory.sps.iter().find(|(bb, _)| { - bb.serial_number == sled.serial_number() - && bb.part_number == sled.part_number() - }) { - return Some(sp); - } - - // Fall back to serial-only match - inventory - .sps - .iter() - .find(|(bb, _)| bb.serial_number == sled.serial_number()) - .map(|(_, sp)| sp) - } - - /// Map a single sled to switch port(s), validating against backplane map. - /// Returns Ok(Some(ports)) on success, Ok(None) if validation failed. - fn map_sled_to_ports( - &self, - opctx: &OpContext, - sled: &Sled, - sp_slot: u32, - backplane_map: &BackplaneMap, - ) -> Result>, anyhow::Error> { - let port_id = dpd_client::types::PortId::Rear( - dpd_client::types::Rear::try_from(format!("rear{sp_slot}")) - .context("invalid rear port number")?, - ); - - // Validate against hardware backplane map - if !backplane_map.contains_key(&port_id) { - warn!( - opctx.log, - "sled sp_slot validation failed (not in hardware backplane map)"; - "sled_id" => %sled.id(), - "sp_slot" => sp_slot, - "expected_port" => %format!("rear{}", sp_slot), - "reason" => "inventory_sp_slot_out_of_range_for_platform", - "action" => "skipped_sled_in_mapping_cache" - ); - return Ok(None); - } - - debug!( - opctx.log, - "mapped sled to rear port via inventory"; - "sled_id" => %sled.id(), - "sp_slot" => sp_slot, - "rear_port" => %format!("rear{}", sp_slot) - ); - - Ok(Some(vec![SwitchBackplanePort { - port_id, - link_id: dpd_client::types::LinkId(0), - direction: dpd_client::types::Direction::Underlay, - }])) - } - - /// Build sled-to-port mappings for all sleds using inventory and backplane data. - /// Returns (mappings, validation_failures). - fn build_sled_mappings( - &self, - opctx: &OpContext, - sleds: &[Sled], - inventory: &nexus_types::inventory::Collection, - backplane_map: &BackplaneMap, - ) -> Result< - (HashMap>, usize), - anyhow::Error, - > { - sleds.iter().try_fold( - (HashMap::new(), 0), - |(mut mappings, mut validation_failures), sled| { - let Some(sp) = self.find_sp_for_sled(inventory, sled) else { - debug!( - opctx.log, - "no SP data found for sled in current inventory collection"; - "sled_id" => %sled.id(), - "serial_number" => sled.serial_number(), - "part_number" => sled.part_number() - ); - return Ok((mappings, validation_failures)); - }; - - match self.map_sled_to_ports( - opctx, - sled, - sp.sp_slot.into(), - backplane_map, - )? { - Some(ports) => { - mappings.insert(sled.id(), ports); - } - None => { - validation_failures += 1; - } - } - - Ok((mappings, validation_failures)) - }, - ) - } - - /// Refresh the sled-to-switch-port mapping cache using inventory data. - /// - /// Maps each sled to its physical rear (backplane) port on the switch by: - /// 1. Getting sled's baseboard serial/part from the sled record - /// 2. Looking up the service processor (SP) in inventory for that baseboard - /// (SP information is collected from MGS by the inventory collector) - /// 3. Using `sp.sp_slot` (cubby number) to determine the rear port identifier - /// 4. Creating `PortId::Rear(RearPort::try_from(format!("rear{sp_slot}")))` - /// - /// On the Dendrite side (switch's DPD daemon), a similar mapping is performed: - /// - /// ```rust,ignore - /// // From dendrite/dpd/src/port_map.rs rev_ab_port_map() - /// for entry in SIDECAR_REV_AB_BACKPLANE_MAP.iter() { - /// let port = PortId::Rear(RearPort::try_from(entry.cubby).unwrap()); - /// inner.insert(port, Connector::QSFP(entry.tofino_connector.into())); - /// } - /// ``` - /// - /// Where `entry.cubby` is the physical cubby/slot number (same as our `sp_slot`), - /// and this maps it to a `PortId::Rear` that DPD can program on the Tofino ASIC. - async fn refresh_sled_mapping_cache( - &self, - opctx: &OpContext, - dataplane_client: &MulticastDataplaneClient, - ) -> Result<(), anyhow::Error> { - // Fetch required data - let inventory = self - .datastore - .inventory_get_latest_collection(opctx) - .await - .context("failed to get latest inventory collection")? - .ok_or_else(|| { - anyhow::Error::msg("no inventory collection available") - })?; - - // First attempt with current backplane map - let mut backplane_map = - self.fetch_backplane_map(opctx, dataplane_client).await?; - - let sleds = self - .datastore - .sled_list_all_batched(opctx, SledFilter::InService) - .await - .context("failed to list in-service sleds for inventory mapping")?; - - // Build sled → port mappings - let (mut mappings, mut validation_failures) = self - .build_sled_mappings(opctx, &sleds, &inventory, &backplane_map)?; - - // If we had validation failures, invalidate backplane cache and retry once - if validation_failures > 0 { - info!( - opctx.log, - "sled validation failures detected: invalidating backplane cache and retrying"; - "validation_failures" => validation_failures - ); - - // Invalidate the backplane cache - self.invalidate_backplane_cache().await; - - // Fetch fresh backplane map - backplane_map = self - .fetch_backplane_map(opctx, dataplane_client) - .await - .context( - "failed to fetch fresh backplane map after invalidation", - )?; - - // Retry mapping with fresh backplane data - (mappings, validation_failures) = self.build_sled_mappings( - opctx, - &sleds, - &inventory, - &backplane_map, - )?; - - // Log sleds that still fail with fresh backplane data - if validation_failures > 0 { - warn!( - opctx.log, - "some sleds still fail validation with fresh backplane map"; - "validation_failures" => validation_failures - ); - } - } - - // Update cache - let sled_count = mappings.len(); - let mut cache = self.sled_mapping_cache.write().await; - *cache = (Instant::now(), mappings); - - // Log results - if validation_failures > 0 { - warn!( - opctx.log, - "sled mapping cache refreshed with validation failures"; - "total_sleds" => sleds.len(), - "mapped_sleds" => sled_count, - "validation_failures" => validation_failures - ); - } else { - info!( - opctx.log, - "sled mapping cache refreshed successfully"; - "total_sleds" => sleds.len(), - "mapped_sleds" => sled_count - ); - } - Ok(()) } - /// Cleanup a member that is marked for deletion (time_deleted set). - /// - /// This includes unsubscribing a member from its VMM, removing - /// it from the dataplane, and hard-deleting the DB row. - async fn cleanup_deleted_member( - &self, - ctx: &MemberReconcileCtx<'_>, - ) -> Result<(), anyhow::Error> { - let MemberReconcileCtx { opctx, group, member, sled_client, .. } = ctx; - // Unsubscribe from sled-agent (best-effort, VMM may be gone). - if let Some(sled_id) = member.sled_id { - if let Err(e) = sled_client - .unsubscribe_vmm(opctx, group, member, sled_id.into(), None) - .await - { - debug!( - opctx.log, - "failed to unsubscribe VMM during member cleanup"; - "member_id" => %member.id, - "sled_id" => %sled_id, - "error" => %e - ); - } - } - - // Use the consolidated cleanup helper with strict error handling - self.cleanup_member_from_dataplane(ctx).await - } - /// Get all multicast groups that need member reconciliation. /// Returns "Creating", "Active", and "Deleting" groups. async fn get_reconcilable_groups( diff --git a/nexus/src/app/background/tasks/multicast/mod.rs b/nexus/src/app/background/tasks/multicast/mod.rs index 6ab622179be..8a6a80b5a61 100644 --- a/nexus/src/app/background/tasks/multicast/mod.rs +++ b/nexus/src/app/background/tasks/multicast/mod.rs @@ -18,9 +18,8 @@ //! - Database state (groups, members, routing configuration) //! - Dataplane state (match-action tables via Dendrite/DPD) //! - Instance lifecycle (start/stop/migrate affecting group membership) -//! - Network topology (sled-to-switch mappings, port configurations) //! -//! ## Architecture: RPW +/- Sagas +//! ## Architecture: RPW and Sagas //! //! **Sagas handle immediate operations:** //! - Instance lifecycle events (start/stop/delete) @@ -32,6 +31,8 @@ //! - Dataplane state convergence //! - Group and Member state checks and transitions ("Joining" → "Joined" → "Left") //! - Drift detection and correction +//! - Switch zone coordination: MRIB route programming through MGD +//! (mg-lower then derives underlay members from DDM peer subscriptions) //! - Cleanup of orphaned resources //! //! ## Multicast Group Architecture @@ -97,7 +98,8 @@ //! 4. **OPTE decapsulation** removes Geneve/IPv6/Ethernet outer headers on target sleds //! 5. **Instance delivery** of inner (guest-facing) packet to guest //! -//! TODO: Other traffic flows like egress from instances will be documented separately. +//! TODO: Egress (instance → external) is not yet supported. See RFD 488 +//! (§sect-external-mcast) for the design. //! //! ## Reconciliation Components //! @@ -105,9 +107,45 @@ //! - **Group lifecycle**: "Creating" → "Active" → "Deleting" → hard-deleted //! - **Member lifecycle**: "Joining" → "Joined" → "Left" → soft-deleted → hard-deleted //! - **Dataplane updates**: DPD API calls for P4 table updates +//! - **MRIB programming**: multicast routing entries written through +//! MGD, diffed against a per-pass snapshot and withdrawn when no +//! "Joined" members remain so DDM peers stop sending traffic //! - **Sled propagation**: M2P mappings and forwarding entries pushed to sled-agents -//! - **OPTE subscriptions**: Per-VMM multicast group subscriptions on target sleds -//! - **Topology mapping**: Sled-to-switch-port resolution (with caching) +//! - **OPTE subscriptions**: Per-instance multicast group subscriptions +//! on target sleds (keyed at the sled by the active VMM's propolis-id) +//! +//! ## RPW Saga Coordination +//! +//! The reconciler launches sagas for transactional operations +//! (e.g. external+underlay group ensure). By default sagas retry +//! independently and the next reconciler tick observes the resulting +//! state. +//! +//! For group creation, the reconciler instead drains saga completion +//! within the same pass so [`reconcile_member_states`] and +//! [`reconcile_active_groups`] can converge in one tick. The motivation +//! is operator-visible latency: members see multicast settle within a +//! single reconciler interval of joining, rather than waiting an +//! additional tick for the saga's effects to be observed. +//! +//! This same-pass drain is bounded by the enclosing `buffer_unordered` +//! concurrency (one slot per in-flight saga), so multiple groups still +//! progress in parallel. A saga failure propagates to that group's per-iteration +//! result and is logged. The pass does not abort, and member / active +//! reconciliation still runs for the groups that succeeded. +//! +//! Saga completion is unbounded. Steno retries transient errors +//! indefinitely, so a wedged dependency holds a `buffer_unordered` +//! slot until the saga unwinds. The `group_concurrency_limit` (see +//! [`MulticastGroupReconcilerConfig`] for the default) caps concurrent +//! slots, but a slow saga in the creating phase delays the start of +//! [`reconcile_member_states`] and [`reconcile_active_groups`] for +//! this tick. +//! +//! This mirrors the `saga_run` + drain pattern used by +//! [`instance_reincarnation`] and [`instance_updater`], but interleaves +//! start-and-await per group inside `buffer_unordered` rather than +//! batch-starting then batch-draining like those tasks do. //! //! ## Deletion Semantics: Groups vs Members //! @@ -124,57 +162,39 @@ //! - RPW can transition back to "Joining" when instance becomes valid //! - Instance deleted: state="Left", time_deleted=SET (permanent soft-delete) //! - Cannot be reactivated (new attach creates new member record) -//! - RPW removes DPD configuration +//! - RPW removes OPTE subscriptions and sled-agent multicast state //! - Cleanup task eventually hard-deletes the row //! //! [RFC 7346]: https://www.rfc-editor.org/rfc/rfc7346 +//! [`UNDERLAY_MULTICAST_SUBNET`]: omicron_common::address::UNDERLAY_MULTICAST_SUBNET +//! [`reconcile_member_states`]: MulticastGroupReconciler::reconcile_member_states +//! [`reconcile_active_groups`]: MulticastGroupReconciler::reconcile_active_groups +//! [`instance_reincarnation`]: crate::app::background::tasks::instance_reincarnation +//! [`instance_updater`]: crate::app::background::tasks::instance_updater +//! [`MulticastGroupReconcilerConfig`]: nexus_config::MulticastGroupReconcilerConfig -use std::collections::{BTreeMap, HashMap}; -use std::net::{IpAddr, Ipv6Addr}; use std::sync::Arc; -use std::time::{Duration, Instant}; use futures::FutureExt; use futures::future::BoxFuture; use internal_dns_resolver::Resolver; use serde_json::json; use slog::{error, info}; -use tokio::sync::RwLock; -use tokio::sync::watch::Receiver; use nexus_db_model::MulticastGroup; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; use nexus_types::internal_api::background::MulticastGroupReconcilerStatus; -use nexus_types::inventory::{Collection, SpType}; -use omicron_common::address::UNDERLAY_MULTICAST_SUBNET; -use omicron_uuid_kinds::SledUuid; -use sled_hardware_types::BaseboardId; use crate::app::background::BackgroundTask; use crate::app::multicast::dataplane::MulticastDataplaneClient; use crate::app::multicast::sled::MulticastSledClient; +use crate::app::multicast::switch_zone::MulticastSwitchZoneClient; use crate::app::saga::StartSaga; pub(crate) mod groups; pub(crate) mod members; - -/// Type alias for the sled mapping cache. -type SledMappingCache = - Arc>)>>; - -/// Type alias for the backplane map cache. -type BackplaneMapCache = Arc< - RwLock< - Option<( - Instant, - BTreeMap< - dpd_client::types::PortId, - dpd_client::types::BackplaneLink, - >, - )>, - >, ->; +mod mrib; /// Result of processing a state transition for multicast entities. #[derive(Debug)] @@ -189,45 +209,21 @@ pub(crate) enum StateTransition { EntityGone, } -/// Switch port configuration for multicast group members. -#[derive(Clone, Debug)] -pub(crate) struct SwitchBackplanePort { - /// Switch port ID - pub port_id: dpd_client::types::PortId, - /// Switch link ID - pub link_id: dpd_client::types::LinkId, - /// Direction for multicast traffic (External or Underlay) - pub direction: dpd_client::types::Direction, -} - /// Background task that reconciles multicast group state with Dendrite /// configuration using the Saga + RPW hybrid pattern. pub(crate) struct MulticastGroupReconciler { datastore: Arc, resolver: Resolver, sagas: Arc, - /// Receiver for inventory updates from the inventory loader background task. - rx_inventory: Receiver>>, - /// Cache for sled-to-backplane-port mappings. - /// Maps sled_id → rear backplane ports for multicast traffic routing. - sled_mapping_cache: SledMappingCache, - sled_cache_ttl: Duration, - /// Cache for backplane hardware topology from DPD. - /// Maps PortId → BackplaneLink for platform-specific port validation. - backplane_map_cache: BackplaneMapCache, - backplane_cache_ttl: Duration, /// Maximum number of members to process concurrently per group. member_concurrency_limit: usize, /// Maximum number of groups to process concurrently. group_concurrency_limit: usize, - /// Whether multicast functionality is enabled (or not). + /// Grace period before an orphaned "Creating" group with no members is + /// reaped by the emptiness sweep (`cleanup_empty_groups`). + orphan_grace_period: chrono::TimeDelta, + /// Whether multicast functionality is enabled. enabled: bool, - /// Last seen sled baseboard→sp_slot mappings for cache invalidation. - /// - /// We track sled locations (keyed by baseboard identity), as sled - /// physical locations rarely change. Caches are only invalidated - /// when `sp_slot` values differ. - last_seen_sled_slots: HashMap, u16>, } impl MulticastGroupReconciler { @@ -235,27 +231,19 @@ impl MulticastGroupReconciler { datastore: Arc, resolver: Resolver, sagas: Arc, - rx_inventory: Receiver>>, enabled: bool, - sled_cache_ttl: Duration, - backplane_cache_ttl: Duration, + group_concurrency_limit: usize, + member_concurrency_limit: usize, + orphan_grace_period: chrono::TimeDelta, ) -> Self { Self { datastore, resolver, sagas, - rx_inventory, - sled_mapping_cache: Arc::new(RwLock::new(( - Instant::now(), - HashMap::new(), - ))), - sled_cache_ttl, - backplane_map_cache: Arc::new(RwLock::new(None)), - backplane_cache_ttl, - member_concurrency_limit: 100, - group_concurrency_limit: 100, + member_concurrency_limit, + group_concurrency_limit, + orphan_grace_period, enabled, - last_seen_sled_slots: HashMap::new(), } } @@ -266,184 +254,6 @@ impl MulticastGroupReconciler { pub(crate) fn get_multicast_tag(group: &MulticastGroup) -> Option<&str> { group.tag.as_deref() } - - /// Invalidate the backplane map cache, forcing refresh on next access. - /// - /// Called when: - /// - Sled validation fails (sp_slot not in cached backplane map) - /// - Need to refresh topology data after detecting potential changes - pub(crate) async fn invalidate_backplane_cache(&self) { - let mut cache = self.backplane_map_cache.write().await; - *cache = None; // Clear the cache entirely - } - - /// Invalidate the sled mapping cache, forcing refresh on next access. - /// - /// Called when: - /// - Backplane topology changes detected (different port count/layout) - /// - Need to re-validate sled mappings against new topology - pub(crate) async fn invalidate_sled_mapping_cache(&self) { - let mut cache = self.sled_mapping_cache.write().await; - // Set timestamp to past to force refresh on next check - *cache = (Instant::now() - self.sled_cache_ttl, cache.1.clone()); - } - - /// Check if sled locations changed and invalidate caches if so. - /// - /// Compares actual serial→sp_slot mappings since sled locations rarely - /// change. Uses the inventory watch channel for cheap access to latest - /// inventory. - async fn check_sled_locations_for_cache_invalidation( - &mut self, - opctx: &OpContext, - ) { - // Get inventory from watch channel (cheap Arc::clone, no DB query) - let Some(inventory) = - self.rx_inventory.borrow_and_update().as_ref().map(Arc::clone) - else { - debug!( - opctx.log, - "skipping cache invalidation check: no inventory available" - ); - return; - }; - - // Build current baseboard→sp_slot mapping for sleds only - let current_sled_slots: HashMap, u16> = inventory - .sps - .iter() - .filter(|(_, sp)| sp.sp_type == SpType::Sled) - .map(|(baseboard, sp)| (Arc::clone(baseboard), sp.sp_slot)) - .collect(); - - if current_sled_slots != self.last_seen_sled_slots { - // Skip invalidation on first run (just initializing) - if !self.last_seen_sled_slots.is_empty() { - info!( - opctx.log, - "invalidating multicast caches due to sled location change"; - "previous_sled_count" => self.last_seen_sled_slots.len(), - "current_sled_count" => current_sled_slots.len() - ); - self.invalidate_backplane_cache().await; - self.invalidate_sled_mapping_cache().await; - } - self.last_seen_sled_slots = current_sled_slots; - } - } -} - -/// Maps an external multicast address to an underlay address in ff04::/64. -/// -/// Maps external addresses into [`UNDERLAY_MULTICAST_SUBNET`] (ff04::/64, -/// a subset of the admin-local scope ff04::/16 per RFC 7346) using XOR-fold. This prefix is static -/// for consistency across racks. -/// -/// See [RFC 7346] for IPv6 multicast admin-local scope. -/// -/// # Salt Parameter (Collision Avoidance) -/// -/// The `salt` enables collision avoidance via XOR perturbation. XOR is bijective: -/// distinct salts produce distinct outputs (since `a ⊕ b = a ⊕ c` implies `b = c`), -/// guaranteeing 256 unique addresses per external IP. -/// -/// This is mathematically equivalent to [binary probing] in hash table literature -/// (`h_i(x) := h(x) ⊕ i`), though the domain context differs in that we're mapping -/// into a sparse 2^64 IPv6 address space rather than probing array slots. -/// -/// ```text -/// Salt perturbation example (h = 0xa): -/// ┌──────┬─────────┬────────┐ -/// │ salt │ h ⊕ salt│ output │ -/// ├──────┼─────────┼────────┤ -/// │ 0 │ 0xa ⊕ 0 │ 0xa │ -/// │ 1 │ 0xa ⊕ 1 │ 0xb │ -/// │ 2 │ 0xa ⊕ 2 │ 0x8 │ -/// │ 3 │ 0xa ⊕ 3 │ 0x9 │ -/// │ 4 │ 0xa ⊕ 4 │ 0xe │ -/// │ 5 │ 0xa ⊕ 5 │ 0xf │ -/// │ 6 │ 0xa ⊕ 6 │ 0xc │ -/// │ 7 │ 0xa ⊕ 7 │ 0xd │ -/// └──────┴─────────┴────────┘ -/// Outputs: [a, b, 8, 9, e, f, c, d] (scattered, not sequential) -/// ``` -/// -/// On collision (i.e., underlay IP already in use), we increment salt and retry. -/// This stores the successful salt with the group for deterministic -/// reconstruction. -/// -/// # Implementation -/// -/// ```text -/// underlay_ip = ff04:: | ((xor_fold(external_ip) ⊕ salt) & HOST_MASK) -/// ``` -/// -/// - IPv4: embedded directly (32 bits fits in 64-bit host space) -/// - IPv6: XOR upper and lower 64-bit halves to fold 128→64 bits -/// - Salt ∈ [0, 255]: XORed into host bits for collision retry -/// -/// The `& HOST_MASK` guarantees the result stays within ff04::/64, our static -/// underlay subnet. -/// -/// [RFC 7346]: https://www.rfc-editor.org/rfc/rfc7346 -/// [binary probing]: https://courses.grainger.illinois.edu/CS473/fa2025/notes/05-hashing.pdf -fn map_external_to_underlay_ip(external_ip: IpAddr, salt: u8) -> IpAddr { - // Derive constants from the default underlay multicast subnet - const HOST_BITS: u32 = 128 - UNDERLAY_MULTICAST_SUBNET.width() as u32; - let prefix_base = - u128::from_be_bytes(UNDERLAY_MULTICAST_SUBNET.addr().octets()); - - map_external_to_underlay_ip_impl(prefix_base, HOST_BITS, external_ip, salt) -} - -/// Core implementation: maps external multicast IP to underlay IPv6 address. -/// -/// Separated for testing purposes. -/// -/// Parameters: -/// - `prefix_base`: Network prefix as u128 (e.g., ff04:: → 0xff04_0000_...) -/// - `host_bits`: Number of host bits (e.g., 64 for a /64 prefix) -/// - `external_ip`: The external multicast address to map -/// - `salt`: XOR perturbation for collision avoidance (0-255) -/// -/// Returns: The mapped underlay IPv6 address -fn map_external_to_underlay_ip_impl( - prefix_base: u128, - host_bits: u32, - external_ip: IpAddr, - salt: u8, -) -> IpAddr { - let host_mask: u128 = - if host_bits >= 128 { u128::MAX } else { (1u128 << host_bits) - 1 }; - - // Derive host value from external IP - let host_value: u128 = match external_ip { - IpAddr::V4(ipv4) => { - // IPv4 (32 bits) fits directly in host space - u128::from(u32::from_be_bytes(ipv4.octets())) - } - IpAddr::V6(ipv6) => { - // XOR-fold 128 bits → host_bits (upper ^ lower). - // This ensures different external addresses (even with identical - // lower bits but different scopes) map to different underlay IPs. - let full = u128::from_be_bytes(ipv6.octets()); - if host_bits >= 128 { - full - } else { - (full >> host_bits) ^ (full & host_mask) - } - } - }; - - // XOR salt for collision avoidance retry, masked to stay in host bits. - // The salt is applied after folding, ensuring different salts produce - // different underlay IPs while staying within the prefix. - let salted = (host_value ^ u128::from(salt)) & host_mask; - - // Combine prefix + host (masking guarantees result stays in prefix) - let underlay = prefix_base | salted; - - IpAddr::V6(Ipv6Addr::from(underlay.to_be_bytes())) } impl BackgroundTask for MulticastGroupReconciler { @@ -516,7 +326,23 @@ impl MulticastGroupReconciler { trace!(opctx.log, "starting multicast reconciliation pass"); - self.check_sled_locations_for_cache_invalidation(opctx).await; + // Per-pass client construction policy: + // + // - DPD (dataplane): fail-closed. Required by every step. A + // pass without DPD has nothing useful to do. + // - sled-agent: never fails. The wrapper builds per-sled + // clients on demand, so construction is infallible. + // - MGD MRIB: fail-open. Only three steps are MRIB-coupled + // (member states, active reconciliation, deleting + // reconciliation). Creating-group reconciliation and the two + // cleanup steps run regardless. Subsequent passes retry the + // gated steps when MRIB returns. + // + // The non-gated cleanup steps never touch the dataplane. + // `cleanup_empty_groups` only marks "Deleting", and the terminal + // "Deleting" → "Deleted" transition lives in the gated + // `reconcile_deleting_groups`. A group therefore cannot vanish + // from the reconciler's view while its MRIB route still exists. // Create dataplane client (across switches) once for the entire // reconciliation pass (in case anything has changed) @@ -543,6 +369,28 @@ impl MulticastGroupReconciler { self.resolver.clone(), ); + // Create MGD MRIB client for multicast route distribution + // via DDM. `mg-lower` syncs MRIB changes to DDM automatically. + // + // Construction failure (e.g., transient DNS resolution returning + // no switch zones) skips MRIB-coupled work this pass but lets + // creating-group and cleanup paths progress. Subsequent passes + // will retry. + let switch_zone_client = match MulticastSwitchZoneClient::new( + self.resolver.clone(), + opctx.log.clone(), + ) + .await + { + Ok(client) => Some(client), + Err(e) => { + let msg = + format!("failed to create multicast MRIB client: {e:#}"); + status.errors.push(msg); + None + } + }; + // Process creating groups match self.reconcile_creating_groups(opctx).await { Ok(count) => status.groups_created += count, @@ -552,12 +400,14 @@ impl MulticastGroupReconciler { } } - // Process member state changes - match self - .reconcile_member_states(opctx, &dataplane_client, &sled_client) - .await - { - Ok(count) => status.members_processed += count, + // Process member state changes. Underlay dataplane members are + // programmed by ddmd from DDM peer subscriptions; the reconciler only + // advances member DB state and manages OPTE subscriptions plus + // M2P/forwarding propagation via sled-agent. + match self.reconcile_member_states(opctx, &sled_client).await { + Ok(counts) => { + status.members_processed += counts.processed; + } Err(e) => { let msg = format!("failed to reconcile member states: {e:#}"); status.errors.push(msg); @@ -586,28 +436,48 @@ impl MulticastGroupReconciler { } } - // Reconcile active groups (verify state, update dataplane as needed) - match self - .reconcile_active_groups(opctx, &dataplane_client, &sled_client) - .await - { - Ok(count) => status.groups_verified += count, - Err(e) => { - let msg = format!("failed to reconcile active groups: {e:#}"); - status.errors.push(msg); + // Reconcile active groups + if let Some(switch_zone_client) = &switch_zone_client { + match self + .reconcile_active_groups( + opctx, + &dataplane_client, + &sled_client, + switch_zone_client, + ) + .await + { + Ok(count) => status.groups_verified += count, + Err(e) => { + let msg = + format!("failed to reconcile active groups: {e:#}"); + status.errors.push(msg); + } } + } else { + status.skipped.push("reconcile_active_groups".to_string()); } - // Process deleting groups (DPD cleanup + hard-delete from DB) - match self - .reconcile_deleting_groups(opctx, &dataplane_client, &sled_client) - .await - { - Ok(count) => status.groups_deleted += count, - Err(e) => { - let msg = format!("failed to reconcile deleting groups: {e:#}"); - status.errors.push(msg); + // Process deleting groups + if let Some(switch_zone_client) = &switch_zone_client { + match self + .reconcile_deleting_groups( + opctx, + &dataplane_client, + &sled_client, + switch_zone_client, + ) + .await + { + Ok(count) => status.groups_deleted += count, + Err(e) => { + let msg = + format!("failed to reconcile deleting groups: {e:#}"); + status.errors.push(msg); + } } + } else { + status.skipped.push("reconcile_deleting_groups".to_string()); } trace!( @@ -628,11 +498,12 @@ impl MulticastGroupReconciler { #[cfg(test)] mod tests { - use super::*; - use std::collections::HashSet; - use std::net::{Ipv4Addr, Ipv6Addr}; + use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + use crate::app::multicast::{ + map_external_to_underlay_ip, map_external_to_underlay_ip_impl, + }; use ipnet::Ipv6Net; use omicron_common::address::IPV6_ADMIN_SCOPED_MULTICAST_PREFIX; diff --git a/nexus/src/app/background/tasks/multicast/mrib.rs b/nexus/src/app/background/tasks/multicast/mrib.rs new file mode 100644 index 00000000000..fc88252afbc --- /dev/null +++ b/nexus/src/app/background/tasks/multicast/mrib.rs @@ -0,0 +1,186 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! MRIB route reconciliation for active multicast groups. +//! +//! This diffs the desired switch MRIB state, derived from group, member, and +//! source filter records, against a per-pass snapshot fetched by the +//! caller, then issues add/remove RPCs to converge. Best-effort: +//! failures are logged and retried on the next reconciler pass. + +use std::collections::HashSet; +use std::net::{IpAddr, Ipv6Addr}; + +use slog::{debug, warn}; +use slog_error_chain::InlineErrorChain; +use uuid::Uuid; + +use nexus_db_model::{MulticastGroup, MulticastGroupMemberState}; +use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::DataStore; +use nexus_db_queries::db::datastore::multicast::members::SourceFilterState; +use nexus_types::identity::Resource; +use omicron_common::api::external::DataPageParams; +use omicron_uuid_kinds::{GenericUuid, MulticastGroupUuid}; + +use crate::app::multicast::switch_zone::{ + MribRouteIndex, MulticastSwitchZoneClient, +}; + +/// Reconcile MRIB routes for a single active group against the per-pass +/// switch snapshot. Withdraws routes when no "Joined" members remain so +/// peer sleds stop sending traffic. +pub(super) async fn reconcile_group( + opctx: &OpContext, + datastore: &DataStore, + switch_zone_client: &MulticastSwitchZoneClient, + mrib_route_index: Option<&MribRouteIndex>, + group: &MulticastGroup, + source_filter: &SourceFilterState, + underlay_group_id: Uuid, +) { + let group_id = MulticastGroupUuid::from_untyped_uuid(group.id()); + + let members = match datastore + .multicast_group_members_list( + opctx, + group_id, + &DataPageParams::max_page(), + ) + .await + { + Ok(m) => m, + Err(e) => { + warn!( + opctx.log, + "failed to list members for MRIB reconcile, skipping"; + "group_id" => %group.id(), + "error" => InlineErrorChain::new(&e), + ); + return; + } + }; + let has_joined = + members.iter().any(|m| m.state == MulticastGroupMemberState::Joined); + + let underlay_group = match datastore + .underlay_multicast_group_fetch(opctx, underlay_group_id) + .await + { + Ok(g) => g, + Err(e) => { + warn!( + opctx.log, + "failed to fetch underlay group for MRIB reconcile, skipping"; + "group_id" => %group.id(), + "underlay_group_id" => %underlay_group_id, + "error" => InlineErrorChain::new(&e), + ); + return; + } + }; + + let IpAddr::V6(underlay_ip) = underlay_group.multicast_ip.ip() else { + warn!( + opctx.log, + "underlay multicast group has non-IPv6 address"; + "group_id" => %group.id(), + "underlay_ip" => %underlay_group.multicast_ip.ip(), + ); + return; + }; + + converge_routes( + opctx, + switch_zone_client, + mrib_route_index, + group, + source_filter, + underlay_ip, + has_joined, + ) + .await; +} + +/// Diff the per-pass MRIB snapshot against the desired route set and +/// issue add/remove RPCs to converge. +async fn converge_routes( + opctx: &OpContext, + switch_zone_client: &MulticastSwitchZoneClient, + mrib_route_index: Option<&MribRouteIndex>, + group: &MulticastGroup, + source_filter: &SourceFilterState, + underlay_ip: Ipv6Addr, + has_joined: bool, +) { + let group_ip = group.multicast_ip.ip(); + let current = mrib_route_index + .and_then(|index| index.get(&group_ip)) + .cloned() + .unwrap_or_default(); + let current_sources = current.keys().copied().collect::>(); + let desired: HashSet> = if has_joined { + source_filter + .specific_sources + .iter() + .map(|s| Some(*s)) + .chain(source_filter.has_any_source_member.then_some(None)) + .collect() + } else { + HashSet::new() + }; + + // Ensure desired routes exist. + for source in &desired { + let current_switches = current.get(source).cloned().unwrap_or_default(); + if current_switches.len() == switch_zone_client.switch_count() + && current_switches.values().all(|c| *c == underlay_ip) + { + continue; + } + if let Err(e) = + switch_zone_client.add_route(group_ip, underlay_ip, *source).await + { + warn!( + opctx.log, + "failed to ensure MRIB route"; + "group_id" => %group.id(), + "source" => ?source, + "error" => %e, + ); + } + } + + // Remove routes that are no longer desired. The per-pass snapshot lets us + // reconcile against current switch state without per-group RPCs. + for source in current_sources.difference(&desired) { + if let Err(e) = switch_zone_client.remove_route(group_ip, *source).await + { + warn!( + opctx.log, + "failed to remove stale MRIB route"; + "group_id" => %group.id(), + "source" => ?source, + "error" => %e, + ); + } + } + + // Surface RPF flux for diagnostics. The route lands in `mrib_in` + // after `add_route` but only flows once promoted to `mrib_loc`. + for source in &desired { + if !switch_zone_client + .route_active_on_all_switches(group_ip, *source) + .await + { + debug!( + opctx.log, + "MRIB route not yet RPF-verified on all switches"; + "group_id" => %group.id(), + "group_ip" => %group_ip, + "source" => ?source, + ); + } + } +} diff --git a/nexus/src/app/background/tasks/sync_switch_configuration.rs b/nexus/src/app/background/tasks/sync_switch_configuration.rs index 2fb4aa26260..e5f36a66834 100644 --- a/nexus/src/app/background/tasks/sync_switch_configuration.rs +++ b/nexus/src/app/background/tasks/sync_switch_configuration.rs @@ -37,7 +37,6 @@ use mg_api_types::bgp::policy::{ ImportExportPolicy4 as MgImportExportPolicy4, ImportExportPolicy6 as MgImportExportPolicy6, }; -use mg_api_types::rdb::prefix::{Prefix, Prefix4, Prefix6}; use mg_api_types::rib::BestpathFanoutRequest; use mg_api_types::static_routes::{ AddStaticRoute4Request, AddStaticRoute6Request, StaticRoute4, @@ -551,7 +550,7 @@ impl BackgroundTask for SwitchPortSettingsManager { let mut switch_bgp_config: HashMap = HashMap::new(); // Prefixes are associated to BgpConfig via the config id - let mut bgp_announce_prefixes: HashMap> = HashMap::new(); + let mut bgp_announce_prefixes: HashMap> = HashMap::new(); for (switch_slot, port, change) in &changes { let PortSettingsChange::Apply(settings) = change else { @@ -644,19 +643,13 @@ impl BackgroundTask for SwitchPortSettingsManager { }, }; - let mut prefixes: Vec = vec![]; + let mut prefixes: Vec = vec![]; for announcement in &announcements { - match announcement.network.ip() { - IpAddr::V4(value) => { - let prefix = Prefix4 { value, length: announcement.network.prefix() }; - prefixes.push(Prefix::V4(prefix)); - }, - IpAddr::V6(value) => { - let prefix = Prefix6 { value, length: announcement.network.prefix() }; - prefixes.push(Prefix::V6(prefix)); - }, - }; + prefixes.push(IpNet::new_unchecked( + announcement.network.ip(), + announcement.network.prefix(), + )); } bgp_announce_prefixes.insert(bgp_config.bgp_announce_set_id, prefixes); } @@ -718,10 +711,7 @@ impl BackgroundTask for SwitchPortSettingsManager { .filter_map(|x| match x.prefix { IpNetwork::V4(p) => Some( - Prefix4{ - length: p.prefix(), - value: p.ip(), - } + Ipv4Net::new_unchecked(p.ip(), p.prefix()) ), IpNetwork::V6(_) => None, } @@ -740,10 +730,7 @@ impl BackgroundTask for SwitchPortSettingsManager { .filter_map(|x| match x.prefix { IpNetwork::V6(p) => Some( - Prefix6{ - length: p.prefix(), - value: p.ip(), - } + Ipv6Net::new_unchecked(p.ip(), p.prefix()) ), IpNetwork::V4(_) => None, } @@ -786,10 +773,7 @@ impl BackgroundTask for SwitchPortSettingsManager { .filter_map(|x| match x.prefix { IpNetwork::V4(p) => Some( - Prefix4{ - length: p.prefix(), - value: p.ip(), - } + Ipv4Net::new_unchecked(p.ip(), p.prefix()) ), IpNetwork::V6(_) => None, } @@ -808,10 +792,7 @@ impl BackgroundTask for SwitchPortSettingsManager { .filter_map(|x| match x.prefix { IpNetwork::V6(p) => Some( - Prefix6{ - length: p.prefix(), - value: p.ip(), - } + Ipv6Net::new_unchecked(p.ip(), p.prefix()) ), IpNetwork::V4(_) => None, } @@ -828,7 +809,7 @@ impl BackgroundTask for SwitchPortSettingsManager { // now that the peer passes the above validations, add it to the list for configuration let peer_config = BgpPeerConfig { name: format!("{ip}"), - host: SocketAddr::new(ip.into(), 179).to_string(), + host: SocketAddr::new(ip.into(), 179), hold_time: peer.hold_time.into(), idle_hold_time: peer.idle_hold_time.into(), delay_open: peer.delay_open.into(), @@ -1040,21 +1021,7 @@ impl BackgroundTask for SwitchPortSettingsManager { let announcements = bgp_announce_prefixes .get(&config.bgp_announce_set_id) .expect("bgp config is present but announce set is not populated") - .iter() - .map(|prefix| { - match prefix { - Prefix::V4(prefix4) => { - let net = Ipv4Net::new(prefix4.value, prefix4.length) - .expect("Prefix4 and Ipv4Net's value types have diverged"); - IpNet::V4(net) - }, - Prefix::V6(prefix6) => { - let net = Ipv6Net::new(prefix6.value, prefix6.length) - .expect("Prefix6 and Ipv6Net's value types have diverged"); - IpNet::V6(net) - }, - } - }).collect(); + .clone(); let max_paths = match MaxPathConfig::new(*config.max_paths) { @@ -1860,7 +1827,7 @@ fn build_sled_agent_clients( #[derive(PartialEq, Eq, Hash, Debug)] struct SwitchStaticRouteV4 { nexthop: IpAddr, - prefix: Prefix4, + prefix: Ipv4Net, vlan: Option, priority: u8, } @@ -1868,7 +1835,7 @@ struct SwitchStaticRouteV4 { #[derive(PartialEq, Eq, Hash, Debug)] struct SwitchStaticRouteV6 { nexthop: Ipv6Addr, - prefix: Prefix6, + prefix: Ipv6Net, vlan: Option, priority: u8, } @@ -2043,10 +2010,7 @@ fn static_routes_in_db( }; routes.insert(SwitchStaticRoute::V4(SwitchStaticRouteV4 { nexthop: IpAddr::V4(nexthop), - prefix: Prefix4 { - value: dst, - length: route.dst.prefix(), - }, + prefix: Ipv4Net::new_unchecked(dst, route.dst.prefix()), vlan: route.vid.map(|x| x.0), priority, })); @@ -2063,10 +2027,7 @@ fn static_routes_in_db( }; routes.insert(SwitchStaticRoute::V6(SwitchStaticRouteV6 { nexthop, - prefix: Prefix6 { - value: dst, - length: route.dst.prefix(), - }, + prefix: Ipv6Net::new_unchecked(dst, route.dst.prefix()), vlan: route.vid.map(|x| x.0), priority, })); @@ -2078,10 +2039,7 @@ fn static_routes_in_db( }; routes.insert(SwitchStaticRoute::V4(SwitchStaticRouteV4 { nexthop: IpAddr::V6(nexthop), - prefix: Prefix4 { - value: dst, - length: route.dst.prefix(), - }, + prefix: Ipv4Net::new_unchecked(dst, route.dst.prefix()), vlan: route.vid.map(|x| x.0), priority, })); @@ -2325,7 +2283,7 @@ async fn static_routes_on_switch( )); } IpAddr::V6(addr) => { - if let Ok(dst) = destination.parse::() { + if let Ok(dst) = destination.parse::() { flattened.insert(SwitchStaticRoute::V6( SwitchStaticRouteV6 { nexthop: addr, @@ -2336,7 +2294,7 @@ async fn static_routes_on_switch( )); continue; }; - if let Ok(dst) = destination.parse::() { + if let Ok(dst) = destination.parse::() { flattened.insert(SwitchStaticRoute::V4( SwitchStaticRouteV4 { nexthop: IpAddr::V6(addr), diff --git a/nexus/src/app/bgp.rs b/nexus/src/app/bgp.rs index 0ca95b2f294..aead8895865 100644 --- a/nexus/src/app/bgp.rs +++ b/nexus/src/app/bgp.rs @@ -198,22 +198,10 @@ impl super::Nexus { for (peer_id, exports) in exported { for ex in exports.iter() { - let prefix = match ex { - mg_api_types::rdb::prefix::Prefix::V4(v4) => { - oxnet::IpNet::V4(oxnet::Ipv4Net::new_unchecked( - v4.value, v4.length, - )) - } - mg_api_types::rdb::prefix::Prefix::V6(v6) => { - oxnet::IpNet::V6(oxnet::Ipv6Net::new_unchecked( - v6.value, v6.length, - )) - } - }; let export = networking::BgpExported { peer_id: peer_id.clone(), switch: switch_slot, - prefix, + prefix: *ex, }; result.push(export); } diff --git a/nexus/src/app/instance.rs b/nexus/src/app/instance.rs index 662b1b067ee..cf9060d0ca4 100644 --- a/nexus/src/app/instance.rs +++ b/nexus/src/app/instance.rs @@ -1050,6 +1050,7 @@ impl super::Nexus { { if let (InstanceStateChangeError::SledAgent(inner), Some(vmm)) = (&e, state.vmm()) + && inner.vmm_gone() { if let Some(reason) = inner.vmm_failure_reason() { let _ = self @@ -1167,14 +1168,22 @@ impl super::Nexus { return Err(e); } - // Idempotent stop: with no active VMM, the instance-update saga will - // not fire (no terminal transition to drive it), so nudge the - // reconciler to converge any stale "Joined" rows now rather than wait - // a full reconciler tick. - if state.vmm().is_none() && self.multicast_enabled() { - self.background_tasks.task_multicast_reconciler.activate(); + // Detach multicast members (state -> "Left", clear `sled_id`) only + // after sled-agent has acknowledged the Stop request. Doing it + // before the request would tear down M2P/forwarding for a guest + // that is still running if the request fails. + if self.multicast_enabled() { + self.db_datastore + .multicast_group_members_detach_by_instance( + opctx, + InstanceUuid::from_untyped_uuid(authz_instance.id()), + ) + .await?; } + // Activate multicast reconciler to handle switch-level changes + self.background_tasks.task_multicast_reconciler.activate(); + self.db_datastore .instance_fetch_with_vmm(opctx, &authz_instance) .await diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index 9863cbf7ffd..26d08daaa20 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -34,7 +34,6 @@ use nexus_mgs_updates::MgsUpdateDriver; use nexus_types::deployment::PendingMgsUpdates; use nexus_types::deployment::ReconfiguratorConfigParam; -use omicron_common::address::MGS_PORT; use omicron_common::api::external::ByteCount; use omicron_common::api::external::Error; use omicron_uuid_kinds::OmicronZoneUuid; @@ -1251,6 +1250,15 @@ pub(crate) async fn dpd_clients( } }; + // Per-request bounds so a stalled DPD connection can't hang an RPW + // iteration or saga action indefinitely. Matches the timeout pair on + // the shared Nexus `reqwest_client`. + let reqwest_client = reqwest::ClientBuilder::new() + .connect_timeout(std::time::Duration::from_secs(15)) + .timeout(std::time::Duration::from_secs(15)) + .build() + .map_err(|e| format!("failed to build DPD reqwest client: {e}"))?; + let clients: Vec<(SocketAddrV6, dpd_client::Client)> = dpd_socketaddrs .iter() .map(|socket_addr| { @@ -1261,8 +1269,9 @@ pub(crate) async fn dpd_clients( )), }; - let client = dpd_client::Client::new( + let client = dpd_client::Client::new_with_client( &format!("http://{socket_addr}"), + reqwest_client.clone(), client_state, ); @@ -1330,29 +1339,28 @@ pub(crate) async fn lldpd_clients( Ok(clients) } -/// Look up Dendrite addresses in DNS then determine the switch location of -/// any addresses we're able to resolve the SwitchSlot for. If a switch -/// zone is down, the resolution process will fail and the entry will be -/// missing from the result. +#[derive(Clone, Debug)] +pub(crate) struct SwitchZoneTarget { + pub(crate) target: String, + pub(crate) addr: Ipv6Addr, +} + +/// Look up switch zones in DNS, then determine the switch location of any +/// zones we're able to resolve the `SwitchSlot` for. If a switch zone is down, +/// the resolution process will fail and the entry will be missing from the +/// result. /// /// # Errors -/// If we fail to resolve the ipv6 addresses of the Dendrite service we -/// return an error +/// If we fail to resolve the MGS SRV records for switch zones, return an error. async fn switch_zone_address_mappings( resolver: &internal_dns_resolver::Resolver, log: &slog::Logger, ) -> Result, String> { - let switch_zone_addresses = match resolver - .lookup_all_ipv6(ServiceName::Dendrite) - .await - { - Ok(addrs) => addrs, - Err(e) => { - error!(log, "failed to resolve addresses for Dendrite services"; "error" => %e); - return Err(e.to_string()); - } - }; - Ok(map_switch_zone_addrs(&log, switch_zone_addresses, resolver).await) + Ok(switch_zone_targets(resolver, log) + .await? + .into_iter() + .map(|(slot, endpoint)| (slot, endpoint.addr)) + .collect()) } // TODO: #3596 Allow updating of Nexus from `handoff_to_nexus()` @@ -1364,40 +1372,52 @@ async fn switch_zone_address_mappings( // up switch addresses as a whole, since how DNS is currently setup for // Dendrite is insufficient for what we need. // -/// Query MGS in each switch zone to learn which switch slot is being managed by -/// the services located on a given ipv6 address. This information can be used -/// along with the well known port numbers to target a specific switch + service -/// combination. +/// Query MGS in each switch zone to learn which switch slot is managed by each +/// service target. /// /// We return whatever we're able to successfully resolve. In the event of -/// a communication timeout or other failure with MGS, the SwitchSlot -> Ipv6Addr -/// mapping will be missing from the returned HashMap. Callers will need to inspect +/// a communication timeout or other failure with MGS, the corresponding entry +/// will be missing from the returned `HashMap`. Callers will need to inspect /// the contents to ensure what they expect to be there is actually there. -async fn map_switch_zone_addrs( - log: &Logger, - switch_zone_addresses: Vec, +pub(crate) async fn switch_zone_targets( resolver: &internal_dns_resolver::Resolver, -) -> HashMap { + log: &Logger, +) -> Result, String> { use gateway_client::Client as MgsClient; + info!(log, "Determining switch slots managed by switch zones"); - let mut switch_zone_addrs = HashMap::new(); - - for addr in switch_zone_addresses { - let port = match resolver - .lookup_all_socket_v6(ServiceName::ManagementGatewayService) - .await - { - Ok(addrs) => { - let port_map: HashMap = addrs - .into_iter() - .map(|sockaddr| (*sockaddr.ip(), sockaddr.port())) - .collect(); - - *port_map.get(&addr).unwrap_or(&MGS_PORT) + let mgs_targets = match resolver + .lookup_srv(ServiceName::ManagementGatewayService) + .await + { + Ok(targets) => targets, + Err(e) => { + error!(log, "failed to resolve MGS service targets"; "error" => %e); + return Err(e.to_string()); + } + }; + + let mut switch_zone_targets = HashMap::new(); + + for (target, port) in mgs_targets { + let addr = match resolver.ipv6_lookup(&target).await { + Ok(Some(addr)) => addr, + Ok(None) => { + warn!( + log, + "MGS SRV target resolved without an IPv6 address"; + "target" => &target, + ); + continue; } Err(e) => { - error!(log, "failed to resolve MGS addresses"; "error" => %e); - MGS_PORT + warn!( + log, + "failed to resolve IPv6 address for MGS target"; + "target" => &target, + "error" => %e, + ); + continue; } }; @@ -1406,14 +1426,22 @@ async fn map_switch_zone_addrs( log.new(o!("component" => "MgsClient")), ); - info!(log, "determining switch slot managed by switch zone"; "zone_address" => #?addr); + info!( + log, + "determining switch slot managed by switch zone"; + "target" => &target, + "zone_address" => #?addr, + "mgs_port" => port, + ); let switch_slot = match mgs_client.sp_local_switch_id().await { Ok(switch) => { info!( log, "identified switch slot for switch zone"; "slot" => #?switch, - "zone_address" => #?addr + "target" => &target, + "zone_address" => #?addr, + "mgs_port" => port, ); switch.slot } @@ -1421,19 +1449,22 @@ async fn map_switch_zone_addrs( error!( log, "failed to identify switch slot for switch zone"; + "target" => &target, "zone_address" => #?addr, + "mgs_port" => port, "reason" => #?e ); continue; } }; + let endpoint = SwitchZoneTarget { target, addr }; match switch_slot { 0 => { - switch_zone_addrs.insert(SwitchSlot::Switch0, addr); + switch_zone_targets.insert(SwitchSlot::Switch0, endpoint); } 1 => { - switch_zone_addrs.insert(SwitchSlot::Switch1, addr); + switch_zone_targets.insert(SwitchSlot::Switch1, endpoint); } _ => { warn!( @@ -1447,10 +1478,10 @@ async fn map_switch_zone_addrs( info!( log, "completed mapping switch zones to switch slots"; - "mappings" => #?switch_zone_addrs + "mappings" => #?switch_zone_targets ); - switch_zone_addrs + Ok(switch_zone_targets) } /// Begin configuring an external HTTP client, returning a diff --git a/nexus/src/app/multicast/dataplane.rs b/nexus/src/app/multicast/dataplane.rs index 8d858154b2a..f7e169035ae 100644 --- a/nexus/src/app/multicast/dataplane.rs +++ b/nexus/src/app/multicast/dataplane.rs @@ -2,10 +2,16 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -//! Shared multicast dataplane operations for sagas and reconciler. +//! Shared multicast group dataplane operations for sagas and reconciler. //! -//! Unified interface for multicast group and member operations in the -//! dataplane (DPD - Data Plane Daemon). +//! Unified interface for multicast group operations in the dataplane +//! (DPD - Data Plane Daemon): creating, updating, and deleting external and +//! underlay groups, including source filters. +//! +//! This client does not program member state. Under RFD 488, `ddmd` derives +//! rear-port underlay members from DDM peer subscriptions and mg-lower programs +//! them into DPD; OPTE/front-port member subscriptions are handled by the sled +//! client. //! //! ## VNI and Forwarding Model //! @@ -40,6 +46,7 @@ use std::collections::HashMap; use std::net::IpAddr; +use std::time::Duration; use futures::future::try_join_all; use oxnet::MulticastMac; @@ -49,10 +56,10 @@ use dpd_client::Error as DpdError; use dpd_client::types::{ ExternalForwarding, InternalForwarding, IpSrc, MacAddr, MulticastGroupCreateExternalEntry, MulticastGroupCreateUnderlayEntry, - MulticastGroupExternalResponse, MulticastGroupMember, - MulticastGroupResponse, MulticastGroupUnderlayResponse, - MulticastGroupUpdateExternalEntry, MulticastGroupUpdateUnderlayEntry, - MulticastTag, NatTarget, UnderlayMulticastIpv6, Vni, + MulticastGroupExternalResponse, MulticastGroupResponse, + MulticastGroupUnderlayResponse, MulticastGroupUpdateExternalEntry, + MulticastGroupUpdateUnderlayEntry, MulticastTag, NatTarget, + UnderlayMulticastIpv6, Vni, }; use internal_dns_resolver::Resolver; @@ -130,16 +137,11 @@ pub(crate) type MulticastDataplaneResult = Result; /// This handles multicast group and member operations across all switches /// in the rack, with automatic error handling and rollback. /// -/// TODO: Add `switch_port_uplinks` configuration to multicast groups for egress -/// multicast traffic (instances → switches → external hosts). -/// -/// Current implementation handles ingress (external → switches → instances) -/// using rear ports with [`dpd_client::types::Direction::Underlay`]. For egress, -/// we need: -/// - Group-level uplink configuration (which front ports to use) -/// - Uplink members with [`dpd_client::types::Direction::External`] added to -/// underlay groups -/// - Integration with existing `switch_ports_with_uplinks()` for port discovery +/// TODO: Egress (instance → external) is not yet supported. The current +/// implementation only handles ingress (external → switches → instances) +/// using rear ports with [`dpd_client::types::Direction::Underlay`]. See +/// RFD 488 (§sect-external-mcast) for the egress design, which will use +/// front-port uplink members with [`dpd_client::types::Direction::External`]. pub(crate) struct MulticastDataplaneClient { dpd_clients: HashMap, log: Logger, @@ -154,6 +156,15 @@ pub(crate) struct GroupUpdateParams<'a> { pub source_filter: &'a SourceFilterState, } +/// Bound DPD client construction. On timeout (or DNS failure) we yield +/// an empty client map rather than failing the pass: group operations +/// skip with no switches, but DB-only member-state transitions +/// ("Joining" → "Left" when the instance is stopped) still proceed. +const DPD_CLIENT_BUILD_TIMEOUT: Duration = + // Caps the internal-DNS retry budget for `_dendrite._tcp` so a DPD + // outage doesn't starve the bg task's idle window. + Duration::from_secs(5); + impl MulticastDataplaneClient { /// Create a new client - builds fresh DPD clients for current switch /// topology. @@ -161,31 +172,35 @@ impl MulticastDataplaneClient { resolver: Resolver, log: Logger, ) -> MulticastDataplaneResult { - let dpd_clients = dpd_clients(&resolver, &log).await.map_err(|e| { - error!( - log, - "failed to build DPD clients"; - "error" => %e - ); - Error::internal_error("failed to build DPD clients") - })?; + let dpd_clients = match tokio::time::timeout( + DPD_CLIENT_BUILD_TIMEOUT, + dpd_clients(&resolver, &log), + ) + .await + { + Ok(Ok(clients)) => clients, + Ok(Err(e)) => { + warn!( + log, + "failed to build DPD clients, continuing with empty \ + client map"; + "error" => %e, + ); + HashMap::new() + } + Err(_) => { + warn!( + log, + "timed out building DPD clients, continuing with empty \ + client map"; + "timeout" => ?DPD_CLIENT_BUILD_TIMEOUT, + ); + HashMap::new() + } + }; Ok(Self { dpd_clients, log }) } - /// Select a single switch deterministically for read operations. - /// - /// Used when all switches should have identical state and we only need - /// to query one. Selects the first switch in sorted order by location - /// for consistency across invocations. - fn select_one_switch( - &self, - ) -> MulticastDataplaneResult<(&SwitchSlot, &dpd_client::Client)> { - self.dpd_clients - .iter() - .min_by_key(|(loc, _)| *loc) - .ok_or_else(|| Error::internal_error("no DPD clients available")) - } - /// Compute DPD source filter from aggregated member source state. /// /// For SSM addresses, always returns specific sources. For ASM addresses, @@ -251,7 +266,7 @@ impl MulticastDataplaneClient { .into_inner()) } Err(e) => { - error!( + warn!( self.log, "underlay create failed"; "underlay_ip" => %ip, @@ -299,7 +314,7 @@ impl MulticastDataplaneClient { Ok(response.into_inner().into_external_response()?) } Err(e) => { - error!( + warn!( self.log, "external create failed"; "external_ip" => %create.group_ip, @@ -367,7 +382,7 @@ impl MulticastDataplaneClient { } } Err(e) => { - error!( + warn!( self.log, "external update failed"; "external_ip" => %group_ip, @@ -387,9 +402,14 @@ impl MulticastDataplaneClient { /// Apply multicast group configuration across switches (via DPD). /// - /// The `source_filter` contains the aggregated source filtering state from - /// all members. If any member wants "any source", switch-level filtering - /// is disabled. + /// # Arguments + /// + /// * `external_group`: Customer-visible group (overlay IP, VNI, identity). + /// * `underlay_group`: Underlay group derived from `external_group` + /// (ff04::/64 address that DPD programs into the dataplane). + /// * `source_filter`: Aggregated source filtering state across all + /// members. If any member requests "any source", switch-level + /// filtering is disabled. pub(crate) async fn create_groups( &self, external_group: &ExternalMulticastGroup, @@ -458,8 +478,9 @@ impl MulticastDataplaneClient { .await?; // TODO: `vlan_id` is `None` because egress VLAN tagging is not - // yet supported. When egress support is added, this should - // be populated from group configuration. + // yet supported. See RFD 488 (§sect-external-mcast) for + // the egress design. When egress support lands, this + // should be populated from group configuration. let external_entry = MulticastGroupCreateExternalEntry { group_ip: external_group_ip, external_forwarding: ExternalForwarding { @@ -661,8 +682,9 @@ impl MulticastDataplaneClient { // Prepare external update/create entries with pre-computed data. // // TODO: `vlan_id` is `None` because egress VLAN tagging is not - // yet supported. When egress support is added, this should - // be populated from group configuration. + // yet supported. See RFD 488 (§sect-external-mcast) for + // the egress design. When egress support lands, this + // should be populated from group configuration. let external_forwarding = ExternalForwarding { vlan_id: None }; let internal_forwarding = @@ -734,278 +756,15 @@ impl MulticastDataplaneClient { Ok((underlay_last, external_last)) } - /// Modify multicast group members across all switches in parallel. - async fn modify_group_membership( - &self, - underlay_group: &UnderlayMulticastGroup, - member: MulticastGroupMember, - operation_name: &str, - modify_fn: F, - ) -> MulticastDataplaneResult<()> - where - F: Fn( - Vec, - MulticastGroupMember, - ) -> Vec - + Clone - + Send - + 'static, - { - let dpd_clients = &self.dpd_clients; - let operation_name = operation_name.to_string(); - - let modify_ops = dpd_clients.iter().map(|(switch_slot, client)| { - let underlay_ip = underlay_group.multicast_ip.ip(); - let member = member.clone(); - let log = self.log.clone(); - let modify_fn = modify_fn.clone(); - let operation_name = operation_name.clone(); - - async move { - let underlay_ip_admin = underlay_ip.into_underlay_multicast()?; - - // Get current underlay group state, create if missing - let current_group_res = client - .multicast_group_get_underlay(&underlay_ip_admin) - .await; - - let (current_members, current_tag) = match current_group_res { - Ok(response) => { - let inner = response.into_inner(); - (inner.members, inner.tag) - } - Err(DpdError::ErrorResponse(ref resp)) - if resp.status() == reqwest::StatusCode::NOT_FOUND => - { - // Underlay group doesn't exist -> recreate it with empty members. - // This can happen when all members have left and the group was - // cleaned up by DPD, yet the external group still exists. - info!( - log, - "underlay group not found, creating before member operation"; - "underlay_ip" => %underlay_ip, - "switch" => ?switch_slot, - "operation" => %operation_name, - "dpd_operation" => "modify_group_membership_recreate" - ); - - let create_entry = MulticastGroupCreateUnderlayEntry { - group_ip: underlay_ip_admin.clone(), - members: Vec::new(), - tag: underlay_group.tag.clone(), - }; - - match client - .multicast_group_create_underlay(&create_entry) - .await - { - Ok(response) => { - let created = response.into_inner(); - (created.members, created.tag) - } - Err(DpdError::ErrorResponse(ref resp)) - if resp.status() == reqwest::StatusCode::CONFLICT => - { - // Race condition: another request created the group - // between our GET (404) and CREATE. Re-fetch it. - debug!( - log, - "underlay group created by concurrent request, fetching"; - "underlay_ip" => %underlay_ip, - "switch" => ?switch_slot, - "dpd_operation" => "modify_group_membership_recreate" - ); - - let fetched = client - .multicast_group_get_underlay(&underlay_ip_admin) - .await - .map_err(|e| { - error!( - log, - "underlay re-fetch after conflict failed"; - "underlay_ip" => %underlay_ip, - "switch" => ?switch_slot, - "error" => %e, - "dpd_operation" => "modify_group_membership_recreate" - ); - Error::internal_error(&format!( - "underlay re-fetch after conflict failed on {switch_slot:?}: {e}" - )) - })? - .into_inner(); - - (fetched.members, fetched.tag) - } - Err(e) => { - error!( - log, - "underlay recreate failed"; - "underlay_ip" => %underlay_ip, - "switch" => ?switch_slot, - "error" => %e, - "dpd_operation" => "modify_group_membership_recreate" - ); - return Err(Error::internal_error(&format!( - "underlay recreate failed on {switch_slot:?}: {e}" - ))); - } - } - } - Err(e) => { - error!( - log, - "underlay get failed"; - "underlay_ip" => %underlay_ip, - "switch" => ?switch_slot, - "error" => %e, - "dpd_operation" => "modify_group_membership_get" - ); - return Err(Error::internal_error(&format!( - "underlay get failed on {switch_slot:?}: {e}" - ))); - } - }; - - // Extract member info for logging before consuming member - let member_port_id = member.port_id.clone(); - let member_link_id = member.link_id; - let member_direction = member.direction; - - // Pre-compute dpd_operation for logging - let dpd_operation_done = - format!("{operation_name}_member_in_underlay_group"); - - // Apply the modification function - let updated_members = modify_fn(current_members, member); - - let update_entry = MulticastGroupUpdateUnderlayEntry { - members: updated_members, - }; - - let tag: MulticastTag = current_tag.clone().try_into() - .map_err(|e| Error::internal_error( - &format!("invalid multicast tag: {e}") - ))?; - - let update_res = client - .multicast_group_update_underlay( - &underlay_ip_admin, &tag, &update_entry) - .await; - - update_res.map_err(|e| { - error!( - log, - "underlay member modify failed"; - "operation_name" => %operation_name, - "underlay_ip" => %underlay_ip, - "switch" => ?switch_slot, - "error" => %e, - "dpd_operation" => "modify_group_membership_update" - ); - Error::internal_error(&format!( - "underlay member modify failed on {switch_slot:?}: {e}" - )) - })?; - - info!( - log, - "DPD multicast member operation completed on switch"; - "operation_name" => %operation_name, - "underlay_group_ip" => %underlay_ip, - "member_port_id" => %member_port_id, - "member_link_id" => %member_link_id, - "member_direction" => ?member_direction, - "switch_slot" => ?switch_slot, - "dpd_operation" => %dpd_operation_done - ); - - Ok::<(), Error>(()) - } - }); - - try_join_all(modify_ops).await?; - Ok(()) - } - - /// Add a member to a multicast group in the dataplane. - pub(crate) async fn add_member( - &self, - underlay_group: &UnderlayMulticastGroup, - member: MulticastGroupMember, - ) -> MulticastDataplaneResult<()> { - info!( - self.log, - "DPD multicast member addition initiated across rack switches"; - "underlay_group_id" => %underlay_group.id, - "underlay_multicast_ip" => %underlay_group.multicast_ip, - "member_port_id" => %member.port_id, - "member_link_id" => %member.link_id, - "member_direction" => ?member.direction, - "switch_count" => self.switch_count(), - "dpd_operation" => "update_underlay_group_members" - ); - - self.modify_group_membership( - underlay_group, - member, - "add", - |mut existing_members, new_member| { - // Add to existing members (avoiding duplicates) - if !existing_members.iter().any(|m| { - m.port_id == new_member.port_id - && m.link_id == new_member.link_id - && m.direction == new_member.direction - }) { - existing_members.push(new_member); - } - existing_members - }, - ) - .await - } - - /// Remove a member from a multicast group in the dataplane. - pub(crate) async fn remove_member( - &self, - underlay_group: &UnderlayMulticastGroup, - member: MulticastGroupMember, - ) -> MulticastDataplaneResult<()> { - info!( - self.log, - "DPD multicast member removal initiated across rack switches"; - "underlay_group_id" => %underlay_group.id, - "underlay_multicast_ip" => %underlay_group.multicast_ip, - "member_port_id" => %member.port_id, - "member_link_id" => %member.link_id, - "member_direction" => ?member.direction, - "switch_count" => self.switch_count(), - "dpd_operation" => "update_underlay_group_members" - ); - - self.modify_group_membership( - underlay_group, - member, - "remove", - |existing_members, target_member| { - // Filter out the target member - existing_members - .into_iter() - .filter(|m| { - !(m.port_id == target_member.port_id - && m.link_id == target_member.link_id - && m.direction == target_member.direction) - }) - .collect() - }, - ) - .await - } - /// Detect and log cross-switch drift for multicast groups. /// - /// We logs errors if: + /// Detection-only. Logs errors when: /// - Group is present on some switches but missing on others (presence drift) /// - Group has different configurations across switches (config drift) + /// + /// Drift correction is handled separately by the active-group reconciler + /// (`groups.rs::reconcile_active_groups`), which re-pushes the + /// authoritative DB state to all switches on the next pass. fn log_drift_issues<'a>( &self, group_ip: IpAddr, @@ -1052,9 +811,11 @@ impl MulticastDataplaneClient { /// Fetch external multicast group DPD state for RPW drift detection. /// /// Queries all switches to detect configuration drift. If any switch has - /// different state (missing group, different config), it will return the - /// found state, so the reconciler can initiate an UPDATE - /// saga that will fix all switches atomically. + /// different state (missing group, different config), returns the found + /// state so the reconciler can re-issue the dataplane operations on the + /// next pass and converge to the intended configuration. Drift repair + /// follows the RPW convergence model rather than an atomic cross-switch + /// saga, so callers should expect *N*-pass convergence on partial failure. pub(crate) async fn fetch_external_group_for_drift_check( &self, group_ip: IpAddr, @@ -1151,147 +912,6 @@ impl MulticastDataplaneClient { Ok(Some(first_config.clone().into_external_response()?)) } - /// Fetch the hardware backplane map from DPD for topology validation. - /// - /// Queries a single switch to get the backplane topology map, which should - /// be identical across all switches. Used by the reconciler to validate that - /// inventory `sp_slot` values are within the valid range for - /// the current hardware. - pub(crate) async fn fetch_backplane_map( - &self, - ) -> MulticastDataplaneResult< - std::collections::BTreeMap< - dpd_client::types::PortId, - dpd_client::types::BackplaneLink, - >, - > { - let (switch_slot, client) = self.select_one_switch()?; - - debug!( - self.log, - "fetching backplane map from DPD for topology validation"; - "switch" => ?switch_slot, - "query_scope" => "single_switch", - "dpd_operation" => "fetch_backplane_map" - ); - - match client.backplane_map().await { - Ok(response) => { - let backplane_map_raw = response.into_inner(); - - // Convert HashMap to BTreeMap - // DPD returns string keys like "rear0", "rear1" - parse them to PortId - let backplane_map: std::collections::BTreeMap<_, _> = backplane_map_raw - .into_iter() - .filter_map(|(port_str, link)| { - match dpd_client::types::PortId::try_from(port_str.as_str()) { - Ok(port_id) => Some((port_id, link)), - Err(e) => { - error!( - self.log, - "failed to parse port ID from backplane map"; - "port_str" => %port_str, - "error" => %e, - "dpd_operation" => "fetch_backplane_map" - ); - None - } - } - }) - .collect(); - - debug!( - self.log, - "backplane map fetched from DPD"; - "switch" => ?switch_slot, - "port_count" => backplane_map.len(), - "dpd_operation" => "fetch_backplane_map" - ); - Ok(backplane_map) - } - Err(e) => { - error!( - self.log, - "backplane map fetch failed"; - "switch" => ?switch_slot, - "error" => %e, - "dpd_operation" => "fetch_backplane_map" - ); - Err(Error::internal_error(&format!( - "failed to fetch backplane map from DPD: {e}" - ))) - } - } - } - - /// Fetch current underlay group members from a single switch. - /// - /// Used by the reconciler to detect stale ports that need to be removed - /// when a member's physical location changes. Queries a single switch - /// since all switches should have identical underlay state. - /// - /// For determinism in drift checks, we select the first switch in sorted - /// order by switch location. - pub(crate) async fn fetch_underlay_members( - &self, - underlay_ip: IpAddr, - ) -> MulticastDataplaneResult>> { - let (switch_slot, client) = self.select_one_switch()?; - - debug!( - self.log, - "fetching underlay group members from DPD for drift detection"; - "underlay_ip" => %underlay_ip, - "switch" => ?switch_slot, - "dpd_operation" => "fetch_underlay_members" - ); - - match client - .multicast_group_get_underlay( - &underlay_ip.into_underlay_multicast()?, - ) - .await - { - Ok(response) => { - let members = response.into_inner().members; - debug!( - self.log, - "underlay group members fetched from DPD"; - "underlay_ip" => %underlay_ip, - "switch" => ?switch_slot, - "member_count" => members.len(), - "dpd_operation" => "fetch_underlay_members" - ); - Ok(Some(members)) - } - Err(DpdError::ErrorResponse(resp)) - if resp.status() == reqwest::StatusCode::NOT_FOUND => - { - debug!( - self.log, - "underlay group not found on switch"; - "underlay_ip" => %underlay_ip, - "switch" => ?switch_slot, - "dpd_operation" => "fetch_underlay_members" - ); - Ok(None) - } - Err(e) => { - error!( - self.log, - "underlay group fetch failed"; - "underlay_ip" => %underlay_ip, - "switch" => ?switch_slot, - "error" => %e, - "dpd_operation" => "fetch_underlay_members" - ); - Err(Error::internal_error(&format!( - "failed to fetch underlay group from DPD: {e}" - ))) - } - } - } - pub(crate) async fn remove_groups( &self, tag: &str, diff --git a/nexus/src/app/multicast/mod.rs b/nexus/src/app/multicast/mod.rs index a5eea32b8d4..3354dffbce4 100644 --- a/nexus/src/app/multicast/mod.rs +++ b/nexus/src/app/multicast/mod.rs @@ -19,11 +19,12 @@ //! linked to it. Cross-silo multicast is enabled by linking the same pool to //! multiple silos. This is the same model used for unicast IP pools. //! -//! # Lifecycle +//! # Implicit lifecycle //! -//! Groups are created implicitly when the first member joins (via member-add) -//! and deleted when the last member leaves. The group's IP is allocated from -//! the multicast pool on creation and returned on deletion. +//! Groups have an implicit lifecycle: they are created when the first member +//! joins (via member-add) and deleted when the last member leaves. The group's +//! IP is allocated from the multicast pool on creation and returned on +//! deletion. //! //! Groups use their UUID as the dpd tag for switch configuration. This avoids //! races when group names are reused after deletion. @@ -47,7 +48,7 @@ //! //! [`UNDERLAY_MULTICAST_SUBNET`]: omicron_common::address::UNDERLAY_MULTICAST_SUBNET -use std::net::IpAddr; +use std::net::{IpAddr, Ipv6Addr}; use std::sync::Arc; use ref_cast::RefCast; @@ -60,19 +61,25 @@ use nexus_db_queries::context::OpContext; use nexus_db_queries::db::datastore::multicast::ExternalMulticastGroupWithSources; use nexus_db_queries::{authz, db}; use nexus_types::external_api::multicast; +use nexus_types::internal_api::views::{ + MulticastDdmPeer, MulticastDdmPeerStatus, MulticastDdmPeersView, +}; use nexus_types::multicast::MulticastGroupCreate; use omicron_common::address::{ - MAX_SOURCE_IPS_PER_GROUP, MAX_SOURCE_IPS_PER_MEMBER, is_ssm_address, + MAX_SOURCE_IPS_PER_GROUP, MAX_SOURCE_IPS_PER_MEMBER, + UNDERLAY_MULTICAST_SUBNET, is_ssm_address, }; use omicron_common::api::external::{ self, CreateResult, DataPageParams, DeleteResult, IdentityMetadataCreateParams, ListResultVec, LookupResult, http_pagination::PaginatedBy, }; +use omicron_ddm_admin_client::PeerStatus; use omicron_uuid_kinds::{GenericUuid, InstanceUuid, MulticastGroupUuid}; pub(crate) mod dataplane; pub(crate) mod sled; +pub(crate) mod switch_zone; /// Validate that SSM addresses have source IPs. /// @@ -194,8 +201,9 @@ impl super::Nexus { /// multicast pool linked to the caller's silo. /// /// Note: SSM validation is done at member join time, not group creation. - /// Groups don't store sources directly; sources are per-member. The group's - /// `source_ips` view field shows the union of all active member sources. + /// Groups don't store sources directly. Sources are per-member. The + /// group's `source_ips` view field shows the union of all active member + /// sources. pub(crate) async fn multicast_group_create( &self, opctx: &OpContext, @@ -341,7 +349,7 @@ impl super::Nexus { /// # Authorization /// /// Requires `Modify` on the instance. Groups are fleet-scoped resources - /// readable by any authenticated user; authorization is enforced on the + /// readable by any authenticated user. Authorization is enforced on the /// instance being attached. /// /// # Behavior @@ -350,7 +358,7 @@ impl super::Nexus { /// - **ID joins**: The group must already exist (returns error otherwise) /// - **Source IPs**: Optional for ASM, required for SSM addresses (232/8, ff3x::/32) /// - **IP version**: Required when joining by name if multiple default pools exist - pub(crate) async fn instance_join_multicast_group( + pub(crate) async fn vmm_join_multicast_group( self: &Arc, opctx: &OpContext, group_identifier: &multicast::MulticastGroupIdentifier, @@ -766,7 +774,7 @@ impl super::Nexus { /// - **Idempotent**: Returns success if the member doesn't exist /// - **Implicit deletion**: If this was the last member, marks the group /// for deletion (reconciler completes cleanup) - pub(crate) async fn instance_leave_multicast_group( + pub(crate) async fn vmm_leave_multicast_group( self: &Arc, opctx: &OpContext, group_lookup: &lookup::MulticastGroup<'_>, @@ -805,10 +813,12 @@ impl super::Nexus { .multicast_group_member_delete_by_id(opctx, member.id) .await?; - // Atomically mark group for deletion if this was the last member. - // The NOT EXISTS guard in the datastore method prevents race conditions - // where a concurrent join could slip in between a "list members" check - // and the mark-for-removal call. + // If this was the last member, eagerly mark the group for deletion so + // it starts deleting within the current reconciler interval instead of + // waiting for the next sweep. This is a best-effort latency + // optimization. `cleanup_empty_groups` is the authority for the + // emptiness decision (including the NOT EXISTS race guard) and reaps + // the group either way. if self .db_datastore .mark_multicast_group_for_removal_if_no_members( @@ -883,6 +893,59 @@ impl super::Nexus { ) .await } + + /// Read-only view of DDM underlay peers across all switch zones. + /// + /// Under RFD 488, `mg-lower` derives underlay multicast members from these + /// DDM peer subscriptions. This surfaces the live per-switch-zone peer set + /// for operators via `omdb nexus multicast ddm-peers`. + pub(crate) async fn multicast_ddm_peers( + &self, + opctx: &OpContext, + ) -> LookupResult { + opctx.authorize(authz::Action::Read, &authz::FLEET).await?; + + let switch_zone_client = switch_zone::MulticastSwitchZoneClient::new( + self.resolver().clone(), + opctx.log.clone(), + ) + .await + .map_err(|e| { + external::Error::internal_error(&format!( + "failed to build switch-zone clients: {e}" + )) + })?; + + let peers = switch_zone_client + .ddm_peers_by_switch() + .await + .into_iter() + .map(|(slot, peer)| { + let (status, status_duration) = match peer.status { + PeerStatus::Init(d) => (MulticastDdmPeerStatus::Init, d), + PeerStatus::Solicit(d) => { + (MulticastDdmPeerStatus::Solicit, d) + } + PeerStatus::Exchange(d) => { + (MulticastDdmPeerStatus::Exchange, d) + } + PeerStatus::Expired(d) => { + (MulticastDdmPeerStatus::Expired, d) + } + }; + MulticastDdmPeer { + switch: format!("{slot:?}"), + addr: peer.addr, + host: peer.host, + status, + status_duration, + if_name: peer.if_name, + } + }) + .collect(); + + Ok(MulticastDdmPeersView { peers }) + } } /// Validate that source IPs match the multicast group's address family. @@ -938,6 +1001,76 @@ fn generate_group_name_from_ip( }) } +/// Maps an external multicast address to an underlay address in ff04::/64. +/// +/// Maps external addresses into [`UNDERLAY_MULTICAST_SUBNET`] (ff04::/64, +/// a subset of the admin-local scope ff04::/16 per RFC 7346) using XOR-fold. +/// This prefix is static for consistency across racks. +/// +/// See [RFC 7346] for IPv6 multicast admin-local scope. +/// +/// # Salt Parameter (Collision Avoidance) +/// +/// The `salt` enables collision avoidance via XOR perturbation. XOR is +/// bijective: distinct salts produce distinct outputs (since +/// `a ^ b = a ^ c` implies `b = c`), guaranteeing 256 unique addresses +/// per external IP. +/// +/// On collision (underlay IP already in use), the caller increments +/// salt and retries. The successful salt is stored with the group for +/// deterministic reconstruction. +/// +/// # Implementation +/// +/// ```text +/// underlay_ip = ff04:: | ((xor_fold(external_ip) ^ salt) & HOST_MASK) +/// ``` +/// +/// - IPv4: embedded directly (32 bits fits in 64-bit host space) +/// - IPv6: XOR upper and lower 64-bit halves to fold 128 to 64 bits +/// - Salt in [0, 255]: XORed into host bits for collision retry +/// +/// The `& HOST_MASK` guarantees the result stays within ff04::/64. +/// +/// [RFC 7346]: https://www.rfc-editor.org/rfc/rfc7346 +pub(crate) fn map_external_to_underlay_ip( + external_ip: IpAddr, + salt: u8, +) -> IpAddr { + const HOST_BITS: u32 = 128 - UNDERLAY_MULTICAST_SUBNET.width() as u32; + let prefix_base = + u128::from_be_bytes(UNDERLAY_MULTICAST_SUBNET.addr().octets()); + + map_external_to_underlay_ip_impl(prefix_base, HOST_BITS, external_ip, salt) +} + +/// Core implementation separated for testing with custom prefix/host_bits. +pub(crate) fn map_external_to_underlay_ip_impl( + prefix_base: u128, + host_bits: u32, + external_ip: IpAddr, + salt: u8, +) -> IpAddr { + let host_mask: u128 = + if host_bits >= 128 { u128::MAX } else { (1u128 << host_bits) - 1 }; + + let host_value: u128 = match external_ip { + IpAddr::V4(ipv4) => u128::from(u32::from_be_bytes(ipv4.octets())), + IpAddr::V6(ipv6) => { + let full = u128::from_be_bytes(ipv6.octets()); + if host_bits >= 128 { + full + } else { + (full >> host_bits) ^ (full & host_mask) + } + } + }; + + let salted = (host_value ^ u128::from(salt)) & host_mask; + let underlay = prefix_base | salted; + IpAddr::V6(Ipv6Addr::from(underlay.to_be_bytes())) +} + #[cfg(test)] mod tests { use super::*; diff --git a/nexus/src/app/multicast/sled.rs b/nexus/src/app/multicast/sled.rs index 60d93d4ba8b..d74d7ab0546 100644 --- a/nexus/src/app/multicast/sled.rs +++ b/nexus/src/app/multicast/sled.rs @@ -18,17 +18,23 @@ //! [`dataplane`]: super::dataplane use std::collections::BTreeSet; +use std::collections::HashMap; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; use std::net::{IpAddr, Ipv6Addr}; use std::sync::Arc; use anyhow::Context; +use futures::future::join_all; +use omicron_common::api::external; +use sled_agent_types::early_networking::SwitchSlot; use slog::{debug, info, warn}; +use nexus_db_lookup::LookupPath; use nexus_db_model::{ MulticastGroup, MulticastGroupMember, MulticastGroupMemberState, }; +use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; use nexus_types::deployment::SledFilter; @@ -87,23 +93,6 @@ impl MulticastSledClient { .await } - /// Look up the current `propolis_id` for an instance. - async fn lookup_propolis_id( - &self, - opctx: &OpContext, - instance_id: InstanceUuid, - ) -> Result, anyhow::Error> { - let instance_state = self - .datastore - .instance_get_state(opctx, &instance_id) - .await - .context("failed to look up instance state")?; - - Ok(instance_state - .and_then(|s| s.propolis_id) - .map(PropolisUuid::from_untyped_uuid)) - } - /// Build the membership descriptor sent to sled-agent for /// subscribe/unsubscribe calls. fn membership_for( @@ -116,37 +105,45 @@ impl MulticastSledClient { } } - /// Subscribe a VMM to a multicast group via sled-agent. + /// Subscribe an instance's active VMM OPTE port to a multicast group. /// - /// Looks up the instance's current `propolis_id` and calls the sled-agent - /// endpoint to configure OPTE port-level multicast filters. The member's - /// per-instance source IPs are passed for SSM filtering. - pub(crate) async fn subscribe_vmm( + /// Sled-agent resolves the active Propolis under its per-instance state + /// lock and configures OPTE port-level multicast filters. The member's + /// per-instance source IPs are passed for SSM filtering. If no active + /// VMM is registered the call is a noop since the OPTE port is gone. + pub(crate) async fn subscribe_instance( &self, opctx: &OpContext, group: &MulticastGroup, member: &MulticastGroupMember, sled_id: SledUuid, - cached_propolis_id: Option, ) -> Result<(), anyhow::Error> { let instance_id = InstanceUuid::from_untyped_uuid(member.parent_id); - // If the instance has no propolis_id (already stopped/destroyed), - // the OPTE port is gone and there's nothing to subscribe. - let propolis_id = match cached_propolis_id { - Some(id) => id, - None => match self.lookup_propolis_id(opctx, instance_id).await? { - Some(id) => id, - None => { - debug!( - opctx.log, - "no propolis_id for instance, skipping subscribe"; - "member_id" => %member.id, - "instance_id" => %instance_id - ); - return Ok(()); - } - }, + + // Read-only dispatch lookup, not a mutation. Authorization for the + // underlying state change is gated at member-create. + let (.., authz_instance) = LookupPath::new(opctx, &self.datastore) + .instance_id(member.parent_id) + .lookup_for(authz::Action::Read) + .await + .context("failed to look up instance for multicast subscribe")?; + let instance_and_vmm = self + .datastore + .instance_fetch_with_vmm(opctx, &authz_instance) + .await + .context("failed to fetch instance with active VMM")?; + let Some(vmm) = instance_and_vmm.vmm().as_ref() else { + debug!( + opctx.log, + "instance has no active VMM; skipping multicast subscribe"; + "member_id" => %member.id, + "instance_id" => %instance_id, + ); + return Ok(()); }; + // A racing live migration may invalidate this propolis_id. The + // next reconciler pass converges against the new active VMM. + let propolis_id = PropolisUuid::from_untyped_uuid(vmm.id); let client = self .sled_client(opctx, sled_id) @@ -162,8 +159,9 @@ impl MulticastSledClient { debug!( opctx.log, - "subscribed VMM to multicast group via sled-agent"; + "subscribed instance to multicast group via sled-agent"; "member_id" => %member.id, + "instance_id" => %instance_id, "propolis_id" => %propolis_id, "sled_id" => %sled_id, "group_ip" => %group.multicast_ip @@ -172,37 +170,39 @@ impl MulticastSledClient { Ok(()) } - /// Unsubscribe a VMM from a multicast group via sled-agent. + /// Unsubscribe an instance's active VMM OPTE port from a multicast group. /// /// Best-effort since if the VMM or sled is already gone, the unsubscribe - /// is effectively a no-op since the OPTE port was destroyed. - pub(crate) async fn unsubscribe_vmm( + /// is effectively a noop because the OPTE port was destroyed. + pub(crate) async fn unsubscribe_instance( &self, opctx: &OpContext, group: &MulticastGroup, member: &MulticastGroupMember, sled_id: SledUuid, - cached_propolis_id: Option, ) -> Result<(), anyhow::Error> { let instance_id = InstanceUuid::from_untyped_uuid(member.parent_id); - // If the instance has no propolis_id (already stopped/destroyed), - // the OPTE port is gone and there's nothing to unsubscribe. - let propolis_id = match cached_propolis_id { - Some(id) => id, - None => match self.lookup_propolis_id(opctx, instance_id).await? { - Some(id) => id, - None => { - debug!( - opctx.log, - "no propolis_id for instance, skipping unsubscribe"; - "member_id" => %member.id, - "instance_id" => %instance_id - ); - return Ok(()); - } - }, + let (.., authz_instance) = LookupPath::new(opctx, &self.datastore) + .instance_id(member.parent_id) + .lookup_for(authz::Action::Read) + .await + .context("failed to look up instance for multicast unsubscribe")?; + let instance_and_vmm = self + .datastore + .instance_fetch_with_vmm(opctx, &authz_instance) + .await + .context("failed to fetch instance with active VMM")?; + let Some(vmm) = instance_and_vmm.vmm().as_ref() else { + debug!( + opctx.log, + "instance has no active VMM; skipping multicast unsubscribe"; + "member_id" => %member.id, + "instance_id" => %instance_id, + ); + return Ok(()); }; + let propolis_id = PropolisUuid::from_untyped_uuid(vmm.id); let client = self .sled_client(opctx, sled_id) @@ -218,8 +218,9 @@ impl MulticastSledClient { debug!( opctx.log, - "unsubscribed VMM from multicast group via sled-agent"; + "unsubscribed instance from multicast group via sled-agent"; "member_id" => %member.id, + "instance_id" => %instance_id, "propolis_id" => %propolis_id, "sled_id" => %sled_id, "group_ip" => %group.multicast_ip @@ -249,23 +250,15 @@ impl MulticastSledClient { opctx: &OpContext, group: &MulticastGroup, ) -> Result<(), anyhow::Error> { - let underlay_group_id = group - .underlay_group_id - .context("group missing underlay_group_id")?; - - let underlay_group = self - .datastore - .underlay_multicast_group_fetch(opctx, underlay_group_id) + let underlay_ip = self + .resolve_underlay_ip(opctx, group) .await - .context("failed to fetch underlay group")?; - - let underlay_ip = match underlay_group.multicast_ip.ip() { - IpAddr::V6(v6) => v6, - other => anyhow::bail!( - "underlay multicast address for group {} is {other}, expected IPv6", - group.id() - ), - }; + .with_context(|| { + format!( + "failed to resolve underlay multicast address for group {}", + group.id() + ) + })?; let group_ip = group.multicast_ip.ip(); @@ -322,17 +315,9 @@ impl MulticastSledClient { .map_err(|e| anyhow::anyhow!(e)) .context("failed to resolve switch zone addresses")?; - // Hash the group UUID to distribute switch selection across both - // switches. All Nexuses compute the same hash for a given group, - // so they agree on the mapping without coordination. - let mut hasher = DefaultHasher::new(); - group_id.hash(&mut hasher); - let idx = (hasher.finish() as usize) % switch_zone_addrs.len(); - let switch_ip = switch_zone_addrs - .iter() - .nth(idx) - .map(|(_, ip)| *ip) - .context("no switch zone found for forwarding next hop")?; + let switch_ip = + select_forwarding_switch_ip(group_id, &switch_zone_addrs) + .context("no switch zone found for forwarding next hop")?; let convergence_params = GroupConvergenceParams { group_ip, @@ -342,9 +327,11 @@ impl MulticastSledClient { switch_ip, }; - let mut failed_sleds: usize = 0; - - for sled in &all_sleds { + // Fan out per-sled convergence so a large rack doesn't pay + // N sequential RPC round-trips. Each sled's RPC is independent, so + // we accumulate per-sled failures rather than fail-fast. + let convergence_params = &convergence_params; + let results = join_all(all_sleds.iter().map(|sled| async move { let sled_id: SledUuid = sled.id(); let client = match self.sled_client(opctx, sled_id).await { Ok(c) => c, @@ -356,13 +343,11 @@ impl MulticastSledClient { "sled_id" => %sled_id, "error" => %e ); - failed_sleds += 1; - continue; + return Err(()); } }; - if let Err(e) = - converge_sled_m2p_and_forwarding(&client, &convergence_params) + converge_sled_m2p_and_forwarding(&client, convergence_params) .await { warn!( @@ -372,9 +357,13 @@ impl MulticastSledClient { "group_ip" => %group_ip, "error" => %e ); - failed_sleds += 1; + return Err(()); } - } + Ok(()) + })) + .await; + + let failed_sleds = results.iter().filter(|r| r.is_err()).count(); info!( opctx.log, @@ -399,6 +388,45 @@ impl MulticastSledClient { Ok(()) } + async fn resolve_underlay_ip( + &self, + opctx: &OpContext, + group: &MulticastGroup, + ) -> Result { + let underlay_group_id = group + .underlay_group_id + .context("group missing underlay_group_id")?; + + match self + .datastore + .underlay_multicast_group_fetch(opctx, underlay_group_id) + .await + { + Ok(underlay_group) => match underlay_group.multicast_ip.ip() { + IpAddr::V6(v6) => Ok(v6), + other => anyhow::bail!( + "underlay multicast address for group {} is {other}, \ + expected IPv6", + group.id() + ), + }, + Err(external::Error::ObjectNotFound { .. }) => { + let salt = group.underlay_salt.map_or(0, |s| *s); + match super::map_external_to_underlay_ip( + group.multicast_ip.ip(), + salt, + ) { + IpAddr::V6(v6) => Ok(v6), + IpAddr::V4(_) => anyhow::bail!( + "computed IPv4 underlay address for group {}", + group.id() + ), + } + } + Err(e) => Err(e).context("failed to fetch underlay group"), + } + } + /// Clear M2P mappings and forwarding entries from all sleds for /// this group. /// @@ -553,3 +581,66 @@ async fn converge_forwarding( Ok(()) } + +fn select_forwarding_switch_ip( + group_id: MulticastGroupUuid, + switch_zone_addrs: &HashMap, +) -> Option { + let mut ordered_switches: Vec<_> = switch_zone_addrs.iter().collect(); + ordered_switches.sort_by_key(|(slot, _)| **slot); + + if ordered_switches.is_empty() { + return None; + } + + // Hash the group UUID to distribute switch selection across both + // switches. Ordering by slot keeps the selection stable across + // reconciliation passes and Nexus instances. + let mut hasher = DefaultHasher::new(); + group_id.hash(&mut hasher); + let idx = (hasher.finish() as usize) % ordered_switches.len(); + Some(*ordered_switches[idx].1) +} + +#[cfg(test)] +mod tests { + use super::select_forwarding_switch_ip; + + use std::collections::HashMap; + use std::net::Ipv6Addr; + + use omicron_uuid_kinds::{GenericUuid, MulticastGroupUuid}; + use sled_agent_types::early_networking::SwitchSlot; + use uuid::Uuid; + + #[test] + fn select_forwarding_switch_ip_returns_none_when_empty() { + let group_id = MulticastGroupUuid::from_untyped_uuid(Uuid::new_v4()); + let switch_zone_addrs = HashMap::new(); + + assert_eq!( + select_forwarding_switch_ip(group_id, &switch_zone_addrs), + None + ); + } + + #[test] + fn select_forwarding_switch_ip_is_stable_across_map_order() { + let group_id = MulticastGroupUuid::from_untyped_uuid(Uuid::new_v4()); + let switch0 = Ipv6Addr::LOCALHOST; + let switch1 = Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 2); + + let mut first = HashMap::new(); + first.insert(SwitchSlot::Switch0, switch0); + first.insert(SwitchSlot::Switch1, switch1); + + let mut second = HashMap::new(); + second.insert(SwitchSlot::Switch1, switch1); + second.insert(SwitchSlot::Switch0, switch0); + + assert_eq!( + select_forwarding_switch_ip(group_id, &first), + select_forwarding_switch_ip(group_id, &second) + ); + } +} diff --git a/nexus/src/app/multicast/switch_zone.rs b/nexus/src/app/multicast/switch_zone.rs new file mode 100644 index 00000000000..6d963d01328 --- /dev/null +++ b/nexus/src/app/multicast/switch_zone.rs @@ -0,0 +1,457 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Switch zone service clients for multicast operations. +//! +//! Wraps MGD (for MRIB programming) and DDM (for the read-only peer +//! view) on the switch zone. Built per reconciliation pass. +//! +//! - **MRIB**: Nexus → MGD MRIB → mg-lower → DDM → peer sleds +//! - **Peers**: under RFD 488 mg-lower derives underlay members from DDM +//! peer subscriptions, so the reconciler does not resolve sled→port; the +//! DDM client only backs the `omdb nexus multicast ddm-peers` view + +use std::collections::HashMap; +use std::net::{IpAddr, Ipv6Addr, SocketAddrV6}; +use std::time::Duration; + +use anyhow::anyhow; +use futures::future::{join_all, try_join_all}; +use internal_dns_resolver::Resolver; +use sled_agent_types::early_networking::SwitchSlot; +use slog::{Logger, debug, warn}; + +use internal_dns_types::names::ServiceName; +use mg_admin_client::types::{ + MribAddStaticRequest, MribDeleteStaticRequest, MulticastRouteKey, + MulticastRouteKeyV4, MulticastRouteKeyV6, StaticMulticastRouteInput, +}; +use omicron_common::address::{DDMD_PORT, MGD_PORT}; +use omicron_ddm_admin_client::PeerInfo; + +use crate::app::switch_zone_targets; + +/// Client for switch zone services used by the multicast reconciler. +/// +/// Provides access to MGD (MRIB route programming) and DDM (the +/// read-only peer view; not used for sled-to-port resolution). +/// +/// Built per reconciliation pass, similar to [`MulticastDataplaneClient`]. +/// +/// Note: per [omicron#10167], system-level networking (uplinkd, system-zone +/// NAT, BGP, BFD) is migrating from Nexus RPWs to sled-agent reconcilers +/// that operate based on data in the bootstore. Multicast is +/// **instance networking** (group state derives from per-instance memberships), +/// so this client's direct-to-MGD path is intentional and should be preserved +/// by the migration. +/// +/// If a follow-on to [omicron#10167] routes MRIB writes through +/// sled-agent, the reconciler logic stays in Nexus and only the wire +/// surface changes: Nexus would call a sled-agent endpoint that fronts +/// MGD instead of MGD directly. +/// +/// [`MulticastDataplaneClient`]: super::dataplane::MulticastDataplaneClient +/// [omicron#10167]: https://github.com/oxidecomputer/omicron/issues/10167 +pub(crate) struct MulticastSwitchZoneClient { + mgd_clients: HashMap, + ddm_clients: HashMap, + log: Logger, +} + +pub(crate) type MribRouteIndex = + HashMap, HashMap>>; + +// Mirrors `MulticastDataplaneClient::new`'s timeout. +const SWITCH_ZONE_BUILD_TIMEOUT: Duration = Duration::from_secs(5); + +// Per-switch bound on the live DDM peer query so an unreachable switch zone +// degrades to a skipped switch rather than stalling the whole request. +// +// Budget: progenitor-generated clients (omdb's lockstep client included) +// default to a 15s reqwest timeout, so the handler's worst case of +// SWITCH_ZONE_BUILD_TIMEOUT + one concurrent DDM_PEERS_QUERY_TIMEOUT (13s) +// must stay under it with margin for DNS and HTTP overhead. +const DDM_PEERS_QUERY_TIMEOUT: Duration = Duration::from_secs(8); + +impl MulticastSwitchZoneClient { + /// Build MGD and DDM clients for all switch zones. + /// + /// Resolves service ports from DNS rather than hardcoding them, + /// falling back to the well-known port constants when DNS lookup + /// fails. This allows the test harness to run MGD and DDM on + /// dynamic ports. + /// + /// Returns an error when no switch zones resolve, so the reconciler + /// retries rather than silently treating writes as noops. + pub(crate) async fn new( + resolver: Resolver, + log: Logger, + ) -> Result { + match tokio::time::timeout( + SWITCH_ZONE_BUILD_TIMEOUT, + Self::build(resolver, log.clone()), + ) + .await + { + Ok(result) => result, + Err(_) => Err(format!( + "timed out building switch-zone clients after \ + {SWITCH_ZONE_BUILD_TIMEOUT:?}" + )), + } + } + + async fn build(resolver: Resolver, log: Logger) -> Result { + let switch_zones = switch_zone_targets(&resolver, &log).await?; + + if switch_zones.is_empty() { + return Err( + "no switch zones resolved for multicast operations".to_string() + ); + } + + // Resolve MGD and DDM sockets from DNS, keyed by SRV target. This + // preserves distinct switch zones that share an IPv6 address in tests + // and differ only by port. + let mgd_socket_map = + resolve_service_sockets(&resolver, &log, ServiceName::Mgd).await; + let ddm_socket_map = + resolve_service_sockets(&resolver, &log, ServiceName::Ddm).await; + + let mgd_clients = switch_zones + .iter() + .map(|(slot, endpoint)| { + let socketaddr = mgd_socket_map + .get(&endpoint.target) + .copied() + .unwrap_or_else(|| { + SocketAddrV6::new(endpoint.addr, MGD_PORT, 0, 0) + }); + ( + *slot, + mg_admin_client::Client::new( + &format!("http://{socketaddr}"), + log.clone(), + ), + ) + }) + .collect(); + + let ddm_clients = switch_zones + .iter() + .filter_map(|(slot, endpoint)| { + let from_dns = ddm_socket_map.get(&endpoint.target).copied(); + let socketaddr = from_dns.unwrap_or_else(|| { + SocketAddrV6::new(endpoint.addr, DDMD_PORT, 0, 0) + }); + match omicron_ddm_admin_client::Client::new(&log, socketaddr) { + Ok(c) => Some((*slot, c)), + Err(e) => { + warn!( + log, + "failed to build DDM client for switch zone"; + "switch" => ?slot, + "error" => %e, + ); + None + } + } + }) + .collect(); + + Ok(Self { mgd_clients, ddm_clients, log }) + } + + /// Add a multicast route to the MRIB on all switches in parallel. + /// + /// `mg-lower` watches the MRIB and automatically advertises the + /// route via DDM to peer sleds. Short-circuits on the first switch + /// failure as the reconciler retries the full set on the next pass. + /// + /// # Arguments + /// + /// * `group_ip`: Overlay multicast group address. + /// * `underlay_ip`: Underlay IPv6 address (ff04::/64) the route + /// forwards to. + /// * `source`: SSM source for source-specific routes; `None` for ASM. + pub(crate) async fn add_route( + &self, + group_ip: IpAddr, + underlay_ip: Ipv6Addr, + source: Option, + ) -> Result<(), anyhow::Error> { + let route_key = make_route_key(group_ip, source); + + let request = MribAddStaticRequest { + routes: vec![StaticMulticastRouteInput { + key: route_key, + underlay_group: underlay_ip, + }], + }; + + try_join_all(self.mgd_clients.iter().map(|(slot, client)| { + let request = &request; + async move { + client.static_add_mcast_route(request).await.map_err(|e| { + warn!( + self.log, + "mgd static_add_mcast_route failed"; + "switch" => ?slot, + "group_ip" => %group_ip, + "error" => %e, + ); + anyhow!( + "mgd static_add_mcast_route failed on switch {slot:?}: {e}" + ) + })?; + debug!( + self.log, + "added multicast route to MRIB"; + "switch" => ?slot, + "group_ip" => %group_ip, + "underlay_ip" => %underlay_ip, + ); + Ok::<(), anyhow::Error>(()) + } + })) + .await?; + Ok(()) + } + + /// Remove a multicast route from the MRIB on all switches in parallel. + /// + /// `mg-lower` detects the removal and withdraws the DDM + /// advertisement from peer sleds. Short-circuits on the first + /// switch failure as the reconciler retries on the next pass. + pub(crate) async fn remove_route( + &self, + group_ip: IpAddr, + source: Option, + ) -> Result<(), anyhow::Error> { + let route_key = make_route_key(group_ip, source); + + let request = MribDeleteStaticRequest { keys: vec![route_key] }; + + try_join_all(self.mgd_clients.iter().map(|(slot, client)| { + let request = &request; + async move { + client.static_remove_mcast_route(request).await.map_err( + |e| { + warn!( + self.log, + "mgd static_remove_mcast_route failed"; + "switch" => ?slot, + "group_ip" => %group_ip, + "error" => %e, + ); + anyhow!( + "mgd static_remove_mcast_route failed on switch {slot:?}: {e}" + ) + }, + )?; + debug!( + self.log, + "removed multicast route from MRIB"; + "switch" => ?slot, + "group_ip" => %group_ip, + ); + Ok::<(), anyhow::Error>(()) + } + })) + .await?; + Ok(()) + } + + /// List static multicast routes from all reachable switches and + /// index them by group|source|switch. + pub(crate) async fn list_routes_indexed( + &self, + ) -> Result { + let mut index = MribRouteIndex::new(); + + for (slot, client) in &self.mgd_clients { + match client.static_list_mcast_routes().await { + Ok(routes) => { + for route in routes.into_inner() { + let (group_ip, source) = route_identifier(&route.key); + index + .entry(group_ip) + .or_default() + .entry(source) + .or_default() + .insert(*slot, route.underlay_group); + } + } + Err(e) => { + warn!( + self.log, + "failed to list multicast routes from switch zone"; + "switch" => ?slot, + "error" => %e, + ); + } + } + } + + Ok(index) + } + + pub(crate) fn switch_count(&self) -> usize { + self.mgd_clients.len() + } + + /// Whether a multicast route is present in `mrib_loc` (RPF-verified) + /// on every configured switch. + /// + /// Returns `false` when the route is missing on any switch, including + /// switches that fail the RPC. The reconciler interprets `false` as + /// not-yet-forwarding (still in `mrib_in`, de-promoted by the RPF + /// revalidator, or simply unreachable) and retries on the next pass. + pub(crate) async fn route_active_on_all_switches( + &self, + group_ip: IpAddr, + source: Option, + ) -> bool { + let vni = u32::from( + omicron_common::api::external::Vni::DEFAULT_MULTICAST_VNI, + ); + + for (slot, client) in &self.mgd_clients { + match client + .get_mrib_selected( + None, + Some(&group_ip), + None, + source.as_ref(), + Some(vni), + ) + .await + { + Ok(resp) => { + if resp.into_inner().is_empty() { + return false; + } + } + Err(e) => { + warn!( + self.log, + "mgd get_mrib_selected failed"; + "switch" => ?slot, + "group_ip" => %group_ip, + "error" => %e, + ); + return false; + } + } + } + + true + } + + /// Query DDM peers from all switch zones, tagged with the switch slot + /// they were observed from so callers can present per-switch topology. + /// + /// Errors from individual switch zones are logged and skipped rather + /// than failing the whole query. + /// + /// Switches are queried concurrently so the worst-case latency is one + /// per-switch [`DDM_PEERS_QUERY_TIMEOUT`] rather than the sum across + /// switches. A sequential scan of two slow switches exceeds omdb's + /// client timeout. + pub(crate) async fn ddm_peers_by_switch( + &self, + ) -> Vec<(SwitchSlot, PeerInfo)> { + let per_switch = + self.ddm_clients.iter().map(|(slot, client)| async move { + match tokio::time::timeout( + DDM_PEERS_QUERY_TIMEOUT, + client.get_peers(), + ) + .await + { + Ok(Ok(peers)) => peers + .into_values() + .map(|peer| (*slot, peer)) + .collect::>(), + Ok(Err(e)) => { + warn!( + self.log, + "failed to get DDM peers from switch zone"; + "switch" => ?slot, + "error" => %e, + ); + Vec::new() + } + Err(_) => { + warn!( + self.log, + "timed out querying DDM peers from switch zone"; + "switch" => ?slot, + "timeout" => ?DDM_PEERS_QUERY_TIMEOUT, + ); + Vec::new() + } + } + }); + + join_all(per_switch).await.into_iter().flatten().collect() + } +} + +fn make_route_key( + group_ip: IpAddr, + source: Option, +) -> MulticastRouteKey { + let vni = + u32::from(omicron_common::api::external::Vni::DEFAULT_MULTICAST_VNI); + match group_ip { + IpAddr::V4(v4) => MulticastRouteKey::V4(MulticastRouteKeyV4 { + group: v4, + source: source.and_then(|s| match s { + IpAddr::V4(s4) => Some(s4), + _ => None, + }), + vni, + }), + IpAddr::V6(v6) => MulticastRouteKey::V6(MulticastRouteKeyV6 { + group: v6, + source: source.and_then(|s| match s { + IpAddr::V6(s6) => Some(s6), + _ => None, + }), + vni, + }), + } +} + +/// Resolve service sockets from DNS, returning a map of SRV target to socket. +async fn resolve_service_sockets( + resolver: &Resolver, + log: &Logger, + service: ServiceName, +) -> HashMap { + match resolver.lookup_all_socket_v6_by_target(service).await { + Ok(pairs) => pairs.into_iter().collect(), + Err(e) => { + warn!( + log, + "failed to resolve service sockets from DNS, using defaults"; + "service" => ?service, + "error" => %e, + ); + HashMap::new() + } + } +} + +fn route_identifier(key: &MulticastRouteKey) -> (IpAddr, Option) { + match key { + MulticastRouteKey::V4(k) => { + (IpAddr::V4(k.group), k.source.map(IpAddr::V4)) + } + MulticastRouteKey::V6(k) => { + (IpAddr::V6(k.group), k.source.map(IpAddr::V6)) + } + } +} diff --git a/nexus/src/app/sagas/multicast_group_dpd_ensure.rs b/nexus/src/app/sagas/multicast_group_dpd_ensure.rs index 17c1fc2b3a1..3f2f74526af 100644 --- a/nexus/src/app/sagas/multicast_group_dpd_ensure.rs +++ b/nexus/src/app/sagas/multicast_group_dpd_ensure.rs @@ -150,19 +150,21 @@ async fn mgde_fetch_group_data( .await .map_err(saga_action_failed)?; - // Validate groups are in correct state + // "Active" is allowed for crash recovery. Rejecting would tear + // down correctly-applied DPD state. match external_group.state { - nexus_db_model::MulticastGroupState::Creating => {} + nexus_db_model::MulticastGroupState::Creating + | nexus_db_model::MulticastGroupState::Active => {} other_state => { warn!( osagactx.log(), - "external group not in 'Creating' state for DPD"; + "external group not in 'Creating' or 'Active' state for DPD"; "external_group_id" => %params.external_group_id, "external_group_name" => external_group.name().as_str(), "current_state" => ?other_state ); return Err(saga_action_failed(Error::internal_error(&format!( - "External group {} is in state {other_state:?}, expected 'Creating'", + "External group {} is in state {other_state:?}, expected 'Creating' or 'Active'", params.external_group_id )))); } @@ -454,12 +456,16 @@ mod test { ); } - /// Test that the saga rejects external groups that are not in "Creating" state. + /// Test that the saga accepts "Active" groups (idempotent crash recovery) + /// but still rejects groups that are no longer in flight. /// - /// The saga validates that external groups are in "Creating" state before applying - /// DPD configuration. This test verifies that validation works correctly. + /// `mgde_fetch_group_data` allows "Creating" and "Active" states. Re-running the + /// saga over a group whose `mgde_update_group_state` already committed + /// must succeed through the original DAG so recovery does not roll back + /// correctly-applied DPD state. Other states (e.g., "Deleted") are still + /// out of scope and must be rejected. #[nexus_test(server = crate::Server)] - async fn test_saga_rejects_non_creating_state( + async fn test_saga_accepts_active_rejects_terminal_state( cptestctx: &ControlPlaneTestContext, ) { let client = &cptestctx.external_client; @@ -539,19 +545,44 @@ mod test { .await .expect("Group should transition to Active state"); - // Try to run saga on Active group - should fail + // Re-running the saga on an "Active" group simulates crash-recovery + // re-execution: every action is idempotent, so the saga must succeed + // through the original DAG rather than triggering rollback that would + // tear down correctly-applied DPD state. let params = Params { serialized_authn: Serialized::for_opctx(&opctx), external_group_id: external_group.id(), underlay_group_id: underlay_group.id, }; + nexus + .sagas + .saga_execute::(params) + .await + .expect("Saga should re-run idempotently against an Active group"); + + // Transition the group to "Deleting" and re-run the saga. The saga + // must refuse to run against a group that is no longer in "Creating" + // or "Active". + let marked = datastore + .mark_multicast_group_for_removal_if_no_members(&opctx, group_id) + .await + .expect("group should mark for removal"); + assert!(marked, "group should transition to Deleting"); + + let params = Params { + serialized_authn: Serialized::for_opctx(&opctx), + external_group_id: external_group.id(), + underlay_group_id: underlay_group.id, + }; let result = nexus .sagas .saga_execute::(params) .await; - - // Saga should reject Active group - assert!(result.is_err(), "Saga should reject group in Active state"); + assert!( + result.is_err(), + "Saga should reject group that is no longer in 'Creating' or \ + 'Active'", + ); } } diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 9395235e79d..e3ae2490b9b 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -5282,7 +5282,7 @@ impl NexusExternalApi for NexusExternalApiImpl { let instance_lookup = nexus.instance_lookup(&opctx, instance_selector)?; let result = nexus - .instance_join_multicast_group( + .vmm_join_multicast_group( &opctx, &path.multicast_group, &instance_lookup, @@ -5320,7 +5320,7 @@ impl NexusExternalApi for NexusExternalApiImpl { let group_lookup = nexus.multicast_group_lookup(&opctx, &group_selector).await?; nexus - .instance_leave_multicast_group( + .vmm_leave_multicast_group( &opctx, &group_lookup, &instance_lookup, @@ -5391,7 +5391,7 @@ impl NexusExternalApi for NexusExternalApiImpl { let result = apictx .context .nexus - .instance_join_multicast_group( + .vmm_join_multicast_group( &opctx, &path.multicast_group, &instance_lookup, diff --git a/nexus/src/lockstep_api/http_entrypoints.rs b/nexus/src/lockstep_api/http_entrypoints.rs index 04df1612332..5218505f7de 100644 --- a/nexus/src/lockstep_api/http_entrypoints.rs +++ b/nexus/src/lockstep_api/http_entrypoints.rs @@ -46,6 +46,7 @@ use nexus_types::internal_api::params::RackInitializationRequest; use nexus_types::internal_api::views::BackgroundTask; use nexus_types::internal_api::views::DemoSaga; use nexus_types::internal_api::views::MgsUpdateDriverStatus; +use nexus_types::internal_api::views::MulticastDdmPeersView; use nexus_types::internal_api::views::QuiesceStatus; use nexus_types::internal_api::views::Saga; use nexus_types::internal_api::views::SupportBundleInfo; @@ -315,6 +316,22 @@ impl NexusLockstepApi for NexusLockstepApiImpl { .await } + async fn multicast_ddm_peers( + rqctx: RequestContext, + ) -> Result, HttpError> { + let apictx = &rqctx.context().context; + let handler = async { + let opctx = + crate::context::op_context_for_internal_api(&rqctx).await; + let nexus = &apictx.nexus; + Ok(HttpResponseOk(nexus.multicast_ddm_peers(&opctx).await?)) + }; + apictx + .internal_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + // APIs for managing blueprints async fn blueprint_list( rqctx: RequestContext, diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index e57f667fe8e..b45eed486bc 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -9,6 +9,7 @@ use omicron_common::api::external::IdentityMetadata; use omicron_sled_agent::sim; use omicron_test_utils::dev::poll::{CondCheckError, wait_for_condition}; use omicron_uuid_kinds::GenericUuid; +use std::collections::BTreeMap; use std::fmt::Debug; use std::net::Ipv6Addr; use std::time::Duration; @@ -119,21 +120,36 @@ async fn wait_for_producer_impl( .expect("Failed to find producer within time limit"); } -/// Build a DPD client for test validation using the first running dendrite instance +/// Build a DPD client for `Switch0` in the test fixture. +/// +/// Deterministic by default. Tests that need to validate state on every +/// switch in a multi-switch fixture should use [`dpd_clients_by_switch`] +/// instead and iterate, since each switch independently programs its own +/// underlay group / NAT / forwarding state. pub fn dpd_client( cptestctx: &ControlPlaneTestContext, ) -> dpd_client::Client { - // Get the first available dendrite instance and extract the values we need - let dendrite_guard = cptestctx.dendrite.read().unwrap(); - let (switch_slot, dendrite_instance) = dendrite_guard - .iter() - .next() - .expect("No dendrite instances running for test"); + use sled_agent_types::early_networking::SwitchSlot; + dpd_client_for(cptestctx, SwitchSlot::Switch0) +} - // Copy the values we need while the guard is still alive - let switch_slot = *switch_slot; - let port = dendrite_instance.port; - drop(dendrite_guard); +/// Build a DPD client targeting a specific switch slot. +pub fn dpd_client_for( + cptestctx: &ControlPlaneTestContext, + switch_slot: sled_agent_types::early_networking::SwitchSlot, +) -> dpd_client::Client { + let port = { + let dendrite = cptestctx.dendrite.read().unwrap(); + dendrite + .get(&switch_slot) + .unwrap_or_else(|| { + panic!( + "no dendrite instance running for {switch_slot:?} in \ + test fixture", + ) + }) + .port + }; let client_state = dpd_client::ClientState { tag: String::from("nexus-test"), @@ -147,6 +163,42 @@ pub fn dpd_client( dpd_client::Client::new(&format!("http://[{addr}]:{port}"), client_state) } +/// Build DPD clients for every switch slot in the test fixture, ordered by +/// [`SwitchSlot`]. +/// +/// Use this when validating a per-switch invariant (e.g., "every switch has +/// the full underlay-member set"). The [`SwitchSlot`] ordering keeps log output +/// and assertions stable across test passes. +/// +/// [`SwitchSlot`]: sled_agent_types::early_networking::SwitchSlot +pub fn dpd_clients_by_switch( + cptestctx: &ControlPlaneTestContext, +) -> BTreeMap +{ + let dendrite = cptestctx.dendrite.read().unwrap(); + dendrite + .iter() + .map(|(slot, instance)| (*slot, instance.port)) + .collect::>() + .into_iter() + .map(|(slot, port)| { + let client_state = dpd_client::ClientState { + tag: String::from("nexus-test"), + log: cptestctx.logctx.log.new(slog::o!( + "component" => "DpdClient", + "switch_slot" => format!("{slot:?}"), + )), + }; + let addr = Ipv6Addr::LOCALHOST; + let client = dpd_client::Client::new( + &format!("http://[{addr}]:{port}"), + client_state, + ); + (slot, client) + }) + .collect() +} + #[cfg(test)] mod test { use crate::TEST_SUITE_PASSWORD; diff --git a/nexus/test-utils/src/starter.rs b/nexus/test-utils/src/starter.rs index bb84aed361c..ebb97f2be45 100644 --- a/nexus/test-utils/src/starter.rs +++ b/nexus/test-utils/src/starter.rs @@ -455,9 +455,33 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { let mgs_addr = SocketAddrV6::new(Ipv6Addr::LOCALHOST, mgs.port, 0, 0).into(); + // Point mgd's lower half at this switch's dendrite and ddmd so the + // mg-lower-mrib thread (illumos only) syncs MRIB entries to ddmd as + // multicast origination. + // + // Both daemons must be started first. + let dendrite_addr = self + .dendrite + .read() + .unwrap() + .get(&switch_slot) + .map(|d| SocketAddrV6::new(Ipv6Addr::LOCALHOST, d.port, 0, 0)) + .map(std::net::SocketAddr::V6); + let ddm_addr = self + .ddm + .get(&switch_slot) + .map(|d| SocketAddrV6::new(Ipv6Addr::LOCALHOST, d.port, 0, 0)) + .map(std::net::SocketAddr::V6); + // Set up an instance of mgd - let mgd = - dev::maghemite::MgdInstance::start(0, mgs_addr).await.unwrap(); + let mgd = dev::maghemite::MgdInstance::start( + 0, + mgs_addr, + dendrite_addr, + ddm_addr, + ) + .await + .unwrap(); let port = mgd.port; self.mgd.insert(switch_slot, mgd); let address = SocketAddrV6::new(Ipv6Addr::LOCALHOST, port, 0, 0); @@ -470,13 +494,13 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { pub async fn start_ddm(&mut self, switch_slot: SwitchSlot) { let log = &self.logctx.log; - debug!(log, "Starting DDM sim"; "switch_slot" => ?switch_slot); + debug!(log, "Starting ddmd"; "switch_slot" => ?switch_slot); let ddm = dev::maghemite::DdmInstance::start().await.unwrap(); let port = ddm.port; self.ddm.insert(switch_slot, ddm); - debug!(log, "DDM sim started"; "port" => port); + debug!(log, "ddmd started"; "port" => port); } pub async fn record_switch_dns( @@ -1657,15 +1681,15 @@ pub(crate) async fn setup_with_config_impl( }), ), ( - "start_mgd_switch0", + "start_ddm_switch0", Box::new(|builder| { - builder.start_mgd(SwitchSlot::Switch0).boxed() + builder.start_ddm(SwitchSlot::Switch0).boxed() }), ), ( - "start_ddm_switch0", + "start_mgd_switch0", Box::new(|builder| { - builder.start_ddm(SwitchSlot::Switch0).boxed() + builder.start_mgd(SwitchSlot::Switch0).boxed() }), ), ( @@ -1707,15 +1731,15 @@ pub(crate) async fn setup_with_config_impl( }), ), ( - "start_mgd_switch1", + "start_ddm_switch1", Box::new(|builder| { - builder.start_mgd(SwitchSlot::Switch1).boxed() + builder.start_ddm(SwitchSlot::Switch1).boxed() }), ), ( - "start_ddm_switch1", + "start_mgd_switch1", Box::new(|builder| { - builder.start_ddm(SwitchSlot::Switch1).boxed() + builder.start_mgd(SwitchSlot::Switch1).boxed() }), ), ( diff --git a/nexus/tests/config.test.toml b/nexus/tests/config.test.toml index ca3d010f105..7a5864b943b 100644 --- a/nexus/tests/config.test.toml +++ b/nexus/tests/config.test.toml @@ -213,9 +213,6 @@ fm.sitrep_gc_period_secs = 600 fm.rendezvous_period_secs = 300 probe_distributor.period_secs = 60 multicast_reconciler.period_secs = 60 -# Use shorter TTLs for tests to ensure cache invalidation logic is exercised -multicast_reconciler.sled_cache_ttl_secs = 60 -multicast_reconciler.backplane_cache_ttl_secs = 120 trust_quorum.period_secs = 60 attached_subnet_manager.period_secs = 60 session_cleanup.period_secs = 300 diff --git a/nexus/tests/integration_tests/initialization.rs b/nexus/tests/integration_tests/initialization.rs index 714880feb37..8a1cfb5c3ba 100644 --- a/nexus/tests/integration_tests/initialization.rs +++ b/nexus/tests/integration_tests/initialization.rs @@ -153,16 +153,18 @@ async fn test_nexus_boots_before_dendrite() { starter.start_dendrite(SwitchSlot::Switch1).await; info!(log, "Started Dendrite"); - info!(log, "Starting mgd"); - starter.start_mgd(SwitchSlot::Switch0).await; - starter.start_mgd(SwitchSlot::Switch1).await; - info!(log, "Started mgd"); - + // ddmd must start before mgd so `start_mgd` can pass `--ddm-addr` and wire + // up the mg-lower-mrib DDM sync, matching the default starter sequence. info!(log, "Starting ddm"); starter.start_ddm(SwitchSlot::Switch0).await; starter.start_ddm(SwitchSlot::Switch1).await; info!(log, "Started ddm"); + info!(log, "Starting mgd"); + starter.start_mgd(SwitchSlot::Switch0).await; + starter.start_mgd(SwitchSlot::Switch1).await; + info!(log, "Started mgd"); + info!(log, "Populating internal DNS records"); starter .record_switch_dns( @@ -200,10 +202,12 @@ async fn nexus_schema_test_setup( starter.start_gateway(SwitchSlot::Switch1, None, sp_conf).await; starter.start_dendrite(SwitchSlot::Switch0).await; starter.start_dendrite(SwitchSlot::Switch1).await; - starter.start_mgd(SwitchSlot::Switch0).await; - starter.start_mgd(SwitchSlot::Switch1).await; + // ddmd must start before mgd so `start_mgd` can pass `--ddm-addr` and wire + // up the mg-lower-mrib DDM sync, matching the default starter sequence. starter.start_ddm(SwitchSlot::Switch0).await; starter.start_ddm(SwitchSlot::Switch1).await; + starter.start_mgd(SwitchSlot::Switch0).await; + starter.start_mgd(SwitchSlot::Switch1).await; starter.populate_internal_dns().await; } diff --git a/nexus/tests/integration_tests/multicast/api.rs b/nexus/tests/integration_tests/multicast/api.rs index fa862ddd9fd..5c88d638468 100644 --- a/nexus/tests/integration_tests/multicast/api.rs +++ b/nexus/tests/integration_tests/multicast/api.rs @@ -1,8 +1,6 @@ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -// -// Copyright 2025 Oxide Computer Company //! Tests for multicast API behavior and functionality. //! diff --git a/nexus/tests/integration_tests/multicast/authorization.rs b/nexus/tests/integration_tests/multicast/authorization.rs index d52bea57ee3..9d36b596edf 100644 --- a/nexus/tests/integration_tests/multicast/authorization.rs +++ b/nexus/tests/integration_tests/multicast/authorization.rs @@ -513,8 +513,8 @@ async fn test_silo_user_multicast_permissions( start: false, auto_restart_policy: Default::default(), anti_affinity_groups: Vec::new(), - enable_jumbo_frames: false, multicast_groups: Vec::new(), + enable_jumbo_frames: false, }; let instance2: Instance = NexusRequest::objects_post( client, diff --git a/nexus/tests/integration_tests/multicast/cache_invalidation.rs b/nexus/tests/integration_tests/multicast/cache_invalidation.rs deleted file mode 100644 index de744c19d9d..00000000000 --- a/nexus/tests/integration_tests/multicast/cache_invalidation.rs +++ /dev/null @@ -1,645 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -//! Integration tests for multicast reconciler cache invalidation. -//! -//! Tests inventory and backplane caches used by the multicast reconciler: -//! -//! - Sled move detection: When a sled moves to a different switch port, the -//! reconciler detects this via inventory and updates DPD port mappings -//! - Cache TTL refresh: Verifies caches are refreshed when TTL expires -//! - Backplane cache expiry: Tests that stale backplane mappings are cleaned up - -use http::{Method, StatusCode}; - -use gateway_client::types::{PowerState, RotState, SpState}; -use nexus_db_lookup::LookupPath; -use nexus_db_queries::context::OpContext; -use nexus_test_utils::resource_helpers::{ - create_default_ip_pools, create_project, -}; -use nexus_test_utils_macros::nexus_test; -use nexus_types::deployment::SledFilter; -use nexus_types::external_api::sled; -use nexus_types::inventory::SpType; -use omicron_nexus::Server; -use omicron_nexus::TestInterfaces; -use omicron_uuid_kinds::{GenericUuid, InstanceUuid, MulticastGroupUuid}; - -use super::*; -use crate::integration_tests::instances::instance_wait_for_state; - -/// Test that multicast operations can handle physical sled movement. -/// -/// This test simulates a sled being physically moved to a different rack slot: -/// - Create a multicast group and instance, wait for member to join -/// - Verify the member is programmed on the correct rear port (based on original `sp_slot`) -/// - Run reconciler multiple times without inventory change to verify no spurious invalidation -/// - Insert a new inventory collection with a different `sp_slot` for the same sled -/// - Reconciler detects sled location change and invalidates caches automatically -/// - Verify DPD now uses the new rear port matching the new `sp_slot` -#[nexus_test(server = Server)] -async fn test_sled_move_updates_multicast_port_mapping( - cptestctx: &ControlPlaneTestContext, -) { - const PROJECT_NAME: &str = "test-project"; - const GROUP_NAME: &str = "sled-move-test-group"; - const INSTANCE_NAME: &str = "sled-move-test-instance"; - - ensure_multicast_test_ready(cptestctx).await; - - let client = &cptestctx.external_client; - let nexus = &cptestctx.server.server_context().nexus; - let datastore = nexus.datastore(); - let log = &cptestctx.logctx.log; - let opctx = OpContext::for_tests(log.clone(), datastore.clone()); - - // Create project and pools in parallel - ops::join3( - create_default_ip_pools(client), - create_project(client, PROJECT_NAME), - create_multicast_ip_pool(client, "sled-move-pool"), - ) - .await; - - // Create instance (no multicast groups at creation - implicit model) - let instance = instance_for_multicast_groups( - cptestctx, - PROJECT_NAME, - INSTANCE_NAME, - true, - &[], - ) - .await; - - // Add instance to multicast group via instance-centric API - multicast_group_attach(&cptestctx, PROJECT_NAME, INSTANCE_NAME, GROUP_NAME) - .await; - wait_for_group_active(client, GROUP_NAME).await; - - let instance_uuid = InstanceUuid::from_untyped_uuid(instance.identity.id); - - // Wait for member to join - wait_for_member_state( - cptestctx, - GROUP_NAME, - instance.identity.id, - nexus_db_model::MulticastGroupMemberState::Joined, - ) - .await; - - // Verify initial port mapping (based on current inventory `sp_slot`) - verify_inventory_based_port_mapping(cptestctx, &instance_uuid) - .await - .expect("Should verify initial port mapping"); - - // Run reconciler again without new inventory to establish the - // baseline collection ID in the reconciler. Running it twice ensures the - // first run sets `last_seen_collection_id`, and the second run confirms - // no unnecessary cache invalidation occurs when collection is unchanged. - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - - // Verify port mapping is unchanged (no spurious cache invalidation) - verify_inventory_based_port_mapping(cptestctx, &instance_uuid) - .await - .expect("Port mapping should be unchanged when inventory unchanged"); - - // Assert that the member is in "Joined" state - let members_before = list_multicast_group_members(client, GROUP_NAME).await; - assert_eq!(members_before.len(), 1, "should have exactly one member"); - assert_eq!( - members_before[0].state, "Joined", - "member should be in Joined state before sled move" - ); - - // Get the sled this instance is running on - let sled_id = nexus - .active_instance_info(&instance_uuid, None) - .await - .expect("Active instance info should be available") - .expect("Instance should be on a sled") - .sled_id; - - // Get sled baseboard information - let sleds = datastore - .sled_list_all_batched(&opctx, SledFilter::InService) - .await - .expect("Should list in-service sleds"); - let sled = sleds - .into_iter() - .find(|s| s.id() == sled_id) - .expect("Should find sled in database"); - - // Get current inventory to see the original sp_slot - let original_inventory = datastore - .inventory_get_latest_collection(&opctx) - .await - .expect("Should fetch latest inventory collection") - .expect("Inventory collection should exist"); - - let original_sp = original_inventory - .sps - .iter() - .find(|(bb, _)| bb.serial_number == sled.serial_number()) - .map(|(_, sp)| sp) - .expect("Should find SP for sled in original inventory"); - - let original_slot = original_sp.sp_slot; - let sled_serial = sled.serial_number().to_string(); - let sled_part_number = sled.part_number().to_string(); - - // Verify DPD has the original port before the move - let dpd = nexus_test_utils::dpd_client(cptestctx); - let original_port_id = dpd_client::types::PortId::Rear( - dpd_client::types::Rear::try_from(format!("rear{original_slot}")) - .expect("Should be valid rear port string"), - ); - - // Determine a valid target slot by querying DPD's backplane map. - // Prefer a different slot if available; otherwise fall back to the same. - let backplane = dpd - .backplane_map() - .await - .expect("Should fetch backplane map") - .into_inner(); - let mut valid_slots: Vec = backplane - .keys() - .filter_map(|k| { - k.strip_prefix("rear").and_then(|s| s.parse::().ok()) - }) - .collect(); - valid_slots.sort_unstable(); - valid_slots.dedup(); - let new_slot = valid_slots - .iter() - .copied() - .find(|s| *s != original_slot) - .unwrap_or(original_slot); - - // Build a new inventory collection with the sled in a different slot - let mut builder = nexus_inventory::CollectionBuilder::new("sled-move-test"); - builder.found_sp_state( - "test-sp", - SpType::Sled, - new_slot, - SpState { - serial_number: sled_serial, - model: sled_part_number, - power_state: PowerState::A0, - revision: 0, - base_mac_address: [0; 6], - hubris_archive_id: "test-hubris".to_string(), - rot: RotState::CommunicationFailed { - message: "test-rot-state".to_string(), - }, - }, - ); - - let new_collection = builder.build(); - - // Insert the new inventory collection - datastore - .inventory_insert_collection(&opctx, &new_collection) - .await - .expect("Should insert new inventory collection"); - - // Activate the inventory loader to update the watch channel with the new - // collection, then activate the reconciler which will detect the sled - // location change and invalidate caches. - activate_inventory_loader(&cptestctx.lockstep_client).await; - nexus.invalidate_multicast_caches(); - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - - // Verify that DPD now uses the new rear port (matching new `sp_slot`) - // This helper reads the latest inventory and asserts DPD has a member - // on rear{`sp_slot`}, so it will verify the new mapping is right - verify_inventory_based_port_mapping(cptestctx, &instance_uuid) - .await - .expect("Port mapping should be updated after cache invalidation"); - - // Assert that the member is still in "Joined" state after the move - let members_after = list_multicast_group_members(client, GROUP_NAME).await; - assert_eq!(members_after.len(), 1, "should still have exactly one member"); - assert_eq!( - members_after[0].state, "Joined", - "member should still be in Joined state after sled move" - ); - assert_eq!( - members_after[0].instance_id, instance.identity.id, - "member should still reference the same instance" - ); - - // Verify stale port cleanup: fetch DPD state and ensure old port was removed - let members = datastore - .multicast_group_members_list_by_instance( - &opctx, - instance_uuid, - &DataPageParams::max_page(), - ) - .await - .expect("Should list multicast members for instance"); - let member = members - .first() - .expect("Instance should have at least one multicast membership"); - - let external_group = datastore - .multicast_group_fetch( - &opctx, - MulticastGroupUuid::from_untyped_uuid(member.external_group_id), - ) - .await - .expect("Should fetch external multicast group"); - let underlay_group_id = external_group - .underlay_group_id - .expect("External group should have underlay_group_id"); - - let underlay_group = datastore - .underlay_multicast_group_fetch(&opctx, underlay_group_id) - .await - .expect("Should fetch underlay multicast group"); - - let dpd_client = nexus_test_utils::dpd_client(cptestctx); - let underlay_group_response = dpd_client - .multicast_group_get(&underlay_group.multicast_ip.ip()) - .await - .expect("DPD multicast_group_get should succeed") - .into_inner(); - - let dpd_members = match underlay_group_response { - dpd_client::types::MulticastGroupResponse::Underlay { - members, .. - } => members, - dpd_client::types::MulticastGroupResponse::External { .. } => { - panic!("Expected Underlay group, got External"); - } - }; - - // Verify that the old port membership has been removed (stale port cleanup) - let has_old_port_member = dpd_members.iter().any(|m| { - matches!(m.direction, dpd_client::types::Direction::Underlay) - && m.port_id == original_port_id - }); - - assert!( - !has_old_port_member, - "Old underlay member with rear{original_slot} should have been removed after sled move" - ); -} - -/// Test for cache TTL behavior. -/// -/// This test verifies that both sled and backplane cache TTL expiry work correctly: -/// -/// Sled cache TTL with inventory change: -/// - Start test server with short TTLs (sled=2s, backplane=1s) -/// - Create multicast group and instance, wait for member to join -/// - Insert new inventory with different `sp_slot` (simulating sled move) -/// - Wait for sled cache TTL to expire -/// - Verify DPD uses the new rear port after reconciler refreshes cache -/// -/// Backplane cache TTL without change: -/// - Wait for backplane cache TTL to expire (tests independent expiry) -/// - Activate reconciler (refreshes expired backplane cache from DPD) -/// - Verify port mapping still works after cache refresh -#[tokio::test] -async fn test_cache_ttl_behavior() { - const PROJECT_NAME: &str = "ttl-test-project"; - const GROUP_NAME: &str = "ttl-test-group"; - const INSTANCE_NAME: &str = "ttl-test-instance"; - - // Start test server with custom config - let cptestctx = - nexus_test_utils::ControlPlaneBuilder::new("test_cache_ttl_behavior") - .customize_nexus_config(&|config| { - // Set short cache TTLs for testing - config - .pkg - .background_tasks - .multicast_reconciler - .sled_cache_ttl_secs = - chrono::TimeDelta::seconds(2).to_std().unwrap(); - config - .pkg - .background_tasks - .multicast_reconciler - .backplane_cache_ttl_secs = - chrono::TimeDelta::seconds(1).to_std().unwrap(); - - // Ensure multicast is enabled - config.pkg.multicast.enabled = true; - }) - .start::() - .await; - - ensure_multicast_test_ready(&cptestctx).await; - - // Local handles for DB and opctx - let nexus = &cptestctx.server.server_context().nexus; - let datastore = nexus.datastore(); - let opctx = - OpContext::for_tests(cptestctx.logctx.log.clone(), datastore.clone()); - - let client = &cptestctx.external_client; - - // Create project and pools in parallel - ops::join3( - create_default_ip_pools(client), - create_project(client, PROJECT_NAME), - create_multicast_ip_pool(client, "ttl-test-pool"), - ) - .await; - - // Create instance (no multicast groups at creation - implicit model) - let instance = instance_for_multicast_groups( - &cptestctx, - PROJECT_NAME, - INSTANCE_NAME, - true, - &[], - ) - .await; - - // Add instance to multicast group via instance-centric API - multicast_group_attach(&cptestctx, PROJECT_NAME, INSTANCE_NAME, GROUP_NAME) - .await; - wait_for_group_active(client, GROUP_NAME).await; - - let instance_uuid = InstanceUuid::from_untyped_uuid(instance.identity.id); - - // Wait for member to join - wait_for_member_state( - &cptestctx, - GROUP_NAME, - instance.identity.id, - nexus_db_model::MulticastGroupMemberState::Joined, - ) - .await; - - // Verify initial port mapping (this populates both caches) - verify_inventory_based_port_mapping(&cptestctx, &instance_uuid) - .await - .expect("Should verify initial port mapping"); - - // Test sled cache TTL with inventory change - - // Get the sled this instance is running on - let sled_id = nexus - .active_instance_info(&instance_uuid, None) - .await - .expect("Active instance info should be available") - .expect("Instance should be on a sled") - .sled_id; - - // Get sled baseboard information - let sleds = datastore - .sled_list_all_batched(&opctx, SledFilter::InService) - .await - .expect("Should list in-service sleds"); - let sled = sleds - .into_iter() - .find(|s| s.id() == sled_id) - .expect("Should find sled in database"); - - // Get current inventory to see the original sp_slot - let original_inventory = datastore - .inventory_get_latest_collection(&opctx) - .await - .expect("Should fetch latest inventory collection") - .expect("Inventory collection should exist"); - - let original_sp = original_inventory - .sps - .iter() - .find(|(bb, _)| bb.serial_number == sled.serial_number()) - .map(|(_, sp)| sp) - .expect("Should find SP for sled in original inventory"); - - let original_slot = original_sp.sp_slot; - let sled_serial = sled.serial_number().to_string(); - let sled_part_number = sled.part_number().to_string(); - - // Determine a valid target slot by querying DPD's backplane map. - let dpd = nexus_test_utils::dpd_client(&cptestctx); - let backplane = dpd - .backplane_map() - .await - .expect("Should fetch backplane map") - .into_inner(); - let mut valid_slots: Vec = backplane - .keys() - .filter_map(|k| { - k.strip_prefix("rear").and_then(|s| s.parse::().ok()) - }) - .collect(); - valid_slots.sort_unstable(); - valid_slots.dedup(); - let new_slot = valid_slots - .iter() - .copied() - .find(|s| *s != original_slot) - .unwrap_or(original_slot); - - // Build a new inventory collection with the sled in a different slot - let mut builder = - nexus_inventory::CollectionBuilder::new("ttl-refresh-test"); - builder.found_sp_state( - "test-sp", - SpType::Sled, - new_slot, - SpState { - serial_number: sled_serial, - model: sled_part_number, - power_state: PowerState::A0, - revision: 0, - base_mac_address: [0; 6], - hubris_archive_id: "test-hubris".to_string(), - rot: RotState::CommunicationFailed { - message: "test-rot-state".to_string(), - }, - }, - ); - - let new_collection = builder.build(); - - // Insert the new inventory collection - datastore - .inventory_insert_collection(&opctx, &new_collection) - .await - .expect("Should insert new inventory collection"); - - // Wait for sled cache TTL to expire (2 seconds) - tokio::time::sleep(std::time::Duration::from_millis(2500)).await; - - wait_for_condition_with_reconciler( - &cptestctx.lockstep_client, - || async { - // Try to verify the inventory-based port mapping - // This will succeed once DPD has been updated with the new rear port - match verify_inventory_based_port_mapping( - &cptestctx, - &instance_uuid, - ) - .await - { - Ok(()) => Ok(()), - Err(_) => { - // Not yet updated, reconciler needs another cycle - Err(CondCheckError::::NotYet) - } - } - }, - &POLL_INTERVAL, - &MULTICAST_OPERATION_TIMEOUT, - ) - .await - .expect("DPD should update with new rear port after sled cache TTL expiry"); - - // Test backplane cache TTL without change - - // Wait for backplane cache TTL to expire (1 second) - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - - // Force cache access by activating reconciler - // This will cause the reconciler to check backplane cache, find it expired, - // and refresh from DPD. - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - - // Verify member is still on the right port after backplane cache refresh - verify_inventory_based_port_mapping(&cptestctx, &instance_uuid) - .await - .expect("Port mapping should work after backplane cache TTL expiry"); - - // Verify member is still in "Joined" state after all cache operations - let members = list_multicast_group_members(client, GROUP_NAME).await; - assert_eq!(members.len(), 1, "should still have exactly one member"); - assert_eq!( - members[0].state, "Joined", - "member should remain in Joined state after cache operations" - ); - assert_eq!( - members[0].instance_id, instance.identity.id, - "member should still reference the same instance" - ); - - cptestctx.teardown().await; -} - -/// Verify expunged sleds are excluded from multicast cache after refresh. -#[nexus_test(extra_sled_agents = 1)] -async fn test_sled_expunge_removes_from_multicast_cache( - cptestctx: &ControlPlaneTestContext, -) { - const PROJECT_NAME: &str = "expunge-test-project"; - const GROUP_NAME: &str = "expunge-test-group"; - const INSTANCE_NAME: &str = "expunge-test-instance"; - - ensure_multicast_test_ready(cptestctx).await; - - let client = &cptestctx.external_client; - let nexus = &cptestctx.server.server_context().nexus; - let datastore = nexus.datastore(); - let opctx = - OpContext::for_tests(cptestctx.logctx.log.clone(), datastore.clone()); - - // Make the second sled non-provisionable so instances go to the first sled - let (authz_sled, ..) = LookupPath::new(&opctx, datastore) - .sled_id(cptestctx.second_sled_id()) - .lookup_for(nexus_db_queries::authz::Action::Modify) - .await - .expect("lookup authz_sled"); - datastore - .sled_set_provision_policy( - &opctx, - &authz_sled, - nexus_types::external_api::sled::SledProvisionPolicy::NonProvisionable, - ) - .await - .expect("set sled provision policy"); - - ops::join3( - create_default_ip_pools(client), - create_project(client, PROJECT_NAME), - create_multicast_ip_pool(client, "expunge-test-pool"), - ) - .await; - - let instance = instance_for_multicast_groups( - cptestctx, - PROJECT_NAME, - INSTANCE_NAME, - true, - &[], - ) - .await; - - multicast_group_attach(&cptestctx, PROJECT_NAME, INSTANCE_NAME, GROUP_NAME) - .await; - wait_for_group_active(client, GROUP_NAME).await; - - let instance_uuid = InstanceUuid::from_untyped_uuid(instance.identity.id); - - wait_for_member_state( - cptestctx, - GROUP_NAME, - instance.identity.id, - nexus_db_model::MulticastGroupMemberState::Joined, - ) - .await; - - verify_inventory_based_port_mapping(&cptestctx, &instance_uuid) - .await - .expect("Should verify initial port mapping"); - - let first_sled_id = cptestctx.first_sled_id(); - cptestctx - .lockstep_client - .make_request( - Method::POST, - "/sleds/expunge", - Some(sled::SledSelector { sled: first_sled_id }), - StatusCode::OK, - ) - .await - .expect("Failed to expunge sled"); - - // Wait for instance to fail (instance-watcher marks instances on expunged sleds as "Failed") - instance_wait_for_state(client, instance_uuid, InstanceState::Failed).await; - - // Manually invalidate caches. - // - // Inventory-based invalidation is tested in - // `test_sled_move_updates_multicast_port_mapping`. This test verifies cache - // refresh uses SledFilter::InService, which excludes expunged sleds. - nexus.invalidate_multicast_caches(); - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - - wait_for_member_state( - cptestctx, - GROUP_NAME, - instance.identity.id, - nexus_db_model::MulticastGroupMemberState::Left, - ) - .await; - - let in_service_sleds = datastore - .sled_list_all_batched(&opctx, SledFilter::InService) - .await - .expect("Failed to list in-service sleds"); - - assert!( - !in_service_sleds.iter().any(|s| s.id() == first_sled_id), - "Expunged sled should not appear in InService sled list" - ); - - let all_sleds = datastore - .sled_list_all_batched(&opctx, SledFilter::All) - .await - .expect("Failed to list all sleds"); - - assert!( - all_sleds.iter().any(|s| s.id() == first_sled_id), - "Expunged sled should still appear in All filter" - ); -} diff --git a/nexus/tests/integration_tests/multicast/failures.rs b/nexus/tests/integration_tests/multicast/failures.rs index a0f70b79320..6e7d0047ef2 100644 --- a/nexus/tests/integration_tests/multicast/failures.rs +++ b/nexus/tests/integration_tests/multicast/failures.rs @@ -1,8 +1,6 @@ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -// -// Copyright 2025 Oxide Computer Company //! Integration tests for multicast group failure and recovery scenarios. //! @@ -57,7 +55,7 @@ use crate::integration_tests::instances as instance_helpers; /// When DPD is unavailable during group activation: /// - Group stays in Creating state /// - Member stays in Joining state -/// - After DPD recovery, group becomes Active and member becomes Joined +/// - After DPD recovery, group activation completes and member becomes Joined #[nexus_test] async fn test_dpd_failure_during_creating_state( cptestctx: &ControlPlaneTestContext, @@ -214,6 +212,16 @@ async fn test_dpd_failure_during_active_state( /// When DPD is unavailable during implicit group deletion: /// - Group stays in Deleting state (cannot complete cleanup) /// - After DPD recovery, deletion completes +/// +/// This also exercises a partial-cleanup retry invariant: the deletion +/// path for groups in "Deleting" is sequential (MRIB withdrawal, sled +/// M2P/forwarding clear, DPD cleanup, DB delete) and returns early on first +/// failure. With DPD stopped, MGD-side MRIB removal succeeds and DPD +/// removal fails, so the group stays in "Deleting". After DPD recovery +/// the next reconciler pass must re-issue MRIB removals on already-empty +/// routes without erroring, which depends on +/// `mg_admin_client::static_remove_mcast_route` being idempotent +/// (verified at the RDB layer in Maghemite). #[nexus_test] async fn test_dpd_failure_during_deleting_state( cptestctx: &ControlPlaneTestContext, @@ -231,13 +239,19 @@ async fn test_dpd_failure_during_deleting_state( ) .await; + // The single converging pass needs sled inventory and DPD ready before + // the dpd-ensure saga runs. + ensure_inventory_ready(cptestctx).await; + ensure_dpd_ready(cptestctx).await; + // Create instance and add to group create_instance(client, project_name, instance_name).await; multicast_group_attach(cptestctx, project_name, instance_name, group_name) .await; - // Wait for group to reach Active state - wait_for_group_active(client, group_name).await; + let active_group = wait_for_group_active(client, group_name).await; + let multicast_ip = active_group.multicast_ip; + assert_mrib_route_exists(cptestctx, multicast_ip).await; // Stop DPD before triggering deletion cptestctx.stop_dendrite(SwitchSlot::Switch0).await; @@ -294,11 +308,22 @@ async fn test_dpd_failure_during_deleting_state( assert_eq!(group.identity.name.as_str(), group_name); } + // Even though the deletion did not complete, MRIB removal + // ran before the DPD step failed. The route must already be gone. + assert_mrib_route_absent(cptestctx, multicast_ip).await; + // Restart DPD and activate reconciler to complete deletion cptestctx.restart_dendrite(SwitchSlot::Switch0).await; activate_multicast_reconciler(&cptestctx.lockstep_client).await; cleanup_instances(cptestctx, client, project_name, &[instance_name]).await; wait_for_group_deleted(cptestctx, group_name).await; + + // The second reconciler pass re-issues MRIB removal on already-empty routes. + // If MGD treated the missing route as an error, the pass would short-circuit + // at MRIB and never retry the DPD/DB cleanup, leaving the group stuck in + // a "Deleting" state and `wait_for_group_deleted` above would time out. The + // route must remain absent and not accidentally re-install. + assert_mrib_route_absent(cptestctx, multicast_ip).await; } #[nexus_test] @@ -365,12 +390,11 @@ async fn test_multicast_group_members_during_dpd_failure( created_group.identity.id ); - // Verify member state during DPD failure - // Instance is running, so member has sled_id, but DPD is unavailable so it - // can't be programmed against. + // Verify member state during DPD failure. The instance is running, but + // the group cannot become Active until DPD group programming succeeds. assert_eq!( members_during_failure[0].state, "Joining", - "Member should be Joining when DPD unavailable (waiting to be programmed)" + "Member should be Joining while group activation waits on DPD" ); // Verify group is still in "Creating" state @@ -813,12 +837,17 @@ async fn test_drift_correction_missing_group_in_dpd( // This leaves the group "Active" in DB but missing from DPD cptestctx.restart_dendrite(SwitchSlot::Switch0).await; - // Verify group is missing from DPD after restart (drift exists) + // Verify group is missing from DPD after restart (drift exists). A + // reconciler pass queued by the earlier attach can race this check and + // re-program the group first. That is the drift correction under test + // arriving early, so tolerate it rather than flake. let dpd_client = nexus_test_utils::dpd_client(cptestctx); - assert!( - dpd_client.multicast_group_get(&multicast_ip).await.is_err(), - "Group should not exist in DPD after restart (this is the drift)" - ); + if dpd_client.multicast_group_get(&multicast_ip).await.is_ok() { + eprintln!( + "drift already corrected by a background reconciler pass; \ + continuing" + ); + } // Activate reconciler - should detect missing group and re-program DPD activate_multicast_reconciler(&cptestctx.lockstep_client).await; @@ -851,7 +880,7 @@ async fn test_drift_correction_missing_group_in_dpd( /// Test member state transition: "Joining" → "Left" when instance becomes invalid. /// -/// When a member is in "Joining" state (waiting for DPD programming) and the +/// When a member is in "Joining" state (waiting for group activation) and the /// instance becomes invalid (stopped/failed), the RPW should transition the /// member to "Left" state. #[nexus_test] @@ -875,17 +904,20 @@ async fn test_member_joining_to_left_on_instance_stop( let instance = create_instance(client, project_name, instance_name).await; let instance_id = InstanceUuid::from_untyped_uuid(instance.identity.id); - // Stop DPD to prevent member from transitioning to "Joined" + // Stop DPD to keep the group from transitioning to "Active". cptestctx.stop_dendrite(SwitchSlot::Switch0).await; - // Add instance to group - member will be stuck in "Joining" since DPD is down + // Add instance to group. The member remains in "Joining" while DPD is + // unavailable for group activation. multicast_group_attach(cptestctx, project_name, instance_name, group_name) .await; - // Run reconciler - member should stay in "Joining" since DPD is unavailable + // Run reconciler. Member should stay in "Joining" while the group is + // still waiting for DPD. wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - // Verify member is in "Joining" state (can't reach Joined without DPD) + // Verify member is in "Joining" state (can't reach Joined until the group + // becomes Active). let members = list_multicast_group_members(client, group_name).await; assert_eq!(members.len(), 1); assert_eq!( @@ -943,7 +975,7 @@ async fn test_member_joining_to_left_on_instance_stop( /// /// When a member is in "Left" state and the instance starts running, the member /// should stay in "Left" until the group becomes "Active". This prevents -/// premature member activation when the group hasn't been programmed in DPD. +/// premature member activation while DPD group programming is still pending. #[nexus_test] async fn test_left_member_waits_for_group_active( cptestctx: &ControlPlaneTestContext, @@ -990,8 +1022,27 @@ async fn test_left_member_waits_for_group_active( put_upsert::<_, MulticastGroupMember>(client, &join_url, &join_params) .await; - // Verify group is stuck in "Creating" (DPD is down) - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; + // Join records the member as "Joining". The reconciler demotes to + // "Left" once it observes the stopped instance. This transition is + // independent of DPD, so the assertion can advance even with + // Dendrite stopped. + wait_for_condition( + || async { + let members = + list_multicast_group_members(client, group_name).await; + match members.first() { + Some(m) if m.state == "Left" => Ok(()), + _ => Err(CondCheckError::<()>::NotYet), + } + }, + &POLL_INTERVAL, + &MULTICAST_OPERATION_TIMEOUT, + ) + .await + .expect( + "member should reach Left after reconciler observes stopped instance", + ); + let group: MulticastGroup = object_get(client, &format!("/v1/multicast-groups/{group_name}")).await; assert_eq!( @@ -999,7 +1050,6 @@ async fn test_left_member_waits_for_group_active( "Group should be stuck in Creating without DPD" ); - // Verify member is in "Left" state (stopped instance) let members = list_multicast_group_members(client, group_name).await; assert_eq!(members.len(), 1); assert_eq!( @@ -1021,19 +1071,15 @@ async fn test_left_member_waits_for_group_active( .unwrap(); instance_wait_for_running_with_simulation(cptestctx, instance_id).await; - // Run reconciler - member should stay in Left because group is not Active - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - - // Verify member stays in "Left" (waiting for group to become Active) - let members_after = list_multicast_group_members(client, group_name).await; - assert_eq!(members_after.len(), 1); - assert_eq!( - members_after[0].state, "Left", - "Member should stay in Left while group is Creating, got: {}", - members_after[0].state - ); + activate_multicast_reconciler(&cptestctx.lockstep_client).await; + wait_for_member_state( + cptestctx, + group_name, + instance.identity.id, + nexus_db_model::MulticastGroupMemberState::Left, + ) + .await; - // Verify group is still Creating let group_after: MulticastGroup = object_get(client, &format!("/v1/multicast-groups/{group_name}")).await; assert_eq!( diff --git a/nexus/tests/integration_tests/multicast/groups.rs b/nexus/tests/integration_tests/multicast/groups.rs index 9ae620b7dc2..79d253ee5ae 100644 --- a/nexus/tests/integration_tests/multicast/groups.rs +++ b/nexus/tests/integration_tests/multicast/groups.rs @@ -1,8 +1,6 @@ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -// -// Copyright 2025 Oxide Computer Company //! Integration tests for multicast group APIs and IP pool operations. //! @@ -283,31 +281,26 @@ async fn test_multicast_group_member_operations( }) .expect("Should find underlay group IP in DPD response"); - // Get the underlay group directly - let underlay_group = dpd_client + // Get the underlay group directly. Group-level programming is all Nexus + // owns in DPD. Per-member underlay forwarding (rear ports, Underlay + // direction) is programmed by `ddmd` from DDM peer subscriptions, so member + // contents are not asserted here. + dpd_client .multicast_group_get_underlay(&underlay_ip) .await .expect("Should get underlay group from DPD"); - assert_eq!( - underlay_group.members.len(), - 1, - "Underlay group should have exactly 1 member after member addition" - ); - - // Assert all underlay members use rear (backplane) ports with Underlay direction - for member in &underlay_group.members { - assert!( - matches!(member.port_id, dpd_client::types::PortId::Rear(_)), - "Underlay member should use rear (backplane) port, got: {:?}", - member.port_id - ); - assert_eq!( - member.direction, - dpd_client::types::Direction::Underlay, - "Underlay member should have Underlay direction" - ); - } + // MRIB: an any-source join yields exactly the (*,G) wildcard route on + // every switch, targeting the same admin-local underlay group that DPD + // programmed. mg-lower advertises this route to peer sleds via DDM. + assert_mrib_route_sources( + cptestctx, + external_multicast_ip, + &[None], + Some(underlay_ip.0), + "wildcard MRIB route once the member is joined", + ) + .await; // Test removing instance from multicast group using instance-centric DELETE let member_remove_url = format!( @@ -327,6 +320,9 @@ async fn test_multicast_group_member_operations( // Wait for both Nexus group and DPD group to be deleted wait_for_group_deleted(cptestctx, group_name).await; wait_for_group_deleted_from_dpd(cptestctx, external_multicast_ip).await; + + // MRIB route is withdrawn along with the dataplane teardown. + assert_mrib_route_absent(cptestctx, external_multicast_ip).await; } #[nexus_test] @@ -358,11 +354,12 @@ async fn test_instance_multicast_endpoints( let instance = create_instance(client, project_name, instance_name).await; let instance_id = InstanceUuid::from_untyped_uuid(instance.identity.id); - // Simulate and wait for instance to be fully running with sled_id assigned + // Simulate and wait for instance to be fully running. The first + // `wait_for_member_state` @ "Joined" also waits for sled assignment + // before its assertion. let nexus = &cptestctx.server.server_context().nexus; instance_simulate(nexus, &instance_id).await; instance_wait_for_state(client, instance_id, InstanceState::Running).await; - wait_for_instance_sled_assignment(cptestctx, &instance_id).await; // Case: List instance multicast groups (should be empty initially) let instance_groups_url = format!( @@ -652,6 +649,9 @@ async fn test_instance_deletion_removes_multicast_memberships( assert_eq!(members.len(), 1, "Instance should be a member of the group"); assert_eq!(members[0].instance_id, instance.identity.id); + // Verify MRIB route exists while group is active with a joined member. + assert_mrib_route_exists(cptestctx, multicast_ip).await; + // Case: Instance deletion should clean up multicast memberships cleanup_instances(cptestctx, client, project_name, &[instance_name]).await; @@ -666,6 +666,9 @@ async fn test_instance_deletion_removes_multicast_memberships( // Wait for reconciler to clean up DPD state (activates reconciler repeatedly until DPD confirms deletion) wait_for_group_deleted_from_dpd(cptestctx, multicast_ip).await; + + // Verify MRIB route was withdrawn after group deletion. + assert_mrib_route_absent(cptestctx, multicast_ip).await; } /// Test that the multicast_ip field is correctly populated in MulticastGroupMember API responses. @@ -1057,7 +1060,7 @@ async fn test_ssm_source_ip_behavior(cptestctx: &ControlPlaneTestContext) { .flatten() .filter_map(|src| match src { dpd_types::IpSrc::Exact(ip) => Some(*ip), - dpd_types::IpSrc::Any => None, // Any-source (ASM) wildcard + dpd_types::IpSrc::Any => None, // any-source (ASM) wildcard }) .collect(); dpd_source_ips.sort(); @@ -1067,6 +1070,72 @@ async fn test_ssm_source_ip_behavior(cptestctx: &ControlPlaneTestContext) { "DPD external group sources should be union of all member sources" ); + // MRIB: one (S,G) static route per source in the union, on every switch. + // Routes require a "Joined" member, so converge membership first. + wait_for_all_members_joined(cptestctx, ssm_union_ip).await; + let union_mrib: Vec> = + expected_union_sources.iter().copied().map(Some).collect(); + assert_mrib_route_sources( + cptestctx, + multicast_ip, + &union_mrib, + None, + "one (S,G) MRIB route per source in the union", + ) + .await; + + // Case: (S,G) source-set narrowing on member detach. + // As specific-source members leave, the DPD union must shrink to the + // remaining members' sources. SSM groups never wildcard, so the DPD + // source list stays `Some(...)` throughout. + multicast_group_detach( + client, + project_name, + instance_names[2], + ssm_union_ip, + ) + .await; + let mut expected_after_inst3 = vec![source1, source2]; + expected_after_inst3.sort(); + wait_for_dpd_source_filter( + cptestctx, + multicast_ip, + Some(expected_after_inst3), + "DPD union should shrink to {source1, source2} after inst-3 detaches", + ) + .await; + assert_mrib_route_sources( + cptestctx, + multicast_ip, + &[Some(source1), Some(source2)], + None, + "MRIB (S,G) routes narrow after inst-3 detaches", + ) + .await; + + multicast_group_detach( + client, + project_name, + instance_names[1], + ssm_union_ip, + ) + .await; + wait_for_dpd_source_filter( + cptestctx, + multicast_ip, + Some(vec![source1]), + "DPD union should shrink to {source1} after inst-2 detaches", + ) + .await; + assert_mrib_route_sources( + cptestctx, + multicast_ip, + &[Some(source1)], + None, + "MRIB (S,G) routes narrow after inst-2 detaches", + ) + .await; + // Case: IPv6 source with IPv4 group should fail let ipv4_ssm_ip = "232.1.0.20"; let ipv6_source: IpAddr = "2001:db8::1".parse().unwrap(); @@ -1224,6 +1293,312 @@ async fn test_ssm_source_ip_behavior(cptestctx: &ControlPlaneTestContext) { for (group_name, _) in &group_configs { wait_for_group_deleted(cptestctx, group_name).await; } + + // MRIB routes are withdrawn with group deletion. + assert_mrib_route_absent(cptestctx, multicast_ip).await; +} + +/// Read the DPD external group source filter as a sorted [`Option>`]. +/// +/// `None` indicates DPD-level source filtering is disabled (the (*,G) case +/// produced by [`compute_sources_for_dpd`] when any ASM member has empty +/// `source_ips`). `Some(sorted)` is the (S,G) union written by Nexus. +/// +/// [`dpd_types::IpSrc::Any`] entries are automatically filtered out. +/// Nexus only emits [`dpd_types::IpSrc::Exact`] today. +async fn dpd_external_source_filter( + cptestctx: &ControlPlaneTestContext, + multicast_ip: IpAddr, +) -> Option> { + let dpd_response = dpd_client(cptestctx) + .multicast_group_get(&multicast_ip) + .await + .expect("DPD should have external group") + .into_inner(); + match dpd_response { + dpd_types::MulticastGroupResponse::External { sources, .. } => sources + .map(|srcs| { + let mut ips: Vec = srcs + .iter() + .filter_map(|src| match src { + dpd_types::IpSrc::Exact(ip) => Some(*ip), + dpd_types::IpSrc::Any => None, + }) + .collect(); + ips.sort(); + ips + }), + dpd_types::MulticastGroupResponse::Underlay { .. } => { + panic!("Expected External group from DPD, got Underlay") + } + } +} + +/// Activate the multicast reconciler and poll DPD until the external group's +/// source filter matches what's expected. Use this after an attach/detach where the +/// test asserts on DPD's converged (S,G) / (*,G) state. +async fn wait_for_dpd_source_filter( + cptestctx: &ControlPlaneTestContext, + multicast_ip: IpAddr, + expected: Option>, + msg: &str, +) { + activate_then_wait_for_condition( + &cptestctx.lockstep_client, + || async { + let actual = + dpd_external_source_filter(cptestctx, multicast_ip).await; + if actual == expected { + Ok(()) + } else { + Err(CondCheckError::<()>::NotYet) + } + }, + &POLL_INTERVAL, + &POLL_TIMEOUT, + ) + .await + .unwrap_or_else(|err| { + panic!( + "{msg}: expected {expected:?}, last observed mismatch ({err:?})", + ) + }); +} + +/// Test ASM source-filter transitions across (*,G) and (S,G). +/// +/// Source filtering for an ASM group is the union of all members' source IPs, +/// unless any member has empty `source_ips`, in which case the switch-level +/// filter is disabled (DPD `sources = None`, i.e. (*,G)). This test exercises +/// the transitions on a single group: +/// +/// 1. Specific-only union grows. Members join with disjoint sources. +/// 2. Widen (S,G) -> (*,G). An any-source member joins, and DPD `sources` +/// becomes `None`. +/// 3. Two-any-source aggregation. A second any-source member joins, then the +/// first leaves. DPD must remain `None` to prove `has_any_source_member` +/// is OR-aggregated across live members rather than a stuck flag. +/// 4. Narrow (*,G) -> (S,G). The last any-source member detaches, and DPD +/// `sources` returns to the remaining specific-source union. +/// 5. Specific union shrinks. A specific-source member detaches, and the DPD +/// union contracts. +#[nexus_test] +async fn test_asm_source_filter_transitions( + cptestctx: &ControlPlaneTestContext, +) { + let client = &cptestctx.external_client; + let project_name = "asm-src-transitions"; + + ops::join3( + create_project(&client, project_name), + create_default_ip_pools(&client), + create_multicast_ip_pool_with_range( + &client, + "asm-transitions-pool", + (224, 2, 0, 0), + (224, 2, 0, 100), + ), + ) + .await; + + // Instance names encode each member's role in the test sequence. The + // `specific-` prefix denotes a member that subscribes to a single source + // (an (S,G) contributor). The `any-source-` prefix denotes a member that + // subscribes to all sources, which forces the group's switch-level filter + // off (turning the group into (*,G)). The trailing letter or digit + // distinguishes peers that play the same role: two specific-source + // members exercise union growth and shrink, and two any-source members + // exercise OR-aggregation of `has_any_source_member` across live members. + let specific_a = "asm-specific-source-a"; + let specific_b = "asm-specific-source-b"; + let any_source_1 = "asm-any-source-1"; + let any_source_2 = "asm-any-source-2"; + let instance_names = [specific_a, specific_b, any_source_1, any_source_2]; + for name in &instance_names { + create_instance(client, project_name, name).await; + } + + let group_ip_str = "224.2.0.10"; + let group_ip: IpAddr = group_ip_str.parse().unwrap(); + let source_a = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)); + let source_b = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2)); + + // Step 1: specific-source-a joins. DPD union is {source_a}. + multicast_group_attach_with_sources( + cptestctx, + project_name, + specific_a, + group_ip_str, + Some(vec![source_a]), + ) + .await; + wait_for_group_active(client, group_ip_str).await; + wait_for_dpd_source_filter( + cptestctx, + group_ip, + Some(vec![source_a]), + "DPD should program (S,G) with the only specific source", + ) + .await; + + // MRIB routes require a "Joined" member, so converge membership once + // here. specific-source-a never detaches, so the group keeps a joined + // member for the remainder of the test. + wait_for_all_members_joined(cptestctx, group_ip_str).await; + assert_mrib_route_sources( + cptestctx, + group_ip, + &[Some(source_a)], + None, + "one (S,G) MRIB route for the only specific source", + ) + .await; + + // specific-source-b joins, so the DPD union grows to {source_a, source_b}. + multicast_group_attach_with_sources( + cptestctx, + project_name, + specific_b, + group_ip_str, + Some(vec![source_b]), + ) + .await; + let mut expected_specific = vec![source_a, source_b]; + expected_specific.sort(); + wait_for_dpd_source_filter( + cptestctx, + group_ip, + Some(expected_specific.clone()), + "DPD union should grow to include both specific sources", + ) + .await; + assert_mrib_route_sources( + cptestctx, + group_ip, + &[Some(source_a), Some(source_b)], + None, + "MRIB (S,G) routes grow with the specific union", + ) + .await; + + // Step 2: any-source-1 joins. DPD widens (S,G) to (*,G). + multicast_group_attach_with_sources( + cptestctx, + project_name, + any_source_1, + group_ip_str, + None, + ) + .await; + wait_for_dpd_source_filter( + cptestctx, + group_ip, + None, + "DPD source filter should be disabled once any member is any-source", + ) + .await; + + // MRIB and DPD diverge for mixed membership: DPD collapses to (*,G) + // (filter disabled), while the MRIB keeps one route per specific source + // plus the any-source wildcard so each subscription stays addressable. + assert_mrib_route_sources( + cptestctx, + group_ip, + &[Some(source_a), Some(source_b), None], + None, + "MRIB keeps specific routes plus the wildcard for mixed membership", + ) + .await; + + // Step 3: any-source-2 joins, then any-source-1 detaches. DPD must stay + // (*,G) across the swap: the group is any-source as long as any live + // member is any-source, independent of join/detach order. + multicast_group_attach_with_sources( + cptestctx, + project_name, + any_source_2, + group_ip_str, + None, + ) + .await; + wait_for_dpd_source_filter( + cptestctx, + group_ip, + None, + "DPD should stay disabled with two any-source members", + ) + .await; + assert_mrib_route_sources( + cptestctx, + group_ip, + &[Some(source_a), Some(source_b), None], + None, + "MRIB wildcard persists with two any-source members", + ) + .await; + + multicast_group_detach(client, project_name, any_source_1, group_ip_str) + .await; + wait_for_dpd_source_filter( + cptestctx, + group_ip, + None, + "DPD should stay disabled while any-source-2 is still any-source", + ) + .await; + assert_mrib_route_sources( + cptestctx, + group_ip, + &[Some(source_a), Some(source_b), None], + None, + "MRIB wildcard persists while any-source-2 remains", + ) + .await; + + // Step 4: any-source-2 detaches. DPD narrows (*,G) to (S,G) back to the + // remaining specific-source union. + multicast_group_detach(client, project_name, any_source_2, group_ip_str) + .await; + wait_for_dpd_source_filter( + cptestctx, + group_ip, + Some(expected_specific), + "DPD source filter should re-enable with the remaining specific union", + ) + .await; + assert_mrib_route_sources( + cptestctx, + group_ip, + &[Some(source_a), Some(source_b)], + None, + "MRIB wildcard withdrawn once no member is any-source", + ) + .await; + + // Step 5: specific-source-b detaches. DPD union contracts to {source_a}. + multicast_group_detach(client, project_name, specific_b, group_ip_str) + .await; + wait_for_dpd_source_filter( + cptestctx, + group_ip, + Some(vec![source_a]), + "DPD union should contract after the second specific member leaves", + ) + .await; + assert_mrib_route_sources( + cptestctx, + group_ip, + &[Some(source_a)], + None, + "MRIB (S,G) routes contract with the specific union", + ) + .await; + + cleanup_instances(cptestctx, client, project_name, &instance_names).await; + wait_for_group_deleted(cptestctx, group_ip_str).await; + + // MRIB routes are withdrawn with group deletion. + assert_mrib_route_absent(cptestctx, group_ip).await; } /// Test default pool behavior when no pool is specified on member join. diff --git a/nexus/tests/integration_tests/multicast/instances.rs b/nexus/tests/integration_tests/multicast/instances.rs index 579bca00a16..33216ef7327 100644 --- a/nexus/tests/integration_tests/multicast/instances.rs +++ b/nexus/tests/integration_tests/multicast/instances.rs @@ -1,8 +1,6 @@ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/ -// -// Copyright 2025 Oxide Computer Company +// file, You can obtain one at https://mozilla.org/MPL/2.0/. //! Tests multicast group + instance integration. //! @@ -21,10 +19,12 @@ //! - Instance reconfigure adding SSM: Must specify sources for new SSM groups //! - SSM sources are per-member (S,G subscription model) +use std::collections::BTreeSet; use std::net::IpAddr; use http::{Method, StatusCode}; +use nexus_db_model::MulticastGroupMemberState; use nexus_db_queries::context::OpContext; use nexus_test_utils::http_testing::{AuthnMode, NexusRequest, RequestBuilder}; use nexus_test_utils::resource_helpers::{ @@ -47,7 +47,7 @@ use omicron_common::api::external::{ Nullable, }; use omicron_nexus::TestInterfaces; -use omicron_uuid_kinds::{GenericUuid, InstanceUuid}; +use omicron_uuid_kinds::{GenericUuid, InstanceUuid, MulticastGroupUuid}; use super::*; use crate::integration_tests::instances::{ @@ -137,7 +137,7 @@ async fn test_multicast_lifecycle(cptestctx: &ControlPlaneTestContext) { "group-lifecycle-1", instances[0].identity.id, // Instance is stopped, so should be "Left" - nexus_db_model::MulticastGroupMemberState::Left, + MulticastGroupMemberState::Left, ) .await; @@ -163,16 +163,15 @@ async fn test_multicast_lifecycle(cptestctx: &ControlPlaneTestContext) { ) .await; - // Verify both instances are attached to group-lifecycle-2 - for i in 0..2 { - wait_for_member_state( - cptestctx, - "group-lifecycle-2", - instances[i + 1].identity.id, - nexus_db_model::MulticastGroupMemberState::Left, // Stopped instances - ) + // Verify both instances are attached to group-lifecycle-2 ("Left" + // because the instances are stopped). + let expected_left: Vec<_> = (0..2) + .map(|i| { + (instances[i + 1].identity.id, MulticastGroupMemberState::Left) + }) + .collect(); + wait_for_members_state(cptestctx, "group-lifecycle-2", &expected_left) .await; - } // Multi-group attachment (instance to multiple groups) // Attach instance-multi-groups to group-lifecycle-3 (implicitly creates the group) @@ -205,7 +204,7 @@ async fn test_multicast_lifecycle(cptestctx: &ControlPlaneTestContext) { cptestctx, group_name, instances[3].identity.id, - nexus_db_model::MulticastGroupMemberState::Left, // Stopped instance + MulticastGroupMemberState::Left, // Stopped instance ) .await; } @@ -426,7 +425,7 @@ async fn test_multicast_group_attach_multiple( cptestctx, group_name, instance.identity.id, - nexus_db_model::MulticastGroupMemberState::Left, + MulticastGroupMemberState::Left, ) .await; } @@ -469,7 +468,7 @@ async fn test_multicast_group_attach_multiple( /// The system handles multiple instances joining simultaneously, rapid attach/detach /// cycles, and concurrent operations during reconciler processing. These scenarios /// expose race conditions in member state transitions, reconciler processing, and -/// DPD synchronization that sequential tests can't catch. +/// sled-agent multicast state convergence that sequential tests can't catch. #[nexus_test] async fn test_multicast_concurrent_operations( cptestctx: &ControlPlaneTestContext, @@ -525,17 +524,13 @@ async fn test_multicast_concurrent_operations( ) .await; - // Verify all members reached correct state despite concurrent operations - for instance in instances.iter() { - wait_for_member_state( - cptestctx, - "concurrent-test-group", - instance.identity.id, - // create_instance() starts instances, so they should be Joined - nexus_db_model::MulticastGroupMemberState::Joined, - ) - .await; - } + // Verify all members reached correct state despite concurrent operations. + // create_instance() starts instances, so they should all be Joined. + let expected: Vec<_> = instances + .iter() + .map(|i| (i.identity.id, MulticastGroupMemberState::Joined)) + .collect(); + wait_for_members_state(cptestctx, "concurrent-test-group", &expected).await; // Verify final member count matches expected (all 4 instances) let members = @@ -605,16 +600,12 @@ async fn test_multicast_concurrent_operations( let post_rapid_members = list_multicast_group_members(client, "concurrent-test-group").await; - // Wait for all remaining members to reach "Joined" state - for member in &post_rapid_members { - wait_for_member_state( - cptestctx, - "concurrent-test-group", - member.instance_id, - nexus_db_model::MulticastGroupMemberState::Joined, - ) - .await; - } + // Wait for all remaining members to reach "Joined" state. + let expected: Vec<_> = post_rapid_members + .iter() + .map(|m| (m.instance_id, MulticastGroupMemberState::Joined)) + .collect(); + wait_for_members_state(cptestctx, "concurrent-test-group", &expected).await; // Cleanup and delete instances (group is implicitly deleted when last member removed) cleanup_instances(cptestctx, client, PROJECT_NAME, &instance_names).await; @@ -688,7 +679,7 @@ async fn test_multicast_member_cleanup_instance_never_started( cptestctx, group_name, instance.identity.id, - nexus_db_model::MulticastGroupMemberState::Left, + MulticastGroupMemberState::Left, ) .await; @@ -748,11 +739,13 @@ async fn test_multicast_member_cleanup_instance_never_started( /// Test multicast group membership during instance migration. /// /// This test verifies two migration scenarios: -/// 1. Single instance migration: membership persists, DPD is updated, port mapping works +/// 1. Single instance migration: membership persists and sled-agent state +/// follows the instance placement /// 2. Concurrent migrations: multiple instances migrate simultaneously without interference /// -/// The RPW reconciler detects `sled_id` changes and updates DPD configuration on -/// both source and target switches to maintain uninterrupted multicast traffic. +/// The RPW reconciler detects `sled_id` changes, updates member placement, and +/// re-converges OPTE subscription plus M2P/forwarding state. Per-member +/// rear-port DPD programming is owned by `ddmd` from DDM peer subscriptions. #[nexus_test(extra_sled_agents = 1)] async fn test_multicast_migration_scenarios( cptestctx: &ControlPlaneTestContext, @@ -810,16 +803,15 @@ async fn test_multicast_migration_scenarios( cptestctx, group1_name, instance1.identity.id, - nexus_db_model::MulticastGroupMemberState::Joined, + MulticastGroupMemberState::Joined, ) .await; - // Verify DPD before migration - let dpd_client = nexus_test_utils::dpd_client(cptestctx); - dpd_client - .multicast_group_get(&multicast_ip) - .await - .expect("Group should exist in DPD before migration"); + for (slot, dpd) in nexus_test_utils::dpd_clients_by_switch(cptestctx) { + dpd.multicast_group_get(&multicast_ip).await.unwrap_or_else(|e| { + panic!("{slot:?}: group should exist in DPD before migration: {e}") + }); + } // Migrate instance let source_sled = nexus @@ -869,22 +861,22 @@ async fn test_multicast_migration_scenarios( .sled_id; assert_eq!(post_sled, target_sled, "Instance should be on target sled"); - wait_for_multicast_reconciler(lockstep_client).await; wait_for_member_state( cptestctx, group1_name, instance1.identity.id, - nexus_db_model::MulticastGroupMemberState::Joined, + MulticastGroupMemberState::Joined, ) .await; - verify_inventory_based_port_mapping(cptestctx, &instance1_id) - .await - .expect("Port mapping should be updated"); - dpd_client - .multicast_group_get(&multicast_ip) - .await - .expect("Group should exist in DPD after migration"); + // Group-level DPD state is all Nexus owns. The rear-port move to the + // target sled is owned by `ddmd`, derived from DDM peer subscriptions, and + // is not asserted here. + for (slot, dpd) in nexus_test_utils::dpd_clients_by_switch(cptestctx) { + dpd.multicast_group_get(&multicast_ip).await.unwrap_or_else(|e| { + panic!("{slot:?}: group should exist in DPD after migration: {e}") + }); + } // Verify sled-agent state after migration: the target sled should // have the VMM subscription and M2P mapping. The source sled should @@ -917,15 +909,11 @@ async fn test_multicast_migration_scenarios( } }; - // Target sled should have the VMM subscription after the - // reconciler pushes it via verify_members. Poll because the - // reconciler may still be propagating state to the sled-agent. - let post_info = nexus - .active_instance_info(&instance1_id, None) - .await - .unwrap() - .unwrap(); - + // Target sled should have the VMM subscription. The target VMM + // receives memberships at registration, applied when OPTE ports + // are created. The reconciler's drift path is a backstop when the + // instance-update saga's eager sled_id write fails. Poll because + // registration and reconciliation may still be settling. let target_agent = cptestctx .sled_agents .iter() @@ -933,24 +921,36 @@ async fn test_multicast_migration_scenarios( .unwrap() .sled_agent(); - wait_for_condition_with_reconciler( + let propolis1_id = cptestctx + .server + .server_context() + .nexus + .active_instance_info(&instance1_id, None) + .await + .unwrap() + .expect("instance1 should have an active VMM") + .propolis_id; + + activate_then_wait_for_condition( &cptestctx.lockstep_client, || async { - let groups = target_agent.multicast_groups.lock().unwrap(); - let has_sub = - groups.get(&post_info.propolis_id).map_or(false, |g| { - g.iter().any(|m| m.group_ip == multicast_ip) - }); + let groups = + target_agent.instance_multicast_groups.lock().unwrap(); + let has_sub = groups.get(&propolis1_id).map_or(false, |g| { + g.iter().any(|m| m.group_ip == multicast_ip) + }); if has_sub { Ok(()) } else { Err(CondCheckError::NotYet::<()>) } }, &POLL_INTERVAL, &POLL_TIMEOUT, ) .await - .expect("Target sled should have VMM subscription after migration"); + .expect( + "Target sled should have instance subscription after migration", + ); // Target sled should have M2P mapping. - wait_for_condition_with_reconciler( + activate_then_wait_for_condition( &cptestctx.lockstep_client, || async { let m2p = target_agent.m2p_mappings.lock().unwrap(); @@ -1005,21 +1005,18 @@ async fn test_multicast_migration_scenarios( .map(|i| InstanceUuid::from_untyped_uuid(i.identity.id)) .collect(); - // Start all instances via simulation - for &instance_id in &instance_ids { + // Start all instances via simulation in parallel. + ops::join_all(instance_ids.iter().map(|&instance_id| async move { instance_simulate(nexus, &instance_id).await; instance_wait_for_state(client, instance_id, InstanceState::Running) .await; - } - for inst in &instances { - wait_for_member_state( - cptestctx, - group2_name, - inst.identity.id, - nexus_db_model::MulticastGroupMemberState::Joined, - ) - .await; - } + })) + .await; + let expected_joined: Vec<_> = instances + .iter() + .map(|inst| (inst.identity.id, MulticastGroupMemberState::Joined)) + .collect(); + wait_for_members_state(cptestctx, group2_name, &expected_joined).await; // Get source/target sleds for each instance let mut source_sleds = Vec::new(); @@ -1054,30 +1051,39 @@ async fn test_multicast_migration_scenarios( r.expect("Migration should initiate"); } - // Complete all migrations - for (i, &instance_id) in instance_ids.iter().enumerate() { - let info = nexus - .active_instance_info(&instance_id, None) - .await - .unwrap() - .unwrap(); - vmm_simulate_on_sled( - cptestctx, - nexus, - source_sleds[i], - info.propolis_id, - ) - .await; - vmm_simulate_on_sled( - cptestctx, - nexus, - target_sleds[i], - info.dst_propolis_id.unwrap(), - ) - .await; - instance_wait_for_state(client, instance_id, InstanceState::Running) + // Complete all migrations in parallel. + ops::join_all(instance_ids.iter().enumerate().map(|(i, &instance_id)| { + let source_sled = source_sleds[i]; + let target_sled = target_sleds[i]; + async move { + let info = nexus + .active_instance_info(&instance_id, None) + .await + .unwrap() + .unwrap(); + vmm_simulate_on_sled( + cptestctx, + nexus, + source_sled, + info.propolis_id, + ) .await; - } + vmm_simulate_on_sled( + cptestctx, + nexus, + target_sled, + info.dst_propolis_id.unwrap(), + ) + .await; + instance_wait_for_state( + client, + instance_id, + InstanceState::Running, + ) + .await; + } + })) + .await; // Verify all on target sleds for (i, &instance_id) in instance_ids.iter().enumerate() { @@ -1095,8 +1101,6 @@ async fn test_multicast_migration_scenarios( ); } - wait_for_multicast_reconciler(lockstep_client).await; - let post_members = list_multicast_group_members(client, group2_name).await; assert_eq!( post_members.len(), @@ -1104,15 +1108,12 @@ async fn test_multicast_migration_scenarios( "Both members should persist after concurrent migration" ); - for inst in &instances { - wait_for_member_state( - cptestctx, - group2_name, - inst.identity.id, - nexus_db_model::MulticastGroupMemberState::Joined, - ) + let post_migration_joined: Vec<_> = instances + .iter() + .map(|inst| (inst.identity.id, MulticastGroupMemberState::Joined)) + .collect(); + wait_for_members_state(cptestctx, group2_name, &post_migration_joined) .await; - } // Cleanup cleanup_instances( @@ -1716,18 +1717,28 @@ async fn test_ssm_without_sources_fails_create_and_reconfigure( /// /// This tests the invariant that `multicast_group_member_delete_by_group_and_instance` /// filters by both `group_id` and `instance_id`, not just `group_id`. This is -/// important for saga undo correctness: if Instance B's create saga fails after -/// joining a group, the undo must not affect Instance A's existing membership +/// important for saga undo correctness: if instance B's create saga fails after +/// joining a group, the undo must not affect instance A's existing membership /// in the same group. -#[nexus_test] +/// +/// This test forces a multi-sled layout via migration (A and B share a sled, +/// C sits on the other) so the member rows Nexus tracks span both shared and +/// solo placement when B is deleted. Per-member rear-port forwarding in DPD +/// is owned by `ddmd`, derived from DDM peer subscriptions, and is not asserted +/// here. The +/// group-level invariants are that the group stays "Active", its underlay +/// group survives in DPD on every switch, and its MRIB route persists. +#[nexus_test(extra_sled_agents = 1)] async fn test_instance_delete_preserves_other_memberships( cptestctx: &ControlPlaneTestContext, ) { + ensure_multicast_test_ready(cptestctx).await; + let client = &cptestctx.external_client; + let nexus = &cptestctx.server.server_context().nexus; let project_name = "delete-preserve-project"; let group_name = "delete-preserve-group"; - // Setup: create project and multicast pool ops::join3( create_default_ip_pools(client), create_project(client, project_name), @@ -1740,53 +1751,145 @@ async fn test_instance_delete_preserves_other_memberships( ) .await; - // Create Instance A and join it to the multicast group - create_instance(client, project_name, "instance-a").await; - multicast_group_attach(cptestctx, project_name, "instance-a", group_name) - .await; + let available_sleds = + [cptestctx.first_sled_id(), cptestctx.second_sled_id()]; + + // Bring up A, B, C as "Running" with the group attached. + let instances = ["instance-a", "instance-b", "instance-c"].iter().map( + |name| async move { + let inst = instance_for_multicast_groups( + cptestctx, + project_name, + name, + true, + &[group_name], + ) + .await; + let id = InstanceUuid::from_untyped_uuid(inst.identity.id); + instance_simulate(nexus, &id).await; + instance_wait_for_state(client, id, InstanceState::Running).await; + (inst, id) + }, + ); + let started: Vec<(Instance, InstanceUuid)> = ops::join_all(instances).await; + let (instance_a, instance_a_uuid) = &started[0]; + let (_instance_b, instance_b_uuid) = &started[1]; + let (instance_c, instance_c_uuid) = &started[2]; + wait_for_group_active(client, group_name).await; + let initial_joined: Vec<_> = started + .iter() + .map(|(inst, _)| (inst.identity.id, MulticastGroupMemberState::Joined)) + .collect(); + wait_for_members_state(cptestctx, group_name, &initial_joined).await; + + // Pick a "shared" sled (where A and B will live) and a "solo" sled + // (where C will live), based on A's current placement. + let shared_sled = nexus + .active_instance_info(instance_a_uuid, None) + .await + .unwrap() + .expect("instance A should be on a sled") + .sled_id; + let solo_sled = *available_sleds + .iter() + .find(|&&s| s != shared_sled) + .expect("two distinct sleds expected"); + + migrate_instance_to(cptestctx, *instance_b_uuid, shared_sled).await; + migrate_instance_to(cptestctx, *instance_c_uuid, solo_sled).await; + + // After migration, the reconciler must observe each member's new + // `sled_id` before the rear-port snapshot. We explicitly + // poll until the DB row matches the post-migration placement for + // every member. + let expected_placement = [ + (instance_a.identity.id, shared_sled), + (instance_b_uuid.into_untyped_uuid(), shared_sled), + (instance_c.identity.id, solo_sled), + ]; + wait_for_member_sled_ids(cptestctx, group_name, &expected_placement).await; - // Verify Instance A is a member let members_before = list_multicast_group_members(client, group_name).await; - assert_eq!(members_before.len(), 1, "Instance A should be a member"); - let instance_a_id = members_before[0].instance_id; + assert_eq!( + members_before.len(), + 3, + "all three instances should be members" + ); - // Create Instance B and join it to the same group - create_instance(client, project_name, "instance-b").await; - multicast_group_attach(cptestctx, project_name, "instance-b", group_name) - .await; + let group_view = get_multicast_group(client, group_name).await; + let multicast_ip = group_view.multicast_ip; + let underlay_admin_ip = + fetch_underlay_admin_ip(cptestctx, multicast_ip).await; + + // Group-level DPD state is all Nexus owns. Per-member rear-port + // forwarding is `ddmd`'s job from DDM peer subscriptions and is not + // asserted here. Assert only that the underlay group exists on every + // switch. + for (slot, dpd) in nexus_test_utils::dpd_clients_by_switch(cptestctx) { + dpd.multicast_group_get_underlay(&underlay_admin_ip) + .await + .unwrap_or_else(|e| { + panic!( + "{slot:?}: underlay group should exist in DPD before \ + B's deletion: {e}" + ) + }); + } - // Verify both instances are now members - let members_with_b = list_multicast_group_members(client, group_name).await; - assert_eq!(members_with_b.len(), 2, "Both instances should be members"); + // The ddm-peers view backs `omdb nexus multicast ddm-peers`. No rear-port + // peers form under this harness, so assert only that the read path is + // reachable and well-formed. + assert_ddm_peers_view_reachable(cptestctx).await; - // Delete Instance B, only removing B's membership, not A's + // Delete instance B. A still occupies the shared sled and C its own, + // so the group must stay "Active" with its dataplane state intact. cleanup_instances(cptestctx, client, project_name, &["instance-b"]).await; - // Verify that Instance A's membership must still exist - let members_after_b_delete = + let members_after_b = list_multicast_group_members(client, group_name).await; - - assert_eq!( - members_after_b_delete.len(), - 1, - "Instance A's membership should survive Instance B's deletion" + assert_eq!(members_after_b.len(), 2, "A and C must survive B's deletion"); + let remaining_instance_ids: BTreeSet<_> = + members_after_b.iter().map(|m| m.instance_id).collect(); + assert!( + remaining_instance_ids.contains(&instance_a.identity.id), + "A's membership must remain" ); - assert_eq!( - members_after_b_delete[0].instance_id, instance_a_id, - "The remaining member should be Instance A" + assert!( + remaining_instance_ids.contains(&instance_c.identity.id), + "C's membership must remain" ); - // Verify the group is still active (not deleted due to last member leaving) let group = get_multicast_group(client, group_name).await; - assert_eq!( - group.state, "Active", - "Group should still be active since Instance A is still a member" - ); + assert_eq!(group.state, "Active"); - // Cleanup: delete Instance A, which should trigger group deletion - cleanup_instances(cptestctx, client, project_name, &["instance-a"]).await; + // B's deletion must not disturb the group's dataplane state: the + // underlay group survives on every switch (A holds the shared sled) and + // the MRIB route persists. + for (slot, dpd) in nexus_test_utils::dpd_clients_by_switch(cptestctx) { + dpd.multicast_group_get_underlay(&underlay_admin_ip) + .await + .unwrap_or_else(|e| { + panic!( + "{slot:?}: underlay group should survive B's deletion: \ + {e}" + ) + }); + } + assert_mrib_route_exists(cptestctx, multicast_ip).await; + + // Cleanup: deleting A and C empties the group, so the sweep tears it + // down everywhere. + cleanup_instances( + cptestctx, + client, + project_name, + &["instance-a", "instance-c"], + ) + .await; wait_for_group_deleted(cptestctx, group_name).await; + wait_for_group_deleted_from_dpd(cptestctx, multicast_ip).await; + assert_mrib_route_absent(cptestctx, multicast_ip).await; } /// Test IPv6 multicast group lifecycle: create, start, stop, delete. @@ -1871,13 +1974,12 @@ async fn test_multicast_ipv6_lifecycle(cptestctx: &ControlPlaneTestContext) { .expect("Start should succeed"); instance_simulate(nexus, &instance_id).await; instance_wait_for_state(client, instance_id, InstanceState::Running).await; - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; wait_for_member_state( cptestctx, group_name, instance.identity.id, - nexus_db_model::MulticastGroupMemberState::Joined, + MulticastGroupMemberState::Joined, ) .await; @@ -1896,13 +1998,12 @@ async fn test_multicast_ipv6_lifecycle(cptestctx: &ControlPlaneTestContext) { instance_simulate(nexus, &instance_id).await; instance_wait_for_state(client, instance_id, InstanceState::Stopped).await; - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; wait_for_member_state( cptestctx, group_name, instance.identity.id, - nexus_db_model::MulticastGroupMemberState::Left, + MulticastGroupMemberState::Left, ) .await; @@ -1968,7 +2069,7 @@ async fn test_group_with_all_members_left(cptestctx: &ControlPlaneTestContext) { cptestctx, group_name, instance1.identity.id, - nexus_db_model::MulticastGroupMemberState::Joined, + MulticastGroupMemberState::Joined, ) .await; @@ -1999,7 +2100,7 @@ async fn test_group_with_all_members_left(cptestctx: &ControlPlaneTestContext) { cptestctx, group_name, instance2.identity.id, - nexus_db_model::MulticastGroupMemberState::Joined, + MulticastGroupMemberState::Joined, ) .await; @@ -2021,21 +2122,14 @@ async fn test_group_with_all_members_left(cptestctx: &ControlPlaneTestContext) { instance_wait_for_state(client, id, InstanceState::Stopped).await; } - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - // Verify both members are "Left" - wait_for_member_state( + wait_for_members_state( cptestctx, group_name, - instance1.identity.id, - nexus_db_model::MulticastGroupMemberState::Left, - ) - .await; - wait_for_member_state( - cptestctx, - group_name, - instance2.identity.id, - nexus_db_model::MulticastGroupMemberState::Left, + &[ + (instance1.identity.id, MulticastGroupMemberState::Left), + (instance2.identity.id, MulticastGroupMemberState::Left), + ], ) .await; @@ -2046,6 +2140,10 @@ async fn test_group_with_all_members_left(cptestctx: &ControlPlaneTestContext) { "Group should remain Active when all members are Left" ); + // With no member "Joined", the reconciler withdraws the MRIB routes + // even though the group itself stays "Active". + assert_mrib_route_absent(cptestctx, group.multicast_ip).await; + // Start one instance again - member should go back to "Joined" let start_url = format!("/v1/instances/left-instance-1/start?project={project_name}"); @@ -2061,13 +2159,22 @@ async fn test_group_with_all_members_left(cptestctx: &ControlPlaneTestContext) { instance_simulate(nexus, &id1).await; instance_wait_for_state(client, id1, InstanceState::Running).await; - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; wait_for_member_state( cptestctx, group_name, instance1.identity.id, - nexus_db_model::MulticastGroupMemberState::Joined, + MulticastGroupMemberState::Joined, + ) + .await; + + // The rejoin restores the any-source wildcard route. + assert_mrib_route_sources( + cptestctx, + group.multicast_ip, + &[None], + None, + "wildcard MRIB route restored after a member rejoins", ) .await; @@ -2081,3 +2188,116 @@ async fn test_group_with_all_members_left(cptestctx: &ControlPlaneTestContext) { .await; wait_for_group_deleted(cptestctx, group_name).await; } + +/// Test that the reconciler sweep reaps an emptied "Active" group on its own. +/// +/// The synchronous mark in the detach path is a best-effort latency +/// optimization, and `cleanup_empty_groups` makes the authoritative emptiness +/// decision. This +/// test soft-deletes the last member row directly in the datastore, bypassing +/// the detach API and its fast-path mark, then verifies a reconciler sweep +/// alone marks the group "Deleting" and removes the dataplane state from DPD. +#[nexus_test] +async fn test_empty_active_group_reaped_by_sweep( + cptestctx: &ControlPlaneTestContext, +) { + ensure_multicast_test_ready(cptestctx).await; + + let client = &cptestctx.external_client; + let project_name = "sweep-reap-project"; + let group_name = "sweep-reap-group"; + let instance_name = "sweep-reap-instance"; + + ops::join3( + create_default_ip_pools(client), + create_project(client, project_name), + create_multicast_ip_pool(client, "sweep-reap-pool"), + ) + .await; + + let instance = instance_for_multicast_groups( + cptestctx, + project_name, + instance_name, + true, + &[], + ) + .await; + + multicast_group_attach(cptestctx, project_name, instance_name, group_name) + .await; + wait_for_group_active(client, group_name).await; + + let nexus = &cptestctx.server.server_context().nexus; + let instance_id = InstanceUuid::from_untyped_uuid(instance.identity.id); + instance_simulate(nexus, &instance_id).await; + instance_wait_for_state(client, instance_id, InstanceState::Running).await; + wait_for_all_members_joined(cptestctx, group_name).await; + + // Capture the underlay multicast IP before deletion so DPD removal can + // be asserted after the group row is gone. + let datastore = nexus.datastore(); + let opctx = + OpContext::for_tests(cptestctx.logctx.log.clone(), datastore.clone()); + let group_view = get_multicast_group(client, group_name).await; + let external_group = datastore + .multicast_group_lookup_by_ip(&opctx, group_view.multicast_ip) + .await + .expect("should lookup external multicast group by IP"); + let underlay_group_id = external_group + .underlay_group_id + .expect("external group should have underlay_group_id"); + let underlay_group = datastore + .underlay_multicast_group_fetch(&opctx, underlay_group_id) + .await + .expect("should fetch underlay multicast group"); + let underlay_multicast_ip = underlay_group.multicast_ip.ip(); + let underlay_v6 = match underlay_multicast_ip { + IpAddr::V6(v6) => v6, + IpAddr::V4(_) => panic!("underlay multicast IP must be IPv6"), + }; + + // While a group is "Active", Nexus programs the handoff to mg-lower: a + // wildcard-source MRIB route targeting the admin-local underlay group on + // every switch's mgd. + assert_mrib_route_sources( + cptestctx, + group_view.multicast_ip, + &[None], + Some(underlay_v6), + "wildcard MRIB route while group is \"Active\"", + ) + .await; + + // Soft-delete the member row directly. The detach API's fast-path mark + // never runs, so only the sweep can observe the group empty. The NOT + // EXISTS guard in `mark_multicast_group_for_removal_if_no_members` + // ignores soft-deleted rows, so no hard-delete pass is required first. + let group_id = + MulticastGroupUuid::from_untyped_uuid(group_view.identity.id); + let member = datastore + .multicast_group_member_get_by_group_and_instance( + &opctx, + group_id, + instance_id, + ) + .await + .expect("should query group member") + .expect("member should exist"); + datastore + .multicast_group_member_delete_by_id(&opctx, member.id) + .await + .expect("should soft-delete member"); + + // One reconciler pass runs `cleanup_empty_groups` (marks "Deleting") + // before `reconcile_deleting_groups` (tears down dataplane state), so + // the assertions below need no extra orchestration: group gone from + // the API, MRIB route withdrawn from every mgd, and the underlay group + // removed from DPD. + wait_for_group_deleted(cptestctx, group_name).await; + assert_mrib_route_absent(cptestctx, group_view.multicast_ip).await; + wait_for_group_deleted_from_dpd(cptestctx, underlay_multicast_ip).await; + + // The instance itself is unaffected by the group reap. + cleanup_instances(cptestctx, client, project_name, &[instance_name]).await; +} diff --git a/nexus/tests/integration_tests/multicast/mod.rs b/nexus/tests/integration_tests/multicast/mod.rs index 64bdd012daa..1445333211a 100644 --- a/nexus/tests/integration_tests/multicast/mod.rs +++ b/nexus/tests/integration_tests/multicast/mod.rs @@ -10,14 +10,24 @@ //! - IP pool setup: `create_multicast_ip_pool`, `create_multicast_ip_pool_with_range` //! - Reconciler control: `wait_for_multicast_reconciler`, `activate_multicast_reconciler` //! - State waiters: `wait_for_group_active`, `wait_for_member_state`, etc. -//! - DPD verification: `verify_inventory_based_port_mapping`, `wait_for_group_deleted_from_dpd` +//! - Dataplane verification: `wait_for_group_deleted_from_dpd`, MRIB route +//! asserts //! - Instance helpers: `instance_for_multicast_groups`, `cleanup_instances` //! - Attach/detach: `multicast_group_attach`, `multicast_group_detach` +//! +//! ## Test boundary +//! +//! The harness runs `ddmd --api-only`, which is only the admin API (no peering +//! daemon). With the Dendrite-stubbed dataplane, there are no real rear-port +//! tfports, so rear-port DDM peering never forms. Coverage stops at the +//! Omicron-owned boundary, which is the group/member state machine, the +//! Nexus-to-mgd MRIB write, and the forwarding and OPTE state pushed to +//! sled-agents. The dataplane tail past that point is Maghemite/Dendrite's +//! domain. use std::future::Future; use std::net::IpAddr; -use std::sync::{Arc, Mutex}; -use std::time::{Duration, Instant}; +use std::time::Duration; use dropshot::test_util::ClientTestContext; use http::{Method, StatusCode}; @@ -41,7 +51,8 @@ use nexus_types::external_api::multicast::{ InstanceMulticastGroupJoin, MulticastGroup, MulticastGroupIdentifier, MulticastGroupJoinSpec, MulticastGroupMember, }; -use nexus_types::identity::{Asset, Resource}; +use nexus_types::identity::Resource; +use nexus_types::internal_api::params::InstanceMigrateRequest; use nexus_types_versions::latest::instance::Instance; use omicron_common::api::external::{ ByteCount, DataPageParams, Hostname, IdentityMetadataCreateParams, @@ -49,7 +60,9 @@ use omicron_common::api::external::{ }; use omicron_nexus::TestInterfaces; use omicron_test_utils::dev::poll::{self, CondCheckError, wait_for_condition}; -use omicron_uuid_kinds::{GenericUuid, InstanceUuid, MulticastGroupUuid}; +use omicron_uuid_kinds::{ + GenericUuid, InstanceUuid, MulticastGroupUuid, SledUuid, +}; use crate::integration_tests::instances as instance_helpers; use sled_agent_client::TestInterfaces as SledAgentTestInterfaces; @@ -60,7 +73,6 @@ pub(crate) type ControlPlaneTestContext = mod api; mod authorization; -mod cache_invalidation; mod enablement; mod failures; mod groups; @@ -73,6 +85,11 @@ const POLL_INTERVAL: Duration = Duration::from_millis(50); const POLL_TIMEOUT: Duration = Duration::from_secs(30); const MULTICAST_OPERATION_TIMEOUT: Duration = Duration::from_secs(120); +// `POLL_TIMEOUT` tracks the 30-second `wait_for_condition` timeout used across +// the nexus integration suite (reconfigurator, quiesce, instances). Pin it so +// this suite does not silently diverge from that convention. +const _: () = assert!(POLL_TIMEOUT.as_secs() == 30); + /// Generic helper for PUT upsert requests that return 201 Created. /// /// Useful for idempotent create-or-update APIs like multicast group join. @@ -246,28 +263,16 @@ pub(crate) async fn activate_multicast_reconciler( .await } -/// Activates the inventory loader and waits for it to complete. +/// Activate the multicast reconciler once, then poll `condition` until it +/// holds (or `timeout` elapses). /// -/// This ensures the watch channel has the latest inventory collection from the database. -pub(crate) async fn activate_inventory_loader( - lockstep_client: &ClientTestContext, -) -> nexus_lockstep_client::types::BackgroundTask { - nexus_test_utils::background::activate_background_task( - lockstep_client, - "inventory_loader", - ) - .await -} - -/// Wait for a condition to be true, activating the reconciler periodically. -/// -/// This is like `wait_for_condition` but activates the multicast reconciler -/// periodically (not on every poll) to drive state changes. We activate the -/// reconciler every 500ms. -/// -/// Useful for tests that need to wait for reconciler-driven state changes -/// (e.g., member state transitions). -pub(crate) async fn wait_for_condition_with_reconciler( +/// This is for tests that expect convergence in a single reconciler pass. +/// We poll after the activation to absorb read-after-write visibility lag +/// (DB commits, sled-agent state propagation), not to wait for further +/// reconciler iterations. If a `condition` only holds after multiple +/// passes, the test writer should orchestrate explicitly, activating per +/// step and asserting intermediate state in between steps. +pub(crate) async fn activate_then_wait_for_condition( lockstep_client: &ClientTestContext, condition: F, poll_interval: &Duration, @@ -277,37 +282,8 @@ where F: Fn() -> Fut, Fut: Future>>, { - // Activate reconciler less frequently than we check the condition - // This reduces overhead while still driving state changes forward - const RECONCILER_ACTIVATION_INTERVAL: Duration = Duration::from_millis(500); - - let last_reconciler_activation = Arc::new(Mutex::new(Instant::now())); - - // First, wait for any already-activated reconciler run to complete. - // This tests explicit activation paths (saga completions, etc.). - wait_for_multicast_reconciler(lockstep_client).await; - - wait_for_condition( - || async { - // Only activate reconciler if enough time has passed - let now = Instant::now(); - let should_activate = { - let last = last_reconciler_activation.lock().unwrap(); - now.duration_since(*last) >= RECONCILER_ACTIVATION_INTERVAL - }; - - if should_activate { - // Use activate to drive progress - activate_multicast_reconciler(lockstep_client).await; - *last_reconciler_activation.lock().unwrap() = now; - } - - condition().await - }, - poll_interval, - timeout, - ) - .await + activate_multicast_reconciler(lockstep_client).await; + wait_for_condition(condition, poll_interval, timeout).await } /// Ensure inventory collection has completed with SP data for all sleds. @@ -315,8 +291,9 @@ where /// This function verifies that inventory has SP data for EVERY in-service sled, /// not just that inventory completed. /// -/// This is required for multicast member operations which map `sled_id` to -/// `sp_slot` to switch ports via inventory. +/// Multicast member operations resolve sled placement and switch zones from +/// inventory. Waiting here keeps tests from racing against early inventory +/// collection before they drive group/member reconciliation. pub(crate) async fn ensure_inventory_ready( cptestctx: &ControlPlaneTestContext, ) { @@ -392,7 +369,7 @@ pub(crate) async fn ensure_inventory_ready( Err(CondCheckError::::NotYet) } }, - &Duration::from_millis(500), + &POLL_INTERVAL, &MULTICAST_OPERATION_TIMEOUT, ) .await @@ -413,14 +390,14 @@ pub(crate) async fn ensure_inventory_ready( /// Ensure multicast test prerequisites are ready. /// -/// This combines inventory collection (for sled → switch port mapping) and -/// DPD readiness (for switch operations) into a single call. Use this at the -/// beginning of multicast tests that will add instances to groups. +/// This combines inventory collection and DPD readiness (for switch +/// operations) into a single call. Use this at the beginning of multicast +/// tests that will add instances to groups. pub(crate) async fn ensure_multicast_test_ready( cptestctx: &ControlPlaneTestContext, ) { - ensure_inventory_ready(cptestctx).await; - ensure_dpd_ready(cptestctx).await; + ops::join2(ensure_inventory_ready(cptestctx), ensure_dpd_ready(cptestctx)) + .await; } /// Ensure DPD (switch infrastructure) is ready and responsive. @@ -455,7 +432,7 @@ pub(crate) async fn ensure_dpd_ready(cptestctx: &ControlPlaneTestContext) { } } }, - &Duration::from_millis(200), + &POLL_INTERVAL, &POLL_TIMEOUT, ) .await @@ -559,11 +536,20 @@ pub(crate) async fn wait_for_group_active( .await } -/// Wait for a specific member to reach the expected state -/// (e.g., Joined, Joining, Left). +/// Wait for a multicast group member to reach the expected state. /// -/// For "Joined" state, this function uses `wait_for_condition_with_reconciler` -/// to ensure the reconciler processes member state transitions. +/// Ensures inventory and DPD are ready, drives one reconciler activation, +/// then asserts the member is observable in `expected_state`. If the state +/// does not match after the pass, fails loudly rather than retrying via +/// reactivation. +/// +/// We poll briefly after the pass to absorb DB read-after-write lag, +/// not to wait for further reconciler iterations. +/// +/// Tests that genuinely need multi-step convergence (e.g., recovery from +/// an injected external failure) must orchestrate explicitly, driving each +/// step with `activate_multicast_reconciler` and assert the intermediate +/// state in between steps. pub(crate) async fn wait_for_member_state( cptestctx: &ControlPlaneTestContext, group_name: &str, @@ -574,98 +560,148 @@ pub(crate) async fn wait_for_member_state( let lockstep_client = &cptestctx.lockstep_client; let expected_state_as_str = expected_state.to_string(); - // For "Joined" state, ensure instance has a sled_id assigned - // (no need to check inventory again since ensure_inventory_ready() already - // verified all sleds have SP data at test setup) + // "Joined" converges from a DB CAS gated on instance validity and sled + // assignment. Reaching it in a single pass needs DPD up (the group's + // Creating→Active saga uses it) and the instance assigned to a sled, so + // wait for both before driving the pass. + // + // "Joining" and "Left" converge from DB-only transitions, so + // don't gate those as failure-mode tests rely on being able to wait + // on them with working DPD stopped. if expected_state == nexus_db_model::MulticastGroupMemberState::Joined { let instance_uuid = InstanceUuid::from_untyped_uuid(instance_id); - wait_for_instance_sled_assignment(cptestctx, &instance_uuid).await; + ops::join2( + ensure_dpd_ready(cptestctx), + wait_for_instance_sled_assignment(cptestctx, &instance_uuid), + ) + .await; } + // Drive one converging pass. This explicit activate guarantees a fresh + // pass runs after this point regardless of whether the API call that + // triggered the test already activated the reconciler. + activate_multicast_reconciler(lockstep_client).await; + + // Verify the post-pass state. Treat read-after-write visibility lag as + // `NotYet`, but treat any other observed state as a permanent failure. let check_member = || async { let members = list_multicast_group_members(client, group_name).await; - - // If we're looking for "Joined" state, we need to ensure the member exists first - // and then wait for the reconciler to process it - if expected_state == nexus_db_model::MulticastGroupMemberState::Joined { - if let Some(member) = - members.iter().find(|m| m.instance_id == instance_id) - { - match member.state.as_str() { - "Joined" => Ok(member.clone()), - "Joining" => { - // Member exists and is in transition - wait a bit more - Err(CondCheckError::NotYet) - } - "Left" => { - // Member in Left state, reconciler needs to process instance start - wait more - Err(CondCheckError::NotYet) - } - other_state => Err(CondCheckError::Failed(format!( - "Member {instance_id} in group {group_name} has unexpected state '{other_state}', expected 'Left', 'Joining' or 'Joined'" - ))), - } - } else { - // Member doesn't exist yet - wait for it to be created - Err(CondCheckError::NotYet) - } - } else { - // For other states, just look for exact match - if let Some(member) = - members.iter().find(|m| m.instance_id == instance_id) - { - if member.state == expected_state_as_str { - Ok(member.clone()) - } else { - Err(CondCheckError::NotYet) - } - } else { - Err(CondCheckError::NotYet) + match members.iter().find(|m| m.instance_id == instance_id) { + Some(member) if member.state == expected_state_as_str => { + Ok(member.clone()) } + Some(member) => Err(CondCheckError::Failed(format!( + "member {instance_id} in group {group_name} reached state \ + '{}' after one reconciler pass, expected '{expected_state_as_str}'", + member.state + ))), + None => Err(CondCheckError::NotYet), } }; - // Use reconciler-activating wait for "Joined" state - let res = if expected_state - == nexus_db_model::MulticastGroupMemberState::Joined + match wait_for_condition( + check_member, + &POLL_INTERVAL, + &MULTICAST_OPERATION_TIMEOUT, + ) + .await { - wait_for_condition_with_reconciler( - lockstep_client, - check_member, - &POLL_INTERVAL, - &MULTICAST_OPERATION_TIMEOUT, - ) - .await - } else { - wait_for_condition( - check_member, - &POLL_INTERVAL, - &MULTICAST_OPERATION_TIMEOUT, - ) - .await - }; - - match res { Ok(member) => member, Err(poll::Error::TimedOut(elapsed)) => { panic!( - "member {instance_id} in group {group_name} did not reach state '{expected_state_as_str}' within {elapsed:?}", + "member {instance_id} in group {group_name} did not appear within {elapsed:?}", ); } Err(poll::Error::PermanentError(err)) => { panic!( - "failed waiting for member {instance_id} in group {group_name} to reach state '{expected_state_as_str}': {err:?}", + "reconciler did not converge member {instance_id} in group \ + {group_name} to '{expected_state_as_str}': {err}", ); } } } -/// Wait for an instance to have a sled_id assigned. +/// Wait for a batch of multicast group members to reach their respective +/// expected states after a single reconciler pass. +/// +/// Like [`wait_for_member_state`] but checks multiple members after +/// one shared reconciler pass. Panics if any member ends up in an +/// unexpected state. +pub(crate) async fn wait_for_members_state( + cptestctx: &ControlPlaneTestContext, + group_name: &str, + expected: &[(Uuid, nexus_db_model::MulticastGroupMemberState)], +) -> Vec { + let client = &cptestctx.external_client; + let lockstep_client = &cptestctx.lockstep_client; + + let joined_instances: Vec = expected + .iter() + .filter(|(_, state)| { + *state == nexus_db_model::MulticastGroupMemberState::Joined + }) + .map(|(id, _)| InstanceUuid::from_untyped_uuid(*id)) + .collect(); + + if !joined_instances.is_empty() { + let assign_waits = joined_instances + .iter() + .map(|uuid| wait_for_instance_sled_assignment(cptestctx, uuid)); + ops::join2(ensure_dpd_ready(cptestctx), ops::join_all(assign_waits)) + .await; + } + + activate_multicast_reconciler(lockstep_client).await; + + let check = || async { + let members = list_multicast_group_members(client, group_name).await; + let mut resolved = Vec::with_capacity(expected.len()); + for (instance_id, expected_state) in expected { + let expected_str = expected_state.to_string(); + match members.iter().find(|m| m.instance_id == *instance_id) { + Some(member) if member.state == expected_str => { + resolved.push(member.clone()); + } + Some(member) => { + return Err(CondCheckError::Failed(format!( + "member {instance_id} in group {group_name} reached \ + state '{}' after one reconciler pass, expected \ + '{expected_str}'", + member.state + ))); + } + None => return Err(CondCheckError::NotYet), + } + } + Ok(resolved) + }; + + match wait_for_condition( + check, + &POLL_INTERVAL, + &MULTICAST_OPERATION_TIMEOUT, + ) + .await + { + Ok(members) => members, + Err(poll::Error::TimedOut(elapsed)) => panic!( + "members in group {group_name} did not all appear within \ + {elapsed:?} (expected {expected:?})", + ), + Err(poll::Error::PermanentError(err)) => panic!( + "reconciler did not converge members in group {group_name} \ + (expected {expected:?}): {err}", + ), + } +} + +/// Wait for an instance to have a `sled_id` assigned. /// /// This is a stricter check than `instance_wait_for_vmm_registration` - it ensures /// that not only does the VMM exist and is not in "Creating" state, but also that /// the VMM has been assigned to a specific sled. This is critical for multicast -/// member join operations which need the sled_id to program switch ports. +/// member join operations, which need the `sled_id` to subscribe the VMM's OPTE +/// port and propagate forwarding state. pub(crate) async fn wait_for_instance_sled_assignment( cptestctx: &ControlPlaneTestContext, instance_id: &InstanceUuid, @@ -892,138 +928,6 @@ pub(crate) async fn wait_for_instance_stopped( } } -/// Verify that inventory-based sled-to-switch-port mapping is correct. -/// -/// This validates the entire flow: -/// instance → sled → inventory → sp_slot → rear{N} → DPD underlay member -pub(crate) async fn verify_inventory_based_port_mapping( - cptestctx: &ControlPlaneTestContext, - instance_uuid: &InstanceUuid, -) -> Result<(), String> { - let nexus = &cptestctx.server.server_context().nexus; - let datastore = nexus.datastore(); - let opctx = - OpContext::for_tests(cptestctx.logctx.log.clone(), datastore.clone()); - - // Get sled_id for the running instance - let sled_id = nexus - .active_instance_info(instance_uuid, None) - .await - .map_err(|e| format!("active_instance_info failed: {e}"))? - .ok_or_else(|| "instance not on a sled".to_string())? - .sled_id; - - // Get the multicast member for this instance to find its external_group_id - let members = datastore - .multicast_group_members_list_by_instance( - &opctx, - *instance_uuid, - &DataPageParams::max_page(), - ) - .await - .map_err(|e| format!("list members failed: {e}"))?; - - let member = members - .first() - .ok_or_else(|| "no multicast membership found".to_string())?; - - let external_group_id = member.external_group_id; - - // Fetch the external multicast group to get underlay_group_id - let external_group = datastore - .multicast_group_fetch( - &opctx, - MulticastGroupUuid::from_untyped_uuid(external_group_id), - ) - .await - .map_err(|e| format!("fetch external group failed: {e}"))?; - - let underlay_group_id = external_group - .underlay_group_id - .ok_or_else(|| "external group has no underlay_group_id".to_string())?; - - // Fetch the underlay group to get its multicast IP - let underlay_group = datastore - .underlay_multicast_group_fetch(&opctx, underlay_group_id) - .await - .map_err(|e| format!("fetch underlay group failed: {e}"))?; - - let underlay_multicast_ip = underlay_group.multicast_ip.ip(); - - // Fetch latest inventory collection - let inventory = datastore - .inventory_get_latest_collection(&opctx) - .await - .map_err(|e| format!("fetch inventory failed: {e}"))? - .ok_or_else(|| "no inventory collection".to_string())?; - - // Get the sled record to find its baseboard info - let sleds = datastore - .sled_list_all_batched(&opctx, SledFilter::InService) - .await - .map_err(|e| format!("list sleds failed: {e}"))?; - let sled = sleds - .into_iter() - .find(|s| s.id() == sled_id) - .ok_or_else(|| "sled not found".to_string())?; - - // Find SP for this sled using baseboard matching (serial + part number) - let sp = inventory - .sps - .iter() - .find(|(bb, _)| { - bb.serial_number == sled.serial_number() - && bb.part_number == sled.part_number() - }) - .or_else(|| { - // Fallback to serial-only match if exact match not found - inventory - .sps - .iter() - .find(|(bb, _)| bb.serial_number == sled.serial_number()) - }) - .map(|(_, sp)| sp) - .ok_or_else(|| "SP not found for sled".to_string())?; - - let expected_rear_port = sp.sp_slot; - - // Fetch DPD underlay group configuration using the underlay multicast IP - let dpd_client = nexus_test_utils::dpd_client(cptestctx); - let underlay_group_response = dpd_client - .multicast_group_get(&underlay_multicast_ip) - .await - .map_err(|e| format!("DPD query failed: {e}"))? - .into_inner(); - - // Extract underlay members from the response - let members = match underlay_group_response { - dpd_client::types::MulticastGroupResponse::Underlay { - members, .. - } => members, - dpd_client::types::MulticastGroupResponse::External { .. } => { - return Err("Expected Underlay group, got External".to_string()); - } - }; - - // Construct the expected `PortId` for comparison - let expected_port_id = dpd_client::types::PortId::Rear( - dpd_client::types::Rear::try_from(format!("rear{expected_rear_port}")) - .map_err(|e| format!("invalid rear port: {e}"))?, - ); - - // Check if DPD has an underlay member with the expected rear port - let has_expected_member = members.iter().any(|m| { - matches!(m.direction, dpd_client::types::Direction::Underlay) - && m.port_id == expected_port_id - }); - - if has_expected_member { - Ok(()) - } else { - Err(format!("DPD does not have member on rear{expected_rear_port}")) - } -} - /// Wait for a multicast group to have a specific number of members. /// /// Note: For expected_count=0 (last member removed), use `wait_for_group_deleted` @@ -1062,7 +966,11 @@ pub(crate) async fn wait_for_member_count( } } -/// Wait for a multicast group to be deleted (returns 404). +/// Wait for a multicast group to be fully deleted (returns 404). +/// +/// Drives one reconciler activation for groups in "Deleting" state and +/// asserts the group is removed via the public list endpoint. Polling +/// around the API check is only for read-after-write visibility. pub(crate) async fn wait_for_group_deleted( cptestctx: &ControlPlaneTestContext, group_name: &str, @@ -1070,22 +978,25 @@ pub(crate) async fn wait_for_group_deleted( let client = &cptestctx.external_client; let lockstep_client = &cptestctx.lockstep_client; - match wait_for_condition_with_reconciler( - lockstep_client, - || async { - let group_url = mcast_group_url(group_name); - let response = NexusRequest::new( - RequestBuilder::new(client, Method::GET, &group_url) - .expect_status(Some(StatusCode::NOT_FOUND)), - ) - .authn_as(AuthnMode::PrivilegedUser) - .execute() - .await; - match response { - Ok(_) => Ok(()), - Err(_) => Err(CondCheckError::<()>::NotYet), - } - }, + activate_multicast_reconciler(lockstep_client).await; + + let check = || async { + let group_url = mcast_group_url(group_name); + let response = NexusRequest::new( + RequestBuilder::new(client, Method::GET, &group_url) + .expect_status(Some(StatusCode::NOT_FOUND)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await; + match response { + Ok(_) => Ok(()), + Err(_) => Err(CondCheckError::<()>::NotYet), + } + }; + + match wait_for_condition( + check, &POLL_INTERVAL, &MULTICAST_OPERATION_TIMEOUT, ) @@ -1093,7 +1004,10 @@ pub(crate) async fn wait_for_group_deleted( { Ok(_) => {} Err(poll::Error::TimedOut(elapsed)) => { - panic!("group {group_name} was not deleted within {elapsed:?}",); + panic!( + "group {group_name} was not deleted within {elapsed:?} after \ + one reconciler pass", + ); } Err(poll::Error::PermanentError(err)) => { panic!( @@ -1103,10 +1017,12 @@ pub(crate) async fn wait_for_group_deleted( } } -/// Wait for a multicast group to be deleted from DPD (dataplane) with reconciler activation. +/// Wait for a multicast group to be removed from DPD (dataplane). /// -/// This function waits for the DPD to report that the multicast group no longer exists -/// (returns 404), while periodically activating the reconciler to drive the cleanup process. +/// Drives one reconciler activation and asserts the DPD multicast group +/// GET returns 404 for the underlay IP (the dataplane cleanup that the +/// "Deleting" → removal transition performs). Polling around the DPD +/// GET is only for read-after-write visibility. pub(crate) async fn wait_for_group_deleted_from_dpd( cptestctx: &ControlPlaneTestContext, multicast_ip: std::net::IpAddr, @@ -1114,17 +1030,17 @@ pub(crate) async fn wait_for_group_deleted_from_dpd( let lockstep_client = &cptestctx.lockstep_client; let dpd_client = nexus_test_utils::dpd_client(cptestctx); - match wait_for_condition_with_reconciler( - lockstep_client, - || async { - match dpd_client.multicast_group_get(&multicast_ip).await { - Ok(_) => { - // Group still exists in DPD - not yet deleted - Err(CondCheckError::<()>::NotYet) - } - Err(_) => Ok(()), // Group doesn't exist - deleted - } - }, + activate_multicast_reconciler(lockstep_client).await; + + let check = || async { + match dpd_client.multicast_group_get(&multicast_ip).await { + Ok(_) => Err(CondCheckError::<()>::NotYet), + Err(_) => Ok(()), + } + }; + + match wait_for_condition( + check, &POLL_INTERVAL, &MULTICAST_OPERATION_TIMEOUT, ) @@ -1133,7 +1049,8 @@ pub(crate) async fn wait_for_group_deleted_from_dpd( Ok(_) => {} Err(poll::Error::TimedOut(elapsed)) => { panic!( - "group with IP {multicast_ip} was not deleted from DPD within {elapsed:?}", + "group with IP {multicast_ip} was not deleted from DPD within \ + {elapsed:?} after one reconciler pass", ); } Err(poll::Error::PermanentError(err)) => { @@ -1144,6 +1061,31 @@ pub(crate) async fn wait_for_group_deleted_from_dpd( } } +/// Wait for every current member of a group to reach "Joined" after a +/// single reconciler pass. +/// +/// Lists the group's members and delegates to [`wait_for_members_state`], +/// which also waits on DPD readiness and sled assignment for "Joined" +/// members. +/// +/// Vacuous for a group with no members: the listed set is empty, so this +/// returns immediately. Callers must establish membership first. +pub(crate) async fn wait_for_all_members_joined( + cptestctx: &ControlPlaneTestContext, + group_name: &str, +) { + let members = + list_multicast_group_members(&cptestctx.external_client, group_name) + .await; + let expected: Vec<_> = members + .iter() + .map(|m| { + (m.instance_id, nexus_db_model::MulticastGroupMemberState::Joined) + }) + .collect(); + wait_for_members_state(cptestctx, group_name, &expected).await; +} + /// Create an instance with multicast groups. pub(crate) async fn instance_for_multicast_groups( cptestctx: &ControlPlaneTestContext, @@ -1152,11 +1094,14 @@ pub(crate) async fn instance_for_multicast_groups( start: bool, multicast_group_names: &[&str], ) -> Instance { - // Ensure inventory and DPD are ready before creating instances with multicast groups - // Inventory is needed for sled → switch port mapping, DPD for switch operations + // Ensure inventory and DPD are ready before creating instances with + // multicast groups. if !multicast_group_names.is_empty() { - ensure_inventory_ready(cptestctx).await; - ensure_dpd_ready(cptestctx).await; + ops::join2( + ensure_inventory_ready(cptestctx), + ensure_dpd_ready(cptestctx), + ) + .await; } let client = &cptestctx.external_client; @@ -1352,6 +1297,187 @@ pub(crate) async fn cleanup_instances( ops::join_all(delete_futures).await; } +/// Wait until each listed member's stored `sled_id` matches the expected +/// post-migration sled. +/// +/// [`wait_for_member_state`] for "Joined" is satisfied as soon as the +/// member is in "Joined", which can happen with the *pre-migration* +/// `sled_id` still recorded if the reconciler has not yet (re-)observed +/// the new active VMM. +/// +/// This tests that reading member placement after migration must wait +/// until the DB row reflects the new sled. +/// +/// Here, we drive one reconciler activation. The members reconciler +/// detects the `member.sled_id != live_vmm.sled_id` skew, updates the DB +/// placement, propagates M2P/forwarding, and subscribes the new VMM's OPTE +/// port via sled-agent. Old-sled OPTE state is reclaimed by VMM teardown, and +/// rear-port underlay membership is `ddmd`-owned. The row is settled by the +/// time this returns. Polling around the read is only for read-after-write +/// visibility. +pub(crate) async fn wait_for_member_sled_ids( + cptestctx: &ControlPlaneTestContext, + group_name: &str, + expected: &[(Uuid, SledUuid)], +) { + let lockstep_client = &cptestctx.lockstep_client; + let nexus = &cptestctx.server.server_context().nexus; + let datastore = nexus.datastore(); + let opctx = + OpContext::for_tests(cptestctx.logctx.log.clone(), datastore.clone()); + + let group_id = { + let group = + get_multicast_group(&cptestctx.external_client, group_name).await; + group.identity.id + }; + + activate_multicast_reconciler(lockstep_client).await; + + let check = || async { + let members = datastore + .multicast_group_members_list( + &opctx, + MulticastGroupUuid::from_untyped_uuid(group_id), + &DataPageParams::max_page(), + ) + .await + .map_err(|e| { + CondCheckError::Failed(format!("list members failed: {e}")) + })?; + + for (instance_id, expected_sled) in expected { + let member = members + .iter() + .find(|m| m.parent_id == *instance_id) + .ok_or(CondCheckError::NotYet)?; + let sled_id = member.sled_id.ok_or(CondCheckError::NotYet)?; + if sled_id.into_untyped_uuid() != expected_sled.into_untyped_uuid() + { + return Err(CondCheckError::Failed(format!( + "member for instance {instance_id} reached sled_id \ + {sled_id:?} after one reconciler pass, expected \ + {expected_sled:?}" + ))); + } + } + Ok::<_, CondCheckError>(()) + }; + + wait_for_condition(check, &POLL_INTERVAL, &MULTICAST_OPERATION_TIMEOUT) + .await + .unwrap_or_else(|e| { + panic!( + "members in group {group_name} did not reach expected sled \ + assignments {expected:?}: {e:?}" + ) + }); +} + +/// Migrate an instance to a specific target sled. +/// +/// Noop if the instance is already on `target_sled`. Otherwise drives +/// the standard `request → simulate-source → simulate-target` sequence +/// used by other integration tests, returning when the instance is +/// "Running" on `target_sled`. +pub(crate) async fn migrate_instance_to( + cptestctx: &ControlPlaneTestContext, + instance_id: InstanceUuid, + target_sled: SledUuid, +) { + let client = &cptestctx.external_client; + let lockstep_client = &cptestctx.lockstep_client; + let nexus = &cptestctx.server.server_context().nexus; + + let info = nexus + .active_instance_info(&instance_id, None) + .await + .unwrap() + .expect("instance should be on a sled"); + if info.sled_id == target_sled { + return; + } + let source_sled = info.sled_id; + + let migrate_url = format!("/instances/{instance_id}/migrate"); + NexusRequest::new( + RequestBuilder::new(lockstep_client, Method::POST, &migrate_url) + .body(Some(&InstanceMigrateRequest { dst_sled_id: target_sled })) + .expect_status(Some(StatusCode::OK)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("should initiate migration"); + + let info = + nexus.active_instance_info(&instance_id, None).await.unwrap().unwrap(); + let src_propolis = info.propolis_id; + let dst_propolis = info.dst_propolis_id.unwrap(); + + instance_helpers::vmm_simulate_on_sled( + cptestctx, + nexus, + source_sled, + src_propolis, + ) + .await; + instance_helpers::instance_wait_for_state( + client, + instance_id, + InstanceState::Migrating, + ) + .await; + + instance_helpers::vmm_simulate_on_sled( + cptestctx, + nexus, + target_sled, + dst_propolis, + ) + .await; + instance_helpers::instance_wait_for_state( + client, + instance_id, + InstanceState::Running, + ) + .await; +} + +/// Resolve the underlay admin-local IPv6 address for a multicast group +/// given its external multicast IP. +pub(crate) async fn fetch_underlay_admin_ip( + cptestctx: &ControlPlaneTestContext, + external_multicast_ip: IpAddr, +) -> dpd_client::types::UnderlayMulticastIpv6 { + let nexus = &cptestctx.server.server_context().nexus; + let datastore = nexus.datastore(); + let opctx = + OpContext::for_tests(cptestctx.logctx.log.clone(), datastore.clone()); + + let external_group = datastore + .multicast_group_lookup_by_ip(&opctx, external_multicast_ip) + .await + .expect("should look up external multicast group by IP"); + let underlay_group_id = external_group + .underlay_group_id + .expect("external group should have underlay_group_id"); + let underlay_group = datastore + .underlay_multicast_group_fetch(&opctx, underlay_group_id) + .await + .expect("should fetch underlay multicast group"); + + match underlay_group.multicast_ip.ip() { + IpAddr::V6(v6) => { + dpd_client::types::UnderlayMulticastIpv6::try_from(v6) + .expect("underlay IP should be admin-local IPv6") + } + IpAddr::V4(other) => { + panic!("expected IPv6 underlay address, got {other}") + } + } +} + /// Stop multiple instances, poking the simulated sled-agent while waiting. pub(crate) async fn stop_instances( cptestctx: &ControlPlaneTestContext, @@ -1451,8 +1577,8 @@ pub(crate) async fn multicast_group_attach_bulk( group_name: &str, ) { // Check inventory and DPD readiness once for all attachments - ensure_inventory_ready(cptestctx).await; - ensure_dpd_ready(cptestctx).await; + ops::join2(ensure_inventory_ready(cptestctx), ensure_dpd_ready(cptestctx)) + .await; let attach_futures = instance_names.iter().map(|instance_name| { multicast_group_attach( @@ -1538,3 +1664,252 @@ pub(crate) mod ops { tokio::join!(op1, op2, op3, op4) } } + +/// Assert that *every* MGD in the fixture has an MRIB route for `group_ip`. +/// +/// Iterates every switch zone present in `cptestctx.mgd`, so multi-switch +/// fixtures (`extra_sled_agents > 0`) catch a route that is programmed only +/// on a subset of switches. +pub(crate) async fn assert_mrib_route_exists( + cptestctx: &nexus_test_utils::ControlPlaneTestContext< + omicron_nexus::Server, + >, + group_ip: IpAddr, +) { + for_each_mgd(cptestctx, |slot, mgd_client| async move { + wait_for_condition::<_, (), _, _>( + || async { + // A transient mgd error is not-yet, not a panic. + let routes = match mgd_client.static_list_mcast_routes().await { + Ok(routes) => routes.into_inner(), + Err(_) => return Err(CondCheckError::NotYet), + }; + if routes + .iter() + .any(|r| mrib_route_matches_group(&r.key, group_ip)) + { + Ok(()) + } else { + Err(CondCheckError::NotYet) + } + }, + &POLL_INTERVAL, + &MULTICAST_OPERATION_TIMEOUT, + ) + .await + .unwrap_or_else(|e| { + panic!("mgd on {slot:?} never had a route for {group_ip}: {e:?}") + }); + }) + .await; +} + +/// Assert that *no* MGD in the fixture has an MRIB route for `group_ip`. +pub(crate) async fn assert_mrib_route_absent( + cptestctx: &nexus_test_utils::ControlPlaneTestContext< + omicron_nexus::Server, + >, + group_ip: IpAddr, +) { + for_each_mgd(cptestctx, |slot, mgd_client| async move { + wait_for_condition::<_, (), _, _>( + || async { + // A transient mgd error is not-yet, not a panic. + let routes = match mgd_client.static_list_mcast_routes().await { + Ok(routes) => routes.into_inner(), + Err(_) => return Err(CondCheckError::NotYet), + }; + if routes + .iter() + .any(|r| mrib_route_matches_group(&r.key, group_ip)) + { + Err(CondCheckError::NotYet) + } else { + Ok(()) + } + }, + &POLL_INTERVAL, + &MULTICAST_OPERATION_TIMEOUT, + ) + .await + .unwrap_or_else(|e| { + panic!("mgd on {slot:?} still had a route for {group_ip}: {e:?}") + }); + }) + .await; +} + +/// Assert that every MGD in the fixture converges to exactly +/// `expected_sources` for `group_ip`. +/// +/// `None` in `expected_sources` is the any-source wildcard route. The +/// reconciler installs one MRIB route per specific source plus the wildcard +/// when any member is any-source, so this is strictly finer grained than +/// [`assert_mrib_route_exists`]. An empty `expected_sources` is equivalent +/// to [`assert_mrib_route_absent`]. +/// +/// All routes for the group must target a single admin-local (ff04::/16) +/// underlay group, and `expected_underlay` when provided. +/// +/// Routes only exist for groups with at least one "Joined" member, so +/// callers asserting a non-empty set must first converge membership (e.g., +/// via [`wait_for_member_state`] or [`wait_for_all_members_joined`]). +/// Polling absorbs read-after-write visibility lag from a pass the caller +/// already drove. This does not activate the reconciler itself. +pub(crate) async fn assert_mrib_route_sources( + cptestctx: &nexus_test_utils::ControlPlaneTestContext< + omicron_nexus::Server, + >, + group_ip: IpAddr, + expected_sources: &[Option], + expected_underlay: Option, + msg: &str, +) { + let expected: std::collections::BTreeSet> = + expected_sources.iter().copied().collect(); + let expected = &expected; + // Each switch's converged underlay target, collected so cross-switch + // agreement can be asserted after the per-switch loop. + let underlays = std::sync::Mutex::new(std::collections::BTreeSet::< + std::net::Ipv6Addr, + >::new()); + let underlays = &underlays; + for_each_mgd(cptestctx, |slot, mgd_client| async move { + let underlay = wait_for_condition::<_, (), _, _>( + || async { + // A transient mgd error is not-yet, not a panic. + let routes = match mgd_client.static_list_mcast_routes().await { + Ok(routes) => routes.into_inner(), + Err(_) => return Err(CondCheckError::NotYet), + }; + let group_routes: Vec<_> = routes + .iter() + .filter(|r| mrib_route_matches_group(&r.key, group_ip)) + .collect(); + let sources: std::collections::BTreeSet> = + group_routes + .iter() + .map(|r| mrib_route_source(&r.key)) + .collect(); + if sources != *expected { + return Err(CondCheckError::NotYet); + } + for route in &group_routes { + if route.underlay_group.segments()[0] != 0xff04 { + return Err(CondCheckError::NotYet); + } + if expected_underlay + .is_some_and(|exp| exp != route.underlay_group) + { + return Err(CondCheckError::NotYet); + } + if route.underlay_group != group_routes[0].underlay_group { + return Err(CondCheckError::NotYet); + } + } + Ok(group_routes.first().map(|r| r.underlay_group)) + }, + &POLL_INTERVAL, + &MULTICAST_OPERATION_TIMEOUT, + ) + .await + .unwrap_or_else(|e| { + panic!( + "{msg}: mgd on {slot:?} never converged to MRIB sources \ + {expected_sources:?} (underlay {expected_underlay:?}) for \ + {group_ip}: {e:?}" + ) + }); + if let Some(underlay) = underlay { + underlays.lock().unwrap().insert(underlay); + } + }) + .await; + + // All switches must agree on a single underlay target. Per-switch + // checks above only enforce internal consistency. + if !expected.is_empty() { + let underlays = underlays.lock().unwrap(); + assert_eq!( + underlays.len(), + 1, + "{msg}: switches disagree on the underlay group for {group_ip}: \ + {underlays:?}", + ); + } +} + +/// Assert the ddm-peers lockstep view is reachable. +/// +/// The harness runs `ddmd --api-only` over a stubbed dataplane, so no rear-port +/// peers form and the returned set is empty. Fetching it still exercises the +/// full read path that backs `omdb nexus multicast ddm-peers`: Nexus +/// authorizes, builds a DDM client per switch zone, queries each `ddmd` admin +/// API, and aggregates the view. +async fn assert_ddm_peers_view_reachable(cptestctx: &ControlPlaneTestContext) { + let url = format!("http://{}", cptestctx.lockstep_client.bind_address); + let client = + nexus_lockstep_client::Client::new(&url, cptestctx.logctx.log.clone()); + client + .multicast_ddm_peers() + .await + .expect("should fetch ddm-peers view from lockstep API"); +} + +fn mrib_route_source( + key: &mg_admin_client::types::MulticastRouteKey, +) -> Option { + match key { + mg_admin_client::types::MulticastRouteKey::V4(k) => { + k.source.map(IpAddr::V4) + } + mg_admin_client::types::MulticastRouteKey::V6(k) => { + k.source.map(IpAddr::V6) + } + } +} + +/// Run a function `f` against every MGD client in the fixture, in `SwitchSlot` +/// order. +async fn for_each_mgd( + cptestctx: &nexus_test_utils::ControlPlaneTestContext< + omicron_nexus::Server, + >, + f: F, +) where + F: Fn( + sled_agent_types::early_networking::SwitchSlot, + mg_admin_client::Client, + ) -> Fut, + Fut: Future, +{ + assert!( + !cptestctx.mgd.is_empty(), + "multicast MRIB assertions require at least one MGD in the test \ + fixture", + ); + let switches: std::collections::BTreeMap<_, _> = + cptestctx.mgd.iter().collect(); + for (slot, mgd) in switches { + let mgd_client = mg_admin_client::Client::new( + &format!("http://[::1]:{}", mgd.port), + cptestctx.logctx.log.clone(), + ); + f(*slot, mgd_client).await; + } +} + +fn mrib_route_matches_group( + key: &mg_admin_client::types::MulticastRouteKey, + group_ip: IpAddr, +) -> bool { + match (key, group_ip) { + (mg_admin_client::types::MulticastRouteKey::V4(k), IpAddr::V4(ip)) => { + k.group == ip + } + (mg_admin_client::types::MulticastRouteKey::V6(k), IpAddr::V6(ip)) => { + k.group == ip + } + _ => false, + } +} diff --git a/nexus/tests/integration_tests/multicast/networking_integration.rs b/nexus/tests/integration_tests/multicast/networking_integration.rs index 0fe2477e52f..a5c29afe5af 100644 --- a/nexus/tests/integration_tests/multicast/networking_integration.rs +++ b/nexus/tests/integration_tests/multicast/networking_integration.rs @@ -115,8 +115,6 @@ async fn test_multicast_external_ip_scenarios( instance_wait_for_running_with_simulation(cptestctx, instance_uuid) .await; - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - // Add instance to multicast group via instance-centric API multicast_group_attach( cptestctx, @@ -187,9 +185,6 @@ async fn test_multicast_external_ip_scenarios( ); object_delete(client, &external_ip_detach_url).await; - // Wait for operations to settle - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - // Verify multicast membership is still intact after external IP removal let members_after_detach = list_multicast_group_members(client, group_name).await; @@ -262,8 +257,6 @@ async fn test_multicast_external_ip_scenarios( instance_wait_for_running_with_simulation(cptestctx, instance_uuid) .await; - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - // Add instance to multicast group via instance-centric API multicast_group_attach( cptestctx, @@ -309,9 +302,6 @@ async fn test_multicast_external_ip_scenarios( .await .unwrap(); - // Wait for dataplane configuration to settle - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - // Verify multicast state is preserved let members_with_ip = list_multicast_group_members(client, group_name).await; @@ -343,9 +333,6 @@ async fn test_multicast_external_ip_scenarios( ); object_delete(client, &external_ip_detach_url).await; - // Wait for operations to settle - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - // Verify multicast state is still preserved let members_without_ip = list_multicast_group_members(client, group_name).await; @@ -426,8 +413,6 @@ async fn test_multicast_external_ip_scenarios( instance_wait_for_running_with_simulation(cptestctx, instance_uuid) .await; - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - // Verify external IP was allocated at creation let external_ips_after_start = fetch_instance_external_ips(client, instance_name, project_name) @@ -546,7 +531,6 @@ async fn test_multicast_with_floating_ip_basic( let instance_id = instance.identity.id; let instance_uuid = InstanceUuid::from_untyped_uuid(instance_id); - wait_for_instance_sled_assignment(cptestctx, &instance_uuid).await; instance_wait_for_running_with_simulation(cptestctx, instance_uuid).await; // Ensure multicast test prerequisites (inventory + DPD) are ready @@ -555,24 +539,7 @@ async fn test_multicast_with_floating_ip_basic( // Add instance to multicast group via instance-centric API multicast_group_attach(cptestctx, project_name, instance_name, group_name) .await; - // Group activation is reconciler-driven; explicitly drive it to avoid flakes. - wait_for_condition_with_reconciler( - &cptestctx.lockstep_client, - || async { - let group = get_multicast_group(client, group_name).await; - if group.state == "Active" { - Ok(()) - } else { - Err(CondCheckError::::NotYet) - } - }, - &POLL_INTERVAL, - &MULTICAST_OPERATION_TIMEOUT, - ) - .await - .unwrap_or_else(|e| { - panic!("group {group_name} did not reach Active state in time: {e:?}") - }); + wait_for_group_active(client, group_name).await; // Wait for multicast member to reach "Joined" state wait_for_member_state( @@ -587,10 +554,11 @@ async fn test_multicast_with_floating_ip_basic( let members = list_multicast_group_members(client, group_name).await; assert_eq!(members.len(), 1, "Should have one multicast member"); - // Verify that inventory-based mapping correctly mapped sled → switch port - verify_inventory_based_port_mapping(cptestctx, &instance_uuid) - .await - .expect("Port mapping verification should succeed"); + // The Nexus-owned dataplane handoff: an MRIB route in mgd for the + // group. Per-member rear-port programming in DPD is `ddmd`'s job from + // DDM peer subscriptions and is not asserted in omicron tests. + let group_view = get_multicast_group(client, group_name).await; + assert_mrib_route_exists(cptestctx, group_view.multicast_ip).await; // Attach floating IP to the same instance let attach_url = format!( @@ -780,14 +748,11 @@ async fn test_multicast_sled_agent_m2p_and_subscriptions( let instance_id = InstanceUuid::from_untyped_uuid(instance.identity.id); instance_wait_for_running_with_simulation(cptestctx, instance_id).await; - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - // Attach instance to a multicast group. multicast_group_attach(cptestctx, project_name, instance_name, group_name) .await; wait_for_group_active(client, group_name).await; - // Wait for the member to reach "Joined" state (reconciler processes it). wait_for_member_state( cptestctx, group_name, @@ -824,6 +789,9 @@ async fn test_multicast_sled_agent_m2p_and_subscriptions( other => panic!("Expected IPv6 underlay address, got {other}"), }; + // Verify MRIB route was programmed on mgd. + assert_mrib_route_exists(cptestctx, multicast_ip).await; + // Verify M2P mapping on the sim sled-agent. let sled_agent = cptestctx.first_sled_agent(); { @@ -854,21 +822,25 @@ async fn test_multicast_sled_agent_m2p_and_subscriptions( // Verify per-VMM multicast subscription on the sim sled-agent. { - let info = nexus + let propolis_id = cptestctx + .server + .server_context() + .nexus .active_instance_info(&instance_id, None) .await .unwrap() - .expect("Running instance should have active info"); + .expect("instance should have an active VMM") + .propolis_id; - let groups = sled_agent.multicast_groups.lock().unwrap(); + let groups = sled_agent.instance_multicast_groups.lock().unwrap(); let vmm_groups = groups - .get(&info.propolis_id) - .expect("Sled-agent should have multicast groups for propolis"); + .get(&propolis_id) + .expect("Sled-agent should have multicast groups for VMM"); assert!( vmm_groups.iter().any(|m| m.group_ip == multicast_ip), - "VMM should be subscribed to multicast group {multicast_ip}, \ - got: {vmm_groups:?}" + "VMM should be subscribed to multicast group \ + {multicast_ip}, got: {vmm_groups:?}" ); } @@ -907,7 +879,7 @@ async fn test_multicast_sled_agent_m2p_and_subscriptions( // M2P and forwarding should be cleared since there are no "Joined" // members remaining. - wait_for_condition_with_reconciler( + activate_then_wait_for_condition( &cptestctx.lockstep_client, || async { let m2p = sled_agent.m2p_mappings.lock().unwrap(); @@ -924,7 +896,7 @@ async fn test_multicast_sled_agent_m2p_and_subscriptions( .expect("M2P should be cleared when no Joined members remain"); // Forwarding should also be cleared when no "Joined" members remain. - wait_for_condition_with_reconciler( + activate_then_wait_for_condition( &cptestctx.lockstep_client, || async { let fwd = sled_agent.mcast_fwd.lock().unwrap(); @@ -960,6 +932,9 @@ async fn test_multicast_sled_agent_m2p_and_subscriptions( got: {fwd:?}" ); } + + // Verify MRIB route was withdrawn after group deletion. + assert_mrib_route_absent(cptestctx, multicast_ip).await; } /// Verify M2P and forwarding entries propagate to all sleds, not just the @@ -1013,8 +988,6 @@ async fn test_multicast_multi_sled_m2p_propagation( let instance_id = InstanceUuid::from_untyped_uuid(instance.identity.id); instance_wait_for_running_with_simulation(cptestctx, instance_id).await; - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - // Attach to a multicast group. multicast_group_attach(cptestctx, project_name, instance_name, group_name) .await; @@ -1064,21 +1037,29 @@ async fn test_multicast_multi_sled_m2p_propagation( let hosting_sled_id = info.sled_id; + // Verify MRIB route was programmed. + assert_mrib_route_exists(cptestctx, multicast_ip).await; + // M2P and forwarding are pushed to all sleds (like V2P). Any // instance on any sled may send to a multicast group; without the // M2P mapping OPTE's overlay layer silently drops the packet. // Forwarding entries let sender sleds replicate to member sleds. + // + // The pass that promotes a member to "Joined" cannot also push its + // M2P/forwarding state. In `reconcile_member_states`, + // `propagate_m2p_and_forwarding` still observes `member_sleds == 0` + // because the member is "Joining" in the DB at that point. + // + // The push therefore lands in the next pass, via + // `reconcile_active_groups`. Drive that pass once here, then poll + // per-sled state without reactivating. + activate_multicast_reconciler(&cptestctx.lockstep_client).await; + for (i, sled_agent) in cptestctx.sled_agents.iter().enumerate() { let agent = sled_agent.sled_agent(); - // Wait for M2P on every sled. The reconciler may need an - // additional pass after the member reaches "Joined": during - // reconcile_member_states, propagate_m2p_and_forwarding may - // see member_sleds=0 (member still "Joining" in DB), so the - // actual push happens in reconcile_active_groups or the next - // full pass. - wait_for_condition_with_reconciler( - &cptestctx.lockstep_client, + // Wait for M2P on every sled. + wait_for_condition( || async { let m2p = agent.m2p_mappings.lock().unwrap(); if m2p.contains(&(multicast_ip, underlay_ipv6)) { @@ -1099,8 +1080,7 @@ async fn test_multicast_multi_sled_m2p_propagation( // one sled, the hosting sled's forwarding has no next hops // (local delivery via subscription). Non-hosting sleds list // the hosting sled as a next hop so senders can reach it. - wait_for_condition_with_reconciler( - &cptestctx.lockstep_client, + wait_for_condition( || async { let fwd = agent.mcast_fwd.lock().unwrap(); if fwd.contains_key(&underlay_ipv6) { @@ -1119,8 +1099,10 @@ async fn test_multicast_multi_sled_m2p_propagation( ) }); - let fwd = agent.mcast_fwd.lock().unwrap(); - let next_hops = &fwd[&underlay_ipv6]; + let next_hops = { + let fwd = agent.mcast_fwd.lock().unwrap(); + fwd[&underlay_ipv6].clone() + }; // Every sled gets a single next hop pointing at a switch. // The switch replicates to member sled ports via DPD config. assert_eq!( @@ -1129,6 +1111,25 @@ async fn test_multicast_multi_sled_m2p_propagation( "Sled {i} should have 1 next_hop (a switch), \ got: {next_hops:?}" ); + + // Read back through the public `list_mcast_fwd` API, which is what + // the reconciler diffs against. Exactly one underlay entry should + // exist with the same next hops as the raw map. A second entry, or + // extra next hops, would mean stale forwarding leaked past + // `converge_forwarding`'s clear-before-set. + let listed = agent.list_mcast_fwd().expect("list_mcast_fwd"); + assert_eq!( + listed.len(), + 1, + "Sled {i} should have exactly 1 forwarding entry, \ + got: {listed:?}" + ); + let entry = &listed[0]; + assert_eq!(entry.underlay, underlay_ipv6, "Sled {i} underlay mismatch"); + assert_eq!( + entry.next_hops, next_hops, + "Sled {i} list_mcast_fwd next hops diverge from raw map" + ); } // Verify per-VMM subscription on the hosting sled only. @@ -1140,11 +1141,22 @@ async fn test_multicast_multi_sled_m2p_propagation( .unwrap() .sled_agent(); - wait_for_condition_with_reconciler( + let propolis_id = cptestctx + .server + .server_context() + .nexus + .active_instance_info(&instance_id, None) + .await + .unwrap() + .expect("instance should have an active VMM") + .propolis_id; + + activate_then_wait_for_condition( &cptestctx.lockstep_client, || async { - let groups = hosting_agent.multicast_groups.lock().unwrap(); - match groups.get(&info.propolis_id) { + let groups = + hosting_agent.instance_multicast_groups.lock().unwrap(); + match groups.get(&propolis_id) { Some(vmm_groups) if vmm_groups .iter() @@ -1169,9 +1181,12 @@ async fn test_multicast_multi_sled_m2p_propagation( cleanup_instances(cptestctx, client, project_name, &[instance_name]).await; wait_for_group_deleted(cptestctx, group_name).await; + // Verify MRIB route removed after group deletion. + assert_mrib_route_absent(cptestctx, multicast_ip).await; + // Verify cleanup on every sled: M2P and forwarding removed. for (i, sled_agent) in all_sled_agents.iter().enumerate() { - wait_for_condition_with_reconciler( + activate_then_wait_for_condition( &cptestctx.lockstep_client, || async { let m2p = sled_agent.m2p_mappings.lock().unwrap(); @@ -1372,13 +1387,19 @@ async fn test_multicast_cross_sled_forwarding( other => panic!("Expected IPv6 underlay address, got {other}"), }; + // Verify MRIB route was programmed for the group. + assert_mrib_route_exists(cptestctx, group_view.multicast_ip).await; + // Wait for forwarding entries on both sleds, then verify each sled's // forwarding lists exactly the other sled (not itself). + // + // Both members are already "Joined" by this point, so one reconciler + // pass computes both sleds' next-hop sets. let agent_a = cptestctx.sled_agents[0].sled_agent(); let agent_b = cptestctx.sled_agents[1].sled_agent(); for (label, agent) in [("sled A", &agent_a), ("sled B", &agent_b)] { - wait_for_condition_with_reconciler( + activate_then_wait_for_condition( &cptestctx.lockstep_client, || async { let fwd = agent.mcast_fwd.lock().unwrap(); @@ -1405,6 +1426,9 @@ async fn test_multicast_cross_sled_forwarding( ) .await; wait_for_group_deleted(cptestctx, group_name).await; + + // Verify MRIB route removed after group deletion. + assert_mrib_route_absent(cptestctx, group_view.multicast_ip).await; } /// Verify multicast state is re-established after simulated cold start. @@ -1454,8 +1478,6 @@ async fn test_multicast_cold_start_reestablishment( let instance_id = InstanceUuid::from_untyped_uuid(instance.identity.id); instance_wait_for_running_with_simulation(cptestctx, instance_id).await; - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - multicast_group_attach(cptestctx, project_name, instance_name, group_name) .await; wait_for_group_active(client, group_name).await; @@ -1495,6 +1517,9 @@ async fn test_multicast_cold_start_reestablishment( other => panic!("Expected IPv6 underlay address, got {other}"), }; + // Verify MRIB route was programmed. + assert_mrib_route_exists(cptestctx, multicast_ip).await; + // M2P and forwarding are pushed to all sleds. Verify at least the // hosting sled has M2P before we clear state. let pre_info = nexus @@ -1510,7 +1535,7 @@ async fn test_multicast_cold_start_reestablishment( .unwrap() .sled_agent(); - wait_for_condition_with_reconciler( + activate_then_wait_for_condition( &cptestctx.lockstep_client, || async { let m2p = pre_hosting_agent.m2p_mappings.lock().unwrap(); @@ -1555,7 +1580,7 @@ async fn test_multicast_cold_start_reestablishment( for sled_agent in &all_sled_agents { sled_agent.m2p_mappings.lock().unwrap().clear(); sled_agent.mcast_fwd.lock().unwrap().clear(); - sled_agent.multicast_groups.lock().unwrap().clear(); + sled_agent.instance_multicast_groups.lock().unwrap().clear(); } // Restart the instance. @@ -1611,10 +1636,16 @@ async fn test_multicast_cold_start_reestablishment( ) .await; + // Verify MRIB route re-established after cold start. + assert_mrib_route_exists(cptestctx, multicast_ip).await; + + // Drive one pass so the per-sled polls below honor the + // single-pass contract (same reason as the join path above). + activate_multicast_reconciler(&cptestctx.lockstep_client).await; + // Verify M2P and forwarding re-established on all sleds. for (i, sled_agent) in all_sled_agents.iter().enumerate() { - wait_for_condition_with_reconciler( - &cptestctx.lockstep_client, + wait_for_condition( || async { let m2p = sled_agent.m2p_mappings.lock().unwrap(); if m2p.contains(&(multicast_ip, underlay_ipv6)) { @@ -1631,8 +1662,7 @@ async fn test_multicast_cold_start_reestablishment( panic!("Sled {i} M2P not re-established within timeout: {e:?}") }); - wait_for_condition_with_reconciler( - &cptestctx.lockstep_client, + wait_for_condition( || async { let fwd = sled_agent.mcast_fwd.lock().unwrap(); if fwd.contains_key(&underlay_ipv6) { @@ -1667,11 +1697,14 @@ async fn test_multicast_cold_start_reestablishment( .unwrap() .sled_agent(); - wait_for_condition_with_reconciler( + let post_propolis_id = post_info.propolis_id; + + activate_then_wait_for_condition( &cptestctx.lockstep_client, || async { - let groups = post_hosting_agent.multicast_groups.lock().unwrap(); - match groups.get(&post_info.propolis_id) { + let groups = + post_hosting_agent.instance_multicast_groups.lock().unwrap(); + match groups.get(&post_propolis_id) { Some(vmm_groups) if vmm_groups .iter() @@ -1688,7 +1721,7 @@ async fn test_multicast_cold_start_reestablishment( .await .unwrap_or_else(|e| { panic!( - "New VMM should be subscribed to {multicast_ip} after restart: \ + "Instance should be subscribed to {multicast_ip} after restart: \ {e:?}" ) }); @@ -1696,4 +1729,7 @@ async fn test_multicast_cold_start_reestablishment( // Cleanup. cleanup_instances(cptestctx, client, project_name, &[instance_name]).await; wait_for_group_deleted(cptestctx, group_name).await; + + // Verify MRIB route removed after group deletion. + assert_mrib_route_absent(cptestctx, multicast_ip).await; } diff --git a/nexus/types/src/internal_api/background.rs b/nexus/types/src/internal_api/background.rs index d4fd3840c15..09a0ad58726 100644 --- a/nexus/types/src/internal_api/background.rs +++ b/nexus/types/src/internal_api/background.rs @@ -167,6 +167,11 @@ pub struct MulticastGroupReconcilerStatus { pub members_deleted: usize, /// Number of empty groups marked for deletion (implicit deletion). pub empty_groups_marked: usize, + /// Reconciliation steps skipped this pass because their downstream + /// client was unavailable. Distinguishes "no work needed" (counters + /// at 0, `skipped` empty) from "work was deferred" (counters at 0, + /// step name in `skipped`). + pub skipped: Vec, /// Errors that occurred during reconciliation operations. pub errors: Vec, } diff --git a/nexus/types/src/internal_api/views.rs b/nexus/types/src/internal_api/views.rs index 3ac998134eb..9f8da9105b5 100644 --- a/nexus/types/src/internal_api/views.rs +++ b/nexus/types/src/internal_api/views.rs @@ -1263,6 +1263,64 @@ pub struct SupportBundleInfo { pub fm_case_id: Option, } +/// DDM session state of an underlay peer. +/// +/// Mirrors the variants of the DDM admin API's `PeerStatus` without the +/// state-duration payload, which is carried separately on +/// [`MulticastDdmPeer::status_duration`] so consumers can match on the +/// state directly. +#[derive( + Clone, Copy, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq, +)] +#[serde(rename_all = "snake_case")] +pub enum MulticastDdmPeerStatus { + Init, + Solicit, + Exchange, + Expired, +} + +impl Display for MulticastDdmPeerStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Init => write!(f, "init"), + Self::Solicit => write!(f, "solicit"), + Self::Exchange => write!(f, "exchange"), + Self::Expired => write!(f, "expired"), + } + } +} + +/// A single DDM underlay peer as observed from a switch zone. +/// +/// Under RFD 488, `mg-lower` derives underlay multicast members from the +/// DDM peer subscriptions reported here. The `if_name` field names the switch +/// interface the peer was discovered on (e.g. "tfportrear0_0"). +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +pub struct MulticastDdmPeer { + /// The switch zone the peer was observed from. + pub switch: String, + /// The peer's underlay address. + pub addr: Ipv6Addr, + /// The peer's reported hostname. + pub host: String, + /// The peer's DDM session state. + pub status: MulticastDdmPeerStatus, + /// How long the peer has held its current session state. + pub status_duration: Duration, + /// The switch interface the peer was discovered on, if known. + pub if_name: Option, +} + +/// Read-only view of DDM underlay peers across all switch zones. +/// +/// Surfaced via `omdb nexus multicast ddm-peers` for operators inspecting the +/// underlay topology that drives multicast member programming. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +pub struct MulticastDdmPeersView { + pub peers: Vec, +} + #[cfg(test)] mod test { use super::CompletedAttempt; diff --git a/openapi/nexus-lockstep.json b/openapi/nexus-lockstep.json index 905a4f8ac31..454a3f39741 100644 --- a/openapi/nexus-lockstep.json +++ b/openapi/nexus-lockstep.json @@ -1077,6 +1077,30 @@ } } }, + "/multicast/ddm-peers": { + "get": { + "summary": "Read-only view of DDM underlay peers across switch zones", + "operationId": "multicast_ddm_peers", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MulticastDdmPeersView" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/oximeter/read-policy": { "get": { "summary": "Get the current oximeter read policy", @@ -5947,6 +5971,78 @@ "waiting" ] }, + "MulticastDdmPeer": { + "description": "A single DDM underlay peer as observed from a switch zone.\n\nUnder RFD 488, `mg-lower` derives underlay multicast members from the DDM peer subscriptions reported here. The `if_name` field names the switch interface the peer was discovered on (e.g. \"tfportrear0_0\").", + "type": "object", + "properties": { + "addr": { + "description": "The peer's underlay address.", + "type": "string", + "format": "ipv6" + }, + "host": { + "description": "The peer's reported hostname.", + "type": "string" + }, + "if_name": { + "nullable": true, + "description": "The switch interface the peer was discovered on, if known.", + "type": "string" + }, + "status": { + "description": "The peer's DDM session state.", + "allOf": [ + { + "$ref": "#/components/schemas/MulticastDdmPeerStatus" + } + ] + }, + "status_duration": { + "description": "How long the peer has held its current session state.", + "allOf": [ + { + "$ref": "#/components/schemas/Duration" + } + ] + }, + "switch": { + "description": "The switch zone the peer was observed from.", + "type": "string" + } + }, + "required": [ + "addr", + "host", + "status", + "status_duration", + "switch" + ] + }, + "MulticastDdmPeerStatus": { + "description": "DDM session state of an underlay peer.\n\nMirrors the variants of the DDM admin API's `PeerStatus` without the state-duration payload, which is carried separately on [`MulticastDdmPeer::status_duration`] so consumers can match on the state directly.", + "type": "string", + "enum": [ + "init", + "solicit", + "exchange", + "expired" + ] + }, + "MulticastDdmPeersView": { + "description": "Read-only view of DDM underlay peers across all switch zones.\n\nSurfaced via `omdb nexus multicast ddm-peers` for operators inspecting the underlay topology that drives multicast member programming.", + "type": "object", + "properties": { + "peers": { + "type": "array", + "items": { + "$ref": "#/components/schemas/MulticastDdmPeer" + } + } + }, + "required": [ + "peers" + ] + }, "MupdateOverrideUuid": { "x-rust-type": { "crate": "omicron-uuid-kinds", diff --git a/package-manifest.toml b/package-manifest.toml index ba07f9baf7a..0b47696b6c0 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -683,10 +683,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "5aaa7c9bc2d0e541d29604453dd98cb32676b347" +source.commit = "4cc569836dc1c5d13a5f2396de310220eafee2fe" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm-gz.sha256.txt -source.sha256 = "e4d20f7f4ff7933e95abff79e7467e4b1a0302066f63be6b14cde739d3899afe" +source.sha256 = "dfca8cc5bd9f2d789e53bad2b416bc2d5b5d5c016e4a1f295a05cbb7c4c1f2aa" output.type = "tarball" [package.mg-ddm] @@ -699,10 +699,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "5aaa7c9bc2d0e541d29604453dd98cb32676b347" +source.commit = "4cc569836dc1c5d13a5f2396de310220eafee2fe" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "5e05778f0ec0385b2e097f82bd6776942ba93cdf4118c4f72f54c612636d3372" +source.sha256 = "922df0577243659de65390384a5c704fad82e55003cdb444981c98e6d00a70d1" output.type = "zone" output.intermediate_only = true @@ -714,10 +714,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "5aaa7c9bc2d0e541d29604453dd98cb32676b347" +source.commit = "4cc569836dc1c5d13a5f2396de310220eafee2fe" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mgd.sha256.txt -source.sha256 = "ae057ecdaed165fbd3eaaa9459c934249d7a366a51bda5c62584fa4bc011be8e" +source.sha256 = "c4ebe683e5f2e2c0745656845739332a5a2b76a9c78fc502e3fe5cf168a9cde4" output.type = "zone" output.intermediate_only = true diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index f652a60bd12..4049393494c 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -8159,6 +8159,18 @@ CREATE TYPE IF NOT EXISTS omicron.public.multicast_group_member_state AS ENUM ( 'left' ); +-- Origin of a multicast group membership. +-- +-- 'static' is administratively-configured membership created through the +-- control plane API (instance attach); it does not expire. 'igmp_snooped' is +-- dynamic soft-state learned from snooped IGMP/MLD reports (RFD 488). Origin +-- distinguishes which rows a future soft-state reaper may expire: snooped rows +-- expire on their query-interval-derived hold-down, static rows never do. +CREATE TYPE IF NOT EXISTS omicron.public.multicast_group_member_origin AS ENUM ( + 'static', + 'igmp_snooped' +); + /* * External multicast groups (customer-facing, allocated from IP pools) * Following the bifurcated design from RFD 488 @@ -8286,7 +8298,10 @@ CREATE TABLE IF NOT EXISTS omicron.public.multicast_group_member ( /* Empty array means any source is allowed (ASM) */ /* Non-empty array enables source filtering (IGMPv3/MLDv2) */ /* The group's source_ips in API views is the union of all active members */ - source_ips INET[] NOT NULL DEFAULT ARRAY[]::INET[] + source_ips INET[] NOT NULL DEFAULT ARRAY[]::INET[], + + /* Origin of this membership (static API vs snooped IGMP/MLD soft-state) */ + membership_origin omicron.public.multicast_group_member_origin NOT NULL DEFAULT 'static' ); /* External Multicast Group Indexes */ @@ -8713,7 +8728,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '266.0.0', NULL) + (TRUE, NOW(), NOW(), '267.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/schema/crdb/multicast-member-origin/up01.sql b/schema/crdb/multicast-member-origin/up01.sql new file mode 100644 index 00000000000..21ddab0bb1b --- /dev/null +++ b/schema/crdb/multicast-member-origin/up01.sql @@ -0,0 +1,4 @@ +CREATE TYPE IF NOT EXISTS omicron.public.multicast_group_member_origin AS ENUM ( + 'static', + 'igmp_snooped' +); diff --git a/schema/crdb/multicast-member-origin/up02.sql b/schema/crdb/multicast-member-origin/up02.sql new file mode 100644 index 00000000000..14ae66dfa29 --- /dev/null +++ b/schema/crdb/multicast-member-origin/up02.sql @@ -0,0 +1,2 @@ +ALTER TABLE omicron.public.multicast_group_member + ADD COLUMN IF NOT EXISTS membership_origin omicron.public.multicast_group_member_origin NOT NULL DEFAULT 'static'; diff --git a/sled-agent/early-networking/src/lib.rs b/sled-agent/early-networking/src/lib.rs index 3cb3421ebdc..0e586e60e25 100644 --- a/sled-agent/early-networking/src/lib.rs +++ b/sled-agent/early-networking/src/lib.rs @@ -28,7 +28,6 @@ use mg_api_types::bgp::policy::{ ImportExportPolicy4 as MgImportExportPolicy4, ImportExportPolicy6 as MgImportExportPolicy6, }; -use mg_api_types::rdb::prefix::{Prefix, Prefix4, Prefix6}; use mg_api_types::rib::BestpathFanoutRequest; use mg_api_types::static_routes::{ AddStaticRoute4Request, AddStaticRoute6Request, StaticRoute4, @@ -41,7 +40,7 @@ use omicron_common::backoff::{ BackoffError, ExponentialBackoff, ExponentialBackoffBuilder, retry_notify, }; use omicron_ddm_admin_client::DdmError; -use oxnet::IpNet; +use oxnet::{IpNet, Ipv4Net, Ipv6Net}; use sled_agent_types::early_networking::{ BfdMode, BgpConfig, BgpPeerConfig, ImportExportPolicy, LinkFec, LinkSpeed, PortConfig, RouterPeerType, SwitchSlot, UplinkAddress, @@ -55,7 +54,7 @@ use slog::o; use slog::warn; use slog_error_chain::InlineErrorChain; use std::collections::{HashMap, HashSet}; -use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::time::{Duration, Instant}; use thiserror::Error; use tokio::sync::watch; @@ -465,10 +464,7 @@ impl<'a> EarlyNetworkSetup<'a> { MgImportExportPolicy4::Allow( list.iter() .filter_map(|x| match x { - IpNet::V4(p) => Some(Prefix4 { - length: p.width(), - value: p.addr(), - }), + IpNet::V4(p) => Some(*p), IpNet::V6(_) => None, }) .collect(), @@ -483,10 +479,7 @@ impl<'a> EarlyNetworkSetup<'a> { MgImportExportPolicy4::Allow( list.iter() .filter_map(|x| match x { - IpNet::V4(p) => Some(Prefix4 { - length: p.width(), - value: p.addr(), - }), + IpNet::V4(p) => Some(*p), IpNet::V6(_) => None, }) .collect(), @@ -506,10 +499,7 @@ impl<'a> EarlyNetworkSetup<'a> { MgImportExportPolicy6::Allow( list.iter() .filter_map(|x| match x { - IpNet::V6(p) => Some(Prefix6 { - length: p.width(), - value: p.addr(), - }), + IpNet::V6(p) => Some(*p), IpNet::V4(_) => None, }) .collect(), @@ -524,10 +514,7 @@ impl<'a> EarlyNetworkSetup<'a> { MgImportExportPolicy6::Allow( list.iter() .filter_map(|x| match x { - IpNet::V6(p) => Some(Prefix6 { - length: p.width(), - value: p.addr(), - }), + IpNet::V6(p) => Some(*p), IpNet::V4(_) => None, }) .collect(), @@ -574,7 +561,7 @@ impl<'a> EarlyNetworkSetup<'a> { RouterPeerType::Numbered { ip: addr } => { let bpc = MgBgpPeerConfig { name: format!("{}", addr), - host: format!("{}:179", addr), + host: SocketAddr::new(addr.into(), 179), hold_time: peer .hold_time .unwrap_or(BgpPeerConfig::DEFAULT_HOLD_TIME), @@ -696,20 +683,7 @@ impl<'a> EarlyNetworkSetup<'a> { code: x.clone(), asn: config.asn, }), - originate: config - .originate - .iter() - .map(|x| match x { - IpNet::V4(ipv4_net) => Prefix::V4(Prefix4 { - length: ipv4_net.width(), - value: ipv4_net.addr(), - }), - IpNet::V6(ipv6_net) => Prefix::V6(Prefix6 { - length: ipv6_net.width(), - value: ipv6_net.addr(), - }), - }) - .collect(), + originate: config.originate.clone(), }; let fanout = BestpathFanoutRequest { @@ -753,10 +727,10 @@ impl<'a> EarlyNetworkSetup<'a> { match (r.nexthop, r.destination.addr()) { (IpAddr::V4(nexthop), IpAddr::V4(dest_addr)) => { - let prefix = Prefix4 { - value: dest_addr, - length: r.destination.width(), - }; + let prefix = Ipv4Net::new_unchecked( + dest_addr, + r.destination.width(), + ); let sr = StaticRoute4 { nexthop: IpAddr::V4(nexthop), prefix, @@ -766,10 +740,10 @@ impl<'a> EarlyNetworkSetup<'a> { rq.routes.list.push(sr); } (IpAddr::V6(nexthop), IpAddr::V4(dest_addr)) => { - let prefix = Prefix4 { - value: dest_addr, - length: r.destination.width(), - }; + let prefix = Ipv4Net::new_unchecked( + dest_addr, + r.destination.width(), + ); let sr = StaticRoute4 { nexthop: IpAddr::V6(nexthop), prefix, @@ -779,10 +753,10 @@ impl<'a> EarlyNetworkSetup<'a> { rq.routes.list.push(sr); } (IpAddr::V6(nexthop), IpAddr::V6(dest_addr)) => { - let prefix = Prefix6 { - value: dest_addr, - length: r.destination.width(), - }; + let prefix = Ipv6Net::new_unchecked( + dest_addr, + r.destination.width(), + ); let sr = StaticRoute6 { nexthop, prefix, diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index fd8dfb57bf0..11b3634f33c 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -83,7 +83,7 @@ use trust_quorum_types::messages::{ use trust_quorum_types::status::{CommitStatus, CoordinatorStatus, NodeStatus}; // Fixed identifiers for prior versions only -use sled_agent_types_versions::{v1, v20, v25, v26, v30, v33, v39}; +use sled_agent_types_versions::{v1, v7, v20, v25, v26, v30, v33, v39}; use sled_diagnostics::{ SledDiagnosticsCommandHttpOutput, SledDiagnosticsQueryOutput, }; @@ -717,11 +717,11 @@ impl SledAgentApi for SledAgentImpl { body: TypedBody, ) -> Result { let sa = rqctx.context(); - let id = path_params.into_inner().propolis_id; + let propolis_id = path_params.into_inner().propolis_id; let membership = body.into_inner(); sa.latencies() .instrument_dropshot_handler(&rqctx, async { - sa.instance_join_multicast_group(id, &membership).await?; + sa.vmm_join_multicast_group(propolis_id, &membership).await?; Ok(HttpResponseUpdatedNoContent()) }) .await @@ -733,11 +733,59 @@ impl SledAgentApi for SledAgentImpl { body: TypedBody, ) -> Result { let sa = rqctx.context(); - let id = path_params.into_inner().propolis_id; + let propolis_id = path_params.into_inner().propolis_id; let membership = body.into_inner(); sa.latencies() .instrument_dropshot_handler(&rqctx, async { - sa.instance_leave_multicast_group(id, &membership).await?; + sa.vmm_leave_multicast_group(propolis_id, &membership).await?; + Ok(HttpResponseUpdatedNoContent()) + }) + .await + } + + async fn vmm_join_multicast_group_v7( + rqctx: RequestContext, + path_params: Path, + body: TypedBody, + ) -> Result { + let sa = rqctx.context(); + let propolis_id = path_params.into_inner().propolis_id; + let membership = match body.into_inner() { + v7::instance::InstanceMulticastBody::Join(m) => m, + v7::instance::InstanceMulticastBody::Leave(_) => { + return Err(HttpError::for_bad_request( + None, + "Join endpoint cannot process Leave operations".to_string(), + )); + } + }; + sa.latencies() + .instrument_dropshot_handler(&rqctx, async { + sa.vmm_join_multicast_group(propolis_id, &membership).await?; + Ok(HttpResponseUpdatedNoContent()) + }) + .await + } + + async fn vmm_leave_multicast_group_v7( + rqctx: RequestContext, + path_params: Path, + body: TypedBody, + ) -> Result { + let sa = rqctx.context(); + let propolis_id = path_params.into_inner().propolis_id; + let membership = match body.into_inner() { + v7::instance::InstanceMulticastBody::Leave(m) => m, + v7::instance::InstanceMulticastBody::Join(_) => { + return Err(HttpError::for_bad_request( + None, + "Leave endpoint cannot process Join operations".to_string(), + )); + } + }; + sa.latencies() + .instrument_dropshot_handler(&rqctx, async { + sa.vmm_leave_multicast_group(propolis_id, &membership).await?; Ok(HttpResponseUpdatedNoContent()) }) .await diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index 0bd39b28c41..1e6a277f6df 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -3011,6 +3011,7 @@ mod tests { let port_manager = PortManager::new( log.new(o!("component" => "PortManager")), Ipv6Addr::new(0xfd00, 0x1de, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01), + &[], ); let cleanup_context = CleanupContext::default(); diff --git a/sled-agent/src/instance_manager.rs b/sled-agent/src/instance_manager.rs index 28f4bf9a8d7..b885328a604 100644 --- a/sled-agent/src/instance_manager.rs +++ b/sled-agent/src/instance_manager.rs @@ -18,7 +18,7 @@ use illumos_utils::opte::PortManager; use illumos_utils::running_zone::ZoneBuilderFactory; use omicron_common::api::external::ByteCount; use omicron_common::api::internal::shared::SledIdentifiers; -use omicron_uuid_kinds::PropolisUuid; +use omicron_uuid_kinds::{InstanceUuid, PropolisUuid}; use oxnet::IpNet; use sled_agent_config_reconciler::AvailableDatasetsReceiver; use sled_agent_config_reconciler::CurrentlyManagedZpoolsReceiver; @@ -43,6 +43,9 @@ pub enum Error { #[error("VMM with ID {0} not found")] NoSuchVmm(PropolisUuid), + #[error("No active VMM for instance {0}")] + NoActiveVmmForInstance(InstanceUuid), + #[error("OPTE port management error")] Opte(#[from] illumos_utils::opte::Error), @@ -303,7 +306,12 @@ impl InstanceManager { rx.await? } - pub async fn join_multicast_group( + /// Subscribe a VMM's OPTE port to a multicast group. + /// + /// Keyed by `propolis_id` rather than `instance_id` so the dispatch is + /// unambiguous during live migration, when source and target VMMs for + /// the same instance can both be registered. + pub async fn join_multicast_group_by_vmm( &self, propolis_id: PropolisUuid, membership: &InstanceMulticastMembership, @@ -311,7 +319,7 @@ impl InstanceManager { let (tx, rx) = oneshot::channel(); self.inner .tx - .send(InstanceManagerRequest::JoinMulticastGroup { + .send(InstanceManagerRequest::JoinMulticastGroupByVmm { propolis_id, membership: membership.clone(), tx, @@ -322,7 +330,11 @@ impl InstanceManager { rx.await? } - pub async fn leave_multicast_group( + /// Unsubscribe a VMM's OPTE port from a multicast group. + /// + /// See [`Self::join_multicast_group_by_vmm`] for the serialization + /// and migration-safety properties. + pub async fn leave_multicast_group_by_vmm( &self, propolis_id: PropolisUuid, membership: &InstanceMulticastMembership, @@ -330,7 +342,7 @@ impl InstanceManager { let (tx, rx) = oneshot::channel(); self.inner .tx - .send(InstanceManagerRequest::LeaveMulticastGroup { + .send(InstanceManagerRequest::LeaveMulticastGroupByVmm { propolis_id, membership: membership.clone(), tx, @@ -482,12 +494,12 @@ enum InstanceManagerRequest { RefreshExternalIps { tx: oneshot::Sender>, }, - JoinMulticastGroup { + JoinMulticastGroupByVmm { propolis_id: PropolisUuid, membership: InstanceMulticastMembership, tx: oneshot::Sender>, }, - LeaveMulticastGroup { + LeaveMulticastGroupByVmm { propolis_id: PropolisUuid, membership: InstanceMulticastMembership, tx: oneshot::Sender>, @@ -630,11 +642,11 @@ impl InstanceManagerRunner { Some(RefreshExternalIps { tx }) => { self.refresh_external_ips(tx) }, - Some(JoinMulticastGroup { propolis_id, membership, tx }) => { - self.join_multicast_group(tx, propolis_id, &membership) - }, - Some(LeaveMulticastGroup { propolis_id, membership, tx }) => { - self.leave_multicast_group(tx, propolis_id, &membership) + Some(JoinMulticastGroupByVmm { propolis_id, membership, tx }) => { + self.join_multicast_group_by_vmm(tx, propolis_id, &membership) + } + Some(LeaveMulticastGroupByVmm { propolis_id, membership, tx }) => { + self.leave_multicast_group_by_vmm(tx, propolis_id, &membership) } Some(GetState { propolis_id, tx }) => { // TODO(eliza): it could potentially be nice to @@ -903,27 +915,39 @@ impl InstanceManagerRunner { Ok(()) } - fn join_multicast_group( + /// Forward a join to the VMM's instance task. The lookup runs inside + /// this dispatcher loop, so it is serialized with `EnsureRegistered`, + /// `EnsureUnregistered`, and other per-VMM state changes. If no VMM + /// is registered with this `propolis_id` the call is a noop, since + /// there is no OPTE port to update. + /// + /// `jobs` is keyed by `propolis_id`, so the lookup is unambiguous + /// even during live migration when source and target VMMs for the + /// same `instance_id` are both registered. + fn join_multicast_group_by_vmm( &self, tx: oneshot::Sender>, propolis_id: PropolisUuid, membership: &InstanceMulticastMembership, ) -> Result<(), Error> { - let Some(instance) = self.get_propolis(propolis_id) else { - return Err(Error::NoSuchVmm(propolis_id)); + let Some(instance) = self.jobs.get(&propolis_id) else { + return tx.send(Ok(())).map_err(|_| Error::FailedSendClientClosed); }; instance.join_multicast_group(tx, membership)?; Ok(()) } - fn leave_multicast_group( + /// Forward a leave to the VMM's instance task. See + /// [`Self::join_multicast_group_by_vmm`] for serialization and + /// no-active-VMM behavior. + fn leave_multicast_group_by_vmm( &self, tx: oneshot::Sender>, propolis_id: PropolisUuid, membership: &InstanceMulticastMembership, ) -> Result<(), Error> { - let Some(instance) = self.get_propolis(propolis_id) else { - return Err(Error::NoSuchVmm(propolis_id)); + let Some(instance) = self.jobs.get(&propolis_id) else { + return tx.send(Ok(())).map_err(|_| Error::FailedSendClientClosed); }; instance.leave_multicast_group(tx, membership)?; Ok(()) diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index a57dc102f85..8976c8dd375 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -201,7 +201,7 @@ impl SledAgentApi for SledAgentSimImpl { let sa = rqctx.context(); let propolis_id = path_params.into_inner().propolis_id; let membership = body.into_inner(); - sa.instance_join_multicast_group(propolis_id, &membership).await?; + sa.vmm_join_multicast_group(propolis_id, &membership).await?; Ok(HttpResponseUpdatedNoContent()) } @@ -213,7 +213,7 @@ impl SledAgentApi for SledAgentSimImpl { let sa = rqctx.context(); let propolis_id = path_params.into_inner().propolis_id; let membership = body.into_inner(); - sa.instance_leave_multicast_group(propolis_id, &membership).await?; + sa.vmm_leave_multicast_group(propolis_id, &membership).await?; Ok(HttpResponseUpdatedNoContent()) } diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index 0d36dd6591f..e8a812c3cec 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -111,8 +111,8 @@ pub struct SledAgent { /// subnets attached to instances. pub attached_subnets: Mutex>>, - /// multicast group memberships for instances - pub multicast_groups: + /// multicast group memberships, keyed by VMM (propolis). + pub instance_multicast_groups: Mutex>>, pub vpc_routes: Mutex>, config: Config, @@ -198,7 +198,7 @@ impl SledAgent { mcast_fwd: Mutex::new(HashMap::new()), external_ips: Mutex::new(HashMap::new()), attached_subnets: Mutex::new(HashMap::new()), - multicast_groups: Mutex::new(HashMap::new()), + instance_multicast_groups: Mutex::new(HashMap::new()), vpc_routes: Mutex::new(HashMap::new()), mock_propolis: futures::lock::Mutex::new(None), config: config.clone(), @@ -355,6 +355,17 @@ impl SledAgent { self.simulated_upstairs.map_id_to_vcr(*id, &vcr); } + // Real sled-agent applies multicast memberships when it creates OPTE + // ports at registration, so migration targets receive them without an + // explicit join call. Mirror that here. + if !local_config.multicast_groups.is_empty() { + let mut groups = self.instance_multicast_groups.lock().unwrap(); + groups + .entry(propolis_id) + .or_default() + .extend(local_config.multicast_groups.iter().cloned()); + } + let mut routes = self.vpc_routes.lock().unwrap(); for nic in &local_config.nics { let kinds = std::iter::once(RouterKind::System) @@ -701,7 +712,20 @@ impl SledAgent { req: &McastForwardingEntry, ) -> Result<(), Error> { let mut fwd = self.mcast_fwd.lock().unwrap(); - fwd.insert(req.underlay, req.next_hops.clone()); + // Match legit OPTE semantics: next hops accumulate, keyed by next-hop + // address. A next hop whose address already exists replaces that + // entry; a new address is appended. Next hops absent from `req` are + // left in place. Only `clear_mcast_fwd` removes them. A full-vector + // replace here would diverge from xde and hide the stale-next-hop + // accumulation that `converge_forwarding`'s clear-before-set guards + // against. + let entry = fwd.entry(req.underlay).or_default(); + for hop in &req.next_hops { + match entry.iter_mut().find(|h| h.next_hop == hop.next_hop) { + Some(existing) => *existing = hop.clone(), + None => entry.push(hop.clone()), + } + } Ok(()) } @@ -859,41 +883,33 @@ impl SledAgent { Ok(()) } - pub async fn instance_join_multicast_group( + pub async fn vmm_join_multicast_group( &self, propolis_id: PropolisUuid, membership: &InstanceMulticastMembership, ) -> Result<(), Error> { if !self.vmms.contains_key(&propolis_id.into_untyped_uuid()).await { return Err(Error::internal_error( - "can't join multicast group for VMM that's not registered", + "can't alter multicast membership for VMM that's not registered", )); } - - let mut groups = self.multicast_groups.lock().unwrap(); - let my_groups = groups.entry(propolis_id).or_default(); - - my_groups.insert(membership.clone()); - + let mut groups = self.instance_multicast_groups.lock().unwrap(); + groups.entry(propolis_id).or_default().insert(membership.clone()); Ok(()) } - pub async fn instance_leave_multicast_group( + pub async fn vmm_leave_multicast_group( &self, propolis_id: PropolisUuid, membership: &InstanceMulticastMembership, ) -> Result<(), Error> { if !self.vmms.contains_key(&propolis_id.into_untyped_uuid()).await { return Err(Error::internal_error( - "can't leave multicast group for VMM that's not registered", + "can't alter multicast membership for VMM that's not registered", )); } - - let mut groups = self.multicast_groups.lock().unwrap(); - let my_groups = groups.entry(propolis_id).or_default(); - - my_groups.remove(membership); - + let mut groups = self.instance_multicast_groups.lock().unwrap(); + groups.entry(propolis_id).or_default().remove(membership); Ok(()) } diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 1889c8ecbd8..376a093af4a 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -587,6 +587,7 @@ impl SledAgent { let port_manager = PortManager::new( parent_log.new(o!("component" => "PortManager")), *sled_address.ip(), + &underlay_nics, ); // The VMM reservoir is configured with respect to what's left after @@ -1042,27 +1043,36 @@ impl SledAgent { } /// Subscribe a VMM's OPTE port to a multicast group. - pub async fn instance_join_multicast_group( + /// + /// Keyed by `propolis_id` rather than `instance_id` so the dispatch + /// is unambiguous during live migration (source and target VMMs for + /// the same instance can both be registered). Dispatch runs under + /// the instance manager's per-VMM lock so OPTE port mutation is + /// serialized with other state changes for the same VMM. + pub async fn vmm_join_multicast_group( &self, propolis_id: PropolisUuid, membership: &InstanceMulticastMembership, ) -> Result<(), Error> { self.inner .instances - .join_multicast_group(propolis_id, membership) + .join_multicast_group_by_vmm(propolis_id, membership) .await .map_err(|e| Error::Instance(e)) } /// Unsubscribe a VMM's OPTE port from a multicast group. - pub async fn instance_leave_multicast_group( + /// + /// See [`Self::vmm_join_multicast_group`] for the migration-safety + /// and no-active-VMM semantics. + pub async fn vmm_leave_multicast_group( &self, propolis_id: PropolisUuid, membership: &InstanceMulticastMembership, ) -> Result<(), Error> { self.inner .instances - .leave_multicast_group(propolis_id, membership) + .leave_multicast_group_by_vmm(propolis_id, membership) .await .map_err(|e| Error::Instance(e)) } diff --git a/sled-agent/types/versions/src/mcast_m2p_forwarding/mod.rs b/sled-agent/types/versions/src/mcast_m2p_forwarding/mod.rs index 8c9d1bb1c4a..f84aead96a1 100644 --- a/sled-agent/types/versions/src/mcast_m2p_forwarding/mod.rs +++ b/sled-agent/types/versions/src/mcast_m2p_forwarding/mod.rs @@ -4,7 +4,9 @@ //! Version `MCAST_M2P_FORWARDING` of the Sled Agent API. //! -//! Adds multicast-to-physical mapping and forwarding types used by -//! the multicast-to-physical and forwarding endpoints. +//! This version adds the multicast-to-physical (M2P) mapping and +//! forwarding types used by the new networking endpoints. It also changes +//! the multicast subscription endpoints to take an `InstanceMulticastMembership` +//! request body in place of `InstanceMulticastBody`. pub mod multicast; diff --git a/sled-agent/types/versions/src/mcast_m2p_forwarding/multicast.rs b/sled-agent/types/versions/src/mcast_m2p_forwarding/multicast.rs index 5c2247c1159..5ed65f52955 100644 --- a/sled-agent/types/versions/src/mcast_m2p_forwarding/multicast.rs +++ b/sled-agent/types/versions/src/mcast_m2p_forwarding/multicast.rs @@ -2,8 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -// Copyright 2026 Oxide Computer Company - //! Multicast networking types for the sled-agent API. //! //! These types support overlay-to-underlay multicast mapping and diff --git a/smf/nexus/multi-sled/config-partial.toml b/smf/nexus/multi-sled/config-partial.toml index 2cbbfc36034..9aa43043f71 100644 --- a/smf/nexus/multi-sled/config-partial.toml +++ b/smf/nexus/multi-sled/config-partial.toml @@ -112,12 +112,6 @@ fm.rendezvous_period_secs = 300 probe_distributor.period_secs = 60 multicast_reconciler.period_secs = 60 trust_quorum.period_secs = 60 -# TTL for sled-to-backplane-port mapping cache -# Default: 3600 seconds (1 hour) - detects new sleds and inventory changes -# multicast_reconciler.sled_cache_ttl_secs = 3600 -# TTL for backplane topology cache (static platform configuration) -# Default: 86400 seconds (24 hours) - refreshed on-demand when validation fails -# multicast_reconciler.backplane_cache_ttl_secs = 86400 attached_subnet_manager.period_secs = 60 session_cleanup.period_secs = 300 session_cleanup.max_delete_per_activation = 10000 diff --git a/smf/nexus/single-sled/config-partial.toml b/smf/nexus/single-sled/config-partial.toml index 93e255dfbb0..6c6baf80b81 100644 --- a/smf/nexus/single-sled/config-partial.toml +++ b/smf/nexus/single-sled/config-partial.toml @@ -112,12 +112,6 @@ fm.rendezvous_period_secs = 300 probe_distributor.period_secs = 60 trust_quorum.period_secs = 60 multicast_reconciler.period_secs = 60 -# TTL for sled-to-backplane-port mapping cache -# Default: 3600 seconds (1 hour) - detects new sleds and inventory changes -# multicast_reconciler.sled_cache_ttl_secs = 3600 -# TTL for backplane topology cache (static platform configuration) -# Default: 86400 seconds (24 hours) - refreshed on-demand when validation fails -# multicast_reconciler.backplane_cache_ttl_secs = 86400 attached_subnet_manager.period_secs = 60 session_cleanup.period_secs = 300 session_cleanup.max_delete_per_activation = 10000 diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml index ad4b8d303c7..a53a72af2b2 100644 --- a/test-utils/Cargo.toml +++ b/test-utils/Cargo.toml @@ -26,6 +26,8 @@ pem.workspace = true regex.workspace = true ring.workspace = true rustls.workspace = true +schemars.workspace = true +serde.workspace = true slog.workspace = true subprocess.workspace = true tempfile.workspace = true diff --git a/test-utils/src/dev/maghemite.rs b/test-utils/src/dev/maghemite.rs index a7091d38508..7386bae9a37 100644 --- a/test-utils/src/dev/maghemite.rs +++ b/test-utils/src/dev/maghemite.rs @@ -36,13 +36,24 @@ pub struct MgdInstance { } impl MgdInstance { + /// Start an `mgd` process. + /// + /// `mgs_addr` is the required management-gateway (MGS) endpoint mgd is + /// bound to. In the harness it is the switch's gateway instance. + /// + /// `dendrite_addr` and `ddm_addr` point mgd's lower half (illumos + /// only) at a dpd and a DDM admin endpoint respectively. When set, + /// the `mg-lower-mrib` thread syncs the local MRIB to the DDM + /// endpoint as multicast origination. pub async fn start( mut port: u16, mgs_addr: SocketAddr, + dendrite_addr: Option, + ddm_addr: Option, ) -> Result { let temp_dir = TempDir::new()?; - let args = vec![ + let mut args = vec![ "run".to_string(), "--admin-addr".into(), "::1".into(), @@ -58,6 +69,14 @@ impl MgdInstance { "--mgs-addr".into(), mgs_addr.to_string(), ]; + if let Some(addr) = dendrite_addr { + args.push("--dendrite-addr".into()); + args.push(addr.to_string()); + } + if let Some(addr) = ddm_addr { + args.push("--ddm-addr".into()); + args.push(addr.to_string()); + } let child = tokio::process::Command::new("mgd") .args(&args) @@ -198,7 +217,7 @@ impl PortProbe { } } - async fn probe(&self) -> Result, anyhow::Error> { + async fn probe(&self) -> Result, anyhow::Error> { let output = tokio::process::Command::new(self.command) .args(&self.args) .output() @@ -207,40 +226,57 @@ impl PortProbe { if !output.status.success() { // The probe command can transiently fail (process exiting, // permissions, etc.); leave it to the caller to retry. - return Ok(None); + return Ok(Vec::new()); } let text = std::str::from_utf8(&output.stdout) .with_context(|| format!("{} output not utf8", self.command))?; - Ok(text + let mut ports: Vec = text .lines() .filter(|line| { self.pid_marker.as_deref().is_none_or(|m| line.contains(m)) }) - .find_map(|line| { + .filter_map(|line| { self.port_re .captures(line)? .get(1)? .as_str() .parse::() .ok() - })) + }) + .collect(); + ports.sort_unstable(); + ports.dedup(); + Ok(ports) } } -/// Ask the kernel which TCP port `pid` is listening on. Retries until either -/// the port is found or [`DDMD_TIMEOUT`] elapses, since `ddmd` may not have -/// finished binding when we first probe. -async fn find_listening_port(pid: u32) -> Result { +/// Ask the kernel which TCP port `pid` is listening on, once it accepts. +/// +/// `ddmd --api-only` binds a single Dropshot server, the versioned admin API, +/// but does not log its `local_addr` at the daemon's error-only log level, so +/// the kernel is the only source for the port. A bare `GET` confirms the +/// server is accepting connections (any HTTP response, including a 404 for the +/// unknown path) before the port is returned, since the kernel can report a +/// socket as listening before Dropshot accepts on it. +/// +/// Retries until a port responds or [`DDMD_TIMEOUT`] elapses. +async fn find_ddm_admin_port(pid: u32) -> Result { let probe = PortProbe::for_pid(pid); + let client = reqwest::Client::new(); let deadline = Instant::now() + DDMD_TIMEOUT; loop { - if let Some(port) = probe.probe().await? { - return Ok(port); + for port in probe.probe().await? { + let url = format!("http://[::1]:{port}/"); + // Any HTTP response means the admin server is up; a connect error + // means it is bound but not accepting yet, so retry. + if client.get(&url).send().await.is_ok() { + return Ok(port); + } } if Instant::now() >= deadline { anyhow::bail!( - "kernel reports no listening TCP port for pid {pid} after \ - {DDMD_TIMEOUT:?}" + "kernel reports no responding admin port for pid {pid} \ + after {DDMD_TIMEOUT:?}" ); } sleep(Duration::from_millis(100)).await; @@ -288,19 +324,22 @@ const DDMD_TIMEOUT: Duration = Duration::from_secs(5); /// Test fixture that spawns and supervises a legit `ddmd` subprocess. /// -/// Owns a `tokio::process::Child` and a tempdir; discovers the bound admin -/// port by scraping dropshot's startup `local_addr` records; kills the child -/// on `cleanup`/`Drop`. Mirrors `MgdInstance`. +/// Owns a `tokio::process::Child` and a tempdir. Discovers the bound admin +/// port by asking the kernel which TCP port the pid is listening on (see +/// [`find_ddm_admin_port`]), and kills the child on `cleanup`/`Drop`. Mirrors +/// `MgdInstance`. /// /// `ddmd` runs in sled global zones and switch zones in production. Spawned -/// here with `--api-only`, which serves only the admin API and skips the -/// discovery / exchange / routing daemons that need real network interfaces -/// and illumos-only kernel facilities. Only switch-zone instances -/// are registered in internal DNS as `ServiceName::Ddm`; sled-global-zone -/// instances are accessed locally by their own host (RSS, sled-agent's -/// prefix advertisement, etc.) and don't need DNS publication. +/// here with `--api-only`, which serves only the admin API: it does not run +/// DDM peer discovery, the peering handshake, or the transit-routing tail that +/// programs the underlay dataplane, all of which need real network interfaces +/// and illumos-only kernel facilities. No rear-port peers form, so the harness +/// exercises the admin surface alone. Only switch-zone instances are +/// registered in internal DNS as `ServiceName::Ddm`. Sled-global-zone +/// instances are accessed locally by their own host (RSS, sled-agent's prefix +/// advertisement, etc.) and don't need DNS publication. pub struct DdmInstance { - /// Port number the ddmd instance is listening on. + /// Port number the ddmd admin server is listening on. pub port: u16, /// Arguments provided to the `ddmd` cli command. pub args: Vec, @@ -323,7 +362,8 @@ impl DdmInstance { /// /// Instead, this fixture asks the kernel directly via `pfiles` (illumos), /// `ss` (Linux), or `lsof` (macOS), which reports the listening TCP port - /// for `ddmd`'s pid regardless of what `ddmd` chose to log. + /// for `ddmd`'s pid regardless of what `ddmd` chose to log (see + /// [`find_ddm_admin_port`]). /// /// Tracked upstream as oxidecomputer/maghemite#740. Once unified logging /// levels land, this fixture can revert to the simpler log-scrape pattern @@ -341,7 +381,7 @@ impl DdmInstance { temp_dir.path().display().to_string(), ]; - let child = tokio::process::Command::new("ddmd") + let mut child = tokio::process::Command::new("ddmd") .args(&args) .stdin(Stdio::null()) .stdout(Stdio::from(redirect_file(temp_dir.path(), "ddmd_stdout")?)) @@ -353,16 +393,32 @@ impl DdmInstance { let pid = child.id().context("ddmd child has no pid (already exited?)")?; - let temp_dir = temp_dir.keep(); - let port = find_listening_port(pid).await.with_context(|| { - format!( - "failed to discover ddmd listening port for pid {pid} \ - (see {}/ddmd_stdout, ddmd_stderr)", - temp_dir.display() - ) - })?; + // Discover the listening port before calling `temp_dir.keep()` or + // building `DdmInstance`. On this path neither the instance nor its + // `Drop` impl for cleanup exists. Reap the child and capture `ddmd`'s + // stderr before `temp_dir` drops and removes the scratch directory; + // otherwise, a discovery failure would leak a running `ddmd` and its + // tempdir. + let port = match find_ddm_admin_port(pid).await { + Ok(port) => port, + Err(e) => { + let _ = child.start_kill(); + let _ = child.wait().await; + let stderr = std::fs::read_to_string( + temp_dir.path().join("ddmd_stderr"), + ) + .unwrap_or_default(); + return Err(e).with_context(|| { + format!( + "failed to discover ddmd listening port for pid \ + {pid}; ddmd stderr: {stderr}" + ) + }); + } + }; + let temp_dir = temp_dir.keep(); Ok(Self { port, args, child: Some(child), data_dir: Some(temp_dir) }) } diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version index 6bbbc579b69..2a05d43db44 100644 --- a/tools/maghemite_ddm_openapi_version +++ b/tools/maghemite_ddm_openapi_version @@ -1 +1 @@ -COMMIT="5aaa7c9bc2d0e541d29604453dd98cb32676b347" +COMMIT="4cc569836dc1c5d13a5f2396de310220eafee2fe" diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version index 6bbbc579b69..2a05d43db44 100644 --- a/tools/maghemite_mg_openapi_version +++ b/tools/maghemite_mg_openapi_version @@ -1 +1 @@ -COMMIT="5aaa7c9bc2d0e541d29604453dd98cb32676b347" +COMMIT="4cc569836dc1c5d13a5f2396de310220eafee2fe" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums index 10cfd99f5b0..fa7e0190f3f 100644 --- a/tools/maghemite_mgd_checksums +++ b/tools/maghemite_mgd_checksums @@ -1,4 +1,4 @@ -CIDL_SHA256="ae057ecdaed165fbd3eaaa9459c934249d7a366a51bda5c62584fa4bc011be8e" -MGD_LINUX_SHA256="bbf6fcace4a288286a112055761a9b01b791b22ad319e816046ffd097dc3d2c3" -MG_DDM_SHA256="5e05778f0ec0385b2e097f82bd6776942ba93cdf4118c4f72f54c612636d3372" -DDMD_LINUX_SHA256="4b1dcc489960e51b6d7a8ae095c64636bb17e41c0dd356c5ab1521b80dc5ed0c" \ No newline at end of file +CIDL_SHA256="c4ebe683e5f2e2c0745656845739332a5a2b76a9c78fc502e3fe5cf168a9cde4" +MGD_LINUX_SHA256="b7a6c6b835bbf10b09024590093210bcbe44b40528e8a06a7495bc9b06b2238e" +MG_DDM_SHA256="922df0577243659de65390384a5c704fad82e55003cdb444981c98e6d00a70d1" +DDMD_LINUX_SHA256="0a570cf2fe15feb9c2c7a25d0fc95b94ddfa6985cbf3f7b8709ee835c9bf0ad2" \ No newline at end of file