diff --git a/.envrc b/.envrc index 9731258f343..a6ab4f52174 100644 --- a/.envrc +++ b/.envrc @@ -5,6 +5,7 @@ PATH_add out/cockroachdb/bin PATH_add out/clickhouse PATH_add out/dendrite-stub/bin PATH_add out/mgd/root/opt/oxide/mgd/bin +PATH_add out/mg-ddm/root/opt/oxide/mg-ddm/bin if [ "$OMICRON_USE_FLAKE" = 1 ] && nix flake show &> /dev/null then diff --git a/Cargo.toml b/Cargo.toml index d4f6ec9d247..73c418a103d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -456,7 +456,7 @@ cfg-if = "1.0" chrono = { version = "0.4", features = [ "serde" ] } chrono-tz = { version = "0.10.4", features = [ "serde" ] } ciborium = "0.2.2" -clap = { version = "4.5", features = ["cargo", "derive", "env", "wrap_help"] } +clap = { version = "4.6", features = ["cargo", "derive", "env", "wrap_help"] } clickana = { path = "dev-tools/clickana" } clickhouse-admin-api = { path = "clickhouse-admin/api" } clickhouse-admin-keeper-client = { path = "clients/clickhouse-admin-keeper-client" } diff --git a/clients/ddm-admin-client/src/lib.rs b/clients/ddm-admin-client/src/lib.rs index 7a8b56d499d..466a8883918 100644 --- a/clients/ddm-admin-client/src/lib.rs +++ b/clients/ddm-admin-client/src/lib.rs @@ -13,6 +13,7 @@ pub use ddm_admin_client::types; use ddm_admin_client::Client as InnerClient; use either::Either; +use omicron_common::address::DDMD_PORT; use oxnet::Ipv6Net; use sled_hardware_types::underlay::BOOTSTRAP_MASK; use sled_hardware_types::underlay::BOOTSTRAP_PREFIX; @@ -26,9 +27,6 @@ use thiserror::Error; use crate::types::EnableStatsRequest; -// TODO-cleanup Is it okay to hardcode this port number here? -const DDMD_PORT: u16 = 8000; - #[derive(Debug, Error, SlogInlineError)] pub enum DdmError { #[error("Failed to construct an HTTP client:")] diff --git a/dev-tools/downloader/src/lib.rs b/dev-tools/downloader/src/lib.rs index 44fb340de28..a7e5a65683b 100644 --- a/dev-tools/downloader/src/lib.rs +++ b/dev-tools/downloader/src/lib.rs @@ -69,6 +69,9 @@ enum Target { /// Maghemite mgd binary MaghemiteMgd, + /// Maghemite ddmd binary + MaghemiteDdmd, + /// SoftNPU, an admin program (scadm) and a pre-compiled P4 program. Softnpu, @@ -137,6 +140,7 @@ pub async fn run_cmd(args: DownloadArgs) -> Result<()> { Target::Console => downloader.download_console().await, Target::DendriteStub => downloader.download_dendrite_stub().await, Target::MaghemiteMgd => downloader.download_maghemite_mgd().await, + Target::MaghemiteDdmd => downloader.download_maghemite_ddmd().await, Target::Softnpu => downloader.download_softnpu().await, Target::TransceiverControl => { downloader.download_transceiver_control().await @@ -946,6 +950,84 @@ impl Downloader<'_> { Ok(()) } + async fn download_maghemite_ddmd(&self) -> Result<()> { + let download_dir = self.output_dir.join("downloads"); + tokio::fs::create_dir_all(&download_dir).await?; + + let checksums_path = self.versions_dir.join("maghemite_mgd_checksums"); + let [mg_ddm_sha2, ddmd_linux_sha2] = get_values_from_file( + ["MG_DDM_SHA256", "DDMD_LINUX_SHA256"], + &checksums_path, + ) + .await?; + let commit_path = + self.versions_dir.join("maghemite_ddm_openapi_version"); + let [commit] = get_values_from_file(["COMMIT"], &commit_path).await?; + + let repo = "oxidecomputer/maghemite"; + let base_url = format!("{BUILDOMAT_URL}/{repo}/image/{commit}"); + + let filename = "mg-ddm.tar.gz"; + let tarball_path = download_dir.join(filename); + download_file_and_verify( + &self.log, + &tarball_path, + &format!("{base_url}/{filename}"), + ChecksumAlgorithm::Sha2, + &mg_ddm_sha2, + ) + .await?; + unpack_tarball(&self.log, &tarball_path, &download_dir).await?; + + let destination_dir = self.output_dir.join("mg-ddm"); + let _ = tokio::fs::remove_dir_all(&destination_dir).await; + tokio::fs::create_dir_all(&destination_dir).await?; + copy_dir_all( + &download_dir.join("root"), + &destination_dir.join("root"), + )?; + + let binary_dir = destination_dir.join("root/opt/oxide/mg-ddm/bin"); + + match os_name()? { + Os::Linux => { + let filename = "ddmd"; + let path = download_dir.join(filename); + download_file_and_verify( + &self.log, + &path, + &format!( + "{BUILDOMAT_URL}/{repo}/linux/{commit}/{filename}" + ), + ChecksumAlgorithm::Sha2, + &ddmd_linux_sha2, + ) + .await?; + set_permissions(&path, 0o755).await?; + tokio::fs::copy(path, binary_dir.join(filename)).await?; + } + Os::Mac => { + info!( + self.log, + "Building maghemite ddmd from source for macOS" + ); + + let binaries = [("ddmd", &["--no-default-features"][..])]; + + let built_binaries = self + .build_from_git("maghemite", &commit, &binaries) + .await?; + + let dest = binary_dir.join("ddmd"); + tokio::fs::copy(&built_binaries[0], &dest).await?; + set_permissions(&dest, 0o755).await?; + } + Os::Illumos => (), + } + + Ok(()) + } + async fn download_softnpu(&self) -> Result<()> { let destination_dir = self.output_dir.join("npuzone"); tokio::fs::create_dir_all(&destination_dir).await?; diff --git a/env.sh b/env.sh index 6a84c35902a..114b53f07ed 100644 --- a/env.sh +++ b/env.sh @@ -12,6 +12,7 @@ export PATH="$OMICRON_WS/out/cockroachdb/bin:$PATH" export PATH="$OMICRON_WS/out/clickhouse:$PATH" export PATH="$OMICRON_WS/out/dendrite-stub/bin:$PATH" export PATH="$OMICRON_WS/out/mgd/root/opt/oxide/mgd/bin:$PATH" +export PATH="$OMICRON_WS/out/mg-ddm/root/opt/oxide/mg-ddm/bin:$PATH" # if xtrace was set previously, do not unset it case $OLD_SHELL_OPTS in diff --git a/internal-dns/types/src/config.rs b/internal-dns/types/src/config.rs index d5bef144343..5b4f736e2c5 100644 --- a/internal-dns/types/src/config.rs +++ b/internal-dns/types/src/config.rs @@ -163,6 +163,20 @@ pub struct DnsConfigBuilder { service_instances_sleds: BTreeMap>, } +/// Ports for the per-switch services published in internal DNS by +/// [`DnsConfigBuilder::host_zone_switch`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct HostSwitchZonePorts { + /// Dendrite (`dpd`) admin API port. + pub dendrite: u16, + /// Management Gateway Service (`mgs`) port. + pub mgs: u16, + /// Maghemite `mgd` admin API port. + pub mgd: u16, + /// Maghemite `ddmd` admin API port. + pub ddm: u16, +} + /// Describes a host of type "sled" in the control plane DNS zone #[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] pub struct Sled(SledUuid); @@ -396,10 +410,14 @@ impl DnsConfigBuilder { &mut self, sled_id: SledUuid, switch_zone_ip: Ipv6Addr, - dendrite_port: u16, - mgs_port: u16, - mgd_port: u16, + ports: HostSwitchZonePorts, ) -> anyhow::Result<()> { + let HostSwitchZonePorts { + dendrite: dendrite_port, + mgs: mgs_port, + mgd: mgd_port, + ddm: ddm_port, + } = ports; let zone = self.host_dendrite(sled_id, switch_zone_ip)?; self.service_backend_zone(ServiceName::Dendrite, &zone, dendrite_port)?; self.service_backend_zone( @@ -407,7 +425,8 @@ impl DnsConfigBuilder { &zone, mgs_port, )?; - self.service_backend_zone(ServiceName::Mgd, &zone, mgd_port) + self.service_backend_zone(ServiceName::Mgd, &zone, mgd_port)?; + self.service_backend_zone(ServiceName::Ddm, &zone, ddm_port) } /// Higher-level shorthand for adding a Nexus zone with both its internal @@ -731,7 +750,9 @@ impl DnsConfigBuilder { #[cfg(test)] mod test { - use super::{DnsConfigBuilder, Host, ServiceName}; + use super::{ + DnsConfigBuilder, DnsRecord, Host, HostSwitchZonePorts, ServiceName, + }; use crate::{config::Zone, names::DNS_ZONE}; use omicron_common::api::external::Generation; use omicron_uuid_kinds::{OmicronZoneUuid, SledUuid}; @@ -779,6 +800,8 @@ mod test { "_oximeter-reader._tcp", ); assert_eq!(ServiceName::Dendrite.dns_name(), "_dendrite._tcp",); + assert_eq!(ServiceName::Mgd.dns_name(), "_mgd._tcp",); + assert_eq!(ServiceName::Ddm.dns_name(), "_ddm._tcp",); assert_eq!( ServiceName::CruciblePantry.dns_name(), "_crucible-pantry._tcp", @@ -796,6 +819,71 @@ mod test { ); } + #[test] + fn host_zone_switch_publishes_all_services() { + let sled_uuid: SledUuid = + "001de000-51ed-4000-8000-000000000001".parse().unwrap(); + let switch_zone_ip = Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1); + + // Use distinct port numbers so an arg-order swap in `host_zone_switch` + // surfaces as a port mismatch on the affected service. + let dendrite_port = 11; + let mgs_port = 13; + let mgd_port = 17; + let ddm_port = 19; + + let mut builder = DnsConfigBuilder::new(); + builder + .host_zone_switch( + sled_uuid, + switch_zone_ip, + HostSwitchZonePorts { + dendrite: dendrite_port, + mgs: mgs_port, + mgd: mgd_port, + ddm: ddm_port, + }, + ) + .unwrap(); + + let config = builder.build_full_config_for_initial_generation(); + + let mut by_name: BTreeMap<&str, &[DnsRecord]> = BTreeMap::new(); + for zone in &config.zones { + for (name, records) in &zone.records { + by_name.insert(name.as_str(), records.as_slice()); + } + } + + for (expected_name, expected_port) in [ + ("_dendrite._tcp", dendrite_port), + ("_mgs._tcp", mgs_port), + ("_mgd._tcp", mgd_port), + ("_ddm._tcp", ddm_port), + ] { + let records = by_name.get(expected_name).unwrap_or_else(|| { + panic!( + "expected {expected_name} in published switch-zone \ + services; got {by_name:?}" + ) + }); + let srv_port = records + .iter() + .find_map(|r| match r { + DnsRecord::Srv(s) => Some(s.port), + _ => None, + }) + .unwrap_or_else(|| { + panic!("no SRV record for {expected_name}: {records:?}") + }); + + assert_eq!( + srv_port, expected_port, + "wrong SRV port for {expected_name}" + ); + } + } + #[test] fn display_hosts() { let sled_uuid = SledUuid::nil(); diff --git a/internal-dns/types/src/names.rs b/internal-dns/types/src/names.rs index 73b2439e48e..105d0222f3c 100644 --- a/internal-dns/types/src/names.rs +++ b/internal-dns/types/src/names.rs @@ -75,6 +75,7 @@ pub enum ServiceName { BoundaryNtp, InternalNtp, Mgd, + Ddm, } impl ServiceName { @@ -116,6 +117,7 @@ impl ServiceName { ServiceName::BoundaryNtp => "boundary-ntp", ServiceName::InternalNtp => "internal-ntp", ServiceName::Mgd => "mgd", + ServiceName::Ddm => "ddm", } } @@ -144,7 +146,8 @@ impl ServiceName { | ServiceName::CruciblePantry | ServiceName::BoundaryNtp | ServiceName::InternalNtp - | ServiceName::Mgd => { + | ServiceName::Mgd + | ServiceName::Ddm => { format!("_{}._tcp", self.service_kind()) } ServiceName::SledAgent(id) => { diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index cf894eb6b76..395375abc7d 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -63,6 +63,7 @@ jiff.workspace = true lldpd_client.workspace = true macaddr.workspace = true maplit.workspace = true +mg-api-types.workspace = true nexus-background-task-interface.workspace = true # Not under "dev-dependencies"; these also need to be implemented for # integration tests. @@ -96,7 +97,6 @@ qorb.workspace = true rand.workspace = true range-requests.workspace = true ref-cast.workspace = true -mg-api-types.workspace = true regex.workspace = true reqwest = { workspace = true, features = ["http2", "json"] } ring.workspace = true diff --git a/nexus/reconfigurator/execution/src/dns.rs b/nexus/reconfigurator/execution/src/dns.rs index 685c7c85e6f..0a85c4dd114 100644 --- a/nexus/reconfigurator/execution/src/dns.rs +++ b/nexus/reconfigurator/execution/src/dns.rs @@ -988,9 +988,8 @@ mod test { // the previous pass (i.e., that corresponds to an Omicron zone). // // There are some ServiceNames missing here because they are not part of - // our representative config (e.g., ClickhouseKeeper) or they don't - // currently have DNS record at all (e.g., SledAgent, Maghemite, Mgd, - // Tfport). + // our representative config (e.g., ClickhouseKeeper) or because they + // do not currently have a DNS record at all (e.g., SledAgent). let mut srv_kinds_expected = BTreeSet::from([ ServiceName::Clickhouse, ServiceName::ClickhouseNative, @@ -1001,6 +1000,8 @@ mod test { ServiceName::NexusLockstep, ServiceName::Oximeter, ServiceName::Dendrite, + ServiceName::Mgd, + ServiceName::Ddm, ServiceName::CruciblePantry, ServiceName::BoundaryNtp, ServiceName::InternalNtp, diff --git a/nexus/reconfigurator/execution/src/test_utils.rs b/nexus/reconfigurator/execution/src/test_utils.rs index cd46adacd0b..fdb17289225 100644 --- a/nexus/reconfigurator/execution/src/test_utils.rs +++ b/nexus/reconfigurator/execution/src/test_utils.rs @@ -113,10 +113,12 @@ pub fn overridables_for_test( let dendrite_port = cptestctx.dendrite.read().unwrap().get(&switch_slot).unwrap().port; let mgd_port = cptestctx.mgd.get(&switch_slot).unwrap().port; + let ddm_port = cptestctx.ddm.get(&switch_slot).unwrap().port; overrides.override_switch_zone_ip(sled_id, ip); overrides.override_dendrite_port(sled_id, dendrite_port); overrides.override_mgs_port(sled_id, mgs_port); overrides.override_mgd_port(sled_id, mgd_port); + overrides.override_ddm_port(sled_id, ddm_port); } overrides } diff --git a/nexus/reconfigurator/planning/src/example.rs b/nexus/reconfigurator/planning/src/example.rs index a1f865e2934..7dbbf3640dc 100644 --- a/nexus/reconfigurator/planning/src/example.rs +++ b/nexus/reconfigurator/planning/src/example.rs @@ -1854,7 +1854,8 @@ mod tests { | ServiceName::RepoDepot | ServiceName::ManagementGatewayService | ServiceName::Dendrite - | ServiceName::Mgd => { + | ServiceName::Mgd + | ServiceName::Ddm => { out.insert(service, Ok(())); } // InternalNtp is too large to fit in a single DNS packet and diff --git a/nexus/test-utils/src/nexus_test.rs b/nexus/test-utils/src/nexus_test.rs index 28ec67fb3d9..1032d809dc1 100644 --- a/nexus/test-utils/src/nexus_test.rs +++ b/nexus/test-utils/src/nexus_test.rs @@ -117,6 +117,7 @@ pub struct ControlPlaneTestContext { /// Ports of stopped dendrite instances (for use by start_dendrite) pub stopped_dendrite_ports: RwLock>, pub mgd: HashMap, + pub ddm: HashMap, pub external_dns_zone_name: String, pub external_dns: TransientDnsServer, pub internal_dns: TransientDnsServer, @@ -320,6 +321,9 @@ impl ControlPlaneTestContext { for (_, mut mgd) in self.mgd { mgd.cleanup().await.unwrap(); } + for (_, mut ddm) in self.ddm { + ddm.cleanup().await.unwrap(); + } self.logctx.cleanup_successful(); } } diff --git a/nexus/test-utils/src/starter.rs b/nexus/test-utils/src/starter.rs index 3822be75d1f..bb84aed361c 100644 --- a/nexus/test-utils/src/starter.rs +++ b/nexus/test-utils/src/starter.rs @@ -23,6 +23,7 @@ use futures::future::BoxFuture; use gateway_test_utils::setup::GatewayTestContext; use iddqd::IdOrdMap; use internal_dns_types::config::DnsConfigBuilder; +use internal_dns_types::config::HostSwitchZonePorts; use internal_dns_types::names::DNS_ZONE_EXTERNAL_TESTING; use internal_dns_types::names::ServiceName; use nexus_config::Database; @@ -146,6 +147,7 @@ pub struct ControlPlaneStarter<'a, N: NexusServer> { pub gateway: BTreeMap, pub dendrite: RwLock>, pub mgd: HashMap, + pub ddm: HashMap, // NOTE: Only exists after starting Nexus, until external Nexus is // initialized. @@ -203,6 +205,7 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { gateway: BTreeMap::new(), dendrite: RwLock::new(HashMap::new()), mgd: HashMap::new(), + ddm: HashMap::new(), nexus_internal: None, nexus_internal_addr: None, external_dns_zone_name: None, @@ -465,6 +468,17 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { self.config.pkg.mgd.insert(switch_slot, config); } + pub async fn start_ddm(&mut self, switch_slot: SwitchSlot) { + let log = &self.logctx.log; + debug!(log, "Starting DDM sim"; "switch_slot" => ?switch_slot); + + let ddm = dev::maghemite::DdmInstance::start().await.unwrap(); + let port = ddm.port; + self.ddm.insert(switch_slot, ddm); + + debug!(log, "DDM sim started"; "port" => port); + } + pub async fn record_switch_dns( &mut self, sled_id: SledUuid, @@ -483,9 +497,18 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { .host_zone_switch( sled_id, Ipv6Addr::LOCALHOST, - self.dendrite.read().unwrap().get(&switch_slot).unwrap().port, - self.gateway.get(&switch_slot).unwrap().port, - self.mgd.get(&switch_slot).unwrap().port, + HostSwitchZonePorts { + dendrite: self + .dendrite + .read() + .unwrap() + .get(&switch_slot) + .unwrap() + .port, + mgs: self.gateway.get(&switch_slot).unwrap().port, + mgd: self.mgd.get(&switch_slot).unwrap().port, + ddm: self.ddm.get(&switch_slot).unwrap().port, + }, ) .unwrap() } @@ -1254,6 +1277,7 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { dendrite: RwLock::new(self.dendrite.into_inner().unwrap()), stopped_dendrite_ports: RwLock::new(HashMap::new()), mgd: self.mgd, + ddm: self.ddm, external_dns_zone_name: self.external_dns_zone_name.unwrap(), external_dns: self.external_dns.unwrap(), internal_dns: self.internal_dns.unwrap(), @@ -1295,6 +1319,9 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { for (_, mut mgd) in self.mgd { mgd.cleanup().await.unwrap(); } + for (_, mut ddm) in self.ddm { + ddm.cleanup().await.unwrap(); + } self.logctx.cleanup_successful(); } @@ -1635,6 +1662,12 @@ pub(crate) async fn setup_with_config_impl( builder.start_mgd(SwitchSlot::Switch0).boxed() }), ), + ( + "start_ddm_switch0", + Box::new(|builder| { + builder.start_ddm(SwitchSlot::Switch0).boxed() + }), + ), ( "record_switch_dns", Box::new(|builder| { @@ -1679,6 +1712,12 @@ pub(crate) async fn setup_with_config_impl( builder.start_mgd(SwitchSlot::Switch1).boxed() }), ), + ( + "start_ddm_switch1", + Box::new(|builder| { + builder.start_ddm(SwitchSlot::Switch1).boxed() + }), + ), ( "record_switch_dns", Box::new(|builder| { diff --git a/nexus/tests/integration_tests/initialization.rs b/nexus/tests/integration_tests/initialization.rs index 350757cf1de..714880feb37 100644 --- a/nexus/tests/integration_tests/initialization.rs +++ b/nexus/tests/integration_tests/initialization.rs @@ -158,6 +158,11 @@ async fn test_nexus_boots_before_dendrite() { starter.start_mgd(SwitchSlot::Switch1).await; info!(log, "Started mgd"); + info!(log, "Starting ddm"); + starter.start_ddm(SwitchSlot::Switch0).await; + starter.start_ddm(SwitchSlot::Switch1).await; + info!(log, "Started ddm"); + info!(log, "Populating internal DNS records"); starter .record_switch_dns( @@ -197,6 +202,8 @@ async fn nexus_schema_test_setup( starter.start_dendrite(SwitchSlot::Switch1).await; starter.start_mgd(SwitchSlot::Switch0).await; starter.start_mgd(SwitchSlot::Switch1).await; + starter.start_ddm(SwitchSlot::Switch0).await; + starter.start_ddm(SwitchSlot::Switch1).await; starter.populate_internal_dns().await; } diff --git a/nexus/types/src/deployment/execution/dns.rs b/nexus/types/src/deployment/execution/dns.rs index 009377fd8d9..c901dcc92f7 100644 --- a/nexus/types/src/deployment/execution/dns.rs +++ b/nexus/types/src/deployment/execution/dns.rs @@ -155,9 +155,7 @@ pub fn blueprint_internal_dns_config( dns_builder.host_zone_switch( scrimlet.id(), switch_zone_ip, - overrides.dendrite_port(scrimlet.id()), - overrides.mgs_port(scrimlet.id()), - overrides.mgd_port(scrimlet.id()), + overrides.host_switch_zone_ports(scrimlet.id()), )?; } diff --git a/nexus/types/src/deployment/execution/overridables.rs b/nexus/types/src/deployment/execution/overridables.rs index 881a7c49bdd..bf46374d1dc 100644 --- a/nexus/types/src/deployment/execution/overridables.rs +++ b/nexus/types/src/deployment/execution/overridables.rs @@ -2,6 +2,8 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use internal_dns_types::config::HostSwitchZonePorts; +use omicron_common::address::DDMD_PORT; use omicron_common::address::DENDRITE_PORT; use omicron_common::address::Ipv6Subnet; use omicron_common::address::MGD_PORT; @@ -29,6 +31,8 @@ pub struct Overridables { pub mgs_ports: BTreeMap, /// map: sled id -> TCP port on which that sled's MGD is listening pub mgd_ports: BTreeMap, + /// map: sled id -> TCP port on which that sled's DDM is listening + pub ddm_ports: BTreeMap, /// map: sled id -> IP address of the sled's switch zone pub switch_zone_ips: BTreeMap, } @@ -67,6 +71,32 @@ impl Overridables { self.mgd_ports.get(&sled_id).copied().unwrap_or(MGD_PORT) } + /// Specify the TCP port on which this sled's DDM is listening + pub fn override_ddm_port(&mut self, sled_id: SledUuid, port: u16) { + self.ddm_ports.insert(sled_id, port); + } + + /// Returns the TCP port on which this sled's DDM is listening + pub fn ddm_port(&self, sled_id: SledUuid) -> u16 { + self.ddm_ports.get(&sled_id).copied().unwrap_or(DDMD_PORT) + } + + /// Returns the per-switch-zone service ports for this sled. + /// + /// Bundles the four switch-zone admin ports into a single + /// [`HostSwitchZonePorts`] so callers cannot swap fields by accident. + pub fn host_switch_zone_ports( + &self, + sled_id: SledUuid, + ) -> HostSwitchZonePorts { + HostSwitchZonePorts { + dendrite: self.dendrite_port(sled_id), + mgs: self.mgs_port(sled_id), + mgd: self.mgd_port(sled_id), + ddm: self.ddm_port(sled_id), + } + } + /// Specify the IP address of this switch zone pub fn override_switch_zone_ip( &mut self, diff --git a/sled-agent/early-networking/Cargo.toml b/sled-agent/early-networking/Cargo.toml index 4d662b6ea03..8fb7b9e21f7 100644 --- a/sled-agent/early-networking/Cargo.toml +++ b/sled-agent/early-networking/Cargo.toml @@ -19,11 +19,11 @@ http.workspace = true internal-dns-resolver.workspace = true internal-dns-types.workspace = true mg-admin-client.workspace = true +mg-api-types.workspace = true omicron-common.workspace = true omicron-ddm-admin-client.workspace = true omicron-workspace-hack.workspace = true oxnet.workspace = true -mg-api-types.workspace = true sled-agent-types.workspace = true slog.workspace = true slog-error-chain.workspace = true diff --git a/sled-agent/rack-setup/src/plan/service.rs b/sled-agent/rack-setup/src/plan/service.rs index d0ea30fc4fa..7a8cbcb0174 100644 --- a/sled-agent/rack-setup/src/plan/service.rs +++ b/sled-agent/rack-setup/src/plan/service.rs @@ -13,7 +13,7 @@ use iddqd::errors::DuplicateItem; use iddqd::id_upcast; use illumos_utils::zpool::ZpoolName; use internal_dns_types::config::{ - DnsConfigBuilder, DnsConfigParams, Host, Zone, + DnsConfigBuilder, DnsConfigParams, Host, HostSwitchZonePorts, Zone, }; use internal_dns_types::names::ServiceName; use nexus_types::deployment::LastAllocatedSubnetIpOffset; @@ -29,10 +29,10 @@ use nexus_types::deployment::{ }; use nexus_types::external_api::sled::SledState; use omicron_common::address::{ - CP_SERVICES_RESERVED_ADDRESSES, DENDRITE_PORT, DNS_HTTP_PORT, DNS_PORT, - Ipv6Subnet, MGD_PORT, MGS_PORT, NEXUS_INTERNAL_PORT, NEXUS_LOCKSTEP_PORT, - NTP_PORT, NUM_SOURCE_NAT_PORTS, REPO_DEPOT_PORT, ReservedRackSubnet, - SLED_PREFIX, SLED_RESERVED_ADDRESSES, get_sled_address, + CP_SERVICES_RESERVED_ADDRESSES, DDMD_PORT, DENDRITE_PORT, DNS_HTTP_PORT, + DNS_PORT, Ipv6Subnet, MGD_PORT, MGS_PORT, NEXUS_INTERNAL_PORT, + NEXUS_LOCKSTEP_PORT, NTP_PORT, NUM_SOURCE_NAT_PORTS, REPO_DEPOT_PORT, + ReservedRackSubnet, SLED_PREFIX, SLED_RESERVED_ADDRESSES, get_sled_address, get_switch_zone_address, }; use omicron_common::api::external::{Generation, MacAddr, Vni}; @@ -338,9 +338,12 @@ impl ServicePlan { .host_zone_switch( sled.sled_id, address, - DENDRITE_PORT, - MGS_PORT, - MGD_PORT, + HostSwitchZonePorts { + dendrite: DENDRITE_PORT, + mgs: MGS_PORT, + mgd: MGD_PORT, + ddm: DDMD_PORT, + }, ) .unwrap(); } diff --git a/test-utils/src/dev/maghemite.rs b/test-utils/src/dev/maghemite.rs index eaae1af8cd4..a7091d38508 100644 --- a/test-utils/src/dev/maghemite.rs +++ b/test-utils/src/dev/maghemite.rs @@ -135,11 +135,123 @@ fn redirect_file( .with_context(|| format!("open \"{}\"", out_path.display())) } +/// Per-OS configuration for asking the kernel which TCP port a pid is +/// listening on. +struct PortProbe { + /// Command to invoke (`pfiles` on illumos, `ss` on Linux). + command: &'static str, + /// Arguments to pass to `command`. + args: Vec, + /// Substring that must appear on a line for it to belong to the target + /// pid. `None` when the command is already pid-scoped (e.g. `pfiles + /// `). + pid_marker: Option, + /// Pattern that extracts the listening port into capture group 1. + port_re: regex::Regex, +} + +impl PortProbe { + #[cfg(target_os = "illumos")] + fn for_pid(pid: u32) -> Self { + // `pfiles ` lines look like: + // sockname: AF_INET6 ::1 port: 41065 + Self { + command: "pfiles", + args: vec![pid.to_string()], + pid_marker: None, + port_re: regex::Regex::new( + r"sockname:\s+AF_INET6\s+::1?\s+port:\s+(\d+)", + ) + .unwrap(), + } + } + + #[cfg(target_os = "linux")] + fn for_pid(pid: u32) -> Self { + // `ss -tlnpH` lines look like: + // LISTEN 0 128 [::1]:41065 [::]:* users:(("ddmd",pid=12345,fd=8)) + Self { + command: "ss", + args: vec!["-tlnpH".to_string()], + pid_marker: Some(format!("pid={pid}")), + port_re: regex::Regex::new(r"\[::1?\]:(\d+)").unwrap(), + } + } + + #[cfg(target_os = "macos")] + fn for_pid(pid: u32) -> Self { + // `lsof` is part of base macOS, so this compiles and runs without + // any extra install. `lsof -nP -p -iTCP -sTCP:LISTEN` lines + // look like: + // ddmd 12345 user 10u IPv6 0x... 0t0 TCP [::1]:41065 (LISTEN) + Self { + command: "lsof", + args: vec![ + "-nP".to_string(), + "-p".to_string(), + pid.to_string(), + "-iTCP".to_string(), + "-sTCP:LISTEN".to_string(), + ], + pid_marker: None, + port_re: regex::Regex::new(r"\[::1?\]:(\d+)").unwrap(), + } + } + + async fn probe(&self) -> Result, anyhow::Error> { + let output = tokio::process::Command::new(self.command) + .args(&self.args) + .output() + .await + .with_context(|| format!("running {}", self.command))?; + if !output.status.success() { + // The probe command can transiently fail (process exiting, + // permissions, etc.); leave it to the caller to retry. + return Ok(None); + } + let text = std::str::from_utf8(&output.stdout) + .with_context(|| format!("{} output not utf8", self.command))?; + Ok(text + .lines() + .filter(|line| { + self.pid_marker.as_deref().is_none_or(|m| line.contains(m)) + }) + .find_map(|line| { + self.port_re + .captures(line)? + .get(1)? + .as_str() + .parse::() + .ok() + })) + } +} + +/// Ask the kernel which TCP port `pid` is listening on. Retries until either +/// the port is found or [`DDMD_TIMEOUT`] elapses, since `ddmd` may not have +/// finished binding when we first probe. +async fn find_listening_port(pid: u32) -> Result { + let probe = PortProbe::for_pid(pid); + let deadline = Instant::now() + DDMD_TIMEOUT; + loop { + if let Some(port) = probe.probe().await? { + return Ok(port); + } + if Instant::now() >= deadline { + anyhow::bail!( + "kernel reports no listening TCP port for pid {pid} after \ + {DDMD_TIMEOUT:?}" + ); + } + sleep(Duration::from_millis(100)).await; + } +} + async fn discover_port(logfile: String) -> Result { let timeout = Instant::now() + MGD_TIMEOUT; tokio::time::timeout_at(timeout, find_mgd_port_in_log(logfile)) .await - .context("time out while discovering mgd port number")? + .context("time out while discovering port number")? } async fn find_mgd_port_in_log(logfile: String) -> Result { @@ -169,6 +281,126 @@ async fn find_mgd_port_in_log(logfile: String) -> Result { } } +/// Specifies the amount of time we will wait for `ddmd` to bind its admin +/// port, confirmed by asking the kernel which TCP port `ddmd`'s pid is +/// listening on. +const DDMD_TIMEOUT: Duration = Duration::from_secs(5); + +/// Test fixture that spawns and supervises a legit `ddmd` subprocess. +/// +/// Owns a `tokio::process::Child` and a tempdir; discovers the bound admin +/// port by scraping dropshot's startup `local_addr` records; kills the child +/// on `cleanup`/`Drop`. Mirrors `MgdInstance`. +/// +/// `ddmd` runs in sled global zones and switch zones in production. Spawned +/// here with `--api-only`, which serves only the admin API and skips the +/// discovery / exchange / routing daemons that need real network interfaces +/// and illumos-only kernel facilities. Only switch-zone instances +/// are registered in internal DNS as `ServiceName::Ddm`; sled-global-zone +/// instances are accessed locally by their own host (RSS, sled-agent's +/// prefix advertisement, etc.) and don't need DNS publication. +pub struct DdmInstance { + /// Port number the ddmd instance is listening on. + pub port: u16, + /// Arguments provided to the `ddmd` cli command. + pub args: Vec, + /// Child process spawned by running `ddmd`. + pub child: Option, + /// Temporary directory where logging output and other files generated by + /// `ddmd` are stored. + pub data_dir: Option, +} + +impl DdmInstance { + /// Start a `ddmd` instance with `--api-only`, bound to an auto-assigned + /// admin port on localhost. + /// + /// `MgdInstance` discovers its admin port by scraping `local_addr` from + /// `mgd`'s startup logs. That approach does not work for `ddmd`: the + /// dropshot endpoint-registration records that carry `local_addr` are + /// emitted at debug level, and `ddmd`'s slog drain is currently focused + /// on errors, so those lines never reach stdout. + /// + /// Instead, this fixture asks the kernel directly via `pfiles` (illumos), + /// `ss` (Linux), or `lsof` (macOS), which reports the listening TCP port + /// for `ddmd`'s pid regardless of what `ddmd` chose to log. + /// + /// Tracked upstream as oxidecomputer/maghemite#740. Once unified logging + /// levels land, this fixture can revert to the simpler log-scrape pattern + /// that `MgdInstance` uses. + pub async fn start() -> Result { + let temp_dir = TempDir::new()?; + + let args = vec![ + "--admin-addr".to_string(), + "::1".into(), + "--admin-port".into(), + "0".into(), + "--api-only".into(), + "--data-dir".into(), + temp_dir.path().display().to_string(), + ]; + + let child = tokio::process::Command::new("ddmd") + .args(&args) + .stdin(Stdio::null()) + .stdout(Stdio::from(redirect_file(temp_dir.path(), "ddmd_stdout")?)) + .stderr(Stdio::from(redirect_file(temp_dir.path(), "ddmd_stderr")?)) + .spawn() + .with_context(|| { + format!("failed to spawn `ddmd` (with args: {:?})", &args) + })?; + + let pid = + child.id().context("ddmd child has no pid (already exited?)")?; + let temp_dir = temp_dir.keep(); + + let port = find_listening_port(pid).await.with_context(|| { + format!( + "failed to discover ddmd listening port for pid {pid} \ + (see {}/ddmd_stdout, ddmd_stderr)", + temp_dir.display() + ) + })?; + + Ok(Self { port, args, child: Some(child), data_dir: Some(temp_dir) }) + } + + pub async fn cleanup(&mut self) -> Result<(), anyhow::Error> { + if let Some(mut child) = self.child.take() { + child.start_kill().context("Sending SIGKILL to child")?; + child.wait().await.context("waiting for child")?; + } + if let Some(dir) = self.data_dir.take() { + std::fs::remove_dir_all(&dir).with_context(|| { + format!("cleaning up temporary directory {}", dir.display()) + })?; + } + Ok(()) + } +} + +impl Drop for DdmInstance { + fn drop(&mut self) { + if self.child.is_some() || self.data_dir.is_some() { + eprintln!( + "WARN: dropped DdmInstance without cleaning it up first \ + (there may still be a child process running and a \ + temporary directory leaked)" + ); + if let Some(child) = self.child.as_mut() { + let _ = child.start_kill(); + } + if let Some(path) = self.data_dir.take() { + eprintln!( + "WARN: ddmd temporary directory leaked: {}", + path.display() + ); + } + } + } +} + #[cfg(test)] mod tests { use super::find_mgd_port_in_log; @@ -189,6 +421,16 @@ mod tests { .expect("Cannot find 'mgd' on PATH. Refer to README.md for installation instructions"); } + #[tokio::test] + async fn test_ddmd_in_path() { + tokio::process::Command::new("ddmd") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("Cannot find 'ddmd' on PATH. Refer to README.md for installation instructions"); + } + #[tokio::test] async fn test_discover_local_listening_port() { // Write some data to a fake log file diff --git a/tools/install_builder_prerequisites.sh b/tools/install_builder_prerequisites.sh index 0f1df7d2528..d79a923ca1f 100755 --- a/tools/install_builder_prerequisites.sh +++ b/tools/install_builder_prerequisites.sh @@ -230,6 +230,7 @@ retry xtask download \ console \ dendrite-stub \ maghemite-mgd \ + maghemite-ddmd \ transceiver-control # Validate the PATH: diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums index ea34493cd53..f048fb28e64 100644 --- a/tools/maghemite_mgd_checksums +++ b/tools/maghemite_mgd_checksums @@ -1,2 +1,4 @@ CIDL_SHA256="5a0c407a91c1fb68af5770a33549e9d1695653f476520562a087eb9f42bd9d25" -MGD_LINUX_SHA256="29e426fdd80e880ca93b0817a9c91149e01aadf131c9be59749da78da9678c28" \ No newline at end of file +MGD_LINUX_SHA256="29e426fdd80e880ca93b0817a9c91149e01aadf131c9be59749da78da9678c28" +MG_DDM_SHA256="7680d445cce94bbfa2ea1b952086d2ca7c0410cc3f34f3386befc4a16c38261d" +DDMD_LINUX_SHA256="8240ccfa3029630d29c947ac58b9c9a7f8b2c999f6b5b75112af7ca1ee35d798" \ No newline at end of file diff --git a/tools/update_maghemite.sh b/tools/update_maghemite.sh index 0051397b51d..0a482cb4440 100755 --- a/tools/update_maghemite.sh +++ b/tools/update_maghemite.sh @@ -54,6 +54,12 @@ function update_mgd { SHA_LINUX=$(get_sha "$REPO" "$TARGET_COMMIT" "mgd" "linux") OUTPUT_LINUX=$(printf "MGD_LINUX_SHA256=\"%s\"\n" "$SHA_LINUX") + SHA_MG_DDM=$(get_sha "$REPO" "$TARGET_COMMIT" "mg-ddm" "image") + OUTPUT_MG_DDM=$(printf "MG_DDM_SHA256=\"%s\"\n" "$SHA_MG_DDM") + + SHA_DDMD_LINUX=$(get_sha "$REPO" "$TARGET_COMMIT" "ddmd" "linux") + OUTPUT_DDMD_LINUX=$(printf "DDMD_LINUX_SHA256=\"%s\"\n" "$SHA_DDMD_LINUX") + if [ -n "$DRY_RUN" ]; then MGD_PATH="/dev/null" else @@ -61,7 +67,7 @@ function update_mgd { fi echo "Updating Maghemite mgd from: $TARGET_COMMIT" set -x - printf "$OUTPUT\n$OUTPUT_LINUX" > $MGD_PATH + printf "$OUTPUT\n$OUTPUT_LINUX\n$OUTPUT_MG_DDM\n$OUTPUT_DDMD_LINUX" > $MGD_PATH set +x }