diff --git a/.envrc b/.envrc index 9731258f343..245e44825bf 100644 --- a/.envrc +++ b/.envrc @@ -5,6 +5,7 @@ PATH_add out/cockroachdb/bin PATH_add out/clickhouse PATH_add out/dendrite-stub/bin PATH_add out/mgd/root/opt/oxide/mgd/bin +PATH_add out/lldp/root/opt/oxide/bin if [ "$OMICRON_USE_FLAKE" = 1 ] && nix flake show &> /dev/null then diff --git a/Cargo.lock b/Cargo.lock index e83da904c92..a3b843eb6af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6387,6 +6387,18 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "loopback-ip-mgr" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/falcon?rev=aa44de0ab2edd702322ef60dd9fec1d1e70348ad#aa44de0ab2edd702322ef60dd9fec1d1e70348ad" +dependencies = [ + "libc", + "libnet", + "network-interface", + "oxnet", + "slog", +] + [[package]] name = "lpc55_areas" version = "0.2.5" @@ -6727,6 +6739,17 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca2b420f638f07fe83056b55ea190bb815f609ec5a35e7017884a10f78839c9e" +[[package]] +name = "network-interface" +version = "0.1.7" +source = "git+https://github.com/oxidecomputer/network-interface?branch=illumos#5a696e910333bdc50ef56cebe9cdd78e40127d87" +dependencies = [ + "cc", + "libc", + "thiserror 1.0.69", + "winapi", +] + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -7717,6 +7740,7 @@ dependencies = [ "illumos-utils", "internal-dns-resolver", "internal-dns-types", + "loopback-ip-mgr", "nexus-client", "nexus-config", "nexus-db-queries", @@ -8469,6 +8493,8 @@ dependencies = [ "gateway-client", "gateway-test-utils", "libc", + "loopback-ip-mgr", + "mg-admin-client", "nexus-config", "nexus-test-interface", "nexus-test-utils", @@ -8481,6 +8507,10 @@ dependencies = [ "oxide-tokio-rt", "pq-sys", "signal-hook-tokio", + "sled-agent-types", + "slog", + "slog-async", + "slog-term", "subprocess", "tokio", "tokio-postgres", @@ -9317,6 +9347,7 @@ dependencies = [ "hex", "http", "libc", + "loopback-ip-mgr", "nexus-config", "omicron-common", "omicron-workspace-hack", diff --git a/Cargo.toml b/Cargo.toml index 0a9cc97e5c7..69897c155af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -613,6 +613,7 @@ ntp-admin-types = { path = "ntp-admin/types" } ntp-admin-types-versions = { path = "ntp-admin/types/versions" } mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "7696ee48d5ee29a917dea459e281fe2e8ff20513" } ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "7696ee48d5ee29a917dea459e281fe2e8ff20513" } +loopback-ip-mgr = { git = "https://github.com/oxidecomputer/falcon", rev = "aa44de0ab2edd702322ef60dd9fec1d1e70348ad" } multimap = "0.10.1" nexus-auth = { path = "nexus/auth" } nexus-background-task-interface = { path = "nexus/background-task-interface" } diff --git a/dev-tools/downloader/src/lib.rs b/dev-tools/downloader/src/lib.rs index 44fb340de28..c2a8df82b58 100644 --- a/dev-tools/downloader/src/lib.rs +++ b/dev-tools/downloader/src/lib.rs @@ -74,6 +74,9 @@ enum Target { /// Transceiver Control binary TransceiverControl, + + /// LLDP binary + LLDP, } #[derive(Parser)] @@ -136,6 +139,7 @@ pub async fn run_cmd(args: DownloadArgs) -> Result<()> { Target::Cockroach => downloader.download_cockroach().await, Target::Console => downloader.download_console().await, Target::DendriteStub => downloader.download_dendrite_stub().await, + Target::LLDP => downloader.download_lldp().await, Target::MaghemiteMgd => downloader.download_maghemite_mgd().await, Target::Softnpu => downloader.download_softnpu().await, Target::TransceiverControl => { @@ -870,6 +874,80 @@ impl Downloader<'_> { Ok(()) } + async fn download_lldp(&self) -> std::result::Result<(), anyhow::Error> { + let download_dir = self.output_dir.join("downloads"); + tokio::fs::create_dir_all(&download_dir).await?; + + let checksums_path = self.versions_dir.join("lldp_checksums"); + let [lldp_sha2, lldp_linux_sha2] = get_values_from_file( + ["CIDL_SHA256", "LINUX_SHA256"], + &checksums_path, + ) + .await?; + let commit_path = self.versions_dir.join("lldp_openapi_version"); + let [commit] = get_values_from_file(["COMMIT"], &commit_path).await?; + + let repo = "oxidecomputer/lldp"; + let base_url = format!("{BUILDOMAT_URL}/{repo}/image/{commit}"); + + let filename = "lldp.tar.gz"; + let tarball_path = download_dir.join(filename); + download_file_and_verify( + &self.log, + &tarball_path, + &format!("{base_url}/{filename}"), + ChecksumAlgorithm::Sha2, + &lldp_sha2, + ) + .await?; + unpack_tarball(&self.log, &tarball_path, &download_dir).await?; + + let destination_dir = self.output_dir.join("lldp"); + let _ = tokio::fs::remove_dir_all(&destination_dir).await; + tokio::fs::create_dir_all(&destination_dir).await?; + copy_dir_all( + &download_dir.join("root"), + &destination_dir.join("root"), + )?; + + let binary_dir = destination_dir.join("root/opt/oxide/bin"); + + match os_name()? { + Os::Linux => { + let filename = "lldpd"; + let path = download_dir.join(filename); + download_file_and_verify( + &self.log, + &path, + &format!( + "{BUILDOMAT_URL}/{repo}/linux/{commit}/{filename}" + ), + ChecksumAlgorithm::Sha2, + &lldp_linux_sha2, + ) + .await?; + set_permissions(&path, 0o755).await?; + tokio::fs::copy(path, binary_dir.join(filename)).await?; + } + Os::Mac => { + info!(self.log, "Building lldp from source for macOS"); + + let binaries = [("lldpd", &["--no-default-features"][..])]; + + let built_binaries = + self.build_from_git("lldp", &commit, &binaries).await?; + + // Copy built binary to binary_dir + let dest = binary_dir.join("lldp"); + tokio::fs::copy(&built_binaries[0], &dest).await?; + set_permissions(&dest, 0o755).await?; + } + Os::Illumos => (), + } + + Ok(()) + } + async fn download_maghemite_mgd(&self) -> Result<()> { let download_dir = self.output_dir.join("downloads"); tokio::fs::create_dir_all(&download_dir).await?; diff --git a/dev-tools/omicron-dev/Cargo.toml b/dev-tools/omicron-dev/Cargo.toml index a21ed7e90a6..c21ac112781 100644 --- a/dev-tools/omicron-dev/Cargo.toml +++ b/dev-tools/omicron-dev/Cargo.toml @@ -19,15 +19,22 @@ futures.workspace = true gateway-client.workspace = true gateway-test-utils.workspace = true libc.workspace = true +loopback-ip-mgr.workspace = true +sled-agent-types.workspace = true +mg-admin-client.workspace = true nexus-config.workspace = true nexus-test-interface.workspace = true nexus-test-utils = { workspace = true, features = ["omicron-dev"] } +omicron-test-utils.workspace = true omicron-nexus.workspace = true omicron-workspace-hack.workspace = true oxide-tokio-rt.workspace = true # See omicron-rpaths for more about the "pq-sys" dependency. pq-sys = "*" signal-hook-tokio.workspace = true +slog.workspace=true +slog-term.workspace=true +slog-async.workspace=true tokio.workspace = true toml.workspace = true @@ -38,7 +45,6 @@ fmd-adm-sys.workspace = true [dev-dependencies] expectorate.workspace = true omicron-dev-lib.workspace = true -omicron-test-utils.workspace = true oxide-client.workspace = true subprocess.workspace = true tokio-postgres.workspace = true diff --git a/dev-tools/omicron-dev/src/main.rs b/dev-tools/omicron-dev/src/main.rs index 4f52e32e07e..45b0b957747 100644 --- a/dev-tools/omicron-dev/src/main.rs +++ b/dev-tools/omicron-dev/src/main.rs @@ -13,7 +13,14 @@ use nexus_config::NexusConfig; use nexus_test_interface::NexusServer; use nexus_test_utils::resource_helpers::DiskTest; use signal_hook_tokio::Signals; -use std::fs; +use sled_agent_types::early_networking::SwitchSlot; +use slog::{Drain, o}; +use std::{ + collections::BTreeMap, + fs, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV6}, + sync::{Arc, Mutex}, +}; const DEFAULT_NEXUS_CONFIG: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../nexus/examples/config.toml"); @@ -37,6 +44,7 @@ impl OmicronDevApp { async fn exec(&self) -> Result<(), anyhow::Error> { match &self.command { OmicronDevCmd::RunAll(args) => args.exec().await, + OmicronDevCmd::RunMultiple(args) => args.exec().await, } } } @@ -45,6 +53,8 @@ impl OmicronDevApp { enum OmicronDevCmd { /// Run a full simulated control plane RunAll(RunAllArgs), + /// Run multiple simulated control planes + RunMultiple(RunMultipleArgs), } #[derive(Clone, Debug, Args)] @@ -60,24 +70,23 @@ struct RunAllArgs { nexus_config: Utf8PathBuf, } +trait Configurable { + fn nexus_config(&self) -> &Utf8PathBuf; +} + +impl Configurable for RunAllArgs { + fn nexus_config(&self) -> &Utf8PathBuf { + &self.nexus_config + } +} + impl RunAllArgs { async fn exec(&self) -> Result<(), anyhow::Error> { // Start a stream listening for SIGINT - let signals = - Signals::new(&[SIGINT]).expect("failed to wait for SIGINT"); - let mut signal_stream = signals.fuse(); + let mut signal_stream = start_stream(); // Read configuration. - let config_str = fs::read_to_string(&self.nexus_config)?; - let mut config: NexusConfig = toml::from_str(&config_str).context( - format!("parsing config: {}", self.nexus_config.as_str()), - )?; - config.pkg.log = dropshot::ConfigLogging::File { - // See LogContext::new(), - path: "UNUSED".to_string().into(), - level: dropshot::ConfigLoggingLevel::Trace, - if_exists: dropshot::ConfigLoggingIfExists::Fail, - }; + let mut config = read_config(self)?; if let Some(p) = self.nexus_listen_port { config @@ -91,7 +100,7 @@ impl RunAllArgs { println!("omicron-dev: setting up all services ... "); let cptestctx = nexus_test_utils::omicron_dev_setup_with_config::< omicron_nexus::Server, - >(&mut config, 0, self.gateway_config.clone()) + >(&mut config, 1, self.gateway_config.clone()) .await .context("error setting up services")?; @@ -179,6 +188,28 @@ impl RunAllArgs { switch_slot, ); } + for (location, dendrite) in cptestctx.dendrite.read().unwrap().iter() { + println!( + "omicron-dev: dendrite: http://[::1]:{} ({:?})", + dendrite.port, location, + ); + } + for (location, lldpd) in &cptestctx.lldpd { + println!( + "omicron-dev: lldp: http://[::1]:{} ({:?})", + lldpd.port, location, + ); + } + for (location, mgd) in &cptestctx.mgd { + println!( + "omicron-dev: mgd api: http://[::1]:{} ({:?})", + mgd.port, location, + ); + println!( + "omicron-dev: mgd bgp-dispatcher: tcp {} ({:?})", + mgd.bgp_dispatcher_addr, location, + ); + } println!( "omicron-dev: silo name: {}", cptestctx.silo_name, @@ -194,10 +225,561 @@ impl RunAllArgs { assert_eq!(caught_signal.unwrap(), SIGINT); eprintln!( "omicron-dev: caught signal, shutting down and removing \ - temporary directory" + temporary directory" ); cptestctx.teardown().await; + + Ok(()) + } +} + +fn slot_index(slot: SwitchSlot) -> u32 { + match slot { + SwitchSlot::Switch0 => 0, + SwitchSlot::Switch1 => 1, + } +} + +fn ipv4_as_u32(a: u8, b: u8, c: u8, d: u8) -> u32 { + (u32::from(a) << 24) + | (u32::from(b) << 16) + | (u32::from(c) << 8) + | u32::from(d) +} + +fn make_neighbor( + local_asn: u32, + name: String, + peer_addr: SocketAddr, + src_addr: Option, +) -> mg_admin_client::types::Neighbor { + mg_admin_client::types::Neighbor { + asn: local_asn, + name, + group: "omicron-dev".to_string(), + host: peer_addr.to_string(), + hold_time: 6000, + keepalive: 2000, + connect_retry: 3000, + idle_hold_time: 0, + delay_open: 0, + resolution: 100, + passive: false, + enforce_first_as: false, + deterministic_collision_resolution: true, + communities: vec![], + ipv4_unicast: Some(mg_admin_client::types::Ipv4UnicastConfig { + import_policy: + mg_admin_client::types::ImportExportPolicy4::NoFiltering, + export_policy: + mg_admin_client::types::ImportExportPolicy4::NoFiltering, + nexthop: None, + }), + ipv6_unicast: None, + md5_auth_key: None, + min_ttl: None, + remote_asn: None, + local_pref: None, + multi_exit_discriminator: None, + connect_retry_jitter: None, + idle_hold_jitter: None, + src_addr, + src_port: None, + vlan_id: None, + } +} + +async fn setup_bgp_peering( + log: &slog::Logger, + peer_routers: &[omicron_test_utils::dev::maghemite::MgdInstance], + contexts: &[nexus_test_utils::ControlPlaneTestContext< + omicron_nexus::Server, + >], +) -> Result<(), anyhow::Error> { + for (rack_n, ctx) in contexts.iter().enumerate() { + for (slot, rack_mgd) in &ctx.mgd { + let slot_idx = slot_index(*slot); + let rack_asn = 65000 + rack_n as u32; + let rack_id = ipv4_as_u32(127, 2, rack_n as u8, slot_idx as u8); + + let rack_api_addr = + SocketAddrV6::new(Ipv6Addr::LOCALHOST, rack_mgd.port, 0, 0); + let rack_client = mg_admin_client::Client::new( + &format!("http://{rack_api_addr}"), + slog::Logger::new( + log, + slog::o!( + "component" => "MgdClient", + "rack" => rack_n, + "slot" => format!("{:?}", slot), + ), + ), + ); + + rack_client + .create_router(&mg_admin_client::types::Router { + asn: rack_asn, + id: rack_id, + graceful_shutdown: false, + listen: rack_mgd.bgp_dispatcher_addr.to_string(), + }) + .await + .with_context(|| { + format!("create_router for rack {rack_n} {slot:?}") + })?; + + for (peer_n, peer_mgd) in peer_routers.iter().enumerate() { + let peer_asn = 65100 + peer_n as u32; + + let peer_api_addr = + SocketAddrV6::new(Ipv6Addr::LOCALHOST, peer_mgd.port, 0, 0); + let peer_client = mg_admin_client::Client::new( + &format!("http://{peer_api_addr}"), + slog::Logger::new( + log, + slog::o!( + "component" => "MgdClient", + "peer_router" => peer_n, + ), + ), + ); + + // rack mgd → peer router + // src_addr ensures the outgoing connection uses the rack + // mgd's own loopback IP as the source, so the peer router + // dispatcher can match the incoming connection to the right + // session (keyed by the rack mgd's IP). + rack_client + .create_neighbor(&make_neighbor( + rack_asn, + format!("peer-router-{peer_n}"), + peer_mgd.bgp_dispatcher_addr, + Some(rack_mgd.bgp_dispatcher_addr.ip()), + )) + .await + .with_context(|| { + format!( + "rack {rack_n} {slot:?} create_neighbor \ + peer-router-{peer_n}" + ) + })?; + + // peer router → rack mgd + // src_addr ensures the outgoing connection uses the peer + // router's own loopback IP as the source, so the rack mgd + // dispatcher can match the incoming connection to the right + // session (keyed by the peer router's IP). + peer_client + .create_neighbor(&make_neighbor( + peer_asn, + format!("rack-{rack_n}-{slot:?}"), + rack_mgd.bgp_dispatcher_addr, + Some(peer_mgd.bgp_dispatcher_addr.ip()), + )) + .await + .with_context(|| { + format!( + "peer-router-{peer_n} create_neighbor \ + rack-{rack_n}-{slot:?}" + ) + })?; + } + } + } + Ok(()) +} + +fn start_stream() -> futures::stream::Fuse { + let signals = Signals::new(&[SIGINT]).expect("failed to wait for SIGINT"); + signals.fuse() +} + +fn read_config(args: &dyn Configurable) -> Result { + let config_str = fs::read_to_string(&args.nexus_config())?; + let mut config: NexusConfig = toml::from_str(&config_str) + .context(format!("parsing config: {}", args.nexus_config().as_str()))?; + config.pkg.log = dropshot::ConfigLogging::File { + // See LogContext::new(), + path: "UNUSED".to_string().into(), + level: dropshot::ConfigLoggingLevel::Trace, + if_exists: dropshot::ConfigLoggingIfExists::Fail, + }; + Ok(config) +} + +/// Number of simulated racks (1–100). +/// +/// Racks are assigned BGP ASNs from the range 65000–65099 (one per rack). +/// Limiting to 100 keeps rack ASNs entirely below the peer-router range. +#[derive(Clone, Copy, Debug)] +struct RackCount(u8); + +impl std::str::FromStr for RackCount { + type Err = String; + fn from_str(s: &str) -> Result { + let n: u8 = s.parse().map_err(|e| format!("invalid rack count: {e}"))?; + if n < 1 || n > 100 { + return Err(format!("rack count must be between 1 and 100, got {n}")); + } + Ok(RackCount(n)) + } +} + +impl std::fmt::Display for RackCount { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +/// Number of simulated peer routers (0–100). +/// +/// Peer routers are assigned BGP ASNs from the range 65100–65199 (one per +/// peer). Limiting to 100 prevents us from running out of ASNs. +#[derive(Clone, Copy, Debug)] +struct PeerRouterCount(u8); + +impl std::str::FromStr for PeerRouterCount { + type Err = String; + fn from_str(s: &str) -> Result { + let n: u8 = s.parse().map_err(|e| format!("invalid peer router count: {e}"))?; + if n > 100 { + return Err(format!( + "peer router count must be between 0 and 100, got {n}" + )); + } + Ok(PeerRouterCount(n)) + } +} + +impl std::fmt::Display for PeerRouterCount { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +#[derive(Clone, Debug, Args)] +struct RunMultipleArgs { + /// Override the gateway server configuration file. + #[clap(long, default_value = DEFAULT_SP_SIM_CONFIG)] + gateway_config: Utf8PathBuf, + /// Override the nexus configuration file. + #[clap(long, default_value = DEFAULT_NEXUS_CONFIG)] + nexus_config: Utf8PathBuf, + /// Number of "racks" to launch (1–100) + #[clap(long, default_value_t = RackCount(1))] + count: RackCount, + /// Launch peer router mgd instances (0–100) + #[clap(long, default_value_t = PeerRouterCount(0))] + peer_routers: PeerRouterCount, +} + +impl Configurable for RunMultipleArgs { + fn nexus_config(&self) -> &Utf8PathBuf { + &self.nexus_config + } +} + +impl RunMultipleArgs { + async fn exec(&self) -> Result<(), anyhow::Error> { + let decorator = slog_term::TermDecorator::new().build(); + let drain = slog_term::FullFormat::new(decorator).build().fuse(); + let drain = slog_async::Async::new(drain).build().fuse(); + + let log = slog::Logger::root(drain, o!()); + + // Start a stream listening for SIGINT + let mut signal_stream = start_stream(); + + // Read configuration. + let mut config = read_config(self)?; + + // Disable physical disk adoption: DiskTest manages disks explicitly, + // and the adoption task (period = 30 s) races with DiskTest::new when + // setup takes ~30 s per rack. Unit tests disable this for the same + // reason via config.test.toml. + config.pkg.background_tasks.physical_disk_adoption.disable = true; + + let mut contexts = vec![]; + + let mut peer_routers = vec![]; + let mut peer_router_loopback_allocations = vec![]; + + let loopback_manager = Arc::new(Mutex::new( + loopback_ip_mgr::LoopbackIpManager::new("lo0", log.clone()), + )); + + for n in 0..self.peer_routers.0 { + let mgd_bgp_addr = + SocketAddr::new(Ipv4Addr::new(127, 0, n, 1).into(), 1049); + + // Allocate the loopback IP for this peer router's BGP listener. + // 127.0.0.1 (n == 0) is always present and treated as a no-op by + // the manager; addresses for n > 0 are added to the interface and + // removed when the allocation is dropped. + let alloc = loopback_ip_mgr::LoopbackIpManager::allocate( + loopback_manager.clone(), + &[mgd_bgp_addr.ip()], + ) + .expect("allocate loopback IP for peer router BGP listener"); + peer_router_loopback_allocations.push(alloc); + + let mgd = omicron_test_utils::dev::maghemite::MgdInstance::start( + 0, + mgd_bgp_addr, + None, + ) + .await + .context("failed to start peer router mgd")?; + + println!( + "peer router {n}: mgd api: http://[::1]:{}", + mgd.port + ); + + println!( + "peer router {n}: mgd bgp-dispatcher: tcp {}", + mgd_bgp_addr + ); + + if let Some(dir) = &mgd.data_dir { + println!( + "peer router {n} tmp dir: {}", + dir.display() + ); + } + + let api_socket_addr = + SocketAddrV6::new(Ipv6Addr::LOCALHOST, mgd.port, 0, 0); + + let client = mg_admin_client::Client::new( + &format!("http://{api_socket_addr}"), + slog::Logger::new( + &log, + o!( + "component" => "MgdClient", + "peer_router" => n, + ), + ), + ); + + let router = mg_admin_client::types::Router { + asn: 65100 + u32::from(n), + id: 65100 + u32::from(n), + graceful_shutdown: true, + listen: mgd.bgp_dispatcher_addr.to_string(), + }; + + println!( + "peer router {n}: bgp ASN: {}\n", + router.asn, + ); + + client.create_router(&router).await?; + + peer_routers.push(mgd); + } + + for n in 0..self.count.0 { + config + .deployment + .dropshot_external + .dropshot + .bind_address + .set_ip("0.0.0.0".parse().unwrap()); + config + .deployment + .dropshot_external + .dropshot + .bind_address + .set_port(0); + + config.deployment.dropshot_internal.bind_address.set_port(0); + config.deployment.dropshot_lockstep.bind_address.set_port(0); + config.deployment.techport_external_server_port = 0; + + // Assign unique loopback IPs to this rack's pair of mgd BGP + // dispatchers so they can coexist with other rack instances and + // with the peer routers above. + let mut mgd_bgp_addrs = BTreeMap::new(); + mgd_bgp_addrs + .insert(SwitchSlot::Switch0, Ipv4Addr::new(127, 2, n, 0)); + mgd_bgp_addrs + .insert(SwitchSlot::Switch1, Ipv4Addr::new(127, 2, n, 1)); + + println!("\nomicron-dev: setting up all services for rack {n}... "); + let cptestctx = + nexus_test_utils::omicron_dev_setup_with_bgp_loopbacks::< + omicron_nexus::Server, + >( + &mut config, + 1, + self.gateway_config.clone(), + loopback_manager.clone(), + mgd_bgp_addrs, + ) + .await + .context("error setting up services")?; + + println!("omicron-dev: Adding disks to first sled agent"); + + // This is how our integration tests are identifying that "disks exist" + // within the database. + // + // This inserts: + // - DEFAULT_ZPOOL_COUNT zpools, each of which contains: + // - A crucible dataset + // - A debug dataset + DiskTest::new(&cptestctx).await; + + println!("omicron-dev: services are running."); + + // Print out basic information about what was started. + // NOTE: The stdout strings here are not intended to be stable, but they + // are used by the test suite. + let addr = cptestctx.external_client.bind_address; + println!("omicron-dev: nexus external API: {:?}", addr); + println!( + "omicron-dev: nexus internal API: {:?}", + cptestctx.server.get_http_server_internal_address(), + ); + println!( + "omicron-dev: nexus lockstep API: {:?}", + cptestctx.server.get_http_server_lockstep_address(), + ); + println!( + "omicron-dev: cockroachdb pid: {}", + cptestctx.database.pid(), + ); + println!( + "omicron-dev: cockroachdb URL: {}", + cptestctx.database.pg_config() + ); + println!( + "omicron-dev: cockroachdb directory: {}", + cptestctx.database.temp_dir().display() + ); + println!( + "omicron-dev: clickhouse native addr: {}", + cptestctx.clickhouse.native_address(), + ); + println!( + "omicron-dev: clickhouse http addr: {}", + cptestctx.clickhouse.http_address(), + ); + println!( + "omicron-dev: internal DNS HTTP: http://{}", + cptestctx.internal_dns.dropshot_server.local_addr() + ); + println!( + "omicron-dev: internal DNS: {}", + cptestctx.internal_dns.dns_server.local_address() + ); + println!( + "omicron-dev: external DNS name: {}", + cptestctx.external_dns_zone_name, + ); + println!( + "omicron-dev: external DNS HTTP: http://{}", + cptestctx.external_dns.dropshot_server.local_addr() + ); + println!( + "omicron-dev: external DNS: {}", + cptestctx.external_dns.dns_server.local_address() + ); + println!( + "omicron-dev: e.g. `dig @{} -p {} {}.sys.{}`", + cptestctx.external_dns.dns_server.local_address().ip(), + cptestctx.external_dns.dns_server.local_address().port(), + cptestctx.silo_name, + cptestctx.external_dns_zone_name, + ); + for (location, gateway) in &cptestctx.gateway { + println!( + "omicron-dev: management gateway: {} ({:?})", + gateway.client.baseurl(), + location, + ); + } + for (location, dendrite) in + cptestctx.dendrite.read().unwrap().iter() + { + println!( + "omicron-dev: dendrite: http://[::1]:{} ({:?})", + dendrite.port, location, + ); + } + for (location, lldpd) in &cptestctx.lldpd { + println!( + "omicron-dev: lldp: http://[::1]:{} ({:?})", + lldpd.port, location, + ); + } + for (location, mgd) in &cptestctx.mgd { + println!( + "omicron-dev: mgd api: http://[::1]:{} ({:?})", + mgd.port, location, + ); + + println!( + "omicron-dev: mgd bgp-dispatcher: tcp {} ({:?})", + mgd.bgp_dispatcher_addr, location, + ); + + if let Some(dir) = &mgd.data_dir { + println!( + "omicron-dev: mgd tmp dir: {} ({:?})", + dir.display(), + location, + ); + } + } + println!( + "omicron-dev: silo name: {}", + cptestctx.silo_name, + ); + println!( + "omicron-dev: privileged user name: {}", + cptestctx.user_name.as_ref(), + ); + println!( + "omicron-dev: privileged password: {}", + cptestctx.password + ); + contexts.push(cptestctx); + } + + if self.peer_routers.0 > 0 { + setup_bgp_peering(&log, &peer_routers, &contexts) + .await + .context("setting up BGP peering")?; + println!("\nomicron-dev: BGP peering configured.\n"); + } + + // Wait for a signal. + let caught_signal = signal_stream.next().await; + assert_eq!(caught_signal.unwrap(), SIGINT); + eprintln!( + "omicron-dev: caught signal, shutting down and removing \ + temporary directory" + ); + + for context in contexts { + context.teardown().await; + } + + for mut router in peer_routers { + if let Err(e) = router.cleanup().await { + eprintln!("error cleaning up peer router: {e}") + } + } + + // Loopback allocations for both peer routers and rack mgd instances + // are released here. The rack allocations were already released by the + // ControlPlaneTestContext teardown above (via MgdInstance drop), so + // this only actively removes the peer router addresses. + drop(peer_router_loopback_allocations); + Ok(()) } } diff --git a/dev-tools/omicron-dev/tests/output/cmd-omicron-dev-noargs-stderr b/dev-tools/omicron-dev/tests/output/cmd-omicron-dev-noargs-stderr index 4d19049a326..1204d5a274c 100644 --- a/dev-tools/omicron-dev/tests/output/cmd-omicron-dev-noargs-stderr +++ b/dev-tools/omicron-dev/tests/output/cmd-omicron-dev-noargs-stderr @@ -3,8 +3,9 @@ Tools for working with a local Omicron deployment Usage: omicron-dev Commands: - run-all Run a full simulated control plane - help Print this message or the help of the given subcommand(s) + run-all Run a full simulated control plane + run-multiple Run multiple simulated control planes + help Print this message or the help of the given subcommand(s) Options: -h, --help Print help diff --git a/env.sh b/env.sh index 6a84c35902a..1ced95c37bb 100644 --- a/env.sh +++ b/env.sh @@ -12,6 +12,7 @@ export PATH="$OMICRON_WS/out/cockroachdb/bin:$PATH" export PATH="$OMICRON_WS/out/clickhouse:$PATH" export PATH="$OMICRON_WS/out/dendrite-stub/bin:$PATH" export PATH="$OMICRON_WS/out/mgd/root/opt/oxide/mgd/bin:$PATH" +export PATH="$OMICRON_WS/out/lldp/root/opt/oxide/bin:$PATH" # if xtrace was set previously, do not unset it case $OLD_SHELL_OPTS in diff --git a/internal-dns/types/src/config.rs b/internal-dns/types/src/config.rs index d5bef144343..6f31b585d53 100644 --- a/internal-dns/types/src/config.rs +++ b/internal-dns/types/src/config.rs @@ -399,6 +399,7 @@ impl DnsConfigBuilder { dendrite_port: u16, mgs_port: u16, mgd_port: u16, + lldpd_port: u16, ) -> anyhow::Result<()> { let zone = self.host_dendrite(sled_id, switch_zone_ip)?; self.service_backend_zone(ServiceName::Dendrite, &zone, dendrite_port)?; @@ -407,7 +408,8 @@ impl DnsConfigBuilder { &zone, mgs_port, )?; - self.service_backend_zone(ServiceName::Mgd, &zone, mgd_port) + self.service_backend_zone(ServiceName::Mgd, &zone, mgd_port)?; + self.service_backend_zone(ServiceName::Lldpd, &zone, lldpd_port) } /// Higher-level shorthand for adding a Nexus zone with both its internal diff --git a/internal-dns/types/src/names.rs b/internal-dns/types/src/names.rs index 73b2439e48e..509f5b37f8e 100644 --- a/internal-dns/types/src/names.rs +++ b/internal-dns/types/src/names.rs @@ -75,6 +75,7 @@ pub enum ServiceName { BoundaryNtp, InternalNtp, Mgd, + Lldpd, } impl ServiceName { @@ -116,6 +117,7 @@ impl ServiceName { ServiceName::BoundaryNtp => "boundary-ntp", ServiceName::InternalNtp => "internal-ntp", ServiceName::Mgd => "mgd", + ServiceName::Lldpd => "lldpd", } } @@ -144,6 +146,7 @@ impl ServiceName { | ServiceName::CruciblePantry | ServiceName::BoundaryNtp | ServiceName::InternalNtp + | ServiceName::Lldpd | ServiceName::Mgd => { format!("_{}._tcp", self.service_kind()) } diff --git a/nexus-config/src/nexus_config.rs b/nexus-config/src/nexus_config.rs index 56a10a87773..8a64e39f3ba 100644 --- a/nexus-config/src/nexus_config.rs +++ b/nexus-config/src/nexus_config.rs @@ -263,7 +263,13 @@ pub struct DpdConfig { pub address: SocketAddr, } -/// Configuration for the `Dendrite` dataplane daemon. +/// Configuration for the `LLDP` daemon. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct LldpdConfig { + pub address: SocketAddr, +} + +/// Configuration for the `Maghemite` routing daemon. #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct MgdConfig { pub address: SocketAddr, @@ -1088,6 +1094,9 @@ pub struct PackageConfig { /// `Dendrite` dataplane daemon configuration #[serde(default)] pub dendrite: HashMap, + /// `LLDP` daemon configuration + #[serde(default)] + pub lldpd: HashMap, /// Maghemite mgd daemon configuration #[serde(default)] pub mgd: HashMap, @@ -1271,6 +1280,8 @@ mod test { type = "from_dns" [dendrite.switch0] address = "[::1]:12224" + [lldpd.switch0] + address = "[::1]:12230" [mgd.switch0] address = "[::1]:4676" [initial_reconfigurator_config] @@ -1438,6 +1449,13 @@ mod test { .unwrap(), } )]), + lldpd: HashMap::from([( + SwitchSlot::Switch0, + LldpdConfig { + address: SocketAddr::from_str("[::1]:12230") + .unwrap(), + } + )]), mgd: HashMap::from([( SwitchSlot::Switch0, MgdConfig { diff --git a/nexus/reconfigurator/execution/src/test_utils.rs b/nexus/reconfigurator/execution/src/test_utils.rs index cd46adacd0b..e4052b53e0d 100644 --- a/nexus/reconfigurator/execution/src/test_utils.rs +++ b/nexus/reconfigurator/execution/src/test_utils.rs @@ -113,10 +113,12 @@ pub fn overridables_for_test( let dendrite_port = cptestctx.dendrite.read().unwrap().get(&switch_slot).unwrap().port; let mgd_port = cptestctx.mgd.get(&switch_slot).unwrap().port; + let lldpd_port = cptestctx.lldpd.get(&switch_slot).unwrap().port; overrides.override_switch_zone_ip(sled_id, ip); overrides.override_dendrite_port(sled_id, dendrite_port); overrides.override_mgs_port(sled_id, mgs_port); overrides.override_mgd_port(sled_id, mgd_port); + overrides.override_lldpd_port(sled_id, lldpd_port); } overrides } diff --git a/nexus/reconfigurator/planning/src/example.rs b/nexus/reconfigurator/planning/src/example.rs index a1f865e2934..87b209ce32e 100644 --- a/nexus/reconfigurator/planning/src/example.rs +++ b/nexus/reconfigurator/planning/src/example.rs @@ -1854,6 +1854,7 @@ mod tests { | ServiceName::RepoDepot | ServiceName::ManagementGatewayService | ServiceName::Dendrite + | ServiceName::Lldpd | ServiceName::Mgd => { out.insert(service, Ok(())); } diff --git a/nexus/src/app/bgp.rs b/nexus/src/app/bgp.rs index 53cc41996c5..d46a1b63995 100644 --- a/nexus/src/app/bgp.rs +++ b/nexus/src/app/bgp.rs @@ -104,7 +104,7 @@ impl super::Nexus { let mut result = Vec::new(); for (switch_slot, client) in self.mg_clients().await.map_err(|e| { external::Error::internal_error(&format!( - "failed to get mg clients: {e}" + "failed to get mgd clients: {e}" )) })? { let router_info = match client.read_routers().await { @@ -160,7 +160,7 @@ impl super::Nexus { let mut result = vec![]; for (switch_slot, client) in self.mg_clients().await.map_err(|e| { external::Error::internal_error(&format!( - "failed to get mg clients: {e}" + "failed to get mgd clients: {e}" )) })? { let router_info = match client.read_routers().await { @@ -233,7 +233,7 @@ impl super::Nexus { let mut result = Vec::new(); for (switch_slot, client) in self.mg_clients().await.map_err(|e| { external::Error::internal_error(&format!( - "failed to get mg clients: {e}" + "failed to get mgd clients: {e}" )) })? { let history = match client @@ -276,7 +276,7 @@ impl super::Nexus { let mut result = Vec::new(); for (switch_slot, client) in self.mg_clients().await.map_err(|e| { external::Error::internal_error(&format!( - "failed to get mg clients: {e}" + "failed to get mgd clients: {e}" )) })? { let mut imported: Vec = Vec::new(); diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index 6778ea72598..bd082caebef 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -1319,33 +1319,74 @@ pub(crate) async fn dpd_clients( Ok(mappings) } -// We currently ignore the rack_id argument here, as the shared -// switch_zone_address_mappings function doesn't allow filtering on the rack ID. -// Since we only have a single rack, this is OK for now. -// TODO: https://github.com/oxidecomputer/omicron/issues/1276 -// /// Returns a mapping of clients for the LLDP daemons of reachable switch zones. /// If we are unable to communicate with the switch zone and determine the mapping /// of SwitchSlot -> Zone Underlay Address, we omit an entry for that client. pub(crate) async fn lldpd_clients( resolver: &internal_dns_resolver::Resolver, + // TODO: https://github.com/oxidecomputer/omicron/issues/1276 _rack_id: Uuid, log: &slog::Logger, ) -> Result, String> { - let mappings = switch_zone_address_mappings(resolver, log).await?; - let log = log.new(o!( "component" => "LldpdClient")); - let port = lldpd_client::default_port(); - let clients: HashMap = mappings + let lldpd_socketaddrs = match resolver + .lookup_all_socket_v6(ServiceName::Lldpd) + .await + { + Ok(addrs) => addrs, + Err(e) => { + error!(log, "failed to resolve addresses for LLDP services"; "error" => %e); + return Err(e.to_string()); + } + }; + + let clients: Vec<(SocketAddrV6, lldpd_client::Client)> = lldpd_socketaddrs .iter() - .map(|(location, addr)| { - let lldpd_client = lldpd_client::Client::new( - &format!("http://[{addr}]:{port}"), - log.clone(), + .map(|socket_addr| { + let client = lldpd_client::Client::new( + &format!("http://{socket_addr}"), + log.new(o!( + "component" => "LldpClient" + )), ); - (*location, lldpd_client) + + (*socket_addr, client) }) .collect(); - Ok(clients) + + let mut mappings: HashMap = + HashMap::new(); + + for (addr, client) in clients { + let switch_slot = match client.switch_identifiers().await { + Ok(response) => response.slot, + Err(e) => { + error!( + log, + "failed to determine switch slot for lldpd"; + "error" => %e, + "addr" => %addr, + ); + continue; + } + }; + + let location = match switch_slot { + Some(0) => SwitchSlot::Switch0, + Some(1) => SwitchSlot::Switch1, + Some(v) => { + warn!(log, "unexpected value for switch slot: {v}"); + continue; + } + None => { + warn!(log, "Lldpd has not learned switch slot from MGS"); + continue; + } + }; + + mappings.insert(location, client); + } + + Ok(mappings) } /// Look up Dendrite addresses in DNS then determine the switch location of diff --git a/nexus/test-utils/Cargo.toml b/nexus/test-utils/Cargo.toml index 4cd399aa910..a67f58a0e0b 100644 --- a/nexus/test-utils/Cargo.toml +++ b/nexus/test-utils/Cargo.toml @@ -43,6 +43,7 @@ omicron-cockroach-admin.workspace = true omicron-common.workspace = true omicron-passwords.workspace = true omicron-sled-agent.workspace = true +loopback-ip-mgr.workspace = true omicron-test-utils.workspace = true omicron-uuid-kinds.workspace = true omicron-workspace-hack.workspace = true diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index e57f667fe8e..d1dab460a2b 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -29,6 +29,8 @@ pub use nexus_test::ControlPlaneBuilder; pub use nexus_test::ControlPlaneTestContext; pub use nexus_test::load_test_config; #[cfg(feature = "omicron-dev")] +pub use nexus_test::omicron_dev_setup_with_bgp_loopbacks; +#[cfg(feature = "omicron-dev")] pub use nexus_test::omicron_dev_setup_with_config; pub use starter::ControlPlaneStarter; pub use starter::ControlPlaneTestContextSledAgent; diff --git a/nexus/test-utils/src/nexus_test.rs b/nexus/test-utils/src/nexus_test.rs index 693aea88732..5ec65cb9c61 100644 --- a/nexus/test-utils/src/nexus_test.rs +++ b/nexus/test-utils/src/nexus_test.rs @@ -36,6 +36,10 @@ use oximeter_producer::Server as ProducerServer; use sled_agent_types::early_networking::SwitchSlot; use std::collections::BTreeMap; use std::collections::HashMap; +#[cfg(feature = "omicron-dev")] +use std::net::Ipv4Addr; +#[cfg(feature = "omicron-dev")] +use std::sync::Mutex; use std::sync::{Arc, RwLock}; use std::time::Duration; use transient_dns_server::TransientDnsServer; @@ -114,6 +118,7 @@ pub struct ControlPlaneTestContext { pub producer: ProducerServer, pub gateway: BTreeMap, pub dendrite: RwLock>, + pub lldpd: HashMap, /// Ports of stopped dendrite instances (for use by start_dendrite) pub stopped_dendrite_ports: RwLock>, pub mgd: HashMap, @@ -320,6 +325,9 @@ impl ControlPlaneTestContext { for (_, mut mgd) in self.mgd { mgd.cleanup().await.unwrap(); } + for (_, mut lldpd) in self.lldpd { + lldpd.cleanup().await.unwrap(); + } self.logctx.cleanup_successful(); } } @@ -356,22 +364,55 @@ pub async fn omicron_dev_setup_with_config( gateway_config_file: Utf8PathBuf, ) -> Result> { let starter = ControlPlaneStarter::::new("omicron-dev", config); + omicron_dev_setup_impl(starter, extra_sled_agents, gateway_config_file) + .await +} + +/// Like [`omicron_dev_setup_with_config`], but assigns unique loopback IPs to +/// each rack's pair of mgd BGP dispatchers so that multiple control plane +/// instances can coexist on the same host. +/// +/// `mgd_bgp_addrs` maps each [`SwitchSlot`] to the IPv4 address that the +/// corresponding mgd instance should bind its BGP dispatcher on. The addresses +/// must exist on a loopback interface; use the supplied `mgd_bgp_loopback` +/// manager to allocate them (it handles install/uninstall with cross-process +/// reference counting). +#[cfg(feature = "omicron-dev")] +pub async fn omicron_dev_setup_with_bgp_loopbacks( + config: &mut NexusConfig, + extra_sled_agents: u16, + gateway_config_file: Utf8PathBuf, + mgd_bgp_loopback: Arc>, + mgd_bgp_addrs: BTreeMap, +) -> Result> { + let mut starter = ControlPlaneStarter::::new("omicron-dev", config); + starter.mgd_bgp_loopback = Some(mgd_bgp_loopback); + starter.mgd_bgp_addrs = mgd_bgp_addrs; + omicron_dev_setup_impl(starter, extra_sled_agents, gateway_config_file) + .await +} - let log = &starter.logctx.log; +#[cfg(feature = "omicron-dev")] +async fn omicron_dev_setup_impl<'a, N: NexusServer>( + starter: ControlPlaneStarter<'a, N>, + extra_sled_agents: u16, + gateway_config_file: Utf8PathBuf, +) -> Result> { + // Clone the logger so we can use it without holding a borrow on `starter` + // while moving it into setup_with_config_impl below. + let log = starter.logctx.log.clone(); slog::debug!(log, "Ensuring seed tarball exists"); - // Start up a ControlPlaneTestContext, which tautologically sets up - // everything needed for a simulated control plane. let why_invalidate = omicron_test_utils::dev::seed::should_invalidate_seed(); let (seed_tar, status) = omicron_test_utils::dev::seed::ensure_seed_tarball_exists( - log, + &log, why_invalidate, ) .await .context("error ensuring seed tarball exists")?; - status.log(log, &seed_tar); + status.log(&log, &seed_tar); Ok(setup_with_config_impl( starter, diff --git a/nexus/test-utils/src/starter.rs b/nexus/test-utils/src/starter.rs index 3822be75d1f..1187d104e74 100644 --- a/nexus/test-utils/src/starter.rs +++ b/nexus/test-utils/src/starter.rs @@ -28,6 +28,7 @@ use internal_dns_types::names::ServiceName; use nexus_config::Database; use nexus_config::DpdConfig; use nexus_config::InternalDns; +use nexus_config::LldpdConfig; use nexus_config::MgdConfig; use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES; use nexus_config::NexusConfig; @@ -107,8 +108,9 @@ use std::collections::BTreeSet; use std::collections::HashMap; use std::fmt::Debug; use std::iter::{once, repeat, zip}; +use std::net::SocketAddrV4; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV6}; -use std::sync::{Arc, RwLock}; +use std::sync::{Arc, Mutex, RwLock}; use std::time::Duration; use transient_dns_server::TransientDnsServer; use uuid::Uuid; @@ -146,6 +148,7 @@ pub struct ControlPlaneStarter<'a, N: NexusServer> { pub gateway: BTreeMap, pub dendrite: RwLock>, pub mgd: HashMap, + pub lldpd: HashMap, // NOTE: Only exists after starting Nexus, until external Nexus is // initialized. @@ -168,6 +171,13 @@ pub struct ControlPlaneStarter<'a, N: NexusServer> { pub password: Option, pub simulated_upstairs: Arc, + + /// When set, `start_mgd` allocates a unique loopback IP for each switch + /// slot's mgd BGP dispatcher from `mgd_bgp_addrs` instead of using + /// 127.0.0.1. Normal integration tests leave this `None`. + pub mgd_bgp_loopback: + Option>>, + pub mgd_bgp_addrs: BTreeMap, } type StepInitFn<'a, N> = Box< @@ -202,6 +212,7 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { producer: None, gateway: BTreeMap::new(), dendrite: RwLock::new(HashMap::new()), + lldpd: HashMap::new(), mgd: HashMap::new(), nexus_internal: None, nexus_internal_addr: None, @@ -218,6 +229,8 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { simulated_upstairs: Arc::new(sim::SimulatedUpstairs::new( simulated_upstairs_log, )), + mgd_bgp_loopback: None, + mgd_bgp_addrs: BTreeMap::new(), } } @@ -445,16 +458,65 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { self.config.pkg.dendrite.insert(switch_slot, config); } + pub async fn start_lldp(&mut self, switch_slot: SwitchSlot) { + let log = &self.logctx.log; + debug!(log, "Starting LLDP for {switch_slot:?}"); + let mgs = self.gateway.get(&switch_slot).unwrap(); + let mgs_addr = + SocketAddrV6::new(Ipv6Addr::LOCALHOST, mgs.port, 0, 0).into(); + + let dpd_port = + self.dendrite.read().unwrap().get(&switch_slot).unwrap().port; + + // Set up an instance of lldpd + let lldpd = + dev::lldp::LldpdInstance::start(0, dpd_port, Some(mgs_addr)) + .await + .unwrap(); + + let port = lldpd.port; + self.lldpd.insert(switch_slot, lldpd); + let address = SocketAddrV6::new(Ipv6Addr::LOCALHOST, port, 0, 0); + + debug!(log, "lldp port is {port}"); + + let config = LldpdConfig { address: std::net::SocketAddr::V6(address) }; + self.config.pkg.lldpd.insert(switch_slot, config); + } + pub async fn start_mgd(&mut self, switch_slot: SwitchSlot) { let log = &self.logctx.log; debug!(log, "Starting mgd"; "switch_slot" => ?switch_slot); let mgs = self.gateway.get(&switch_slot).unwrap(); - let mgs_addr = + let mgs_addr: std::net::SocketAddr = SocketAddrV6::new(Ipv6Addr::LOCALHOST, mgs.port, 0, 0).into(); - // Set up an instance of mgd - let mgd = - dev::maghemite::MgdInstance::start(0, mgs_addr).await.unwrap(); + // If a loopback manager and per-slot address were provided, allocate a + // unique loopback IP for this instance's BGP dispatcher so that + // multiple control plane instances can coexist on the same host. + // Otherwise, fall back to 127.0.0.1, which is always present and + // sufficient for single-mgd development and normal integration tests. + let (bgp_addr, bgp_loopback_allocation) = match ( + &self.mgd_bgp_loopback, + self.mgd_bgp_addrs.get(&switch_slot), + ) { + (Some(mgr), Some(&ip)) => { + let alloc = loopback_ip_mgr::LoopbackIpManager::allocate( + mgr.clone(), + &[IpAddr::V4(ip)], + ) + .expect("allocate loopback IP for mgd BGP dispatcher"); + (SocketAddr::new(IpAddr::V4(ip), 1049), Some(alloc)) + } + _ => (SocketAddrV4::new(Ipv4Addr::LOCALHOST, 0).into(), None), + }; + + let mut mgd = + dev::maghemite::MgdInstance::start(0, bgp_addr, Some(mgs_addr)) + .await + .unwrap(); + mgd.bgp_loopback_allocation = bgp_loopback_allocation; + let port = mgd.port; self.mgd.insert(switch_slot, mgd); let address = SocketAddrV6::new(Ipv6Addr::LOCALHOST, port, 0, 0); @@ -486,6 +548,7 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { self.dendrite.read().unwrap().get(&switch_slot).unwrap().port, self.gateway.get(&switch_slot).unwrap().port, self.mgd.get(&switch_slot).unwrap().port, + self.lldpd.get(&switch_slot).unwrap().port, ) .unwrap() } @@ -1252,6 +1315,7 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { logctx: self.logctx, gateway: self.gateway, dendrite: RwLock::new(self.dendrite.into_inner().unwrap()), + lldpd: self.lldpd, stopped_dendrite_ports: RwLock::new(HashMap::new()), mgd: self.mgd, external_dns_zone_name: self.external_dns_zone_name.unwrap(), @@ -1295,6 +1359,9 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { for (_, mut mgd) in self.mgd { mgd.cleanup().await.unwrap(); } + for (_, mut lldpd) in self.lldpd { + lldpd.cleanup().await.unwrap(); + } self.logctx.cleanup_successful(); } @@ -1538,6 +1605,7 @@ pub(crate) async fn setup_with_config_impl( extra_sled_agents: u16, gateway_config_file: Utf8PathBuf, second_nexus: bool, + // peer_routers: bool, ) -> ControlPlaneTestContext { const STEP_TIMEOUT: Duration = Duration::from_secs(600); @@ -1629,6 +1697,12 @@ pub(crate) async fn setup_with_config_impl( builder.start_dendrite(SwitchSlot::Switch0).boxed() }), ), + ( + "start_lldpd_switch0", + Box::new(|builder| { + builder.start_lldp(SwitchSlot::Switch0).boxed() + }), + ), ( "start_mgd_switch0", Box::new(|builder| { @@ -1673,6 +1747,12 @@ pub(crate) async fn setup_with_config_impl( builder.start_dendrite(SwitchSlot::Switch1).boxed() }), ), + ( + "start_lldpd_switch1", + Box::new(|builder| { + builder.start_lldp(SwitchSlot::Switch1).boxed() + }), + ), ( "start_mgd_switch1", Box::new(|builder| { diff --git a/nexus/tests/integration_tests/initialization.rs b/nexus/tests/integration_tests/initialization.rs index 350757cf1de..a9504f4cd32 100644 --- a/nexus/tests/integration_tests/initialization.rs +++ b/nexus/tests/integration_tests/initialization.rs @@ -153,6 +153,11 @@ async fn test_nexus_boots_before_dendrite() { starter.start_dendrite(SwitchSlot::Switch1).await; info!(log, "Started Dendrite"); + info!(log, "Starting lldp"); + starter.start_lldp(SwitchSlot::Switch0).await; + starter.start_lldp(SwitchSlot::Switch1).await; + info!(log, "Started lldp"); + info!(log, "Starting mgd"); starter.start_mgd(SwitchSlot::Switch0).await; starter.start_mgd(SwitchSlot::Switch1).await; diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index 4f96fc310c3..5c1b3b2f8fd 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -53,6 +53,8 @@ async fn test_setup<'a>( starter.start_gateway(SwitchSlot::Switch1, None, sp_conf).await; starter.start_dendrite(SwitchSlot::Switch0).await; starter.start_dendrite(SwitchSlot::Switch1).await; + starter.start_lldp(SwitchSlot::Switch0).await; + starter.start_lldp(SwitchSlot::Switch1).await; starter.start_mgd(SwitchSlot::Switch0).await; starter.start_mgd(SwitchSlot::Switch1).await; starter.populate_internal_dns().await; diff --git a/nexus/types/src/deployment/execution/dns.rs b/nexus/types/src/deployment/execution/dns.rs index 009377fd8d9..2344131bdfb 100644 --- a/nexus/types/src/deployment/execution/dns.rs +++ b/nexus/types/src/deployment/execution/dns.rs @@ -158,6 +158,7 @@ pub fn blueprint_internal_dns_config( overrides.dendrite_port(scrimlet.id()), overrides.mgs_port(scrimlet.id()), overrides.mgd_port(scrimlet.id()), + overrides.lldpd_port(scrimlet.id()), )?; } diff --git a/nexus/types/src/deployment/execution/overridables.rs b/nexus/types/src/deployment/execution/overridables.rs index 881a7c49bdd..154902d1a8c 100644 --- a/nexus/types/src/deployment/execution/overridables.rs +++ b/nexus/types/src/deployment/execution/overridables.rs @@ -4,6 +4,7 @@ use omicron_common::address::DENDRITE_PORT; use omicron_common::address::Ipv6Subnet; +use omicron_common::address::LLDP_PORT; use omicron_common::address::MGD_PORT; use omicron_common::address::MGS_PORT; use omicron_common::address::SLED_PREFIX; @@ -31,6 +32,8 @@ pub struct Overridables { pub mgd_ports: BTreeMap, /// map: sled id -> IP address of the sled's switch zone pub switch_zone_ips: BTreeMap, + /// map: sled id -> TCP port on which that sled's LLDP is listening + pub lldpd_ports: BTreeMap, } pub static DEFAULT: LazyLock = @@ -67,6 +70,16 @@ impl Overridables { self.mgd_ports.get(&sled_id).copied().unwrap_or(MGD_PORT) } + /// Specify the TCP port on which this sled's LLDP is listening + pub fn override_lldpd_port(&mut self, sled_id: SledUuid, port: u16) { + self.lldpd_ports.insert(sled_id, port); + } + + /// Returns the TCP port on which this sled's LLDP is listening + pub fn lldpd_port(&self, sled_id: SledUuid) -> u16 { + self.lldpd_ports.get(&sled_id).copied().unwrap_or(LLDP_PORT) + } + /// Specify the IP address of this switch zone pub fn override_switch_zone_ip( &mut self, diff --git a/package-manifest.toml b/package-manifest.toml index 204a5436de3..2989bc62503 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -725,8 +725,8 @@ output.intermediate_only = true service_name = "lldp" source.type = "prebuilt" source.repo = "lldp" -source.commit = "61479b6922f9112fbe1e722414d2b8055212cb12" -source.sha256 = "8f988c0b0fa3ad4121ab0e825298601035e56c5c054bdc3a1dfb4d6c8fd5b300" +source.commit = "d22509dfdb051321b859e924948605115691b93c" +source.sha256 = "cf93d374b751a0b760dd5adb3737cd5b239c2ba427749164974b262c0c446593" output.type = "zone" output.intermediate_only = true diff --git a/sled-agent/rack-setup/src/plan/service.rs b/sled-agent/rack-setup/src/plan/service.rs index 48de2b1d3e9..e7e790520eb 100644 --- a/sled-agent/rack-setup/src/plan/service.rs +++ b/sled-agent/rack-setup/src/plan/service.rs @@ -28,6 +28,7 @@ use nexus_types::deployment::{ PendingMgsUpdates, blueprint_zone_type, }; use nexus_types::external_api::sled::SledState; +use omicron_common::address::LLDP_PORT; use omicron_common::address::{ CP_SERVICES_RESERVED_ADDRESSES, DENDRITE_PORT, DNS_HTTP_PORT, DNS_PORT, Ipv6Subnet, MGD_PORT, MGS_PORT, NEXUS_INTERNAL_PORT, NEXUS_LOCKSTEP_PORT, @@ -341,6 +342,7 @@ impl ServicePlan { DENDRITE_PORT, MGS_PORT, MGD_PORT, + LLDP_PORT, ) .unwrap(); } diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml index ad4b8d303c7..a700fa0ea98 100644 --- a/test-utils/Cargo.toml +++ b/test-utils/Cargo.toml @@ -19,6 +19,7 @@ headers.workspace = true hex.workspace = true http.workspace = true libc.workspace = true +loopback-ip-mgr.workspace = true nexus-config.workspace = true omicron-common.workspace = true oxide-tokio-rt.workspace = true diff --git a/test-utils/src/dev/lldp.rs b/test-utils/src/dev/lldp.rs new file mode 100644 index 00000000000..6144c302445 --- /dev/null +++ b/test-utils/src/dev/lldp.rs @@ -0,0 +1,216 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Tools for managing LLDP during development + +use std::net::{IpAddr, Ipv6Addr, SocketAddr}; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::time::Duration; + +use anyhow::Context; +use tempfile::TempDir; +use tokio::{ + fs::File, + io::{AsyncBufReadExt, BufReader}, + time::{Instant, sleep}, +}; + +/// Specifies the amount of time we will wait for `lldpd` to launch, +/// which is currently confirmed by watching `lldpd`'s log output +/// for a message specifying the address and port `lldpd` is listening on. +pub const LLDPD_TIMEOUT: Duration = Duration::new(5, 0); + +pub struct LldpdInstance { + /// Port number the mgd instance is listening on. This can be provided + /// manually, or dynamically determined if a value of 0 is provided. + pub port: u16, + /// Arguments provided to the `lldpd` cli command. + pub args: Vec, + /// Child process spawned by running `lldpd` + pub child: Option, + /// Temporary directory where logging output and other files generated by + /// `lldpd` are stored. + pub data_dir: Option, +} + +impl LldpdInstance { + pub async fn start( + mut port: u16, + dpd_port: u16, + mgs_address: Option, + ) -> Result { + let temp_dir = TempDir::new()?; + let listen_addr = + SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), port); + + let mut args = vec![ + "run".to_string(), + "--listen-addr".into(), + listen_addr.to_string(), + "--port".to_string(), + dpd_port.to_string(), + ]; + + if let Some(socket_addr) = mgs_address { + args.push("--mgs-addr".to_string()); + args.push(socket_addr.to_string()); + } + + let child = tokio::process::Command::new("lldpd") + .args(&args) + .stdin(Stdio::null()) + .stdout(Stdio::from(redirect_file( + temp_dir.path(), + "lldpd_stdout", + )?)) + .stderr(Stdio::from(redirect_file( + temp_dir.path(), + "lldpd_stderr", + )?)) + .spawn() + .with_context(|| { + format!("failed to spawn `lldpd` (with args: {:?})", &args) + })?; + + let child = Some(child); + + let temp_dir = temp_dir.keep(); + if port == 0 { + port = discover_port( + temp_dir.join("lldpd_stdout").display().to_string(), + ) + .await + .with_context(|| { + format!( + "failed to discover lldpd port from files in {}", + temp_dir.display() + ) + })?; + } + + Ok(Self { port, args, child, data_dir: Some(temp_dir) }) + } + + pub async fn cleanup(&mut self) -> Result<(), anyhow::Error> { + if let Some(mut child) = self.child.take() { + child.start_kill().context("Sending SIGKILL to child")?; + child.wait().await.context("waiting for child")?; + } + if let Some(dir) = self.data_dir.take() { + std::fs::remove_dir_all(&dir).with_context(|| { + format!("cleaning up temporary directory {}", dir.display()) + })?; + } + Ok(()) + } +} + +impl Drop for LldpdInstance { + fn drop(&mut self) { + if self.child.is_some() || self.data_dir.is_some() { + eprintln!( + "WARN: dropped LldpdInstance without cleaning it up first \ + (there may still be a child process running and a \ + temporary directory leaked)" + ); + if let Some(child) = self.child.as_mut() { + let _ = child.start_kill(); + } + if let Some(path) = self.data_dir.take() { + eprintln!( + "WARN: lldpd temporary directory leaked: {}", + path.display() + ); + } + } + } +} + +fn redirect_file( + temp_dir_path: &Path, + label: &str, +) -> Result { + let out_path = temp_dir_path.join(label); + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&out_path) + .with_context(|| format!("open \"{}\"", out_path.display())) +} + +async fn discover_port(logfile: String) -> Result { + let timeout = Instant::now() + LLDPD_TIMEOUT; + tokio::time::timeout_at(timeout, find_lldpd_port_in_log(logfile)) + .await + .context("time out while discovering lldpd port number")? +} + +async fn find_lldpd_port_in_log(logfile: String) -> Result { + let re = regex::Regex::new(r#""local_addr":"\[::1?\]:([0-9]+)""#).unwrap(); + let mut reader = BufReader::new(File::open(&logfile).await?); + let mut lines = reader.lines(); + loop { + match lines.next_line().await? { + Some(line) => { + if let Some(cap) = re.captures(&line) { + // unwrap on get(1) should be ok, since captures() returns + // `None` if there are no matches found + let port = cap.get(1).unwrap(); + let result = port.as_str().parse::()?; + return Ok(result); + } + } + None => { + sleep(Duration::from_millis(10)).await; + + // We might have gotten a partial line; close the file, reopen + // it, and start reading again from the beginning. + reader = BufReader::new(File::open(&logfile).await?); + lines = reader.lines(); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::find_lldpd_port_in_log; + use std::io::Write; + use std::process::Stdio; + use tempfile::NamedTempFile; + + const EXPECTED_PORT: u16 = 12230; + + #[tokio::test] + async fn test_lldpd_in_path() { + // With no arguments, we expect to see the default help message. + tokio::process::Command::new("lldpd") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("Cannot find 'lldpd' on PATH. Refer to README.md for installation instructions"); + } + + #[tokio::test] + async fn test_discover_local_listening_port() { + // Write some data to a fake log file + // This line is representative of the kind of output that lldpd currently logs + let line = r#"{"msg":"listening","v":0,"name":"lldpd","level":30,"time":"2025-12-23T00:07:09.226947807Z","hostname":"sled03","pid":10187,"local_addr":"[::]:12230","server_id": +"1","unit":"api-server"}"#; + let mut file = NamedTempFile::new().unwrap(); + writeln!(file, "A garbage line").unwrap(); + writeln!(file, "{}", line).unwrap(); + writeln!(file, "Another garbage line").unwrap(); + file.flush().unwrap(); + + assert_eq!( + find_lldpd_port_in_log(file.path().display().to_string()) + .await + .unwrap(), + EXPECTED_PORT + ); + } +} diff --git a/test-utils/src/dev/maghemite.rs b/test-utils/src/dev/maghemite.rs index eaae1af8cd4..4e8a85a55ac 100644 --- a/test-utils/src/dev/maghemite.rs +++ b/test-utils/src/dev/maghemite.rs @@ -4,6 +4,7 @@ //! Tools for managing Maghemite during development +use loopback_ip_mgr::IpAllocation; use std::net::SocketAddr; use std::path::{Path, PathBuf}; use std::process::Stdio; @@ -20,12 +21,15 @@ use tokio::{ /// Specifies the amount of time we will wait for `mgd` to launch, /// which is currently confirmed by watching `mgd`'s log output /// for a message specifying the address and port `mgd` is listening on. -pub const MGD_TIMEOUT: Duration = Duration::new(5, 0); +pub const MGD_TIMEOUT: Duration = Duration::new(10, 0); pub struct MgdInstance { /// Port number the mgd instance is listening on. This can be provided /// manually, or dynamically determined if a value of 0 is provided. pub port: u16, + /// `SocketAddr` number that the mgd bgp dispatcher is listening on. This can be provided + /// manually, or dynamically determined if a value of 0 is provided. + pub bgp_dispatcher_addr: SocketAddr, /// Arguments provided to the `mgd` cli command. pub args: Vec, /// Child process spawned by running `mgd` @@ -33,32 +37,42 @@ pub struct MgdInstance { /// Temporary directory where logging output and other files generated by /// `mgd` are stored. pub data_dir: Option, + /// RAII guard keeping a loopback IP allocated for the BGP dispatcher + /// address. Held here so the address is released when the instance is + /// cleaned up. `None` when using an address that is always present (e.g. + /// 127.0.0.1). + pub bgp_loopback_allocation: Option, } impl MgdInstance { pub async fn start( mut port: u16, - mgs_addr: SocketAddr, + bgp_dispatcher_addr: SocketAddr, + mgs_address: Option, ) -> Result { let temp_dir = TempDir::new()?; - let args = vec![ + let mut args = vec![ "run".to_string(), "--admin-addr".into(), "::1".into(), "--admin-port".into(), port.to_string(), - "--no-bgp-dispatcher".into(), "--data-dir".into(), temp_dir.path().display().to_string(), "--rack-uuid".into(), uuid::Uuid::new_v4().to_string(), "--sled-uuid".into(), uuid::Uuid::new_v4().to_string(), - "--mgs-addr".into(), - mgs_addr.to_string(), + "--bgp-dispatcher-addr".into(), + bgp_dispatcher_addr.to_string(), ]; + if let Some(socket_addr) = mgs_address { + args.push("--mgs-addr".to_string()); + args.push(socket_addr.to_string()); + } + let child = tokio::process::Command::new("mgd") .args(&args) .stdin(Stdio::null()) @@ -85,7 +99,14 @@ impl MgdInstance { })?; } - Ok(Self { port, args, child, data_dir: Some(temp_dir) }) + Ok(Self { + port, + bgp_dispatcher_addr, + args, + child, + data_dir: Some(temp_dir), + bgp_loopback_allocation: None, + }) } pub async fn cleanup(&mut self) -> Result<(), anyhow::Error> { @@ -144,9 +165,16 @@ async fn discover_port(logfile: String) -> Result { async fn find_mgd_port_in_log(logfile: String) -> Result { let re = regex::Regex::new(r#""local_addr":"\[::1?\]:([0-9]+)""#).unwrap(); + find_pattern_in_logfile(logfile, re).await +} + +async fn find_pattern_in_logfile( + logfile: String, + re: regex::Regex, +) -> Result { let mut reader = BufReader::new(File::open(&logfile).await?); let mut lines = reader.lines(); - loop { + let result = loop { match lines.next_line().await? { Some(line) => { if let Some(cap) = re.captures(&line) { @@ -154,7 +182,7 @@ async fn find_mgd_port_in_log(logfile: String) -> Result { // `None` if there are no matches found let port = cap.get(1).unwrap(); let result = port.as_str().parse::()?; - return Ok(result); + break result; } } None => { @@ -166,12 +194,13 @@ async fn find_mgd_port_in_log(logfile: String) -> Result { lines = reader.lines(); } } - } + }; + Ok(result) } #[cfg(test)] mod tests { - use super::find_mgd_port_in_log; + use super::discover_port; use std::io::Write; use std::process::Stdio; use tempfile::NamedTempFile; @@ -201,9 +230,7 @@ mod tests { file.flush().unwrap(); assert_eq!( - find_mgd_port_in_log(file.path().display().to_string()) - .await - .unwrap(), + discover_port(file.path().display().to_string()).await.unwrap(), EXPECTED_PORT ); } diff --git a/test-utils/src/dev/mod.rs b/test-utils/src/dev/mod.rs index 5a37ac2549e..6e573d03c54 100644 --- a/test-utils/src/dev/mod.rs +++ b/test-utils/src/dev/mod.rs @@ -9,6 +9,7 @@ pub mod clickhouse; pub mod db; pub mod dendrite; pub mod falcon; +pub mod lldp; pub mod maghemite; pub mod poll; #[cfg(feature = "seed-gen")] diff --git a/tools/install_builder_prerequisites.sh b/tools/install_builder_prerequisites.sh index 0f1df7d2528..efb0ab75fea 100755 --- a/tools/install_builder_prerequisites.sh +++ b/tools/install_builder_prerequisites.sh @@ -230,6 +230,7 @@ retry xtask download \ console \ dendrite-stub \ maghemite-mgd \ + lldp \ transceiver-control # Validate the PATH: @@ -239,6 +240,8 @@ expected_in_path=( 'cockroach' 'clickhouse' 'dpd' + 'mgd' + 'lldpd' ) function show_hint diff --git a/tools/lldp_checksums b/tools/lldp_checksums new file mode 100644 index 00000000000..ba1f5b5a701 --- /dev/null +++ b/tools/lldp_checksums @@ -0,0 +1,2 @@ +CIDL_SHA256="5b08d32dab6ae0bb6a885f629f9c21d209692b768fe02b33d3625d593542ebd8" +LINUX_SHA256="3a841d1e537a4a6a6228ee2dcad2b80248c789876fa3ad564e5caf9797f0e068" diff --git a/tools/lldp_openapi_version b/tools/lldp_openapi_version new file mode 100644 index 00000000000..5b46a3e7278 --- /dev/null +++ b/tools/lldp_openapi_version @@ -0,0 +1 @@ +COMMIT="81d801175cd4df29aeee5dc5303bafcfe54560fe" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums index 470facaa671..0423051d5e1 100644 --- a/tools/maghemite_mgd_checksums +++ b/tools/maghemite_mgd_checksums @@ -1,2 +1,2 @@ CIDL_SHA256="301d31ca481e4822f69484feacca31dd08a7c4aae87d96641d384bda3178d2f3" -MGD_LINUX_SHA256="95f9759a5fde2784d148c81df2218d29adde1d27fb72d5dbcf534de6450f0f7c" \ No newline at end of file +MGD_LINUX_SHA256="95f9759a5fde2784d148c81df2218d29adde1d27fb72d5dbcf534de6450f0f7c"