sled-agent: configure whether physical disks should be detected

Author: emilyalbini

dev-tools/xtask/src/virtual_hardware.rs

@@ -496,8 +496,17 @@ fn get_gateway_mac(
 /// to parse these fields correctly.
 #[derive(Clone, Debug, Deserialize)]
 struct SledAgentConfig {
-    /// Optional list of virtual devices to be used as "discovered disks".
-    pub vdevs: Option<Vec<Utf8PathBuf>>,
+    external_disks: SledAgentExternalDisks,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "snake_case", tag = "kind")]
+enum SledAgentExternalDisks {
+    Virtual {
+        vdevs: Vec<Utf8PathBuf>,
+    },
+    #[serde(other)]
+    Unknown,
 }
 
 impl SledAgentConfig {
@@ -515,7 +524,8 @@ fn ensure_vdevs(
 ) -> Result<()> {
     let config = SledAgentConfig::read(sled_agent_config)?;
 
-    let Some(vdevs) = &config.vdevs else {
+    let SledAgentExternalDisks::Virtual { vdevs } = &config.external_disks
+    else {
         bail!("No vdevs found in this configuration");
     };
 
@@ -565,7 +575,7 @@ fn destroy_vdevs(
 
     // Remove the vdev files themselves, if they are regular files
     let config = SledAgentConfig::read(sled_agent_config)?;
-    if let Some(vdevs) = &config.vdevs {
+    if let SledAgentExternalDisks::Virtual { vdevs } = &config.external_disks {
         for vdev in vdevs {
             let vdev_path = if vdev.is_absolute() {
                 vdev.to_owned()

installinator/src/hardware.rs

@@ -7,6 +7,7 @@ use anyhow::Result;
 use anyhow::anyhow;
 use anyhow::ensure;
 use omicron_common::disk::DiskVariant;
+use sled_hardware::ExternalDisks;
 use sled_hardware::HardwareManager;
 use sled_hardware::SledMode;
 use sled_storage::config::MountConfig;
@@ -25,10 +26,12 @@ impl Hardware {
             .context("failed to detect whether host is an oxide sled")?;
         ensure!(is_oxide_sled, "hardware scan only supported on oxide sleds");
 
-        let hardware = HardwareManager::new(log, SledMode::Auto, vec![])
-            .map_err(|err| {
-                anyhow!("failed to create HardwareManager: {err}")
-            })?;
+        let hardware = HardwareManager::new(
+            log,
+            SledMode::Auto,
+            ExternalDisks::DetectPhysical,
+        )
+        .map_err(|err| anyhow!("failed to create HardwareManager: {err}"))?;
 
         let disks: Vec<RawDisk> =
             hardware.disks().into_values().map(|disk| disk.into()).collect();

sled-agent/src/config.rs

@@ -15,7 +15,7 @@ use illumos_utils::dladm::PhysicalLink;
 use omicron_common::vlan::VlanID;
 use serde::Deserialize;
 use sled_hardware::DataLinks;
-use sled_hardware::UnparsedDisk;
+use sled_hardware::ExternalDisks;
 use sprockets_tls::keys::SprocketsConfig;
 
 #[derive(Clone, Debug, Deserialize)]
@@ -88,12 +88,8 @@ pub struct Config {
     pub swap_device_size_gb: Option<u32>,
     /// Optional VLAN ID to be used for tagging guest VNICs.
     pub vlan: Option<VlanID>,
-    /// Optional list of virtual devices to be used as "discovered disks".
-    pub vdevs: Option<Vec<Utf8PathBuf>>,
-    /// Optional list of real devices to be injected as observed disks during
-    /// device polling.
-    #[serde(default)]
-    pub nongimlet_observed_disks: Option<Vec<UnparsedDisk>>,
+    /// The source of external disks to use.
+    pub external_disks: ExternalDisks,
     /// Optionally skip waiting for time synchronization
     pub skip_timesync: Option<bool>,
 

sled-agent/src/long_running_tasks.rs

@@ -35,7 +35,7 @@ use sled_agent_health_monitor::HealthMonitorHandle;
 use sled_agent_measurements::MeasurementsHandle;
 use sled_agent_resolvable_files::ZoneImageSourceResolver;
 use sled_agent_types::zone_bundle::CleanupContext;
-use sled_hardware::{HardwareManager, SledMode, UnparsedDisk};
+use sled_hardware::{ExternalDisks, HardwareManager, SledMode};
 use sled_storage::config::MountConfig;
 use sled_storage::disk::RawSyntheticDisk;
 use slog::{Logger, info};
@@ -125,11 +125,9 @@ pub async fn spawn_all_longrunning_tasks(
             log,
         );
 
-    let nongimlet_observed_disks =
-        config.nongimlet_observed_disks.clone().unwrap_or(vec![]);
-
     let hardware_manager =
-        spawn_hardware_manager(log, sled_mode, nongimlet_observed_disks).await;
+        spawn_hardware_manager(log, sled_mode, config.external_disks.clone())
+            .await;
 
     // Start monitoring for hardware changes, adding some synthetic disks if
     // necessary.
@@ -231,7 +229,7 @@ fn spawn_key_manager(
 async fn spawn_hardware_manager(
     log: &Logger,
     sled_mode: SledMode,
-    nongimlet_observed_disks: Vec<UnparsedDisk>,
+    external_disks: ExternalDisks,
 ) -> HardwareManager {
     // The `HardwareManager` does not use the the "task/handle" pattern
     // and spawns its worker task inside `HardwareManager::new`. Instead of returning
@@ -241,10 +239,10 @@ async fn spawn_hardware_manager(
     //
     // There are pros and cons to both methods, but the reason to mention it here is that
     // the handle in this case is the `HardwareManager` itself.
-    info!(log, "Starting HardwareManager"; "sled_mode" => ?sled_mode, "nongimlet_observed_disks" => ?nongimlet_observed_disks);
+    info!(log, "Starting HardwareManager"; "sled_mode" => ?sled_mode, "external_disks" => ?external_disks);
     let log = log.clone();
     tokio::task::spawn_blocking(move || {
-        HardwareManager::new(&log, sled_mode, nongimlet_observed_disks).unwrap()
+        HardwareManager::new(&log, sled_mode, external_disks).unwrap()
     })
     .await
     .unwrap()
@@ -363,7 +361,7 @@ async fn upsert_synthetic_disks_if_needed(
     raw_disks_tx: &RawDisksSender,
     config: &Config,
 ) {
-    if let Some(vdevs) = &config.vdevs {
+    if let ExternalDisks::Virtual { vdevs } = &config.external_disks {
         for (i, vdev) in vdevs.iter().enumerate() {
             info!(
                 log,

sled-hardware/src/illumos/mod.rs

@@ -3,6 +3,7 @@
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 use crate::DiskFirmware;
+use crate::ExternalDisks;
 use crate::HardwareView;
 use crate::TofinoSnapshot;
 use crate::TofinoView;
@@ -124,7 +125,10 @@ struct HardwareSnapshot {
 
 impl HardwareSnapshot {
     // Walk the device tree to capture a view of the current hardware.
-    fn new(log: &Logger) -> Result<Self, Error> {
+    fn new(
+        log: &Logger,
+        external_disks: &ExternalDisks,
+    ) -> Result<Self, Error> {
         let mut device_info =
             DevInfo::new_force_load().map_err(Error::DevInfo)?;
 
@@ -168,17 +172,27 @@ impl HardwareSnapshot {
 
         // Monitor for block devices.
         let mut disks = HashMap::new();
-        let mut node_walker = device_info.walk_driver("blkdev");
-        while let Some(node) =
-            node_walker.next().transpose().map_err(Error::DevInfo)?
-        {
-            poll_blkdev_node(
-                &log,
-                sled_type,
-                &mut disks,
-                node,
-                boot_storage_unit,
-            )?;
+        match external_disks {
+            ExternalDisks::DetectPhysical => {
+                let mut node_walker = device_info.walk_driver("blkdev");
+                while let Some(node) =
+                    node_walker.next().transpose().map_err(Error::DevInfo)?
+                {
+                    poll_blkdev_node(
+                        &log,
+                        sled_type,
+                        &mut disks,
+                        node,
+                        boot_storage_unit,
+                    )?;
+                }
+            }
+            ExternalDisks::HardcodedPhysical { disks: hardcoded_disks } => {
+                for disk in hardcoded_disks {
+                    disks.insert(disk.identity().clone(), disk.clone());
+                }
+            }
+            ExternalDisks::Virtual { .. } => {}
         }
 
         Ok(Self { tofino, disks, baseboard })
@@ -504,10 +518,10 @@ fn poll_blkdev_node(
 fn poll_device_tree(
     log: &Logger,
     hardware_view_tx: &watch::Sender<HardwareView>,
-    nonsled_observed_disks: &[UnparsedDisk],
+    external_disks: &ExternalDisks,
 ) -> Result<(), Error> {
     // Construct a view of hardware by walking the device tree.
-    let polled_hw = match HardwareSnapshot::new(log) {
+    let polled_hw = match HardwareSnapshot::new(log, external_disks) {
         Ok(polled_hw) => polled_hw,
 
         Err(e) => {
@@ -529,16 +543,19 @@ fn poll_device_tree(
 
                     // For platforms that don't support the HardwareSnapshot
                     // functionality, sled-agent can be supplied a fixed list of
-                    // UnparsedDisks. Add those to the HardwareSnapshot here if they
-                    // are missing (which they will be for non-sleds).
-                    for observed_disk in nonsled_observed_disks {
-                        let identity = observed_disk.identity();
-                        if !inner.disks.contains_key(identity) {
-                            inner.disks.insert(
-                                identity.clone(),
-                                observed_disk.clone(),
-                            );
-                            did_modify = true;
+                    // UnparsedDisks. Add those to the HardwareSnapshot here if
+                    // they are missing (which they will be for non-sleds).
+                    if let ExternalDisks::HardcodedPhysical { disks } =
+                        external_disks
+                    {
+                        for disk in disks {
+                            let identity = disk.identity();
+                            if !inner.disks.contains_key(identity) {
+                                inner
+                                    .disks
+                                    .insert(identity.clone(), disk.clone());
+                                did_modify = true;
+                            }
                         }
                     }
 
@@ -747,11 +764,10 @@ fn parse_smbios_output(log: &Logger, output: String) -> Option<Baseboard> {
 fn hardware_tracking_task(
     log: Logger,
     hardware_view_tx: watch::Sender<HardwareView>,
-    nonsled_observed_disks: Vec<UnparsedDisk>,
+    external_disks: ExternalDisks,
 ) {
     loop {
-        match poll_device_tree(&log, &hardware_view_tx, &nonsled_observed_disks)
-        {
+        match poll_device_tree(&log, &hardware_view_tx, &external_disks) {
             // We've already warned about `NotAnOxideSled` by this point,
             // so let's not spam the logs.
             Ok(_) | Err(Error::NotAnOxideSled(_)) => (),
@@ -785,7 +801,7 @@ impl HardwareManager {
     pub fn new(
         log: &Logger,
         sled_mode: SledMode,
-        nonsled_observed_disks: Vec<UnparsedDisk>,
+        external_disks: ExternalDisks,
     ) -> Result<Self, String> {
         let log = log.new(o!("component" => "HardwareManager"));
         info!(log, "Creating HardwareManager");
@@ -837,8 +853,7 @@ impl HardwareManager {
         // This mitigates issues where the Sled Agent could try to propagate
         // an "empty" view of hardware to other consumers before the first
         // query.
-        match poll_device_tree(&log, &hardware_view_tx, &nonsled_observed_disks)
-        {
+        match poll_device_tree(&log, &hardware_view_tx, &external_disks) {
             Ok(_) => (),
             // Allow non-sled devices to proceed with a "null" view of
             // hardware, otherwise they won't be able to start.
@@ -865,11 +880,7 @@ impl HardwareManager {
         });
 
         std::thread::spawn(move || {
-            hardware_tracking_task(
-                log,
-                hardware_view_tx,
-                nonsled_observed_disks,
-            );
+            hardware_tracking_task(log, hardware_view_tx, external_disks);
         });
 
         Ok(Self { hardware_view_rx })

sled-hardware/src/lib.rs

@@ -65,6 +65,18 @@ pub enum DataLinks {
     Physical,
 }
 
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case", tag = "kind")]
+pub enum ExternalDisks {
+    /// Instead of detecting physical disks, use the provided file-backed vdevs.
+    Virtual { vdevs: Vec<String> },
+    /// Instead of detecting physical disks, inject the provided ones during
+    /// device polling.
+    HardcodedPhysical { disks: Vec<UnparsedDisk> },
+    /// Detect physical external disks connected to the sled.
+    DetectPhysical,
+}
+
 /// Configuration for forcing a sled to run as a Scrimlet or compute Sled
 #[derive(Copy, Clone, Debug)]
 pub enum SledMode {

sled-hardware/src/non_illumos/mod.rs

@@ -5,7 +5,7 @@
 use std::collections::HashMap;
 
 use crate::disk::{DiskPaths, Partition, PooledDiskError, UnparsedDisk};
-use crate::{HardwareView, SledMode};
+use crate::{ExternalDisks, HardwareView, SledMode};
 use omicron_common::disk::{DiskIdentity, DiskVariant};
 use omicron_uuid_kinds::ZpoolUuid;
 use sled_hardware_types::{Baseboard, SledCpuFamily};
@@ -33,7 +33,7 @@ impl HardwareManager {
     pub fn new(
         _log: &Logger,
         _sled_mode: SledMode,
-        _nonsled_observed_disks: Vec<UnparsedDisk>,
+        _external_disks: ExternalDisks,
     ) -> Result<Self, String> {
         unimplemented!("Accessing hardware unsupported on non-illumos");
     }

smf/sled-agent/gimlet-standalone/config.toml

@@ -63,6 +63,9 @@ swap_device_size_gb = 256
 # $ dladm show-phys -p -o LINK
 # data_link = "igb0"
 
+[external_disks]
+kind = "detect_physical"
+
 [data_links]
 kind = "virtual"
 devices = ["net0", "net1"]

smf/sled-agent/gimlet/config.toml

@@ -59,6 +59,9 @@ vmm_reservoir_percentage = 86.3
 # allocated.
 swap_device_size_gb = 256
 
+[external_disks]
+kind = "detect_physical"
+
 [data_links]
 kind = "physical"