Add support for re-adopting physical disks (#10221)

This change implements the determinations in RFD 663. It allows
re-adopting physical disks in the control plane after the control plane
level disk in the `physical_disk` table is expunged.

It does this by forcing manual adoption of disks by an operator, where
requests are placed in the `physical_disk_adoption_request` table.
A disk will now only be adopted or re-adopted by the disk adoption
background task if its physical vendor/model/serial information is
present in a `physical_disk_adoption_request` row.

The typical flow for an operator is to list uninitialized disks and then
issue an adoption request via the external API.

We also allow automatic re-adoption of disks if they have never been seen by the
same sled. For instance, if a sled is expunged and zpools destroyed, re-adding the
sled will give it a different id and therefore allow adoption of disks with the same 
physical id.

---------

Co-authored-by: Andy Fiddaman <illumos@fiddaman.net>

Author: andrewjstone

dev-tools/omdb/src/bin/omdb/nexus.rs

@@ -65,6 +65,7 @@ use nexus_types::internal_api::background::InstanceReincarnationStatus;
 use nexus_types::internal_api::background::InstanceUpdaterStatus;
 use nexus_types::internal_api::background::InventoryLoadStatus;
 use nexus_types::internal_api::background::LookupRegionPortStatus;
+use nexus_types::internal_api::background::PhysicalDiskAdoptionStatus;
 use nexus_types::internal_api::background::ProbeDistributorStatus;
 use nexus_types::internal_api::background::ReadOnlyRegionReplacementStartStatus;
 use nexus_types::internal_api::background::RegionReplacementDriverStatus;
@@ -1300,6 +1301,9 @@ fn print_task_details(bgtask: &BackgroundTask, details: &serde_json::Value) {
         "phantom_disks" => {
             print_task_phantom_disks(details);
         }
+        "physical_disk_adoption" => {
+            print_task_physical_disk_adoption(details);
+        }
         "probe_distributor" => {
             print_task_probe_distributor(details);
         }
@@ -3949,6 +3953,25 @@ fn print_task_trust_quorum_manager(details: &serde_json::Value) {
     }
 }
 
+fn print_task_physical_disk_adoption(details: &serde_json::Value) {
+    let status = match serde_json::from_value::<PhysicalDiskAdoptionStatus>(
+        details.clone(),
+    ) {
+        Ok(status) => status,
+        Err(error) => {
+            eprintln!(
+                "warning: failed to interpret task details: {:?}: {:#?}",
+                error, details
+            );
+            return;
+        }
+    };
+    println!("physical disks added: {}", status.disks_added);
+    for error in status.errors {
+        println!("{ERRICON} {error}");
+    }
+}
+
 const ERRICON: &str = "/!\\";
 
 fn warn_if_nonzero(n: usize) -> &'static str {

dev-tools/omdb/tests/successes.out

@@ -852,7 +852,8 @@ task: "physical_disk_adoption"
   configured period: every <REDACTED_DURATION>s
   last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
     started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
-    last completion reported error: task disabled
+physical disks added: 0
+/!\ task disabled
 
 task: "probe_distributor"
   configured period: every <REDACTED_DURATION>m
@@ -1534,7 +1535,8 @@ task: "physical_disk_adoption"
   configured period: every <REDACTED_DURATION>s
   last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
     started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
-    last completion reported error: task disabled
+physical disks added: 0
+/!\ task disabled
 
 task: "probe_distributor"
   configured period: every <REDACTED_DURATION>m

nexus/db-model/src/physical_disk.rs

@@ -6,12 +6,15 @@ use super::{
     Generation, PhysicalDiskKind, PhysicalDiskPolicy, PhysicalDiskState,
 };
 use crate::DbTypedUuid;
+use crate::InvPhysicalDisk;
 use crate::collection::DatastoreCollectionConfig;
 use chrono::{DateTime, Utc};
 use db_macros::Asset;
 use nexus_db_schema::schema::{physical_disk, zpool};
 use nexus_types::external_api::physical_disk as physical_disk_types;
+use nexus_types::external_api::physical_disk::PhysicalDiskManufacturerIdentity;
 use nexus_types::identity::Asset;
+use omicron_uuid_kinds::PhysicalDiskAdoptionRequestKind;
 use omicron_uuid_kinds::PhysicalDiskKind as PhysicalDiskUuidKind;
 use omicron_uuid_kinds::PhysicalDiskUuid;
 use omicron_uuid_kinds::SledKind;
@@ -38,8 +41,20 @@ pub struct PhysicalDisk {
 }
 
 impl PhysicalDisk {
-    /// Creates a new in-service, active disk
-    pub fn new(
+    /// Creates a new in-service, active disk from an inventory disk
+    pub fn new(inv_disk: InvPhysicalDisk) -> Self {
+        Self::from_parts(
+            PhysicalDiskUuid::new_v4(),
+            inv_disk.vendor,
+            inv_disk.serial,
+            inv_disk.model,
+            inv_disk.variant,
+            inv_disk.sled_id.into(),
+        )
+    }
+
+    /// Creates a new in-service, active disk from individual fields
+    pub fn from_parts(
         id: PhysicalDiskUuid,
         vendor: String,
         serial: String,
@@ -74,6 +89,14 @@ impl PhysicalDisk {
     }
 }
 
+impl From<PhysicalDisk>
+    for physical_disk_types::PhysicalDiskManufacturerIdentity
+{
+    fn from(value: PhysicalDisk) -> Self {
+        Self { vendor: value.vendor, serial: value.serial, model: value.model }
+    }
+}
+
 impl From<PhysicalDisk> for physical_disk_types::PhysicalDisk {
     fn from(disk: PhysicalDisk) -> Self {
         Self {
@@ -89,6 +112,34 @@ impl From<PhysicalDisk> for physical_disk_types::PhysicalDisk {
     }
 }
 
+/// A request to adopt a physical disk into the control plane.
+#[derive(Queryable, Insertable, Debug, Clone, Selectable)]
+#[diesel(table_name = nexus_db_schema::schema::physical_disk_adoption_request)]
+pub struct PhysicalDiskAdoptionRequest {
+    pub id: DbTypedUuid<PhysicalDiskAdoptionRequestKind>,
+    pub vendor: String,
+    pub serial: String,
+    pub model: String,
+    pub time_created: DateTime<Utc>,
+    pub time_deleted: Option<DateTime<Utc>>,
+}
+
+impl From<PhysicalDiskAdoptionRequest>
+    for physical_disk_types::PhysicalDiskAdoptionRequest
+{
+    fn from(req: PhysicalDiskAdoptionRequest) -> Self {
+        Self {
+            id: req.id.into(),
+            disk_id: PhysicalDiskManufacturerIdentity {
+                vendor: req.vendor,
+                serial: req.serial,
+                model: req.model,
+            },
+            time_created: req.time_created,
+        }
+    }
+}
+
 impl DatastoreCollectionConfig<super::Zpool> for PhysicalDisk {
     type CollectionId = DbTypedUuid<PhysicalDiskUuidKind>;
     type GenerationNumberColumn = physical_disk::dsl::rcgen;

nexus/db-model/src/schema_versions.rs

@@ -16,7 +16,7 @@ use std::{collections::BTreeMap, sync::LazyLock};
 ///
 /// This must be updated when you change the database schema.  Refer to
 /// schema/crdb/README.adoc in the root of this repository for details.
-pub const SCHEMA_VERSION: Version = Version::new(256, 0, 0);
+pub const SCHEMA_VERSION: Version = Version::new(257, 0, 0);
 
 /// List of all past database schema versions, in *reverse* order
 ///
@@ -28,6 +28,7 @@ pub static KNOWN_VERSIONS: LazyLock<Vec<KnownVersion>> = LazyLock::new(|| {
         // |  leaving the first copy as an example for the next person.
         // v
         // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"),
+        KnownVersion::new(257, "add-disk-adoption-requests"),
         KnownVersion::new(256, "bgp-unnumbered-peer-cleanup"),
         KnownVersion::new(255, "blueprint-add-external-networking-generation"),
         KnownVersion::new(

nexus/db-queries/src/db/datastore/mod.rs

@@ -1013,7 +1013,7 @@ mod test {
         kind: PhysicalDiskKind,
         serial: String,
     ) -> PhysicalDiskUuid {
-        let physical_disk = PhysicalDisk::new(
+        let physical_disk = PhysicalDisk::from_parts(
             PhysicalDiskUuid::new_v4(),
             TEST_VENDOR.into(),
             serial,