merge main into multicast-e2e + update latest mcast dendrite/mags

Author: zeeshanlakhani

Cargo.lock

@@ -163,6 +163,7 @@ members = [
     "sled-storage",
     "sled-storage/zfs-test-harness",
     "sp-sim",
+    "support-bundle-collection",
     "test-utils",
     "trust-quorum",
     "trust-quorum/gfss",
@@ -351,6 +352,7 @@ default-members = [
     "sled-storage",
     "sled-storage/zfs-test-harness",
     "sp-sim",
+    "support-bundle-collection",
     "trust-quorum",
     "trust-quorum/gfss",
     "trust-quorum/protocol",
@@ -484,7 +486,7 @@ crucible-common = { git = "https://github.com/oxidecomputer/crucible", rev = "3c
 # NOTE: See above!
 csv = "1.3.1"
 curve25519-dalek = "4"
-daft = { version = "0.1.4", features = ["derive", "newtype-uuid1", "oxnet01", "uuid1"] }
+daft = { version = "0.1.7", features = ["derive", "newtype-uuid1", "oxnet01", "uuid1"] }
 display-error-chain = "0.2.2"
 omicron-ddm-admin-client = { path = "clients/ddm-admin-client" }
 datatest-stable = "0.3.2"
@@ -501,10 +503,10 @@ digest = "0.10.7"
 dns-server = { path = "dns-server" }
 dns-server-api = { path = "dns-server-api" }
 dns-service-client = { path = "clients/dns-service-client" }
-dpd-client = { git = "https://github.com/oxidecomputer/dendrite", rev = "b6d8ba7957c8ed1e6b29fa4b2d85ad2c2248565e" }
+dpd-client = { git = "https://github.com/oxidecomputer/dendrite", rev = "90efdc6cf618523320bc403f1b9484dbd7f762be" }
 dropshot = { version = "0.17.0", features = [ "usdt-probes" ] }
-dropshot-api-manager = "0.7.1"
-dropshot-api-manager-types = "0.7.1"
+dropshot-api-manager = "0.7.2"
+dropshot-api-manager-types = "0.7.2"
 dyn-clone = "1.0.20"
 either = "1.15.0"
 ereport-types = { path = "ereport/types" }
@@ -611,9 +613,9 @@ ntp-admin-client = { path = "clients/ntp-admin-client" }
 ntp-admin-v1-client = { path = "clients/ntp-admin-v1-client" }
 ntp-admin-types = { path = "ntp-admin/types" }
 ntp-admin-types-versions = { path = "ntp-admin/types/versions" }
-mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "864e0e150a90fd838199ceb38c2ad8a2be92a270" }
-mg-api-types = { git = "https://github.com/oxidecomputer/maghemite", rev = "864e0e150a90fd838199ceb38c2ad8a2be92a270" }
-ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "864e0e150a90fd838199ceb38c2ad8a2be92a270" }
+mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "5060d5864df864741dac8a0a4078fa4ecc3c73c8" }
+mg-api-types = { git = "https://github.com/oxidecomputer/maghemite", rev = "5060d5864df864741dac8a0a4078fa4ecc3c73c8" }
+ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "5060d5864df864741dac8a0a4078fa4ecc3c73c8" }
 multimap = "0.10.1"
 nexus-auth = { path = "nexus/auth" }
 nexus-background-task-interface = { path = "nexus/background-task-interface" }
@@ -830,6 +832,7 @@ strum = { version = "0.27.2", features = [ "derive" ] }
 subprocess = "0.2.9"
 subtle = "2.6.1"
 supports-color = "3.0.2"
+support-bundle-collection = { path = "support-bundle-collection" }
 support-bundle-viewer = "0.1.2"
 swrite = "0.1.0"
 sync-ptr = "0.1.4"

Cargo.toml

@@ -48,6 +48,8 @@ serde_json.workspace = true
 slog-term.workspace = true
 subprocess.workspace = true
 url.workspace = true
+uuid.workspace = true
+chrono-tz.workspace = true
 
 [lints]
 workspace = true

clickhouse-admin/Cargo.toml

@@ -783,12 +783,19 @@ mod tests {
     use crate::context::ServerContext;
     use crate::context::USAGE_UPDATE_INTERVAL;
     use camino::Utf8PathBuf;
+    use chrono::NaiveDate;
+    use chrono::TimeZone;
+    use chrono_tz::Tz;
     use clickhouse_admin_types::CLICKHOUSE_SERVER_CONFIG_FILE;
     use clickhouse_admin_types::retention::Days;
     use clickhouse_admin_types::retention::RetentionPolicyRequest;
     use dropshot::ErrorStatusCode;
     use omicron_common::api::external::Generation;
     use omicron_test_utils::dev;
+    use oximeter_db::native::block::Block;
+    use oximeter_db::native::block::Column;
+    use oximeter_db::native::block::Precision;
+    use oximeter_db::native::block::ValueArray;
     use slog::info;
     use std::str::FromStr;
 
@@ -967,12 +974,15 @@ mod tests {
         assert!(usage.last_success.is_some());
 
         // Wait until we actually do compute the usage again.
+        //
+        // From `grep -c "CREATE TABLE" oximeter/db/schema/single-node/db-init.sql`.
+        const N_EXPECTED_TABLES: usize = 39;
         let usage = dev::poll::wait_for_condition(
             || async {
                 let usage = context.database_usage();
                 match &usage.last_success {
                     Some(success) => {
-                        if success.tables.is_empty() {
+                        if success.tables.len() < N_EXPECTED_TABLES {
                             Err(dev::poll::CondCheckError::<()>::NotYet)
                         } else {
                             Ok(usage)
@@ -998,7 +1008,74 @@ mod tests {
         assert!(
             tables.contains_key(&String::from("oximeter.measurements_f64"))
         );
-        assert!(tables.contains_key(&String::from("oximeter.version")));
+
+        // Insert some rows in the version table, so we can see it updated.
+        let version_table = String::from("oximeter.version");
+        let version_usage =
+            tables.get(&version_table).expect("Should have this table");
+        let naive_dt = NaiveDate::from_ymd_opt(2020, 1, 1)
+            .unwrap()
+            .and_hms_opt(12, 12, 12)
+            .unwrap();
+        let timestamp = Tz::UTC.from_local_datetime(&naive_dt).unwrap();
+        oximeter_db::native::Connection::new(
+            clickhouse.native_address().into(),
+        )
+        .await
+        .expect("Should be able to make client")
+        .insert(
+            uuid::Uuid::new_v4(),
+            "INSERT INTO oximeter.version FORMAT NATIVE",
+            Block {
+                name: String::new(),
+                info: Default::default(),
+                columns: [
+                    (
+                        "value".to_string(),
+                        Column::from(ValueArray::UInt64(vec![1, 2, 3])),
+                    ),
+                    (
+                        "timestamp".to_string(),
+                        Column::from(ValueArray::DateTime64 {
+                            precision: Precision::new(9).unwrap(),
+                            tz: Tz::UTC,
+                            values: vec![timestamp; 3],
+                        }),
+                    ),
+                ]
+                .into(),
+            },
+        )
+        .await
+        .expect("Should be able to insert data");
+
+        // Check again, waiting until we get more than the previous usage.
+        let usage = dev::poll::wait_for_condition(
+            || async {
+                let usage = context.database_usage();
+                match &usage.last_success {
+                    Some(success) => {
+                        let Some(usage) = success.tables.get(&version_table)
+                        else {
+                            return Err(
+                                dev::poll::CondCheckError::<()>::NotYet,
+                            );
+                        };
+                        if usage.n_rows > version_usage.n_rows {
+                            Ok(usage.clone())
+                        } else {
+                            Err(dev::poll::CondCheckError::<()>::NotYet)
+                        }
+                    }
+                    None => Err(dev::poll::CondCheckError::<()>::NotYet),
+                }
+            },
+            &std::time::Duration::from_millis(50),
+            &(2 * USAGE_UPDATE_INTERVAL),
+        )
+        .await
+        .expect("New rows didn't show up on time");
+        assert_eq!(usage.n_rows, 4);
 
         // Kill the database, and wait for another collection. This one should
         // fail.

clickhouse-admin/src/context.rs

@@ -26,6 +26,7 @@ dropshot.workspace = true
 futures.workspace = true
 hex.workspace = true
 http.workspace = true
+humantime.workspace = true
 iddqd.workspace = true
 ipnetwork.workspace = true
 itertools.workspace = true

common/Cargo.toml

@@ -69,6 +69,25 @@ pub fn now_db_precision() -> chrono::DateTime<chrono::Utc> {
     ts - std::time::Duration::from_nanos(u64::from(only_nanos))
 }
 
+/// Format a [`std::time::Duration`] as a human-readable string
+/// (e.g. `"1h 5m 23ms"`), truncated to millisecond precision.
+pub fn format_duration_ms(duration: std::time::Duration) -> String {
+    // Ignore units smaller than a millisecond.
+    let elapsed = std::time::Duration::from_millis(
+        u64::try_from(duration.as_millis()).unwrap_or(u64::MAX),
+    );
+    humantime::format_duration(elapsed).to_string()
+}
+
+/// Format a [`chrono::TimeDelta`] as a human-readable string (see
+/// [`format_duration_ms`]).
+pub fn format_time_delta(time_delta: chrono::TimeDelta) -> String {
+    match time_delta.to_std() {
+        Ok(d) => format_duration_ms(d),
+        Err(_) => String::from("<time delta out of range>"),
+    }
+}
+
 pub const OMICRON_DPD_TAG: &str = "omicron";
 
 /// A wrapper struct that does nothing other than elide the inner value from

common/src/lib.rs

@@ -51,6 +51,15 @@ pub enum ZpoolName {
     Internal(InternalZpoolUuid),
 }
 
+impl fmt::Display for ZpoolKind {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            ZpoolKind::External => write!(f, "external"),
+            ZpoolKind::Internal => write!(f, "internal"),
+        }
+    }
+}
+
 const ZPOOL_NAME_REGEX: &str = r"^ox[ip]_[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$";
 
 /// Custom JsonSchema implementation to encode the constraints on Name.

common/src/zpool_name.rs

@@ -49,7 +49,7 @@ Management Gateway Service (client: gateway-client)
     consumed by: dpd (dendrite/dpd) via 1 path
     consumed by: lldpd (lldp/lldpd) via 1 path
     consumed by: mgd (maghemite/mgd) via 1 path
-    consumed by: omicron-nexus (omicron/nexus) via 5 paths
+    consumed by: omicron-nexus (omicron/nexus) via 6 paths
     consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path
     consumed by: wicketd (omicron/wicketd) via 3 paths
 
@@ -99,7 +99,7 @@ Repo Depot API (client: repo-depot-client)
     consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path
 
 Sled Agent (client: sled-agent-client)
-    consumed by: omicron-nexus (omicron/nexus) via 8 paths
+    consumed by: omicron-nexus (omicron/nexus) via 9 paths
 
 Wicketd (client: wicketd-client)
 

dev-tools/ls-apis/tests/api_dependencies.out

@@ -7,15 +7,13 @@
 use crate::Omdb;
 use crate::check_allow_destructive::DestructiveOperationToken;
 use anyhow::Context;
-use chrono::TimeDelta;
 use chrono::Utc;
 use clap::Args;
 use clap::Subcommand;
 use nexus_lockstep_client::types::PendingRecovery;
 use nexus_lockstep_client::types::QuiesceState;
 use nexus_lockstep_client::types::QuiesceStatus;
 use nexus_lockstep_client::types::SagaQuiesceStatus;
-use std::time::Duration;
 
 #[derive(Debug, Args)]
 pub struct QuiesceArgs {
@@ -77,7 +75,7 @@ async fn quiesce_show(
             println!(
                 "quiescing since {} ({} ago)",
                 humantime::format_rfc3339_millis(time_requested.into()),
-                format_time_delta(now - time_requested),
+                omicron_common::format_time_delta(now - time_requested),
             );
             println!("details: waiting for running sagas to finish");
         }
@@ -89,14 +87,16 @@ async fn quiesce_show(
             println!(
                 "quiescing since {} ({} ago)",
                 humantime::format_rfc3339_millis(time_requested.into()),
-                format_time_delta(now - time_requested),
+                omicron_common::format_time_delta(now - time_requested),
             );
             println!(
                 "details: waiting for database connections to be released"
             );
             println!(
                 "    previously: waiting for sagas took {}",
-                format_duration_ms(duration_draining_sagas.into()),
+                omicron_common::format_duration_ms(
+                    duration_draining_sagas.into()
+                ),
             );
         }
         QuiesceState::RecordingQuiesce {
@@ -108,15 +108,17 @@ async fn quiesce_show(
             println!(
                 "quiescing since {} ({} ago)",
                 humantime::format_rfc3339_millis(time_requested.into()),
-                format_time_delta(now - time_requested),
+                omicron_common::format_time_delta(now - time_requested),
             );
             println!(
                 "    waiting for sagas took {}",
-                format_duration_ms(duration_draining_sagas.into()),
+                omicron_common::format_duration_ms(
+                    duration_draining_sagas.into()
+                ),
             );
             println!(
                 "    waiting for db quiesce took {}",
-                format_duration_ms(duration_draining_db.into()),
+                omicron_common::format_duration_ms(duration_draining_db.into()),
             );
         }
         QuiesceState::Quiesced {
@@ -130,23 +132,27 @@ async fn quiesce_show(
             println!(
                 "quiesced since {} ({} ago)",
                 humantime::format_rfc3339_millis(time_quiesced.into()),
-                format_time_delta(now - time_quiesced),
+                omicron_common::format_time_delta(now - time_quiesced),
             );
             println!(
                 "    waiting for sagas took {}",
-                format_duration_ms(duration_draining_sagas.into()),
+                omicron_common::format_duration_ms(
+                    duration_draining_sagas.into()
+                ),
             );
             println!(
                 "    waiting for db quiesce took {}",
-                format_duration_ms(duration_draining_db.into()),
+                omicron_common::format_duration_ms(duration_draining_db.into()),
             );
             println!(
                 "    recording quiesce took {}",
-                format_duration_ms(duration_recording_quiesce.into()),
+                omicron_common::format_duration_ms(
+                    duration_recording_quiesce.into()
+                ),
             );
             println!(
                 "    total quiesce time: {}",
-                format_duration_ms(duration_total.into()),
+                omicron_common::format_duration_ms(duration_total.into()),
             );
         }
     }
@@ -225,7 +231,7 @@ async fn quiesce_show(
             "    claim {} held since {} ({} ago)",
             claim.id,
             claim.held_since,
-            format_time_delta(Utc::now() - claim.held_since),
+            omicron_common::format_time_delta(Utc::now() - claim.held_since),
         );
         if args.stacks {
             println!("    acquired by:");
@@ -243,18 +249,3 @@ async fn quiesce_start(
     client.quiesce_start().await.context("quiescing Nexus")?;
     quiesce_show(client, &QuiesceShowArgs { stacks: false }).await
 }
-
-fn format_duration_ms(duration: Duration) -> String {
-    // Ignore units smaller than a millisecond.
-    let elapsed = Duration::from_millis(
-        u64::try_from(duration.as_millis()).unwrap_or(u64::MAX),
-    );
-    humantime::format_duration(elapsed).to_string()
-}
-
-fn format_time_delta(time_delta: TimeDelta) -> String {
-    match time_delta.to_std() {
-        Ok(d) => format_duration_ms(d),
-        Err(_) => String::from("<time delta out of range>"),
-    }
-}