Add omdb command to view network config from the bootstore (#10225)

Closes #9931.

Author: jgallagher

Cargo.lock

@@ -221,6 +221,12 @@ impl NodeHandle {
         rx.await?
     }
 
+    /// Get the current contents of the watch channel containing the network
+    /// config
+    pub fn network_config_contents(&self) -> Option<NetworkConfig> {
+        self.network_config_rx.borrow().clone()
+    }
+
     /// Subscribe to the watch channel containing the network config
     pub fn network_config_subscribe(
         &self,

bootstore/src/schemes/v0/peer.rs

@@ -216,7 +216,13 @@ pub const MIN_PORT: u16 = u16::MIN;
 pub const DNS_PORT: u16 = 53;
 pub const DNS_HTTP_PORT: u16 = 5353;
 pub const SLED_AGENT_PORT: u16 = 12345;
+pub const BOOTSTRAP_AGENT_RACK_INIT_PORT: u16 = 12346;
+pub const BOOTSTORE_PORT: u16 = 12347;
 pub const REPO_DEPOT_PORT: u16 = 12348;
+pub const TRUST_QUORUM_PORT: u16 = 12349;
+
+pub const BOOTSTRAP_AGENT_HTTP_PORT: u16 = 80;
+pub const BOOTSTRAP_AGENT_LOCKSTEP_PORT: u16 = 8080;
 
 pub const COCKROACH_PORT: u16 = 32221;
 pub const COCKROACH_ADMIN_PORT: u16 = 32222;

common/src/address.rs

@@ -15,6 +15,8 @@ vergen-gitcl.workspace = true
 anyhow.workspace = true
 async-bb8-diesel.workspace = true
 async-trait.workspace = true
+base64.workspace = true
+bootstrap-agent-lockstep-client.workspace = true
 bytes.workspace = true
 camino.workspace = true
 chrono.workspace = true

dev-tools/omdb/Cargo.toml

@@ -8,7 +8,11 @@ use crate::Omdb;
 use crate::check_allow_destructive::DestructiveOperationToken;
 use crate::helpers::CONNECTION_OPTIONS_HEADING;
 use anyhow::Context;
+use anyhow::anyhow;
 use anyhow::bail;
+use base64::Engine;
+use bootstrap_agent_lockstep_client::types::ReplicatedNetworkConfig;
+use bootstrap_agent_lockstep_client::types::ReplicatedNetworkConfigContents;
 use clap::Args;
 use clap::Subcommand;
 use sled_agent_client::types::NodeStatus;
@@ -18,6 +22,8 @@ use sled_agent_client::types::OperatorSwitchZonePolicy;
 #[derive(Debug, Args)]
 pub struct SledAgentArgs {
     /// URL of the Sled internal API
+    ///
+    /// This is typically port 12345 on a sled's underlay network IP address.
     #[clap(
         long,
         env = "OMDB_SLED_AGENT_URL",
@@ -26,6 +32,17 @@ pub struct SledAgentArgs {
     )]
     sled_agent_url: Option<String>,
 
+    /// URL of the sled bootstrap-agent-lockstep API
+    ///
+    /// This is typically port 8080 on a sled's bootstrap network IP address.
+    #[clap(
+        long,
+        env = "OMDB_BOOTSTRAP_LOCKSTEP_URL",
+        global = true,
+        help_heading = CONNECTION_OPTIONS_HEADING,
+    )]
+    bootstrap_lockstep_url: Option<String>,
+
     #[command(subcommand)]
     command: SledAgentCommands,
 }
@@ -41,6 +58,10 @@ enum SledAgentCommands {
     #[clap(subcommand)]
     SwitchZonePolicy(SwitchZonePolicyCommands),
 
+    /// print information about the replicated network config
+    #[clap(subcommand)]
+    NetworkConfig(NetworkConfigCommands),
+
     /// print information about the local bootstore node
     #[clap(subcommand)]
     Bootstore(BootstoreCommands),
@@ -71,6 +92,12 @@ enum SwitchZonePolicyCommands {
     DangerDangerDisable,
 }
 
+#[derive(Debug, Subcommand)]
+enum NetworkConfigCommands {
+    /// show the current contents of the replicated network config
+    Show,
+}
+
 #[derive(Debug, Subcommand)]
 enum BootstoreCommands {
     /// show the internal state of the local bootstore node
@@ -102,30 +129,50 @@ impl SledAgentArgs {
         omdb: &Omdb,
         log: &slog::Logger,
     ) -> Result<(), anyhow::Error> {
-        // This is a little goofy. The sled URL is required, but can come
-        // from the environment, in which case it won't be on the command line.
-        let Some(sled_agent_url) = &self.sled_agent_url else {
-            bail!(
-                "sled URL must be specified with --sled-agent-url or \
-                OMDB_SLED_AGENT_URL"
-            );
+        // We require either a sled_agent_url or a bootstrap_lockstep_url, but
+        // we don't know which one until we match on `self.command` below. Wrap
+        // the conversion from URL-to-client in a closure so we only bail out if
+        // we're missing a URL argument we actually need.
+
+        // Helper to create a sled-agent client.
+        let make_sa_client = || -> anyhow::Result<_> {
+            let Some(sled_agent_url) = &self.sled_agent_url else {
+                bail!(
+                    "sled URL must be specified with --sled-agent-url or \
+                     OMDB_SLED_AGENT_URL"
+                );
+            };
+            Ok(sled_agent_client::Client::new(sled_agent_url, log.clone()))
+        };
+
+        // Helper to create a bootstrap-agent-lockstep client.
+        let make_ba_lockstep_client = || -> anyhow::Result<_> {
+            let Some(bootstrap_lockstep_url) = &self.bootstrap_lockstep_url
+            else {
+                bail!(
+                    "bootstrap lockstep URL must be specified with \
+                     --bootstrap-lockstep-url or OMDB_BOOTSTRAP_LOCKSTEP_URL"
+                );
+            };
+            Ok(bootstrap_agent_lockstep_client::Client::new(
+                bootstrap_lockstep_url,
+                log.clone(),
+            ))
         };
-        let client =
-            sled_agent_client::Client::new(sled_agent_url, log.clone());
 
         match &self.command {
             SledAgentCommands::Zones(ZoneCommands::List) => {
-                cmd_zones_list(&client).await
+                cmd_zones_list(&make_sa_client()?).await
             }
             SledAgentCommands::SwitchZonePolicy(
                 SwitchZonePolicyCommands::Get,
-            ) => cmd_switch_zone_policy_get(&client).await,
+            ) => cmd_switch_zone_policy_get(&make_sa_client()?).await,
             SledAgentCommands::SwitchZonePolicy(
                 SwitchZonePolicyCommands::Enable,
             ) => {
                 let token = omdb.check_allow_destructive()?;
                 cmd_switch_zone_policy_put(
-                    &client,
+                    &make_sa_client()?,
                     OperatorSwitchZonePolicy::StartIfSwitchPresent,
                     token,
                 )
@@ -136,21 +183,24 @@ impl SledAgentArgs {
             ) => {
                 let token = omdb.check_allow_destructive()?;
                 cmd_switch_zone_policy_put(
-                    &client,
+                    &make_sa_client()?,
                     OperatorSwitchZonePolicy::StopDespiteSwitchPresence,
                     token,
                 )
                 .await
             }
+            SledAgentCommands::NetworkConfig(NetworkConfigCommands::Show) => {
+                cmd_network_config_show(&make_ba_lockstep_client()?).await
+            }
             SledAgentCommands::Bootstore(BootstoreCommands::Status) => {
-                cmd_bootstore_status(&client).await
+                cmd_bootstore_status(&make_sa_client()?).await
             }
             SledAgentCommands::TrustQuorum(TrustQuorumCommands::Status) => {
-                cmd_trust_quorum_status(&client).await
+                cmd_trust_quorum_status(&make_sa_client()?).await
             }
             SledAgentCommands::TrustQuorum(
                 TrustQuorumCommands::ProxyStatus(args),
-            ) => cmd_trust_quorum_proxy_status(&client, args).await,
+            ) => cmd_trust_quorum_proxy_status(&make_sa_client()?, args).await,
         }
     }
 }
@@ -212,6 +262,58 @@ async fn cmd_switch_zone_policy_put(
     cmd_switch_zone_policy_get(client).await
 }
 
+/// Runs `omdb sled-agent network-config show`
+async fn cmd_network_config_show(
+    client: &bootstrap_agent_lockstep_client::Client,
+) -> anyhow::Result<()> {
+    let ReplicatedNetworkConfig { contents } = client
+        .network_config_contents_for_debug()
+        .await
+        .context("failed to fetch network config contents")?
+        .into_inner();
+    let Some(contents) = contents else {
+        println!(
+            "no network contents available yet - \
+             this is normal if RSS has not yet started"
+        );
+        return Ok(());
+    };
+
+    let ReplicatedNetworkConfigContents { base64_blob, generation } = contents;
+    println!("network config generation: {generation}");
+
+    let blob = {
+        let mut output = Vec::new();
+        match base64::engine::general_purpose::STANDARD
+            .decode_vec(&base64_blob, &mut output)
+        {
+            Ok(()) => output,
+            Err(err) => {
+                println!("raw data (expected base64): {base64_blob}");
+                return Err(
+                    anyhow!(err).context("failed to decode base64 blob")
+                );
+            }
+        }
+    };
+
+    let decoded: serde_json::Value = match serde_json::from_slice(&blob) {
+        Ok(value) => value,
+        Err(err) => {
+            println!(
+                "raw blob (expected JSON): {}",
+                String::from_utf8_lossy(&blob)
+            );
+            return Err(anyhow!(err).context("failed to decode blob as JSON"));
+        }
+    };
+
+    println!("network config contents:");
+    println!("{decoded:#}"); // `:#` format induces pretty-printing
+
+    Ok(())
+}
+
 /// Runs `omdb sled-agent bootstore status`
 async fn cmd_bootstore_status(
     client: &sled_agent_client::Client,

dev-tools/omdb/src/bin/omdb/sled_agent.rs

@@ -1122,18 +1122,23 @@ Usage: omdb sled-agent [OPTIONS] <COMMAND>
 Commands:
   zones               print information about zones
   switch-zone-policy  control the switch zone policy
+  network-config      print information about the replicated network config
   bootstore           print information about the local bootstore node
   trust-quorum        print information about the local trust quorum node
   help                Print this message or the help of the given subcommand(s)
 
 Options:
       --log-level <LOG_LEVEL>  log level filter [env: LOG_LEVEL=] [default: warn]
       --color <COLOR>          Color output [default: auto] [possible values: auto, always, never]
-  -h, --help                   Print help
+  -h, --help                   Print help (see more with '--help')
 
 Connection Options:
-      --sled-agent-url <SLED_AGENT_URL>  URL of the Sled internal API [env: OMDB_SLED_AGENT_URL=]
-      --dns-server <DNS_SERVER>          [env: OMDB_DNS_SERVER=]
+      --sled-agent-url <SLED_AGENT_URL>
+          URL of the Sled internal API [env: OMDB_SLED_AGENT_URL=]
+      --bootstrap-lockstep-url <BOOTSTRAP_LOCKSTEP_URL>
+          URL of the sled bootstrap-agent-lockstep API [env: OMDB_BOOTSTRAP_LOCKSTEP_URL=]
+      --dns-server <DNS_SERVER>
+          [env: OMDB_DNS_SERVER=]
 
 Safety Options:
   -w, --destructive  Allow potentially-destructive subcommands
@@ -1155,11 +1160,15 @@ Commands:
 Options:
       --log-level <LOG_LEVEL>  log level filter [env: LOG_LEVEL=] [default: warn]
       --color <COLOR>          Color output [default: auto] [possible values: auto, always, never]
-  -h, --help                   Print help
+  -h, --help                   Print help (see more with '--help')
 
 Connection Options:
-      --sled-agent-url <SLED_AGENT_URL>  URL of the Sled internal API [env: OMDB_SLED_AGENT_URL=]
-      --dns-server <DNS_SERVER>          [env: OMDB_DNS_SERVER=]
+      --sled-agent-url <SLED_AGENT_URL>
+          URL of the Sled internal API [env: OMDB_SLED_AGENT_URL=]
+      --bootstrap-lockstep-url <BOOTSTRAP_LOCKSTEP_URL>
+          URL of the sled bootstrap-agent-lockstep API [env: OMDB_BOOTSTRAP_LOCKSTEP_URL=]
+      --dns-server <DNS_SERVER>
+          [env: OMDB_DNS_SERVER=]
 
 Safety Options:
   -w, --destructive  Allow potentially-destructive subcommands

dev-tools/omdb/tests/usage_errors.out

@@ -10,6 +10,31 @@
     "version": "0.0.1"
   },
   "paths": {
+    "/debug/network-config-contents": {
+      "get": {
+        "summary": "Get the current contents of the network config kept in the replicated",
+        "description": "bootstore.\n\nThis should ONLY be used for debugging (e.g., via `omdb`). Nexus, RSS, and sled-agent should never access this endpoint in production - the bootstore contents should be treated as \"write only\" from their point of view.",
+        "operationId": "network_config_contents_for_debug",
+        "responses": {
+          "200": {
+            "description": "successful operation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ReplicatedNetworkConfig"
+                }
+              }
+            }
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      }
+    },
     "/rack-initialize": {
       "get": {
         "summary": "Get the current status of rack initialization or reset.",
@@ -1205,6 +1230,39 @@
           "user_password_hash"
         ]
       },
+      "ReplicatedNetworkConfig": {
+        "description": "Wrapper for optional contents of the replicated network config.",
+        "type": "object",
+        "properties": {
+          "contents": {
+            "nullable": true,
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/ReplicatedNetworkConfigContents"
+              }
+            ]
+          }
+        }
+      },
+      "ReplicatedNetworkConfigContents": {
+        "description": "Contents of the replicated network config.\n\nAnalogous to the bootstore `NetworkConfig` type.",
+        "type": "object",
+        "properties": {
+          "base64_blob": {
+            "description": "The `NetworkConfig` arbitrary blob, encoded using the standard base64 alphabet.\n\nCurrently, the contents are expected to be JSON, but serialization/deserialization of the contents is performed outside the replication engine, which just deals with a binary blob.",
+            "type": "string"
+          },
+          "generation": {
+            "type": "integer",
+            "format": "uint64",
+            "minimum": 0
+          }
+        },
+        "required": [
+          "base64_blob",
+          "generation"
+        ]
+      },
       "RouteConfig": {
         "type": "object",
         "properties": {