Implement TqSecretRetriever for Trust Quorum secret retrieval (#9649)

- Implements `TqSecretRetriever` analogous to `LrtqSecretRetriever`, but
using Trust Quorum's `NodeTaskHandle` instead of bootstore's
`NodeHandle`.
- Builds atop this to implement `TqOrLrtqSecretRetriever` which
dynamically switches between them based on rack state.
- Switches the `SecretRetriever` trait to use `&mut self`, which
eliminates the need for interior mutability, but which complicates the
previous use of a `OnceLock` to do late-binding of the secret retriever
(needed because of order of initialization during boot).
- Eliminates that global `OnceLock` and instead threads the late-binding
of secret retriever through long running task handles to achieve the
same effect.

This closes #9586.

Author: plaidfinch

Cargo.lock

@@ -18,4 +18,3 @@ thiserror.workspace = true
 tokio.workspace = true
 zeroize.workspace = true
 omicron-workspace-hack.workspace = true
-

key-manager/Cargo.toml

@@ -306,18 +306,23 @@ pub enum SecretRetrieverError {
 
     #[error("Bootstore error: {0}")]
     Bootstore(String),
+
+    #[error("Trust quorum error: {0}")]
+    TrustQuorum(String),
 }
 
 /// A mechanism for retrieving a secrets to use as input key material to HKDF-
 /// Extract.
 #[async_trait]
-pub trait SecretRetriever {
+pub trait SecretRetriever: Send + Sync + 'static {
     /// Return the latest secret
     ////
     /// This is useful when a new entity is being encrypted and there is no need
     /// for a reconfiguration. When an entity is already encrypted, and needs to
     /// be decrypted, the user should instead call the [`SecretRetriever::get`].
-    async fn get_latest(&self) -> Result<VersionedIkm, SecretRetrieverError>;
+    async fn get_latest(
+        &mut self,
+    ) -> Result<VersionedIkm, SecretRetrieverError>;
 
     /// Get the secret for the given epoch
     ///
@@ -331,7 +336,7 @@ pub trait SecretRetriever {
     /// Return an error if its not possible to recover the old secret given the
     /// latest secret.
     async fn get(
-        &self,
+        &mut self,
         epoch: u64,
     ) -> Result<SecretState, SecretRetrieverError>;
 }
@@ -363,15 +368,15 @@ mod tests {
     #[async_trait]
     impl SecretRetriever for TestSecretRetriever {
         async fn get_latest(
-            &self,
+            &mut self,
         ) -> Result<VersionedIkm, SecretRetrieverError> {
             let salt = [0u8; 32];
             let (epoch, bytes) = self.ikms.last_key_value().unwrap();
             Ok(VersionedIkm::new(*epoch, salt, bytes))
         }
 
         async fn get(
-            &self,
+            &mut self,
             epoch: u64,
         ) -> Result<SecretState, SecretRetrieverError> {
             let salt = [0u8; 32];

key-manager/src/lib.rs

@@ -97,6 +97,7 @@ sled-hardware.workspace = true
 sled-hardware-types.workspace = true
 sled-storage.workspace = true
 sp-sim.workspace = true
+secrecy.workspace = true
 slog.workspace = true
 slog-async.workspace = true
 slog-dtrace.workspace = true

sled-agent/Cargo.toml

@@ -2356,7 +2356,7 @@ mod illumos_tests {
     #[async_trait::async_trait]
     impl SecretRetriever for HardcodedSecretRetriever {
         async fn get_latest(
-            &self,
+            &mut self,
         ) -> Result<key_manager::VersionedIkm, SecretRetrieverError> {
             let epoch = 0;
             let salt = [0u8; 32];
@@ -2366,7 +2366,7 @@ mod illumos_tests {
         }
 
         async fn get(
-            &self,
+            &mut self,
             epoch: u64,
         ) -> Result<key_manager::SecretState, SecretRetrieverError> {
             if epoch != 0 {

sled-agent/config-reconciler/src/dataset_serialization_task.rs

@@ -0,0 +1,97 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! A secret retriever that waits to be configured before use.
+//!
+//! Created early in boot (before we know which retriever type to use),
+//! configured later once we have the sled agent request.
+
+use async_trait::async_trait;
+use key_manager::{
+    SecretRetriever, SecretRetrieverError, SecretState, VersionedIkm,
+};
+use std::sync::{Arc, Mutex};
+use tokio::sync::oneshot;
+
+/// A secret retriever that waits to be configured before use.
+///
+/// Created early in boot (before we know which retriever type to use),
+/// configured later once we have the sled agent request.
+pub struct ConfigurableSecretRetriever {
+    inner: Option<Box<dyn SecretRetriever>>,
+    config_rx: Option<oneshot::Receiver<Box<dyn SecretRetriever>>>,
+}
+
+/// Handle to configure a [`ConfigurableSecretRetriever`].
+///
+/// Cloneable, but `init` can only succeed once (panics on second call).
+#[derive(Clone)]
+pub struct ConfigurableSecretRetrieverHandle {
+    #[allow(clippy::type_complexity)]
+    tx: Arc<Mutex<Option<oneshot::Sender<Box<dyn SecretRetriever>>>>>,
+}
+
+impl ConfigurableSecretRetriever {
+    pub fn new() -> (Self, ConfigurableSecretRetrieverHandle) {
+        let (tx, rx) = oneshot::channel();
+        (
+            Self { inner: None, config_rx: Some(rx) },
+            ConfigurableSecretRetrieverHandle {
+                tx: Arc::new(Mutex::new(Some(tx))),
+            },
+        )
+    }
+}
+
+impl ConfigurableSecretRetrieverHandle {
+    /// Configure the pending retriever with the actual implementation.
+    ///
+    /// Panics if called twice or if the corresponding
+    /// [`ConfigurableSecretRetriever`] was dropped.
+    pub fn init(&self, retriever: impl SecretRetriever) {
+        self.tx
+            .lock()
+            .unwrap()
+            .take()
+            .expect("PendingSecretRetriever already configured")
+            .send(Box::new(retriever))
+            .unwrap_or_else(|_| {
+                panic!("PendingSecretRetriever dropped before configure")
+            });
+    }
+}
+
+#[async_trait]
+impl SecretRetriever for ConfigurableSecretRetriever {
+    async fn get_latest(
+        &mut self,
+    ) -> Result<VersionedIkm, SecretRetrieverError> {
+        self.ensure_configured().await?;
+        self.inner.as_mut().unwrap().get_latest().await
+    }
+
+    async fn get(
+        &mut self,
+        epoch: u64,
+    ) -> Result<SecretState, SecretRetrieverError> {
+        self.ensure_configured().await?;
+        self.inner.as_mut().unwrap().get(epoch).await
+    }
+}
+
+impl ConfigurableSecretRetriever {
+    async fn ensure_configured(&mut self) -> Result<(), SecretRetrieverError> {
+        if self.inner.is_none() {
+            let rx = self
+                .config_rx
+                .take()
+                .ok_or(SecretRetrieverError::RackNotInitialized)?;
+            self.inner = Some(
+                rx.await
+                    .map_err(|_| SecretRetrieverError::RackNotInitialized)?,
+            );
+        }
+        Ok(())
+    }
+}