feat: store credentials in Postgres with envelope encryption

Carbide can now keep its credentials in Postgres instead of Vault,
encrypted so that a copy of the database on its own gives up nothing.
This swaps out Vault KV as the credential store; Vault still issues the
PKI certificates.

The encryption is layered. Every credential gets its own data key, and
that data key is wrapped by a key encryption key (KEK) that lives
outside the database -- in carbide's own config or an external KMS. The
credential's path is mixed into the encryption as well, so a row lifted
out and dropped onto another path will not decrypt. The KEK is the only
thing an operator has to protect; everything else lives in Postgres.

`PostgresCredentialManager` implements the same `CredentialManager`
traits the rest of the system already uses, so nothing downstream had to
change. It keeps the two reader behaviors callers learned from Vault:
the newest write for a path wins, and an empty-password entry reads as
"no credential" (several delete paths still record that tombstone). The
store is an append-only journal -- one row per write, newest-per-path on
read -- which is what the upcoming rotation work uses for history and
rollback.

Where the wrapping keys come from is pluggable through `KmsBackend`:
local key material (`integrated`), or Vault/OpenBao Transit, and more
than one provider at once. `[secrets.routing]` maps path prefixes to the
KEK that encrypts new writes under them. Startup cross-checks the routing
against the providers, so a misspelled, duplicated, or colliding key
fails the boot instead of some write hours later.

Moving an existing site off Vault is a one-time import at startup, and it
is deliberately all-or-nothing: any list or read failure, or an empty
listing, aborts the boot rather than recording a half-finished import as
done, and only one replica runs it at a time behind a Postgres advisory
lock. Once it completes, Vault is out of the credential chain entirely --
with `[secrets]` set, the chain is env -> file -> postgres and nothing
falls back to Vault. Prerequisites that live outside this process
(services that still read Vault directly, mixed-fleet rolling upgrades)
are spelled out on `SecretsConfig`.

Rotating a KEK is a config change plus one command: point the route at
the new key and run `carbide-admin-cli secrets re-wrap`. Only the
per-row data-key wrapping is redone; the encrypted values are never
touched. The re-wrap makes its KMS calls outside the write transaction,
runs one at a time, and reports how many rows still sit on a retired key
so you know when the old KEK is safe to remove.

Tests cover the manager round-trip, journal write-order under equal
timestamps, rotation rollback by deleting the newest entry, tombstone
reads, create conflicts, re-wrap idempotence and counting, and a
path-binding regression that confirms a transplanted row will not
decrypt -- plus an end-to-end Vault import against a real Vault dev
server.

This supports #353.

Signed-off-by: Chet Nichols III <chetn@nvidia.com>

Author: chet

Cargo.lock

@@ -81,3 +81,4 @@ admin:
   - dev-env
   - ssh
   - jump
+  - secrets

crates/admin-cli/cli_domains.yaml

@@ -25,9 +25,9 @@ use crate::{
     ipxe_template, jump, machine, machine_interfaces, machine_validation, managed_host,
     managed_switch, mlx, network_devices, network_security_group, network_segment, nvl_domain,
     nvl_logical_partition, nvl_partition, nvlink_nmxc_endpoints, operating_system, os_image, ping,
-    power_shelf, rack, redfish, resource_pool, rms, route_server, scout_stream, set, site_explorer,
-    sku, spx_partition, ssh, switch, tenant, tenant_keyset, tpm_ca, trim_table, version, vpc,
-    vpc_peering, vpc_prefix,
+    power_shelf, rack, redfish, resource_pool, rms, route_server, scout_stream, secrets, set,
+    site_explorer, sku, spx_partition, ssh, switch, tenant, tenant_keyset, tpm_ca, trim_table,
+    version, vpc, vpc_peering, vpc_prefix,
 };
 
 #[derive(Parser, Debug)]
@@ -202,6 +202,8 @@ pub enum CliCommand {
     ExtensionService(extension_service::Cmd),
     #[clap(about = "Firmware related actions", subcommand)]
     Firmware(firmware::Cmd),
+    #[clap(about = "Secrets management", subcommand)]
+    Secrets(secrets::Cmd),
     #[clap(
         about = "Regenerate the docs/manuals/nico-admin-cli markdown reference",
         hide = true

crates/admin-cli/src/cfg/cli_options.rs

@@ -103,6 +103,7 @@ mod rms;
 mod route_server;
 mod rpc;
 mod scout_stream;
+mod secrets;
 mod set;
 mod site_explorer;
 mod sku;
@@ -274,6 +275,7 @@ async fn main() -> color_eyre::Result<()> {
         CliCommand::ResourcePool(cmd) => cmd.dispatch(ctx).await?,
         CliCommand::RouteServer(cmd) => cmd.dispatch(ctx).await?,
         CliCommand::ScoutStream(cmd) => cmd.dispatch(ctx).await?,
+        CliCommand::Secrets(cmd) => cmd.dispatch(ctx).await?,
         CliCommand::Set(cmd) => cmd.dispatch(ctx).await?,
         CliCommand::Ssh(cmd) => cmd.dispatch(ctx).await?,
         CliCommand::SiteExplorer(cmd) => cmd.dispatch(ctx).await?,

crates/admin-cli/src/main.rs

@@ -0,0 +1,31 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+mod re_wrap;
+
+use clap::Parser;
+
+use crate::cfg::dispatch::Dispatch;
+
+#[derive(Parser, Debug, Clone, Dispatch)]
+#[clap(rename_all = "kebab_case")]
+pub enum Cmd {
+    #[clap(about = "Re-wrap secret DEKs to use the \
+                 currently active KEK per routing \
+                 config")]
+    ReWrap(re_wrap::Args),
+}

crates/admin-cli/src/secrets/mod.rs

@@ -0,0 +1,38 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+use clap::Parser;
+
+#[derive(Parser, Debug, Clone)]
+#[command(after_long_help = "\
+EXAMPLES:
+
+Re-wrap every credential whose KEK no longer matches the routing config
+(run this after rotating a key in [secrets.routing]):
+    $ nico-admin-cli secrets re-wrap
+
+Use a smaller batch size to lighten load on an external KMS:
+    $ nico-admin-cli secrets re-wrap --batch-size 25
+
+")]
+pub struct Args {
+    #[clap(
+        long,
+        help = "Rows scanned per batch during the walk. The server applies its own default and limits."
+    )]
+    pub batch_size: Option<u32>,
+}

crates/admin-cli/src/secrets/re_wrap/args.rs

@@ -0,0 +1,42 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+use crate::errors::CarbideCliResult;
+use crate::rpc::ApiClient;
+
+pub async fn re_wrap(api_client: &ApiClient, batch_size: Option<u32>) -> CarbideCliResult<()> {
+    let request = ::rpc::forge::ReWrapSecretsRequest { batch_size };
+
+    let resp = api_client.0.re_wrap_secrets(request).await?;
+
+    println!(
+        "Re-wrap complete: {} re-wrapped, {} already current",
+        resp.re_wrapped, resp.already_current
+    );
+    if resp.stale_remaining == 0 {
+        println!(
+            "No rows remain on KEKs outside the routing config; unrouted KEKs can be retired."
+        );
+    } else {
+        println!(
+            "{} rows are still wrapped by KEKs outside the routing config -- \
+             concurrent writers likely landed rows mid-walk; run re-wrap again.",
+            resp.stale_remaining
+        );
+    }
+    Ok(())
+}

crates/admin-cli/src/secrets/re_wrap/cmd.rs

@@ -0,0 +1,31 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+pub mod args;
+mod cmd;
+
+pub use args::Args;
+
+use crate::cfg::run::Run;
+use crate::cfg::runtime::RuntimeContext;
+use crate::errors::CarbideCliResult;
+
+impl Run for Args {
+    async fn run(self, ctx: &mut RuntimeContext) -> CarbideCliResult<()> {
+        cmd::re_wrap(&ctx.api_client, self.batch_size).await
+    }
+}

crates/admin-cli/src/secrets/re_wrap/mod.rs

@@ -42,6 +42,7 @@ carbide-ib-fabric = { path = "../ib-fabric" }
 carbide-ib-partition-controller = { path = "../ib-partition-controller" }
 carbide-ipmi = { path = "../ipmi" }
 carbide-ipxe-renderer = { path = "../ipxe-renderer" }
+carbide-kms-provider = { path = "../kms-provider" }
 carbide-libmlx = { path = "../libmlx" }
 carbide-machine-controller = { path = "../machine-controller" }
 carbide-measured-boot = { path = "../measured-boot", features = ["sqlx"] }
@@ -180,7 +181,9 @@ tracing-subscriber = { workspace = true, features = [
 tss-esapi = { workspace = true, optional = true }
 url = { workspace = true, features = ["serde"] }
 uuid = { workspace = true, features = ["v4", "serde"] }
+vaultrs = { workspace = true }
 x509-parser = { workspace = true, features = ["verify"] }
+zeroize = { workspace = true }
 
 [features]
 default = ["linux-build"]

crates/api-core/Cargo.toml

@@ -88,6 +88,7 @@ pub struct Api {
     pub(crate) metric_emitter: ApiMetricsEmitter,
     pub(crate) component_manager: Option<component_manager::component_manager::ComponentManager>,
     pub(crate) bms_client: OnceLock<Arc<BmsDsxExchangeHandle>>,
+    pub(crate) secrets_context: Option<crate::secrets::SecretsContext>,
 }
 
 pub(crate) type ScoutStreamType =
@@ -1444,6 +1445,13 @@ impl Forge for Api {
         crate::handlers::credential::delete_credential(self, request).await
     }
 
+    async fn re_wrap_secrets(
+        &self,
+        request: Request<rpc::ReWrapSecretsRequest>,
+    ) -> Result<Response<rpc::ReWrapSecretsResponse>, Status> {
+        crate::handlers::secrets::re_wrap_secrets(self, request).await
+    }
+
     /// get_route_servers returns a list of all configured route server
     /// entries for all source types.
     async fn get_route_servers(