Bump trustee to v0.20.0 & Synchronize Trustee Resources via KBS API

All trustee resources, including the resource policy, attestation policy, RV, LUKS key, and AK, are now synchronized using the KBS API.
The main motivation for this is replacing the patch mechanism that causes a restart of the trustee deployment.
Also, this abstraction over the trustee backend simplifies the process of upgrading to future versions of trustee.

 Config:
  - Update kbs-config.toml to v0.20.0 format
  - Store reference values in dedicated ConfigMap (trustee-rv-data)
    instead of trustee-data

API-driven resource management:
  - Add KBS API client functions: send_secret, delete_secret,
    register_ak, sync_resource_policy, sync_attestation_policy,
    sync_reference_values
  - Replace volume-mount approach for secrets with KBS API calls
  - Add trustee deployment reconciler that syncs policies, reference
    values, and secrets when deployment becomes available

Dependencies:
  - Add kbs-client and jsonwebtoken crates
  - Bump trustee image tag to v0.20.0

Signed-off-by: Roy Kaufman <rkaufman@redhat.com>

Author: iroykaufman

Cargo.lock

@@ -28,8 +28,10 @@ OPERATOR_IMAGE ?= $(REGISTRY)/trusted-cluster-operator:$(TAG)
 COMPUTE_PCRS_IMAGE=$(REGISTRY)/compute-pcrs:$(TAG)
 REG_SERVER_IMAGE=$(REGISTRY)/registration-server:$(TAG)
 ATTESTATION_KEY_REGISTER_IMAGE=$(REGISTRY)/attestation-key-register:$(TAG)
-TRUSTEE_IMAGE ?= quay.io/trusted-execution-clusters/key-broker-service:v0.17.0
+
+TRUSTEE_IMAGE ?= quay.io/trusted-execution-clusters/key-broker-service:v0.20.0
 TEST_IMAGE ?= quay.io/trusted-execution-clusters/fedora-coreos-kubevirt:42.20260622
+
 # tagged as 42.20251012.2.0
 APPROVED_IMAGE ?= quay.io/trusted-execution-clusters/fedora-coreos@sha256:6997f51fd27d1be1b5fc2e6cc3ebf16c17eb94d819b5d44ea8d6cf5f826ee773
 

Makefile

@@ -31,6 +31,9 @@ serde_json.workspace = true
 thiserror = "2.0.18"
 tokio.workspace = true
 toml = "1.1.2"
+kbs-client = {git = "https://github.com/confidential-containers/trustee.git", rev = "e65897a9ad4eb3ac69fa2ec75ed831200eb2acd7", default-features = false, features = ["native-tls"] }
+jsonwebtoken = { version = "10.4.0", default-features = false, features = ["use_pem"] }
+jsonwebtoken-openssl = "1.0.0"
 
 [dev-dependencies]
 http.workspace = true

operator/Cargo.toml

@@ -331,7 +331,7 @@ async fn secret_reconcile(
         match ev {
             Event::Apply(_secret) => {
                 // On creation/update, just update the trustee deployment volumes
-                trustee::update_attestation_keys(&ctx)
+                trustee::update_attestation_keys(ctx.client.clone())
                     .await
                     .map(|_| Action::await_change())
                     .map_err(|e| {
@@ -345,7 +345,7 @@ async fn secret_reconcile(
                     "AttestationKey secret {secret_name} is being deleted, updating trustee deployment volumes"
                 );
                 // Update trustee deployment - secrets with deletion_timestamp will be filtered out
-                trustee::update_attestation_keys(&ctx)
+                trustee::update_attestation_keys(ctx.client.clone())
                     .await
                     .map(|_| Action::await_change())
                     .map_err(|e| {

operator/src/attestation_key_register.rs

@@ -2,36 +2,35 @@
 sockets = ["0.0.0.0:8080"]
 
 [admin]
-type = "DenyAll"
+authorization_mode = "AuthenticatedAuthorization"
+
+  [admin.authentication.bearer_jwt]
+  identity_providers = [
+    { public_key_uri = "/opt/trustee/keys/public.pub" }
+  ]
+
+  [admin.authorization.regex_acl]
+  acls = [{ role = "admin", allowed_endpoints = "^/kbs/.+$" }]
 
 [attestation_token]
-insecure_key = true
-attestation_token_type = "CoCo"
+insecure_header_jwk = true
 
 [attestation_service]
 type = "coco_as_builtin"
-work_dir = "/opt/trustee"
-policy_engine = "opa"
+timeout = 5
 
   [attestation_service.attestation_token_broker]
-  type = "Ear"
-  policy_dir = "/opt/trustee/policies"
-
-  [attestation_service.attestation_token_config]
   duration_min = 5
 
   [attestation_service.rvps_config]
   type = "BuiltIn"
 
-    [attestation_service.rvps_config.storage]
-    type = "LocalJson"
-    file_path = "/opt/trustee/reference-values.json"
-
-
 [[plugins]]
 name = "resource"
-type = "LocalFs"
-dir_path = "/opt/trustee/kbs-repository"
+storage_backend_type = "kvstorage"
+
+[storage_backend]
+storage_type = "LocalJson"
 
-[policy_engine]
-policy_path = "/opt/trustee/policy.rego"
+  [storage_backend.backends.local_json]
+  file_dir_path = "/opt/trustee/storage"

operator/src/kbs-config.toml

@@ -28,12 +28,11 @@ mod register_server;
 #[cfg(test)]
 mod test_utils;
 mod trustee;
-
 use crate::conditions::*;
 use operator::*;
 
 /// Default tag for Trustee image
-const TRUSTEE_VERSION: &str = "v0.17.0";
+const TRUSTEE_VERSION: &str = "v0.20.0";
 /// Default version tag for operator-managed component images
 const COMPONENT_VERSION: &str = "v0.2.0";
 /// Default registry
@@ -132,7 +131,6 @@ async fn reconcile(
         update_status!(clusters, name, status)?;
     }
 
-
     if let Err(e) = install_components(&kube_client, &cluster).await {
         // warn with `:?` to also get context
         warn!("Installation of a component failed: {e:?}\nRequeueing...");
@@ -167,17 +165,18 @@ async fn install_trustee_configuration(
         .context("Failed to create the KBS configuration configmap")?;
     info!("Generated configmap for the KBS configuration");
 
-    trustee::generate_attestation_policy(client.clone(), owner_reference.clone())
+    reference_values::create_pcrs_config_map(client.clone())
         .await
-        .context("Failed to create the attestation policy configmap")?;
-    info!("Generated configmap for the attestation policy");
-
-    match trustee::generate_trustee_auth_keys_secret(client.clone(), owner_reference.clone()).await
-    {
-        Ok(_) => info!("Generate auth keys for the KBS API",),
-        Err(e) => error!("Failed to create the auth keys: {e}"),
-    }
-
+        .context("Failed to create the PCRs configmap")?;
+    info!("Created bare configmap for PCRs");
+    trustee::generate_trustee_auth_keys_secret(client.clone(), owner_reference.clone())
+        .await
+        .context("Failed to create the auth keys")?;
+    info!("Generate auth keys for the KBS API");
+    trustee::generate_rv_data(client.clone(), owner_reference.clone())
+        .await
+        .context("Failed to create the reference values configmap")?;
+    info!("Created configmap for reference values");
     let kbs_port = cluster.spec.trustee_kbs_port;
     trustee::generate_kbs_service(client.clone(), owner_reference.clone(), kbs_port)
         .await
@@ -253,7 +252,7 @@ async fn install_attestation_key_register(
 #[tokio::main]
 async fn main() -> Result<()> {
     env_logger::Builder::from_env(Env::default().default_filter_or("info")).init();
-
+    let _ = jsonwebtoken_openssl::install_default();
     let kube_client = Client::try_default().await?;
     info!("trusted execution clusters operator",);
 
@@ -284,9 +283,9 @@ async fn main() -> Result<()> {
     attestation_key_register::launch_ak_controller(ak_ctx.clone()).await;
     attestation_key_register::launch_machine_ak_controller(ak_ctx.clone()).await;
     attestation_key_register::launch_secret_ak_controller(ak_ctx).await;
-    reference_values::create_pcrs_config_map(kube_client.clone()).await?;
     reference_values::launch_rv_image_controller(kube_client.clone()).await;
     reference_values::launch_rv_job_controller(kube_client.clone()).await;
+    trustee::launch_trustee_sync_controller(kube_client.clone()).await;
 
     Controller::new(cl, watcher::Config::default())
         .run(reconcile, controller_error_policy, ctx)
@@ -300,7 +299,7 @@ async fn main() -> Result<()> {
 mod tests {
     use http::{Method, Request, StatusCode};
     use k8s_openapi::api::apps::v1::Deployment;
-    use k8s_openapi::api::core::v1::{ConfigMap, Service};
+    use k8s_openapi::api::core::v1::{ConfigMap, Secret, Service};
     use k8s_openapi::{apimachinery::pkg::apis::meta::v1::Time, jiff::Timestamp};
     use kube::client::Body;
 
@@ -440,25 +439,26 @@ mod tests {
         };
 
         let clos = async |req: Request<Body>, ctr| {
-            if ctr < 8 && req.method() == Method::POST {
+            if ctr < 10 && req.method() == Method::POST {
                 use serde_json::to_string;
                 let resp = match ctr {
-                    // Trustee
-                    0 => to_string(&ConfigMap::default()),
-                    1 => to_string(&ConfigMap::default()),
-                    2 => to_string(&Service::default()),
-                    3 => to_string(&Deployment::default()),
-                    // Registration server
-                    4 => to_string(&Deployment::default()),
-                    5 => to_string(&Service::default()),
-                    // Attestation key register server
-                    6 => to_string(&Deployment::default()),
-                    7 => to_string(&Service::default()),
+                    // install_trustee_configuration
+                    0 => to_string(&ConfigMap::default()), // trustee-data
+                    1 => to_string(&ConfigMap::default()), // image-pcrs
+                    2 => to_string(&Secret::default()),    // trustee-auth
+                    3 => to_string(&ConfigMap::default()), // trustee-rv-data
+                    4 => to_string(&Service::default()),   // kbs-service
+                    5 => to_string(&Deployment::default()), // trustee-deployment
+                    // install_register_server
+                    6 => to_string(&Deployment::default()), // register-server
+                    7 => to_string(&Service::default()),    // register-server-svc
+                    // install_attestation_key_register
+                    8 => to_string(&Deployment::default()), // ak-register
+                    9 => to_string(&Service::default()),    // ak-register-svc
                     _ => unreachable!("unexpected counter {ctr}"),
                 };
                 Ok(resp.unwrap())
-
-            } else if ctr == 8 && req.method() == Method::PATCH {
+            } else if ctr == 10 && req.method() == Method::PATCH {
                 let body = req.into_body().collect_bytes().await.unwrap().to_vec();
                 let body = String::from_utf8_lossy(&body);
                 assert!(body.contains("ForeignCondition"),);
@@ -489,7 +489,7 @@ mod tests {
         cluster.status = Some(TrustedExecutionClusterStatus {
             conditions: Some(vec![pre_existing_installed, foreign_condition]),
         });
-        count_check!(9, clos, |client| {
+        count_check!(11, clos, |client| {
             let result = reconcile(Arc::new(cluster), Arc::new(dummy_cluster_ctx(client))).await;
             assert_eq!(result.unwrap(), Action::await_change());
         });

operator/src/main.rs

@@ -220,7 +220,6 @@ async fn adopt_approved_image(
     Ok(())
 }
 
-
 async fn image_reconcile(
     image: Arc<ApprovedImage>,
     client: Arc<Client>,
@@ -479,12 +478,13 @@ mod tests {
                 Ok(serde_json::to_string(&dummy_pcrs_map()).unwrap())
             }
             (2, &Method::GET) | (3, &Method::PUT) => {
-                assert!(req.uri().path().contains(trustee::TRUSTEE_DATA_MAP));
+                assert!(req.uri().path().contains(trustee::TRUSTEE_RV_MAP));
                 Ok(serde_json::to_string(&dummy_trustee_map()).unwrap())
             }
+            (4, &Method::GET) => Err(StatusCode::NOT_FOUND),
             _ => panic!("unexpected API interaction: {req:?}, counter {ctr}"),
         };
-        count_check!(4, clos, |client| {
+        count_check!(5, clos, |client| {
             let job = Arc::new(dummy_job());
             let result = job_reconcile(job, Arc::new(client)).await.unwrap();
             assert_eq!(result, Action::await_change());
@@ -551,7 +551,6 @@ mod tests {
         test_error_method!(clos, Method::PATCH);
     }
 
-
     // handle_new_image and its caller image_add_reconcile are
     // inherently online functions and not tested here
 
@@ -566,12 +565,13 @@ mod tests {
                 Ok(serde_json::to_string(&dummy_pcrs_map()).unwrap())
             }
             (3, &Method::GET) | (4, &Method::PUT) => {
-                assert!(req.uri().path().contains(trustee::TRUSTEE_DATA_MAP));
+                assert!(req.uri().path().contains(trustee::TRUSTEE_RV_MAP));
                 Ok(serde_json::to_string(&dummy_trustee_map()).unwrap())
             }
+            (5, &Method::GET) => Err(StatusCode::NOT_FOUND),
             _ => panic!("unexpected API interaction: {req:?}, counter {ctr}"),
         };
-        count_check!(5, clos, |client| {
+        count_check!(6, clos, |client| {
             assert!(image_remove_reconcile(client, image, cluster).await.is_ok());
         });
     }

operator/src/reference_values.rs

@@ -139,7 +139,7 @@ async fn keygen_reconcile(
                 async {
                     let owner_reference = generate_owner_reference(&Arc::unwrap_or_clone(machine))?;
                     trustee::generate_secret(kube_client.clone(), id, owner_reference).await?;
-                    trustee::mount_secret(kube_client, id).await
+                    trustee::send_secret(kube_client, id).await
                 }
                 .await
                 .map(|_| Action::await_change())
@@ -185,10 +185,12 @@ async fn keygen_reconcile(
                     }
                 }
 
-                trustee::unmount_secret(kube_client, id)
-                    .await
-                    .map(|_| Action::await_change())
-                    .map_err(|e| finalizer::Error::<ControllerError>::CleanupFailed(e.into()))
+                trustee::delete_secret(kube_client, id).await.map_err(|e| {
+                    finalizer::Error::<ControllerError>::CleanupFailed(
+                        anyhow!("failed to delete secret for machine {id}: {e}").into(),
+                    )
+                })?;
+                Ok(Action::await_change())
             }
         }
     })

operator/src/register_server.rs

@@ -3,7 +3,10 @@
 // SPDX-License-Identifier: MIT
 
 use compute_pcrs_lib::Pcr;
-use k8s_openapi::{api::core::v1::ConfigMap, jiff::Timestamp};
+use k8s_openapi::{
+    api::core::v1::{ConfigMap, Secret},
+    jiff::Timestamp,
+};
 use std::collections::BTreeMap;
 
 use crate::trustee;
@@ -41,6 +44,26 @@ pub fn dummy_trustee_map() -> ConfigMap {
     }
 }
 
+pub fn dummy_trustee_auth() -> Secret {
+    let key_pair =
+        trustee::generate_ed25519_key_pair().expect("Failed to generate ed25519 key pair");
+    let data = BTreeMap::from([
+        (
+            trustee::TRUSTEE_AUTH_PRIV_KEY.to_string(),
+            k8s_openapi::ByteString(key_pair.private_key_pem),
+        ),
+        (
+            trustee::TRUSTEE_AUTH_PUB_KEY.to_string(),
+            k8s_openapi::ByteString(key_pair.public_key_pem),
+        ),
+    ]);
+
+    Secret {
+        data: Some(data),
+        ..Default::default()
+    }
+}
+
 pub fn dummy_pcrs_map() -> ConfigMap {
     let data = BTreeMap::from([(
         PCR_CONFIG_FILE.to_string(),

operator/src/test_utils.rs

@@ -9,6 +9,7 @@ executables := 3 if {
   input.tpm.pcr04 in query_reference_value("tpm_pcr4")
   input.tpm.pcr14 in query_reference_value("tpm_pcr14")
 
+  input.tpm.ak_public in query_reference_value("trusted_aks")
 }
 # Azure SNP vTPM validation
 executables := 3 if {