From 2f61d4c9e8ba605e100be234e4bbc42e24071c51 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Thu, 9 Jul 2026 12:38:23 +0200 Subject: [PATCH 1/4] rvs: Reconcile images on jobs too so that status is updated in a timely manner Signed-off-by: Jakob Naucke --- operator/src/reference_values.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/operator/src/reference_values.rs b/operator/src/reference_values.rs index f202c533..f065d8ff 100644 --- a/operator/src/reference_values.rs +++ b/operator/src/reference_values.rs @@ -337,8 +337,11 @@ async fn image_remove_reconcile( pub async fn launch_rv_image_controller(client: Client) { let images: Api = Api::default_namespaced(client.clone()); + let jobs: Api = Api::default_namespaced(client.clone()); + let wc = watcher::Config::default().labels(&format!("{JOB_LABEL_KEY}={PCR_COMMAND_NAME}")); tokio::spawn( Controller::new(images, Default::default()) + .owns(jobs, wc) .run(image_reconcile, controller_error_policy, Arc::new(client)) .for_each(controller_info), ); From 9edc00413810aaa6bfb9ec32dd117556d4842be9 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Fri, 26 Jun 2026 16:03:27 +0200 Subject: [PATCH 2/4] Remove await_change As per kube-rs docs, await_change is unadvisable for cases where eventual consistency is desired because reconcile events can be lost occasionally. We saw this happen in tests where owned resources with finalizers stayed behind after TEC deletion. Instead, requeue after 1 minute (ongoing deletion cases) or 5 minutes (resources applied cases). This is shorter than kube-rs's "sane default" of 60 minutes, but aligns better with testing, user expectations, and is also what KubeVirt appears to be doing. Signed-off-by: Jakob Naucke --- operator/src/attestation_key_register.rs | 14 +++++++------- operator/src/main.rs | 12 ++++++------ operator/src/reference_values.rs | 8 ++++---- operator/src/register_server.rs | 10 +++++----- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/operator/src/attestation_key_register.rs b/operator/src/attestation_key_register.rs index 332e5124..24945010 100644 --- a/operator/src/attestation_key_register.rs +++ b/operator/src/attestation_key_register.rs @@ -195,10 +195,10 @@ async fn ak_reconcile( for machine in ctx.machine_store.state() { if ak.spec.uuid.as_ref() == Some(&machine.spec.id) { approve_ak(&ak, &machine, &ctx).await?; - return Ok(Action::await_change()); + return Ok(Action::requeue(Duration::from_secs(300))); } } - Ok(Action::await_change()) + Ok(Action::requeue(Duration::from_secs(300))) } async fn machine_reconcile( @@ -216,7 +216,7 @@ async fn machine_reconcile( "Machine {} is being deleted, updating attestation key volumes", machine.metadata.name.clone().unwrap_or_default() ); - return Ok(Action::await_change()); + return Ok(Action::requeue(Duration::from_secs(60))); } for ak in ctx.ak_store.state() { @@ -224,10 +224,10 @@ async fn machine_reconcile( && *ak_uuid == machine.spec.id { approve_ak(&ak, &machine, &ctx).await?; - return Ok(Action::await_change()); + return Ok(Action::requeue(Duration::from_secs(300))); } } - Ok(Action::await_change()) + Ok(Action::requeue(Duration::from_secs(300))) } async fn approve_ak(ak: &AttestationKey, machine: &Machine, ctx: &AkContextData) -> Result<()> { @@ -333,7 +333,7 @@ async fn secret_reconcile( // On creation/update, just update the trustee deployment volumes trustee::update_attestation_keys(&ctx) .await - .map(|_| Action::await_change()) + .map(|_| Action::requeue(Duration::from_secs(300))) .map_err(|e| { eprintln!("Error updating attestation key volumes on secret apply: {e}"); finalizer::Error::::ApplyFailed(e.into()) @@ -347,7 +347,7 @@ async fn secret_reconcile( // Update trustee deployment - secrets with deletion_timestamp will be filtered out trustee::update_attestation_keys(&ctx) .await - .map(|_| Action::await_change()) + .map(|_| Action::requeue(Duration::from_secs(300))) .map_err(|e| { eprintln!( "Error updating attestation key volumes during secret deletion: {e}" diff --git a/operator/src/main.rs b/operator/src/main.rs index 03ff138e..3e06b5c7 100644 --- a/operator/src/main.rs +++ b/operator/src/main.rs @@ -99,11 +99,11 @@ async fn reconcile( if changed { update_status!(clusters, name, TrustedExecutionClusterStatus { conditions })?; } - return Ok(Action::await_change()); + return Ok(Action::requeue(Duration::from_secs(60))); } if is_installed(cluster.status.clone()) { - return Ok(Action::await_change()); + return Ok(Action::requeue(Duration::from_secs(300))); } if ctx.tec_store.state().len() > 1 { @@ -145,7 +145,7 @@ async fn reconcile( let status = TrustedExecutionClusterStatus { conditions }; update_status!(clusters, name, status)?; } - Ok(Action::await_change()) + Ok(Action::requeue(Duration::from_secs(300))) } async fn install_components(client: &Client, cluster: &TrustedExecutionCluster) -> Result<()> { @@ -336,7 +336,7 @@ mod tests { let mut cluster = dummy_cluster(); cluster.metadata.deletion_timestamp = Some(Time(Timestamp::now())); let result = reconcile(Arc::new(cluster), Arc::new(dummy_cluster_ctx(client))).await; - assert_eq!(result.unwrap(), Action::await_change()); + assert_eq!(result.unwrap(), Action::requeue(Duration::from_secs(60))); }); } @@ -416,7 +416,7 @@ mod tests { conditions: Some(vec![foreign_condition]), }); let result = reconcile(Arc::new(cluster), Arc::new(dummy_cluster_ctx(client))).await; - assert_eq!(result.unwrap(), Action::await_change()); + assert_eq!(result.unwrap(), Action::requeue(Duration::from_secs(60))); }); } @@ -493,7 +493,7 @@ mod tests { }); count_check!(10, clos, |client| { let result = reconcile(Arc::new(cluster), Arc::new(dummy_cluster_ctx(client))).await; - assert_eq!(result.unwrap(), Action::await_change()); + assert_eq!(result.unwrap(), Action::requeue(Duration::from_secs(300))); }); } diff --git a/operator/src/reference_values.rs b/operator/src/reference_values.rs index f065d8ff..d75c2c1d 100644 --- a/operator/src/reference_values.rs +++ b/operator/src/reference_values.rs @@ -291,7 +291,7 @@ async fn image_add_reconcile( } let (action, reason) = match handle_new_image(client.clone(), image).await { - Ok(reason) => (Action::await_change(), reason), + Ok(reason) => (Action::requeue(Duration::from_secs(300)), reason), Err(e) => { warn!("PCR computation for {name} failed: {e}"); let action = Action::requeue(Duration::from_secs(60)); @@ -320,7 +320,7 @@ async fn image_remove_reconcile( let name = image.metadata.name.as_ref().unwrap_or(&default); if cluster.is_none() { info!("No TrustedExecutionCluster found, skipping disallow_image for {name}"); - return Ok(Action::await_change()); + return Ok(Action::requeue(Duration::from_secs(60))); } let cluster = cluster.unwrap(); let tec_name = cluster.metadata.name.unwrap_or("".to_string()); @@ -329,10 +329,10 @@ async fn image_remove_reconcile( "TrustedExecutionCluster {tec_name} is being deleted, \ skipping disallow_image for {name}" ); - return Ok(Action::await_change()); + return Ok(Action::requeue(Duration::from_secs(60))); } disallow_image(client, name).await?; - Ok(Action::await_change()) + Ok(Action::requeue(Duration::from_secs(60))) } pub async fn launch_rv_image_controller(client: Client) { diff --git a/operator/src/register_server.rs b/operator/src/register_server.rs index 2a57ded2..0720d145 100644 --- a/operator/src/register_server.rs +++ b/operator/src/register_server.rs @@ -20,7 +20,7 @@ use kube::runtime::{ }; use kube::{Api, Client, Resource}; use log::info; -use std::{collections::BTreeMap, sync::Arc}; +use std::{collections::BTreeMap, sync::Arc, time::Duration}; use crate::trustee; use operator::*; @@ -142,7 +142,7 @@ async fn keygen_reconcile( trustee::mount_secret(kube_client, id).await } .await - .map(|_| Action::await_change()) + .map(|_| Action::requeue(Duration::from_secs(300))) .map_err(|e| finalizer::Error::::ApplyFailed(e.into())) } Event::Cleanup(machine) => { @@ -168,7 +168,7 @@ async fn keygen_reconcile( skipping unmount_secret for Machine {}", machine.metadata.name.as_deref().unwrap_or("unknown") ); - return Ok(Action::await_change()); + return Ok(Action::requeue(Duration::from_secs(60))); } Err(kube::Error::Api(ae)) if ae.code == 404 => { // TEC already deleted, skip unmount_secret @@ -177,7 +177,7 @@ async fn keygen_reconcile( skipping unmount_secret for Machine {}", machine.metadata.name.as_deref().unwrap_or("unknown") ); - return Ok(Action::await_change()); + return Ok(Action::requeue(Duration::from_secs(60))); } _ => { // TEC exists and is not being deleted, proceed with unmount_secret @@ -187,7 +187,7 @@ async fn keygen_reconcile( trustee::unmount_secret(kube_client, id) .await - .map(|_| Action::await_change()) + .map(|_| Action::requeue(Duration::from_secs(60))) .map_err(|e| finalizer::Error::::CleanupFailed(e.into())) } } From 7ae47da0e32ae8af17653ff501930d3550eaee29 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Tue, 7 Jul 2026 15:40:18 +0200 Subject: [PATCH 3/4] tests: Increase deletion timeouts of owned CRs requeue is 5 minutes, so wait 6 minutes for deletion if the event is missed. Also applies to ApprovedImage status update. Signed-off-by: Jakob Naucke --- test_utils/src/lib.rs | 2 +- tests/trusted_execution_cluster.rs | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test_utils/src/lib.rs b/test_utils/src/lib.rs index 53d75d37..3dbfa2fe 100644 --- a/test_utils/src/lib.rs +++ b/test_utils/src/lib.rs @@ -470,7 +470,7 @@ impl TestContext { pub async fn cleanup(&self) -> Result<()> { self.delete_trusted_execution_cluster().await?; - let timeout = scaled_duration(60); + let timeout = scaled_duration(360); let msg = format!("Resources were left behind after {timeout:?}"); let poller = Poller::new().with_timeout(timeout).with_error_message(msg); let chk = || async move { diff --git a/tests/trusted_execution_cluster.rs b/tests/trusted_execution_cluster.rs index f57c6d73..10553d2b 100644 --- a/tests/trusted_execution_cluster.rs +++ b/tests/trusted_execution_cluster.rs @@ -112,10 +112,10 @@ named_test!( wait_for_resource_deleted(&deployments_api, REGISTER_SERVER_DEPLOYMENT, timeout).await?; let images_api: Api = Api::namespaced(client.clone(), namespace); - wait_for_resource_deleted(&images_api, APPROVED_IMAGE_NAME, scaled_timeout(120)).await?; + wait_for_resource_deleted(&images_api, APPROVED_IMAGE_NAME, scaled_timeout(360)).await?; - wait_for_resource_deleted(&machines, &machine_name, scaled_timeout(120)).await?; - wait_for_resource_deleted(&attestation_keys, &ak_name, scaled_timeout(120)).await?; + wait_for_resource_deleted(&machines, &machine_name, scaled_timeout(360)).await?; + wait_for_resource_deleted(&attestation_keys, &ak_name, scaled_timeout(360)).await?; let secrets_api: Api = Api::namespaced(client.clone(), namespace); wait_for_resource_deleted(&secrets_api, &ak_name, scaled_timeout(120)).await?; @@ -328,9 +328,9 @@ async fn test_attestation_key_lifecycle() -> anyhow::Result<()> { machines.delete(&machine_name, &dp).await?; test_ctx.info(format!("Deleted Machine: {machine_name}")); - wait_for_resource_deleted(&machines, &machine_name, scaled_timeout(120)).await?; + wait_for_resource_deleted(&machines, &machine_name, scaled_timeout(360)).await?; test_ctx.info("Machine successfully deleted"); - wait_for_resource_deleted(&attestation_keys, &ak_name, scaled_timeout(120)).await?; + wait_for_resource_deleted(&attestation_keys, &ak_name, scaled_timeout(360)).await?; test_ctx.info("AttestationKey successfully deleted"); wait_for_resource_deleted(&secrets_api, &ak_name, scaled_timeout(120)).await?; test_ctx.info("Secret successfully deleted"); @@ -367,7 +367,7 @@ async fn test_nonexistent_approved_image() -> anyhow::Result<()> { }; let done = await_condition(images, "coreos1", is_pending); let ctx = "waiting for ApprovedImage coreos1 to be PodPending"; - timeout(scaled_duration(30), done).await.context(ctx)??; + timeout(scaled_duration(360), done).await.context(ctx)??; test_ctx.cleanup().await?; Ok(()) @@ -398,7 +398,7 @@ async fn test_approved_image_readoption() -> anyhow::Result<()> { test_ctx.info(format!("Deleting TrustedExecutionCluster {TEC_NAME}")); clusters.delete(TEC_NAME, &Default::default()).await?; wait_for_resource_deleted(&configmaps, TRUSTEE_CONFIG_MAP, scaled_timeout(60)).await?; - wait_for_resource_deleted(&images, APPROVED_IMAGE_NAME, scaled_timeout(60)).await?; + wait_for_resource_deleted(&images, APPROVED_IMAGE_NAME, scaled_timeout(360)).await?; test_ctx.info(format!("Configmap {TRUSTEE_CONFIG_MAP} was removed")); let image = ApprovedImage { From ef6d5ca5f393ffc13f3c5c81f2277d2439d1ef00 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Mon, 13 Jul 2026 10:28:50 +0200 Subject: [PATCH 4/4] operator/ak-reg: Use warn!, not eprintln! in consistency with remainder codebase Signed-off-by: Jakob Naucke --- operator/src/attestation_key_register.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/operator/src/attestation_key_register.rs b/operator/src/attestation_key_register.rs index 24945010..f83ddb71 100644 --- a/operator/src/attestation_key_register.rs +++ b/operator/src/attestation_key_register.rs @@ -25,7 +25,7 @@ use kube::{ watcher, }, }; -use log::info; +use log::{info, warn}; use serde_json::json; use std::{collections::BTreeMap, sync::Arc, time::Duration}; @@ -335,7 +335,7 @@ async fn secret_reconcile( .await .map(|_| Action::requeue(Duration::from_secs(300))) .map_err(|e| { - eprintln!("Error updating attestation key volumes on secret apply: {e}"); + warn!("Error updating attestation key volumes on secret apply: {e}"); finalizer::Error::::ApplyFailed(e.into()) }) } @@ -349,9 +349,7 @@ async fn secret_reconcile( .await .map(|_| Action::requeue(Duration::from_secs(300))) .map_err(|e| { - eprintln!( - "Error updating attestation key volumes during secret deletion: {e}" - ); + warn!("Error updating attestation key volumes during secret deletion: {e}"); finalizer::Error::::CleanupFailed(e.into()) }) }