diff --git a/operator/src/attestation_key_register.rs b/operator/src/attestation_key_register.rs index 332e5124..f83ddb71 100644 --- a/operator/src/attestation_key_register.rs +++ b/operator/src/attestation_key_register.rs @@ -25,7 +25,7 @@ use kube::{ watcher, }, }; -use log::info; +use log::{info, warn}; use serde_json::json; use std::{collections::BTreeMap, sync::Arc, time::Duration}; @@ -195,10 +195,10 @@ async fn ak_reconcile( for machine in ctx.machine_store.state() { if ak.spec.uuid.as_ref() == Some(&machine.spec.id) { approve_ak(&ak, &machine, &ctx).await?; - return Ok(Action::await_change()); + return Ok(Action::requeue(Duration::from_secs(300))); } } - Ok(Action::await_change()) + Ok(Action::requeue(Duration::from_secs(300))) } async fn machine_reconcile( @@ -216,7 +216,7 @@ async fn machine_reconcile( "Machine {} is being deleted, updating attestation key volumes", machine.metadata.name.clone().unwrap_or_default() ); - return Ok(Action::await_change()); + return Ok(Action::requeue(Duration::from_secs(60))); } for ak in ctx.ak_store.state() { @@ -224,10 +224,10 @@ async fn machine_reconcile( && *ak_uuid == machine.spec.id { approve_ak(&ak, &machine, &ctx).await?; - return Ok(Action::await_change()); + return Ok(Action::requeue(Duration::from_secs(300))); } } - Ok(Action::await_change()) + Ok(Action::requeue(Duration::from_secs(300))) } async fn approve_ak(ak: &AttestationKey, machine: &Machine, ctx: &AkContextData) -> Result<()> { @@ -333,9 +333,9 @@ async fn secret_reconcile( // On creation/update, just update the trustee deployment volumes trustee::update_attestation_keys(&ctx) .await - .map(|_| Action::await_change()) + .map(|_| Action::requeue(Duration::from_secs(300))) .map_err(|e| { - eprintln!("Error updating attestation key volumes on secret apply: {e}"); + warn!("Error updating attestation key volumes on secret apply: {e}"); finalizer::Error::::ApplyFailed(e.into()) }) } @@ -347,11 +347,9 @@ async fn secret_reconcile( // Update trustee deployment - secrets with deletion_timestamp will be filtered out trustee::update_attestation_keys(&ctx) .await - .map(|_| Action::await_change()) + .map(|_| Action::requeue(Duration::from_secs(300))) .map_err(|e| { - eprintln!( - "Error updating attestation key volumes during secret deletion: {e}" - ); + warn!("Error updating attestation key volumes during secret deletion: {e}"); finalizer::Error::::CleanupFailed(e.into()) }) } diff --git a/operator/src/main.rs b/operator/src/main.rs index 03ff138e..3e06b5c7 100644 --- a/operator/src/main.rs +++ b/operator/src/main.rs @@ -99,11 +99,11 @@ async fn reconcile( if changed { update_status!(clusters, name, TrustedExecutionClusterStatus { conditions })?; } - return Ok(Action::await_change()); + return Ok(Action::requeue(Duration::from_secs(60))); } if is_installed(cluster.status.clone()) { - return Ok(Action::await_change()); + return Ok(Action::requeue(Duration::from_secs(300))); } if ctx.tec_store.state().len() > 1 { @@ -145,7 +145,7 @@ async fn reconcile( let status = TrustedExecutionClusterStatus { conditions }; update_status!(clusters, name, status)?; } - Ok(Action::await_change()) + Ok(Action::requeue(Duration::from_secs(300))) } async fn install_components(client: &Client, cluster: &TrustedExecutionCluster) -> Result<()> { @@ -336,7 +336,7 @@ mod tests { let mut cluster = dummy_cluster(); cluster.metadata.deletion_timestamp = Some(Time(Timestamp::now())); let result = reconcile(Arc::new(cluster), Arc::new(dummy_cluster_ctx(client))).await; - assert_eq!(result.unwrap(), Action::await_change()); + assert_eq!(result.unwrap(), Action::requeue(Duration::from_secs(60))); }); } @@ -416,7 +416,7 @@ mod tests { conditions: Some(vec![foreign_condition]), }); let result = reconcile(Arc::new(cluster), Arc::new(dummy_cluster_ctx(client))).await; - assert_eq!(result.unwrap(), Action::await_change()); + assert_eq!(result.unwrap(), Action::requeue(Duration::from_secs(60))); }); } @@ -493,7 +493,7 @@ mod tests { }); count_check!(10, clos, |client| { let result = reconcile(Arc::new(cluster), Arc::new(dummy_cluster_ctx(client))).await; - assert_eq!(result.unwrap(), Action::await_change()); + assert_eq!(result.unwrap(), Action::requeue(Duration::from_secs(300))); }); } diff --git a/operator/src/reference_values.rs b/operator/src/reference_values.rs index f202c533..d75c2c1d 100644 --- a/operator/src/reference_values.rs +++ b/operator/src/reference_values.rs @@ -291,7 +291,7 @@ async fn image_add_reconcile( } let (action, reason) = match handle_new_image(client.clone(), image).await { - Ok(reason) => (Action::await_change(), reason), + Ok(reason) => (Action::requeue(Duration::from_secs(300)), reason), Err(e) => { warn!("PCR computation for {name} failed: {e}"); let action = Action::requeue(Duration::from_secs(60)); @@ -320,7 +320,7 @@ async fn image_remove_reconcile( let name = image.metadata.name.as_ref().unwrap_or(&default); if cluster.is_none() { info!("No TrustedExecutionCluster found, skipping disallow_image for {name}"); - return Ok(Action::await_change()); + return Ok(Action::requeue(Duration::from_secs(60))); } let cluster = cluster.unwrap(); let tec_name = cluster.metadata.name.unwrap_or("".to_string()); @@ -329,16 +329,19 @@ async fn image_remove_reconcile( "TrustedExecutionCluster {tec_name} is being deleted, \ skipping disallow_image for {name}" ); - return Ok(Action::await_change()); + return Ok(Action::requeue(Duration::from_secs(60))); } disallow_image(client, name).await?; - Ok(Action::await_change()) + Ok(Action::requeue(Duration::from_secs(60))) } pub async fn launch_rv_image_controller(client: Client) { let images: Api = Api::default_namespaced(client.clone()); + let jobs: Api = Api::default_namespaced(client.clone()); + let wc = watcher::Config::default().labels(&format!("{JOB_LABEL_KEY}={PCR_COMMAND_NAME}")); tokio::spawn( Controller::new(images, Default::default()) + .owns(jobs, wc) .run(image_reconcile, controller_error_policy, Arc::new(client)) .for_each(controller_info), ); diff --git a/operator/src/register_server.rs b/operator/src/register_server.rs index 2a57ded2..0720d145 100644 --- a/operator/src/register_server.rs +++ b/operator/src/register_server.rs @@ -20,7 +20,7 @@ use kube::runtime::{ }; use kube::{Api, Client, Resource}; use log::info; -use std::{collections::BTreeMap, sync::Arc}; +use std::{collections::BTreeMap, sync::Arc, time::Duration}; use crate::trustee; use operator::*; @@ -142,7 +142,7 @@ async fn keygen_reconcile( trustee::mount_secret(kube_client, id).await } .await - .map(|_| Action::await_change()) + .map(|_| Action::requeue(Duration::from_secs(300))) .map_err(|e| finalizer::Error::::ApplyFailed(e.into())) } Event::Cleanup(machine) => { @@ -168,7 +168,7 @@ async fn keygen_reconcile( skipping unmount_secret for Machine {}", machine.metadata.name.as_deref().unwrap_or("unknown") ); - return Ok(Action::await_change()); + return Ok(Action::requeue(Duration::from_secs(60))); } Err(kube::Error::Api(ae)) if ae.code == 404 => { // TEC already deleted, skip unmount_secret @@ -177,7 +177,7 @@ async fn keygen_reconcile( skipping unmount_secret for Machine {}", machine.metadata.name.as_deref().unwrap_or("unknown") ); - return Ok(Action::await_change()); + return Ok(Action::requeue(Duration::from_secs(60))); } _ => { // TEC exists and is not being deleted, proceed with unmount_secret @@ -187,7 +187,7 @@ async fn keygen_reconcile( trustee::unmount_secret(kube_client, id) .await - .map(|_| Action::await_change()) + .map(|_| Action::requeue(Duration::from_secs(60))) .map_err(|e| finalizer::Error::::CleanupFailed(e.into())) } } diff --git a/test_utils/src/lib.rs b/test_utils/src/lib.rs index 53d75d37..3dbfa2fe 100644 --- a/test_utils/src/lib.rs +++ b/test_utils/src/lib.rs @@ -470,7 +470,7 @@ impl TestContext { pub async fn cleanup(&self) -> Result<()> { self.delete_trusted_execution_cluster().await?; - let timeout = scaled_duration(60); + let timeout = scaled_duration(360); let msg = format!("Resources were left behind after {timeout:?}"); let poller = Poller::new().with_timeout(timeout).with_error_message(msg); let chk = || async move { diff --git a/tests/trusted_execution_cluster.rs b/tests/trusted_execution_cluster.rs index f57c6d73..10553d2b 100644 --- a/tests/trusted_execution_cluster.rs +++ b/tests/trusted_execution_cluster.rs @@ -112,10 +112,10 @@ named_test!( wait_for_resource_deleted(&deployments_api, REGISTER_SERVER_DEPLOYMENT, timeout).await?; let images_api: Api = Api::namespaced(client.clone(), namespace); - wait_for_resource_deleted(&images_api, APPROVED_IMAGE_NAME, scaled_timeout(120)).await?; + wait_for_resource_deleted(&images_api, APPROVED_IMAGE_NAME, scaled_timeout(360)).await?; - wait_for_resource_deleted(&machines, &machine_name, scaled_timeout(120)).await?; - wait_for_resource_deleted(&attestation_keys, &ak_name, scaled_timeout(120)).await?; + wait_for_resource_deleted(&machines, &machine_name, scaled_timeout(360)).await?; + wait_for_resource_deleted(&attestation_keys, &ak_name, scaled_timeout(360)).await?; let secrets_api: Api = Api::namespaced(client.clone(), namespace); wait_for_resource_deleted(&secrets_api, &ak_name, scaled_timeout(120)).await?; @@ -328,9 +328,9 @@ async fn test_attestation_key_lifecycle() -> anyhow::Result<()> { machines.delete(&machine_name, &dp).await?; test_ctx.info(format!("Deleted Machine: {machine_name}")); - wait_for_resource_deleted(&machines, &machine_name, scaled_timeout(120)).await?; + wait_for_resource_deleted(&machines, &machine_name, scaled_timeout(360)).await?; test_ctx.info("Machine successfully deleted"); - wait_for_resource_deleted(&attestation_keys, &ak_name, scaled_timeout(120)).await?; + wait_for_resource_deleted(&attestation_keys, &ak_name, scaled_timeout(360)).await?; test_ctx.info("AttestationKey successfully deleted"); wait_for_resource_deleted(&secrets_api, &ak_name, scaled_timeout(120)).await?; test_ctx.info("Secret successfully deleted"); @@ -367,7 +367,7 @@ async fn test_nonexistent_approved_image() -> anyhow::Result<()> { }; let done = await_condition(images, "coreos1", is_pending); let ctx = "waiting for ApprovedImage coreos1 to be PodPending"; - timeout(scaled_duration(30), done).await.context(ctx)??; + timeout(scaled_duration(360), done).await.context(ctx)??; test_ctx.cleanup().await?; Ok(()) @@ -398,7 +398,7 @@ async fn test_approved_image_readoption() -> anyhow::Result<()> { test_ctx.info(format!("Deleting TrustedExecutionCluster {TEC_NAME}")); clusters.delete(TEC_NAME, &Default::default()).await?; wait_for_resource_deleted(&configmaps, TRUSTEE_CONFIG_MAP, scaled_timeout(60)).await?; - wait_for_resource_deleted(&images, APPROVED_IMAGE_NAME, scaled_timeout(60)).await?; + wait_for_resource_deleted(&images, APPROVED_IMAGE_NAME, scaled_timeout(360)).await?; test_ctx.info(format!("Configmap {TRUSTEE_CONFIG_MAP} was removed")); let image = ApprovedImage {