fix(restore): make deployment-ready timeout configurable via env (#47)

Author: passcod

src/bin/operator.rs

@@ -20,7 +20,7 @@ use tower_http::trace::TraceLayer;
 use tracing::{debug, info, warn};
 
 use postgres_restore_operator::{
-	context::{Context, DEFAULT_KOPIA_IMAGE},
+	context::{Context, DEFAULT_DEPLOYMENT_READY_TIMEOUT_SECS, DEFAULT_KOPIA_IMAGE},
 	controllers,
 	types::{PostgresPhysicalReplica, PostgresPhysicalRestore},
 };
@@ -169,12 +169,28 @@ async fn main() -> anyhow::Result<()> {
 
 	annotate_own_pod(&client, &namespace).await;
 
+	let deployment_ready_timeout_secs = std::env::var("DEPLOYMENT_READY_TIMEOUT_SECS")
+		.ok()
+		.and_then(|v| {
+			v.parse::<u64>()
+				.map_err(
+					|e| warn!(value = v, error = %e, "invalid DEPLOYMENT_READY_TIMEOUT_SECS, using default"),
+				)
+				.ok()
+		})
+		.unwrap_or(DEFAULT_DEPLOYMENT_READY_TIMEOUT_SECS);
+	info!(
+		deployment_ready_timeout_secs,
+		"deployment readiness timeout configured"
+	);
+
 	let ctx = Arc::new(Context::new(
 		client.clone(),
 		max_concurrent_restores,
 		kopia_image,
 		use_port_forward,
 		callback_base_url,
+		deployment_ready_timeout_secs,
 	));
 
 	// Heartbeat: a background task updates this timestamp every 5s.
@@ -494,6 +510,7 @@ mod tests {
 			DEFAULT_KOPIA_IMAGE.to_string(),
 			false,
 			"http://test.svc:8080".to_string(),
+			DEFAULT_DEPLOYMENT_READY_TIMEOUT_SECS,
 		));
 		let heartbeat = Arc::new(AtomicI64::new(Timestamp::now().as_second()));
 		let state = ServerState { heartbeat, ctx };

src/context.rs

@@ -10,6 +10,7 @@ use kube::runtime::events::{Recorder, Reporter};
 use crate::{controllers::jobs::CallbackStore, metrics::Metrics};
 
 pub const DEFAULT_KOPIA_IMAGE: &str = "kopia/kopia:0.22.3";
+pub const DEFAULT_DEPLOYMENT_READY_TIMEOUT_SECS: u64 = 30 * 60;
 
 pub struct Context {
 	pub client: Client,
@@ -27,6 +28,12 @@ pub struct Context {
 	/// Base URL the operator is reachable at from within the cluster,
 	/// e.g. `http://postgres-restore-operator.pgro-system.svc:8080`.
 	pub callback_base_url: String,
+	/// Seconds to wait for a restore's postgres Deployment to become Ready
+	/// after the kopia restore Job completes, before marking the restore
+	/// Failed. Configurable via the `DEPLOYMENT_READY_TIMEOUT_SECS` env
+	/// var so it can be raised for replicas with large data dirs (slower
+	/// WAL replay) without needing a code release.
+	pub deployment_ready_timeout_secs: u64,
 	/// Unix timestamp of the last successful entry into a reconcile function.
 	/// Used by `/livez` to detect a stuck reconciliation loop.
 	pub last_reconcile: Arc<AtomicI64>,
@@ -39,6 +46,7 @@ impl Context {
 		kopia_image: String,
 		use_port_forward: bool,
 		callback_base_url: String,
+		deployment_ready_timeout_secs: u64,
 	) -> Self {
 		let reporter = Reporter::from("postgres-restore-operator");
 		let recorder = Recorder::new(client.clone(), reporter);
@@ -55,6 +63,7 @@ impl Context {
 			snapshot_results: Arc::new(CallbackStore::default()),
 			schema_migration_results: Arc::new(CallbackStore::default()),
 			callback_base_url,
+			deployment_ready_timeout_secs,
 			last_reconcile: Arc::new(AtomicI64::new(Timestamp::now().as_second())),
 		}
 	}

src/controllers/restore.rs

@@ -766,13 +766,18 @@ async fn reconcile_ready(
 		return Ok(Action::requeue(Duration::from_secs(5)));
 	}
 
-	// Check for timeout (10 minutes)
+	// Deployment readiness timeout. Replicas with larger data dirs need
+	// time for postgres to open the data dir and replay WAL after a
+	// fresh kopia restore. Default is 30 minutes; tunable per-cluster
+	// via the `DEPLOYMENT_READY_TIMEOUT_SECS` env var so larger replicas
+	// can raise it without a code release.
 	if let Some(created_at) = restore.status.as_ref().and_then(|s| s.restored_at.as_ref()) {
 		let elapsed = Timestamp::now().duration_since(created_at.0);
-		if elapsed > SignedDuration::from_secs(10 * 60) {
+		let timeout_secs = ctx.deployment_ready_timeout_secs;
+		if elapsed > SignedDuration::from_secs(timeout_secs as i64) {
 			warn!(
 				restore = name,
-				"deployment not ready after 10 minutes, marking as Failed"
+				timeout_secs, "deployment not ready within configured timeout, marking as Failed"
 			);
 			return fail_restore(
 				ctx,