fix(pegboard): align drain grace defaults with stop threshold

NathanFlurry · NathanFlurry · commit e1be3d68d3be · 2026-05-04T09:12:10.000-07:00
diff --git a/engine/packages/api-types/src/namespaces/runner_configs.rs b/engine/packages/api-types/src/namespaces/runner_configs.rs
@@ -68,7 +68,7 @@ impl Into<rivet_types::runner_configs::RunnerConfig> for RunnerConfig {
 				request_lifespan,
 				max_concurrent_actors: max_concurrent_actors.unwrap_or(max_runners as u64),
 				// Default to deprecated config value (config.pegboard.serverless_drain_grace_period)
-				drain_grace_period: drain_grace_period.unwrap_or(10),
+				drain_grace_period: drain_grace_period.unwrap_or(30 * 60),
 				slots_per_runner,
 				min_runners: min_runners.unwrap_or_default(),
 				max_runners,
diff --git a/engine/packages/config/src/config/pegboard.rs b/engine/packages/config/src/config/pegboard.rs
@@ -173,7 +173,7 @@ impl Pegboard {
 	/// When changing this default, update
 	/// website/src/content/docs/actors/versions.mdx (SIGTERM Handling section).
 	pub fn actor_stop_threshold(&self) -> i64 {
-		self.actor_stop_threshold.unwrap_or(30_000)
+		self.actor_stop_threshold.unwrap_or(30 * 60 * 1000)
 	}
 
 	pub fn actor_retry_duration_threshold(&self) -> i64 {
diff --git a/engine/packages/engine/tests/runner/api_runner_configs_upsert.rs b/engine/packages/engine/tests/runner/api_runner_configs_upsert.rs
@@ -669,3 +669,49 @@ fn upsert_runner_config_serverless_slots_per_runner_zero() {
 		);
 	});
 }
+
+#[test]
+fn upsert_runner_config_serverless_drain_grace_period_exceeds_actor_stop_threshold() {
+	common::run(common::TestOpts::new(1), |ctx| async move {
+		let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await;
+
+		let runner_name = "long-drain-runner";
+		let mut datacenters = HashMap::new();
+		datacenters.insert(
+			"dc-1".to_string(),
+			rivet_api_types::namespaces::runner_configs::RunnerConfig {
+				kind: rivet_api_types::namespaces::runner_configs::RunnerConfigKind::Serverless {
+					url: "http://example.com".to_string(),
+					headers: None,
+					request_lifespan: 30,
+					max_concurrent_actors: Some(5),
+					drain_grace_period: Some(30 * 60 + 1),
+					slots_per_runner: 1,
+					min_runners: Some(1),
+					max_runners: 5,
+					runners_margin: Some(2),
+					metadata_poll_interval: None,
+				},
+				metadata: None,
+				drain_on_version_upgrade: true,
+			},
+		);
+
+		let result = common::api::public::runner_configs_upsert(
+			ctx.leader_dc().guard_port(),
+			rivet_api_peer::runner_configs::UpsertPath {
+				runner_name: runner_name.to_string(),
+			},
+			rivet_api_peer::runner_configs::UpsertQuery {
+				namespace: namespace.clone(),
+			},
+			rivet_api_public::runner_configs::upsert::UpsertRequest { datacenters },
+		)
+		.await;
+
+		assert!(
+			result.is_err(),
+			"Upsert should fail when drain_grace_period exceeds actor_stop_threshold"
+		);
+	});
+}
diff --git a/engine/packages/pegboard/src/ops/runner_config/upsert.rs b/engine/packages/pegboard/src/ops/runner_config/upsert.rs
@@ -29,6 +29,7 @@ pub async fn pegboard_runner_config_upsert(ctx: &OperationCtx, input: &Input) ->
 		RunnerConfigKind::Serverless {
 			url,
 			headers,
+			drain_grace_period,
 			slots_per_runner,
 			..
 		} => {
@@ -79,6 +80,17 @@ pub async fn pegboard_runner_config_upsert(ctx: &OperationCtx, input: &Input) ->
 				}
 				.build());
 			}
+
+			let actor_stop_threshold_ms = ctx.config().pegboard().actor_stop_threshold();
+			let drain_grace_period_ms = i64::from(*drain_grace_period) * 1000;
+			if drain_grace_period_ms > actor_stop_threshold_ms {
+				return Err(errors::RunnerConfig::Invalid {
+					reason: format!(
+						"`drain_grace_period` cannot be greater than `actor_stop_threshold` ({drain_grace_period_ms}ms > {actor_stop_threshold_ms}ms)"
+					),
+				}
+				.build());
+			}
 		}
 	}
 
diff --git a/engine/sdks/rust/data/src/versioned/namespace_runner_config.rs b/engine/sdks/rust/data/src/versioned/namespace_runner_config.rs
@@ -309,7 +309,7 @@ impl NamespaceRunnerConfig {
 							// Default to max_runners for v4 -> v5 migration
 							max_concurrent_actors: serverless.max_runners as u64,
 							// Default to deprecated config value (config.pegboard.serverless_drain_grace_period)
-							drain_grace_period: 10,
+							drain_grace_period: 30 * 60,
 							slots_per_runner: serverless.slots_per_runner,
 							min_runners: serverless.min_runners,
 							max_runners: serverless.max_runners,
diff --git a/rivetkit-typescript/packages/rivetkit/src/serverless/configure.ts b/rivetkit-typescript/packages/rivetkit/src/serverless/configure.ts
@@ -62,7 +62,7 @@ export async function configureServerlessPool(
 				serverless: {
 					url: customConfig.url,
 					headers,
-					request_lifespan: customConfig.requestLifespan ?? 15 * 60,
+					request_lifespan: customConfig.requestLifespan ?? 60 * 60,
 					drain_grace_period: customConfig.drainGracePeriod,
 					metadata_poll_interval:
 						customConfig.metadataPollInterval ?? 1000,
diff --git a/rivetkit-typescript/packages/rivetkit/tests/platforms/shared-platform-harness.ts b/rivetkit-typescript/packages/rivetkit/tests/platforms/shared-platform-harness.ts
@@ -497,7 +497,7 @@ export async function createPlatformServerlessRunner({
 				serverless: {
 					url: serverlessUrl,
 					headers: headers ?? {},
-					request_lifespan: requestLifespan ?? 15 * 60,
+					request_lifespan: requestLifespan ?? 60 * 60,
 					drain_grace_period: drainGracePeriod,
 					metadata_poll_interval: metadataPollInterval ?? 1_000,
 					max_runners: maxRunners ?? 100_000,
diff --git a/website/src/content/docs/actors/limits.mdx b/website/src/content/docs/actors/limits.mdx
@@ -138,15 +138,15 @@ These timeouts control how actors are shut down when a serverless request reache
 
 | Name | Soft Limit | Hard Limit | Description |
 |------|------------|------------|-------------|
-| Request lifespan | 900 seconds (15 min) | — | Total lifespan of a serverless request before drain begins. Configurable via `requestLifespan` in [`configurePool`](/docs/connect/registry-configuration). |
+| Request lifespan | 3600 seconds (60 min) | — | Total lifespan of a serverless request before drain begins. Configurable via `requestLifespan` in [`configurePool`](/docs/connect/registry-configuration). |
 | Serverless drain grace period | — | 10 seconds | Time reserved at the end of a request for actors to stop gracefully. Configurable via [engine config](/docs/self-hosting/configuration) (`pegboard.serverless_drain_grace_period`). |
 
 ### Actor Lifecycle
 
 | Name | Soft Limit | Hard Limit | Description |
 |------|------------|------------|-------------|
 | Actor start threshold | — | 30 seconds | Maximum time for an actor to start before it is considered lost and rescheduled. |
-| Actor stop threshold | — | 30 seconds | Maximum time for an actor to stop before it is considered lost. |
+| Actor stop threshold | — | 30 minutes | Maximum time for an actor to stop before it is considered lost. |
 
 ## Increasing Limits
 
diff --git a/website/src/content/docs/actors/versions.mdx b/website/src/content/docs/actors/versions.mdx
@@ -215,7 +215,7 @@ The `drainOnVersionUpgrade` option controls whether old actors are stopped when
 | Value | Behavior |
 |-------|----------|
 | `false` (default in [runner mode](/docs/general/runtime-modes)) | Old actors continue running. New actors go to new version. Versions coexist. |
-| `true` (default in [serverless mode](/docs/general/runtime-modes)) | Old actors receive stop signal and have 30s to finish gracefully. |
+| `true` (default in [serverless mode](/docs/general/runtime-modes)) | Old actors receive stop signal and have 30m to finish gracefully. |
 
 ## Upgrading Actor State
 
@@ -343,11 +343,11 @@ When a runner process receives SIGTERM, it gracefully stops all actors before ex
 
 - Each actor's `onSleep` hook is called, giving it time to save state
 - Actors are rescheduled to other available runners
-- The runner waits up to **120 seconds** for all actors to finish stopping
+- The runner waits up to **30 minutes** for all actors to finish stopping
 - If the process is force-killed before actors finish (e.g. SIGKILL), actors are rescheduled with a crash backoff penalty instead of a clean handoff
 
 <Note>
-Ensure your platform's shutdown grace period is at least **130 seconds** to give actors time to stop cleanly.
+Ensure your platform's shutdown grace period is at least **30 minutes** to give actors time to stop cleanly.
 </Note>
 
 ### Shutdown Timeouts
@@ -356,7 +356,7 @@ Several timeouts control how long each part of the shutdown process can take:
 
 | Timeout | Default | Description | Configuration |
 |---------|---------|-------------|---------------|
-| `actor_stop_threshold` | 30s | Engine-side limit on how long each actor has to stop before being marked lost | [Engine config](/docs/self-hosting/configuration) (`pegboard.actor_stop_threshold`) |
+| `actor_stop_threshold` | 30m | Engine-side limit on how long each actor has to stop before being marked lost | [Engine config](/docs/self-hosting/configuration) (`pegboard.actor_stop_threshold`) |
 | `sleepGracePeriod` | 15s | Total graceful sleep budget for `onSleep`, `waitUntil`, `keepAwake`, and async raw WebSocket handlers | [Actor options](/docs/actors/lifecycle#options) |
 | `runner_lost_threshold` | 15s | Fallback detection if the runner dies without graceful shutdown | [Engine config](/docs/self-hosting/configuration) (`pegboard.runner_lost_threshold`) |
 
diff --git a/website/src/content/docs/connect/kubernetes.mdx b/website/src/content/docs/connect/kubernetes.mdx
@@ -79,9 +79,9 @@ spec:
         app: rivetkit-app
     spec:
       # Allow enough time for actors to gracefully stop on SIGTERM.
-      # The runner waits up to 120s for actors to finish; 130s provides buffer.
+      # The runner waits up to 30m for actors to finish.
       # See: /docs/actors/versions#graceful-shutdown-sigterm
-      terminationGracePeriodSeconds: 130
+      terminationGracePeriodSeconds: 1800
       containers:
         - name: rivetkit-app
           image: registry.example.com/your-team/rivetkit-app:latest
diff --git a/website/src/content/docs/general/production-checklist.mdx b/website/src/content/docs/general/production-checklist.mdx
@@ -26,7 +26,7 @@ We recommend passing this page to your coding agent to verify your configuration
 
 ### Runner
 
-- **Set a graceful shutdown period of at least 130 seconds.** Runners need time to drain actors during upgrades. In Kubernetes, set `terminationGracePeriodSeconds: 130` on the pod spec.
+- **Set a graceful shutdown period of at least 30 minutes.** Runners need time to drain actors during upgrades. In Kubernetes, set `terminationGracePeriodSeconds: 1800` on the pod spec.
 
 ## Actors
 
diff --git a/website/src/content/docs/general/runtime-modes.mdx b/website/src/content/docs/general/runtime-modes.mdx
@@ -79,9 +79,9 @@ Read more about [how we handle timeouts](/blog/2025-10-20-how-we-built-websocket
 
 #### Shutdown Sequence
 
-Each serverless request has a configurable lifespan (`requestLifespan`, default: 15 minutes). Set this to match your platform's function timeout (e.g. `requestLifespan: 300` for Vercel Pro).
+Each serverless request has a configurable lifespan (`requestLifespan`, default: 60 minutes). Set this to match your platform's function timeout (e.g. `requestLifespan: 3600` for Vercel Pro).
 
-When the request nears its lifespan, the engine reserves a grace period (`serverless_drain_grace_period`, default: 10 seconds) at the end to gracefully stop actors. For example, with a 300-second lifespan, actors begin stopping at 290 seconds. After the full lifespan elapses, the connection is forcibly closed and any remaining actors are rescheduled.
+When the request nears its lifespan, the engine reserves a grace period (`serverless_drain_grace_period`, default: 10 seconds) at the end to gracefully stop actors. For example, with a 3600-second lifespan, actors begin stopping at 3590 seconds. After the full lifespan elapses, the connection is forcibly closed and any remaining actors are rescheduled.
 
 See [Limits](/docs/actors/limits#serverless-shutdown) for configuration details.
 
diff --git a/website/src/content/docs/self-hosting/kubernetes.mdx b/website/src/content/docs/self-hosting/kubernetes.mdx
@@ -103,9 +103,9 @@ spec:
         app: rivetkit-app
     spec:
       # Allow enough time for actors to gracefully stop on SIGTERM.
-      # The runner waits up to 120s for actors to finish; 130s provides buffer.
+      # The runner waits up to 30m for actors to finish.
       # See: /docs/actors/versions#graceful-shutdown-sigterm
-      terminationGracePeriodSeconds: 130
+      terminationGracePeriodSeconds: 1800
       containers:
         - name: rivetkit-app
           image: registry.example.com/your-team/rivetkit-app:latest

Original file line number	Diff line number	Diff line change
`@@ -173,7 +173,7 @@ impl Pegboard {`
`173`	`173`	`/// When changing this default, update`
`174`	`174`	`/// website/src/content/docs/actors/versions.mdx (SIGTERM Handling section).`
`175`	`175`	`pub fn actor_stop_threshold(&self) -> i64 {`
`176`		`- self.actor_stop_threshold.unwrap_or(30_000)`
	`176`	`+ self.actor_stop_threshold.unwrap_or(30 * 60 * 1000)`
`177`	`177`	`}`
`178`	`178`
`179`	`179`	`pub fn actor_retry_duration_threshold(&self) -> i64 {`
Original file line number	Diff line number	Diff line change
`@@ -29,6 +29,7 @@ pub async fn pegboard_runner_config_upsert(ctx: &OperationCtx, input: &Input) ->`
`29`	`29`	`RunnerConfigKind::Serverless {`
`30`	`30`	`url,`
`31`	`31`	`headers,`
	`32`	`+ drain_grace_period,`
`32`	`33`	`slots_per_runner,`
`33`	`34`	`..`
`34`	`35`	`} => {`
`@@ -79,6 +80,17 @@ pub async fn pegboard_runner_config_upsert(ctx: &OperationCtx, input: &Input) ->`
`79`	`80`	`}`
`80`	`81`	`.build());`
`81`	`82`	`}`
	`83`	`+`
	`84`	`+ let actor_stop_threshold_ms = ctx.config().pegboard().actor_stop_threshold();`
	`85`	`+ let drain_grace_period_ms = i64::from(drain_grace_period) 1000;`
	`86`	`+ if drain_grace_period_ms > actor_stop_threshold_ms {`
	`87`	`+ return Err(errors::RunnerConfig::Invalid {`
	`88`	`+ reason: format!(`
	`89`	+ "`drain_grace_period` cannot be greater than `actor_stop_threshold` ({drain_grace_period_ms}ms > {actor_stop_threshold_ms}ms)"
	`90`	`+ ),`
	`91`	`+ }`
	`92`	`+ .build());`
	`93`	`+ }`
`82`	`94`	`}`
`83`	`95`	`}`
`84`	`96`