test: stabilize rolling restart graceful shutdown e2e (llm-d#545)

Make the rolling restart shutdown test deterministic by slowing the simulator so requests stay in flight until SIGTERM arrives. This removes the timing race that let the batch complete before orphan recovery could be exercised.

Signed-off-by: Jooyeon Mok <jmok@redhat.com>

Author: j-mok-dev

test/e2e/e2e_test.go

@@ -54,6 +54,7 @@ var (
 
 	// testSimService* hold the Kubernetes service names for the simulators.
 	// Must match VLLM_SIM_*_NAME in dev-common.sh (overridable via env).
+	testSimService     = getEnvOrDefault("TEST_SIM_SERVICE", "vllm-sim")
 	testSimService429  = getEnvOrDefault("TEST_SIM_SERVICE_429", "vllm-sim-429")
 	testSimServiceAIMD = getEnvOrDefault("TEST_SIM_SERVICE_AIMD", "vllm-sim-aimd")
 

test/e2e/processor_graceful_shutdown_test.go

@@ -96,17 +96,31 @@ func doTestPodDeleteMidJob(t *testing.T) {
 		finalBatch.RequestCounts.Total)
 }
 
-// doTestRollingRestartReEnqueue submits a batch with the same slow-request
-// pattern as doTestPodDeleteMidJob, triggers a rolling restart of the processor
-// deployment, and verifies the GC reconciler transitions the orphaned job to
-// failed. Same SIGTERM -> orphan -> reconciler path, different trigger.
+// doTestRollingRestartReEnqueue submits a batch, triggers a rolling restart of
+// the processor deployment, and verifies the GC reconciler transitions the
+// orphaned job to failed. Same SIGTERM -> orphan -> reconciler path as
+// PodDeleteMidJob, different trigger.
+//
+// Rolling restart delivers SIGTERM only after the new pod is Ready (~12s).
+// To guarantee requests are still in-flight when SIGTERM arrives, we set the
+// simulator's inter-token latency to 30s via /admin/config. This makes even a
+// single generated token take longer than the new-pod startup delay, eliminating
+// the race between request completion and SIGTERM delivery.
 func doTestRollingRestartReEnqueue(t *testing.T) {
 	t.Helper()
 
 	if !testKubectlAvailable {
 		t.Skip("kubectl not available, skipping rolling restart test")
 	}
 
+	// Slow down the simulator so requests cannot complete before SIGTERM arrives.
+	setSimAdminConfig(t, testSimService, `{"inter-token-latency":"30s"}`)
+	t.Cleanup(func() {
+		if err := trySetSimAdminConfig(t, testSimService, `{"inter-token-latency":"100ms"}`); err != nil {
+			t.Errorf("cleanup: failed to restore %s inter-token-latency: %v", testSimService, err)
+		}
+	})
+
 	var lines []string
 	for i := 1; i <= 10; i++ {
 		lines = append(lines, fmt.Sprintf(