Migrate OCP-44493: configurable terminationGracePeriod for liveness and startup probes

Migrates test from openshift-tests-private to origin.

Test validates probe-level terminationGracePeriodSeconds for:
- Liveness probes with probe-level terminationGracePeriodSeconds (10s)
- Startup probes with probe-level terminationGracePeriodSeconds (10s)
- Liveness probes without probe-level (falls back to pod-level 60s)

The test creates pods with failing probes and verifies the time
difference between probe failure (Killing event) and container
restart (Started event) matches the expected termination grace
period within acceptable range.

Event matching logic parses 'oc describe pod' output for:
- Killing events with container name
- Started events after restart

Updates:
- Add test to test/extended/node/node_e2e/node.go
- Document test in test/extended/node/README.md

Relates: https://issues.redhat.com/browse/OCPBUGS-44493
Signed-off-by: Bhargavi Gudi <BhargaviGudi@users.noreply.github.com>

Author: BhargaviGudi

test/extended/node/README.md

@@ -19,6 +19,7 @@ This directory contains OpenShift end-to-end tests for node-related features.
 - **image_volume.go** - Tests mounting container images as volumes in pods, including subPath and error handling
 - **node_swap.go** - Tests default kubelet swap settings (failSwapOn and swapBehavior) and rejection of user overrides
 - **zstd_chunked.go** - Tests building and running images with zstd:chunked compression format
+- **node_e2e/node.go** - Probe-level terminationGracePeriodSeconds (OCP-44493) - Tests configurable termination grace period for liveness and startup probes [Lifecycle:informing]
 
 ## Directory Structure
 

test/extended/node/node_e2e/node.go

@@ -9,13 +9,17 @@ import (
 
 	g "github.com/onsi/ginkgo/v2"
 	o "github.com/onsi/gomega"
+	ote "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo"
 
 	configv1 "github.com/openshift/api/config/v1"
 	"github.com/openshift/origin/test/extended/imagepolicy"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	utilrand "k8s.io/apimachinery/pkg/util/rand"
 	"k8s.io/apimachinery/pkg/util/wait"
 	e2e "k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/utils/ptr"
 
 	nodeutils "github.com/openshift/origin/test/extended/node"
 	exutil "github.com/openshift/origin/test/extended/util"
@@ -164,6 +168,220 @@ var _ = g.Describe("[sig-node] [Jira:Node/Kubelet] Kubelet, CRI-O, CPU manager",
 		e2e.Logf("/dev/fuse mount output: %s", output)
 		o.Expect(output).To(o.ContainSubstring("fuse"), "dev fuse is not mounted inside pod")
 	})
+
+	//author: bgudi@redhat.com
+	g.It("[OTP] add configurable terminationGracePeriodSeconds to liveness and startup probes [OCP-44493]", ote.Informing(), func() {
+		ctx := context.Background()
+		var err error
+
+		oc.SetupProject()
+		namespace := oc.Namespace()
+
+		// Helper to verify probe termination period by checking Events API
+		verifyProbeTermination := func(podName, containerName string, expectedTerminationSec int) error {
+			return wait.PollUntilContextTimeout(ctx, 10*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) {
+				// Get events using the Events API
+				events, err := oc.KubeClient().CoreV1().Events(namespace).List(ctx, metav1.ListOptions{
+					FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.kind=Pod", podName),
+				})
+				if err != nil {
+					e2e.Logf("Error getting events: %v", err)
+					return false, nil
+				}
+
+				// Look for probe failure (Killing) and container restart (Started) events
+				var killingEvent, startedEvent *corev1.Event
+				for i := range events.Items {
+					event := &events.Items[i]
+					if event.Reason == "Killing" && strings.Contains(event.Message, containerName) &&
+						strings.Contains(event.Message, "failed") && strings.Contains(event.Message, "probe") {
+						if killingEvent == nil || event.LastTimestamp.Time.After(killingEvent.LastTimestamp.Time) {
+							killingEvent = event
+						}
+					}
+					if event.Reason == "Started" && strings.Contains(event.Message, "Started container") {
+						// Find Started event after the Killing event
+						if killingEvent != nil && event.FirstTimestamp.Time.After(killingEvent.LastTimestamp.Time) {
+							if startedEvent == nil || event.FirstTimestamp.Time.Before(startedEvent.FirstTimestamp.Time) {
+								startedEvent = event
+							}
+						}
+					}
+				}
+
+				if killingEvent == nil || startedEvent == nil {
+					e2e.Logf("Waiting for probe failure (Killing) and container restart (Started) events")
+					return false, nil
+				}
+
+				e2e.Logf("Killing event: %s at %v", killingEvent.Message, killingEvent.LastTimestamp)
+				e2e.Logf("Started event: %s at %v", startedEvent.Message, startedEvent.FirstTimestamp)
+
+				// Calculate time difference in seconds
+				timeDiff := int(startedEvent.FirstTimestamp.Sub(killingEvent.LastTimestamp.Time).Seconds())
+				e2e.Logf("Time difference: %d seconds (expected: %d ±10 seconds)", timeDiff, expectedTerminationSec)
+
+				// Use gomega BeNumerically for range checking
+				o.Expect(timeDiff).To(o.BeNumerically(">=", expectedTerminationSec-3),
+					"Time difference is less than expected minimum")
+				o.Expect(timeDiff).To(o.BeNumerically("<=", expectedTerminationSec+10),
+					"Time difference is greater than expected maximum")
+
+				e2e.Logf("Termination grace period check passed")
+				return true, nil
+			})
+		}
+
+		g.By("Test liveness probe with probe-level terminationGracePeriodSeconds")
+		livenessPod := &corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "liveness-probe",
+				Namespace: namespace,
+			},
+			Spec: corev1.PodSpec{
+				TerminationGracePeriodSeconds: ptr.To[int64](60),
+				SecurityContext: &corev1.PodSecurityContext{
+					RunAsNonRoot: ptr.To(true),
+					SeccompProfile: &corev1.SeccompProfile{
+						Type: corev1.SeccompProfileTypeRuntimeDefault,
+					},
+				},
+				Containers: []corev1.Container{
+					{
+						Name:  "test",
+						Image: "quay.io/openshifttest/nginx-alpine@sha256:04f316442d48ba60e3ea0b5a67eb89b0b667abf1c198a3d0056ca748736336a0",
+						SecurityContext: &corev1.SecurityContext{
+							AllowPrivilegeEscalation: ptr.To(false),
+							Capabilities: &corev1.Capabilities{
+								Drop: []corev1.Capability{"ALL"},
+							},
+						},
+						Command: []string{"sh", "-c", "sleep 100000000"},
+						Ports: []corev1.ContainerPort{
+							{ContainerPort: 8080},
+						},
+						LivenessProbe: &corev1.Probe{
+							ProbeHandler: corev1.ProbeHandler{
+								HTTPGet: &corev1.HTTPGetAction{
+									Path: "/healthz",
+									Port: intstr.FromInt(8080),
+								},
+							},
+							FailureThreshold:              1,
+							PeriodSeconds:                 60,
+							TerminationGracePeriodSeconds: ptr.To[int64](10),
+						},
+					},
+				},
+			},
+		}
+
+		_, err = oc.KubeClient().CoreV1().Pods(namespace).Create(ctx, livenessPod, metav1.CreateOptions{})
+		o.Expect(err).NotTo(o.HaveOccurred(), "failed to create liveness probe pod")
+
+		err = verifyProbeTermination("liveness-probe", "test", 10)
+		o.Expect(err).NotTo(o.HaveOccurred(), "liveness probe termination grace period not honored")
+
+		g.By("Test startup probe with probe-level terminationGracePeriodSeconds")
+		startupPod := &corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "startup-probe",
+				Namespace: namespace,
+			},
+			Spec: corev1.PodSpec{
+				TerminationGracePeriodSeconds: ptr.To[int64](60),
+				SecurityContext: &corev1.PodSecurityContext{
+					RunAsNonRoot: ptr.To(true),
+					SeccompProfile: &corev1.SeccompProfile{
+						Type: corev1.SeccompProfileTypeRuntimeDefault,
+					},
+				},
+				Containers: []corev1.Container{
+					{
+						Name:  "teststartup",
+						Image: "quay.io/openshifttest/nginx-alpine@sha256:04f316442d48ba60e3ea0b5a67eb89b0b667abf1c198a3d0056ca748736336a0",
+						SecurityContext: &corev1.SecurityContext{
+							AllowPrivilegeEscalation: ptr.To(false),
+							Capabilities: &corev1.Capabilities{
+								Drop: []corev1.Capability{"ALL"},
+							},
+						},
+						Command: []string{"sh", "-c", "sleep 100000000"},
+						Ports: []corev1.ContainerPort{
+							{ContainerPort: 8080},
+						},
+						StartupProbe: &corev1.Probe{
+							ProbeHandler: corev1.ProbeHandler{
+								HTTPGet: &corev1.HTTPGetAction{
+									Path: "/healthz",
+									Port: intstr.FromInt(8080),
+								},
+							},
+							FailureThreshold:              1,
+							PeriodSeconds:                 60,
+							TerminationGracePeriodSeconds: ptr.To[int64](10),
+						},
+					},
+				},
+			},
+		}
+
+		_, err = oc.KubeClient().CoreV1().Pods(namespace).Create(ctx, startupPod, metav1.CreateOptions{})
+		o.Expect(err).NotTo(o.HaveOccurred(), "failed to create startup probe pod")
+
+		err = verifyProbeTermination("startup-probe", "teststartup", 10)
+		o.Expect(err).NotTo(o.HaveOccurred(), "startup probe termination grace period not honored")
+
+		g.By("Test liveness probe without probe-level terminationGracePeriodSeconds (should use pod-level)")
+		livenessPodNoProbeTerm := &corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "liveness-probe-no-term",
+				Namespace: namespace,
+			},
+			Spec: corev1.PodSpec{
+				TerminationGracePeriodSeconds: ptr.To[int64](60),
+				SecurityContext: &corev1.PodSecurityContext{
+					RunAsNonRoot: ptr.To(true),
+					SeccompProfile: &corev1.SeccompProfile{
+						Type: corev1.SeccompProfileTypeRuntimeDefault,
+					},
+				},
+				Containers: []corev1.Container{
+					{
+						Name:  "test",
+						Image: "quay.io/openshifttest/nginx-alpine@sha256:04f316442d48ba60e3ea0b5a67eb89b0b667abf1c198a3d0056ca748736336a0",
+						SecurityContext: &corev1.SecurityContext{
+							AllowPrivilegeEscalation: ptr.To(false),
+							Capabilities: &corev1.Capabilities{
+								Drop: []corev1.Capability{"ALL"},
+							},
+						},
+						Command: []string{"sh", "-c", "sleep 100000000"},
+						Ports: []corev1.ContainerPort{
+							{ContainerPort: 8080},
+						},
+						LivenessProbe: &corev1.Probe{
+							ProbeHandler: corev1.ProbeHandler{
+								HTTPGet: &corev1.HTTPGetAction{
+									Path: "/healthz",
+									Port: intstr.FromInt(8080),
+								},
+							},
+							FailureThreshold: 1,
+							PeriodSeconds:    60,
+							// No TerminationGracePeriodSeconds - should use pod-level (60s)
+						},
+					},
+				},
+			},
+		}
+
+		_, err = oc.KubeClient().CoreV1().Pods(namespace).Create(ctx, livenessPodNoProbeTerm, metav1.CreateOptions{})
+		o.Expect(err).NotTo(o.HaveOccurred(), "failed to create liveness probe pod without probe termination")
+
+		err = verifyProbeTermination("liveness-probe-no-term", "test", 60)
+		o.Expect(err).NotTo(o.HaveOccurred(), "liveness probe should use pod-level termination grace period when probe-level not set")
+	})
 })
 
 // author: asahay@redhat.com