Skip to content

Commit 588d9fb

Browse files
BhargaviGudiBhargaviGudi
committed
Migrate OCP-44493: configurable terminationGracePeriod for liveness and startup probes
Migrates test from openshift-tests-private to origin. Test validates probe-level terminationGracePeriodSeconds for: - Liveness probes with probe-level terminationGracePeriodSeconds (10s) - Startup probes with probe-level terminationGracePeriodSeconds (10s) - Liveness probes without probe-level (falls back to pod-level 60s) The test creates pods with failing probes and verifies the time difference between probe failure (Killing event) and container restart (Started event) matches the expected termination grace period within acceptable range. Event matching logic parses 'oc describe pod' output for: - Killing events with container name - Started events after restart Updates: - Add test to test/extended/node/node_e2e/node.go - Document test in test/extended/node/README.md Relates: https://issues.redhat.com/browse/OCPBUGS-44493 Signed-off-by: Bhargavi Gudi <BhargaviGudi@users.noreply.github.com>
1 parent 8c50bc4 commit 588d9fb

3 files changed

Lines changed: 334 additions & 0 deletions

File tree

‎test/extended/node/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ This directory contains OpenShift end-to-end tests for node-related features.
2020
- **image_volume.go** - Tests mounting container images as volumes in pods, including subPath and error handling
2121
- **node_swap.go** - Tests default kubelet swap settings (failSwapOn and swapBehavior) and rejection of user overrides
2222
- **zstd_chunked.go** - Tests building and running images with zstd:chunked compression format
23+
- **node_e2e/probe_termination.go** - Probe-level terminationGracePeriodSeconds (OCP-44493) - Tests configurable termination grace period for liveness and startup probes. Includes 3 test cases: probe-level config for liveness probe, probe-level config for startup probe, and fallback to pod-level config when probe-level is not set [Lifecycle:informing]
2324

2425
## Directory Structure
2526

‎test/extended/node/node_e2e/probe_termination.go‎

Lines changed: 316 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,316 @@
1+
package node
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"strings"
7+
"time"
8+
9+
g "github.com/onsi/ginkgo/v2"
10+
o "github.com/onsi/gomega"
11+
ote "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo"
12+
13+
corev1 "k8s.io/api/core/v1"
14+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
15+
"k8s.io/apimachinery/pkg/util/intstr"
16+
"k8s.io/apimachinery/pkg/util/wait"
17+
e2e "k8s.io/kubernetes/test/e2e/framework"
18+
"k8s.io/utils/ptr"
19+
20+
nodeutils "github.com/openshift/origin/test/extended/node"
21+
exutil "github.com/openshift/origin/test/extended/util"
22+
)
23+
24+
var _ = g.Describe("[sig-node] Probe configuration", func() {
25+
var (
26+
oc = exutil.NewCLIWithoutNamespace("probe-termination")
27+
)
28+
29+
g.BeforeEach(func() {
30+
isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient())
31+
o.Expect(err).NotTo(o.HaveOccurred())
32+
if isMicroShift {
33+
g.Skip("Skipping test on MicroShift cluster")
34+
}
35+
})
36+
37+
//author: bgudi@redhat.com
38+
g.It("[OTP] Liveness probe should respect probe-level terminationGracePeriodSeconds [OCP-44493]", ote.Informing(), func() {
39+
ctx := context.Background()
40+
41+
oc.SetupProject()
42+
namespace := oc.Namespace()
43+
44+
g.By("Create pod with liveness probe having probe-level terminationGracePeriodSeconds=10s")
45+
pod := &corev1.Pod{
46+
ObjectMeta: metav1.ObjectMeta{
47+
Name: "liveness-probe-level",
48+
Namespace: namespace,
49+
},
50+
Spec: corev1.PodSpec{
51+
TerminationGracePeriodSeconds: ptr.To[int64](60),
52+
SecurityContext: &corev1.PodSecurityContext{
53+
RunAsNonRoot: ptr.To(true),
54+
SeccompProfile: &corev1.SeccompProfile{
55+
Type: corev1.SeccompProfileTypeRuntimeDefault,
56+
},
57+
},
58+
Containers: []corev1.Container{
59+
{
60+
Name: "test",
61+
Image: "quay.io/openshifttest/nginx-alpine@sha256:04f316442d48ba60e3ea0b5a67eb89b0b667abf1c198a3d0056ca748736336a0",
62+
SecurityContext: &corev1.SecurityContext{
63+
AllowPrivilegeEscalation: ptr.To(false),
64+
Capabilities: &corev1.Capabilities{
65+
Drop: []corev1.Capability{"ALL"},
66+
},
67+
},
68+
Command: []string{"sh", "-c", "sleep 100000000"},
69+
Ports: []corev1.ContainerPort{
70+
{ContainerPort: 8080},
71+
},
72+
LivenessProbe: &corev1.Probe{
73+
ProbeHandler: corev1.ProbeHandler{
74+
HTTPGet: &corev1.HTTPGetAction{
75+
Path: "/healthz",
76+
Port: intstr.FromInt(8080),
77+
},
78+
},
79+
InitialDelaySeconds: 5,
80+
FailureThreshold: 1,
81+
PeriodSeconds: 60,
82+
TerminationGracePeriodSeconds: ptr.To[int64](10),
83+
},
84+
},
85+
},
86+
},
87+
}
88+
89+
_, err := oc.KubeClient().CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{})
90+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to create liveness probe pod")
91+
92+
g.By("Verify probe-level terminationGracePeriodSeconds is honored (10s)")
93+
expectedSec := 10
94+
minSec := expectedSec - 3
95+
maxSec := expectedSec + 10
96+
timeDiff, err := verifyProbeTermination(ctx, oc, namespace, "liveness-probe-level", "test", expectedSec)
97+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to get probe termination events")
98+
o.Expect(timeDiff).To(o.BeNumerically(">=", minSec), fmt.Sprintf("time difference %ds is less than expected minimum %ds", timeDiff, minSec))
99+
o.Expect(timeDiff).To(o.BeNumerically("<=", maxSec), fmt.Sprintf("time difference %ds is greater than expected maximum %ds", timeDiff, maxSec))
100+
})
101+
102+
//author: bgudi@redhat.com
103+
g.It("[OTP] Startup probe should respect probe-level terminationGracePeriodSeconds [OCP-44493]", ote.Informing(), func() {
104+
ctx := context.Background()
105+
106+
oc.SetupProject()
107+
namespace := oc.Namespace()
108+
109+
g.By("Create pod with startup probe having probe-level terminationGracePeriodSeconds=10s")
110+
pod := &corev1.Pod{
111+
ObjectMeta: metav1.ObjectMeta{
112+
Name: "startup-probe-level",
113+
Namespace: namespace,
114+
},
115+
Spec: corev1.PodSpec{
116+
TerminationGracePeriodSeconds: ptr.To[int64](60),
117+
SecurityContext: &corev1.PodSecurityContext{
118+
RunAsNonRoot: ptr.To(true),
119+
SeccompProfile: &corev1.SeccompProfile{
120+
Type: corev1.SeccompProfileTypeRuntimeDefault,
121+
},
122+
},
123+
Containers: []corev1.Container{
124+
{
125+
Name: "teststartup",
126+
Image: "quay.io/openshifttest/nginx-alpine@sha256:04f316442d48ba60e3ea0b5a67eb89b0b667abf1c198a3d0056ca748736336a0",
127+
SecurityContext: &corev1.SecurityContext{
128+
AllowPrivilegeEscalation: ptr.To(false),
129+
Capabilities: &corev1.Capabilities{
130+
Drop: []corev1.Capability{"ALL"},
131+
},
132+
},
133+
Command: []string{"sh", "-c", "sleep 100000000"},
134+
Ports: []corev1.ContainerPort{
135+
{ContainerPort: 8080},
136+
},
137+
StartupProbe: &corev1.Probe{
138+
ProbeHandler: corev1.ProbeHandler{
139+
HTTPGet: &corev1.HTTPGetAction{
140+
Path: "/healthz",
141+
Port: intstr.FromInt(8080),
142+
},
143+
},
144+
InitialDelaySeconds: 5,
145+
FailureThreshold: 1,
146+
PeriodSeconds: 60,
147+
TerminationGracePeriodSeconds: ptr.To[int64](10),
148+
},
149+
},
150+
},
151+
},
152+
}
153+
154+
_, err := oc.KubeClient().CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{})
155+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to create startup probe pod")
156+
157+
g.By("Verify probe-level terminationGracePeriodSeconds is honored (10s)")
158+
expectedSec := 10
159+
minSec := expectedSec - 3
160+
maxSec := expectedSec + 10
161+
timeDiff, err := verifyProbeTermination(ctx, oc, namespace, "startup-probe-level", "teststartup", expectedSec)
162+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to get probe termination events")
163+
o.Expect(timeDiff).To(o.BeNumerically(">=", minSec), fmt.Sprintf("time difference %ds is less than expected minimum %ds", timeDiff, minSec))
164+
o.Expect(timeDiff).To(o.BeNumerically("<=", maxSec), fmt.Sprintf("time difference %ds is greater than expected maximum %ds", timeDiff, maxSec))
165+
})
166+
167+
//author: bgudi@redhat.com
168+
g.It("[OTP] Liveness probe should fall back to pod-level terminationGracePeriodSeconds when probe-level is not set [OCP-44493]", ote.Informing(), func() {
169+
ctx := context.Background()
170+
171+
oc.SetupProject()
172+
namespace := oc.Namespace()
173+
174+
g.By("Create pod with liveness probe without probe-level terminationGracePeriodSeconds")
175+
pod := &corev1.Pod{
176+
ObjectMeta: metav1.ObjectMeta{
177+
Name: "liveness-pod-level",
178+
Namespace: namespace,
179+
},
180+
Spec: corev1.PodSpec{
181+
TerminationGracePeriodSeconds: ptr.To[int64](60),
182+
SecurityContext: &corev1.PodSecurityContext{
183+
RunAsNonRoot: ptr.To(true),
184+
SeccompProfile: &corev1.SeccompProfile{
185+
Type: corev1.SeccompProfileTypeRuntimeDefault,
186+
},
187+
},
188+
Containers: []corev1.Container{
189+
{
190+
Name: "test",
191+
Image: "quay.io/openshifttest/nginx-alpine@sha256:04f316442d48ba60e3ea0b5a67eb89b0b667abf1c198a3d0056ca748736336a0",
192+
SecurityContext: &corev1.SecurityContext{
193+
AllowPrivilegeEscalation: ptr.To(false),
194+
Capabilities: &corev1.Capabilities{
195+
Drop: []corev1.Capability{"ALL"},
196+
},
197+
},
198+
Command: []string{"sh", "-c", "sleep 100000000"},
199+
Ports: []corev1.ContainerPort{
200+
{ContainerPort: 8080},
201+
},
202+
LivenessProbe: &corev1.Probe{
203+
ProbeHandler: corev1.ProbeHandler{
204+
HTTPGet: &corev1.HTTPGetAction{
205+
Path: "/healthz",
206+
Port: intstr.FromInt(8080),
207+
},
208+
},
209+
InitialDelaySeconds: 5,
210+
FailureThreshold: 1,
211+
PeriodSeconds: 60,
212+
// No TerminationGracePeriodSeconds - should use pod-level (60s)
213+
},
214+
},
215+
},
216+
},
217+
}
218+
219+
_, err := oc.KubeClient().CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{})
220+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to create liveness probe pod without probe-level termination")
221+
222+
g.By("Verify pod-level terminationGracePeriodSeconds is used (60s)")
223+
expectedSec := 60
224+
minSec := expectedSec - 3
225+
maxSec := expectedSec + 10
226+
timeDiff, err := verifyProbeTermination(ctx, oc, namespace, "liveness-pod-level", "test", expectedSec)
227+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to get probe termination events")
228+
o.Expect(timeDiff).To(o.BeNumerically(">=", minSec), fmt.Sprintf("time difference %ds is less than expected minimum %ds", timeDiff, minSec))
229+
o.Expect(timeDiff).To(o.BeNumerically("<=", maxSec), fmt.Sprintf("time difference %ds is greater than expected maximum %ds", timeDiff, maxSec))
230+
})
231+
})
232+
233+
// findLatestEventByReason finds the latest event matching the given reason and message filter
234+
func findLatestEventByReason(events *corev1.EventList, reason string, msgFilter func(string) bool) *corev1.Event {
235+
var latestEvent *corev1.Event
236+
for i := range events.Items {
237+
event := &events.Items[i]
238+
if event.Reason == reason && msgFilter(event.Message) {
239+
if latestEvent == nil || event.LastTimestamp.Time.After(latestEvent.LastTimestamp.Time) {
240+
latestEvent = event
241+
}
242+
}
243+
}
244+
return latestEvent
245+
}
246+
247+
// findEarliestEventAfter finds the earliest event matching the reason and filter that occurred after the given time
248+
// Uses LastTimestamp to properly detect repeated events (container restarts)
249+
func findEarliestEventAfter(events *corev1.EventList, reason string, msgFilter func(string) bool, afterTime time.Time) *corev1.Event {
250+
var earliestEvent *corev1.Event
251+
for i := range events.Items {
252+
event := &events.Items[i]
253+
// Use LastTimestamp instead of FirstTimestamp to catch repeated events (e.g., container restarts)
254+
// FirstTimestamp stays at the original event time, but LastTimestamp updates on each occurrence
255+
if event.Reason == reason && msgFilter(event.Message) && event.LastTimestamp.Time.After(afterTime) {
256+
if earliestEvent == nil || event.LastTimestamp.Time.Before(earliestEvent.LastTimestamp.Time) {
257+
earliestEvent = event
258+
}
259+
}
260+
}
261+
return earliestEvent
262+
}
263+
264+
// verifyProbeTermination verifies that the probe termination grace period is honored
265+
// by checking the time difference between probe failure (Killing) and container restart (Started) events
266+
// Returns the time difference in seconds, or an error if events are not found
267+
func verifyProbeTermination(ctx context.Context, oc *exutil.CLI, namespace, podName, containerName string, expectedTerminationSec int) (int, error) {
268+
var timeDiff int
269+
// Timeout needs to account for: pod start (~30s) + probe period (60s) + termination (up to 60s) + restart (~30s) = ~3 minutes minimum
270+
// Use 6 minutes to be safe for tests with 60s termination grace period
271+
err := wait.PollUntilContextTimeout(ctx, 10*time.Second, 6*time.Minute, true, func(ctx context.Context) (bool, error) {
272+
// Get events using the Events API
273+
events, err := oc.KubeClient().CoreV1().Events(namespace).List(ctx, metav1.ListOptions{
274+
FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.kind=Pod", podName),
275+
})
276+
if err != nil {
277+
e2e.Logf("Error getting events: %v", err)
278+
return false, nil
279+
}
280+
281+
// Find probe failure (Killing) event for the container
282+
killingEvent := findLatestEventByReason(events, "Killing", func(msg string) bool {
283+
return strings.Contains(msg, containerName) &&
284+
strings.Contains(msg, "failed") &&
285+
strings.Contains(msg, "probe")
286+
})
287+
288+
if killingEvent == nil {
289+
e2e.Logf("Waiting for probe failure (Killing) event")
290+
return false, nil
291+
}
292+
293+
// Find container restart (Started) event that occurred after the Killing event
294+
startedEvent := findEarliestEventAfter(events, "Started", func(msg string) bool {
295+
return strings.Contains(msg, "Container started")
296+
}, killingEvent.LastTimestamp.Time)
297+
298+
if startedEvent == nil {
299+
e2e.Logf("Waiting for container restart (Started) event after Killing event")
300+
return false, nil
301+
}
302+
303+
e2e.Logf("Killing event: %s at %v", killingEvent.Message, killingEvent.LastTimestamp)
304+
e2e.Logf("Started event: %s at %v", startedEvent.Message, startedEvent.LastTimestamp)
305+
306+
// Calculate time difference using the helper function
307+
timeDiff = int(nodeutils.CalculateEventTimeDiff(killingEvent, startedEvent).Seconds())
308+
e2e.Logf("Time difference: %d seconds (expected: %d ±10 seconds)", timeDiff, expectedTerminationSec)
309+
310+
return true, nil
311+
})
312+
if err != nil {
313+
return 0, err
314+
}
315+
return timeDiff, nil
316+
}

‎test/extended/node/node_utils.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -764,3 +764,20 @@ func GetFirstReadyWorkerNode(oc *exutil.CLI) string {
764764
o.Expect(false).To(o.BeTrue(), "no Ready worker node found among %v", workers)
765765
return "" // unreachable; satisfies compiler
766766
}
767+
768+
// CalculateEventTimeDiff calculates the time difference between two Kubernetes events.
769+
// It uses LastTimestamp for both events to handle repeated events correctly.
770+
// For repeated events (like container restarts), Kubernetes reuses the event object and updates
771+
// LastTimestamp while keeping FirstTimestamp at the original occurrence.
772+
// Falls back to FirstTimestamp if LastTimestamp is zero.
773+
func CalculateEventTimeDiff(startEvent, endEvent *corev1.Event) time.Duration {
774+
startTime := startEvent.LastTimestamp.Time
775+
if startTime.IsZero() {
776+
startTime = startEvent.FirstTimestamp.Time
777+
}
778+
endTime := endEvent.LastTimestamp.Time
779+
if endTime.IsZero() {
780+
endTime = endEvent.FirstTimestamp.Time
781+
}
782+
return endTime.Sub(startTime)
783+
}

0 commit comments

Comments
 (0)