Skip to content

Commit fa2d6a8

Browse files
BhargaviGudiBhargaviGudi
committed
Migrate OCP-44493: configurable terminationGracePeriod for liveness and startup probes
Migrates test from openshift-tests-private to origin. Test validates probe-level terminationGracePeriodSeconds for: - Liveness probes with probe-level terminationGracePeriodSeconds (10s) - Startup probes with probe-level terminationGracePeriodSeconds (10s) - Liveness probes without probe-level (falls back to pod-level 60s) The test creates pods with failing probes and verifies the time difference between probe failure (Killing event) and container restart (Started event) matches the expected termination grace period within acceptable range. Event matching logic parses 'oc describe pod' output for: - Killing events with container name - Started events after restart Updates: - Add test to test/extended/node/node_e2e/node.go - Document test in test/extended/node/README.md Relates: https://issues.redhat.com/browse/OCPBUGS-44493 Signed-off-by: Bhargavi Gudi <BhargaviGudi@users.noreply.github.com>
1 parent 8c50bc4 commit fa2d6a8

4 files changed

Lines changed: 317 additions & 0 deletions

File tree

‎test/extended/node/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ This directory contains OpenShift end-to-end tests for node-related features.
2020
- **image_volume.go** - Tests mounting container images as volumes in pods, including subPath and error handling
2121
- **node_swap.go** - Tests default kubelet swap settings (failSwapOn and swapBehavior) and rejection of user overrides
2222
- **zstd_chunked.go** - Tests building and running images with zstd:chunked compression format
23+
- **node_e2e/probe_termination.go** - Probe-level terminationGracePeriodSeconds (OCP-44493) - Tests configurable termination grace period for liveness and startup probes. Includes 3 test cases: probe-level config for liveness probe, probe-level config for startup probe, and fallback to pod-level config when probe-level is not set [Lifecycle:informing]
2324

2425
## Directory Structure
2526

‎test/extended/node/node_e2e/node.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,7 @@ var _ = g.Describe("[sig-node] [Jira:Node/Kubelet] Kubelet, CRI-O, CPU manager",
164164
e2e.Logf("/dev/fuse mount output: %s", output)
165165
o.Expect(output).To(o.ContainSubstring("fuse"), "dev fuse is not mounted inside pod")
166166
})
167+
167168
})
168169

169170
// author: asahay@redhat.com

‎test/extended/node/node_e2e/probe_termination.go‎

Lines changed: 300 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,300 @@
1+
package node
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"strings"
7+
"time"
8+
9+
g "github.com/onsi/ginkgo/v2"
10+
o "github.com/onsi/gomega"
11+
ote "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo"
12+
13+
corev1 "k8s.io/api/core/v1"
14+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
15+
"k8s.io/apimachinery/pkg/util/intstr"
16+
"k8s.io/apimachinery/pkg/util/wait"
17+
e2e "k8s.io/kubernetes/test/e2e/framework"
18+
"k8s.io/utils/ptr"
19+
20+
nodeutils "github.com/openshift/origin/test/extended/node"
21+
exutil "github.com/openshift/origin/test/extended/util"
22+
)
23+
24+
var _ = g.Describe("[sig-node] Probe configuration", func() {
25+
var (
26+
oc = exutil.NewCLIWithoutNamespace("probe-termination")
27+
)
28+
29+
//author: bgudi@redhat.com
30+
g.It("[OTP] Liveness probe should respect probe-level terminationGracePeriodSeconds [OCP-44493]", ote.Informing(), func() {
31+
ctx := context.Background()
32+
33+
oc.SetupProject()
34+
namespace := oc.Namespace()
35+
36+
g.By("Create pod with liveness probe having probe-level terminationGracePeriodSeconds=10s")
37+
pod := &corev1.Pod{
38+
ObjectMeta: metav1.ObjectMeta{
39+
Name: "liveness-probe-level",
40+
Namespace: namespace,
41+
},
42+
Spec: corev1.PodSpec{
43+
TerminationGracePeriodSeconds: ptr.To[int64](60),
44+
SecurityContext: &corev1.PodSecurityContext{
45+
RunAsNonRoot: ptr.To(true),
46+
SeccompProfile: &corev1.SeccompProfile{
47+
Type: corev1.SeccompProfileTypeRuntimeDefault,
48+
},
49+
},
50+
Containers: []corev1.Container{
51+
{
52+
Name: "test",
53+
Image: "quay.io/openshifttest/nginx-alpine@sha256:04f316442d48ba60e3ea0b5a67eb89b0b667abf1c198a3d0056ca748736336a0",
54+
SecurityContext: &corev1.SecurityContext{
55+
AllowPrivilegeEscalation: ptr.To(false),
56+
Capabilities: &corev1.Capabilities{
57+
Drop: []corev1.Capability{"ALL"},
58+
},
59+
},
60+
Command: []string{"sh", "-c", "sleep 100000000"},
61+
Ports: []corev1.ContainerPort{
62+
{ContainerPort: 8080},
63+
},
64+
LivenessProbe: &corev1.Probe{
65+
ProbeHandler: corev1.ProbeHandler{
66+
HTTPGet: &corev1.HTTPGetAction{
67+
Path: "/healthz",
68+
Port: intstr.FromInt(8080),
69+
},
70+
},
71+
FailureThreshold: 1,
72+
PeriodSeconds: 60,
73+
TerminationGracePeriodSeconds: ptr.To[int64](10),
74+
},
75+
},
76+
},
77+
},
78+
}
79+
80+
_, err := oc.KubeClient().CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{})
81+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to create liveness probe pod")
82+
83+
g.By("Verify probe-level terminationGracePeriodSeconds is honored (10s)")
84+
expectedSec := 10
85+
minSec := expectedSec - 3
86+
maxSec := expectedSec + 10
87+
timeDiff, err := verifyProbeTermination(ctx, oc, namespace, "liveness-probe-level", "test", expectedSec)
88+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to get probe termination events")
89+
o.Expect(timeDiff).To(o.BeNumerically(">=", minSec), fmt.Sprintf("time difference %ds is less than expected minimum %ds", timeDiff, minSec))
90+
o.Expect(timeDiff).To(o.BeNumerically("<=", maxSec), fmt.Sprintf("time difference %ds is greater than expected maximum %ds", timeDiff, maxSec))
91+
})
92+
93+
//author: bgudi@redhat.com
94+
g.It("[OTP] Startup probe should respect probe-level terminationGracePeriodSeconds [OCP-44493]", ote.Informing(), func() {
95+
ctx := context.Background()
96+
97+
oc.SetupProject()
98+
namespace := oc.Namespace()
99+
100+
g.By("Create pod with startup probe having probe-level terminationGracePeriodSeconds=10s")
101+
pod := &corev1.Pod{
102+
ObjectMeta: metav1.ObjectMeta{
103+
Name: "startup-probe-level",
104+
Namespace: namespace,
105+
},
106+
Spec: corev1.PodSpec{
107+
TerminationGracePeriodSeconds: ptr.To[int64](60),
108+
SecurityContext: &corev1.PodSecurityContext{
109+
RunAsNonRoot: ptr.To(true),
110+
SeccompProfile: &corev1.SeccompProfile{
111+
Type: corev1.SeccompProfileTypeRuntimeDefault,
112+
},
113+
},
114+
Containers: []corev1.Container{
115+
{
116+
Name: "teststartup",
117+
Image: "quay.io/openshifttest/nginx-alpine@sha256:04f316442d48ba60e3ea0b5a67eb89b0b667abf1c198a3d0056ca748736336a0",
118+
SecurityContext: &corev1.SecurityContext{
119+
AllowPrivilegeEscalation: ptr.To(false),
120+
Capabilities: &corev1.Capabilities{
121+
Drop: []corev1.Capability{"ALL"},
122+
},
123+
},
124+
Command: []string{"sh", "-c", "sleep 100000000"},
125+
Ports: []corev1.ContainerPort{
126+
{ContainerPort: 8080},
127+
},
128+
StartupProbe: &corev1.Probe{
129+
ProbeHandler: corev1.ProbeHandler{
130+
HTTPGet: &corev1.HTTPGetAction{
131+
Path: "/healthz",
132+
Port: intstr.FromInt(8080),
133+
},
134+
},
135+
FailureThreshold: 1,
136+
PeriodSeconds: 60,
137+
TerminationGracePeriodSeconds: ptr.To[int64](10),
138+
},
139+
},
140+
},
141+
},
142+
}
143+
144+
_, err := oc.KubeClient().CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{})
145+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to create startup probe pod")
146+
147+
g.By("Verify probe-level terminationGracePeriodSeconds is honored (10s)")
148+
expectedSec := 10
149+
minSec := expectedSec - 3
150+
maxSec := expectedSec + 10
151+
timeDiff, err := verifyProbeTermination(ctx, oc, namespace, "startup-probe-level", "teststartup", expectedSec)
152+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to get probe termination events")
153+
o.Expect(timeDiff).To(o.BeNumerically(">=", minSec), fmt.Sprintf("time difference %ds is less than expected minimum %ds", timeDiff, minSec))
154+
o.Expect(timeDiff).To(o.BeNumerically("<=", maxSec), fmt.Sprintf("time difference %ds is greater than expected maximum %ds", timeDiff, maxSec))
155+
})
156+
157+
//author: bgudi@redhat.com
158+
g.It("[OTP] Liveness probe should fall back to pod-level terminationGracePeriodSeconds when probe-level is not set [OCP-44493]", ote.Informing(), func() {
159+
ctx := context.Background()
160+
161+
oc.SetupProject()
162+
namespace := oc.Namespace()
163+
164+
g.By("Create pod with liveness probe without probe-level terminationGracePeriodSeconds")
165+
pod := &corev1.Pod{
166+
ObjectMeta: metav1.ObjectMeta{
167+
Name: "liveness-pod-level",
168+
Namespace: namespace,
169+
},
170+
Spec: corev1.PodSpec{
171+
TerminationGracePeriodSeconds: ptr.To[int64](60),
172+
SecurityContext: &corev1.PodSecurityContext{
173+
RunAsNonRoot: ptr.To(true),
174+
SeccompProfile: &corev1.SeccompProfile{
175+
Type: corev1.SeccompProfileTypeRuntimeDefault,
176+
},
177+
},
178+
Containers: []corev1.Container{
179+
{
180+
Name: "test",
181+
Image: "quay.io/openshifttest/nginx-alpine@sha256:04f316442d48ba60e3ea0b5a67eb89b0b667abf1c198a3d0056ca748736336a0",
182+
SecurityContext: &corev1.SecurityContext{
183+
AllowPrivilegeEscalation: ptr.To(false),
184+
Capabilities: &corev1.Capabilities{
185+
Drop: []corev1.Capability{"ALL"},
186+
},
187+
},
188+
Command: []string{"sh", "-c", "sleep 100000000"},
189+
Ports: []corev1.ContainerPort{
190+
{ContainerPort: 8080},
191+
},
192+
LivenessProbe: &corev1.Probe{
193+
ProbeHandler: corev1.ProbeHandler{
194+
HTTPGet: &corev1.HTTPGetAction{
195+
Path: "/healthz",
196+
Port: intstr.FromInt(8080),
197+
},
198+
},
199+
FailureThreshold: 1,
200+
PeriodSeconds: 60,
201+
// No TerminationGracePeriodSeconds - should use pod-level (60s)
202+
},
203+
},
204+
},
205+
},
206+
}
207+
208+
_, err := oc.KubeClient().CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{})
209+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to create liveness probe pod without probe-level termination")
210+
211+
g.By("Verify pod-level terminationGracePeriodSeconds is used (60s)")
212+
expectedSec := 60
213+
minSec := expectedSec - 3
214+
maxSec := expectedSec + 10
215+
timeDiff, err := verifyProbeTermination(ctx, oc, namespace, "liveness-pod-level", "test", expectedSec)
216+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to get probe termination events")
217+
o.Expect(timeDiff).To(o.BeNumerically(">=", minSec), fmt.Sprintf("time difference %ds is less than expected minimum %ds", timeDiff, minSec))
218+
o.Expect(timeDiff).To(o.BeNumerically("<=", maxSec), fmt.Sprintf("time difference %ds is greater than expected maximum %ds", timeDiff, maxSec))
219+
})
220+
})
221+
222+
// findLatestEventByReason finds the latest event matching the given reason and message filter
223+
func findLatestEventByReason(events *corev1.EventList, reason string, msgFilter func(string) bool) *corev1.Event {
224+
var latestEvent *corev1.Event
225+
for i := range events.Items {
226+
event := &events.Items[i]
227+
if event.Reason == reason && msgFilter(event.Message) {
228+
if latestEvent == nil || event.LastTimestamp.Time.After(latestEvent.LastTimestamp.Time) {
229+
latestEvent = event
230+
}
231+
}
232+
}
233+
return latestEvent
234+
}
235+
236+
// findEarliestEventAfter finds the earliest event matching the reason and filter that occurred after the given time
237+
func findEarliestEventAfter(events *corev1.EventList, reason string, msgFilter func(string) bool, afterTime time.Time) *corev1.Event {
238+
var earliestEvent *corev1.Event
239+
for i := range events.Items {
240+
event := &events.Items[i]
241+
if event.Reason == reason && msgFilter(event.Message) && event.FirstTimestamp.Time.After(afterTime) {
242+
if earliestEvent == nil || event.FirstTimestamp.Time.Before(earliestEvent.FirstTimestamp.Time) {
243+
earliestEvent = event
244+
}
245+
}
246+
}
247+
return earliestEvent
248+
}
249+
250+
// verifyProbeTermination verifies that the probe termination grace period is honored
251+
// by checking the time difference between probe failure (Killing) and container restart (Started) events
252+
// Returns the time difference in seconds, or an error if events are not found
253+
func verifyProbeTermination(ctx context.Context, oc *exutil.CLI, namespace, podName, containerName string, expectedTerminationSec int) (int, error) {
254+
var timeDiff int
255+
err := wait.PollUntilContextTimeout(ctx, 10*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) {
256+
// Get events using the Events API
257+
events, err := oc.KubeClient().CoreV1().Events(namespace).List(ctx, metav1.ListOptions{
258+
FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.kind=Pod", podName),
259+
})
260+
if err != nil {
261+
e2e.Logf("Error getting events: %v", err)
262+
return false, nil
263+
}
264+
265+
// Find probe failure (Killing) event for the container
266+
killingEvent := findLatestEventByReason(events, "Killing", func(msg string) bool {
267+
return strings.Contains(msg, containerName) &&
268+
strings.Contains(msg, "failed") &&
269+
strings.Contains(msg, "probe")
270+
})
271+
272+
if killingEvent == nil {
273+
e2e.Logf("Waiting for probe failure (Killing) event")
274+
return false, nil
275+
}
276+
277+
// Find container restart (Started) event that occurred after the Killing event
278+
startedEvent := findEarliestEventAfter(events, "Started", func(msg string) bool {
279+
return strings.Contains(msg, "Started container")
280+
}, killingEvent.LastTimestamp.Time)
281+
282+
if startedEvent == nil {
283+
e2e.Logf("Waiting for container restart (Started) event after Killing event")
284+
return false, nil
285+
}
286+
287+
e2e.Logf("Killing event: %s at %v", killingEvent.Message, killingEvent.LastTimestamp)
288+
e2e.Logf("Started event: %s at %v", startedEvent.Message, startedEvent.FirstTimestamp)
289+
290+
// Calculate time difference using the helper function
291+
timeDiff = int(nodeutils.CalculateEventTimeDiff(killingEvent, startedEvent).Seconds())
292+
e2e.Logf("Time difference: %d seconds (expected: %d ±10 seconds)", timeDiff, expectedTerminationSec)
293+
294+
return true, nil
295+
})
296+
if err != nil {
297+
return 0, err
298+
}
299+
return timeDiff, nil
300+
}

‎test/extended/node/node_utils.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -764,3 +764,18 @@ func GetFirstReadyWorkerNode(oc *exutil.CLI) string {
764764
o.Expect(false).To(o.BeTrue(), "no Ready worker node found among %v", workers)
765765
return "" // unreachable; satisfies compiler
766766
}
767+
768+
// CalculateEventTimeDiff calculates the time difference between two Kubernetes events.
769+
// It uses LastTimestamp for the start event and FirstTimestamp for the end event.
770+
// Falls back to FirstTimestamp/LastTimestamp respectively if the primary timestamp is zero.
771+
func CalculateEventTimeDiff(startEvent, endEvent *corev1.Event) time.Duration {
772+
startTime := startEvent.LastTimestamp.Time
773+
if startTime.IsZero() {
774+
startTime = startEvent.FirstTimestamp.Time
775+
}
776+
endTime := endEvent.FirstTimestamp.Time
777+
if endTime.IsZero() {
778+
endTime = endEvent.LastTimestamp.Time
779+
}
780+
return endTime.Sub(startTime)
781+
}

0 commit comments

Comments
 (0)