Skip to content

Commit b3b98a0

Browse files
openshift-merge-bot[bot]openshift-merge-bot[bot]
authored
Merge pull request #31170 from BhargaviGudi/migrate-44493
OCPNODE-4529: Migrate test case 44493 - configurable terminationGracePeriodSeconds for probes
2 parents c7deaba + 1c1f28d commit b3b98a0

3 files changed

Lines changed: 302 additions & 0 deletions

File tree

‎test/extended/node/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ This directory contains OpenShift end-to-end tests for node-related features.
2222
- **image_volume.go** - Tests mounting container images as volumes in pods, including subPath and error handling
2323
- **node_swap.go** - Tests default kubelet swap settings (failSwapOn and swapBehavior) and rejection of user overrides
2424
- **zstd_chunked.go** - Tests building and running images with zstd:chunked compression format
25+
- **node_e2e/probe_termination.go** - Probe-level terminationGracePeriodSeconds (OCP-44493) - Tests configurable termination grace period for liveness and startup probes. Includes 3 test cases: probe-level config for liveness probe, probe-level config for startup probe, and fallback to pod-level config when probe-level is not set [Lifecycle:informing]
2526

2627
## Directory Structure
2728

‎test/extended/node/node_e2e/probe_termination.go‎

Lines changed: 284 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,284 @@
1+
package node
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"strings"
7+
"time"
8+
9+
g "github.com/onsi/ginkgo/v2"
10+
o "github.com/onsi/gomega"
11+
ote "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo"
12+
13+
corev1 "k8s.io/api/core/v1"
14+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
15+
"k8s.io/apimachinery/pkg/util/intstr"
16+
"k8s.io/apimachinery/pkg/util/wait"
17+
e2e "k8s.io/kubernetes/test/e2e/framework"
18+
"k8s.io/utils/ptr"
19+
20+
nodeutils "github.com/openshift/origin/test/extended/node"
21+
exutil "github.com/openshift/origin/test/extended/util"
22+
"github.com/openshift/origin/test/extended/util/image"
23+
)
24+
25+
var _ = g.Describe("[sig-node] Probe configuration", func() {
26+
var (
27+
oc = exutil.NewCLIWithoutNamespace("probe-termination")
28+
)
29+
30+
g.BeforeEach(func() {
31+
isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient())
32+
o.Expect(err).NotTo(o.HaveOccurred())
33+
if isMicroShift {
34+
g.Skip("Skipping test on MicroShift cluster")
35+
}
36+
})
37+
38+
//author: bgudi@redhat.com
39+
g.It("[OTP] Liveness probe should respect probe-level terminationGracePeriodSeconds [OCP-44493]", ote.Informing(), func() {
40+
ctx := context.Background()
41+
42+
oc.SetupProject()
43+
namespace := oc.Namespace()
44+
45+
g.By("Create pod with liveness probe having probe-level terminationGracePeriodSeconds=10s")
46+
pod := &corev1.Pod{
47+
ObjectMeta: metav1.ObjectMeta{
48+
Name: "liveness-probe-level",
49+
Namespace: namespace,
50+
},
51+
Spec: corev1.PodSpec{
52+
TerminationGracePeriodSeconds: ptr.To[int64](60),
53+
Containers: []corev1.Container{
54+
{
55+
Name: "test",
56+
Image: image.ShellImage(),
57+
Command: []string{"sh", "-c", "sleep 100000000"},
58+
Ports: []corev1.ContainerPort{
59+
{ContainerPort: 8080},
60+
},
61+
LivenessProbe: &corev1.Probe{
62+
ProbeHandler: corev1.ProbeHandler{
63+
HTTPGet: &corev1.HTTPGetAction{
64+
Path: "/healthz",
65+
Port: intstr.FromInt(8080),
66+
},
67+
},
68+
InitialDelaySeconds: 5,
69+
FailureThreshold: 1,
70+
PeriodSeconds: 60,
71+
TerminationGracePeriodSeconds: ptr.To[int64](10),
72+
},
73+
},
74+
},
75+
},
76+
}
77+
78+
_, err := oc.KubeClient().CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{})
79+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to create liveness probe pod")
80+
81+
g.By("Verify probe-level terminationGracePeriodSeconds is honored (10s)")
82+
expectedSec := 10
83+
// Allow asymmetric tolerance: -3s for event timing precision, +10s for container cleanup overhead
84+
minSec := expectedSec - 3
85+
maxSec := expectedSec + 10
86+
timeDiff, err := verifyProbeTermination(ctx, oc, namespace, "liveness-probe-level", "test", expectedSec)
87+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to get probe termination events")
88+
o.Expect(timeDiff).To(o.BeNumerically(">=", minSec), fmt.Sprintf("time difference %ds is less than expected minimum %ds", timeDiff, minSec))
89+
o.Expect(timeDiff).To(o.BeNumerically("<=", maxSec), fmt.Sprintf("time difference %ds is greater than expected maximum %ds", timeDiff, maxSec))
90+
})
91+
92+
//author: bgudi@redhat.com
93+
g.It("[OTP] Startup probe should respect probe-level terminationGracePeriodSeconds [OCP-44493]", ote.Informing(), func() {
94+
ctx := context.Background()
95+
96+
oc.SetupProject()
97+
namespace := oc.Namespace()
98+
99+
g.By("Create pod with startup probe having probe-level terminationGracePeriodSeconds=10s")
100+
pod := &corev1.Pod{
101+
ObjectMeta: metav1.ObjectMeta{
102+
Name: "startup-probe-level",
103+
Namespace: namespace,
104+
},
105+
Spec: corev1.PodSpec{
106+
TerminationGracePeriodSeconds: ptr.To[int64](60),
107+
Containers: []corev1.Container{
108+
{
109+
Name: "teststartup",
110+
Image: image.ShellImage(),
111+
Command: []string{"sh", "-c", "sleep 100000000"},
112+
Ports: []corev1.ContainerPort{
113+
{ContainerPort: 8080},
114+
},
115+
StartupProbe: &corev1.Probe{
116+
ProbeHandler: corev1.ProbeHandler{
117+
HTTPGet: &corev1.HTTPGetAction{
118+
Path: "/healthz",
119+
Port: intstr.FromInt(8080),
120+
},
121+
},
122+
InitialDelaySeconds: 5,
123+
FailureThreshold: 1,
124+
PeriodSeconds: 60,
125+
TerminationGracePeriodSeconds: ptr.To[int64](10),
126+
},
127+
},
128+
},
129+
},
130+
}
131+
132+
_, err := oc.KubeClient().CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{})
133+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to create startup probe pod")
134+
135+
g.By("Verify probe-level terminationGracePeriodSeconds is honored (10s)")
136+
expectedSec := 10
137+
// Allow asymmetric tolerance: -3s for event timing precision, +10s for container cleanup overhead
138+
minSec := expectedSec - 3
139+
maxSec := expectedSec + 10
140+
timeDiff, err := verifyProbeTermination(ctx, oc, namespace, "startup-probe-level", "teststartup", expectedSec)
141+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to get probe termination events")
142+
o.Expect(timeDiff).To(o.BeNumerically(">=", minSec), fmt.Sprintf("time difference %ds is less than expected minimum %ds", timeDiff, minSec))
143+
o.Expect(timeDiff).To(o.BeNumerically("<=", maxSec), fmt.Sprintf("time difference %ds is greater than expected maximum %ds", timeDiff, maxSec))
144+
})
145+
146+
//author: bgudi@redhat.com
147+
g.It("[OTP] Liveness probe should fall back to pod-level terminationGracePeriodSeconds when probe-level is not set [OCP-44493]", ote.Informing(), func() {
148+
ctx := context.Background()
149+
150+
oc.SetupProject()
151+
namespace := oc.Namespace()
152+
153+
g.By("Create pod with liveness probe without probe-level terminationGracePeriodSeconds")
154+
pod := &corev1.Pod{
155+
ObjectMeta: metav1.ObjectMeta{
156+
Name: "liveness-pod-level",
157+
Namespace: namespace,
158+
},
159+
Spec: corev1.PodSpec{
160+
TerminationGracePeriodSeconds: ptr.To[int64](60),
161+
Containers: []corev1.Container{
162+
{
163+
Name: "test",
164+
Image: image.ShellImage(),
165+
Command: []string{"sh", "-c", "sleep 100000000"},
166+
Ports: []corev1.ContainerPort{
167+
{ContainerPort: 8080},
168+
},
169+
LivenessProbe: &corev1.Probe{
170+
ProbeHandler: corev1.ProbeHandler{
171+
HTTPGet: &corev1.HTTPGetAction{
172+
Path: "/healthz",
173+
Port: intstr.FromInt(8080),
174+
},
175+
},
176+
InitialDelaySeconds: 5,
177+
FailureThreshold: 1,
178+
PeriodSeconds: 60,
179+
// No TerminationGracePeriodSeconds - should use pod-level (60s)
180+
},
181+
},
182+
},
183+
},
184+
}
185+
186+
_, err := oc.KubeClient().CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{})
187+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to create liveness probe pod without probe-level termination")
188+
189+
g.By("Verify pod-level terminationGracePeriodSeconds is used (60s)")
190+
expectedSec := 60
191+
// Allow asymmetric tolerance: -3s for event timing precision, +10s for container cleanup overhead
192+
minSec := expectedSec - 3
193+
maxSec := expectedSec + 10
194+
timeDiff, err := verifyProbeTermination(ctx, oc, namespace, "liveness-pod-level", "test", expectedSec)
195+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to get probe termination events")
196+
o.Expect(timeDiff).To(o.BeNumerically(">=", minSec), fmt.Sprintf("time difference %ds is less than expected minimum %ds", timeDiff, minSec))
197+
o.Expect(timeDiff).To(o.BeNumerically("<=", maxSec), fmt.Sprintf("time difference %ds is greater than expected maximum %ds", timeDiff, maxSec))
198+
})
199+
})
200+
201+
// findLatestEventByReason finds the latest event matching the given reason and message filter
202+
func findLatestEventByReason(events *corev1.EventList, reason string, msgFilter func(string) bool) *corev1.Event {
203+
var latestEvent *corev1.Event
204+
for i := range events.Items {
205+
event := &events.Items[i]
206+
if event.Reason == reason && msgFilter(event.Message) {
207+
if latestEvent == nil || event.LastTimestamp.Time.After(latestEvent.LastTimestamp.Time) {
208+
latestEvent = event
209+
}
210+
}
211+
}
212+
return latestEvent
213+
}
214+
215+
// findEarliestEventAfter finds the earliest event matching the reason and filter that occurred after the given time
216+
// Uses LastTimestamp to properly detect repeated events (container restarts)
217+
func findEarliestEventAfter(events *corev1.EventList, reason string, msgFilter func(string) bool, afterTime time.Time) *corev1.Event {
218+
var earliestEvent *corev1.Event
219+
for i := range events.Items {
220+
event := &events.Items[i]
221+
// Use LastTimestamp instead of FirstTimestamp to catch repeated events (e.g., container restarts)
222+
// FirstTimestamp stays at the original event time, but LastTimestamp updates on each occurrence
223+
if event.Reason == reason && msgFilter(event.Message) && event.LastTimestamp.Time.After(afterTime) {
224+
if earliestEvent == nil || event.LastTimestamp.Time.Before(earliestEvent.LastTimestamp.Time) {
225+
earliestEvent = event
226+
}
227+
}
228+
}
229+
return earliestEvent
230+
}
231+
232+
// verifyProbeTermination verifies that the probe termination grace period is honored
233+
// by checking the time difference between probe failure (Killing) and container restart (Started) events
234+
// Returns the time difference in seconds, or an error if events are not found
235+
func verifyProbeTermination(ctx context.Context, oc *exutil.CLI, namespace, podName, containerName string, expectedTerminationSec int) (int, error) {
236+
var timeDiff int
237+
// Timeout needs to account for: pod start (~30s) + probe period (60s) + termination (up to 60s) + restart (~30s) = ~3 minutes minimum
238+
// Use 6 minutes to be safe for tests with 60s termination grace period
239+
err := wait.PollUntilContextTimeout(ctx, 10*time.Second, 6*time.Minute, true, func(ctx context.Context) (bool, error) {
240+
// Get events using the Events API
241+
events, err := oc.KubeClient().CoreV1().Events(namespace).List(ctx, metav1.ListOptions{
242+
FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.kind=Pod", podName),
243+
})
244+
if err != nil {
245+
e2e.Logf("Error getting events: %v", err)
246+
return false, nil
247+
}
248+
249+
// Find probe failure (Killing) event for the container
250+
killingEvent := findLatestEventByReason(events, "Killing", func(msg string) bool {
251+
return strings.Contains(msg, containerName) &&
252+
strings.Contains(msg, "failed") &&
253+
strings.Contains(msg, "probe")
254+
})
255+
256+
if killingEvent == nil {
257+
e2e.Logf("Waiting for probe failure (Killing) event")
258+
return false, nil
259+
}
260+
261+
// Find container restart (Started) event that occurred after the Killing event
262+
startedEvent := findEarliestEventAfter(events, "Started", func(msg string) bool {
263+
return strings.Contains(msg, "Container started")
264+
}, killingEvent.LastTimestamp.Time)
265+
266+
if startedEvent == nil {
267+
e2e.Logf("Waiting for container restart (Started) event after Killing event")
268+
return false, nil
269+
}
270+
271+
e2e.Logf("Killing event: %s at %v", killingEvent.Message, killingEvent.LastTimestamp)
272+
e2e.Logf("Started event: %s at %v", startedEvent.Message, startedEvent.LastTimestamp)
273+
274+
// Calculate time difference using the helper function
275+
timeDiff = int(nodeutils.CalculateEventTimeDiff(killingEvent, startedEvent).Seconds())
276+
e2e.Logf("Time difference: %d seconds (expected: %d ±10 seconds)", timeDiff, expectedTerminationSec)
277+
278+
return true, nil
279+
})
280+
if err != nil {
281+
return 0, err
282+
}
283+
return timeDiff, nil
284+
}

‎test/extended/node/node_utils.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -764,3 +764,20 @@ func GetFirstReadyWorkerNode(oc *exutil.CLI) string {
764764
o.Expect(false).To(o.BeTrue(), "no Ready worker node found among %v", workers)
765765
return "" // unreachable; satisfies compiler
766766
}
767+
768+
// CalculateEventTimeDiff calculates the time difference between two Kubernetes events.
769+
// It uses LastTimestamp for both events to handle repeated events correctly.
770+
// For repeated events (like container restarts), Kubernetes reuses the event object and updates
771+
// LastTimestamp while keeping FirstTimestamp at the original occurrence.
772+
// Falls back to FirstTimestamp if LastTimestamp is zero.
773+
func CalculateEventTimeDiff(startEvent, endEvent *corev1.Event) time.Duration {
774+
startTime := startEvent.LastTimestamp.Time
775+
if startTime.IsZero() {
776+
startTime = startEvent.FirstTimestamp.Time
777+
}
778+
endTime := endEvent.LastTimestamp.Time
779+
if endTime.IsZero() {
780+
endTime = endEvent.FirstTimestamp.Time
781+
}
782+
return endTime.Sub(startTime)
783+
}

0 commit comments

Comments
 (0)