Skip to content

Commit 3426c8b

Browse files
mdboothmdbooth
committed
Ignore NodeReady=False during a CNI rollout
A CNI rollout is expected to cause each Node to become briefly not Ready while the ovn-kubernetes pod is replaced. This change adds an additional exclusion to the node watch test for NodeReady=False when: * the network ClusterOperator reports Progressing=True, and * the message contains NetworkPluginNotReady reported by cri-o
1 parent 135cf91 commit 3426c8b

2 files changed

Lines changed: 150 additions & 3 deletions

File tree

‎pkg/monitortests/node/watchnodes/node.go‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,7 @@ func startNodeMonitoring(ctx context.Context, m monitorapi.RecorderWriter, clien
154154
// We want to fail the monitor test if a node goes not ready
155155
// if it is unexpected.
156156
// Unexpected in this case means that it went not ready outside
157-
// of a MCO config update.
157+
// of a MCO config update or CNI rollout.
158158
func(node, oldNode *corev1.Node) []monitorapi.Interval {
159159
var intervals []monitorapi.Interval
160160

@@ -167,11 +167,16 @@ func startNodeMonitoring(ctx context.Context, m monitorapi.RecorderWriter, clien
167167

168168
now := time.Now()
169169
if isOldNodeReady && !isNewNodeReady && isConfigTheSame && !isNodeUnscheduable {
170+
msg := monitorapi.NewMessage().Reason(monitorapi.NodeUnexpectedReadyReason).
171+
HumanMessage("unexpected node not ready")
172+
// Extract the NodeReady condition message for downstream filtering
173+
if c := findNodeCondition(node.Status.Conditions, corev1.NodeReady, 0); c != nil && c.Message != "" {
174+
msg = msg.WithAnnotation(monitorapi.AnnotationCause, c.Message)
175+
}
170176
intervals = append(intervals,
171177
monitorapi.NewInterval(monitorapi.SourceUnexpectedReady, monitorapi.Error).
172178
Locator(monitorapi.NewLocator().NodeFromName(node.Name)).
173-
Message(monitorapi.NewMessage().Reason(monitorapi.NodeUnexpectedReadyReason).
174-
HumanMessage("unexpected node not ready")).
179+
Message(msg).
175180
Display().
176181
Build(now, now))
177182
}
@@ -370,6 +375,13 @@ func reportUnexpectedNodeDownFailures(intervals monitorapi.Intervals, targetedRe
370375
return false
371376
})
372377

378+
// Get all network ClusterOperator Progressing=True intervals
379+
networkProgressingIntervals := intervals.Filter(func(eventInterval monitorapi.Interval) bool {
380+
return eventInterval.Locator.Keys[monitorapi.LocatorClusterOperatorKey] == "network" &&
381+
eventInterval.Message.Annotations[monitorapi.AnnotationCondition] == "Progressing" &&
382+
eventInterval.Message.Annotations[monitorapi.AnnotationStatus] == "True"
383+
})
384+
373385
// We need to build a map of node to machine name
374386
nodeNameToMachineName := map[string]string{}
375387
// Given the deleted machine, store the deleted intervals.
@@ -394,6 +406,15 @@ func reportUnexpectedNodeDownFailures(intervals monitorapi.Intervals, targetedRe
394406
machineDeletingIntervals := machineNameToDeletePhases[machineNameForNode]
395407

396408
if !intervalStartDuring(unexpectedNodeUnready, machineDeletingIntervals) {
409+
// Skip NotReady events caused by NetworkPluginNotReady during network operator rollout
410+
// NetworkPluginNotReady is a RuntimeStatus reported by cri-o and exposed by kubelet in the condition's message.
411+
conditionMsg := unexpectedNodeUnready.Message.Annotations[monitorapi.AnnotationCause]
412+
if strings.Contains(conditionMsg, "NetworkPluginNotReady") {
413+
if intervalStartDuring(unexpectedNodeUnready, networkProgressingIntervals) {
414+
continue
415+
}
416+
}
417+
397418
failures = append(failures, fmt.Sprintf("%v - %v at from: %v - to: %v", unexpectedNodeUnready.Locator.OldLocator(), unexpectedNodeUnready.Message.OldMessage(), unexpectedNodeUnready.From, unexpectedNodeUnready.To))
398419
}
399420
}

‎pkg/monitortests/node/watchnodes/node_test.go‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -321,6 +321,132 @@ func TestReportUnexpectedNodeDownFailures(t *testing.T) {
321321
expected: []string{},
322322
unexpectedReason: monitorapi.NodeUnexpectedUnreachableReason,
323323
},
324+
{
325+
name: "node unexpected ready caused by NetworkPluginNotReady during network operator rollout",
326+
rawIntervals: monitorapi.Intervals{
327+
// The UnexpectedNotReady interval with NetworkPluginNotReady in AnnotationCause
328+
{
329+
Condition: monitorapi.Condition{
330+
Level: monitorapi.Error,
331+
Locator: monitorapi.Locator{
332+
Type: monitorapi.LocatorTypeNode,
333+
Keys: map[monitorapi.LocatorKey]string{
334+
"node": "node1",
335+
},
336+
},
337+
Message: monitorapi.Message{
338+
Reason: monitorapi.NodeUnexpectedReadyReason,
339+
HumanMessage: "unexpected node not ready",
340+
Annotations: map[monitorapi.AnnotationKey]string{
341+
monitorapi.AnnotationReason: "UnexpectedNotReady",
342+
monitorapi.AnnotationCause: "container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?",
343+
},
344+
},
345+
},
346+
From: utility.SystemdJournalLogTime("Nov 11 19:46:00", 2024),
347+
To: utility.SystemdJournalLogTime("Nov 11 19:46:00", 2024),
348+
},
349+
// The network CO Progressing=True interval that brackets it
350+
{
351+
Condition: monitorapi.Condition{
352+
Level: monitorapi.Warning,
353+
Locator: monitorapi.Locator{
354+
Type: monitorapi.LocatorTypeClusterOperator,
355+
Keys: map[monitorapi.LocatorKey]string{
356+
monitorapi.LocatorClusterOperatorKey: "network",
357+
},
358+
},
359+
Message: monitorapi.Message{
360+
HumanMessage: "Progressing because OVN is updating",
361+
Annotations: map[monitorapi.AnnotationKey]string{
362+
monitorapi.AnnotationCondition: "Progressing",
363+
monitorapi.AnnotationStatus: "True",
364+
},
365+
},
366+
},
367+
From: utility.SystemdJournalLogTime("Nov 11 19:45:00", 2024),
368+
To: utility.SystemdJournalLogTime("Nov 11 19:47:00", 2024),
369+
},
370+
},
371+
expected: []string{},
372+
unexpectedReason: monitorapi.NodeUnexpectedReadyReason,
373+
},
374+
{
375+
name: "node unexpected ready caused by NetworkPluginNotReady WITHOUT network operator rollout",
376+
rawIntervals: monitorapi.Intervals{
377+
{
378+
Condition: monitorapi.Condition{
379+
Level: monitorapi.Error,
380+
Locator: monitorapi.Locator{
381+
Type: monitorapi.LocatorTypeNode,
382+
Keys: map[monitorapi.LocatorKey]string{
383+
"node": "node1",
384+
},
385+
},
386+
Message: monitorapi.Message{
387+
Reason: monitorapi.NodeUnexpectedReadyReason,
388+
HumanMessage: "unexpected node not ready",
389+
Annotations: map[monitorapi.AnnotationKey]string{
390+
monitorapi.AnnotationReason: "UnexpectedNotReady",
391+
monitorapi.AnnotationCause: "container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?",
392+
},
393+
},
394+
},
395+
From: utility.SystemdJournalLogTime("Nov 11 19:46:00", 2024),
396+
To: utility.SystemdJournalLogTime("Nov 11 19:46:00", 2024),
397+
},
398+
},
399+
expected: []string{"node/node1 - cause/container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started? reason/UnexpectedNotReady unexpected node not ready at from: 2024-11-11 19:46:00 +0000 UTC - to: 2024-11-11 19:46:00 +0000 UTC"},
400+
unexpectedReason: monitorapi.NodeUnexpectedReadyReason,
401+
},
402+
{
403+
name: "node unexpected ready NOT caused by NetworkPluginNotReady during network operator rollout",
404+
rawIntervals: monitorapi.Intervals{
405+
{
406+
Condition: monitorapi.Condition{
407+
Level: monitorapi.Error,
408+
Locator: monitorapi.Locator{
409+
Type: monitorapi.LocatorTypeNode,
410+
Keys: map[monitorapi.LocatorKey]string{
411+
"node": "node1",
412+
},
413+
},
414+
Message: monitorapi.Message{
415+
Reason: monitorapi.NodeUnexpectedReadyReason,
416+
HumanMessage: "unexpected node not ready",
417+
Annotations: map[monitorapi.AnnotationKey]string{
418+
monitorapi.AnnotationReason: "UnexpectedNotReady",
419+
monitorapi.AnnotationCause: "kubelet stopped posting node status",
420+
},
421+
},
422+
},
423+
From: utility.SystemdJournalLogTime("Nov 11 19:46:00", 2024),
424+
To: utility.SystemdJournalLogTime("Nov 11 19:46:00", 2024),
425+
},
426+
{
427+
Condition: monitorapi.Condition{
428+
Level: monitorapi.Warning,
429+
Locator: monitorapi.Locator{
430+
Type: monitorapi.LocatorTypeClusterOperator,
431+
Keys: map[monitorapi.LocatorKey]string{
432+
monitorapi.LocatorClusterOperatorKey: "network",
433+
},
434+
},
435+
Message: monitorapi.Message{
436+
HumanMessage: "Progressing because OVN is updating",
437+
Annotations: map[monitorapi.AnnotationKey]string{
438+
monitorapi.AnnotationCondition: "Progressing",
439+
monitorapi.AnnotationStatus: "True",
440+
},
441+
},
442+
},
443+
From: utility.SystemdJournalLogTime("Nov 11 19:45:00", 2024),
444+
To: utility.SystemdJournalLogTime("Nov 11 19:47:00", 2024),
445+
},
446+
},
447+
expected: []string{"node/node1 - cause/kubelet stopped posting node status reason/UnexpectedNotReady unexpected node not ready at from: 2024-11-11 19:46:00 +0000 UTC - to: 2024-11-11 19:46:00 +0000 UTC"},
448+
unexpectedReason: monitorapi.NodeUnexpectedReadyReason,
449+
},
324450
}
325451

326452
for _, tc := range testCases {

0 commit comments

Comments
 (0)