Skip to content

Commit 0a5d587

Browse files
committed
feat(karpenter): add envtest and e2e tests for kubelet config
Add envtest suites for CEL validation rules including threshold ordering, field promotion ratcheting, and generated ratcheting tests. Add e2e test infrastructure for verifying kubelet config on nodes. Signed-off-by: John Kyros <jkyros@redhat.com>
1 parent 5fefe11 commit 0a5d587

11 files changed

Lines changed: 1426 additions & 24 deletions

File tree

Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,7 @@ karpenter-api: $(CONTROLLER_GEN) $(YQ)
200200
karpenter-operator/hack/adjust-cel.sh
201201
$(CONTROLLER_GEN) $(CRD_OPTIONS) paths="./api/karpenter/..." output:crd:artifacts:config=karpenter-operator/controllers/karpenter/assets
202202
cp karpenter-operator/controllers/karpenter/assets/karpenter.hypershift.openshift.io_openshiftec2nodeclasses.yaml karpenter-operator/controllers/karpenter/assets/zz_generated.crd-manifests/openshiftec2nodeclasses.crd.yaml
203+
$(GO) run ./hack/kubelet-ratcheting-gen/main.go
203204

204205
.PHONY: control-plane-operator
205206
control-plane-operator:

api/karpenter/v1/kubelet_config.go

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ func init() {
4242
// +kubebuilder:validation:XValidation:rule="!has(self.podsPerCore) || !has(self.maxPods) || self.podsPerCore <= self.maxPods",message="podsPerCore must not exceed maxPods"
4343
// +kubebuilder:validation:XValidation:rule="!has(self.evictionSoft) || (has(self.evictionSoftGracePeriod) && self.evictionSoft.all(e, e in self.evictionSoftGracePeriod))",message="evictionSoft entry does not have a matching evictionSoftGracePeriod"
4444
// +kubebuilder:validation:XValidation:rule="!has(self.evictionSoftGracePeriod) || (has(self.evictionSoft) && self.evictionSoftGracePeriod.all(e, e in self.evictionSoft))",message="evictionSoftGracePeriod entry does not have a matching evictionSoft"
45-
// +kubebuilder:validation:XValidation:rule="!has(self.evictionHard) || !has(self.evictionSoft) || self.evictionHard.all(key, !(key in self.evictionSoft) || (self.evictionSoft[key].endsWith('%') && self.evictionHard[key].endsWith('%')) ? (self.evictionSoft[key].size() <= 1 || self.evictionHard[key].size() <= 1 || double(self.evictionSoft[key].substring(0, self.evictionSoft[key].size() - 1)) >= double(self.evictionHard[key].substring(0, self.evictionHard[key].size() - 1))) : (!(isQuantity(self.evictionSoft[key]) && isQuantity(self.evictionHard[key])) || quantity(self.evictionSoft[key]).compareTo(quantity(self.evictionHard[key])) >= 0))",message="evictionSoft threshold must be greater than or equal to evictionHard threshold for the same signal (soft eviction should fire before hard)"
4645
type KubeletConfiguration struct {
4746
// maxPods is the maximum number of pods that can run on a node.
4847
// The value must be between 1 and 2500, inclusive.
Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
// kubelet-ratcheting-gen generates the envtest ratcheting testsuite for KubeletConfiguration.
2+
// It reflects over KubeletConfiguration to enumerate all typed fields, verifies each has a
3+
// fixture value, and writes a testsuite YAML that simulates a pre-upgrade → post-upgrade
4+
// CRD schema swap. Run via: go run ./hack/kubelet-ratcheting-gen/main.go (from repo root).
5+
//
6+
// When adding a new typed field to KubeletConfiguration:
7+
// 1. Add a fixture value to the fixtures map below.
8+
// 2. Run `make karpenter-api` to regenerate the testsuite.
9+
// 3. Commit the updated testsuite YAML.
10+
package main
11+
12+
import (
13+
"fmt"
14+
"os"
15+
"reflect"
16+
"strings"
17+
"text/template"
18+
19+
karpenterv1 "github.com/openshift/hypershift/api/karpenter/v1"
20+
)
21+
22+
const outputPath = "karpenter-operator/controllers/karpenter/assets/tests/" +
23+
"openshiftec2nodeclasses.karpenter.hypershift.openshift.io/" +
24+
"stable.openshiftec2nodeclasses.kubelet-ratcheting.testsuite.yaml"
25+
26+
// fixtures maps each KubeletConfiguration JSON field name to a valid YAML snippet.
27+
// Values must satisfy all CRD validation rules (min/max, CEL cross-field constraints).
28+
// Add an entry here when promoting a new field from overflow to typed.
29+
var fixtures = map[string]string{
30+
"maxPods": "110",
31+
"podsPerCore": "10",
32+
"systemReserved": "cpu: \"100m\"\n memory: \"256Mi\"",
33+
"kubeReserved": "cpu: \"200m\"\n memory: \"512Mi\"",
34+
"evictionHard": "memory.available: \"100Mi\"\n nodefs.available: \"10%\"",
35+
"evictionSoft": "memory.available: \"200Mi\"",
36+
"evictionSoftGracePeriod": "memory.available: \"30s\"",
37+
"evictionMaxPodGracePeriod": "60",
38+
"imageGCHighThresholdPercent": "85",
39+
"imageGCLowThresholdPercent": "80",
40+
"cpuCFSQuota": "true",
41+
}
42+
43+
// mapFields are fields whose fixture value is a YAML mapping (rendered with extra indentation).
44+
var mapFields = map[string]bool{
45+
"systemReserved": true,
46+
"kubeReserved": true,
47+
"evictionHard": true,
48+
"evictionSoft": true,
49+
"evictionSoftGracePeriod": true,
50+
}
51+
52+
const testsuiteTmpl = `# Code generated by hack/kubelet-ratcheting-gen. DO NOT EDIT.
53+
# To regenerate: make karpenter-api
54+
# To add a new field: add a fixture in hack/kubelet-ratcheting-gen/main.go, then regenerate.
55+
apiVersion: apiextensions.k8s.io/v1
56+
name: "OpenshiftEC2NodeClass kubelet ratcheting validation"
57+
crdName: openshiftec2nodeclasses.karpenter.hypershift.openshift.io
58+
version: v1
59+
# Verifies that all currently-typed kubelet fields survive a CRD schema upgrade.
60+
# initialCRDPatches strip the entire kubelet typed-field schema back to a pure
61+
# "type: object, x-kubernetes-preserve-unknown-fields: true" — the state that existed
62+
# before any fields were promoted from overflow. The initial object is created with all
63+
# typed fields set (they land in overflow under the patched schema). The patches are then
64+
# reverted and the update is applied, verifying every field survives the upgrade seamlessly.
65+
tests:
66+
onUpdate:
67+
- name: When all typed kubelet fields were set before they were typed they should remain valid after CRD upgrade
68+
initialCRDPatches:
69+
# Strip all typed field schemas, leaving type: object, x-kubernetes-preserve-unknown-fields: true.
70+
- op: remove
71+
path: /spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/kubelet/properties
72+
# Strip all CEL cross-field validation rules.
73+
- op: remove
74+
path: /spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/kubelet/x-kubernetes-validations
75+
initial: |
76+
apiVersion: karpenter.hypershift.openshift.io/v1
77+
kind: OpenshiftEC2NodeClass
78+
spec:
79+
subnetSelectorTerms:
80+
- tags:
81+
env: test
82+
securityGroupSelectorTerms:
83+
- id: sg-0123456789abcdef0
84+
kubelet:
85+
{{- range .Fields}}
86+
{{.Name}}:{{if .IsMap}}
87+
{{.Value}}{{else}} {{.Value}}{{end}}
88+
{{- end}}
89+
updated: |
90+
apiVersion: karpenter.hypershift.openshift.io/v1
91+
kind: OpenshiftEC2NodeClass
92+
spec:
93+
subnetSelectorTerms:
94+
- tags:
95+
env: test
96+
securityGroupSelectorTerms:
97+
- id: sg-0123456789abcdef0
98+
kubelet:
99+
{{- range .Fields}}
100+
{{.Name}}:{{if .IsMap}}
101+
{{.Value}}{{else}} {{.Value}}{{end}}
102+
{{- end}}
103+
`
104+
105+
type field struct {
106+
Name string
107+
Value string
108+
IsMap bool
109+
}
110+
111+
func main() {
112+
if err := run(); err != nil {
113+
fmt.Fprintf(os.Stderr, "error: %v\n", err)
114+
os.Exit(1)
115+
}
116+
}
117+
118+
func run() error {
119+
fields, err := collectFields()
120+
if err != nil {
121+
return err
122+
}
123+
124+
tmpl, err := template.New("testsuite").Parse(testsuiteTmpl)
125+
if err != nil {
126+
return fmt.Errorf("parsing template: %w", err)
127+
}
128+
129+
f, err := os.Create(outputPath)
130+
if err != nil {
131+
return fmt.Errorf("creating output file %s: %w", outputPath, err)
132+
}
133+
defer f.Close()
134+
135+
if err := tmpl.Execute(f, struct{ Fields []field }{Fields: fields}); err != nil {
136+
return fmt.Errorf("executing template: %w", err)
137+
}
138+
139+
fmt.Printf("generated %s (%d fields)\n", outputPath, len(fields))
140+
return nil
141+
}
142+
143+
// collectFields reflects over KubeletConfiguration, enumerates typed fields by JSON tag,
144+
// and looks up each one in the fixtures map. Missing fixtures cause a hard failure.
145+
func collectFields() ([]field, error) {
146+
t := reflect.TypeOf(karpenterv1.KubeletConfiguration{})
147+
var fields []field
148+
var missing []string
149+
150+
for i := range t.NumField() {
151+
f := t.Field(i)
152+
tag := f.Tag.Get("json")
153+
if tag == "" || tag == "-" {
154+
continue
155+
}
156+
name, _, _ := strings.Cut(tag, ",")
157+
if name == "" || name == "-" {
158+
continue
159+
}
160+
161+
val, ok := fixtures[name]
162+
if !ok {
163+
missing = append(missing, fmt.Sprintf("%s (json:%q)", f.Name, name))
164+
continue
165+
}
166+
fields = append(fields, field{Name: name, Value: val, IsMap: mapFields[name]})
167+
}
168+
169+
if len(missing) > 0 {
170+
return nil, fmt.Errorf(
171+
"no fixture value for the following KubeletConfiguration fields — "+
172+
"add them to the fixtures map in hack/kubelet-ratcheting-gen/main.go:\n %s",
173+
strings.Join(missing, "\n "),
174+
)
175+
}
176+
177+
return fields, nil
178+
}

karpenter-operator/controllers/karpenter/assets/karpenter.hypershift.openshift.io_openshiftec2nodeclasses.yaml

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -396,17 +396,6 @@ spec:
396396
evictionSoft
397397
rule: '!has(self.evictionSoftGracePeriod) || (has(self.evictionSoft)
398398
&& self.evictionSoftGracePeriod.all(e, e in self.evictionSoft))'
399-
- message: evictionSoft threshold must be greater than or equal to
400-
evictionHard threshold for the same signal (soft eviction should
401-
fire before hard)
402-
rule: '!has(self.evictionHard) || !has(self.evictionSoft) || self.evictionHard.all(key,
403-
!(key in self.evictionSoft) || (self.evictionSoft[key].endsWith(''%'')
404-
&& self.evictionHard[key].endsWith(''%'')) ? (self.evictionSoft[key].size()
405-
<= 1 || self.evictionHard[key].size() <= 1 || double(self.evictionSoft[key].substring(0,
406-
self.evictionSoft[key].size() - 1)) >= double(self.evictionHard[key].substring(0,
407-
self.evictionHard[key].size() - 1))) : (!(isQuantity(self.evictionSoft[key])
408-
&& isQuantity(self.evictionHard[key])) || quantity(self.evictionSoft[key]).compareTo(quantity(self.evictionHard[key]))
409-
>= 0))'
410399
metadataOptions:
411400
description: |-
412401
metadataOptions contains parameters for specifying the exposure of the

0 commit comments

Comments
 (0)