Skip to content

Commit 4dc1948

Browse files
jlebonjlebon
committed
tests: switch to custom FCOS image with CRI-O via coreos-assembler
Build a custom FCOS qcow2 with CRI-O and Kubernetes packages baked in using coreos-assembler. This replaces the vanilla FCOS + containerd approach, eliminating the SELinux permissive workaround (CRI-O has native SELinux support and properly labels privileged containers). The test VM image build is now a multi-stage Containerfile: 1. Build a bootable container (FROM fcos, add K8s + CRI-O repos, dnf install) via inner podman build 2. Use cosa import + cosa osbuild qemu to produce the qcow2 3. Copy the qcow2 into the final test harness image The config.bu is simplified: no more K8s binary downloads, containerd config, CNI plugins install, or SELinux permissive hack. Also fix daemon DaemonSet ServiceAccount name mismatch with kustomize namePrefix by making it configurable via DAEMON_SERVICE_ACCOUNT env var, and removing the operator-managed ServiceAccount (now deployed by kustomize). Fix DAEMON_SERVICE_ACCOUNT substitution in build-installer. Assisted-by: OpenCode (Claude Opus 4.6)
1 parent eb33283 commit 4dc1948

12 files changed

Lines changed: 125 additions & 152 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,4 @@
55
# kubebuilder build artifacts
66
/bin/
77
/cover.out
8+
/dist/

‎Makefile‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,12 @@ build-installer: manifests generate kustomize ## Generate a consolidated YAML wi
171171
cd config/manager && "$(KUSTOMIZE)" edit set image controller=${IMG}
172172
@sed -i 's|value: daemon:latest|value: $(DAEMON_IMG)|' config/manager/manager.yaml
173173
"$(KUSTOMIZE)" build config/default > dist/install.yaml
174+
@# Kustomize applies namePrefix to resource names (including the
175+
@# ServiceAccount) but not to env var values. Patch the generated
176+
@# install.yaml so DAEMON_SERVICE_ACCOUNT matches the actual SA name.
177+
@grep -q 'namePrefix' config/default/kustomization.yaml && \
178+
prefix=$$(grep 'namePrefix:' config/default/kustomization.yaml | awk '{print $$2}') && \
179+
sed -i "s|value: bootc-daemon|value: $${prefix}bootc-daemon|" dist/install.yaml || true
174180
@sed -i 's|value: $(DAEMON_IMG)|value: daemon:latest|' config/manager/manager.yaml
175181

176182
##@ Deployment

‎cmd/operator/main.go‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,11 @@ func main() {
202202
os.Exit(1)
203203
}
204204

205+
daemonServiceAccount := os.Getenv("DAEMON_SERVICE_ACCOUNT")
206+
if daemonServiceAccount == "" {
207+
daemonServiceAccount = "bootc-daemon"
208+
}
209+
205210
if err := (&controller.BootcNodePoolReconciler{
206211
Client: mgr.GetClient(),
207212
Scheme: mgr.GetScheme(),
@@ -218,10 +223,11 @@ func main() {
218223
// The reconciler ensures the DaemonSet exists with the correct
219224
// image and watches for drift.
220225
dsReconciler := &controller.DaemonSetReconciler{
221-
Client: mgr.GetClient(),
222-
Scheme: mgr.GetScheme(),
223-
Namespace: operatorNamespace,
224-
Image: daemonImage,
226+
Client: mgr.GetClient(),
227+
Scheme: mgr.GetScheme(),
228+
Namespace: operatorNamespace,
229+
Image: daemonImage,
230+
ServiceAccount: daemonServiceAccount,
225231
}
226232
if err := dsReconciler.SetupWithManager(mgr); err != nil {
227233
setupLog.Error(err, "Failed to create controller", "controller", "DaemonSet")

‎config/manager/manager.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ spec:
6868
env:
6969
- name: DAEMON_IMAGE
7070
value: daemon:latest
71+
- name: DAEMON_SERVICE_ACCOUNT
72+
value: bootc-daemon
7173
- name: OPERATOR_NAMESPACE
7274
valueFrom:
7375
fieldRef:

‎config/rbac/role.yaml‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -36,17 +36,6 @@ rules:
3636
- get
3737
- list
3838
- watch
39-
- apiGroups:
40-
- ""
41-
resources:
42-
- serviceaccounts
43-
verbs:
44-
- create
45-
- get
46-
- list
47-
- patch
48-
- update
49-
- watch
5039
- apiGroups:
5140
- ""
5241
- events.k8s.io

‎internal/controller/daemonset.go‎

Lines changed: 5 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -52,13 +52,13 @@ const (
5252
// The DaemonSet is a cluster-singleton owned by the operator.
5353
type DaemonSetReconciler struct {
5454
client.Client
55-
Scheme *runtime.Scheme
56-
Namespace string
57-
Image string
55+
Scheme *runtime.Scheme
56+
Namespace string
57+
Image string
58+
ServiceAccount string
5859
}
5960

6061
// +kubebuilder:rbac:groups=apps,resources=daemonsets,verbs=get;list;watch;create;update;patch;delete
61-
// +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update;patch
6262

6363
// Reconcile ensures the daemon DaemonSet exists and has the correct
6464
// image. It creates the DaemonSet if it does not exist, and updates
@@ -71,11 +71,6 @@ func (r *DaemonSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
7171
return ctrl.Result{}, nil
7272
}
7373

74-
// Ensure the ServiceAccount exists.
75-
if err := r.ensureServiceAccount(ctx); err != nil {
76-
return ctrl.Result{}, fmt.Errorf("ensuring ServiceAccount: %w", err)
77-
}
78-
7974
// Check if the DaemonSet already exists.
8075
existing := &appsv1.DaemonSet{}
8176
err := r.Get(ctx, types.NamespacedName{Name: daemonSetName, Namespace: r.Namespace}, existing)
@@ -105,11 +100,6 @@ func (r *DaemonSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
105100
func (r *DaemonSetReconciler) EnsureDaemonSet(ctx context.Context) error {
106101
log := logf.FromContext(ctx).WithName("daemonset")
107102

108-
// Ensure the ServiceAccount exists first.
109-
if err := r.ensureServiceAccount(ctx); err != nil {
110-
return fmt.Errorf("ensuring ServiceAccount: %w", err)
111-
}
112-
113103
existing := &appsv1.DaemonSet{}
114104
err := r.Get(ctx, types.NamespacedName{Name: daemonSetName, Namespace: r.Namespace}, existing)
115105
if errors.IsNotFound(err) {
@@ -128,24 +118,6 @@ func (r *DaemonSetReconciler) EnsureDaemonSet(ctx context.Context) error {
128118
return r.updateDaemonSetIfNeeded(ctx, existing)
129119
}
130120

131-
// ensureServiceAccount creates the daemon ServiceAccount if it does
132-
// not exist.
133-
func (r *DaemonSetReconciler) ensureServiceAccount(ctx context.Context) error {
134-
sa := &corev1.ServiceAccount{}
135-
err := r.Get(ctx, types.NamespacedName{Name: daemonSetName, Namespace: r.Namespace}, sa)
136-
if errors.IsNotFound(err) {
137-
sa = &corev1.ServiceAccount{
138-
ObjectMeta: metav1.ObjectMeta{
139-
Name: daemonSetName,
140-
Namespace: r.Namespace,
141-
Labels: daemonLabels(),
142-
},
143-
}
144-
return r.Create(ctx, sa)
145-
}
146-
return err
147-
}
148-
149121
// updateDaemonSetIfNeeded updates the DaemonSet's daemon container
150122
// image if it differs from the desired image.
151123
func (r *DaemonSetReconciler) updateDaemonSetIfNeeded(ctx context.Context, existing *appsv1.DaemonSet) error {
@@ -190,7 +162,7 @@ func (r *DaemonSetReconciler) buildDaemonSet() *appsv1.DaemonSet {
190162
},
191163
},
192164
Spec: corev1.PodSpec{
193-
ServiceAccountName: daemonSetName,
165+
ServiceAccountName: r.ServiceAccount,
194166
// Run on all nodes except those with the skip label.
195167
Affinity: &corev1.Affinity{
196168
NodeAffinity: &corev1.NodeAffinity{

‎internal/controller/daemonset_test.go‎

Lines changed: 7 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,11 @@ var _ = Describe("DaemonSet Reconciler", func() {
4343
BeforeEach(func() {
4444
ctx = context.Background()
4545
reconciler = &DaemonSetReconciler{
46-
Client: k8sClient,
47-
Scheme: k8sClient.Scheme(),
48-
Namespace: testNamespace,
49-
Image: testDaemonImg,
46+
Client: k8sClient,
47+
Scheme: k8sClient.Scheme(),
48+
Namespace: testNamespace,
49+
Image: testDaemonImg,
50+
ServiceAccount: daemonSetName,
5051
}
5152
})
5253

@@ -61,15 +62,6 @@ var _ = Describe("DaemonSet Reconciler", func() {
6162
_ = k8sClient.Delete(ctx, ds)
6263
}
6364

64-
// Clean up the ServiceAccount if it exists.
65-
sa := &corev1.ServiceAccount{}
66-
err = k8sClient.Get(ctx, types.NamespacedName{
67-
Name: daemonSetName,
68-
Namespace: testNamespace,
69-
}, sa)
70-
if err == nil {
71-
_ = k8sClient.Delete(ctx, sa)
72-
}
7365
})
7466

7567
Context("EnsureDaemonSet", func() {
@@ -114,13 +106,8 @@ var _ = Describe("DaemonSet Reconciler", func() {
114106
Expect(ds.Spec.Template.Spec.Tolerations).To(HaveLen(1))
115107
Expect(string(ds.Spec.Template.Spec.Tolerations[0].Operator)).To(Equal(string(corev1.TolerationOpExists)))
116108

117-
// Verify ServiceAccount was created.
118-
sa := &corev1.ServiceAccount{}
119-
Expect(k8sClient.Get(ctx, types.NamespacedName{
120-
Name: daemonSetName,
121-
Namespace: testNamespace,
122-
}, sa)).To(Succeed())
123-
Expect(sa.Labels["app.kubernetes.io/name"]).To(Equal(daemonSetName))
109+
// Verify ServiceAccountName is set on the pod spec.
110+
Expect(ds.Spec.Template.Spec.ServiceAccountName).To(Equal(daemonSetName))
124111
})
125112

126113
It("should be idempotent when DaemonSet already exists", func() {

‎tests/k8s/Containerfile‎

Lines changed: 31 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,42 @@
1-
FROM quay.io/fedora/fedora-minimal
2-
1+
# Stage 1: runtime tools for running the test VM
2+
FROM quay.io/fedora/fedora-minimal AS base
33
ARG DNF_FLAGS="-y --setopt=install_weak_deps=False"
44
RUN dnf install ${DNF_FLAGS} \
55
qemu-system-x86-core qemu-img butane virtiofsd \
66
openssh-clients curl xz jq && \
77
dnf clean all
88

9-
# Download the latest stable FCOS qemu qcow2 image.
10-
RUN STREAM=https://builds.coreos.fedoraproject.org/streams/stable.json && \
11-
URL=$(curl -sSfL "${STREAM}" | \
12-
jq -r '.architectures.x86_64.artifacts.qemu.formats["qcow2.xz"].disk.location') && \
13-
mkdir -p /usr/share/fcos && \
14-
curl -sSfL "${URL}" | xzcat > /usr/share/fcos/image.qcow2
9+
# Stage 2: build a custom FCOS qcow2 with CRI-O + Kubernetes baked in.
10+
# This stage requires --device /dev/kvm and --security-opt label=disable
11+
# at build time (passed by run.sh).
12+
FROM quay.io/coreos-assembler/coreos-assembler:latest AS qcow2-builder
13+
USER root
14+
COPY Containerfile.fcos /tmp/Containerfile.fcos
15+
# Build the bootable container image using an inner podman build.
16+
# Run as root to avoid user namespace issues inside the build container.
17+
# Save as OCI archive so the builder user (cosa) can access it.
18+
RUN podman build --storage-driver overlay \
19+
-t localhost/fcos-k8s:latest -f /tmp/Containerfile.fcos /tmp && \
20+
podman save --format oci-archive -o /tmp/fcos-k8s.ociarchive \
21+
localhost/fcos-k8s:latest && \
22+
podman rmi localhost/fcos-k8s:latest
23+
# Import the OCI archive into cosa and build a qcow2.
24+
# cosa needs builds/ and tmp/ dirs; we skip `cosa init` since we don't
25+
# need a config git repo for the import-only workflow.
26+
USER builder
27+
WORKDIR /srv
28+
RUN mkdir -p builds tmp cache && \
29+
cosa import oci-archive:/tmp/fcos-k8s.ociarchive && \
30+
cosa osbuild qemu
1531

32+
# Stage 3: final test harness image
33+
FROM base
34+
RUN mkdir -p /usr/share/fcos
35+
# Copy the qcow2 from the cosa build. The exact filename depends on the
36+
# FCOS version, so use a glob.
37+
COPY --from=qcow2-builder /srv/builds/latest/x86_64/*-qemu.x86_64.qcow2 \
38+
/usr/share/fcos/image.qcow2
1639
COPY config.bu /usr/share/fcos/config.bu
1740
COPY entrypoint.sh /entrypoint.sh
1841
RUN chmod +x /entrypoint.sh
19-
2042
ENTRYPOINT ["/entrypoint.sh"]

‎tests/k8s/Containerfile.fcos‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Bootable container image: FCOS + CRI-O + Kubernetes
2+
# Built during `podman build` of the test harness and converted to a qcow2
3+
# by coreos-assembler.
4+
FROM quay.io/fedora/fedora-coreos:stable
5+
6+
# Kubernetes v1.35 repo
7+
RUN cat > /etc/yum.repos.d/kubernetes.repo <<'EOF'
8+
[kubernetes]
9+
name=Kubernetes
10+
baseurl=https://pkgs.k8s.io/core:/stable:/v1.35/rpm/
11+
enabled=1
12+
gpgcheck=1
13+
gpgkey=https://pkgs.k8s.io/core:/stable:/v1.35/rpm/repodata/repomd.xml.key
14+
EOF
15+
16+
# CRI-O v1.35 repo
17+
RUN cat > /etc/yum.repos.d/cri-o.repo <<'EOF'
18+
[cri-o]
19+
name=CRI-O
20+
baseurl=https://download.opensuse.org/repositories/isv:/cri-o:/stable:/v1.35/rpm/
21+
enabled=1
22+
gpgcheck=1
23+
gpgkey=https://download.opensuse.org/repositories/isv:/cri-o:/stable:/v1.35/rpm/repodata/repomd.xml.key
24+
EOF
25+
26+
RUN dnf install -y kubectl kubelet kubeadm cri-o cri-tools && \
27+
dnf clean all && \
28+
systemctl enable crio

0 commit comments

Comments
 (0)