Skip to content

Commit 430a929

Browse files
tg123tg123
committed
Add Windows e2e tests via kind-in-WSL
1 parent c662869 commit 430a929

4 files changed

Lines changed: 269 additions & 0 deletions

File tree

‎.gitattributes‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,11 @@
4343
*.fsx text=auto
4444
*.hs text=auto
4545

46+
# Shell scripts must keep LF endings even when checked out on Windows; a CRLF
47+
# turns `set -euxo pipefail` into `pipefail\r` (invalid option) once the file is
48+
# copied into WSL.
49+
*.sh text eol=lf
50+
4651
*.csproj text=auto
4752
*.vbproj text=auto
4853
*.fsproj text=auto

‎.github/e2e/bootstrap-kind.sh‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
#!/usr/bin/env bash
2+
# Bring up a single node Kubernetes control plane inside the kindest/node rootfs
3+
# that oci-to-wsl imported into WSL. kind normally runs this image as a Docker
4+
# container and drives kubeadm from the host through `docker exec`; here WSL
5+
# boots the image's systemd directly, so the equivalent bootstrap is performed
6+
# in place.
7+
set -euxo pipefail
8+
9+
# --------------------------------------------------------------------------
10+
# Reproduce the setup that kind's container entrypoint (/usr/local/bin/entrypoint)
11+
# performs when running kindest/node inside Docker. WSL boots systemd directly
12+
# from the image's rootfs, so these steps must be done explicitly.
13+
# --------------------------------------------------------------------------
14+
15+
# 1. Mount propagation — kubeadm and kubelet need shared mounts.
16+
mount --make-rshared / 2>/dev/null || true
17+
18+
# 2. /dev/kmsg — kubelet reads kernel messages from this device and exits
19+
# immediately if it is missing. WSL2 does not create it by default.
20+
if [ ! -e /dev/kmsg ]; then
21+
ln -sf /dev/console /dev/kmsg
22+
fi
23+
24+
# 3. Kernel parameters required by kube-proxy and networking.
25+
sysctl -w net.ipv4.ip_forward=1 || true
26+
sysctl -w net.ipv4.conf.all.forwarding=1 || true
27+
sysctl -w net.ipv6.conf.all.forwarding=1 2>/dev/null || true
28+
29+
# 4. Ensure /run is a tmpfs (systemd usually handles this, but be safe).
30+
if ! mountpoint -q /run; then
31+
mount -t tmpfs tmpfs /run
32+
mkdir -p /run/lock
33+
fi
34+
35+
# 5. Ensure a machine-id exists (kubelet uses it as node identity).
36+
if [ ! -s /etc/machine-id ]; then
37+
systemd-machine-id-setup 2>/dev/null || uuidgen | tr -d '-' > /etc/machine-id
38+
fi
39+
40+
# 6. containerd configuration — use cgroupfs driver. WSL2's systemd has limited
41+
# cgroup delegation, so the cgroupfs driver is more reliable here.
42+
mkdir -p /etc/containerd
43+
containerd config default \
44+
| sed 's/SystemdCgroup = true/SystemdCgroup = false/' \
45+
> /etc/containerd/config.toml
46+
47+
# 7. Start containerd.
48+
systemctl enable --now containerd
49+
systemctl restart containerd
50+
51+
# Wait for the container runtime to accept requests.
52+
for _ in {1..60}; do
53+
if ctr --namespace k8s.io version >/dev/null 2>&1; then
54+
break
55+
fi
56+
sleep 2
57+
done
58+
59+
# 8. The kindest/node image ships a kubelet drop-in (11-kind.conf) whose
60+
# ExecStartPre runs /kind/bin/create-kubelet-cgroup-v2.sh. That script
61+
# expects Docker-managed cgroup namespaces and fails in bare WSL2.
62+
# Patch it out so the kubelet service can start.
63+
if [ -f /etc/systemd/system/kubelet.service.d/11-kind.conf ]; then
64+
sed -i '/create-kubelet-cgroup-v2/d' /etc/systemd/system/kubelet.service.d/11-kind.conf
65+
systemctl daemon-reload
66+
fi
67+
systemctl stop kubelet 2>/dev/null || true
68+
69+
# Initialize the control plane using a kubeadm config that forces cgroupfs and
70+
# disables swap checking.
71+
set +e
72+
kubeadm init \
73+
--ignore-preflight-errors=all \
74+
--config /dev/stdin <<'KUBEADM_CONFIG'
75+
apiVersion: kubeadm.k8s.io/v1beta4
76+
kind: InitConfiguration
77+
nodeRegistration:
78+
criSocket: unix:///run/containerd/containerd.sock
79+
ignorePreflightErrors:
80+
- all
81+
---
82+
apiVersion: kubeadm.k8s.io/v1beta4
83+
kind: ClusterConfiguration
84+
networking:
85+
podSubnet: 10.244.0.0/16
86+
apiServer:
87+
certSANs:
88+
- "127.0.0.1"
89+
- "localhost"
90+
---
91+
apiVersion: kubelet.config.k8s.io/v1beta1
92+
kind: KubeletConfiguration
93+
cgroupDriver: cgroupfs
94+
failSwapOn: false
95+
KUBEADM_CONFIG
96+
rc=$?
97+
set -e
98+
99+
if [ $rc -ne 0 ]; then
100+
echo "=== kubeadm init failed (rc=$rc) — collecting diagnostics ===" >&2
101+
systemctl status kubelet --no-pager 2>&1 || true
102+
journalctl -xeu kubelet --no-pager -n 80 2>&1 || true
103+
echo "=== cgroup info ===" >&2
104+
mount | grep cgroup || true
105+
cat /proc/self/cgroup 2>/dev/null || true
106+
ls /sys/fs/cgroup/ 2>/dev/null || true
107+
exit 1
108+
fi
109+
110+
export KUBECONFIG=/etc/kubernetes/admin.conf
111+
112+
# Single node cluster: allow workloads to schedule on the control-plane node.
113+
kubectl taint nodes --all node-role.kubernetes.io/control-plane- 2>/dev/null || true
114+
115+
# The bundled kindnet manifest (/kind/manifests/default-cni.yaml) contains Go
116+
# template placeholders (e.g. {{ .PodSubnet }}) that kind normally renders at
117+
# cluster creation time. Render the template and apply; if that fails, install
118+
# a standalone CNI config for host-local networking on this single node.
119+
if ! sed -e 's/{{ \.PodSubnet }}/10.244.0.0\/16/g' \
120+
-e 's/{{ \.PodSubnet}}/10.244.0.0\/16/g' \
121+
-e 's/{{\.PodSubnet}}/10.244.0.0\/16/g' \
122+
-e '/^{{/d' -e '/^}}/d' \
123+
/kind/manifests/default-cni.yaml | kubectl apply -f - 2>&1; then
124+
echo "kindnet manifest apply failed; installing host-local CNI config directly"
125+
mkdir -p /etc/cni/net.d
126+
cat > /etc/cni/net.d/10-kindnet.conflist <<'CNI'
127+
{
128+
"cniVersion": "0.4.0",
129+
"name": "kindnet",
130+
"plugins": [
131+
{
132+
"type": "ptp",
133+
"ipMasq": false,
134+
"ipam": { "type": "host-local", "dataDir": "/run/cni-ipam-state", "routes": [{"dst": "0.0.0.0/0"}], "ranges": [[{"subnet": "10.244.0.0/24"}]] }
135+
},
136+
{ "type": "portmap", "capabilities": {"portMappings": true} }
137+
]
138+
}
139+
CNI
140+
fi
141+
142+
# Assign podCIDR to the node so kubelet can allocate pod IPs.
143+
NODE=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}')
144+
kubectl patch node "$NODE" -p '{"spec":{"podCIDR":"10.244.0.0/24","podCIDRs":["10.244.0.0/24"]}}' 2>/dev/null || true
145+
146+
kubectl wait --for=condition=Ready nodes --all --timeout=300s

‎.github/e2e/kind-wsl.yaml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# oci-to-wsl profile that imports the kindest/node image into WSL as a
2+
# systemd-enabled distribution. Unlike the regular "kind in Docker" flow, the
3+
# Kubernetes node runs natively inside WSL, so no Docker daemon or nested
4+
# containers are required on the Windows runner.
5+
name: kind
6+
image: kindest/node:v1.32.2
7+
install_dir: C:\WSL\kind
8+
9+
# kindest/node expects an init system; WSL boots systemd when configured.
10+
wsl_conf:
11+
mode: merge
12+
content: |
13+
[boot]
14+
systemd=true
15+
16+
# Stage the bootstrap script that turns the imported rootfs into a running
17+
# single node control plane. Path is resolved relative to this profile.
18+
files:
19+
- src: ./bootstrap-kind.sh
20+
dst: /usr/local/bin/bootstrap-kind.sh
21+
mode: "0755"

‎.github/workflows/buildtest.yaml‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,103 @@ jobs:
8787
exit 1
8888
fi
8989
90+
# Run the same end to end suite as the Linux `e2e` job, but on Windows. There
91+
# is no Linux Kubernetes distribution that runs natively on a Windows runner,
92+
# so the kindest/node image (which already bundles containerd, the kubelet and
93+
# the control-plane components) is imported straight into WSL with oci-to-wsl.
94+
# WSL2 is enabled by default on the runner, so no separate WSL provisioning is
95+
# needed, and the API server is reachable from the Windows host (which runs the
96+
# test process) through WSL2 localhost forwarding.
97+
e2e-windows-wsl:
98+
runs-on: windows-latest
99+
name: E2E (kind in WSL)
100+
steps:
101+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
102+
with:
103+
fetch-depth: 0
104+
- name: Setup dotnet
105+
uses: actions/setup-dotnet@9a946fdbd5fb07b82b2f5a4466058b876ab72bb2 # v5.3.0
106+
with:
107+
dotnet-version: |
108+
8.0.x
109+
9.0.x
110+
10.0.x
111+
- name: Install oci-to-wsl
112+
shell: pwsh
113+
run: |
114+
$url = "https://github.com/tg123/oci-to-wsl/releases/download/v0.0.1/oci-to-wsl_windows_x86_64.zip"
115+
$expected = "79772846F1CE4A38E5B2841EA2406A070F4DB64F3DBF3EA32F04B519111FD003"
116+
Invoke-WebRequest $url -OutFile oci-to-wsl.zip
117+
$actual = (Get-FileHash oci-to-wsl.zip -Algorithm SHA256).Hash
118+
if ($actual -ne $expected) {
119+
throw "oci-to-wsl checksum mismatch: expected $expected, got $actual"
120+
}
121+
Expand-Archive -Force oci-to-wsl.zip -DestinationPath .
122+
- name: Start kind cluster in WSL
123+
shell: pwsh
124+
run: |
125+
# Import kindest/node into WSL as a systemd-enabled distro named "kind".
126+
.\oci-to-wsl.exe --profile .github/e2e/kind-wsl.yaml
127+
128+
# Reboot the distro so the systemd configuration written by the profile
129+
# takes effect.
130+
wsl --shutdown
131+
132+
# WSL terminates the lightweight VM once its last session exits, which
133+
# tears down the API server between steps and makes localhost:6443
134+
# connections fail with "actively refused". Hold an idle session open in
135+
# the background so the cluster (and WSL2 localhost forwarding) stay up
136+
# for the duration of the host-side test steps.
137+
Start-Process -FilePath wsl -ArgumentList '-d','kind','-u','root','--','sleep','infinity'
138+
139+
# Bring up the single node control plane in place.
140+
wsl -d kind -u root -- bash /usr/local/bin/bootstrap-kind.sh
141+
142+
# Export a kubeconfig for the Windows host. kubeadm points it at the WSL
143+
# eth0 address; rewrite it to localhost, which WSL2 forwards into WSL.
144+
wsl -d kind -u root -- cat /etc/kubernetes/admin.conf | Out-File -Encoding ascii kind.kubeconfig
145+
(Get-Content kind.kubeconfig) -replace 'server: https://[^ ]+', 'server: https://127.0.0.1:6443' | Set-Content kind.kubeconfig
146+
147+
# Wait until the API server is reachable from the Windows host through
148+
# the forwarded port before handing off to the test steps.
149+
$ok = $false
150+
foreach ($i in 1..60) {
151+
if ((Test-NetConnection -ComputerName 127.0.0.1 -Port 6443 -WarningAction SilentlyContinue).TcpTestSucceeded) {
152+
$ok = $true
153+
break
154+
}
155+
Start-Sleep -Seconds 5
156+
}
157+
if (-not $ok) {
158+
throw "API server on 127.0.0.1:6443 was not reachable from the Windows host"
159+
}
160+
- name: Test
161+
shell: pwsh
162+
env:
163+
K8S_E2E_MINIKUBE: "1"
164+
KUBECONFIG: ${{ github.workspace }}\kind.kubeconfig
165+
run: |
166+
"" | Out-File -Encoding ascii skip.log
167+
dotnet test tests/E2E.Tests --logger "SkipTestLogger;file=$PWD/skip.log" -p:BuildInParallel=false
168+
if ((Get-Item skip.log).Length -gt 0) {
169+
Get-Content skip.log
170+
Write-Error "CASES MUST NOT BE SKIPPED"
171+
exit 1
172+
}
173+
- name: AOT Test
174+
shell: pwsh
175+
env:
176+
K8S_E2E_MINIKUBE: "1"
177+
KUBECONFIG: ${{ github.workspace }}\kind.kubeconfig
178+
run: |
179+
"" | Out-File -Encoding ascii skip.log
180+
dotnet test tests/E2E.Aot.Tests --logger "SkipTestLogger;file=$PWD/skip.log" -p:BuildInParallel=false
181+
if ((Get-Item skip.log).Length -gt 0) {
182+
Get-Content skip.log
183+
Write-Error "CASES MUST NOT BE SKIPPED"
184+
exit 1
185+
}
186+
90187
on:
91188
pull_request:
92189
types: [assigned, opened, synchronize, reopened]

0 commit comments

Comments
 (0)