Skip to content

Commit 6336c12

Browse files
tg123tg123
committed
Add Windows e2e tests via kind-in-WSL
1 parent c662869 commit 6336c12

4 files changed

Lines changed: 188 additions & 0 deletions

File tree

‎.gitattributes‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,11 @@
4343
*.fsx text=auto
4444
*.hs text=auto
4545

46+
# Shell scripts must keep LF endings even when checked out on Windows; a CRLF
47+
# turns `set -euxo pipefail` into `pipefail\r` (invalid option) once the file is
48+
# copied into WSL.
49+
*.sh text eol=lf
50+
4651
*.csproj text=auto
4752
*.vbproj text=auto
4853
*.fsproj text=auto

‎.github/e2e/bootstrap-kind.sh‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
#!/usr/bin/env bash
2+
# Bring up a single node Kubernetes control plane inside the kindest/node rootfs
3+
# that oci-to-wsl imported into WSL. kind normally runs this image as a Docker
4+
# container and drives kubeadm from the host through `docker exec`; here WSL
5+
# boots the image's systemd directly, so the equivalent bootstrap is performed
6+
# in place.
7+
set -euxo pipefail
8+
9+
# The kindest/node entrypoint (which WSL does not run) makes the root mount
10+
# shared and enables IPv4 forwarding; reproduce the parts kubeadm and the
11+
# kubelet rely on.
12+
mount --make-rshared / 2>/dev/null || true
13+
sysctl -w net.ipv4.ip_forward=1 || true
14+
15+
# Ensure cgroups v2 unified hierarchy is fully mounted. WSL2 with systemd
16+
# typically mounts this, but the kubelet needs specific controllers delegated.
17+
if [ ! -d /sys/fs/cgroup/kubelet.slice ]; then
18+
mkdir -p /sys/fs/cgroup/kubelet.slice
19+
fi
20+
21+
# The kindest/node image ships a kubelet config that expects the systemd cgroup
22+
# driver. Ensure containerd is also configured for systemd cgroups (the image
23+
# default may use cgroupfs when not running inside Docker/kind).
24+
mkdir -p /etc/containerd
25+
if [ -f /etc/containerd/config.toml ]; then
26+
sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
27+
else
28+
containerd config default | sed 's/SystemdCgroup = false/SystemdCgroup = true/' > /etc/containerd/config.toml
29+
fi
30+
31+
# containerd ships as a systemd unit in the node image.
32+
systemctl enable --now containerd
33+
systemctl restart containerd
34+
35+
# Wait for the container runtime to accept requests.
36+
for _ in {1..60}; do
37+
if ctr --namespace k8s.io version >/dev/null 2>&1; then
38+
break
39+
fi
40+
sleep 2
41+
done
42+
43+
# Ensure the kubelet uses the systemd cgroup driver to match containerd.
44+
mkdir -p /var/lib/kubelet
45+
cat > /var/lib/kubelet/kubeadm-flags.env <<'KUBELET_FLAGS'
46+
KUBELET_KUBEADM_ARGS="--container-runtime-endpoint=unix:///run/containerd/containerd.sock --cgroup-driver=systemd"
47+
KUBELET_FLAGS
48+
49+
# Initialize the control plane. Preflight errors are ignored because the WSL
50+
# environment intentionally differs from a vanilla node (swap on, kernel
51+
# modules provided by the Windows-side kernel, and so on).
52+
kubeadm init \
53+
--ignore-preflight-errors=all \
54+
--pod-network-cidr=10.244.0.0/16 \
55+
--apiserver-cert-extra-sans=127.0.0.1,localhost
56+
57+
export KUBECONFIG=/etc/kubernetes/admin.conf
58+
59+
# Single node cluster: allow workloads to schedule on the control-plane node.
60+
kubectl taint nodes --all node-role.kubernetes.io/control-plane- 2>/dev/null || true
61+
62+
# kindest/node bundles the default CNI (kindnet) manifest.
63+
kubectl apply -f /kind/manifests/default-cni.yaml
64+
65+
kubectl wait --for=condition=Ready nodes --all --timeout=180s

‎.github/e2e/kind-wsl.yaml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# oci-to-wsl profile that imports the kindest/node image into WSL as a
2+
# systemd-enabled distribution. Unlike the regular "kind in Docker" flow, the
3+
# Kubernetes node runs natively inside WSL, so no Docker daemon or nested
4+
# containers are required on the Windows runner.
5+
name: kind
6+
image: kindest/node:v1.32.2
7+
install_dir: C:\WSL\kind
8+
9+
# kindest/node expects an init system; WSL boots systemd when configured.
10+
wsl_conf:
11+
mode: merge
12+
content: |
13+
[boot]
14+
systemd=true
15+
16+
# Stage the bootstrap script that turns the imported rootfs into a running
17+
# single node control plane. Path is resolved relative to this profile.
18+
files:
19+
- src: ./bootstrap-kind.sh
20+
dst: /usr/local/bin/bootstrap-kind.sh
21+
mode: "0755"

‎.github/workflows/buildtest.yaml‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,103 @@ jobs:
8787
exit 1
8888
fi
8989
90+
# Run the same end to end suite as the Linux `e2e` job, but on Windows. There
91+
# is no Linux Kubernetes distribution that runs natively on a Windows runner,
92+
# so the kindest/node image (which already bundles containerd, the kubelet and
93+
# the control-plane components) is imported straight into WSL with oci-to-wsl.
94+
# WSL2 is enabled by default on the runner, so no separate WSL provisioning is
95+
# needed, and the API server is reachable from the Windows host (which runs the
96+
# test process) through WSL2 localhost forwarding.
97+
e2e-windows-wsl:
98+
runs-on: windows-latest
99+
name: E2E (kind in WSL)
100+
steps:
101+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
102+
with:
103+
fetch-depth: 0
104+
- name: Setup dotnet
105+
uses: actions/setup-dotnet@9a946fdbd5fb07b82b2f5a4466058b876ab72bb2 # v5.3.0
106+
with:
107+
dotnet-version: |
108+
8.0.x
109+
9.0.x
110+
10.0.x
111+
- name: Install oci-to-wsl
112+
shell: pwsh
113+
run: |
114+
$url = "https://github.com/tg123/oci-to-wsl/releases/download/v0.0.1/oci-to-wsl_windows_x86_64.zip"
115+
$expected = "79772846F1CE4A38E5B2841EA2406A070F4DB64F3DBF3EA32F04B519111FD003"
116+
Invoke-WebRequest $url -OutFile oci-to-wsl.zip
117+
$actual = (Get-FileHash oci-to-wsl.zip -Algorithm SHA256).Hash
118+
if ($actual -ne $expected) {
119+
throw "oci-to-wsl checksum mismatch: expected $expected, got $actual"
120+
}
121+
Expand-Archive -Force oci-to-wsl.zip -DestinationPath .
122+
- name: Start kind cluster in WSL
123+
shell: pwsh
124+
run: |
125+
# Import kindest/node into WSL as a systemd-enabled distro named "kind".
126+
.\oci-to-wsl.exe --profile .github/e2e/kind-wsl.yaml
127+
128+
# Reboot the distro so the systemd configuration written by the profile
129+
# takes effect.
130+
wsl --shutdown
131+
132+
# WSL terminates the lightweight VM once its last session exits, which
133+
# tears down the API server between steps and makes localhost:6443
134+
# connections fail with "actively refused". Hold an idle session open in
135+
# the background so the cluster (and WSL2 localhost forwarding) stay up
136+
# for the duration of the host-side test steps.
137+
Start-Process -FilePath wsl -ArgumentList '-d','kind','-u','root','--','sleep','infinity'
138+
139+
# Bring up the single node control plane in place.
140+
wsl -d kind -u root -- bash /usr/local/bin/bootstrap-kind.sh
141+
142+
# Export a kubeconfig for the Windows host. kubeadm points it at the WSL
143+
# eth0 address; rewrite it to localhost, which WSL2 forwards into WSL.
144+
wsl -d kind -u root -- cat /etc/kubernetes/admin.conf | Out-File -Encoding ascii kind.kubeconfig
145+
(Get-Content kind.kubeconfig) -replace 'server: https://[^ ]+', 'server: https://127.0.0.1:6443' | Set-Content kind.kubeconfig
146+
147+
# Wait until the API server is reachable from the Windows host through
148+
# the forwarded port before handing off to the test steps.
149+
$ok = $false
150+
foreach ($i in 1..60) {
151+
if ((Test-NetConnection -ComputerName 127.0.0.1 -Port 6443 -WarningAction SilentlyContinue).TcpTestSucceeded) {
152+
$ok = $true
153+
break
154+
}
155+
Start-Sleep -Seconds 5
156+
}
157+
if (-not $ok) {
158+
throw "API server on 127.0.0.1:6443 was not reachable from the Windows host"
159+
}
160+
- name: Test
161+
shell: pwsh
162+
env:
163+
K8S_E2E_MINIKUBE: "1"
164+
KUBECONFIG: ${{ github.workspace }}\kind.kubeconfig
165+
run: |
166+
"" | Out-File -Encoding ascii skip.log
167+
dotnet test tests/E2E.Tests --logger "SkipTestLogger;file=$PWD/skip.log" -p:BuildInParallel=false
168+
if ((Get-Item skip.log).Length -gt 0) {
169+
Get-Content skip.log
170+
Write-Error "CASES MUST NOT BE SKIPPED"
171+
exit 1
172+
}
173+
- name: AOT Test
174+
shell: pwsh
175+
env:
176+
K8S_E2E_MINIKUBE: "1"
177+
KUBECONFIG: ${{ github.workspace }}\kind.kubeconfig
178+
run: |
179+
"" | Out-File -Encoding ascii skip.log
180+
dotnet test tests/E2E.Aot.Tests --logger "SkipTestLogger;file=$PWD/skip.log" -p:BuildInParallel=false
181+
if ((Get-Item skip.log).Length -gt 0) {
182+
Get-Content skip.log
183+
Write-Error "CASES MUST NOT BE SKIPPED"
184+
exit 1
185+
}
186+
90187
on:
91188
pull_request:
92189
types: [assigned, opened, synchronize, reopened]

0 commit comments

Comments
 (0)