Skip to content

Commit 4de7f9b

Browse files
tg123tg123
committed
Add Windows e2e tests via kind-in-WSL
1 parent c662869 commit 4de7f9b

10 files changed

Lines changed: 534 additions & 123 deletions

File tree

‎.gitattributes‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,11 @@
4343
*.fsx text=auto
4444
*.hs text=auto
4545

46+
# Shell scripts must keep LF endings even when checked out on Windows; a CRLF
47+
# turns `set -euxo pipefail` into `pipefail\r` (invalid option) once the file is
48+
# copied into WSL.
49+
*.sh text eol=lf
50+
4651
*.csproj text=auto
4752
*.vbproj text=auto
4853
*.fsproj text=auto

‎.github/e2e/bootstrap-kind.sh‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
#!/usr/bin/env bash
2+
# Bring up a single node Kubernetes control plane inside the kindest/node rootfs
3+
# that oci-to-wsl imported into WSL. kind normally runs this image as a Docker
4+
# container and drives kubeadm from the host through `docker exec`; here WSL
5+
# boots the image's systemd directly, so the equivalent bootstrap is performed
6+
# in place.
7+
set -euxo pipefail
8+
9+
# --------------------------------------------------------------------------
10+
# Reproduce the setup that kind's container entrypoint (/usr/local/bin/entrypoint)
11+
# performs when running kindest/node inside Docker. WSL boots systemd directly
12+
# from the image's rootfs, so these steps must be done explicitly.
13+
# --------------------------------------------------------------------------
14+
15+
# 1. Mount propagation — kubeadm and kubelet need shared mounts.
16+
mount --make-rshared / 2>/dev/null || true
17+
18+
# 2. /dev/kmsg — kubelet reads kernel messages from this device and exits
19+
# immediately if it is missing. WSL2 does not create it by default.
20+
if [ ! -e /dev/kmsg ]; then
21+
ln -sf /dev/console /dev/kmsg
22+
fi
23+
24+
# 3. Kernel parameters required by kube-proxy and networking.
25+
sysctl -w net.ipv4.ip_forward=1 || true
26+
sysctl -w net.ipv4.conf.all.forwarding=1 || true
27+
sysctl -w net.ipv6.conf.all.forwarding=1 2>/dev/null || true
28+
29+
# 4. Ensure /run is a tmpfs (systemd usually handles this, but be safe).
30+
if ! mountpoint -q /run; then
31+
mount -t tmpfs tmpfs /run
32+
mkdir -p /run/lock
33+
fi
34+
35+
# 5. Ensure a machine-id exists (kubelet uses it as node identity).
36+
if [ ! -s /etc/machine-id ]; then
37+
systemd-machine-id-setup 2>/dev/null || uuidgen | tr -d '-' > /etc/machine-id
38+
fi
39+
40+
# 6. containerd configuration — use cgroupfs driver. WSL2's systemd has limited
41+
# cgroup delegation, so the cgroupfs driver is more reliable here.
42+
mkdir -p /etc/containerd
43+
containerd config default \
44+
| sed 's/SystemdCgroup = true/SystemdCgroup = false/' \
45+
> /etc/containerd/config.toml
46+
47+
# 7. Start containerd.
48+
systemctl enable --now containerd
49+
systemctl restart containerd
50+
51+
# Wait for the container runtime to accept requests.
52+
containerd_ready=0
53+
for _ in {1..60}; do
54+
if ctr --namespace k8s.io version >/dev/null 2>&1; then
55+
containerd_ready=1
56+
break
57+
fi
58+
sleep 2
59+
done
60+
if [ "$containerd_ready" -ne 1 ]; then
61+
echo "containerd did not become ready after 120s" >&2
62+
systemctl status containerd --no-pager || true
63+
journalctl -xeu containerd --no-pager -n 80 || true
64+
exit 1
65+
fi
66+
67+
# 8. The kindest/node image ships a kubelet drop-in (11-kind.conf) whose
68+
# ExecStartPre runs /kind/bin/create-kubelet-cgroup-v2.sh. That script
69+
# expects Docker-managed cgroup namespaces and fails in bare WSL2.
70+
# Patch it out so the kubelet service can start.
71+
if [ -f /etc/systemd/system/kubelet.service.d/11-kind.conf ]; then
72+
sed -i '/create-kubelet-cgroup-v2/d' /etc/systemd/system/kubelet.service.d/11-kind.conf
73+
systemctl daemon-reload
74+
fi
75+
systemctl stop kubelet 2>/dev/null || true
76+
77+
# Initialize the control plane using a kubeadm config that forces cgroupfs and
78+
# disables swap checking.
79+
set +e
80+
kubeadm init \
81+
--ignore-preflight-errors=all \
82+
--config /dev/stdin <<'KUBEADM_CONFIG'
83+
apiVersion: kubeadm.k8s.io/v1beta4
84+
kind: InitConfiguration
85+
nodeRegistration:
86+
criSocket: unix:///run/containerd/containerd.sock
87+
ignorePreflightErrors:
88+
- all
89+
---
90+
apiVersion: kubeadm.k8s.io/v1beta4
91+
kind: ClusterConfiguration
92+
networking:
93+
podSubnet: 10.244.0.0/16
94+
apiServer:
95+
certSANs:
96+
- "127.0.0.1"
97+
- "localhost"
98+
---
99+
apiVersion: kubelet.config.k8s.io/v1beta1
100+
kind: KubeletConfiguration
101+
cgroupDriver: cgroupfs
102+
failSwapOn: false
103+
KUBEADM_CONFIG
104+
rc=$?
105+
set -e
106+
107+
if [ $rc -ne 0 ]; then
108+
echo "=== kubeadm init failed (rc=$rc) — collecting diagnostics ===" >&2
109+
systemctl status kubelet --no-pager 2>&1 || true
110+
journalctl -xeu kubelet --no-pager -n 80 2>&1 || true
111+
echo "=== cgroup info ===" >&2
112+
mount | grep cgroup || true
113+
cat /proc/self/cgroup 2>/dev/null || true
114+
ls /sys/fs/cgroup/ 2>/dev/null || true
115+
exit 1
116+
fi
117+
118+
export KUBECONFIG=/etc/kubernetes/admin.conf
119+
120+
# Single node cluster: allow workloads to schedule on the control-plane node.
121+
kubectl taint nodes --all node-role.kubernetes.io/control-plane- 2>/dev/null || true
122+
123+
# The bundled kindnet manifest (/kind/manifests/default-cni.yaml) contains Go
124+
# template placeholders (e.g. {{ .PodSubnet }}) that kind normally renders at
125+
# cluster creation time. Render the template and apply; if that fails, install
126+
# a standalone CNI config for host-local networking on this single node.
127+
if ! sed -e 's/{{ \.PodSubnet }}/10.244.0.0\/16/g' \
128+
-e 's/{{ \.PodSubnet}}/10.244.0.0\/16/g' \
129+
-e 's/{{\.PodSubnet}}/10.244.0.0\/16/g' \
130+
-e '/^{{/d' -e '/^}}/d' \
131+
/kind/manifests/default-cni.yaml | kubectl apply -f - 2>&1; then
132+
echo "kindnet manifest apply failed; installing host-local CNI config directly"
133+
mkdir -p /etc/cni/net.d
134+
cat > /etc/cni/net.d/10-kindnet.conflist <<'CNI'
135+
{
136+
"cniVersion": "0.4.0",
137+
"name": "kindnet",
138+
"plugins": [
139+
{
140+
"type": "ptp",
141+
"ipMasq": false,
142+
"ipam": { "type": "host-local", "dataDir": "/run/cni-ipam-state", "routes": [{"dst": "0.0.0.0/0"}], "ranges": [[{"subnet": "10.244.0.0/24"}]] }
143+
},
144+
{ "type": "portmap", "capabilities": {"portMappings": true} }
145+
]
146+
}
147+
CNI
148+
fi
149+
150+
# Assign podCIDR to the node so kubelet can allocate pod IPs.
151+
NODE=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}')
152+
kubectl patch node "$NODE" -p '{"spec":{"podCIDR":"10.244.0.0/24","podCIDRs":["10.244.0.0/24"]}}' 2>/dev/null || true
153+
154+
kubectl wait --for=condition=Ready nodes --all --timeout=300s

‎.github/e2e/kind-wsl.yaml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# oci-to-wsl profile that imports the kindest/node image into WSL as a
2+
# systemd-enabled distribution. Unlike the regular "kind in Docker" flow, the
3+
# Kubernetes node runs natively inside WSL, so no Docker daemon or nested
4+
# containers are required on the Windows runner.
5+
name: kind
6+
image: kindest/node:v1.32.2
7+
install_dir: C:\WSL\kind
8+
9+
# kindest/node expects an init system; WSL boots systemd when configured.
10+
wsl_conf:
11+
mode: merge
12+
content: |
13+
[boot]
14+
systemd=true
15+
16+
# Stage the bootstrap script that turns the imported rootfs into a running
17+
# single node control plane. Path is resolved relative to this profile.
18+
files:
19+
- src: ./bootstrap-kind.sh
20+
dst: /usr/local/bin/bootstrap-kind.sh
21+
mode: "0755"

‎.github/workflows/buildtest.yaml‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,84 @@ jobs:
8787
exit 1
8888
fi
8989
90+
# Run the .NET Framework 4.8 e2e suite (KubernetesClient.Classic) against a
91+
# real Kubernetes cluster on Windows. There is no native Windows Kubernetes
92+
# distribution, so the kindest/node image (which bundles containerd, kubelet
93+
# and the control-plane components) is imported into WSL2 with oci-to-wsl.
94+
# The Linux e2e job already covers the modern (net8/9/10) client libraries.
95+
# WSL2 requires windows-2025 (windows-2022 only has WSL1).
96+
e2e-windows-wsl:
97+
runs-on: windows-2025
98+
name: e2e classic (wsl)
99+
steps:
100+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
101+
with:
102+
fetch-depth: 0
103+
- name: Setup dotnet
104+
uses: actions/setup-dotnet@9a946fdbd5fb07b82b2f5a4466058b876ab72bb2 # v5.3.0
105+
with:
106+
dotnet-version: 10.0.x
107+
- name: Import kindest/node into WSL
108+
uses: tg123/oci-to-wsl@902e4b0d7f3e7608dbd581e9e2fd473907a279a8 # main
109+
with:
110+
profile: .github/e2e/kind-wsl.yaml
111+
- name: Start kind cluster in WSL
112+
shell: pwsh
113+
run: |
114+
# Reboot the distro so the systemd configuration written by the profile
115+
# takes effect.
116+
wsl --shutdown
117+
118+
# WSL terminates the lightweight VM once its last session exits, which
119+
# tears down the API server between steps. Hold an idle session open in
120+
# the background so the cluster stays up for the host-side test steps.
121+
Start-Process -FilePath wsl -ArgumentList '-d','kind','-u','root','--','sleep','infinity'
122+
123+
# Bring up the single node control plane in place.
124+
wsl -d kind -u root -- bash /usr/local/bin/bootstrap-kind.sh
125+
126+
# Get the WSL2 VM's IP address. In NAT mode the host can reach this
127+
# IP directly via the virtual switch.
128+
$wslIp = ((wsl -d kind -- hostname -I).Trim() -split '\s+', 0, 'RegexMatch' | Where-Object { $_ -match '^\d+\.\d+\.\d+\.\d+$' } | Select-Object -First 1)
129+
if (-not $wslIp) { throw "Could not determine WSL IP" }
130+
Write-Output "WSL IP: $wslIp"
131+
132+
# Export a kubeconfig for the Windows host, rewriting the server URL
133+
# to the WSL VM's IP address reachable from Windows.
134+
wsl -d kind -u root -- cat /etc/kubernetes/admin.conf | Out-File -Encoding ascii kind.kubeconfig
135+
(Get-Content kind.kubeconfig) -replace 'server: https://[^ ]+', "server: https://${wslIp}:6443" | Set-Content kind.kubeconfig
136+
137+
# .NET Framework 4.8 does not support custom trust stores, so skip TLS
138+
# verification for the self-signed K8s CA (this is a CI-only cluster).
139+
(Get-Content kind.kubeconfig) -replace 'certificate-authority-data:.*', 'insecure-skip-tls-verify: true' | Set-Content kind.kubeconfig
140+
141+
# Wait until the API server is reachable from the Windows host.
142+
$ok = $false
143+
foreach ($i in 1..60) {
144+
if ((Test-NetConnection -ComputerName $wslIp -Port 6443 -WarningAction SilentlyContinue).TcpTestSucceeded) {
145+
$ok = $true
146+
break
147+
}
148+
Start-Sleep -Seconds 5
149+
}
150+
if (-not $ok) {
151+
throw "API server on ${wslIp}:6443 was not reachable from the Windows host"
152+
}
153+
- name: Test
154+
shell: pwsh
155+
env:
156+
K8S_E2E_MINIKUBE: "1"
157+
KUBECONFIG: ${{ github.workspace }}\kind.kubeconfig
158+
run: |
159+
[IO.File]::WriteAllText("$PWD/skip.log", "")
160+
dotnet test tests/E2E.Classic.Tests --logger "SkipTestLogger;file=$PWD/skip.log" -p:BuildInParallel=false
161+
$skipContent = [IO.File]::ReadAllText("$PWD/skip.log").Trim()
162+
if ($skipContent.Length -gt 0) {
163+
Write-Output $skipContent
164+
Write-Error "CASES MUST NOT BE SKIPPED"
165+
exit 1
166+
}
167+
90168
on:
91169
pull_request:
92170
types: [assigned, opened, synchronize, reopened]

‎.gitignore‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,7 @@ bin/
1717
launchSettings.json
1818
*.DotSettings
1919

20-
*.sln
20+
*.sln
21+
# VS test runner local config
22+
Minikube.runsettings
23+
testEnvironments.json

‎kubernetes-client.proj‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
<Project Sdk="Microsoft.Build.Traversal">
22
<ItemGroup>
33
<ProjectReference Include="src/**/*.csproj" />
4-
<ProjectReference Include="tests/**/*.csproj" />
4+
<ProjectReference Include="tests/**/*.csproj" Exclude="tests/E2E.Classic.Tests/**/*.csproj" />
5+
<ProjectReference Include="tests/E2E.Classic.Tests/**/*.csproj" Condition="'$(OS)' == 'Windows_NT'" />
56
<ProjectReference Include="examples/**/*.csproj" />
67
</ItemGroup>
78
</Project>

‎tests/E2E.Classic.Tests/E2E.Classic.Tests.csproj‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
<PropertyGroup>
3+
<IsPackable>false</IsPackable>
4+
<RootNamespace>k8s.E2E</RootNamespace>
5+
<TargetFramework>net48</TargetFramework>
6+
</PropertyGroup>
7+
8+
<ItemGroup>
9+
<PackageReference Include="Microsoft.NET.Test.Sdk" />
10+
11+
<PackageReference Include="xunit" />
12+
<PackageReference Include="xunit.runner.visualstudio">
13+
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
14+
</PackageReference>
15+
</ItemGroup>
16+
17+
<ItemGroup>
18+
<ProjectReference Include="..\..\src\KubernetesClient.Classic\KubernetesClient.Classic.csproj" />
19+
<ProjectReference Include="..\SkipTestLogger\SkipTestLogger.csproj" />
20+
</ItemGroup>
21+
22+
<ItemGroup>
23+
<Reference Include="System.Net.Http" />
24+
</ItemGroup>
25+
26+
<ItemGroup>
27+
<Compile Include="..\E2E.Tests\MinikubeFactAttribute.cs" />
28+
<Compile Include="..\E2E.Tests\Onebyone.cs" />
29+
</ItemGroup>
30+
</Project>

0 commit comments

Comments
 (0)