diff --git a/.gitattributes b/.gitattributes index 65f0cf8c6..3bd3ea991 100644 --- a/.gitattributes +++ b/.gitattributes @@ -43,6 +43,11 @@ *.fsx text=auto *.hs text=auto +# Shell scripts must keep LF endings even when checked out on Windows; a CRLF +# turns `set -euxo pipefail` into `pipefail\r` (invalid option) once the file is +# copied into WSL. +*.sh text eol=lf + *.csproj text=auto *.vbproj text=auto *.fsproj text=auto diff --git a/.github/e2e/bootstrap-kind.sh b/.github/e2e/bootstrap-kind.sh new file mode 100755 index 000000000..0e5dbab84 --- /dev/null +++ b/.github/e2e/bootstrap-kind.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +# Bring up a single node Kubernetes control plane inside the kindest/node rootfs +# that oci-to-wsl imported into WSL. kind normally runs this image as a Docker +# container and drives kubeadm from the host through `docker exec`; here WSL +# boots the image's systemd directly, so the equivalent bootstrap is performed +# in place. +set -euxo pipefail + +# -------------------------------------------------------------------------- +# Reproduce the setup that kind's container entrypoint (/usr/local/bin/entrypoint) +# performs when running kindest/node inside Docker. WSL boots systemd directly +# from the image's rootfs, so these steps must be done explicitly. +# -------------------------------------------------------------------------- + +# 1. Mount propagation — kubeadm and kubelet need shared mounts. +mount --make-rshared / 2>/dev/null || true + +# 2. /dev/kmsg — kubelet reads kernel messages from this device and exits +# immediately if it is missing. WSL2 does not create it by default. +if [ ! -e /dev/kmsg ]; then + ln -sf /dev/console /dev/kmsg +fi + +# 3. Kernel parameters required by kube-proxy and networking. +sysctl -w net.ipv4.ip_forward=1 || true +sysctl -w net.ipv4.conf.all.forwarding=1 || true +sysctl -w net.ipv6.conf.all.forwarding=1 2>/dev/null || true + +# 4. Ensure /run is a tmpfs (systemd usually handles this, but be safe). +if ! mountpoint -q /run; then + mount -t tmpfs tmpfs /run + mkdir -p /run/lock +fi + +# 5. Ensure a machine-id exists (kubelet uses it as node identity). +if [ ! -s /etc/machine-id ]; then + systemd-machine-id-setup 2>/dev/null || uuidgen | tr -d '-' > /etc/machine-id +fi + +# 6. containerd configuration — use cgroupfs driver. WSL2's systemd has limited +# cgroup delegation, so the cgroupfs driver is more reliable here. +mkdir -p /etc/containerd +containerd config default \ + | sed 's/SystemdCgroup = true/SystemdCgroup = false/' \ + > /etc/containerd/config.toml + +# 7. Start containerd. +systemctl enable --now containerd +systemctl restart containerd + +# Wait for the container runtime to accept requests. +containerd_ready=0 +for _ in {1..60}; do + if ctr --namespace k8s.io version >/dev/null 2>&1; then + containerd_ready=1 + break + fi + sleep 2 +done +if [ "$containerd_ready" -ne 1 ]; then + echo "containerd did not become ready after 120s" >&2 + systemctl status containerd --no-pager || true + journalctl -xeu containerd --no-pager -n 80 || true + exit 1 +fi + +# 8. The kindest/node image ships a kubelet drop-in (11-kind.conf) whose +# ExecStartPre runs /kind/bin/create-kubelet-cgroup-v2.sh. That script +# expects Docker-managed cgroup namespaces and fails in bare WSL2. +# Patch it out so the kubelet service can start. +if [ -f /etc/systemd/system/kubelet.service.d/11-kind.conf ]; then + sed -i '/create-kubelet-cgroup-v2/d' /etc/systemd/system/kubelet.service.d/11-kind.conf + systemctl daemon-reload +fi +systemctl stop kubelet 2>/dev/null || true + +# Initialize the control plane using a kubeadm config that forces cgroupfs and +# disables swap checking. +set +e +kubeadm init \ + --ignore-preflight-errors=all \ + --config /dev/stdin <<'KUBEADM_CONFIG' +apiVersion: kubeadm.k8s.io/v1beta4 +kind: InitConfiguration +nodeRegistration: + criSocket: unix:///run/containerd/containerd.sock + ignorePreflightErrors: + - all +--- +apiVersion: kubeadm.k8s.io/v1beta4 +kind: ClusterConfiguration +networking: + podSubnet: 10.244.0.0/16 +apiServer: + certSANs: + - "127.0.0.1" + - "localhost" +--- +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +cgroupDriver: cgroupfs +failSwapOn: false +KUBEADM_CONFIG +rc=$? +set -e + +if [ $rc -ne 0 ]; then + echo "=== kubeadm init failed (rc=$rc) — collecting diagnostics ===" >&2 + systemctl status kubelet --no-pager 2>&1 || true + journalctl -xeu kubelet --no-pager -n 80 2>&1 || true + echo "=== cgroup info ===" >&2 + mount | grep cgroup || true + cat /proc/self/cgroup 2>/dev/null || true + ls /sys/fs/cgroup/ 2>/dev/null || true + exit 1 +fi + +export KUBECONFIG=/etc/kubernetes/admin.conf + +# Single node cluster: allow workloads to schedule on the control-plane node. +kubectl taint nodes --all node-role.kubernetes.io/control-plane- 2>/dev/null || true + +# The bundled kindnet manifest (/kind/manifests/default-cni.yaml) contains Go +# template placeholders (e.g. {{ .PodSubnet }}) that kind normally renders at +# cluster creation time. Render the template and apply; if that fails, install +# a standalone CNI config for host-local networking on this single node. +if ! sed -e 's/{{ \.PodSubnet }}/10.244.0.0\/16/g' \ + -e 's/{{ \.PodSubnet}}/10.244.0.0\/16/g' \ + -e 's/{{\.PodSubnet}}/10.244.0.0\/16/g' \ + -e '/^{{/d' -e '/^}}/d' \ + /kind/manifests/default-cni.yaml | kubectl apply -f - 2>&1; then + echo "kindnet manifest apply failed; installing host-local CNI config directly" + mkdir -p /etc/cni/net.d + cat > /etc/cni/net.d/10-kindnet.conflist <<'CNI' +{ + "cniVersion": "0.4.0", + "name": "kindnet", + "plugins": [ + { + "type": "ptp", + "ipMasq": false, + "ipam": { "type": "host-local", "dataDir": "/run/cni-ipam-state", "routes": [{"dst": "0.0.0.0/0"}], "ranges": [[{"subnet": "10.244.0.0/24"}]] } + }, + { "type": "portmap", "capabilities": {"portMappings": true} } + ] +} +CNI +fi + +# Assign podCIDR to the node so kubelet can allocate pod IPs. +NODE=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') +kubectl patch node "$NODE" -p '{"spec":{"podCIDR":"10.244.0.0/24","podCIDRs":["10.244.0.0/24"]}}' 2>/dev/null || true + +kubectl wait --for=condition=Ready nodes --all --timeout=300s diff --git a/.github/e2e/kind-wsl.yaml b/.github/e2e/kind-wsl.yaml new file mode 100644 index 000000000..bf6a71c22 --- /dev/null +++ b/.github/e2e/kind-wsl.yaml @@ -0,0 +1,21 @@ +# oci-to-wsl profile that imports the kindest/node image into WSL as a +# systemd-enabled distribution. Unlike the regular "kind in Docker" flow, the +# Kubernetes node runs natively inside WSL, so no Docker daemon or nested +# containers are required on the Windows runner. +name: kind +image: kindest/node:v1.32.2 +install_dir: C:\WSL\kind + +# kindest/node expects an init system; WSL boots systemd when configured. +wsl_conf: + mode: merge + content: | + [boot] + systemd=true + +# Stage the bootstrap script that turns the imported rootfs into a running +# single node control plane. Path is resolved relative to this profile. +files: + - src: ./bootstrap-kind.sh + dst: /usr/local/bin/bootstrap-kind.sh + mode: "0755" diff --git a/.github/workflows/buildtest.yaml b/.github/workflows/buildtest.yaml index 19c13b36b..7de489af0 100644 --- a/.github/workflows/buildtest.yaml +++ b/.github/workflows/buildtest.yaml @@ -87,6 +87,84 @@ jobs: exit 1 fi + # Run the .NET Framework 4.8 e2e suite (KubernetesClient.Classic) against a + # real Kubernetes cluster on Windows. There is no native Windows Kubernetes + # distribution, so the kindest/node image (which bundles containerd, kubelet + # and the control-plane components) is imported into WSL2 with oci-to-wsl. + # The Linux e2e job already covers the modern (net8/9/10) client libraries. + # WSL2 requires windows-2025 (windows-2022 only has WSL1). + e2e-windows-wsl: + runs-on: windows-2025 + name: e2e classic (wsl) + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + - name: Setup dotnet + uses: actions/setup-dotnet@9a946fdbd5fb07b82b2f5a4466058b876ab72bb2 # v5.3.0 + with: + dotnet-version: 10.0.x + - name: Import kindest/node into WSL + uses: tg123/oci-to-wsl@902e4b0d7f3e7608dbd581e9e2fd473907a279a8 # main + with: + profile: .github/e2e/kind-wsl.yaml + - name: Start kind cluster in WSL + shell: pwsh + run: | + # Reboot the distro so the systemd configuration written by the profile + # takes effect. + wsl --shutdown + + # WSL terminates the lightweight VM once its last session exits, which + # tears down the API server between steps. Hold an idle session open in + # the background so the cluster stays up for the host-side test steps. + Start-Process -FilePath wsl -ArgumentList '-d','kind','-u','root','--','sleep','infinity' + + # Bring up the single node control plane in place. + wsl -d kind -u root -- bash /usr/local/bin/bootstrap-kind.sh + + # Get the WSL2 VM's IP address. In NAT mode the host can reach this + # IP directly via the virtual switch. + $wslIp = ((wsl -d kind -- hostname -I).Trim() -split '\s+', 0, 'RegexMatch' | Where-Object { $_ -match '^\d+\.\d+\.\d+\.\d+$' } | Select-Object -First 1) + if (-not $wslIp) { throw "Could not determine WSL IP" } + Write-Output "WSL IP: $wslIp" + + # Export a kubeconfig for the Windows host, rewriting the server URL + # to the WSL VM's IP address reachable from Windows. + wsl -d kind -u root -- cat /etc/kubernetes/admin.conf | Out-File -Encoding ascii kind.kubeconfig + (Get-Content kind.kubeconfig) -replace 'server: https://[^ ]+', "server: https://${wslIp}:6443" | Set-Content kind.kubeconfig + + # .NET Framework 4.8 does not support custom trust stores, so skip TLS + # verification for the self-signed K8s CA (this is a CI-only cluster). + (Get-Content kind.kubeconfig) -replace 'certificate-authority-data:.*', 'insecure-skip-tls-verify: true' | Set-Content kind.kubeconfig + + # Wait until the API server is reachable from the Windows host. + $ok = $false + foreach ($i in 1..60) { + if ((Test-NetConnection -ComputerName $wslIp -Port 6443 -WarningAction SilentlyContinue).TcpTestSucceeded) { + $ok = $true + break + } + Start-Sleep -Seconds 5 + } + if (-not $ok) { + throw "API server on ${wslIp}:6443 was not reachable from the Windows host" + } + - name: Test + shell: pwsh + env: + K8S_E2E_MINIKUBE: "1" + KUBECONFIG: ${{ github.workspace }}\kind.kubeconfig + run: | + [IO.File]::WriteAllText("$PWD/skip.log", "") + dotnet test tests/E2E.Classic.Tests --logger "SkipTestLogger;file=$PWD/skip.log" -p:BuildInParallel=false + $skipContent = [IO.File]::ReadAllText("$PWD/skip.log").Trim() + if ($skipContent.Length -gt 0) { + Write-Output $skipContent + Write-Error "CASES MUST NOT BE SKIPPED" + exit 1 + } + on: pull_request: types: [assigned, opened, synchronize, reopened] diff --git a/kubernetes-client.proj b/kubernetes-client.proj index 9f634d328..4d4e982e9 100644 --- a/kubernetes-client.proj +++ b/kubernetes-client.proj @@ -1,7 +1,8 @@ - + + diff --git a/tests/E2E.Classic.Tests/E2E.Classic.Tests.csproj b/tests/E2E.Classic.Tests/E2E.Classic.Tests.csproj new file mode 100644 index 000000000..a2202c9b0 --- /dev/null +++ b/tests/E2E.Classic.Tests/E2E.Classic.Tests.csproj @@ -0,0 +1,30 @@ + + + false + k8s.E2E + net48 + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + + diff --git a/tests/E2E.Classic.Tests/MinikubeTests.cs b/tests/E2E.Classic.Tests/MinikubeTests.cs new file mode 100644 index 000000000..5ad8b11cb --- /dev/null +++ b/tests/E2E.Classic.Tests/MinikubeTests.cs @@ -0,0 +1,240 @@ +using k8s.Autorest; +using k8s.Models; +using System; +using System.Collections.Concurrent; +using System.IO; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace k8s.E2E +{ + [Collection(nameof(Onebyone))] + public class MinikubeTests + { + static MinikubeTests() + { + // .NET Framework 4.8 defaults to TLS 1.0/1.1; K8s API server requires TLS 1.2+ + ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12; + } + + [MinikubeFact] + public void SimpleTest() + { + var namespaceParameter = "default"; + var podName = "k8scsharp-e2e-pod"; + + using var client = CreateClient(); + + void Cleanup() + { + var pods = client.CoreV1.ListNamespacedPod(namespaceParameter); + while (pods.Items.Any(p => p.Metadata.Name == podName)) + { + try + { + client.CoreV1.DeleteNamespacedPod(podName, namespaceParameter); + } + catch (HttpOperationException e) + { + if (e.Response.StatusCode == System.Net.HttpStatusCode.NotFound) + { + return; + } + } + } + } + + try + { + Cleanup(); + + client.CoreV1.CreateNamespacedPod( + new V1Pod() + { + Metadata = new V1ObjectMeta { Name = podName, }, + Spec = new V1PodSpec + { + Containers = new[] { new V1Container() { Name = "k8scsharp-e2e", Image = "nginx", }, }, + }, + }, + namespaceParameter); + + var pods = client.CoreV1.ListNamespacedPod(namespaceParameter); + Assert.Contains(pods.Items, p => p.Metadata.Name == podName); + } + finally + { + Cleanup(); + } + } + + [MinikubeFact] + public async Task LogStreamTestAsync() + { + var namespaceParameter = "default"; + var podName = "k8scsharp-e2e-logstream-pod"; + + using var client = CreateClient(); + + void Cleanup() + { + var pods = client.CoreV1.ListNamespacedPod(namespaceParameter); + while (pods.Items.Any(p => p.Metadata.Name == podName)) + { + try + { + client.CoreV1.DeleteNamespacedPod(podName, namespaceParameter); + } + catch (HttpOperationException e) + { + if (e.Response.StatusCode == System.Net.HttpStatusCode.NotFound) + { + return; + } + } + } + } + + try + { + Cleanup(); + + client.CoreV1.CreateNamespacedPod( + new V1Pod() + { + Metadata = new V1ObjectMeta { Name = podName, }, + Spec = new V1PodSpec + { + Containers = new[] + { + new V1Container() + { + Name = "k8scsharp-e2e-logstream", + Image = "busybox", + Command = new[] { "ping" }, + Args = new[] { "-i", "10", "127.0.0.1" }, + }, + }, + }, + }, + namespaceParameter); + + var lines = new ConcurrentQueue(); + using var started = new ManualResetEvent(false); + + async Task Pod() + { + var deadline = DateTime.UtcNow.AddMinutes(2); + while (true) + { + var pods = client.CoreV1.ListNamespacedPod(namespaceParameter); + var pod = pods.Items.First(p => p.Metadata.Name == podName); + if (pod.Status.Phase == "Running") + { + return pod; + } + + if (DateTime.UtcNow > deadline) + { + throw new TimeoutException($"Pod {podName} did not become Running within 2 minutes (last phase: {pod.Status.Phase})"); + } + + await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); + } + } + + var pod = await Pod().ConfigureAwait(false); + var stream = client.CoreV1.ReadNamespacedPodLog(pod.Metadata.Name, pod.Metadata.NamespaceProperty, follow: true); + using var reader = new StreamReader(stream); + + var copytask = Task.Run(() => + { + try + { + for (; ; ) + { + var line = reader.ReadLine(); + if (line == null) + { + break; + } + + lines.Enqueue(line); + started.Set(); + } + } + catch (ObjectDisposedException) + { + // Reader was disposed during test teardown; normal shutdown. + } + finally + { + started.Set(); + } + }); + + Assert.True(started.WaitOne(TimeSpan.FromMinutes(2))); + await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); + Assert.Null(copytask.Exception); + Assert.Equal(2, lines.Count); + await Task.Delay(TimeSpan.FromSeconds(11)).ConfigureAwait(false); + Assert.Equal(3, lines.Count); + } + finally + { + Cleanup(); + } + } + + [MinikubeFact] + public async Task DatetimeFieldTest() + { + using var kubernetes = CreateClient(); + + await kubernetes.CoreV1.CreateNamespacedEventAsync( + new Corev1Event + { + InvolvedObject = new V1ObjectReference + { + ApiVersion = "v1alpha1", + Kind = "Test", + Name = "test", + NamespaceProperty = "default", + ResourceVersion = "1", + Uid = "1", + }, + Metadata = new V1ObjectMeta + { + GenerateName = "started-", + }, + Action = "STARTED", + Type = "Normal", + Reason = "STARTED", + Message = "Started", + EventTime = DateTime.Now, + FirstTimestamp = DateTime.Now, + LastTimestamp = DateTime.Now, + ReportingComponent = "37", + ReportingInstance = "38", + }, + "default" + ).ConfigureAwait(false); + } + + [MinikubeFact] + public async Task VersionTestAsync() + { + using var client = CreateClient(); + var version = await client.Version.GetCodeAsync().ConfigureAwait(false); + Assert.NotNull(version); + } + + public static IKubernetes CreateClient() + { + return new Kubernetes(KubernetesClientConfiguration.BuildDefaultConfig()); + } + } +} diff --git a/tests/KubernetesClient.Classic.Tests/KubernetesClient.Classic.Tests.csproj b/tests/KubernetesClient.Classic.Tests/KubernetesClient.Classic.Tests.csproj deleted file mode 100644 index c3fb239da..000000000 --- a/tests/KubernetesClient.Classic.Tests/KubernetesClient.Classic.Tests.csproj +++ /dev/null @@ -1,37 +0,0 @@ - - - false - k8s.Tests - net8.0;net9.0;net10.0 - net8.0;net9.0;net10.0;net48 - - - - - - - - - - - - - - - - - - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - - - - - diff --git a/tests/KubernetesClient.Classic.Tests/SimpleTests.cs b/tests/KubernetesClient.Classic.Tests/SimpleTests.cs deleted file mode 100644 index 3e17a5854..000000000 --- a/tests/KubernetesClient.Classic.Tests/SimpleTests.cs +++ /dev/null @@ -1,84 +0,0 @@ -using k8s.Models; -using System.IO; -using System.Net; -using System.Net.Sockets; -using System.Threading.Tasks; -using Xunit; - -namespace k8s.tests; - -public class SimpleTests -{ - // TODO: fail to setup asp.net core 6 on net48 - private class DummyHttpServer : System.IDisposable - { - private readonly TcpListener server; - private readonly Task loop; - private volatile bool running = false; - - public string Addr => $"http://{server.LocalEndpoint}"; - - public DummyHttpServer(object obj) - { - server = new TcpListener(IPAddress.Parse("127.0.0.1"), 0); - server.Start(); - running = true; - loop = Task.Run(async () => - { - while (running) - { - var result = KubernetesJson.Serialize(obj); - - var client = await server.AcceptTcpClientAsync().ConfigureAwait(false); - var stream = client.GetStream(); - stream.Read(new byte[1024], 0, 1024); // TODO ensure full header - - var writer = new StreamWriter(stream); - await writer.WriteLineAsync("HTTP/1.0 200 OK").ConfigureAwait(false); - await writer.WriteLineAsync("Content-Length: " + result.Length).ConfigureAwait(false); - await writer.WriteLineAsync("Content-Type: application/json").ConfigureAwait(false); - await writer.WriteLineAsync().ConfigureAwait(false); - await writer.WriteLineAsync(result).ConfigureAwait(false); - - await writer.FlushAsync().ConfigureAwait(false); - client.Close(); - } - }); - } - - public void Dispose() - { - try - { - running = false; - server.Stop(); -#if NET8_0_OR_GREATER - server.Dispose(); -#endif - loop.Wait(); - loop.Dispose(); - } - catch - { - // ignore - } - } - } - - [Fact] - public async Task QueryPods() - { - using var server = new DummyHttpServer(new V1Pod() - { - Metadata = new V1ObjectMeta() - { - Name = "pod0", - }, - }); - var client = new Kubernetes(new KubernetesClientConfiguration { Host = server.Addr }); - - var pod = await client.CoreV1.ReadNamespacedPodAsync("pod", "default").ConfigureAwait(true); - - Assert.Equal("pod0", pod.Metadata.Name); - } -}