Merge pull request #605 from vdice/ci/update-smoketest-add-wasmtime-coverage

ci(*): update helm-chart-smoketest.yml, add wasmtime-v1 coverage

Author: vdice

.github/workflows/helm-chart-smoketest.yml

@@ -11,6 +11,8 @@ env:
   MICROK8S_CHANNEL: 1.32/stable
   SHIM_SPIN_VERSION: v0.23.0
   DOCKER_BUILD_SUMMARY: false
+  # A comma-delimited list of shim files in config/samples to test
+  SAMPLE_SHIMS: sample_shim_spin.yaml,sample_shim_wasmtime.yaml
 
 jobs:
   build-images:
@@ -156,61 +158,68 @@ jobs:
             --set rcm.shimDownloaderConfig.content.sleepDuration=3 \
             deploy/helm
 
-      - name: apply Spin shim
+      - name: apply shims
+        working-directory: config/samples
         run: |
-          shim_file=config/samples/test_shim_spin.yaml
-          if [[ "${{ matrix.config.type }}" == "microk8s" ]]; then
-            cp $shim_file config/samples/test_shim_spin_microk8s.yaml
-            shim_file=config/samples/test_shim_spin_microk8s.yaml
-            # update file to remove the 'containerdRuntimeOptions' field
-            # as there is a known bug that MicroK8s containerd does not pass the options
-            yq -i 'del(.spec.containerdRuntimeOptions)' $shim_file
-          fi
-          kubectl apply -f $shim_file
-
-      - name: label nodes
-        run: kubectl label node --all spin=true
-
-      - name: verify only one installer pod with Succeeded status
-        run: |
-          timeout 60s bash -c 'until [[ "$(kubectl -n rcm get $(kubectl get pods -n rcm --no-headers -o name | grep install | head -n1) -o jsonpath="{.status.phase}" 2>/dev/null)" == "Succeeded" ]]; do sleep 2; done'
+          IFS=',' read -ra shims <<< "${{ env.SAMPLE_SHIMS }}"
+          for shim_file in "${shims[@]}"; do
+            if [[ "${{ matrix.config.type }}" == "microk8s" || "${{ matrix.config.type }}" == "k3d" ]]; then
+              cp $shim_file $shim_file.amended
+              shim_file=$shim_file.amended
+              # update file to remove the 'containerdRuntimeOptions' field
+              # as there is a known bug that MicroK8s containerd does not pass the options
+              # and k3d uses cgroupfs, not systemd
+              yq -i 'del(.spec.containerdRuntimeOptions)' $shim_file
+            fi
+            kubectl apply -f $shim_file
+          done
+
+      - name: label nodes and wait for shims to be ready
+        run: ./scripts/sample-shims-label-nodes.sh
 
       - name: verify configMap for shim-downloader is added
         run: |
           timeout 60s bash -c 'until [[ $(kubectl -n rcm get $(kubectl get pods -n rcm --no-headers -o name | grep install | head -n1) -o jsonpath="{.spec.initContainers[0].envFrom[0].configMapRef.name}" 2>/dev/null) == "configmap-test" ]]; do sleep 2; done'
 
-      - name: run Spin App
+      # TODO: unify testdata/apps to all model the same behavor, eg simple web server, etc
+      - name: run and verify spin app
         run: |
-          kubectl apply -f testdata/apps/spin-app.yaml
-          kubectl rollout status deployment wasm-spin --timeout 180s
-          kubectl get pods -A
-          kubectl port-forward svc/wasm-spin 8083:80 &
-          timeout 90s bash -c 'until curl -f -vvv http://localhost:8083/hello; do sleep 2; done'
+          kubectl apply -f testdata/apps/spin-v2-app.yaml
+          kubectl rollout status deployment spin-v2-app --timeout 180s
+          kubectl port-forward svc/spin-v2-app 8083:80 &
+          timeout 60s bash -c 'until curl -f -vvv http://localhost:8083/hello; do sleep 2; done'
 
-      - name: restart system containerd
-        if: matrix.config.type == 'microk8s'
-        run: sudo systemctl start containerd
+      - name: run and verify wasmtime app
+        run: |
+          kubectl apply -f testdata/apps/wasmtime-v1-app.yaml
+          kubectl rollout status deployment wasmtime-v1-app --timeout 180s
+          pod=$(kubectl get pod -l app=wasmtime-v1-app -o name)
+          timeout 60s bash -c 'until kubectl logs $1 | grep -q "This is a song that never ends."; do sleep 2; done' -- $pod
 
       - name: debug
         if: failure()
         run: |
           kubectl get pods -A
-          kubectl describe shim spin-v2
-          kubectl describe runtimeclass wasmtime-spin-v2
+          kubectl describe shims || true
+          kubectl describe runtimeclasses || true
 
           # Get install pod logs
-          install_pod=$(kubectl get pods -n rcm --no-headers -o name | awk '{if ($1 ~ "-spin-v2-install") print $0}')
-          kubectl describe -n rcm $install_pod || true
-          kubectl logs -n rcm -c downloader $install_pod || true
-          kubectl logs -n rcm -c provisioner $install_pod || true
+          IFS=',' read -ra shims <<< "${{ env.SAMPLE_SHIMS }}"
+          for shim_file in "${shims[@]}"; do
+            shim_name="$(cat $shim_file | yq '.metadata.name')"
+            install_pod="$(kubectl get pods -n rcm --no-headers -o name | grep $shim_name-install)"
+            kubectl describe -n rcm $install_pod || true
+            kubectl logs -n rcm -c downloader $install_pod || true
+            kubectl logs -n rcm -c provisioner $install_pod || true
+          done
 
           # RCM pod logs
           kubectl logs -n rcm -l app.kubernetes.io/name=runtime-class-manager || true
           kubectl describe -n rcm pod -l app.kubernetes.io/name=runtime-class-manager || true
 
           # App logs
-          kubectl logs -l app=wasm-spin || true
-          kubectl describe pod -l app=wasm-spin || true
-
-      - name: Verify curl
-        run: curl localhost:8083/hello
+          for app_file in testdata/apps/spin-v2-app.yaml testdata/apps/wasmtime-v1-app.yaml; do
+            app=$(cat $app_file | yq 'select(.kind == "Deployment") | .metadata.name')
+            kubectl logs -l app=$app || true
+            kubectl describe pod -l app=$app || true
+          done

config/samples/sample_shim_wasmtime.yaml

@@ -23,7 +23,7 @@ spec:
 
   runtimeClass:
     name: wasmtime-v1
-    handler: wasmtime
+    handler: wasmtime-v1
 
   rolloutStrategy:
     type: recreate

scripts/sample-shims-label-nodes.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+set -euo pipefail
+
+SCRIPT_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
+
+# Currently used in .github/workflows/helm-chart-smoketest.yml
+# Iterates through all sample shims and applies their labels to all nodes
+# using the current kubernetes context.
+# Waits for all corresponding Shim resources to be ready, else fails.
+
+IFS=',' read -ra shims <<< "${SAMPLE_SHIMS}"
+for shim_file in "${shims[@]}"; do
+  cd "${SCRIPT_DIR}/../config/samples"
+  label="$(cat $shim_file | yq '.spec.nodeSelector' | tr -d '"' | tr -d '[:space:]' | sed s/:/=/g)"
+  kubectl label node --all $label
+
+  shim_name="$(cat $shim_file | yq '.metadata.name')"
+  timeout=300
+  SECONDS=0 # Reset the internal bash timer to 0
+  success=false
+
+  echo "Waiting for the $shim_name shim to be ready/installed..."
+
+  while [[ $SECONDS -lt $timeout ]]; do
+    # Fetch both nodes and nodesReady
+    read -r nodes nodesReady <<< $(kubectl get shim "$shim_name" \
+      -o jsonpath='{.status.nodes} {.status.nodesReady}' 2>/dev/null)
+
+    # Check to see if all nodes are ready
+    if [[ -n "$nodes" ]] && [[ -n "$nodesReady" ]] && [[ "$nodes" -eq "$nodesReady" ]]; then
+      echo "Success: all nodes have the $shim_name shim installed."
+      success=true
+      break
+    fi
+
+    sleep 2
+  done
+
+  if [[ "${success}" != "true" ]]; then
+    echo "Error: Timed out after ${timeout}s waiting for the $shim_name shim to be ready."
+    exit 1
+  fi
+done

testdata/apps/spin-app.yaml testdata/apps/spin-v2-app.yamltestdata/apps/spin-app.yaml renamed to testdata/apps/spin-v2-app.yaml

@@ -2,16 +2,16 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: wasm-spin
+  name: spin-v2-app
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app: wasm-spin
+      app: spin-v2-app
   template:
     metadata:
       labels:
-        app: wasm-spin
+        app: spin-v2-app
     spec:
       runtimeClassName: wasmtime-spin-v2
       containers:
@@ -22,11 +22,11 @@ spec:
 apiVersion: v1
 kind: Service
 metadata:
-  name: wasm-spin
+  name: spin-v2-app
 spec:
   ports:
     - protocol: TCP
       port: 80
       targetPort: 80
   selector:
-    app: wasm-spin
+    app: spin-v2-app

testdata/apps/wasmtime-v1-app.yaml

@@ -0,0 +1,18 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: wasmtime-v1-app
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: wasmtime-v1-app
+  template:
+    metadata:
+      labels:
+        app: wasmtime-v1-app
+    spec:
+      runtimeClassName: wasmtime-v1
+      containers:
+      - name: wasmtime-v1-app
+        image: ghcr.io/containerd/runwasi/wasi-demo-app:0.2.0