ref(smoketest): break out lengthy script; also remove containerdruntimeoptions if k3d

Signed-off-by: Vaughn Dice <vdice@akamai.com>

Author: vdice

.github/workflows/helm-chart-smoketest.yml

@@ -11,6 +11,8 @@ env:
   MICROK8S_CHANNEL: 1.32/stable
   SHIM_SPIN_VERSION: v0.23.0
   DOCKER_BUILD_SUMMARY: false
+  # A comma-delimited list of shim files in config/samples to test
+  SAMPLE_SHIMS: sample_shim_spin.yaml,sample_shim_wasmtime.yaml
 
 jobs:
   build-images:
@@ -156,58 +158,28 @@ jobs:
             --set rcm.shimDownloaderConfig.content.sleepDuration=3 \
             deploy/helm
 
-      - name: verify configMap for shim-downloader is added
-        run: |
-          timeout 60s bash -c 'until [[ $(kubectl -n rcm get $(kubectl get pods -n rcm --no-headers -o name | grep install | head -n1) -o jsonpath="{.spec.initContainers[0].envFrom[0].configMapRef.name}" 2>/dev/null) == "configmap-test" ]]; do sleep 2; done'
-
       - name: apply shims
+        working-directory: config/samples
         run: |
-          for shim_file in $(ls config/samples/sample_shim*); do
-            if [[ "${{ matrix.config.type }}" == "microk8s" ]]; then
-              cp $shim_file $shim_file.microk8s
-              shim_file=$shim_file.microk8s
+          IFS=',' read -ra shims <<< "${{ env.SAMPLE_SHIMS }}"
+          for shim_file in "${shims[@]}"; do
+            if [[ "${{ matrix.config.type }}" == "microk8s" || "${{ matrix.config.type }}" == "k3d" ]]; then
+              cp $shim_file $shim_file.amended
+              shim_file=$shim_file.amended
               # update file to remove the 'containerdRuntimeOptions' field
               # as there is a known bug that MicroK8s containerd does not pass the options
+              # and k3d uses cgroupfs, not systemd
               yq -i 'del(.spec.containerdRuntimeOptions)' $shim_file
             fi
             kubectl apply -f $shim_file
           done
 
-      - name: label nodes and wait for shim to be ready
-        run: |
-          for shim_file in $(ls config/samples/sample_shim*); do
-            label="$(cat $shim_file | yq '.spec.nodeSelector' | tr -d '"' | tr -d '[:space:]' | sed s/:/=/g)"
-            kubectl label node --all $label
+      - name: label nodes and wait for shims to be ready
+        run: ./scripts/sample-shims-label-nodes.sh
 
-            shim_name="$(cat $shim_file | yq '.metadata.name')"
-            # TODO: k3d can take a long round of failed install pods (exit code 6 when curling the artifact?)
-            # Once this behavior is diagnosed and resolved, we should be able to shorten this timeout substantially
-            timeout=600
-            SECONDS=0 # Reset the internal bash timer to 0
-            success=false
-
-            echo "Waiting for the $shim_name shim to be ready/installed..."
-
-            while [[ $SECONDS -lt $timeout ]]; do
-              # Fetch both nodes and nodesReady
-              read -r nodes nodesReady <<< $(kubectl get shim "$shim_name" \
-                -o jsonpath='{.status.nodes} {.status.nodesReady}' 2>/dev/null)
-
-              # Check to see if all nodes are ready
-              if [[ -n "$nodes" ]] && [[ -n "$nodesReady" ]] && [[ "$nodes" -eq "$nodesReady" ]]; then
-                echo "Success: all nodes have the $shim_name shim installed."
-                success=true
-                break
-              fi
-
-              sleep 2
-            done
-
-            if [[ "${success}" != "true" ]]; then
-              echo "Error: Timed out after ${timeout}s waiting for the $shim_name shim to be ready."
-              exit 1
-            fi
-          done
+      - name: verify configMap for shim-downloader is added
+        run: |
+          timeout 60s bash -c 'until [[ $(kubectl -n rcm get $(kubectl get pods -n rcm --no-headers -o name | grep install | head -n1) -o jsonpath="{.spec.initContainers[0].envFrom[0].configMapRef.name}" 2>/dev/null) == "configmap-test" ]]; do sleep 2; done'
 
       # TODO: unify testdata/apps to all model the same behavor, eg simple web server, etc
       - name: run and verify spin app
@@ -228,11 +200,12 @@ jobs:
         if: failure()
         run: |
           kubectl get pods -A
-          kubectl describe shims
-          kubectl describe runtimeclasses
+          kubectl describe shims || true
+          kubectl describe runtimeclasses || true
 
           # Get install pod logs
-          for shim_file in $(ls config/samples/sample_shim*); do
+          IFS=',' read -ra shims <<< "${{ env.SAMPLE_SHIMS }}"
+          for shim_file in "${shims[@]}"; do
             shim_name="$(cat $shim_file | yq '.metadata.name')"
             install_pod="$(kubectl get pods -n rcm --no-headers -o name | grep $shim_name-install)"
             kubectl describe -n rcm $install_pod || true
@@ -245,7 +218,7 @@ jobs:
           kubectl describe -n rcm pod -l app.kubernetes.io/name=runtime-class-manager || true
 
           # App logs
-          for app_file in $(ls testdata/apps/*); do
+          for app_file in testdata/apps/spin-v2-app.yaml testdata/apps/wasmtime-v1-app.yaml; do
             app=$(cat $app_file | yq 'select(.kind == "Deployment") | .metadata.name')
             kubectl logs -l app=$app || true
             kubectl describe pod -l app=$app || true

scripts/sample-shims-label-nodes.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+set -euo pipefail
+
+SCRIPT_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
+
+# Currently used in .github/workflows/helm-chart-smoketest.yml
+# Iterates through all sample shims and applies their labels to all nodes
+# using the current kubernetes context.
+# Waits for all corresponding Shim resources to be ready, else fails.
+
+IFS=',' read -ra shims <<< "${SAMPLE_SHIMS}"
+for shim_file in "${shims[@]}"; do
+  cd "${SCRIPT_DIR}/../config/samples"
+  label="$(cat $shim_file | yq '.spec.nodeSelector' | tr -d '"' | tr -d '[:space:]' | sed s/:/=/g)"
+  kubectl label node --all $label
+
+  shim_name="$(cat $shim_file | yq '.metadata.name')"
+  timeout=300
+  SECONDS=0 # Reset the internal bash timer to 0
+  success=false
+
+  echo "Waiting for the $shim_name shim to be ready/installed..."
+
+  while [[ $SECONDS -lt $timeout ]]; do
+    # Fetch both nodes and nodesReady
+    read -r nodes nodesReady <<< $(kubectl get shim "$shim_name" \
+      -o jsonpath='{.status.nodes} {.status.nodesReady}' 2>/dev/null)
+
+    # Check to see if all nodes are ready
+    if [[ -n "$nodes" ]] && [[ -n "$nodesReady" ]] && [[ "$nodes" -eq "$nodesReady" ]]; then
+      echo "Success: all nodes have the $shim_name shim installed."
+      success=true
+      break
+    fi
+
+    sleep 2
+  done
+
+  if [[ "${success}" != "true" ]]; then
+    echo "Error: Timed out after ${timeout}s waiting for the $shim_name shim to be ready."
+    exit 1
+  fi
+done

testdata/apps/wasmtime-v1-app.yaml

@@ -15,4 +15,4 @@ spec:
       runtimeClassName: wasmtime-v1
       containers:
       - name: wasmtime-v1-app
-        image: ghcr.io/containerd/runwasi/wasi-demo-app:latest
+        image: ghcr.io/containerd/runwasi/wasi-demo-app:0.2.0