test-k8s-deployment.yaml

name: CI Test K8s

on:
  pull_request:
    branches:
      - main
      - release/*
    paths:
      - "k8s/**"
      - ".github/workflows/test-k8s-deployment.yaml"
  push:
    branches:
      - main
      - release/*
    paths:
      - "k8s/**"
      - ".github/workflows/test-k8s-deployment.yaml"
  workflow_dispatch: {}

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-test-deployment
  cancel-in-progress: true

jobs:
  k8s-ci-test:
    runs-on: ubuntu-latest
    timeout-minutes: 45
    strategy:
      fail-fast: false
      matrix:
        function:
          - hello-world
          - llm-internal-calvin
          - opencode-headless
          - twilio-sms
          - llm-external
          - send-email-link
          - crypto-login
          - github-repo-creator
          - pytorch-gpu
          - runtime-script
          - rust-hello-world
          - simple-bash
          - simple-email
          - stripe-function

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup kind cluster
        uses: helm/kind-action@v1
        with:
          cluster_name: local
          wait: 120s

      - name: Install jq (for resource diagnostics)
        run: |
          sudo apt-get update
          sudo apt-get install -y jq

      - name: Verify cluster
        run: |
          kubectl version
          kubectl get nodes -o wide

      - name: Install Knative (operators-knative-only)
        run: |
          cd k8s/scripts/setup
          make operators-knative-only

      - name: Trim Knative resources for CI
        run: |
          set -e

          echo "=== Shrinking Knative control-plane resources ==="

          echo "Patching knative-serving activator..."
          kubectl -n knative-serving set resources deploy/activator \
            --requests=cpu=50m,memory=80Mi --limits=cpu=200m,memory=256Mi \
            || echo "WARN: failed to patch activator"

          echo "Patching knative-serving autoscaler..."
          kubectl -n knative-serving set resources deploy/autoscaler \
            --requests=cpu=25m,memory=80Mi --limits=cpu=200m,memory=256Mi \
            || echo "WARN: failed to patch autoscaler"

          echo "Patching knative-serving controller..."
          kubectl -n knative-serving set resources deploy/controller \
            --requests=cpu=50m,memory=100Mi --limits=cpu=300m,memory=512Mi \
            || echo "WARN: failed to patch controller"

          echo "Patching knative-serving webhook..."
          kubectl -n knative-serving set resources deploy/webhook \
            --requests=cpu=25m,memory=80Mi --limits=cpu=200m,memory=256Mi \
            || echo "WARN: failed to patch webhook"

          echo "Patching knative net-kourier controller..."
          kubectl -n knative-serving set resources deploy/net-kourier-controller \
            --requests=cpu=25m,memory=80Mi --limits=cpu=200m,memory=256Mi \
            || echo "WARN: failed to patch net-kourier-controller"

          echo "Patching kourier gateway..."
          kubectl -n kourier-system set resources deploy/3scale-kourier-gateway \
            --requests=cpu=25m,memory=80Mi --limits=cpu=200m,memory=256Mi \
            || echo "WARN: failed to patch 3scale-kourier-gateway"

          echo "Optionally shrinking coredns..."
          kubectl -n kube-system scale deploy/coredns --replicas=1 || echo "WARN: failed to scale coredns"
          kubectl -n kube-system set resources deploy/coredns \
            --requests=cpu=50m,memory=70Mi --limits=cpu=200m,memory=170Mi \
            || echo "WARN: failed to patch coredns"

          echo "Restarting control-plane pods so resource changes take effect..."
          kubectl -n knative-serving rollout restart deploy/activator deploy/autoscaler deploy/controller deploy/webhook deploy/net-kourier-controller || true
          kubectl -n kourier-system rollout restart deploy/3scale-kourier-gateway || true
          kubectl -n kube-system rollout restart deploy/coredns || true

      - name: Create ghcr-pull image pull secret (optional)
        run: |
          set -e
          if [ -n "${GHCR_USERNAME:-}" ] && [ -n "${GHCR_TOKEN:-}" ]; then
            echo "Creating ghcr-pull secret in default namespace"
            kubectl create namespace default --dry-run=client -o yaml | kubectl apply -f -
            kubectl create secret docker-registry ghcr-pull \
              --docker-server=ghcr.io \
              --docker-username="${GHCR_USERNAME}" \
              --docker-password="${GHCR_TOKEN}" \
              --docker-email="${GHCR_EMAIL:-devnull@example.com}" \
              --dry-run=client -o yaml | kubectl apply -n default -f -
          else
            echo "GHCR_USERNAME/GHCR_TOKEN not set; assuming dashboard/db-job images are public."
          fi
        env:
          GHCR_USERNAME: ${{ secrets.GH_USERNAME }}
          GHCR_TOKEN: ${{ secrets.GH_PAT_TOKEN }}
          GHCR_EMAIL: ${{ secrets.GH_EMAIL }}

      - name: Sleep for a bit before install
        run: |
          sleep 10

      - name: Apply CI overlay (on top of local)
        run: |
          cd k8s
          kubectl kustomize overlays/ci --load-restrictor=LoadRestrictionsNone | kubectl apply -f -
      
      - name: Sleep for a bit
        run: |
          sleep 100

      - name: Dump pod resource requests/limits (all namespaces)
        if: always()
        run: |
          echo "=== Per-pod resource requests/limits ==="
          kubectl get pods -A -o json \
            | jq -r '
                .items[]
                | .metadata.namespace as $ns
                | .metadata.name as $pod
                | .spec.containers[]
                | [$ns, $pod, .name,
                   (.resources.requests.cpu // "0"),
                   (.resources.requests.memory // "0"),
                   (.resources.limits.cpu // "0"),
                   (.resources.limits.memory // "0")]
                | @tsv
              ' \
            | column -t

          echo
          echo "=== Aggregated resource requests per namespace (CPU cores, MiB) ==="
          kubectl get pods -A -o json \
            | jq -r '
                [ .items[]
                  | .metadata.namespace as $ns
                  | {
                      ns: $ns,
                      cpu: ([.spec.containers[].resources.requests.cpu // "0"]
                            | map(
                                if test("m$") then (sub("m$";"") | tonumber / 1000)
                                elif . == "0" then 0
                                else tonumber
                                end
                              )
                            | add),
                      memMi: ([.spec.containers[].resources.requests.memory // "0"]
                              | map(
                                  if test("Gi$") then (sub("Gi$";"") | tonumber * 1024)
                                  elif test("Mi$") then (sub("Mi$";"") | tonumber)
                                  elif . == "0" then 0
                                  else 0
                                  end
                                )
                              | add)
                    }
                ]
                | group_by(.ns)
                | map({ns: .[0].ns, cpu: (map(.cpu) | add), memMi: (map(.memMi) | add)})
                | sort_by(.ns)
                | .[]
                | [.ns, ( .cpu | tostring ), ( .memMi | tostring )]
                | @tsv
              ' \
            | awk 'BEGIN{print "NAMESPACE\tCPU_CORES\tMEM_MIB"} {print}'

      - name: Wait for local workloads
        continue-on-error: true
        run: |
          set -e
          echo "Namespaces:" && kubectl get ns
          echo "All pods (initial):" && kubectl get pods -A

          echo "All pods (final):" && kubectl get pods -A
          echo "Knative services:" && kubectl get ksvc -A || true

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '20'

      - name: Install pnpm
        uses: pnpm/action-setup@v2
        with:
          version: 9

      - name: Install dependencies
        run: pnpm install --no-frozen-lockfile

      - name: Build and Load Test Runner Image
        run: |
          make build-test-runner KIND_CLUSTER_NAME=local

      - name: Run K8s Tests
        run: |
          # Ensure kubectl proxy port is available or managed by the runner
          pnpm exec ts-node scripts/test-runner.ts --function ${{ matrix.function }}


      - name: Dump diagnostics on failure
        if: always()
        run: |
          echo "=== Namespaces ==="
          kubectl get ns

          echo "=== Pods (all namespaces) ==="
          kubectl get pods -A -o wide || true

          echo "=== Events (default) ==="
          kubectl get events --sort-by=.lastTimestamp || true

          echo "=== Deployments (default) ==="
          kubectl get deploy -o wide || true

          echo "=== Logs for default pods ==="
          for pod in $(kubectl get pods -o jsonpath='{.items[*].metadata.name}'); do
            echo "--------------------------------------------------"
            echo "Logs for $pod"
            kubectl describe pod "$pod" || true
            kubectl logs "$pod" --all-containers --tail=200 || true
          done