Skip to content

Commit 741236d

Browse files
himsnghhimsngh
committed
feat(ci): automate dynamic e2e test kapp-controller against latest & minimum supported kubernetes releases
Signed-off-by: Himanshu Singh <himansh.singh3@gmail.com>
1 parent 7d885ad commit 741236d

2 files changed

Lines changed: 330 additions & 2 deletions

File tree

‎.github/workflows/nightly-build-kind.yml‎

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
name: Nightly Build Kind Images
2+
3+
on:
4+
schedule:
5+
- cron: '30 20 * * *'
6+
workflow_dispatch:
7+
8+
env:
9+
REGISTRY: ghcr.io
10+
11+
jobs:
12+
resolve-versions:
13+
runs-on: ubuntu-latest
14+
timeout-minutes: 10
15+
outputs:
16+
matrix: ${{ steps.set-matrix.outputs.matrix }}
17+
timestamp: ${{ steps.get-date.outputs.timestamp }}
18+
steps:
19+
- name: Get Date Timestamp
20+
id: get-date
21+
run: echo "timestamp=$(date +'%Y%m%d')" >> "$GITHUB_OUTPUT"
22+
23+
- name: Resolve Kubernetes Versions
24+
id: set-matrix
25+
run: |
26+
set -euo pipefail
27+
28+
echo "Fetching upstream Kubernetes branches..."
29+
BRANCHES=$(git ls-remote --heads https://github.com/kubernetes/kubernetes.git \
30+
| awk '{print $2}' | grep -Eo 'release-1\.[0-9]+$' | sort -V)
31+
32+
if [[ -z "$BRANCHES" ]]; then
33+
echo "::error::Failed to fetch Kubernetes branches."
34+
exit 1
35+
fi
36+
37+
LATEST_BRANCH=$(echo "$BRANCHES" | tail -n 1)
38+
MIN_BRANCH=$(echo "$BRANCHES" | tail -n 4 | head -n 1)
39+
40+
echo "Resolved -> Latest: $LATEST_BRANCH | Min (N-3): $MIN_BRANCH"
41+
42+
MATRIX_JSON=$(jq -nc \
43+
--arg latest "$LATEST_BRANCH" \
44+
--arg min "$MIN_BRANCH" \
45+
'{"k8s_branch": [$latest, $min]}')
46+
47+
echo "matrix=$MATRIX_JSON" >> "$GITHUB_OUTPUT"
48+
49+
build-and-push:
50+
needs: resolve-versions
51+
runs-on: ubuntu-latest
52+
timeout-minutes: 90
53+
strategy:
54+
fail-fast: false
55+
matrix: ${{ fromJson(needs.resolve-versions.outputs.matrix) }}
56+
permissions:
57+
contents: read
58+
packages: write
59+
steps:
60+
- name: Free Disk Space
61+
run: |
62+
docker system prune -af --volumes || true
63+
sudo rm -rf /usr/share/dotnet /usr/local/lib/android || true
64+
65+
- name: Setup Environment Variables
66+
run: |
67+
set -euo pipefail
68+
OWNER_LC=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
69+
echo "IMAGE_BASE=${{ env.REGISTRY }}/${OWNER_LC}/kindest-node" >> "$GITHUB_ENV"
70+
71+
BRANCH_SHA=$(git ls-remote https://github.com/kubernetes/kubernetes.git refs/heads/${{ matrix.k8s_branch }} | awk '{print $1}' | cut -c1-7)
72+
echo "BRANCH_SHA=$BRANCH_SHA" >> "$GITHUB_ENV"
73+
74+
- name: Login to GitHub Container Registry
75+
uses: docker/login-action@v3
76+
with:
77+
registry: ghcr.io
78+
username: ${{ github.actor }}
79+
password: ${{ secrets.GITHUB_TOKEN }}
80+
81+
- name: Check GHCR for Cached Build
82+
id: check-cache
83+
run: |
84+
set -euo pipefail
85+
TIMESTAMP="${{ needs.resolve-versions.outputs.timestamp }}"
86+
87+
CACHE_IMAGE="${{ env.IMAGE_BASE }}:${{ matrix.k8s_branch }}-${{ env.BRANCH_SHA }}"
88+
DAILY_IMAGE="${{ env.IMAGE_BASE }}:${{ matrix.k8s_branch }}-${TIMESTAMP}"
89+
90+
echo "CACHE_IMAGE=$CACHE_IMAGE" >> "$GITHUB_ENV"
91+
echo "DAILY_IMAGE=$DAILY_IMAGE" >> "$GITHUB_ENV"
92+
93+
export DOCKER_CLI_EXPERIMENTAL=enabled
94+
if docker manifest inspect "$CACHE_IMAGE" > /dev/null 2>&1; then
95+
echo "Cache Hit! Image already exists: $CACHE_IMAGE"
96+
echo "skip_build=true" >> "$GITHUB_OUTPUT"
97+
else
98+
echo "Cache Miss. Proceeding with source build."
99+
echo "skip_build=false" >> "$GITHUB_OUTPUT"
100+
fi
101+
102+
- name: Checkout Kubernetes
103+
if: steps.check-cache.outputs.skip_build == 'false'
104+
uses: actions/checkout@v4
105+
with:
106+
repository: kubernetes/kubernetes
107+
ref: ${{ matrix.k8s_branch }}
108+
path: kubernetes
109+
fetch-depth: 0
110+
111+
- name: Checkout Kind
112+
if: steps.check-cache.outputs.skip_build == 'false'
113+
uses: actions/checkout@v4
114+
with:
115+
repository: kubernetes-sigs/kind
116+
path: kind
117+
118+
- name: Install Go
119+
if: steps.check-cache.outputs.skip_build == 'false'
120+
uses: actions/setup-go@v5
121+
with:
122+
go-version: '1.24'
123+
124+
- name: Build and Push (Cache Miss)
125+
if: steps.check-cache.outputs.skip_build == 'false'
126+
env:
127+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
128+
run: |
129+
set -euo pipefail
130+
/usr/local/bin/kind build node-image --type=source "$PWD/kubernetes" --image "${{ env.CACHE_IMAGE }}"
131+
132+
echo "Re-authenticating to GHCR to prevent session timeout..."
133+
echo "$GH_TOKEN" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
134+
135+
docker push "${{ env.CACHE_IMAGE }}"
136+
137+
docker tag "${{ env.CACHE_IMAGE }}" "${{ env.DAILY_IMAGE }}"
138+
docker push "${{ env.DAILY_IMAGE }}"
139+
140+
- name: Retag Existing Build (Cache Hit)
141+
if: steps.check-cache.outputs.skip_build == 'true'
142+
run: |
143+
set -euo pipefail
144+
docker pull "${{ env.CACHE_IMAGE }}"
145+
docker tag "${{ env.CACHE_IMAGE }}" "${{ env.DAILY_IMAGE }}"
146+
docker push "${{ env.DAILY_IMAGE }}"
147+
148+
trigger-tests:
149+
needs: [resolve-versions, build-and-push]
150+
runs-on: ubuntu-latest
151+
timeout-minutes: 10
152+
if: success()
153+
permissions:
154+
actions: write
155+
contents: read
156+
steps:
157+
- name: Checkout Code
158+
uses: actions/checkout@v4
159+
160+
- name: Trigger Downstream E2E Tests
161+
env:
162+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
163+
TIMESTAMP: ${{ needs.resolve-versions.outputs.timestamp }}
164+
WORKFLOW_BRANCH: ${{ github.ref_name }}
165+
run: |
166+
set -euo pipefail
167+
168+
ALL_TAGS=$(git ls-remote --tags --refs https://github.com/carvel-dev/kapp-controller.git \
169+
| awk -F/ '{print $3}' | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | sort -V)
170+
171+
MINORS=$(echo "$ALL_TAGS" | cut -d. -f1,2 | uniq | tail -n 2)
172+
173+
DYNAMIC_TAGS=""
174+
for minor in $MINORS; do
175+
LATEST_PATCH=$(echo "$ALL_TAGS" | grep "^${minor}\." | tail -n 1)
176+
DYNAMIC_TAGS="$DYNAMIC_TAGS $LATEST_PATCH"
177+
done
178+
179+
for tag in $DYNAMIC_TAGS; do
180+
echo "Dispatching E2E test for kapp-controller: $tag"
181+
182+
gh workflow run nightly-e2e.yml \
183+
--ref "$WORKFLOW_BRANCH" \
184+
-f image_timestamp="$TIMESTAMP" \
185+
-f kapp_ctrl_ref="$tag" \
186+
-f registry_user="${{ github.repository_owner }}"
187+
188+
sleep 3
189+
done

‎.github/workflows/nightly-e2e.yml‎

Lines changed: 141 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
name: Nightly E2E Tests
2+
23
on:
34
workflow_dispatch:
45
inputs:
@@ -13,8 +14,146 @@ on:
1314
description: 'User to pull image from'
1415
required: false
1516
default: 'carvel-dev'
17+
1618
jobs:
17-
dummy:
19+
resolve-versions:
1820
runs-on: ubuntu-latest
21+
timeout-minutes: 5
22+
outputs:
23+
matrix: ${{ steps.set-matrix.outputs.matrix }}
1924
steps:
20-
- run: echo "Registering inputs on main"
25+
- name: Resolve Kubernetes Versions
26+
id: set-matrix
27+
run: |
28+
set -euo pipefail
29+
# Recalculating the matrix locally to avoid passing complex JSON arrays through GH API string inputs.
30+
BRANCHES=$(git ls-remote --heads https://github.com/kubernetes/kubernetes.git \
31+
| awk '{print $2}' | grep -Eo 'release-1\.[0-9]+$' | sort -V)
32+
33+
LATEST_BRANCH=$(echo "$BRANCHES" | tail -n 1)
34+
MIN_BRANCH=$(echo "$BRANCHES" | tail -n 4 | head -n 1)
35+
36+
MATRIX_JSON=$(jq -nc \
37+
--arg latest "$LATEST_BRANCH" \
38+
--arg min "$MIN_BRANCH" \
39+
'{"k8s_version": [$latest, $min]}')
40+
41+
echo "matrix=$MATRIX_JSON" >> "$GITHUB_OUTPUT"
42+
43+
e2e-tests:
44+
needs: resolve-versions
45+
runs-on: ubuntu-latest
46+
timeout-minutes: 60
47+
strategy:
48+
fail-fast: false
49+
matrix: ${{ fromJson(needs.resolve-versions.outputs.matrix) }}
50+
steps:
51+
- name: Setup Environment Variables
52+
run: |
53+
set -euo pipefail
54+
RAW_USER="${{ inputs.registry_user || github.repository_owner }}"
55+
USER_LC=$(echo "$RAW_USER" | tr '[:upper:]' '[:lower:]')
56+
57+
TIMESTAMP="${{ inputs.image_timestamp }}"
58+
TAG="${{ matrix.k8s_version }}-${TIMESTAMP}"
59+
FULL_IMAGE="ghcr.io/${USER_LC}/kindest-node:$TAG"
60+
61+
echo "FULL_IMAGE=$FULL_IMAGE" >> "$GITHUB_ENV"
62+
63+
- name: Checkout Kapp-Controller
64+
uses: actions/checkout@v4
65+
with:
66+
# Clones the fork/branch first. Target refs/tags are fetched manually below.
67+
fetch-depth: 0
68+
69+
- name: Fetch Upstream Tags
70+
run: |
71+
set -euo pipefail
72+
echo "Adding upstream remote..."
73+
git remote add upstream https://github.com/carvel-dev/kapp-controller.git || true
74+
75+
echo "Fetching upstream tags..."
76+
git fetch upstream --tags
77+
78+
TARGET_REF="${{ inputs.kapp_ctrl_ref || 'develop' }}"
79+
echo "Checking out target ref: $TARGET_REF"
80+
81+
# Safely checks out upstream tags or falls back to the fork's local branch
82+
git checkout "$TARGET_REF"
83+
84+
- name: Setup Go
85+
uses: actions/setup-go@v5
86+
with:
87+
go-version-file: 'go.mod'
88+
89+
- name: Login to GitHub Container Registry
90+
uses: docker/login-action@v3
91+
with:
92+
registry: ghcr.io
93+
username: ${{ github.actor }}
94+
password: ${{ secrets.GITHUB_TOKEN }}
95+
96+
- name: Verify Image Existence (Pre-flight Check)
97+
run: |
98+
set -euo pipefail
99+
echo "Verifying image exists in GHCR: ${{ env.FULL_IMAGE }}"
100+
export DOCKER_CLI_EXPERIMENTAL=enabled
101+
if ! docker manifest inspect "${{ env.FULL_IMAGE }}" > /dev/null 2>&1; then
102+
echo "::error::Image ${{ env.FULL_IMAGE }} does not exist. Halting cluster creation."
103+
exit 1
104+
fi
105+
106+
- name: Create Kind Cluster
107+
run: |
108+
set -euo pipefail
109+
echo "Bootstrapping cluster using verified image: ${{ env.FULL_IMAGE }}"
110+
/usr/local/bin/kind create cluster --image "${{ env.FULL_IMAGE }}" --name kinder --wait 1m
111+
112+
/usr/local/bin/kind get kubeconfig --name kinder > kubeconfig.yaml
113+
echo "KUBECONFIG=$PWD/kubeconfig.yaml" >> "$GITHUB_ENV"
114+
115+
- name: Install Dependencies
116+
run: ./hack/install-deps.sh
117+
118+
- name: Build, Load and Deploy Kapp-Controller
119+
run: |
120+
set -euo pipefail
121+
122+
VERSION="${{ inputs.kapp_ctrl_ref || 'develop' }}"
123+
VERSION=${VERSION#v}
124+
if [[ "$VERSION" == "develop" || "$VERSION" == "HEAD" ]]; then
125+
VERSION="0.100.0+develop"
126+
fi
127+
128+
echo "Injecting Build Version: $VERSION"
129+
sed -i "s|function get_kappctrl_ver() {|function get_kappctrl_ver() { echo \"$VERSION\"; return; |g" hack/version-util.sh
130+
131+
echo "Compiling manifests..."
132+
ytt -f config/config -f config/values-schema.yml -f config-dev -v dev.version="$VERSION" | kbld -f- > kbld.out 2> kbldmeta.out
133+
134+
echo "Extracting image from kbld metadata..."
135+
BUILT_IMAGE=$(awk '/final: kapp-controller ->/ {print $NF}' kbldmeta.out | tail -n 1)
136+
137+
if [[ -z "$BUILT_IMAGE" ]]; then
138+
echo "::error::Failed to extract built image from kbld output."
139+
exit 1
140+
fi
141+
142+
echo "Loading $BUILT_IMAGE into kind..."
143+
/usr/local/bin/kind load docker-image "$BUILT_IMAGE" --name kinder
144+
145+
echo "Deploying to cluster..."
146+
kapp deploy -a kc -f kbld.out -c -y
147+
148+
- name: Install Secretgen Controller
149+
run: |
150+
set -euo pipefail
151+
export KAPPCTRL_E2E_SECRETGEN_CONTROLLER=true
152+
source ./hack/secretgen-controller.sh
153+
deploy_secretgen-controller
154+
155+
- name: Run E2E Tests
156+
run: |
157+
set -euo pipefail
158+
mkdir -p tmp
159+
KAPPCTRL_E2E_NAMESPACE=kappctrl-test eval './hack/test-e2e.sh'

0 commit comments

Comments
 (0)