Skip to content

Commit d4e22be

Browse files
GrigoryPervakovGrigoryPervakov
authored
test: pre pull clickhouse images in e2e test, test campatible versions only with manifests installation (#165)
1 parent 97ddba1 commit d4e22be

7 files changed

Lines changed: 62 additions & 15 deletions

File tree

‎.github/workflows/ci.yaml‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -177,15 +177,18 @@ jobs:
177177
fail-fast: false
178178
matrix:
179179
include:
180-
- name: "Minimal supported kubernetes with LTS ClickHouse release"
180+
- name: minimal-k8s-all-deploy-methods
181181
k8s_image: v1.28.15
182182
clickhouse_version: "26.3"
183-
- name: "Latest kubernetes version with LTS ClickHouse release"
183+
deploy_target: test-compat-e2e
184+
- name: maximal-k8s-all-deploy-methods
184185
k8s_image: v1.35.1
185186
clickhouse_version: "26.3"
186-
- name: "Supported ClickHouse versions"
187+
deploy_target: test-compat-e2e
188+
- name: supported-clickhouse-compatibility
187189
k8s_image: v1.30.13
188190
clickhouse_version: "26.3,26.2,26.1,25.8"
191+
deploy_target: test-compat-e2e-manifest
189192
steps:
190193
- name: Checkout code
191194
uses: actions/checkout@v6
@@ -222,15 +225,15 @@ jobs:
222225
done
223226
224227
- name: Run compatibility e2e tests
225-
run: make test-compat-e2e
228+
run: make ${{ matrix.deploy_target }}
226229
env:
227230
CLICKHOUSE_VERSION: ${{ matrix.clickhouse_version }}
228231

229232
- name: Upload test report
230233
uses: actions/upload-artifact@v6
231234
if: ${{ !cancelled() }}
232235
with:
233-
name: compat-e2e-report-${{ strategy.job-index }}
236+
name: e2e-report-${{ matrix.name }}
234237
path: "**/report/*"
235238
if-no-files-found: error
236239
overwrite: true
@@ -241,7 +244,7 @@ jobs:
241244
fail-fast: false
242245
matrix:
243246
scope: [ test-keeper-e2e, test-clickhouse-e2e ]
244-
runs-on: [self-hosted, func-tester]
247+
runs-on: [self-hosted, amd-medium]
245248
steps:
246249
- name: Checkout code
247250
uses: actions/checkout@v6
@@ -279,12 +282,6 @@ jobs:
279282
runs-on: ubuntu-latest
280283
if: ${{ !cancelled() }}
281284
steps:
282-
- name: Download all compat test reports
283-
uses: actions/download-artifact@v8
284-
with:
285-
pattern: compat-e2e-report-*
286-
287-
288285
- name: Download all e2e test reports
289286
uses: actions/download-artifact@v8
290287
with:

‎Makefile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,10 @@ test-clickhouse-e2e: ## Run clickhouse e2e tests.
161161
test-compat-e2e: ## Run compatibility e2e tests (requires CLICKHOUSE_VERSION env var).
162162
go test ./test/deploy/ -test.timeout 30m -v --ginkgo.v --ginkgo.junit-report=report/junit-report.xml
163163

164+
.PHONY: test-compat-e2e-manifest # Run compatibility smoke tests (manifests deployment only).
165+
test-compat-e2e-manifest: ## Run compatibility e2e tests using manifests deployment only (requires CLICKHOUSE_VERSION env var).
166+
go test ./test/deploy/ -test.timeout 30m -v --ginkgo.v --ginkgo.label-filter=manifest --ginkgo.junit-report=report/junit-report.xml
167+
164168
.PHONY: lint
165169
lint: golangci-lint codespell actionlint ## Run golangci-lint linter, codespell, and actionlint
166170
$(GOLANGCI_LINT) run

‎ci/actionlint.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
self-hosted-runner:
22
labels:
3-
- func-tester
3+
- amd-medium

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ require (
1919
github.com/sethvargo/go-envconfig v1.3.0
2020
github.com/testcontainers/testcontainers-go v0.41.0
2121
go.uber.org/zap v1.27.1
22+
golang.org/x/sync v0.20.0
2223
gopkg.in/yaml.v2 v2.4.0
2324
k8s.io/api v0.35.3
2425
k8s.io/apimachinery v0.35.3
@@ -134,7 +135,6 @@ require (
134135
golang.org/x/mod v0.34.0 // indirect
135136
golang.org/x/net v0.52.0 // indirect
136137
golang.org/x/oauth2 v0.36.0 // indirect
137-
golang.org/x/sync v0.20.0 // indirect
138138
golang.org/x/sys v0.42.0 // indirect
139139
golang.org/x/term v0.41.0 // indirect
140140
golang.org/x/text v0.35.0 // indirect

‎test/deploy/deploy_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ var _ = JustAfterEach(func(ctx context.Context) {
130130
testutil.DumpNamespaceDiagnostics(ctx, config, k8sClient, ns, reportDir)
131131
})
132132

133-
var _ = Describe("Manifests deployment", Ordered, func() {
133+
var _ = Describe("Manifests deployment", Ordered, Label("manifest"), func() {
134134
namespace := "clickhouse-operator-system"
135135

136136
BeforeAll(func(ctx context.Context) {

‎test/e2e/e2e_suite_test.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,15 @@ var _ = BeforeSuite(func(ctx context.Context) {
9797
})
9898
Expect(err).NotTo(HaveOccurred())
9999

100+
By("pre-loading clickhouse images into kind")
101+
102+
imagePuller := testutil.PreloadImages(ctx, []string{
103+
"docker.io/clickhouse/clickhouse-server:" + BaseVersion,
104+
"docker.io/clickhouse/clickhouse-server:" + UpdateVersion,
105+
"docker.io/clickhouse/clickhouse-keeper:" + BaseVersion,
106+
"docker.io/clickhouse/clickhouse-keeper:" + UpdateVersion,
107+
})
108+
100109
By("installing CRDs")
101110
Expect(testutil.InstallCRDs(ctx)).To(Succeed())
102111
DeferCleanup(func(ctx context.Context) {
@@ -139,6 +148,10 @@ var _ = BeforeSuite(func(ctx context.Context) {
139148
DeferCleanup(func() {
140149
cancel()
141150
})
151+
152+
if err = imagePuller.Wait(); err != nil {
153+
GinkgoWriter.Printf("failed to pre pull images: %s", err)
154+
}
142155
})
143156

144157
var _ = JustAfterEach(func(ctx context.Context) {

‎test/testutil/utils.go‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"path/filepath"
1515
"reflect"
1616
"regexp"
17+
"runtime"
1718
"slices"
1819
"strings"
1920
"time"
@@ -22,6 +23,7 @@ import (
2223
cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1"
2324
. "github.com/onsi/ginkgo/v2"
2425
. "github.com/onsi/gomega"
26+
"golang.org/x/sync/errgroup"
2527
appsv1 "k8s.io/api/apps/v1"
2628
batchv1 "k8s.io/api/batch/v1"
2729
corev1 "k8s.io/api/core/v1"
@@ -466,6 +468,37 @@ func SetupCA(ctx context.Context, k8sClient client.Client, namespace string, suf
466468
})
467469
}
468470

471+
// PreloadImages pulls each image from the registry and loads it into the kind cluster.
472+
// All images are processed in parallel.
473+
func PreloadImages(ctx context.Context, images []string) *errgroup.Group {
474+
g, ctx := errgroup.WithContext(ctx)
475+
476+
for _, image := range images {
477+
g.Go(func() error {
478+
// Remove any cached manifest-index
479+
_ = exec.CommandContext(ctx, "docker", "image", "rm", image).Run()
480+
481+
By("pulling image:" + image)
482+
483+
pull := exec.CommandContext(ctx, "docker", "pull", "--platform", "linux/"+runtime.GOARCH, image)
484+
if out, err := pull.CombinedOutput(); err != nil {
485+
return fmt.Errorf("docker pull %s: %w\n%s", image, err, out)
486+
}
487+
488+
By("loading image into kind: " + image)
489+
490+
load := exec.CommandContext(ctx, "kind", "load", "docker-image", image)
491+
if out, err := load.CombinedOutput(); err != nil {
492+
return fmt.Errorf("kind load %s: %w\n%s", image, err, out)
493+
}
494+
495+
return nil
496+
})
497+
}
498+
499+
return g
500+
}
501+
469502
// EnsureNamespace ensures the test namespace is created and active.
470503
func EnsureNamespace(ctx context.Context, k8sClient client.Client, name string) {
471504
DeferCleanup(func(ctx context.Context) {

0 commit comments

Comments
 (0)