Skip to content

Commit 7b954bb

Browse files
ntnnntnn
authored
KCP e2e test (#333)
* Bump kcp-dev/multicluster-provider to latest main Signed-off-by: Nelo-T. Wallus <red.brush9525@fastmail.com> Signed-off-by: Nelo-T. Wallus <n.wallus@sap.com> * Return generated id and secret in CreateDexClient Signed-off-by: Nelo-T. Wallus <red.brush9525@fastmail.com> Signed-off-by: Nelo-T. Wallus <n.wallus@sap.com> * Add cookie encryption key for backend Signed-off-by: Nelo-T. Wallus <red.brush9525@fastmail.com> Signed-off-by: Nelo-T. Wallus <n.wallus@sap.com> * Add framework.ApplyFiles and .ApplyManifests Signed-off-by: Nelo-T. Wallus <red.brush9525@fastmail.com> * Update framework.StartBackend and split CRD installation Signed-off-by: Nelo-T. Wallus <red.brush9525@fastmail.com> Signed-off-by: Nelo-T. Wallus <n.wallus@sap.com> * Add logging config for controller-runtime Signed-off-by: Nelo-T. Wallus <red.brush9525@fastmail.com> * Add dex as dependency for kcp contrib tests Signed-off-by: Nelo-T. Wallus <red.brush9525@fastmail.com> Signed-off-by: Nelo-T. Wallus <n.wallus@sap.com> * Add WriteKubeconfig Signed-off-by: Nelo-T. Wallus <red.brush9525@fastmail.com> Signed-off-by: Nelo-T. Wallus <n.wallus@sap.com> * Bump kcp to v0.28.3 Signed-off-by: Nelo-T. Wallus <red.brush9525@fastmail.com> Signed-off-by: Nelo-T. Wallus <n.wallus@sap.com> * Pin kcp to a functional sdk/testing commit Signed-off-by: Nelo-T. Wallus <red.brush9525@fastmail.com> Signed-off-by: Nelo-T. Wallus <n.wallus@sap.com> * Update kcp readme Signed-off-by: Nelo-T. Wallus <red.brush9525@fastmail.com> Signed-off-by: Nelo-T. Wallus <n.wallus@sap.com> * Add kcp e2e test Signed-off-by: Nelo-T. Wallus <red.brush9525@fastmail.com> Signed-off-by: Nelo-T. Wallus <n.wallus@sap.com> * Just ignore dex on non-linux Signed-off-by: Nelo-T. Wallus <red.brush9525@fastmail.com> Signed-off-by: Nelo-T. Wallus <n.wallus@sap.com> * Drop dependency on build step for contrib kcp e2e tests Signed-off-by: Nelo-T. Wallus <red.brush9525@fastmail.com> Signed-off-by: Nelo-T. Wallus <n.wallus@sap.com> --------- Signed-off-by: Nelo-T. Wallus <red.brush9525@fastmail.com> Signed-off-by: Nelo-T. Wallus <n.wallus@sap.com>
1 parent 3abf1be commit 7b954bb

18 files changed

Lines changed: 1018 additions & 75 deletions

File tree

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ CODE_GENERATOR_BIN := code-generator
7777
CODE_GENERATOR := $(TOOLS_GOBIN_DIR)/$(CODE_GENERATOR_BIN)-$(CODE_GENERATOR_VER)
7878
export CODE_GENERATOR # so hack scripts can use it
7979

80-
KCP_VER := v0.28.0
80+
KCP_VER := v0.28.3
8181
KCP_BIN := kcp
8282
KCP := $(TOOLS_GOBIN_DIR)/$(KCP_BIN)-$(KCP_VER)
8383
KCP_CMD ?= $(KCP)
@@ -286,7 +286,7 @@ CONTRIBS_E2E := $(patsubst %,test-e2e-contrib-%,$(CONTRIBS))
286286

287287
.PHONY: test-e2e-contribs $(CONTRIBS_E2E)
288288
test-e2e-contribs: $(CONTRIBS_E2E) ## Run e2e tests for external integrations
289-
test-e2e-contrib-kcp: build $(KCP)
289+
test-e2e-contrib-kcp: $(DEX) $(KCP)
290290
$(CONTRIBS_E2E):
291291
cd contrib/$(patsubst test-e2e-contrib-%,%,$@) && $(GO_TEST) -race -count $(COUNT) -p $(E2E_PARALLELISM) -parallel $(E2E_PARALLELISM) ./test/e2e/...
292292

‎contrib/kcp/README.md‎

Lines changed: 70 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,23 @@ It will do the following:
2323

2424
# How to run
2525

26+
## Preparation
27+
2628
1. Start dex
29+
30+
```bash
31+
make run-dex
32+
```
33+
2734
2. Start kcp
28-
3. Bootstrap kcp:
35+
36+
```bash
37+
make run-kcp
38+
```
39+
40+
## Backend
41+
42+
2. Bootstrap kcp:
2943
```bash
3044
cp .kcp/admin.kubeconfig .kcp/backend.kubeconfig
3145
export KUBECONFIG=.kcp/backend.kubeconfig
@@ -49,18 +63,22 @@ k ws use :root:kube-bind
4963
--schema-source apiresourceschemas
5064
```
5165

66+
This process will keep running, so open a new terminal.
67+
68+
## Provider
5269

5370
5. Copy the kubeconfig to the provider and create provider workspace:
5471
```bash
5572
cp .kcp/admin.kubeconfig .kcp/provider.kubeconfig
5673
export KUBECONFIG=.kcp/provider.kubeconfig
5774
k ws use :root
58-
kubectl ws create provider --enter
75+
kubectl create-workspace provider --enter
5976
```
6077

6178
6. Bind the APIExport to the provider workspace
6279
```bash
63-
kubectl kcp bind apiexport root:kube-bind:kube-bind.io --accept-permission-claim clusterrolebindings.rbac.authorization.k8s.io \
80+
kubectl kcp bind apiexport root:kube-bind:kube-bind.io \
81+
--accept-permission-claim clusterrolebindings.rbac.authorization.k8s.io \
6482
--accept-permission-claim clusterroles.rbac.authorization.k8s.io \
6583
--accept-permission-claim customresourcedefinitions.apiextensions.k8s.io \
6684
--accept-permission-claim serviceaccounts.core \
@@ -77,7 +95,6 @@ kubectl kcp bind apiexport root:kube-bind:kube-bind.io --accept-permission-claim
7795
kubectl create -f contrib/kcp/deploy/examples/apiexport.yaml
7896
kubectl create -f contrib/kcp/deploy/examples/apiresourceschema-cowboys.yaml
7997
kubectl create -f contrib/kcp/deploy/examples/apiresourceschema-sheriffs.yaml
80-
# recursive bind
8198
kubectl kcp bind apiexport root:provider:cowboys-stable
8299
```
83100

@@ -86,9 +103,11 @@ kubectl kcp bind apiexport root:provider:cowboys-stable
86103
```bash
87104
kubectl get logicalcluster
88105
# NAME PHASE URL AGE
89-
# cluster Ready https://192.168.2.166:6443/clusters/1d5vpxvdpy0opbj1
106+
# cluster Ready https://192.168.2.166:6443/clusters/1d5vpxvdpy0opbj1
90107
```
91108

109+
## Consumer
110+
92111
9. Now we gonna initiate consumer:
93112
```bash
94113
cp .kcp/admin.kubeconfig .kcp/consumer.kubeconfig
@@ -104,15 +123,24 @@ kubectl ws create consumer --enter
104123

105124
# Extract secret for binding process. Note that secret name is not the same as output from command above. Check secret
106125
# name by running `kubectl get secret -n kube-bind`
107-
kubectl get secret kubeconfig-hxwlc -n kube-bind -o jsonpath='{.data.kubeconfig}' | base64 -d > remote.kubeconfig
126+
kubectl get secrets -n kube-bind -o jsonpath='{.items[0].data.kubeconfig}' | base64 -d > remote.kubeconfig
108127

109-
./bin/kubectl-bind apiservice --remote-kubeconfig remote.kubeconfig -f contrib/kcp/deploy/examples/apiserviceexport-namespaced.yaml --skip-konnector --remote-namespace kube-bind-697cb
128+
namespace=$(yq '.contexts[0].context.namespace' remote.kubeconfig)
110129

111-
export KUBECONFIG=.kcp/consumer.kubeconfig
112-
go run ./cmd/konnector/ --lease-namespace default
130+
./bin/kubectl-bind apiservice -v 6 --remote-kubeconfig remote.kubeconfig -f apiserviceexport.yaml --skip-konnector --remote-namespace "$namespace"
131+
```
132+
133+
This will keep running, so switch to a new terminal.
113134

135+
### Consumer Konnector
114136

115-
11. (Optional) Add second consumer to test
137+
Start konnector:
138+
139+
```bash
140+
./bin/konnector --lease-namespace default --kubeconfig .kcp/consumer.kubeconfig
141+
```
142+
143+
Optionally add second consumer to test
116144

117145
```bash
118146
cp .kcp/admin.kubeconfig .kcp/consumer2.kubeconfig
@@ -121,27 +149,53 @@ kubectl ws use :root
121149
kubectl ws create consumer2 --enter
122150

123151
./bin/kubectl-bind http://127.0.0.1:8080/clusters/2vgrh380y0cq38du/exports --dry-run -o yaml > apiserviceexport2.yaml
124-
kubectl get secret kubeconfig-wvvsb -n kube-bind -o jsonpath='{.data.kubeconfig}' | base64 -d > remote2.kubeconfig
125152

126-
./bin/kubectl-bind apiservice --remote-kubeconfig remote2.kubeconfig -f apiserviceexport.yaml --skip-konnector --remote-namespace kube-bind-m5zx4
153+
kubectl get secrets -n kube-bind -o jsonpath='{.items[0].data.kubeconfig}' | base64 -d > remote2.kubeconfig
127154

155+
./bin/kubectl-bind apiservice -v 6 --remote-kubeconfig remote2.kubeconfig -f apiserviceexport2.yaml --skip-konnector --remote-namespace "$(yq '.contexts[0].context.namespace' remote2.kubeconfig)"
128156

129-
export KUBECONFIG=.kcp/consumer2.kubeconfig
130-
go run ./cmd/konnector/ --lease-namespace default --server-address :8091
157+
./bin/konnector --lease-namespace default --kubeconfig .kcp/consumer2.kubeconfig --server-address :8091
131158
```
132159

160+
This will keep running, so switch to a new terminal.
161+
162+
## Testing
163+
133164
Create objects:
134165
```
166+
export KUBECONFIG=.kcp/consumer.kubeconfig
135167
kubectl apply -f contrib/kcp/deploy/examples/cowboy.yaml
168+
kubectl apply -f contrib/kcp/deploy/examples/sheriff.yaml
136169
```
137170

138171

139172
## Debug
140173

141174
```bash
175+
142176
cp .kcp/admin.kubeconfig .kcp/debug.kubeconfig
143177
export KUBECONFIG=.kcp/debug.kubeconfig
144178
k ws use :root:kube-bind
145179

146-
k -s "$(kubectl get apiexportendpointslice kube-bind.io -o jsonpath="{.status.endpoints[0].url}")/clusters/*" api-resources
147-
k -s "$(kubectl get apiexportendpointslice kube-bind.io -o jsonpath="{.status.endpoints[0].url}")/clusters/*" get crd
180+
k -s "$(kubectl get apiexportendpointslice kube-bind.io -o jsonpath='{.status.endpoints[0].url}')/clusters/*" api-resources
181+
182+
k -s "$(kubectl get apiexportendpointslice kube-bind.io -o jsonpath='{.status.endpoints[0].url}')/clusters/*" get crd
183+
184+
namespace=$(yq '.contexts[0].context.namespace' remote.kubeconfig)
185+
186+
# some claimed objects
187+
188+
kubectl create cm provider -n "$namespace-default"
189+
kubectl label cm provider app=wildwest -n "$namespace-default"
190+
191+
kubectl create cm consumer -n default
192+
kubectl label cm consumer app=wildwest -n default
193+
194+
kubectl create secret generic provider-secret
195+
kubectl label secret provider-secret app=wildwest
196+
197+
kubectl create namespace bob
198+
kubectl create secret generic wildwest-secrets1 -n bob
199+
kubectl label secret wildwest-secrets1 app=wildwest -n bob
200+
201+
```

‎contrib/kcp/go.mod‎

Lines changed: 46 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,37 +8,58 @@ replace (
88
github.com/kube-bind/kube-bind/sdk => ../../sdk
99
)
1010

11+
// kcp pinned to a commit on main as sdk/testing requires
12+
// features not in the latest release
13+
// Can use versioned when v0.28.2 releases
14+
replace github.com/kcp-dev/kcp/sdk => github.com/kcp-dev/kcp/sdk v0.28.1-0.20251003164010-742ce0ea6b8c
15+
1116
require (
17+
github.com/headzoo/surf v1.0.1
1218
github.com/kcp-dev/client-go v0.0.0-20250728134101-0355faa9361b
13-
github.com/kcp-dev/kcp v0.28.1
19+
github.com/kcp-dev/kcp v0.28.3
1420
github.com/kcp-dev/kcp/sdk v0.28.1
1521
github.com/kcp-dev/logicalcluster/v3 v3.0.5
22+
github.com/kube-bind/kube-bind v0.0.0-00010101000000-000000000000
23+
github.com/kube-bind/kube-bind/sdk v0.4.1
1624
github.com/spf13/pflag v1.0.7
25+
github.com/stretchr/testify v1.10.0
1726
k8s.io/apiextensions-apiserver v0.33.3
1827
k8s.io/apimachinery v0.33.3
1928
k8s.io/apiserver v0.33.3
29+
k8s.io/cli-runtime v0.32.0
2030
k8s.io/client-go v0.33.3
2131
k8s.io/component-base v0.33.3
2232
k8s.io/klog/v2 v2.130.1
33+
sigs.k8s.io/yaml v1.4.0
2334
)
2435

2536
require (
2637
cel.dev/expr v0.19.1 // indirect
38+
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
2739
github.com/NYTimes/gziphandler v1.1.1 // indirect
40+
github.com/PuerkitoBio/goquery v1.8.0 // indirect
41+
github.com/andybalholm/cascadia v1.3.1 // indirect
2842
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
2943
github.com/beorn7/perks v1.0.1 // indirect
3044
github.com/blang/semver/v4 v4.0.0 // indirect
3145
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
3246
github.com/cespare/xxhash/v2 v2.3.0 // indirect
47+
github.com/coreos/go-oidc/v3 v3.15.0 // indirect
3348
github.com/coreos/go-semver v0.3.1 // indirect
3449
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
3550
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
51+
github.com/dexidp/dex/api/v2 v2.3.0 // indirect
52+
github.com/egymgmbh/go-prefix-writer v0.0.0-20180609083313-7326ea162eca // indirect
3653
github.com/emicklei/go-restful/v3 v3.12.1 // indirect
54+
github.com/evanphx/json-patch/v5 v5.9.11 // indirect
3755
github.com/felixge/httpsnoop v1.0.4 // indirect
3856
github.com/fsnotify/fsnotify v1.7.0 // indirect
3957
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
58+
github.com/go-errors/errors v1.4.2 // indirect
59+
github.com/go-jose/go-jose/v4 v4.0.5 // indirect
4060
github.com/go-logr/logr v1.4.2 // indirect
4161
github.com/go-logr/stdr v1.2.2 // indirect
62+
github.com/go-logr/zapr v1.3.0 // indirect
4263
github.com/go-openapi/jsonpointer v0.21.0 // indirect
4364
github.com/go-openapi/jsonreference v0.21.0 // indirect
4465
github.com/go-openapi/swag v0.23.0 // indirect
@@ -48,27 +69,44 @@ require (
4869
github.com/google/cel-go v0.23.2 // indirect
4970
github.com/google/gnostic-models v0.6.9 // indirect
5071
github.com/google/go-cmp v0.7.0 // indirect
72+
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
5173
github.com/google/uuid v1.6.0 // indirect
74+
github.com/gorilla/mux v1.8.0 // indirect
75+
github.com/gorilla/securecookie v1.1.1 // indirect
76+
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
5277
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
5378
github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 // indirect
5479
github.com/inconshreveable/mousetrap v1.1.0 // indirect
5580
github.com/josharian/intern v1.0.0 // indirect
5681
github.com/json-iterator/go v1.1.12 // indirect
5782
github.com/kcp-dev/apimachinery/v2 v2.0.1-0.20250728122101-adbf20db3e51 // indirect
83+
github.com/kcp-dev/kcp/pkg/apis v0.11.0 // indirect
84+
github.com/kcp-dev/multicluster-provider v0.2.1-0.20251002133408-9a8d21dc2872 // indirect
85+
github.com/kube-bind/kube-bind/cli v0.0.0-20250515145715-d9f20e7c840d // indirect
5886
github.com/kylelemons/godebug v1.1.0 // indirect
87+
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
5988
github.com/mailru/easyjson v0.9.0 // indirect
89+
github.com/martinlindhe/base36 v1.1.1 // indirect
90+
github.com/mdp/qrterminal/v3 v3.2.0 // indirect
91+
github.com/moby/term v0.5.0 // indirect
6092
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
6193
github.com/modern-go/reflect2 v1.0.2 // indirect
94+
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
6295
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
6396
github.com/onsi/gomega v1.36.2 // indirect
97+
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
6498
github.com/pkg/errors v0.9.1 // indirect
99+
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
65100
github.com/prometheus/client_golang v1.22.0 // indirect
66101
github.com/prometheus/client_model v0.6.1 // indirect
67102
github.com/prometheus/common v0.62.0 // indirect
68103
github.com/prometheus/procfs v0.15.1 // indirect
69104
github.com/spf13/cobra v1.9.1 // indirect
70105
github.com/stoewer/go-strcase v1.3.0 // indirect
106+
github.com/vmihailenco/msgpack/v4 v4.3.12 // indirect
107+
github.com/vmihailenco/tagparser v0.1.1 // indirect
71108
github.com/x448/float16 v0.8.4 // indirect
109+
github.com/xlab/treeprint v1.2.0 // indirect
72110
go.etcd.io/etcd/api/v3 v3.5.21 // indirect
73111
go.etcd.io/etcd/client/pkg/v3 v3.5.21 // indirect
74112
go.etcd.io/etcd/client/v3 v3.5.21 // indirect
@@ -93,6 +131,8 @@ require (
93131
golang.org/x/term v0.32.0 // indirect
94132
golang.org/x/text v0.25.0 // indirect
95133
golang.org/x/time v0.11.0 // indirect
134+
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
135+
google.golang.org/appengine v1.6.7 // indirect
96136
google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422 // indirect
97137
google.golang.org/genproto/googleapis/rpc v0.0.0-20250106144421-5f5ef82da422 // indirect
98138
google.golang.org/grpc v1.70.0 // indirect
@@ -103,11 +143,15 @@ require (
103143
k8s.io/api v0.33.3 // indirect
104144
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
105145
k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect
146+
rsc.io/qr v0.2.0 // indirect
106147
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect
148+
sigs.k8s.io/controller-runtime v0.21.0 // indirect
107149
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
150+
sigs.k8s.io/kustomize/api v0.19.0 // indirect
151+
sigs.k8s.io/kustomize/kyaml v0.19.0 // indirect
152+
sigs.k8s.io/multicluster-runtime v0.21.0-alpha.9.0.20251002124257-36facc7fbe82 // indirect
108153
sigs.k8s.io/randfill v1.0.0 // indirect
109154
sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
110-
sigs.k8s.io/yaml v1.4.0 // indirect
111155
)
112156

113157
replace (

0 commit comments

Comments
 (0)