Skip to content

Commit cecc1b8

Browse files
kahirokunnkahirokunn
committed
add multi-cluster deployment support via Cluster Inventory API
Signed-off-by: kahirokunn <okinakahiro@gmail.com>
1 parent 9c66d7b commit cecc1b8

264 files changed

Lines changed: 17715 additions & 4100 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitignore‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,4 +83,10 @@ Session.vim
8383
# hack/generate-helm.sh copies config/charts/ to charts/ for helm package.
8484
# Ignore that build output; config/charts/ (the source) is tracked.
8585
/charts/
86-
.tmp/
86+
.tmp/
87+
88+
# Go binary
89+
/operator
90+
91+
# kodata latest (generated locally by fetcher)
92+
cmd/operator/kodata/*/latest/

‎cmd/operator/main.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ limitations under the License.
1717
package main
1818

1919
import (
20+
_ "knative.dev/operator/pkg/reconciler/common" // init() registers flags and caching scheme.
2021
"knative.dev/operator/pkg/reconciler/knativeeventing"
2122
"knative.dev/operator/pkg/reconciler/knativeserving"
2223
kubefilteredfactory "knative.dev/pkg/client/injection/kube/informers/factory/filtered"
@@ -25,6 +26,7 @@ import (
2526
)
2627

2728
func main() {
29+
// Do not call flag.Parse(); sharedmain parses flags.
2830
ctx := signals.NewContext()
2931
ctx = kubefilteredfactory.WithSelectors(ctx,
3032
knativeserving.Selector,

‎cmd/operator/main_test.go‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
/*
2+
Copyright 2026 The Knative Authors
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package main
18+
19+
import (
20+
"os/exec"
21+
"path/filepath"
22+
"strings"
23+
"testing"
24+
"time"
25+
)
26+
27+
// TestFlagsAcceptedBySharedmain guards against an early flag.Parse() regression
28+
// by building the binary and invoking each late-registered sharedmain flag.
29+
func TestFlagsAcceptedBySharedmain(t *testing.T) {
30+
bin := filepath.Join(t.TempDir(), "knative-operator")
31+
build := exec.Command("go", "build", "-o", bin, ".")
32+
if out, err := build.CombinedOutput(); err != nil {
33+
t.Fatalf("go build failed: %v\n%s", err, out)
34+
}
35+
36+
flags := []string{
37+
"--kubeconfig=/nonexistent",
38+
"--disable-ha",
39+
"--server=https://example.invalid",
40+
"--cluster=remote",
41+
"--kube-api-burst=200",
42+
"--kube-api-qps=100",
43+
}
44+
for _, fl := range flags {
45+
t.Run(strings.TrimPrefix(strings.SplitN(fl, "=", 2)[0], "--"), func(t *testing.T) {
46+
cmd := exec.Command(bin, fl)
47+
cmd.Env = append(cmd.Environ(), "KUBECONFIG=/dev/null")
48+
done := make(chan struct{})
49+
var out []byte
50+
var runErr error
51+
go func() {
52+
out, runErr = cmd.CombinedOutput()
53+
close(done)
54+
}()
55+
select {
56+
case <-done:
57+
case <-time.After(15 * time.Second):
58+
_ = cmd.Process.Kill()
59+
<-done
60+
}
61+
_ = runErr
62+
if strings.Contains(string(out), "flag provided but not defined") {
63+
t.Fatalf("%s rejected as undefined — early flag.Parse() regression?\n%s", fl, out)
64+
}
65+
})
66+
}
67+
}

‎config/charts/knative-operator/templates/access-providers-config.yaml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# Copyright 2025 The Knative Authors
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
{{- $mc := .Values.knative_operator.multicluster | default dict }}
16+
{{- if $mc.enabled }}
17+
{{- if not $mc.accessProvidersConfig }}
18+
{{- fail "knative_operator.multicluster.enabled is true but knative_operator.multicluster.accessProvidersConfig is empty — either provide an access-providers config or disable multi-cluster support" }}
19+
{{- end }}
20+
{{- $mountPaths := list }}
21+
{{- range ($mc.plugins | default (list)) }}
22+
{{- $mountPaths = append $mountPaths .mountPath }}
23+
{{- end }}
24+
{{- $cfg := $mc.accessProvidersConfig | default dict }}
25+
{{- range ($cfg.providers | default (list)) }}
26+
{{- $cmd := (.execConfig | default dict).command | default "" }}
27+
{{- if $cmd }}
28+
{{- $cmdDir := dir $cmd }}
29+
{{- if not (has $cmdDir $mountPaths) }}
30+
{{- fail (printf "multicluster validation error: provider %q command %q has parent dir %q which does not match any plugins[].mountPath (have %v); execConfig.command parent directory must equal a plugin mountPath" .name $cmd $cmdDir $mountPaths) }}
31+
{{- end }}
32+
{{- end }}
33+
{{- end }}
34+
apiVersion: v1
35+
kind: ConfigMap
36+
metadata:
37+
name: clusterprofile-provider-file
38+
namespace: "{{ .Release.Namespace }}"
39+
labels:
40+
app.kubernetes.io/name: knative-operator
41+
app.kubernetes.io/version: "{{ .Chart.Version }}"
42+
data:
43+
config.json: {{ $mc.accessProvidersConfig | default dict | mustToJson | quote }}
44+
{{- end }}

‎config/charts/knative-operator/templates/crds/knativeeventings.yaml‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,11 @@ spec:
3232
singular: knativeeventing
3333
scope: Namespaced
3434
versions:
35-
- name: v1beta1
35+
- additionalPrinterColumns:
36+
- jsonPath: .spec.clusterProfileRef.name
37+
name: Target Cluster
38+
type: string
39+
name: v1beta1
3640
schema:
3741
openAPIV3Schema:
3842
description: KnativeEventing is the Schema for the eventings API
@@ -69,6 +73,25 @@ spec:
6973
- URL
7074
type: object
7175
type: array
76+
clusterProfileRef:
77+
description: |-
78+
ClusterProfileRef is an optional reference to a ClusterProfile resource
79+
(multicluster.x-k8s.io/v1alpha1). When set, the operator reconciles
80+
the component on the remote cluster described by the referenced
81+
ClusterProfile instead of the local cluster.
82+
properties:
83+
name:
84+
description: Name is the name of the ClusterProfile resource.
85+
minLength: 1
86+
type: string
87+
namespace:
88+
description: Namespace is the namespace of the ClusterProfile resource.
89+
minLength: 1
90+
type: string
91+
required:
92+
- name
93+
- namespace
94+
type: object
7295
config:
7396
additionalProperties:
7497
additionalProperties:
@@ -3503,6 +3526,9 @@ spec:
35033526
type: object
35043527
type: array
35053528
type: object
3529+
x-kubernetes-validations:
3530+
- message: spec.clusterProfileRef is immutable
3531+
rule: (!has(self.clusterProfileRef) && !has(oldSelf.clusterProfileRef)) || (has(self.clusterProfileRef) && has(oldSelf.clusterProfileRef) && self.clusterProfileRef == oldSelf.clusterProfileRef)
35063532
status:
35073533
description: KnativeEventingStatus defines the observed state of KnativeEventing
35083534
properties:

‎config/charts/knative-operator/templates/crds/knativeservings.yaml‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,11 @@ spec:
3232
singular: knativeserving
3333
scope: Namespaced
3434
versions:
35-
- name: v1beta1
35+
- additionalPrinterColumns:
36+
- jsonPath: .spec.clusterProfileRef.name
37+
name: Target Cluster
38+
type: string
39+
name: v1beta1
3640
schema:
3741
openAPIV3Schema:
3842
description: KnativeServing is the Schema for the knativeservings API
@@ -69,6 +73,25 @@ spec:
6973
- URL
7074
type: object
7175
type: array
76+
clusterProfileRef:
77+
description: |-
78+
ClusterProfileRef is an optional reference to a ClusterProfile resource
79+
(multicluster.x-k8s.io/v1alpha1). When set, the operator reconciles
80+
the component on the remote cluster described by the referenced
81+
ClusterProfile instead of the local cluster.
82+
properties:
83+
name:
84+
description: Name is the name of the ClusterProfile resource.
85+
minLength: 1
86+
type: string
87+
namespace:
88+
description: Namespace is the namespace of the ClusterProfile resource.
89+
minLength: 1
90+
type: string
91+
required:
92+
- name
93+
- namespace
94+
type: object
7295
config:
7396
additionalProperties:
7497
additionalProperties:
@@ -3774,6 +3797,9 @@ spec:
37743797
type: object
37753798
type: array
37763799
type: object
3800+
x-kubernetes-validations:
3801+
- message: spec.clusterProfileRef is immutable
3802+
rule: (!has(self.clusterProfileRef) && !has(oldSelf.clusterProfileRef)) || (has(self.clusterProfileRef) && has(oldSelf.clusterProfileRef) && self.clusterProfileRef == oldSelf.clusterProfileRef)
37773803
status:
37783804
description: KnativeServingStatus defines the observed state of KnativeServing
37793805
properties:

‎config/charts/knative-operator/templates/operator.yaml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -855,5 +855,30 @@ spec:
855855
ports:
856856
- name: metrics
857857
containerPort: 9090
858+
{{- $mc := .Values.knative_operator.multicluster | default dict }}
859+
{{- if $mc.enabled }}
860+
args:
861+
- --clusterprofile-provider-file=/etc/cluster-inventory/config.json
862+
volumeMounts:
863+
- name: access-config
864+
mountPath: /etc/cluster-inventory
865+
readOnly: true
866+
{{- range ($mc.plugins | default list) }}
867+
- name: {{ .name }}
868+
mountPath: {{ .mountPath }}
869+
readOnly: true
870+
{{- end }}
871+
{{- end }}
872+
{{- if $mc.enabled }}
873+
volumes:
874+
- name: access-config
875+
configMap:
876+
name: clusterprofile-provider-file
877+
{{- range ($mc.plugins | default list) }}
878+
- name: {{ .name }}
879+
image:
880+
reference: {{ .image }}
881+
{{- end }}
882+
{{- end }}
858883

859884
---

‎config/charts/knative-operator/templates/rbac/eventing-operator-role.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -401,3 +401,12 @@ rules:
401401
- list
402402
- get
403403
- watch
404+
# for multicluster support
405+
- apiGroups:
406+
- multicluster.x-k8s.io
407+
resources:
408+
- clusterprofiles
409+
verbs:
410+
- get
411+
- list
412+
- watch

‎config/charts/knative-operator/templates/rbac/serving-operator-role.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,15 @@ rules:
252252
- pods
253253
verbs:
254254
- get
255+
# for multicluster support
256+
- apiGroups:
257+
- multicluster.x-k8s.io
258+
resources:
259+
- clusterprofiles
260+
verbs:
261+
- get
262+
- list
263+
- watch
255264
# Copyright 2020 The Knative Authors
256265
#
257266
# Licensed under the Apache License, Version 2.0 (the "License");

‎config/charts/knative-operator/values.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,24 @@
11
knative_operator:
2+
# Multi-cluster (Cluster Inventory API): when enabled, the chart mounts
3+
# access provider config and optional plugin images, and sets
4+
# --clusterprofile-provider-file on the operator. ClusterProfile.status
5+
# accessProviders are not managed by this chart.
6+
multicluster:
7+
enabled: false
8+
accessProvidersConfig: {}
9+
# plugins[] uses the Kubernetes "image" volume type
10+
plugins: []
11+
# accessProvidersConfig:
12+
# providers:
13+
# - name: token-secretreader
14+
# execConfig:
15+
# apiVersion: client.authentication.k8s.io/v1
16+
# command: /access-plugins/token-secretreader/kubeconfig-secretreader-plugin
17+
# provideClusterInfo: true
18+
# plugins:
19+
# - name: token-secretreader
20+
# image: ghcr.io/example/plugin:v1.0.0
21+
# mountPath: /access-plugins/token-secretreader
222
knative_operator:
323
image: gcr.io/knative-releases/knative.dev/operator/cmd/operator
424
tag: {{ tag }}

0 commit comments

Comments
 (0)