Skip to content

Commit 28729ba

Browse files
kahirokunnkahirokunn
committed
add multi-cluster deployment support via Cluster Inventory API
Signed-off-by: kahirokunn <okinakahiro@gmail.com>
1 parent 786194b commit 28729ba

212 files changed

Lines changed: 16137 additions & 4095 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎cmd/operator/main.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,12 @@ limitations under the License.
1717
package main
1818

1919
import (
20+
"flag"
21+
"log"
22+
"os"
23+
24+
// Register --clusterprofile-provider-file before sharedmain parses flags.
25+
"knative.dev/operator/pkg/reconciler/common"
2026
"knative.dev/operator/pkg/reconciler/knativeeventing"
2127
"knative.dev/operator/pkg/reconciler/knativeserving"
2228
kubefilteredfactory "knative.dev/pkg/client/injection/kube/informers/factory/filtered"
@@ -25,6 +31,15 @@ import (
2531
)
2632

2733
func main() {
34+
// Parse flags early so we can validate --clusterprofile-provider-file
35+
// before sharedmain starts the controllers. sharedmain.MainWithContext
36+
// tolerates a second flag.Parse() call, so we can safely do this here.
37+
flag.Parse()
38+
if err := common.ValidateClusterProfileProviderFile(); err != nil {
39+
log.Printf("invalid multi-cluster configuration: %v", err)
40+
os.Exit(2)
41+
}
42+
2843
ctx := signals.NewContext()
2944
ctx = kubefilteredfactory.WithSelectors(ctx,
3045
knativeserving.Selector,

‎config/charts/knative-operator/templates/crds/knativeeventings.yaml‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,11 @@ spec:
3232
singular: knativeeventing
3333
scope: Namespaced
3434
versions:
35-
- name: v1beta1
35+
- additionalPrinterColumns:
36+
- jsonPath: .spec.clusterProfileRef.name
37+
name: Target Cluster
38+
type: string
39+
name: v1beta1
3640
schema:
3741
openAPIV3Schema:
3842
description: KnativeEventing is the Schema for the eventings API
@@ -69,6 +73,25 @@ spec:
6973
- URL
7074
type: object
7175
type: array
76+
clusterProfileRef:
77+
description: |-
78+
ClusterProfileRef is an optional reference to a ClusterProfile resource
79+
(multicluster.x-k8s.io/v1alpha1). When set, the operator reconciles
80+
the component on the remote cluster described by the referenced
81+
ClusterProfile instead of the local cluster.
82+
properties:
83+
name:
84+
description: Name is the name of the ClusterProfile resource.
85+
minLength: 1
86+
type: string
87+
namespace:
88+
description: Namespace is the namespace of the ClusterProfile resource.
89+
minLength: 1
90+
type: string
91+
required:
92+
- name
93+
- namespace
94+
type: object
7295
config:
7396
additionalProperties:
7497
additionalProperties:
@@ -3363,6 +3386,9 @@ spec:
33633386
type: object
33643387
type: array
33653388
type: object
3389+
x-kubernetes-validations:
3390+
- message: spec.clusterProfileRef is immutable
3391+
rule: (!has(self.clusterProfileRef) && !has(oldSelf.clusterProfileRef)) || (has(self.clusterProfileRef) && has(oldSelf.clusterProfileRef) && self.clusterProfileRef == oldSelf.clusterProfileRef)
33663392
status:
33673393
description: KnativeEventingStatus defines the observed state of KnativeEventing
33683394
properties:

‎config/charts/knative-operator/templates/crds/knativeservings.yaml‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,11 @@ spec:
3232
singular: knativeserving
3333
scope: Namespaced
3434
versions:
35-
- name: v1beta1
35+
- additionalPrinterColumns:
36+
- jsonPath: .spec.clusterProfileRef.name
37+
name: Target Cluster
38+
type: string
39+
name: v1beta1
3640
schema:
3741
openAPIV3Schema:
3842
description: KnativeServing is the Schema for the knativeservings API
@@ -69,6 +73,25 @@ spec:
6973
- URL
7074
type: object
7175
type: array
76+
clusterProfileRef:
77+
description: |-
78+
ClusterProfileRef is an optional reference to a ClusterProfile resource
79+
(multicluster.x-k8s.io/v1alpha1). When set, the operator reconciles
80+
the component on the remote cluster described by the referenced
81+
ClusterProfile instead of the local cluster.
82+
properties:
83+
name:
84+
description: Name is the name of the ClusterProfile resource.
85+
minLength: 1
86+
type: string
87+
namespace:
88+
description: Namespace is the namespace of the ClusterProfile resource.
89+
minLength: 1
90+
type: string
91+
required:
92+
- name
93+
- namespace
94+
type: object
7295
config:
7396
additionalProperties:
7497
additionalProperties:
@@ -3796,6 +3819,9 @@ spec:
37963819
type: object
37973820
type: array
37983821
type: object
3822+
x-kubernetes-validations:
3823+
- message: spec.clusterProfileRef is immutable
3824+
rule: (!has(self.clusterProfileRef) && !has(oldSelf.clusterProfileRef)) || (has(self.clusterProfileRef) && has(oldSelf.clusterProfileRef) && self.clusterProfileRef == oldSelf.clusterProfileRef)
37993825
status:
38003826
description: KnativeServingStatus defines the observed state of KnativeServing
38013827
properties:

‎config/charts/knative-operator/templates/credential-providers-config.yaml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# Copyright 2025 The Knative Authors
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
{{- $mc := .Values.knative_operator.multicluster | default dict }}
16+
{{- if $mc.enabled }}
17+
{{- if not $mc.accessProvidersConfig }}
18+
{{- fail "knative_operator.multicluster.enabled is true but knative_operator.multicluster.accessProvidersConfig is empty — either provide an access-providers config or disable multi-cluster support" }}
19+
{{- end }}
20+
{{- $mountPaths := list }}
21+
{{- range ($mc.plugins | default (list)) }}
22+
{{- $mountPaths = append $mountPaths .mountPath }}
23+
{{- end }}
24+
{{- $cfg := $mc.accessProvidersConfig | default dict }}
25+
{{- range ($cfg.providers | default (list)) }}
26+
{{- $cmd := (.execConfig | default dict).command | default "" }}
27+
{{- if $cmd }}
28+
{{- $cmdDir := dir $cmd }}
29+
{{- if not (has $cmdDir $mountPaths) }}
30+
{{- fail (printf "multicluster validation error: provider %q command %q has parent dir %q which does not match any plugins[].mountPath (have %v); execConfig.command parent directory must equal a plugin mountPath" .name $cmd $cmdDir $mountPaths) }}
31+
{{- end }}
32+
{{- end }}
33+
{{- end }}
34+
apiVersion: v1
35+
kind: ConfigMap
36+
metadata:
37+
name: clusterprofile-provider-file
38+
namespace: "{{ .Release.Namespace }}"
39+
labels:
40+
app.kubernetes.io/name: knative-operator
41+
app.kubernetes.io/version: "{{ .Chart.Version }}"
42+
data:
43+
config.json: {{ $mc.accessProvidersConfig | default dict | mustToJson | quote }}
44+
{{- end }}

‎config/charts/knative-operator/templates/operator.yaml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -855,5 +855,30 @@ spec:
855855
ports:
856856
- name: metrics
857857
containerPort: 9090
858+
{{- $mc := .Values.knative_operator.multicluster | default dict }}
859+
{{- if $mc.enabled }}
860+
args:
861+
- --clusterprofile-provider-file=/etc/cluster-inventory/config.json
862+
volumeMounts:
863+
- name: cred-config
864+
mountPath: /etc/cluster-inventory
865+
readOnly: true
866+
{{- range ($mc.plugins | default list) }}
867+
- name: {{ .name }}
868+
mountPath: {{ .mountPath }}
869+
readOnly: true
870+
{{- end }}
871+
{{- end }}
872+
{{- if $mc.enabled }}
873+
volumes:
874+
- name: cred-config
875+
configMap:
876+
name: clusterprofile-provider-file
877+
{{- range ($mc.plugins | default list) }}
878+
- name: {{ .name }}
879+
image:
880+
reference: {{ .image }}
881+
{{- end }}
882+
{{- end }}
858883

859884
---

‎config/charts/knative-operator/templates/rbac/eventing-operator-role.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -401,3 +401,12 @@ rules:
401401
- list
402402
- get
403403
- watch
404+
# for multicluster support
405+
- apiGroups:
406+
- multicluster.x-k8s.io
407+
resources:
408+
- clusterprofiles
409+
verbs:
410+
- get
411+
- list
412+
- watch

‎config/charts/knative-operator/templates/rbac/serving-operator-role.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,15 @@ rules:
252252
- pods
253253
verbs:
254254
- get
255+
# for multicluster support
256+
- apiGroups:
257+
- multicluster.x-k8s.io
258+
resources:
259+
- clusterprofiles
260+
verbs:
261+
- get
262+
- list
263+
- watch
255264
# Copyright 2020 The Knative Authors
256265
#
257266
# Licensed under the Apache License, Version 2.0 (the "License");

‎config/charts/knative-operator/values.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,24 @@
11
knative_operator:
2+
# Multi-cluster (Cluster Inventory API): when enabled, the chart mounts
3+
# access provider config and optional plugin images, and sets
4+
# --clusterprofile-provider-file on the operator. ClusterProfile.status
5+
# accessProviders are not managed by this chart.
6+
multicluster:
7+
enabled: false
8+
accessProvidersConfig: {}
9+
# plugins[] uses the Kubernetes "image" volume type
10+
plugins: []
11+
# accessProvidersConfig:
12+
# providers:
13+
# - name: token-secretreader
14+
# execConfig:
15+
# apiVersion: client.authentication.k8s.io/v1
16+
# command: /credential-plugins/token-secretreader/kubeconfig-secretreader-plugin
17+
# provideClusterInfo: true
18+
# plugins:
19+
# - name: token-secretreader
20+
# image: ghcr.io/example/plugin:v1.0.0
21+
# mountPath: /credential-plugins/token-secretreader
222
knative_operator:
323
image: gcr.io/knative-releases/knative.dev/operator/cmd/operator
424
tag: {{ tag }}

‎config/crd/bases/operator.knative.dev_knativeeventings.yaml‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,11 @@ spec:
2828
singular: knativeeventing
2929
scope: Namespaced
3030
versions:
31-
- name: v1beta1
31+
- additionalPrinterColumns:
32+
- jsonPath: .spec.clusterProfileRef.name
33+
name: Target Cluster
34+
type: string
35+
name: v1beta1
3236
schema:
3337
openAPIV3Schema:
3438
description: KnativeEventing is the Schema for the eventings API
@@ -66,6 +70,26 @@ spec:
6670
- URL
6771
type: object
6872
type: array
73+
clusterProfileRef:
74+
description: |-
75+
ClusterProfileRef is an optional reference to a ClusterProfile resource
76+
(multicluster.x-k8s.io/v1alpha1). When set, the operator reconciles
77+
the component on the remote cluster described by the referenced
78+
ClusterProfile instead of the local cluster.
79+
properties:
80+
name:
81+
description: Name is the name of the ClusterProfile resource.
82+
minLength: 1
83+
type: string
84+
namespace:
85+
description: Namespace is the namespace of the ClusterProfile
86+
resource.
87+
minLength: 1
88+
type: string
89+
required:
90+
- name
91+
- namespace
92+
type: object
6993
config:
7094
additionalProperties:
7195
additionalProperties:
@@ -3524,6 +3548,11 @@ spec:
35243548
type: object
35253549
type: array
35263550
type: object
3551+
x-kubernetes-validations:
3552+
- message: spec.clusterProfileRef is immutable
3553+
rule: (!has(self.clusterProfileRef) && !has(oldSelf.clusterProfileRef))
3554+
|| (has(self.clusterProfileRef) && has(oldSelf.clusterProfileRef)
3555+
&& self.clusterProfileRef == oldSelf.clusterProfileRef)
35273556
status:
35283557
description: KnativeEventingStatus defines the observed state of KnativeEventing
35293558
properties:

‎config/crd/bases/operator.knative.dev_knativeservings.yaml‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,11 @@ spec:
2828
singular: knativeserving
2929
scope: Namespaced
3030
versions:
31-
- name: v1beta1
31+
- additionalPrinterColumns:
32+
- jsonPath: .spec.clusterProfileRef.name
33+
name: Target Cluster
34+
type: string
35+
name: v1beta1
3236
schema:
3337
openAPIV3Schema:
3438
description: KnativeServing is the Schema for the knativeservings API
@@ -66,6 +70,26 @@ spec:
6670
- URL
6771
type: object
6872
type: array
73+
clusterProfileRef:
74+
description: |-
75+
ClusterProfileRef is an optional reference to a ClusterProfile resource
76+
(multicluster.x-k8s.io/v1alpha1). When set, the operator reconciles
77+
the component on the remote cluster described by the referenced
78+
ClusterProfile instead of the local cluster.
79+
properties:
80+
name:
81+
description: Name is the name of the ClusterProfile resource.
82+
minLength: 1
83+
type: string
84+
namespace:
85+
description: Namespace is the namespace of the ClusterProfile
86+
resource.
87+
minLength: 1
88+
type: string
89+
required:
90+
- name
91+
- namespace
92+
type: object
6993
config:
7094
additionalProperties:
7195
additionalProperties:
@@ -4043,6 +4067,11 @@ spec:
40434067
type: object
40444068
type: array
40454069
type: object
4070+
x-kubernetes-validations:
4071+
- message: spec.clusterProfileRef is immutable
4072+
rule: (!has(self.clusterProfileRef) && !has(oldSelf.clusterProfileRef))
4073+
|| (has(self.clusterProfileRef) && has(oldSelf.clusterProfileRef)
4074+
&& self.clusterProfileRef == oldSelf.clusterProfileRef)
40464075
status:
40474076
description: KnativeServingStatus defines the observed state of KnativeServing
40484077
properties:

0 commit comments

Comments
 (0)