Add explicit default cluster defenition with internal_replication=1 and generated cluster secret (#89)

Author: GrigoryPervakov

.golangci.yml

@@ -106,9 +106,6 @@ linters:
     gosec:
       excludes:
         - G204
-    godot:
-      exclude:
-        - "^ *\\+operator-sdk:"
     ireturn:
       allow:
         - error
@@ -117,6 +114,7 @@ linters:
         - generic
         - Option$
         - github.com\/ClickHouse\/clickhouse-go/v2\.Conn
+        - github.com\/ClickHouse\/clickhouse-go\/v2\/lib\/driver\.Rows
         - k8s.io
     wsl_v5:
       allow-whole-block: true

internal/controller/clickhouse/config.go

@@ -166,19 +166,28 @@ type configGeneratorFunc func(tmpl *template.Template, r *clickhouseReconciler,
 type baseConfigParams struct {
 	Path   string
 	Log    controller.LoggerConfig
-	Macros map[string]any
+	Macros []macro
+
+	KeeperNodes       []keeperNode
+	KeeperIdentityEnv string
 
-	KeeperNodes               []keeperNode
-	KeeperIdentityEnv         string
 	DistributedDDLPath        string
 	DistributedDDLProfileName string
 	UsersXMLPath              string
 	UsersZookeeperPath        string
 	UDFZookeeperPath          string
 
+	ClusterSecretEnv string
+	ManagementPort   uint16
+	ClusterHosts     [][]string
+
 	OpenSSL controller.OpenSSLConfig
 }
 
+type macro struct {
+	Name  string
+	Value any
+}
 type keeperNode struct {
 	Host   string
 	Port   int32
@@ -226,23 +235,38 @@ func baseConfigGenerator(tmpl *template.Template, r *clickhouseReconciler, id v1
 		openSSL.Client.PreferServerCiphers = true
 	}
 
+	clusterHosts := make([][]string, r.Cluster.Shards())
+	for shard := range r.Cluster.Shards() {
+		hosts := make([]string, r.Cluster.Replicas())
+		for replica := range r.Cluster.Replicas() {
+			hosts[replica] = r.Cluster.HostnameByID(v1.ClickHouseReplicaID{ShardID: shard, Index: replica})
+		}
+
+		clusterHosts[shard] = hosts
+	}
+
 	params := baseConfigParams{
 		Path: internal.ClickHouseDataPath,
 		Log:  controller.GenerateLoggerConfig(r.Cluster.Spec.Settings.Logger, LogPath, "clickhouse-server"),
-		Macros: map[string]any{
-			"cluster": DefaultClusterName,
-			"shard":   id.ShardID,
-			"replica": id.Index,
+		Macros: []macro{
+			{Name: "cluster", Value: DefaultClusterName},
+			{Name: "shard", Value: id.ShardID},
+			{Name: "replica", Value: id.Index},
 		},
 
-		KeeperNodes:               keeperNodes,
-		KeeperIdentityEnv:         EnvKeeperIdentity,
+		KeeperNodes:       keeperNodes,
+		KeeperIdentityEnv: EnvKeeperIdentity,
+
 		DistributedDDLPath:        KeeperPathDistributedDDL,
 		DistributedDDLProfileName: DefaultProfileName,
 		UsersXMLPath:              UsersFileName,
 		UsersZookeeperPath:        KeeperPathUsers,
 		UDFZookeeperPath:          KeeperPathUDF,
 
+		ClusterSecretEnv: EnvClusterSecret,
+		ManagementPort:   PortManagement,
+		ClusterHosts:     clusterHosts,
+
 		OpenSSL: openSSL,
 	}
 
@@ -259,13 +283,28 @@ type networkConfigParams struct {
 	InterserverHTTPUser           string
 	InterserverHTTPPasswordEnvVar string
 	ManagementPort                uint16
-	Protocols                     map[string]protocol
+	Protocols                     []namedProtocol
+}
+
+type namedProtocol struct {
+	Name     string
+	Protocol protocol
 }
 
 func networkConfigGenerator(tmpl *template.Template, r *clickhouseReconciler, _ v1.ClickHouseReplicaID) (string, error) {
-	protocols := buildProtocols(r.Cluster)
-	delete(protocols, "interserver")
-	delete(protocols, "management")
+	var protocols []namedProtocol
+	for name, proto := range buildProtocols(r.Cluster) {
+		if name == "interserver" || name == "management" {
+			continue
+		}
+
+		protocols = append(protocols, namedProtocol{
+			Name:     name,
+			Protocol: proto,
+		})
+	}
+
+	controllerutil.SortKey(protocols, func(p namedProtocol) string { return p.Name })
 
 	params := networkConfigParams{
 		InterserverHTTPPort:           PortInterserver,

internal/controller/clickhouse/constants.go

@@ -47,10 +47,12 @@ const (
 	EnvInterserverPassword = "CLICKHOUSE_INTERSERVER_PASSWORD"
 	EnvDefaultUserPassword = "CLICKHOUSE_DEFAULT_USER_PASSWORD"
 	EnvKeeperIdentity      = "CLICKHOUSE_KEEPER_IDENTITY"
+	EnvClusterSecret       = "CLICKHOUSE_CLUSTER_SECRET"
 
 	SecretKeyInterserverPassword = "interserver-password"
 	SecretKeyManagementPassword  = "management-password"
 	SecretKeyKeeperIdentity      = "keeper-identity"
+	SecretKeyClusterSecret       = "cluster-secret"
 )
 
 var (
@@ -59,5 +61,14 @@ var (
 		SecretKeyInterserverPassword: "%s",
 		SecretKeyManagementPassword:  "%s",
 		SecretKeyKeeperIdentity:      "clickhouse:%s",
+		SecretKeyClusterSecret:       "%s",
+	}
+	secretsToEnvMapping = []struct {
+		Key string
+		Env string
+	}{
+		{Key: SecretKeyInterserverPassword, Env: EnvInterserverPassword},
+		{Key: SecretKeyKeeperIdentity, Env: EnvKeeperIdentity},
+		{Key: SecretKeyClusterSecret, Env: EnvClusterSecret},
 	}
 )

internal/controller/clickhouse/templates.go

@@ -7,17 +7,17 @@ import (
 	"path"
 	"strconv"
 
-	v1 "github.com/ClickHouse/clickhouse-operator/api/v1alpha1"
-	"github.com/ClickHouse/clickhouse-operator/internal"
-	"github.com/ClickHouse/clickhouse-operator/internal/controller"
-	"github.com/ClickHouse/clickhouse-operator/internal/controllerutil"
-
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	policyv1 "k8s.io/api/policy/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/ptr"
+
+	v1 "github.com/ClickHouse/clickhouse-operator/api/v1alpha1"
+	"github.com/ClickHouse/clickhouse-operator/internal"
+	"github.com/ClickHouse/clickhouse-operator/internal/controller"
+	"github.com/ClickHouse/clickhouse-operator/internal/controllerutil"
 )
 
 func templateHeadlessService(cr *v1.ClickHouseCluster) *corev1.Service {
@@ -240,28 +240,6 @@ func templateStatefulSet(r *clickhouseReconciler, id v1.ClickHouseReplicaID) (*a
 				Name:  "CLICKHOUSE_SKIP_USER_SETUP",
 				Value: "1",
 			},
-			{
-				Name: EnvInterserverPassword,
-				ValueFrom: &corev1.EnvVarSource{
-					SecretKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{
-							Name: r.Cluster.SecretName(),
-						},
-						Key: SecretKeyInterserverPassword,
-					},
-				},
-			},
-			{
-				Name: EnvKeeperIdentity,
-				ValueFrom: &corev1.EnvVarSource{
-					SecretKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{
-							Name: r.Cluster.SecretName(),
-						},
-						Key: SecretKeyKeeperIdentity,
-					},
-				},
-			},
 		}, r.Cluster.Spec.ContainerTemplate.Env...),
 		Ports: []corev1.ContainerPort{
 			{
@@ -287,6 +265,20 @@ func templateStatefulSet(r *clickhouseReconciler, id v1.ClickHouseReplicaID) (*a
 		},
 	}
 
+	for _, secret := range secretsToEnvMapping {
+		container.Env = append(container.Env, corev1.EnvVar{
+			Name: secret.Env,
+			ValueFrom: &corev1.EnvVarSource{
+				SecretKeyRef: &corev1.SecretKeySelector{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: r.Cluster.SecretName(),
+					},
+					Key: secret.Key,
+				},
+			},
+		})
+	}
+
 	container.Ports = make([]corev1.ContainerPort, 0, len(protocols))
 	for name, protocol := range protocols {
 		if protocol.Port == 0 {

internal/controller/clickhouse/templates/base.yaml.tmpl

@@ -2,8 +2,8 @@ path: {{ .Path }}
 logger:
 {{ yaml .Log | indent 2 }}
 macros:
-  {{- range $key, $value := .Macros }}
-  {{ $key }}: {{ $value }}
+  {{- range $i, $m := .Macros }}
+  {{ $m.Name }}: {{ $m.Value }}
   {{- end }}
 
 {{- /* Replication settings */}}
@@ -19,6 +19,21 @@ zookeeper:
   identity:
     "@from_env": {{ .KeeperIdentityEnv }}
 
+remote_servers:
+  default:
+    secret:
+      "@from_env": {{ .ClusterSecretEnv }}
+    shard:
+    {{- $ctx := .}}
+    {{- range $shard, $replicas := .ClusterHosts }}
+      - internal_replication: true
+        replica:
+        {{- range $i, $replica := $replicas }}
+          - host: {{ $replica }}
+            port: {{ $ctx.ManagementPort }}
+        {{- end }}
+    {{- end }}
+
 distributed_ddl:
   path: {{ .DistributedDDLPath }}
   profile: {{ .DistributedDDLProfileName }}

internal/controller/clickhouse/templates/network.yaml.tmpl

@@ -12,17 +12,17 @@ interserver_http_credentials:
 tcp_port: {{ .ManagementPort }}
 
 protocols:
-{{- range $name, $protocol := .Protocols }}
-  {{ $name }}:
-    type: {{ $protocol.Type }}
-    {{- if ne $protocol.Impl "" }}
-    impl: {{ $protocol.Impl }}
+{{- range $protocol := .Protocols }}
+  {{ $protocol.Name }}:
+    type: {{ $protocol.Protocol.Type }}
+    {{- if ne $protocol.Protocol.Impl "" }}
+    impl: {{ $protocol.Protocol.Impl }}
     {{- end }}
-    {{- if ne $protocol.Port 0 }}
-    port: {{ $protocol.Port }}
+    {{- if ne $protocol.Protocol.Port 0 }}
+    port: {{ $protocol.Protocol.Port }}
     {{- end }}
-    {{- if ne $protocol.Description "" }}
-    description: {{ $protocol.Description }}
+    {{- if ne $protocol.Protocol.Description "" }}
+    description: {{ $protocol.Protocol.Description }}
     {{- end }}
 {{- end }}