Skip to content

Commit f9eeb08

Browse files
scanhex12GrigoryPervakov
scanhex12
and
GrigoryPervakov
authored
feat: named collections in keeper with key encryption (#111)
* squash: named_collections_encryption_key Squashed commits: f06d079 Konstantin Vedernikov - start e0f2b48 Konstantin Vedernikov - fix review e79beac Konstantin Vedernikov - fix 0761dbe Konstantin Vedernikov - fix test 0ebfa59 Konstantin Vedernikov - replace image tag with version 61b7129 Konstantin Vedernikov - new template for named collections ff93bfd Konstantin Vedernikov - style 674a0f1 Konstantin Vedernikov - fix 6b1e7b8 Konstantin Vedernikov - better 6b52f7f Pervakov Grigorii - fix(docs): .containerTemplate.resources docstring (#135) cebaa52 scanhex12 - fix review b7176a1 scanhex12 - remove extra fixes 4ebde29 scanhex12 - return back some changes e77dc2a scanhex12 - better c04ed9a scanhex12 - add version to tests 25f7808 scanhex12 - new config file into e2e test 3bf93db scanhex12 - lint 41042db scanhex12 - fix review issues f5eea12 scanhex12 - fix review a2b4c8e scanhex12 - better 42ed945 scanhex12 - fix review * Cleanup and fix issues * wait for version probe job --------- Co-authored-by: Pervakov Grigorii <pervakov.grigory@gmail.com>
1 parent d4e22be commit f9eeb08

14 files changed

Lines changed: 726 additions & 385 deletions

File tree

‎internal/controller/clickhouse/config.go‎

Lines changed: 70 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,14 @@ import (
1515
"github.com/ClickHouse/clickhouse-operator/internal/controller"
1616
"github.com/ClickHouse/clickhouse-operator/internal/controller/keeper"
1717
"github.com/ClickHouse/clickhouse-operator/internal/controllerutil"
18+
"github.com/ClickHouse/clickhouse-operator/internal/upgrade"
1819
)
1920

2021
var (
2122
//go:embed templates/base.yaml.tmpl
2223
baseConfigTemplateStr string
24+
//go:embed templates/named_collections.yaml.tmpl
25+
namedCollectionsTemplateStr string
2326
//go:embed templates/network.yaml.tmpl
2427
networkConfigTemplateStr string
2528
//go:embed templates/log_tables.yaml.tmpl
@@ -33,11 +36,41 @@ var (
3336
)
3437

3538
func init() {
39+
templateFuncs := template.FuncMap{
40+
"yaml": func(v any) (string, error) {
41+
data, err := yaml.Marshal(v)
42+
return string(data), err
43+
},
44+
"indent": func(countRaw any, strRaw any) (string, error) {
45+
count, ok := countRaw.(int)
46+
if !ok {
47+
return "", fmt.Errorf("indent: expected int for indentation value, got %T", countRaw)
48+
}
49+
50+
str, ok := strRaw.(string)
51+
if !ok {
52+
return "", fmt.Errorf("indent: expected string for content value, got %T", strRaw)
53+
}
54+
55+
builder := strings.Builder{}
56+
indentation := strings.Repeat(" ", count)
57+
58+
for line := range strings.SplitSeq(str, "\n") {
59+
if _, err := fmt.Fprintf(&builder, "%s%s\n", indentation, line); err != nil {
60+
return "", fmt.Errorf("failed to write indented line: %w", err)
61+
}
62+
}
63+
64+
return builder.String(), nil
65+
},
66+
}
67+
3668
for _, templateSpec := range []struct {
3769
Path string
3870
Filename string
3971
Raw string
4072
Generator configGeneratorFunc
73+
Enabled func(r *clickhouseReconciler) bool
4174
}{{
4275
Path: ConfigPath,
4376
Filename: ConfigFileName,
@@ -53,6 +86,14 @@ func init() {
5386
Filename: "00-logs-tables.yaml",
5487
Raw: logTablesConfigTemplateStr,
5588
Generator: logTablesConfigGenerator,
89+
}, {
90+
Path: path.Join(ConfigPath, ConfigDPath),
91+
Filename: "00-named-collections.yaml",
92+
Raw: namedCollectionsTemplateStr,
93+
Generator: namedCollectionsConfigGenerator,
94+
Enabled: func(r *clickhouseReconciler) bool {
95+
return upgrade.VersionAtLeast(r.Cluster.Status.Version, minVersionNamedCollections)
96+
},
5697
}, {
5798
Path: ConfigPath,
5899
Filename: UsersFileName,
@@ -64,68 +105,35 @@ func init() {
64105
Raw: clientConfigTemplateStr,
65106
Generator: clientConfigGenerator,
66107
}} {
67-
tmpl := template.New("").Funcs(template.FuncMap{
68-
"yaml": func(v any) (string, error) {
69-
data, err := yaml.Marshal(v)
70-
return string(data), err
71-
},
72-
"indent": func(countRaw any, strRaw any) (string, error) {
73-
count, ok := countRaw.(int)
74-
if !ok {
75-
return "", fmt.Errorf("indent: expected int for indentation value, got %T", countRaw)
76-
}
77-
78-
str, ok := strRaw.(string)
79-
if !ok {
80-
return "", fmt.Errorf("indent: expected string for content value, got %T", strRaw)
81-
}
82-
83-
builder := strings.Builder{}
84-
indentation := strings.Repeat(" ", count)
85-
86-
for line := range strings.SplitSeq(str, "\n") {
87-
if _, err := fmt.Fprintf(&builder, "%s%s\n", indentation, line); err != nil {
88-
return "", fmt.Errorf("failed to write indented line: %w", err)
89-
}
90-
}
91-
92-
return builder.String(), nil
93-
},
94-
})
95-
if _, err := tmpl.Parse(templateSpec.Raw); err != nil {
96-
panic(fmt.Sprintf("failed to parse template %s: %v", templateSpec.Filename, err))
97-
}
108+
tmpl := template.Must(template.New("").Funcs(templateFuncs).Parse(templateSpec.Raw))
98109

99110
generators = append(generators, &templateConfigGenerator{
100111
filename: templateSpec.Filename,
101112
path: templateSpec.Path,
102113
template: tmpl,
103114
generator: templateSpec.Generator,
115+
enabled: templateSpec.Enabled,
104116
})
105117
}
106118

107119
generators = append(generators,
108120
&extraConfigGenerator{
109121
Name: ExtraConfigFileName,
110122
ConfigSubPath: ConfigDPath,
111-
Getter: func(r *clickhouseReconciler) []byte {
112-
return r.Cluster.Spec.Settings.ExtraConfig.Raw
113-
},
123+
Getter: func(r *clickhouseReconciler) []byte { return r.Cluster.Spec.Settings.ExtraConfig.Raw },
114124
},
115125
&extraConfigGenerator{
116126
Name: ExtraUsersConfigFileName,
117127
ConfigSubPath: UsersDPath,
118-
Getter: func(r *clickhouseReconciler) []byte {
119-
return r.Cluster.Spec.Settings.ExtraUsersConfig.Raw
120-
},
128+
Getter: func(r *clickhouseReconciler) []byte { return r.Cluster.Spec.Settings.ExtraUsersConfig.Raw },
121129
})
122130
}
123131

124132
type configGenerator interface {
125133
Filename() string
126134
Path() string
127135
ConfigKey() string
128-
Exists(r *clickhouseReconciler) bool
136+
Enabled(r *clickhouseReconciler) bool
129137
Generate(r *clickhouseReconciler, id v1.ClickHouseReplicaID) (string, error)
130138
}
131139

@@ -134,6 +142,7 @@ type templateConfigGenerator struct {
134142
path string
135143
template *template.Template
136144
generator configGeneratorFunc
145+
enabled func(r *clickhouseReconciler) bool
137146
}
138147

139148
func (g *templateConfigGenerator) Filename() string {
@@ -148,8 +157,8 @@ func (g *templateConfigGenerator) ConfigKey() string {
148157
return controllerutil.PathToName(path.Join(g.path, g.filename))
149158
}
150159

151-
func (g *templateConfigGenerator) Exists(*clickhouseReconciler) bool {
152-
return true
160+
func (g *templateConfigGenerator) Enabled(r *clickhouseReconciler) bool {
161+
return g.enabled == nil || g.enabled(r)
153162
}
154163

155164
func (g *templateConfigGenerator) Generate(r *clickhouseReconciler, id v1.ClickHouseReplicaID) (string, error) {
@@ -389,6 +398,25 @@ func clientConfigGenerator(tmpl *template.Template, r *clickhouseReconciler, _ v
389398
return builder.String(), nil
390399
}
391400

401+
type namedCollectionsConfigParams struct {
402+
NamedCollectionsKeyEnv string
403+
NamedCollectionsPath string
404+
}
405+
406+
func namedCollectionsConfigGenerator(tmpl *template.Template, _ *clickhouseReconciler, _ v1.ClickHouseReplicaID) (string, error) {
407+
params := namedCollectionsConfigParams{
408+
NamedCollectionsKeyEnv: EnvNamedCollectionsKey,
409+
NamedCollectionsPath: KeeperPathNamedCollections,
410+
}
411+
412+
builder := strings.Builder{}
413+
if err := tmpl.Execute(&builder, params); err != nil {
414+
return "", fmt.Errorf("template named collections config: %w", err)
415+
}
416+
417+
return builder.String(), nil
418+
}
419+
392420
type extraConfigGenerator struct {
393421
Name string
394422
ConfigSubPath string
@@ -407,12 +435,12 @@ func (g *extraConfigGenerator) ConfigKey() string {
407435
return g.Name
408436
}
409437

410-
func (g *extraConfigGenerator) Exists(r *clickhouseReconciler) bool {
438+
func (g *extraConfigGenerator) Enabled(r *clickhouseReconciler) bool {
411439
return len(g.Getter(r)) > 0
412440
}
413441

414442
func (g *extraConfigGenerator) Generate(r *clickhouseReconciler, _ v1.ClickHouseReplicaID) (string, error) {
415-
if !g.Exists(r) {
443+
if !g.Enabled(r) {
416444
return "", errors.New("extra config generator called, but no extra config provided")
417445
}
418446

‎internal/controller/clickhouse/config_test.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,9 @@ var _ = Describe("ConfigGenerator", func() {
2929
},
3030
},
3131
},
32+
Status: v1.ClickHouseClusterStatus{
33+
Version: "25.12.1.1",
34+
},
3235
},
3336
keeper: v1.KeeperCluster{
3437
Spec: v1.KeeperClusterSpec{
@@ -39,7 +42,7 @@ var _ = Describe("ConfigGenerator", func() {
3942

4043
for _, generator := range generators {
4144
It("should generate config: "+generator.Filename(), func() {
42-
Expect(generator.Exists(&ctx)).To(BeTrue())
45+
Expect(generator.Enabled(&ctx)).To(BeTrue())
4346
data, err := generator.Generate(&ctx, v1.ClickHouseReplicaID{})
4447
Expect(err).ToNot(HaveOccurred())
4548

‎internal/controller/clickhouse/constants.go‎

Lines changed: 52 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,13 @@
11
package clickhouse
22

33
import (
4+
"fmt"
5+
46
"github.com/blang/semver/v4"
7+
8+
v1 "github.com/ClickHouse/clickhouse-operator/api/v1alpha1"
9+
"github.com/ClickHouse/clickhouse-operator/internal/controllerutil"
10+
"github.com/ClickHouse/clickhouse-operator/internal/upgrade"
511
)
612

713
const (
@@ -32,10 +38,11 @@ const (
3238

3339
LogPath = "/var/log/clickhouse-server/"
3440

35-
DefaultClusterName = "default"
36-
KeeperPathUsers = "/clickhouse/access"
37-
KeeperPathUDF = "/clickhouse/user_defined"
38-
KeeperPathDistributedDDL = "/clickhouse/task_queue/ddl"
41+
DefaultClusterName = "default"
42+
KeeperPathUsers = "/clickhouse/access"
43+
KeeperPathUDF = "/clickhouse/user_defined"
44+
KeeperPathDistributedDDL = "/clickhouse/task_queue/ddl"
45+
KeeperPathNamedCollections = "/clickhouse/named_collections"
3946

4047
ContainerName = "clickhouse-server"
4148
DefaultRevisionHistory = 10
@@ -49,27 +56,55 @@ const (
4956
EnvDefaultUserPassword = "CLICKHOUSE_DEFAULT_USER_PASSWORD"
5057
EnvKeeperIdentity = "CLICKHOUSE_KEEPER_IDENTITY"
5158
EnvClusterSecret = "CLICKHOUSE_CLUSTER_SECRET"
59+
EnvNamedCollectionsKey = "CLICKHOUSE_NAMED_COLLECTIONS_KEY"
5260

5361
SecretKeyInterserverPassword = "interserver-password"
5462
SecretKeyManagementPassword = "management-password"
5563
SecretKeyKeeperIdentity = "keeper-identity"
5664
SecretKeyClusterSecret = "cluster-secret"
65+
SecretKeyNamedCollectionsKey = "named-collections-key"
66+
67+
// NamedCollectionsKeyByteLen is the AES-128 key size in bytes (16 bytes = 32 hex chars).
68+
NamedCollectionsKeyByteLen = 16
5769
)
5870

71+
type secretSpec struct {
72+
Key string
73+
Env string
74+
Format string
75+
Generate func() any
76+
Enabled func(cluster *v1.ClickHouseCluster) bool
77+
}
78+
79+
func (s *secretSpec) generate() []byte {
80+
var arg any
81+
if s.Generate != nil {
82+
arg = s.Generate()
83+
} else {
84+
arg = controllerutil.GeneratePassword()
85+
}
86+
87+
return fmt.Appendf(nil, s.Format, arg)
88+
}
89+
90+
func (s *secretSpec) enabled(cluster *v1.ClickHouseCluster) bool {
91+
return s.Enabled == nil || s.Enabled(cluster)
92+
}
93+
5994
var (
95+
// minVersionNamedCollections is the minimum ClickHouse version that supports keeper_encrypted for named collections.
96+
minVersionNamedCollections = upgrade.ClickHouseVersion{Major: 25, Minor: 12} //nolint:mnd
6097
breakingStatefulSetVersion, _ = semver.Parse("0.0.1")
61-
secretsToGenerate = map[string]string{
62-
SecretKeyInterserverPassword: "%s",
63-
SecretKeyManagementPassword: "%s",
64-
SecretKeyKeeperIdentity: "clickhouse:%s",
65-
SecretKeyClusterSecret: "%s",
66-
}
67-
secretsToEnvMapping = []struct {
68-
Key string
69-
Env string
70-
}{
71-
{Key: SecretKeyInterserverPassword, Env: EnvInterserverPassword},
72-
{Key: SecretKeyKeeperIdentity, Env: EnvKeeperIdentity},
73-
{Key: SecretKeyClusterSecret, Env: EnvClusterSecret},
98+
clusterSecrets = []secretSpec{
99+
{Key: SecretKeyInterserverPassword, Env: EnvInterserverPassword, Format: "%s"},
100+
{Key: SecretKeyManagementPassword, Format: "%s"},
101+
{Key: SecretKeyKeeperIdentity, Env: EnvKeeperIdentity, Format: "clickhouse:%s"},
102+
{Key: SecretKeyClusterSecret, Env: EnvClusterSecret, Format: "%s"},
103+
{Key: SecretKeyNamedCollectionsKey, Env: EnvNamedCollectionsKey, Format: "%x",
104+
Generate: func() any { return controllerutil.GenerateRandomBytes(NamedCollectionsKeyByteLen) },
105+
Enabled: func(cluster *v1.ClickHouseCluster) bool {
106+
return upgrade.VersionAtLeast(cluster.Status.Version, minVersionNamedCollections)
107+
},
108+
},
74109
}
75110
)

0 commit comments

Comments
 (0)