Skip to content

Commit 3ce23ba

Browse files
discosturclaude
discostur
and
claude
committed
Add automated backup and restore via dedicated CRDs
Introduces operator-managed backup and restore for ClickHouse using clickhouse-backup, exposed through three new custom resources in the clickhouse.altinity.com/v1 API group: - ClickHouseBackup (chb): one-off backup -> Kubernetes Job - ClickHouseBackupSchedule (chbs): recurring backup -> managed CronJob - ClickHouseRestore (chr): one-off restore -> Kubernetes Job The controllers follow the existing ClickHouseKeeper controller-runtime pattern. clickhouse-backup runs as a sidecar (a documented prerequisite); the generated jobs trigger it remotely through the system.backup_actions integration table, so no backup logic is reimplemented in the operator. Cluster-aware: backs up one replica per shard for Replicated* tables (AllReplicas opt-in for non-replicated data); on restore it applies the schema on all replicas and the data on the first replica of each shard, letting native replication synchronize the rest. Restore safety follows the conventions of mature DB operators: preflight validation (target CHI Completed, topology reachable) and an overwrite guard that refuses a non-empty target unless overwrite=true. Includes the CRDs, RBAC (incl. batch jobs/cronjobs), regenerated install bundles and Helm chart, documentation and examples, Go unit tests and a TestFlows e2e test. Refs #1795, #862. Supersedes the gRPC-plugin approach of #1798. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Signed-off-by: Kilian Ries <mail@kilian-ries.de>
1 parent dae4e5b commit 3ce23ba

42 files changed

Lines changed: 6453 additions & 3 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎cmd/operator/app/main.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ func Run() {
9999
launchClickHouse(ctx, &wg)
100100
launchClickHouseReconcilerMetricsExporter(ctx, &wg)
101101
launchKeeper(ctx, &wg)
102+
launchBackup(ctx, &wg)
102103

103104
// Wait for completion
104105
<-ctx.Done()
@@ -142,6 +143,25 @@ func launchKeeper(ctx context.Context, wg *sync.WaitGroup) {
142143
}()
143144
}
144145

146+
func launchBackup(ctx context.Context, wg *sync.WaitGroup) {
147+
backupErr := initBackup(ctx)
148+
wg.Add(1)
149+
go func() {
150+
defer wg.Done()
151+
if backupErr == nil {
152+
log.Info("Starting backup")
153+
backupErr = runBackup(ctx)
154+
if backupErr == nil {
155+
log.Info("Starting backup OK")
156+
} else {
157+
log.Warning("Starting backup FAILED with err: %v", backupErr)
158+
}
159+
} else {
160+
log.Warning("Starting backup skipped due to failed initialization with err: %v", backupErr)
161+
}
162+
}()
163+
}
164+
145165
// setupSignalsNotification sets up OS signals
146166
func setupSignalsNotification(cancel context.CancelFunc) {
147167
stopChan := make(chan os.Signal, 2)

‎cmd/operator/app/thread_backup.go‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package app
16+
17+
import (
18+
"context"
19+
20+
"github.com/go-logr/logr"
21+
22+
batchv1 "k8s.io/api/batch/v1"
23+
apiMachineryRuntime "k8s.io/apimachinery/pkg/runtime"
24+
clientGoScheme "k8s.io/client-go/kubernetes/scheme"
25+
ctrl "sigs.k8s.io/controller-runtime"
26+
ctrlRuntime "sigs.k8s.io/controller-runtime"
27+
"sigs.k8s.io/controller-runtime/pkg/cache"
28+
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
29+
30+
api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1"
31+
"github.com/altinity/clickhouse-operator/pkg/chop"
32+
backup "github.com/altinity/clickhouse-operator/pkg/controller/chbackup"
33+
)
34+
35+
var (
36+
backupScheme *apiMachineryRuntime.Scheme
37+
backupManager ctrlRuntime.Manager
38+
backupLogger logr.Logger
39+
)
40+
41+
func initBackup(ctx context.Context) error {
42+
var err error
43+
44+
backupLogger = ctrl.Log.WithName("backup-runner")
45+
46+
backupScheme = apiMachineryRuntime.NewScheme()
47+
if err = clientGoScheme.AddToScheme(backupScheme); err != nil {
48+
backupLogger.Error(err, "init backup - unable to clientGoScheme.AddToScheme")
49+
return err
50+
}
51+
// Registers ClickHouseInstallation along with ClickHouseBackup/Schedule/Restore kinds.
52+
if err = api.AddToScheme(backupScheme); err != nil {
53+
backupLogger.Error(err, "init backup - unable to api.AddToScheme")
54+
return err
55+
}
56+
57+
defaultNamespaces := make(map[string]cache.Config)
58+
for _, ns := range chop.Config().GetCacheNamespaces() {
59+
defaultNamespaces[ns] = cache.Config{}
60+
}
61+
backupManager, err = ctrlRuntime.NewManager(ctrlRuntime.GetConfigOrDie(), ctrlRuntime.Options{
62+
Scheme: backupScheme,
63+
Cache: cache.Options{
64+
DefaultNamespaces: defaultNamespaces,
65+
},
66+
// Disable the metrics listener: the keeper manager owns it on this pod.
67+
Metrics: metricsserver.Options{BindAddress: "0"},
68+
})
69+
if err != nil {
70+
backupLogger.Error(err, "init backup - unable to ctrlRuntime.NewManager")
71+
return err
72+
}
73+
74+
if err = ctrlRuntime.
75+
NewControllerManagedBy(backupManager).
76+
For(&api.ClickHouseBackup{}).
77+
Owns(&batchv1.Job{}).
78+
Complete(&backup.BackupController{
79+
Client: backupManager.GetClient(),
80+
Scheme: backupManager.GetScheme(),
81+
}); err != nil {
82+
backupLogger.Error(err, "init backup - unable to build ClickHouseBackup controller")
83+
return err
84+
}
85+
86+
if err = ctrlRuntime.
87+
NewControllerManagedBy(backupManager).
88+
For(&api.ClickHouseBackupSchedule{}).
89+
Owns(&batchv1.CronJob{}).
90+
Complete(&backup.ScheduleController{
91+
Client: backupManager.GetClient(),
92+
Scheme: backupManager.GetScheme(),
93+
}); err != nil {
94+
backupLogger.Error(err, "init backup - unable to build ClickHouseBackupSchedule controller")
95+
return err
96+
}
97+
98+
if err = ctrlRuntime.
99+
NewControllerManagedBy(backupManager).
100+
For(&api.ClickHouseRestore{}).
101+
Owns(&batchv1.Job{}).
102+
Complete(&backup.RestoreController{
103+
Client: backupManager.GetClient(),
104+
Scheme: backupManager.GetScheme(),
105+
}); err != nil {
106+
backupLogger.Error(err, "init backup - unable to build ClickHouseRestore controller")
107+
return err
108+
}
109+
110+
// Initialization successful
111+
return nil
112+
}
113+
114+
func runBackup(ctx context.Context) error {
115+
if err := backupManager.Start(ctx); err != nil {
116+
backupLogger.Error(err, "run backup - unable to backupManager.Start")
117+
return err
118+
}
119+
// Run successful
120+
return nil
121+
}

‎deploy/builder/cat-clickhouse-operator-install-yaml.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,14 @@ if [[ "${MANIFEST_PRINT_CRD}" == "yes" ]]; then
154154
cat "${TEMPLATES_DIR}/${SECTION_FILE_NAME}" | \
155155
OPERATOR_VERSION="${OPERATOR_VERSION}" \
156156
envsubst
157+
158+
# Render Backup CRDs (ClickHouseBackup, ClickHouseBackupSchedule, ClickHouseRestore)
159+
SECTION_FILE_NAME="clickhouse-operator-install-yaml-template-01-section-crd-04-backup.yaml"
160+
ensure_file "${TEMPLATES_DIR}" "${SECTION_FILE_NAME}" "${REPO_PATH_TEMPLATES_PATH}"
161+
render_separator
162+
cat "${TEMPLATES_DIR}/${SECTION_FILE_NAME}" | \
163+
OPERATOR_VERSION="${OPERATOR_VERSION}" \
164+
envsubst
157165
fi
158166

159167
if [[ "${MANIFEST_PRINT_RBAC_CLUSTERED}" == "yes" || "${MANIFEST_PRINT_RBAC_NAMESPACED}" == "yes" ]]; then

0 commit comments

Comments
 (0)