Skip to content

Commit 56b62c0

Browse files
discosturclaude
discostur
and
claude
committed
Add automated backup and restore via dedicated CRDs
Introduces operator-managed backup and restore for ClickHouse using clickhouse-backup, exposed through three new custom resources in the clickhouse.altinity.com/v1 API group: - ClickHouseBackup (chb): one-off backup -> Kubernetes Job - ClickHouseBackupSchedule (chbs): recurring backup -> managed CronJob - ClickHouseRestore (chr): one-off restore -> Kubernetes Job The controllers follow the existing ClickHouseKeeper controller-runtime pattern. clickhouse-backup runs as a sidecar (a documented prerequisite); the generated jobs trigger it remotely through the system.backup_actions integration table, so no backup logic is reimplemented in the operator. Cluster-aware: backs up one replica per shard for Replicated* tables (AllReplicas opt-in for non-replicated data); on restore it applies the schema on the first replica per shard via ON CLUSTER (requires the sidecar's restore_schema_on_cluster) and the data on the first replica, letting native replication synchronize the remaining replicas. Restore safety follows the conventions of mature DB operators: preflight validation (target CHI Completed, topology reachable) and an overwrite guard that refuses a non-empty target unless overwrite=true. Also adds: selective (tables/partitions) and incremental (--diff-from-remote) backups; remote-backup retention (keepLastRemote); optional post-backup verification; Prometheus metrics on the operator's existing :9999 endpoint plus Kubernetes Events; and annotation-driven bootstrap-from-backup for new installations. Compression and encryption are documented as clickhouse-backup sidecar settings. Includes the CRDs, RBAC (incl. batch jobs/cronjobs), regenerated install bundles and Helm chart, documentation and examples, Go unit tests and a TestFlows e2e test. Refs #1795, #862. Supersedes the gRPC-plugin approach of #1798. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Signed-off-by: Kilian Ries <mail@kilian-ries.de>
1 parent dae4e5b commit 56b62c0

46 files changed

Lines changed: 7579 additions & 3 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎cmd/operator/app/main.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ func Run() {
9999
launchClickHouse(ctx, &wg)
100100
launchClickHouseReconcilerMetricsExporter(ctx, &wg)
101101
launchKeeper(ctx, &wg)
102+
launchBackup(ctx, &wg)
102103

103104
// Wait for completion
104105
<-ctx.Done()
@@ -142,6 +143,25 @@ func launchKeeper(ctx context.Context, wg *sync.WaitGroup) {
142143
}()
143144
}
144145

146+
func launchBackup(ctx context.Context, wg *sync.WaitGroup) {
147+
backupErr := initBackup(ctx)
148+
wg.Add(1)
149+
go func() {
150+
defer wg.Done()
151+
if backupErr == nil {
152+
log.Info("Starting backup")
153+
backupErr = runBackup(ctx)
154+
if backupErr == nil {
155+
log.Info("Starting backup OK")
156+
} else {
157+
log.Warning("Starting backup FAILED with err: %v", backupErr)
158+
}
159+
} else {
160+
log.Warning("Starting backup skipped due to failed initialization with err: %v", backupErr)
161+
}
162+
}()
163+
}
164+
145165
// setupSignalsNotification sets up OS signals
146166
func setupSignalsNotification(cancel context.CancelFunc) {
147167
stopChan := make(chan os.Signal, 2)

‎cmd/operator/app/thread_backup.go‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package app
16+
17+
import (
18+
"context"
19+
20+
"github.com/go-logr/logr"
21+
22+
batchv1 "k8s.io/api/batch/v1"
23+
apiMachineryRuntime "k8s.io/apimachinery/pkg/runtime"
24+
clientGoScheme "k8s.io/client-go/kubernetes/scheme"
25+
ctrl "sigs.k8s.io/controller-runtime"
26+
ctrlRuntime "sigs.k8s.io/controller-runtime"
27+
"sigs.k8s.io/controller-runtime/pkg/cache"
28+
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
29+
30+
api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1"
31+
"github.com/altinity/clickhouse-operator/pkg/chop"
32+
backup "github.com/altinity/clickhouse-operator/pkg/controller/chbackup"
33+
)
34+
35+
var (
36+
backupScheme *apiMachineryRuntime.Scheme
37+
backupManager ctrlRuntime.Manager
38+
backupLogger logr.Logger
39+
)
40+
41+
func initBackup(ctx context.Context) error {
42+
var err error
43+
44+
backupLogger = ctrl.Log.WithName("backup-runner")
45+
46+
backupScheme = apiMachineryRuntime.NewScheme()
47+
if err = clientGoScheme.AddToScheme(backupScheme); err != nil {
48+
backupLogger.Error(err, "init backup - unable to clientGoScheme.AddToScheme")
49+
return err
50+
}
51+
// Registers ClickHouseInstallation along with ClickHouseBackup/Schedule/Restore kinds.
52+
if err = api.AddToScheme(backupScheme); err != nil {
53+
backupLogger.Error(err, "init backup - unable to api.AddToScheme")
54+
return err
55+
}
56+
57+
defaultNamespaces := make(map[string]cache.Config)
58+
for _, ns := range chop.Config().GetCacheNamespaces() {
59+
defaultNamespaces[ns] = cache.Config{}
60+
}
61+
backupManager, err = ctrlRuntime.NewManager(ctrlRuntime.GetConfigOrDie(), ctrlRuntime.Options{
62+
Scheme: backupScheme,
63+
Cache: cache.Options{
64+
DefaultNamespaces: defaultNamespaces,
65+
},
66+
// Disable the metrics listener: the keeper manager owns it on this pod.
67+
Metrics: metricsserver.Options{BindAddress: "0"},
68+
})
69+
if err != nil {
70+
backupLogger.Error(err, "init backup - unable to ctrlRuntime.NewManager")
71+
return err
72+
}
73+
74+
recorder := backupManager.GetEventRecorderFor("clickhouse-backup")
75+
76+
if err = ctrlRuntime.
77+
NewControllerManagedBy(backupManager).
78+
For(&api.ClickHouseBackup{}).
79+
Owns(&batchv1.Job{}).
80+
Complete(&backup.BackupController{
81+
Client: backupManager.GetClient(),
82+
Scheme: backupManager.GetScheme(),
83+
Recorder: recorder,
84+
}); err != nil {
85+
backupLogger.Error(err, "init backup - unable to build ClickHouseBackup controller")
86+
return err
87+
}
88+
89+
if err = ctrlRuntime.
90+
NewControllerManagedBy(backupManager).
91+
For(&api.ClickHouseBackupSchedule{}).
92+
Owns(&batchv1.CronJob{}).
93+
Complete(&backup.ScheduleController{
94+
Client: backupManager.GetClient(),
95+
Scheme: backupManager.GetScheme(),
96+
}); err != nil {
97+
backupLogger.Error(err, "init backup - unable to build ClickHouseBackupSchedule controller")
98+
return err
99+
}
100+
101+
if err = ctrlRuntime.
102+
NewControllerManagedBy(backupManager).
103+
For(&api.ClickHouseRestore{}).
104+
Owns(&batchv1.Job{}).
105+
Complete(&backup.RestoreController{
106+
Client: backupManager.GetClient(),
107+
Scheme: backupManager.GetScheme(),
108+
Recorder: recorder,
109+
}); err != nil {
110+
backupLogger.Error(err, "init backup - unable to build ClickHouseRestore controller")
111+
return err
112+
}
113+
114+
// Bootstrap-from-backup: watch CHIs and auto-restore when annotated.
115+
if err = ctrlRuntime.
116+
NewControllerManagedBy(backupManager).
117+
For(&api.ClickHouseInstallation{}).
118+
Complete(&backup.BootstrapController{
119+
Client: backupManager.GetClient(),
120+
Scheme: backupManager.GetScheme(),
121+
Recorder: recorder,
122+
}); err != nil {
123+
backupLogger.Error(err, "init backup - unable to build bootstrap controller")
124+
return err
125+
}
126+
127+
// Initialization successful
128+
return nil
129+
}
130+
131+
func runBackup(ctx context.Context) error {
132+
if err := backupManager.Start(ctx); err != nil {
133+
backupLogger.Error(err, "run backup - unable to backupManager.Start")
134+
return err
135+
}
136+
// Run successful
137+
return nil
138+
}

‎deploy/builder/cat-clickhouse-operator-install-yaml.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,14 @@ if [[ "${MANIFEST_PRINT_CRD}" == "yes" ]]; then
154154
cat "${TEMPLATES_DIR}/${SECTION_FILE_NAME}" | \
155155
OPERATOR_VERSION="${OPERATOR_VERSION}" \
156156
envsubst
157+
158+
# Render Backup CRDs (ClickHouseBackup, ClickHouseBackupSchedule, ClickHouseRestore)
159+
SECTION_FILE_NAME="clickhouse-operator-install-yaml-template-01-section-crd-04-backup.yaml"
160+
ensure_file "${TEMPLATES_DIR}" "${SECTION_FILE_NAME}" "${REPO_PATH_TEMPLATES_PATH}"
161+
render_separator
162+
cat "${TEMPLATES_DIR}/${SECTION_FILE_NAME}" | \
163+
OPERATOR_VERSION="${OPERATOR_VERSION}" \
164+
envsubst
157165
fi
158166

159167
if [[ "${MANIFEST_PRINT_RBAC_CLUSTERED}" == "yes" || "${MANIFEST_PRINT_RBAC_NAMESPACED}" == "yes" ]]; then

0 commit comments

Comments
 (0)