|
| 1 | +//go:build integration |
| 2 | + |
| 3 | +package main |
| 4 | + |
| 5 | +import ( |
| 6 | + "encoding/json" |
| 7 | + "os" |
| 8 | + "strings" |
| 9 | + "testing" |
| 10 | + "time" |
| 11 | + |
| 12 | + "github.com/rs/zerolog/log" |
| 13 | + |
| 14 | + "github.com/Altinity/clickhouse-backup/v2/pkg/metadata" |
| 15 | +) |
| 16 | + |
| 17 | +// TestServerWatchRBACConfigs - `server --watch --rbac --configs --named-collections` shall pass RBAC, |
| 18 | +// configs and named collections flags into the watch goroutine, see https://github.com/Altinity/clickhouse-backup/issues/955 |
| 19 | +func TestServerWatchRBACConfigs(t *testing.T) { |
| 20 | + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "20.4") < 0 { |
| 21 | + t.Skipf("Test skipped, RBAC not available for %s version", os.Getenv("CLICKHOUSE_VERSION")) |
| 22 | + } |
| 23 | + withNamedCollections := compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "22.12") >= 0 |
| 24 | + env, r := NewTestEnvironment(t) |
| 25 | + defer env.Cleanup(t, r) |
| 26 | + env.connectWithWait(t, r, 0*time.Second, 1*time.Second, 1*time.Minute) |
| 27 | + |
| 28 | + prefix := "watch955" |
| 29 | + dbName := "test_watch955" |
| 30 | + cleanBackups := func() { |
| 31 | + for _, location := range []string{"local", "remote"} { |
| 32 | + out, _ := env.DockerExecOut("clickhouse-backup", "bash", "-ce", "clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml list "+location+" 2>/dev/null | cut -d ' ' -f 1 | grep '^"+prefix+"-' || true") |
| 33 | + for _, backupName := range strings.Fields(out) { |
| 34 | + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-ce", "clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml delete "+location+" "+backupName+" 2>/dev/null || true") |
| 35 | + } |
| 36 | + } |
| 37 | + // backups killed mid-flight may not show up in `list local`, remove leftovers from all backup disks |
| 38 | + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-ce", "rm -rf /var/lib/clickhouse/backup/"+prefix+"-* /hdd1_data/backup/"+prefix+"-* /hdd2_data/backup/"+prefix+"-* 2>/dev/null || true") |
| 39 | + } |
| 40 | + cleanBackups() |
| 41 | + r.NoError(env.dropDatabase(dbName, true)) |
| 42 | + |
| 43 | + // the pooled env has no user tables, and create_remote fails with `no tables for backup` without at least one |
| 44 | + env.queryWithNoError(t, r, "CREATE DATABASE "+dbName) |
| 45 | + env.queryWithNoError(t, r, "CREATE TABLE "+dbName+".t1 (id UInt64) ENGINE=MergeTree() ORDER BY id") |
| 46 | + env.queryWithNoError(t, r, "INSERT INTO "+dbName+".t1 SELECT number FROM numbers(1000)") |
| 47 | + // at least one SQL-managed RBAC object, so rbac_size in the backup metadata is guaranteed > 0 |
| 48 | + env.queryWithNoError(t, r, "CREATE USER IF NOT EXISTS test_watch_rbac_955 IDENTIFIED WITH no_password") |
| 49 | + watchFlags := "--rbac --configs" |
| 50 | + if withNamedCollections { |
| 51 | + // IF NOT EXISTS / IF EXISTS for NAMED COLLECTION appeared only in 23.6, drop leftovers with plain DROP and ignore errors |
| 52 | + if dropErr := env.ch.Query("DROP NAMED COLLECTION test_watch_nc_955"); dropErr != nil { |
| 53 | + log.Debug().Msgf("DROP NAMED COLLECTION test_watch_nc_955 error (ignored): %v", dropErr) |
| 54 | + } |
| 55 | + env.queryWithNoError(t, r, "CREATE NAMED COLLECTION test_watch_nc_955 AS key1 = 'value1'") |
| 56 | + watchFlags += " --named-collections" |
| 57 | + } |
| 58 | + |
| 59 | + log.Debug().Msgf("Run `clickhouse-backup server --watch %s` in background", watchFlags) |
| 60 | + env.DockerExecBackgroundNoError(r, "clickhouse-backup", "bash", "-ce", |
| 61 | + "BACKUPS_TO_KEEP_REMOTE=0 clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml server --watch --watch-interval=30m --full-interval=24h --watch-backup-name-template="+prefix+"-{type}-{time:20060102150405} "+watchFlags+" &>>/tmp/watch_rbac_configs.log") |
| 62 | + defer func() { |
| 63 | + // [c]lickhouse regexp bracket trick, so pkill doesn't match its own `bash -ce` command line and kill itself with SIGTERM; |
| 64 | + // wait until the server process actually exits, a backup in flight during pkill can re-create local files after cleanup |
| 65 | + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-ce", "pkill -f '[c]lickhouse-backup.*server' || true; for i in $(seq 1 30); do pgrep -f '[c]lickhouse-backup.*server' >/dev/null || break; sleep 1; done") |
| 66 | + out, _ := env.DockerExecOut("clickhouse-backup", "bash", "-ce", "cat /tmp/watch_rbac_configs.log; rm -f /tmp/watch_rbac_configs.log") |
| 67 | + log.Debug().Msg(out) |
| 68 | + cleanBackups() |
| 69 | + if dropErr := env.ch.Query("DROP USER IF EXISTS test_watch_rbac_955"); dropErr != nil { |
| 70 | + log.Warn().Msgf("DROP USER test_watch_rbac_955 error: %v", dropErr) |
| 71 | + } |
| 72 | + if withNamedCollections { |
| 73 | + if dropErr := env.ch.Query("DROP NAMED COLLECTION test_watch_nc_955"); dropErr != nil { |
| 74 | + log.Warn().Msgf("DROP NAMED COLLECTION test_watch_nc_955 error: %v", dropErr) |
| 75 | + } |
| 76 | + } |
| 77 | + r.NoError(env.dropDatabase(dbName, true)) |
| 78 | + }() |
| 79 | + |
| 80 | + // watch creates the first full backup right after server startup, wait until create_remote + delete local finishes |
| 81 | + backupName := "" |
| 82 | + deadline := time.Now().Add(5 * time.Minute) |
| 83 | + for time.Now().Before(deadline) { |
| 84 | + out, _ := env.DockerExecOut("clickhouse-backup", "bash", "-ce", "clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml list remote 2>/dev/null | cut -d ' ' -f 1 | grep '^"+prefix+"-full-' || true") |
| 85 | + if names := strings.Fields(out); len(names) > 0 { |
| 86 | + backupName = names[0] |
| 87 | + break |
| 88 | + } |
| 89 | + time.Sleep(5 * time.Second) |
| 90 | + } |
| 91 | + r.NotEmpty(backupName, "expect full backup created by `server --watch`") |
| 92 | + |
| 93 | + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "download", backupName) |
| 94 | + metadataStr, err := env.DockerExecOut("clickhouse-backup", "cat", "/var/lib/clickhouse/backup/"+backupName+"/metadata.json") |
| 95 | + r.NoError(err, "cat metadata.json: %s", metadataStr) |
| 96 | + var backupMetadata metadata.BackupMetadata |
| 97 | + r.NoError(json.Unmarshal([]byte(metadataStr), &backupMetadata)) |
| 98 | + r.Greater(backupMetadata.RBACSize, uint64(0), "expect rbac_size > 0 in %s: %s", backupName, metadataStr) |
| 99 | + r.Greater(backupMetadata.ConfigSize, uint64(0), "expect config_size > 0 in %s: %s", backupName, metadataStr) |
| 100 | + if withNamedCollections { |
| 101 | + r.Greater(backupMetadata.NamedCollectionsSize, uint64(0), "expect named_collections_size > 0 in %s: %s", backupName, metadataStr) |
| 102 | + } |
| 103 | +} |
0 commit comments