Skip to content

Commit 6b1cecb

Browse files
dvasilasdvasilas
authored
Merge pull request #77 from scality/improvement/complete-crr
WKBCH-25: Complete CRR support
2 parents 3733d90 + d098b2c commit 6b1cecb

10 files changed

Lines changed: 171 additions & 7 deletions

File tree

‎cmd/config.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ type EnvironmentConfig struct {
6060
Clickhouse ClickhouseConfig `yaml:"clickhouse"`
6161
Fluentbit FluentbitConfig `yaml:"fluentbit"`
6262
Nginx NginxConfig `yaml:"nginx"`
63+
64+
HostUID int `yaml:"-"`
65+
HostGID int `yaml:"-"`
6366
}
6467

6568
type GlobalConfig struct {
@@ -357,6 +360,8 @@ func DefaultEnvironmentConfig() EnvironmentConfig {
357360

358361
func LoadEnvironmentConfig(path string) (EnvironmentConfig, error) {
359362
cfg := DefaultEnvironmentConfig()
363+
cfg.HostUID = os.Getuid()
364+
cfg.HostGID = os.Getgid()
360365

361366
if path == "" {
362367
return cfg, nil

‎cmd/configure.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ func generateBackbeatConfig(cfg EnvironmentConfig, path string) error {
146146
"config.json",
147147
"config.notification.json",
148148
"notificationCredentials.json",
149+
"admin-backbeat.json",
149150
}
150151

151152
return renderTemplates(cfg, "templates/backbeat", filepath.Join(path, "backbeat"), templates)

‎scripts/enable-crr.sh‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
#!/usr/bin/env bash
2+
# enable-crr.sh — create source + destination buckets and configure replication.
3+
#
4+
# Usage:
5+
# scripts/enable-crr.sh --source <bucket> --destination <bucket> \
6+
# [--prefix <pfx>] [--endpoint <url>]
7+
#
8+
# Defaults:
9+
# --endpoint http://127.0.0.1:8000
10+
# --prefix "" (replicate everything)
11+
#
12+
# Idempotent: re-running is a no-op once resources exist.
13+
14+
set -eu
15+
16+
# Pinned to match templates/vault/create-management-account.sh and the
17+
# replication-role accountSeed in templates/vault/config.json.
18+
ROLE_ARN="arn:aws:iam::123456789012:role/scality-internal/replication-role"
19+
20+
SOURCE=""
21+
DESTINATION=""
22+
PREFIX=""
23+
ENDPOINT="http://127.0.0.1:8000"
24+
25+
while [ $# -gt 0 ]; do
26+
case "$1" in
27+
--source) SOURCE="$2"; shift 2 ;;
28+
--destination) DESTINATION="$2"; shift 2 ;;
29+
--prefix) PREFIX="$2"; shift 2 ;;
30+
--endpoint) ENDPOINT="$2"; shift 2 ;;
31+
-h|--help)
32+
sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//'
33+
exit 0
34+
;;
35+
*) echo "unknown flag: $1" >&2; exit 2 ;;
36+
esac
37+
done
38+
39+
if [ -z "$SOURCE" ] || [ -z "$DESTINATION" ]; then
40+
echo "error: --source and --destination are required" >&2
41+
exit 2
42+
fi
43+
44+
# testaccount credentials are fixed in templates/vault/create-management-account.sh
45+
export AWS_ACCESS_KEY_ID="WBTKACCESSI9O3YKIRQ0"
46+
export AWS_SECRET_ACCESS_KEY="ICxmNTBbOqijy4rMq/MOP1EPlTMqfsEBLjROcAbN"
47+
export AWS_DEFAULT_REGION="us-east-1"
48+
49+
AWS="aws --endpoint-url $ENDPOINT"
50+
51+
create_bucket() {
52+
local bucket="$1"
53+
if $AWS s3api create-bucket --bucket "$bucket" >/dev/null 2>&1; then
54+
echo "[crr] created bucket $bucket"
55+
else
56+
# swallow "already exists and owned by you" — treat anything else as fatal
57+
if $AWS s3api head-bucket --bucket "$bucket" >/dev/null 2>&1; then
58+
echo "[crr] bucket $bucket already exists"
59+
else
60+
echo "error: failed to create bucket $bucket" >&2
61+
$AWS s3api create-bucket --bucket "$bucket"
62+
exit 1
63+
fi
64+
fi
65+
}
66+
67+
enable_versioning() {
68+
local bucket="$1"
69+
$AWS s3api put-bucket-versioning \
70+
--bucket "$bucket" \
71+
--versioning-configuration Status=Enabled
72+
echo "[crr] versioning enabled on $bucket"
73+
}
74+
75+
create_bucket "$SOURCE"
76+
create_bucket "$DESTINATION"
77+
enable_versioning "$SOURCE"
78+
enable_versioning "$DESTINATION"
79+
80+
REPLICATION_CONFIG=$(cat <<EOF
81+
{
82+
"Role": "${ROLE_ARN},${ROLE_ARN}",
83+
"Rules": [
84+
{
85+
"ID": "workbench-crr",
86+
"Status": "Enabled",
87+
"Prefix": "${PREFIX}",
88+
"Destination": {
89+
"Bucket": "arn:aws:s3:::${DESTINATION}",
90+
"StorageClass": "sf"
91+
}
92+
}
93+
]
94+
}
95+
EOF
96+
)
97+
98+
$AWS s3api put-bucket-replication \
99+
--bucket "$SOURCE" \
100+
--replication-configuration "$REPLICATION_CONFIG"
101+
102+
echo "[crr] replication configured: $SOURCE -> $DESTINATION (prefix='$PREFIX')"

‎templates/backbeat/admin-backbeat.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"D4IT2AWSB588GO5J9T00":"UEEu8tYlsOGGrgf4DAiSZD6apVNPUWqRiPG0nTB6"}

‎templates/backbeat/config.json‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@
2121
"host": "127.0.0.1",
2222
"port": 8500
2323
},
24+
"redis": {
25+
"host": "127.0.0.1",
26+
"port": 6379
27+
},
2428
"replicationGroupId": "RG001 ",
2529
"queuePopulator": {
2630
"cronRule": "*/5 * * * * *",
@@ -59,7 +63,7 @@
5963
"host": "127.0.0.1",
6064
"port": 8500,
6165
"adminPort": 8600,
62-
"adminCredentialsFile": "/home/scality/backbeat/node_modules/vaultclient/tests/utils/admincredentials.json"
66+
"adminCredentialsFile": "/conf/admin-backbeat.json"
6367
}
6468
}
6569
},
@@ -74,7 +78,7 @@
7478
"host": "127.0.0.1",
7579
"port": 8500,
7680
"adminPort": 8600,
77-
"adminCredentialsFile": "/home/scality/backbeat/node_modules/vaultclient/tests/utils/admincredentials.json"
81+
"adminCredentialsFile": "/conf/admin-backbeat.json"
7882
}
7983
}
8084
},
@@ -176,7 +180,7 @@
176180
"port": 8500
177181
}
178182
},
179-
"backlogControl": { "enabled": true },
183+
"backlogControl": { "enabled": false },
180184
"cronRule": "*/5 * * * * *",
181185
"concurrency": 10,
182186
"bucketSource": "bucketd",

‎templates/global/defaults.env‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,6 @@ NGINX_IMAGE="{{ .Nginx.Image }}"
1515

1616
METADATA_S3_DB_VERSION="{{ .S3Metadata.VFormat }}"
1717
CLOUDSERVER_ENABLE_NULL_VERSION_COMPAT_MODE="{{ .Cloudserver.EnableNullVersionCompatMode }}"
18+
19+
HOST_UID="{{ .HostUID }}"
20+
HOST_GID="{{ .HostGID }}"

‎templates/global/docker-compose.yaml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,7 @@ services:
142142
BASE_IMAGE: ${VAULT_IMAGE}
143143
container_name: workbench-setup-vault
144144
network_mode: host
145+
user: "${HOST_UID}:${HOST_GID}"
145146
volumes:
146147
- ./config/vault/management-creds.json:/conf/management-creds.json:ro
147148
- ./config/backbeat:/conf/backbeat:rw
@@ -183,6 +184,7 @@ services:
183184
BASE_IMAGE: ${SCUBA_IMAGE}
184185
container_name: workbench-setup-scuba
185186
network_mode: host
187+
user: "${HOST_UID}:${HOST_GID}"
186188
depends_on:
187189
setup-vault:
188190
condition: service_completed_successfully
@@ -198,13 +200,18 @@ services:
198200
depends_on:
199201
setup-vault:
200202
condition: service_completed_successfully
203+
setup-kafka:
204+
condition: service_completed_successfully
205+
redis:
206+
condition: service_healthy
201207
environment:
202208
SUPERVISORD_CONF: supervisord.conf
203209
BACKBEAT_CONFIG_FILE: /conf/config.json
204210
volumes:
205211
- ./config/backbeat/supervisord.conf:/conf/supervisord.conf:ro
206212
- ./config/backbeat/config.json:/conf/config.json:ro
207213
- ./config/backbeat/config.notification.json:/conf/config.notification.json:ro
214+
- ./config/backbeat/admin-backbeat.json:/conf/admin-backbeat.json:ro
208215
- ./config/backbeat/env:/conf/env:ro
209216
- ./logs/backbeat:/logs
210217
profiles:
@@ -278,7 +285,7 @@ services:
278285
backbeat-data-mover
279286
backbeat-replication-status
280287
backbeat-replication-failed
281-
backbeat-metrics-group-crr
288+
backbeat-metrics
282289
CREATE_ZOOKEEPER_PATHS: 'true'
283290
ZOOKEEPER_ENDPOINT: 127.0.0.1:2181/backbeat
284291
depends_on:

‎templates/kafka/setup.sh‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,9 @@ if [[ "$CREATE_ZOOKEEPER_PATHS" == "true" ]]; then
4646
fi
4747

4848
echo "[setup] Creating Zookeeper paths..."
49+
# zookeeper-shell.sh exits non-zero when any 'create' returns NodeExists,
50+
# which is the expected outcome on re-runs against a persisted volume.
51+
set +e
4952
zookeeper-shell.sh localhost:2181/backbeat <<EOF
5053
create /
5154
create /bucket-notification
@@ -61,14 +64,15 @@ create /queue-populator/raft-id-dispatcher
6164
create /queue-populator/raft-id-dispatcher/owners
6265
create /queue-populator/raft-id-dispatcher/leaders
6366
create /queue-populator/raft-id-dispatcher/provisions
64-
create /queue-populator/raft-id-dispatcher/provisions/0
6567
create /queue-populator/raft-id-dispatcher/provisions/1
6668
create /queue-populator/raft-id-dispatcher/provisions/2
69+
create /queue-populator/raft-id-dispatcher/provisions/3
6770
create /lifecycle
6871
create /lifecycle/conductor
6972
create /lifecycle/conductor/election
7073
quit
7174
EOF
75+
set -e
7276
echo "[setup] Zookeeper paths created."
7377
echo
7478
fi

‎templates/vault/config.json‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,38 @@
115115
]
116116
}
117117
}
118+
},
119+
{
120+
"role": {
121+
"roleName": "replication-role",
122+
"trustPolicy": {
123+
"Version": "2012-10-17",
124+
"Statement": [
125+
{
126+
"Effect": "Allow",
127+
"Principal": {
128+
"AWS": "arn:aws:iam::000000000000:user/root"
129+
},
130+
"Action": "sts:AssumeRole",
131+
"Condition": {}
132+
}
133+
]
134+
}
135+
},
136+
"permissionPolicy": {
137+
"policyName": "replication-policy",
138+
"policyDocument": {
139+
"Version": "2012-10-17",
140+
"Statement": [
141+
{
142+
"Sid": "ReplicationFullAccess",
143+
"Effect": "Allow",
144+
"Action": ["s3:*"],
145+
"Resource": ["*"]
146+
}
147+
]
148+
}
149+
}
118150
}
119151
],
120152
"utapi": {

‎templates/vault/create-management-account.sh‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,16 +56,20 @@ fi
5656
echo "[setup] Management account and access key setup completed successfully"
5757

5858
# === Create test data account ===
59-
# This account is seeded with the lifecycle role via vault's accountSeeds,
60-
# which allows backbeat to AssumeRole into it for lifecycle operations.
59+
# This account is seeded with the lifecycle and replication roles via vault's
60+
# accountSeeds. We pin the account ID so downstream tooling (e.g. the
61+
# enable-crr.sh helper) can reference role ARNs in this account by a stable
62+
# value.
6163
TEST_ACCOUNT_ACCESS_KEY="WBTKACCESSI9O3YKIRQ0"
6264
TEST_ACCOUNT_SECRET_KEY="ICxmNTBbOqijy4rMq/MOP1EPlTMqfsEBLjROcAbN"
65+
TEST_ACCOUNT_ID="123456789012"
6366

6467
echo "[setup] Creating test data account..."
6568
resp=$(./node_modules/vaultclient/bin/vaultclient \
6669
create-account \
6770
--name testaccount \
6871
--email testaccount@test.com \
72+
--accountid "$TEST_ACCOUNT_ID" \
6973
--host 127.0.0.1 \
7074
--port 8600 2>&1) || {
7175
if echo "$resp" | grep -q "EntityAlreadyExists"; then
@@ -221,6 +225,7 @@ if [ -f "$BACKBEAT_CONFIG_FILE" ]; then
221225

222226
mv /tmp/backbeat-config.updated.json "$BACKBEAT_CONFIG_FILE"
223227
echo "[setup] Backbeat config.json updated with lifecycle credentials"
228+
224229
fi
225230

226231
echo "[setup] Setup completed successfully"

0 commit comments

Comments
 (0)