Skip to content

Commit 3035c4e

Browse files
bert-ebert-e
committed
Merge branch 'w/9.0/improvement/CLDSRV-657-kms-tests' into tmp/octopus/w/9.1/improvement/CLDSRV-657-kms-tests
2 parents 9db7225 + 67aaf0a commit 3035c4e

File tree

23 files changed

+2463
-3
lines changed

23 files changed

+2463
-3
lines changed

‎.github/actions/setup-ci/action.yaml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,10 @@ runs:
77
steps:
88
- name: Setup etc/hosts
99
shell: bash
10-
run: sudo echo "127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com" | sudo tee -a /etc/hosts
10+
run: |
11+
sudo echo "127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com" | sudo tee -a /etc/hosts
12+
sudo echo "127.0.0.1 pykmip.local" | sudo tee -a /etc/hosts
13+
for i in `seq 1 50`; do sudo echo "127.0.0.$i $i.pykmip.local" | sudo tee -a /etc/hosts ; done
1114
- name: Setup Credentials
1215
shell: bash
1316
run: bash .github/scripts/credentials.bash

‎.github/docker/admin.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"accessKey": "D4IT2AWSB588GO5J9T00",
3+
"secretKeyValue": "UEEu8tYlsOGGrgf4DAiSZD6apVNPUWqRiPG0nTB6"
4+
}

‎.github/docker/creds.env‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,3 +34,4 @@ gcpbackendmismatch_GCP_SERVICE_KEY
3434
gcpbackend_GCP_SERVICE_KEYFILE
3535
gcpbackendmismatch_GCP_SERVICE_KEYFILE
3636
gcpbackendnoproxy_GCP_SERVICE_KEYFILE
37+
AWS_SDK_JS_SUPPRESS_MAINTENANCE_MODE_MESSAGE=1

‎.github/docker/docker-compose.sse.yaml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
services:
2+
vault-sse-before-migration:
3+
image: ${VAULT_IMAGE}
4+
profiles: ['sse-migration']
5+
# root because S3C images needs ownership permission on files and mounted paths
6+
user: root
7+
command: sh -c "chmod 400 tests/utils/keyfile && yarn start > /artifacts/vault.log 2> /artifacts/vault-stderr.log"
8+
network_mode: "host"
9+
volumes:
10+
- /tmp/artifacts/${JOB_NAME}:/artifacts
11+
- ./vault-config.json:/conf/config.json:ro
12+
- ./vault-db:/data
13+
environment:
14+
- VAULT_DB_BACKEND=LEVELDB
15+
- CI=true
16+
- ENABLE_LOCAL_CACHE=true
17+
- REDIS_HOST=0.0.0.0
18+
- REDIS_PORT=6379
19+
depends_on:
20+
- redis
21+
vault-sse-migration:
22+
extends: vault-sse-before-migration
23+
profiles: ['sse-migration']
24+
command: sh -c "chmod 400 tests/utils/keyfile && yarn start > /artifacts/vault.migration.log 2> /artifacts/vault-stderr.migration.log"
25+
environment:
26+
- KMS_BACKEND
27+
cloudserver-sse-before-migration:
28+
extends:
29+
file: docker-compose.yaml
30+
service: cloudserver
31+
profiles: [sse-migration]
32+
volumes:
33+
# using artesca container
34+
- ../../localData:/usr/src/app/localData
35+
- ../../localMetadata:/usr/src/app/localMetadata
36+
- ../../tests/functional/sse-kms-migration/config.json:/conf/config.json
37+
environment:
38+
- S3_CONFIG_FILE=/conf/config.json
39+
- S3VAULT=scality
40+
depends_on:
41+
- vault-sse-before-migration
42+
cloudserver-sse-migration:
43+
extends: cloudserver-sse-before-migration
44+
profiles: [sse-migration]
45+
command: sh -c "yarn start > /artifacts/s3.migration.log 2> /artifacts/s3-stderr.migration.log"
46+
environment:
47+
- S3KMS
48+
depends_on: !override
49+
- redis
50+
- vault-sse-migration

‎.github/docker/docker-compose.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ services:
44
command: |
55
bash -c "
66
# Using tini to handle signals properly
7-
tini -s -g -- npx nyc --clean --silent yarn start > /artifacts/s3.log &
7+
tini -s -g -- npx nyc --clean --silent yarn start > /artifacts/s3.log 2> /artifacts/s3-stderr.log &
88
PID=$$!
99
generate_coverage() {
1010
echo 'Stopping NodeJS processes...'
@@ -94,6 +94,10 @@ services:
9494
- ../pykmip/certs:/ssl
9595
- ../pykmip/policy.json:/etc/pykmip/policies/policy.json
9696
- ../pykmip/server.conf:/etc/pykmip/server.conf
97+
localkms:
98+
network_mode: "host"
99+
profiles: ['localkms']
100+
image: ${KMS_IMAGE:-nsmithuk/local-kms:3.11.7}
97101
mongo:
98102
network_mode: "host"
99103
profiles: ['mongo', 'ceph']

‎.github/docker/ensure-version.sh‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/bin/bash
2+
3+
# Make sure the good container version runs before running sse migration tests
4+
5+
set -o pipefail;
6+
7+
CONTAINER=$1
8+
EXPECTED_VERSION=$2
9+
10+
# run jq outside container as some container might not have it
11+
VERSION=$(docker compose exec $1 cat package.json | jq -r .version)
12+
if [[ "$VERSION" != "$EXPECTED_VERSION" ]]; then
13+
echo "bad version of container $CONTAINER. Should be $EXPECTED_VERSION. Was $VERSION" >&2
14+
exit 1
15+
else
16+
echo OK $VERSION
17+
fi

‎.github/docker/local.sh‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
#!/bin/bash
2+
set -e -o pipefail
3+
4+
# run kms migration tests locally
5+
# in .github/docker
6+
7+
export S3BACKEND=file
8+
export S3METADATA=file
9+
export S3VAULT=scality
10+
export MPU_TESTING="yes"
11+
12+
export CLOUDSERVER_IMAGE_BEFORE_SSE_MIGRATION=ghcr.io/scality/cloudserver:9.0.8
13+
export CLOUDSERVER_IMAGE_ORIGINAL=ghcr.io/scality/cloudserver:9.0.16
14+
15+
export VAULT_IMAGE_BEFORE_SSE_MIGRATION=ghcr.io/scality/vault:7.70.31
16+
export VAULT_IMAGE_ORIGINAL=ghcr.io/scality/vault:7.70.32
17+
export KMS_IMAGE=nsmithuk/local-kms:3.11.7
18+
19+
export S3_CONFIG_FILE="$(pwd)/../../tests/functional/sse-kms-migration/config.json"
20+
21+
export kmsContainer=localkms
22+
export kmsProvider=aws
23+
export kmsPort=8080
24+
25+
# export kmsContainer=pykmip
26+
# export kmsProvider=kmip
27+
# export kmsPort=5696
28+
29+
export kmsHideScalityArn=true
30+
export globalEncryptionEnabled=true
31+
32+
export JOB_NAME=sse-kms-migration-tests-$kmsHideScalityArn-$kmsProvider
33+
34+
mkdir -p /tmp/artifacts/$JOB_NAME
35+
mkdir -p /tmp/ssl-kmip
36+
37+
export CLOUDSERVER_IMAGE=$CLOUDSERVER_IMAGE_BEFORE_SSE_MIGRATION
38+
export VAULT_IMAGE=$VAULT_IMAGE_BEFORE_SSE_MIGRATION
39+
40+
export KMS_AWS_SECRET_ACCESS_KEY=123
41+
export KMS_AWS_ACCESS_KEY_ID=456
42+
43+
export COMPOSE_FILE=docker-compose.yaml:docker-compose.sse.yaml
44+
45+
function stop_all() {
46+
docker compose -p docker down
47+
}
48+
49+
function rm_all() {
50+
sudo rm -rf ./vault-db/
51+
sudo rm -rf ../../localData/*
52+
sudo rm -rf ../../localMetadata/*
53+
}
54+
55+
function start_all_before_migration() {
56+
cd ../../tests/functional/sse-kms-migration
57+
pwd
58+
cp configs/base.json config.json
59+
cd ../../../.github/docker
60+
61+
docker compose up -d redis vault-sse-before-migration cloudserver-sse-before-migration
62+
bash ../../wait_for_local_port.bash 8500 40
63+
bash ../../wait_for_local_port.bash 8000 40
64+
# HAVE vaultclient bin in your PATH or an alias
65+
# alias vaultclient="~/scality/vaultclient/bin/vaultclient"
66+
export PATH="$PATH:~/scality/vaultclient/bin/"
67+
vaultclient --config admin.json delete-account --name test || true
68+
vaultclient --config admin.json create-account --name test --email test@scality.com
69+
vaultclient --config admin.json generate-account-access-key --name test --accesskey TESTAK00000000000000 --secretkey TESTSK0000000000000000000000000000000000
70+
vaultclient --config admin.json get-account --account-name test
71+
}
72+
73+
function run_before_migration() {
74+
echo ===== RUN BEFORE MIGRATION =====
75+
cd ../..
76+
77+
AWS_ENDPOINT_URL=http://0:8000 AWS_DEFAULT_REGION=us-east-1 AWS_ACCESS_KEY_ID=TESTAK00000000000000 AWS_SECRET_ACCESS_KEY=TESTSK0000000000000000000000000000000000 aws s3 ls
78+
yarn run ft_sse_before_migration | tee /tmp/artifacts/$JOB_NAME/beforeMigration.log
79+
}
80+
81+
function run_latest_images() {
82+
export CLOUDSERVER_IMAGE=$CLOUDSERVER_IMAGE_ORIGINAL
83+
export VAULT_IMAGE=$VAULT_IMAGE_ORIGINAL
84+
export S3KMS=$kmsProvider # S3
85+
export KMS_BACKEND=$([[ "$kmsProvider" == "aws" ]] && echo "aws") # vault only aws is supported
86+
87+
cd tests/functional/sse-kms-migration
88+
jq -s "
89+
.[0] * .[1] * .[2] *
90+
{ kmsHideScalityArn: $kmsHideScalityArn } *
91+
{ globalEncryptionEnabled: $globalEncryptionEnabled }
92+
" \
93+
configs/base.json \
94+
configs/$kmsProvider.json \
95+
configs/sseMigration.json \
96+
> config.json
97+
cd ../../../
98+
cd .github/docker
99+
# copy kmip certs
100+
sudo cp -r ../pykmip/certs/* /tmp/ssl-kmip
101+
102+
docker compose down cloudserver-sse-before-migration vault-sse-before-migration
103+
docker compose up -d $kmsContainer vault-sse-migration cloudserver-sse-migration
104+
105+
bash ../../wait_for_local_port.bash $kmsPort 40
106+
bash ../../wait_for_local_port.bash 8500 40
107+
bash ../../wait_for_local_port.bash 8000 40
108+
}
109+
110+
function run_migration() {
111+
echo ==== RUN MIGRATION ====
112+
cd ../..
113+
export S3KMS=$kmsProvider # S3
114+
export KMS_BACKEND=$([[ "$kmsProvider" == "aws" ]] && echo "aws") # vault only aws is supported
115+
yarn run ft_sse_migration | tee /tmp/artifacts/$JOB_NAME/migration.log
116+
}
117+
118+
function run_after_migration() {
119+
sleep 1
120+
export S3KMS=$kmsProvider # S3
121+
export KMS_BACKEND=$([[ "$kmsProvider" == "aws" ]] && echo "aws") # vault only aws is supported
122+
yarn run ft_sse_arn | tee /tmp/artifacts/$JOB_NAME/arnPrefix.log
123+
}
124+
125+
stop_all
126+
rm_all
127+
start_all_before_migration
128+
run_before_migration
129+
run_latest_images
130+
run_migration
131+
run_after_migration

‎.github/docker/vault-config.json‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
{
2+
"clusters": 2,
3+
"healthChecks": {
4+
"allowFrom": ["127.0.0.1/8", "::1"]
5+
},
6+
"interfaces": {
7+
"S3": {
8+
"address": "0.0.0.0",
9+
"port": 8500,
10+
"allowFrom": ["0.0.0.0/8", "::1"]
11+
},
12+
"administration": {
13+
"address": "0.0.0.0",
14+
"port": 8600
15+
},
16+
"sts": {
17+
"address": "127.0.0.1",
18+
"port": 8800
19+
}
20+
},
21+
"map": ["127.0.0.1:4300", "127.0.0.2:4301", "127.0.0.3:4302", "127.0.0.4:4303", "127.0.0.5:4304"],
22+
"keyFilePath": "./tests/utils/keyfile",
23+
"adminCredentialsFilePath": "./tests/utils/admincredentials.json.encrypted",
24+
"log": {
25+
"level": "info",
26+
"dump": "error"
27+
},
28+
"accountSeeds": [
29+
{
30+
"role": {
31+
"roleName": "scality-role1",
32+
"trustPolicy": {
33+
"Version": "2012-10-17",
34+
"Statement": [
35+
{
36+
"Effect": "Allow",
37+
"Principal": { "AWS": "arn:aws:iam::000000000000:user/root" },
38+
"Action": "sts:AssumeRole",
39+
"Condition": {}
40+
}
41+
]
42+
}
43+
},
44+
"permissionPolicy": {
45+
"policyName": "scality-policy1",
46+
"policyDocument": {
47+
"Version": "2012-10-17",
48+
"Statement": [
49+
{
50+
"Sid": "FullAccess",
51+
"Effect": "Allow",
52+
"Action": ["s3:*"],
53+
"Resource": ["*"]
54+
}
55+
]
56+
}
57+
}
58+
}
59+
],
60+
"utapi": {
61+
"host": "127.0.0.1",
62+
"port": 8100
63+
},
64+
"scuba": {
65+
"host": "127.0.0.1",
66+
"port": 8100
67+
},
68+
"kmsAWS": {
69+
"noAwsArn": true,
70+
"providerName": "local",
71+
"region": "us-east-1",
72+
"endpoint": "http://0:8080",
73+
"ak": "456",
74+
"sk": "123"
75+
}
76+
}

0 commit comments

Comments
 (0)