Skip to content

Commit 5b8ad15

Browse files
BourgoisMickaelBourgoisMickael
committed
CLDSRV-717: Test S3C setup like in Integration
1 parent 0cbe863 commit 5b8ad15

8 files changed

Lines changed: 366 additions & 0 deletions

File tree

‎.github/actions/setup-ci/action.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ runs:
99
shell: bash
1010
run: |
1111
sudo echo "127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com" | sudo tee -a /etc/hosts
12+
sudo echo "127.0.0.1 bucketwebsitetester.s3-website-us-east-1.scality.com" | sudo tee -a /etc/hosts
1213
sudo echo "127.0.0.1 pykmip.local" | sudo tee -a /etc/hosts
1314
for i in `seq 1 50`; do sudo echo "127.0.0.$i $i.pykmip.local" | sudo tee -a /etc/hosts ; done
1415
- name: Setup Credentials

‎.github/docker/config.s3c.json‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
{
2+
"port": 8000,
3+
"maxScannedLifecycleListingEntries": 10000,
4+
"metricsPort": 8001,
5+
"internalPort": 8004,
6+
"replicationGroupId": "RG001",
7+
"restEndpoints": {
8+
"s3.amazonaws.com": "dc-1",
9+
"localhost": "dc-1",
10+
"scality.scality.com": "dc-1",
11+
"127.0.0.1": "dc-1",
12+
"node1.scality.com": "dc-1"
13+
},
14+
"websiteEndpoints": ["s3-website-us-east-1.scality.com"],
15+
"bucketd": {
16+
"bootstrap": ["0.0.0.0:9000"]
17+
},
18+
"vaultd": {
19+
"host": "127.0.0.1",
20+
"port": 8500
21+
},
22+
"clusters": 5,
23+
"kmsHideScalityArn": true,
24+
"healthChecks": {
25+
"allowFrom": ["127.0.0.1", "::1"]
26+
},
27+
"localCache": {
28+
"host": "localhost",
29+
"port": 6379,
30+
"password": ""
31+
},
32+
"log": {
33+
"logLevel": "info",
34+
"dumpLevel": "error"
35+
},
36+
"replicationEndpoints": [
37+
{"site": "zenko", "servers": ["127.0.0.1:9080"], "echo": false, "default": true},
38+
{"site": "us-east-2", "servers": ["127.0.0.1:9080"]}
39+
],
40+
"requests": {
41+
"extractClientIPFromHeader": "x-forwarded-for",
42+
"extractProtocolFromHeader": "x-forwarded-proto",
43+
"trustedProxyCIDRs": [
44+
"127.0.0.1/32",
45+
"::ffff:127.0.0.1/128",
46+
"127.0.0.1"
47+
],
48+
"viaProxy": true
49+
},
50+
"multiObjectDeleteEnableOptimizations": false,
51+
"supportedLifecycleRules": [
52+
"Expiration",
53+
"NoncurrentVersionExpiration",
54+
"AbortIncompleteMultipartUpload"
55+
],
56+
"bucketNotificationDestinations": [
57+
{
58+
"resource": "target1",
59+
"type": "dummy",
60+
"host": "localhost:6000"
61+
}
62+
]
63+
}

‎.github/docker/docker-compose.sse.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,18 @@ services:
3434
- ../../localData:/usr/src/app/localData
3535
- ../../localMetadata:/usr/src/app/localMetadata
3636
- ../../tests/functional/sse-kms-migration/config.json:/conf/config.json
37+
- ../../tests/locationConfig/locationConfigS3C.json:/conf/locationConfig.json
3738
environment:
3839
- S3_CONFIG_FILE=/conf/config.json
40+
- S3_LOCATION_FILE=/conf/locationConfig.json
3941
cloudserver-sse-migration:
4042
extends: cloudserver-sse-before-migration
4143
profiles: [sse-migration]
4244
command: sh -c "yarn start > /artifacts/s3.migration.log 2> /artifacts/s3-stderr.migration.log"
45+
metadata-standalone:
46+
image: ghcr.io/scality/metadata:8.11.0-standalone
47+
profiles: ['metadata-standalone']
48+
network_mode: 'host'
49+
volumes:
50+
- ./md-config-v0.json:/mnt/standalone_workdir/config.json:ro
51+
- /tmp/artifacts/${JOB_NAME}/md:/mnt/standalone_workdir/log

‎.github/docker/md-config-v0.json‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
{
2+
"raftSessions": 1,
3+
"raftMembers": 5,
4+
"bucketdCount": 1,
5+
"bucketdWorkers": 1,
6+
"basePorts": {
7+
"bucketd": 9000,
8+
"repd": 4200,
9+
"repdAdmin": 4250
10+
},
11+
"logLevel": "info",
12+
"env": {
13+
"METADATA_NEW_BUCKETS_VFORMAT": "v0",
14+
"S3_VERSION_ID_ENCODING_TYPE":"hex"
15+
},
16+
"migration": {
17+
"deploy": false,
18+
"raftSessions": 0,
19+
"raftMembers": 5,
20+
"bucketdCount": 1,
21+
"bucketdWorkers": 1,
22+
"basePorts": {
23+
"bucketd": 9001,
24+
"repd": 4700,
25+
"repdAdmin": 4750
26+
},
27+
"logLevel": "info"
28+
}
29+
}

‎.github/docker/setup-s3c.sh‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/bin/bash
2+
3+
# setup S3C environment just like
4+
# https://github.com/scality/Integration/blob/development/9.5/tests/setup-environment/index.js
5+
6+
set -e -o pipefail
7+
8+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
9+
10+
VAULTCLIENT=$SCRIPT_DIR/../../node_modules/vaultclient/bin/vaultclient
11+
CONFIG=$SCRIPT_DIR/admin.json
12+
13+
echo "Deleting and setting up S3C accounts like Integration (follow conf/authdata.json)"
14+
15+
$VAULTCLIENT --config $CONFIG delete-account --name Bart || true
16+
$VAULTCLIENT --config $CONFIG create-account \
17+
--name Bart \
18+
--email sampleaccount1@sampling.com \
19+
--accountid 123456789012 \
20+
--canonicalid 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be
21+
$VAULTCLIENT --config $CONFIG generate-account-access-key \
22+
--name Bart \
23+
--accesskey ACC1AK00000000000000 \
24+
--secretkey ACC1SK0000000000000000000000000000000000
25+
26+
$VAULTCLIENT --config $CONFIG delete-account --name Lisa || true
27+
$VAULTCLIENT --config $CONFIG create-account \
28+
--name Lisa \
29+
--email sampleaccount2@sampling.com \
30+
--accountid 123456789013 \
31+
--canonicalid 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf
32+
$VAULTCLIENT --config $CONFIG generate-account-access-key \
33+
--name Lisa \
34+
--accesskey ACC2AK00000000000000 \
35+
--secretkey ACC2SK0000000000000000000000000000000000
36+
37+
# Replication account for backbeat replication tests
38+
$VAULTCLIENT --config $CONFIG delete-account --name Replication || true
39+
# Cannot use url as canonicalid for service account
40+
$VAULTCLIENT --config $CONFIG create-account \
41+
--name Replication \
42+
--email inspector@replication.info \
43+
--accountid 123456789015 \
44+
--canonicalid 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2ba
45+
$VAULTCLIENT --config $CONFIG generate-account-access-key \
46+
--name Replication \
47+
--accesskey ACCREPAK000000000000 \
48+
--secretkey ACCREPSK00000000000000000000000000000000
49+
50+
echo "Copying s3c credentials to mem credentials"
51+
cp \
52+
$SCRIPT_DIR/../../tests/functional/aws-node-sdk/lib/json/s3c_credentials.json \
53+
$SCRIPT_DIR/../../tests/functional/aws-node-sdk/lib/json/mem_credentials.json
54+
55+
echo "Update conf/authdata.json account Replication canonicalID"
56+
REP_CANONICAL_ID=$(
57+
$VAULTCLIENT --config $CONFIG get-account --account-name Replication | jq -r .canonicalId
58+
)
59+
60+
# might need to undo changes if script was already ran before (manuallyoutside CI)
61+
git checkout -- $SCRIPT_DIR/../../conf/authdata.json || true
62+
sed -i "s/http:\/\/acs.zenko.io\/accounts\/service\/replication/$REP_CANONICAL_ID/g" $SCRIPT_DIR/../../conf/authdata.json

‎.github/workflows/tests.yaml‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -451,6 +451,114 @@ jobs:
451451
source: /tmp/artifacts
452452
if: always()
453453

454+
# Configure and run as Integration run S3C tests
455+
s3c-ft-tests:
456+
runs-on: ubuntu-24.04
457+
needs: build
458+
env:
459+
S3BACKEND: file
460+
S3DATA: scality
461+
S3METADATA: scality
462+
S3VAULT: scality
463+
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
464+
MPU_TESTING: "yes"
465+
ENABLE_NULL_VERSION_COMPAT_MODE: true
466+
COMPOSE_FILE: docker-compose.yaml:docker-compose.sse.yaml
467+
S3_VERSION_ID_ENCODING_TYPE: hex
468+
JOB_NAME: ${{ github.job }}
469+
VAULT_IMAGE: ghcr.io/scality/vault:7.76.0
470+
S3_END_TO_END: true
471+
S3_TESTVAL_OWNERCANONICALID: 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be
472+
steps:
473+
- name: Checkout
474+
uses: actions/checkout@v4
475+
- name: Login to Registry
476+
uses: docker/login-action@v3
477+
with:
478+
registry: ghcr.io
479+
username: ${{ github.repository_owner }}
480+
password: ${{ github.token }}
481+
- name: Setup CI environment
482+
uses: ./.github/actions/setup-ci
483+
- name: Copy S3C config
484+
run: cp .github/docker/config.s3c.json tests/functional/sse-kms-migration/config.json
485+
- name: Setup CI services
486+
run: docker compose up -d --quiet-pull redis sproxyd metadata-standalone vault-sse-before-migration cloudserver-sse-before-migration
487+
working-directory: .github/docker
488+
- name: Wait for services to be ready
489+
run: |-
490+
set -o pipefail;
491+
bash wait_for_local_port.bash 81 40
492+
bash wait_for_local_port.bash 9000 40
493+
bash wait_for_local_port.bash 8000 40
494+
- name: Setup vault credentials like S3C Integration
495+
run: ./setup-s3c.sh
496+
working-directory: .github/docker
497+
- name: Run cloudserver-object tests
498+
env:
499+
S3_CONFIG_FILE: ${{ github.workspace }}/.github/docker/config.s3c.json
500+
S3_LOCATION_FILE: ${{ github.workspace }}/tests/locationConfig/locationConfigS3C.json
501+
run: |-
502+
set -o pipefail;
503+
yarn run ft_awssdk_objects_misc | tee /tmp/artifacts/${{ github.job }}/ft_awssdk_objects_misc.log
504+
- name: Run cloudserver-version tests
505+
env:
506+
S3_CONFIG_FILE: ${{ github.workspace }}/.github/docker/config.s3c.json
507+
S3_LOCATION_FILE: ${{ github.workspace }}/tests/locationConfig/locationConfigS3C.json
508+
run: |-
509+
set -o pipefail;
510+
yarn run ft_awssdk_versioning | tee /tmp/artifacts/${{ github.job }}/ft_awssdk_versioning.log
511+
- name: Run cloudserver-bucket tests
512+
env:
513+
S3_CONFIG_FILE: ${{ github.workspace }}/.github/docker/config.s3c.json
514+
S3_LOCATION_FILE: ${{ github.workspace }}/tests/locationConfig/locationConfigS3C.json
515+
run: |-
516+
set -o pipefail;
517+
yarn run ft_awssdk_buckets | tee /tmp/artifacts/${{ github.job }}/ft_awssdk_buckets.log
518+
- name: Run cloudserver-routes (metadata) tests
519+
env:
520+
S3_CONFIG_FILE: ${{ github.workspace }}/.github/docker/config.s3c.json
521+
S3_LOCATION_FILE: ${{ github.workspace }}/tests/locationConfig/locationConfigS3C.json
522+
run: |-
523+
set -o pipefail;
524+
yarn run ft_node_routes | tee /tmp/artifacts/${{ github.job }}/ft_node_routes.log
525+
- name: Run backbeat route tests
526+
env:
527+
S3_CONFIG_FILE: ${{ github.workspace }}/.github/docker/config.s3c.json
528+
S3_LOCATION_FILE: ${{ github.workspace }}/tests/locationConfig/locationConfigS3C.json
529+
run: |-
530+
set -o pipefail;
531+
yarn run ft_route_backbeat | tee /tmp/artifacts/${{ github.job }}/ft_route_backbeat.log
532+
- name: Run backbeat tests
533+
env:
534+
S3_CONFIG_FILE: ${{ github.workspace }}/.github/docker/config.s3c.json
535+
S3_LOCATION_FILE: ${{ github.workspace }}/tests/locationConfig/locationConfigS3C.json
536+
run: |-
537+
set -o pipefail;
538+
yarn run ft_backbeat | tee /tmp/artifacts/${{ github.job }}/ft_backbeat.log
539+
- name: Cleanup and upload coverage
540+
uses: ./.github/actions/cleanup-and-coverage
541+
with:
542+
codecov-token: ${{ secrets.CODECOV_TOKEN }}
543+
flags: s3c-ft-tests
544+
if: always()
545+
- name: Upload test results to Codecov
546+
uses: codecov/test-results-action@v1
547+
with:
548+
token: ${{ secrets.CODECOV_TOKEN }}
549+
files: '**/junit/*junit*.xml'
550+
flags: s3c-ft-tests
551+
if: always() && !cancelled()
552+
- name: Upload logs to artifacts
553+
uses: scality/action-artifacts@v4
554+
with:
555+
method: upload
556+
url: https://artifacts.scality.net
557+
user: ${{ secrets.ARTIFACTS_USER }}
558+
password: ${{ secrets.ARTIFACTS_PASSWORD }}
559+
source: /tmp/artifacts
560+
if: always()
561+
454562
utapi-v2-tests:
455563
runs-on: ubuntu-24.04
456564
needs: build

‎tests/functional/aws-node-sdk/lib/json/s3c_credentials.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"default": {
3+
"accessKey": "ACC1AK00000000000000",
4+
"secretKey": "ACC1SK0000000000000000000000000000000000"
5+
},
6+
"lisa": {
7+
"accessKey": "ACC2AK00000000000000",
8+
"secretKey": "ACC2SK0000000000000000000000000000000000"
9+
},
10+
"replication": {
11+
"accessKey": "ACCREPAK000000000000",
12+
"secretKey": "ACCREPSK00000000000000000000000000000000"
13+
}
14+
}

‎tests/locationConfig/locationConfigS3C.json‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
{
2+
"dc-1": {
3+
"type": "scality",
4+
"objectId": "dc-1",
5+
"legacyAwsBehavior": true,
6+
"details": {
7+
"connector": {
8+
"sproxyd": {
9+
"bootstrap": ["localhost:81"],
10+
"path": "/proxy/dc1/"
11+
}
12+
}
13+
}
14+
},
15+
"sproxyd": {
16+
"type": "scality",
17+
"objectId": "legacy",
18+
"legacyAwsBehavior": false,
19+
"details": {
20+
"connector": {
21+
"sproxyd": {
22+
"bootstrap": ["localhost:81"],
23+
"path": "/proxy/dc1/"
24+
}
25+
}
26+
}
27+
},
28+
"us-east-1": {
29+
"type": "scality",
30+
"objectId": "us-east-1",
31+
"legacyAwsBehavior": true,
32+
"details": {
33+
"connector": {
34+
"sproxyd": {
35+
"bootstrap": ["localhost:81"],
36+
"path": "/proxy/dc1/"
37+
}
38+
}
39+
}
40+
},
41+
"us-east-2": {
42+
"type": "scality",
43+
"objectId": "us-east-2",
44+
"legacyAwsBehavior": true,
45+
"details": {
46+
"connector": {
47+
"sproxyd": {
48+
"bootstrap": ["localhost:81"],
49+
"path": "/proxy/dc1/"
50+
}
51+
}
52+
}
53+
},
54+
"scality-us-west-1": {
55+
"type": "scality",
56+
"objectId": "scality-us-west-1",
57+
"legacyAwsBehavior": false,
58+
"details": {
59+
"connector": {
60+
"sproxyd": {
61+
"bootstrap": ["localhost:81"],
62+
"path": "/proxy/dc1/"
63+
}
64+
}
65+
}
66+
},
67+
"STANDARD": {
68+
"type": "scality",
69+
"objectId": "STANDARD",
70+
"legacyAwsBehavior": false,
71+
"details": {
72+
"connector": {
73+
"sproxyd": {
74+
"bootstrap": ["localhost:81"],
75+
"path": "/proxy/dc1/"
76+
}
77+
}
78+
}
79+
}
80+
}

0 commit comments

Comments
 (0)