fixup CI to merge configs for new matrix

Author: BourgoisMickael

.github/docker/docker-compose.sse.yaml

@@ -33,7 +33,7 @@ services:
       # using artesca container
       - ../../localData:/usr/src/app/localData
       - ../../localMetadata:/usr/src/app/localMetadata
-      - ../../tests/functional/sse-kms-migration/config.${SSE_CONF}.json:/conf/config.json
+      - ../../tests/functional/sse-kms-migration/config.json:/conf/config.json
     environment:
       - S3_CONFIG_FILE=/conf/config.json
       - S3VAULT=scality

.github/workflows/tests.yaml

@@ -372,8 +372,10 @@ jobs:
         source: /tmp/artifacts
       if: always()
 
+  # This test with the final yarn run ft_sse_arn covers more code than the kmip tests
   sse-kms-migration-tests:
     strategy:
+      fail-fast: false # prevent cancel if one 1 matrix option fails
       matrix:
         kms:
           - provider: aws
@@ -387,7 +389,7 @@ jobs:
             globalEncryptionEnabled: false
           - kmsHideScalityArn: false
             globalEncryptionEnabled: false
-          - kmsHideScalityArn: true
+          - kmsHideScalityArn: false
             globalEncryptionEnabled: true
         # This matrix should create 6 different jobs
         # No need to test globalEncryption with hidden arn (not related)
@@ -418,9 +420,9 @@ jobs:
         sse-kms-migration-${{
           matrix.kms.provider
         }}-${{
-          matrix.kmsHideScalityArn && 'hideArn' || 'showArn'
+          matrix.opts.kmsHideScalityArn && 'hideArn' || 'showArn'
         }}${{
-          matrix.globalEncryptionEnabled && 'global' || ''
+          matrix.opts.globalEncryptionEnabled && '-global' || ''
         }}
       COMPOSE_FILE: docker-compose.yaml:docker-compose.sse.yaml
     steps:
@@ -437,21 +439,23 @@ jobs:
         password: ${{ github.token }}
     - name: Setup CI environment
       uses: ./.github/actions/setup-ci
-    - name: Copy KMIP certs (kmip always configured even when aws is used)
+    - name: Copy KMIP certs
       run: cp -r ./certs /tmp/ssl-kmip
       working-directory: .github/pykmip
     - name: Setup matrix job artifacts directory
       shell: bash
       run: |
         set -exu
-        mkdir -p /tmp/artifacts/${{ matrix.job-name }}/
+        mkdir -p /tmp/artifacts/${{ env.JOB_NAME }}/
+    - name: Copy base config
+      run: cp configs/base.json config.json
+      working-directory: tests/functional/sse-kms-migration
     - name: Setup CI services (with old cloudserver image before sse migration)
       run: docker compose up -d --quiet-pull redis vault-sse-before-migration cloudserver-sse-before-migration
       working-directory: .github/docker
       env:
         CLOUDSERVER_IMAGE: ${{ env.CLOUDSERVER_IMAGE_BEFORE_SSE_MIGRATION }}
         VAULT_IMAGE: ${{ env.VAULT_IMAGE_BEFORE_SSE_MIGRATION }}
-        SSE_CONF: before
     - name: Wait for services vault and s3
       run: |-
         bash wait_for_local_port.bash 8500 40
@@ -472,21 +476,32 @@ jobs:
       shell: bash # for pipefail
       env:
         # yarn run does a cd into the test folder
-        S3_CONFIG_FILE: config.before.json
+        S3_CONFIG_FILE: config.json
         S3KMS: file
-      run: yarn run ft_sse_before_migration | tee /tmp/artifacts/${{ matrix.job-name }}/beforeMigration.log
+      run: yarn run ft_sse_before_migration | tee /tmp/artifacts/${{ env.JOB_NAME }}/beforeMigration.log
+    - name: Merge config.json files for options
+      run: |
+        jq -s '
+          .[0] * .[1] * .[2] *
+          { kmsHideScalityArn: ${{ matrix.opts.kmsHideScalityArn }} } *
+          { globalEncryptionEnabled: ${{ matrix.opts.globalEncryptionEnabled }} }
+        ' \
+          configs/base.json \
+          configs/${{ matrix.kms.provider }}.json \
+          configs/sseMigration.json \
+          > config.json
+      working-directory: tests/functional/sse-kms-migration
     - name: Replace old cloudserver image with current one
       run: |-
         docker compose down cloudserver-sse-before-migration vault-sse-before-migration
-        docker compose up -d --quiet-pull ${{ matrix.kmsContainer }} vault-sse-migration cloudserver-sse-migration
+        docker compose up -d --quiet-pull ${{ matrix.kms.container }} vault-sse-migration cloudserver-sse-migration
       working-directory: .github/docker
       env:
-        SSE_CONF: sseMigration.${{ matrix.kmsHideScalityArn }}
-        S3KMS: ${{ matrix.kmsProvider }} # S3
-        KMS_BACKEND: ${{ matrix.kmsProvider == 'aws' && 'aws' || '' }} # vault only supports aws
+        S3KMS: ${{ matrix.kms.provider }} # S3
+        KMS_BACKEND: ${{ matrix.kms.provider == 'aws' && 'aws' || '' }} # vault only supports aws
     - name: Wait for services kms vault and s3
       run: |-
-        bash wait_for_local_port.bash ${{ matrix.kmsPort }} 40
+        bash wait_for_local_port.bash ${{ matrix.kms.port }} 40
         bash wait_for_local_port.bash 8500 40
         bash wait_for_local_port.bash 8000 40
     - name: Ensure latest version of cloudserver and vault is used
@@ -500,31 +515,31 @@ jobs:
         # Functional tests needs access to the running config to use the same
         # KMS provider and sseMigration
         # yarn run does a cd into the test folder
-        S3_CONFIG_FILE: config.sseMigration.${{ matrix.kmsHideScalityArn }}.json
-        S3KMS: ${{ matrix.kmsProvider }}
-      run: yarn run ft_sse_migration | tee /tmp/artifacts/${{ matrix.job-name }}/migration.log
+        S3_CONFIG_FILE: config.json
+        S3KMS: ${{ matrix.kms.provider }}
+      run: yarn run ft_sse_migration | tee /tmp/artifacts/${{ env.JOB_NAME }}/migration.log
     - name: Run SSE arnPrefix tests
       shell: bash # for pipefail
       env:
         # Functional tests needs access to the running config to use the same
         # KMS provider and sseMigration
         # yarn run does a cd into the test folder
-        S3_CONFIG_FILE: config.sseMigration.${{ matrix.kmsHideScalityArn }}.json
-        S3KMS: ${{ matrix.kmsProvider }}
-      run: yarn run ft_sse_arn | tee /tmp/artifacts/${{ matrix.job-name }}/arnPrefix.log
+        S3_CONFIG_FILE: config.json
+        S3KMS: ${{ matrix.kms.provider }}
+      run: yarn run ft_sse_arn | tee /tmp/artifacts/${{ env.JOB_NAME }}/arnPrefix.log
     - name: Print docker compose logs
       run: |-
         docker compose logs \
           cloudserver-sse-before-migration \
           cloudserver-sse-migration \
           vault-sse-before-migration \
           vault-sse-migration \
-          ${{ matrix.kmsContainer == 'localkms' && 'localkms' || '' }}
+          ${{ matrix.kms.container == 'localkms' && 'localkms' || '' }}
       # pykmip logs are already uploaded to artifacts, but not localkms
       working-directory: .github/docker
       if: failure()
     - name: Remove empty artifact files to simplify viewing artifacts
-      run: find /tmp/artifacts/${{ matrix.job-name }}/ -size 0 -delete
+      run: find /tmp/artifacts/${{ env.JOB_NAME }}/ -size 0 -delete
       if: always()
     - name: Upload logs to artifacts
       uses: scality/action-artifacts@v4