Skip to content

Commit 0323321

Browse files
BourgoisMickaelBourgoisMickael
committed
CLDSRV-732: Fix flakiness starting cloudserver
The CI fails to wait for S3 after 40s recently s3-stderr has those logs: ``` npm error code E403 npm error 403 403 Forbidden - GET https://registry.npmjs.org/nyc npm error 403 In most cases, you or one of your dependencies are requesting npm error 403 a package version that is forbidden by your security policy, or npm error 403 on a server you do not have access to. npm error A complete log of this run can be found in: /root/.npm/_logs/2025-08-26T12_00_55_746Z-debug-0.log ``` Build another image to include nyc and a command for coverage during tests
1 parent 0cbe863 commit 0323321

File tree

5 files changed

+61
-26
lines changed

5 files changed

+61
-26
lines changed

‎.github/docker/docker-compose.yaml‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,6 @@
11
services:
22
cloudserver:
33
image: ${CLOUDSERVER_IMAGE}
4-
command: |
5-
bash -c "
6-
# Using tini to handle signals properly
7-
tini -s -g -- npx nyc --clean --silent yarn start > /artifacts/s3.log 2> /artifacts/s3-stderr.log &
8-
PID=$$!
9-
generate_coverage() {
10-
echo 'Stopping NodeJS processes...'
11-
kill -TERM $$PID 2>/dev/null || true
12-
wait $$PID
13-
echo 'Generating coverage report...'
14-
npx nyc report --report-dir /coverage/test --reporter=lcov --reporter=text-summary
15-
}
16-
trap generate_coverage SIGTERM
17-
wait $$PID
18-
"
194
network_mode: "host"
205
volumes:
216
- /tmp/ssl:/ssl

‎.github/workflows/release.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,10 +48,21 @@ jobs:
4848
with:
4949
context: .
5050
push: true
51+
target: production
5152
tags: ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}
5253
cache-from: type=gha
5354
cache-to: type=gha,mode=max
5455

56+
- name: Build and push test coverage image
57+
uses: docker/build-push-action@v5
58+
with:
59+
context: .
60+
push: true
61+
target: testcoverage
62+
tags: ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}-testcoverage
63+
cache-from: type=gha
64+
cache-to: type=gha,mode=max
65+
5566
- name: Build and push federation image
5667
uses: docker/build-push-action@v5
5768
with:

‎.github/workflows/tests.yaml‎

Lines changed: 26 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,7 @@ jobs:
178178
with:
179179
push: true
180180
context: .
181+
target: production
181182
provenance: false
182183
tags: |
183184
ghcr.io/${{ github.repository }}:${{ github.sha }}
@@ -187,6 +188,21 @@ jobs:
187188
cache-from: type=gha,scope=cloudserver
188189
cache-to: type=gha,mode=max,scope=cloudserver
189190

191+
- name: Build and push cloudserver image test coverage
192+
uses: docker/build-push-action@v5
193+
with:
194+
push: true
195+
context: .
196+
target: testcoverage
197+
provenance: false
198+
tags: |
199+
ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage
200+
labels: |
201+
git.repository=${{ github.repository }}
202+
git.commit-sha=${{ github.sha }}
203+
cache-from: type=gha,scope=cloudserver
204+
cache-to: type=gha,mode=max,scope=cloudserver
205+
190206
- name: Build and push federation image
191207
uses: docker/build-push-action@v5
192208
with:
@@ -229,7 +245,7 @@ jobs:
229245
runs-on: ubuntu-24.04
230246
needs: build
231247
env:
232-
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
248+
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage
233249
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
234250
S3BACKEND: mem
235251
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
@@ -295,7 +311,7 @@ jobs:
295311
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
296312
DEFAULT_BUCKET_KEY_FORMAT: v0
297313
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
298-
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
314+
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage
299315
JOB_NAME: ${{ github.job }}
300316
steps:
301317
- name: Checkout
@@ -348,7 +364,7 @@ jobs:
348364
DEFAULT_BUCKET_KEY_FORMAT: v1
349365
METADATA_MAX_CACHED_BUCKETS: 1
350366
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
351-
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
367+
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage
352368
JOB_NAME: ${{ github.job }}
353369
steps:
354370
- name: Checkout
@@ -405,7 +421,7 @@ jobs:
405421
env:
406422
S3BACKEND: file
407423
S3VAULT: mem
408-
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
424+
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage
409425
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
410426
MPU_TESTING: "yes"
411427
ENABLE_NULL_VERSION_COMPAT_MODE: "${{ matrix.enable-null-compat }}"
@@ -459,7 +475,7 @@ jobs:
459475
ENABLE_UTAPI_V2: t
460476
S3BACKEND: mem
461477
BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket
462-
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
478+
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage
463479
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
464480
JOB_NAME: ${{ github.job }}
465481
steps:
@@ -515,7 +531,7 @@ jobs:
515531
SCUBA_HOST: localhost
516532
SCUBA_PORT: 8100
517533
SCUBA_HEALTHCHECK_FREQUENCY: 100
518-
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
534+
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage
519535
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
520536
JOB_NAME: ${{ github.job }}
521537
steps:
@@ -562,7 +578,7 @@ jobs:
562578
S3BACKEND: file
563579
S3VAULT: mem
564580
MPU_TESTING: "yes"
565-
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
581+
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage
566582
PYKMIP_IMAGE: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
567583
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
568584
JOB_NAME: ${{ github.job }}
@@ -622,7 +638,7 @@ jobs:
622638
S3BACKEND: file
623639
S3VAULT: mem
624640
MPU_TESTING: true
625-
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
641+
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage
626642
PYKMIP_IMAGE: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
627643
JOB_NAME: ${{ github.job }}
628644
COMPOSE_FILE: docker-compose.yaml:docker-compose.sse.yaml
@@ -688,7 +704,7 @@ jobs:
688704
MPU_TESTING: "yes"
689705
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigCeph.json
690706
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
691-
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
707+
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage
692708
JOB_NAME: ${{ github.job }}
693709
ENABLE_NULL_VERSION_COMPAT_MODE: true # needed with mongodb backend
694710
steps:
@@ -809,7 +825,7 @@ jobs:
809825
VAULT_VERSION_CURRENT: 7.70.32
810826
CLOUDSERVER_IMAGE_BEFORE_SSE_MIGRATION: ghcr.io/${{ github.repository }}:9.0.8
811827
VAULT_IMAGE_BEFORE_SSE_MIGRATION: ghcr.io/scality/vault:7.70.31
812-
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
828+
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage
813829
VAULT_IMAGE: ghcr.io/scality/vault:7.70.32
814830
KMS_IMAGE: nsmithuk/local-kms:3.11.7
815831
MPU_TESTING: "yes"

‎Dockerfile‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ COPY package.json yarn.lock /usr/src/app/
3030
RUN yarn install --production --ignore-optional --frozen-lockfile --ignore-engines --network-concurrency 1
3131

3232
################################################################################
33-
FROM node:${NODE_VERSION}
33+
FROM node:${NODE_VERSION} AS production
3434

3535
ENV NO_PROXY=localhost,127.0.0.1
3636
ENV no_proxy=localhost,127.0.0.1
@@ -55,3 +55,10 @@ VOLUME ["/usr/src/app/localData","/usr/src/app/localMetadata"]
5555
ENTRYPOINT ["tini", "-g", "--", "/usr/src/app/docker-entrypoint.sh"]
5656

5757
CMD [ "yarn", "start" ]
58+
59+
################################################################################
60+
FROM production AS testcoverage
61+
62+
RUN yarn global add nyc
63+
64+
CMD [ "./docker-test-with-coverage.sh" ]

‎docker-test-with-coverage.sh‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/bin/bash
2+
3+
nyc --clean --silent yarn start > /artifacts/s3.log 2> /artifacts/s3-stderr.log &
4+
5+
PID=$!
6+
7+
generate_coverage() {
8+
echo 'Stopping NodeJS processes...'
9+
kill -TERM $PID 2>/dev/null || true
10+
wait $PID
11+
echo 'Generating coverage report...'
12+
nyc report --report-dir /coverage/test --reporter=lcov --reporter=text-summary
13+
}
14+
15+
trap generate_coverage SIGTERM
16+
wait $PID

0 commit comments

Comments
 (0)