👷 create a new script to clean old bucket from GCP

Issue: CLDSRV-867

Author: DarkIsDude

.github/scripts/cleanupOldGCPBuckets.js

@@ -0,0 +1,137 @@
+/* eslint-disable no-console */
+'use strict';
+
+const {
+    S3Client,
+    ListBucketsCommand,
+    ListObjectsV2Command,
+    DeleteObjectsCommand,
+    DeleteBucketCommand,
+    ListMultipartUploadsCommand,
+    AbortMultipartUploadCommand,
+} = require('@aws-sdk/client-s3');
+
+const GCP_ENDPOINT = 'https://storage.googleapis.com';
+const BUCKET_PREFIX = 'cldsrvci-';
+const ONE_WEEK_MS = 7 * 24 * 60 * 60 * 1000;
+
+function buildClient() {
+    const accessKeyId = process.env.AWS_GCP_BACKEND_ACCESS_KEY;
+    const secretAccessKey = process.env.AWS_GCP_BACKEND_SECRET_KEY;
+
+    if (!accessKeyId || !secretAccessKey) {
+        console.error(
+            'Missing required environment variables: ' +
+            'AWS_GCP_BACKEND_ACCESS_KEY and AWS_GCP_BACKEND_SECRET_KEY'
+        );
+        process.exit(1);
+    }
+
+    return new S3Client({
+        endpoint: GCP_ENDPOINT,
+        region: 'us-east-1',
+        credentials: { accessKeyId, secretAccessKey },
+        forcePathStyle: true,
+        disableS3ExpressSessionAuth: true,
+        requestChecksumCalculation: 'WHEN_REQUIRED',
+        responseChecksumValidation: 'WHEN_REQUIRED',
+    });
+}
+
+async function abortMultipartUploads(client, bucketName) {
+    let uploadIdMarker;
+    let keyMarker;
+
+    do {
+        const res = await client.send(new ListMultipartUploadsCommand({
+            Bucket: bucketName,
+            UploadIdMarker: uploadIdMarker,
+            KeyMarker: keyMarker,
+        }));
+
+        for (const upload of res.Uploads || []) {
+            console.log(
+                `  Aborting MPU: ${upload.Key} (${upload.UploadId})`
+            );
+            await client.send(new AbortMultipartUploadCommand({
+                Bucket: bucketName,
+                Key: upload.Key,
+                UploadId: upload.UploadId,
+            }));
+        }
+
+        uploadIdMarker = res.NextUploadIdMarker;
+        keyMarker = res.NextKeyMarker;
+    } while (uploadIdMarker);
+}
+
+async function deleteAllObjects(client, bucketName) {
+    let continuationToken;
+
+    do {
+        const res = await client.send(new ListObjectsV2Command({
+            Bucket: bucketName,
+            ContinuationToken: continuationToken,
+        }));
+
+        const objects = res.Contents || [];
+        if (objects.length > 0) {
+            console.log(`  Deleting ${objects.length} object(s)...`);
+            await client.send(new DeleteObjectsCommand({
+                Bucket: bucketName,
+                Delete: {
+                    Objects: objects.map(o => ({ Key: o.Key })),
+                    Quiet: true,
+                },
+            }));
+        }
+
+        continuationToken = res.NextContinuationToken;
+    } while (continuationToken);
+}
+
+async function cleanupBucket(client, bucketName) {
+    console.log(`Cleaning up bucket: ${bucketName}`);
+    try {
+        await abortMultipartUploads(client, bucketName);
+        await deleteAllObjects(client, bucketName);
+        await client.send(new DeleteBucketCommand({ Bucket: bucketName }));
+        console.log(`Deleted bucket: ${bucketName}`);
+    } catch (err) {
+        console.error(`Failed to delete bucket ${bucketName}: ${err.message}`);
+    }
+}
+
+async function main() {
+    const client = buildClient();
+
+    console.log('Listing GCP buckets...');
+    const { Buckets = [] } = await client.send(new ListBucketsCommand({}));
+
+    const now = Date.now();
+    const stale = Buckets.filter(b =>
+        b.Name.startsWith(BUCKET_PREFIX) &&
+        now - new Date(b.CreationDate).getTime() > ONE_WEEK_MS
+    );
+
+    if (stale.length === 0) {
+        console.log('No stale GCP CI buckets found.');
+        return;
+    }
+
+    console.log(
+        `Found ${stale.length} stale GCP CI bucket(s) to clean up: ${ 
+        stale.map(b => b.Name).join(', ')}`
+    );
+
+    for (const bucket of stale) {
+        await cleanupBucket(client, bucket.Name);
+    }
+
+    console.log('GCP CI bucket cleanup complete.');
+}
+
+main().catch(err => {
+    console.error('GCP cleanup script failed:', err);
+    process.exit(1);
+});

.github/workflows/tests.yaml

@@ -1026,3 +1026,26 @@ jobs:
         password: ${{ secrets.ARTIFACTS_PASSWORD }}
         source: /tmp/artifacts
       if: always()
+
+  cleanup-gcp-buckets:
+    runs-on: ubuntu-24.04
+    needs:
+      - multiple-backend
+      - mongo-v0-ft-tests
+      - mongo-v1-ft-tests
+      - file-ft-tests
+    if: always()
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - uses: actions/setup-node@v4
+      with:
+        node-version: '22'
+        cache: yarn
+    - name: Install dependencies
+      run: yarn install --frozen-lockfile --network-concurrency 1
+    - name: Delete stale GCP CI buckets
+      run: yarn run cleanup_gcp_buckets
+      env:
+        AWS_GCP_BACKEND_ACCESS_KEY: ${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY }}
+        AWS_GCP_BACKEND_SECRET_KEY: ${{ secrets.AWS_GCP_BACKEND_SECRET_KEY }}

package.json

@@ -105,7 +105,7 @@
     "ft_node_routes": "cd tests/functional/raw-node && yarn run test-routes",
     "ft_route_backbeat": "cd tests/multipleBackend/routes && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 routeBackbeat.js routeBackbeatForReplication.js --exit",
     "ft_gcp": "cd tests/functional/raw-node && yarn run test-gcp",
-    "cleanup_gcp_buckets": "node scripts/cleanupOldGCPBuckets.js",
+    "cleanup_gcp_buckets": "node .github/scripts/cleanupOldGCPBuckets.js",
     "ft_healthchecks": "cd tests/functional/healthchecks && yarn test",
     "ft_s3cmd": "cd tests/functional/s3cmd && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit",
     "ft_s3curl": "cd tests/functional/s3curl && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit",