Merge branch 'feature/CLDSRV-812/list-objects-v2-optional-permissions' into feature/CLDSRV-813/optional-attributes-response

Author: DarkIsDude

Dockerfile

@@ -38,10 +38,11 @@ ENV no_proxy=localhost,127.0.0.1
 EXPOSE 8000
 EXPOSE 8002
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
         jq \
         tini \
+        python3-redis \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /usr/src/app

lib/Config.js

@@ -1877,6 +1877,14 @@ class Config extends EventEmitter {
             this.rateLimiting = parseRateLimitConfig(config.rateLimiting, this.clusters = this.clusters || 1);
         }
 
+
+        if (config.capabilities) {
+            if (config.capabilities.locationTypes) {
+                config.capabilities.locationTypes = new Set(config.capabilities.locationTypes);
+            }
+            this.capabilities = config.capabilities;
+        }
+
         return config;
     }
 

lib/api/api.js

@@ -363,10 +363,15 @@ const api = {
         });
         }; // End of processRequest helper function
 
-        // Cache-only rate limit check (fast path, no metadata fetch)
-        // If config is cached, apply rate limiting now
-        // If not cached, metadata validation functions will handle it
-        if (request.bucketName && config.rateLimiting?.enabled && !rateLimitApiActions.includes(apiMethod)) {
+        const applyRateLimit = request.bucketName
+            && config.rateLimiting?.enabled
+            && !rateLimitApiActions.includes(apiMethod) // Don't limit any rate limit admin actions
+            && !request.isInternalServiceRequest;       // Don't limit any calls from internal services
+
+        if (applyRateLimit) {
+            // Cache-only rate limit check (fast path, no metadata fetch)
+            // If config is cached, apply rate limiting now
+            // If not cached, metadata validation functions will handle it
             const cachedConfig = getRateLimitFromCache(request.bucketName);
 
             if (cachedConfig !== undefined) {

lib/api/apiUtils/object/versioning.js

@@ -98,6 +98,7 @@ function _storeNullVersionMD(bucketName, objKey, nullVersionId, objMD, log, cb)
             isNull2: true,
         });
     }
+    nullVersionMD.originOp = 's3:StoreNullVersion';
     metadata.putObjectMD(bucketName, objKey, nullVersionMD, { versionId }, log, err => {
         if (err) {
             log.debug('error from metadata storing null version as new version',

lib/api/apiUtils/rateLimit/cache.js

@@ -39,10 +39,22 @@ function expireCachedConfigs(now) {
     return toRemove.length;
 }
 
+/**
+ * Invalidate cached config for a specific bucket
+ *
+ * @param {string} bucketName - Bucket name
+ * @returns {boolean} True if entry was found and removed
+ */
+function invalidateCachedConfig(bucketName) {
+    const cacheKey = `bucket:${bucketName}`;
+    return configCache.delete(cacheKey);
+}
+
 module.exports = {
     setCachedConfig,
     getCachedConfig,
     expireCachedConfigs,
+    invalidateCachedConfig,
 
     // Do not access directly
     // Used only for tests

lib/api/apiUtils/rateLimit/tokenBucket.js

@@ -247,6 +247,16 @@ function getTokenBucket(bucketName, limitConfig, log) {
             bufferSize: bucket.bufferSize,
             refillThreshold: bucket.refillThreshold,
         });
+    } else if (bucket.limitConfig.limit !== limitConfig.limit) {
+        // Update limit config when it changes dynamically
+        const oldLimit = bucket.limitConfig.limit;
+        bucket.limitConfig = limitConfig;
+
+        log.info('Updated token bucket limit config', {
+            bucketName,
+            oldLimit,
+            newLimit: limitConfig.limit,
+        });
     }
 
     return bucket;
@@ -286,9 +296,20 @@ function cleanupTokenBuckets(maxIdleMs = 60000) {
     return toRemove.length;
 }
 
+/**
+ * Remove a specific token bucket (used when rate limit config is deleted)
+ *
+ * @param {string} bucketName - Bucket name
+ * @returns {boolean} True if bucket was found and removed
+ */
+function removeTokenBucket(bucketName) {
+    return tokenBuckets.delete(bucketName);
+}
+
 module.exports = {
     WorkerTokenBucket,
     getTokenBucket,
     getAllTokenBuckets,
     cleanupTokenBuckets,
+    removeTokenBucket,
 };

lib/api/bucketDeleteRateLimit.js

@@ -4,6 +4,8 @@ const metadata = require('../metadata/wrapper');
 const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 const { isRateLimitServiceUser } = require('./apiUtils/authorization/serviceUser');
+const { invalidateCachedConfig } = require('./apiUtils/rateLimit/cache');
+const { removeTokenBucket } = require('./apiUtils/rateLimit/tokenBucket');
 
 /**
  * bucketDeleteRateLimit - Delete the bucket rate limit configuration
@@ -49,6 +51,10 @@ function bucketDeleteRateLimit(authInfo, request, log, callback) {
             if (err) {
                 return callback(err, corsHeaders);
             }
+            // Invalidate cache and remove token bucket
+            invalidateCachedConfig(bucketName);
+            removeTokenBucket(bucketName);
+            log.debug('invalidated rate limit cache and token bucket for bucket', { bucketName });
             // TODO: implement Utapi metric support
             return callback(null, corsHeaders);
         });

lib/api/bucketPutRateLimit.js

@@ -6,6 +6,7 @@ const { config } = require('../Config');
 const metadata = require('../metadata/wrapper');
 const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
 const { isRateLimitServiceUser } = require('./apiUtils/authorization/serviceUser');
+const { invalidateCachedConfig } = require('./apiUtils/rateLimit/cache');
 
 function parseRequestBody(requestBody, callback) {
     try {
@@ -92,6 +93,9 @@ function bucketPutRateLimit(authInfo, request, log, callback) {
                 { error: err, method: 'bucketPutRateLimit' });
             return callback(err, corsHeaders);
         }
+        // Invalidate cache so new limit takes effect immediately
+        invalidateCachedConfig(bucketName);
+        log.debug('invalidated rate limit cache for bucket', { bucketName });
         // TODO: implement Utapi metric support
         return callback(null, corsHeaders);
     });

lib/api/objectGet.js

@@ -139,6 +139,11 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
 
         const objLength = (objMD.location === null ?
                            0 : parseInt(objMD['content-length'], 10));
+        // Store full object size for server access logs
+        if (request.serverAccessLog) {
+            // eslint-disable-next-line no-param-reassign
+            request.serverAccessLog.objectSize = objLength;
+        }
         let byteRange;
         const streamingParams = {};
         if (request.headers.range) {

lib/metadata/metadataUtils.js

@@ -226,7 +226,8 @@ function checkRateLimitIfNeeded(bucket, bucketName, request, log, callback) {
     // Skip if already checked or not enabled
     if (request.rateLimitAlreadyChecked
         || !config.rateLimiting?.enabled
-        || rateLimitApiActions.includes(request.apiMethod)) {
+        || rateLimitApiActions.includes(request.apiMethod)
+        || request.isInternalServiceRequest) {
         return process.nextTick(callback, null);
     }