Add aws default credentials provider (#3842)

ianwow · web-flow · commit c191cd5ded58 · 2025-04-25T12:27:06.000+02:00
diff --git a/docs/application-settings.md b/docs/application-settings.md
@@ -277,8 +277,8 @@ The general idea is that you'll place all the account-specific settings in a sep
 ```yaml
 settings:
   s3:
-      accessKeyId: <S3 access key>
-      secretAccessKey: <S3 access key>
+      accessKeyId: <S3 access key> # optional
+      secretAccessKey: <S3 access key> #optional
       endpoint: <endpoint> # http://s3.storage.com
       bucket: <bucket name> # prebid-application-settings
       region: <region name> # if not provided AWS_GLOBAL will be used. Example value: 'eu-central-1'
@@ -298,6 +298,14 @@ settings:
           timeout: 5000
 ```
 
+If `accessKeyId` and `secretAccessKey` are not specified in the Prebid Server configuration  then AWS credentials will be looked up in this order:
+- Java System Properties - `aws.accessKeyId` and `aws.secretAccessKey`
+- Environment Variables - `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
+- Web Identity Token credentials from system properties or environment variables
+- Credential profiles file at the default location (`~/.aws/credentials`) shared by all AWS SDKs and the AWS CLI
+- Credentials delivered through the Amazon EC2 container service if "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" environment variable is set and security manager has permission to access the variable,
+- Instance profile credentials delivered through the Amazon EC2 metadata service
+
 ### File format
 
 We recommend using the `json` format for your account configuration. A minimal configuration may look like this.
diff --git a/docs/config-app.md b/docs/config-app.md
@@ -391,8 +391,8 @@ contain 'WHERE last_updated > ?' for MySQL and 'WHERE last_updated > $1' for Pos
 
 For S3 storage configuration
 - `settings.in-memory-cache.s3-update.refresh-rate` - refresh period in ms for stored request updates in S3
-- `settings.s3.access-key-id` - an access key
-- `settings.s3.secret-access-key` - a secret access key
+- `settings.s3.access-key-id` - an access key (optional)
+- `settings.s3.secret-access-key` - a secret access key (optional)
 - `settings.s3.region` - a region, AWS_GLOBAL by default
 - `settings.s3.endpoint` - an endpoint
 - `settings.s3.bucket` - a bucket name
@@ -402,6 +402,15 @@ For S3 storage configuration
 - `settings.s3.stored-requests-dir` - a directory with stored requests
 - `settings.s3.stored-responses-dir` - a directory with stored responses
 
+If `settings.s3.access-key-id` and `settings.s3.secret-access-key` are not specified in the Prebid Server configuration then AWS credentials will be looked up in this order:
+- Java System Properties - `aws.accessKeyId` and `aws.secretAccessKey`
+- Environment Variables - `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
+- Web Identity Token credentials from system properties or environment variables 
+- Credential profiles file at the default location (`~/.aws/credentials`) shared by all AWS SDKs and the AWS CLI
+- Credentials delivered through the Amazon EC2 container service if "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" environment variable is set and security manager has permission to access the variable,
+- Instance profile credentials delivered through the Amazon EC2 metadata service
+
+
 For targeting available next options:
 - `settings.targeting.truncate-attr-chars` - set the max length for names of targeting keywords (0 means no truncation).
 
diff --git a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java
@@ -1,5 +1,8 @@
 package org.prebid.server.spring.config;
 
+import org.apache.commons.lang3.StringUtils;
+import org.prebid.server.log.Logger;
+import org.prebid.server.log.LoggerFactory;
 import io.vertx.core.Vertx;
 import io.vertx.core.file.FileSystem;
 import lombok.Data;
@@ -40,9 +43,12 @@
 import org.springframework.stereotype.Component;
 import org.springframework.validation.annotation.Validated;
 import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
 import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
 import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.core.exception.SdkClientException;
 
 import javax.validation.constraints.Min;
 import javax.validation.constraints.NotBlank;
@@ -58,6 +64,8 @@
 @UtilityClass
 public class SettingsConfiguration {
 
+    private static final Logger logger = LoggerFactory.getLogger(SettingsConfiguration.class);
+
     @Configuration
     @ConditionalOnProperty(prefix = "settings.filesystem",
             name = {"settings-filename", "stored-requests-dir", "stored-imps-dir"})
@@ -233,18 +241,30 @@ static class S3SettingsConfiguration {
 
         @Component
         @ConfigurationProperties(prefix = "settings.s3")
-        @ConditionalOnProperty(prefix = "settings.s3", name = {"accessKeyId", "secretAccessKey"})
         @Validated
         @Data
         @NoArgsConstructor
         protected static class S3ConfigurationProperties {
 
-            @NotBlank
+            /**
+             * If accessKeyId and secretAccessKey are provided in the
+             * configuration file then they will be used. Otherwise, the
+             * DefaultCredentialsProvider will look for credentials in this order:
+             *
+             * - Java System Properties
+             * - Environment Variables
+             * - Web Identity Token
+             * - AWS credentials file (~/.aws/credentials)
+             * - ECS container credentials
+             * - EC2 instance profile
+             */
             private String accessKeyId;
-
-            @NotBlank
             private String secretAccessKey;
 
+            private boolean useStaticCredentials() {
+                return StringUtils.isNotBlank(accessKeyId) && StringUtils.isNotBlank(secretAccessKey);
+            }
+
             /**
              * If not provided AWS_GLOBAL will be used as a region
              */
@@ -274,22 +294,33 @@ protected static class S3ConfigurationProperties {
 
         @Bean
         S3AsyncClient s3AsyncClient(S3ConfigurationProperties s3ConfigurationProperties) throws URISyntaxException {
-            final AwsBasicCredentials credentials = AwsBasicCredentials.create(
-                    s3ConfigurationProperties.getAccessKeyId(),
-                    s3ConfigurationProperties.getSecretAccessKey());
             final Region awsRegion = Optional.ofNullable(s3ConfigurationProperties.getRegion())
                     .map(Region::of)
                     .orElse(Region.AWS_GLOBAL);
 
-            return S3AsyncClient
-                    .builder()
-                    .credentialsProvider(StaticCredentialsProvider.create(credentials))
+            return S3AsyncClient.builder()
+                    .credentialsProvider(awsCredentialsProvider(s3ConfigurationProperties))
                     .endpointOverride(new URI(s3ConfigurationProperties.getEndpoint()))
                     .forcePathStyle(s3ConfigurationProperties.getForcePathStyle())
                     .region(awsRegion)
                     .build();
         }
 
+        private static AwsCredentialsProvider awsCredentialsProvider(S3ConfigurationProperties config) {
+            final AwsCredentialsProvider credentialsProvider = config.useStaticCredentials()
+                    ? StaticCredentialsProvider.create(
+                            AwsBasicCredentials.create(config.getAccessKeyId(), config.getSecretAccessKey()))
+                    : DefaultCredentialsProvider.create();
+
+            try {
+                credentialsProvider.resolveCredentials();
+            } catch (SdkClientException e) {
+                logger.error("Failed to resolve AWS credentials", e);
+            }
+
+            return credentialsProvider;
+        }
+
         @Bean
         S3ApplicationSettings s3ApplicationSettings(S3AsyncClient s3AsyncClient,
                                                     S3ConfigurationProperties s3ConfigurationProperties,
diff --git a/src/test/java/org/prebid/server/hooks/execution/HookStageExecutorTest.java b/src/test/java/org/prebid/server/hooks/execution/HookStageExecutorTest.java
@@ -654,7 +654,7 @@ public void shouldExecuteEntrypointHooksToleratingTimeoutAndFailedFuture(VertxTe
                                             assertThat(hookOutcome.getStatus())
                                                     .isEqualTo(ExecutionStatus.execution_failure);
                                             assertThat(hookOutcome.getMessage()).isEqualTo("Failed after a while");
-                                            assertThat(hookOutcome.getExecutionTime()).isBetween(50L, 70L);
+                                            assertThat(hookOutcome.getExecutionTime()).isBetween(50L, 80L);
                                         });
 
                                         assertThat(group0Hooks.get(1)).satisfies(hookOutcome -> {
