(feat) adding support for endpoint_url in pipeline config

Signed-off-by: Aman Dhingra <amandhingra00@gmail.com>

Author: amdhing

data-prepper-plugins/dynamodb-source/src/main/java/org/opensearch/dataprepper/plugins/source/dynamodb/ClientFactory.java

@@ -13,9 +13,13 @@
 import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
 import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
+import software.amazon.awssdk.services.dynamodb.DynamoDbClientBuilder;
 import software.amazon.awssdk.services.dynamodb.streams.DynamoDbStreamsClient;
+import software.amazon.awssdk.services.dynamodb.streams.DynamoDbStreamsClientBuilder;
 import software.amazon.awssdk.services.s3.S3Client;
 
+import java.net.URI;
+
 public class ClientFactory {
 
     private final AwsCredentialsProvider awsCredentialsProvider;
@@ -37,18 +41,28 @@ public ClientFactory(final AwsCredentialsSupplier awsCredentialsSupplier,
 
 
     public DynamoDbStreamsClient buildDynamoDbStreamClient() {
-        return DynamoDbStreamsClient.builder()
+        DynamoDbStreamsClientBuilder clientBuilder = DynamoDbStreamsClient.builder()
                 .credentialsProvider(awsCredentialsProvider)
-                .region(awsAuthenticationConfig.getAwsRegion())
-                .build();
+                .region(awsAuthenticationConfig.getAwsRegion());
+
+        if (awsAuthenticationConfig.getEndpointUrl() != null && !awsAuthenticationConfig.getEndpointUrl().isEmpty()) {
+            clientBuilder.endpointOverride(URI.create(awsAuthenticationConfig.getEndpointUrl()));
+        }
+
+        return clientBuilder.build();
     }
 
 
     public DynamoDbClient buildDynamoDBClient() {
-        return DynamoDbClient.builder()
+        DynamoDbClientBuilder clientBuilder = DynamoDbClient.builder()
                 .region(awsAuthenticationConfig.getAwsRegion())
-                .credentialsProvider(awsCredentialsProvider)
-                .build();
+                .credentialsProvider(awsCredentialsProvider);
+
+        if (awsAuthenticationConfig.getEndpointUrl() != null && !awsAuthenticationConfig.getEndpointUrl().isEmpty()) {
+            clientBuilder.endpointOverride(URI.create(awsAuthenticationConfig.getEndpointUrl()));
+        }
+
+        return clientBuilder.build();
     }
 
 

data-prepper-plugins/dynamodb-source/src/main/java/org/opensearch/dataprepper/plugins/source/dynamodb/configuration/AwsAuthenticationConfig.java

@@ -29,6 +29,10 @@ public class AwsAuthenticationConfig {
     @Size(max = 5, message = "sts_header_overrides supports a maximum of 5 headers to override")
     private Map<String, String> awsStsHeaderOverrides;
 
+    @JsonProperty("endpoint_url")
+    @Size(min = 10, max = 2048, message = "endpoint_url length should be between 10 and 2048 characters")
+    private String endpoint_url;
+
     public String getAwsStsRoleArn() {
         return awsStsRoleArn;
     }
@@ -44,5 +48,9 @@ public Region getAwsRegion() {
     public Map<String, String> getAwsStsHeaderOverrides() {
         return awsStsHeaderOverrides;
     }
+
+    public String getEndpointUrl() {
+        return endpoint_url;
+    }
 }