diff --git a/data-prepper-plugins/s3-common/build.gradle b/data-prepper-plugins/s3-common/build.gradle new file mode 100644 index 0000000000..68f520b889 --- /dev/null +++ b/data-prepper-plugins/s3-common/build.gradle @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +dependencies { + compileOnly 'org.projectlombok:lombok:1.18.30' + annotationProcessor 'org.projectlombok:lombok:1.18.30' + implementation project(path: ':data-prepper-plugins:common') + implementation 'software.amazon.awssdk:sts' + implementation 'software.amazon.awssdk:s3' + implementation 'com.fasterxml.jackson.core:jackson-databind' + implementation 'org.apache.httpcomponents:httpcore:4.4.16' + implementation 'dev.failsafe:failsafe:3.3.2' + implementation 'io.micrometer:micrometer-core' + testImplementation libs.commons.lang3 + implementation libs.parquet.common + testImplementation testLibs.bundles.junit +} + +jacocoTestCoverageVerification { + dependsOn jacocoTestReport + violationRules { + rule { //in addition to core projects rule + limit { + minimum = 0.90 + } + } + } +} \ No newline at end of file diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/configuration/AwsAuthenticationOptions.java b/data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/config/AwsAuthenticationOptions.java similarity index 91% rename from data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/configuration/AwsAuthenticationOptions.java rename to data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/config/AwsAuthenticationOptions.java index 32f922f9c2..bfaa6e20e1 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/configuration/AwsAuthenticationOptions.java +++ b/data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/config/AwsAuthenticationOptions.java @@ -1,14 +1,19 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ -package org.opensearch.dataprepper.plugins.source.s3.configuration; +package org.opensearch.dataprepper.plugins.s3.common.config; import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.constraints.Size; -import software.amazon.awssdk.arns.Arn; import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.arns.Arn; import java.util.Map; import java.util.Optional; @@ -68,4 +73,3 @@ public Map getAwsStsHeaderOverrides() { return awsStsHeaderOverrides; } } - diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ownership/BucketOwnerProvider.java b/data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/ownership/BucketOwnerProvider.java similarity index 68% rename from data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ownership/BucketOwnerProvider.java rename to data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/ownership/BucketOwnerProvider.java index 7b7fc25fdb..cab2f6ba2c 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ownership/BucketOwnerProvider.java +++ b/data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/ownership/BucketOwnerProvider.java @@ -1,9 +1,14 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ -package org.opensearch.dataprepper.plugins.source.s3.ownership; +package org.opensearch.dataprepper.plugins.s3.common.ownership; import java.util.Optional; diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ownership/MappedBucketOwnerProvider.java b/data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/ownership/MappedBucketOwnerProvider.java similarity index 64% rename from data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ownership/MappedBucketOwnerProvider.java rename to data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/ownership/MappedBucketOwnerProvider.java index 8150f8584c..21f931c403 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ownership/MappedBucketOwnerProvider.java +++ b/data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/ownership/MappedBucketOwnerProvider.java @@ -1,21 +1,30 @@ -package org.opensearch.dataprepper.plugins.source.s3.ownership; +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.s3.common.ownership; import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.Optional; - /** * Implements {@link BucketOwnerProvider} using a mapping of bucket * names to account Ids for the bucket owners. Uses a delegate * {@link BucketOwnerProvider} as a fallback when the bucket is not * found in the map. */ -class MappedBucketOwnerProvider implements BucketOwnerProvider { +public class MappedBucketOwnerProvider implements BucketOwnerProvider { private final Map bucketOwnershipMap; private final BucketOwnerProvider fallbackProvider; - MappedBucketOwnerProvider(Map bucketOwnershipMap, BucketOwnerProvider fallbackProvider) { + public MappedBucketOwnerProvider(Map bucketOwnershipMap, BucketOwnerProvider fallbackProvider) { this.bucketOwnershipMap = new HashMap<>(Objects.requireNonNull(bucketOwnershipMap)); this.fallbackProvider = Objects.requireNonNull(fallbackProvider); } diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ownership/NoOwnershipBucketOwnerProvider.java b/data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/ownership/NoOwnershipBucketOwnerProvider.java similarity index 55% rename from data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ownership/NoOwnershipBucketOwnerProvider.java rename to data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/ownership/NoOwnershipBucketOwnerProvider.java index 4f1324dc1a..670c71cd6f 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ownership/NoOwnershipBucketOwnerProvider.java +++ b/data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/ownership/NoOwnershipBucketOwnerProvider.java @@ -1,9 +1,14 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ -package org.opensearch.dataprepper.plugins.source.s3.ownership; +package org.opensearch.dataprepper.plugins.s3.common.ownership; import java.util.Optional; @@ -11,7 +16,7 @@ * An implementation of {@link BucketOwnerProvider} which does not provide * a bucket owner, effectively skipping owner validation. */ -class NoOwnershipBucketOwnerProvider implements BucketOwnerProvider { +public class NoOwnershipBucketOwnerProvider implements BucketOwnerProvider{ @Override public Optional getBucketOwner(final String bucket) { return Optional.empty(); diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ownership/StaticBucketOwnerProvider.java b/data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/ownership/StaticBucketOwnerProvider.java similarity index 59% rename from data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ownership/StaticBucketOwnerProvider.java rename to data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/ownership/StaticBucketOwnerProvider.java index 0ef04ccffd..b5bf89b316 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ownership/StaticBucketOwnerProvider.java +++ b/data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/ownership/StaticBucketOwnerProvider.java @@ -1,18 +1,19 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ -package org.opensearch.dataprepper.plugins.source.s3.ownership; +package org.opensearch.dataprepper.plugins.s3.common.ownership; import java.util.Objects; import java.util.Optional; -/** - * An implementation of {@link BucketOwnerProvider} which provides the - * same owner for all buckets. - */ -class StaticBucketOwnerProvider implements BucketOwnerProvider { +public class StaticBucketOwnerProvider implements BucketOwnerProvider { private final String accountId; public StaticBucketOwnerProvider(final String accountId) { diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3InputFile.java b/data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/source/S3InputFile.java similarity index 89% rename from data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3InputFile.java rename to data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/source/S3InputFile.java index 3f355ee2f8..da0a1b1381 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3InputFile.java +++ b/data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/source/S3InputFile.java @@ -1,8 +1,18 @@ -package org.opensearch.dataprepper.plugins.source.s3; +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.s3.common.source; import org.apache.parquet.io.SeekableInputStream; import org.opensearch.dataprepper.model.io.InputFile; -import org.opensearch.dataprepper.plugins.source.s3.ownership.BucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import software.amazon.awssdk.services.s3.model.HeadObjectResponse; diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3InputStream.java b/data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/source/S3InputStream.java similarity index 98% rename from data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3InputStream.java rename to data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/source/S3InputStream.java index ad7ef86799..d2520e546d 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3InputStream.java +++ b/data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/source/S3InputStream.java @@ -1,4 +1,14 @@ -package org.opensearch.dataprepper.plugins.source.s3; +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.s3.common.source; import com.google.common.base.Preconditions; import com.google.common.io.ByteStreams; @@ -8,7 +18,7 @@ import dev.failsafe.function.CheckedSupplier; import org.apache.http.ConnectionClosedException; import org.apache.parquet.io.SeekableInputStream; -import org.opensearch.dataprepper.plugins.source.s3.ownership.BucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.core.sync.ResponseTransformer; diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectPluginMetrics.java b/data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/source/S3ObjectPluginMetrics.java similarity index 95% rename from data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectPluginMetrics.java rename to data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/source/S3ObjectPluginMetrics.java index ff2073f4d0..3f34b39641 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectPluginMetrics.java +++ b/data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/source/S3ObjectPluginMetrics.java @@ -1,8 +1,14 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ -package org.opensearch.dataprepper.plugins.source.s3; + +package org.opensearch.dataprepper.plugins.s3.common.source; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.DistributionSummary; @@ -94,4 +100,4 @@ public Counter getS3ObjectsThrottledCounter() { public Counter getS3ObjectsDeleteFailed() { return s3ObjectsDeleteFailed; } public Counter getS3ObjectReadFailedCounter() { return s3ObjectReadFailedCounter; } -} \ No newline at end of file +} diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectReference.java b/data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/source/S3ObjectReference.java similarity index 79% rename from data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectReference.java rename to data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/source/S3ObjectReference.java index 382e015bf5..f11b81e419 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectReference.java +++ b/data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/source/S3ObjectReference.java @@ -1,9 +1,16 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ -package org.opensearch.dataprepper.plugins.source.s3; +package org.opensearch.dataprepper.plugins.s3.common.source; + +import lombok.Getter; import java.util.Objects; import java.util.Optional; @@ -11,8 +18,10 @@ /** * Reference to an S3 object. */ -class S3ObjectReference { +public class S3ObjectReference { + @Getter private final String bucketName; + @Getter private final String key; private final String owner; @@ -22,20 +31,12 @@ private S3ObjectReference(final String bucketName, final String key, final Strin this.owner = owner; } - static Builder bucketAndKey(final String bucketName, final String key) { + public static Builder bucketAndKey(final String bucketName, final String key) { Objects.requireNonNull(bucketName, "bucketName must be non null"); Objects.requireNonNull(key, "key must be non null"); return new Builder(bucketName, key); } - String getBucketName() { - return bucketName; - } - - String getKey() { - return key; - } - Optional getBucketOwner() { return Optional.ofNullable(owner); } diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/StsArnRole.java b/data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/source/StsArnRole.java similarity index 74% rename from data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/StsArnRole.java rename to data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/source/StsArnRole.java index fa1bb236d4..4970a211d4 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/StsArnRole.java +++ b/data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/source/StsArnRole.java @@ -1,4 +1,14 @@ -package org.opensearch.dataprepper.plugins.source.s3; +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.s3.common.source; import software.amazon.awssdk.arns.Arn; diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/configuration/AwsAuthenticationOptionsTest.java b/data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/config/AwsAuthenticationOptionsTest.java similarity index 92% rename from data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/configuration/AwsAuthenticationOptionsTest.java rename to data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/config/AwsAuthenticationOptionsTest.java index 29fc197207..e090828a46 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/configuration/AwsAuthenticationOptionsTest.java +++ b/data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/config/AwsAuthenticationOptionsTest.java @@ -1,9 +1,14 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ -package org.opensearch.dataprepper.plugins.source.s3.configuration; +package org.opensearch.dataprepper.plugins.s3.common.config; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -69,4 +74,4 @@ private void reflectivelySetField(final AwsAuthenticationOptions awsAuthenticati field.setAccessible(false); } } -} \ No newline at end of file +} diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/ownership/MappedBucketOwnerProviderTest.java b/data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/ownership/MappedBucketOwnerProviderTest.java similarity index 93% rename from data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/ownership/MappedBucketOwnerProviderTest.java rename to data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/ownership/MappedBucketOwnerProviderTest.java index 89c1b97585..56fed70cb6 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/ownership/MappedBucketOwnerProviderTest.java +++ b/data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/ownership/MappedBucketOwnerProviderTest.java @@ -1,4 +1,14 @@ -package org.opensearch.dataprepper.plugins.source.s3.ownership; +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.s3.common.ownership; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; @@ -117,4 +127,4 @@ void getBucketOwner_returns_empty_when_not_in_map_nor_in_fallback() { assertThat(optionalOwner.isPresent(), equalTo(false)); } } -} \ No newline at end of file +} diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/ownership/NoOwnershipBucketOwnerProviderTest.java b/data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/ownership/NoOwnershipBucketOwnerProviderTest.java similarity index 77% rename from data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/ownership/NoOwnershipBucketOwnerProviderTest.java rename to data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/ownership/NoOwnershipBucketOwnerProviderTest.java index 432fac5723..9dbc644d4a 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/ownership/NoOwnershipBucketOwnerProviderTest.java +++ b/data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/ownership/NoOwnershipBucketOwnerProviderTest.java @@ -1,9 +1,14 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ -package org.opensearch.dataprepper.plugins.source.s3.ownership; +package org.opensearch.dataprepper.plugins.s3.common.ownership; import org.junit.jupiter.api.Test; @@ -26,4 +31,4 @@ void getBucketOwner_returns_empty() { assertThat(optionalOwner, notNullValue()); assertThat(optionalOwner.isPresent(), equalTo(false)); } -} \ No newline at end of file +} diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/ownership/StaticBucketOwnerProviderTest.java b/data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/ownership/StaticBucketOwnerProviderTest.java similarity index 84% rename from data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/ownership/StaticBucketOwnerProviderTest.java rename to data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/ownership/StaticBucketOwnerProviderTest.java index cf1937410b..c3238e52cd 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/ownership/StaticBucketOwnerProviderTest.java +++ b/data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/ownership/StaticBucketOwnerProviderTest.java @@ -1,9 +1,14 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ -package org.opensearch.dataprepper.plugins.source.s3.ownership; +package org.opensearch.dataprepper.plugins.s3.common.ownership; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -42,4 +47,4 @@ void getBucketOwner_returns_the_predefined_accountId() { assertThat(optionalOwner.isPresent(), equalTo(true)); assertThat(optionalOwner.get(), equalTo(accountId)); } -} \ No newline at end of file +} diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3InputFileTest.java b/data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/source/S3InputFileTest.java similarity index 91% rename from data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3InputFileTest.java rename to data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/source/S3InputFileTest.java index f77e9892d2..9960adeab5 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3InputFileTest.java +++ b/data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/source/S3InputFileTest.java @@ -1,10 +1,20 @@ -package org.opensearch.dataprepper.plugins.source.s3; +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.s3.common.source; import org.apache.parquet.io.SeekableInputStream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; -import org.opensearch.dataprepper.plugins.source.s3.ownership.BucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import software.amazon.awssdk.services.s3.model.HeadObjectResponse; @@ -16,7 +26,7 @@ import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertAll; -import static org.mockito.Mockito.any; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -100,5 +110,4 @@ public void testNewStream() { assertThat(seekableInputStream.getClass(), equalTo(S3InputStream.class)); } - } diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3InputStreamTest.java b/data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/source/S3InputStreamTest.java similarity index 96% rename from data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3InputStreamTest.java rename to data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/source/S3InputStreamTest.java index d42804f073..22f55733a1 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3InputStreamTest.java +++ b/data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/source/S3InputStreamTest.java @@ -1,4 +1,14 @@ -package org.opensearch.dataprepper.plugins.source.s3; +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.s3.common.source; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.DistributionSummary; @@ -10,7 +20,7 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.dataprepper.plugins.source.s3.ownership.BucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider; import software.amazon.awssdk.core.sync.ResponseTransformer; import software.amazon.awssdk.http.HttpStatusCode; import software.amazon.awssdk.services.s3.S3Client; @@ -197,8 +207,7 @@ void testReadEndOfFile() throws IOException { @MethodSource("retryableExceptions") void testReadSucceedsAfterRetries(final Class retryableExceptionClass) throws IOException { InputStream mockInputStream = mock(InputStream.class); - when(s3Client.getObject(any(GetObjectRequest.class), any(ResponseTransformer.class))) - .thenReturn(mockInputStream); + when(s3Client.getObject(any(GetObjectRequest.class), any(ResponseTransformer.class))).thenReturn(mockInputStream); when(mockInputStream.read()) .thenThrow(retryableExceptionClass) @@ -222,8 +231,7 @@ void testReadSucceedsAfterRetries(final Class retryableExce @MethodSource("retryableExceptions") void testReadFailsAfterRetries(final Class retryableExceptionClass) throws IOException { InputStream mockInputStream = mock(InputStream.class); - when(s3Client.getObject(any(GetObjectRequest.class), any(ResponseTransformer.class))) - .thenReturn(mockInputStream); + when(s3Client.getObject(any(GetObjectRequest.class), any(ResponseTransformer.class))).thenReturn(mockInputStream); when(mockInputStream.read()).thenThrow(retryableExceptionClass); @@ -235,7 +243,7 @@ void testReadFailsAfterRetries(final Class retryableExcepti verify(mockInputStream, times(RETRIES + 1)).read(); verify(mockInputStream, times(RETRIES + 2)).close(); verify(s3Client, times(RETRIES + 2)) - .getObject(any(GetObjectRequest.class), any(ResponseTransformer.class)); + .getObject(any(GetObjectRequest.class), any(ResponseTransformer.class)); } @Test @@ -277,8 +285,7 @@ void testReadAllBytesSucceedsAfterRetries(final Class retry throws IOException { InputStream mockInputStream = mock(InputStream.class); - when(s3Client.getObject(any(GetObjectRequest.class), any(ResponseTransformer.class))) - .thenReturn(mockInputStream); + when(s3Client.getObject(any(GetObjectRequest.class), any(ResponseTransformer.class))).thenReturn(mockInputStream); when(mockInputStream.readAllBytes()) .thenThrow(retryableExceptionClass) @@ -304,8 +311,7 @@ void testReadAllBytesFailsAfterRetries(final Class retryabl throws IOException { InputStream mockInputStream = mock(InputStream.class); - when(s3Client.getObject(any(GetObjectRequest.class), any(ResponseTransformer.class))) - .thenReturn(mockInputStream); + when(s3Client.getObject(any(GetObjectRequest.class), any(ResponseTransformer.class))).thenReturn(mockInputStream); when(mockInputStream.readAllBytes()).thenThrow(retryableExceptionClass); @@ -542,8 +548,8 @@ void testReadAfterSeekBackwardsWorks() throws IOException { void testS3ObjectsFailedNotFoundCounter() { when(s3Client.getObject(any(GetObjectRequest.class), any(ResponseTransformer.class))).thenThrow( S3Exception.builder() - .statusCode(HttpStatusCode.NOT_FOUND) - .build()); + .statusCode(HttpStatusCode.NOT_FOUND) + .build()); final S3InputStream s3InputStream = createObjectUnderTest(); assertThrows(IOException.class, () -> s3InputStream.read()); // Force opening the stream @@ -555,8 +561,8 @@ void testS3ObjectsFailedNotFoundCounter() { void testS3ObjectsFailedAccessDeniedCounter() { when(s3Client.getObject(any(GetObjectRequest.class), any(ResponseTransformer.class))).thenThrow( S3Exception.builder() - .statusCode(HttpStatusCode.FORBIDDEN) - .build()); + .statusCode(HttpStatusCode.FORBIDDEN) + .build()); final S3InputStream s3InputStream = createObjectUnderTest(); assertThrows(IOException.class, () -> s3InputStream.read()); // Force opening the stream @@ -568,7 +574,7 @@ void testS3ObjectsFailedAccessDeniedCounter() { void testS3ObjectsThrottledCounter() { S3Exception throttledException = mock(S3Exception.class); when(throttledException.isThrottlingException()).thenReturn(true); - + when(s3Client.getObject(any(GetObjectRequest.class), any(ResponseTransformer.class))).thenThrow(throttledException); final S3InputStream s3InputStream = createObjectUnderTest(); diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectPluginMetricsTest.java b/data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/source/S3ObjectPluginMetricsTest.java similarity index 92% rename from data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectPluginMetricsTest.java rename to data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/source/S3ObjectPluginMetricsTest.java index fee484d264..5d8eda8055 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectPluginMetricsTest.java +++ b/data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/source/S3ObjectPluginMetricsTest.java @@ -1,8 +1,14 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ -package org.opensearch.dataprepper.plugins.source.s3; + +package org.opensearch.dataprepper.plugins.s3.common.source; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.DistributionSummary; @@ -48,4 +54,5 @@ public void s3ObjectPluginMetricsTest(){ assertThat(metrics.getS3ObjectsThrottledCounter(),sameInstance(counter)); assertThat(metrics.getS3ObjectReadFailedCounter(),sameInstance(counter)); } + } diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectReferenceTest.java b/data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/source/S3ObjectReferenceTest.java similarity index 91% rename from data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectReferenceTest.java rename to data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/source/S3ObjectReferenceTest.java index cc142dc9ca..97d5b15005 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectReferenceTest.java +++ b/data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/source/S3ObjectReferenceTest.java @@ -1,9 +1,14 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ -package org.opensearch.dataprepper.plugins.source.s3; +package org.opensearch.dataprepper.plugins.s3.common.source; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -69,4 +74,4 @@ void toString_returns_string_with_bucket_and_key() { assertThat(objectUnderTest.toString(), containsString(bucketName)); assertThat(objectUnderTest.toString(), containsString(key)); } -} \ No newline at end of file +} diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/StsArnRoleTest.java b/data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/source/StsArnRoleTest.java similarity index 91% rename from data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/StsArnRoleTest.java rename to data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/source/StsArnRoleTest.java index 8a4cbe5395..7bb08ff483 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/StsArnRoleTest.java +++ b/data-prepper-plugins/s3-common/src/test/java/org/opensearch/dataprepper/plugins/s3/common/source/StsArnRoleTest.java @@ -1,9 +1,14 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ -package org.opensearch.dataprepper.plugins.source.s3; +package org.opensearch.dataprepper.plugins.s3.common.source; import org.apache.commons.lang3.RandomStringUtils; import org.junit.jupiter.api.Test; @@ -73,5 +78,4 @@ void getAccountId_returns_accountId_part() { private String randomAccountId() { return RandomStringUtils.randomNumeric(12); } - -} \ No newline at end of file +} diff --git a/data-prepper-plugins/s3-source/build.gradle b/data-prepper-plugins/s3-source/build.gradle index e226c984e5..2c44e51716 100644 --- a/data-prepper-plugins/s3-source/build.gradle +++ b/data-prepper-plugins/s3-source/build.gradle @@ -12,6 +12,7 @@ dependencies { implementation project(':data-prepper-plugins:buffer-common') implementation project(':data-prepper-plugins:common') implementation project(':data-prepper-plugins:sqs-common') + implementation project(':data-prepper-plugins:s3-common') implementation libs.armeria.core implementation 'io.micrometer:micrometer-core' diff --git a/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3DataSelectionIT.java b/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3DataSelectionIT.java index 772746b164..7af5222c05 100644 --- a/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3DataSelectionIT.java +++ b/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3DataSelectionIT.java @@ -17,8 +17,10 @@ import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.plugins.codec.CompressionOption; +import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectPluginMetrics; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectReference; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3DataSelection; -import org.opensearch.dataprepper.plugins.source.s3.ownership.BucketOwnerProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; diff --git a/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3MetricsIT.java b/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3MetricsIT.java index 2031754a56..8e15123ce1 100644 --- a/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3MetricsIT.java +++ b/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3MetricsIT.java @@ -15,8 +15,10 @@ import org.opensearch.dataprepper.model.buffer.Buffer; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectPluginMetrics; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectReference; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3DataSelection; -import org.opensearch.dataprepper.plugins.source.s3.ownership.BucketOwnerProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; diff --git a/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectWorkerIT.java b/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectWorkerIT.java index 3579eb6e55..77c1b7c398 100644 --- a/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectWorkerIT.java +++ b/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectWorkerIT.java @@ -22,8 +22,10 @@ import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.plugins.codec.CompressionOption; +import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectPluginMetrics; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectReference; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3DataSelection; -import org.opensearch.dataprepper.plugins.source.s3.ownership.BucketOwnerProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; diff --git a/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanObjectWorkerIT.java b/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanObjectWorkerIT.java index c898cd3193..90511d5675 100644 --- a/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanObjectWorkerIT.java +++ b/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanObjectWorkerIT.java @@ -40,6 +40,8 @@ import org.opensearch.dataprepper.model.source.coordinator.SourceCoordinator; import org.opensearch.dataprepper.core.parser.model.SourceCoordinationConfig; import org.opensearch.dataprepper.plugins.codec.CompressionOption; +import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectPluginMetrics; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3ScanBucketOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3ScanBucketOptions; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3ScanScanOptions; @@ -47,7 +49,6 @@ import org.opensearch.dataprepper.plugins.source.s3.configuration.S3SelectCSVOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3SelectJsonOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3SelectSerializationFormatOption; -import org.opensearch.dataprepper.plugins.source.s3.ownership.BucketOwnerProvider; import org.opensearch.dataprepper.plugins.sourcecoordinator.inmemory.InMemorySourceCoordinationStore; import org.opensearch.dataprepper.core.sourcecoordination.LeaseBasedSourceCoordinator; import software.amazon.awssdk.regions.Region; diff --git a/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3SelectObjectWorkerIT.java b/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3SelectObjectWorkerIT.java index c1f6a27a7f..b589df5a37 100644 --- a/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3SelectObjectWorkerIT.java +++ b/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3SelectObjectWorkerIT.java @@ -21,11 +21,13 @@ import org.opensearch.dataprepper.model.buffer.Buffer; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectPluginMetrics; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectReference; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3SelectCSVOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3DataSelection; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3SelectJsonOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3SelectSerializationFormatOption; -import org.opensearch.dataprepper.plugins.source.s3.ownership.BucketOwnerProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; diff --git a/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/SqsServiceIT.java b/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/SqsServiceIT.java index 0e4749b125..57ca606503 100644 --- a/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/SqsServiceIT.java +++ b/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/SqsServiceIT.java @@ -18,10 +18,11 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; +import org.opensearch.dataprepper.plugins.s3.common.config.AwsAuthenticationOptions; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectReference; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3DataSelection; import org.opensearch.dataprepper.plugins.source.s3.configuration.NotificationSourceOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.OnErrorOption; -import org.opensearch.dataprepper.plugins.source.s3.configuration.AwsAuthenticationOptions; import org.opensearch.dataprepper.plugins.source.s3.configuration.SqsOptions; import org.opensearch.dataprepper.plugins.source.sqs.common.SqsBackoff; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; diff --git a/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorkerIT.java b/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorkerIT.java index 61362129c1..6a08a42acf 100644 --- a/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorkerIT.java +++ b/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorkerIT.java @@ -24,6 +24,7 @@ import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.event.JacksonEvent; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectReference; import org.opensearch.dataprepper.plugins.source.s3.configuration.NotificationSourceOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.OnErrorOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3DataSelection; diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/AwsAuthenticationAdapter.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/AwsAuthenticationAdapter.java index 16c0197f94..c60dca8d47 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/AwsAuthenticationAdapter.java +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/AwsAuthenticationAdapter.java @@ -7,7 +7,7 @@ import org.opensearch.dataprepper.aws.api.AwsCredentialsOptions; import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; -import org.opensearch.dataprepper.plugins.source.s3.configuration.AwsAuthenticationOptions; +import org.opensearch.dataprepper.plugins.s3.common.config.AwsAuthenticationOptions; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; class AwsAuthenticationAdapter { diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/EventMetadataModifier.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/EventMetadataModifier.java index 8e384a72c8..b86acade2a 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/EventMetadataModifier.java +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/EventMetadataModifier.java @@ -6,6 +6,7 @@ package org.opensearch.dataprepper.plugins.source.s3; import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectReference; import java.util.Objects; import java.util.function.BiConsumer; diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectHandler.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectHandler.java index 88c50ee3a0..8f1de524b5 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectHandler.java +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectHandler.java @@ -6,6 +6,7 @@ import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; import org.opensearch.dataprepper.model.source.coordinator.SourceCoordinator; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectReference; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3DataSelection; import java.io.IOException; diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectRequest.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectRequest.java index 211d81a871..10a679c036 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectRequest.java +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectRequest.java @@ -9,10 +9,12 @@ import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.plugins.codec.CompressionOption; +import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectPluginMetrics; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectReference; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3SelectCSVOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3SelectJsonOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3SelectSerializationFormatOption; -import org.opensearch.dataprepper.plugins.source.s3.ownership.BucketOwnerProvider; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.CompressionType; diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectWorker.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectWorker.java index 7b27713548..d973da3213 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectWorker.java +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectWorker.java @@ -15,7 +15,10 @@ import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.source.coordinator.SourceCoordinator; import org.opensearch.dataprepper.plugins.codec.CompressionOption; -import org.opensearch.dataprepper.plugins.source.s3.ownership.BucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.source.S3InputFile; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectPluginMetrics; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectReference; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3DataSelection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanPartitionCreationSupplier.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanPartitionCreationSupplier.java index b95c737209..b7745ae434 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanPartitionCreationSupplier.java +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanPartitionCreationSupplier.java @@ -7,10 +7,10 @@ import org.opensearch.dataprepper.model.source.coordinator.PartitionIdentifier; import org.opensearch.dataprepper.model.source.coordinator.SourceCoordinator; +import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider; import org.opensearch.dataprepper.plugins.source.s3.configuration.FolderPartitioningOptions; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3ScanKeyPathOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3ScanSchedulingOptions; -import org.opensearch.dataprepper.plugins.source.s3.ownership.BucketOwnerProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.s3.S3Client; diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanService.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanService.java index 9460d68b34..069c385e37 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanService.java +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanService.java @@ -8,8 +8,8 @@ import org.opensearch.dataprepper.common.concurrent.BackgroundThreadFactory; import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; import org.opensearch.dataprepper.model.source.coordinator.SourceCoordinator; +import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3ScanBucketOptions; -import org.opensearch.dataprepper.plugins.source.s3.ownership.BucketOwnerProvider; import java.time.Duration; import java.time.LocalDateTime; diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3SelectObjectWorker.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3SelectObjectWorker.java index e06c0d2e5c..1aba0bd162 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3SelectObjectWorker.java +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3SelectObjectWorker.java @@ -18,11 +18,13 @@ import org.opensearch.dataprepper.model.log.JacksonLog; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.source.coordinator.SourceCoordinator; +import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectPluginMetrics; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectReference; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3SelectCSVOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3SelectJsonOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3DataSelection; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3SelectSerializationFormatOption; -import org.opensearch.dataprepper.plugins.source.s3.ownership.BucketOwnerProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.s3.S3AsyncClient; diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3Service.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3Service.java index 8c3eb7d6f1..82fa023ed7 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3Service.java +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3Service.java @@ -6,6 +6,7 @@ package org.opensearch.dataprepper.plugins.source.s3; import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectReference; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3DataSelection; import java.io.IOException; diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3Source.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3Source.java index 4b7b6e72ac..265d94f5d9 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3Source.java +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3Source.java @@ -20,12 +20,14 @@ import org.opensearch.dataprepper.model.source.Source; import org.opensearch.dataprepper.model.source.coordinator.SourceCoordinator; import org.opensearch.dataprepper.model.source.coordinator.UsesSourceCoordination; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectPluginMetrics; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectReference; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3ScanScanOptions; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3SelectCSVOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3SelectJsonOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3SelectOptions; -import org.opensearch.dataprepper.plugins.source.s3.ownership.BucketOwnerProvider; import org.opensearch.dataprepper.plugins.source.s3.ownership.ConfigBucketOwnerProviderFactory; +import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.services.s3.model.CompressionType; diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3SourceConfig.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3SourceConfig.java index 280024176d..1a11788aa0 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3SourceConfig.java +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3SourceConfig.java @@ -14,7 +14,7 @@ import org.opensearch.dataprepper.aws.validator.AwsAccountId; import org.opensearch.dataprepper.model.configuration.PluginModel; import org.opensearch.dataprepper.plugins.codec.CompressionOption; -import org.opensearch.dataprepper.plugins.source.s3.configuration.AwsAuthenticationOptions; +import org.opensearch.dataprepper.plugins.s3.common.config.AwsAuthenticationOptions; import org.opensearch.dataprepper.plugins.source.s3.configuration.NotificationSourceOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.NotificationTypeOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.OnErrorOption; diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ScanObjectWorker.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ScanObjectWorker.java index 6ea8841635..574ab64bef 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ScanObjectWorker.java +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ScanObjectWorker.java @@ -14,13 +14,14 @@ import org.opensearch.dataprepper.model.source.coordinator.exceptions.PartitionNotFoundException; import org.opensearch.dataprepper.model.source.coordinator.exceptions.PartitionNotOwnedException; import org.opensearch.dataprepper.model.source.coordinator.exceptions.PartitionUpdateException; +import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectReference; import org.opensearch.dataprepper.plugins.source.s3.configuration.FolderPartitioningOptions; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3ScanSchedulingOptions; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3DataSelection; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3ScanBucketOptions; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3ScanBucketOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3ScanKeyPathOption; -import org.opensearch.dataprepper.plugins.source.s3.ownership.BucketOwnerProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.s3.S3Client; diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorker.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorker.java index 8c1e155595..2aabff0484 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorker.java +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorker.java @@ -11,6 +11,7 @@ import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectReference; import org.opensearch.dataprepper.plugins.source.s3.configuration.NotificationSourceOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.OnErrorOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.SqsOptions; diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ownership/ConfigBucketOwnerProviderFactory.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ownership/ConfigBucketOwnerProviderFactory.java index acb5137e66..d3fe99bdee 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ownership/ConfigBucketOwnerProviderFactory.java +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ownership/ConfigBucketOwnerProviderFactory.java @@ -6,9 +6,13 @@ package org.opensearch.dataprepper.plugins.source.s3.ownership; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; +import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.ownership.MappedBucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.ownership.NoOwnershipBucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.ownership.StaticBucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.source.StsArnRole; import org.opensearch.dataprepper.plugins.source.s3.S3SourceConfig; import org.opensearch.dataprepper.plugins.source.s3.SqsQueueUrl; -import org.opensearch.dataprepper.plugins.source.s3.StsArnRole; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import java.net.MalformedURLException; diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/AwsAuthenticationAdapterTest.java b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/AwsAuthenticationAdapterTest.java index a412ca7de7..dbdefc3d17 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/AwsAuthenticationAdapterTest.java +++ b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/AwsAuthenticationAdapterTest.java @@ -15,7 +15,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.aws.api.AwsCredentialsOptions; import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; -import org.opensearch.dataprepper.plugins.source.s3.configuration.AwsAuthenticationOptions; +import org.opensearch.dataprepper.plugins.s3.common.config.AwsAuthenticationOptions; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.regions.Region; diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/EventMetadataModifierTest.java b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/EventMetadataModifierTest.java index 1eab96160f..5bdf73e002 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/EventMetadataModifierTest.java +++ b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/EventMetadataModifierTest.java @@ -18,6 +18,7 @@ import static org.mockito.Mockito.never; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectReference; import java.util.UUID; import java.util.stream.Stream; diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectRequestTest.java b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectRequestTest.java index 766e3b4ca8..35b68141f9 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectRequestTest.java +++ b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectRequestTest.java @@ -11,10 +11,12 @@ import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.plugins.codec.CompressionOption; +import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectPluginMetrics; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectReference; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3SelectCSVOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3SelectJsonOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3SelectSerializationFormatOption; -import org.opensearch.dataprepper.plugins.source.s3.ownership.BucketOwnerProvider; import org.opensearch.dataprepper.test.helper.ReflectivelySetField; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectWorkerTest.java b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectWorkerTest.java index 3c5db8dcad..cbc3bec38a 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectWorkerTest.java +++ b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectWorkerTest.java @@ -28,7 +28,10 @@ import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.source.coordinator.SourceCoordinator; import org.opensearch.dataprepper.plugins.codec.CompressionOption; -import org.opensearch.dataprepper.plugins.source.s3.ownership.BucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.source.S3InputFile; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectPluginMetrics; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectReference; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3DataSelection; import software.amazon.awssdk.core.sync.ResponseTransformer; import software.amazon.awssdk.services.s3.S3Client; diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanObjectWorkerTest.java b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanObjectWorkerTest.java index 3e0cd2131b..7ccb00bed4 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanObjectWorkerTest.java +++ b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanObjectWorkerTest.java @@ -24,6 +24,8 @@ import org.opensearch.dataprepper.model.source.coordinator.exceptions.PartitionNotFoundException; import org.opensearch.dataprepper.model.source.coordinator.exceptions.PartitionNotOwnedException; import org.opensearch.dataprepper.model.source.coordinator.exceptions.PartitionUpdateException; +import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectReference; import org.opensearch.dataprepper.plugins.source.s3.configuration.FolderPartitioningOptions; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3ScanScanOptions; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3ScanBucketOptions; @@ -31,7 +33,6 @@ import org.opensearch.dataprepper.plugins.source.s3.configuration.S3ScanSchedulingOptions; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3DataSelection; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3ScanKeyPathOption; -import org.opensearch.dataprepper.plugins.source.s3.ownership.BucketOwnerProvider; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanPartitionCreationSupplierTest.java b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanPartitionCreationSupplierTest.java index 5567d25437..a49894c289 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanPartitionCreationSupplierTest.java +++ b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanPartitionCreationSupplierTest.java @@ -14,11 +14,11 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.model.source.coordinator.PartitionIdentifier; import org.opensearch.dataprepper.model.source.coordinator.SourceCoordinator; +import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider; import org.opensearch.dataprepper.plugins.source.s3.configuration.FolderPartitioningOptions; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3ScanBucketOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3ScanKeyPathOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3ScanSchedulingOptions; -import org.opensearch.dataprepper.plugins.source.s3.ownership.BucketOwnerProvider; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanServiceTest.java b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanServiceTest.java index 68a81fd0e5..c888a7ca2e 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanServiceTest.java +++ b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanServiceTest.java @@ -13,11 +13,11 @@ import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; import org.opensearch.dataprepper.model.source.coordinator.SourceCoordinator; +import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3ScanBucketOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3ScanBucketOptions; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3ScanKeyPathOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3ScanScanOptions; -import org.opensearch.dataprepper.plugins.source.s3.ownership.BucketOwnerProvider; import java.time.Duration; import java.time.LocalDateTime; diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3SelectObjectWorkerTest.java b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3SelectObjectWorkerTest.java index c5d2ccf562..ac95eb771d 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3SelectObjectWorkerTest.java +++ b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3SelectObjectWorkerTest.java @@ -18,11 +18,13 @@ import org.opensearch.dataprepper.model.buffer.Buffer; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectPluginMetrics; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectReference; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3SelectCSVOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3SelectJsonOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3SelectSerializationFormatOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3DataSelection; -import org.opensearch.dataprepper.plugins.source.s3.ownership.BucketOwnerProvider; import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.model.CompressionType; diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ServiceTest.java b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ServiceTest.java index 69ba677f24..b99dd41898 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ServiceTest.java +++ b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ServiceTest.java @@ -5,6 +5,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectReference; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3DataSelection; import java.io.IOException; diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorkerTest.java b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorkerTest.java index d7ad742298..32b956d3a2 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorkerTest.java +++ b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorkerTest.java @@ -27,6 +27,7 @@ import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; import org.opensearch.dataprepper.model.acknowledgements.ProgressCheck; +import org.opensearch.dataprepper.plugins.s3.common.source.S3ObjectReference; import org.opensearch.dataprepper.plugins.source.s3.configuration.NotificationSourceOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.OnErrorOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.S3DataSelection; diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/ownership/ConfigBucketOwnerProviderFactoryTest.java b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/ownership/ConfigBucketOwnerProviderFactoryTest.java index 2668a9d9b2..96b94f8326 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/ownership/ConfigBucketOwnerProviderFactoryTest.java +++ b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/ownership/ConfigBucketOwnerProviderFactoryTest.java @@ -14,8 +14,12 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; import org.opensearch.dataprepper.plugins.source.s3.S3SourceConfig; -import org.opensearch.dataprepper.plugins.source.s3.configuration.AwsAuthenticationOptions; import org.opensearch.dataprepper.plugins.source.s3.configuration.SqsOptions; +import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.ownership.MappedBucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.ownership.NoOwnershipBucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.ownership.StaticBucketOwnerProvider; +import org.opensearch.dataprepper.plugins.s3.common.config.AwsAuthenticationOptions; import software.amazon.awssdk.auth.credentials.AwsCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; diff --git a/settings.gradle b/settings.gradle index f161e8c7d5..90e002e5de 100644 --- a/settings.gradle +++ b/settings.gradle @@ -192,6 +192,7 @@ include 'data-prepper-plugins:http-source-common' include 'data-prepper-plugins:http-common' include 'data-prepper-plugins:aws-lambda' include 'data-prepper-plugins:ml-inference-processor' +include 'data-prepper-plugins:s3-common' //include 'data-prepper-plugins:dummy-plugin' include 'data-prepper-plugin-schema' include 'data-prepper-plugins:kinesis-source'