M365 Crawler Metric, Buffer, and Unit Test Updates

Signed-off-by: Alex Christensen <alchrisk@amazon.com>

Author: chrisale000

.github/workflows/data-prepper-aws-secrets-e2e-tests.yml

@@ -29,20 +29,21 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: configure aws credentials
         id: creds
-        uses: aws-actions/configure-aws-credentials@v4
+        uses: aws-actions/configure-aws-credentials@v5
         with:
           role-to-assume: ${{ secrets.TEST_IAM_ROLE_ARN }}
           aws-region: ${{ secrets.TEST_REGION }}
           output-credentials: true
-      - name: get caller identity 1
-        run: |
-          aws sts get-caller-identity
       - name: Configure AWS Credentials file
         run: |
           aws configure set default.region ${{ secrets.TEST_REGION }}
           aws configure set default.aws_access_key_id ${{ steps.creds.outputs.aws-access-key-id }}
           aws configure set default.aws_secret_access_key ${{ steps.creds.outputs.aws-secret-access-key }}
           aws configure set default.aws_session_token ${{ steps.creds.outputs.aws-session-token }}
+      - name: Fix AWS credentials permissions for Docker
+        run: |
+          chmod 644 ~/.aws/credentials ~/.aws/config
+          ls -la ~/.aws/
       - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:

.github/workflows/kinesis-source-integration-tests.yml

@@ -30,15 +30,12 @@ jobs:
 
       - name: configure aws credentials
         id: creds
-        uses: aws-actions/configure-aws-credentials@v4
+        uses: aws-actions/configure-aws-credentials@v5
         with:
           role-to-assume: ${{ secrets.TEST_IAM_ROLE_ARN }}
           aws-region: ${{ secrets.TEST_REGION }}
           output-credentials: true
 
-      - name: get caller identity 1
-        run: |
-          aws sts get-caller-identity
       - name: Configure AWS Credentials file
         run: |
           aws configure set default.region ${{ secrets.TEST_REGION }}

.github/workflows/opensearch-sink-opensearch-integration-tests.yml

@@ -20,7 +20,7 @@ jobs:
     strategy:
       matrix:
         java: [11]
-        opensearch: [1.0.1, 1.1.0, 1.2.4, 1.3.14, 2.0.1, 2.1.0, 2.3.0, 2.5.0, 2.7.0, 2.9.0, 2.11.1]
+        opensearch: [1.0.1, 1.1.0, 1.2.4, 1.3.14, 2.0.1, 2.1.0, 2.3.0, 2.5.0, 2.7.0, 2.9.0, 2.11.1, 2.12.0, 2.19.3]
       fail-fast: false
 
     runs-on: ubuntu-latest
@@ -32,15 +32,22 @@ jobs:
           java-version: ${{ matrix.java }}
       - name: Checkout Data Prepper
         uses: actions/checkout@v2
+      - name: Set password for OpenSearch version
+        run: |
+          if [[ "${{ matrix.opensearch }}" =~ ^2\.(1[2-9]|[2-9][0-9]) ]]; then
+            echo "PASSWORD=yourStrongPassword123!" >> $GITHUB_ENV
+          else
+            echo "PASSWORD=admin" >> $GITHUB_ENV
+          fi
       - name: Run OpenSearch docker
         run: |
           docker pull opensearchproject/opensearch:${{ matrix.opensearch }}
-          docker run -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" -d opensearchproject/opensearch:${{ matrix.opensearch }}
+          docker run -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" -e "OPENSEARCH_INITIAL_ADMIN_PASSWORD=yourStrongPassword123!" -d opensearchproject/opensearch:${{ matrix.opensearch }}
           sleep 90
       - name: Run OpenSearch tests
         run: |
-          ./gradlew :data-prepper-plugins:opensearch:integrationTest --tests "org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchIT" -Dtests.opensearch.host=localhost:9200 -Dtests.opensearch.user=admin -Dtests.opensearch.password=admin
-          ./gradlew :data-prepper-plugins:opensearch:integrationTest -Dtests.opensearch.host=localhost:9200 -Dtests.opensearch.user=admin -Dtests.opensearch.password=admin -Dtests.opensearch.bundle=true -Dtests.opensearch.version=opensearch:${{ matrix.opensearch }}
+          ./gradlew :data-prepper-plugins:opensearch:integrationTest --tests "org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchIT" -Dtests.opensearch.host=localhost:9200 -Dtests.opensearch.user=admin -Dtests.opensearch.password="$PASSWORD"
+          ./gradlew :data-prepper-plugins:opensearch:integrationTest -Dtests.opensearch.host=localhost:9200 -Dtests.opensearch.user=admin -Dtests.opensearch.password="$PASSWORD" -Dtests.opensearch.bundle=true -Dtests.opensearch.version=opensearch:${{ matrix.opensearch }}
       - name: Upload Unit Test Results
         if: always()
         uses: actions/upload-artifact@v4

data-prepper-api/build.gradle

@@ -19,6 +19,7 @@ dependencies {
     testImplementation project(':data-prepper-test:test-common')
     testImplementation 'org.skyscreamer:jsonassert:1.5.3'
     testImplementation libs.commons.io
+    testImplementation 'org.hibernate.validator:hibernate-validator:8.0.1.Final'
 }
 
 jacocoTestCoverageVerification {

data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/DataPrepperVersion.java

@@ -1,17 +1,25 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
 package org.opensearch.dataprepper.model.configuration;
 
 import java.util.Objects;
 import java.util.Optional;
+import java.util.ServiceLoader;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 public class DataPrepperVersion {
-    private static final String CURRENT_VERSION = "2.13";
-
     private static final String FULL_FORMAT = "%d.%d";
     private static final String SHORTHAND_FORMAT = "%d";
-    private static final String VERSION_PATTERN_STRING = "^((\\d+)(\\.(\\d+))?)$";
-    private static final Pattern VERSION_PATTERN = Pattern.compile(VERSION_PATTERN_STRING);
+    private static final Pattern VERSION_PATTERN = Pattern.compile("^((\\d+)(\\.(\\d+))?)$");
+    private static final Pattern VERSION_FULL_PATTERN = Pattern.compile("^((\\d+)(\\.(\\d+))?(\\.(\\d+))?)(-SNAPSHOT)?$");
     private static final int MAJOR_VERSION_PATTERN_POSITION = 2;
     private static final int MINOR_VERSION_PATTERN_POSITION = 4;
 
@@ -20,21 +28,27 @@ public class DataPrepperVersion {
 
     private static DataPrepperVersion instance;
 
-    private DataPrepperVersion(int majorVersion, Integer minorVersion) {
+    private DataPrepperVersion(final int majorVersion, final Integer minorVersion) {
         this.minorVersion = minorVersion;
         this.majorVersion = majorVersion;
     }
 
-    public static DataPrepperVersion getCurrentVersion() {
-        if (Objects.isNull(instance)) {
-            instance = parse(CURRENT_VERSION);
+    public static synchronized DataPrepperVersion getCurrentVersion() {
+        if (instance == null) {
+            final String versionString = ServiceLoader.load(VersionProvider.class).findFirst()
+                    .orElseThrow(() -> new RuntimeException("No Data Prepper version available."))
+                    .getVersionString();
+            instance = parse(versionString, VERSION_FULL_PATTERN);
         }
         return instance;
     }
 
     public static DataPrepperVersion parse(final String version) {
+        return parse(version, VERSION_PATTERN);
+    }
 
-        final Matcher result = VERSION_PATTERN.matcher(version);
+    private static DataPrepperVersion parse(final String version, final Pattern versionPattern) {
+        final Matcher result = versionPattern.matcher(version);
         if(result.find()) {
             String major = result.group(MAJOR_VERSION_PATTERN_POSITION);
             final int foundMajorVersion = Integer.parseInt(major);

data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/VersionProvider.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.dataprepper.model.configuration;
+
+/**
+ * An interface for providing the current Data Prepper version
+ * as a Java SPI. This is intended only to be used by Data Prepper
+ * core or within tests.
+ *
+ * @since 2.13
+ */
+public interface VersionProvider {
+    /**
+     * Gets the current Data Prepper version as a string.
+     * @return The Data Prepper version.
+     */
+    String getVersionString();
+}

data-prepper-api/src/main/java/org/opensearch/dataprepper/model/constraints/ByteCountMax.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.dataprepper.model.constraints;
+
+import jakarta.validation.Constraint;
+import jakarta.validation.Payload;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Use this annotation to ensure that the byte count does not exceed
+ * a specified maximum value.
+ *
+ * @since 2.13
+ */
+@Constraint(validatedBy = {ByteCountMaxValidator.class})
+@Target({ElementType.METHOD, ElementType.FIELD, ElementType.TYPE_USE})
+@Retention(RetentionPolicy.RUNTIME)
+public @interface ByteCountMax {
+    /**
+     * The maximum value defined as a {@link org.opensearch.dataprepper.model.types.ByteCount}
+     * string.
+     *
+     * @return The byte count string
+     * @since 2.13
+     */
+    String value();
+
+    String message() default "The provided byte count exceeds the maximum allowed value.";
+
+    Class<?>[] groups() default {};
+
+    Class<? extends Payload>[] payload() default {};
+}

data-prepper-api/src/main/java/org/opensearch/dataprepper/model/constraints/ByteCountMaxValidator.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.dataprepper.model.constraints;
+
+import jakarta.validation.ConstraintValidator;
+import jakarta.validation.ConstraintValidatorContext;
+import org.opensearch.dataprepper.model.types.ByteCount;
+
+public class ByteCountMaxValidator implements ConstraintValidator<ByteCountMax, ByteCount> {
+    private ByteCount maxByteCount;
+
+    @Override
+    public void initialize(final ByteCountMax constraint) {
+        maxByteCount = ByteCount.parse(constraint.value());
+    }
+
+    @Override
+    public boolean isValid(final ByteCount byteCount, final ConstraintValidatorContext context) {
+        if (byteCount == null) {
+            return true;
+        }
+
+        if (byteCount.compareTo(maxByteCount) > 0) {
+            if (context != null) {
+                context.disableDefaultConstraintViolation();
+                context.buildConstraintViolationWithTemplate(
+                        String.format("The provided byte count %s exceeds maximum of %s", byteCount, maxByteCount))
+                        .addConstraintViolation();
+            }
+
+            return false;
+        }
+
+        return true;
+    }
+}

data-prepper-api/src/main/java/org/opensearch/dataprepper/model/constraints/ByteCountMin.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.dataprepper.model.constraints;
+
+import jakarta.validation.Constraint;
+import jakarta.validation.Payload;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Use this annotation to ensure that the byte count is not below
+ * a specified maximum value.
+ *
+ * @since 2.13
+ */
+@Constraint(validatedBy = {ByteCountMinValidator.class})
+@Target({ElementType.METHOD, ElementType.FIELD, ElementType.TYPE_USE})
+@Retention(RetentionPolicy.RUNTIME)
+public @interface ByteCountMin {
+    /**
+     * The minimum value defined as a {@link org.opensearch.dataprepper.model.types.ByteCount}
+     * string.
+     *
+     * @return The byte count string
+     * @since 2.13
+     */
+    String value();
+
+    String message() default "The provided byte count is below the minimum allowed value.";
+
+    Class<?>[] groups() default {};
+
+    Class<? extends Payload>[] payload() default {};
+}

data-prepper-api/src/main/java/org/opensearch/dataprepper/model/constraints/ByteCountMinValidator.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.dataprepper.model.constraints;
+
+import jakarta.validation.ConstraintValidator;
+import jakarta.validation.ConstraintValidatorContext;
+import org.opensearch.dataprepper.model.types.ByteCount;
+
+public class ByteCountMinValidator implements ConstraintValidator<ByteCountMin, ByteCount> {
+    private ByteCount minByteCount;
+
+    @Override
+    public void initialize(final ByteCountMin constraint) {
+        minByteCount = ByteCount.parse(constraint.value());
+    }
+
+    @Override
+    public boolean isValid(final ByteCount byteCount, final ConstraintValidatorContext context) {
+        if (byteCount == null) {
+            return true;
+        }
+
+        if (byteCount.compareTo(minByteCount) < 0) {
+            if (context != null) {
+                context.disableDefaultConstraintViolation();
+                context.buildConstraintViolationWithTemplate(
+                        String.format("The provided byte count %s is below minimum of %s", byteCount, minByteCount))
+                        .addConstraintViolation();
+            }
+
+            return false;
+        }
+
+        return true;
+    }
+}