From 271ee907da74adfa86a06393fb6c377c76b3a12c Mon Sep 17 00:00:00 2001 From: Taylor Gray Date: Wed, 1 Mar 2023 15:33:38 -0600 Subject: [PATCH 1/2] Add grok pattern validatior to data-prepper-api --- data-prepper-api/build.gradle | 1 + .../validations/GrokPatternValidator.java | 46 +++++++++++++++ .../validations/GrokPatternValidatorTest.java | 57 +++++++++++++++++++ 3 files changed, 104 insertions(+) create mode 100644 data-prepper-api/src/main/java/org/opensearch/dataprepper/validations/GrokPatternValidator.java create mode 100644 data-prepper-api/src/test/java/org/opensearch/dataprepper/validations/GrokPatternValidatorTest.java diff --git a/data-prepper-api/build.gradle b/data-prepper-api/build.gradle index 253c94d058..467efca09b 100644 --- a/data-prepper-api/build.gradle +++ b/data-prepper-api/build.gradle @@ -4,6 +4,7 @@ */ dependencies { + implementation "io.krakens:java-grok:0.1.9" implementation 'io.micrometer:micrometer-core' implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/validations/GrokPatternValidator.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/validations/GrokPatternValidator.java new file mode 100644 index 0000000000..c46abafd9c --- /dev/null +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/validations/GrokPatternValidator.java @@ -0,0 +1,46 @@ +package org.opensearch.dataprepper.validations; + +import io.krakens.grok.api.GrokCompiler; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public final class GrokPatternValidator { + + private static final GrokCompiler grokCompiler = GrokCompiler.newInstance(); + + static { + grokCompiler.registerDefaultPatterns(); + } + + private GrokPatternValidator() { + + } + + public static List validatePatterns(final Map additionalPatternsToRegister, final List matchPatterns) { + final List incompatiblePatternErrors = new ArrayList<>(); + additionalPatternsToRegister.forEach((patternName, pattern) -> { + try { + grokCompiler.register(patternName, pattern); + } catch (final NullPointerException | IllegalArgumentException e) { + final String errorMessage = String.format("The grok pattern with name \"%s\" and pattern \"%s\" is invalid with the following error: %s", patternName, pattern, e.getMessage()); + incompatiblePatternErrors.add(errorMessage); + } + }); + + if (incompatiblePatternErrors.isEmpty()) { + matchPatterns.forEach(pattern -> { + try { + grokCompiler.compile(pattern); + } catch (final IllegalArgumentException e) { + final String errorMessage = String.format("The grok match pattern \"%s\" is invalid with the following error: %s", pattern, e.getMessage()); + incompatiblePatternErrors.add(errorMessage); + } + }); + } + + return incompatiblePatternErrors; + } + +} diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/validations/GrokPatternValidatorTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/validations/GrokPatternValidatorTest.java new file mode 100644 index 0000000000..175f74a3e3 --- /dev/null +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/validations/GrokPatternValidatorTest.java @@ -0,0 +1,57 @@ +package org.opensearch.dataprepper.validations; + +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.startsWith; + +public class GrokPatternValidatorTest { + + @Test + void validatePatterns_with_invalid_match_patterns_returns_expected_messages() { + final Map additionalPatterns = Map.of( + "PATTERN1", "one_%{WORD}" + ); + + final List matchPatterns = List.of("some_pattern", "%{PATTERN1}", "%{WORD} %{PATTERN10}", "%{ANOTHER_PATTERN}"); + final List validationErrors = GrokPatternValidator.validatePatterns(additionalPatterns, matchPatterns); + + assertThat(validationErrors, notNullValue()); + assertThat(validationErrors.size(), equalTo(2)); + assertThat(validationErrors.get(0), startsWith("The grok match pattern \"%{WORD} %{PATTERN10}\" is invalid")); + assertThat(validationErrors.get(1), startsWith("The grok match pattern \"%{ANOTHER_PATTERN}\" is invalid")); + } + + @Test + void validatePatterns_with_no_invalid_patterns_returns_empty_list() { + final Map additionalPatterns = Map.of( + "PATTERN1", "one_%{WORD}", + "PATTERN2", "test_%{NUMBER}" + ); + + final List matchPatterns = List.of("some_pattern", "%{PATTERN1}", "%{WORD} %{PATTERN2}"); + + final List errorMessages = GrokPatternValidator.validatePatterns(additionalPatterns, matchPatterns); + assertThat(errorMessages, notNullValue()); + assertThat(errorMessages.size(), equalTo(0)); + } + + @Test + void validatePatterns_with_invalid_additional_patterns_returns_expected_messages() { + final Map additionalPatterns = new HashMap<>(); + additionalPatterns.put("test", null); + + final List matchPatterns = List.of("some_pattern", "%{PATTERN1}", "%{WORD} %{PATTERN2}"); + + final List errorMessages = GrokPatternValidator.validatePatterns(additionalPatterns, matchPatterns); + assertThat(errorMessages, notNullValue()); + assertThat(errorMessages.size(), equalTo(1)); + assertThat(errorMessages.get(0), startsWith("The grok pattern with name \"test\" and pattern \"null\"")); + } +} From d32384671be7678b0de37e09ba9c5f5e178624ec Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Mar 2023 22:09:49 +0000 Subject: [PATCH 2/2] Bump net.bytebuddy:byte-buddy-agent in /data-prepper-plugins/opensearch Bumps [net.bytebuddy:byte-buddy-agent](https://github.com/raphw/byte-buddy) from 1.12.22 to 1.14.0. - [Release notes](https://github.com/raphw/byte-buddy/releases) - [Changelog](https://github.com/raphw/byte-buddy/blob/master/release-notes.md) - [Commits](https://github.com/raphw/byte-buddy/compare/byte-buddy-1.12.22...byte-buddy-1.14.0) --- updated-dependencies: - dependency-name: net.bytebuddy:byte-buddy-agent dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/opensearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/opensearch/build.gradle b/data-prepper-plugins/opensearch/build.gradle index ef2257cc8d..607704ae53 100644 --- a/data-prepper-plugins/opensearch/build.gradle +++ b/data-prepper-plugins/opensearch/build.gradle @@ -59,7 +59,7 @@ dependencies { implementation 'software.amazon.awssdk:apache-client' testImplementation 'commons-io:commons-io:2.11.0' testImplementation 'net.bytebuddy:byte-buddy:1.12.22' - testImplementation 'net.bytebuddy:byte-buddy-agent:1.12.22' + testImplementation 'net.bytebuddy:byte-buddy-agent:1.14.0' testImplementation 'org.slf4j:slf4j-simple:2.0.6' }