Skip to content

Commit 9071bab

Browse files
srikanthpadakantisrikanthpadakanti
authored
Merge branch 'opensearch-project:main' into file-tail-source-6782
2 parents 59db55c + 513dcfb commit 9071bab

50 files changed

Lines changed: 1852 additions & 548 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/geoip/processor/GeoIPProcessor.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ private void processRecords(Collection<Record<Event>> records, GeoIPDatabaseRead
148148
//Lookup from DB
149149
if (ipAddress != null && !ipAddress.isEmpty()) {
150150
try {
151-
final Optional<InetAddress> optionalInetAddress = GeoInetAddress.usableInetFromString(ipAddress);
151+
final Optional<InetAddress> optionalInetAddress = GeoInetAddress.usableInetFromString(ipAddress, geoIPProcessorConfig.getLookupPrivateAddresses());
152152
if (optionalInetAddress.isPresent()) {
153153
geoData = geoIPDatabaseReader.getGeoData(optionalInetAddress.get(), fields, databases);
154154
if (geoData.isEmpty()) {

‎data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/geoip/processor/GeoIPProcessorConfig.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,11 @@ public class GeoIPProcessorConfig {
5252
})
5353
private String whenCondition;
5454

55+
@JsonProperty("lookup_private_addresses")
56+
@JsonPropertyDescription("When set to <code>true</code>, the processor will also enrich private/internal IP addresses (e.g. 10.x, 172.16.x, 192.168.x, loopback). " +
57+
"This is useful when using custom MMDB files enriched with internal IPAM data. Default is <code>false</code>.")
58+
private boolean lookupPrivateAddresses = false;
59+
5560
/**
5661
* Get List of entries
5762
* @return List of EntryConfig
@@ -91,4 +96,8 @@ public List<String> getTagsOnNoValidIp() {
9196
public String getWhenCondition() {
9297
return whenCondition;
9398
}
99+
100+
public boolean getLookupPrivateAddresses() {
101+
return lookupPrivateAddresses;
102+
}
94103
}

‎data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/geoip/processor/GeoInetAddress.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,17 @@
1919
class GeoInetAddress {
2020

2121
static Optional<InetAddress> usableInetFromString(final String ipAddress) {
22+
return usableInetFromString(ipAddress, false);
23+
}
24+
25+
static Optional<InetAddress> usableInetFromString(final String ipAddress, final boolean lookupPrivateAddresses) {
2226
final InetAddress address;
2327
try {
2428
address = InetAddresses.forString(ipAddress);
2529
} catch (final IllegalArgumentException e) {
2630
return Optional.empty();
2731
}
28-
if (isPublicIpAddress(address))
32+
if (isPublicIpAddress(address) || lookupPrivateAddresses)
2933
return Optional.of(address);
3034
return Optional.empty();
3135
}

‎data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/geoip/processor/GeoIPProcessorConfigTest.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ void testDefaultConfig() {
3232
assertThat(geoIPProcessorConfig.getTagsOnEngineFailure(), equalTo(null));
3333
assertThat(geoIPProcessorConfig.getTagsOnIPNotFound(), equalTo(null));
3434
assertThat(geoIPProcessorConfig.getWhenCondition(), equalTo(null));
35+
assertThat(geoIPProcessorConfig.getLookupPrivateAddresses(), equalTo(false));
3536
}
3637

3738
@Test
@@ -51,4 +52,10 @@ void testGetEntries() throws NoSuchFieldException, IllegalAccessException {
5152
assertThat(geoIPProcessorConfig.getTagsOnIPNotFound(), equalTo(tagsOnIPNotFound));
5253
assertThat(geoIPProcessorConfig.getWhenCondition(), equalTo(whenCondition));
5354
}
55+
56+
@Test
57+
void testLookupPrivateAddressesWhenSet() throws NoSuchFieldException, IllegalAccessException {
58+
ReflectivelySetField.setField(GeoIPProcessorConfig.class, geoIPProcessorConfig, "lookupPrivateAddresses", true);
59+
assertThat(geoIPProcessorConfig.getLookupPrivateAddresses(), equalTo(true));
60+
}
5461
}

‎data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/geoip/processor/GeoIPProcessorTest.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
import static org.junit.jupiter.api.Assertions.assertThrows;
4848
import static org.junit.jupiter.api.Assertions.assertTrue;
4949
import static org.mockito.ArgumentMatchers.any;
50+
import static org.mockito.ArgumentMatchers.anyBoolean;
5051
import static org.mockito.Mockito.doThrow;
5152
import static org.mockito.Mockito.lenient;
5253
import static org.mockito.Mockito.mockStatic;
@@ -313,7 +314,7 @@ void doExecuteTest_should_not_add_geodata_if_ip_address_is_not_public() {
313314
when(geoIPProcessorConfig.getTagsOnNoValidIp()).thenReturn(testTags);
314315

315316
try (final MockedStatic<GeoInetAddress> ipValidationCheckMockedStatic = mockStatic(GeoInetAddress.class)) {
316-
ipValidationCheckMockedStatic.when(() -> GeoInetAddress.usableInetFromString(any())).thenReturn(Optional.empty());
317+
ipValidationCheckMockedStatic.when(() -> GeoInetAddress.usableInetFromString(any(), anyBoolean())).thenReturn(Optional.empty());
317318

318319
when(geoIPProcessorConfig.getEntries()).thenReturn(List.of(entry));
319320
when(entry.getSource()).thenReturn(SOURCE);

‎data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/geoip/processor/GeoInetAddressTest.java‎

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,4 +45,63 @@ void ipValidationcheckTest_invalid(String invalidIpAddress) {
4545
assertThat(actual, notNullValue());
4646
assertThat(actual.isPresent(), equalTo(false));
4747
}
48-
}
48+
49+
@ParameterizedTest
50+
@ValueSource(strings = {"10.0.0.1", "172.16.0.1", "192.168.1.1"})
51+
void privateIpAcceptedWhenLookupEnabled(String privateIpAddress) {
52+
final Optional<InetAddress> actual = GeoInetAddress.usableInetFromString(privateIpAddress, true);
53+
assertThat(actual, notNullValue());
54+
assertThat(actual.isPresent(), equalTo(true));
55+
assertThat(actual.get(), equalTo(InetAddresses.forString(privateIpAddress)));
56+
}
57+
58+
@ParameterizedTest
59+
@ValueSource(strings = {"10.0.0.1", "172.16.0.1", "192.168.1.1"})
60+
void privateIpRejectedWhenLookupDisabled(String privateIpAddress) {
61+
final Optional<InetAddress> actual = GeoInetAddress.usableInetFromString(privateIpAddress, false);
62+
assertThat(actual, notNullValue());
63+
assertThat(actual.isPresent(), equalTo(false));
64+
}
65+
66+
@ParameterizedTest
67+
@ValueSource(strings = {"127.0.0.1", "::1"})
68+
void loopbackAcceptedWhenLookupEnabled(String loopbackAddress) {
69+
final Optional<InetAddress> actual = GeoInetAddress.usableInetFromString(loopbackAddress, true);
70+
assertThat(actual, notNullValue());
71+
assertThat(actual.isPresent(), equalTo(true));
72+
}
73+
74+
@ParameterizedTest
75+
@ValueSource(strings = {"93.184.216.34", "2607:f8b0:4005:805::200e"})
76+
void publicIpWorksInBothModes(String publicIpAddress) {
77+
final Optional<InetAddress> withFlag = GeoInetAddress.usableInetFromString(publicIpAddress, true);
78+
final Optional<InetAddress> withoutFlag = GeoInetAddress.usableInetFromString(publicIpAddress, false);
79+
assertThat(withFlag.isPresent(), equalTo(true));
80+
assertThat(withoutFlag.isPresent(), equalTo(true));
81+
}
82+
83+
@ParameterizedTest
84+
@ValueSource(strings = {"123", "255.255.255.999", "true", "[1,2,3]"})
85+
void invalidIpRejectedInBothModes(String invalidIpAddress) {
86+
final Optional<InetAddress> withFlag = GeoInetAddress.usableInetFromString(invalidIpAddress, true);
87+
final Optional<InetAddress> withoutFlag = GeoInetAddress.usableInetFromString(invalidIpAddress, false);
88+
assertThat(withFlag.isPresent(), equalTo(false));
89+
assertThat(withoutFlag.isPresent(), equalTo(false));
90+
}
91+
92+
@ParameterizedTest
93+
@ValueSource(strings = {"224.0.0.1", "239.255.255.250"})
94+
void multicastAcceptedWhenLookupEnabled(String multicastAddress) {
95+
final Optional<InetAddress> actual = GeoInetAddress.usableInetFromString(multicastAddress, true);
96+
assertThat(actual, notNullValue());
97+
assertThat(actual.isPresent(), equalTo(true));
98+
}
99+
100+
@ParameterizedTest
101+
@ValueSource(strings = {"169.254.1.1", "fe80::1"})
102+
void linkLocalAcceptedWhenLookupEnabled(String linkLocalAddress) {
103+
final Optional<InetAddress> actual = GeoInetAddress.usableInetFromString(linkLocalAddress, true);
104+
assertThat(actual, notNullValue());
105+
assertThat(actual.isPresent(), equalTo(true));
106+
}
107+
}

‎data-prepper-plugins/http-client-common/build.gradle‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*
9+
*/
10+
11+
dependencies {
12+
implementation libs.armeria.core
13+
implementation 'com.fasterxml.jackson.core:jackson-databind'
14+
implementation 'jakarta.validation:jakarta.validation-api'
15+
}
16+
17+
jacocoTestCoverageVerification {
18+
dependsOn jacocoTestReport
19+
violationRules {
20+
rule {
21+
limit {
22+
minimum = 1.0
23+
}
24+
}
25+
}
26+
}

‎data-prepper-plugins/http-client-common/src/main/java/org/opensearch/dataprepper/plugins/http/client/auth/BasicAuthHttpRequestAuthenticator.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*
9+
*/
10+
11+
package org.opensearch.dataprepper.plugins.http.client.auth;
12+
13+
import com.linecorp.armeria.common.HttpHeaderNames;
14+
import com.linecorp.armeria.common.HttpHeadersBuilder;
15+
16+
import java.nio.charset.StandardCharsets;
17+
import java.util.Base64;
18+
19+
public class BasicAuthHttpRequestAuthenticator implements HttpRequestAuthenticator {
20+
21+
private final String headerValue;
22+
23+
public BasicAuthHttpRequestAuthenticator(final String username, final String password) {
24+
if (username == null || username.isBlank()) {
25+
throw new IllegalArgumentException("Username must not be null or blank");
26+
}
27+
if (password == null) {
28+
throw new IllegalArgumentException("Password must not be null");
29+
}
30+
final String credentials = username + ":" + password;
31+
this.headerValue = "Basic " + Base64.getEncoder()
32+
.encodeToString(credentials.getBytes(StandardCharsets.UTF_8));
33+
}
34+
35+
@Override
36+
public void applyAuth(final HttpHeadersBuilder builder) {
37+
builder.add(HttpHeaderNames.AUTHORIZATION, headerValue);
38+
}
39+
}

‎data-prepper-plugins/http-client-common/src/main/java/org/opensearch/dataprepper/plugins/http/client/auth/BearerTokenHttpRequestAuthenticator.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*
9+
*/
10+
11+
package org.opensearch.dataprepper.plugins.http.client.auth;
12+
13+
import com.linecorp.armeria.common.HttpHeaderNames;
14+
import com.linecorp.armeria.common.HttpHeadersBuilder;
15+
16+
public class BearerTokenHttpRequestAuthenticator implements HttpRequestAuthenticator {
17+
18+
private final String headerValue;
19+
20+
public BearerTokenHttpRequestAuthenticator(final String token) {
21+
if (token == null || token.isBlank()) {
22+
throw new IllegalArgumentException("Bearer token must not be null or blank");
23+
}
24+
this.headerValue = "Bearer " + token;
25+
}
26+
27+
@Override
28+
public void applyAuth(final HttpHeadersBuilder builder) {
29+
builder.add(HttpHeaderNames.AUTHORIZATION, headerValue);
30+
}
31+
}

‎data-prepper-plugins/http-client-common/src/main/java/org/opensearch/dataprepper/plugins/http/client/auth/HttpClientAuthenticationConfig.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*
9+
*/
10+
11+
package org.opensearch.dataprepper.plugins.http.client.auth;
12+
13+
import com.fasterxml.jackson.annotation.JsonProperty;
14+
import jakarta.validation.Valid;
15+
import jakarta.validation.constraints.NotBlank;
16+
17+
public class HttpClientAuthenticationConfig {
18+
19+
@Valid
20+
@JsonProperty("http_basic")
21+
private HttpBasicConfig httpBasic;
22+
23+
@JsonProperty("bearer_token")
24+
@NotBlank
25+
private String bearerToken;
26+
27+
public HttpBasicConfig getHttpBasic() {
28+
return httpBasic;
29+
}
30+
31+
public String getBearerToken() {
32+
return bearerToken;
33+
}
34+
35+
public static class HttpBasicConfig {
36+
37+
@JsonProperty("username")
38+
@NotBlank
39+
private String username;
40+
41+
@JsonProperty("password")
42+
@NotBlank
43+
private String password;
44+
45+
public String getUsername() {
46+
return username;
47+
}
48+
49+
public String getPassword() {
50+
return password;
51+
}
52+
}
53+
}

0 commit comments

Comments
 (0)