diff --git a/data-prepper-plugins/opensearch/build.gradle b/data-prepper-plugins/opensearch/build.gradle index 39c6477a12..92b34ff02d 100644 --- a/data-prepper-plugins/opensearch/build.gradle +++ b/data-prepper-plugins/opensearch/build.gradle @@ -46,6 +46,7 @@ dependencies { testImplementation 'net.bytebuddy:byte-buddy:1.15.11' testImplementation 'net.bytebuddy:byte-buddy-agent:1.15.11' testImplementation testLibs.slf4j.simple + testImplementation 'org.wiremock:wiremock:3.10.0' } sourceSets { diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchIT.java index 23fbc4c915..34c66e23c3 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchIT.java @@ -32,6 +32,7 @@ public void testOpenSearchConnection() throws IOException { builder.withUsername(user); builder.withPassword(password); } + builder.withInsecure(true); final AwsCredentialsSupplier awsCredentialsSupplier = mock(AwsCredentialsSupplier.class); final RestHighLevelClient client = builder.build().createClient(awsCredentialsSupplier); diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 2146b6ac20..564ed55fce 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -1691,6 +1691,7 @@ private Map initializeConfigurationMetadata(final String indexTy metadata.put(IndexConfiguration.INDEX_ALIAS, indexAlias); metadata.put(IndexConfiguration.TEMPLATE_FILE, templateFilePath); metadata.put(IndexConfiguration.FLUSH_TIMEOUT, -1); + metadata.put("insecure", true); final String user = System.getProperty("tests.opensearch.user"); final String password = System.getProperty("tests.opensearch.password"); if (user != null) { diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java index f33ed06be8..8461bc37c8 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java @@ -384,8 +384,18 @@ private void checkProxyPort(final int port) { } private void attachSSLContext(final HttpAsyncClientBuilder httpClientBuilder) { - final SSLContext sslContext = certPath != null ? getCAStrategy(certPath) : getTrustAllStrategy(); - httpClientBuilder.setSSLContext(sslContext); + final SSLContext sslContext; + if(certPath != null) { + sslContext = getCAStrategy(certPath); + } else if(this.insecure) { + sslContext = getTrustAllStrategy(); + } else { + sslContext = null; + } + if(sslContext != null) { + httpClientBuilder.setSSLContext(sslContext); + } + if (this.insecure) { httpClientBuilder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE); } @@ -439,7 +449,7 @@ private OpenSearchTransport createOpenSearchTransport(final RestHighLevelClient transportOptions.setRequestCompressionSize(Integer.MAX_VALUE); } - return new AwsSdk2Transport(createSdkHttpClient(), HttpHost.create(hosts.get(0)).getHostName(), + return new AwsSdk2Transport(createSdkHttpClient(), HttpHost.create(hosts.get(0)).toHostString(), serviceName, Region.of(awsRegion), transportOptions.build()); } else { return new RestClientTransport( @@ -461,11 +471,13 @@ private SdkHttpClient createSdkHttpClient() { } private void attachSSLContext(final ApacheHttpClient.Builder apacheHttpClientBuilder) { - TrustManager[] trustManagers = createTrustManagers(certPath); - apacheHttpClientBuilder.tlsTrustManagersProvider(() -> trustManagers); + TrustManager[] trustManagers = createTrustManagers(certPath, insecure); + if(trustManagers.length > 0) { + apacheHttpClientBuilder.tlsTrustManagersProvider(() -> trustManagers); + } } - private static TrustManager[] createTrustManagers(final Path certPath) { + private static TrustManager[] createTrustManagers(final Path certPath, final boolean insecure) { if (certPath != null) { LOG.info("Using the cert provided in the config."); try (InputStream certificateInputStream = Files.newInputStream(certPath)) { @@ -481,8 +493,11 @@ private static TrustManager[] createTrustManagers(final Path certPath) { } catch (Exception ex) { throw new RuntimeException(ex.getMessage(), ex); } - } else { + } else if(insecure) { + LOG.info("Using the trust all strategy"); return new TrustManager[] { new X509TrustAllManager() }; + } else { + return new TrustManager[0]; } } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchClientFactory.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchClientFactory.java index 2ee6c59b87..06e2ab3f07 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchClientFactory.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchClientFactory.java @@ -271,7 +271,9 @@ private void setConnectAndSocketTimeout(final org.elasticsearch.client.RestClien private void attachSSLContext(final NettyNioAsyncHttpClient.Builder asyncClientBuilder, final OpenSearchSourceConfiguration openSearchSourceConfiguration) { TrustManager[] trustManagers = createTrustManagers(openSearchSourceConfiguration.getConnectionConfiguration()); - asyncClientBuilder.tlsTrustManagersProvider(() -> trustManagers); + if (trustManagers.length > 0) { + asyncClientBuilder.tlsTrustManagersProvider(() -> trustManagers); + } } private void attachSSLContext(final HttpAsyncClientBuilder httpClientBuilder, final OpenSearchSourceConfiguration openSearchSourceConfiguration) { @@ -287,31 +289,37 @@ private void attachSSLContext(final HttpAsyncClientBuilder httpClientBuilder, fi private TrustManager[] createTrustManagers(final ConnectionConfiguration connectionConfiguration) { final Path certPath = connectionConfiguration.getCertPath(); - if (Objects.nonNull(certPath)) { + final String certificate = connectionConfiguration.getCertificate(); + if (certPath != null) { return TrustStoreProvider.createTrustManager(certPath); - } else if (Objects.nonNull(connectionConfiguration.getCertificate())) { - if (PemObjectValidator.isPemObject(connectionConfiguration.getCertificate())) { - return TrustStoreProvider.createTrustManager(connectionConfiguration.getCertificate()); + } else if (certificate != null) { + if (PemObjectValidator.isPemObject(certificate)) { + return TrustStoreProvider.createTrustManager(certificate); } else { - return TrustStoreProvider.createTrustManager(Path.of(connectionConfiguration.getCertificate())); - } - } else { + return TrustStoreProvider.createTrustManager(Path.of(certificate));} + } else if (connectionConfiguration.isInsecure()) { return TrustStoreProvider.createTrustAllManager(); + + } else { + return new TrustManager[0]; } } private SSLContext getCAStrategy(final ConnectionConfiguration connectionConfiguration) { final Path certPath = connectionConfiguration.getCertPath(); - if (Objects.nonNull(certPath)) { + final String certificate = connectionConfiguration.getCertificate(); + if (certPath != null) { return TrustStoreProvider.createSSLContext(certPath); - } else if (Objects.nonNull(connectionConfiguration.getCertificate())) { - if (PemObjectValidator.isPemObject(connectionConfiguration.getCertificate())) { - return TrustStoreProvider.createSSLContext(connectionConfiguration.getCertificate()); + } else if (certificate != null) { + if (PemObjectValidator.isPemObject(certificate)) { + return TrustStoreProvider.createSSLContext(certificate); } else { return TrustStoreProvider.createSSLContext(Path.of(connectionConfiguration.getCertificate())); } + } else if (connectionConfiguration.isInsecure()) { + return TrustStoreProvider.createSSLContextWithTrustAllStrategy(); } else { - return TrustStoreProvider.createSSLContextWithTrustAllStrategy(); + return null; } } } diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java index 9a4cf44b3e..8c891564b3 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java @@ -150,7 +150,6 @@ void testCreateOpenSearchClientAwsServerlessDefault() throws IOException { when(awsCredentialsSupplier.getProvider(any())).thenReturn(awsCredentialsProvider); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); - when(apacheHttpClientBuilder.tlsTrustManagersProvider(any())).thenReturn(apacheHttpClientBuilder); when(apacheHttpClientBuilder.build()).thenReturn(apacheHttpClient); final OpenSearchClient openSearchClient; try (final MockedStatic apacheHttpClientMockedStatic = mockStatic(ApacheHttpClient.class)) { @@ -160,7 +159,6 @@ void testCreateOpenSearchClientAwsServerlessDefault() throws IOException { assertNotNull(openSearchClient); assertThat(openSearchClient._transport(), instanceOf(AwsSdk2Transport.class)); assertThat(openSearchClient._transport().jsonpMapper(), instanceOf(PreSerializedJsonpMapper.class)); - verify(apacheHttpClientBuilder).tlsTrustManagersProvider(any()); verify(apacheHttpClientBuilder).build(); openSearchClient.shutdown(); client.close(); diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration_ServerTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration_ServerTest.java new file mode 100644 index 0000000000..115e1afcec --- /dev/null +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration_ServerTest.java @@ -0,0 +1,186 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.sink.opensearch; + +import com.github.tomakehurst.wiremock.WireMockServer; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.client.core.MainResponse; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch.core.InfoResponse; +import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; + +import javax.net.ssl.SSLHandshakeException; +import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import java.util.UUID; + +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.jsonResponse; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class ConnectionConfiguration_ServerTest { + private static WireMockServer wireMockServer; + + @Mock + private AwsCredentialsSupplier awsCredentialsSupplier; + + private String host; + + private String clusterUuid; + + @BeforeAll + static void setUpAll() { + wireMockServer = new WireMockServer(options() + .httpDisabled(true) + .dynamicHttpsPort() + .keystorePath("src/test/resources/test_keystore.jks") + .keystorePassword("password") + .keyManagerPassword("password") + ); + + wireMockServer.start(); + } + + @AfterAll + static void tearDownAll() { + wireMockServer.stop(); + } + + @BeforeEach + void setUp() { + host = "https://localhost:" + wireMockServer.httpsPort(); + + clusterUuid = UUID.randomUUID().toString(); + final Map responseBody = Map.of( + "name", "opensearch", + "cluster_name", "opensearch", + "cluster_uuid", clusterUuid, + "version", Map.of( + "number", "2.10.0", + "build_hash", "abcdefg", + "build_date", "20241212", + "build_type", "testing", + "distribution", "datapreppertesting", + "build_snapshot", "false", + "lucene_version", "8", + "minimum_wire_compatibility_version", "2.10.0", + "minimum_index_compatibility_version", "2.10.0" + ), + "tagline", "You Know, for Search" + ); + wireMockServer.stubFor(get("/").willReturn(jsonResponse(responseBody, 200))); + } + + @Nested + class DefaultConfiguration { + private ConnectionConfiguration createObjectUnderTest() { + return new ConnectionConfiguration.Builder(Collections.singletonList(host)) + .build(); + } + + @Test + void createClient_will_not_trust_self_signed_certificates_by_default() { + final RestHighLevelClient client = createObjectUnderTest().createClient(awsCredentialsSupplier); + assertThat(client, notNullValue()); + + assertThrows(SSLHandshakeException.class, () -> client.info(RequestOptions.DEFAULT)); + } + + @Test + void createOpenSearchClient_will_not_trust_self_signed_certificates_by_default() { + final ConnectionConfiguration objectUnderTest = createObjectUnderTest(); + final OpenSearchClient openSearchClient = objectUnderTest.createOpenSearchClient(objectUnderTest.createClient(awsCredentialsSupplier), awsCredentialsSupplier); + assertThat(openSearchClient, notNullValue()); + + assertThrows(SSLHandshakeException.class, openSearchClient::info); + } + } + + @Nested + class DefaultSigV4Configuration { + @BeforeEach + void setUp() { + when(awsCredentialsSupplier.getProvider(any())).thenReturn(AnonymousCredentialsProvider.create()); + } + + private ConnectionConfiguration createObjectUnderTest() { + return new ConnectionConfiguration.Builder(Collections.singletonList(host)) + .withAwsSigv4(true) + .withAwsRegion("us-east-1") + .build(); + } + + @Test + void createClient_will_not_trust_self_signed_certificates_by_default() { + final RestHighLevelClient client = createObjectUnderTest().createClient(awsCredentialsSupplier); + assertThat(client, notNullValue()); + + assertThrows(SSLHandshakeException.class, () -> client.info(RequestOptions.DEFAULT)); + } + + @Test + void createOpenSearchClient_will_not_trust_self_signed_certificates_by_default() { + final ConnectionConfiguration objectUnderTest = createObjectUnderTest(); + final OpenSearchClient openSearchClient = objectUnderTest.createOpenSearchClient(objectUnderTest.createClient(awsCredentialsSupplier), awsCredentialsSupplier); + assertThat(openSearchClient, notNullValue()); + + assertThrows(SSLHandshakeException.class, openSearchClient::info); + } + } + + @Nested + class InsecureConfiguration { + private ConnectionConfiguration createObjectUnderTest() { + return new ConnectionConfiguration.Builder(Collections.singletonList(host)) + .withInsecure(true) + .build(); + } + + @Test + void createClient_will_trust_self_signed_certificates_if_insecure() throws IOException { + final RestHighLevelClient client = createObjectUnderTest().createClient(awsCredentialsSupplier); + assertThat(client, notNullValue()); + + final MainResponse infoResponse = client.info(RequestOptions.DEFAULT); + + assertThat(infoResponse, notNullValue()); + assertThat(infoResponse.getClusterName(), equalTo("opensearch")); + assertThat(infoResponse.getClusterUuid(), equalTo(clusterUuid)); + } + + + @Test + void createOpenSearchClient_will_trust_self_signed_certificates_if_insecure() throws IOException { + final ConnectionConfiguration objectUnderTest = createObjectUnderTest(); + final OpenSearchClient openSearchClient = objectUnderTest.createOpenSearchClient(objectUnderTest.createClient(awsCredentialsSupplier), awsCredentialsSupplier); + assertThat(openSearchClient, notNullValue()); + + final InfoResponse infoResponse = openSearchClient.info(); + + assertThat(infoResponse, notNullValue()); + assertThat(infoResponse.clusterName(), equalTo("opensearch")); + assertThat(infoResponse.clusterUuid(), equalTo(clusterUuid)); + } + } +} \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchClientFactoryTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchClientFactoryTest.java index 647dd09227..ecc7d54787 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchClientFactoryTest.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchClientFactoryTest.java @@ -27,6 +27,8 @@ import software.amazon.awssdk.regions.Region; import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; + import java.nio.file.Path; import java.time.Duration; import java.util.Collections; @@ -41,6 +43,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; @@ -409,9 +412,40 @@ void createSdkAsyncHttpClient_with_self_signed_certificate() { lenient().when(openSearchSourceConfiguration.getConnectionConfiguration()).thenReturn(connectionConfiguration); lenient().when(connectionConfiguration.getCertPath()).thenReturn(path); try (MockedStatic trustStoreProviderMockedStatic = mockStatic(TrustStoreProvider.class)) { + TrustManager[] mockTrustManagers = new TrustManager[] { mock(TrustManager.class) }; + trustStoreProviderMockedStatic.when(() -> TrustStoreProvider.createTrustManager(path)).thenReturn(mockTrustManagers); final SdkAsyncHttpClient sdkAsyncHttpClient = createObjectUnderTest().createSdkAsyncHttpClient(openSearchSourceConfiguration); assertThat(sdkAsyncHttpClient, notNullValue()); trustStoreProviderMockedStatic.verify(() -> TrustStoreProvider.createTrustManager(path)); } } + @Test + void createSdkAsyncHttpClient_with_secure_configuration_and_no_cert_path_does_not_trust_all_managers() { + when(connectionConfiguration.getCertPath()).thenReturn(null); + when(connectionConfiguration.isInsecure()).thenReturn(false); + when(connectionConfiguration.getConnectTimeout()).thenReturn(Duration.ofSeconds(30)); + try (MockedStatic trustStoreProviderMockedStatic = mockStatic(TrustStoreProvider.class)) { + final SdkAsyncHttpClient sdkAsyncHttpClient = createObjectUnderTest().createSdkAsyncHttpClient(openSearchSourceConfiguration); + assertThat(sdkAsyncHttpClient, notNullValue()); + trustStoreProviderMockedStatic.verify(() -> TrustStoreProvider.createTrustAllManager(), never()); + trustStoreProviderMockedStatic.verify(() -> TrustStoreProvider.createTrustManager(any(Path.class)), never()); + } + } + + @Test + void createSdkAsyncHttpClient_with_insecure_configuration_and_no_cert_path_trusts_all_managers() { + when(connectionConfiguration.getCertPath()).thenReturn(null); + when(connectionConfiguration.isInsecure()).thenReturn(true); + when(connectionConfiguration.getConnectTimeout()).thenReturn(Duration.ofSeconds(30)); + try (MockedStatic trustStoreProviderMockedStatic = mockStatic(TrustStoreProvider.class)) { + TrustManager[] mockTrustManagers = new TrustManager[] { mock(TrustManager.class) }; + trustStoreProviderMockedStatic.when(() -> TrustStoreProvider.createTrustAllManager()) + .thenReturn(mockTrustManagers); + final SdkAsyncHttpClient sdkAsyncHttpClient = createObjectUnderTest().createSdkAsyncHttpClient(openSearchSourceConfiguration); + assertThat(sdkAsyncHttpClient, notNullValue()); + trustStoreProviderMockedStatic.verify(() -> TrustStoreProvider.createTrustAllManager(), times(1)); + trustStoreProviderMockedStatic.verify(() -> TrustStoreProvider.createTrustManager(any(Path.class)), never()); + } + } + } diff --git a/data-prepper-plugins/opensearch/src/test/resources/test_keystore.jks b/data-prepper-plugins/opensearch/src/test/resources/test_keystore.jks new file mode 100644 index 0000000000..77002f6532 Binary files /dev/null and b/data-prepper-plugins/opensearch/src/test/resources/test_keystore.jks differ diff --git a/e2e-test/log/src/integrationTest/java/org/opensearch/dataprepper/integration/log/EndToEndBasicLogTest.java b/e2e-test/log/src/integrationTest/java/org/opensearch/dataprepper/integration/log/EndToEndBasicLogTest.java index 294af39288..2571aceaa9 100644 --- a/e2e-test/log/src/integrationTest/java/org/opensearch/dataprepper/integration/log/EndToEndBasicLogTest.java +++ b/e2e-test/log/src/integrationTest/java/org/opensearch/dataprepper/integration/log/EndToEndBasicLogTest.java @@ -132,6 +132,7 @@ private RestHighLevelClient prepareOpenSearchRestHighLevelClient() { Collections.singletonList("https://127.0.0.1:9200")); builder.withUsername("admin"); builder.withPassword("admin"); + builder.withInsecure(true); return builder.build().createClient(null); } diff --git a/e2e-test/log/src/integrationTest/java/org/opensearch/dataprepper/integration/log/ParallelGrokStringSubstituteLogTest.java b/e2e-test/log/src/integrationTest/java/org/opensearch/dataprepper/integration/log/ParallelGrokStringSubstituteLogTest.java index e4f7b87f8e..203597d7db 100644 --- a/e2e-test/log/src/integrationTest/java/org/opensearch/dataprepper/integration/log/ParallelGrokStringSubstituteLogTest.java +++ b/e2e-test/log/src/integrationTest/java/org/opensearch/dataprepper/integration/log/ParallelGrokStringSubstituteLogTest.java @@ -100,6 +100,7 @@ private RestHighLevelClient prepareOpenSearchRestHighLevelClient() { Collections.singletonList("https://127.0.0.1:9200")); builder.withUsername("admin"); builder.withPassword("admin"); + builder.withInsecure(true); final AwsCredentialsSupplier awsCredentialsSupplier = mock(AwsCredentialsSupplier.class); return builder.build().createClient(awsCredentialsSupplier); } diff --git a/e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline-date-pattern-index.yml b/e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline-date-pattern-index.yml index 7d1ecb150d..16026e682f 100644 --- a/e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline-date-pattern-index.yml +++ b/e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline-date-pattern-index.yml @@ -11,5 +11,6 @@ grok-pipeline: hosts: [ "https://node-0.example.com:9200" ] username: "admin" password: "admin" + insecure: true index: "test-grok-index-%{yyyy.MM.dd}" flush_timeout: 5000 diff --git a/e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline-with-aws-secrets.yml b/e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline-with-aws-secrets.yml index b05b774386..2246db5af5 100644 --- a/e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline-with-aws-secrets.yml +++ b/e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline-with-aws-secrets.yml @@ -17,5 +17,6 @@ grok-pipeline: hosts: [ "https://node-0.example.com:9200" ] username: "${{aws_secrets:opensearch-sink:username}}" password: "${{aws_secrets:opensearch-sink:password}}" + insecure: true index: "test-grok-index" flush_timeout: 5000 \ No newline at end of file diff --git a/e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline.yml b/e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline.yml index f9bbc0506c..1eee55f52e 100644 --- a/e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline.yml +++ b/e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline.yml @@ -12,5 +12,6 @@ grok-pipeline: hosts: [ "https://node-0.example.com:9200" ] username: "admin" password: "admin" + insecure: true index: "test-grok-index" flush_timeout: 5000 diff --git a/e2e-test/log/src/integrationTest/resources/parallel-grok-substitute-e2e-pipeline.yml b/e2e-test/log/src/integrationTest/resources/parallel-grok-substitute-e2e-pipeline.yml index 0d4ef4260e..3488e684ab 100644 --- a/e2e-test/log/src/integrationTest/resources/parallel-grok-substitute-e2e-pipeline.yml +++ b/e2e-test/log/src/integrationTest/resources/parallel-grok-substitute-e2e-pipeline.yml @@ -22,6 +22,7 @@ pipeline2: hosts: [ "https://node-0.example.com:9200" ] username: "admin" password: "admin" + insecure: true index: "test-substitute-index" flush_timeout: 5000 @@ -38,5 +39,6 @@ pipeline3: hosts: [ "https://node-0.example.com:9200" ] username: "admin" password: "admin" + insecure: true index: "test-grok-index" flush_timeout: 5000 diff --git a/e2e-test/peerforwarder/src/integrationTest/java/org/opensearch/dataprepper/integration/peerforwarder/EndToEndLogMetricsTest.java b/e2e-test/peerforwarder/src/integrationTest/java/org/opensearch/dataprepper/integration/peerforwarder/EndToEndLogMetricsTest.java index 397b0280de..f324d75980 100644 --- a/e2e-test/peerforwarder/src/integrationTest/java/org/opensearch/dataprepper/integration/peerforwarder/EndToEndLogMetricsTest.java +++ b/e2e-test/peerforwarder/src/integrationTest/java/org/opensearch/dataprepper/integration/peerforwarder/EndToEndLogMetricsTest.java @@ -181,6 +181,7 @@ private RestHighLevelClient prepareOpenSearchRestHighLevelClient() { Collections.singletonList("https://127.0.0.1:9200")); builder.withUsername("admin"); builder.withPassword("admin"); + builder.withInsecure(true); final AwsCredentialsSupplier awsCredentialsSupplier = mock(AwsCredentialsSupplier.class); return builder.build().createClient(awsCredentialsSupplier); } diff --git a/e2e-test/peerforwarder/src/integrationTest/java/org/opensearch/dataprepper/integration/peerforwarder/EndToEndPeerForwarderTest.java b/e2e-test/peerforwarder/src/integrationTest/java/org/opensearch/dataprepper/integration/peerforwarder/EndToEndPeerForwarderTest.java index ad24b3889b..a39a41cc5c 100644 --- a/e2e-test/peerforwarder/src/integrationTest/java/org/opensearch/dataprepper/integration/peerforwarder/EndToEndPeerForwarderTest.java +++ b/e2e-test/peerforwarder/src/integrationTest/java/org/opensearch/dataprepper/integration/peerforwarder/EndToEndPeerForwarderTest.java @@ -117,6 +117,7 @@ private RestHighLevelClient prepareOpenSearchRestHighLevelClient() { Collections.singletonList("https://127.0.0.1:9200")); builder.withUsername("admin"); builder.withPassword("admin"); + builder.withInsecure(true); final AwsCredentialsSupplier awsCredentialsSupplier = mock(AwsCredentialsSupplier.class); return builder.build().createClient(awsCredentialsSupplier); } diff --git a/e2e-test/peerforwarder/src/integrationTest/resources/aggregate-e2e-pipeline.yml b/e2e-test/peerforwarder/src/integrationTest/resources/aggregate-e2e-pipeline.yml index 476340ddd8..1c863d6241 100644 --- a/e2e-test/peerforwarder/src/integrationTest/resources/aggregate-e2e-pipeline.yml +++ b/e2e-test/peerforwarder/src/integrationTest/resources/aggregate-e2e-pipeline.yml @@ -12,5 +12,6 @@ aggregate-pipeline: hosts: [ "https://node-0.example.com:9200" ] username: "admin" password: "admin" + insecure: true index: "test-peer-forwarder-index" flush_timeout: 5000 \ No newline at end of file diff --git a/e2e-test/peerforwarder/src/integrationTest/resources/log-metrics-pipeline.yml b/e2e-test/peerforwarder/src/integrationTest/resources/log-metrics-pipeline.yml index f7a77414fc..b9dac3ddc3 100644 --- a/e2e-test/peerforwarder/src/integrationTest/resources/log-metrics-pipeline.yml +++ b/e2e-test/peerforwarder/src/integrationTest/resources/log-metrics-pipeline.yml @@ -16,5 +16,6 @@ aggregate-pipeline: hosts: [ "https://node-0.example.com:9200" ] username: "admin" password: "admin" + insecure: true index: "test-log-metrics-index" flush_timeout: 5000 diff --git a/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java b/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java index 6a8d033572..f7c956417a 100644 --- a/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java +++ b/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java @@ -115,6 +115,7 @@ public void testPipelineEndToEnd() { Collections.singletonList("https://127.0.0.1:9200")); builder.withUsername("admin"); builder.withPassword("admin"); + builder.withInsecure(true); final RestHighLevelClient restHighLevelClient = builder.build().createClient(null); // Wait for data to flow through pipeline and be indexed by ES await().atLeast(3, TimeUnit.SECONDS).atMost(20, TimeUnit.SECONDS).untilAsserted( diff --git a/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndServiceMapTest.java b/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndServiceMapTest.java index aead979e6c..ef9396d040 100644 --- a/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndServiceMapTest.java +++ b/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndServiceMapTest.java @@ -81,6 +81,7 @@ public void testPipelineEndToEnd() { Collections.singletonList("https://127.0.0.1:9200")); builder.withUsername("admin"); builder.withPassword("admin"); + builder.withInsecure(true); final AwsCredentialsSupplier awsCredentialsSupplier = mock(AwsCredentialsSupplier.class); final RestHighLevelClient restHighLevelClient = builder.build().createClient(awsCredentialsSupplier); diff --git a/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline-from-build.yml b/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline-from-build.yml index 55e7ac7423..c869a38a16 100644 --- a/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline-from-build.yml +++ b/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline-from-build.yml @@ -18,5 +18,6 @@ raw-pipeline: hosts: [ "https://node-0.example.com:9200" ] username: "admin" password: "admin" + insecure: true index_type: trace-analytics-raw flush_timeout: 5000 \ No newline at end of file diff --git a/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml b/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml index d09631885c..4bbbab2a83 100644 --- a/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml +++ b/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml @@ -18,5 +18,6 @@ raw-pipeline: hosts: [ "https://node-0.example.com:9200" ] username: "admin" password: "admin" + insecure: true index_type: trace-analytics-raw flush_timeout: 5000 diff --git a/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline.yml b/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline.yml index 2f036a7208..35d8d060d2 100644 --- a/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline.yml +++ b/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline.yml @@ -16,10 +16,12 @@ raw-pipeline: hosts: [ "https://node-0.example.com:9200" ] username: "admin" password: "admin" + insecure: true sink: - opensearch: hosts: [ "https://node-0.example.com:9200" ] username: "admin" password: "admin" + insecure: true index_type: trace-analytics-raw flush_timeout: 5000 diff --git a/e2e-test/trace/src/integrationTest/resources/service-map-e2e-pipeline.yml b/e2e-test/trace/src/integrationTest/resources/service-map-e2e-pipeline.yml index 5d934e0e95..326ee02cc8 100644 --- a/e2e-test/trace/src/integrationTest/resources/service-map-e2e-pipeline.yml +++ b/e2e-test/trace/src/integrationTest/resources/service-map-e2e-pipeline.yml @@ -18,5 +18,6 @@ service-map-pipeline: hosts: ["https://node-0.example.com:9200"] username: "admin" password: "admin" + insecure: true index_type: trace-analytics-service-map flush_timeout: 5000