diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index 9e5cc0599dfc4..7da83e10c8071 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -126,6 +126,8 @@ on: value: ${{ jobs.int_tests.outputs.nats }} nginx: value: ${{ jobs.int_tests.outputs.nginx }} + opensearch: + value: ${{ jobs.int_tests.outputs.opensearch }} opentelemetry: value: ${{ jobs.int_tests.outputs.opentelemetry }} postgres: @@ -346,6 +348,7 @@ jobs: mqtt: ${{ steps.filter.outputs.mqtt }} nats: ${{ steps.filter.outputs.nats }} nginx: ${{ steps.filter.outputs.nginx }} + opensearch: ${{ steps.filter.outputs.opensearch }} opentelemetry: ${{ steps.filter.outputs.opentelemetry }} postgres: ${{ steps.filter.outputs.postgres }} prometheus: ${{ steps.filter.outputs.prometheus }} @@ -413,6 +416,7 @@ jobs: "nats": ${{ steps.filter.outputs.nats }}, "nginx": ${{ steps.filter.outputs.nginx }}, "opentelemetry": ${{ steps.filter.outputs.opentelemetry }}, + "opensearch": ${{ steps.filter.outputs.opensearch }}, "postgres": ${{ steps.filter.outputs.postgres }}, "prometheus": ${{ steps.filter.outputs.prometheus }}, "pulsar": ${{ steps.filter.outputs.pulsar }}, diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index bfbe82668c652..1cb89338d4b97 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -128,6 +128,7 @@ jobs: "mongodb", "nats", "nginx", + "opensearch", "opentelemetry", "postgres", "prometheus", diff --git a/Cargo.toml b/Cargo.toml index 31da1df0dda70..f02bcce0d1c34 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1073,6 +1073,7 @@ datadog-traces-integration-tests = ["sources-datadog_agent", "sinks-datadog_trac docker-logs-integration-tests = ["sources-docker_logs", "unix"] doris-integration-tests = ["sinks-doris"] es-integration-tests = ["sinks-elasticsearch", "aws-core"] +opensearch-integration-tests = [] eventstoredb_metrics-integration-tests = ["sources-eventstoredb_metrics"] fluent-integration-tests = ["docker", "sources-fluent"] gcp-cloud-storage-integration-tests = ["sinks-gcp"] diff --git a/Makefile b/Makefile index bbe578ea375df..4fc92596cbbf4 100644 --- a/Makefile +++ b/Makefile @@ -279,7 +279,7 @@ test-behavior: test-behavior-transforms test-behavior-formats test-behavior-conf .PHONY: test-integration test-integration: ## Runs all integration tests test-integration: test-integration-amqp test-integration-appsignal test-integration-aws test-integration-axiom test-integration-azure test-integration-chronicle test-integration-clickhouse -test-integration: test-integration-databend test-integration-docker-logs test-integration-elasticsearch +test-integration: test-integration-databend test-integration-docker-logs test-integration-elasticsearch test-integration-opensearch test-integration: test-integration-eventstoredb test-integration-fluent test-integration-gcp test-integration-greptimedb test-integration-humio test-integration-http-client test-integration-influxdb test-integration: test-integration-kafka test-integration-logstash test-integration-loki test-integration-mongodb test-integration-nats test-integration: test-integration-nginx test-integration-opentelemetry test-integration-postgres test-integration-prometheus test-integration-pulsar diff --git a/changelog.d/20174_opensearch_version_detection.fix.md b/changelog.d/20174_opensearch_version_detection.fix.md new file mode 100644 index 0000000000000..a6e28a818fe16 --- /dev/null +++ b/changelog.d/20174_opensearch_version_detection.fix.md @@ -0,0 +1,3 @@ +The ElasticSearch sink chooses the compatible API version 7 when connecting to OpenSearch. + +authors: tstenner diff --git a/src/sinks/elasticsearch/common.rs b/src/sinks/elasticsearch/common.rs index c279a202cf841..3d4d4e847d414 100644 --- a/src/sinks/elasticsearch/common.rs +++ b/src/sinks/elasticsearch/common.rs @@ -151,6 +151,8 @@ impl ElasticsearchCommon { ElasticsearchApiVersion::V6 => 6, ElasticsearchApiVersion::V7 => 7, ElasticsearchApiVersion::V8 => 8, + // Opensearch 3.x is incompatible with Elasticsearch 8.x APIs + ElasticsearchApiVersion::OS3 => 7, ElasticsearchApiVersion::Auto => { match get_version( &base_url, @@ -381,6 +383,7 @@ async fn get_version( ) -> crate::Result { #[derive(Deserialize)] struct Version { + distribution: Option, number: Option, } #[derive(Deserialize)] @@ -405,6 +408,13 @@ async fn get_version( let mut body = body.collect().await?.aggregate(); let body = body.copy_to_bytes(body.remaining()); let ResponsePayload { version } = serde_json::from_slice(&body)?; + if let Some(version) = version.as_ref() + && let Some(distribution) = version.distribution.as_ref() + && distribution == "opensearch" + { + // OpenSearch versions 1.x and 2.x are compatible with Elasticsearch API version 7.x + return Ok(7); + } if let Some(version) = version && let Some(number) = version.number { diff --git a/src/sinks/elasticsearch/integration_tests.rs b/src/sinks/elasticsearch/integration_tests.rs index 68095e8d9d3e6..5a42745396d9a 100644 --- a/src/sinks/elasticsearch/integration_tests.rs +++ b/src/sinks/elasticsearch/integration_tests.rs @@ -31,6 +31,12 @@ use crate::{ tls::{self, TlsConfig}, }; +const INITIAL_ADMIN_PASSWORD: &str = "Vector2026!"; + +fn http_username() -> String { + std::env::var("ELASTICSEARCH_USERNAME").unwrap_or_else(|_| "elastic".into()) +} + fn aws_server() -> String { std::env::var("ELASTICSEARCH_AWS_ADDRESS").unwrap_or_else(|_| "http://localhost:4571".into()) } @@ -264,15 +270,14 @@ async fn auto_version_http() { .expect("Config error"); } -#[cfg(feature = "aws-core")] #[tokio::test] async fn auto_version_https() { trace_init(); let config = ElasticsearchConfig { auth: Some(ElasticsearchAuthConfig::Basic { - user: "elastic".to_string(), - password: "vector".to_string().into(), + user: http_username(), + password: INITIAL_ADMIN_PASSWORD.to_string().into(), }), endpoints: vec![https_server()], doc_type: "log_lines".to_string(), @@ -291,6 +296,7 @@ async fn auto_version_https() { } #[cfg(feature = "aws-core")] +#[cfg_attr(feature = "opensearch-integration-tests", ignore)] #[tokio::test] async fn auto_version_aws() { trace_init(); @@ -376,8 +382,8 @@ async fn insert_events_over_https() { run_insert_tests( ElasticsearchConfig { auth: Some(ElasticsearchAuthConfig::Basic { - user: "elastic".to_string(), - password: "vector".to_string().into(), + user: http_username(), + password: INITIAL_ADMIN_PASSWORD.to_string().into(), }), endpoints: vec![https_server()], doc_type: "log_lines".into(), @@ -396,6 +402,7 @@ async fn insert_events_over_https() { } #[cfg(feature = "aws-core")] +#[cfg_attr(feature = "opensearch-integration-tests", ignore)] #[tokio::test] async fn insert_events_on_aws() { trace_init(); @@ -422,6 +429,7 @@ async fn insert_events_on_aws() { } #[cfg(feature = "aws-core")] +#[cfg_attr(feature = "opensearch-integration-tests", ignore)] #[tokio::test] async fn insert_events_on_aws_with_compression() { trace_init(); @@ -487,8 +495,9 @@ async fn insert_events_with_failure_and_gzip_compression() { #[tokio::test] async fn insert_events_in_data_stream() { trace_init(); - let template_index = format!("my-template-{}", gen_index()); - let stream_index = format!("my-stream-{}", gen_index()); + let index = gen_index(); + let template_index = format!("my-template-{}", index); + let stream_index = format!("my-stream-{}", index); let cfg = ElasticsearchConfig { endpoints: vec![http_server()], @@ -497,6 +506,10 @@ async fn insert_events_in_data_stream() { index: Template::try_from(stream_index.clone()).expect("unable to parse template"), ..Default::default() }, + data_stream: Some(DataStreamConfig { + namespace: index, + ..Default::default() + }), batch: batch_settings(), ..Default::default() }; @@ -522,8 +535,8 @@ async fn distributed_insert_events() { // Assumes that behind https_server and http_server addresses lies the same server let mut config = ElasticsearchConfig { auth: Some(ElasticsearchAuthConfig::Basic { - user: "elastic".into(), - password: "vector".to_string().into(), + user: http_username(), + password: INITIAL_ADMIN_PASSWORD.to_string().into(), }), endpoints: vec![https_server(), http_server()], doc_type: "log_lines".into(), @@ -548,8 +561,8 @@ async fn distributed_insert_events_failover() { let mut config = ElasticsearchConfig { auth: Some(ElasticsearchAuthConfig::Basic { - user: "elastic".into(), - password: "vector".to_string().into(), + user: http_username(), + password: INITIAL_ADMIN_PASSWORD.to_string().into(), }), // Valid endpoints and some random non elasticsearch endpoint endpoints: vec![ @@ -627,14 +640,6 @@ async fn run_insert_tests_with_config( let common = ElasticsearchCommon::parse_single(config) .await .expect("Config error"); - let index = match config.mode { - // Data stream mode uses an index name generated from the event. - ElasticsearchMode::DataStream => format!( - "{}", - Utc::now().format(".ds-logs-generic-default-%Y.%m.%d-000001") - ), - ElasticsearchMode::Bulk => config.bulk.index.to_string(), - }; let base_url = common.base_url.clone(); let cx = SinkContext::default(); @@ -654,7 +659,10 @@ async fn run_insert_tests_with_config( let events = events.map(move |mut events| { if doit { events.iter_logs_mut().for_each(|log| { + // _type: 1 is invalid for elasticsearch log.insert("_type", 1); + // opensearch has a 512 character limit on _id + log.insert("_id", "invalid id because it is too long".repeat(16)); }); } doit = true; @@ -678,10 +686,19 @@ async fn run_insert_tests_with_config( // make sure writes are all visible flush(common).await.expect("Flushing writes failed"); + let index = match config.mode { + ElasticsearchMode::DataStream => config + .data_stream + .as_ref() + .map(|ds| format!("logs-generic-{}", ds.namespace)) + .unwrap(), + ElasticsearchMode::Bulk => config.bulk.index.to_string(), + }; + let client = create_http_client(); let mut response = client .get(format!("{base_url}/{index}/_search")) - .basic_auth("elastic", Some("vector")) + .basic_auth(http_username(), Some(INITIAL_ADMIN_PASSWORD)) .json(&json!({ "query": { "query_string": { "query": "*" } } })) @@ -770,7 +787,7 @@ async fn run_insert_tests_with_multiple_endpoints(config: &ElasticsearchConfig) for base_url in base_urls { if let Ok(response) = client .get(format!("{base_url}/{index}/_search")) - .basic_auth("elastic", Some("vector")) + .basic_auth(http_username(), Some(INITIAL_ADMIN_PASSWORD)) .json(&json!({ "query": { "query_string": { "query": "*" } } })) diff --git a/src/sinks/elasticsearch/mod.rs b/src/sinks/elasticsearch/mod.rs index 1466c415dbd1b..3f2484da4dd08 100644 --- a/src/sinks/elasticsearch/mod.rs +++ b/src/sinks/elasticsearch/mod.rs @@ -312,6 +312,9 @@ pub enum ElasticsearchApiVersion { V7, /// Use the Elasticsearch 8.x API. V8, + #[serde(rename = "os3")] + /// Use the Opensearch 3.x API + OS3, } #[derive(Debug, Snafu)] diff --git a/tests/data/ca/intermediate_server/certs/opensearch-secure-chain.cert.pem b/tests/data/ca/intermediate_server/certs/opensearch-secure-chain.cert.pem new file mode 100644 index 0000000000000..2b2dfbb27b136 --- /dev/null +++ b/tests/data/ca/intermediate_server/certs/opensearch-secure-chain.cert.pem @@ -0,0 +1,98 @@ +-----BEGIN CERTIFICATE----- +MIIFtzCCA5+gAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwajESMBAGA1UEAwwJVmVj +dG9yIENBMQ8wDQYDVQQLDAZWZWN0b3IxEDAOBgNVBAoMB0RhdGFkb2cxETAPBgNV +BAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazELMAkGA1UEBhMCVVMwHhcN +MjIwNjA3MjIyNzUzWhcNMzIwNjA0MjIyNzUzWjBrMQswCQYDVQQGEwJVUzERMA8G +A1UECAwITmV3IFlvcmsxEDAOBgNVBAoMB0RhdGFkb2cxDzANBgNVBAsMBlZlY3Rv +cjEmMCQGA1UEAwwdVmVjdG9yIEludGVybWVkaWF0ZSBTZXJ2ZXIgQ0EwggIiMA0G +CSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCy/mB0/ZwfgKrSZPQIFaGPtRA9xL2N +o2SsHndZ8k2XOCV225Hb2fzNH+o2WGNSjwmGjLP/uXb47KH0cHCAyFGzSjp++8/O +zoZaFiO0P5El02hQxmoabO3Cqu/N62EFsLfpSM828JM6YOn9p+WXUDn1+YPNoOOE +H142p4/RjFnXNHkzR3geXU4Pfi3KXDrMi8vK42lDqXPLPs6rhreBAfQ2dsYyqhz6 +tg6FzZuXxxzEYyYtNgGh+zTji99WCBMLbCmRcDurRjdTDO7m4O3PrwbGUy0xdLeb +HJiNGvUDCPH4bfwLiNqwVIZY38RBCAqbCnrqRhDaZIfAUev4mq3Kqh6KUeO/U7Vx +/5J5rL5ApREKOfWPATHMprBuEU2rs3N+MPBA04HoiFlu311urCxVEA1qsZCTkoCg +GHuDIVSU4E4hT4co95/J0to4zWgPlfPg1+cXyU8lAIMe7JdCGkG9cDe7Umw/GSbt +ZdoCMQZ6WyyiW2Hw+7sFD3V3VzYa5YA/rjKZRduPmGWKrs+mAa5J5pM2M22rrjbd +EpfTHWLS9s6cPN3/jxpCxn6Hv/KhIYRAcIterugag1+clvS1ajVjxBRavOxPBsf+ +hYdh7S5NTZnT98gjkc3yOuGQm7BPtXau+IYZRlWcB0dJ4/E2P69hmWQezSo9VVWh +5/K1RkbPvqTGZQIDAQABo2YwZDAdBgNVHQ4EFgQUPD06L8zVggN9mcRY8eHbNu+t +DUEwHwYDVR0jBBgwFoAURTWK6ARqnZkz8rktUc5PrtasIh8wEgYDVR0TAQH/BAgw +BgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBAGqaGBuL +2J6Na8RHx/GmSeuZFiVcWhmd/I9bVpeMSYHSZujA2nay6OGaUYs0Lq/G5OKgsuT9 +AIHnsn7VUE1zqoDfXac/K8sXlOig8if7rTb+06jgymaP1YSELg3R+pBsdkZnXVil +izh/9FvzoyV+QQlIhojqCIybVFgxa1XFHq4QCPhDfwkg+tp9RctfwNmWgsJ63H19 +RmxN+H2xIrySvObwXnB4j6D4wvgu468QXQMEuSsnLcIQFg6Zteqe8fixbqTiOTBf +Dk1k+EpB9VMEkIPvMdfa48vseXdBEe6Ma9zGuJC76q4q1ZapVLTvOUP5Y24khlgd +cj5tfP7o7yc6HqymfXAcD1lzP2JQhqaRxA4I18Nrd+aHi+G1EM2c3cicuD3n6Iw9 +9oqdCwmMfS25fv5cyA5B6hRusIZ9wRopTi7at+JHl0GIt/FelaTYI7kRmAqgakQe +oEKLpXcH8lRJW802DmXm7ka4eQzwxa7Ngyf8O+JOFtGO0+EshuLJovxiPl6IyLyG +NJ/dHq3ad+46YVManbHdyjHxgT5PSvJFkq0Yluvf44NIyP5QRTCAvfH76bu7hXgS +QoQj5t5ILn6meQRTR79r2iwpQTanPLTEdoZvmrE4TeUBev9BA5KpiPPA3i3ZF/oV +0EYorXCNri7M/jylGW7AuWvNUyaVR6xgxAn6 +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFujCCA6KgAwIBAgIJAKhPL9BkNaFGMA0GCSqGSIb3DQEBCwUAMGoxEjAQBgNV +BAMMCVZlY3RvciBDQTEPMA0GA1UECwwGVmVjdG9yMRAwDgYDVQQKDAdEYXRhZG9n +MREwDwYDVQQIDAhOZXcgWW9yazERMA8GA1UEBwwITmV3IFlvcmsxCzAJBgNVBAYT +AlVTMB4XDTIyMDYwNzIyMjc1MloXDTQyMDYwMjIyMjc1MlowajESMBAGA1UEAwwJ +VmVjdG9yIENBMQ8wDQYDVQQLDAZWZWN0b3IxEDAOBgNVBAoMB0RhdGFkb2cxETAP +BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazELMAkGA1UEBhMCVVMw +ggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC9c1T+NXTNmqiiV36NSEJt +7mo0cyv8Byk2ZGdC85vHBm45QDY5USoh0vgonzPpWgSMggPn1WbR0f1y+LBwXdlM ++ZyZh2RVVeUrSjJ88lLHVn4DfywpdDkwQaFj1VmOsj2I9rMMrgc5x5n1Hj7lwZ+t +uPVSAGmgKp4iFfzLph9r/rjP1TUAnVUComfTUVS+Gd7zoGPOc14cMJXG6g2P2aAU +P6dg5uQlTxRmagnlx7bwm3lRwv6LMtnAdnjwBDBxr933nucAnk21GgE92GejiO3Z +OwlzIdzBI23lPcWi5pq+vCTgAArNq24W1Ha+7Jn5QewNTGKFyyYAJetZAwCUR8QS +Ip++2GE2pNhaGqcV5u1Tbwl02eD6p2qRqjfgLxmb+aC6xfl0n9kiFGPZppjCqDEW +sw+gX66nf+qxZVRWpJon2kWcFvhTnLqoa3T3+9+KIeamz2lW6wxMnki/Co2EA1Wa +mmedaUUcRPCgMx9aCktRkMyH6bEY8/vfJ07juxUsszOc46T00Scmn6Vkuo9Uc3Kf +2Q2N6Wo4jtyAiMO4gAwq5kzzpBAhNgRfLHOb83r2gAUj2Y4Vln/UUR/KR8ZbJi4i +r1BjX16Lz3yblJXXb1lp4uZynlbHNaAevXyGlRqHddM2ykKtAX/vgJcZRGSvms11 +uce/cqzrzx60AhpLRma5CwIDAQABo2MwYTAdBgNVHQ4EFgQURTWK6ARqnZkz8rkt +Uc5PrtasIh8wHwYDVR0jBBgwFoAURTWK6ARqnZkz8rktUc5PrtasIh8wDwYDVR0T +AQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBAEf5 +TR3hq/DtSAmsYotu1lAWz/OlTpG+7AdqSOHB878X4ETN3xaQ+KWvSwvf0K70ZDTV +tFOTh/r43cpzPifPKd1P+2ctnQEzrBtAacvyETLq1ABRK9VJOtfJ6Xk5KZXPhKdY +t353PQgBgW8YzQ2adq2B7FtgIlX7f1DIndjcMZBbolETR6xt9QwB/UnPI7Mwt01T ++bCBhr1fWAbZ4YAMlQ0xRam4qUOTjxgfmePrmSrv4HO7cXHMsRMLiXk+BLcx959/ +K/B6xzpzn6366Eqnqlo/uDiMpo5ud2I/Snz5PduB6oLztPMEf/8RmkG5tpHXYdWr +tM64WqNGO+ikluIrrtYvtyZS4DfsLAMfMYZcxX/Uw56gHo0i2c8I6+6JvGWdvOJ0 +FjrsKeIQoRlV77z025kI4V9jKi3XNMEsAIH+W7KNSut0X80yX7SugvQGoe0GDkXu +0fy8hMC3uTN2LEycYFRRfoIeKPLi6OZFK0PdS2E15d8PEU3n3W4eBCPgMtmiOKLY +d8QNBC8XLAuBoK9R8luCJpOJWUcFXjLpjcDab4V2hKTuAs+GQyDh/Xx4wF1yHX0r +zIkyN0EkOD/SvD8X4uFaM4mdsAh+ucn4ryUV7i5PgvDM9z4InHAMAee1ebBl0U+h ++NzMWF5c5OwxD5o6/Wh1HopmzJiVNT2v9u0kHT/f +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFjjCCA3agAwIBAgICEAowDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCVVMx +ETAPBgNVBAgMCE5ldyBZb3JrMRAwDgYDVQQKDAdEYXRhZG9nMQ8wDQYDVQQLDAZW +ZWN0b3IxJjAkBgNVBAMMHVZlY3RvciBJbnRlcm1lZGlhdGUgU2VydmVyIENBMB4X +DTI2MDQwOTE5MjU0M1oXDTM2MDQwNjE5MjU0M1owcjELMAkGA1UEBhMCVVMxETAP +BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazEQMA4GA1UECgwHRGF0 +YWRvZzEPMA0GA1UECwwGVmVjdG9yMRowGAYDVQQDDBFvcGVuc2VhcmNoLXNlY3Vy +ZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM0RfZCpkjHw7iE3QoPD +L96COa6k1mxeWT+f/y6oc/BfGb0PHjQwxbVLdtoGpDPDha+PQUjZRtTSe2RicFSm +CN4QGMUUqRAs7sEVKWLibg9Sc7sftXJbTqNw7rB9rTIeSoqlVGCsJLok6iVsfX4L +dmHjMTrPA2PLWKqkQKAFccE8uY6uxWO77ohTy4DTUT5tRZpkGGJNohBiXtZg0UIE +wgH/aS2SzC7hXxniwgZCQ64l0yD2x43kdlPu+F3p0meyhQ2SUDnm0QrGkZqQ4X31 +iYim5tGoXR1m/NOtX4YSifX74W/PTrn7GpxCRvyyeLxsSl+Tr9B4yVOrZx3EDUtr +4E0CAwEAAaOCATMwggEvMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQDAgZAMDMG +CWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBTZXJ2ZXIgQ2VydGlmaWNh +dGUwHQYDVR0OBBYEFIn3xriMRcm2mwj3p6oyQ4K2pmGWMIGVBgNVHSMEgY0wgYqA +FDw9Oi/M1YIDfZnEWPHh2zbvrQ1BoW6kbDBqMRIwEAYDVQQDDAlWZWN0b3IgQ0Ex +DzANBgNVBAsMBlZlY3RvcjEQMA4GA1UECgwHRGF0YWRvZzERMA8GA1UECAwITmV3 +IFlvcmsxETAPBgNVBAcMCE5ldyBZb3JrMQswCQYDVQQGEwJVU4ICEAAwDgYDVR0P +AQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA0GCSqGSIb3DQEBCwUAA4IC +AQBRU9BpRpnLefDFYb6+7Aen9ceO6WY9DPggRDn9MxtzW6a5XgIYPVxax5w2vjbv +AbX60dgP/5WqIciTty3D/vvmRbqTrAqwFCkZqSGTDQcggO9WBHgujocoyeoskIVw +iEzrie6Ae34R+QrTFcLmR9QVeJzaszuPT+iU5TLFudbWxIUM+vx1l5dv5R0ZNctO +461Fovz7cX3sIBj4LVhzJ0Cf0oJBkcfR3QQYsj8euc0XODyTP1wqWeh+09/7NP+f +daCyGes8nZmb2c5uPwpA2avSFMC7hdn7iF92kLCSczc8ZRIqX+TuD9lBEpWyaqLi +tNn5XRKsdTEOftBO+kjgZQDJpPHTszdWi7ZUNypHOx3ZZZBB9l/vbaKM1f6GssNv +LsbNuYeAT+nrydJ0DIyCtsYpTAVp6e6zHpCvx6MIg70oWeexI6QyfafF+0+568KB +ABecFyAp+GuXQIDpu/mTlV0ejxJrBUIecbc7XD2g5IPkTT/PF8rHqka9mlchV/69 +KzIDVFXjcsBv3lLWM4/ztR6ujIxn/FEhdMP3XF4BR+HQXJ4oKJafr5BBlugJV/Ka +7f9FQMTBO+9b2fx5bK0QoVjlD4RY88fH3UBf5nsAtDrc2fzS8GUWV7A3mOmXf3Qp +/8RE3Q/Q4GNqcN4QQ02yM2ILH5G76WmuwEaIJTXSxS1wmg== +-----END CERTIFICATE----- diff --git a/tests/data/ca/intermediate_server/certs/opensearch-secure.cert.pem b/tests/data/ca/intermediate_server/certs/opensearch-secure.cert.pem new file mode 100644 index 0000000000000..d64b33a60beb1 --- /dev/null +++ b/tests/data/ca/intermediate_server/certs/opensearch-secure.cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFjjCCA3agAwIBAgICEAowDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCVVMx +ETAPBgNVBAgMCE5ldyBZb3JrMRAwDgYDVQQKDAdEYXRhZG9nMQ8wDQYDVQQLDAZW +ZWN0b3IxJjAkBgNVBAMMHVZlY3RvciBJbnRlcm1lZGlhdGUgU2VydmVyIENBMB4X +DTI2MDQwOTE5MjU0M1oXDTM2MDQwNjE5MjU0M1owcjELMAkGA1UEBhMCVVMxETAP +BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazEQMA4GA1UECgwHRGF0 +YWRvZzEPMA0GA1UECwwGVmVjdG9yMRowGAYDVQQDDBFvcGVuc2VhcmNoLXNlY3Vy +ZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM0RfZCpkjHw7iE3QoPD +L96COa6k1mxeWT+f/y6oc/BfGb0PHjQwxbVLdtoGpDPDha+PQUjZRtTSe2RicFSm +CN4QGMUUqRAs7sEVKWLibg9Sc7sftXJbTqNw7rB9rTIeSoqlVGCsJLok6iVsfX4L +dmHjMTrPA2PLWKqkQKAFccE8uY6uxWO77ohTy4DTUT5tRZpkGGJNohBiXtZg0UIE +wgH/aS2SzC7hXxniwgZCQ64l0yD2x43kdlPu+F3p0meyhQ2SUDnm0QrGkZqQ4X31 +iYim5tGoXR1m/NOtX4YSifX74W/PTrn7GpxCRvyyeLxsSl+Tr9B4yVOrZx3EDUtr +4E0CAwEAAaOCATMwggEvMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQDAgZAMDMG +CWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBTZXJ2ZXIgQ2VydGlmaWNh +dGUwHQYDVR0OBBYEFIn3xriMRcm2mwj3p6oyQ4K2pmGWMIGVBgNVHSMEgY0wgYqA +FDw9Oi/M1YIDfZnEWPHh2zbvrQ1BoW6kbDBqMRIwEAYDVQQDDAlWZWN0b3IgQ0Ex +DzANBgNVBAsMBlZlY3RvcjEQMA4GA1UECgwHRGF0YWRvZzERMA8GA1UECAwITmV3 +IFlvcmsxETAPBgNVBAcMCE5ldyBZb3JrMQswCQYDVQQGEwJVU4ICEAAwDgYDVR0P +AQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA0GCSqGSIb3DQEBCwUAA4IC +AQBRU9BpRpnLefDFYb6+7Aen9ceO6WY9DPggRDn9MxtzW6a5XgIYPVxax5w2vjbv +AbX60dgP/5WqIciTty3D/vvmRbqTrAqwFCkZqSGTDQcggO9WBHgujocoyeoskIVw +iEzrie6Ae34R+QrTFcLmR9QVeJzaszuPT+iU5TLFudbWxIUM+vx1l5dv5R0ZNctO +461Fovz7cX3sIBj4LVhzJ0Cf0oJBkcfR3QQYsj8euc0XODyTP1wqWeh+09/7NP+f +daCyGes8nZmb2c5uPwpA2avSFMC7hdn7iF92kLCSczc8ZRIqX+TuD9lBEpWyaqLi +tNn5XRKsdTEOftBO+kjgZQDJpPHTszdWi7ZUNypHOx3ZZZBB9l/vbaKM1f6GssNv +LsbNuYeAT+nrydJ0DIyCtsYpTAVp6e6zHpCvx6MIg70oWeexI6QyfafF+0+568KB +ABecFyAp+GuXQIDpu/mTlV0ejxJrBUIecbc7XD2g5IPkTT/PF8rHqka9mlchV/69 +KzIDVFXjcsBv3lLWM4/ztR6ujIxn/FEhdMP3XF4BR+HQXJ4oKJafr5BBlugJV/Ka +7f9FQMTBO+9b2fx5bK0QoVjlD4RY88fH3UBf5nsAtDrc2fzS8GUWV7A3mOmXf3Qp +/8RE3Q/Q4GNqcN4QQ02yM2ILH5G76WmuwEaIJTXSxS1wmg== +-----END CERTIFICATE----- diff --git a/tests/data/ca/intermediate_server/index.txt b/tests/data/ca/intermediate_server/index.txt index b2a0481603b78..897e77a2aa3db 100644 --- a/tests/data/ca/intermediate_server/index.txt +++ b/tests/data/ca/intermediate_server/index.txt @@ -7,3 +7,4 @@ V 320731200837Z 1006 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/C V 330412000039Z 1007 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=rabbitmq V 341228053159Z 1008 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=pulsar V 360301205253Z 1009 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=azurite +V 360406192543Z 100A unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=opensearch-secure diff --git a/tests/data/ca/intermediate_server/index.txt.old b/tests/data/ca/intermediate_server/index.txt.old index 0402ad0946130..b2a0481603b78 100644 --- a/tests/data/ca/intermediate_server/index.txt.old +++ b/tests/data/ca/intermediate_server/index.txt.old @@ -6,3 +6,4 @@ V 320613195253Z 1005 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/C V 320731200837Z 1006 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=dufs-https V 330412000039Z 1007 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=rabbitmq V 341228053159Z 1008 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=pulsar +V 360301205253Z 1009 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=azurite diff --git a/tests/data/ca/intermediate_server/private/opensearch-secure.key.pem b/tests/data/ca/intermediate_server/private/opensearch-secure.key.pem new file mode 100644 index 0000000000000..a668c3e1455d0 --- /dev/null +++ b/tests/data/ca/intermediate_server/private/opensearch-secure.key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDNEX2QqZIx8O4h +N0KDwy/egjmupNZsXlk/n/8uqHPwXxm9Dx40MMW1S3baBqQzw4Wvj0FI2UbU0ntk +YnBUpgjeEBjFFKkQLO7BFSli4m4PUnO7H7VyW06jcO6wfa0yHkqKpVRgrCS6JOol +bH1+C3Zh4zE6zwNjy1iqpECgBXHBPLmOrsVju+6IU8uA01E+bUWaZBhiTaIQYl7W +YNFCBMIB/2ktkswu4V8Z4sIGQkOuJdMg9seN5HZT7vhd6dJnsoUNklA55tEKxpGa +kOF99YmIpubRqF0dZvzTrV+GEon1++Fvz065+xqcQkb8sni8bEpfk6/QeMlTq2cd +xA1La+BNAgMBAAECggEAAPi4vhM3mJn6HtkqrQIjO2wlhtDMHUZTKOyCon900So6 +Ln2gWIH0pde+L+lBJOejvgGxPv2cisyWfhLDUntY5mhwHZrr9CIqXWrSv9+vws3S +s4vCadDJ7hdOYjKuO+MN2ltPK4gVQnaDT3sUmyDFxk/zv0IU04e2m1Rc5AUylUZI +snEBu61wdqv2X1BtxxDvegfAF2UtD8kyyWNrLjXWq94T5O3xj+WOd1pJs73Eq5YA +Z75EINYWmb5hAhDnTZOkXqsnvx3rlO/LQxRFGoqZ26cTmIkiVGcT/hy5JaqafCiT +g+nsMMFfel+j6cQPh4wSxbZBtJ5lxzRHhYYVnRtHUQKBgQD8EGBaHx4nh7fgW4Ex +5fHkBEYLDCnu7cuoe1ulSK6xtWjqHHdYTFuFHUl57Kx/npfxzpdEO0BMRZzj+RXB +JSY2udSkfT6D/+vpIQrrd4D7xz3bO7vuhKhUA3kTncG6NMsOKlAO7mZ/m/7RoMka +WmckPEc+hel/p8u52SOFa0d1CQKBgQDQRT/YxYE3c69B5mmLWzEkmnb+ycII6XxE +6yJiSHhghAjh/4PBhG9KmkbJ7lMOYkW+53ZKwwD4UEuwQ+wVz5QuvapdmXrKexXb +SNpRspomfJ7X9pwoEtfkblNprfvD/vnaszQiyrQnyIOH/LTFWoPa4BRyN+yJ5Yoo +RwNayLvGJQKBgFd4+2bq+5TuRUJHnvsI+DuPvBrjWr3+qblLLc+a/HzRHpp3zHbI +8M2uDyyPlIEpMUOJ1PaptTOCyOJYM1N2p98/9+opSIOYUW7yt45lR06Yov+sqBgq +0xtspRzIl00eOwUsa+Hym4FXSkPHD2Lk0nxBqtuorxJM0acqEmifhQm5AoGBAJ8G +kVa6kAzAR2zBO4vNyLtTOlMXyRG2/k09YRa8kTXY+1NdH8SOk8/Z9v1O/Zo5DVpU +MC2rsjzoSrK/Wi/ZPSNnbQNlIq4ix7CVcyX4gt2qQDtUsS2kHbbtSsGhuGCSe0Mz +nDaIooDSCSS3n2wlzhI9ET+ZVgKBAzn35ybVqRMFAoGBAPVUiuuyCCAjh6lY9+LA +3vCZgnUAOxTEMRGE0b0bE3266mIp6dTyJBm/TCkm7MG+u8GhmmddL/UlcgOTV+G8 +tEZHYZMRy2Cj9B+FrzIoAnpQv1Z0RsfkDV6pZ6weCcigF1BJWziKzvP24hBXiont +egkOLFdymlnzak0dafwCsnep +-----END PRIVATE KEY----- diff --git a/tests/data/ca/intermediate_server/private/opensearch-secure.p12 b/tests/data/ca/intermediate_server/private/opensearch-secure.p12 new file mode 100644 index 0000000000000..da3edfa9b761c Binary files /dev/null and b/tests/data/ca/intermediate_server/private/opensearch-secure.p12 differ diff --git a/tests/data/ca/intermediate_server/serial b/tests/data/ca/intermediate_server/serial index 4e75b247d2247..3bc64e9ec5f89 100644 --- a/tests/data/ca/intermediate_server/serial +++ b/tests/data/ca/intermediate_server/serial @@ -1 +1 @@ -100A +100B diff --git a/tests/data/ca/intermediate_server/serial.old b/tests/data/ca/intermediate_server/serial.old index 6cb3869343bf8..4e75b247d2247 100644 --- a/tests/data/ca/intermediate_server/serial.old +++ b/tests/data/ca/intermediate_server/serial.old @@ -1 +1 @@ -1009 +100A diff --git a/tests/integration/elasticsearch/config/compose.yaml b/tests/integration/elasticsearch/config/compose.yaml index cb6d0bcc2f392..d1082d0fdda41 100644 --- a/tests/integration/elasticsearch/config/compose.yaml +++ b/tests/integration/elasticsearch/config/compose.yaml @@ -21,7 +21,7 @@ services: - xpack.security.transport.ssl.enabled=true - xpack.security.transport.ssl.certificate=certs/intermediate_server/certs/elasticsearch-secure-chain.cert.pem - xpack.security.transport.ssl.key=certs/intermediate_server/private/elasticsearch-secure.key.pem - - ELASTIC_PASSWORD=vector + - ELASTIC_PASSWORD=Vector2026! - ES_JAVA_OPTS=-Xms400m -Xmx400m volumes: - ../../../data/ca:/usr/share/elasticsearch/config/certs:ro diff --git a/tests/integration/elasticsearch/config/test.yaml b/tests/integration/elasticsearch/config/test.yaml index 984932cb08540..899231b6a5da0 100644 --- a/tests/integration/elasticsearch/config/test.yaml +++ b/tests/integration/elasticsearch/config/test.yaml @@ -9,6 +9,7 @@ env: ELASTICSEARCH_AWS_ADDRESS: http://localstack:4571 ELASTICSEARCH_HTTPS_ADDRESS: https://elasticsearch-secure:9200 ELASTICSEARCH_HTTP_ADDRESS: http://elasticsearch:9200 + ELASTICSEARCH_USERNAME: elastic matrix: version: [7.13.1] diff --git a/tests/integration/opensearch/config/compose.yaml b/tests/integration/opensearch/config/compose.yaml new file mode 100644 index 0000000000000..af4e67b2206b7 --- /dev/null +++ b/tests/integration/opensearch/config/compose.yaml @@ -0,0 +1,27 @@ +services: + opensearch: + image: docker.io/opensearchproject/opensearch:${CONFIG_VERSION} + environment: + - discovery.type=single-node + - DISABLE_SECURITY_PLUGIN=true + - DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI=true + - OPENSEARCH_JAVA_OPTS=-Xms400m -Xmx400m + opensearch-secure: + image: docker.io/opensearchproject/opensearch:${CONFIG_VERSION} + environment: + - discovery.type=single-node + - http.port=9201 + - plugins.anomaly_detection.enabled=false + - plugins.security.ssl.http.enabled=true + - plugins.security.ssl.http.keystore_type=PKCS12 + - plugins.security.ssl.http.keystore_password=NOPASS + - plugins.security.ssl.http.keystore_filepath=certs/intermediate_server/private/opensearch-secure.p12 + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=Vector2026! + - DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI=true + - OPENSEARCH_JAVA_OPTS=-Xms400m -Xmx400m + volumes: + - ../../../data/ca:/usr/share/opensearch/config/certs:ro +networks: + default: + name: ${VECTOR_NETWORK} + external: true diff --git a/tests/integration/opensearch/config/test.yaml b/tests/integration/opensearch/config/test.yaml new file mode 100644 index 0000000000000..00caaa6b72f4b --- /dev/null +++ b/tests/integration/opensearch/config/test.yaml @@ -0,0 +1,21 @@ +features: + - es-integration-tests + - opensearch-integration-tests + +test_filter: "::elasticsearch::integration_tests::" + +env: + ELASTICSEARCH_HTTP_ADDRESS: http://opensearch:9200 + ELASTICSEARCH_HTTPS_ADDRESS: https://opensearch-secure:9201 + ELASTICSEARCH_USERNAME: admin + +matrix: + version: [3.6.0] + +# changes to these files/paths will invoke the integration test in CI +# expressions are evaluated using https://github.com/micromatch/picomatch +paths: + - "src/sinks/elasticsearch/**" + - "src/sinks/util/**" + - "tests/data/ca**" + - "tests/integration/opensearch/**" diff --git a/website/cue/reference/components/sinks/generated/elasticsearch.cue b/website/cue/reference/components/sinks/generated/elasticsearch.cue index b2659900df149..911087c4fa889 100644 --- a/website/cue/reference/components/sinks/generated/elasticsearch.cue +++ b/website/cue/reference/components/sinks/generated/elasticsearch.cue @@ -48,9 +48,10 @@ generated: components: sinks: elasticsearch: configuration: { [es_version]: https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-state.html#cluster-state-api-path-params """ - v6: "Use the Elasticsearch 6.x API." - v7: "Use the Elasticsearch 7.x API." - v8: "Use the Elasticsearch 8.x API." + os3: "Use the Opensearch 3.x API" + v6: "Use the Elasticsearch 6.x API." + v7: "Use the Elasticsearch 7.x API." + v8: "Use the Elasticsearch 8.x API." } } }