diff --git a/.gitignore b/.gitignore index 418934fe46..0935ddb9e5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Gradle directories build +bin .gradle gradle/tools diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java index f842eb88e6..279b49ec04 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java @@ -434,6 +434,8 @@ private Map readIndexTemplate(final String templateFile, final I templateURL = loadExistingTemplate(templateType, IndexConstants.RAW_STANDARD_TEMPLATE_FILE); } else if (indexType.equals(IndexType.TRACE_ANALYTICS_SERVICE_MAP)) { templateURL = loadExistingTemplate(templateType, IndexConstants.SERVICE_MAP_DEFAULT_TEMPLATE_FILE); + } else if (indexType.equals(IndexType.OTEL_APM_SERVICE_MAP)) { + templateURL = loadExistingTemplate(templateType, IndexConstants.OTEL_APM_SERVICE_MAP_TEMPLATE_FILE); } else if (indexType.equals(IndexType.LOG_ANALYTICS)) { templateURL = loadExistingTemplate(templateType, IndexConstants.LOGS_DEFAULT_TEMPLATE_FILE); } else if (indexType.equals(IndexType.LOG_ANALYTICS_PLAIN)) { diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConstants.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConstants.java index 4a27cf2baf..2df48191e7 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConstants.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConstants.java @@ -37,10 +37,15 @@ public class IndexConstants { public static final String ISM_ROLLOVER_ALIAS_SETTING = "opendistro.index_state_management.rollover_alias"; // TODO: extract out version number into version enum public static final String SERVICE_MAP_DEFAULT_TEMPLATE_FILE = "otel-v1-apm-service-map-index-template.json"; + public static final String OTEL_APM_SERVICE_MAP_TEMPLATE_FILE = "otel-v2-apm-service-map-index-template.json"; + public static final String OTEL_APM_SERVICE_MAP_ISM_POLICY = "otel-v2-apm-service-map-policy"; + public static final String OTEL_APM_SERVICE_MAP_ISM_FILE_NO_ISM_TEMPLATE = "otel-v2-apm-service-map-policy-no-ism-template.json"; + public static final String OTEL_APM_SERVICE_MAP_ISM_FILE_WITH_ISM_TEMPLATE = "otel-v2-apm-service-map-policy-with-ism-template.json"; static { // TODO: extract out version number into version enum TYPE_TO_DEFAULT_ALIAS.put(IndexType.TRACE_ANALYTICS_SERVICE_MAP, "otel-v1-apm-service-map"); + TYPE_TO_DEFAULT_ALIAS.put(IndexType.OTEL_APM_SERVICE_MAP, "otel-v2-apm-service-map"); TYPE_TO_DEFAULT_ALIAS.put(IndexType.TRACE_ANALYTICS_RAW, "otel-v1-apm-span"); TYPE_TO_DEFAULT_ALIAS.put(IndexType.TRACE_ANALYTICS_RAW_PLAIN, "otel-v1-apm-span"); TYPE_TO_DEFAULT_ALIAS.put(IndexType.LOG_ANALYTICS, "logs-otel-v1"); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexManagerFactory.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexManagerFactory.java index dc4ee94d09..3781bd9b06 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexManagerFactory.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexManagerFactory.java @@ -59,6 +59,10 @@ public final IndexManager getIndexManager(final IndexType indexType, indexManager = new TraceAnalyticsServiceMapIndexManager( restHighLevelClient, openSearchClient, openSearchSinkConfiguration, clusterSettingsParser, templateStrategy, indexAlias); break; + case OTEL_APM_SERVICE_MAP: + indexManager = new OTelAPMServiceMapIndexManager( + restHighLevelClient, openSearchClient, openSearchSinkConfiguration, clusterSettingsParser, templateStrategy, indexAlias); + break; case LOG_ANALYTICS: case LOG_ANALYTICS_PLAIN: indexManager = new LogAnalyticsIndexManager( @@ -151,6 +155,24 @@ public TraceAnalyticsServiceMapIndexManager(final RestHighLevelClient restHighLe } } + private static class OTelAPMServiceMapIndexManager extends AbstractIndexManager { + + public OTelAPMServiceMapIndexManager(final RestHighLevelClient restHighLevelClient, + final OpenSearchClient openSearchClient, + final OpenSearchSinkConfiguration openSearchSinkConfiguration, + final ClusterSettingsParser clusterSettingsParser, + final TemplateStrategy templateStrategy, + final String indexAlias) { + super(restHighLevelClient, openSearchClient, openSearchSinkConfiguration, clusterSettingsParser, templateStrategy, indexAlias); + this.ismPolicyManagementStrategy = new IsmPolicyManagement( + openSearchClient, + restHighLevelClient, + IndexConstants.OTEL_APM_SERVICE_MAP_ISM_POLICY, + IndexConstants.OTEL_APM_SERVICE_MAP_ISM_FILE_WITH_ISM_TEMPLATE, + IndexConstants.OTEL_APM_SERVICE_MAP_ISM_FILE_NO_ISM_TEMPLATE); + } + } + private static class LogAnalyticsIndexManager extends AbstractIndexManager { public LogAnalyticsIndexManager(final RestHighLevelClient restHighLevelClient, diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexType.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexType.java index c00e5c6415..3977dc398c 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexType.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexType.java @@ -15,6 +15,7 @@ public enum IndexType { TRACE_ANALYTICS_RAW("trace-analytics-raw"), TRACE_ANALYTICS_RAW_PLAIN("trace-analytics-plain-raw"), TRACE_ANALYTICS_SERVICE_MAP("trace-analytics-service-map"), + OTEL_APM_SERVICE_MAP("otel-v2-apm-service-map"), LOG_ANALYTICS("log-analytics"), LOG_ANALYTICS_PLAIN("log-analytics-plain"), METRIC_ANALYTICS("metric-analytics"), diff --git a/data-prepper-plugins/opensearch/src/main/resources/index-template/otel-v2-apm-service-map-index-template.json b/data-prepper-plugins/opensearch/src/main/resources/index-template/otel-v2-apm-service-map-index-template.json new file mode 100644 index 0000000000..a66dc22ce9 --- /dev/null +++ b/data-prepper-plugins/opensearch/src/main/resources/index-template/otel-v2-apm-service-map-index-template.json @@ -0,0 +1,146 @@ +{ + "version": 0, + "index_patterns": ["otel-v2-apm-service-map*"], + "template": { + "mappings": { + "date_detection": false, + "dynamic_templates": [ + { + "long_service_group_by_attributes": { + "path_match": "service.groupByAttributes.*", + "match_mapping_type": "long", + "mapping": { + "type": "long" + } + } + }, + { + "double_service_group_by_attributes": { + "path_match": "service.groupByAttributes.*", + "match_mapping_type": "double", + "mapping": { + "type": "double" + } + } + }, + { + "string_service_group_by_attributes": { + "path_match": "service.groupByAttributes.*", + "match_mapping_type": "string", + "mapping": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + { + "long_operation_remote_service_group_by_attributes": { + "path_match": "operation.remoteService.groupByAttributes.*", + "match_mapping_type": "long", + "mapping": { + "type": "long" + } + } + }, + { + "double_operation_remote_service_group_by_attributes": { + "path_match": "operation.remoteService.groupByAttributes.*", + "match_mapping_type": "double", + "mapping": { + "type": "double" + } + } + }, + { + "string_operation_remote_service_group_by_attributes": { + "path_match": "operation.remoteService.groupByAttributes.*", + "match_mapping_type": "string", + "mapping": { + "ignore_above": 256, + "type": "keyword" + } + } + } + ], + "_source": { + "enabled": true + }, + "properties": { + "eventType": { + "type": "keyword" + }, + "hashCode": { + "type": "keyword" + }, + "operation": { + "properties": { + "name": { + "type": "keyword" + }, + "remoteOperationName": { + "type": "keyword" + }, + "remoteService": { + "properties": { + "groupByAttributes": { + "type": "object", + "dynamic": "true" + }, + "keyAttributes": { + "properties": { + "environment": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + } + } + } + } + }, + "remoteService": { + "properties": { + "groupByAttributes": { + "type": "object", + "dynamic": "true" + }, + "keyAttributes": { + "properties": { + "environment": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + } + } + }, + "service": { + "properties": { + "groupByAttributes": { + "type": "object", + "dynamic": "true" + }, + "keyAttributes": { + "properties": { + "environment": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + } + } + }, + "timestamp": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + } + } + } + } +} diff --git a/data-prepper-plugins/opensearch/src/main/resources/otel-v2-apm-service-map-index-template.json b/data-prepper-plugins/opensearch/src/main/resources/otel-v2-apm-service-map-index-template.json new file mode 100644 index 0000000000..bcb7214a89 --- /dev/null +++ b/data-prepper-plugins/opensearch/src/main/resources/otel-v2-apm-service-map-index-template.json @@ -0,0 +1,143 @@ +{ + "version": 0, + "mappings": { + "date_detection": false, + "dynamic_templates": [ + { + "long_service_group_by_attributes": { + "path_match": "service.groupByAttributes.*", + "match_mapping_type": "long", + "mapping": { + "type": "long" + } + } + }, + { + "double_service_group_by_attributes": { + "path_match": "service.groupByAttributes.*", + "match_mapping_type": "double", + "mapping": { + "type": "double" + } + } + }, + { + "string_service_group_by_attributes": { + "path_match": "service.groupByAttributes.*", + "match_mapping_type": "string", + "mapping": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + { + "long_operation_remote_service_group_by_attributes": { + "path_match": "operation.remoteService.groupByAttributes.*", + "match_mapping_type": "long", + "mapping": { + "type": "long" + } + } + }, + { + "double_operation_remote_service_group_by_attributes": { + "path_match": "operation.remoteService.groupByAttributes.*", + "match_mapping_type": "double", + "mapping": { + "type": "double" + } + } + }, + { + "string_operation_remote_service_group_by_attributes": { + "path_match": "operation.remoteService.groupByAttributes.*", + "match_mapping_type": "string", + "mapping": { + "ignore_above": 256, + "type": "keyword" + } + } + } + ], + "_source": { + "enabled": true + }, + "properties": { + "eventType": { + "type": "keyword" + }, + "hashCode": { + "type": "keyword" + }, + "operation": { + "properties": { + "name": { + "type": "keyword" + }, + "remoteOperationName": { + "type": "keyword" + }, + "remoteService": { + "properties": { + "groupByAttributes": { + "type": "object", + "dynamic": "true" + }, + "keyAttributes": { + "properties": { + "environment": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + } + } + } + } + }, + "remoteService": { + "properties": { + "groupByAttributes": { + "type": "object", + "dynamic": "true" + }, + "keyAttributes": { + "properties": { + "environment": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + } + } + }, + "service": { + "properties": { + "groupByAttributes": { + "type": "object", + "dynamic": "true" + }, + "keyAttributes": { + "properties": { + "environment": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + } + } + }, + "timestamp": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + } + } + } +} diff --git a/data-prepper-plugins/opensearch/src/main/resources/otel-v2-apm-service-map-policy-no-ism-template.json b/data-prepper-plugins/opensearch/src/main/resources/otel-v2-apm-service-map-policy-no-ism-template.json new file mode 100644 index 0000000000..58436521a2 --- /dev/null +++ b/data-prepper-plugins/opensearch/src/main/resources/otel-v2-apm-service-map-policy-no-ism-template.json @@ -0,0 +1,19 @@ +{ + "policy": { + "description": "Managing OpenTelemetry APM service map indices", + "default_state": "current_write_index", + "states": [ + { + "name": "current_write_index", + "actions": [ + { + "rollover": { + "min_size": "50gb", + "min_index_age": "24h" + } + } + ] + } + ] + } +} diff --git a/data-prepper-plugins/opensearch/src/main/resources/otel-v2-apm-service-map-policy-with-ism-template.json b/data-prepper-plugins/opensearch/src/main/resources/otel-v2-apm-service-map-policy-with-ism-template.json new file mode 100644 index 0000000000..5b95252576 --- /dev/null +++ b/data-prepper-plugins/opensearch/src/main/resources/otel-v2-apm-service-map-policy-with-ism-template.json @@ -0,0 +1,22 @@ +{ + "policy": { + "description": "Managing OpenTelemetry APM service map indices", + "default_state": "current_write_index", + "states": [ + { + "name": "current_write_index", + "actions": [ + { + "rollover": { + "min_size": "50gb", + "min_index_age": "24h" + } + } + ] + } + ], + "ism_template": { + "index_patterns": ["otel-v2-apm-service-map-*"] + } + } +} diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexTypeTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexTypeTests.java index d325a98f11..a1c0a7067f 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexTypeTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexTypeTests.java @@ -28,6 +28,7 @@ public void getByValue() { assertEquals(Optional.of(IndexType.TRACE_ANALYTICS_RAW), IndexType.getByValue("trace-analytics-raw")); assertEquals(Optional.of(IndexType.TRACE_ANALYTICS_RAW_PLAIN), IndexType.getByValue("trace-analytics-plain-raw")); assertEquals(Optional.of(IndexType.TRACE_ANALYTICS_SERVICE_MAP), IndexType.getByValue("trace-analytics-service-map")); + assertEquals(Optional.of(IndexType.OTEL_APM_SERVICE_MAP), IndexType.getByValue("otel-v2-apm-service-map")); assertEquals(Optional.of(IndexType.LOG_ANALYTICS), IndexType.getByValue("log-analytics")); assertEquals(Optional.of(IndexType.LOG_ANALYTICS_PLAIN), IndexType.getByValue("log-analytics-plain")); assertEquals(Optional.of(IndexType.METRIC_ANALYTICS), IndexType.getByValue("metric-analytics")); @@ -36,7 +37,7 @@ public void getByValue() { @Test public void getIndexTypeValues() { - assertEquals("[trace-analytics-raw, trace-analytics-plain-raw, trace-analytics-service-map, log-analytics, log-analytics-plain, metric-analytics, metric-analytics-plain, custom, management_disabled]", IndexType.getIndexTypeValues()); + assertEquals("[trace-analytics-raw, trace-analytics-plain-raw, trace-analytics-service-map, otel-v2-apm-service-map, log-analytics, log-analytics-plain, metric-analytics, metric-analytics-plain, custom, management_disabled]", IndexType.getIndexTypeValues()); } @ParameterizedTest