Skip to content

Otel apm OpenSearch template #6435

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
san81 wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Otel apm OpenSearch template #6435

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f1a7299
Otel apm v2 service map index template ism policy files added
san81 Jan 28, 2026
700b04d
removing hashCode as the default documentId
san81 Jan 28, 2026
39af124
test case fix
san81 Jan 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# Gradle directories
build
bin
.gradle
gradle/tools

Expand Down
2 changes: 2 additions & 0 deletions ...ain/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -434,6 +434,8 @@ private Map<String, Object> readIndexTemplate(final String templateFile, final I
templateURL = loadExistingTemplate(templateType, IndexConstants.RAW_STANDARD_TEMPLATE_FILE);
} else if (indexType.equals(IndexType.TRACE_ANALYTICS_SERVICE_MAP)) {
templateURL = loadExistingTemplate(templateType, IndexConstants.SERVICE_MAP_DEFAULT_TEMPLATE_FILE);
} else if (indexType.equals(IndexType.OTEL_APM_SERVICE_MAP)) {
templateURL = loadExistingTemplate(templateType, IndexConstants.OTEL_APM_SERVICE_MAP_TEMPLATE_FILE);
} else if (indexType.equals(IndexType.LOG_ANALYTICS)) {
templateURL = loadExistingTemplate(templateType, IndexConstants.LOGS_DEFAULT_TEMPLATE_FILE);
} else if (indexType.equals(IndexType.LOG_ANALYTICS_PLAIN)) {
Expand Down
5 changes: 5 additions & 0 deletions ...rc/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,15 @@ public class IndexConstants {
public static final String ISM_ROLLOVER_ALIAS_SETTING = "opendistro.index_state_management.rollover_alias";
// TODO: extract out version number into version enum
public static final String SERVICE_MAP_DEFAULT_TEMPLATE_FILE = "otel-v1-apm-service-map-index-template.json";
public static final String OTEL_APM_SERVICE_MAP_TEMPLATE_FILE = "otel-v2-apm-service-map-index-template.json";
public static final String OTEL_APM_SERVICE_MAP_ISM_POLICY = "otel-v2-apm-service-map-policy";
public static final String OTEL_APM_SERVICE_MAP_ISM_FILE_NO_ISM_TEMPLATE = "otel-v2-apm-service-map-policy-no-ism-template.json";
public static final String OTEL_APM_SERVICE_MAP_ISM_FILE_WITH_ISM_TEMPLATE = "otel-v2-apm-service-map-policy-with-ism-template.json";

static {
// TODO: extract out version number into version enum
TYPE_TO_DEFAULT_ALIAS.put(IndexType.TRACE_ANALYTICS_SERVICE_MAP, "otel-v1-apm-service-map");
TYPE_TO_DEFAULT_ALIAS.put(IndexType.OTEL_APM_SERVICE_MAP, "otel-v2-apm-service-map");
TYPE_TO_DEFAULT_ALIAS.put(IndexType.TRACE_ANALYTICS_RAW, "otel-v1-apm-span");
TYPE_TO_DEFAULT_ALIAS.put(IndexType.TRACE_ANALYTICS_RAW_PLAIN, "otel-v1-apm-span");
TYPE_TO_DEFAULT_ALIAS.put(IndexType.LOG_ANALYTICS, "logs-otel-v1");
Expand Down
22 changes: 22 additions & 0 deletions ...in/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexManagerFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,10 @@ public final IndexManager getIndexManager(final IndexType indexType,
indexManager = new TraceAnalyticsServiceMapIndexManager(
restHighLevelClient, openSearchClient, openSearchSinkConfiguration, clusterSettingsParser, templateStrategy, indexAlias);
break;
case OTEL_APM_SERVICE_MAP:
indexManager = new OTelAPMServiceMapIndexManager(
restHighLevelClient, openSearchClient, openSearchSinkConfiguration, clusterSettingsParser, templateStrategy, indexAlias);
break;
case LOG_ANALYTICS:
case LOG_ANALYTICS_PLAIN:
indexManager = new LogAnalyticsIndexManager(
Expand Down Expand Up @@ -151,6 +155,24 @@ public TraceAnalyticsServiceMapIndexManager(final RestHighLevelClient restHighLe
}
}

private static class OTelAPMServiceMapIndexManager extends AbstractIndexManager {

public OTelAPMServiceMapIndexManager(final RestHighLevelClient restHighLevelClient,
final OpenSearchClient openSearchClient,
final OpenSearchSinkConfiguration openSearchSinkConfiguration,
final ClusterSettingsParser clusterSettingsParser,
final TemplateStrategy templateStrategy,
final String indexAlias) {
super(restHighLevelClient, openSearchClient, openSearchSinkConfiguration, clusterSettingsParser, templateStrategy, indexAlias);
this.ismPolicyManagementStrategy = new IsmPolicyManagement(
openSearchClient,
restHighLevelClient,
IndexConstants.OTEL_APM_SERVICE_MAP_ISM_POLICY,
IndexConstants.OTEL_APM_SERVICE_MAP_ISM_FILE_WITH_ISM_TEMPLATE,
IndexConstants.OTEL_APM_SERVICE_MAP_ISM_FILE_NO_ISM_TEMPLATE);
}
}

private static class LogAnalyticsIndexManager extends AbstractIndexManager {

public LogAnalyticsIndexManager(final RestHighLevelClient restHighLevelClient,
Expand Down
1 change: 1 addition & 0 deletions ...rch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ public enum IndexType {
TRACE_ANALYTICS_RAW("trace-analytics-raw"),
TRACE_ANALYTICS_RAW_PLAIN("trace-analytics-plain-raw"),
TRACE_ANALYTICS_SERVICE_MAP("trace-analytics-service-map"),
OTEL_APM_SERVICE_MAP("otel-v2-apm-service-map"),
LOG_ANALYTICS("log-analytics"),
LOG_ANALYTICS_PLAIN("log-analytics-plain"),
METRIC_ANALYTICS("metric-analytics"),
Expand Down
146 changes: 146 additions & 0 deletions .../opensearch/src/main/resources/index-template/otel-v2-apm-service-map-index-template.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
{
"version": 0,
"index_patterns": ["otel-v2-apm-service-map*"],
"template": {
"mappings": {
"date_detection": false,
"dynamic_templates": [
{
"long_service_group_by_attributes": {
"path_match": "service.groupByAttributes.*",
"match_mapping_type": "long",
"mapping": {
"type": "long"
}
}
},
{
"double_service_group_by_attributes": {
"path_match": "service.groupByAttributes.*",
"match_mapping_type": "double",
"mapping": {
"type": "double"
}
}
},
{
"string_service_group_by_attributes": {
"path_match": "service.groupByAttributes.*",
"match_mapping_type": "string",
"mapping": {
"ignore_above": 256,
"type": "keyword"
}
}
},
{
"long_operation_remote_service_group_by_attributes": {
"path_match": "operation.remoteService.groupByAttributes.*",
"match_mapping_type": "long",
"mapping": {
"type": "long"
}
}
},
{
"double_operation_remote_service_group_by_attributes": {
"path_match": "operation.remoteService.groupByAttributes.*",
"match_mapping_type": "double",
"mapping": {
"type": "double"
}
}
},
{
"string_operation_remote_service_group_by_attributes": {
"path_match": "operation.remoteService.groupByAttributes.*",
"match_mapping_type": "string",
"mapping": {
"ignore_above": 256,
"type": "keyword"
}
}
}
],
"_source": {
"enabled": true
},
"properties": {
"eventType": {
"type": "keyword"
},
"hashCode": {
"type": "keyword"
},
"operation": {
"properties": {
"name": {
"type": "keyword"
},
"remoteOperationName": {
"type": "keyword"
},
"remoteService": {
"properties": {
"groupByAttributes": {
"type": "object",
"dynamic": "true"
},
"keyAttributes": {
"properties": {
"environment": {
"type": "keyword"
},
"name": {
"type": "keyword"
}
}
}
}
}
}
},
"remoteService": {
"properties": {
"groupByAttributes": {
"type": "object",
"dynamic": "true"
},
"keyAttributes": {
"properties": {
"environment": {
"type": "keyword"
},
"name": {
"type": "keyword"
}
}
}
}
},
"service": {
"properties": {
"groupByAttributes": {
"type": "object",
"dynamic": "true"
},
"keyAttributes": {
"properties": {
"environment": {
"type": "keyword"
},
"name": {
"type": "keyword"
}
}
}
}
},
"timestamp": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
}
}
}
}
}
143 changes: 143 additions & 0 deletions ...prepper-plugins/opensearch/src/main/resources/otel-v2-apm-service-map-index-template.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
{
"version": 0,
"mappings": {
"date_detection": false,
"dynamic_templates": [
{
"long_service_group_by_attributes": {
"path_match": "service.groupByAttributes.*",
"match_mapping_type": "long",
"mapping": {
"type": "long"
}
}
},
{
"double_service_group_by_attributes": {
"path_match": "service.groupByAttributes.*",
"match_mapping_type": "double",
"mapping": {
"type": "double"
}
}
},
{
"string_service_group_by_attributes": {
"path_match": "service.groupByAttributes.*",
"match_mapping_type": "string",
"mapping": {
"ignore_above": 256,
"type": "keyword"
}
}
},
{
"long_operation_remote_service_group_by_attributes": {
"path_match": "operation.remoteService.groupByAttributes.*",
"match_mapping_type": "long",
"mapping": {
"type": "long"
}
}
},
{
"double_operation_remote_service_group_by_attributes": {
"path_match": "operation.remoteService.groupByAttributes.*",
"match_mapping_type": "double",
"mapping": {
"type": "double"
}
}
},
{
"string_operation_remote_service_group_by_attributes": {
"path_match": "operation.remoteService.groupByAttributes.*",
"match_mapping_type": "string",
"mapping": {
"ignore_above": 256,
"type": "keyword"
}
}
}
],
"_source": {
"enabled": true
},
"properties": {
"eventType": {
"type": "keyword"
},
"hashCode": {
"type": "keyword"
},
"operation": {
"properties": {
"name": {
"type": "keyword"
},
"remoteOperationName": {
"type": "keyword"
},
"remoteService": {
"properties": {
"groupByAttributes": {
"type": "object",
"dynamic": "true"
},
"keyAttributes": {
"properties": {
"environment": {
"type": "keyword"
},
"name": {
"type": "keyword"
}
}
}
}
}
}
},
"remoteService": {
"properties": {
"groupByAttributes": {
"type": "object",
"dynamic": "true"
},
"keyAttributes": {
"properties": {
"environment": {
"type": "keyword"
},
"name": {
"type": "keyword"
}
}
}
}
},
"service": {
"properties": {
"groupByAttributes": {
"type": "object",
"dynamic": "true"
},
"keyAttributes": {
"properties": {
"environment": {
"type": "keyword"
},
"name": {
"type": "keyword"
}
}
}
}
},
"timestamp": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
}
}
}
}
19 changes: 19 additions & 0 deletions ...plugins/opensearch/src/main/resources/otel-v2-apm-service-map-policy-no-ism-template.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
{
"policy": {
"description": "Managing OpenTelemetry APM service map indices",
"default_state": "current_write_index",
"states": [
{
"name": "current_write_index",
"actions": [
{
"rollover": {
"min_size": "50gb",
"min_index_age": "24h"
}
}
]
}
]
}
}
Loading
Loading