Introducing Otel apm service map processor

.gitignore

@@ -1,5 +1,6 @@
 # Gradle directories
 build
+bin
 .gradle
 gradle/tools
 

data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java

@@ -434,6 +434,8 @@ private Map<String, Object> readIndexTemplate(final String templateFile, final I
                 templateURL = loadExistingTemplate(templateType, IndexConstants.RAW_STANDARD_TEMPLATE_FILE);
             } else if (indexType.equals(IndexType.TRACE_ANALYTICS_SERVICE_MAP)) {
                 templateURL = loadExistingTemplate(templateType, IndexConstants.SERVICE_MAP_DEFAULT_TEMPLATE_FILE);
+            } else if (indexType.equals(IndexType.OTEL_APM_SERVICE_MAP)) {
+                templateURL = loadExistingTemplate(templateType, IndexConstants.OTEL_APM_SERVICE_MAP_TEMPLATE_FILE);
             } else if (indexType.equals(IndexType.LOG_ANALYTICS)) {
                 templateURL = loadExistingTemplate(templateType, IndexConstants.LOGS_DEFAULT_TEMPLATE_FILE);
             } else if (indexType.equals(IndexType.LOG_ANALYTICS_PLAIN)) {

data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConstants.java

@@ -37,10 +37,12 @@ public class IndexConstants {
   public static final String ISM_ROLLOVER_ALIAS_SETTING = "opendistro.index_state_management.rollover_alias";
   // TODO: extract out version number into version enum
   public static final String SERVICE_MAP_DEFAULT_TEMPLATE_FILE = "otel-v1-apm-service-map-index-template.json";
+  public static final String OTEL_APM_SERVICE_MAP_TEMPLATE_FILE = "otel-apm-service-map-index-template.json";
 
   static {
     // TODO: extract out version number into version enum
     TYPE_TO_DEFAULT_ALIAS.put(IndexType.TRACE_ANALYTICS_SERVICE_MAP, "otel-v1-apm-service-map");
+    TYPE_TO_DEFAULT_ALIAS.put(IndexType.OTEL_APM_SERVICE_MAP, "otel-apm-service-map");
     TYPE_TO_DEFAULT_ALIAS.put(IndexType.TRACE_ANALYTICS_RAW, "otel-v1-apm-span");
     TYPE_TO_DEFAULT_ALIAS.put(IndexType.TRACE_ANALYTICS_RAW_PLAIN, "otel-v1-apm-span");
     TYPE_TO_DEFAULT_ALIAS.put(IndexType.LOG_ANALYTICS, "logs-otel-v1");

data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexManagerFactory.java

@@ -59,6 +59,10 @@ public final IndexManager getIndexManager(final IndexType indexType,
                 indexManager = new TraceAnalyticsServiceMapIndexManager(
                         restHighLevelClient, openSearchClient, openSearchSinkConfiguration, clusterSettingsParser, templateStrategy, indexAlias);
                 break;
+            case OTEL_APM_SERVICE_MAP:
+                indexManager = new OTelAPMServiceMapIndexManager(
+                        restHighLevelClient, openSearchClient, openSearchSinkConfiguration, clusterSettingsParser, templateStrategy, indexAlias);
+                break;
             case LOG_ANALYTICS:
             case LOG_ANALYTICS_PLAIN:
                 indexManager = new LogAnalyticsIndexManager(
@@ -151,6 +155,19 @@ public TraceAnalyticsServiceMapIndexManager(final RestHighLevelClient restHighLe
         }
     }
 
+    private static class OTelAPMServiceMapIndexManager extends AbstractIndexManager {
+
+        public OTelAPMServiceMapIndexManager(final RestHighLevelClient restHighLevelClient,
+                                            final OpenSearchClient openSearchClient,
+                                            final OpenSearchSinkConfiguration openSearchSinkConfiguration,
+                                            final ClusterSettingsParser clusterSettingsParser,
+                                            final TemplateStrategy templateStrategy,
+                                            final String indexAlias) {
+            super(restHighLevelClient, openSearchClient, openSearchSinkConfiguration, clusterSettingsParser, templateStrategy, indexAlias);
+            this.ismPolicyManagementStrategy = new NoIsmPolicyManagement(openSearchClient, restHighLevelClient);
+        }
+    }
+
     private static class LogAnalyticsIndexManager extends AbstractIndexManager {
 
         public LogAnalyticsIndexManager(final RestHighLevelClient restHighLevelClient,

data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexType.java

@@ -15,6 +15,7 @@ public enum IndexType {
     TRACE_ANALYTICS_RAW("trace-analytics-raw"),
     TRACE_ANALYTICS_RAW_PLAIN("trace-analytics-plain-raw"),
     TRACE_ANALYTICS_SERVICE_MAP("trace-analytics-service-map"),
+    OTEL_APM_SERVICE_MAP("otel-apm-service-map"),
     LOG_ANALYTICS("log-analytics"),
     LOG_ANALYTICS_PLAIN("log-analytics-plain"),
     METRIC_ANALYTICS("metric-analytics"),

data-prepper-plugins/opensearch/src/main/resources/index-template/otel-apm-service-map-index-template.json

@@ -0,0 +1,141 @@
+{
+  "version": 0,
+  "index_patterns": ["otel-apm-service-map*"],
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "long_service_group_by_attributes": {
+          "path_match": "service.groupByAttributes.*",
+          "match_mapping_type": "long",
+          "mapping": {
+            "type": "long"
+          }
+        }
+      },
+      {
+        "double_service_group_by_attributes": {
+          "path_match": "service.groupByAttributes.*",
+          "match_mapping_type": "double",
+          "mapping": {
+            "type": "double"
+          }
+        }
+      },
+      {
+        "string_service_group_by_attributes": {
+          "path_match": "service.groupByAttributes.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "ignore_above": 256,
+            "type": "keyword"
+          }
+        }
+      },
+      {
+        "long_operation_remote_service_group_by_attributes": {
+          "path_match": "operation.remoteService.groupByAttributes.*",
+          "match_mapping_type": "long",
+          "mapping": {
+            "type": "long"
+          }
+        }
+      },
+      {
+        "double_operation_remote_service_group_by_attributes": {
+          "path_match": "operation.remoteService.groupByAttributes.*",
+          "match_mapping_type": "double",
+          "mapping": {
+            "type": "double"
+          }
+        }
+      },
+      {
+        "string_operation_remote_service_group_by_attributes": {
+          "path_match": "operation.remoteService.groupByAttributes.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "ignore_above": 256,
+            "type": "keyword"
+          }
+        }
+      }
+    ],
+    "date_detection": false,
+    "properties": {
+      "eventType": {
+        "type": "keyword"
+      },
+      "hashCode": {
+        "type": "keyword"
+      },
+      "operation": {
+        "properties": {
+          "name": {
+            "type": "keyword"
+          },
+          "remoteOperationName": {
+            "type": "keyword"
+          },
+          "remoteService": {
+            "properties": {
+              "groupByAttributes": {
+                "type": "object",
+                "dynamic": "true"
+              },
+              "keyAttributes": {
+                "properties": {
+                  "environment": {
+                    "type": "keyword"
+                  },
+                  "name": {
+                    "type": "keyword"
+                  }
+                }
+              }
+            }
+          }
+        }
+      },
+      "remoteService": {
+        "properties": {
+          "groupByAttributes": {
+            "type": "object",
+            "dynamic": "true"
+          },
+          "keyAttributes": {
+            "properties": {
+              "environment": {
+                "type": "keyword"
+              },
+              "name": {
+                "type": "keyword"
+              }
+            }
+          }
+        }
+      },
+      "service": {
+        "properties": {
+          "groupByAttributes": {
+            "type": "object",
+            "dynamic": "true"
+          },
+          "keyAttributes": {
+            "properties": {
+              "environment": {
+                "type": "keyword"
+              },
+              "name": {
+                "type": "keyword"
+              }
+            }
+          }
+        }
+      },
+      "timestamp": {
+        "type": "date",
+        "format": "strict_date_optional_time||epoch_millis"
+      }
+    }
+  }
+}

data-prepper-plugins/opensearch/src/main/resources/otel-apm-service-map-index-template.json

@@ -0,0 +1,141 @@
+{
+  "version": 0,
+  "index_patterns": ["otel-apm-service-map*"],
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "long_service_group_by_attributes": {
+          "path_match": "service.groupByAttributes.*",
+          "match_mapping_type": "long",
+          "mapping": {
+            "type": "long"
+          }
+        }
+      },
+      {
+        "double_service_group_by_attributes": {
+          "path_match": "service.groupByAttributes.*",
+          "match_mapping_type": "double",
+          "mapping": {
+            "type": "double"
+          }
+        }
+      },
+      {
+        "string_service_group_by_attributes": {
+          "path_match": "service.groupByAttributes.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "ignore_above": 256,
+            "type": "keyword"
+          }
+        }
+      },
+      {
+        "long_operation_remote_service_group_by_attributes": {
+          "path_match": "operation.remoteService.groupByAttributes.*",
+          "match_mapping_type": "long",
+          "mapping": {
+            "type": "long"
+          }
+        }
+      },
+      {
+        "double_operation_remote_service_group_by_attributes": {
+          "path_match": "operation.remoteService.groupByAttributes.*",
+          "match_mapping_type": "double",
+          "mapping": {
+            "type": "double"
+          }
+        }
+      },
+      {
+        "string_operation_remote_service_group_by_attributes": {
+          "path_match": "operation.remoteService.groupByAttributes.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "ignore_above": 256,
+            "type": "keyword"
+          }
+        }
+      }
+    ],
+    "date_detection": false,
+    "properties": {
+      "eventType": {
+        "type": "keyword"
+      },
+      "hashCode": {
+        "type": "keyword"
+      },
+      "operation": {
+        "properties": {
+          "name": {
+            "type": "keyword"
+          },
+          "remoteOperationName": {
+            "type": "keyword"
+          },
+          "remoteService": {
+            "properties": {
+              "groupByAttributes": {
+                "type": "object",
+                "dynamic": "true"
+              },
+              "keyAttributes": {
+                "properties": {
+                  "environment": {
+                    "type": "keyword"
+                  },
+                  "name": {
+                    "type": "keyword"
+                  }
+                }
+              }
+            }
+          }
+        }
+      },
+      "remoteService": {
+        "properties": {
+          "groupByAttributes": {
+            "type": "object",
+            "dynamic": "true"
+          },
+          "keyAttributes": {
+            "properties": {
+              "environment": {
+                "type": "keyword"
+              },
+              "name": {
+                "type": "keyword"
+              }
+            }
+          }
+        }
+      },
+      "service": {
+        "properties": {
+          "groupByAttributes": {
+            "type": "object",
+            "dynamic": "true"
+          },
+          "keyAttributes": {
+            "properties": {
+              "environment": {
+                "type": "keyword"
+              },
+              "name": {
+                "type": "keyword"
+              }
+            }
+          }
+        }
+      },
+      "timestamp": {
+        "type": "date",
+        "format": "strict_date_optional_time||epoch_millis"
+      }
+    }
+  }
+}

data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexTypeTests.java

@@ -28,6 +28,7 @@ public void getByValue() {
         assertEquals(Optional.of(IndexType.TRACE_ANALYTICS_RAW), IndexType.getByValue("trace-analytics-raw"));
         assertEquals(Optional.of(IndexType.TRACE_ANALYTICS_RAW_PLAIN), IndexType.getByValue("trace-analytics-plain-raw"));
         assertEquals(Optional.of(IndexType.TRACE_ANALYTICS_SERVICE_MAP), IndexType.getByValue("trace-analytics-service-map"));
+        assertEquals(Optional.of(IndexType.OTEL_APM_SERVICE_MAP), IndexType.getByValue("otel-apm-service-map"));
         assertEquals(Optional.of(IndexType.LOG_ANALYTICS), IndexType.getByValue("log-analytics"));
         assertEquals(Optional.of(IndexType.LOG_ANALYTICS_PLAIN), IndexType.getByValue("log-analytics-plain"));
         assertEquals(Optional.of(IndexType.METRIC_ANALYTICS), IndexType.getByValue("metric-analytics"));
@@ -36,7 +37,7 @@ public void getByValue() {
 
     @Test
     public void getIndexTypeValues() {
-        assertEquals("[trace-analytics-raw, trace-analytics-plain-raw, trace-analytics-service-map, log-analytics, log-analytics-plain, metric-analytics, metric-analytics-plain, custom, management_disabled]", IndexType.getIndexTypeValues());
+        assertEquals("[trace-analytics-raw, trace-analytics-plain-raw, trace-analytics-service-map, otel-apm-service-map, log-analytics, log-analytics-plain, metric-analytics, metric-analytics-plain, custom, management_disabled]", IndexType.getIndexTypeValues());
     }
 
     @ParameterizedTest