Skip to content

Commit 9d527ae

Browse files
ps48ps48
authored
Fix and extend AWS environment derivation for OTLP resource attributes (#6857)
* Fix and extend AWS environment derivation for OTLP resource attributes Signed-off-by: ps48 <pshenoy36@gmail.com> * added @nested, remove unused imports Signed-off-by: ps48 <pshenoy36@gmail.com> --------- Signed-off-by: ps48 <pshenoy36@gmail.com>
1 parent 2f21000 commit 9d527ae

7 files changed

Lines changed: 636 additions & 16 deletions

File tree

‎data-prepper-plugins/otel-apm-service-map-processor/README.md‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,38 @@ The `metric_timestamp_granularity` option controls the truncation granularity fo
7575

7676
In `arrival_time` mode, granularity has minimal impact since all spans in a window share the same `clock.instant()` — each window always produces one data point per label combination regardless of truncation.
7777

78+
## Environment Derivation
79+
80+
For every emitted `NodeOperationDetail` the processor populates `sourceNode.keyAttributes.environment` (and the matching `targetNode.keyAttributes.environment` on edge events) by inspecting OTel resource attributes — and where applicable span attributes — on the underlying spans. The same value is also written to each raw span as `attributes.derived.environment` by the companion `otel_traces` processor.
81+
82+
Lookup precedence (first match wins):
83+
84+
1. AWS platform detection from `cloud.platform` (and a few platform-specific signals listed below).
85+
2. `deployment.environment.name` from `resource.attributes`.
86+
3. `deployment.environment` from `resource.attributes` (legacy OTel key).
87+
4. Default fallback: `generic:default`.
88+
89+
For every AWS attribute the processor checks both span-level attributes and `resource.attributes` (since most OTel SDKs emit them in the resource).
90+
91+
| Resource (or span) attributes | `environment` value |
92+
|---|---|
93+
| `cloud.platform=aws_api_gateway` (+ optional `aws.api_gateway.stage=<s>`) | `api-gateway:<s>` |
94+
| `cloud.platform=aws_ec2` | `ec2:default` |
95+
| `cloud.platform=aws_ecs` + `aws.ecs.launchtype=fargate` | `ecs-fargate:default` |
96+
| `cloud.platform=aws_ecs` + `aws.ecs.launchtype=ec2` | `ecs-ec2:default` |
97+
| `cloud.platform=aws_ecs` (launchtype absent or other) | `ecs:default` |
98+
| `cloud.platform=aws_eks` | `eks:default` |
99+
| `cloud.platform=aws_elastic_beanstalk` | `elastic-beanstalk:default` |
100+
| `cloud.platform=aws_lambda` | `lambda:default` |
101+
| `cloud.resource_id` starts with `arn:aws:lambda:` | `lambda:default` |
102+
| `aws.lambda.invoked_arn` is set | `lambda:default` |
103+
| `cloud.provider=aws` + `faas.name=<n>` | `lambda:default` |
104+
| `deployment.environment.name=<env>` | `<env>` |
105+
| `deployment.environment=<env>` | `<env>` (legacy) |
106+
| (none of the above) | `generic:default` |
107+
108+
Attribute names follow the OpenTelemetry semantic conventions for [cloud](https://opentelemetry.io/docs/specs/semconv/registry/attributes/cloud/) and [AWS](https://opentelemetry.io/docs/specs/semconv/registry/attributes/aws/).
109+
78110
## Pipeline Examples
79111

80112
### Basic Pipeline

‎data-prepper-plugins/otel-apm-service-map-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/otel_apm_service_map/OTelApmServiceMapProcessor.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -356,9 +356,9 @@ private Map<String, Object> extractSpanAttributes(final Span span) {
356356
combinedAttributes.put("resource", resource);
357357
}
358358
final Map<String, Object> scope = span.getScope();
359-
if (scope != null ) {
359+
if (scope != null) {
360360
final Map<String, Object> scopeAttributes = (Map<String, Object>)scope.get("attributes");
361-
if (attributes != null) {
361+
if (scopeAttributes != null) {
362362
combinedAttributes.putAll(scopeAttributes);
363363
}
364364
}

‎data-prepper-plugins/otel-apm-service-map-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/otel_apm_service_map/OTelApmServiceMapProcessorTest.java‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1329,6 +1329,73 @@ void testSpanEndTimeMode_usesSpanEndTime() {
13291329
spanEndProcessor.shutdown();
13301330
}
13311331

1332+
@Test
1333+
void doExecute_emitsServiceMapEventWithEnvironmentFromResource_whenScopeHasNullAttributes() {
1334+
// Regression for issue #6786 (Bug 1): when the instrumentation scope has no
1335+
// "attributes" key, OTelApmServiceMapProcessor.extractSpanAttributes used to
1336+
// throw on putAll(null) and the catch returned an empty map, dropping the
1337+
// resource and its deployment.environment.name.
1338+
when(clock.instant())
1339+
.thenReturn(testTime)
1340+
.thenReturn(testTime)
1341+
.thenReturn(testTime.plusSeconds(65))
1342+
.thenReturn(testTime.plusSeconds(65))
1343+
.thenReturn(testTime.plusSeconds(65))
1344+
.thenReturn(testTime.plusSeconds(65))
1345+
.thenReturn(testTime.plusSeconds(130))
1346+
.thenReturn(testTime.plusSeconds(130))
1347+
.thenReturn(testTime.plusSeconds(130))
1348+
.thenReturn(testTime.plusSeconds(130));
1349+
1350+
final BaseEventBuilder<Event> eventBuilder = mock(EventBuilder.class, RETURNS_DEEP_STUBS);
1351+
when(eventFactory.eventBuilder(any())).thenReturn(eventBuilder);
1352+
doAnswer((a) -> {
1353+
eventMetadata = a.getArgument(0);
1354+
return eventBuilder;
1355+
}).when(eventBuilder).withEventMetadata(any());
1356+
doAnswer((a) -> {
1357+
eventData = a.getArgument(0);
1358+
return eventBuilder;
1359+
}).when(eventBuilder).withData(any());
1360+
doAnswer((a) -> JacksonEvent.builder()
1361+
.withEventMetadata(eventMetadata).withData(eventData).build())
1362+
.when(eventBuilder).build();
1363+
1364+
final File isolatedDir = new File(tempDir, "scope-null-attrs-" + System.nanoTime());
1365+
isolatedDir.mkdirs();
1366+
final OTelApmServiceMapProcessor isolatedProcessor = new OTelApmServiceMapProcessor(
1367+
Duration.ofSeconds(60), isolatedDir, clock, 1, eventFactory, pluginMetrics);
1368+
1369+
final Span leafServerSpan = createMockSpanWithIds("leaf-service", "GET /api/test",
1370+
"SPAN_KIND_SERVER", "1111111111111111", "", "aaaaaaaaaaaaaaaa");
1371+
final Map<String, Object> resourceAttrs = new HashMap<>();
1372+
resourceAttrs.put("deployment.environment.name", "production");
1373+
final Map<String, Object> resource = new HashMap<>();
1374+
resource.put("attributes", resourceAttrs);
1375+
when(leafServerSpan.getResource()).thenReturn(resource);
1376+
// Scope present with NO "attributes" key — the trigger for Bug 1.
1377+
final Map<String, Object> scope = new HashMap<>();
1378+
scope.put("name", "my-tracer");
1379+
when(leafServerSpan.getScope()).thenReturn(scope);
1380+
1381+
try {
1382+
isolatedProcessor.doExecute(Collections.singletonList(new Record<>(leafServerSpan)));
1383+
isolatedProcessor.doExecute(Collections.emptyList());
1384+
final Collection<Record<Event>> result = isolatedProcessor.doExecute(Collections.emptyList());
1385+
1386+
assertNotNull(result);
1387+
final Record<Event> serviceMapEvent = result.stream()
1388+
.filter(r -> r.getData().getMetadata() != null
1389+
&& "SERVICE_MAP".equals(r.getData().getMetadata().getEventType()))
1390+
.findFirst()
1391+
.orElseThrow(() -> new AssertionError("Expected at least one SERVICE_MAP event"));
1392+
assertThat(serviceMapEvent.getData().get("sourceNode/keyAttributes/environment", String.class),
1393+
equalTo("production"));
1394+
} finally {
1395+
isolatedProcessor.shutdown();
1396+
}
1397+
}
1398+
13321399
// Helper method to create mock spans
13331400
private Span createMockSpan(String serviceName, String operationName, String spanKind) {
13341401
Span mockSpan = mock(Span.class);

‎data-prepper-plugins/otel-proto-common/src/main/java/org/opensearch/dataprepper/plugins/otel/common/OTelSpanDerivationUtil.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,13 @@ public static void deriveAttributesForSpan(final Span span) {
109109
putAttribute(span, DERIVED_REMOTE_SERVICE_ATTRIBUTE, remoteOperationAndService.getService());
110110
}
111111

112-
final String environment = computeEnvironment(spanAttributes);
112+
// Build combined attributes including resource for environment derivation
113+
final Map<String, Object> combinedAttributes = spanAttributes != null ? new HashMap<>(spanAttributes) : new HashMap<>();
114+
final Map<String, Object> resource = span.getResource();
115+
if (resource != null) {
116+
combinedAttributes.put("resource", resource);
117+
}
118+
final String environment = computeEnvironment(combinedAttributes);
113119

114120
// Add derived attributes using our safe attribute setting method
115121
putAttribute(span, DERIVED_FAULT_ATTRIBUTE, String.valueOf(errorFault.fault));

‎data-prepper-plugins/otel-proto-common/src/main/java/org/opensearch/dataprepper/plugins/otel/common/ServiceEnvironmentProviders.java‎

Lines changed: 60 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -39,26 +39,73 @@ public static String getDeploymentEnvironment(final Map<String, Object> spanAttr
3939
return "generic:default";
4040
}
4141

42+
/**
43+
* Get an attribute by checking span-level attributes first, then resource attributes.
44+
*/
45+
private static String getAttributeFromSpanOrResource(final Map<String, Object> spanAttributes,
46+
final Map<String, Object> resourceAttributes,
47+
final String key) {
48+
String value = getStringAttribute(spanAttributes, key);
49+
if (value == null && resourceAttributes != null) {
50+
value = getStringAttribute(resourceAttributes, key);
51+
}
52+
return value;
53+
}
54+
55+
/**
56+
* Try to extract resource attributes from the nested structure: spanAttributes["resource"]["attributes"]
57+
*/
58+
private static Map<String, Object> extractResourceAttributes(final Map<String, Object> spanAttributes) {
59+
try {
60+
@SuppressWarnings("unchecked")
61+
Map<String, Object> resourceAttrs = (Map<String, Object>)
62+
((Map<String, Object>) spanAttributes.get("resource")).get("attributes");
63+
return resourceAttrs;
64+
} catch (Exception ignored) {
65+
return null;
66+
}
67+
}
68+
4269
public static String getAwsServiceEnvironment(final Map<String, Object> spanAttributes) {
4370
try {
44-
String cloudPlatform = getStringAttribute(spanAttributes, "cloud.platform");
45-
if (cloudPlatform != null && cloudPlatform.equals("aws_api_gateway")) {
46-
return "api-gateway:"+getStringAttribute(spanAttributes, "aws.api_gateway.stage");
71+
final Map<String, Object> resourceAttributes = extractResourceAttributes(spanAttributes);
72+
73+
String cloudPlatform = getAttributeFromSpanOrResource(spanAttributes, resourceAttributes, "cloud.platform");
74+
if ("aws_api_gateway".equals(cloudPlatform)) {
75+
String stage = getAttributeFromSpanOrResource(spanAttributes, resourceAttributes, "aws.api_gateway.stage");
76+
return "api-gateway:" + stage;
4777
}
48-
if (cloudPlatform != null && cloudPlatform.equals("aws_ec2")) {
78+
if ("aws_ec2".equals(cloudPlatform)) {
4979
return "ec2:default";
5080
}
51-
String cloudResourceId = getStringAttribute(spanAttributes, "cloud.resource_id");
52-
String invokedArn = getStringAttribute(spanAttributes, "aws.lambda.invoked_arn");
53-
if (cloudResourceId != null && cloudResourceId.startsWith("arn:aws:lambda:") || invokedArn != null) {
81+
if ("aws_ecs".equals(cloudPlatform)) {
82+
String launchType = getAttributeFromSpanOrResource(spanAttributes, resourceAttributes, "aws.ecs.launchtype");
83+
if ("fargate".equals(launchType)) {
84+
return "ecs-fargate:default";
85+
}
86+
if ("ec2".equals(launchType)) {
87+
return "ecs-ec2:default";
88+
}
89+
return "ecs:default";
90+
}
91+
if ("aws_eks".equals(cloudPlatform)) {
92+
return "eks:default";
93+
}
94+
if ("aws_elastic_beanstalk".equals(cloudPlatform)) {
95+
return "elastic-beanstalk:default";
96+
}
97+
if ("aws_lambda".equals(cloudPlatform)) {
5498
return "lambda:default";
5599
}
56-
57-
@SuppressWarnings("unchecked")
58-
Map<String, Object> resourceAttributes = (Map<String, Object>)
59-
((Map<String, Object>) spanAttributes.get("resource")).get("attributes");
60-
String cloudProvider = getStringAttribute(resourceAttributes, "cloud.provider");
61-
String faasName = getStringAttribute(resourceAttributes, "faas.name");
100+
101+
String cloudResourceId = getAttributeFromSpanOrResource(spanAttributes, resourceAttributes, "cloud.resource_id");
102+
String invokedArn = getAttributeFromSpanOrResource(spanAttributes, resourceAttributes, "aws.lambda.invoked_arn");
103+
if ((cloudResourceId != null && cloudResourceId.startsWith("arn:aws:lambda:")) || invokedArn != null) {
104+
return "lambda:default";
105+
}
106+
107+
String cloudProvider = resourceAttributes != null ? getStringAttribute(resourceAttributes, "cloud.provider") : null;
108+
String faasName = resourceAttributes != null ? getStringAttribute(resourceAttributes, "faas.name") : null;
62109
if (cloudProvider != null && cloudProvider.equals("aws") && faasName != null) {
63110
return "lambda:default";
64111
}

0 commit comments

Comments
 (0)