From ef24dae074d8652a94a7d5bb34616ddea6a7e3a2 Mon Sep 17 00:00:00 2001 From: Krishna Kondaka Date: Mon, 21 Apr 2025 17:57:13 -0700 Subject: [PATCH] Add missing OTEL standard fields (#5635) Signed-off-by: Krishna Kondaka (cherry picked from commit ac2714533ec1bbfbae149cf4c9c38080d88b5bdb) --- .../model/metric/JacksonMetric.java | 22 ++++++- .../dataprepper/model/metric/Metric.java | 12 +++- .../dataprepper/model/trace/JacksonSpan.java | 24 ++++++- .../dataprepper/model/trace/Span.java | 14 +++- .../model/metric/JacksonGaugeTest.java | 8 +++ .../model/trace/JacksonSpanTest.java | 8 +++ .../otel/codec/OTelProtoStandardCodec.java | 65 +++++++++++++------ .../codec/OTelProtoOpensearchCodecTest.java | 2 +- .../codec/OTelProtoStandardCodecTest.java | 39 ++++++++++- .../test/resources/test-gauge-metrics.json | 23 +++++++ ...-histogram-metrics-no-explicit-bounds.json | 23 +++++++ .../resources/test-histogram-metrics.json | 25 +++++++ .../test-request-both-span-types.json | 2 + .../src/test/resources/test-request-log.json | 6 +- .../test-request-multiple-traces.json | 2 + .../test/resources/test-request-no-spans.json | 5 +- .../src/test/resources/test-request.json | 2 + .../src/test/resources/test-sum-metrics.json | 23 +++++++ 18 files changed, 268 insertions(+), 37 deletions(-) diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/JacksonMetric.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/JacksonMetric.java index 9307afbcd7..92fd9dd7ba 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/JacksonMetric.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/JacksonMetric.java @@ -37,6 +37,7 @@ public abstract class JacksonMetric extends JacksonEvent implements Metric { protected static final String SCHEMA_URL_KEY = "schemaUrl"; protected static final String EXEMPLARS_KEY = "exemplars"; protected static final String FLAGS_KEY = "flags"; + protected static final String METADATA_KEY = "metadata"; private boolean flattenAttributes; protected JacksonMetric(Builder builder, boolean flattenAttributes) { @@ -101,6 +102,11 @@ public String getKind() { return this.get(KIND_KEY, String.class); } + @Override + public Map getMetricMetadata() { + return this.get(METADATA_KEY, Map.class); + } + @Override public String getStartTime() { return this.get(START_TIME_KEY, String.class); @@ -256,7 +262,7 @@ public T withServiceName(final String serviceName) { } /** - * Sets the scope of the log event + * Sets the scope of the metric event * * @param scope scope to be set * @return the builder @@ -268,7 +274,19 @@ public T withScope(final Map scope) { } /** - * Sets the resource of the log event + * Sets the metadata of the metric event + * + * @param metadata metadata to be set + * @return the builder + * @since 2.11 + */ + public T withMetricMetadata(final Map metadata) { + put(METADATA_KEY, metadata); + return getThis(); + } + + /** + * Sets the resource of the metric event * * @param resource resource to be set * @return the builder diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/Metric.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/Metric.java index 192311d9e7..a7d67f74d0 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/Metric.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/Metric.java @@ -116,7 +116,7 @@ enum KIND {GAUGE, HISTOGRAM, EXPONENTIAL_HISTOGRAM, SUM, SUMMARY} Integer getFlags(); /** - * Gets the scope of this log event. + * Gets the scope of this metric event. * * @return the scope * @since 2.11 @@ -124,7 +124,15 @@ enum KIND {GAUGE, HISTOGRAM, EXPONENTIAL_HISTOGRAM, SUM, SUMMARY} Map getScope(); /** - * Gets the resource of this log event. + * Gets the metadata of this metric event. + * + * @return the metadata + * @since 2.11 + */ + Map getMetricMetadata(); + + /** + * Gets the resource of this metric event. * * @return the resource * @since 2.11 diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/trace/JacksonSpan.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/trace/JacksonSpan.java index a38e90d491..e2935475f0 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/trace/JacksonSpan.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/trace/JacksonSpan.java @@ -48,6 +48,7 @@ public class JacksonSpan extends JacksonEvent implements Span { protected static final String ATTRIBUTES_KEY = "attributes"; private static final String DROPPED_ATTRIBUTES_COUNT_KEY = "droppedAttributesCount"; private static final String EVENTS_KEY = "events"; + private static final String SCHEMA_URL_KEY = "schemaUrl"; private static final String DROPPED_EVENTS_COUNT_KEY = "droppedEventsCount"; private static final String LINKS_KEY = "links"; private static final String DROPPED_LINKS_COUNT_KEY = "droppedLinksCount"; @@ -91,6 +92,11 @@ protected void checkAndSetDefaultValues() { putIfAbsent(DROPPED_EVENTS_COUNT_KEY, Integer.class, 0); } + @Override + public String getSchemaUrl() { + return this.get(SCHEMA_URL_KEY, String.class); + } + @Override public String getTraceId() { return this.get(TRACE_ID_KEY, String.class); @@ -406,7 +412,19 @@ public Builder withFlags(final Integer flags) { } /** - * Sets the status of the log event + * Sets the schema url of span + * + * @param schemaUrl schema url + * @return returns the builder + * @since 2.11 + */ + public Builder withSchemaUrl(final String schemaUrl) { + data.put(SCHEMA_URL_KEY, schemaUrl); + return this; + } + + /** + * Sets the status of the span event * * @param status status to be set * @return the builder @@ -418,7 +436,7 @@ public Builder withStatus(final Map status) { } /** - * Sets the scope of the log event + * Sets the scope of the span event * * @param scope scope to be set * @return the builder @@ -430,7 +448,7 @@ public Builder withScope(final Map scope) { } /** - * Sets the resource of the log event + * Sets the resource of the span event * * @param resource resource to be set * @return the builder diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/trace/Span.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/trace/Span.java index 2fc6a70d0a..dd90a559fe 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/trace/Span.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/trace/Span.java @@ -168,7 +168,15 @@ public interface Span extends Event { void setServiceName(final String serviceName); /** - * Gets the scope of this log event. + * Gets the schema url of this span event. + * + * @return the schema url + * @since 2.11 + */ + String getSchemaUrl(); + + /** + * Gets the scope of this span event. * * @return the scope * @since 2.11 @@ -176,7 +184,7 @@ public interface Span extends Event { Map getScope(); /** - * Gets the resource of this log event. + * Gets the resource of this span event. * * @return the resource * @since 2.11 @@ -184,7 +192,7 @@ public interface Span extends Event { Map getResource(); /** - * Gets the status of this log event. + * Gets the status of this span event. * * @return the status * @since 2.11 diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/metric/JacksonGaugeTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/metric/JacksonGaugeTest.java index 8d08740f9d..b87be47df7 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/metric/JacksonGaugeTest.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/metric/JacksonGaugeTest.java @@ -40,6 +40,7 @@ class JacksonGaugeTest { "key2", TEST_KEY2); protected static final Map TEST_SCOPE = ImmutableMap.of("name", UUID.randomUUID().toString(), "version", UUID.randomUUID().toString(), "attributes", List.of(Map.of("key", UUID.randomUUID().toString(), "value", UUID.randomUUID().toString()))); protected static final Map TEST_RESOURCE = ImmutableMap.of("attributes", List.of(Map.of("key", UUID.randomUUID().toString(), "value", UUID.randomUUID().toString()))); + protected static final Map TEST_METADATA = ImmutableMap.of("metadataKey1", UUID.randomUUID().toString(), "metadataKey2", UUID.randomUUID().toString(), "metadataKey3", UUID.randomUUID().toString()); protected static final String TEST_SERVICE_NAME = "service"; protected static final String TEST_NAME = "name"; protected static final String TEST_DESCRIPTION = "description"; @@ -72,6 +73,7 @@ public void setup() { .withUnit(TEST_UNIT_NAME) .withScope(TEST_SCOPE) .withResource(TEST_RESOURCE) + .withMetricMetadata(TEST_METADATA) .withValue(TEST_VALUE) .withServiceName(TEST_SERVICE_NAME) .withExemplars(TEST_EXEMPLARS) @@ -160,6 +162,12 @@ public void testGetResource() { assertThat(resource, is(equalTo(TEST_RESOURCE))); } + @Test + public void testGetMetricMetadata() { + final Map metadata = gauge.getMetricMetadata(); + assertThat(metadata, is(equalTo(TEST_METADATA))); + } + @Test public void testGetTime() { final String endTime = gauge.getTime(); diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/trace/JacksonSpanTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/trace/JacksonSpanTest.java index b3402877b4..5c98f21bf5 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/trace/JacksonSpanTest.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/trace/JacksonSpanTest.java @@ -47,6 +47,7 @@ public class JacksonSpanTest { protected static final String TEST_PARENT_SPAN_ID = UUID.randomUUID().toString(); protected static final String TEST_NAME = UUID.randomUUID().toString(); protected static final String TEST_KIND = UUID.randomUUID().toString(); + protected static final String TEST_SCHEMA_URL = UUID.randomUUID().toString(); protected static final int TEST_FLAGS = 10; protected static final String TEST_START_TIME = UUID.randomUUID().toString(); protected static final String TEST_END_TIME = UUID.randomUUID().toString(); @@ -106,6 +107,7 @@ public void setup() { .withResource(TEST_RESOURCE) .withStatus(TEST_STATUS) .withStartTime(TEST_START_TIME) + .withSchemaUrl(TEST_SCHEMA_URL) .withEndTime(TEST_END_TIME) .withAttributes(TEST_ATTRIBUTES) .withDroppedAttributesCount(TEST_DROPPED_ATTRIBUTES_COUNT) @@ -168,6 +170,12 @@ public void testGetFlags() { assertThat(flags, is(equalTo(TEST_FLAGS))); } + @Test + public void testGetSchemaUrl() { + final String schemaUrl = jacksonSpan.getSchemaUrl(); + assertThat(schemaUrl, is(equalTo(TEST_SCHEMA_URL))); + } + @Test public void testGetStartTime() { final String GetStartTime = jacksonSpan.getStartTime(); diff --git a/data-prepper-plugins/otel-proto-common/src/main/java/org/opensearch/dataprepper/plugins/otel/codec/OTelProtoStandardCodec.java b/data-prepper-plugins/otel-proto-common/src/main/java/org/opensearch/dataprepper/plugins/otel/codec/OTelProtoStandardCodec.java index 3b4877ea19..d2cb0db9c3 100644 --- a/data-prepper-plugins/otel-proto-common/src/main/java/org/opensearch/dataprepper/plugins/otel/codec/OTelProtoStandardCodec.java +++ b/data-prepper-plugins/otel-proto-common/src/main/java/org/opensearch/dataprepper/plugins/otel/codec/OTelProtoStandardCodec.java @@ -78,7 +78,7 @@ import java.util.stream.Stream; /** - * OTelProtoStandardCodec is for encoding/decoding between DataPrepper OTEL representation and standard + * OTelProtoStandardCodec is for encoding/decoding between DataPrepper OTEL representation and standard * {@link io.opentelemetry.proto} as per the Standard Spec. */ public class OTelProtoStandardCodec { @@ -97,6 +97,7 @@ public class OTelProtoStandardCodec { static final String NAME_KEY = "name"; + static final String SCHEMA_URL_KEY = "schemaUrl"; static final String VERSION_KEY = "version"; static final String DROPPED_ATTRIBUTES_COUNT_KEY = "droppedAttributesCount"; @@ -178,8 +179,7 @@ public List parseExportLogsServiceRequest(final ExportLogsServ protected Stream parseResourceLogs(ResourceLogs rs, final Instant timeReceived) { final String serviceName = getServiceName(rs.getResource()).orElse(null); - final Map resourceAttributes = getResourceAttributes(rs.getResource()); - final String schemaUrl = rs.getSchemaUrl(); + final Map resourceAttributes = getResourceAttributes(rs.getResource(), rs.getSchemaUrl()); Stream mappedScopeListLogs = rs.getScopeLogsList() .stream() @@ -188,7 +188,7 @@ protected Stream parseResourceLogs(ResourceLogs rs, final Inst serviceName, getInstrumentationScopeAttributes(sls.getScope()), resourceAttributes, - schemaUrl, + sls.getSchemaUrl(), timeReceived); }) .flatMap(Collection::stream); @@ -226,7 +226,7 @@ protected Map convertKeyValueToAttributes(List attribu protected List parseResourceSpans(final ResourceSpans resourceSpans, final Instant timeReceived) { final String serviceName = getServiceName(resourceSpans.getResource()).orElse(null); - final Map resourceAttributes = getResourceAttributes(resourceSpans.getResource()); + final Map resourceAttributes = getResourceAttributes(resourceSpans.getResource(), resourceSpans.getSchemaUrl()); if (!resourceSpans.getScopeSpansList().isEmpty()) { return parseScopeSpans(resourceSpans.getScopeSpansList(), serviceName, resourceAttributes, timeReceived); } @@ -237,7 +237,7 @@ protected List parseResourceSpans(final ResourceSpans resourceSpans, final private List parseScopeSpans(final List scopeSpansList, final String serviceName, final Map resourceAttributes, final Instant timeReceived) { return scopeSpansList.stream() - .map(scopeSpans -> parseSpans(scopeSpans.getSpansList(), scopeSpans.getScope(), + .map(scopeSpans -> parseSpans(scopeSpans.getSpansList(), scopeSpans.getScope(), scopeSpans.getSchemaUrl(), OTelProtoStandardCodec::getInstrumentationScopeAttributes, serviceName, resourceAttributes, timeReceived)) .flatMap(Collection::stream) .collect(Collectors.toList()); @@ -281,13 +281,14 @@ private Map> splitSpansByTrac } private List parseSpans(final List spans, final T scope, - final Function> scopeAttributesGetter, - final String serviceName, final Map resourceAttributes, - final Instant timeReceived) { + final String schemaUrl, + final Function> scopeAttributesGetter, + final String serviceName, final Map resourceAttributes, + final Instant timeReceived) { return spans.stream() .map(span -> { final Map scopeAttributes = scopeAttributesGetter.apply(scope); - return parseSpan(span, scopeAttributes, serviceName, resourceAttributes, timeReceived); + return parseSpan(span, scopeAttributes, schemaUrl, serviceName, resourceAttributes, timeReceived); }) .collect(Collectors.toList()); } @@ -322,14 +323,19 @@ private Map getStatus(final io.opentelemetry.proto.trace.v1.Stat return Map.of("message", status.getMessage(), "code", status.getCodeValue()); } - protected Span parseSpan(final io.opentelemetry.proto.trace.v1.Span sp, final Map instrumentationScopeAttributes, - final String serviceName, final Map resourceAttributes, final Instant timeReceived) { + protected Span parseSpan(final io.opentelemetry.proto.trace.v1.Span sp, + final Map instrumentationScopeAttributes, + final String schemaUrl, + final String serviceName, + final Map resourceAttributes, + final Instant timeReceived) { Span span = JacksonStandardSpan.builder() .withSpanId(convertByteStringToString(sp.getSpanId())) .withTraceId(convertByteStringToString(sp.getTraceId())) .withTraceState(sp.getTraceState()) .withParentSpanId(convertByteStringToString(sp.getParentSpanId())) .withName(sp.getName()) + .withSchemaUrl(schemaUrl) .withKind(sp.getKind().name()) .withStartTime(convertUnixNanosToISO8601(sp.getStartTimeUnixNano())) .withEndTime(convertUnixNanosToISO8601(sp.getEndTimeUnixNano())) @@ -399,9 +405,9 @@ protected Map getSpanAttributes(final io.opentelemetry.proto.tra return span.getAttributesList().stream().collect(Collectors.toMap(i -> i.getKey(), i -> convertAnyValue(i.getValue()))); } - protected Map getResourceAttributes(final Resource resource) { + protected Map getResourceAttributes(final Resource resource, final String schemaUrl) { Map attributes = resource.getAttributesList().stream().collect(Collectors.toMap(i -> i.getKey(), i -> convertAnyValue(i.getValue()))); - return Map.of(ATTRIBUTES_KEY, attributes, DROPPED_ATTRIBUTES_COUNT_KEY, resource.getDroppedAttributesCount()); + return Map.of(ATTRIBUTES_KEY, attributes, DROPPED_ATTRIBUTES_COUNT_KEY, resource.getDroppedAttributesCount(), SCHEMA_URL_KEY, schemaUrl); } /** @@ -483,7 +489,7 @@ public Collection> parseExportMetricsServiceRequest( final boolean flattenAttributes) { Collection> recordsOut = new ArrayList<>(); for (ResourceMetrics rs : request.getResourceMetricsList()) { - final Map resourceAttributes = getResourceAttributes(rs.getResource()); + final Map resourceAttributes = getResourceAttributes(rs.getResource(), rs.getSchemaUrl()); final String serviceName = getServiceName(rs.getResource()).orElse(null); for (ScopeMetrics sm : rs.getScopeMetricsList()) { @@ -528,6 +534,11 @@ private List> processMetricsList( return recordsOut; } + private Map getMetricMetadata(io.opentelemetry.proto.metrics.v1.Metric metric) { + List metadata = metric.getMetadataList(); + return convertKeyValueToAttributes(metadata); + } + private List> mapGauge( io.opentelemetry.proto.metrics.v1.Metric metric, String serviceName, @@ -544,6 +555,7 @@ private List> mapGauge( .withTime(convertUnixNanosToISO8601(dp.getTimeUnixNano())) .withValue(getValueAsDouble(dp)) .withScope(ils) + .withMetricMetadata(getMetricMetadata(metric)) .withResource(resourceAttributes) .withAttributes(convertKeyValueToAttributes(dp.getAttributesList())) .withSchemaUrl(schemaUrl) @@ -576,6 +588,7 @@ private List> mapSum( .withScope(ils) .withResource(resourceAttributes) .withSchemaUrl(schemaUrl) + .withMetricMetadata(getMetricMetadata(metric)) .withExemplars(convertExemplars(dp.getExemplarsList())) .withFlags(dp.getFlags()) .withTimeReceived(timeReceived) @@ -600,6 +613,7 @@ private List> mapSummary( .withTime(convertUnixNanosToISO8601(dp.getTimeUnixNano())) .withCount(dp.getCount()) .withSum(dp.getSum()) + .withMetricMetadata(getMetricMetadata(metric)) .withQuantiles(getQuantileValues(dp.getQuantileValuesList())) .withQuantilesValueCount(dp.getQuantileValuesCount()) .withScope(ils) @@ -630,12 +644,15 @@ private List> mapHistogram( .withStartTime(convertUnixNanosToISO8601(dp.getStartTimeUnixNano())) .withTime(convertUnixNanosToISO8601(dp.getTimeUnixNano())) .withSum(dp.getSum()) + .withMin(dp.getMin()) + .withMax(dp.getMax()) .withCount(dp.getCount()) .withBucketCount(dp.getBucketCountsCount()) .withExplicitBoundsCount(dp.getExplicitBoundsCount()) .withAggregationTemporality(metric.getHistogram().getAggregationTemporality().toString()) .withBucketCountsList(dp.getBucketCountsList()) .withExplicitBoundsList(dp.getExplicitBoundsList()) + .withMetricMetadata(getMetricMetadata(metric)) .withScope(ils) .withResource(resourceAttributes) .withAttributes(convertKeyValueToAttributes(dp.getAttributesList())) @@ -683,8 +700,12 @@ private List> mapExponentialHistogram( .withStartTime(convertUnixNanosToISO8601(dp.getStartTimeUnixNano())) .withTime(convertUnixNanosToISO8601(dp.getTimeUnixNano())) .withSum(dp.getSum()) + .withMin(dp.getMin()) + .withMax(dp.getMax()) .withCount(dp.getCount()) + .withMetricMetadata(getMetricMetadata(metric)) .withZeroCount(dp.getZeroCount()) + .withZeroThreshold(dp.getZeroThreshold()) .withScale(dp.getScale()) .withPositive(dp.getPositive().getBucketCountsList()) .withPositiveOffset(dp.getPositive().getOffset()) @@ -719,9 +740,15 @@ public ResourceSpans convertToResourceSpans(final Span span) throws UnsupportedE final Map allAttributes = span.getAttributes(); final Resource resource = constructResource(span.getResource()); rsBuilder.setResource(resource); + if (span.getResource() != null) { + rsBuilder.setSchemaUrl((String)span.getResource().get(SCHEMA_URL_KEY)); + } final ScopeSpans.Builder scopeSpansBuilder = ScopeSpans.newBuilder(); final InstrumentationScope instrumentationScope = constructInstrumentationScope(span.getScope()); scopeSpansBuilder.setScope(instrumentationScope); + if (span.getSchemaUrl() != null) { + scopeSpansBuilder.setSchemaUrl(span.getSchemaUrl()); + } final io.opentelemetry.proto.trace.v1.Span otelProtoSpan = constructSpan(span); scopeSpansBuilder.addSpans(otelProtoSpan); rsBuilder.addScopeSpans(scopeSpansBuilder); @@ -736,7 +763,7 @@ protected Resource constructResource(final Map resourceAttrs) th if (attributes != null) { resourceAttributes = convertAttributesToKeyValue(attributes); } - final Integer droppedAttributesCount = (Integer)resourceAttrs.get(DROPPED_ATTRIBUTES_COUNT_KEY); + final Integer droppedAttributesCount = (Integer)resourceAttrs.get(DROPPED_ATTRIBUTES_COUNT_KEY); if (droppedAttributesCount != null) { rsBuilder.setDroppedAttributesCount(droppedAttributesCount); } @@ -774,7 +801,7 @@ protected InstrumentationScope constructInstrumentationScope(Map if (version != null) { builder.setVersion(version); } - final Integer droppedAttributesCount = (Integer)scopeAttributes.get(DROPPED_ATTRIBUTES_COUNT_KEY); + final Integer droppedAttributesCount = (Integer)scopeAttributes.get(DROPPED_ATTRIBUTES_COUNT_KEY); if (droppedAttributesCount != null) { builder.setDroppedAttributesCount(droppedAttributesCount); } @@ -842,7 +869,7 @@ protected io.opentelemetry.proto.trace.v1.Span.Link convertSpanLink(final Link l return builder.build(); } - protected io.opentelemetry.proto.trace.v1.Span constructSpan(final Span span) + protected io.opentelemetry.proto.trace.v1.Span constructSpan(final Span span) throws DecoderException, UnsupportedEncodingException { final Map allAttributes = span.getAttributes(); io.opentelemetry.proto.trace.v1.Span.Builder builder = io.opentelemetry.proto.trace.v1.Span.newBuilder() @@ -880,7 +907,7 @@ protected AnyValue objectToAnyValue(final Object obj) throws UnsupportedEncoding anyValueBuilder.setDoubleValue((Double) obj); } else if (obj instanceof List) { List arrayValues = new ArrayList<>(); - + for (Object aobj : (List)obj) { arrayValues.add(objectToAnyValue(aobj)); } diff --git a/data-prepper-plugins/otel-proto-common/src/test/java/org/opensearch/dataprepper/plugins/otel/codec/OTelProtoOpensearchCodecTest.java b/data-prepper-plugins/otel-proto-common/src/test/java/org/opensearch/dataprepper/plugins/otel/codec/OTelProtoOpensearchCodecTest.java index 97950bc571..26d65cfd5b 100644 --- a/data-prepper-plugins/otel-proto-common/src/test/java/org/opensearch/dataprepper/plugins/otel/codec/OTelProtoOpensearchCodecTest.java +++ b/data-prepper-plugins/otel-proto-common/src/test/java/org/opensearch/dataprepper/plugins/otel/codec/OTelProtoOpensearchCodecTest.java @@ -466,7 +466,7 @@ private void validateLog(OpenTelemetryLog logRecord) { assertThat(logRecord.getObservedTime(), is("2020-05-24T14:00:02Z")); assertThat(logRecord.getBody(), is("Log value")); assertThat(logRecord.getDroppedAttributesCount(), is(3)); - assertThat(logRecord.getSchemaUrl(), is("schemaurl")); + assertThat(logRecord.getSchemaUrl(), is("resourceSchemaUrl")); assertThat(logRecord.getSeverityNumber(), is(5)); assertThat(logRecord.getSeverityText(), is("Severity value")); assertThat(logRecord.getTraceId(), is("ba1a1c23b4093b63")); diff --git a/data-prepper-plugins/otel-proto-common/src/test/java/org/opensearch/dataprepper/plugins/otel/codec/OTelProtoStandardCodecTest.java b/data-prepper-plugins/otel-proto-common/src/test/java/org/opensearch/dataprepper/plugins/otel/codec/OTelProtoStandardCodecTest.java index 59254056b4..7403789e7f 100644 --- a/data-prepper-plugins/otel-proto-common/src/test/java/org/opensearch/dataprepper/plugins/otel/codec/OTelProtoStandardCodecTest.java +++ b/data-prepper-plugins/otel-proto-common/src/test/java/org/opensearch/dataprepper/plugins/otel/codec/OTelProtoStandardCodecTest.java @@ -232,9 +232,11 @@ private void validateSpans(final List spans) { } Map resource = span.getResource(); assertThat(resource.containsKey(OTelProtoStandardCodec.ATTRIBUTES_KEY), is(true)); + assertThat((String)resource.get(OTelProtoStandardCodec.SCHEMA_URL_KEY), equalTo("resourceSchemaUrl")); Map attributes = (Map)resource.get(OTelProtoStandardCodec.ATTRIBUTES_KEY); assertThat(attributes.containsKey("service.name"), is(true)); Map scope = span.getScope(); + //assertThat((String)span.get(OTelProtoStandardCodec.SCHEMA_URL_KEY, String.class), equalTo("spansSchemaUrl")); assertThat(scope.containsKey(OTelProtoStandardCodec.NAME_KEY), is(true)); } } @@ -330,7 +332,7 @@ public void testKeyValueListAsResourceAttributes() throws JsonProcessingExceptio .setStringValue("EaglesService").build()).build(); final Map resourceAttributes = decoderUnderTest.getResourceAttributes(Resource.newBuilder() - .addAllAttributes(Arrays.asList(spanAttribute1, spanAttribute2)).build()); + .addAllAttributes(Arrays.asList(spanAttribute1, spanAttribute2)).build(), "resourceSchemaUrl"); final Map actual = (Map)resourceAttributes.get(OTelProtoStandardCodec.ATTRIBUTES_KEY); assertThat(actual.get(spanAttribute2.getKey()), equalTo(spanAttribute2.getValue().getStringValue())); assertThat(actual.containsKey(spanAttribute1.getKey()), is(true)); @@ -360,7 +362,7 @@ public void testArrayOfValueAsResourceAttributes() throws JsonProcessingExceptio .setArrayValue(arrayValue)).build(); final Map resource = decoderUnderTest.getResourceAttributes(Resource.newBuilder() - .addAllAttributes(Collections.singletonList(spanAttribute1)).build()); + .addAllAttributes(Collections.singletonList(spanAttribute1)).build(), "resourceSchemaUrl"); final Map actual = (Map)resource.get(OTelProtoStandardCodec.ATTRIBUTES_KEY); assertThat(actual.containsKey(spanAttribute1.getKey()), is(true)); final List actualValue = (List)actual.get(spanAttribute1.getKey()); @@ -439,11 +441,12 @@ private void validateLog(OpenTelemetryLog logRecord) { assertThat(logRecord.getObservedTime(), is("2020-05-24T14:00:02Z")); assertThat(logRecord.getBody(), is("Log value")); assertThat(logRecord.getDroppedAttributesCount(), is(3)); - assertThat(logRecord.getSchemaUrl(), is("schemaurl")); + assertThat(logRecord.getSchemaUrl(), is("logsSchemaUrl")); assertThat(logRecord.getSeverityNumber(), is(5)); assertThat(logRecord.getSeverityText(), is("Severity value")); assertThat(logRecord.getTraceId(), is("ba1a1c23b4093b63")); assertThat(logRecord.getSpanId(), is("2cc83ac90ebc469c")); + assertThat((String)logRecord.get(OTelProtoStandardCodec.SCHEMA_URL_KEY, String.class), equalTo("logsSchemaUrl")); Map scope = logRecord.getScope(); Map scopeAttributes = (Map)scope.get(OTelProtoStandardCodec.ATTRIBUTES_KEY); assertThat(scopeAttributes.get("my.scope.attribute"), is("log scope attribute")); @@ -454,7 +457,10 @@ private void validateLog(OpenTelemetryLog logRecord) { assertThat(attributes.get("statement.params"), is("us-east-1")); Map resource = logRecord.getResource(); Map resourceAttributes = (Map)resource.get(OTelProtoStandardCodec.ATTRIBUTES_KEY); + assertThat((String)resource.get(OTelProtoStandardCodec.SCHEMA_URL_KEY), equalTo("resourceSchemaUrl")); assertThat(resourceAttributes.get("service.name"), is("service")); + assertThat(resource.get(OTelProtoStandardCodec.DROPPED_ATTRIBUTES_COUNT_KEY), equalTo(0)); + assertThat(resource.get(OTelProtoStandardCodec.SCHEMA_URL_KEY), equalTo("resourceSchemaUrl")); } @Test @@ -498,12 +504,19 @@ private void validateGaugeMetricRequest(Collection> met assertThat(metric.getUnit(), equalTo("1")); assertThat(metric.getName(), equalTo("counter-int")); JacksonGauge gauge = (JacksonGauge)metric; + assertThat((String)gauge.get(OTelProtoStandardCodec.SCHEMA_URL_KEY, String.class), equalTo("metricsSchemaUrl")); + Map resource = gauge.getResource(); + assertThat((String)resource.get(OTelProtoStandardCodec.SCHEMA_URL_KEY), equalTo("resourceSchemaUrl")); assertThat(gauge.getValue(), equalTo(123.0)); Map scope = gauge.getScope(); Map scopeAttributes = (Map)scope.get(OTelProtoStandardCodec.ATTRIBUTES_KEY); assertThat(scopeAttributes.get("my.scope.attribute"), is("gauge scope attribute")); assertThat(scope.get(OTelProtoStandardCodec.NAME_KEY), is("my.library")); assertThat(scope.get(OTelProtoStandardCodec.VERSION_KEY), is("1.0.0")); + Map metadata = gauge.getMetricMetadata(); + assertThat(metadata.get("metadataKey1"), equalTo("metadataValue1")); + assertThat(metadata.get("metadataKey2"), equalTo(200L)); + assertThat(metadata.get("metadataKey3"), equalTo("metadataValue3")); } private void validateSumMetricRequest(Collection> metrics) { @@ -514,12 +527,19 @@ private void validateSumMetricRequest(Collection> metri assertThat(metric.getUnit(), equalTo("1")); assertThat(metric.getName(), equalTo("sum-int")); JacksonSum sum = (JacksonSum)metric; + assertThat((String)sum.get(OTelProtoStandardCodec.SCHEMA_URL_KEY, String.class), equalTo("metricsSchemaUrl")); + Map resource = sum.getResource(); + assertThat((String)resource.get(OTelProtoStandardCodec.SCHEMA_URL_KEY), equalTo("resourceSchemaUrl")); assertThat(sum.getValue(), equalTo(456.0)); Map scope = sum.getScope(); Map scopeAttributes = (Map)scope.get(OTelProtoStandardCodec.ATTRIBUTES_KEY); assertThat(scopeAttributes.get("my.scope.attribute"), is("sum scope attribute")); assertThat(scope.get(OTelProtoStandardCodec.NAME_KEY), is("my.library")); assertThat(scope.get(OTelProtoStandardCodec.VERSION_KEY), is("1.0.0")); + Map metadata = sum.getMetricMetadata(); + assertThat(metadata.get("metadataKey1"), equalTo("metadataValue1")); + assertThat(metadata.get("metadataKey2"), equalTo(200L)); + assertThat(metadata.get("metadataKey3"), equalTo("metadataValue3")); } private void validateHistogramMetricRequest(Collection> metrics) { @@ -530,8 +550,13 @@ private void validateHistogramMetricRequest(Collection> assertThat(metric.getUnit(), equalTo("1")); assertThat(metric.getName(), equalTo("histogram-int")); JacksonHistogram histogram = (JacksonHistogram)metric; + assertThat((String)histogram.get(OTelProtoStandardCodec.SCHEMA_URL_KEY, String.class), equalTo("metricsSchemaUrl")); + Map resource = histogram.getResource(); + assertThat((String)resource.get(OTelProtoStandardCodec.SCHEMA_URL_KEY), equalTo("resourceSchemaUrl")); assertThat(histogram.getSum(), equalTo(100.0)); assertThat(histogram.getCount(), equalTo(30L)); + assertThat(histogram.getMin(), equalTo(40.0)); + assertThat(histogram.getMax(), equalTo(60.0)); assertThat(histogram.getExemplars(), equalTo(Collections.emptyList())); assertThat(histogram.getExplicitBoundsList(), equalTo(List.of(1.0, 2.0, 3.0, 4.0))); assertThat(histogram.getExplicitBoundsCount(), equalTo(4)); @@ -543,6 +568,10 @@ private void validateHistogramMetricRequest(Collection> assertThat(scopeAttributes.get("my.scope.attribute"), is("histogram scope attribute")); assertThat(scope.get(OTelProtoStandardCodec.NAME_KEY), is("my.library")); assertThat(scope.get(OTelProtoStandardCodec.VERSION_KEY), is("1.0.0")); + Map metadata = histogram.getMetricMetadata(); + assertThat(metadata.get("metadataKey1"), equalTo("metadataValue1")); + assertThat(metadata.get("metadataKey2"), equalTo(200L)); + assertThat(metadata.get("metadataKey3"), equalTo("metadataValue3")); } private void validateHistogramMetricRequestNoExplicitBounds(Collection> metrics) { @@ -566,6 +595,10 @@ private void validateHistogramMetricRequestNoExplicitBounds(Collection metadata = histogram.getMetricMetadata(); + assertThat(metadata.get("metadataKey1"), equalTo("metadataValue1")); + assertThat(metadata.get("metadataKey2"), equalTo(200L)); + assertThat(metadata.get("metadataKey3"), equalTo("metadataValue3")); } diff --git a/data-prepper-plugins/otel-proto-common/src/test/resources/test-gauge-metrics.json b/data-prepper-plugins/otel-proto-common/src/test/resources/test-gauge-metrics.json index 0cdbad5be3..de939c4a5e 100644 --- a/data-prepper-plugins/otel-proto-common/src/test/resources/test-gauge-metrics.json +++ b/data-prepper-plugins/otel-proto-common/src/test/resources/test-gauge-metrics.json @@ -2,6 +2,7 @@ "resourceMetrics": [ { "resource": { + "droppedAttributesCount": 0, "attributes": [ { "key": "resource-attr", @@ -11,8 +12,10 @@ } ] }, + "schemaUrl": "resourceSchemaUrl", "scopeMetrics": [ { + "schemaUrl": "metricsSchemaUrl", "scope": { "name": "my.library", "version": "1.0.0", @@ -29,6 +32,26 @@ { "name": "counter-int", "unit": 1, + "metadata": [ + { + "key": "metadataKey1", + "value": { + "stringValue": "metadataValue1" + } + }, + { + "key": "metadataKey2", + "value": { + "intValue": "200" + } + }, + { + "key": "metadataKey3", + "value": { + "stringValue": "metadataValue3" + } + } + ], "gauge": { "dataPoints": [ { diff --git a/data-prepper-plugins/otel-proto-common/src/test/resources/test-histogram-metrics-no-explicit-bounds.json b/data-prepper-plugins/otel-proto-common/src/test/resources/test-histogram-metrics-no-explicit-bounds.json index 80c3f17f09..74d81a9bb2 100644 --- a/data-prepper-plugins/otel-proto-common/src/test/resources/test-histogram-metrics-no-explicit-bounds.json +++ b/data-prepper-plugins/otel-proto-common/src/test/resources/test-histogram-metrics-no-explicit-bounds.json @@ -2,6 +2,7 @@ "resourceMetrics": [ { "resource": { + "droppedAttributesCount": 0, "attributes": [ { "key": "resource-attr", @@ -11,8 +12,10 @@ } ] }, + "schemaUrl": "resourceSchemaUrl", "scopeMetrics": [ { + "schemaUrl": "metricsSchemaUrl", "scope": { "name": "my.library", "version": "1.0.0", @@ -29,6 +32,26 @@ { "name": "histogram-int", "unit": 1, + "metadata": [ + { + "key": "metadataKey1", + "value": { + "stringValue": "metadataValue1" + } + }, + { + "key": "metadataKey2", + "value": { + "intValue": "200" + } + }, + { + "key": "metadataKey3", + "value": { + "stringValue": "metadataValue3" + } + } + ], "histogram": { "dataPoints": [ { diff --git a/data-prepper-plugins/otel-proto-common/src/test/resources/test-histogram-metrics.json b/data-prepper-plugins/otel-proto-common/src/test/resources/test-histogram-metrics.json index 09246f0fb9..78458c1fc4 100644 --- a/data-prepper-plugins/otel-proto-common/src/test/resources/test-histogram-metrics.json +++ b/data-prepper-plugins/otel-proto-common/src/test/resources/test-histogram-metrics.json @@ -2,6 +2,7 @@ "resourceMetrics": [ { "resource": { + "droppedAttributesCount": 0, "attributes": [ { "key": "resource-attr", @@ -11,8 +12,10 @@ } ] }, + "schemaUrl": "resourceSchemaUrl", "scopeMetrics": [ { + "schemaUrl": "metricsSchemaUrl", "scope": { "name": "my.library", "version": "1.0.0", @@ -29,6 +32,26 @@ { "name": "histogram-int", "unit": 1, + "metadata": [ + { + "key": "metadataKey1", + "value": { + "stringValue": "metadataValue1" + } + }, + { + "key": "metadataKey2", + "value": { + "intValue": "200" + } + }, + { + "key": "metadataKey3", + "value": { + "stringValue": "metadataValue3" + } + } + ], "histogram": { "dataPoints": [ { @@ -44,6 +67,8 @@ "timeUnixNano": "1581452773000000789", "count": "30", "sum": "100", + "min": "40", + "max": "60", "bucket_counts": [3, 5, 15, 6, 1], "explicit_bounds": [1.0, 2.0, 3.0, 4.0], "exemplars": [] diff --git a/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-both-span-types.json b/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-both-span-types.json index 62559e23eb..02a11ab518 100644 --- a/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-both-span-types.json +++ b/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-both-span-types.json @@ -69,8 +69,10 @@ ], "droppedAttributesCount": 0 }, + "schemaUrl": "resourceSchemaUrl", "scopeSpans": [ { + "schemaUrl": "spansSchemaUrl", "scope": { "name": "io.opentelemetry.auto.spring-webmvc-3.1", "version": "" diff --git a/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-log.json b/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-log.json index bbb68bd061..c427e5944e 100644 --- a/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-log.json +++ b/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-log.json @@ -1,6 +1,7 @@ { "resourceLogs": [{ "resource": { + "droppedAttributesCount": 0, "attributes": [{ "key": "service.name", "value": { @@ -8,7 +9,9 @@ } }] }, + "schemaUrl": "resourceSchemaUrl", "scopeLogs": [{ + "schemaUrl": "logsSchemaUrl", "scope": { "name": "my.library", "version": "1.0.0", @@ -40,7 +43,6 @@ "spanId": "LMg6yQ68Rpw=", "observedTimeUnixNano": "1590328802000000000" }] - }], - "schemaUrl": "schemaurl" + }] }] } diff --git a/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-multiple-traces.json b/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-multiple-traces.json index 64cf75c2d3..2c58d26d7e 100644 --- a/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-multiple-traces.json +++ b/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-multiple-traces.json @@ -69,8 +69,10 @@ ], "droppedAttributesCount": 0 }, + "schemaUrl": "resourceSchemaUrl", "scopeSpans": [ { + "schemaUrl": "spansSchemaUrl", "scope": { "name": "io.opentelemetry.auto.spring-webmvc-3.1", "version": "" diff --git a/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-no-spans.json b/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-no-spans.json index c38b0d5875..bc9eb215e9 100644 --- a/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-no-spans.json +++ b/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-no-spans.json @@ -68,7 +68,8 @@ } ], "droppedAttributesCount": 0 - } + }, + "schemaUrl": "resourceSchemaUrl" } ] -} \ No newline at end of file +} diff --git a/data-prepper-plugins/otel-proto-common/src/test/resources/test-request.json b/data-prepper-plugins/otel-proto-common/src/test/resources/test-request.json index 2fdb687cdb..54d0b3c34b 100644 --- a/data-prepper-plugins/otel-proto-common/src/test/resources/test-request.json +++ b/data-prepper-plugins/otel-proto-common/src/test/resources/test-request.json @@ -69,8 +69,10 @@ ], "droppedAttributesCount": 0 }, + "schemaUrl": "resourceSchemaUrl", "scopeSpans": [ { + "schemaUrl": "spansSchemaUrl", "scope": { "name": "io.opentelemetry.auto.spring-webmvc-3.1", "version": "", diff --git a/data-prepper-plugins/otel-proto-common/src/test/resources/test-sum-metrics.json b/data-prepper-plugins/otel-proto-common/src/test/resources/test-sum-metrics.json index 77d4c73a51..6e77d9c062 100644 --- a/data-prepper-plugins/otel-proto-common/src/test/resources/test-sum-metrics.json +++ b/data-prepper-plugins/otel-proto-common/src/test/resources/test-sum-metrics.json @@ -2,6 +2,7 @@ "resourceMetrics": [ { "resource": { + "droppedAttributesCount": 0, "attributes": [ { "key": "resource-attr", @@ -11,8 +12,10 @@ } ] }, + "schemaUrl": "resourceSchemaUrl", "scopeMetrics": [ { + "schemaUrl": "metricsSchemaUrl", "scope": { "name": "my.library", "version": "1.0.0", @@ -29,6 +32,26 @@ { "name": "sum-int", "unit": 1, + "metadata": [ + { + "key": "metadataKey1", + "value": { + "stringValue": "metadataValue1" + } + }, + { + "key": "metadataKey2", + "value": { + "intValue": "200" + } + }, + { + "key": "metadataKey3", + "value": { + "stringValue": "metadataValue3" + } + } + ], "sum": { "dataPoints": [ {