Skip to content

Commit 20db890

Browse files
TomasLongoTomas Longo
TomasLongo
and
Tomas Longo
authored
Update otel proto buf specification (#5434)
* Update otel proto buf specification Signed-off-by: Tomas Longo <tomas.longo@sap.com> * Remove todos Signed-off-by: Tomas Longo <tomas.longo@sap.com> * remove last remnants of instrumentation library Signed-off-by: Tomas Longo <tomas.longo@sap.com> * Just test the new otel proto buf spec in the e2e raw span test Signed-off-by: Tomas Longo <tomas.longo@sap.com> --------- Signed-off-by: Tomas Longo <tomas.longo@sap.com> Co-authored-by: Tomas Longo <tomas.longo@sap.com>
1 parent 327a638 commit 20db890

29 files changed

Lines changed: 103 additions & 847 deletions

File tree

‎.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ jobs:
1414
strategy:
1515
matrix:
1616
java: [11, 17, 21, docker]
17-
otelVersion: ['0.9.0-alpha', '0.16.0-alpha']
17+
otelVersion: ['1.3.2-alpha']
1818
fail-fast: false
1919

2020
runs-on: ubuntu-latest

‎data-prepper-plugins/kafka-plugins/src/integrationTest/java/org/opensearch/dataprepper/plugins/kafka/buffer/KafkaBufferOTelIT.java‎

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,6 @@
4545
import io.opentelemetry.proto.resource.v1.Resource;
4646
import io.opentelemetry.proto.trace.v1.ScopeSpans;
4747
import io.opentelemetry.proto.trace.v1.ResourceSpans;
48-
import io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans;
49-
import io.opentelemetry.proto.common.v1.InstrumentationLibrary;
5048
import io.opentelemetry.proto.common.v1.InstrumentationScope;
5149
import io.opentelemetry.proto.metrics.v1.Gauge;
5250
import io.opentelemetry.proto.metrics.v1.Sum;
@@ -407,24 +405,9 @@ private ExportTraceServiceRequest createExportTraceRequest() {
407405
.build())
408406
.build();
409407

410-
final InstrumentationLibrarySpans ilSpans = InstrumentationLibrarySpans.newBuilder()
411-
.setInstrumentationLibrary(InstrumentationLibrary.newBuilder()
412-
.setName(ilName)
413-
.setVersion(ilVersion)
414-
.build())
415-
.addSpans(io.opentelemetry.proto.trace.v1.Span.newBuilder()
416-
.setTraceId(ByteString.copyFrom(TraceId2.getBytes()))
417-
.setSpanId(ByteString.copyFrom(SpanId2.getBytes()))
418-
.setKind(io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_INTERNAL)
419-
.setName(ilSpanName)
420-
.setStartTimeUnixNano(currentUnixTimeNano)
421-
.setEndTimeUnixNano(currentUnixTimeNano+TIME_DELTA*1000_000_000)
422-
.build())
423-
.build();
424408
ResourceSpans resourceSpans = ResourceSpans.newBuilder()
425409
.setResource(resource)
426410
.addScopeSpans(scopeSpans)
427-
.addInstrumentationLibrarySpans(ilSpans)
428411
.build();
429412

430413
return ExportTraceServiceRequest.newBuilder()

‎data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsGrpcServiceTest.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,10 @@
1212
import io.micrometer.core.instrument.Timer;
1313
import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest;
1414
import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceResponse;
15-
import io.opentelemetry.proto.logs.v1.InstrumentationLibraryLogs;
1615
import io.opentelemetry.proto.logs.v1.LogRecord;
1716
import io.opentelemetry.proto.logs.v1.ResourceLogs;
17+
import io.opentelemetry.proto.logs.v1.ScopeLogs;
18+
1819
import org.junit.jupiter.api.BeforeEach;
1920
import org.junit.jupiter.api.Test;
2021
import org.junit.jupiter.api.extension.ExtendWith;
@@ -61,9 +62,8 @@
6162
public class OTelLogsGrpcServiceTest {
6263
private static final ExportLogsServiceRequest LOGS_REQUEST = ExportLogsServiceRequest.newBuilder()
6364
.addResourceLogs(ResourceLogs.newBuilder()
64-
.addInstrumentationLibraryLogs(InstrumentationLibraryLogs.newBuilder()
65-
.addLogRecords(LogRecord.newBuilder())
66-
.build())).build();
65+
.addScopeLogs(ScopeLogs.newBuilder().addLogRecords(LogRecord.newBuilder()) .build()))
66+
.build();
6767

6868
private static PluginSetting pluginSetting;
6969
private final int bufferWriteTimeoutInMillis = 100000;

‎data-prepper-plugins/otel-metrics-raw-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/OTelMetricsProtoHelper.java‎

Lines changed: 0 additions & 110 deletions
Original file line numberDiff line numberDiff line change
@@ -8,30 +8,21 @@
88
import com.fasterxml.jackson.core.JsonProcessingException;
99
import com.fasterxml.jackson.databind.ObjectMapper;
1010
import io.opentelemetry.proto.common.v1.AnyValue;
11-
import io.opentelemetry.proto.common.v1.InstrumentationLibrary;
12-
import io.opentelemetry.proto.common.v1.InstrumentationScope;
1311
import io.opentelemetry.proto.common.v1.KeyValue;
1412
import io.opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint;
1513
import io.opentelemetry.proto.metrics.v1.NumberDataPoint;
16-
import io.opentelemetry.proto.metrics.v1.SummaryDataPoint;
17-
import io.opentelemetry.proto.resource.v1.Resource;
1814
import org.apache.commons.codec.binary.Hex;
1915
import org.opensearch.dataprepper.model.metric.Bucket;
2016
import org.opensearch.dataprepper.model.metric.DefaultBucket;
2117
import org.opensearch.dataprepper.model.metric.DefaultExemplar;
22-
import org.opensearch.dataprepper.model.metric.DefaultQuantile;
2318
import org.opensearch.dataprepper.model.metric.Exemplar;
24-
import org.opensearch.dataprepper.model.metric.Quantile;
2519
import org.slf4j.Logger;
2620
import org.slf4j.LoggerFactory;
2721

2822
import java.time.Instant;
2923
import java.util.ArrayList;
30-
import java.util.Collection;
31-
import java.util.HashMap;
3224
import java.util.List;
3325
import java.util.Map;
34-
import java.util.Optional;
3526
import java.util.concurrent.ConcurrentHashMap;
3627
import java.util.function.Function;
3728
import java.util.stream.Collectors;
@@ -40,14 +31,7 @@ public final class OTelMetricsProtoHelper {
4031

4132
private static final Logger LOG = LoggerFactory.getLogger(OTelMetricsProtoHelper.class);
4233
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
43-
private static final String SERVICE_NAME = "service.name";
44-
private static final String METRIC_ATTRIBUTES = "metric.attributes";
45-
static final String RESOURCE_ATTRIBUTES = "resource.attributes";
4634
static final String EXEMPLAR_ATTRIBUTES = "exemplar.attributes";
47-
static final String INSTRUMENTATION_LIBRARY_NAME = "instrumentationLibrary.name";
48-
static final String INSTRUMENTATION_LIBRARY_VERSION = "instrumentationLibrary.version";
49-
static final String INSTRUMENTATION_SCOPE_NAME = "instrumentationScope.name";
50-
static final String INSTRUMENTATION_SCOPE_VERSION = "instrumentationScope.version";
5135

5236
/**
5337
* To make it ES friendly we will replace '.' in keys with '@' in all the Keys in {@link io.opentelemetry.proto.common.v1.KeyValue}
@@ -60,8 +44,6 @@ public final class OTelMetricsProtoHelper {
6044
* Span and Resource attributes are essential for kibana so they should not be nested. SO we will prefix them with "metric.attributes"
6145
* and "resource.attributes" and "exemplar.attributes".
6246
*/
63-
public static final Function<String, String> PREFIX_AND_METRIC_ATTRIBUTES_REPLACE_DOT_WITH_AT = i -> METRIC_ATTRIBUTES + DOT + i.replace(DOT, AT);
64-
public static final Function<String, String> PREFIX_AND_RESOURCE_ATTRIBUTES_REPLACE_DOT_WITH_AT = i -> RESOURCE_ATTRIBUTES + DOT + i.replace(DOT, AT);
6547
public static final Function<String, String> PREFIX_AND_EXEMPLAR_ATTRIBUTES_REPLACE_DOT_WITH_AT = i -> EXEMPLAR_ATTRIBUTES + DOT + i.replace(DOT, AT);
6648

6749
private OTelMetricsProtoHelper() {
@@ -111,30 +93,6 @@ public static Object convertAnyValue(final AnyValue value) {
11193
}
11294
}
11395

114-
/**
115-
* Converts the keys of all attributes in the {@link NumberDataPoint}.
116-
* Also, casts the underlying data into its actual type
117-
*
118-
* @param numberDataPoint The point to process
119-
* @return A Map containing all attributes of `numberDataPoint` with keys converted into an OS-friendly format
120-
*/
121-
public static Map<String, Object> convertKeysOfDataPointAttributes(final NumberDataPoint numberDataPoint) {
122-
return numberDataPoint.getAttributesList().stream()
123-
.collect(Collectors.toMap(i -> PREFIX_AND_METRIC_ATTRIBUTES_REPLACE_DOT_WITH_AT.apply(i.getKey()), i -> convertAnyValue(i.getValue())));
124-
}
125-
126-
/**
127-
* Unpacks the List of {@link KeyValue} object into a Map.
128-
* Converts the keys into an os friendly format and casts the underlying data into its actual type?
129-
*
130-
* @param attributesList The list of {@link KeyValue} objects to process
131-
* @return A Map containing unpacked {@link KeyValue} data
132-
*/
133-
public static Map<String, Object> unpackKeyValueList(List<KeyValue> attributesList) {
134-
return attributesList.stream()
135-
.collect(Collectors.toMap(i -> PREFIX_AND_METRIC_ATTRIBUTES_REPLACE_DOT_WITH_AT.apply(i.getKey()), i -> convertAnyValue(i.getValue())));
136-
}
137-
13896
/**
13997
* Unpacks the List of {@link KeyValue} object into a Map.
14098
* Converts the keys into an os friendly format and casts the underlying data into its actual type?
@@ -184,79 +142,11 @@ public static Double getExemplarValueAsDouble(final io.opentelemetry.proto.metri
184142
}
185143
}
186144

187-
public static Map<String, Object> getResourceAttributes(final Resource resource) {
188-
return resource.getAttributesList().stream()
189-
.collect(Collectors.toMap(i -> PREFIX_AND_RESOURCE_ATTRIBUTES_REPLACE_DOT_WITH_AT.apply(i.getKey()), i -> convertAnyValue(i.getValue())));
190-
}
191-
192-
/**
193-
* Extracts the name and version of the used instrumentation library used
194-
*
195-
* @param instrumentationLibrary instrumentationLibrary
196-
* @return A map, containing information about the instrumentation library
197-
*/
198-
public static Map<String, Object> getInstrumentationLibraryAttributes(final InstrumentationLibrary instrumentationLibrary) {
199-
final Map<String, Object> instrumentationAttr = new HashMap<>();
200-
if (!instrumentationLibrary.getName().isEmpty()) {
201-
instrumentationAttr.put(INSTRUMENTATION_LIBRARY_NAME, instrumentationLibrary.getName());
202-
}
203-
if (!instrumentationLibrary.getVersion().isEmpty()) {
204-
instrumentationAttr.put(INSTRUMENTATION_LIBRARY_VERSION, instrumentationLibrary.getVersion());
205-
}
206-
return instrumentationAttr;
207-
}
208-
209-
/**
210-
* Extracts the name and version of the used instrumentation scope used
211-
*
212-
* @param instrumentationScope instrumentationScope
213-
* @return A map, containing information about the instrumentation scope
214-
*/
215-
public static Map<String, Object> getInstrumentationScopeAttributes(final InstrumentationScope instrumentationScope) {
216-
final Map<String, Object> instrumentationScopeAttr = new HashMap<>();
217-
if (!instrumentationScope.getName().isEmpty()) {
218-
instrumentationScopeAttr.put(INSTRUMENTATION_SCOPE_NAME, instrumentationScope.getName());
219-
}
220-
if (!instrumentationScope.getVersion().isEmpty()) {
221-
instrumentationScopeAttr.put(INSTRUMENTATION_SCOPE_VERSION, instrumentationScope.getVersion());
222-
}
223-
return instrumentationScopeAttr;
224-
}
225-
226145

227146
public static String convertUnixNanosToISO8601(final long unixNano) {
228147
return Instant.ofEpochSecond(0L, unixNano).toString();
229148
}
230149

231-
public static String getStartTimeISO8601(final NumberDataPoint numberDataPoint) {
232-
return convertUnixNanosToISO8601(numberDataPoint.getStartTimeUnixNano());
233-
}
234-
235-
public static String getTimeISO8601(final NumberDataPoint ndp) {
236-
return convertUnixNanosToISO8601(ndp.getTimeUnixNano());
237-
}
238-
239-
public static Optional<String> getServiceName(final Resource resource) {
240-
return resource.getAttributesList().stream()
241-
.filter(keyValue -> keyValue.getKey().equals(SERVICE_NAME) && !keyValue.getValue().getStringValue().isEmpty())
242-
.findFirst()
243-
.map(i -> i.getValue().getStringValue());
244-
}
245-
246-
247-
public static Map<String, Object> mergeAllAttributes(final Collection<Map<String, Object>> attributes) {
248-
return attributes.stream()
249-
.flatMap(map -> map.entrySet().stream())
250-
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
251-
}
252-
253-
254-
public static List<Quantile> getQuantileValues(List<SummaryDataPoint.ValueAtQuantile> quantileValues) {
255-
return quantileValues.stream()
256-
.map(q -> new DefaultQuantile(q.getQuantile(), q.getValue()))
257-
.collect(Collectors.toList());
258-
}
259-
260150
/**
261151
* Create the buckets, see <a href="https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/metrics/v1/metrics.proto">
262152
* the OTel metrics proto spec</a>

‎data-prepper-plugins/otel-metrics-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/MetricsPluginExponentialHistogramTest.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@
1212
import io.opentelemetry.proto.common.v1.KeyValue;
1313
import io.opentelemetry.proto.metrics.v1.ExponentialHistogram;
1414
import io.opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint;
15-
import io.opentelemetry.proto.metrics.v1.InstrumentationLibraryMetrics;
1615
import io.opentelemetry.proto.metrics.v1.ResourceMetrics;
16+
import io.opentelemetry.proto.metrics.v1.ScopeMetrics;
1717
import io.opentelemetry.proto.resource.v1.Resource;
1818
import org.hamcrest.MatcherAssert;
1919
import org.hamcrest.Matchers;
@@ -139,7 +139,7 @@ private ExportMetricsServiceRequest fillServiceRequest(ExponentialHistogram hist
139139
.setName("name")
140140
.setDescription("description")
141141
.build();
142-
InstrumentationLibraryMetrics instLib = InstrumentationLibraryMetrics.newBuilder()
142+
ScopeMetrics scopeMetrics = ScopeMetrics.newBuilder()
143143
.addMetrics(metric).build();
144144

145145
Resource resource = Resource.newBuilder()
@@ -149,7 +149,7 @@ private ExportMetricsServiceRequest fillServiceRequest(ExponentialHistogram hist
149149
).build();
150150
ResourceMetrics resourceMetrics = ResourceMetrics.newBuilder()
151151
.setResource(resource)
152-
.addInstrumentationLibraryMetrics(instLib)
152+
.addScopeMetrics(scopeMetrics)
153153
.build();
154154
return ExportMetricsServiceRequest.newBuilder().addResourceMetrics(resourceMetrics).build();
155155
}

‎data-prepper-plugins/otel-metrics-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/MetricsPluginGaugeTest.java‎

Lines changed: 0 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,10 @@
1010
import com.google.protobuf.ByteString;
1111
import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest;
1212
import io.opentelemetry.proto.common.v1.AnyValue;
13-
import io.opentelemetry.proto.common.v1.InstrumentationLibrary;
1413
import io.opentelemetry.proto.common.v1.InstrumentationScope;
1514
import io.opentelemetry.proto.common.v1.KeyValue;
1615
import io.opentelemetry.proto.metrics.v1.Exemplar;
1716
import io.opentelemetry.proto.metrics.v1.Gauge;
18-
import io.opentelemetry.proto.metrics.v1.InstrumentationLibraryMetrics;
1917
import io.opentelemetry.proto.metrics.v1.NumberDataPoint;
2018
import io.opentelemetry.proto.metrics.v1.ResourceMetrics;
2119
import io.opentelemetry.proto.metrics.v1.ScopeMetrics;
@@ -71,60 +69,6 @@ void init() {
7169
rawProcessor = new OTelMetricsRawProcessor(testsettings, new OtelMetricsRawProcessorConfig());
7270
}
7371

74-
@Test
75-
void testInstrumentationLibrary() throws JsonProcessingException {
76-
NumberDataPoint.Builder p1 = NumberDataPoint.newBuilder().setAsInt(4);
77-
Gauge gauge = Gauge.newBuilder().addDataPoints(p1).build();
78-
79-
io.opentelemetry.proto.metrics.v1.Metric.Builder metric = io.opentelemetry.proto.metrics.v1.Metric.newBuilder()
80-
.setGauge(gauge)
81-
.setUnit("seconds")
82-
.setName("name")
83-
.setDescription("description");
84-
85-
InstrumentationLibraryMetrics isntLib = InstrumentationLibraryMetrics.newBuilder()
86-
.addMetrics(metric)
87-
.setInstrumentationLibrary(InstrumentationLibrary.newBuilder()
88-
.setName("ilname")
89-
.setVersion("ilversion")
90-
.build())
91-
.build();
92-
93-
Resource resource = Resource.newBuilder()
94-
.addAttributes(KeyValue.newBuilder()
95-
.setKey("service.name")
96-
.setValue(AnyValue.newBuilder().setStringValue("service").build())
97-
).build();
98-
99-
ResourceMetrics resourceMetrics = ResourceMetrics.newBuilder()
100-
.addInstrumentationLibraryMetrics(isntLib)
101-
.setResource(resource)
102-
.build();
103-
104-
ExportMetricsServiceRequest exportMetricRequest = ExportMetricsServiceRequest.newBuilder()
105-
.addResourceMetrics(resourceMetrics).build();
106-
107-
Record<ExportMetricsServiceRequest> record = new Record<>(exportMetricRequest);
108-
109-
Collection<Record<? extends Metric>> records = rawProcessor.doExecute(Collections.singletonList(record));
110-
List<Record<? extends Metric>> list = new ArrayList<>(records);
111-
112-
Record<? extends Metric> dataPrepperResult = list.get(0);
113-
ObjectMapper objectMapper = new ObjectMapper();
114-
Map<String, Object> map = objectMapper.readValue(dataPrepperResult.getData().toJsonString(), Map.class);
115-
assertThat(map).contains(entry("kind", Metric.KIND.GAUGE.toString()));
116-
assertThat(map).contains(entry("unit", "seconds"));
117-
assertThat(map).contains(entry("serviceName", "service"));
118-
assertThat(map).contains(entry("resource.attributes.service@name", "service"));
119-
assertThat(map).contains(entry("description", "description"));
120-
assertThat(map).contains(entry("value", 4.0D));
121-
assertThat(map).contains(entry("startTime", "1970-01-01T00:00:00Z"));
122-
assertThat(map).contains(entry("time", "1970-01-01T00:00:00Z"));
123-
assertThat(map).contains(entry("instrumentationLibrary.name", "ilname"));
124-
assertThat(map).contains(entry("instrumentationLibrary.version", "ilversion"));
125-
126-
}
127-
12872
@Test
12973
void testScopeMetricsLibrary() throws JsonProcessingException {
13074
NumberDataPoint.Builder p1 = NumberDataPoint.newBuilder().setAsInt(4);

‎data-prepper-plugins/otel-metrics-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/MetricsPluginHistogramTest.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@
1212
import io.opentelemetry.proto.common.v1.KeyValue;
1313
import io.opentelemetry.proto.metrics.v1.Histogram;
1414
import io.opentelemetry.proto.metrics.v1.HistogramDataPoint;
15-
import io.opentelemetry.proto.metrics.v1.InstrumentationLibraryMetrics;
1615
import io.opentelemetry.proto.metrics.v1.ResourceMetrics;
16+
import io.opentelemetry.proto.metrics.v1.ScopeMetrics;
1717
import io.opentelemetry.proto.resource.v1.Resource;
1818
import org.junit.jupiter.api.BeforeEach;
1919
import org.junit.jupiter.api.Test;
@@ -119,7 +119,7 @@ private ExportMetricsServiceRequest fillServiceRequest(Histogram histogram) {
119119
.setName("name")
120120
.setDescription("description")
121121
.build();
122-
InstrumentationLibraryMetrics instLib = InstrumentationLibraryMetrics.newBuilder()
122+
ScopeMetrics scopeMetrics = ScopeMetrics.newBuilder()
123123
.addMetrics(metric).build();
124124

125125
Resource resource = Resource.newBuilder()
@@ -129,7 +129,7 @@ private ExportMetricsServiceRequest fillServiceRequest(Histogram histogram) {
129129
).build();
130130
ResourceMetrics resourceMetrics = ResourceMetrics.newBuilder()
131131
.setResource(resource)
132-
.addInstrumentationLibraryMetrics(instLib)
132+
.addScopeMetrics(scopeMetrics)
133133
.build();
134134
return ExportMetricsServiceRequest.newBuilder().addResourceMetrics(resourceMetrics).build();
135135
}

0 commit comments

Comments
 (0)