Skip to content

Commit 0d8b933

Browse files
trasktrask
committed
Add unit and integration tests for GenAI exception event extractor
1 parent 970aef2 commit 0d8b933

4 files changed

Lines changed: 136 additions & 13 deletions

File tree

‎instrumentation-api-incubator/src/test/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/internal/GenAiExceptionEventExtractorsTest.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
/*
2+
* Copyright The OpenTelemetry Authors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package io.opentelemetry.instrumentation.api.incubator.semconv.genai.internal;
7+
8+
import static io.opentelemetry.instrumentation.api.internal.SemconvExceptionSignal.emitExceptionAsLogs;
9+
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat;
10+
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo;
11+
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.satisfies;
12+
import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_MESSAGE;
13+
import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_STACKTRACE;
14+
import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_TYPE;
15+
16+
import io.opentelemetry.api.logs.Severity;
17+
import io.opentelemetry.context.Context;
18+
import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
19+
import io.opentelemetry.instrumentation.api.instrumenter.InstrumenterBuilder;
20+
import io.opentelemetry.sdk.logs.data.LogRecordData;
21+
import io.opentelemetry.sdk.testing.junit5.OpenTelemetryExtension;
22+
import java.util.List;
23+
import org.assertj.core.api.AbstractAssert;
24+
import org.junit.jupiter.api.Test;
25+
import org.junit.jupiter.api.extension.RegisterExtension;
26+
27+
class GenAiExceptionEventExtractorsTest {
28+
29+
@RegisterExtension
30+
static final OpenTelemetryExtension otelTesting = OpenTelemetryExtension.create();
31+
32+
@Test
33+
void genAiClientExceptionLog() {
34+
InstrumenterBuilder<String, String> builder =
35+
Instrumenter.builder(otelTesting.getOpenTelemetry(), "test", unused -> "span");
36+
GenAiExceptionEventExtractors.setGenAiClientExceptionEventExtractor(builder);
37+
Instrumenter<String, String> instrumenter = builder.buildInstrumenter();
38+
39+
Context context = instrumenter.start(Context.root(), "request");
40+
IllegalStateException error = new IllegalStateException("test");
41+
instrumenter.end(context, "request", "response", error);
42+
43+
List<LogRecordData> logs = otelTesting.getLogRecords();
44+
if (emitExceptionAsLogs()) {
45+
assertThat(logs).hasSize(1);
46+
assertThat(logs.get(0))
47+
.hasSeverity(Severity.WARN)
48+
.hasEventName("gen_ai.client.operation.exception")
49+
.hasAttributesSatisfyingExactly(
50+
equalTo(EXCEPTION_TYPE, "java.lang.IllegalStateException"),
51+
equalTo(EXCEPTION_MESSAGE, "test"),
52+
satisfies(EXCEPTION_STACKTRACE, AbstractAssert::isNotNull));
53+
} else {
54+
assertThat(logs).isEmpty();
55+
}
56+
}
57+
}

‎instrumentation/openai/openai-java-1.1/javaagent/build.gradle.kts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,4 +29,19 @@ tasks {
2929
systemProperty("otel.instrumentation.genai.capture-message-content", "true")
3030
systemProperty("collectMetadata", otelProps.collectMetadata)
3131
}
32+
33+
val testExceptionSignalLogs by registering(Test::class) {
34+
testClassesDirs = sourceSets.test.get().output.classesDirs
35+
classpath = sourceSets.test.get().runtimeClasspath
36+
37+
systemProperty("otel.instrumentation.genai.capture-message-content", "true")
38+
filter {
39+
includeTestsMatching("EmbeddingsTest")
40+
}
41+
jvmArgs("-Dotel.semconv.exception.signal.preview=logs")
42+
}
43+
44+
check {
45+
dependsOn(testExceptionSignalLogs)
46+
}
3247
}

‎instrumentation/openai/openai-java-1.1/library/build.gradle.kts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,18 @@ tasks {
1313
test {
1414
systemProperty("testLatestDeps", otelProps.testLatestDeps)
1515
}
16+
17+
val testExceptionSignalLogs by registering(Test::class) {
18+
testClassesDirs = sourceSets.test.get().output.classesDirs
19+
classpath = sourceSets.test.get().runtimeClasspath
20+
21+
filter {
22+
includeTestsMatching("EmbeddingsTest")
23+
}
24+
jvmArgs("-Dotel.semconv.exception.signal.preview=logs")
25+
}
26+
27+
check {
28+
dependsOn(testExceptionSignalLogs)
29+
}
1630
}

‎instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractEmbeddingsTest.java‎

Lines changed: 50 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,14 @@
55

66
package io.opentelemetry.instrumentation.openai.v1_1;
77

8+
import static io.opentelemetry.instrumentation.api.internal.SemconvExceptionSignal.emitExceptionAsLogs;
9+
import static io.opentelemetry.instrumentation.api.internal.SemconvExceptionSignal.emitExceptionAsSpanEvents;
10+
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat;
811
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo;
912
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.satisfies;
13+
import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_MESSAGE;
14+
import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_STACKTRACE;
15+
import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_TYPE;
1016
import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GEN_AI_OPERATION_NAME;
1117
import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GEN_AI_PROVIDER_NAME;
1218
import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GEN_AI_REQUEST_ENCODING_FORMATS;
@@ -18,6 +24,7 @@
1824
import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GenAiProviderNameIncubatingValues.OPENAI;
1925
import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GenAiTokenTypeIncubatingValues.INPUT;
2026
import static java.util.Collections.singletonList;
27+
import static java.util.stream.Collectors.toList;
2128
import static org.assertj.core.api.Assertions.assertThat;
2229
import static org.assertj.core.api.Assertions.catchThrowable;
2330

@@ -28,11 +35,15 @@
2835
import com.openai.errors.OpenAIIoException;
2936
import com.openai.models.embeddings.CreateEmbeddingResponse;
3037
import com.openai.models.embeddings.EmbeddingCreateParams;
38+
import io.opentelemetry.api.logs.Severity;
3139
import io.opentelemetry.api.trace.Span;
3240
import io.opentelemetry.api.trace.SpanKind;
3341
import io.opentelemetry.context.Context;
3442
import io.opentelemetry.instrumentation.testing.internal.AutoCleanupExtension;
43+
import io.opentelemetry.sdk.logs.data.LogRecordData;
44+
import java.util.List;
3545
import java.util.concurrent.CompletionException;
46+
import org.awaitility.Awaitility;
3647
import org.junit.jupiter.api.Test;
3748
import org.junit.jupiter.api.extension.RegisterExtension;
3849

@@ -239,19 +250,26 @@ void connectionError() {
239250
trace ->
240251
trace.hasSpansSatisfyingExactly(
241252
maybeWithTransportSpan(
242-
span ->
243-
span.hasName("embeddings text-embedding-3-small")
244-
.hasKind(SpanKind.CLIENT)
245-
.hasException(thrown)
246-
.hasAttributesSatisfyingExactly(
247-
equalTo(GEN_AI_PROVIDER_NAME, OPENAI),
248-
equalTo(GEN_AI_OPERATION_NAME, EMBEDDINGS),
249-
equalTo(GEN_AI_REQUEST_MODEL, MODEL),
250-
// Newer versions of the library populate base64 when unset by
251-
// the user.
252-
satisfies(
253-
GEN_AI_REQUEST_ENCODING_FORMATS,
254-
val -> val.isIn(singletonList("base64"), null))))));
253+
span -> {
254+
span.hasName("embeddings text-embedding-3-small")
255+
.hasKind(SpanKind.CLIENT)
256+
.hasAttributesSatisfyingExactly(
257+
equalTo(GEN_AI_PROVIDER_NAME, OPENAI),
258+
equalTo(GEN_AI_OPERATION_NAME, EMBEDDINGS),
259+
equalTo(GEN_AI_REQUEST_MODEL, MODEL),
260+
// Newer versions of the library populate base64 when unset by
261+
// the user.
262+
satisfies(
263+
GEN_AI_REQUEST_ENCODING_FORMATS,
264+
val -> val.isIn(singletonList("base64"), null)));
265+
if (emitExceptionAsSpanEvents()) {
266+
span.hasException(thrown);
267+
}
268+
})));
269+
270+
if (emitExceptionAsLogs()) {
271+
assertClientExceptionLog();
272+
}
255273

256274
getTesting()
257275
.waitAndAssertMetrics(
@@ -270,4 +288,23 @@ void connectionError() {
270288
equalTo(GEN_AI_OPERATION_NAME, EMBEDDINGS),
271289
equalTo(GEN_AI_REQUEST_MODEL, MODEL)))));
272290
}
291+
292+
private void assertClientExceptionLog() {
293+
Awaitility.await()
294+
.untilAsserted(
295+
() -> {
296+
List<LogRecordData> logs =
297+
getTesting().logRecords().stream()
298+
.filter(log -> "gen_ai.client.operation.exception".equals(log.getEventName()))
299+
.collect(toList());
300+
assertThat(logs).hasSize(1);
301+
assertThat(logs.get(0))
302+
.hasSeverity(Severity.WARN)
303+
.hasEventName("gen_ai.client.operation.exception")
304+
.hasAttributesSatisfyingExactly(
305+
satisfies(EXCEPTION_TYPE, val -> val.isNotNull()),
306+
satisfies(EXCEPTION_MESSAGE, val -> val.isNotNull()),
307+
satisfies(EXCEPTION_STACKTRACE, val -> val.isNotNull()));
308+
});
309+
}
273310
}

0 commit comments

Comments
 (0)