Skip to content

Commit a7571ea

Browse files
traskCopilot
trask
and
Copilot
authored
Emit GenAI exceptions as logs under the opt-in preview (#18893)
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
1 parent 6bc03b7 commit a7571ea

10 files changed

Lines changed: 234 additions & 27 deletions

File tree

‎instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/internal/GenAiExceptionEventExtractors.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/*
2+
* Copyright The OpenTelemetry Authors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package io.opentelemetry.instrumentation.api.incubator.semconv.genai.internal;
7+
8+
import io.opentelemetry.api.logs.Severity;
9+
import io.opentelemetry.instrumentation.api.instrumenter.InstrumenterBuilder;
10+
import io.opentelemetry.instrumentation.api.internal.Experimental;
11+
12+
/**
13+
* This class is internal and is hence not for public use. Its APIs are unstable and can change at
14+
* any time.
15+
*/
16+
public final class GenAiExceptionEventExtractors {
17+
18+
/**
19+
* Configures the GenAI client operation exception event name and severity. Only takes effect when
20+
* emitting exceptions as logs is enabled via the {@code otel.semconv.exception.signal.preview}
21+
* flag.
22+
*/
23+
public static <REQUEST> void setGenAiClientExceptionEventExtractor(
24+
InstrumenterBuilder<REQUEST, ?> builder) {
25+
Experimental.setExceptionEventExtractor(
26+
builder,
27+
(logRecordBuilder, context, request) -> {
28+
logRecordBuilder.setEventName("gen_ai.client.operation.exception");
29+
logRecordBuilder.setSeverity(Severity.WARN);
30+
});
31+
}
32+
33+
private GenAiExceptionEventExtractors() {}
34+
}

‎instrumentation-api-incubator/src/test/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/internal/GenAiExceptionEventExtractorsTest.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
/*
2+
* Copyright The OpenTelemetry Authors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package io.opentelemetry.instrumentation.api.incubator.semconv.genai.internal;
7+
8+
import static io.opentelemetry.instrumentation.api.internal.SemconvExceptionSignal.emitExceptionAsLogs;
9+
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat;
10+
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo;
11+
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.satisfies;
12+
import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_MESSAGE;
13+
import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_STACKTRACE;
14+
import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_TYPE;
15+
16+
import io.opentelemetry.api.logs.Severity;
17+
import io.opentelemetry.context.Context;
18+
import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
19+
import io.opentelemetry.instrumentation.api.instrumenter.InstrumenterBuilder;
20+
import io.opentelemetry.sdk.logs.data.LogRecordData;
21+
import io.opentelemetry.sdk.testing.junit5.OpenTelemetryExtension;
22+
import java.util.List;
23+
import org.assertj.core.api.AbstractAssert;
24+
import org.junit.jupiter.api.Test;
25+
import org.junit.jupiter.api.extension.RegisterExtension;
26+
27+
class GenAiExceptionEventExtractorsTest {
28+
29+
@RegisterExtension
30+
static final OpenTelemetryExtension otelTesting = OpenTelemetryExtension.create();
31+
32+
@Test
33+
void genAiClientExceptionLog() {
34+
InstrumenterBuilder<String, String> builder =
35+
Instrumenter.builder(otelTesting.getOpenTelemetry(), "test", unused -> "span");
36+
GenAiExceptionEventExtractors.setGenAiClientExceptionEventExtractor(builder);
37+
Instrumenter<String, String> instrumenter = builder.buildInstrumenter();
38+
39+
Context context = instrumenter.start(Context.root(), "request");
40+
IllegalStateException error = new IllegalStateException("test");
41+
instrumenter.end(context, "request", "response", error);
42+
43+
List<LogRecordData> logs = otelTesting.getLogRecords();
44+
if (emitExceptionAsLogs()) {
45+
assertThat(logs).hasSize(1);
46+
assertThat(logs.get(0))
47+
.hasSeverity(Severity.WARN)
48+
.hasEventName("gen_ai.client.operation.exception")
49+
.hasAttributesSatisfyingExactly(
50+
equalTo(EXCEPTION_TYPE, "java.lang.IllegalStateException"),
51+
equalTo(EXCEPTION_MESSAGE, "test"),
52+
satisfies(EXCEPTION_STACKTRACE, AbstractAssert::isNotNull));
53+
} else {
54+
assertThat(logs).isEmpty();
55+
}
56+
}
57+
}

‎instrumentation/openai/openai-java-1.1/javaagent/build.gradle.kts‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,22 @@ dependencies {
2222
}
2323

2424
tasks {
25-
test {
25+
withType<Test>().configureEach {
2626
systemProperty("testLatestDeps", otelProps.testLatestDeps)
2727
// TODO run tests both with and without genai message capture
2828

2929
systemProperty("otel.instrumentation.genai.capture-message-content", "true")
3030
systemProperty("collectMetadata", otelProps.collectMetadata)
3131
}
32+
33+
val testExceptionSignalLogs by registering(Test::class) {
34+
testClassesDirs = sourceSets.test.get().output.classesDirs
35+
classpath = sourceSets.test.get().runtimeClasspath
36+
jvmArgs("-Dotel.semconv.exception.signal.preview=logs")
37+
systemProperty("metadataConfig", "otel.semconv.exception.signal.preview=logs")
38+
}
39+
40+
check {
41+
dependsOn(testExceptionSignalLogs)
42+
}
3243
}

‎instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/ChatTest.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,20 @@
55

66
package io.opentelemetry.javaagent.instrumentation.openai.v1_1;
77

8+
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo;
9+
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.satisfies;
10+
import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_MESSAGE;
11+
import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_STACKTRACE;
12+
import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_TYPE;
13+
import static java.util.Collections.singletonList;
14+
815
import com.openai.client.OpenAIClient;
916
import com.openai.client.OpenAIClientAsync;
17+
import io.opentelemetry.api.logs.Severity;
1018
import io.opentelemetry.instrumentation.openai.v1_1.AbstractChatTest;
1119
import io.opentelemetry.instrumentation.testing.junit.AgentInstrumentationExtension;
1220
import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension;
21+
import io.opentelemetry.sdk.testing.assertj.LogRecordDataAssert;
1322
import io.opentelemetry.sdk.testing.assertj.SpanDataAssert;
1423
import java.util.ArrayList;
1524
import java.util.List;
@@ -45,4 +54,18 @@ protected List<Consumer<SpanDataAssert>> maybeWithTransportSpan(Consumer<SpanDat
4554
result.add(s -> s.hasName("POST"));
4655
return result;
4756
}
57+
58+
@Override
59+
protected List<Consumer<LogRecordDataAssert>> transportExceptionLogs() {
60+
return singletonList(
61+
transportLog ->
62+
transportLog
63+
.hasSeverity(Severity.WARN)
64+
.hasEventName("http.client.request.exception")
65+
.hasAttributesSatisfyingExactly(
66+
equalTo(EXCEPTION_TYPE, "java.net.ConnectException"),
67+
satisfies(
68+
EXCEPTION_MESSAGE, val -> val.startsWith("Failed to connect to localhost")),
69+
satisfies(EXCEPTION_STACKTRACE, val -> val.isNotNull())));
70+
}
4871
}

‎instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/EmbeddingsTest.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,20 @@
55

66
package io.opentelemetry.javaagent.instrumentation.openai.v1_1;
77

8+
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo;
9+
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.satisfies;
10+
import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_MESSAGE;
11+
import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_STACKTRACE;
12+
import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_TYPE;
13+
import static java.util.Collections.singletonList;
14+
815
import com.openai.client.OpenAIClient;
916
import com.openai.client.OpenAIClientAsync;
17+
import io.opentelemetry.api.logs.Severity;
1018
import io.opentelemetry.instrumentation.openai.v1_1.AbstractEmbeddingsTest;
1119
import io.opentelemetry.instrumentation.testing.junit.AgentInstrumentationExtension;
1220
import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension;
21+
import io.opentelemetry.sdk.testing.assertj.LogRecordDataAssert;
1322
import io.opentelemetry.sdk.testing.assertj.SpanDataAssert;
1423
import java.util.ArrayList;
1524
import java.util.List;
@@ -45,4 +54,18 @@ protected List<Consumer<SpanDataAssert>> maybeWithTransportSpan(Consumer<SpanDat
4554
result.add(s -> s.hasName("POST"));
4655
return result;
4756
}
57+
58+
@Override
59+
protected List<Consumer<LogRecordDataAssert>> transportExceptionLogs() {
60+
return singletonList(
61+
transportLog ->
62+
transportLog
63+
.hasSeverity(Severity.WARN)
64+
.hasEventName("http.client.request.exception")
65+
.hasAttributesSatisfyingExactly(
66+
equalTo(EXCEPTION_TYPE, "java.net.ConnectException"),
67+
satisfies(
68+
EXCEPTION_MESSAGE, val -> val.startsWith("Failed to connect to localhost")),
69+
satisfies(EXCEPTION_STACKTRACE, val -> val.isNotNull())));
70+
}
4871
}

‎instrumentation/openai/openai-java-1.1/library/build.gradle.kts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,17 @@ dependencies {
1010
}
1111

1212
tasks {
13-
test {
13+
withType<Test>().configureEach {
1414
systemProperty("testLatestDeps", otelProps.testLatestDeps)
1515
}
16+
17+
val testExceptionSignalLogs by registering(Test::class) {
18+
testClassesDirs = sourceSets.test.get().output.classesDirs
19+
classpath = sourceSets.test.get().runtimeClasspath
20+
jvmArgs("-Dotel.semconv.exception.signal.preview=logs")
21+
}
22+
23+
check {
24+
dependsOn(testExceptionSignalLogs)
25+
}
1626
}

‎instrumentation/openai/openai-java-1.1/library/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/OpenAITelemetryBuilder.java‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55

66
package io.opentelemetry.instrumentation.openai.v1_1;
77

8+
import static io.opentelemetry.instrumentation.api.incubator.semconv.genai.internal.GenAiExceptionEventExtractors.setGenAiClientExceptionEventExtractor;
9+
810
import com.google.errorprone.annotations.CanIgnoreReturnValue;
911
import com.openai.models.chat.completions.ChatCompletion;
1012
import com.openai.models.chat.completions.ChatCompletionCreateParams;
@@ -16,6 +18,7 @@
1618
import io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiClientMetrics;
1719
import io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiSpanNameExtractor;
1820
import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
21+
import io.opentelemetry.instrumentation.api.instrumenter.InstrumenterBuilder;
1922
import io.opentelemetry.instrumentation.api.instrumenter.SpanKindExtractor;
2023

2124
/** A builder of {@link OpenAITelemetry}. */
@@ -48,24 +51,28 @@ public OpenAITelemetryBuilder setCaptureMessageContent(boolean captureMessageCon
4851
*/
4952
public OpenAITelemetry build() {
5053
ChatAttributesGetter chatAttributesGetter = new ChatAttributesGetter();
51-
Instrumenter<ChatCompletionCreateParams, ChatCompletion> chatInstrumenter =
54+
InstrumenterBuilder<ChatCompletionCreateParams, ChatCompletion> chatBuilder =
5255
Instrumenter.<ChatCompletionCreateParams, ChatCompletion>builder(
5356
openTelemetry,
5457
INSTRUMENTATION_NAME,
5558
GenAiSpanNameExtractor.create(chatAttributesGetter))
5659
.addAttributesExtractor(GenAiAttributesExtractor.create(chatAttributesGetter))
57-
.addOperationMetrics(GenAiClientMetrics.get())
58-
.buildInstrumenter();
60+
.addOperationMetrics(GenAiClientMetrics.get());
61+
setGenAiClientExceptionEventExtractor(chatBuilder);
62+
Instrumenter<ChatCompletionCreateParams, ChatCompletion> chatInstrumenter =
63+
chatBuilder.buildInstrumenter();
5964

6065
EmbeddingAttributesGetter embeddingAttributesGetter = new EmbeddingAttributesGetter();
61-
Instrumenter<EmbeddingCreateParams, CreateEmbeddingResponse> embeddingsInstrumenter =
66+
InstrumenterBuilder<EmbeddingCreateParams, CreateEmbeddingResponse> embeddingsBuilder =
6267
Instrumenter.<EmbeddingCreateParams, CreateEmbeddingResponse>builder(
6368
openTelemetry,
6469
INSTRUMENTATION_NAME,
6570
GenAiSpanNameExtractor.create(embeddingAttributesGetter))
6671
.addAttributesExtractor(GenAiAttributesExtractor.create(embeddingAttributesGetter))
67-
.addOperationMetrics(GenAiClientMetrics.get())
68-
.buildInstrumenter(SpanKindExtractor.alwaysClient());
72+
.addOperationMetrics(GenAiClientMetrics.get());
73+
setGenAiClientExceptionEventExtractor(embeddingsBuilder);
74+
Instrumenter<EmbeddingCreateParams, CreateEmbeddingResponse> embeddingsInstrumenter =
75+
embeddingsBuilder.buildInstrumenter(SpanKindExtractor.alwaysClient());
6976

7077
Logger eventLogger = openTelemetry.getLogsBridge().get(INSTRUMENTATION_NAME);
7178
return new OpenAITelemetry(

‎instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractChatTest.java‎

Lines changed: 21 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
package io.opentelemetry.instrumentation.openai.v1_1;
77

88
import static io.opentelemetry.api.common.AttributeKey.stringKey;
9+
import static io.opentelemetry.instrumentation.api.internal.SemconvExceptionSignal.emitExceptionAsSpanEvents;
910
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo;
1011
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.satisfies;
1112
import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GEN_AI_OPERATION_NAME;
@@ -67,12 +68,14 @@
6768
import io.opentelemetry.context.Context;
6869
import io.opentelemetry.instrumentation.openai.TestHelper;
6970
import io.opentelemetry.instrumentation.testing.internal.AutoCleanupExtension;
71+
import io.opentelemetry.sdk.testing.assertj.LogRecordDataAssert;
7072
import java.util.ArrayList;
7173
import java.util.HashMap;
7274
import java.util.List;
7375
import java.util.Map;
7476
import java.util.Optional;
7577
import java.util.concurrent.CompletionException;
78+
import java.util.function.Consumer;
7679
import org.junit.jupiter.api.Test;
7780
import org.junit.jupiter.api.extension.RegisterExtension;
7881

@@ -905,7 +908,7 @@ void connectionError() {
905908
trace.hasSpansSatisfyingExactly(
906909
maybeWithTransportSpan(
907910
span ->
908-
span.hasException(thrown)
911+
span.hasException(emitExceptionAsSpanEvents() ? thrown : null)
909912
.hasAttributesSatisfyingExactly(
910913
equalTo(GEN_AI_PROVIDER_NAME, OPENAI),
911914
equalTo(GEN_AI_OPERATION_NAME, CHAT),
@@ -930,14 +933,7 @@ void connectionError() {
930933

931934
SpanContext spanCtx = getTesting().waitForTraces(1).get(0).get(0).getSpanContext();
932935

933-
getTesting()
934-
.waitAndAssertLogRecords(
935-
log ->
936-
log.hasAttributesSatisfyingExactly(
937-
equalTo(GEN_AI_PROVIDER_NAME, OPENAI),
938-
equalTo(stringKey("event.name"), "gen_ai.user.message"))
939-
.hasSpanContext(spanCtx)
940-
.hasBody(Value.of(KeyValue.of("content", Value.of(TEST_CHAT_INPUT)))));
936+
getTesting().waitAndAssertLogRecords(connectionErrorLogs(spanCtx, thrown));
941937
}
942938

943939
@Test
@@ -1618,7 +1614,7 @@ void streamConnectionError() {
16181614
trace.hasSpansSatisfyingExactly(
16191615
maybeWithTransportSpan(
16201616
span ->
1621-
span.hasException(thrown)
1617+
span.hasException(emitExceptionAsSpanEvents() ? thrown : null)
16221618
.hasAttributesSatisfyingExactly(
16231619
equalTo(GEN_AI_PROVIDER_NAME, OPENAI),
16241620
equalTo(GEN_AI_OPERATION_NAME, CHAT),
@@ -1643,14 +1639,21 @@ void streamConnectionError() {
16431639

16441640
SpanContext spanCtx = getTesting().waitForTraces(1).get(0).get(0).getSpanContext();
16451641

1646-
getTesting()
1647-
.waitAndAssertLogRecords(
1648-
log ->
1649-
log.hasAttributesSatisfyingExactly(
1650-
equalTo(GEN_AI_PROVIDER_NAME, OPENAI),
1651-
equalTo(stringKey("event.name"), "gen_ai.user.message"))
1652-
.hasSpanContext(spanCtx)
1653-
.hasBody(Value.of(KeyValue.of("content", Value.of(TEST_CHAT_INPUT)))));
1642+
getTesting().waitAndAssertLogRecords(connectionErrorLogs(spanCtx, thrown));
1643+
}
1644+
1645+
private List<Consumer<LogRecordDataAssert>> connectionErrorLogs(
1646+
SpanContext spanCtx, Throwable thrown) {
1647+
List<Consumer<LogRecordDataAssert>> assertions = new ArrayList<>();
1648+
assertions.add(
1649+
log ->
1650+
log.hasAttributesSatisfyingExactly(
1651+
equalTo(GEN_AI_PROVIDER_NAME, OPENAI),
1652+
equalTo(stringKey("event.name"), "gen_ai.user.message"))
1653+
.hasSpanContext(spanCtx)
1654+
.hasBody(Value.of(KeyValue.of("content", Value.of(TEST_CHAT_INPUT)))));
1655+
assertions.addAll(genAiClientExceptionLogs(thrown));
1656+
return assertions;
16541657
}
16551658

16561659
protected static ChatCompletionMessageParam createUserMessage(String content) {

‎instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractEmbeddingsTest.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55

66
package io.opentelemetry.instrumentation.openai.v1_1;
77

8+
import static io.opentelemetry.instrumentation.api.internal.SemconvExceptionSignal.emitExceptionAsSpanEvents;
9+
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat;
810
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo;
911
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.satisfies;
1012
import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GEN_AI_OPERATION_NAME;
@@ -242,7 +244,7 @@ void connectionError() {
242244
span ->
243245
span.hasName("embeddings text-embedding-3-small")
244246
.hasKind(SpanKind.CLIENT)
245-
.hasException(thrown)
247+
.hasException(emitExceptionAsSpanEvents() ? thrown : null)
246248
.hasAttributesSatisfyingExactly(
247249
equalTo(GEN_AI_PROVIDER_NAME, OPENAI),
248250
equalTo(GEN_AI_OPERATION_NAME, EMBEDDINGS),
@@ -253,6 +255,8 @@ void connectionError() {
253255
GEN_AI_REQUEST_ENCODING_FORMATS,
254256
val -> val.isIn(singletonList("base64"), null))))));
255257

258+
getTesting().waitAndAssertLogRecords(genAiClientExceptionLogs(thrown));
259+
256260
getTesting()
257261
.waitAndAssertMetrics(
258262
INSTRUMENTATION_NAME,

0 commit comments

Comments
 (0)