From eff57222ba7dca35928b93fabcb6abb0a01916f8 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Wed, 3 Jun 2026 11:31:24 -0700 Subject: [PATCH 01/11] Add GenAI exception event extractor helper --- .../GenAiExceptionEventExtractors.java | 38 +++++++++++++++++++ .../openai/v1_1/OpenAITelemetryBuilder.java | 29 ++++++++------ 2 files changed, 55 insertions(+), 12 deletions(-) create mode 100644 instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/internal/GenAiExceptionEventExtractors.java diff --git a/instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/internal/GenAiExceptionEventExtractors.java b/instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/internal/GenAiExceptionEventExtractors.java new file mode 100644 index 000000000000..d0281951139d --- /dev/null +++ b/instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/internal/GenAiExceptionEventExtractors.java @@ -0,0 +1,38 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.instrumentation.api.incubator.semconv.genai.internal; + +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import io.opentelemetry.api.logs.Severity; +import io.opentelemetry.instrumentation.api.instrumenter.InstrumenterBuilder; +import io.opentelemetry.instrumentation.api.internal.Experimental; + +/** + * This class is internal and is hence not for public use. Its APIs are unstable and can change at + * any time. + */ +public final class GenAiExceptionEventExtractors { + + /** + * Configures the GenAI client operation exception event name and severity. Only takes effect when + * emitting exceptions as logs is enabled via the {@code otel.semconv.exception.signal.preview} + * flag. + */ + @CanIgnoreReturnValue + public static + InstrumenterBuilder setGenAiClientExceptionEventExtractor( + InstrumenterBuilder builder) { + Experimental.setExceptionEventExtractor( + builder, + (logRecordBuilder, context, request) -> { + logRecordBuilder.setEventName("gen_ai.client.operation.exception"); + logRecordBuilder.setSeverity(Severity.WARN); + }); + return builder; + } + + private GenAiExceptionEventExtractors() {} +} diff --git a/instrumentation/openai/openai-java-1.1/library/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/OpenAITelemetryBuilder.java b/instrumentation/openai/openai-java-1.1/library/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/OpenAITelemetryBuilder.java index f21429a94d61..df86ab826e04 100644 --- a/instrumentation/openai/openai-java-1.1/library/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/OpenAITelemetryBuilder.java +++ b/instrumentation/openai/openai-java-1.1/library/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/OpenAITelemetryBuilder.java @@ -5,6 +5,8 @@ package io.opentelemetry.instrumentation.openai.v1_1; +import static io.opentelemetry.instrumentation.api.incubator.semconv.genai.internal.GenAiExceptionEventExtractors.setGenAiClientExceptionEventExtractor; + import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.openai.models.chat.completions.ChatCompletion; import com.openai.models.chat.completions.ChatCompletionCreateParams; @@ -49,22 +51,25 @@ public OpenAITelemetryBuilder setCaptureMessageContent(boolean captureMessageCon public OpenAITelemetry build() { ChatAttributesGetter chatAttributesGetter = new ChatAttributesGetter(); Instrumenter chatInstrumenter = - Instrumenter.builder( - openTelemetry, - INSTRUMENTATION_NAME, - GenAiSpanNameExtractor.create(chatAttributesGetter)) - .addAttributesExtractor(GenAiAttributesExtractor.create(chatAttributesGetter)) - .addOperationMetrics(GenAiClientMetrics.get()) + setGenAiClientExceptionEventExtractor( + Instrumenter.builder( + openTelemetry, + INSTRUMENTATION_NAME, + GenAiSpanNameExtractor.create(chatAttributesGetter)) + .addAttributesExtractor(GenAiAttributesExtractor.create(chatAttributesGetter)) + .addOperationMetrics(GenAiClientMetrics.get())) .buildInstrumenter(); EmbeddingAttributesGetter embeddingAttributesGetter = new EmbeddingAttributesGetter(); Instrumenter embeddingsInstrumenter = - Instrumenter.builder( - openTelemetry, - INSTRUMENTATION_NAME, - GenAiSpanNameExtractor.create(embeddingAttributesGetter)) - .addAttributesExtractor(GenAiAttributesExtractor.create(embeddingAttributesGetter)) - .addOperationMetrics(GenAiClientMetrics.get()) + setGenAiClientExceptionEventExtractor( + Instrumenter.builder( + openTelemetry, + INSTRUMENTATION_NAME, + GenAiSpanNameExtractor.create(embeddingAttributesGetter)) + .addAttributesExtractor( + GenAiAttributesExtractor.create(embeddingAttributesGetter)) + .addOperationMetrics(GenAiClientMetrics.get())) .buildInstrumenter(SpanKindExtractor.alwaysClient()); Logger eventLogger = openTelemetry.getLogsBridge().get(INSTRUMENTATION_NAME); From e8e8c77e67bc78e8255d995640fd051f0ff7b299 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Wed, 3 Jun 2026 17:40:06 -0700 Subject: [PATCH 02/11] Add unit and integration tests for GenAI exception event extractor --- .../GenAiExceptionEventExtractorsTest.java | 57 +++++++++++++++++ .../javaagent/build.gradle.kts | 15 +++++ .../openai-java-1.1/library/build.gradle.kts | 14 +++++ .../openai/v1_1/AbstractEmbeddingsTest.java | 63 +++++++++++++++---- 4 files changed, 136 insertions(+), 13 deletions(-) create mode 100644 instrumentation-api-incubator/src/test/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/internal/GenAiExceptionEventExtractorsTest.java diff --git a/instrumentation-api-incubator/src/test/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/internal/GenAiExceptionEventExtractorsTest.java b/instrumentation-api-incubator/src/test/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/internal/GenAiExceptionEventExtractorsTest.java new file mode 100644 index 000000000000..1e4608d8a878 --- /dev/null +++ b/instrumentation-api-incubator/src/test/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/internal/GenAiExceptionEventExtractorsTest.java @@ -0,0 +1,57 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.instrumentation.api.incubator.semconv.genai.internal; + +import static io.opentelemetry.instrumentation.api.internal.SemconvExceptionSignal.emitExceptionAsLogs; +import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat; +import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo; +import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.satisfies; +import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_MESSAGE; +import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_STACKTRACE; +import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_TYPE; + +import io.opentelemetry.api.logs.Severity; +import io.opentelemetry.context.Context; +import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter; +import io.opentelemetry.instrumentation.api.instrumenter.InstrumenterBuilder; +import io.opentelemetry.sdk.logs.data.LogRecordData; +import io.opentelemetry.sdk.testing.junit5.OpenTelemetryExtension; +import java.util.List; +import org.assertj.core.api.AbstractAssert; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +class GenAiExceptionEventExtractorsTest { + + @RegisterExtension + static final OpenTelemetryExtension otelTesting = OpenTelemetryExtension.create(); + + @Test + void genAiClientExceptionLog() { + InstrumenterBuilder builder = + Instrumenter.builder(otelTesting.getOpenTelemetry(), "test", unused -> "span"); + GenAiExceptionEventExtractors.setGenAiClientExceptionEventExtractor(builder); + Instrumenter instrumenter = builder.buildInstrumenter(); + + Context context = instrumenter.start(Context.root(), "request"); + IllegalStateException error = new IllegalStateException("test"); + instrumenter.end(context, "request", "response", error); + + List logs = otelTesting.getLogRecords(); + if (emitExceptionAsLogs()) { + assertThat(logs).hasSize(1); + assertThat(logs.get(0)) + .hasSeverity(Severity.WARN) + .hasEventName("gen_ai.client.operation.exception") + .hasAttributesSatisfyingExactly( + equalTo(EXCEPTION_TYPE, "java.lang.IllegalStateException"), + equalTo(EXCEPTION_MESSAGE, "test"), + satisfies(EXCEPTION_STACKTRACE, AbstractAssert::isNotNull)); + } else { + assertThat(logs).isEmpty(); + } + } +} diff --git a/instrumentation/openai/openai-java-1.1/javaagent/build.gradle.kts b/instrumentation/openai/openai-java-1.1/javaagent/build.gradle.kts index 64f2e53ca232..bfdfe8cfe123 100644 --- a/instrumentation/openai/openai-java-1.1/javaagent/build.gradle.kts +++ b/instrumentation/openai/openai-java-1.1/javaagent/build.gradle.kts @@ -29,4 +29,19 @@ tasks { systemProperty("otel.instrumentation.genai.capture-message-content", "true") systemProperty("collectMetadata", otelProps.collectMetadata) } + + val testExceptionSignalLogs by registering(Test::class) { + testClassesDirs = sourceSets.test.get().output.classesDirs + classpath = sourceSets.test.get().runtimeClasspath + + systemProperty("otel.instrumentation.genai.capture-message-content", "true") + filter { + includeTestsMatching("EmbeddingsTest") + } + jvmArgs("-Dotel.semconv.exception.signal.preview=logs") + } + + check { + dependsOn(testExceptionSignalLogs) + } } diff --git a/instrumentation/openai/openai-java-1.1/library/build.gradle.kts b/instrumentation/openai/openai-java-1.1/library/build.gradle.kts index f8ab3f0299e8..74ef4c4d2cc1 100644 --- a/instrumentation/openai/openai-java-1.1/library/build.gradle.kts +++ b/instrumentation/openai/openai-java-1.1/library/build.gradle.kts @@ -13,4 +13,18 @@ tasks { test { systemProperty("testLatestDeps", otelProps.testLatestDeps) } + + val testExceptionSignalLogs by registering(Test::class) { + testClassesDirs = sourceSets.test.get().output.classesDirs + classpath = sourceSets.test.get().runtimeClasspath + + filter { + includeTestsMatching("EmbeddingsTest") + } + jvmArgs("-Dotel.semconv.exception.signal.preview=logs") + } + + check { + dependsOn(testExceptionSignalLogs) + } } diff --git a/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractEmbeddingsTest.java b/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractEmbeddingsTest.java index f4057d63f28d..19a0b517548e 100644 --- a/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractEmbeddingsTest.java +++ b/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractEmbeddingsTest.java @@ -5,8 +5,14 @@ package io.opentelemetry.instrumentation.openai.v1_1; +import static io.opentelemetry.instrumentation.api.internal.SemconvExceptionSignal.emitExceptionAsLogs; +import static io.opentelemetry.instrumentation.api.internal.SemconvExceptionSignal.emitExceptionAsSpanEvents; +import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat; import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo; import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.satisfies; +import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_MESSAGE; +import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_STACKTRACE; +import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_TYPE; import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GEN_AI_OPERATION_NAME; import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GEN_AI_PROVIDER_NAME; import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GEN_AI_REQUEST_ENCODING_FORMATS; @@ -18,6 +24,7 @@ import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GenAiProviderNameIncubatingValues.OPENAI; import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GenAiTokenTypeIncubatingValues.INPUT; import static java.util.Collections.singletonList; +import static java.util.stream.Collectors.toList; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.catchThrowable; @@ -28,11 +35,15 @@ import com.openai.errors.OpenAIIoException; import com.openai.models.embeddings.CreateEmbeddingResponse; import com.openai.models.embeddings.EmbeddingCreateParams; +import io.opentelemetry.api.logs.Severity; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.context.Context; import io.opentelemetry.instrumentation.testing.internal.AutoCleanupExtension; +import io.opentelemetry.sdk.logs.data.LogRecordData; +import java.util.List; import java.util.concurrent.CompletionException; +import org.awaitility.Awaitility; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -239,19 +250,26 @@ void connectionError() { trace -> trace.hasSpansSatisfyingExactly( maybeWithTransportSpan( - span -> - span.hasName("embeddings text-embedding-3-small") - .hasKind(SpanKind.CLIENT) - .hasException(thrown) - .hasAttributesSatisfyingExactly( - equalTo(GEN_AI_PROVIDER_NAME, OPENAI), - equalTo(GEN_AI_OPERATION_NAME, EMBEDDINGS), - equalTo(GEN_AI_REQUEST_MODEL, MODEL), - // Newer versions of the library populate base64 when unset by - // the user. - satisfies( - GEN_AI_REQUEST_ENCODING_FORMATS, - val -> val.isIn(singletonList("base64"), null)))))); + span -> { + span.hasName("embeddings text-embedding-3-small") + .hasKind(SpanKind.CLIENT) + .hasAttributesSatisfyingExactly( + equalTo(GEN_AI_PROVIDER_NAME, OPENAI), + equalTo(GEN_AI_OPERATION_NAME, EMBEDDINGS), + equalTo(GEN_AI_REQUEST_MODEL, MODEL), + // Newer versions of the library populate base64 when unset by + // the user. + satisfies( + GEN_AI_REQUEST_ENCODING_FORMATS, + val -> val.isIn(singletonList("base64"), null))); + if (emitExceptionAsSpanEvents()) { + span.hasException(thrown); + } + }))); + + if (emitExceptionAsLogs()) { + assertClientExceptionLog(); + } getTesting() .waitAndAssertMetrics( @@ -270,4 +288,23 @@ void connectionError() { equalTo(GEN_AI_OPERATION_NAME, EMBEDDINGS), equalTo(GEN_AI_REQUEST_MODEL, MODEL))))); } + + private void assertClientExceptionLog() { + Awaitility.await() + .untilAsserted( + () -> { + List logs = + getTesting().logRecords().stream() + .filter(log -> "gen_ai.client.operation.exception".equals(log.getEventName())) + .collect(toList()); + assertThat(logs).hasSize(1); + assertThat(logs.get(0)) + .hasSeverity(Severity.WARN) + .hasEventName("gen_ai.client.operation.exception") + .hasAttributesSatisfyingExactly( + satisfies(EXCEPTION_TYPE, val -> val.isNotNull()), + satisfies(EXCEPTION_MESSAGE, val -> val.isNotNull()), + satisfies(EXCEPTION_STACKTRACE, val -> val.isNotNull())); + }); + } } From 2c3d5db5a44278ad815a8eeaab1c468e646c90ab Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Thu, 4 Jun 2026 16:51:52 -0700 Subject: [PATCH 03/11] Mirror test system properties on OpenAI testExceptionSignalLogs tasks --- .../javaagent/build.gradle.kts | 4 +-- .../openai-java-1.1/library/build.gradle.kts | 2 +- .../openai/v1_1/OpenAITelemetryBuilder.java | 36 ++++++++++--------- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/instrumentation/openai/openai-java-1.1/javaagent/build.gradle.kts b/instrumentation/openai/openai-java-1.1/javaagent/build.gradle.kts index bfdfe8cfe123..32d251f44f14 100644 --- a/instrumentation/openai/openai-java-1.1/javaagent/build.gradle.kts +++ b/instrumentation/openai/openai-java-1.1/javaagent/build.gradle.kts @@ -22,7 +22,7 @@ dependencies { } tasks { - test { + withType().configureEach { systemProperty("testLatestDeps", otelProps.testLatestDeps) // TODO run tests both with and without genai message capture @@ -34,11 +34,11 @@ tasks { testClassesDirs = sourceSets.test.get().output.classesDirs classpath = sourceSets.test.get().runtimeClasspath - systemProperty("otel.instrumentation.genai.capture-message-content", "true") filter { includeTestsMatching("EmbeddingsTest") } jvmArgs("-Dotel.semconv.exception.signal.preview=logs") + systemProperty("metadataConfig", "otel.semconv.exception.signal.preview=logs") } check { diff --git a/instrumentation/openai/openai-java-1.1/library/build.gradle.kts b/instrumentation/openai/openai-java-1.1/library/build.gradle.kts index 74ef4c4d2cc1..ce2fe22a4214 100644 --- a/instrumentation/openai/openai-java-1.1/library/build.gradle.kts +++ b/instrumentation/openai/openai-java-1.1/library/build.gradle.kts @@ -10,7 +10,7 @@ dependencies { } tasks { - test { + withType().configureEach { systemProperty("testLatestDeps", otelProps.testLatestDeps) } diff --git a/instrumentation/openai/openai-java-1.1/library/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/OpenAITelemetryBuilder.java b/instrumentation/openai/openai-java-1.1/library/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/OpenAITelemetryBuilder.java index df86ab826e04..c697366aadbd 100644 --- a/instrumentation/openai/openai-java-1.1/library/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/OpenAITelemetryBuilder.java +++ b/instrumentation/openai/openai-java-1.1/library/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/OpenAITelemetryBuilder.java @@ -18,6 +18,7 @@ import io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiClientMetrics; import io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiSpanNameExtractor; import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter; +import io.opentelemetry.instrumentation.api.instrumenter.InstrumenterBuilder; import io.opentelemetry.instrumentation.api.instrumenter.SpanKindExtractor; /** A builder of {@link OpenAITelemetry}. */ @@ -50,27 +51,28 @@ public OpenAITelemetryBuilder setCaptureMessageContent(boolean captureMessageCon */ public OpenAITelemetry build() { ChatAttributesGetter chatAttributesGetter = new ChatAttributesGetter(); + InstrumenterBuilder chatBuilder = + Instrumenter.builder( + openTelemetry, + INSTRUMENTATION_NAME, + GenAiSpanNameExtractor.create(chatAttributesGetter)) + .addAttributesExtractor(GenAiAttributesExtractor.create(chatAttributesGetter)) + .addOperationMetrics(GenAiClientMetrics.get()); + setGenAiClientExceptionEventExtractor(chatBuilder); Instrumenter chatInstrumenter = - setGenAiClientExceptionEventExtractor( - Instrumenter.builder( - openTelemetry, - INSTRUMENTATION_NAME, - GenAiSpanNameExtractor.create(chatAttributesGetter)) - .addAttributesExtractor(GenAiAttributesExtractor.create(chatAttributesGetter)) - .addOperationMetrics(GenAiClientMetrics.get())) - .buildInstrumenter(); + chatBuilder.buildInstrumenter(); EmbeddingAttributesGetter embeddingAttributesGetter = new EmbeddingAttributesGetter(); + InstrumenterBuilder embeddingsBuilder = + Instrumenter.builder( + openTelemetry, + INSTRUMENTATION_NAME, + GenAiSpanNameExtractor.create(embeddingAttributesGetter)) + .addAttributesExtractor(GenAiAttributesExtractor.create(embeddingAttributesGetter)) + .addOperationMetrics(GenAiClientMetrics.get()); + setGenAiClientExceptionEventExtractor(embeddingsBuilder); Instrumenter embeddingsInstrumenter = - setGenAiClientExceptionEventExtractor( - Instrumenter.builder( - openTelemetry, - INSTRUMENTATION_NAME, - GenAiSpanNameExtractor.create(embeddingAttributesGetter)) - .addAttributesExtractor( - GenAiAttributesExtractor.create(embeddingAttributesGetter)) - .addOperationMetrics(GenAiClientMetrics.get())) - .buildInstrumenter(SpanKindExtractor.alwaysClient()); + embeddingsBuilder.buildInstrumenter(SpanKindExtractor.alwaysClient()); Logger eventLogger = openTelemetry.getLogsBridge().get(INSTRUMENTATION_NAME); return new OpenAITelemetry( From 93bd015f17546aba5c9599aad0ac4d1d37a962bf Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Fri, 5 Jun 2026 13:46:47 -0700 Subject: [PATCH 04/11] simplify --- .../openai/v1_1/AbstractEmbeddingsTest.java | 34 ++++++------------- 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractEmbeddingsTest.java b/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractEmbeddingsTest.java index 19a0b517548e..094526895a91 100644 --- a/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractEmbeddingsTest.java +++ b/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractEmbeddingsTest.java @@ -24,7 +24,6 @@ import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GenAiProviderNameIncubatingValues.OPENAI; import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GenAiTokenTypeIncubatingValues.INPUT; import static java.util.Collections.singletonList; -import static java.util.stream.Collectors.toList; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.catchThrowable; @@ -40,10 +39,7 @@ import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.context.Context; import io.opentelemetry.instrumentation.testing.internal.AutoCleanupExtension; -import io.opentelemetry.sdk.logs.data.LogRecordData; -import java.util.List; import java.util.concurrent.CompletionException; -import org.awaitility.Awaitility; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -268,7 +264,16 @@ void connectionError() { }))); if (emitExceptionAsLogs()) { - assertClientExceptionLog(); + getTesting() + .waitAndAssertLogRecords( + logRecord -> + logRecord + .hasSeverity(Severity.WARN) + .hasEventName("gen_ai.client.operation.exception") + .hasAttributesSatisfyingExactly( + equalTo(EXCEPTION_TYPE, thrown.getClass().getName()), + equalTo(EXCEPTION_MESSAGE, thrown.getMessage()), + satisfies(EXCEPTION_STACKTRACE, val -> val.isNotNull()))); } getTesting() @@ -288,23 +293,4 @@ void connectionError() { equalTo(GEN_AI_OPERATION_NAME, EMBEDDINGS), equalTo(GEN_AI_REQUEST_MODEL, MODEL))))); } - - private void assertClientExceptionLog() { - Awaitility.await() - .untilAsserted( - () -> { - List logs = - getTesting().logRecords().stream() - .filter(log -> "gen_ai.client.operation.exception".equals(log.getEventName())) - .collect(toList()); - assertThat(logs).hasSize(1); - assertThat(logs.get(0)) - .hasSeverity(Severity.WARN) - .hasEventName("gen_ai.client.operation.exception") - .hasAttributesSatisfyingExactly( - satisfies(EXCEPTION_TYPE, val -> val.isNotNull()), - satisfies(EXCEPTION_MESSAGE, val -> val.isNotNull()), - satisfies(EXCEPTION_STACKTRACE, val -> val.isNotNull())); - }); - } } From 3d0c76db2d307e5925ed9c49b4cf98d0cb1946d5 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Fri, 5 Jun 2026 13:48:51 -0700 Subject: [PATCH 05/11] Address review comment on GenAI exception spans --- .../instrumentation/openai/v1_1/AbstractEmbeddingsTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractEmbeddingsTest.java b/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractEmbeddingsTest.java index 094526895a91..889d35b8db75 100644 --- a/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractEmbeddingsTest.java +++ b/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractEmbeddingsTest.java @@ -258,9 +258,7 @@ void connectionError() { satisfies( GEN_AI_REQUEST_ENCODING_FORMATS, val -> val.isIn(singletonList("base64"), null))); - if (emitExceptionAsSpanEvents()) { - span.hasException(thrown); - } + span.hasException(emitExceptionAsSpanEvents() ? thrown : null); }))); if (emitExceptionAsLogs()) { From e70f7d3c647ed65aacec8c7736e612f7e41bdc6f Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Fri, 5 Jun 2026 14:41:07 -0700 Subject: [PATCH 06/11] Simplify OpenAI exception log assertions --- .../openai/v1_1/EmbeddingsTest.java | 27 ++++++++++ .../openai/v1_1/AbstractEmbeddingsTest.java | 53 ++++++++++--------- 2 files changed, 55 insertions(+), 25 deletions(-) diff --git a/instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/EmbeddingsTest.java b/instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/EmbeddingsTest.java index ca4db1daaf3d..d53ba7c5c4ca 100644 --- a/instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/EmbeddingsTest.java +++ b/instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/EmbeddingsTest.java @@ -5,11 +5,19 @@ package io.opentelemetry.javaagent.instrumentation.openai.v1_1; +import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo; +import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.satisfies; +import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_MESSAGE; +import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_STACKTRACE; +import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_TYPE; + import com.openai.client.OpenAIClient; import com.openai.client.OpenAIClientAsync; +import io.opentelemetry.api.logs.Severity; import io.opentelemetry.instrumentation.openai.v1_1.AbstractEmbeddingsTest; import io.opentelemetry.instrumentation.testing.junit.AgentInstrumentationExtension; import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension; +import io.opentelemetry.sdk.testing.assertj.LogRecordDataAssert; import io.opentelemetry.sdk.testing.assertj.SpanDataAssert; import java.util.ArrayList; import java.util.List; @@ -45,4 +53,23 @@ protected List> maybeWithTransportSpan(Consumer s.hasName("POST")); return result; } + + @Override + protected List> maybeWithTransportExceptionLog( + Consumer logRecord) { + List> result = new ArrayList<>(); + result.add( + transportLog -> + transportLog + .hasSeverity(Severity.WARN) + .hasEventName("http.client.request.exception") + .hasAttributesSatisfyingExactly( + equalTo(EXCEPTION_TYPE, "java.net.ConnectException"), + satisfies( + EXCEPTION_MESSAGE, val -> val.startsWith("Failed to connect to localhost")), + satisfies(EXCEPTION_STACKTRACE, val -> val.isNotNull())) + .hasTotalAttributeCount(3)); + result.add(logRecord); + return result; + } } diff --git a/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractEmbeddingsTest.java b/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractEmbeddingsTest.java index 889d35b8db75..12370be6130d 100644 --- a/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractEmbeddingsTest.java +++ b/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractEmbeddingsTest.java @@ -10,9 +10,6 @@ import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat; import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo; import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.satisfies; -import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_MESSAGE; -import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_STACKTRACE; -import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_TYPE; import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GEN_AI_OPERATION_NAME; import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GEN_AI_PROVIDER_NAME; import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GEN_AI_REQUEST_ENCODING_FORMATS; @@ -39,7 +36,10 @@ import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.context.Context; import io.opentelemetry.instrumentation.testing.internal.AutoCleanupExtension; +import io.opentelemetry.sdk.testing.assertj.LogRecordDataAssert; +import java.util.List; import java.util.concurrent.CompletionException; +import java.util.function.Consumer; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -246,32 +246,30 @@ void connectionError() { trace -> trace.hasSpansSatisfyingExactly( maybeWithTransportSpan( - span -> { - span.hasName("embeddings text-embedding-3-small") - .hasKind(SpanKind.CLIENT) - .hasAttributesSatisfyingExactly( - equalTo(GEN_AI_PROVIDER_NAME, OPENAI), - equalTo(GEN_AI_OPERATION_NAME, EMBEDDINGS), - equalTo(GEN_AI_REQUEST_MODEL, MODEL), - // Newer versions of the library populate base64 when unset by - // the user. - satisfies( - GEN_AI_REQUEST_ENCODING_FORMATS, - val -> val.isIn(singletonList("base64"), null))); - span.hasException(emitExceptionAsSpanEvents() ? thrown : null); - }))); + span -> + span.hasName("embeddings text-embedding-3-small") + .hasKind(SpanKind.CLIENT) + .hasException(emitExceptionAsSpanEvents() ? thrown : null) + .hasAttributesSatisfyingExactly( + equalTo(GEN_AI_PROVIDER_NAME, OPENAI), + equalTo(GEN_AI_OPERATION_NAME, EMBEDDINGS), + equalTo(GEN_AI_REQUEST_MODEL, MODEL), + // Newer versions of the library populate base64 when unset by + // the user. + satisfies( + GEN_AI_REQUEST_ENCODING_FORMATS, + val -> val.isIn(singletonList("base64"), null)))))); if (emitExceptionAsLogs()) { getTesting() .waitAndAssertLogRecords( - logRecord -> - logRecord - .hasSeverity(Severity.WARN) - .hasEventName("gen_ai.client.operation.exception") - .hasAttributesSatisfyingExactly( - equalTo(EXCEPTION_TYPE, thrown.getClass().getName()), - equalTo(EXCEPTION_MESSAGE, thrown.getMessage()), - satisfies(EXCEPTION_STACKTRACE, val -> val.isNotNull()))); + maybeWithTransportExceptionLog( + logRecord -> + logRecord + .hasSeverity(Severity.WARN) + .hasEventName("gen_ai.client.operation.exception") + .hasException(thrown) + .hasTotalAttributeCount(3))); } getTesting() @@ -291,4 +289,9 @@ void connectionError() { equalTo(GEN_AI_OPERATION_NAME, EMBEDDINGS), equalTo(GEN_AI_REQUEST_MODEL, MODEL))))); } + + protected List> maybeWithTransportExceptionLog( + Consumer logRecord) { + return singletonList(logRecord); + } } From faa79ea91abdceb04cccbce26af9f632e26d365b Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Fri, 5 Jun 2026 17:44:59 -0700 Subject: [PATCH 07/11] Address GenAI exception log preview tests --- .../javaagent/build.gradle.kts | 4 -- .../instrumentation/openai/v1_1/ChatTest.java | 27 ++++++++++ .../openai-java-1.1/library/build.gradle.kts | 4 -- .../openai/v1_1/AbstractChatTest.java | 54 ++++++++++++------- 4 files changed, 63 insertions(+), 26 deletions(-) diff --git a/instrumentation/openai/openai-java-1.1/javaagent/build.gradle.kts b/instrumentation/openai/openai-java-1.1/javaagent/build.gradle.kts index 32d251f44f14..14df0859b470 100644 --- a/instrumentation/openai/openai-java-1.1/javaagent/build.gradle.kts +++ b/instrumentation/openai/openai-java-1.1/javaagent/build.gradle.kts @@ -33,10 +33,6 @@ tasks { val testExceptionSignalLogs by registering(Test::class) { testClassesDirs = sourceSets.test.get().output.classesDirs classpath = sourceSets.test.get().runtimeClasspath - - filter { - includeTestsMatching("EmbeddingsTest") - } jvmArgs("-Dotel.semconv.exception.signal.preview=logs") systemProperty("metadataConfig", "otel.semconv.exception.signal.preview=logs") } diff --git a/instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/ChatTest.java b/instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/ChatTest.java index bdc796c97c1e..61a203ade955 100644 --- a/instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/ChatTest.java +++ b/instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/ChatTest.java @@ -5,11 +5,19 @@ package io.opentelemetry.javaagent.instrumentation.openai.v1_1; +import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo; +import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.satisfies; +import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_MESSAGE; +import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_STACKTRACE; +import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_TYPE; + import com.openai.client.OpenAIClient; import com.openai.client.OpenAIClientAsync; +import io.opentelemetry.api.logs.Severity; import io.opentelemetry.instrumentation.openai.v1_1.AbstractChatTest; import io.opentelemetry.instrumentation.testing.junit.AgentInstrumentationExtension; import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension; +import io.opentelemetry.sdk.testing.assertj.LogRecordDataAssert; import io.opentelemetry.sdk.testing.assertj.SpanDataAssert; import java.util.ArrayList; import java.util.List; @@ -45,4 +53,23 @@ protected List> maybeWithTransportSpan(Consumer s.hasName("POST")); return result; } + + @Override + protected List> maybeWithTransportExceptionLog( + Consumer logRecord) { + List> result = new ArrayList<>(); + result.add( + transportLog -> + transportLog + .hasSeverity(Severity.WARN) + .hasEventName("http.client.request.exception") + .hasAttributesSatisfyingExactly( + equalTo(EXCEPTION_TYPE, "java.net.ConnectException"), + satisfies( + EXCEPTION_MESSAGE, val -> val.startsWith("Failed to connect to localhost")), + satisfies(EXCEPTION_STACKTRACE, val -> val.isNotNull())) + .hasTotalAttributeCount(3)); + result.add(logRecord); + return result; + } } diff --git a/instrumentation/openai/openai-java-1.1/library/build.gradle.kts b/instrumentation/openai/openai-java-1.1/library/build.gradle.kts index ce2fe22a4214..c8bdcce2c9b0 100644 --- a/instrumentation/openai/openai-java-1.1/library/build.gradle.kts +++ b/instrumentation/openai/openai-java-1.1/library/build.gradle.kts @@ -17,10 +17,6 @@ tasks { val testExceptionSignalLogs by registering(Test::class) { testClassesDirs = sourceSets.test.get().output.classesDirs classpath = sourceSets.test.get().runtimeClasspath - - filter { - includeTestsMatching("EmbeddingsTest") - } jvmArgs("-Dotel.semconv.exception.signal.preview=logs") } diff --git a/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractChatTest.java b/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractChatTest.java index 209cc3600ab3..a5065e57655f 100644 --- a/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractChatTest.java +++ b/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractChatTest.java @@ -6,6 +6,8 @@ package io.opentelemetry.instrumentation.openai.v1_1; import static io.opentelemetry.api.common.AttributeKey.stringKey; +import static io.opentelemetry.instrumentation.api.internal.SemconvExceptionSignal.emitExceptionAsLogs; +import static io.opentelemetry.instrumentation.api.internal.SemconvExceptionSignal.emitExceptionAsSpanEvents; import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo; import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.satisfies; import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GEN_AI_OPERATION_NAME; @@ -62,17 +64,20 @@ import com.openai.models.chat.completions.ChatCompletionUserMessageParam; import io.opentelemetry.api.common.KeyValue; import io.opentelemetry.api.common.Value; +import io.opentelemetry.api.logs.Severity; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.context.Context; import io.opentelemetry.instrumentation.openai.TestHelper; import io.opentelemetry.instrumentation.testing.internal.AutoCleanupExtension; +import io.opentelemetry.sdk.testing.assertj.LogRecordDataAssert; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletionException; +import java.util.function.Consumer; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -905,7 +910,7 @@ void connectionError() { trace.hasSpansSatisfyingExactly( maybeWithTransportSpan( span -> - span.hasException(thrown) + span.hasException(emitExceptionAsSpanEvents() ? thrown : null) .hasAttributesSatisfyingExactly( equalTo(GEN_AI_PROVIDER_NAME, OPENAI), equalTo(GEN_AI_OPERATION_NAME, CHAT), @@ -930,14 +935,7 @@ void connectionError() { SpanContext spanCtx = getTesting().waitForTraces(1).get(0).get(0).getSpanContext(); - getTesting() - .waitAndAssertLogRecords( - log -> - log.hasAttributesSatisfyingExactly( - equalTo(GEN_AI_PROVIDER_NAME, OPENAI), - equalTo(stringKey("event.name"), "gen_ai.user.message")) - .hasSpanContext(spanCtx) - .hasBody(Value.of(KeyValue.of("content", Value.of(TEST_CHAT_INPUT))))); + getTesting().waitAndAssertLogRecords(connectionErrorLogs(spanCtx, thrown)); } @Test @@ -1618,7 +1616,7 @@ void streamConnectionError() { trace.hasSpansSatisfyingExactly( maybeWithTransportSpan( span -> - span.hasException(thrown) + span.hasException(emitExceptionAsSpanEvents() ? thrown : null) .hasAttributesSatisfyingExactly( equalTo(GEN_AI_PROVIDER_NAME, OPENAI), equalTo(GEN_AI_OPERATION_NAME, CHAT), @@ -1643,14 +1641,34 @@ void streamConnectionError() { SpanContext spanCtx = getTesting().waitForTraces(1).get(0).get(0).getSpanContext(); - getTesting() - .waitAndAssertLogRecords( - log -> - log.hasAttributesSatisfyingExactly( - equalTo(GEN_AI_PROVIDER_NAME, OPENAI), - equalTo(stringKey("event.name"), "gen_ai.user.message")) - .hasSpanContext(spanCtx) - .hasBody(Value.of(KeyValue.of("content", Value.of(TEST_CHAT_INPUT))))); + getTesting().waitAndAssertLogRecords(connectionErrorLogs(spanCtx, thrown)); + } + + private List> connectionErrorLogs( + SpanContext spanCtx, Throwable thrown) { + List> assertions = new ArrayList<>(); + assertions.add( + log -> + log.hasAttributesSatisfyingExactly( + equalTo(GEN_AI_PROVIDER_NAME, OPENAI), + equalTo(stringKey("event.name"), "gen_ai.user.message")) + .hasSpanContext(spanCtx) + .hasBody(Value.of(KeyValue.of("content", Value.of(TEST_CHAT_INPUT))))); + if (emitExceptionAsLogs()) { + assertions.addAll( + maybeWithTransportExceptionLog( + log -> + log.hasSeverity(Severity.WARN) + .hasEventName("gen_ai.client.operation.exception") + .hasException(thrown) + .hasTotalAttributeCount(3))); + } + return assertions; + } + + protected List> maybeWithTransportExceptionLog( + Consumer logRecord) { + return singletonList(logRecord); } protected static ChatCompletionMessageParam createUserMessage(String content) { From 53edc2cb2de2dd72a6c6209991ed8542d6961e5c Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Fri, 5 Jun 2026 17:54:37 -0700 Subject: [PATCH 08/11] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../javaagent/instrumentation/openai/v1_1/EmbeddingsTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/EmbeddingsTest.java b/instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/EmbeddingsTest.java index d53ba7c5c4ca..529d9c6992bd 100644 --- a/instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/EmbeddingsTest.java +++ b/instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/EmbeddingsTest.java @@ -67,8 +67,7 @@ protected List> maybeWithTransportExceptionLog( equalTo(EXCEPTION_TYPE, "java.net.ConnectException"), satisfies( EXCEPTION_MESSAGE, val -> val.startsWith("Failed to connect to localhost")), - satisfies(EXCEPTION_STACKTRACE, val -> val.isNotNull())) - .hasTotalAttributeCount(3)); + satisfies(EXCEPTION_STACKTRACE, val -> val.isNotNull()))); result.add(logRecord); return result; } From 3f389c6fd77072ff8c58ce5c2219aa7ccf465051 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Fri, 5 Jun 2026 17:56:31 -0700 Subject: [PATCH 09/11] Apply suggestion from @trask --- .../javaagent/instrumentation/openai/v1_1/ChatTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/ChatTest.java b/instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/ChatTest.java index 61a203ade955..2cb46a56242f 100644 --- a/instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/ChatTest.java +++ b/instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/ChatTest.java @@ -67,8 +67,7 @@ protected List> maybeWithTransportExceptionLog( equalTo(EXCEPTION_TYPE, "java.net.ConnectException"), satisfies( EXCEPTION_MESSAGE, val -> val.startsWith("Failed to connect to localhost")), - satisfies(EXCEPTION_STACKTRACE, val -> val.isNotNull())) - .hasTotalAttributeCount(3)); + satisfies(EXCEPTION_STACKTRACE, val -> val.isNotNull()))); result.add(logRecord); return result; } From 1494fdda2a98f17b87a632790d06972c143b3ceb Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Fri, 5 Jun 2026 18:05:25 -0700 Subject: [PATCH 10/11] refactor --- .../instrumentation/openai/v1_1/ChatTest.java | 9 ++--- .../openai/v1_1/EmbeddingsTest.java | 9 ++--- .../openai/v1_1/AbstractChatTest.java | 17 +-------- .../openai/v1_1/AbstractEmbeddingsTest.java | 22 +----------- .../openai/v1_1/AbstractOpenAiTest.java | 35 +++++++++++++++++++ 5 files changed, 43 insertions(+), 49 deletions(-) diff --git a/instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/ChatTest.java b/instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/ChatTest.java index 2cb46a56242f..84e5c4c3295b 100644 --- a/instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/ChatTest.java +++ b/instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/ChatTest.java @@ -10,6 +10,7 @@ import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_MESSAGE; import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_STACKTRACE; import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_TYPE; +import static java.util.Collections.singletonList; import com.openai.client.OpenAIClient; import com.openai.client.OpenAIClientAsync; @@ -55,10 +56,8 @@ protected List> maybeWithTransportSpan(Consumer> maybeWithTransportExceptionLog( - Consumer logRecord) { - List> result = new ArrayList<>(); - result.add( + protected List> transportExceptionLogs() { + return singletonList( transportLog -> transportLog .hasSeverity(Severity.WARN) @@ -68,7 +67,5 @@ protected List> maybeWithTransportExceptionLog( satisfies( EXCEPTION_MESSAGE, val -> val.startsWith("Failed to connect to localhost")), satisfies(EXCEPTION_STACKTRACE, val -> val.isNotNull()))); - result.add(logRecord); - return result; } } diff --git a/instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/EmbeddingsTest.java b/instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/EmbeddingsTest.java index 529d9c6992bd..1d5c9e86b717 100644 --- a/instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/EmbeddingsTest.java +++ b/instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/EmbeddingsTest.java @@ -10,6 +10,7 @@ import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_MESSAGE; import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_STACKTRACE; import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_TYPE; +import static java.util.Collections.singletonList; import com.openai.client.OpenAIClient; import com.openai.client.OpenAIClientAsync; @@ -55,10 +56,8 @@ protected List> maybeWithTransportSpan(Consumer> maybeWithTransportExceptionLog( - Consumer logRecord) { - List> result = new ArrayList<>(); - result.add( + protected List> transportExceptionLogs() { + return singletonList( transportLog -> transportLog .hasSeverity(Severity.WARN) @@ -68,7 +67,5 @@ protected List> maybeWithTransportExceptionLog( satisfies( EXCEPTION_MESSAGE, val -> val.startsWith("Failed to connect to localhost")), satisfies(EXCEPTION_STACKTRACE, val -> val.isNotNull()))); - result.add(logRecord); - return result; } } diff --git a/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractChatTest.java b/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractChatTest.java index a5065e57655f..caa2c3c79619 100644 --- a/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractChatTest.java +++ b/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractChatTest.java @@ -6,7 +6,6 @@ package io.opentelemetry.instrumentation.openai.v1_1; import static io.opentelemetry.api.common.AttributeKey.stringKey; -import static io.opentelemetry.instrumentation.api.internal.SemconvExceptionSignal.emitExceptionAsLogs; import static io.opentelemetry.instrumentation.api.internal.SemconvExceptionSignal.emitExceptionAsSpanEvents; import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo; import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.satisfies; @@ -64,7 +63,6 @@ import com.openai.models.chat.completions.ChatCompletionUserMessageParam; import io.opentelemetry.api.common.KeyValue; import io.opentelemetry.api.common.Value; -import io.opentelemetry.api.logs.Severity; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.context.Context; @@ -1654,23 +1652,10 @@ private List> connectionErrorLogs( equalTo(stringKey("event.name"), "gen_ai.user.message")) .hasSpanContext(spanCtx) .hasBody(Value.of(KeyValue.of("content", Value.of(TEST_CHAT_INPUT))))); - if (emitExceptionAsLogs()) { - assertions.addAll( - maybeWithTransportExceptionLog( - log -> - log.hasSeverity(Severity.WARN) - .hasEventName("gen_ai.client.operation.exception") - .hasException(thrown) - .hasTotalAttributeCount(3))); - } + assertions.addAll(genAiClientExceptionLogs(thrown)); return assertions; } - protected List> maybeWithTransportExceptionLog( - Consumer logRecord) { - return singletonList(logRecord); - } - protected static ChatCompletionMessageParam createUserMessage(String content) { return ChatCompletionMessageParam.ofUser( ChatCompletionUserMessageParam.builder() diff --git a/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractEmbeddingsTest.java b/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractEmbeddingsTest.java index 12370be6130d..ff4ac5547ae7 100644 --- a/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractEmbeddingsTest.java +++ b/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractEmbeddingsTest.java @@ -5,7 +5,6 @@ package io.opentelemetry.instrumentation.openai.v1_1; -import static io.opentelemetry.instrumentation.api.internal.SemconvExceptionSignal.emitExceptionAsLogs; import static io.opentelemetry.instrumentation.api.internal.SemconvExceptionSignal.emitExceptionAsSpanEvents; import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat; import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo; @@ -31,15 +30,11 @@ import com.openai.errors.OpenAIIoException; import com.openai.models.embeddings.CreateEmbeddingResponse; import com.openai.models.embeddings.EmbeddingCreateParams; -import io.opentelemetry.api.logs.Severity; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.context.Context; import io.opentelemetry.instrumentation.testing.internal.AutoCleanupExtension; -import io.opentelemetry.sdk.testing.assertj.LogRecordDataAssert; -import java.util.List; import java.util.concurrent.CompletionException; -import java.util.function.Consumer; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -260,17 +255,7 @@ void connectionError() { GEN_AI_REQUEST_ENCODING_FORMATS, val -> val.isIn(singletonList("base64"), null)))))); - if (emitExceptionAsLogs()) { - getTesting() - .waitAndAssertLogRecords( - maybeWithTransportExceptionLog( - logRecord -> - logRecord - .hasSeverity(Severity.WARN) - .hasEventName("gen_ai.client.operation.exception") - .hasException(thrown) - .hasTotalAttributeCount(3))); - } + getTesting().waitAndAssertLogRecords(genAiClientExceptionLogs(thrown)); getTesting() .waitAndAssertMetrics( @@ -289,9 +274,4 @@ void connectionError() { equalTo(GEN_AI_OPERATION_NAME, EMBEDDINGS), equalTo(GEN_AI_REQUEST_MODEL, MODEL))))); } - - protected List> maybeWithTransportExceptionLog( - Consumer logRecord) { - return singletonList(logRecord); - } } diff --git a/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractOpenAiTest.java b/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractOpenAiTest.java index b505555008cf..f56be7221ce9 100644 --- a/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractOpenAiTest.java +++ b/instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractOpenAiTest.java @@ -5,17 +5,22 @@ package io.opentelemetry.instrumentation.openai.v1_1; +import static io.opentelemetry.instrumentation.api.internal.SemconvExceptionSignal.emitExceptionAsLogs; import static io.opentelemetry.instrumentation.testing.util.TestLatestDeps.testLatestDeps; +import static java.util.Collections.emptyList; import com.openai.client.OpenAIClient; import com.openai.client.OpenAIClientAsync; import com.openai.client.okhttp.OpenAIOkHttpClient; import com.openai.client.okhttp.OpenAIOkHttpClientAsync; +import io.opentelemetry.api.logs.Severity; import io.opentelemetry.instrumentation.openai.TestHelper; import io.opentelemetry.instrumentation.openai.v3_0.OpenAi3TestHelper; import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension; import io.opentelemetry.instrumentation.testing.recording.RecordingExtension; +import io.opentelemetry.sdk.testing.assertj.LogRecordDataAssert; import io.opentelemetry.sdk.testing.assertj.SpanDataAssert; +import java.util.ArrayList; import java.util.List; import java.util.function.Consumer; import org.junit.jupiter.api.extension.RegisterExtension; @@ -90,5 +95,35 @@ protected OpenAIClientAsync getClientAsync() { protected abstract List> maybeWithTransportSpan( Consumer span); + /** + * Returns the log record assertions expected when a GenAI client operation fails. Empty unless + * exceptions are emitted as logs, in which case it contains the {@code + * gen_ai.client.operation.exception} log, preceded by any transport-level exception logs supplied + * by {@link #transportExceptionLogs}. + */ + protected final List> genAiClientExceptionLogs(Throwable thrown) { + if (!emitExceptionAsLogs()) { + return emptyList(); + } + List> logs = new ArrayList<>(transportExceptionLogs()); + logs.add( + logRecord -> + logRecord + .hasSeverity(Severity.WARN) + .hasEventName("gen_ai.client.operation.exception") + .hasException(thrown) + .hasTotalAttributeCount(3)); + return logs; + } + + /** + * Returns the transport-level exception log assertions emitted by additional instrumentation + * (e.g. the HTTP client under the javaagent), ordered before the GenAI exception log. Empty by + * default. + */ + protected List> transportExceptionLogs() { + return emptyList(); + } + @Parameter protected TestType testType; } From e23f93ac35cee4db4c0585d5bbc3755521ea519f Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Sat, 6 Jun 2026 16:38:41 -0700 Subject: [PATCH 11/11] Relax GenAI helper builder type --- .../genai/internal/GenAiExceptionEventExtractors.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/internal/GenAiExceptionEventExtractors.java b/instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/internal/GenAiExceptionEventExtractors.java index d0281951139d..0ec802a003eb 100644 --- a/instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/internal/GenAiExceptionEventExtractors.java +++ b/instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/internal/GenAiExceptionEventExtractors.java @@ -5,7 +5,6 @@ package io.opentelemetry.instrumentation.api.incubator.semconv.genai.internal; -import com.google.errorprone.annotations.CanIgnoreReturnValue; import io.opentelemetry.api.logs.Severity; import io.opentelemetry.instrumentation.api.instrumenter.InstrumenterBuilder; import io.opentelemetry.instrumentation.api.internal.Experimental; @@ -21,17 +20,14 @@ public final class GenAiExceptionEventExtractors { * emitting exceptions as logs is enabled via the {@code otel.semconv.exception.signal.preview} * flag. */ - @CanIgnoreReturnValue - public static - InstrumenterBuilder setGenAiClientExceptionEventExtractor( - InstrumenterBuilder builder) { + public static void setGenAiClientExceptionEventExtractor( + InstrumenterBuilder builder) { Experimental.setExceptionEventExtractor( builder, (logRecordBuilder, context, request) -> { logRecordBuilder.setEventName("gen_ai.client.operation.exception"); logRecordBuilder.setSeverity(Severity.WARN); }); - return builder; } private GenAiExceptionEventExtractors() {}