Emit GenAI exceptions as logs under the opt-in preview

instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/internal/GenAiExceptionEventExtractors.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.api.incubator.semconv.genai.internal;
+
+import io.opentelemetry.api.logs.Severity;
+import io.opentelemetry.instrumentation.api.instrumenter.InstrumenterBuilder;
+import io.opentelemetry.instrumentation.api.internal.Experimental;
+
+/**
+ * This class is internal and is hence not for public use. Its APIs are unstable and can change at
+ * any time.
+ */
+public final class GenAiExceptionEventExtractors {
+
+  /**
+   * Configures the GenAI client operation exception event name and severity. Only takes effect when
+   * emitting exceptions as logs is enabled via the {@code otel.semconv.exception.signal.preview}
+   * flag.
+   */
+  public static <REQUEST> void setGenAiClientExceptionEventExtractor(
+      InstrumenterBuilder<REQUEST, ?> builder) {
+    Experimental.setExceptionEventExtractor(
+        builder,
+        (logRecordBuilder, context, request) -> {
+          logRecordBuilder.setEventName("gen_ai.client.operation.exception");
+          logRecordBuilder.setSeverity(Severity.WARN);
+        });
+  }
+
+  private GenAiExceptionEventExtractors() {}
+}

instrumentation-api-incubator/src/test/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/internal/GenAiExceptionEventExtractorsTest.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.api.incubator.semconv.genai.internal;
+
+import static io.opentelemetry.instrumentation.api.internal.SemconvExceptionSignal.emitExceptionAsLogs;
+import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat;
+import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo;
+import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.satisfies;
+import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_MESSAGE;
+import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_STACKTRACE;
+import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_TYPE;
+
+import io.opentelemetry.api.logs.Severity;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
+import io.opentelemetry.instrumentation.api.instrumenter.InstrumenterBuilder;
+import io.opentelemetry.sdk.logs.data.LogRecordData;
+import io.opentelemetry.sdk.testing.junit5.OpenTelemetryExtension;
+import java.util.List;
+import org.assertj.core.api.AbstractAssert;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+class GenAiExceptionEventExtractorsTest {
+
+  @RegisterExtension
+  static final OpenTelemetryExtension otelTesting = OpenTelemetryExtension.create();
+
+  @Test
+  void genAiClientExceptionLog() {
+    InstrumenterBuilder<String, String> builder =
+        Instrumenter.builder(otelTesting.getOpenTelemetry(), "test", unused -> "span");
+    GenAiExceptionEventExtractors.setGenAiClientExceptionEventExtractor(builder);
+    Instrumenter<String, String> instrumenter = builder.buildInstrumenter();
+
+    Context context = instrumenter.start(Context.root(), "request");
+    IllegalStateException error = new IllegalStateException("test");
+    instrumenter.end(context, "request", "response", error);
+
+    List<LogRecordData> logs = otelTesting.getLogRecords();
+    if (emitExceptionAsLogs()) {
+      assertThat(logs).hasSize(1);
+      assertThat(logs.get(0))
+          .hasSeverity(Severity.WARN)
+          .hasEventName("gen_ai.client.operation.exception")
+          .hasAttributesSatisfyingExactly(
+              equalTo(EXCEPTION_TYPE, "java.lang.IllegalStateException"),
+              equalTo(EXCEPTION_MESSAGE, "test"),
+              satisfies(EXCEPTION_STACKTRACE, AbstractAssert::isNotNull));
+    } else {
+      assertThat(logs).isEmpty();
+    }
+  }
+}

instrumentation/openai/openai-java-1.1/javaagent/build.gradle.kts

@@ -22,11 +22,22 @@ dependencies {
 }
 
 tasks {
-  test {
+  withType<Test>().configureEach {
     systemProperty("testLatestDeps", otelProps.testLatestDeps)
     // TODO run tests both with and without genai message capture
 
     systemProperty("otel.instrumentation.genai.capture-message-content", "true")
     systemProperty("collectMetadata", otelProps.collectMetadata)
   }
+
+  val testExceptionSignalLogs by registering(Test::class) {
+    testClassesDirs = sourceSets.test.get().output.classesDirs
+    classpath = sourceSets.test.get().runtimeClasspath
+    jvmArgs("-Dotel.semconv.exception.signal.preview=logs")
+    systemProperty("metadataConfig", "otel.semconv.exception.signal.preview=logs")
+  }
+
+  check {
+    dependsOn(testExceptionSignalLogs)
+  }
 }

instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/ChatTest.java

@@ -5,11 +5,20 @@
 
 package io.opentelemetry.javaagent.instrumentation.openai.v1_1;
 
+import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo;
+import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.satisfies;
+import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_MESSAGE;
+import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_STACKTRACE;
+import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_TYPE;
+import static java.util.Collections.singletonList;
+
 import com.openai.client.OpenAIClient;
 import com.openai.client.OpenAIClientAsync;
+import io.opentelemetry.api.logs.Severity;
 import io.opentelemetry.instrumentation.openai.v1_1.AbstractChatTest;
 import io.opentelemetry.instrumentation.testing.junit.AgentInstrumentationExtension;
 import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension;
+import io.opentelemetry.sdk.testing.assertj.LogRecordDataAssert;
 import io.opentelemetry.sdk.testing.assertj.SpanDataAssert;
 import java.util.ArrayList;
 import java.util.List;
@@ -45,4 +54,18 @@ protected List<Consumer<SpanDataAssert>> maybeWithTransportSpan(Consumer<SpanDat
     result.add(s -> s.hasName("POST"));
     return result;
   }
+
+  @Override
+  protected List<Consumer<LogRecordDataAssert>> transportExceptionLogs() {
+    return singletonList(
+        transportLog ->
+            transportLog
+                .hasSeverity(Severity.WARN)
+                .hasEventName("http.client.request.exception")
+                .hasAttributesSatisfyingExactly(
+                    equalTo(EXCEPTION_TYPE, "java.net.ConnectException"),
+                    satisfies(
+                        EXCEPTION_MESSAGE, val -> val.startsWith("Failed to connect to localhost")),
+                    satisfies(EXCEPTION_STACKTRACE, val -> val.isNotNull())));
+  }
 }

instrumentation/openai/openai-java-1.1/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/openai/v1_1/EmbeddingsTest.java

@@ -5,11 +5,20 @@
 
 package io.opentelemetry.javaagent.instrumentation.openai.v1_1;
 
+import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo;
+import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.satisfies;
+import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_MESSAGE;
+import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_STACKTRACE;
+import static io.opentelemetry.semconv.ExceptionAttributes.EXCEPTION_TYPE;
+import static java.util.Collections.singletonList;
+
 import com.openai.client.OpenAIClient;
 import com.openai.client.OpenAIClientAsync;
+import io.opentelemetry.api.logs.Severity;
 import io.opentelemetry.instrumentation.openai.v1_1.AbstractEmbeddingsTest;
 import io.opentelemetry.instrumentation.testing.junit.AgentInstrumentationExtension;
 import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension;
+import io.opentelemetry.sdk.testing.assertj.LogRecordDataAssert;
 import io.opentelemetry.sdk.testing.assertj.SpanDataAssert;
 import java.util.ArrayList;
 import java.util.List;
@@ -45,4 +54,18 @@ protected List<Consumer<SpanDataAssert>> maybeWithTransportSpan(Consumer<SpanDat
     result.add(s -> s.hasName("POST"));
     return result;
   }
+
+  @Override
+  protected List<Consumer<LogRecordDataAssert>> transportExceptionLogs() {
+    return singletonList(
+        transportLog ->
+            transportLog
+                .hasSeverity(Severity.WARN)
+                .hasEventName("http.client.request.exception")
+                .hasAttributesSatisfyingExactly(
+                    equalTo(EXCEPTION_TYPE, "java.net.ConnectException"),
+                    satisfies(
+                        EXCEPTION_MESSAGE, val -> val.startsWith("Failed to connect to localhost")),
+                    satisfies(EXCEPTION_STACKTRACE, val -> val.isNotNull())));
+  }
 }

instrumentation/openai/openai-java-1.1/library/build.gradle.kts

@@ -10,7 +10,17 @@ dependencies {
 }
 
 tasks {
-  test {
+  withType<Test>().configureEach {
     systemProperty("testLatestDeps", otelProps.testLatestDeps)
   }
+
+  val testExceptionSignalLogs by registering(Test::class) {
+    testClassesDirs = sourceSets.test.get().output.classesDirs
+    classpath = sourceSets.test.get().runtimeClasspath
+    jvmArgs("-Dotel.semconv.exception.signal.preview=logs")
+  }
+
+  check {
+    dependsOn(testExceptionSignalLogs)
+  }
 }

instrumentation/openai/openai-java-1.1/library/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/OpenAITelemetryBuilder.java

@@ -5,6 +5,8 @@
 
 package io.opentelemetry.instrumentation.openai.v1_1;
 
+import static io.opentelemetry.instrumentation.api.incubator.semconv.genai.internal.GenAiExceptionEventExtractors.setGenAiClientExceptionEventExtractor;
+
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
 import com.openai.models.chat.completions.ChatCompletion;
 import com.openai.models.chat.completions.ChatCompletionCreateParams;
@@ -16,6 +18,7 @@
 import io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiClientMetrics;
 import io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiSpanNameExtractor;
 import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
+import io.opentelemetry.instrumentation.api.instrumenter.InstrumenterBuilder;
 import io.opentelemetry.instrumentation.api.instrumenter.SpanKindExtractor;
 
 /** A builder of {@link OpenAITelemetry}. */
@@ -48,24 +51,28 @@ public OpenAITelemetryBuilder setCaptureMessageContent(boolean captureMessageCon
    */
   public OpenAITelemetry build() {
     ChatAttributesGetter chatAttributesGetter = new ChatAttributesGetter();
-    Instrumenter<ChatCompletionCreateParams, ChatCompletion> chatInstrumenter =
+    InstrumenterBuilder<ChatCompletionCreateParams, ChatCompletion> chatBuilder =
         Instrumenter.<ChatCompletionCreateParams, ChatCompletion>builder(
                 openTelemetry,
                 INSTRUMENTATION_NAME,
                 GenAiSpanNameExtractor.create(chatAttributesGetter))
             .addAttributesExtractor(GenAiAttributesExtractor.create(chatAttributesGetter))
-            .addOperationMetrics(GenAiClientMetrics.get())
-            .buildInstrumenter();
+            .addOperationMetrics(GenAiClientMetrics.get());
+    setGenAiClientExceptionEventExtractor(chatBuilder);
+    Instrumenter<ChatCompletionCreateParams, ChatCompletion> chatInstrumenter =
+        chatBuilder.buildInstrumenter();
 
     EmbeddingAttributesGetter embeddingAttributesGetter = new EmbeddingAttributesGetter();
-    Instrumenter<EmbeddingCreateParams, CreateEmbeddingResponse> embeddingsInstrumenter =
+    InstrumenterBuilder<EmbeddingCreateParams, CreateEmbeddingResponse> embeddingsBuilder =
         Instrumenter.<EmbeddingCreateParams, CreateEmbeddingResponse>builder(
                 openTelemetry,
                 INSTRUMENTATION_NAME,
                 GenAiSpanNameExtractor.create(embeddingAttributesGetter))
             .addAttributesExtractor(GenAiAttributesExtractor.create(embeddingAttributesGetter))
-            .addOperationMetrics(GenAiClientMetrics.get())
-            .buildInstrumenter(SpanKindExtractor.alwaysClient());
+            .addOperationMetrics(GenAiClientMetrics.get());
+    setGenAiClientExceptionEventExtractor(embeddingsBuilder);
+    Instrumenter<EmbeddingCreateParams, CreateEmbeddingResponse> embeddingsInstrumenter =
+        embeddingsBuilder.buildInstrumenter(SpanKindExtractor.alwaysClient());
 
     Logger eventLogger = openTelemetry.getLogsBridge().get(INSTRUMENTATION_NAME);
     return new OpenAITelemetry(

instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractChatTest.java

@@ -6,6 +6,7 @@
 package io.opentelemetry.instrumentation.openai.v1_1;
 
 import static io.opentelemetry.api.common.AttributeKey.stringKey;
+import static io.opentelemetry.instrumentation.api.internal.SemconvExceptionSignal.emitExceptionAsSpanEvents;
 import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo;
 import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.satisfies;
 import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GEN_AI_OPERATION_NAME;
@@ -67,12 +68,14 @@
 import io.opentelemetry.context.Context;
 import io.opentelemetry.instrumentation.openai.TestHelper;
 import io.opentelemetry.instrumentation.testing.internal.AutoCleanupExtension;
+import io.opentelemetry.sdk.testing.assertj.LogRecordDataAssert;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.CompletionException;
+import java.util.function.Consumer;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 
@@ -905,7 +908,7 @@ void connectionError() {
                 trace.hasSpansSatisfyingExactly(
                     maybeWithTransportSpan(
                         span ->
-                            span.hasException(thrown)
+                            span.hasException(emitExceptionAsSpanEvents() ? thrown : null)
                                 .hasAttributesSatisfyingExactly(
                                     equalTo(GEN_AI_PROVIDER_NAME, OPENAI),
                                     equalTo(GEN_AI_OPERATION_NAME, CHAT),
@@ -930,14 +933,7 @@ void connectionError() {
 
     SpanContext spanCtx = getTesting().waitForTraces(1).get(0).get(0).getSpanContext();
 
-    getTesting()
-        .waitAndAssertLogRecords(
-            log ->
-                log.hasAttributesSatisfyingExactly(
-                        equalTo(GEN_AI_PROVIDER_NAME, OPENAI),
-                        equalTo(stringKey("event.name"), "gen_ai.user.message"))
-                    .hasSpanContext(spanCtx)
-                    .hasBody(Value.of(KeyValue.of("content", Value.of(TEST_CHAT_INPUT)))));
+    getTesting().waitAndAssertLogRecords(connectionErrorLogs(spanCtx, thrown));
   }
 
   @Test
@@ -1618,7 +1614,7 @@ void streamConnectionError() {
                 trace.hasSpansSatisfyingExactly(
                     maybeWithTransportSpan(
                         span ->
-                            span.hasException(thrown)
+                            span.hasException(emitExceptionAsSpanEvents() ? thrown : null)
                                 .hasAttributesSatisfyingExactly(
                                     equalTo(GEN_AI_PROVIDER_NAME, OPENAI),
                                     equalTo(GEN_AI_OPERATION_NAME, CHAT),
@@ -1643,14 +1639,21 @@ void streamConnectionError() {
 
     SpanContext spanCtx = getTesting().waitForTraces(1).get(0).get(0).getSpanContext();
 
-    getTesting()
-        .waitAndAssertLogRecords(
-            log ->
-                log.hasAttributesSatisfyingExactly(
-                        equalTo(GEN_AI_PROVIDER_NAME, OPENAI),
-                        equalTo(stringKey("event.name"), "gen_ai.user.message"))
-                    .hasSpanContext(spanCtx)
-                    .hasBody(Value.of(KeyValue.of("content", Value.of(TEST_CHAT_INPUT)))));
+    getTesting().waitAndAssertLogRecords(connectionErrorLogs(spanCtx, thrown));
+  }
+
+  private List<Consumer<LogRecordDataAssert>> connectionErrorLogs(
+      SpanContext spanCtx, Throwable thrown) {
+    List<Consumer<LogRecordDataAssert>> assertions = new ArrayList<>();
+    assertions.add(
+        log ->
+            log.hasAttributesSatisfyingExactly(
+                    equalTo(GEN_AI_PROVIDER_NAME, OPENAI),
+                    equalTo(stringKey("event.name"), "gen_ai.user.message"))
+                .hasSpanContext(spanCtx)
+                .hasBody(Value.of(KeyValue.of("content", Value.of(TEST_CHAT_INPUT)))));
+    assertions.addAll(genAiClientExceptionLogs(thrown));
+    return assertions;
   }
 
   protected static ChatCompletionMessageParam createUserMessage(String content) {

instrumentation/openai/openai-java-1.1/testing/src/main/java/io/opentelemetry/instrumentation/openai/v1_1/AbstractEmbeddingsTest.java

@@ -5,6 +5,8 @@
 
 package io.opentelemetry.instrumentation.openai.v1_1;
 
+import static io.opentelemetry.instrumentation.api.internal.SemconvExceptionSignal.emitExceptionAsSpanEvents;
+import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat;
 import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo;
 import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.satisfies;
 import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GEN_AI_OPERATION_NAME;
@@ -242,7 +244,7 @@ void connectionError() {
                         span ->
                             span.hasName("embeddings text-embedding-3-small")
                                 .hasKind(SpanKind.CLIENT)
-                                .hasException(thrown)
+                                .hasException(emitExceptionAsSpanEvents() ? thrown : null)
                                 .hasAttributesSatisfyingExactly(
                                     equalTo(GEN_AI_PROVIDER_NAME, OPENAI),
                                     equalTo(GEN_AI_OPERATION_NAME, EMBEDDINGS),
@@ -253,6 +255,8 @@ void connectionError() {
                                         GEN_AI_REQUEST_ENCODING_FORMATS,
                                         val -> val.isIn(singletonList("base64"), null))))));
 
+    getTesting().waitAndAssertLogRecords(genAiClientExceptionLogs(thrown));
+
     getTesting()
         .waitAndAssertMetrics(
             INSTRUMENTATION_NAME,