feat(bigquery): add resend attribute to span tracing + integration tests

java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryHelper.java

@@ -27,15 +27,21 @@
 import com.google.cloud.RetryHelper;
 import io.opentelemetry.api.trace.Span;
 import io.opentelemetry.api.trace.Tracer;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.ContextKey;
 import io.opentelemetry.context.Scope;
 import java.io.IOException;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
 public class BigQueryRetryHelper extends RetryHelper {
 
+  public static final ContextKey<AtomicInteger> RETRY_ATTEMPT_KEY =
+      ContextKey.named("bq_retry_attempt");
+
   private static final Logger LOG = Logger.getLogger(BigQueryRetryHelper.class.getName());
 
   public static <V> V runWithRetries(
@@ -54,7 +60,11 @@ public static <V> V runWithRetries(
               .spanBuilder("com.google.cloud.bigquery.BigQueryRetryHelper.runWithRetries")
               .startSpan();
     }
-    try (Scope runWithRetriesScope = runWithRetries != null ? runWithRetries.makeCurrent() : null) {
+    Context retryContext = Context.current().with(RETRY_ATTEMPT_KEY, new AtomicInteger(0));
+    if (runWithRetries != null) {
+      retryContext = retryContext.with(runWithRetries);
+    }
+    try (Scope runWithRetriesScope = retryContext != null ? retryContext.makeCurrent() : null) {
       // Suppressing should be ok as a workaraund. Current and only ResultRetryAlgorithm
       // implementation does not use response at all, so ignoring its type is ok.
       @SuppressWarnings("unchecked")

java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/telemetry/ErrorTypeUtil.java

@@ -16,16 +16,19 @@
 package com.google.cloud.bigquery.telemetry;
 
 import com.google.api.core.BetaApi;
+import com.google.common.annotations.VisibleForTesting;
 
 /**
  * Utility class for identifying exception types for telemetry tracking. TODO: this class should get
  * replaced with gax version when ready work tracked in
  * https://github.com/googleapis/google-cloud-java/issues/12105
  */
 @BetaApi
-class ErrorTypeUtil {
+@VisibleForTesting
+public class ErrorTypeUtil {
 
-  enum ErrorType {
+  @VisibleForTesting
+  public enum ErrorType {
     CLIENT_TIMEOUT,
     CLIENT_CONNECTION_ERROR,
     CLIENT_REQUEST_ERROR,

java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/telemetry/HttpTracingRequestInitializer.java

@@ -19,13 +19,15 @@
 import com.google.api.client.http.*;
 import com.google.api.core.BetaApi;
 import com.google.api.core.InternalApi;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
 import com.google.common.annotations.VisibleForTesting;
 import io.opentelemetry.api.common.AttributeKey;
 import io.opentelemetry.api.trace.Span;
 import io.opentelemetry.api.trace.Tracer;
 import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator;
 import io.opentelemetry.context.Context;
 import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  * HttpRequestInitializer that wraps a delegate initializer, intercepts all HTTP requests, adds
@@ -50,7 +52,7 @@ public class HttpTracingRequestInitializer implements HttpRequestInitializer {
   public static final AttributeKey<Long> HTTP_RESPONSE_BODY_SIZE =
       AttributeKey.longKey("http.response.body.size");
 
-  @VisibleForTesting static final String HTTP_RPC_SYSTEM_NAME = "http";
+  @VisibleForTesting public static final String HTTP_RPC_SYSTEM_NAME = "http";
 
   private static final java.util.Set<String> REDACTED_QUERY_PARAMETERS =
       com.google.common.collect.ImmutableSet.of(
@@ -84,6 +86,14 @@ public void initialize(HttpRequest request) throws IOException {
 
     addInitialHttpAttributesToSpan(span, request);
 
+    AtomicInteger attemptTracker = Context.current().get(BigQueryRetryHelper.RETRY_ATTEMPT_KEY);
+    if (attemptTracker != null) {
+      int attempt = attemptTracker.getAndIncrement();
+      if (attempt > 0) {
+        span.setAttribute(HTTP_REQUEST_RESEND_COUNT, (long) attempt);
+      }
+    }
+
     HttpResponseInterceptor originalInterceptor = request.getResponseInterceptor();
     request.setResponseInterceptor(
         response -> {

java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITOpenTelemetryTest.java

@@ -0,0 +1,286 @@
+/*
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.bigquery.it;
+
+import static com.google.cloud.bigquery.telemetry.ErrorTypeUtil.ErrorType;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
+import com.google.api.gax.retrying.RetrySettings;
+import com.google.cloud.bigquery.BigQuery;
+import com.google.cloud.bigquery.BigQueryException;
+import com.google.cloud.bigquery.telemetry.BigQueryTelemetryTracer;
+import com.google.cloud.bigquery.telemetry.HttpTracingRequestInitializer;
+import com.google.cloud.bigquery.testing.RemoteBigQueryHelper;
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.api.trace.Tracer;
+import io.opentelemetry.sdk.OpenTelemetrySdk;
+import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.data.SpanData;
+import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+public class ITOpenTelemetryTest {
+
+  private static RemoteBigQueryHelper bigqueryHelper;
+  private InMemorySpanExporter memoryExporter;
+  private Tracer tracer;
+
+  @BeforeAll
+  public static void setUpClass() throws IOException {
+    System.setProperty("com.google.cloud.bigquery.http.tracing.dev.enabled", "true");
+    bigqueryHelper = RemoteBigQueryHelper.create();
+  }
+
+  @BeforeEach
+  public void setUp() {
+    memoryExporter = InMemorySpanExporter.create();
+    SdkTracerProvider tracerProvider =
+        SdkTracerProvider.builder()
+            .addSpanProcessor(SimpleSpanProcessor.create(memoryExporter))
+            .build();
+    OpenTelemetry openTelemetry =
+        OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build();
+    tracer = openTelemetry.getTracer("it-otel-test");
+  }
+
+  @Test
+  public void testListDatasetsTraced() {
+    BigQuery bq =
+        bigqueryHelper.getOptions().toBuilder()
+            .setEnableOpenTelemetryTracing(true)
+            .setOpenTelemetryTracer(tracer)
+            .build()
+            .getService();
+
+    bq.listDatasets();
+
+    List<SpanData> spans = memoryExporter.getFinishedSpanItems();
+    assertNotNull(spans);
+    assertFalse(spans.isEmpty(), "Expected at least one span collected");
+
+    boolean foundRpcSpan = false;
+    for (SpanData span : spans) {
+      if (span.getName().equals("com.google.cloud.bigquery.BigQueryRpc.listDatasets")) {
+        foundRpcSpan = true;
+        Map<AttributeKey<?>, Object> attrs = span.getAttributes().asMap();
+        checkGeneralAttributes(attrs);
+        assertEquals("GET", attrs.get(HttpTracingRequestInitializer.HTTP_REQUEST_METHOD));
+        assertEquals("DatasetService", attrs.get(AttributeKey.stringKey("bq.rpc.service")));
+        assertEquals("ListDatasets", attrs.get(AttributeKey.stringKey("bq.rpc.method")));
+        assertEquals("bigquery.googleapis.com", attrs.get(BigQueryTelemetryTracer.SERVER_ADDRESS));
+        assertEquals(200L, attrs.get(HttpTracingRequestInitializer.HTTP_RESPONSE_STATUS_CODE));
+        assertEquals("bigquery.googleapis.com", attrs.get(BigQueryTelemetryTracer.URL_DOMAIN));
+        assertEquals(
+            "https://bigquery.googleapis.com/bigquery/v2/projects/gcloud-devel/datasets?prettyPrint=false",
+            attrs.get(HttpTracingRequestInitializer.URL_FULL));
+        assertEquals(
+            "//bigquery.googleapis.com/projects/gcloud-devel/datasets",
+            attrs.get(BigQueryTelemetryTracer.GCP_RESOURCE_DESTINATION_ID));
+        assertEquals(
+            "projects/{+projectId}/datasets", attrs.get(BigQueryTelemetryTracer.URL_TEMPLATE));
+      }
+    }
+    assertTrue(foundRpcSpan, "Expected to find BigQueryRpc.listDatasets span");
+  }
+
+  @Test
+  public void testGetDatasetNotFoundTraced() {
+    BigQuery bq =
+        bigqueryHelper.getOptions().toBuilder()
+            .setEnableOpenTelemetryTracing(true)
+            .setOpenTelemetryTracer(tracer)
+            .build()
+            .getService();
+
+    bq.getDataset("non_existent_dataset");
+
+    List<SpanData> spans = memoryExporter.getFinishedSpanItems();
+    assertNotNull(spans);
+    assertFalse(spans.isEmpty(), "Expected at least one span collected");
+
+    boolean foundRpcSpan = false;
+    for (SpanData span : spans) {
+      if (span.getName().equals("com.google.cloud.bigquery.BigQueryRpc.getDataset")) {
+        foundRpcSpan = true;
+        Map<AttributeKey<?>, Object> attrs = span.getAttributes().asMap();
+        checkGeneralAttributes(attrs);
+        assertEquals("GET", attrs.get(HttpTracingRequestInitializer.HTTP_REQUEST_METHOD));
+        assertEquals("DatasetService", attrs.get(AttributeKey.stringKey("bq.rpc.service")));
+        assertEquals("GetDataset", attrs.get(AttributeKey.stringKey("bq.rpc.method")));
+        assertEquals(404L, attrs.get(HttpTracingRequestInitializer.HTTP_RESPONSE_STATUS_CODE));
+        assertEquals(
+            "projects/{+projectId}/datasets/{+datasetId}",
+            attrs.get(BigQueryTelemetryTracer.URL_TEMPLATE));
+        assertEquals(
+            "https://bigquery.googleapis.com/bigquery/v2/projects/gcloud-devel/datasets/non_existent_dataset?prettyPrint=false",
+            attrs.get(HttpTracingRequestInitializer.URL_FULL));
+        assertEquals("bigquery.googleapis.com", attrs.get(BigQueryTelemetryTracer.SERVER_ADDRESS));
+        assertEquals("bigquery.googleapis.com", attrs.get(BigQueryTelemetryTracer.URL_DOMAIN));
+        assertEquals(
+            "//bigquery.googleapis.com/projects/gcloud-devel/datasets/non_existent_dataset",
+            attrs.get(BigQueryTelemetryTracer.GCP_RESOURCE_DESTINATION_ID));
+
+        // Error attributes
+        assertEquals("notFound", attrs.get(BigQueryTelemetryTracer.ERROR_TYPE));
+        assertEquals(
+            "Not found: Dataset gcloud-devel:non_existent_dataset",
+            attrs.get(BigQueryTelemetryTracer.STATUS_MESSAGE));
+      }
+    }
+    assertTrue(foundRpcSpan, "Expected to find BigQueryRpc.getDataset span");
+  }
+
+  @Test
+  public void testConnectionErrorRetriesTraced() {
+    // Pass invalid host to force connection error and retries
+    BigQuery bq =
+        bigqueryHelper.getOptions().toBuilder()
+            .setRetrySettings(RetrySettings.newBuilder().setMaxAttempts(5).build())
+            .setEnableOpenTelemetryTracing(true)
+            .setOpenTelemetryTracer(tracer)
+            .setHost("https://invalid-host-name-12345.com:8080")
+            .build()
+            .getService();
+
+    try {
+      bq.listDatasets();
+      fail("Expected BigQueryException due to invalid host");
+    } catch (BigQueryException e) {
+      // Expected
+    }
+
+    List<SpanData> spans = memoryExporter.getFinishedSpanItems();
+    assertNotNull(spans);
+    assertFalse(spans.isEmpty(), "Expected at least one span collected");
+
+    int rpcSpanCount = 0;
+    for (SpanData span : spans) {
+      if (span.getName().equals("com.google.cloud.bigquery.BigQueryRpc.listDatasets")) {
+        rpcSpanCount++;
+        Map<AttributeKey<?>, Object> attrs = span.getAttributes().asMap();
+        checkGeneralAttributes(attrs);
+        assertEquals(
+            "https://invalid-host-name-12345.com:8080/bigquery/v2/projects/gcloud-devel/datasets?prettyPrint=false",
+            (String) attrs.get(HttpTracingRequestInitializer.URL_FULL));
+        assertEquals(
+            "invalid-host-name-12345.com", attrs.get(BigQueryTelemetryTracer.SERVER_ADDRESS));
+        assertEquals(8080L, attrs.get(BigQueryTelemetryTracer.SERVER_PORT));
+        assertEquals("invalid-host-name-12345.com", attrs.get(BigQueryTelemetryTracer.URL_DOMAIN));
+        assertEquals(
+            "projects/{+projectId}/datasets", attrs.get(BigQueryTelemetryTracer.URL_TEMPLATE));
+        assertEquals(
+            "//bigquery.googleapis.com/projects/gcloud-devel/datasets",
+            attrs.get(BigQueryTelemetryTracer.GCP_RESOURCE_DESTINATION_ID));
+        checkRetryAttribute(span, rpcSpanCount);
+
+        // Error attributes
+        assertEquals(
+            "java.net.UnknownHostException", attrs.get(BigQueryTelemetryTracer.EXCEPTION_TYPE));
+        assertEquals(
+            ErrorType.CLIENT_CONNECTION_ERROR.toString(),
+            attrs.get(BigQueryTelemetryTracer.ERROR_TYPE));
+        assertEquals(
+            "UnknownHostException: invalid-host-name-12345.com",
+            attrs.get(BigQueryTelemetryTracer.STATUS_MESSAGE));
+      }
+    }
+    assertEquals(5, rpcSpanCount, "Expected 5 attempts total");
+  }
+
+  @Test
+  public void testSimultaneousCallsDoNotAffectResendCountForEachother() {
+    BigQuery bq =
+        bigqueryHelper.getOptions().toBuilder()
+            .setRetrySettings(RetrySettings.newBuilder().setMaxAttempts(5).build())
+            .setEnableOpenTelemetryTracing(true)
+            .setOpenTelemetryTracer(tracer)
+            .setHost("https://invalid-host-name-123456.com")
+            .build()
+            .getService();
+
+    try {
+      bq.listDatasets();
+      fail("Expected BigQueryException due to invalid host");
+    } catch (BigQueryException e) {
+      // Expected
+    }
+    try {
+      bq.cancel("test-job-id");
+      fail("Expected BigQueryException due to invalid host");
+    } catch (BigQueryException e) {
+      // Expected
+    }
+
+    List<SpanData> spans = memoryExporter.getFinishedSpanItems();
+    assertNotNull(spans);
+    assertFalse(spans.isEmpty(), "Expected at least one span collected");
+
+    int listDataSpanCount = 0;
+    int cancelJobSpanCount = 0;
+    for (SpanData span : spans) {
+      if (span.getName().equals("com.google.cloud.bigquery.BigQueryRpc.listDatasets")) {
+        listDataSpanCount++;
+        checkRetryAttribute(span, listDataSpanCount);
+      } else if (span.getName().equals("com.google.cloud.bigquery.BigQueryRpc.cancelJob")) {
+        cancelJobSpanCount++;
+        checkRetryAttribute(span, cancelJobSpanCount);
+      }
+    }
+    assertEquals(5, listDataSpanCount, "Expected 5 attempts total for listDatasets call");
+    assertEquals(5, cancelJobSpanCount, "Expected 5 attempts total for cancelJob call");
+  }
+
+  private static void checkRetryAttribute(SpanData span, int listDataSpanCount) {
+    Map<AttributeKey<?>, Object> attrs = span.getAttributes().asMap();
+    Long resendCount = (Long) attrs.get(HttpTracingRequestInitializer.HTTP_REQUEST_RESEND_COUNT);
+    if (listDataSpanCount == 1) {
+      assertTrue(resendCount == null || resendCount == 0);
+    } else {
+      assertNotNull(resendCount, "Expected resend count for retry attempt " + listDataSpanCount);
+      assertEquals((long) (listDataSpanCount - 1), resendCount.longValue());
+    }
+  }
+
+  private void checkGeneralAttributes(Map<AttributeKey<?>, Object> attrs) {
+    assertEquals(
+        HttpTracingRequestInitializer.HTTP_RPC_SYSTEM_NAME,
+        attrs.get(BigQueryTelemetryTracer.RPC_SYSTEM_NAME));
+    assertEquals(
+        BigQueryTelemetryTracer.BQ_GCP_CLIENT_SERVICE,
+        attrs.get(BigQueryTelemetryTracer.GCP_CLIENT_SERVICE));
+    assertEquals(
+        BigQueryTelemetryTracer.BQ_GCP_CLIENT_REPO,
+        attrs.get(BigQueryTelemetryTracer.GCP_CLIENT_REPO));
+    assertEquals(
+        BigQueryTelemetryTracer.BQ_GCP_CLIENT_LANGUAGE,
+        attrs.get(BigQueryTelemetryTracer.GCP_CLIENT_LANGUAGE));
+    assertEquals(
+        BigQueryTelemetryTracer.BQ_GCP_CLIENT_ARTIFACT,
+        attrs.get(BigQueryTelemetryTracer.GCP_CLIENT_ARTIFACT));
+    assertNotNull(attrs.get(BigQueryTelemetryTracer.GCP_CLIENT_VERSION));
+  }
+}