feat(bigquery): add resend attribute to span tracing + integration tests (#12313)

ldetmer · gemini-code-assist[bot] · web-flow · commit 167722dd8d27 · 2026-04-01T16:58:22.000-04:00
This adds the http.request.resend_count to span tracing in the case of a retry. We achieve this by adding a context to the parent span that tracks all retries. This remains at null for the initial retry and then gets incremented for all subsequent retries. This PR also introduces integration tests to fully validate the resent attribute. [example trace with resent attribute set ](https://screenshot.googleplex.com/BTjLU6v2stpYPWf.png) --------- Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
diff --git a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryHelper.java b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryHelper.java
@@ -27,15 +27,21 @@
 import com.google.cloud.RetryHelper;
 import io.opentelemetry.api.trace.Span;
 import io.opentelemetry.api.trace.Tracer;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.ContextKey;
 import io.opentelemetry.context.Scope;
 import java.io.IOException;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
 public class BigQueryRetryHelper extends RetryHelper {
 
+  public static final ContextKey<AtomicInteger> RETRY_ATTEMPT_KEY =
+      ContextKey.named("bq_retry_attempt");
+
   private static final Logger LOG = Logger.getLogger(BigQueryRetryHelper.class.getName());
 
   public static <V> V runWithRetries(
@@ -54,7 +60,11 @@ public static <V> V runWithRetries(
               .spanBuilder("com.google.cloud.bigquery.BigQueryRetryHelper.runWithRetries")
               .startSpan();
     }
-    try (Scope runWithRetriesScope = runWithRetries != null ? runWithRetries.makeCurrent() : null) {
+    Context retryContext = Context.current().with(RETRY_ATTEMPT_KEY, new AtomicInteger(0));
+    if (runWithRetries != null) {
+      retryContext = retryContext.with(runWithRetries);
+    }
+    try (Scope runWithRetriesScope = retryContext.makeCurrent()) {
       // Suppressing should be ok as a workaraund. Current and only ResultRetryAlgorithm
       // implementation does not use response at all, so ignoring its type is ok.
       @SuppressWarnings("unchecked")
diff --git a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/telemetry/ErrorTypeUtil.java b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/telemetry/ErrorTypeUtil.java
@@ -16,16 +16,19 @@
 package com.google.cloud.bigquery.telemetry;
 
 import com.google.api.core.BetaApi;
+import com.google.common.annotations.VisibleForTesting;
 
 /**
  * Utility class for identifying exception types for telemetry tracking. TODO: this class should get
  * replaced with gax version when ready work tracked in
  * https://github.com/googleapis/google-cloud-java/issues/12105
  */
 @BetaApi
-class ErrorTypeUtil {
+@VisibleForTesting
+public class ErrorTypeUtil {
 
-  enum ErrorType {
+  @VisibleForTesting
+  public enum ErrorType {
     CLIENT_TIMEOUT,
     CLIENT_CONNECTION_ERROR,
     CLIENT_REQUEST_ERROR,
diff --git a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/telemetry/HttpTracingRequestInitializer.java b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/telemetry/HttpTracingRequestInitializer.java
@@ -19,13 +19,15 @@
 import com.google.api.client.http.*;
 import com.google.api.core.BetaApi;
 import com.google.api.core.InternalApi;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
 import com.google.common.annotations.VisibleForTesting;
 import io.opentelemetry.api.common.AttributeKey;
 import io.opentelemetry.api.trace.Span;
 import io.opentelemetry.api.trace.Tracer;
 import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator;
 import io.opentelemetry.context.Context;
 import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  * HttpRequestInitializer that wraps a delegate initializer, intercepts all HTTP requests, adds
@@ -50,7 +52,7 @@ public class HttpTracingRequestInitializer implements HttpRequestInitializer {
   public static final AttributeKey<Long> HTTP_RESPONSE_BODY_SIZE =
       AttributeKey.longKey("http.response.body.size");
 
-  @VisibleForTesting static final String HTTP_RPC_SYSTEM_NAME = "http";
+  @VisibleForTesting public static final String HTTP_RPC_SYSTEM_NAME = "http";
 
   private static final java.util.Set<String> REDACTED_QUERY_PARAMETERS =
       com.google.common.collect.ImmutableSet.of(
@@ -84,6 +86,14 @@ public void initialize(HttpRequest request) throws IOException {
 
     addInitialHttpAttributesToSpan(span, request);
 
+    AtomicInteger attemptTracker = Context.current().get(BigQueryRetryHelper.RETRY_ATTEMPT_KEY);
+    if (attemptTracker != null) {
+      int attempt = attemptTracker.getAndIncrement();
+      if (attempt > 0) {
+        span.setAttribute(HTTP_REQUEST_RESEND_COUNT, (long) attempt);
+      }
+    }
+
     HttpResponseInterceptor originalInterceptor = request.getResponseInterceptor();
     request.setResponseInterceptor(
         response -> {
diff --git a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITOpenTelemetryTest.java b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITOpenTelemetryTest.java
@@ -0,0 +1,286 @@
+/*
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.bigquery.it;
+
+import static com.google.cloud.bigquery.telemetry.ErrorTypeUtil.ErrorType;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
+import com.google.api.gax.retrying.RetrySettings;
+import com.google.cloud.bigquery.BigQuery;
+import com.google.cloud.bigquery.BigQueryException;
+import com.google.cloud.bigquery.telemetry.BigQueryTelemetryTracer;
+import com.google.cloud.bigquery.telemetry.HttpTracingRequestInitializer;
+import com.google.cloud.bigquery.testing.RemoteBigQueryHelper;
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.api.trace.Tracer;
+import io.opentelemetry.sdk.OpenTelemetrySdk;
+import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.data.SpanData;
+import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+public class ITOpenTelemetryTest {
+
+  private static RemoteBigQueryHelper bigqueryHelper;
+  private InMemorySpanExporter memoryExporter;
+  private Tracer tracer;
+
+  @BeforeAll
+  public static void setUpClass() throws IOException {
+    System.setProperty("com.google.cloud.bigquery.http.tracing.dev.enabled", "true");
+    bigqueryHelper = RemoteBigQueryHelper.create();
+  }
+
+  @BeforeEach
+  public void setUp() {
+    memoryExporter = InMemorySpanExporter.create();
+    SdkTracerProvider tracerProvider =
+        SdkTracerProvider.builder()
+            .addSpanProcessor(SimpleSpanProcessor.create(memoryExporter))
+            .build();
+    OpenTelemetry openTelemetry =
+        OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build();
+    tracer = openTelemetry.getTracer("it-otel-test");
+  }
+
+  @Test
+  public void testListDatasetsTraced() {
+    BigQuery bq =
+        bigqueryHelper.getOptions().toBuilder()
+            .setEnableOpenTelemetryTracing(true)
+            .setOpenTelemetryTracer(tracer)
+            .build()
+            .getService();
+
+    bq.listDatasets();
+
+    List<SpanData> spans = memoryExporter.getFinishedSpanItems();
+    assertNotNull(spans);
+    assertFalse(spans.isEmpty(), "Expected at least one span collected");
+
+    boolean foundRpcSpan = false;
+    for (SpanData span : spans) {
+      if (span.getName().equals("com.google.cloud.bigquery.BigQueryRpc.listDatasets")) {
+        foundRpcSpan = true;
+        Map<AttributeKey<?>, Object> attrs = span.getAttributes().asMap();
+        checkGeneralAttributes(attrs);
+        assertEquals("GET", attrs.get(HttpTracingRequestInitializer.HTTP_REQUEST_METHOD));
+        assertEquals("DatasetService", attrs.get(AttributeKey.stringKey("bq.rpc.service")));
+        assertEquals("ListDatasets", attrs.get(AttributeKey.stringKey("bq.rpc.method")));
+        assertEquals("bigquery.googleapis.com", attrs.get(BigQueryTelemetryTracer.SERVER_ADDRESS));
+        assertEquals(200L, attrs.get(HttpTracingRequestInitializer.HTTP_RESPONSE_STATUS_CODE));
+        assertEquals("bigquery.googleapis.com", attrs.get(BigQueryTelemetryTracer.URL_DOMAIN));
+        assertEquals(
+            "https://bigquery.googleapis.com/bigquery/v2/projects/gcloud-devel/datasets?prettyPrint=false",
+            attrs.get(HttpTracingRequestInitializer.URL_FULL));
+        assertEquals(
+            "//bigquery.googleapis.com/projects/gcloud-devel/datasets",
+            attrs.get(BigQueryTelemetryTracer.GCP_RESOURCE_DESTINATION_ID));
+        assertEquals(
+            "projects/{+projectId}/datasets", attrs.get(BigQueryTelemetryTracer.URL_TEMPLATE));
+      }
+    }
+    assertTrue(foundRpcSpan, "Expected to find BigQueryRpc.listDatasets span");
+  }
+
+  @Test
+  public void testGetDatasetNotFoundTraced() {
+    BigQuery bq =
+        bigqueryHelper.getOptions().toBuilder()
+            .setEnableOpenTelemetryTracing(true)
+            .setOpenTelemetryTracer(tracer)
+            .build()
+            .getService();
+
+    bq.getDataset("non_existent_dataset");
+
+    List<SpanData> spans = memoryExporter.getFinishedSpanItems();
+    assertNotNull(spans);
+    assertFalse(spans.isEmpty(), "Expected at least one span collected");
+
+    boolean foundRpcSpan = false;
+    for (SpanData span : spans) {
+      if (span.getName().equals("com.google.cloud.bigquery.BigQueryRpc.getDataset")) {
+        foundRpcSpan = true;
+        Map<AttributeKey<?>, Object> attrs = span.getAttributes().asMap();
+        checkGeneralAttributes(attrs);
+        assertEquals("GET", attrs.get(HttpTracingRequestInitializer.HTTP_REQUEST_METHOD));
+        assertEquals("DatasetService", attrs.get(AttributeKey.stringKey("bq.rpc.service")));
+        assertEquals("GetDataset", attrs.get(AttributeKey.stringKey("bq.rpc.method")));
+        assertEquals(404L, attrs.get(HttpTracingRequestInitializer.HTTP_RESPONSE_STATUS_CODE));
+        assertEquals(
+            "projects/{+projectId}/datasets/{+datasetId}",
+            attrs.get(BigQueryTelemetryTracer.URL_TEMPLATE));
+        assertEquals(
+            "https://bigquery.googleapis.com/bigquery/v2/projects/gcloud-devel/datasets/non_existent_dataset?prettyPrint=false",
+            attrs.get(HttpTracingRequestInitializer.URL_FULL));
+        assertEquals("bigquery.googleapis.com", attrs.get(BigQueryTelemetryTracer.SERVER_ADDRESS));
+        assertEquals("bigquery.googleapis.com", attrs.get(BigQueryTelemetryTracer.URL_DOMAIN));
+        assertEquals(
+            "//bigquery.googleapis.com/projects/gcloud-devel/datasets/non_existent_dataset",
+            attrs.get(BigQueryTelemetryTracer.GCP_RESOURCE_DESTINATION_ID));
+
+        // Error attributes
+        assertEquals("notFound", attrs.get(BigQueryTelemetryTracer.ERROR_TYPE));
+        assertEquals(
+            "Not found: Dataset gcloud-devel:non_existent_dataset",
+            attrs.get(BigQueryTelemetryTracer.STATUS_MESSAGE));
+      }
+    }
+    assertTrue(foundRpcSpan, "Expected to find BigQueryRpc.getDataset span");
+  }
+
+  @Test
+  public void testConnectionErrorRetriesTraced() {
+    // Pass invalid host to force connection error and retries
+    BigQuery bq =
+        bigqueryHelper.getOptions().toBuilder()
+            .setRetrySettings(RetrySettings.newBuilder().setMaxAttempts(5).build())
+            .setEnableOpenTelemetryTracing(true)
+            .setOpenTelemetryTracer(tracer)
+            .setHost("https://invalid-host-name-12345.com:8080")
+            .build()
+            .getService();
+
+    try {
+      bq.listDatasets();
+      fail("Expected BigQueryException due to invalid host");
+    } catch (BigQueryException e) {
+      // Expected
+    }
+
+    List<SpanData> spans = memoryExporter.getFinishedSpanItems();
+    assertNotNull(spans);
+    assertFalse(spans.isEmpty(), "Expected at least one span collected");
+
+    int rpcSpanCount = 0;
+    for (SpanData span : spans) {
+      if (span.getName().equals("com.google.cloud.bigquery.BigQueryRpc.listDatasets")) {
+        rpcSpanCount++;
+        Map<AttributeKey<?>, Object> attrs = span.getAttributes().asMap();
+        checkGeneralAttributes(attrs);
+        assertEquals(
+            "https://invalid-host-name-12345.com:8080/bigquery/v2/projects/gcloud-devel/datasets?prettyPrint=false",
+            (String) attrs.get(HttpTracingRequestInitializer.URL_FULL));
+        assertEquals(
+            "invalid-host-name-12345.com", attrs.get(BigQueryTelemetryTracer.SERVER_ADDRESS));
+        assertEquals(8080L, attrs.get(BigQueryTelemetryTracer.SERVER_PORT));
+        assertEquals("invalid-host-name-12345.com", attrs.get(BigQueryTelemetryTracer.URL_DOMAIN));
+        assertEquals(
+            "projects/{+projectId}/datasets", attrs.get(BigQueryTelemetryTracer.URL_TEMPLATE));
+        assertEquals(
+            "//bigquery.googleapis.com/projects/gcloud-devel/datasets",
+            attrs.get(BigQueryTelemetryTracer.GCP_RESOURCE_DESTINATION_ID));
+        checkRetryAttribute(span, rpcSpanCount);
+
+        // Error attributes
+        assertEquals(
+            "java.net.UnknownHostException", attrs.get(BigQueryTelemetryTracer.EXCEPTION_TYPE));
+        assertEquals(
+            ErrorType.CLIENT_CONNECTION_ERROR.toString(),
+            attrs.get(BigQueryTelemetryTracer.ERROR_TYPE));
+        assertEquals(
+            "UnknownHostException: invalid-host-name-12345.com",
+            attrs.get(BigQueryTelemetryTracer.STATUS_MESSAGE));
+      }
+    }
+    assertEquals(5, rpcSpanCount, "Expected 5 attempts total");
+  }
+
+  @Test
+  public void testSimultaneousCallsDoNotAffectResendCountForEachother() {
+    BigQuery bq =
+        bigqueryHelper.getOptions().toBuilder()
+            .setRetrySettings(RetrySettings.newBuilder().setMaxAttempts(5).build())
+            .setEnableOpenTelemetryTracing(true)
+            .setOpenTelemetryTracer(tracer)
+            .setHost("https://invalid-host-name-123456.com")
+            .build()
+            .getService();
+
+    try {
+      bq.listDatasets();
+      fail("Expected BigQueryException due to invalid host");
+    } catch (BigQueryException e) {
+      // Expected
+    }
+    try {
+      bq.cancel("test-job-id");
+      fail("Expected BigQueryException due to invalid host");
+    } catch (BigQueryException e) {
+      // Expected
+    }
+
+    List<SpanData> spans = memoryExporter.getFinishedSpanItems();
+    assertNotNull(spans);
+    assertFalse(spans.isEmpty(), "Expected at least one span collected");
+
+    int listDataSpanCount = 0;
+    int cancelJobSpanCount = 0;
+    for (SpanData span : spans) {
+      if (span.getName().equals("com.google.cloud.bigquery.BigQueryRpc.listDatasets")) {
+        listDataSpanCount++;
+        checkRetryAttribute(span, listDataSpanCount);
+      } else if (span.getName().equals("com.google.cloud.bigquery.BigQueryRpc.cancelJob")) {
+        cancelJobSpanCount++;
+        checkRetryAttribute(span, cancelJobSpanCount);
+      }
+    }
+    assertEquals(5, listDataSpanCount, "Expected 5 attempts total for listDatasets call");
+    assertEquals(5, cancelJobSpanCount, "Expected 5 attempts total for cancelJob call");
+  }
+
+  private static void checkRetryAttribute(SpanData span, int listDataSpanCount) {
+    Map<AttributeKey<?>, Object> attrs = span.getAttributes().asMap();
+    Long resendCount = (Long) attrs.get(HttpTracingRequestInitializer.HTTP_REQUEST_RESEND_COUNT);
+    if (listDataSpanCount == 1) {
+      assertTrue(resendCount == null || resendCount == 0);
+    } else {
+      assertNotNull(resendCount, "Expected resend count for retry attempt " + listDataSpanCount);
+      assertEquals((long) (listDataSpanCount - 1), resendCount.longValue());
+    }
+  }
+
+  private void checkGeneralAttributes(Map<AttributeKey<?>, Object> attrs) {
+    assertEquals(
+        HttpTracingRequestInitializer.HTTP_RPC_SYSTEM_NAME,
+        attrs.get(BigQueryTelemetryTracer.RPC_SYSTEM_NAME));
+    assertEquals(
+        BigQueryTelemetryTracer.BQ_GCP_CLIENT_SERVICE,
+        attrs.get(BigQueryTelemetryTracer.GCP_CLIENT_SERVICE));
+    assertEquals(
+        BigQueryTelemetryTracer.BQ_GCP_CLIENT_REPO,
+        attrs.get(BigQueryTelemetryTracer.GCP_CLIENT_REPO));
+    assertEquals(
+        BigQueryTelemetryTracer.BQ_GCP_CLIENT_LANGUAGE,
+        attrs.get(BigQueryTelemetryTracer.GCP_CLIENT_LANGUAGE));
+    assertEquals(
+        BigQueryTelemetryTracer.BQ_GCP_CLIENT_ARTIFACT,
+        attrs.get(BigQueryTelemetryTracer.GCP_CLIENT_ARTIFACT));
+    assertNotNull(attrs.get(BigQueryTelemetryTracer.GCP_CLIENT_VERSION));
+  }
+}
diff --git a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpcTest.java b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpcTest.java
diff --git a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/telemetry/HttpTracingRequestInitializerTest.java b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/telemetry/HttpTracingRequestInitializerTest.java
