Skip to content

Commit b480e7d

Browse files
diegomarquezpdiegomarquezp
committed
impl(o11y): introduce resend_count logic
1 parent 5dce4f9 commit b480e7d

5 files changed

Lines changed: 272 additions & 0 deletions

File tree

‎gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityAttributes.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,4 +81,10 @@ public class ObservabilityAttributes {
8181

8282
/** The url template of the request (e.g. /v1/{name}:access). */
8383
public static final String URL_TEMPLATE_ATTRIBUTE = "url.template";
84+
85+
/** The resend count of the request. Only used in HTTP transport. */
86+
public static final String HTTP_RESEND_COUNT_ATTRIBUTE = "http.request.resend_count";
87+
88+
/** The resend count of the request. Only used in gRPC transport. */
89+
public static final String GRPC_RESEND_COUNT_ATTRIBUTE = "gcp.grpc.resend_count";
8490
}

‎gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityUtils.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,8 @@ static Attributes toOtelAttributes(Map<String, Object> attributes) {
6565
(k, v) -> {
6666
if (v instanceof String) {
6767
attributesBuilder.put(k, (String) v);
68+
} else if (v instanceof Long) {
69+
attributesBuilder.put(k, (Long) v);
6870
} else if (v instanceof Integer) {
6971
attributesBuilder.put(k, (long) (Integer) v);
7072
}

‎gax-java/gax/src/main/java/com/google/api/gax/tracing/SpanTracer.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ public class SpanTracer implements ApiTracer {
5252
private final String attemptSpanName;
5353
private final ApiTracerContext apiTracerContext;
5454
private TraceManager.Span attemptHandle;
55+
private long resendCount;
5556

5657
/**
5758
* Creates a new instance of {@code SpanTracer}.
@@ -65,6 +66,7 @@ public SpanTracer(
6566
this.attemptSpanName = attemptSpanName;
6667
this.apiTracerContext = apiTracerContext;
6768
this.attemptAttributes = new HashMap<>();
69+
this.resendCount = 0;
6870
buildAttributes();
6971
}
7072

@@ -76,15 +78,48 @@ private void buildAttributes() {
7678
@Override
7779
public void attemptStarted(Object request, int attemptNumber) {
7880
Map<String, Object> attemptAttributes = new HashMap<>(this.attemptAttributes);
81+
82+
if (this.resendCount > 0) {
83+
ApiTracerContext.Transport transport = apiTracerContext.transport();
84+
if (transport == ApiTracerContext.Transport.GRPC) {
85+
attemptAttributes.put(
86+
ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE, this.resendCount);
87+
} else if (transport == ApiTracerContext.Transport.HTTP) {
88+
attemptAttributes.put(
89+
ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE, this.resendCount);
90+
}
91+
}
92+
7993
// Start the specific attempt span with the operation span as parent
8094
this.attemptHandle = traceManager.createSpan(attemptSpanName, attemptAttributes);
95+
this.resendCount++;
8196
}
8297

8398
@Override
8499
public void attemptSucceeded() {
85100
endAttempt();
86101
}
87102

103+
@Override
104+
public void attemptCancelled() {
105+
endAttempt();
106+
}
107+
108+
@Override
109+
public void attemptFailedDuration(Throwable error, java.time.Duration delay) {
110+
endAttempt();
111+
}
112+
113+
@Override
114+
public void attemptFailedRetriesExhausted(Throwable error) {
115+
endAttempt();
116+
}
117+
118+
@Override
119+
public void attemptPermanentFailure(Throwable error) {
120+
endAttempt();
121+
}
122+
88123
private void endAttempt() {
89124
if (attemptHandle != null) {
90125
attemptHandle.end();

‎gax-java/gax/src/test/java/com/google/api/gax/tracing/SpanTracerTest.java‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,4 +76,90 @@ void testAttemptStarted_includesLanguageAttribute() {
7676
assertThat(attributesCaptor.getValue())
7777
.containsEntry(SpanTracer.LANGUAGE_ATTRIBUTE, SpanTracer.DEFAULT_LANGUAGE);
7878
}
79+
80+
@Test
81+
void testAttemptStarted_retryAttributes_grpc() {
82+
ApiTracerContext grpcContext =
83+
ApiTracerContext.newBuilder()
84+
.setLibraryMetadata(com.google.api.gax.rpc.LibraryMetadata.empty())
85+
.setTransport(ApiTracerContext.Transport.GRPC)
86+
.build();
87+
SpanTracer grpcTracer = new SpanTracer(recorder, grpcContext, ATTEMPT_SPAN_NAME);
88+
89+
// First attempt, no retry attribute
90+
grpcTracer.attemptStarted(new Object(), 0);
91+
ArgumentCaptor<Map> attributesCaptor = ArgumentCaptor.forClass(Map.class);
92+
verify(recorder).createSpan(eq(ATTEMPT_SPAN_NAME), attributesCaptor.capture());
93+
assertThat(attributesCaptor.getValue())
94+
.doesNotContainKey(ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE);
95+
assertThat(attributesCaptor.getValue())
96+
.doesNotContainKey(ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE);
97+
98+
// First retry
99+
grpcTracer.attemptStarted(new Object(), 0);
100+
verify(recorder, org.mockito.Mockito.times(2))
101+
.createSpan(eq(ATTEMPT_SPAN_NAME), attributesCaptor.capture());
102+
Map<String, Object> capturedAttributes = (Map<String, Object>) attributesCaptor.getValue();
103+
assertThat(capturedAttributes)
104+
.containsEntry(ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE, 1L);
105+
assertThat(capturedAttributes)
106+
.doesNotContainKey(ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE);
107+
108+
// N-th retry
109+
grpcTracer.attemptStarted(new Object(), 0);
110+
grpcTracer.attemptStarted(new Object(), 0);
111+
grpcTracer.attemptStarted(new Object(), 0);
112+
grpcTracer.attemptStarted(new Object(), 0);
113+
verify(recorder, org.mockito.Mockito.times(6))
114+
.createSpan(eq(ATTEMPT_SPAN_NAME), attributesCaptor.capture());
115+
capturedAttributes = (Map<String, Object>) attributesCaptor.getValue();
116+
assertThat(capturedAttributes)
117+
.containsEntry(ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE, 5L);
118+
assertThat(capturedAttributes)
119+
.doesNotContainKey(ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE);
120+
}
121+
122+
@Test
123+
void testAttemptStarted_retryAttributes_http() {
124+
ApiTracerContext httpContext =
125+
ApiTracerContext.newBuilder()
126+
.setLibraryMetadata(com.google.api.gax.rpc.LibraryMetadata.empty())
127+
.setTransport(ApiTracerContext.Transport.HTTP)
128+
.build();
129+
SpanTracer httpTracer = new SpanTracer(recorder, httpContext, ATTEMPT_SPAN_NAME);
130+
ArgumentCaptor<Map> attributesCaptor = ArgumentCaptor.forClass(Map.class);
131+
132+
// First attempt, no retry attribute
133+
httpTracer.attemptStarted(new Object(), 0);
134+
verify(recorder, org.mockito.Mockito.times(1))
135+
.createSpan(eq(ATTEMPT_SPAN_NAME), attributesCaptor.capture());
136+
Map<String, Object> capturedAttributes = (Map<String, Object>) attributesCaptor.getValue();
137+
assertThat(capturedAttributes)
138+
.doesNotContainKey(ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE);
139+
assertThat(capturedAttributes)
140+
.doesNotContainKey(ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE);
141+
142+
// First retry
143+
httpTracer.attemptStarted(new Object(), 0);
144+
verify(recorder, org.mockito.Mockito.times(2))
145+
.createSpan(eq(ATTEMPT_SPAN_NAME), attributesCaptor.capture());
146+
capturedAttributes = (Map<String, Object>) attributesCaptor.getValue();
147+
assertThat(capturedAttributes)
148+
.doesNotContainKey(ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE);
149+
assertThat(capturedAttributes)
150+
.containsEntry(ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE, 1L);
151+
152+
// N-th retry
153+
httpTracer.attemptStarted(new Object(), 0);
154+
httpTracer.attemptStarted(new Object(), 0);
155+
httpTracer.attemptStarted(new Object(), 0);
156+
httpTracer.attemptStarted(new Object(), 0);
157+
verify(recorder, org.mockito.Mockito.times(6))
158+
.createSpan(eq(ATTEMPT_SPAN_NAME), attributesCaptor.capture());
159+
capturedAttributes = (Map<String, Object>) attributesCaptor.getValue();
160+
assertThat(capturedAttributes)
161+
.doesNotContainKey(ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE);
162+
assertThat(capturedAttributes)
163+
.containsEntry(ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE, 5L);
164+
}
79165
}

‎java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelTracing.java‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,14 +31,24 @@
3131
package com.google.showcase.v1beta1.it;
3232

3333
import static com.google.common.truth.Truth.assertThat;
34+
import static org.junit.Assert.assertThrows;
3435

36+
import com.google.api.client.http.javanet.NetHttpTransport;
37+
import com.google.api.gax.core.NoCredentialsProvider;
38+
import com.google.api.gax.retrying.RetrySettings;
39+
import com.google.api.gax.rpc.StatusCode;
40+
import com.google.api.gax.rpc.UnavailableException;
3541
import com.google.api.gax.tracing.ObservabilityAttributes;
3642
import com.google.api.gax.tracing.OpenTelemetryTraceManager;
3743
import com.google.api.gax.tracing.SpanTracer;
3844
import com.google.api.gax.tracing.SpanTracerFactory;
45+
import com.google.rpc.Status;
3946
import com.google.showcase.v1beta1.EchoClient;
4047
import com.google.showcase.v1beta1.EchoRequest;
48+
import com.google.showcase.v1beta1.EchoSettings;
4149
import com.google.showcase.v1beta1.it.util.TestClientInitializer;
50+
import com.google.showcase.v1beta1.stub.EchoStub;
51+
import com.google.showcase.v1beta1.stub.EchoStubSettings;
4252
import io.opentelemetry.api.GlobalOpenTelemetry;
4353
import io.opentelemetry.api.common.AttributeKey;
4454
import io.opentelemetry.api.trace.SpanKind;
@@ -48,6 +58,7 @@
4858
import io.opentelemetry.sdk.trace.data.SpanData;
4959
import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
5060
import java.util.List;
61+
import java.util.Optional;
5162
import org.junit.jupiter.api.AfterEach;
5263
import org.junit.jupiter.api.BeforeEach;
5364
import org.junit.jupiter.api.Test;
@@ -196,4 +207,136 @@ void testTracing_successfulEcho_httpjson() throws Exception {
196207
.isEqualTo("v1beta1/echo:echo");
197208
}
198209
}
210+
211+
@Test
212+
void testTracing_retry_grpc() throws Exception {
213+
final int attempts = 5;
214+
final StatusCode.Code statusCode = StatusCode.Code.UNAVAILABLE;
215+
// A custom EchoClient is used in this test because retries have jitter, and we cannot
216+
// predict the number of attempts that are scheduled for an RPC invocation otherwise.
217+
// The custom retrySettings limit to a set number of attempts before the call gives up.
218+
RetrySettings retrySettings =
219+
RetrySettings.newBuilder()
220+
.setTotalTimeout(org.threeten.bp.Duration.ofMillis(5000L))
221+
.setMaxAttempts(attempts)
222+
.build();
223+
224+
EchoStubSettings.Builder grpcEchoSettingsBuilder = EchoStubSettings.newBuilder();
225+
grpcEchoSettingsBuilder
226+
.echoSettings()
227+
.setRetrySettings(retrySettings)
228+
.setRetryableCodes(statusCode);
229+
EchoSettings grpcEchoSettings = EchoSettings.create(grpcEchoSettingsBuilder.build());
230+
grpcEchoSettings =
231+
grpcEchoSettings.toBuilder()
232+
.setCredentialsProvider(NoCredentialsProvider.create())
233+
.setTransportChannelProvider(EchoSettings.defaultGrpcTransportProviderBuilder().build())
234+
.setEndpoint("localhost:7469")
235+
.build();
236+
237+
SpanTracerFactory tracingFactory =
238+
new SpanTracerFactory(new OpenTelemetryTraceManager(openTelemetrySdk));
239+
240+
EchoStubSettings echoStubSettings =
241+
(EchoStubSettings)
242+
grpcEchoSettings.getStubSettings().toBuilder().setTracerFactory(tracingFactory).build();
243+
EchoStub stub = echoStubSettings.createStub();
244+
EchoClient grpcClient = EchoClient.create(stub);
245+
246+
EchoRequest echoRequest =
247+
EchoRequest.newBuilder()
248+
.setError(Status.newBuilder().setCode(statusCode.ordinal()).build())
249+
.build();
250+
251+
assertThrows(UnavailableException.class, () -> grpcClient.echo(echoRequest));
252+
253+
List<SpanData> spans = spanExporter.getFinishedSpanItems();
254+
assertThat(spans).hasSize(attempts); // Expect exactly one span for the successful retry
255+
256+
// This single span represents the successful retry, which has resend_count=1
257+
for (int resendCount = 1; resendCount < attempts; resendCount++) {
258+
Optional<SpanData> found =
259+
spans.stream()
260+
.filter(
261+
span ->
262+
span.getAttributes()
263+
.asMap()
264+
.getOrDefault(
265+
AttributeKey.longKey(
266+
ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE),
267+
-1L)
268+
.equals(1L))
269+
.findFirst();
270+
assertThat(found).isPresent();
271+
}
272+
}
273+
274+
@Test
275+
void testTracing_retry_httpjson() throws Exception {
276+
final int attempts = 5;
277+
final StatusCode.Code statusCode = StatusCode.Code.UNAVAILABLE;
278+
// A custom EchoClient is used in this test because retries have jitter, and we cannot
279+
// predict the number of attempts that are scheduled for an RPC invocation otherwise.
280+
// The custom retrySettings limit to a set number of attempts before the call gives up.
281+
RetrySettings retrySettings =
282+
RetrySettings.newBuilder()
283+
.setTotalTimeout(org.threeten.bp.Duration.ofMillis(5000L))
284+
.setMaxAttempts(attempts)
285+
.build();
286+
287+
EchoStubSettings.Builder httpJsonEchoSettingsBuilder = EchoStubSettings.newHttpJsonBuilder();
288+
httpJsonEchoSettingsBuilder
289+
.echoSettings()
290+
.setRetrySettings(retrySettings)
291+
.setRetryableCodes(statusCode);
292+
EchoSettings httpJsonEchoSettings = EchoSettings.create(httpJsonEchoSettingsBuilder.build());
293+
httpJsonEchoSettings =
294+
httpJsonEchoSettings.toBuilder()
295+
.setCredentialsProvider(NoCredentialsProvider.create())
296+
.setTransportChannelProvider(
297+
EchoSettings.defaultHttpJsonTransportProviderBuilder()
298+
.setHttpTransport(
299+
new NetHttpTransport.Builder().doNotValidateCertificate().build())
300+
.setEndpoint("http://localhost:7469")
301+
.build())
302+
.build();
303+
304+
SpanTracerFactory tracingFactory =
305+
new SpanTracerFactory(new OpenTelemetryTraceManager(openTelemetrySdk));
306+
307+
EchoStubSettings echoStubSettings =
308+
(EchoStubSettings)
309+
httpJsonEchoSettings.getStubSettings().toBuilder()
310+
.setTracerFactory(tracingFactory)
311+
.build();
312+
EchoStub stub = echoStubSettings.createStub();
313+
EchoClient httpClient = EchoClient.create(stub);
314+
315+
EchoRequest echoRequest =
316+
EchoRequest.newBuilder()
317+
.setError(Status.newBuilder().setCode(statusCode.ordinal()).build())
318+
.build();
319+
320+
assertThrows(UnavailableException.class, () -> httpClient.echo(echoRequest));
321+
322+
List<SpanData> spans = spanExporter.getFinishedSpanItems();
323+
assertThat(spans).hasSize(attempts); // Expect exactly one span for the successful retry
324+
325+
// This single span represents the successful retry, which has resend_count=1
326+
for (int resendCount = 1; resendCount < attempts; resendCount++) {
327+
Optional<SpanData> found =
328+
spans.stream()
329+
.filter(
330+
span ->
331+
span.getAttributes()
332+
.asMap()
333+
.getOrDefault(
334+
AttributeKey.longKey(
335+
ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE),
336+
-1L)
337+
.equals(1L))
338+
.findFirst();
339+
assertThat(found).isPresent();
340+
}
341+
}
199342
}

0 commit comments

Comments
 (0)