Skip to content

tests: Add retry metrics tests to ITOtelGoldenMetrics #12672

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
blakeli0 merged 8 commits into from
Apr 3, 2026
Merged

tests: Add retry metrics tests to ITOtelGoldenMetrics #12672

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
433c431
Add retry metrics tests to ITOtelGoldenMetrics
blakeli0 Apr 3, 2026
7ec89f7
tests: resolve comments
blakeli0 Apr 3, 2026
4ecfe55
tests: fix errors.
blakeli0 Apr 3, 2026
6eaa186
Merge branch 'main' into add-retry-metrics-tests
blakeli0 Apr 3, 2026
3f32b88
tests: Rename tests.
blakeli0 Apr 3, 2026
a64c3da
Migrate to java.time.Duration in ITOtelGoldenMetrics
blakeli0 Apr 3, 2026
6d45b65
Use short name for Duration in ITOtelGoldenMetrics
blakeli0 Apr 3, 2026
3946ce2
Use Duration variant methods in RetrySettings in ITOtelGoldenMetrics
blakeli0 Apr 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
320 changes: 311 additions & 9 deletions ...case/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelGoldenMetrics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,42 +30,43 @@

package com.google.showcase.v1beta1.it;

import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;

import com.google.api.client.http.HttpTransport;
import com.google.api.gax.core.NoCredentialsProvider;
import com.google.api.gax.retrying.RetrySettings;
import com.google.api.gax.rpc.StatusCode;
import com.google.api.gax.rpc.TransportChannelProvider;
import com.google.api.gax.rpc.UnavailableException;
import com.google.api.gax.tracing.GoldenSignalsMetricsTracerFactory;
import com.google.api.gax.tracing.ObservabilityAttributes;
import com.google.common.collect.ImmutableList;
import com.google.rpc.Status;
import com.google.showcase.v1beta1.EchoClient;
import com.google.showcase.v1beta1.EchoRequest;
import com.google.showcase.v1beta1.EchoResponse;
import com.google.showcase.v1beta1.EchoSettings;
import com.google.showcase.v1beta1.it.util.TestClientInitializer;
import com.google.showcase.v1beta1.stub.EchoStubSettings;
import io.grpc.CallOptions;
import io.grpc.Channel;
import io.grpc.ClientCall;
import io.grpc.ClientInterceptor;
import io.grpc.MethodDescriptor;
import io.grpc.Metadata;
import io.grpc.MethodDescriptor;
import io.opentelemetry.api.common.AttributeKey;
import io.opentelemetry.sdk.OpenTelemetrySdk;
import io.opentelemetry.sdk.metrics.SdkMeterProvider;
import io.opentelemetry.sdk.metrics.data.MetricData;
import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.util.Collection;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.util.Collection;

import static com.google.common.truth.Truth.assertThat;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertThrows;

class ITOtelGoldenMetrics {
private static final String SHOWCASE_SERVER_ADDRESS = "localhost";
private static final long SHOWCASE_SERVER_PORT = 7469;
Expand Down Expand Up @@ -379,4 +380,305 @@ public String getHeaderValue(int index) {
.isEqualTo("503");
}
}

@Test
void testMetrics_zeroDeadline_grpc() throws Exception {
Copy link
Copy Markdown
Member

@lqiu96 lqiu96 Apr 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what does zeroDeadline mean? Is this testing that the RPC returns deadline_exceeded status? I think it may be helpful if the test name could be a bit more specific

Copy link
Copy Markdown
Contributor Author

@blakeli0 blakeli0 Apr 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Updated to make it more specific

GoldenSignalsMetricsTracerFactory tracerFactory =
new GoldenSignalsMetricsTracerFactory(openTelemetrySdk);

// Using 1ms as 0ms might be rejected by some validation or trigger immediate failure before
// metrics
RetrySettings zeroRetrySettings =
RetrySettings.newBuilder()
.setInitialRpcTimeout(org.threeten.bp.Duration.ofMillis(1))
.setMaxRpcTimeout(org.threeten.bp.Duration.ofMillis(1))
.setTotalTimeout(org.threeten.bp.Duration.ofMillis(1))
Comment thread
blakeli0 marked this conversation as resolved.
Outdated
.setMaxAttempts(1)
.build();

try (EchoClient client =
TestClientInitializer.createGrpcEchoClientOpentelemetryWithRetrySettings(
tracerFactory, zeroRetrySettings)) {

assertThrows(
Exception.class,
() -> client.echo(EchoRequest.newBuilder().setContent("metrics-test").build()));

Thread.sleep(100);
Comment thread
blakeli0 marked this conversation as resolved.
Collection<MetricData> metrics = metricReader.collectAllMetrics();
assertThat(metrics).isNotEmpty();

MetricData durationMetric =
metrics.stream()
.filter(m -> m.getName().equals("gcp.client.request.duration"))
.findFirst()
.orElseThrow(() -> new AssertionError("Duration metric not found"));

io.opentelemetry.api.common.Attributes attributes =
durationMetric.getHistogramData().getPoints().iterator().next().getAttributes();
Comment thread
blakeli0 marked this conversation as resolved.

assertThat(
attributes.get(
AttributeKey.stringKey(ObservabilityAttributes.RPC_RESPONSE_STATUS_ATTRIBUTE)))
.isEqualTo("DEADLINE_EXCEEDED");
assertThat(
attributes.get(AttributeKey.stringKey(ObservabilityAttributes.ERROR_TYPE_ATTRIBUTE)))
.isEqualTo("DEADLINE_EXCEEDED");
}
}

@Test
void testMetrics_zeroDeadline_httpjson() throws Exception {
GoldenSignalsMetricsTracerFactory tracerFactory =
new GoldenSignalsMetricsTracerFactory(openTelemetrySdk);

RetrySettings zeroRetrySettings =
RetrySettings.newBuilder()
.setInitialRpcTimeout(org.threeten.bp.Duration.ofMillis(1))
.setMaxRpcTimeout(org.threeten.bp.Duration.ofMillis(1))
.setTotalTimeout(org.threeten.bp.Duration.ofMillis(1))
.setMaxAttempts(1)
.build();

try (EchoClient client =
TestClientInitializer.createHttpJsonEchoClientOpentelemetryWithRetrySettings(
tracerFactory, zeroRetrySettings)) {

assertThrows(
Exception.class,
() -> client.echo(EchoRequest.newBuilder().setContent("metrics-test").build()));

Thread.sleep(100);
Collection<MetricData> metrics = metricReader.collectAllMetrics();
assertThat(metrics).isNotEmpty();

MetricData durationMetric =
metrics.stream()
.filter(m -> m.getName().equals("gcp.client.request.duration"))
.findFirst()
.orElseThrow(() -> new AssertionError("Duration metric not found"));

io.opentelemetry.api.common.Attributes attributes =
durationMetric.getHistogramData().getPoints().iterator().next().getAttributes();

assertThat(
attributes.get(
AttributeKey.stringKey(ObservabilityAttributes.RPC_RESPONSE_STATUS_ATTRIBUTE)))
.isEqualTo("DEADLINE_EXCEEDED");
assertThat(
attributes.get(AttributeKey.stringKey(ObservabilityAttributes.ERROR_TYPE_ATTRIBUTE)))
.isEqualTo("504");
}
}

@Test
void testMetrics_retryAndSucceed_grpc() throws Exception {
Copy link
Copy Markdown
Member

@lqiu96 lqiu96 Apr 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same here. Is this test intended to test that that multiple retry attempts results in one value for gcp.client.request.duration and that the final status is OK?

Copy link
Copy Markdown
Contributor Author

@blakeli0 blakeli0 Apr 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, exactly. Making sure that there is only one metric instead of multiple in case of retries.

Copy link
Copy Markdown
Contributor Author

@blakeli0 blakeli0 Apr 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also updated to make it more specific

GoldenSignalsMetricsTracerFactory tracerFactory =
new GoldenSignalsMetricsTracerFactory(openTelemetrySdk);

RetrySettings retrySettings =
RetrySettings.newBuilder()
.setInitialRpcTimeout(org.threeten.bp.Duration.ofMillis(5000L))
.setMaxRpcTimeout(org.threeten.bp.Duration.ofMillis(5000L))
.setTotalTimeout(org.threeten.bp.Duration.ofMillis(5000L))
.setMaxAttempts(3)
.build();

java.util.concurrent.atomic.AtomicInteger attemptCount = new java.util.concurrent.atomic.AtomicInteger(0);

ClientInterceptor interceptor =
new ClientInterceptor() {
@Override
public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(
MethodDescriptor<ReqT, RespT> method, CallOptions callOptions, Channel next) {
int attempt = attemptCount.incrementAndGet();
if (attempt <= 2) {
return new ClientCall<ReqT, RespT>() {
@Override
public void start(Listener<RespT> responseListener, Metadata headers) {
responseListener.onClose(io.grpc.Status.UNAVAILABLE, new Metadata());
}

@Override
public void request(int numMessages) {}

@Override
public void cancel(String message, Throwable cause) {}

@Override
public void halfClose() {}

@Override
public void sendMessage(ReqT message) {}
};
} else {
return next.newCall(method, callOptions);
}
}
};

java.util.Set<StatusCode.Code> retryableCodes = java.util.Collections.singleton(StatusCode.Code.UNAVAILABLE);

try (EchoClient client =
TestClientInitializer.createGrpcEchoClientOpentelemetry(
tracerFactory, retrySettings, retryableCodes, ImmutableList.of(interceptor))) {

client.echo(EchoRequest.newBuilder().setContent("metrics-test").build());

assertThat(attemptCount.get()).isEqualTo(3);

Thread.sleep(100);
Collection<MetricData> metrics = metricReader.collectAllMetrics();
assertThat(metrics).hasSize(1);
Comment thread
blakeli0 marked this conversation as resolved.

MetricData durationMetric =
metrics.stream()
.filter(m -> m.getName().equals("gcp.client.request.duration"))
.findFirst()
.orElseThrow(() -> new AssertionError("Duration metric not found"));

assertThat(durationMetric.getHistogramData().getPoints()).hasSize(1);

io.opentelemetry.api.common.Attributes attributes =
durationMetric.getHistogramData().getPoints().iterator().next().getAttributes();

assertThat(
attributes.get(
AttributeKey.stringKey(ObservabilityAttributes.RPC_RESPONSE_STATUS_ATTRIBUTE)))
.isEqualTo("OK");
}
}

@Test
void testMetrics_retryAndSucceed_httpjson() throws Exception {
GoldenSignalsMetricsTracerFactory tracerFactory =
new GoldenSignalsMetricsTracerFactory(openTelemetrySdk);

RetrySettings retrySettings =
RetrySettings.newBuilder()
.setInitialRpcTimeout(org.threeten.bp.Duration.ofMillis(5000L))
.setMaxRpcTimeout(org.threeten.bp.Duration.ofMillis(5000L))
.setTotalTimeout(org.threeten.bp.Duration.ofMillis(5000L))
.setMaxAttempts(3)
.build();

java.util.concurrent.atomic.AtomicInteger requestCount = new java.util.concurrent.atomic.AtomicInteger(0);

HttpTransport mockTransport =
new HttpTransport() {
@Override
protected com.google.api.client.http.LowLevelHttpRequest buildRequest(
String method, String url) {
int currentCount = requestCount.incrementAndGet();
return new com.google.api.client.http.LowLevelHttpRequest() {
@Override
public void addHeader(String name, String value) {}

@Override
public com.google.api.client.http.LowLevelHttpResponse execute() {
if (currentCount <= 2) {
return new com.google.api.client.http.LowLevelHttpResponse() {
@Override
public InputStream getContent() {
return new ByteArrayInputStream("{}".getBytes(UTF_8));
}

@Override
public String getContentEncoding() { return null; }

@Override
public long getContentLength() { return 2; }

@Override
public String getContentType() { return "application/json"; }

@Override
public String getStatusLine() { return "HTTP/1.1 503 Service Unavailable"; }

@Override
public int getStatusCode() { return 503; }

@Override
public String getReasonPhrase() { return "Service Unavailable"; }

@Override
public int getHeaderCount() { return 0; }

@Override
public String getHeaderName(int index) { return null; }

@Override
public String getHeaderValue(int index) { return null; }
};
} else {
return new com.google.api.client.http.LowLevelHttpResponse() {
@Override
public InputStream getContent() {
return new ByteArrayInputStream("{\"content\":\"metrics-test\"}".getBytes(UTF_8));
}
Comment thread
blakeli0 marked this conversation as resolved.

@Override
public String getContentEncoding() { return null; }

@Override
public long getContentLength() { return 24; }

@Override
public String getContentType() { return "application/json"; }

@Override
public String getStatusLine() { return "HTTP/1.1 200 OK"; }

@Override
public int getStatusCode() { return 200; }

@Override
public String getReasonPhrase() { return "OK"; }

@Override
public int getHeaderCount() { return 0; }

@Override
public String getHeaderName(int index) { return null; }

@Override
public String getHeaderValue(int index) { return null; }
};
}
}
};
}
};

java.util.Set<StatusCode.Code> retryableCodes = java.util.Collections.singleton(StatusCode.Code.UNAVAILABLE);

try (EchoClient client =
TestClientInitializer.createHttpJsonEchoClientOpentelemetry(
tracerFactory, retrySettings, retryableCodes, mockTransport)) {

client.echo(EchoRequest.newBuilder().setContent("metrics-test").build());

assertThat(requestCount.get()).isEqualTo(3);

Thread.sleep(100);
Collection<MetricData> metrics = metricReader.collectAllMetrics();
assertThat(metrics).hasSize(1);

MetricData durationMetric =
metrics.stream()
.filter(m -> m.getName().equals("gcp.client.request.duration"))
.findFirst()
.orElseThrow(() -> new AssertionError("Duration metric not found"));

assertThat(durationMetric.getHistogramData().getPoints()).hasSize(1);

io.opentelemetry.api.common.Attributes attributes =
durationMetric.getHistogramData().getPoints().iterator().next().getAttributes();

assertThat(
attributes.get(
AttributeKey.stringKey(ObservabilityAttributes.RPC_RESPONSE_STATUS_ATTRIBUTE)))
.isEqualTo("OK");
}
}
}
Loading
Loading