Skip to content

Commit 13b4b97

Browse files
committed
xds: Propagate status cause through XdsDepManager
Often there is no cause, but connect(), channel credentials, and call credentials failures on the control plane RPC can include a useful causal exception. This was triggered by seeing an error like below, but it didn't include the cause, which would have included HTTP error information from the failure fetching the credential. ``` UNAVAILABLE: Error retrieving LDS resource xdstp://traffic-director-c2p.xds.googleapis.com/envoy.config.listener.v3.Listener/bigtable.googleapis.com: UNAUTHENTICATED: Failed computing credential metadata nodeID: C2P-798500073 ```
1 parent 8fd809f commit 13b4b97

6 files changed

Lines changed: 119 additions & 27 deletions

File tree

api/src/testFixtures/java/io/grpc/StatusMatcher.java

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
*/
2727
public final class StatusMatcher implements ArgumentMatcher<Status> {
2828
public static StatusMatcher statusHasCode(ArgumentMatcher<Status.Code> codeMatcher) {
29-
return new StatusMatcher(codeMatcher, null);
29+
return new StatusMatcher(codeMatcher, null, null);
3030
}
3131

3232
public static StatusMatcher statusHasCode(Status.Code code) {
@@ -35,17 +35,20 @@ public static StatusMatcher statusHasCode(Status.Code code) {
3535

3636
private final ArgumentMatcher<Status.Code> codeMatcher;
3737
private final ArgumentMatcher<String> descriptionMatcher;
38+
private final ArgumentMatcher<Throwable> causeMatcher;
3839

3940
private StatusMatcher(
4041
ArgumentMatcher<Status.Code> codeMatcher,
41-
ArgumentMatcher<String> descriptionMatcher) {
42+
ArgumentMatcher<String> descriptionMatcher,
43+
ArgumentMatcher<Throwable> causeMatcher) {
4244
this.codeMatcher = checkNotNull(codeMatcher, "codeMatcher");
4345
this.descriptionMatcher = descriptionMatcher;
46+
this.causeMatcher = causeMatcher;
4447
}
4548

4649
public StatusMatcher andDescription(ArgumentMatcher<String> descriptionMatcher) {
4750
checkState(this.descriptionMatcher == null, "Already has a description matcher");
48-
return new StatusMatcher(codeMatcher, descriptionMatcher);
51+
return new StatusMatcher(codeMatcher, descriptionMatcher, causeMatcher);
4952
}
5053

5154
public StatusMatcher andDescription(String description) {
@@ -56,11 +59,21 @@ public StatusMatcher andDescriptionContains(String substring) {
5659
return andDescription(new StringContainsMatcher(substring));
5760
}
5861

62+
public StatusMatcher andCause(ArgumentMatcher<Throwable> causeMatcher) {
63+
checkState(this.causeMatcher == null, "Already has a cause matcher");
64+
return new StatusMatcher(codeMatcher, descriptionMatcher, causeMatcher);
65+
}
66+
67+
public StatusMatcher andCause(Throwable cause) {
68+
return andCause(new EqualsMatcher<>(cause));
69+
}
70+
5971
@Override
6072
public boolean matches(Status status) {
6173
return status != null
6274
&& codeMatcher.matches(status.getCode())
63-
&& (descriptionMatcher == null || descriptionMatcher.matches(status.getDescription()));
75+
&& (descriptionMatcher == null || descriptionMatcher.matches(status.getDescription()))
76+
&& (causeMatcher == null || causeMatcher.matches(status.getCause()));
6477
}
6578

6679
@Override
@@ -72,6 +85,10 @@ public String toString() {
7285
sb.append(", description=");
7386
sb.append(descriptionMatcher);
7487
}
88+
if (causeMatcher != null) {
89+
sb.append(", cause=");
90+
sb.append(causeMatcher);
91+
}
7592
sb.append("}");
7693
return sb.toString();
7794
}

xds/src/main/java/io/grpc/xds/XdsDependencyManager.java

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import io.grpc.Status;
3131
import io.grpc.StatusOr;
3232
import io.grpc.SynchronizationContext;
33+
import io.grpc.internal.GrpcUtil;
3334
import io.grpc.internal.RetryingNameResolver;
3435
import io.grpc.xds.Endpoints.LocalityLbEndpoints;
3536
import io.grpc.xds.VirtualHost.Route.RouteAction.ClusterWeight;
@@ -652,13 +653,10 @@ public void onResourceChanged(StatusOr<T> update) {
652653
data = update;
653654
subscribeToChildren(update.getValue());
654655
} else {
655-
Status status = update.getStatus();
656-
Status translatedStatus = Status.UNAVAILABLE.withDescription(
657-
String.format("Error retrieving %s: %s. Details: %s%s",
658-
toContextString(),
659-
status.getCode(),
660-
status.getDescription() != null ? status.getDescription() : "",
661-
nodeInfo()));
656+
Status translatedStatus = GrpcUtil.statusWithDetails(
657+
Status.Code.UNAVAILABLE,
658+
"Error retrieving " + toContextString() + nodeInfo(),
659+
update.getStatus());
662660

663661
data = StatusOr.fromStatus(translatedStatus);
664662
}

xds/src/test/java/io/grpc/xds/CdsLoadBalancer2Test.java

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -255,9 +255,9 @@ public void nonAggregateCluster_resourceNotExist_returnErrorPicker() {
255255
startXdsDepManager();
256256
verify(helper).updateBalancingState(
257257
eq(ConnectivityState.TRANSIENT_FAILURE), pickerCaptor.capture());
258-
String expectedDescription = "Error retrieving CDS resource " + CLUSTER + ": NOT_FOUND. "
259-
+ "Details: Timed out waiting for resource " + CLUSTER
260-
+ " from xDS server nodeID: " + NODE_ID;
258+
String expectedDescription = "Error retrieving CDS resource " + CLUSTER
259+
+ " nodeID: " + NODE_ID
260+
+ ": NOT_FOUND: Timed out waiting for resource " + CLUSTER + " from xDS server";
261261
Status unavailable = Status.UNAVAILABLE.withDescription(expectedDescription);
262262
assertPickerStatus(pickerCaptor.getValue(), unavailable);
263263
assertThat(childBalancers).isEmpty();
@@ -311,8 +311,9 @@ public void nonAggregateCluster_resourceRevoked() {
311311
controlPlaneService.setXdsConfig(ADS_TYPE_URL_CDS, ImmutableMap.of());
312312

313313
assertThat(childBalancer.shutdown).isTrue();
314-
String expectedDescription = "Error retrieving CDS resource " + CLUSTER + ": NOT_FOUND. "
315-
+ "Details: Resource " + CLUSTER + " does not exist nodeID: " + NODE_ID;
314+
String expectedDescription = "Error retrieving CDS resource " + CLUSTER
315+
+ " nodeID: " + NODE_ID
316+
+ ": NOT_FOUND: Resource " + CLUSTER + " does not exist";
316317
Status unavailable = Status.UNAVAILABLE.withDescription(expectedDescription);
317318
verify(helper).updateBalancingState(
318319
eq(ConnectivityState.TRANSIENT_FAILURE), pickerCaptor.capture());
@@ -515,9 +516,9 @@ public void aggregateCluster_noNonAggregateClusterExits_returnErrorPicker() {
515516

516517
verify(helper).updateBalancingState(
517518
eq(ConnectivityState.TRANSIENT_FAILURE), pickerCaptor.capture());
518-
String expectedDescription = "Error retrieving CDS resource " + cluster1 + ": NOT_FOUND. "
519-
+ "Details: Timed out waiting for resource " + cluster1 + " from xDS server nodeID: "
520-
+ NODE_ID;
519+
String expectedDescription = "Error retrieving CDS resource " + cluster1
520+
+ " nodeID: " + NODE_ID
521+
+ ": NOT_FOUND: Timed out waiting for resource " + cluster1 + " from xDS server";
521522
Status status = Status.UNAVAILABLE.withDescription(expectedDescription);
522523
assertPickerStatus(pickerCaptor.getValue(), status);
523524
assertThat(childBalancers).isEmpty();

xds/src/test/java/io/grpc/xds/ClusterResolverLoadBalancerTest.java

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -697,8 +697,8 @@ public void onlyEdsClusters_resourceNeverExist_returnErrorPicker() {
697697

698698
verify(helper).updateBalancingState(
699699
eq(ConnectivityState.TRANSIENT_FAILURE), pickerCaptor.capture());
700-
String expectedDescription = "Error retrieving CDS resource " + CLUSTER + ": NOT_FOUND. "
701-
+ "Details: Timed out waiting for resource " + CLUSTER + " from xDS server nodeID: node-id";
700+
String expectedDescription = "Error retrieving CDS resource " + CLUSTER + " nodeID: node-id: "
701+
+ "NOT_FOUND: Timed out waiting for resource " + CLUSTER + " from xDS server";
702702
Status expectedError = Status.UNAVAILABLE.withDescription(expectedDescription);
703703
assertPicker(pickerCaptor.getValue(), expectedError, null);
704704
}
@@ -720,8 +720,8 @@ public void cdsMissing_handledDirectly() {
720720
assertThat(childBalancers).hasSize(0); // no child LB policy created
721721
verify(helper).updateBalancingState(
722722
eq(ConnectivityState.TRANSIENT_FAILURE), pickerCaptor.capture());
723-
String expectedDescription = "Error retrieving CDS resource " + CLUSTER + ": NOT_FOUND. "
724-
+ "Details: Timed out waiting for resource " + CLUSTER + " from xDS server nodeID: node-id";
723+
String expectedDescription = "Error retrieving CDS resource " + CLUSTER + " nodeID: node-id: "
724+
+ "NOT_FOUND: Timed out waiting for resource " + CLUSTER + " from xDS server";
725725
Status expectedError = Status.UNAVAILABLE.withDescription(expectedDescription);
726726
assertPicker(pickerCaptor.getValue(), expectedError, null);
727727
assertPicker(pickerCaptor.getValue(), expectedError, null);
@@ -751,8 +751,8 @@ public void cdsRevoked_handledDirectly() {
751751
controlPlaneService.setXdsConfig(ADS_TYPE_URL_CDS, ImmutableMap.of());
752752
verify(helper).updateBalancingState(
753753
eq(ConnectivityState.TRANSIENT_FAILURE), pickerCaptor.capture());
754-
String expectedDescription = "Error retrieving CDS resource " + CLUSTER + ": NOT_FOUND. "
755-
+ "Details: Resource " + CLUSTER + " does not exist nodeID: node-id";
754+
String expectedDescription = "Error retrieving CDS resource " + CLUSTER + " nodeID: node-id: "
755+
+ "NOT_FOUND: Resource " + CLUSTER + " does not exist";
756756
Status expectedError = Status.UNAVAILABLE.withDescription(expectedDescription);
757757
assertPicker(pickerCaptor.getValue(), expectedError, null);
758758
assertThat(childBalancer.shutdown).isTrue();
@@ -767,8 +767,8 @@ public void edsMissing_failsRpcs() {
767767
verify(helper).updateBalancingState(
768768
eq(ConnectivityState.TRANSIENT_FAILURE), pickerCaptor.capture());
769769
String expectedDescription = "Error retrieving EDS resource " + EDS_SERVICE_NAME
770-
+ ": NOT_FOUND. Details: Timed out waiting for resource " + EDS_SERVICE_NAME
771-
+ " from xDS server nodeID: node-id";
770+
+ " nodeID: node-id: "
771+
+ "NOT_FOUND: Timed out waiting for resource " + EDS_SERVICE_NAME + " from xDS server";
772772
Status expectedError = Status.UNAVAILABLE.withDescription(expectedDescription);
773773
assertPicker(pickerCaptor.getValue(), expectedError, null);
774774
}
Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/*
2+
* Copyright 2026 The gRPC Authors
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package io.grpc.xds;
18+
19+
import static java.util.Objects.requireNonNull;
20+
21+
import io.grpc.CallOptions;
22+
import io.grpc.Channel;
23+
import io.grpc.ClientCall;
24+
import io.grpc.ClientInterceptor;
25+
import io.grpc.Metadata;
26+
import io.grpc.MethodDescriptor;
27+
import io.grpc.NoopClientCall;
28+
import io.grpc.Status;
29+
30+
/**
31+
* An interceptor that fails all RPCs with the provided status.
32+
*/
33+
final class FailingClientInterceptor implements ClientInterceptor {
34+
private final Status status;
35+
36+
public FailingClientInterceptor(Status status) {
37+
this.status = requireNonNull(status, "status");
38+
}
39+
40+
@Override
41+
public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(
42+
MethodDescriptor<ReqT, RespT> method, CallOptions callOptions, Channel next) {
43+
return new NoopClientCall<ReqT, RespT>() {
44+
@Override
45+
public void start(Listener<RespT> responseListener, Metadata headers) {
46+
responseListener.onClose(status, new Metadata());
47+
}
48+
};
49+
}
50+
}

xds/src/test/java/io/grpc/xds/XdsDependencyManagerTest.java

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -409,6 +409,32 @@ public void testTcpListenerErrors() {
409409
testWatcher.verifyStats(0, 1);
410410
}
411411

412+
@Test
413+
public void testControlPlaneError() {
414+
Status forcedStatus = Status.NOT_FOUND
415+
.withDescription("expected")
416+
.withCause(new IllegalArgumentException("a random exception"));
417+
xdsClient.shutdown();
418+
xdsClient = XdsTestUtils.createXdsClient(
419+
Collections.singletonList("control-plane"),
420+
serverInfo -> new GrpcXdsTransportFactory.GrpcXdsTransport(
421+
InProcessChannelBuilder.forName(serverInfo.target())
422+
.directExecutor()
423+
.intercept(new FailingClientInterceptor(forcedStatus))
424+
.build()),
425+
fakeClock);
426+
xdsDependencyManager = new XdsDependencyManager(
427+
xdsClient, syncContext, serverName, serverName, nameResolverArgs);
428+
xdsDependencyManager.start(xdsConfigWatcher);
429+
430+
verify(xdsConfigWatcher).onUpdate(
431+
argThat(StatusOrMatcher.hasStatus(
432+
statusHasCode(Status.Code.UNAVAILABLE)
433+
.andDescriptionContains(forcedStatus.getDescription())
434+
.andCause(forcedStatus.getCause()))));
435+
testWatcher.verifyStats(0, 1);
436+
}
437+
412438
@Test
413439
public void testMissingRds() {
414440
String rdsName = "badRdsName";

0 commit comments

Comments
 (0)