Skip to content

Commit 35eb041

Browse files
jinseopkim0jinseopkim0
authored
fix(bigquery): retry cancel job and routine creation on transient HTTP 5xx errors (#13416)
Retries are applied to: ### 1. Read Metadata RPCs * **`bigQueryRpc.getDatasetSkipExceptionTranslation`** (via `getDataset(...)`) * **`bigQueryRpc.getTableSkipExceptionTranslation`** (via `getTable(...)`) * **`bigQueryRpc.getJobSkipExceptionTranslation`** (via `getJob(...)`) * **`bigQueryRpc.getIamPolicySkipExceptionTranslation`** (via `getIamPolicy(...)`) ### 2. List RPCs * **`bigQueryRpc.listDatasetsSkipExceptionTranslation`** (via `listDatasets(...)`) * **`bigQueryRpc.listTablesSkipExceptionTranslation`** (via `listTables(...)`) * **`bigQueryRpc.listJobsSkipExceptionTranslation`** (via `listJobs(...)`) * **`bigQueryRpc.listModelsSkipExceptionTranslation`** (via `listModels(...)`) * **`bigQueryRpc.listRoutinesSkipExceptionTranslation`** (via `listRoutines(...)`) * **`bigQueryRpc.listTableDataSkipExceptionTranslation`** (via `listTableData(...)`) * **`bigQueryRpc.getQueryResultsSkipExceptionTranslation`** (via `getQueryResults(...)`) ### 3. Job Actions * **`bigQueryRpc.cancelSkipExceptionTranslation`** (via `cancel(...)` — resolves `b/467066417`) ### 4. IAM & Permissions * **`bigQueryRpc.testIamPermissionsSkipExceptionTranslation`** (via `testIamPermissions(...)`) ### 5. Routine Creation (Specifically added for UDF Test) * **`bigQueryRpc.createSkipExceptionTranslation` (Routines only)** (via `createRoutine(...)` — resolves `b/467066596`) b/467066417, b/467066596
1 parent 40f72f7 commit 35eb041

3 files changed

Lines changed: 146 additions & 16 deletions

File tree

‎java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -482,7 +482,7 @@ public com.google.api.services.bigquery.model.Routine call() throws IOException
482482
}
483483
},
484484
getOptions().getRetrySettings(),
485-
getOptions().getResultRetryAlgorithm(),
485+
BigQueryRetryHelper.maybeWrapForHttpRetry(getOptions().getResultRetryAlgorithm()),
486486
getOptions().getClock(),
487487
EMPTY_RETRY_CONFIG,
488488
getOptions().isOpenTelemetryTracingEnabled(),
@@ -678,7 +678,7 @@ public com.google.api.services.bigquery.model.Dataset call() throws IOException
678678
}
679679
},
680680
getOptions().getRetrySettings(),
681-
getOptions().getResultRetryAlgorithm(),
681+
BigQueryRetryHelper.maybeWrapForHttpRetry(getOptions().getResultRetryAlgorithm()),
682682
getOptions().getClock(),
683683
EMPTY_RETRY_CONFIG,
684684
getOptions().isOpenTelemetryTracingEnabled(),
@@ -744,7 +744,7 @@ private static Page<Dataset> listDatasets(
744744
}
745745
},
746746
serviceOptions.getRetrySettings(),
747-
serviceOptions.getResultRetryAlgorithm(),
747+
BigQueryRetryHelper.maybeWrapForHttpRetry(serviceOptions.getResultRetryAlgorithm()),
748748
serviceOptions.getClock(),
749749
EMPTY_RETRY_CONFIG,
750750
serviceOptions.isOpenTelemetryTracingEnabled(),
@@ -1217,7 +1217,7 @@ public com.google.api.services.bigquery.model.Table call() throws IOException {
12171217
}
12181218
},
12191219
getOptions().getRetrySettings(),
1220-
getOptions().getResultRetryAlgorithm(),
1220+
BigQueryRetryHelper.maybeWrapForHttpRetry(getOptions().getResultRetryAlgorithm()),
12211221
getOptions().getClock(),
12221222
EMPTY_RETRY_CONFIG,
12231223
getOptions().isOpenTelemetryTracingEnabled(),
@@ -1276,7 +1276,7 @@ public com.google.api.services.bigquery.model.Model call() throws IOException {
12761276
}
12771277
},
12781278
getOptions().getRetrySettings(),
1279-
getOptions().getResultRetryAlgorithm(),
1279+
BigQueryRetryHelper.maybeWrapForHttpRetry(getOptions().getResultRetryAlgorithm()),
12801280
getOptions().getClock(),
12811281
EMPTY_RETRY_CONFIG,
12821282
getOptions().isOpenTelemetryTracingEnabled(),
@@ -1335,7 +1335,7 @@ public com.google.api.services.bigquery.model.Routine call() throws IOException
13351335
}
13361336
},
13371337
getOptions().getRetrySettings(),
1338-
getOptions().getResultRetryAlgorithm(),
1338+
BigQueryRetryHelper.maybeWrapForHttpRetry(getOptions().getResultRetryAlgorithm()),
13391339
getOptions().getClock(),
13401340
EMPTY_RETRY_CONFIG,
13411341
getOptions().isOpenTelemetryTracingEnabled(),
@@ -1553,7 +1553,7 @@ public Tuple<String, Iterable<com.google.api.services.bigquery.model.Table>> cal
15531553
}
15541554
},
15551555
serviceOptions.getRetrySettings(),
1556-
serviceOptions.getResultRetryAlgorithm(),
1556+
BigQueryRetryHelper.maybeWrapForHttpRetry(serviceOptions.getResultRetryAlgorithm()),
15571557
serviceOptions.getClock(),
15581558
EMPTY_RETRY_CONFIG,
15591559
serviceOptions.isOpenTelemetryTracingEnabled(),
@@ -1594,7 +1594,7 @@ public Tuple<String, Iterable<com.google.api.services.bigquery.model.Model>> cal
15941594
}
15951595
},
15961596
serviceOptions.getRetrySettings(),
1597-
serviceOptions.getResultRetryAlgorithm(),
1597+
BigQueryRetryHelper.maybeWrapForHttpRetry(serviceOptions.getResultRetryAlgorithm()),
15981598
serviceOptions.getClock(),
15991599
EMPTY_RETRY_CONFIG,
16001600
serviceOptions.isOpenTelemetryTracingEnabled(),
@@ -1635,7 +1635,7 @@ private static Page<Routine> listRoutines(
16351635
}
16361636
},
16371637
serviceOptions.getRetrySettings(),
1638-
serviceOptions.getResultRetryAlgorithm(),
1638+
BigQueryRetryHelper.maybeWrapForHttpRetry(serviceOptions.getResultRetryAlgorithm()),
16391639
serviceOptions.getClock(),
16401640
EMPTY_RETRY_CONFIG,
16411641
serviceOptions.isOpenTelemetryTracingEnabled(),
@@ -1812,7 +1812,7 @@ public TableDataList call() throws IOException {
18121812
}
18131813
},
18141814
serviceOptions.getRetrySettings(),
1815-
serviceOptions.getResultRetryAlgorithm(),
1815+
BigQueryRetryHelper.maybeWrapForHttpRetry(serviceOptions.getResultRetryAlgorithm()),
18161816
serviceOptions.getClock(),
18171817
EMPTY_RETRY_CONFIG,
18181818
serviceOptions.isOpenTelemetryTracingEnabled(),
@@ -1889,7 +1889,7 @@ public com.google.api.services.bigquery.model.Job call() throws IOException {
18891889
}
18901890
},
18911891
getOptions().getRetrySettings(),
1892-
getOptions().getResultRetryAlgorithm(),
1892+
BigQueryRetryHelper.maybeWrapForHttpRetry(getOptions().getResultRetryAlgorithm()),
18931893
getOptions().getClock(),
18941894
EMPTY_RETRY_CONFIG,
18951895
getOptions().isOpenTelemetryTracingEnabled(),
@@ -1946,7 +1946,7 @@ public Tuple<String, Iterable<com.google.api.services.bigquery.model.Job>> call(
19461946
}
19471947
},
19481948
serviceOptions.getRetrySettings(),
1949-
serviceOptions.getResultRetryAlgorithm(),
1949+
BigQueryRetryHelper.maybeWrapForHttpRetry(serviceOptions.getResultRetryAlgorithm()),
19501950
serviceOptions.getClock(),
19511951
EMPTY_RETRY_CONFIG,
19521952
serviceOptions.isOpenTelemetryTracingEnabled(),
@@ -2001,7 +2001,7 @@ public Boolean call() throws IOException {
20012001
}
20022002
},
20032003
getOptions().getRetrySettings(),
2004-
getOptions().getResultRetryAlgorithm(),
2004+
BigQueryRetryHelper.maybeWrapForHttpRetry(getOptions().getResultRetryAlgorithm()),
20052005
getOptions().getClock(),
20062006
EMPTY_RETRY_CONFIG,
20072007
getOptions().isOpenTelemetryTracingEnabled(),
@@ -2262,7 +2262,7 @@ public GetQueryResultsResponse call() throws IOException {
22622262
}
22632263
},
22642264
serviceOptions.getRetrySettings(),
2265-
serviceOptions.getResultRetryAlgorithm(),
2265+
BigQueryRetryHelper.maybeWrapForHttpRetry(serviceOptions.getResultRetryAlgorithm()),
22662266
serviceOptions.getClock(),
22672267
DEFAULT_RETRY_CONFIG,
22682268
serviceOptions.isOpenTelemetryTracingEnabled(),
@@ -2333,7 +2333,7 @@ public com.google.api.services.bigquery.model.Policy call() throws IOException {
23332333
}
23342334
},
23352335
getOptions().getRetrySettings(),
2336-
getOptions().getResultRetryAlgorithm(),
2336+
BigQueryRetryHelper.maybeWrapForHttpRetry(getOptions().getResultRetryAlgorithm()),
23372337
getOptions().getClock(),
23382338
EMPTY_RETRY_CONFIG,
23392339
getOptions().isOpenTelemetryTracingEnabled(),
@@ -2427,7 +2427,7 @@ public com.google.api.services.bigquery.model.TestIamPermissionsResponse call()
24272427
}
24282428
},
24292429
getOptions().getRetrySettings(),
2430-
getOptions().getResultRetryAlgorithm(),
2430+
BigQueryRetryHelper.maybeWrapForHttpRetry(getOptions().getResultRetryAlgorithm()),
24312431
getOptions().getClock(),
24322432
EMPTY_RETRY_CONFIG,
24332433
getOptions().isOpenTelemetryTracingEnabled(),

‎java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryHelper.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
*/
1616
package com.google.cloud.bigquery;
1717

18+
import com.google.api.client.http.HttpResponseException;
1819
import com.google.api.core.ApiClock;
1920
import com.google.api.gax.retrying.DirectRetryingExecutor;
2021
import com.google.api.gax.retrying.ExponentialRetryAlgorithm;
@@ -23,6 +24,7 @@
2324
import com.google.api.gax.retrying.RetrySettings;
2425
import com.google.api.gax.retrying.RetryingExecutor;
2526
import com.google.api.gax.retrying.RetryingFuture;
27+
import com.google.api.gax.retrying.TimedAttemptSettings;
2628
import com.google.api.gax.retrying.TimedRetryAlgorithm;
2729
import com.google.cloud.RetryHelper;
2830
import io.opentelemetry.api.trace.Span;
@@ -119,6 +121,50 @@ private static <V> V run(
119121
return retryingFuture.get();
120122
}
121123

124+
/**
125+
* Conditionally wraps the provided retry algorithm with a wrapper that retries on transient HTTP
126+
* errors.
127+
*
128+
* <p>Wrapping only occurs if the provided algorithm is the default {@link
129+
* BigQueryBaseService#DEFAULT_BIGQUERY_EXCEPTION_HANDLER}. Custom user-defined retry algorithms
130+
* are returned unmodified to preserve custom retry policies.
131+
*/
132+
@SuppressWarnings("unchecked")
133+
static <V> ResultRetryAlgorithm<V> maybeWrapForHttpRetry(ResultRetryAlgorithm<?> algorithm) {
134+
if (algorithm == BigQueryBaseService.DEFAULT_BIGQUERY_EXCEPTION_HANDLER) {
135+
return wrapDefaultAlgorithm((ResultRetryAlgorithm<V>) algorithm);
136+
}
137+
return (ResultRetryAlgorithm<V>) algorithm;
138+
}
139+
140+
/**
141+
* Wraps the default retry algorithm to additionally retry on transient HTTP status codes 500,
142+
* 502, 503, and 504. Other retry decisions and timing logic are delegated back to the default
143+
* algorithm.
144+
*/
145+
private static <V> ResultRetryAlgorithm<V> wrapDefaultAlgorithm(
146+
ResultRetryAlgorithm<V> defaultAlgorithm) {
147+
return new ResultRetryAlgorithm<V>() {
148+
@Override
149+
public TimedAttemptSettings createNextAttempt(
150+
Throwable previousThrowable, V previousResponse, TimedAttemptSettings previousSettings) {
151+
return defaultAlgorithm.createNextAttempt(
152+
previousThrowable, previousResponse, previousSettings);
153+
}
154+
155+
@Override
156+
public boolean shouldRetry(Throwable previousThrowable, V previousResponse) {
157+
if (previousThrowable instanceof HttpResponseException) {
158+
int statusCode = ((HttpResponseException) previousThrowable).getStatusCode();
159+
if (statusCode == 500 || statusCode == 502 || statusCode == 503 || statusCode == 504) {
160+
return true;
161+
}
162+
}
163+
return defaultAlgorithm.shouldRetry(previousThrowable, previousResponse);
164+
}
165+
};
166+
}
167+
122168
public static class BigQueryRetryHelperException extends RuntimeException {
123169

124170
private static final long serialVersionUID = -8519852520090965314L;

‎java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import static org.junit.jupiter.api.Assertions.assertNotNull;
2727
import static org.junit.jupiter.api.Assertions.assertNull;
2828
import static org.junit.jupiter.api.Assertions.assertSame;
29+
import static org.junit.jupiter.api.Assertions.assertThrows;
2930
import static org.junit.jupiter.api.Assertions.assertTrue;
3031
import static org.mockito.ArgumentMatchers.any;
3132
import static org.mockito.ArgumentMatchers.eq;
@@ -37,7 +38,13 @@
3738
import static org.mockito.Mockito.verify;
3839
import static org.mockito.Mockito.when;
3940

41+
import com.google.api.client.googleapis.json.GoogleJsonError;
42+
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
43+
import com.google.api.client.http.HttpHeaders;
44+
import com.google.api.client.http.HttpResponseException;
4045
import com.google.api.gax.paging.Page;
46+
import com.google.api.gax.retrying.ResultRetryAlgorithm;
47+
import com.google.api.gax.retrying.TimedAttemptSettings;
4148
import com.google.api.services.bigquery.model.ErrorProto;
4249
import com.google.api.services.bigquery.model.GetQueryResultsResponse;
4350
import com.google.api.services.bigquery.model.JobConfigurationQuery;
@@ -989,6 +996,83 @@ void testGetTable() throws IOException {
989996
.getTableSkipExceptionTranslation(PROJECT, DATASET, TABLE, EMPTY_RPC_OPTIONS);
990997
}
991998

999+
@Test
1000+
void testGetTableFailureShouldRetryServerErrors() throws IOException {
1001+
GoogleJsonError error = new GoogleJsonError();
1002+
error.setMessage("Visibility check was unavailable. Please retry the request");
1003+
error.setCode(503);
1004+
GoogleJsonError.ErrorInfo errorInfo = new GoogleJsonError.ErrorInfo();
1005+
errorInfo.setReason("backendError");
1006+
error.setErrors(ImmutableList.of(errorInfo));
1007+
1008+
when(bigqueryRpcMock.getTableSkipExceptionTranslation(
1009+
PROJECT, DATASET, TABLE, EMPTY_RPC_OPTIONS))
1010+
.thenThrow(new GoogleJsonResponseException(serverErrorResponse(), error))
1011+
.thenReturn(TABLE_INFO_WITH_PROJECT.toPb());
1012+
1013+
bigquery =
1014+
options.toBuilder()
1015+
.setRetrySettings(ServiceOptions.getDefaultRetrySettings())
1016+
.build()
1017+
.getService();
1018+
1019+
Table table = bigquery.getTable(DATASET, TABLE);
1020+
1021+
assertEquals(new Table(bigquery, new TableInfo.BuilderImpl(TABLE_INFO_WITH_PROJECT)), table);
1022+
verify(bigqueryRpcMock, times(2))
1023+
.getTableSkipExceptionTranslation(PROJECT, DATASET, TABLE, EMPTY_RPC_OPTIONS);
1024+
}
1025+
1026+
@Test
1027+
void testGetTableFailureWithCustomRetryAlgorithmShouldNotRetry() throws IOException {
1028+
GoogleJsonError error = new GoogleJsonError();
1029+
error.setMessage("Visibility check was unavailable. Please retry the request");
1030+
error.setCode(503);
1031+
GoogleJsonError.ErrorInfo errorInfo = new GoogleJsonError.ErrorInfo();
1032+
errorInfo.setReason("backendError");
1033+
error.setErrors(ImmutableList.of(errorInfo));
1034+
1035+
when(bigqueryRpcMock.getTableSkipExceptionTranslation(
1036+
PROJECT, DATASET, TABLE, EMPTY_RPC_OPTIONS))
1037+
.thenThrow(new GoogleJsonResponseException(serverErrorResponse(), error));
1038+
1039+
ResultRetryAlgorithm<?> customAlgorithm =
1040+
new ResultRetryAlgorithm<Object>() {
1041+
@Override
1042+
public TimedAttemptSettings createNextAttempt(
1043+
Throwable previousThrowable,
1044+
Object previousResponse,
1045+
TimedAttemptSettings previousSettings) {
1046+
return null;
1047+
}
1048+
1049+
@Override
1050+
public boolean shouldRetry(Throwable previousThrowable, Object previousResponse) {
1051+
return false;
1052+
}
1053+
};
1054+
1055+
bigquery =
1056+
options.toBuilder()
1057+
.setRetrySettings(ServiceOptions.getDefaultRetrySettings())
1058+
.setResultRetryAlgorithm(customAlgorithm)
1059+
.build()
1060+
.getService();
1061+
1062+
assertThrows(
1063+
BigQueryException.class,
1064+
() -> {
1065+
bigquery.getTable(DATASET, TABLE);
1066+
});
1067+
1068+
verify(bigqueryRpcMock, times(1))
1069+
.getTableSkipExceptionTranslation(PROJECT, DATASET, TABLE, EMPTY_RPC_OPTIONS);
1070+
}
1071+
1072+
private static HttpResponseException.Builder serverErrorResponse() {
1073+
return new HttpResponseException.Builder(503, "Service Unavailable", new HttpHeaders());
1074+
}
1075+
9921076
@Test
9931077
void testGetModel() throws IOException {
9941078
when(bigqueryRpcMock.getModelSkipExceptionTranslation(

0 commit comments

Comments
 (0)