Skip to content

Commit 8248d8e

Browse files
fix(bigquery): avoid SSLError retry loop (#17489)
b/391888240 Previously, invalid requests like incorrect table schemas would result in SSLErrors, which would then be retried up to the default timeout value. SSLErrors generally won't resolve on retry. This PR: - marks SSLErrors as non-retryable, so it won't waste 5 minutes on retries - catches and re-raises the insert_rows SSLError exception, to add clearer error context about what may be failing
1 parent 58a7ec5 commit 8248d8e

5 files changed

Lines changed: 129 additions & 11 deletions

File tree

packages/google-cloud-bigquery/google/cloud/bigquery/client.py

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4046,15 +4046,22 @@ def insert_rows_json(
40464046
path = "%s/insertAll" % table.path
40474047
# We can always retry, because every row has an insert ID.
40484048
span_attributes = {"path": path}
4049-
response = self._call_api(
4050-
retry,
4051-
span_name="BigQuery.insertRowsJson",
4052-
span_attributes=span_attributes,
4053-
method="POST",
4054-
path=path,
4055-
data=data,
4056-
timeout=timeout,
4057-
)
4049+
try:
4050+
response = self._call_api(
4051+
retry,
4052+
span_name="BigQuery.insertRowsJson",
4053+
span_attributes=span_attributes,
4054+
method="POST",
4055+
path=path,
4056+
data=data,
4057+
timeout=timeout,
4058+
)
4059+
except requests.exceptions.SSLError as exc:
4060+
msg = (
4061+
"An SSL/Connection error occurred while streaming rows. This "
4062+
"could be due to an invalid request (e.g., invalid table schema)."
4063+
)
4064+
raise requests.exceptions.SSLError(msg) from exc
40584065
errors = []
40594066

40604067
for error in response.get("insertErrors", ()):

packages/google-cloud-bigquery/google/cloud/bigquery/retry.py

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,11 @@
4040

4141
_DEFAULT_RETRY_DEADLINE = 10.0 * 60.0 # 10 minutes
4242

43+
# Exceptions that are subclasses of types in _UNSTRUCTURED_RETRYABLE_TYPES
44+
# but should not be retried because they typically indicate persistent
45+
# configuration or security issues.
46+
_UNSTRUCTURED_NON_RETRYABLE_TYPES = (requests.exceptions.SSLError,)
47+
4348
# Ambiguous errors (e.g. internalError, backendError, rateLimitExceeded) retry
4449
# until the full `_DEFAULT_RETRY_DEADLINE`. This is because the
4550
# `jobs.getQueryResults` REST API translates a job failure into an HTTP error.
@@ -64,9 +69,13 @@
6469
def _should_retry(exc):
6570
"""Predicate for determining when to retry.
6671
67-
We retry if and only if the 'reason' is in _RETRYABLE_REASONS or is
68-
in _UNSTRUCTURED_RETRYABLE_TYPES.
72+
We retry if the 'reason' is in _RETRYABLE_REASONS or if the exception
73+
is an instance of one of the _UNSTRUCTURED_RETRYABLE_TYPES, unless it
74+
is explicitly excluded by being in _UNSTRUCTURED_NON_RETRYABLE_TYPES.
6975
"""
76+
if isinstance(exc, _UNSTRUCTURED_NON_RETRYABLE_TYPES):
77+
return False
78+
7079
try:
7180
reason = exc.errors[0]["reason"]
7281
except (AttributeError, IndexError, TypeError, KeyError):
Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import time
16+
from unittest import mock
17+
18+
import pytest
19+
import requests.exceptions
20+
from google.cloud import bigquery
21+
22+
23+
def test_insert_rows_json_ssl_error_no_retry(bigquery_client, dataset_id, project_id):
24+
"""
25+
Verify that SSLError during insert_rows_json is NOT retried and
26+
propagates a descriptive error message immediately.
27+
"""
28+
table_id = f"{project_id}.{dataset_id}.test_ssl_retry_{int(time.time())}"
29+
schema = [bigquery.SchemaField("name", "STRING")]
30+
table = bigquery.Table(table_id, schema=schema)
31+
bigquery_client.create_table(table)
32+
try:
33+
# We mock the api_request to simulate the GFE abruptly closing the connection
34+
# which manifests as a requests.exceptions.SSLError.
35+
bigquery_client._connection.api_request
36+
call_count = 0
37+
38+
def mock_api_request(*args, **kwargs):
39+
nonlocal call_count
40+
call_count += 1
41+
raise requests.exceptions.SSLError("EOF occurred in violation of protocol")
42+
43+
with mock.patch.object(
44+
bigquery_client._connection, "api_request", side_effect=mock_api_request
45+
):
46+
# Use a reasonably short deadline for the test, although it should fail on the first attempt anyway.
47+
retry = bigquery.DEFAULT_RETRY.with_deadline(5.0)
48+
49+
start_time = time.time()
50+
with pytest.raises(requests.exceptions.SSLError) as excinfo:
51+
bigquery_client.insert_rows_json(table, [{"name": "test"}], retry=retry)
52+
duration = time.time() - start_time
53+
54+
# Verification:
55+
# 1. It should NOT have retried (total calls should be 1)
56+
assert call_count == 1
57+
58+
# 2. It should have failed quickly (much less than the 5s deadline)
59+
assert duration < 2.0
60+
61+
# 3. The error message should contain our descriptive wrapping
62+
assert "invalid table schema" in str(excinfo.value)
63+
assert "SSL/Connection error occurred" in str(excinfo.value)
64+
finally:
65+
# Cleanup
66+
bigquery_client.delete_table(table_id)

packages/google-cloud-bigquery/tests/unit/test_client.py

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6840,6 +6840,38 @@ def test_insert_rows_w_wrong_arg(self):
68406840
with self.assertRaises(TypeError):
68416841
client.insert_rows_json(table, ROW)
68426842

6843+
def test_insert_rows_json_w_ssl_error(self):
6844+
from google.cloud.bigquery.dataset import DatasetReference
6845+
from google.cloud.bigquery.schema import SchemaField
6846+
from google.cloud.bigquery.table import Table
6847+
import requests.exceptions
6848+
6849+
PROJECT = "PROJECT"
6850+
DS_ID = "DS_ID"
6851+
TABLE_ID = "TABLE_ID"
6852+
ROWS = [{"full_name": "Bhettye Rhubble", "age": "27", "joined": None}]
6853+
6854+
creds = _make_credentials()
6855+
client = self._make_one(project=PROJECT, credentials=creds, _http=object())
6856+
conn = client._connection = make_connection({})
6857+
6858+
# Make the connection raise an SSLError
6859+
conn.api_request.side_effect = requests.exceptions.SSLError("EOF occurred")
6860+
6861+
table_ref = DatasetReference(PROJECT, DS_ID).table(TABLE_ID)
6862+
schema = [
6863+
SchemaField("full_name", "STRING", mode="REQUIRED"),
6864+
SchemaField("age", "INTEGER", mode="REQUIRED"),
6865+
SchemaField("joined", "TIMESTAMP", mode="NULLABLE"),
6866+
]
6867+
table = Table(table_ref, schema=schema)
6868+
6869+
with self.assertRaises(requests.exceptions.SSLError) as context:
6870+
client.insert_rows_json(table, ROWS)
6871+
6872+
self.assertIn("invalid table schema", str(context.exception))
6873+
self.assertIn("SSL/Connection error occurred", str(context.exception))
6874+
68436875
def test_list_partitions(self):
68446876
from google.cloud.bigquery.table import Table
68456877

packages/google-cloud-bigquery/tests/unit/test_retry.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,10 @@ def test_w_unstructured_requests_connectionerror(self):
5151
exc = requests.exceptions.ConnectionError()
5252
self.assertTrue(self._call_fut(exc))
5353

54+
def test_w_unstructured_requests_sslerror(self):
55+
exc = requests.exceptions.SSLError()
56+
self.assertFalse(self._call_fut(exc))
57+
5458
def test_w_unstructured_requests_chunked_encoding_error(self):
5559
exc = requests.exceptions.ChunkedEncodingError()
5660
self.assertTrue(self._call_fut(exc))

0 commit comments

Comments
 (0)