Skip to content

Commit 960da56

Browse files
fxdgearclaude
andcommitted
fix: retry on RemoteProtocolError when retry_connection_errors is enabled
When a server crashes mid-request (e.g. SIGSEGV in core-product-api), httpx raises RemoteProtocolError ("Server disconnected without sending a response"). This error was falling through to the catch-all Exception handler and being wrapped as PermanentError, bypassing retry even when retry_connection_errors=True. This adds RemoteProtocolError to the list of retriable exceptions alongside ConnectError and TimeoutException, in both sync and async retry paths. Discovered during INC-10207 (40factory) where core-product-api pod crashes caused cascading 504s in the partitioner plugin with no retry. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent fbd80fd commit 960da56

File tree

2 files changed

+139
-0
lines changed

2 files changed

+139
-0
lines changed
Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
"""Tests for retry logic, specifically covering RemoteProtocolError retry behavior."""
2+
3+
import asyncio
4+
from unittest.mock import MagicMock
5+
6+
import httpx
7+
import pytest
8+
9+
from unstructured_client.utils.retries import (
10+
BackoffStrategy,
11+
PermanentError,
12+
Retries,
13+
RetryConfig,
14+
retry,
15+
retry_async,
16+
)
17+
18+
19+
def _make_retries(retry_connection_errors: bool) -> Retries:
20+
return Retries(
21+
config=RetryConfig(
22+
strategy="backoff",
23+
backoff=BackoffStrategy(
24+
initial_interval=100,
25+
max_interval=200,
26+
exponent=1.5,
27+
max_elapsed_time=5000,
28+
),
29+
retry_connection_errors=retry_connection_errors,
30+
),
31+
status_codes=[],
32+
)
33+
34+
35+
class TestRemoteProtocolErrorRetry:
36+
"""Test that RemoteProtocolError (e.g. 'Server disconnected without sending a response')
37+
is retried when retry_connection_errors=True."""
38+
39+
def test_remote_protocol_error_retried_when_enabled(self):
40+
"""RemoteProtocolError should be retried and succeed on subsequent attempt."""
41+
retries_config = _make_retries(retry_connection_errors=True)
42+
43+
mock_response = MagicMock(spec=httpx.Response)
44+
mock_response.status_code = 200
45+
46+
call_count = 0
47+
48+
def func():
49+
nonlocal call_count
50+
call_count += 1
51+
if call_count == 1:
52+
raise httpx.RemoteProtocolError(
53+
"Server disconnected without sending a response."
54+
)
55+
return mock_response
56+
57+
result = retry(func, retries_config)
58+
assert result.status_code == 200
59+
assert call_count == 2
60+
61+
def test_remote_protocol_error_not_retried_when_disabled(self):
62+
"""RemoteProtocolError should raise PermanentError when retry_connection_errors=False."""
63+
retries_config = _make_retries(retry_connection_errors=False)
64+
65+
def func():
66+
raise httpx.RemoteProtocolError(
67+
"Server disconnected without sending a response."
68+
)
69+
70+
with pytest.raises(httpx.RemoteProtocolError):
71+
retry(func, retries_config)
72+
73+
def test_connect_error_still_retried(self):
74+
"""Existing ConnectError retry behavior should be preserved."""
75+
retries_config = _make_retries(retry_connection_errors=True)
76+
77+
mock_response = MagicMock(spec=httpx.Response)
78+
mock_response.status_code = 200
79+
80+
call_count = 0
81+
82+
def func():
83+
nonlocal call_count
84+
call_count += 1
85+
if call_count == 1:
86+
raise httpx.ConnectError("Connection refused")
87+
return mock_response
88+
89+
result = retry(func, retries_config)
90+
assert result.status_code == 200
91+
assert call_count == 2
92+
93+
94+
class TestRemoteProtocolErrorRetryAsync:
95+
"""Async versions of the RemoteProtocolError retry tests."""
96+
97+
def test_remote_protocol_error_retried_async(self):
98+
"""Async: RemoteProtocolError should be retried when retry_connection_errors=True."""
99+
retries_config = _make_retries(retry_connection_errors=True)
100+
101+
mock_response = MagicMock(spec=httpx.Response)
102+
mock_response.status_code = 200
103+
104+
call_count = 0
105+
106+
async def func():
107+
nonlocal call_count
108+
call_count += 1
109+
if call_count == 1:
110+
raise httpx.RemoteProtocolError(
111+
"Server disconnected without sending a response."
112+
)
113+
return mock_response
114+
115+
result = asyncio.run(retry_async(func, retries_config))
116+
assert result.status_code == 200
117+
assert call_count == 2
118+
119+
def test_remote_protocol_error_not_retried_async_when_disabled(self):
120+
"""Async: RemoteProtocolError should not be retried when retry_connection_errors=False."""
121+
retries_config = _make_retries(retry_connection_errors=False)
122+
123+
async def func():
124+
raise httpx.RemoteProtocolError(
125+
"Server disconnected without sending a response."
126+
)
127+
128+
with pytest.raises(httpx.RemoteProtocolError):
129+
asyncio.run(retry_async(func, retries_config))

src/unstructured_client/utils/retries.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,11 @@ def do_request() -> httpx.Response:
8888
if retries.config.retry_connection_errors:
8989
raise
9090

91+
raise PermanentError(exception) from exception
92+
except httpx.RemoteProtocolError as exception:
93+
if retries.config.retry_connection_errors:
94+
raise
95+
9196
raise PermanentError(exception) from exception
9297
except httpx.TimeoutException as exception:
9398
if retries.config.retry_connection_errors:
@@ -137,6 +142,11 @@ async def do_request() -> httpx.Response:
137142
if retries.config.retry_connection_errors:
138143
raise
139144

145+
raise PermanentError(exception) from exception
146+
except httpx.RemoteProtocolError as exception:
147+
if retries.config.retry_connection_errors:
148+
raise
149+
140150
raise PermanentError(exception) from exception
141151
except httpx.TimeoutException as exception:
142152
if retries.config.retry_connection_errors:

0 commit comments

Comments
 (0)