refactor: Separate initial and max timeouts with lower defaults

vdusek · claude · vdusek · commit ac20367e9ef5 · 2026-03-02T15:44:30.000+01:00
Introduce `timeout_max` parameter to cap exponential timeout growth
independently from the initial timeout. Lower defaults to
`DEFAULT_REQUEST_TIMEOUT=10s`, `DEFAULT_REQUEST_TIMEOUT_MAX=600s`,
and `DEFAULT_MAX_RETRIES=4`, producing the sequence [10, 20, 40, 80, 160].

Co-Authored-By: Claude Opus 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/src/apify_client/_apify_client.py b/src/apify_client/_apify_client.py
@@ -10,6 +10,7 @@
     DEFAULT_MAX_RETRIES,
     DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
     DEFAULT_REQUEST_TIMEOUT,
+    DEFAULT_REQUEST_TIMEOUT_MAX,
 )
 from apify_client._docs import docs_group
 from apify_client._http_clients import HttpClient, HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync
@@ -115,6 +116,7 @@ def __init__(
         max_retries: int = DEFAULT_MAX_RETRIES,
         min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
         timeout: timedelta = DEFAULT_REQUEST_TIMEOUT,
+        timeout_max: timedelta = DEFAULT_REQUEST_TIMEOUT_MAX,
         headers: dict[str, str] | None = None,
     ) -> None:
         """Initialize the Apify API client.
@@ -132,7 +134,8 @@ def __init__(
             max_retries: How many times to retry a failed request at most.
             min_delay_between_retries: How long will the client wait between retrying requests
                 (increases exponentially from this value).
-            timeout: The socket timeout of the HTTP requests sent to the Apify API.
+            timeout: The initial socket timeout of the HTTP requests sent to the Apify API.
+            timeout_max: Maximum timeout cap for exponential timeout growth across retries.
             headers: Additional HTTP headers to include in all API requests.
         """
         # We need to do this because of mocking in tests and default mutable arguments.
@@ -192,6 +195,7 @@ def __init__(
         self._max_retries = max_retries
         self._min_delay_between_retries = min_delay_between_retries
         self._timeout = timeout
+        self._timeout_max = timeout_max
         self._headers = headers
 
     @classmethod
@@ -250,6 +254,7 @@ def http_client(self) -> HttpClient:
             self._http_client = ImpitHttpClient(
                 token=self._token,
                 timeout=self._timeout,
+                timeout_max=self._timeout_max,
                 max_retries=self._max_retries,
                 min_delay_between_retries=self._min_delay_between_retries,
                 statistics=self._statistics,
@@ -456,6 +461,7 @@ def __init__(
         max_retries: int = DEFAULT_MAX_RETRIES,
         min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
         timeout: timedelta = DEFAULT_REQUEST_TIMEOUT,
+        timeout_max: timedelta = DEFAULT_REQUEST_TIMEOUT_MAX,
         headers: dict[str, str] | None = None,
     ) -> None:
         """Initialize the Apify API client.
@@ -473,7 +479,8 @@ def __init__(
             max_retries: How many times to retry a failed request at most.
             min_delay_between_retries: How long will the client wait between retrying requests
                 (increases exponentially from this value).
-            timeout: The socket timeout of the HTTP requests sent to the Apify API.
+            timeout: The initial socket timeout of the HTTP requests sent to the Apify API.
+            timeout_max: Maximum timeout cap for exponential timeout growth across retries.
             headers: Additional HTTP headers to include in all API requests.
         """
         # We need to do this because of mocking in tests and default mutable arguments.
@@ -533,6 +540,7 @@ def __init__(
         self._max_retries = max_retries
         self._min_delay_between_retries = min_delay_between_retries
         self._timeout = timeout
+        self._timeout_max = timeout_max
         self._headers = headers
 
     @classmethod
@@ -591,6 +599,7 @@ def http_client(self) -> HttpClientAsync:
             self._http_client = ImpitHttpClientAsync(
                 token=self._token,
                 timeout=self._timeout,
+                timeout_max=self._timeout_max,
                 max_retries=self._max_retries,
                 min_delay_between_retries=self._min_delay_between_retries,
                 statistics=self._statistics,
diff --git a/src/apify_client/_consts.py b/src/apify_client/_consts.py
@@ -23,10 +23,13 @@
 API_VERSION = 'v2'
 """Current Apify API version."""
 
-DEFAULT_REQUEST_TIMEOUT = timedelta(seconds=30)
-"""Default timeout for individual API requests."""
+DEFAULT_REQUEST_TIMEOUT = timedelta(seconds=10)
+"""Default initial timeout for individual API requests."""
 
-DEFAULT_MAX_RETRIES = 8
+DEFAULT_REQUEST_TIMEOUT_MAX = timedelta(seconds=600)
+"""Default maximum timeout cap for individual API requests (limits exponential growth)."""
+
+DEFAULT_MAX_RETRIES = 4
 """Default maximum number of retries for failed requests."""
 
 DEFAULT_MIN_DELAY_BETWEEN_RETRIES = timedelta(milliseconds=500)
diff --git a/src/apify_client/_http_clients/_base.py b/src/apify_client/_http_clients/_base.py
@@ -14,6 +14,7 @@
     DEFAULT_MAX_RETRIES,
     DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
     DEFAULT_REQUEST_TIMEOUT,
+    DEFAULT_REQUEST_TIMEOUT_MAX,
     Timeout,
 )
 from apify_client._docs import docs_group
@@ -90,6 +91,7 @@ def __init__(
         *,
         token: str | None = None,
         timeout: timedelta = DEFAULT_REQUEST_TIMEOUT,
+        timeout_max: timedelta = DEFAULT_REQUEST_TIMEOUT_MAX,
         max_retries: int = DEFAULT_MAX_RETRIES,
         min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
         statistics: ClientStatistics | None = None,
@@ -99,13 +101,15 @@ def __init__(
 
         Args:
             token: Apify API token for authentication.
-            timeout: Request timeout.
+            timeout: Initial request timeout.
+            timeout_max: Maximum timeout cap for exponential timeout growth across retries.
             max_retries: Maximum number of retries for failed requests.
             min_delay_between_retries: Minimum delay between retries.
             statistics: Statistics tracker for API calls. Created automatically if not provided.
             headers: Additional HTTP headers to include in all requests.
         """
         self._timeout = timeout
+        self._timeout_max = timeout_max
         self._max_retries = max_retries
         self._min_delay_between_retries = min_delay_between_retries
         self._statistics = statistics or ClientStatistics()
diff --git a/src/apify_client/_http_clients/_impit.py b/src/apify_client/_http_clients/_impit.py
@@ -14,6 +14,7 @@
     DEFAULT_MAX_RETRIES,
     DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
     DEFAULT_REQUEST_TIMEOUT,
+    DEFAULT_REQUEST_TIMEOUT_MAX,
     Timeout,
 )
 from apify_client._docs import docs_group
@@ -54,18 +55,19 @@ def _compute_timeout(
     client_timeout: timedelta,
     attempt: int,
     timeout: Timeout,
+    timeout_max: timedelta,
 ) -> float:
-    """Compute timeout for a request attempt with exponential increase, bounded by client timeout.
+    """Compute timeout for a request attempt with exponential increase, bounded by timeout_max.
 
     For `'no_timeout'`, returns 0 (impit interprets this as no timeout). For `None`, uses the client-level timeout.
-    For `timedelta` values, doubles the timeout with each attempt but caps at the client-level timeout.
+    For `timedelta` values, doubles the timeout with each attempt but caps at the maximum timeout.
     """
     if timeout == 'no_timeout':
         return 0
 
     timeout_secs = to_seconds(timeout or client_timeout)
-    client_timeout_secs = to_seconds(client_timeout)
-    return min(client_timeout_secs, timeout_secs * 2 ** (attempt - 1))
+    timeout_max_secs = to_seconds(timeout_max)
+    return min(timeout_max_secs, timeout_secs * 2 ** (attempt - 1))
 
 
 @docs_group('HTTP clients')
@@ -82,6 +84,7 @@ def __init__(
         *,
         token: str | None = None,
         timeout: timedelta = DEFAULT_REQUEST_TIMEOUT,
+        timeout_max: timedelta = DEFAULT_REQUEST_TIMEOUT_MAX,
         max_retries: int = DEFAULT_MAX_RETRIES,
         min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
         statistics: ClientStatistics | None = None,
@@ -91,7 +94,8 @@ def __init__(
 
         Args:
             token: Apify API token for authentication.
-            timeout: Default timeout for HTTP requests.
+            timeout: Default initial timeout for HTTP requests.
+            timeout_max: Maximum timeout cap for exponential timeout growth across retries.
             max_retries: Maximum number of retry attempts for failed requests.
             min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
             statistics: Statistics tracker for API calls. Created automatically if not provided.
@@ -100,6 +104,7 @@ def __init__(
         super().__init__(
             token=token,
             timeout=timeout,
+            timeout_max=timeout_max,
             max_retries=max_retries,
             min_delay_between_retries=min_delay_between_retries,
             statistics=statistics,
@@ -109,7 +114,7 @@ def __init__(
         self._impit_client = impit.Client(
             headers=self._headers,
             follow_redirects=True,
-            timeout=to_seconds(self._timeout),
+            timeout=to_seconds(self._timeout_max),
         )
 
     def call(
@@ -212,7 +217,7 @@ def _make_request(
                 url=url_with_params,
                 headers=headers,
                 content=content,
-                timeout=_compute_timeout(self._timeout, attempt, timeout),
+                timeout=_compute_timeout(self._timeout, attempt, timeout, self._timeout_max),
                 stream=stream or False,
             )
 
@@ -309,6 +314,7 @@ def __init__(
         *,
         token: str | None = None,
         timeout: timedelta = DEFAULT_REQUEST_TIMEOUT,
+        timeout_max: timedelta = DEFAULT_REQUEST_TIMEOUT_MAX,
         max_retries: int = DEFAULT_MAX_RETRIES,
         min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
         statistics: ClientStatistics | None = None,
@@ -318,7 +324,8 @@ def __init__(
 
         Args:
             token: Apify API token for authentication.
-            timeout: Default timeout for HTTP requests.
+            timeout: Default initial timeout for HTTP requests.
+            timeout_max: Maximum timeout cap for exponential timeout growth across retries.
             max_retries: Maximum number of retry attempts for failed requests.
             min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
             statistics: Statistics tracker for API calls. Created automatically if not provided.
@@ -327,6 +334,7 @@ def __init__(
         super().__init__(
             token=token,
             timeout=timeout,
+            timeout_max=timeout_max,
             max_retries=max_retries,
             min_delay_between_retries=min_delay_between_retries,
             statistics=statistics,
@@ -336,7 +344,7 @@ def __init__(
         self._impit_async_client = impit.AsyncClient(
             headers=self._headers,
             follow_redirects=True,
-            timeout=to_seconds(self._timeout),
+            timeout=to_seconds(self._timeout_max),
         )
 
     async def call(
@@ -439,7 +447,7 @@ async def _make_request(
                 url=url_with_params,
                 headers=headers,
                 content=content,
-                timeout=_compute_timeout(self._timeout, attempt, timeout),
+                timeout=_compute_timeout(self._timeout, attempt, timeout, self._timeout_max),
                 stream=stream or False,
             )
 
diff --git a/tests/unit/test_client_timeouts.py b/tests/unit/test_client_timeouts.py
@@ -77,12 +77,12 @@ async def test_none_timeout_uses_client_default_async(patch_request: list) -> No
 async def test_dynamic_timeout_async_client(monkeypatch: pytest.MonkeyPatch) -> None:
     """Tests timeout values for request with retriable errors.
 
-    Values should increase with each attempt, starting from initial call value and bounded by the client timeout value.
+    Values should increase with each attempt, starting from initial call value and bounded by timeout_max.
     """
     should_raise_error = iter((True, True, True, False))
     call_timeout = 1
-    client_timeout = 5
-    expected_timeouts = [call_timeout, 2, 4, client_timeout]
+    timeout_max = 5
+    expected_timeouts = [call_timeout, 2, 4, timeout_max]
     retry_counter_mock = Mock()
 
     timeouts = []
@@ -98,9 +98,10 @@ async def mock_request(*_args: Any, **kwargs: Any) -> Response:
 
     monkeypatch.setattr('impit.AsyncClient.request', mock_request)
 
-    response = await ImpitHttpClientAsync(timeout=timedelta(seconds=client_timeout)).call(
-        method='GET', url='http://placeholder.url/async_timeout', timeout=timedelta(seconds=call_timeout)
-    )
+    response = await ImpitHttpClientAsync(
+        timeout=timedelta(seconds=call_timeout),
+        timeout_max=timedelta(seconds=timeout_max),
+    ).call(method='GET', url='http://placeholder.url/async_timeout', timeout=timedelta(seconds=call_timeout))
 
     # Check that the retry counter was called the expected number of times
     # (4 times: 3 retries + 1 final successful call)
@@ -141,12 +142,12 @@ async def mock_request(*_args: Any, **_kwargs: Any) -> Response:
 def test_dynamic_timeout_sync_client(monkeypatch: pytest.MonkeyPatch) -> None:
     """Tests timeout values for request with retriable errors.
 
-    Values should increase with each attempt, starting from initial call value and bounded by the client timeout value.
+    Values should increase with each attempt, starting from initial call value and bounded by timeout_max.
     """
     should_raise_error = iter((True, True, True, False))
     call_timeout = 1
-    client_timeout = 5
-    expected_timeouts = [call_timeout, 2, 4, client_timeout]
+    timeout_max = 5
+    expected_timeouts = [call_timeout, 2, 4, timeout_max]
     retry_counter_mock = Mock()
 
     timeouts = []
@@ -162,9 +163,10 @@ def mock_request(*_args: Any, **kwargs: Any) -> Response:
 
     monkeypatch.setattr('impit.Client.request', mock_request)
 
-    response = ImpitHttpClient(timeout=timedelta(seconds=client_timeout)).call(
-        method='GET', url='http://placeholder.url/sync_timeout', timeout=timedelta(seconds=call_timeout)
-    )
+    response = ImpitHttpClient(
+        timeout=timedelta(seconds=call_timeout),
+        timeout_max=timedelta(seconds=timeout_max),
+    ).call(method='GET', url='http://placeholder.url/sync_timeout', timeout=timedelta(seconds=call_timeout))
 
     # Check that the retry counter was called the expected number of times
     # (4 times: 3 retries + 1 final successful call)
