Skip to content

Commit 2a78994

Browse files
MantisusMantisus
committed
allow overwriting default headers
1 parent 70e328f commit 2a78994

File tree

3 files changed

+82
-30
lines changed

3 files changed

+82
-30
lines changed

‎src/apify_client/_http_client.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -38,29 +38,29 @@ def __init__(
3838
min_delay_between_retries_millis: int = 500,
3939
timeout_secs: int = 360,
4040
stats: Statistics | None = None,
41-
extra_headers: dict | None = None,
41+
headers: dict | None = None,
4242
) -> None:
4343
self.max_retries = max_retries
4444
self.min_delay_between_retries_millis = min_delay_between_retries_millis
4545
self.timeout_secs = timeout_secs
4646

47-
headers = {'Accept': 'application/json, */*'}
47+
default_headers = {'Accept': 'application/json, */*'}
4848

4949
workflow_key = os.getenv('APIFY_WORKFLOW_KEY')
5050
if workflow_key is not None:
51-
headers['X-Apify-Workflow-Key'] = workflow_key
51+
default_headers['X-Apify-Workflow-Key'] = workflow_key
5252

5353
is_at_home = 'APIFY_IS_AT_HOME' in os.environ
5454
python_version = '.'.join([str(x) for x in sys.version_info[:3]])
5555
client_version = metadata.version('apify-client')
5656

5757
user_agent = f'ApifyClient/{client_version} ({sys.platform}; Python/{python_version}); isAtHome/{is_at_home}'
58-
headers['User-Agent'] = user_agent
58+
default_headers['User-Agent'] = user_agent
5959

6060
if token is not None:
61-
headers['Authorization'] = f'Bearer {token}'
61+
default_headers['Authorization'] = f'Bearer {token}'
6262

63-
init_headers = {**(extra_headers or {}), **headers}
63+
init_headers = {**default_headers, **(headers or {})}
6464

6565
self.impit_client = impit.Client(headers=init_headers, follow_redirects=True, timeout=timeout_secs)
6666
self.impit_async_client = impit.AsyncClient(headers=init_headers, follow_redirects=True, timeout=timeout_secs)

‎src/apify_client/client.py‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
from __future__ import annotations
22

3+
import warnings
4+
35
from apify_client._http_client import HTTPClient, HTTPClientAsync
46
from apify_client._statistics import Statistics
57
from apify_client.clients import (
@@ -98,6 +100,18 @@ def _options(self) -> dict:
98100
'http_client': self.http_client,
99101
}
100102

103+
def _check_custom_headers(self, headers: dict) -> None:
104+
default_headers = {'Accept', 'Authorization', 'Accept-Encoding', 'User-Agent'}
105+
overwrite_headers = [key for key in headers if key.title() in default_headers]
106+
if overwrite_headers:
107+
warnings.warn(
108+
f'{", ".join(overwrite_headers)} headers of {self.__class__.__name__} was overridden with an '
109+
'explicit value. A wrong header value can lead to API errors, it is recommended to use the default '
110+
f'value for following headers: {", ".join(default_headers)}.',
111+
category=UserWarning,
112+
stacklevel=2,
113+
)
114+
101115

102116
class ApifyClient(_BaseApifyClient):
103117
"""The Apify API client."""
@@ -113,7 +127,7 @@ def __init__(
113127
max_retries: int | None = 8,
114128
min_delay_between_retries_millis: int | None = 500,
115129
timeout_secs: int | None = DEFAULT_TIMEOUT,
116-
extra_headers: dict | None = None,
130+
headers: dict | None = None,
117131
) -> None:
118132
"""Initialize a new instance.
119133
@@ -127,7 +141,7 @@ def __init__(
127141
min_delay_between_retries_millis: How long will the client wait between retrying requests
128142
(increases exponentially from this value).
129143
timeout_secs: The socket timeout of the HTTP requests sent to the Apify API.
130-
extra_headers: Additional headers to include in all requests.
144+
headers: Set headers to client for all requests.
131145
"""
132146
super().__init__(
133147
token,
@@ -139,13 +153,17 @@ def __init__(
139153
)
140154

141155
self.stats = Statistics()
156+
157+
if headers:
158+
self._check_custom_headers(headers)
159+
142160
self.http_client = HTTPClient(
143161
token=token,
144162
max_retries=self.max_retries,
145163
min_delay_between_retries_millis=self.min_delay_between_retries_millis,
146164
timeout_secs=self.timeout_secs,
147165
stats=self.stats,
148-
extra_headers=extra_headers,
166+
headers=headers,
149167
)
150168

151169
def actor(self, actor_id: str) -> ActorClient:
@@ -304,7 +322,7 @@ def __init__(
304322
max_retries: int | None = 8,
305323
min_delay_between_retries_millis: int | None = 500,
306324
timeout_secs: int | None = DEFAULT_TIMEOUT,
307-
extra_headers: dict | None = None,
325+
headers: dict | None = None,
308326
) -> None:
309327
"""Initialize a new instance.
310328
@@ -318,7 +336,7 @@ def __init__(
318336
min_delay_between_retries_millis: How long will the client wait between retrying requests
319337
(increases exponentially from this value).
320338
timeout_secs: The socket timeout of the HTTP requests sent to the Apify API.
321-
extra_headers: Additional headers to include in all requests.
339+
headers: Set headers to client for all requests.
322340
"""
323341
super().__init__(
324342
token,
@@ -330,13 +348,17 @@ def __init__(
330348
)
331349

332350
self.stats = Statistics()
351+
352+
if headers:
353+
self._check_custom_headers(headers)
354+
333355
self.http_client = HTTPClientAsync(
334356
token=token,
335357
max_retries=self.max_retries,
336358
min_delay_between_retries_millis=self.min_delay_between_retries_millis,
337359
timeout_secs=self.timeout_secs,
338360
stats=self.stats,
339-
extra_headers=extra_headers,
361+
headers=headers,
340362
)
341363

342364
def actor(self, actor_id: str) -> ActorClientAsync:

‎tests/unit/test_client_headers.py‎

Lines changed: 48 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@
66
from importlib import metadata
77
from typing import TYPE_CHECKING
88

9+
import pytest
910
from werkzeug import Request, Response
1011

12+
from apify_client import ApifyClient, ApifyClientAsync
1113
from apify_client._http_client import HTTPClient, HTTPClientAsync
1214

1315
if TYPE_CHECKING:
@@ -69,14 +71,13 @@ def test_default_headers_sync(httpserver: HTTPServer) -> None:
6971
}
7072

7173

72-
async def test_extra_headers_async(httpserver: HTTPServer) -> None:
73-
"""Test that extra headers are sent with each request."""
74+
async def test_headers_async(httpserver: HTTPServer) -> None:
75+
"""Test that custom headers are sent with each request."""
7476

75-
extra_headers = {
76-
'Test-Header': 'blah',
77-
'User-Agent': 'CustomUserAgent/1.0', # Do not override Apify User-Agent
78-
}
79-
client = HTTPClientAsync(token='placeholder_token', extra_headers=extra_headers)
77+
client = HTTPClientAsync(
78+
token='placeholder_token',
79+
headers={'Test-Header': 'blah', 'User-Agent': 'CustomUserAgent/1.0', 'Authorization': 'strange_value'},
80+
)
8081
httpserver.expect_request('/').respond_with_handler(_header_handler)
8182
api_url = httpserver.url_for('/').removesuffix('/')
8283

@@ -86,22 +87,25 @@ async def test_extra_headers_async(httpserver: HTTPServer) -> None:
8687

8788
assert request_headers == {
8889
'Test-Header': 'blah',
89-
'User-Agent': _get_user_agent(), # Do not override Apify User-Agent
90+
'User-Agent': 'CustomUserAgent/1.0',
9091
'Accept': 'application/json, */*',
91-
'Authorization': 'Bearer placeholder_token',
92+
'Authorization': 'strange_value',
9293
'Accept-Encoding': 'gzip, br, zstd, deflate',
9394
'Host': f'{httpserver.host}:{httpserver.port}',
9495
}
9596

9697

97-
def test_extra_headers_sync(httpserver: HTTPServer) -> None:
98-
"""Test that extra headers are sent with each request."""
98+
def test_headers_sync(httpserver: HTTPServer) -> None:
99+
"""Test that custom headers are sent with each request."""
99100

100-
extra_headers = {
101-
'Test-Header': 'blah',
102-
'User-Agent': 'CustomUserAgent/1.0', # Do not override Apify User-Agent
103-
}
104-
client = HTTPClient(token='placeholder_token', extra_headers=extra_headers)
101+
client = HTTPClient(
102+
token='placeholder_token',
103+
headers={
104+
'Test-Header': 'blah',
105+
'User-Agent': 'CustomUserAgent/1.0',
106+
'Authorization': 'strange_value',
107+
},
108+
)
105109
httpserver.expect_request('/').respond_with_handler(_header_handler)
106110
api_url = httpserver.url_for('/').removesuffix('/')
107111

@@ -111,9 +115,35 @@ def test_extra_headers_sync(httpserver: HTTPServer) -> None:
111115

112116
assert request_headers == {
113117
'Test-Header': 'blah',
114-
'User-Agent': _get_user_agent(), # Do not override Apify User-Agent
118+
'User-Agent': 'CustomUserAgent/1.0',
115119
'Accept': 'application/json, */*',
116-
'Authorization': 'Bearer placeholder_token',
120+
'Authorization': 'strange_value',
117121
'Accept-Encoding': 'gzip, br, zstd, deflate',
118122
'Host': f'{httpserver.host}:{httpserver.port}',
119123
}
124+
125+
126+
def test_warning_on_overridden_headers_sync() -> None:
127+
"""Test that warning is raised when default headers are overridden."""
128+
129+
with pytest.warns(UserWarning, match='User-Agent, Authorization headers of ApifyClient'):
130+
ApifyClient(
131+
token='placeholder_token',
132+
headers={
133+
'User-Agent': 'CustomUserAgent/1.0',
134+
'Authorization': 'strange_value',
135+
},
136+
)
137+
138+
139+
async def test_warning_on_overridden_headers_async() -> None:
140+
"""Test that warning is raised when default headers are overridden."""
141+
142+
with pytest.warns(UserWarning, match='User-Agent, Authorization headers of ApifyClientAsync'):
143+
ApifyClientAsync(
144+
token='placeholder_token',
145+
headers={
146+
'User-Agent': 'CustomUserAgent/1.0',
147+
'Authorization': 'strange_value',
148+
},
149+
)

0 commit comments

Comments
 (0)