Add extra_headers argument to allow setting additional user headers

Author: Mantisus

src/apify_client/_http_client.py

@@ -38,6 +38,7 @@ def __init__(
         min_delay_between_retries_millis: int = 500,
         timeout_secs: int = 360,
         stats: Statistics | None = None,
+        extra_headers: dict | None = None,
     ) -> None:
         self.max_retries = max_retries
         self.min_delay_between_retries_millis = min_delay_between_retries_millis
@@ -59,8 +60,10 @@ def __init__(
         if token is not None:
             headers['Authorization'] = f'Bearer {token}'
 
-        self.impit_client = impit.Client(headers=headers, follow_redirects=True, timeout=timeout_secs)
-        self.impit_async_client = impit.AsyncClient(headers=headers, follow_redirects=True, timeout=timeout_secs)
+        init_headers = {**(extra_headers or {}), **headers}
+
+        self.impit_client = impit.Client(headers=init_headers, follow_redirects=True, timeout=timeout_secs)
+        self.impit_async_client = impit.AsyncClient(headers=init_headers, follow_redirects=True, timeout=timeout_secs)
 
         self.stats = stats or Statistics()
 

src/apify_client/client.py

@@ -113,6 +113,7 @@ def __init__(
         max_retries: int | None = 8,
         min_delay_between_retries_millis: int | None = 500,
         timeout_secs: int | None = DEFAULT_TIMEOUT,
+        extra_headers: dict | None = None,
     ) -> None:
         """Initialize a new instance.
 
@@ -126,6 +127,7 @@ def __init__(
             min_delay_between_retries_millis: How long will the client wait between retrying requests
                 (increases exponentially from this value).
             timeout_secs: The socket timeout of the HTTP requests sent to the Apify API.
+            extra_headers: Additional headers to include in all requests.
         """
         super().__init__(
             token,
@@ -143,6 +145,7 @@ def __init__(
             min_delay_between_retries_millis=self.min_delay_between_retries_millis,
             timeout_secs=self.timeout_secs,
             stats=self.stats,
+            extra_headers=extra_headers,
         )
 
     def actor(self, actor_id: str) -> ActorClient:
@@ -301,6 +304,7 @@ def __init__(
         max_retries: int | None = 8,
         min_delay_between_retries_millis: int | None = 500,
         timeout_secs: int | None = DEFAULT_TIMEOUT,
+        extra_headers: dict | None = None,
     ) -> None:
         """Initialize a new instance.
 
@@ -314,6 +318,7 @@ def __init__(
             min_delay_between_retries_millis: How long will the client wait between retrying requests
                 (increases exponentially from this value).
             timeout_secs: The socket timeout of the HTTP requests sent to the Apify API.
+            extra_headers: Additional headers to include in all requests.
         """
         super().__init__(
             token,
@@ -331,6 +336,7 @@ def __init__(
             min_delay_between_retries_millis=self.min_delay_between_retries_millis,
             timeout_secs=self.timeout_secs,
             stats=self.stats,
+            extra_headers=extra_headers,
         )
 
     def actor(self, actor_id: str) -> ActorClientAsync:

tests/unit/test_client_headers.py

@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+import json
+import os
+import sys
+from importlib import metadata
+from typing import TYPE_CHECKING
+
+from werkzeug import Request, Response
+
+from apify_client._http_client import HTTPClient, HTTPClientAsync
+
+if TYPE_CHECKING:
+    from pytest_httpserver import HTTPServer
+
+
+def _header_handler(request: Request) -> Response:
+    return Response(
+        status=200,
+        headers={},
+        response=json.dumps({'received_headers': dict(request.headers)}),
+    )
+
+
+def _get_user_agent() -> str:
+    is_at_home = 'APIFY_IS_AT_HOME' in os.environ
+    python_version = '.'.join([str(x) for x in sys.version_info[:3]])
+    client_version = metadata.version('apify-client')
+    return f'ApifyClient/{client_version} ({sys.platform}; Python/{python_version}); isAtHome/{is_at_home}'
+
+
+async def test_default_headers_async(httpserver: HTTPServer) -> None:
+    """Test that default headers are sent with each request."""
+
+    client = HTTPClientAsync(token='placeholder_token')
+    httpserver.expect_request('/').respond_with_handler(_header_handler)
+    api_url = httpserver.url_for('/').removesuffix('/')
+
+    response = await client.call(method='GET', url=f'{api_url}/')
+
+    request_headers = json.loads(response.text)['received_headers']
+
+    assert request_headers == {
+        'User-Agent': _get_user_agent(),
+        'Accept': 'application/json, */*',
+        'Authorization': 'Bearer placeholder_token',
+        'Accept-Encoding': 'gzip, br, zstd, deflate',
+        'Host': f'{httpserver.host}:{httpserver.port}',
+    }
+
+
+def test_default_headers_sync(httpserver: HTTPServer) -> None:
+    """Test that default headers are sent with each request."""
+
+    client = HTTPClient(token='placeholder_token')
+    httpserver.expect_request('/').respond_with_handler(_header_handler)
+    api_url = httpserver.url_for('/').removesuffix('/')
+
+    response = client.call(method='GET', url=f'{api_url}/')
+
+    request_headers = json.loads(response.text)['received_headers']
+
+    assert request_headers == {
+        'User-Agent': _get_user_agent(),
+        'Accept': 'application/json, */*',
+        'Authorization': 'Bearer placeholder_token',
+        'Accept-Encoding': 'gzip, br, zstd, deflate',
+        'Host': f'{httpserver.host}:{httpserver.port}',
+    }
+
+
+async def test_extra_headers_async(httpserver: HTTPServer) -> None:
+    """Test that extra headers are sent with each request."""
+
+    extra_headers = {
+        'Test-Header': 'blah',
+        'User-Agent': 'CustomUserAgent/1.0',  # Do not override Apify User-Agent
+    }
+    client = HTTPClientAsync(token='placeholder_token', extra_headers=extra_headers)
+    httpserver.expect_request('/').respond_with_handler(_header_handler)
+    api_url = httpserver.url_for('/').removesuffix('/')
+
+    response = await client.call(method='GET', url=f'{api_url}/')
+
+    request_headers = json.loads(response.text)['received_headers']
+
+    assert request_headers == {
+        'Test-Header': 'blah',
+        'User-Agent': _get_user_agent(),  # Do not override Apify User-Agent
+        'Accept': 'application/json, */*',
+        'Authorization': 'Bearer placeholder_token',
+        'Accept-Encoding': 'gzip, br, zstd, deflate',
+        'Host': f'{httpserver.host}:{httpserver.port}',
+    }
+
+
+def test_extra_headers_sync(httpserver: HTTPServer) -> None:
+    """Test that extra headers are sent with each request."""
+
+    extra_headers = {
+        'Test-Header': 'blah',
+        'User-Agent': 'CustomUserAgent/1.0',  # Do not override Apify User-Agent
+    }
+    client = HTTPClient(token='placeholder_token', extra_headers=extra_headers)
+    httpserver.expect_request('/').respond_with_handler(_header_handler)
+    api_url = httpserver.url_for('/').removesuffix('/')
+
+    response = client.call(method='GET', url=f'{api_url}/')
+
+    request_headers = json.loads(response.text)['received_headers']
+
+    assert request_headers == {
+        'Test-Header': 'blah',
+        'User-Agent': _get_user_agent(),  # Do not override Apify User-Agent
+        'Accept': 'application/json, */*',
+        'Authorization': 'Bearer placeholder_token',
+        'Accept-Encoding': 'gzip, br, zstd, deflate',
+        'Host': f'{httpserver.host}:{httpserver.port}',
+    }