Skip to content

Commit bb5a260

Browse files
PijukatelPijukatel
committed
Allow to define public api url different from api url
1 parent 348cb7d commit bb5a260

7 files changed

Lines changed: 110 additions & 30 deletions

File tree

‎src/apify_client/client.py‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ def __init__(
6464
token: str | None = None,
6565
*,
6666
api_url: str | None = None,
67+
api_public_url: str | None = None,
6768
max_retries: int | None = 8,
6869
min_delay_between_retries_millis: int | None = 500,
6970
timeout_secs: int | None = DEFAULT_TIMEOUT,
@@ -72,7 +73,10 @@ def __init__(
7273
7374
Args:
7475
token: The Apify API token.
75-
api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com.
76+
api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com. It can
77+
be internal url that is not globally accessible, in such case `api_public_url` should be set as well.
78+
api_public_url: The globally accessible URL of the Apify API server. It should be set only if the `api_url`
79+
is internal url that is not globally accessible.
7680
max_retries: How many times to retry a failed request at most.
7781
min_delay_between_retries_millis: How long will the client wait between retrying requests
7882
(increases exponentially from this value).
@@ -81,6 +85,7 @@ def __init__(
8185
self.token = token
8286
api_url = (api_url or DEFAULT_API_URL).rstrip('/')
8387
self.base_url = f'{api_url}/{API_VERSION}'
88+
self.public_base_url = (api_public_url or self.base_url).rstrip('/')
8489
self.max_retries = max_retries or 8
8590
self.min_delay_between_retries_millis = min_delay_between_retries_millis or 500
8691
self.timeout_secs = timeout_secs or DEFAULT_TIMEOUT
@@ -103,6 +108,7 @@ def __init__(
103108
token: str | None = None,
104109
*,
105110
api_url: str | None = None,
111+
api_public_url: str | None = None,
106112
max_retries: int | None = 8,
107113
min_delay_between_retries_millis: int | None = 500,
108114
timeout_secs: int | None = DEFAULT_TIMEOUT,
@@ -111,7 +117,10 @@ def __init__(
111117
112118
Args:
113119
token: The Apify API token.
114-
api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com.
120+
api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com. It can
121+
be internal url that is not globally accessible, in such case `api_public_url` should be set as well.
122+
api_public_url: The globally accessible URL of the Apify API server. It should be set only if the `api_url`
123+
is internal url that is not globally accessible.
115124
max_retries: How many times to retry a failed request at most.
116125
min_delay_between_retries_millis: How long will the client wait between retrying requests
117126
(increases exponentially from this value).
@@ -120,6 +129,7 @@ def __init__(
120129
super().__init__(
121130
token,
122131
api_url=api_url,
132+
api_public_url=api_public_url,
123133
max_retries=max_retries,
124134
min_delay_between_retries_millis=min_delay_between_retries_millis,
125135
timeout_secs=timeout_secs,
@@ -286,6 +296,7 @@ def __init__(
286296
token: str | None = None,
287297
*,
288298
api_url: str | None = None,
299+
api_public_url: str | None = None,
289300
max_retries: int | None = 8,
290301
min_delay_between_retries_millis: int | None = 500,
291302
timeout_secs: int | None = DEFAULT_TIMEOUT,
@@ -294,7 +305,10 @@ def __init__(
294305
295306
Args:
296307
token: The Apify API token.
297-
api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com.
308+
api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com. It can
309+
be internal url that is not globally accessible, in such case `api_public_url` should be set as well.
310+
api_public_url: The globally accessible URL of the Apify API server. It should be set only if the `api_url`
311+
is internal url that is not globally accessible.
298312
max_retries: How many times to retry a failed request at most.
299313
min_delay_between_retries_millis: How long will the client wait between retrying requests
300314
(increases exponentially from this value).
@@ -303,6 +317,7 @@ def __init__(
303317
super().__init__(
304318
token,
305319
api_url=api_url,
320+
api_public_url=api_public_url,
306321
max_retries=max_retries,
307322
min_delay_between_retries_millis=min_delay_between_retries_millis,
308323
timeout_secs=timeout_secs,

‎src/apify_client/clients/base/base_client.py‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from __future__ import annotations
22

33
from typing import TYPE_CHECKING, Any
4+
from urllib.parse import urlparse
45

56
from apify_client._logging import WithLogDetailsClient
67
from apify_client._utils import to_safe_id
@@ -18,10 +19,13 @@ class _BaseBaseClient(metaclass=WithLogDetailsClient):
1819
http_client: HTTPClient | HTTPClientAsync
1920
root_client: ApifyClient | ApifyClientAsync
2021

21-
def _url(self, path: str | None = None) -> str:
22-
if path is not None:
23-
return f'{self.url}/{path}'
24-
return self.url
22+
def _url(self, path: str | None = None, *, public: bool = False) -> str:
23+
url = f'{self.url}/{path}' if path is not None else self.url
24+
25+
if public:
26+
url_path = urlparse(url).path
27+
return urlparse(self.root_client.public_base_url)._replace(path=url_path).geturl()
28+
return url
2529

2630
def _params(self, **kwargs: Any) -> dict:
2731
return {

‎src/apify_client/clients/resource_clients/dataset.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -619,7 +619,7 @@ def create_items_public_url(
619619
)
620620
request_params['signature'] = signature
621621

622-
items_public_url = urlparse(self._url('items'))
622+
items_public_url = urlparse(self._url('items', public=True))
623623
filtered_params = {k: v for k, v in request_params.items() if v is not None}
624624
if filtered_params:
625625
items_public_url = items_public_url._replace(query=urlencode(filtered_params))
@@ -1126,7 +1126,7 @@ async def create_items_public_url(
11261126
)
11271127
request_params['signature'] = signature
11281128

1129-
items_public_url = urlparse(self._url('items'))
1129+
items_public_url = urlparse(self._url('items', public=True))
11301130
filtered_params = {k: v for k, v in request_params.items() if v is not None}
11311131
if filtered_params:
11321132
items_public_url = items_public_url._replace(query=urlencode(filtered_params))

‎src/apify_client/clients/resource_clients/key_value_store.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -307,7 +307,7 @@ def create_keys_public_url(
307307
)
308308
request_params['signature'] = signature
309309

310-
keys_public_url = urlparse(self._url('keys'))
310+
keys_public_url = urlparse(self._url('keys', public=True))
311311

312312
filtered_params = {k: v for k, v in request_params.items() if v is not None}
313313
if filtered_params:
@@ -597,7 +597,7 @@ async def create_keys_public_url(
597597
)
598598
request_params['signature'] = signature
599599

600-
keys_public_url = urlparse(self._url('keys'))
600+
keys_public_url = urlparse(self._url('keys', public=True))
601601
filtered_params = {k: v for k, v in request_params.items() if v is not None}
602602
if filtered_params:
603603
keys_public_url = keys_public_url._replace(query=urlencode(filtered_params))

‎tests/integration/conftest.py‎

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,27 @@
77
TOKEN_ENV_VAR = 'APIFY_TEST_USER_API_TOKEN'
88
API_URL_ENV_VAR = 'APIFY_INTEGRATION_TESTS_API_URL'
99

10+
parametrized_api_urls = pytest.mark.parametrize(
11+
('api_url', 'api_public_url'),
12+
[
13+
('https://api.apify.com', 'https://api.apify.com'),
14+
('https://api.apify.com', None),
15+
('https://api.apify.com', 'https://custom-public-url.com'),
16+
],
17+
)
1018

11-
@pytest.fixture
12-
def apify_client() -> ApifyClient:
13-
api_token = os.getenv(TOKEN_ENV_VAR)
14-
api_url = os.getenv(API_URL_ENV_VAR)
1519

16-
if not api_token:
20+
@pytest.fixture
21+
def api_token() -> str:
22+
token = os.getenv(TOKEN_ENV_VAR)
23+
if not token:
1724
raise RuntimeError(f'{TOKEN_ENV_VAR} environment variable is missing, cannot run tests!')
25+
return token
1826

27+
28+
@pytest.fixture
29+
def apify_client(api_token: str) -> ApifyClient:
30+
api_url = os.getenv(API_URL_ENV_VAR)
1931
return ApifyClient(api_token, api_url=api_url)
2032

2133

@@ -25,11 +37,6 @@ def apify_client() -> ApifyClient:
2537
# but `pytest-asyncio` closes the event loop after each test,
2638
# and uses a new one for the next test.
2739
@pytest.fixture
28-
def apify_client_async() -> ApifyClientAsync:
29-
api_token = os.getenv(TOKEN_ENV_VAR)
40+
def apify_client_async(api_token: str) -> ApifyClientAsync:
3041
api_url = os.getenv(API_URL_ENV_VAR)
31-
32-
if not api_token:
33-
raise RuntimeError(f'{TOKEN_ENV_VAR} environment variable is missing, cannot run tests!')
34-
3542
return ApifyClientAsync(api_token, api_url=api_url)

‎tests/integration/test_dataset.py‎

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,11 @@
11
from __future__ import annotations
22

3-
from typing import TYPE_CHECKING
4-
53
import impit
64

5+
from integration.conftest import parametrized_api_urls
76
from integration.integration_test_utils import random_resource_name
87

9-
if TYPE_CHECKING:
10-
from apify_client import ApifyClient, ApifyClientAsync
8+
from apify_client import ApifyClient, ApifyClientAsync
119

1210

1311
class TestDatasetSync:
@@ -47,6 +45,20 @@ def test_dataset_should_create_public_items_non_expiring_url(self, apify_client:
4745
dataset.delete()
4846
assert apify_client.dataset(created_dataset['id']).get() is None
4947

48+
@parametrized_api_urls
49+
def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None:
50+
apify_client = ApifyClient(token=api_token, api_url=api_url, api_public_url=api_public_url)
51+
created_store = apify_client.datasets().get_or_create(name=random_resource_name('key-value-store'))
52+
dataset = apify_client.dataset(created_store['id'])
53+
try:
54+
public_url = dataset.create_items_public_url()
55+
assert public_url == (
56+
f'{api_public_url or api_url}/v2/datasets/'
57+
f'{created_store["id"]}/items?signature={public_url.split("signature=")[1]}'
58+
)
59+
finally:
60+
dataset.delete()
61+
5062

5163
class TestDatasetAsync:
5264
async def test_dataset_should_create_public_items_expiring_url_with_params(
@@ -88,3 +100,17 @@ async def test_dataset_should_create_public_items_non_expiring_url(
88100

89101
await dataset.delete()
90102
assert await apify_client_async.dataset(created_dataset['id']).get() is None
103+
104+
@parametrized_api_urls
105+
async def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None:
106+
apify_client = ApifyClientAsync(token=api_token, api_url=api_url, api_public_url=api_public_url)
107+
created_store = await apify_client.datasets().get_or_create(name=random_resource_name('key-value-store'))
108+
dataset = apify_client.dataset(created_store['id'])
109+
try:
110+
public_url = await dataset.create_items_public_url()
111+
assert public_url == (
112+
f'{api_public_url or api_url}/v2/datasets/'
113+
f'{created_store["id"]}/items?signature={public_url.split("signature=")[1]}'
114+
)
115+
finally:
116+
await dataset.delete()

‎tests/integration/test_key_value_store.py‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,11 @@
11
from __future__ import annotations
22

3-
from typing import TYPE_CHECKING
4-
53
import impit
64

5+
from integration.conftest import parametrized_api_urls
76
from integration.integration_test_utils import random_resource_name
87

9-
if TYPE_CHECKING:
10-
from apify_client import ApifyClient, ApifyClientAsync
8+
from apify_client import ApifyClient, ApifyClientAsync
119

1210

1311
class TestKeyValueStoreSync:
@@ -47,6 +45,20 @@ def test_key_value_store_should_create_public_keys_non_expiring_url(self, apify_
4745
store.delete()
4846
assert apify_client.key_value_store(created_store['id']).get() is None
4947

48+
@parametrized_api_urls
49+
def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None:
50+
apify_client = ApifyClient(token=api_token, api_url=api_url, api_public_url=api_public_url)
51+
created_store = apify_client.key_value_stores().get_or_create(name=random_resource_name('key-value-store'))
52+
kvs = apify_client.key_value_store(created_store['id'])
53+
try:
54+
public_url = kvs.create_keys_public_url()
55+
assert public_url == (
56+
f'{api_public_url or api_url}/v2/key-value-stores/'
57+
f'{created_store["id"]}/keys?signature={public_url.split("signature=")[1]}'
58+
)
59+
finally:
60+
kvs.delete()
61+
5062

5163
class TestKeyValueStoreAsync:
5264
async def test_key_value_store_should_create_expiring_keys_public_url_with_params(
@@ -90,3 +102,19 @@ async def test_key_value_store_should_create_public_keys_non_expiring_url(
90102

91103
await store.delete()
92104
assert await apify_client_async.key_value_store(created_store['id']).get() is None
105+
106+
@parametrized_api_urls
107+
async def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None:
108+
apify_client = ApifyClientAsync(token=api_token, api_url=api_url, api_public_url=api_public_url)
109+
created_store = await apify_client.key_value_stores().get_or_create(
110+
name=random_resource_name('key-value-store')
111+
)
112+
kvs = apify_client.key_value_store(created_store['id'])
113+
try:
114+
public_url = await kvs.create_keys_public_url()
115+
assert public_url == (
116+
f'{api_public_url or api_url}/v2/key-value-stores/'
117+
f'{created_store["id"]}/keys?signature={public_url.split("signature=")[1]}'
118+
)
119+
finally:
120+
await kvs.delete()

0 commit comments

Comments
 (0)