From 40b1b08bb7b1b0f86e85c5cbb49307921219a607 Mon Sep 17 00:00:00 2001 From: lmossman Date: Mon, 4 Aug 2025 17:46:52 -0700 Subject: [PATCH 1/4] don't replace base url in paginator path anymore --- .../requesters/paginators/default_paginator.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py b/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py index ca2405b44..cb1072a8c 100644 --- a/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +++ b/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py @@ -159,14 +159,7 @@ def path( ) -> Optional[str]: token = next_page_token.get("next_page_token") if next_page_token else None if token and self.page_token_option and isinstance(self.page_token_option, RequestPath): - # make additional interpolation context - interpolation_context = get_interpolation_context( - stream_state=stream_state, - stream_slice=stream_slice, - next_page_token=next_page_token, - ) - # Replace url base to only return the path - return str(token).replace(self.url_base.eval(self.config, **interpolation_context), "") # type: ignore # url_base is casted to a InterpolatedString in __post_init__ + return str(token) else: return None From 83365d473e3735b33334f8a38ddcb31307d80e29 Mon Sep 17 00:00:00 2001 From: lmossman Date: Tue, 5 Aug 2025 10:45:25 -0700 Subject: [PATCH 2/4] fix unit test --- .../requesters/paginators/test_default_paginator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py b/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py index 925ce0c59..b0eb12c9d 100644 --- a/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +++ b/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py @@ -36,7 +36,7 @@ ( RequestPath(parameters={}), None, - "/next_url", + "https://airbyte.io/next_url", {"limit": 2}, {}, {}, @@ -128,7 +128,7 @@ ( RequestPath(parameters={}), None, - "/next_url", + "https://airbyte.io/next_url", {"limit": 2}, {}, {}, From 0ecadd5d45aa1cb423a20e3062004983e96be96f Mon Sep 17 00:00:00 2001 From: lmossman Date: Tue, 5 Aug 2025 10:47:11 -0700 Subject: [PATCH 3/4] call join_url on url and path --- .../sources/declarative/requesters/http_requester.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte_cdk/sources/declarative/requesters/http_requester.py b/airbyte_cdk/sources/declarative/requesters/http_requester.py index 6b0e65aab..3ce4c8540 100644 --- a/airbyte_cdk/sources/declarative/requesters/http_requester.py +++ b/airbyte_cdk/sources/declarative/requesters/http_requester.py @@ -168,7 +168,13 @@ def _get_url( next_page_token=next_page_token, ) - full_url = self._join_url(url_base, path) if url_base else url + path if path else url + full_url = ( + self._join_url(url_base, path) + if url_base + else self._join_url(url, path) + if path + else url + ) return full_url From 1117b0e857177ea5c35854da6fd8b8c63a970314 Mon Sep 17 00:00:00 2001 From: lmossman Date: Tue, 5 Aug 2025 11:35:35 -0700 Subject: [PATCH 4/4] remove unnecessary test --- .../paginators/test_default_paginator.py | 39 ------------------- 1 file changed, 39 deletions(-) diff --git a/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py b/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py index b0eb12c9d..ee9f12b12 100644 --- a/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +++ b/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py @@ -539,42 +539,3 @@ def test_path_returns_none_when_option_not_request_path() -> None: ) result = paginator.path(next_page_token) assert result is None - - -def test_path_with_additional_interpolation_context() -> None: - page_token_option = RequestPath(parameters={}) - paginator = DefaultPaginator( - pagination_strategy=Mock(), - config={}, - url_base="https://api.domain.com/{{ stream_slice['campaign_id'] }}", - parameters={}, - page_token_option=page_token_option, - ) - # define stream_state here - stream_state = {"state": "state_value"} - # define stream_slice here - stream_slice = StreamSlice( - partition={ - "campaign_id": "123_abcd", - }, - cursor_slice={ - "start": "A", - "end": "B", - }, - extra_fields={ - "extra_field_A": "value_A", - "extra_field_B": "value_B", - }, - ) - # define next_page_token here - next_page_token = { - "next_page_token": "https://api.domain.com/123_abcd/some_next_page_token_here" - } - - expected_after_interpolation = "/some_next_page_token_here" - - assert expected_after_interpolation == paginator.path( - next_page_token=next_page_token, - stream_state=stream_state, - stream_slice=stream_slice, - )