From efceea2b352fdc34bbdecc124822875a50a6b602 Mon Sep 17 00:00:00 2001 From: maxi297 Date: Mon, 14 Apr 2025 16:06:13 -0400 Subject: [PATCH 1/2] Ensure page token 0 is being passed --- .../retrievers/simple_retriever.py | 4 +-- .../retrievers/test_simple_retriever.py | 34 +++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index b339aaedf..cb4684ba5 100644 --- a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -363,9 +363,7 @@ def _read_pages( ) -> Iterable[Record]: pagination_complete = False initial_token = self._paginator.get_initial_token() - next_page_token: Optional[Mapping[str, Any]] = ( - {"next_page_token": initial_token} if initial_token else None - ) + next_page_token: Optional[Mapping[str, Any]] = {"next_page_token": initial_token} if initial_token is not None else None while not pagination_complete: response = self._fetch_next_page(stream_state, stream_slice, next_page_token) diff --git a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py index 442f8fc22..d862f4c39 100644 --- a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py +++ b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py @@ -810,6 +810,40 @@ def retriever_read_pages(_, __, ___): cursor.close_slice.assert_called_once_with(stream_slice, None) +def test_given_initial_token_is_zero_when_read_records_then_pass_initial_token(): + record_selector = MagicMock() + record_selector.select_records.return_value = [] + cursor = MagicMock(spec=DeclarativeCursor) + paginator = MagicMock() + paginator.get_initial_token.return_value = 0 + paginator.next_page_token.return_value = None + + retriever = SimpleRetriever( + name="stream_name", + primary_key=primary_key, + requester=MagicMock(), + paginator=paginator, + record_selector=record_selector, + stream_slicer=cursor, + cursor=cursor, + parameters={}, + config={}, + ) + stream_slice = StreamSlice(cursor_slice={}, partition={}) + + response = requests.Response() + response.status_code = 200 + response._content = "{}".encode() + + with patch.object( + SimpleRetriever, + "_fetch_next_page", + return_value=response, + ) as fetch_next_page_mock: + list(retriever.read_records(stream_slice=stream_slice, records_schema={})) + fetch_next_page_mock.assert_called_once_with(cursor.get_stream_state(), stream_slice, {"next_page_token": 0}) + + def _generate_slices(number_of_slices): return [{"date": f"2022-01-0{day + 1}"} for day in range(number_of_slices)] From 36a14890ae57f78c667430c1d0af20dde40a33f2 Mon Sep 17 00:00:00 2001 From: octavia-squidington-iii Date: Mon, 14 Apr 2025 20:20:23 +0000 Subject: [PATCH 2/2] Auto-fix lint and format issues --- .../sources/declarative/retrievers/simple_retriever.py | 4 +++- .../sources/declarative/retrievers/test_simple_retriever.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index cb4684ba5..a739a628a 100644 --- a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -363,7 +363,9 @@ def _read_pages( ) -> Iterable[Record]: pagination_complete = False initial_token = self._paginator.get_initial_token() - next_page_token: Optional[Mapping[str, Any]] = {"next_page_token": initial_token} if initial_token is not None else None + next_page_token: Optional[Mapping[str, Any]] = ( + {"next_page_token": initial_token} if initial_token is not None else None + ) while not pagination_complete: response = self._fetch_next_page(stream_state, stream_slice, next_page_token) diff --git a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py index d862f4c39..0425b4e84 100644 --- a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py +++ b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py @@ -841,7 +841,9 @@ def test_given_initial_token_is_zero_when_read_records_then_pass_initial_token() return_value=response, ) as fetch_next_page_mock: list(retriever.read_records(stream_slice=stream_slice, records_schema={})) - fetch_next_page_mock.assert_called_once_with(cursor.get_stream_state(), stream_slice, {"next_page_token": 0}) + fetch_next_page_mock.assert_called_once_with( + cursor.get_stream_state(), stream_slice, {"next_page_token": 0} + ) def _generate_slices(number_of_slices):