fix(pagination): support for pagination with logging enabled (#99)

This commit fixes the api logging configuration by writing a `__deep_copy__` method in the classes to handle the logger cloning as logger can't be pickled. Also, this commit narrows down the pagination type to be applied for subsequent calls after the 1 call.

Author: sufyankhanrao

apimatic_core/logger/configuration/api_logging_configuration.py

@@ -5,7 +5,7 @@
 from apimatic_core.constants.logger_constants import LoggerConstants
 from apimatic_core.logger.default_logger import ConsoleLogger
 from apimatic_core.utilities.api_helper import ApiHelper
-
+import copy
 
 class ApiLoggingConfiguration:
 
@@ -47,6 +47,14 @@ def __init__(self, logger, log_level, mask_sensitive_headers,
         self._request_logging_config = request_logging_config
         self._response_logging_config = response_logging_config
 
+    def __deepcopy__(self, memo={}):
+        return ApiLoggingConfiguration(
+            logger=self._logger,
+            log_level=self._log_level,
+            mask_sensitive_headers=self._mask_sensitive_headers,
+            request_logging_config=copy.deepcopy(self._request_logging_config),
+            response_logging_config=copy.deepcopy(self._response_logging_config)
+        )
 
 class BaseHttpLoggingConfiguration:
 
@@ -178,6 +186,14 @@ def _filter_excluded_headers(self, headers):
                 extracted_headers[key] = value
         return extracted_headers
 
+    def __deepcopy__(self, memo={}):
+        return self.__class__(
+            log_body=self._log_body,
+            log_headers=self._log_headers,
+            headers_to_include=copy.deepcopy(self._headers_to_include),
+            headers_to_exclude=copy.deepcopy(self._headers_to_exclude),
+            headers_to_unmask=copy.deepcopy(self._headers_to_unmask)
+        )
 
 class ApiRequestLoggingConfiguration(BaseHttpLoggingConfiguration):
 
@@ -221,6 +237,15 @@ def get_loggable_url(self, query_url):
 
         return ApiHelper.get_url_without_query(query_url)
 
+    def __deepcopy__(self, memo={}):
+        return ApiRequestLoggingConfiguration(
+            log_body=self._log_body,
+            log_headers=self._log_headers,
+            headers_to_include=copy.deepcopy(self._headers_to_include),
+            headers_to_exclude=copy.deepcopy(self._headers_to_exclude),
+            headers_to_unmask=copy.deepcopy(self._headers_to_unmask),
+            include_query_in_path=self._include_query_in_path
+        )
 
 class ApiResponseLoggingConfiguration(BaseHttpLoggingConfiguration):
 

apimatic_core/pagination/paginated_data.py

@@ -51,6 +51,7 @@ def __init__(self, api_call, paginated_items_converter):
         self._paginated_items_converter = paginated_items_converter
         self._initial_request_builder = api_call.request_builder
         self._last_request_builder = None
+        self._locked_strategy = None
         self._pagination_strategies = self._api_call.get_pagination_strategies
         self._http_call_context =\
             self._api_call.global_configuration.get_http_client_configuration().http_callback or HttpCallContext()
@@ -123,20 +124,39 @@ def _fetch_next_page(self):
         Raises:
             Exception: Propagates any exceptions encountered during the API call execution.
         """
+
+        if self._locked_strategy is not None:
+            return self._execute_strategy(self._locked_strategy)
+
         for pagination_strategy in self._pagination_strategies:
-            request_builder = pagination_strategy.apply(self)
-            if request_builder is None:
+            response = self._execute_strategy(pagination_strategy)
+            if response is None:
                 continue
 
-            self._last_request_builder = request_builder
+            if self._locked_strategy is None:
+                self._locked_strategy = self._get_locked_strategy()
 
-            response = self._api_call.clone(
-                global_configuration=self._global_configuration, request_builder=request_builder
-            ).execute()
-            return pagination_strategy.apply_metadata_wrapper(response)
+            return response
 
         return None
 
+    def _execute_strategy(self, pagination_strategy):
+        request_builder = pagination_strategy.apply(self)
+        if request_builder is None:
+            return None
+
+        self._last_request_builder = request_builder
+
+        response = self._api_call.clone(
+            global_configuration=self._global_configuration, request_builder=request_builder
+        ).execute()
+        return pagination_strategy.apply_metadata_wrapper(response)
+
+    def _get_locked_strategy(self):
+        for pagination_strategy in self._pagination_strategies:
+            if pagination_strategy.is_applicable(self.last_response):
+                return pagination_strategy
+
     def clone(self):
         """
         Creates and returns a new independent PaginatedData instance

apimatic_core/pagination/pagination_strategy.py

@@ -32,6 +32,19 @@ def __init__(self, metadata_wrapper):
 
         self._metadata_wrapper = metadata_wrapper
 
+    @abstractmethod
+    def is_applicable(self, response):
+        """
+        Checks whether the pagination strategy is a valid candidate based on the given HTTP response.
+
+        Args:
+            response: The response from the previous API call.
+
+        Returns:
+            bool: True if this strategy is valid based on the given HTTP response..
+        """
+        ...
+
     @abstractmethod
     def apply(self, paginated_data):
         """

apimatic_core/pagination/strategies/cursor_pagination.py

@@ -37,6 +37,27 @@ def __init__(self, output, input_, metadata_wrapper):
         self._input = input_
         self._cursor_value = None
 
+    def is_applicable(self, response):
+        """
+        Checks whether the cursor pagination strategy is a valid candidate based on the given HTTP response.
+
+        Args:
+            response: The response from the previous API call.
+
+        Returns:
+            bool: True if this strategy is valid based on the given HTTP response..
+        """
+        if response is None:
+            return True
+
+        self._cursor_value = ApiHelper.resolve_response_pointer(
+            self._output,
+            response.text,
+            response.headers
+        )
+
+        return self._cursor_value is not None
+
     def apply(self, paginated_data):
         """
         Advances the pagination by updating the request builder with the next cursor value.

apimatic_core/pagination/strategies/link_pagination.py

@@ -30,6 +30,27 @@ def __init__(self, next_link_pointer, metadata_wrapper):
         self._next_link_pointer = next_link_pointer
         self._next_link = None
 
+    def is_applicable(self, response):
+        """
+        Checks whether the link pagination strategy is a valid candidate based on the given HTTP response.
+
+        Args:
+            response: The response from the previous API call.
+
+        Returns:
+            bool: True if this strategy is valid based on the given HTTP response..
+        """
+        if response is None:
+            return True
+
+        self._next_link = ApiHelper.resolve_response_pointer(
+            self._next_link_pointer,
+            response.text,
+            response.headers
+        )
+
+        return self._next_link is not None
+
     def apply(self, paginated_data):
         """
         Updates the request builder with query parameters from the next page

apimatic_core/pagination/strategies/offset_pagination.py

@@ -31,6 +31,18 @@ def __init__(self, input_, metadata_wrapper):
         self._input = input_
         self._offset = 0
 
+    def is_applicable(self, response):
+        """
+        Checks whether the offset pagination strategy is a valid candidate based on the given HTTP response.
+
+        Args:
+            response: The response from the previous API call.
+
+        Returns:
+            bool: True if this strategy is valid based on the given HTTP response..
+        """
+        return True
+
     def apply(self, paginated_data):
         """
         Updates the request builder to fetch the next page of results using offset-based pagination.

apimatic_core/pagination/strategies/page_pagination.py

@@ -29,6 +29,18 @@ def __init__(self, input_, metadata_wrapper):
         self._input = input_
         self._page_number = 1
 
+    def is_applicable(self, response):
+        """
+        Checks whether the offset pagination strategy is a valid candidate based on the given HTTP response.
+
+        Args:
+            response: The response from the previous API call.
+
+        Returns:
+            bool: True if this strategy is valid based on the given HTTP response..
+        """
+        return True
+
     def apply(self, paginated_data):
         """
         Updates the request builder to fetch the next page of results based on the current paginated data.

setup.py

@@ -12,7 +12,7 @@
 
 setup(
     name='apimatic-core',
-    version='0.2.20',
+    version='0.2.21',
     description='A library that contains core logic and utilities for '
                 'consuming REST APIs using Python SDKs generated by APIMatic.',
     long_description=long_description,

tests/apimatic_core/pagination_tests/strategies/test_cursor_pagination.py

@@ -174,6 +174,16 @@ def test_apply_metadata_wrapper(self, mock_metadata_wrapper, mocker):
         mock_metadata_wrapper.assert_called_once_with(mock_paged_response, "some_cursor_value")
         assert result == "wrapped_response"
 
+
+    def test_is_applicable_for_none_response(self, mock_metadata_wrapper):
+        cp = CursorPagination(
+            input_="$request.query#/cursor",
+            output="$response.body#/next_cursor",
+            metadata_wrapper=mock_metadata_wrapper
+        )
+        result = cp.is_applicable(None)
+        assert result == True
+
     # Test _get_initial_cursor_value
     def test_get_initial_cursor_value_path(self, mocker, mock_request_builder):
         mock_split_into_parts = mocker.patch.object(

tests/apimatic_core/pagination_tests/strategies/test_link_pagination.py

@@ -153,3 +153,11 @@ def test_apply_metadata_wrapper(self, mock_metadata_wrapper, mocker):
 
         mock_metadata_wrapper.assert_called_once_with(mock_paged_response, "https://api.example.com/data?page=2")
         assert result == "wrapped_response_with_link"
+
+    def test_is_applicable_for_none_response(self, mock_metadata_wrapper):
+        lp = LinkPagination(
+            next_link_pointer="$response.body#/next_cursor",
+            metadata_wrapper=mock_metadata_wrapper
+        )
+        result = lp.is_applicable(None)
+        assert result == True