enh: add support for handling paginated odata responses (#31)

Author: anna-follestad-4ss

README.rst

@@ -7,4 +7,4 @@
 
 
 .. _4Insight.io: https://4insight.io
-.. _4Insight REST API: https://4insight.io/#/developer
+.. _4Insight REST API: https://4insight-api-prod.4subsea.net/swagger/index.html

docs/index.rst

@@ -6,7 +6,7 @@
 :py:mod:`fourinsight.api` is mainly used by other high-level 4Insight Python packages. However, it can also be used to make low-level calls.
 
 .. _4Insight.io: https://4insight.io
-.. _4Insight REST API: https://4insight.io/#/developer
+.. _4Insight REST API: https://4insight-api-prod.4subsea.net/swagger/index.html
 
 
 .. toctree::

docs/user_guide/api_calls.rst

@@ -1,18 +1,43 @@
 Make API calls
 ==============
 
-Below example demonstrate how to make an API call to `4Insight.io`_
+The example below demonstrates how to make an API call to `4Insight.io`_
 For the complete overview of available API calls see `4Insight REST API`_. 
 
 
 .. code-block:: python
 
-    response = session.get("https://api.4insight.io/v1.0/Campaigns")
+    response = session.get("https://api.4insight.io/v1.1/Campaigns")
 
     # or with relative url
 
-    response = session.get("/v1.0/Campaigns")
+    response = session.get("/v1.1/Campaigns")
 
 
-.. _4Insight REST API: https://4insight.io/#/developer
+Some API endpoints support OData and have paging, which returns a limited number of responses per page. 
+For these endpoints, the ``get_pages`` method can be useful:
+
+.. code-block:: python
+
+    response = session.get_pages("https://api.4insight.io/v1.1/Campaigns")
+    
+    # Iterate through response pages:
+    for page in response:
+        print(page)
+    
+    # Alternatively, get just the next page:
+    page = next(response)
+
+
+``get_pages`` returns a generator object, from which the user can iterate through the page, or obtain the pages one by one by calling ``next(response)``. 
+
+
+
+
+
+
+
+
+
+.. _4Insight REST API: https://4insight-api-prod.4subsea.net/swagger/index.html
 .. _4Insight.io: https://4insight.io

docs/user_guide/authentication.rst

@@ -62,4 +62,4 @@ Example:
 
 
 .. _4Insight.io: https://4insight.io
-.. _4Insight REST API: https://4insight.io/#/developer
+.. _4Insight REST API: https://4insight-api-prod.4subsea.net/swagger/index.html

fourinsight/api/__init__.py

@@ -4,4 +4,4 @@
 
 logging.getLogger(__name__).addHandler(logging.NullHandler())
 
-__version__ = "0.0.1"
+__version__ = "0.0.3"

fourinsight/api/authenticate.py

@@ -15,6 +15,7 @@
     WebApplicationClient,
 )
 from requests.adapters import HTTPAdapter
+from requests.exceptions import HTTPError
 from requests.packages.urllib3 import Retry
 from requests_oauthlib import OAuth2Session
 
@@ -85,7 +86,7 @@ def token(self):
 
 
 class BaseAuthSession(OAuth2Session, metaclass=ABCMeta):
-    """
+    r"""
     Abstract class for authorized sessions.
 
     Parameters
@@ -101,7 +102,7 @@ class BaseAuthSession(OAuth2Session, metaclass=ABCMeta):
         The provided dict is stored internally in '_session_params'.
     auth_force : bool, optional
         Force re-authenticating the session (default is False)
-    kwargs : keyword arguments
+    **kwargs : keyword arguments
         Keyword arguments passed on to ``requests_oauthlib.OAuth2Session``.
         Here, the mandatory parameters for the authentication client shall be
         provided.
@@ -210,6 +211,38 @@ def _update_args_kwargs(self, args, kwargs):
             kwargs.setdefault(key, self._defaults[key])
         return (method, url, *args[2:]), kwargs
 
+    def get_pages(self, url, **kwargs):
+        r"""
+        Sends GET requests, and returns a generator that lets the user iterate over
+        paginated responses. Note that the endpoint must support OData; the json
+        response should include the a parameter '@odata.nextLink', providing the
+        URL for the next page.
+
+        Parameters
+        ----------
+        url : str
+            API endpoint. To return pages, the endpoint must support OData and contain
+            the parameter '@odata.nextLink'.
+        **kwargs :
+            Optional keyword arguments. Will be passed on to the ``get`` method.
+
+        Yields
+        ------
+        response : obj
+            The response as a :class:`Response` object.
+        """
+
+        while url:
+            response = self.get(url, **kwargs)
+
+            try:
+                response.raise_for_status()
+            except HTTPError:
+                url = None
+            else:
+                url = response.json().get("@odata.nextLink")
+            yield response
+
 
 class UserSession(BaseAuthSession):
     """

tests/test_authenticator.py

@@ -1,9 +1,11 @@
+import inspect
 import json
 import logging
 from io import StringIO
-from unittest.mock import MagicMock, patch
+from unittest.mock import MagicMock, Mock, call, patch
 
 import pytest
+from requests.exceptions import HTTPError
 
 import fourinsight.api as fapi
 from fourinsight.api import authenticate
@@ -523,6 +525,195 @@ def test_request_logger(self, mock_request, mock_fetch, mock_refresh):
         log_out = stream.read()
         assert log_out.startswith("request initiated")
 
+    def test_get_pages(self, mock_fetch, mock_refresh):
+
+        JSON_DATA = [
+            {
+                "value": {"a": 0, "b": "foo", "c": "bar"},
+                "@odata.nextLink": "first/nextlink",
+            },
+            {
+                "value": [
+                    {"a": 123, "b": "baz", "c": "bar"},
+                    {"a": 456, "b": "foo", "c": "bar"},
+                ],
+                "@odata.nextLink": "second/nextlink",
+            },
+            {
+                "value": None,
+                "@odata.nextLink": "third/nextlink",
+            },
+            {
+                "value": {"a": 0, "b": "foo", "c": "bar"},
+            },
+        ]
+
+        mock_response = Mock()
+        mock_response._n_calls = 0
+        mock_response.json.side_effect = lambda *args, **kwargs: JSON_DATA[
+            mock_response._n_calls - 1
+        ]
+
+        def get_side_effect(*args, **kwargs):
+            mock_response._n_calls += 1
+            return mock_response
+
+        with patch.object(
+            authenticate.ClientSession, "get", side_effect=get_side_effect
+        ) as mock_get:
+            session = authenticate.ClientSession("my_client_id", "my_client_secret")
+
+            pages = session.get_pages("foo/bar/baz", baz="foobar")
+
+            for i, page_i in enumerate(pages):
+                assert page_i.json() == JSON_DATA[i]
+                assert page_i is mock_response
+
+            mock_get.assert_has_calls(
+                [
+                    call("foo/bar/baz", baz="foobar"),
+                    call("first/nextlink", baz="foobar"),
+                    call("second/nextlink", baz="foobar"),
+                    call("third/nextlink", baz="foobar"),
+                ]
+            )
+
+    def test_get_pages_last_nextlink_none(self, mock_fetch, mock_refresh):
+
+        JSON_DATA = [
+            {
+                "value": {"a": 0, "b": "foo", "c": "bar"},
+                "@odata.nextLink": "first/nextlink",
+            },
+            {
+                "value": [
+                    {"a": 123, "b": "baz", "c": "bar"},
+                    {"a": 456, "b": "foo", "c": "bar"},
+                ],
+                "@odata.nextLink": "second/nextlink",
+            },
+            {
+                "value": None,
+                "@odata.nextLink": "third/nextlink",
+            },
+            {
+                "value": {"a": 0, "b": "foo", "c": "bar"},
+                "@odata.nextLink": None,
+            },
+        ]
+
+        mock_response = Mock()
+        mock_response._n_calls = 0
+        mock_response.json.side_effect = lambda *args, **kwargs: JSON_DATA[
+            mock_response._n_calls - 1
+        ]
+
+        def get_side_effect(*args, **kwargs):
+            mock_response._n_calls += 1
+            return mock_response
+
+        with patch.object(
+            authenticate.ClientSession, "get", side_effect=get_side_effect
+        ) as mock_get:
+            session = authenticate.ClientSession("my_client_id", "my_client_secret")
+
+            pages = session.get_pages("foo/bar/baz", baz="foobar")
+
+            for i, page_i in enumerate(pages):
+                assert page_i.json() == JSON_DATA[i]
+                assert page_i is mock_response
+
+            mock_get.assert_has_calls(
+                [
+                    call("foo/bar/baz", baz="foobar"),
+                    call("first/nextlink", baz="foobar"),
+                    call("second/nextlink", baz="foobar"),
+                    call("third/nextlink", baz="foobar"),
+                ]
+            )
+
+    def test_get_pages_raises_json_list(self, mock_fetch, mock_refresh):
+
+        JSON_DATA = [
+            {
+                "a": "foo",
+                "b": "bar",
+                "c": "baz",
+            },
+            {
+                "a": "foobar",
+                "b": "baz",
+                "c": "test",
+            },
+        ]
+
+        mock_response = Mock()
+        mock_response.json.return_value = JSON_DATA
+
+        with patch.object(
+            authenticate.ClientSession, "get", return_value=mock_response
+        ):
+            session = authenticate.ClientSession("my_client_id", "my_client_secret")
+
+            pages = session.get_pages("foo/bar/baz", baz="foobar")
+
+            with pytest.raises(AttributeError):
+                next(pages)
+
+    def test_get_pages_raise_for_status(self, mock_fetch, mock_refresh):
+
+        JSON_DATA = [
+            {
+                "value": {"a": 0, "b": "foo", "c": "bar"},
+                "@odata.nextLink": "first/nextlink",
+            },
+            {
+                "value": [
+                    {"a": 123, "b": "baz", "c": "bar"},
+                    {"a": 456, "b": "foo", "c": "bar"},
+                ],
+                "@odata.nextLink": "second/nextlink",
+            },
+            {
+                "value": None,
+                "@odata.nextLink": "third/nextlink",
+            },
+            {
+                "value": {"a": 0, "b": "foo", "c": "bar"},
+                "@odata.nextLink": None,
+            },
+        ]
+
+        mock_response = Mock()
+        mock_response._n_calls = 0
+        mock_response.json.side_effect = lambda *args, **kwargs: JSON_DATA[
+            mock_response._n_calls - 1
+        ]
+
+        def raise_httperror(*args, **kwargs):
+            raise HTTPError()
+
+        mock_response.raise_for_status.side_effect = raise_httperror
+
+        def get_side_effect(*args, **kwargs):
+            mock_response._n_calls += 1
+            return mock_response
+
+        with patch.object(
+            authenticate.ClientSession, "get", side_effect=get_side_effect
+        ) as mock_get:
+            session = authenticate.ClientSession("my_client_id", "my_client_secret")
+
+            pages = session.get_pages("foo/bar/baz", baz="foobar")
+
+            for i, page_i in enumerate(pages):
+                assert page_i.json() == JSON_DATA[i]
+                assert page_i is mock_response
+
+            mock_get.assert_called_once_with("foo/bar/baz", baz="foobar")
+
+            mock_response.raise_for_status.assert_called_once()
+
 
 if __name__ == "__main__":
     pytest.main()