Merge pull request #4 from metaodi/data-loader

metaodi · web-flow · commit 3a880900e9b7 · 2021-09-16T00:19:51.000+02:00
Add pagination support
diff --git a/examples/download_attachent.py b/examples/download_attachent.py
@@ -2,22 +2,31 @@
 import museumpy
 from dotenv import load_dotenv, find_dotenv
 import os
+from pprint import pprint
+import tempfile
 
 load_dotenv(find_dotenv())
 user = os.getenv('MP_USER')
 pw = os.getenv('MP_PASS')
 
-client = museumpy.client(
+client = museumpy.MuseumPlusClient(
     base_url='https://mpzurichrietberg.zetcom.com/MpWeb-mpZurichRietberg',
     requests_kwargs={'auth': (user, pw)},
 )
 
-group_result = client.search('ObjObjectGroupTxt', 'MyGroup')
+group_result = client.search(
+    field='OgrNameTxt',
+    value='Patolu, MAP',
+    module='ObjectGroup'
+)
 group = group_result[0]['raw']
-ref = group['moduleIm']['moduleReference']
+ref = group['moduleItem']['moduleReference']
+
 
-for ref_item in ref['moduleReferenceItem']:
+for ref_item in ref['moduleReferenceItem'][:5]:
     item = client.module_item(ref_item['moduleItemId'], ref['targetModule'])
+    pprint(item, depth=1)
     if item['hasAttachments'] == 'true':
-        attachment_path = client.download_attachment(ref_item['moduleItemId'], ref['targetModule'], 'files')
-        print(f"Attachment downloaded and saved at {attachment_path}")
+        with tempfile.TemporaryDirectory() as tmpdir:
+            attachment_path = client.download_attachment(ref_item['moduleItemId'], ref['targetModule'], tmpdir)
+            print(f"Attachment downloaded and saved at {attachment_path}")
diff --git a/examples/pagination.py b/examples/pagination.py
@@ -0,0 +1,25 @@
+import museumpy
+from dotenv import load_dotenv, find_dotenv
+from pprint import pprint
+import os
+
+load_dotenv(find_dotenv())
+user = os.getenv('MP_USER')
+pw = os.getenv('MP_PASS')
+
+
+client = museumpy.MuseumPlusClient(
+    base_url='https://mpzurichrietberg.zetcom.com/MpWeb-mpZurichRietberg',
+    requests_kwargs={'auth': (user, pw)}
+)
+
+result = client.fulltext_search(
+    query='Patolu',
+    limit=2
+)
+
+print(result)
+print(result.count)
+for rec in result[:5]:
+    pprint(rec, depth=1)
+print(result)
diff --git a/examples/simple_search.py b/examples/simple_search.py
@@ -15,5 +15,5 @@
 )
 
 pprint(records)
-print(len(records))
+print(records.count)
 pprint(records[0], depth=1)
diff --git a/museumpy/__init__.py b/museumpy/__init__.py
@@ -1,7 +1,7 @@
 __version__ = '0.0.3'
 __all__ = ['client', 'errors', 'response', 'xmlparse']
 
-from .errors import MuseumPlusError  # noqa
+from .errors import MuseumpyError  # noqa
 from .client import MuseumPlusClient
 
 def fulltext_search(base_url, query, **kwargs):  # noqa
diff --git a/museumpy/client.py b/museumpy/client.py
@@ -45,61 +45,61 @@ def __init__(self, base_url=None, map_function=None, requests_kwargs=None):
 
     def fulltext_search(self, query, module='Object', limit=100, offset=0):
         url = f"{self.base_url}/ria-ws/application/module/{module}/search"
-        data = FULLTEXT_TEMPLATE.format(
-            module_name=module,
-            limit=limit,
-            offset=offset,
-            query=query,
-        )
-        xml = data.encode("utf-8")
-        xml_response = self._post_xml(url, xml)
-        return response.SearchResponse(xml_response, self.map_function)
+        params = {
+            'module_name': module,
+            'query': query,
+        }
+        data_loader = DataPoster(url, params, FULLTEXT_TEMPLATE, self.requests_kwargs)
+        return response.SearchResponse(data_loader, limit, offset, self.map_function)
 
     def search(self, field, value, module='Object', limit=100, offset=0):
         url = f"{self.base_url}/ria-ws/application/module/{module}/search"
-        data = SEARCH_TEMPLATE.format(
-            module_name=module,
-            limit=limit,
-            offset=offset,
-            field=field,
-            value=value,
-        )
-        xml = data.encode("utf-8")
-        xml_response = self._post_xml(url, xml)
-        return response.SearchResponse(xml_response, self.map_function)
+        params = {
+            'module_name': module,
+            'field': field,
+            'value': value,
+            }
+        data_loader = DataPoster(url, params, SEARCH_TEMPLATE, self.requests_kwargs)
+        return response.SearchResponse(data_loader, limit, offset, self.map_function)
 
     def module_item(self, id, module='Object'):
         url = f"{self.base_url}/ria-ws/application/module/{module}/{id}"
-        xml_response = self._get_xml(url)
-        resp = response.SearchResponse(xml_response)
-        if len(resp) == 1:
+        data_loader = DataLoader(url, self.requests_kwargs)
+        resp = response.SearchResponse(data_loader)
+        if resp.count == 1:
             return resp[0]
         return resp
 
     def download_attachment(self, id, module='Object', dir='.'):
         url = f"{self.base_url}/ria-ws/application/module/{module}/{id}/attachment"
-        return self._download_file(url, dir)
+        data_loader = DataLoader(url, self.requests_kwargs)
+        return data_loader.download_file(url, dir)
 
-    def _download_file(self, url, dir):
-        headers = {'Accept': 'application/octet-stream'}
-        res = self._get_content(url, headers)
-        d = res.headers.get('Content-Disposition')
-        fname = re.findall("filename=(.+)", d)[0]
-        assert fname, "Could not find filename in Content-Disposition header"
-        path = os.path.join(dir, fname)
-        with open(path, 'wb') as f:
-            for chunk in res.iter_content(1024):
-                f.write(chunk)
-        return path
 
-    def _get_xml(self, url):
-        res = self._get_content(url)
+class DataPoster(object):
+    def __init__(self, url, params=None, template=None, requests_kwargs=None):
+        self.session = requests.Session()
+        self.url = url
+        self.params = params
+        self.template = template
+        self.xmlparser = xmlparse.XMLParser()
+        self.requests_kwargs = requests_kwargs or {}
+
+    def load(self, **kwargs):
+        self.params.update(kwargs)
+        xml = self.template.format(**self.params).encode('utf-8')
+        return self._post_xml(self.url, xml)
+
+    def _post_xml(self, url, xml):
+        headers = {'Content-Type': 'application/xml'}
+        res = self._post_content(url, xml, headers)
         return self.xmlparser.parse(res.content)
 
-    def _get_content(self, url, headers={}):
+    def _post_content(self, url, data, headers):
         try:
-            res = self.session.get(
+            res = self.session.post(
                 url,
+                data=data,
                 headers=headers,
                 **self.requests_kwargs
             )
@@ -111,16 +111,38 @@ def _get_content(self, url, headers={}):
 
         return res
 
-    def _post_xml(self, url, xml):
-        headers = {'Content-Type': 'application/xml'}
-        res = self._post_content(url, xml, headers)
+
+class DataLoader(object):
+    def __init__(self, url, requests_kwargs=None):
+        self.session = requests.Session()
+        self.url = url
+        self.xmlparser = xmlparse.XMLParser()
+        self.requests_kwargs = requests_kwargs or {}
+
+    def load(self, **kwargs):
+        xml = self._get_xml(self.url)
+        return xml
+
+    def download_file(self, url, dir):
+        headers = {'Accept': 'application/octet-stream'}
+        res = self._get_content(url, headers)
+        d = res.headers.get('Content-Disposition')
+        fname = re.findall("filename=(.+)", d)[0]
+        assert fname, "Could not find filename in Content-Disposition header"
+        path = os.path.join(dir, fname)
+        with open(path, 'wb') as f:
+            for chunk in res.iter_content(1024):
+                f.write(chunk)
+        return path
+
+    def _get_xml(self, url):
+        res = self._get_content(url)
         return self.xmlparser.parse(res.content)
 
-    def _post_content(self, url, data, headers):
+    def _get_content(self, url, headers={}):
         try:
-            res = self.session.post(
+            res = self.session.get(
                 url,
-                data=data,
                 headers=headers,
                 **self.requests_kwargs
             )
@@ -129,4 +151,5 @@ def _post_content(self, url, data, headers):
             raise errors.MuseumPlusError("HTTP error: %s" % e)
         except requests.exceptions.RequestException as e:
             raise errors.MuseumPlusError("Request error: %s" % e)
+
         return res
diff --git a/museumpy/errors.py b/museumpy/errors.py
@@ -1,10 +1,23 @@
-class MuseumPlusError(Exception):
+class MuseumpyError(Exception):
     """
     General MuseumPlus error class to provide a superclass for all other errors
     """
 
 
-class XMLParsingError(MuseumPlusError):
+class MuseumPlusError(MuseumpyError):
+    """
+    MuseumPlus error raised when an error with the communication with MuseumPlus occurs
+    """
+
+
+class XMLParsingError(MuseumpyError):
     """
     The error raised when parsing the XML.
     """
+
+
+class NoMoreRecordsError(MuseumpyError):
+    """
+    This error is raised if all records have been loaded (or no records are
+    present)
+    """
diff --git a/museumpy/response.py b/museumpy/response.py
@@ -1,24 +1,42 @@
 # -*- coding: utf-8 -*-
 
 from . import xmlparse
+from . import errors
 
 ZETCOM_NS = "http://www.zetcom.com/ria/ws/module"
 
 
 class SearchResponse(object):
-    def __init__(self, xml_response, map_function=None):
+    def __init__(self, data_loader, limit=100, offset=0, map_function=None):
+        self.data_loader = data_loader
         self.xmlparser = xmlparse.XMLParser()
         self.records = []
-        self._extract_records(xml_response, map_function)
+        self.count = 0
+        self.limit = limit
+        self.offset = offset
+        self.map_function = map_function
 
-    def _extract_records(self, xml, map_function):
+        xml = data_loader.load(limit=limit, offset=offset)
+        self._parse_content(xml)
+
+    def _parse_content(self, xml):
+        self.count = self.maybe_int(self.xmlparser.find(xml, f'.//{{{ZETCOM_NS}}}module').attrib['totalSize'])  # noqa
+        self._extract_records(xml)
+
+    def maybe_int(self, s):
+        try:
+            return int(s)
+        except (ValueError, TypeError):
+            return s
+
+    def _extract_records(self, xml):
         new_records = []
         xml_recs = self.xmlparser.findall(xml, f'.//{{{ZETCOM_NS}}}module/{{{ZETCOM_NS}}}moduleItem')  # noqa
         for xml_rec in xml_recs:
             record = self._map_xml(xml_rec)
             record['raw'] = self.xmlparser.todict(xml_rec, xml_attribs=True)
-            if map_function:
-                record = map_function(record, xml_rec)
+            if self.map_function:
+                record = self.map_function(record, xml_rec)
             new_records.append(record)
         self.records.extend(new_records)
 
@@ -93,18 +111,60 @@ def __repr__(self):
         try:
             return (
                 'SearchResponse('
-                'count=%r)'
+                'count=%r,'
+                'limit=%r,'
+                'offset=%r)'
                 ) % (
-                   len(self.records),
+                   self.count,
+                   self.limit,
+                   self.offset
                 )
         except AttributeError:
             return 'SearchResponse(empty)'
 
-    def __len__(self):
-        return len(self.records)
+    def __length_hint__(self):
+        return self.count
 
     def __iter__(self):
-        yield from self.records
+        # use while loop since self.records could grow while iterating
+        i = 0
+        while True:
+            # load new data when near end
+            if i == len(self.records):
+                try:
+                    self._load_new_data()
+                except errors.NoMoreRecordsError:
+                    break
+            yield self.records[i]
+            i += 1
 
     def __getitem__(self, key):
+        if isinstance(key, slice):
+            limit = max(key.start or 0, key.stop or self.count)
+            self._load_new_data_until(limit)
+            count = len(self.records)
+            return [self.records[k] for k in range(*key.indices(count))]
+
+        if not isinstance(key, int):
+            raise TypeError("Index must be an integer or slice")
+
+        limit = key
+        if limit < 0:
+            # if we get a negative index, load all data
+            limit = self.count
+        self._load_new_data_until(limit)
         return self.records[key]
+
+    def _load_new_data_until(self, limit):
+        while limit >= len(self.records):
+            try:
+                self._load_new_data()
+            except errors.NoMoreRecordsError:
+                break
+
+    def _load_new_data(self):
+        self.offset = self.offset + self.limit
+        if self.offset >= self.count:
+            raise errors.NoMoreRecordsError("There are no more records")
+        xml = self.data_loader.load(limit=self.limit, offset=self.offset)
+        self._parse_content(xml)
diff --git a/setup.sh b/setup.sh
@@ -5,4 +5,4 @@ source pyenv/bin/activate
 
 pip install --upgrade pip
 pip install -r requirements.txt
-pip install .
+pip install -e .
diff --git a/tests/client_test.py b/tests/client_test.py
@@ -14,7 +14,8 @@ class TestClient(MuseumpyTestCase):
     def test_simple_search(self):
         client = MuseumPlusClient('http://test.com/MpWeb-test')
         r = client.search(field='TestField', value='TestValue')
-        self.assertEqual(len(r), 1)
+        self.assertEqual(r.__length_hint__(), 1)
+        self.assertEqual(r.count, 1)
 
         self.assertEqual(r[0]['hasAttachments'], 'true')  # noqa
         self.assertEqual(r[0]['ObjCreditlineGrp'], 'Geschenk Gisela Müller und Erich Gross')  # noqa
diff --git a/tests/fixtures/response_multiple_1.xml b/tests/fixtures/response_multiple_1.xml
diff --git a/tests/fixtures/response_multiple_2.xml b/tests/fixtures/response_multiple_2.xml
diff --git a/tests/fixtures/response_multiple_3.xml b/tests/fixtures/response_multiple_3.xml
diff --git a/tests/response_test.py b/tests/response_test.py

Original file line number	Diff line number	Diff line change
`@@ -15,5 +15,5 @@`
`15`	`15`	`)`
`16`	`16`
`17`	`17`	`pprint(records)`
`18`		`-print(len(records))`
	`18`	`+print(records.count)`
`19`	`19`	`pprint(records[0], depth=1)`