Merge pull request #20 from AASHE/chunk-queries

Chunk queries & subscriptions

Author: jamstooks

README.md

@@ -34,11 +34,36 @@ This can now be used to make additional calls using the methods included in
 the WSDL from MemberSuite. For documentation on available methods and their
 usage, see http://api.docs.membersuite.com/
 
-Use request_session() as a model for constructing the headers for 
+Use request_session() as a model for constructing the headers for
 your own functions in your app that follow this method:
 
     1) Call client.construct_concierge_header(url) to generate a new header element, using your method's URL as an argument.
     2) Call client.service.method_name(_soapheaders=[concierge_request_header], method arguments)
     3) Return any relevant data out of the response object
 
 ***IMPORTANT NOTE: In constructing headers, SessionId must appear first.***
+
+## Running tests
+
+To run all tests:
+
+    $ nosetests
+
+To run specific tests, load them by module. For example:
+
+    $ python -m membersuite_api_client.tests.test_subscriptions
+
+## Contributing and Extending
+
+Looking to contribute? The best place to start is in the code base. Notice how
+we created modules for each MemberSuite objects, like `organizations`.
+
+Each module contains `services.py` and `models.py` files.
+
+Your "models" are simply python representations of the MemberSuite objects.
+
+Your "services" provide interfaces to those models in MemberSuite. This is
+where objects are retrieved from MemberSuite and converted to your models for
+use in a python app. It is recommended that the services be classes and if
+you define `result_to_models` and `ms_object_to_model` methods on the class
+you can use the ChunkQueryMixin to make large queries.

membersuite_api_client/mixins.py

@@ -0,0 +1,55 @@
+from retrying import retry
+
+
+class ChunkQueryMixin():
+    """
+    A mixin for API client service classes that makes it easy to consistently
+    request multiple queries from a MemberSuite endpoint.
+
+    Membersuite will often time out on big queries, so this allows us to
+    break it up into smaller requests.
+
+    Design assumptions:
+        - The service defines an `result_to_models` method to "transform"
+        the objects returned by the endpoint
+    """
+
+    def get_long_query(
+            self, base_query, retry_attempts=2, limit_to=200, max_calls=None):
+        """
+        Takes a base query for all objects and recursively requests them
+
+        @base_query - the base query to be executed
+        @retry_attempts - the number of times to retry a query when it fails
+        @limit_to - how many rows to query for in each chunk
+        @max_recursion_depth - None is infinite
+        """
+
+        @retry(stop_max_attempt_number=retry_attempts)
+        def run_query(base_query, start_record, limit_to):
+            # inline method to take advantage of retry
+            result = self.client.runSQL(
+                query=base_query,
+                start_record=start_record,
+                limit_to=limit_to,
+            )
+            return self.result_to_models(result)
+
+        record_index = 0
+        result = run_query(base_query, record_index, limit_to)
+        all_objects = result
+        call_count = 1
+        """
+        continue to run queries as long as we
+            - don't excede the call call_count
+            - don't see results that are less than the limited length (the end)
+        """
+        while (
+                call_count != max_calls and
+                len(result) >= limit_to):
+
+            record_index += len(result)  # should be `limit_to`
+            all_objects += run_query(base_query, record_index, limit_to)
+            call_count += 1
+
+        return all_objects

membersuite_api_client/subscriptions/models.py

@@ -7,9 +7,9 @@
 
 class Subscription(object):
 
-    def __init__(self, id, org, start, end, extra_data={}):
+    def __init__(self, id, org_id, start, end, extra_data={}):
         self.id = id
-        self.org = org
+        self.org_id = org_id
         self.start = start
         self.end = end
         self.extra_data = extra_data  # all other fields, for reference

membersuite_api_client/subscriptions/services.py

@@ -4,62 +4,84 @@
     http://api.docs.membersuite.com/#References/Objects/Subscription.htm
 
     @todo
-        let's define an Organization object, like this Subscription,
-        for this interface
-    @todo
-        confirm owner field is actually the orgnization
-    @todo
-        set up fixtures in MemberSuite for integration testing
+        set up fixtures in MemberSuite sandbox for integration testing
     @todo
         add date modified param for performance
-    @todo
-        additional method for getting all subscriptions for syncing purposes
 """
 
 from .models import Subscription
+from ..exceptions import ExecuteMSQLError
+from ..mixins import ChunkQueryMixin
 from ..utils import convert_ms_object
 
+import datetime
 
-class SubscriptionService(object):
+
+class SubscriptionService(ChunkQueryMixin, object):
 
     def __init__(self, client):
         """
         Accepts a ConciergeClient to connect with MemberSuite
         """
         self.client = client
 
-    def get_org_subscriptions(self, org_id, publication_id=None):
+    def get_subscriptions(
+            self, publication_id=None, org_id=None, since_when=None,
+            retry_attempts=2, limit_to=200, max_calls=None):
         """
-        Get all the subscriptions for a given organization
-
-        Returns a list of subscription objects
+        Fetches all subscriptions from Membersuite of a particular
+        `publication_id` if set.
         """
         query = "SELECT Objects() FROM Subscription"
-        query += " WHERE owner = '%s'" % org_id
-
+        if org_id:
+            query += " WHERE owner = '%s'" % org_id
         if publication_id:
-            query += "AND publication = '%s'" % publication_id
+            query += " AND publication = '%s'" % publication_id
+        if since_when:
+            query += " AND LastModifiedDate > '{since_when} 00:00:00'" \
+                .format(since_when=datetime.date.today() -
+                        datetime.timedelta(days=since_when))
 
-        result = self.client.runSQL(query)
+        # note, get_long_query is overkill when just looking at
+        # one org, but it still only executes once
+        # `get_long_query` uses `result_to_models` to return Subscriptions
+        subscription_list = self.get_long_query(
+            query, retry_attempts=retry_attempts, limit_to=limit_to,
+            max_calls=max_calls)
+
+        return subscription_list
+
+    def result_to_models(self, result):
+        """
+        this is the 'transorm' part of ETL:
+            converts the result of the SQL to Subscription objects
+        """
         mysql_result = result['body']['ExecuteMSQLResult']
 
         if not mysql_result['Errors']:
             obj_result = mysql_result['ResultValue']['ObjectSearchResult']
+            if not obj_result['Objects']:
+                return []
             objects = obj_result['Objects']['MemberSuiteObject']
 
             subscription_list = []
             for obj in objects:
-                sane_obj = convert_ms_object(
-                    obj['Fields']['KeyValueOfstringanyType'])
-                subscription = Subscription(
-                    id=sane_obj['ID'],
-                    org=org_id,
-                    start=sane_obj['StartDate'],
-                    end=sane_obj['TerminationDate'],
-                    extra_data=sane_obj)
+                subscription = self.ms_object_to_model(obj)
                 subscription_list.append(subscription)
 
             return subscription_list
 
         else:
-            return None
+            raise ExecuteMSQLError(result)
+
+    def ms_object_to_model(self, ms_obj):
+        " Converts an individual result to a Subscription Model "
+        sane_obj = convert_ms_object(
+            ms_obj['Fields']['KeyValueOfstringanyType'])
+        subscription = Subscription(
+            id=sane_obj['ID'],
+            org_id=sane_obj['Owner'],
+            start=sane_obj['StartDate'],
+            end=sane_obj['TerminationDate'],
+            extra_data=sane_obj)
+        return subscription

membersuite_api_client/tests/test_subscriptions.py

@@ -15,15 +15,28 @@ def test_get_org_subscriptions(self):
         Get the all subscriptions for an organization
         """
 
-        test_org_id = "6faf90e4-0007-c91c-7dc8-0b3c53985743"
-        subscription_list = self.service.get_org_subscriptions(test_org_id)
-        self.assertNotEqual(subscription_list, None)
+        test_org_id = "6faf90e4-0007-cbaa-6232-0b3c7fa70db7"
+        subscription_list = self.service.get_subscriptions(org_id=test_org_id)
+        self.assertGreaterEqual(len(subscription_list), 2)
 
         # with publication_id
         STARS_PUBLICATION_ID = '6faf90e4-009e-cb9b-7c9e-0b3bcd6dff6a'
-        subscription_list = self.service.get_org_subscriptions(
-            test_org_id, publication_id=STARS_PUBLICATION_ID)
-        self.assertNotEqual(subscription_list, None)
+        subscription_list = self.service.get_subscriptions(
+            org_id=test_org_id, publication_id=STARS_PUBLICATION_ID)
+        self.assertGreaterEqual(len(subscription_list), 2)
+
+        # now for a "long query" - querying ALL subscriptions
+        subscription_list = self.service.get_subscriptions(
+            retry_attempts=2, limit_to=3, max_calls=3)
+        self.assertEqual(len(subscription_list), 9)
+
+        # test a long query that's longer than the number we have
+        # to ensure that edge case stops the queries
+        subscription_list = self.service.get_subscriptions(
+            retry_attempts=5, limit_to=200, max_calls=6)
+        self.assertLess(len(subscription_list), 1500)
+
+        # @todo: test the modified date
 
 
 if __name__ == '__main__':

requirements.txt

@@ -1,3 +1,4 @@
 future==0.16.0
 lxml==3.7.0
+retrying>=1.3.3
 zeep==0.23.0

setup.py

@@ -27,5 +27,6 @@ def read(fname):
       ],
       include_package_data=True,
       install_requires=["future==0.16.0",
+                        "retrying>=1.3.3",
                         "zeep>=0.26"]
       )