Merge branch 'feature/pagination' of github.com:jcarbaugh/python-sunlight into jcarbaugh-feature/pagination

Author: dcloud

examples/paging

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+from __future__ import print_function
+
+import logging
+
+from sunlight import congress, capitolwords
+from sunlight.pagination import PagingService, logger
+
+
+logger.setLevel(logging.DEBUG)
+
+# try to create a paging service with an unpageable one
+try:
+    capitolwords = PagingService(capitolwords)
+except ValueError, ve:
+    print('ValueError: %s' % ve.message)
+
+# create a pageable service
+congress = PagingService(congress)
+
+print(len(list(congress.legislators(limit=1000)))) # page more than known results
+print(len(list(congress.legislators(limit=5))))    # page less than a single page
+print(len(list(congress.legislators(limit=55))))   # page more than a single page
+
+# bypass unpageable methods
+print(len(congress.all_legislators_in_office()))
+
+# page from an arbitrary page
+print(len(list(congress.legislators(limit=100, page=3))))

sunlight/pagination.py

@@ -0,0 +1,88 @@
+import time
+import logging
+
+logger = logging.getLogger('sunlight.paginator')
+
+
+def pageable(func):
+    func.is_pageable = True
+    return func
+
+
+class PagingService(object):
+
+    limit_attr = 'limit'
+    page_attr = 'page'
+    per_page_attr = 'per_page'
+
+    delay = 0.1
+
+    def __init__(self, service=None, delay=None):
+
+        if service:
+            self.service = service
+        else:
+            self.service = self.service_class()
+
+        if not getattr(self.service, 'is_pageable', False):
+            raise ValueError('service must be a pagable service')
+
+        if delay is not None:
+            self.delay = delay
+
+    def __getattr__(self, name):
+
+        attr = getattr(self.service, name, None)
+
+        if callable(attr) and getattr(attr, 'is_pageable', False):
+
+            def pagingfunc(*args, **kwargs):
+
+                count = 0
+
+                page = int(kwargs.get(self.page_attr, 1))
+                per_page = int(kwargs.get(self.per_page_attr, 50))
+                limit = int(kwargs.pop(self.limit_attr, per_page))
+
+                per_page = min(limit, per_page)
+
+                kwargs[self.per_page_attr] = per_page
+                kwargs[self.page_attr] = 1
+
+                stopthepresses = False
+
+                while 1:
+
+                    logger.debug('loading %s page %d' % (name, page))
+
+                    kwargs[self.page_attr] = page
+                    resp = attr(*args, **kwargs)
+
+                    if not resp:
+                        logger.debug('!   %s returned 0 results this iteration, stopping' % name)
+                        stopthepresses = True
+
+                    for rec in resp:
+
+                        yield rec
+
+                        count += 1
+
+                        if count >= limit:
+                            logger.debug('!   count exceeded limit, stopping')
+                            stopthepresses = True
+                            break
+
+                    if count % per_page != 0:
+                        logger.debug('!   %s returned less than number of requested results, stopping' % name)
+                        stopthepresses = True
+
+                    if stopthepresses:
+                        break
+
+                    page += 1
+                    time.sleep(self.delay)
+
+            return pagingfunc
+
+        return attr

sunlight/service.py

@@ -44,6 +44,9 @@ class Service:
     Base class for all the API implementations, as well as a bunch of common
     code on how to actually fetch text over the network.
     """
+
+    is_pageable = False
+
     @debug_cache
     def get(self, top_level_object, **kwargs):
         """

sunlight/services/congress.py

@@ -9,8 +9,8 @@
 """
 
 import sunlight.service
-from sunlight.service import EntityList
-from sunlight.service import EntityDict
+from sunlight.service import EntityDict, EntityList
+from sunlight.pagination import pageable
 import json
 
 
@@ -51,6 +51,9 @@ class Congress(sunlight.service.Service):
     is the place to look for help on field names and examples.
     """
 
+    is_pageable = True
+
+    @pageable
     def legislators(self, **kwargs):
         """
         Search and filter for members of Congress.
@@ -60,6 +63,7 @@ def legislators(self, **kwargs):
         """
         return self.get('legislators', **kwargs)
 
+    @pageable
     def legislator(self, identifier, id_type=LEGISLATOR_ID_TYPES[0], **kwargs):
         """
         Retrieve a member of Congress by a unique identifier. Defaults to
@@ -91,7 +95,6 @@ def legislator(self, identifier, id_type=LEGISLATOR_ID_TYPES[0], **kwargs):
             return EntityDict(results[0], results._meta)
         return None
 
-
     def all_legislators_in_office(self, **kwargs):
         """
         Returns all legislators currently in office (non-paginated response).
@@ -104,6 +107,7 @@ def all_legislators_in_office(self, **kwargs):
         })
         return self.get('legislators', **kwargs)
 
+    @pageable
     def locate_legislators_by_lat_lon(self, lat, lon, **kwargs):
         """
         Find members of Congress by a latitude and longitude.
@@ -117,6 +121,7 @@ def locate_legislators_by_lat_lon(self, lat, lon, **kwargs):
         })
         return self.get('legislators/locate', **kwargs)
 
+    @pageable
     def locate_legislators_by_zip(self, zipcode, **kwargs):
         """
         Find members of Congress by zip code.
@@ -129,6 +134,7 @@ def locate_legislators_by_zip(self, zipcode, **kwargs):
         })
         return self.get('legislators/locate', **kwargs)
 
+    @pageable
     def bills(self, **kwargs):
         """
         Search and filter through bills in Congress.
@@ -138,6 +144,7 @@ def bills(self, **kwargs):
         """
         return self.get('bills', **kwargs)
 
+    @pageable
     def bill(self, bill_id, **kwargs):
         """
         Retrieve a bill by bill_id.
@@ -153,6 +160,7 @@ def bill(self, bill_id, **kwargs):
             return EntityDict(results[0], results._meta)
         return None
 
+    @pageable
     def search_bills(self, query, **kwargs):
         """
         Search the full text of legislation, and other fields.
@@ -165,6 +173,7 @@ def search_bills(self, query, **kwargs):
         })
         return self.get('bills/search', **kwargs)
 
+    @pageable
     def upcoming_bills(self, **kwargs):
         """
         Search and filter through upcoming bills in the House and Senate.
@@ -177,6 +186,7 @@ def upcoming_bills(self, **kwargs):
         """
         return self.get('upcoming_bills', **kwargs)
 
+    @pageable
     def locate_districts_by_lat_lon(self, lat, lon, **kwargs):
         """
         Find congressional districts by a latitude and longitude.
@@ -190,6 +200,7 @@ def locate_districts_by_lat_lon(self, lat, lon, **kwargs):
         })
         return self.get('/districts/locate', **kwargs)
 
+    @pageable
     def locate_districts_by_zip(self, zipcode, **kwargs):
         """
         Find congressional districts by a latitude and longitude.
@@ -202,6 +213,7 @@ def locate_districts_by_zip(self, zipcode, **kwargs):
         })
         return self.get('/districts/locate', **kwargs)
 
+    @pageable
     def committees(self, **kwargs):
         """
         Search and filter through committees in the House and Senate.
@@ -211,6 +223,7 @@ def committees(self, **kwargs):
         """
         return self.get('committees', **kwargs)
 
+    @pageable
     def amendments(self, **kwargs):
         """
         Search and filter through amendments in Congress.
@@ -220,6 +233,7 @@ def amendments(self, **kwargs):
         """
         return self.get('amendments', **kwargs)
 
+    @pageable
     def votes(self, **kwargs):
         """
         Search and filter through votes in Congress.
@@ -229,6 +243,7 @@ def votes(self, **kwargs):
         """
         return self.get('votes', **kwargs)
 
+    @pageable
     def floor_updates(self, **kwargs):
         """
         Search and filter through floor updates in the House and Senate.
@@ -238,6 +253,7 @@ def floor_updates(self, **kwargs):
         """
         return self.get('floor_updates', **kwargs)
 
+    @pageable
     def hearings(self, **kwargs):
         """
         Search and filter through committee hearings in the House and Senate.
@@ -247,6 +263,7 @@ def hearings(self, **kwargs):
         """
         return self.get('hearings', **kwargs)
 
+    @pageable
     def nominations(self, **kwargs):
         """
         Search and filter through presidential nominations in Congress.