Fix out-of-order return

Author: Lyetenth

firststreet/http_util.py

@@ -59,6 +59,10 @@ def __init__(self, api_key, connection_limit, rate_limit, rate_period, version=N
         self.rate_limit = rate_limit
         self.rate_period = rate_period
 
+    async def bound_fetch(self, sem, endpoint, session, throttler):
+        async with sem:
+            return await self.execute(endpoint, session, throttler)
+
     async def endpoint_execute(self, endpoints):
         """Asynchronously calls each endpoint and returns the JSON responses
         Args:
@@ -75,9 +79,12 @@ async def endpoint_execute(self, endpoints):
 
         # Asnycio create tasks for each endpoint
         try:
-            tasks = (asyncio.create_task(self.execute(endpoint, session, throttler)) for endpoint in endpoints)
-            ret = [await t
-                   for t in tqdm.tqdm(self.limited_as_completed(tasks, self.connection_limit), total=len(endpoints))]
+
+            sem = asyncio.Semaphore(self.connection_limit)
+            tasks = [asyncio.create_task(self.bound_fetch(sem, endpoint, session, throttler)) for endpoint in endpoints]
+
+            [await f for f in tqdm.tqdm(asyncio.as_completed(tasks), total=len(endpoints))]
+            ret = [t.result() for t in tasks]
 
         finally:
             await session.close()
@@ -220,39 +227,3 @@ def _network_error(options, rate_limit, error=None, status=None, message=None):
             503: e.OfflineError(message=formatted, attachments={"options": options, "rate_limit": rate_limit}),
         }.get(status,
               e.UnknownError(message=formatted, attachments={"options": options, "rate_limit": rate_limit}))
-
-    @staticmethod
-    def limited_as_completed(coros, limit):
-        """
-        Run the coroutines (or futures) supplied in the
-        iterable coros, ensuring that there are at most
-        limit coroutines running at any time.
-        Return an iterator whose values, when waited for,
-        are Future instances containing the results of
-        the coroutines.
-        Results may be provided in any order, as they
-        become available.
-
-        https://github.com/andybalaam/asyncioplus/blob/master/asyncioplus/limited_as_completed.py
-        """
-        futures = [
-            asyncio.ensure_future(c)
-            for c in islice(coros, 0, limit)
-        ]
-
-        async def first_to_finish():
-            while True:
-                await asyncio.sleep(0)
-                for f in futures:
-                    if f.done():
-                        futures.remove(f)
-                        try:
-                            newf = next(coros)
-                            futures.append(
-                                asyncio.ensure_future(newf))
-                        except StopIteration as e:
-                            pass
-                        return f.result()
-
-        while len(futures) > 0:
-            yield first_to_finish()

tests/api/test_common.py

@@ -26,7 +26,5 @@ def test_historic_geom(self):
         assert his[0].geometry is not None
 
     def test_location_geom(self):
-        loc = fs.location.get_detail([395112095], "property", csv=True)
-        assert loc[0].geometry is None
         loc = fs.location.get_detail([39153531702], "tract", csv=True)
         assert loc[0].geometry is not None

tests/api/test_location.py

@@ -164,8 +164,8 @@ def test_one_of_each(self, tmpdir):
         assert location[0].state is not None
         assert location[0].footprintId is None
         assert location[0].elevation is not None
-        assert location[0].fema is None
-        assert location[0].geometry is None
+        assert location[0].fema is not None
+        assert location[0].geometry is not None
         location = fs.location.get_detail([1206631], "neighborhood", csv=True, output_dir=tmpdir)
         assert len(location) == 1
         assert location[0].valid_id is True
@@ -175,7 +175,7 @@ def test_one_of_each(self, tmpdir):
         assert location[0].subtype is not None
         assert location[0].county is not None
         assert location[0].state is not None
-        assert location[0].geometry is None
+        assert location[0].geometry is not None
         location = fs.location.get_detail([3915406], "city", csv=True, output_dir=tmpdir)
         assert len(location) == 1
         assert location[0].valid_id is True