Merge pull request #22 from ShorensteinCenter/devel

[Bugfix, N/A] Added exponential backoff to async api requests for other error types

Author: williamhakim10

README.md

@@ -36,7 +36,8 @@ These instructions will get you a copy of the project up and running on your loc
 * `SES_REGION_NAME` - AWS Simple Email Service region.
 * `SES_DEFAULT_EMAIL_SOURCE` - The default email address to send from. This email needs to be verified by SES and active outside the SES sandbox.
 * `SES_CONFIGURATION_SET` - SES Configuration set for tracking opens/clicks/etc. Optional.
-* `NO_PROXY` - We use proxies to distribute our MailChimp requests across IP addresses. Set this variable to `True` in order to disable proxying, or modify the `enable_proxy` method in `app/lists.py` according to your proxy setup.
+* `NO_PROXY` - We use proxies to distribute our MailChimp requests across IP addresses. Set this variable to `True` in order to disable proxying, or modify the `enable_proxy` method in `app/lists.py` according to your proxy configuration.
+* `NO_EMAIL` - Suppresses sending of emails. Note that SES is still required, emails will just be logged rather than sent. Optional.
 * `ADMIN_EMAIL` - Email address to send error emails to. Optional.
 
 ##### Upgrade the database

app/emails.py

@@ -1,4 +1,6 @@
 """This module contains functions associated with sending email."""
+import os
+import logging
 import boto3
 from flask import render_template
 from app import app
@@ -32,6 +34,13 @@ def send_email(subject, recipients, template_name, template_context, # pylint: d
 
     with app.app_context():
         html = render_template(template_name, **template_context)
+        if os.environ.get('NO_EMAIL'):
+            logger = logging.getLogger(__name__)
+            logger.warning('NO_EMAIL environment variable set. '
+                           'Suppressing an email with the following params: '
+                           'Sender: %s. Recipients: %s. Subject: %s.',
+                           sender, recipients, subject)
+            return
         ses.send_email(
             Source=sender,
             Destination={'ToAddresses': recipients},

app/lists.py

@@ -12,8 +12,6 @@
 from pandas.io.json import json_normalize
 import numpy as np
 from aiohttp import ClientSession, BasicAuth
-from aiohttp.client_exceptions import (
-    ClientHttpProxyError, ServerDisconnectedError)
 import iso8601
 from celery.utils.log import get_task_logger
 
@@ -247,9 +245,7 @@ async def make_async_request(self, url, params, session, retry=0):
                     error_details)
 
         # Catch proxy problems as well as potential asyncio timeouts/disconnects
-        except (asyncio.TimeoutError, # pylint: disable=invalid-name
-                ClientHttpProxyError,
-                ServerDisconnectedError) as e:
+        except Exception as e: # pylint: disable=invalid-name
 
             exception_type = type(e).__name__
 
@@ -261,10 +257,15 @@ async def make_async_request(self, url, params, session, retry=0):
                 self.logger.warning('Server disconnected! URL: %s. API key: '
                                     '%s.', url, self.api_key)
 
-            else:
+            elif exception_type == 'TimeoutError':
                 self.logger.warning('Asyncio request timed out! URL: %s. '
                                     'API key: %s.', url, self.api_key)
 
+            else:
+                self.logger.warning('An unforseen error type occurred. '
+                                    'Error type: %s. URL: %s. API Key: %s.',
+                                    exception_type, url, self.api_key)
+
             # Retry if we haven't already retried a few times
             if retry < self.MAX_RETRIES: