GCP / GSuite fixes (#436)

* GCP / GSuite fixes

* Ignore noisy GCP warnings
* Include additional filter for "this service is not enabled" errors within the GCP Client
* Swap out oauth libs - Fixes #353

* Remove sum() call from datasets fetching.

* Move warning filtering back into gcp/client.py

Author: ajvb

bin/__init__.py

@@ -15,7 +15,7 @@ def datasets():
             get_dataset(d["datasetReference"]["datasetId"], project_id)
             for d in datasets
         ]
-    return sum(results, [])
+    return results
 
 
 def get_dataset(dataset_id, project_id):

bin/auth/__init__.py

@@ -1,12 +1,21 @@
 import warnings
+import logging
 
 from apiclient.discovery import build as build_service
 from apiclient.errors import HttpError
 
+
+# Filters out the warning about using end user credentials.
 warnings.filterwarnings(
     "ignore", "Your application has authenticated using end user credentials"
 )
 
+# Filters out a warning around a feature we don't use - https://github.com/googleapis/google-api-python-client/issues/299
+logging.getLogger("googleapiclient.discovery_cache").setLevel(logging.ERROR)
+
+# Filters out a warning about not have a default GCP Project ID. Not required for Frost, no need to display.
+logging.getLogger("google.auth._default").setLevel(logging.ERROR)
+
 
 def cache_key(project_id, version, product, subproduct, call="list", id_value="na"):
     """Returns the fullname (directory and filename) for a cached GCP API call.
@@ -276,6 +285,8 @@ def _list_all_items(self, api_entity, call_kwargs, results_key):
                 # This will be thrown if an API is disabled.
                 if "has not been used in project" in e._get_reason():
                     return []
+                if "has not enabled" in e._get_reason():
+                    return []
                 raise e
             items = sum([items], resp.get(results_key, []))
             try:

bin/auth/setup_gsuite.py

@@ -0,0 +1,3 @@
+### EXPERIMENTAL - GSuite tests
+
+These tests and the gsuite client are experimental. The client currently only supports the "application default" google auth credentials.

gcp/bigquery/resources.py

@@ -2,32 +2,14 @@
 import httplib2
 
 from apiclient import discovery
-from oauth2client.file import Storage
+import google.auth
 
-CREDS_NAME = "frost-gsuite-readonly"
 SCOPES = [
     "https://www.googleapis.com/auth/admin.directory.user.readonly",
     "https://www.googleapis.com/auth/admin.directory.group.readonly",
 ]
 
 
-def get_credential_dir():
-    home_dir = os.path.expanduser("~")
-    credential_dir = os.path.join(home_dir, ".credentials")
-    if not os.path.exists(credential_dir):
-        os.makedirs(credential_dir)
-    return credential_dir
-
-
-def get_credential_path(credential_name):
-    return os.path.join(get_credential_dir(), credential_name + ".json")
-
-
-def get_credentials(credential_name):
-    store = Storage(get_credential_path(credential_name))
-    return store.get()
-
-
 class GsuiteClient:
     def __init__(self, domain, offline):
         self.domain = domain
@@ -37,10 +19,10 @@ def __init__(self, domain, offline):
             self.directory_client = self.build_directory_client()
 
     def build_directory_client(self):
-        # TODO: Support passing creds name as config option
-        credentials = get_credentials(CREDS_NAME)
-        http = credentials.authorize(httplib2.Http())
-        return discovery.build("admin", "directory_v1", http=http)
+        # TODO: Support service accounts:
+        #   https://googleapis.github.io/google-api-python-client/docs/oauth-server.html#examples
+        credentials, _ = google.auth.default(scopes=SCOPES)
+        return discovery.build("admin", "directory_v1", credentials=credentials)
 
     def list_users(self):
         """

gcp/client.py

@@ -1,8 +1,9 @@
 botocore==1.12.75
 click==7.1.2
 coverage==4.5.3
-google_api_python_client==1.12.3
-oauth2client==4.1.3
+google-api-python-client==1.12.8
+google-auth==1.24.0
+google-auth-httplib2==0.0.4
 pre-commit==1.17.0
 pytest-cov==2.10.0
 pytest-json==0.4.0