Alpha 6 Release (#140)

* Allow for overriding enforce_json

* Update default API gateway URL

* Fix docstring spelling mistake

* Remove overriding urllib warnings

* Update query example

* Update heading

* Reformat with black

* Bump version

* Reformat with black

* Add transactions attribute

* Implement module-level logging

* Avoid applying header credentials twice when no method credentials are present

* Update HISTORY

Author: sserrata

HISTORY.md

@@ -1,10 +1,23 @@
 # History
 
-## 2.0.0-alpha5 (2020-02-20)
+## 2.0.0-alpha6 (2020-03-04)
+
+-   Chnaged `iter_job_results()` method to allow overriding `enforce_json`
+-   Updated default API gateway URL
+-   Minor corrections to docstrings
+-   Removeed suppression of urllib3 warnings
+-   Updated example scripts
+-   Updated README
+-   Reformatted/refactoring with black
+-   Added `transactions` attribute to `ApiStats` with default value
+-   Reimplemented module-level logging
+-   Applied change to `HTTPClient` to avoid applying header credentials twice when no method credentials are present
+
+## 2.0.0-alpha5 (2020-03-03)
 
 -   Fixed `Credentials` storage adapter import
 
-## 2.0.0-alpha4 (2020-02-20)
+## 2.0.0-alpha4 (2020-03-03)
 
 -   Updated MANIFEST
 

README.md

@@ -1,6 +1,6 @@
 ![Tests](https://github.com/PaloAltoNetworks/pan-cortex-data-lake-python/workflows/Tests/badge.svg) [![PyPI version](https://badge.fury.io/py/pan-cortex-data-lake.svg)](https://badge.fury.io/py/pan-cortex-data-lake)
 
-# Palo Alto Networks Cortex™ Data Lake Python SDK
+# Palo Alto Networks Cortex™ Data Lake SDK
 
 Python idiomatic SDK for the Cortex™ Data Lake.
 

examples/credentials_generate.py

@@ -12,16 +12,14 @@
 curpath = os.path.dirname(os.path.abspath(__file__))
 sys.path[:0] = [os.path.join(curpath, os.pardir)]
 
-from cortex import Credentials
+from pan_cortex_data_lake import Credentials
 
 
 def confirm_write(profile):
     """Prompt user to enter Y or N (case-insensitive) to continue."""
     answer = ""
     while answer not in ["y", "n"]:
-        answer = input(
-            "\nWrite credentials to PROFILE '%s' [Y/N]? " % profile
-        ).lower()
+        answer = input("\nWrite credentials to PROFILE '%s' [Y/N]? " % profile).lower()
     return answer == "y"
 
 
@@ -32,10 +30,12 @@ def main():
         client_secret = getpass.getpass(prompt="CLIENT_SECRET: ")
         refresh_token = getpass.getpass(prompt="REFRESH_TOKEN: ")
         profile = input("PROFILE [default]: ") or None
-        c = Credentials(client_id=client_id,
-                        client_secret=client_secret,
-                        refresh_token=refresh_token,
-                        profile=profile)
+        c = Credentials(
+            client_id=client_id,
+            client_secret=client_secret,
+            refresh_token=refresh_token,
+            profile=profile,
+        )
         if confirm_write(profile):
             print("Writing credentials file...")
             c.write_credentials()

examples/credentials_remove.py

@@ -11,7 +11,7 @@
 curpath = os.path.dirname(os.path.abspath(__file__))
 sys.path[:0] = [os.path.join(curpath, os.pardir)]
 
-from cortex import Credentials
+from pan_cortex_data_lake import Credentials
 
 
 def confirm_delete(profile):

examples/query_all.py

@@ -6,18 +6,19 @@
 import os
 import sys
 import time
+import logging
 
 # Necessary to reference cortex package in relative path
 curpath = os.path.dirname(os.path.abspath(__file__))
 sys.path[:0] = [os.path.join(curpath, os.pardir)]
 
 from pan_cortex_data_lake import Credentials, QueryService
 
-url = "https://cortex-prd1-api.us.cdl.paloaltonetworks.com"  # prod us
+url = "https://api.us.cdl.paloaltonetworks.com"  # prod us
 
 # Create Credentials instance
 # export PAN_DEVELOPER_TOKEN for quick access
-c = Credentials(verify=False)
+c = Credentials()
 
 # Create Query Service instance
 qs = QueryService(url=url, force_trace=True, credentials=c)
@@ -40,9 +41,7 @@
 
 # Iterate through job results (pages)
 print("Iterate through job results: \n")
-for p in qs.iter_job_results(
-    job_id=job_id, page_size=1, result_format="valuesDictionary"
-):
+for p in qs.iter_job_results(job_id=job_id, result_format="valuesDictionary"):
     print("RESULTS: {}\n".format(p.text))
 
 print("STATS: {}".format(qs.stats))

pan_cortex_data_lake/__init__.py

@@ -3,7 +3,7 @@
 """Main package for cortex."""
 
 __author__ = "Palo Alto Networks"
-__version__ = "2.0.0-a5"
+__version__ = "2.0.0-a6"
 
 from .exceptions import (  # noqa: F401
     CortexError,

pan_cortex_data_lake/adapters/adapter.py

@@ -6,7 +6,7 @@
 from abc import ABCMeta, abstractmethod
 
 # Python 2.7 and 3.5+ compatibility
-ABC = ABCMeta('ABC', (object,), {'__slots__': ()})
+ABC = ABCMeta("ABC", (object,), {"__slots__": ()})
 
 
 class StorageAdapter(ABC):  # enforce StorageAdapter interface
@@ -39,8 +39,7 @@ def remove_profile(self, profile=None):
         pass
 
     @abstractmethod
-    def write_credentials(self, credentials=None, profile=None,
-                          cache_token=None):
+    def write_credentials(self, credentials=None, profile=None, cache_token=None):
         """Write credentials.
 
         Write credentials to store.

pan_cortex_data_lake/httpclient.py

@@ -6,15 +6,17 @@
 
 import logging
 
+logger = logging.getLogger(__name__)
+
 import requests
 from requests.adapters import HTTPAdapter
 
-# support ujson in place of standrad json library
+# support ujson in place of standard json library
 try:
     import ujson
 
     requests.models.complexjson = ujson
-    logging.debug("Monkey patched requests with ujson")
+    logger.debug("Monkey patched requests with ujson")
 except ImportError:
     pass
 
@@ -56,8 +58,6 @@ def __init__(self, **kwargs):
             **kwargs: Supported :class:`~requests.Session` and :class:`~requests.adapters.HTTPAdapter` parameters.
 
         """
-        if not logging.getLogger(__name__).isEnabledFor(logging.DEBUG):
-            requests.packages.urllib3.disable_warnings()
         self.kwargs = kwargs.copy()  # used for __repr__
         with requests.Session() as self.session:
             self._default_headers()  # apply default headers
@@ -89,14 +89,13 @@ def __init__(self, **kwargs):
                 self.session.headers.update({"x-envoy-force-trace": ""})
             self.port = kwargs.pop("port", 443)
             self.raise_for_status = kwargs.pop("raise_for_status", False)
-            self.url = kwargs.pop(
-                "url", "https://cortex-prd1-api.us.cdl.paloaltonetworks.com"
-            )
+            self.url = kwargs.pop("url", "https://api.us.cdl.paloaltonetworks.com")
 
             if len(kwargs) > 0:  # Handle invalid kwargs
                 raise UnexpectedKwargsError(kwargs)
 
             if self.credentials:
+                logger.debug("Applying session-level credentials")
                 self._apply_credentials(
                     auto_refresh=self.auto_refresh,
                     credentials=self.credentials,
@@ -139,12 +138,12 @@ def _apply_credentials(auto_refresh=True, credentials=None, headers=None):
         if auto_refresh is True:
             if token is None:
                 token = credentials.refresh(access_token=None, timeout=10)
-                logging.debug("Token refreshed due to 'None' condition")
+                logger.debug("Token refreshed due to 'None' condition")
             elif credentials.jwt_is_expired():
                 token = credentials.refresh(timeout=10)
-                logging.debug("Token refreshed due to 'expired' condition")
+                logger.debug("Token refreshed due to 'expired' condition")
         headers.update({"Authorization": "Bearer {}".format(token)})
-        logging.debug("Credentials applied to authorization header")
+        logger.debug("Credentials applied to authorization header")
 
     def _default_headers(self):
         """Update default headers.
@@ -159,7 +158,7 @@ def _default_headers(self):
                 "User-Agent": "%s/%s" % ("cortex-sdk-python", __version__),
             }
         )
-        logging.debug("Default headers applied: %r" % self.session.headers)
+        logger.debug("Default headers applied: %r" % self.session.headers)
 
     def _send_request(self, enforce_json, method, raise_for_status, url, **kwargs):
         """Send HTTP request.
@@ -221,13 +220,14 @@ def request(self, **kwargs):
 
         # Non-Requests key-word arguments
         auto_refresh = kwargs.pop("auto_refresh", self.auto_refresh)
-        credentials = kwargs.pop("credentials", self.credentials)
+        credentials = kwargs.pop("credentials", None)
         endpoint = kwargs.pop("endpoint", "")  # default to empty endpoint
         enforce_json = kwargs.pop("enforce_json", self.enforce_json)
         raise_for_status = kwargs.pop("raise_for_status", self.raise_for_status)
         url = "{}:{}{}".format(url, self.port, endpoint)
 
         if credentials:
+            logger.debug("Applying method-level credentials")
             self._apply_credentials(
                 auto_refresh=auto_refresh, credentials=credentials, headers=headers
             )

pan_cortex_data_lake/query.py

@@ -220,6 +220,7 @@ def iter_job_results(
         """
         credentials = kwargs.pop("credentials", None)
         params = kwargs.pop("params", {})
+        enforce_json = kwargs.pop("enforce_json", True)
         for name, value in [
             ("maxWait", max_wait),
             ("offset", offset),
@@ -234,7 +235,7 @@ def iter_job_results(
             r = self.get_job_results(
                 job_id=job_id,
                 params=params,
-                enforce_json=True,
+                enforce_json=enforce_json,
                 credentials=credentials,
                 **kwargs
             )

pan_cortex_data_lake/utils.py

@@ -12,6 +12,7 @@ class ApiStats(dict):
 
     def __init__(self, *args, **kwargs):
         super(ApiStats, self).__init__(*args, **kwargs)
+        self.transactions = 0
         for arg in args:
             if isinstance(arg, dict):
                 for k, v in arg.items():