http_client.py

# Unless explicitly stated otherwise all files in this repository are licensed under the BSD-3-Clause License.
# This product includes software developed at Datadog (https://www.datadoghq.com/).
# Copyright 2015-Present Datadog, Inc
"""
Available HTTP Client for Datadog API client.

Priority:
1. `requests` 3p module
2. `urlfetch` 3p module - Google App Engine only
"""
# stdlib
import copy
import logging
import platform
import urllib
from threading import Lock

# 3p
try:
    import requests
    import requests.adapters
except ImportError:
    requests = None  # type: ignore

try:
    from google.appengine.api import urlfetch, urlfetch_errors
except ImportError:
    urlfetch, urlfetch_errors = None, None

try:
    import urllib3  # type: ignore
except ImportError:
    urllib3 = None

# datadog
from datadog.api.exceptions import ProxyError, ClientError, HTTPError, HttpTimeout


log = logging.getLogger("datadog.api")


def _get_user_agent_header():
    from datadog import version

    return "datadogpy/{version} (python {pyver}; os {os}; arch {arch})".format(
        version=version.__version__,
        pyver=platform.python_version(),
        os=platform.system().lower(),
        arch=platform.machine().lower(),
    )


def _remove_context(exc):
    """Python3: remove context from chained exceptions to prevent leaking API keys in tracebacks."""
    exc.__cause__ = None
    return exc


class HTTPClient(object):
    """
    An abstract generic HTTP client. Subclasses must implement the `request` methods.
    """

    @classmethod
    def request(cls, method, url, headers, params, data, timeout, proxies, verify, max_retries):
        """
        Main method to be implemented by HTTP clients.

        The returned data structure has the following fields:
        * `content`: string containing the response from the server
        * `status_code`: HTTP status code returned by the server

        Can raise the following exceptions:
        * `ClientError`: server cannot be contacted
        * `HttpTimeout`: connection timed out
        * `HTTPError`: unexpected HTTP response code
        """
        raise NotImplementedError(u"Must be implemented by HTTPClient subclasses.")


class RequestClient(HTTPClient):
    """
    HTTP client based on 3rd party `requests` module, using a single session.
    This allows us to keep the session alive to spare some execution time.
    """

    _session = None
    _session_lock = Lock()

    @classmethod
    def request(cls, method, url, headers, params, data, timeout, proxies, verify, max_retries):
        try:

            with cls._session_lock:
                if cls._session is None:
                    cls._session = requests.Session()
                    http_adapter = requests.adapters.HTTPAdapter(max_retries=max_retries)
                    cls._session.mount("https://", http_adapter)
                    cls._session.headers.update({"User-Agent": _get_user_agent_header()})

            result = cls._session.request(
                method, url, headers=headers, params=params, data=data, timeout=timeout, proxies=proxies, verify=verify
            )

            result.raise_for_status()

        except requests.exceptions.ProxyError as e:
            raise _remove_context(ProxyError(method, url, e))
        except requests.ConnectionError as e:
            raise _remove_context(ClientError(method, url, e))
        except requests.exceptions.Timeout:
            raise _remove_context(HttpTimeout(method, url, timeout))
        except requests.exceptions.HTTPError as e:
            if e.response.status_code in (400, 401, 403, 404, 409, 429):
                # This gets caught afterwards and raises an ApiError exception
                pass
            else:
                raise _remove_context(HTTPError(e.response.status_code, result.reason))
        except TypeError:
            raise TypeError(
                u"Your installed version of `requests` library seems not compatible with"
                u"Datadog's usage. We recommend upgrading it ('pip install -U requests')."
                u"If you need help or have any question, please contact support@datadoghq.com"
            )

        return result


class URLFetchClient(HTTPClient):
    """
    HTTP client based on Google App Engine `urlfetch` module.
    """

    @classmethod
    def request(cls, method, url, headers, params, data, timeout, proxies, verify, max_retries):
        """
        Wrapper around `urlfetch.fetch` method.

        TO IMPLEMENT:
        * `max_retries`
        """
        # No local certificate file can be used on Google App Engine
        validate_certificate = True if verify else False

        # Encode parameters in the url
        url_with_params = "{url}?{params}".format(url=url, params=urllib.urlencode(params))
        newheaders = copy.deepcopy(headers)
        newheaders["User-Agent"] = _get_user_agent_header()

        try:
            result = urlfetch.fetch(
                url=url_with_params,
                method=method,
                headers=newheaders,
                validate_certificate=validate_certificate,
                deadline=timeout,
                payload=data,
                # setting follow_redirects=False may be slightly faster:
                # https://cloud.google.com/appengine/docs/python/microservice-performance#use_the_shortest_route
                follow_redirects=False,
            )

            cls.raise_on_status(result)

        except urlfetch.DownloadError as e:
            raise ClientError(method, url, e)
        except urlfetch_errors.DeadlineExceededError:
            raise HttpTimeout(method, url, timeout)

        return result

    @classmethod
    def raise_on_status(cls, result):
        """
        Raise on HTTP status code errors.
        """
        status_code = result.status_code

        if (status_code / 100) != 2:
            if status_code in (400, 401, 403, 404, 409, 429):
                pass
            else:
                raise HTTPError(status_code)


class Urllib3Client(HTTPClient):
    """
    HTTP client based on 3rd party `urllib3` module.
    """

    _pool = None
    _pool_lock = Lock()

    @classmethod
    def request(cls, method, url, headers, params, data, timeout, proxies, verify, max_retries):
        """
        Wrapper around `urllib3.PoolManager.request` method. This method will raise
        exceptions for HTTP status codes that are not 2xx.
        """
        try:
            with cls._pool_lock:
                if cls._pool is None:
                    cls._pool = urllib3.PoolManager(
                        retries=max_retries,
                        timeout=timeout,
                        cert_reqs="CERT_REQUIRED" if verify else "CERT_NONE",
                    )

            newheaders = copy.deepcopy(headers)
            newheaders["User-Agent"] = _get_user_agent_header()
            response = cls._pool.request(
                method, url, body=data, fields=params, headers=newheaders
            )
            cls.raise_on_status(response)

        except urllib3.exceptions.ProxyError as e:
            raise _remove_context(ProxyError(method, url, e))
        except urllib3.exceptions.MaxRetryError as e:
            raise _remove_context(ClientError(method, url, e))
        except urllib3.exceptions.TimeoutError as e:
            raise _remove_context(HttpTimeout(method, url, e))
        except urllib3.exceptions.HTTPError as e:
            raise _remove_context(HTTPError(e))

        return response

    @classmethod
    def raise_on_status(cls, response):
        """
        Raise on HTTP status code errors.
        """
        status_code = response.status
        if status_code < 200 or status_code >= 300:
            if status_code not in (400, 401, 403, 404, 409, 429):
                raise HTTPError(status_code, response.reason)


def resolve_http_client():
    """
    Resolve an appropriate HTTP client based the defined priority and user environment.
    """
    if requests:
        log.debug(u"Use `requests` based HTTP client.")
        return RequestClient

    if urlfetch and urlfetch_errors:
        log.debug(u"Use `urlfetch` based HTTP client.")
        return URLFetchClient

    if urllib3:
        log.debug(u"Use `urllib3` based HTTP client.")
        return Urllib3Client

    raise ImportError(
        u"Datadog API client was unable to resolve a HTTP client. " u" Please install `requests` library."
    )