SG-19308 bundles certifi (#235)

* We're bundling certifi to give a more consistent certificate behaviour. We're not actually using the certifi `where()` functionality to get the path to the certificates however, as it can give the wrong path when certifi can be found else where in the `sys.path`. So we are manually pathing to the certificates provided with certifi.
* Removed the rimu tests, as this site is no longer hosted.

Author: pscadding

.travis.yml

@@ -14,10 +14,6 @@ language: python
 # and 3.7 is not supported in trusty.  For now just use trusty to test 2.6, and use
 # the more modern xenial for 2.7 and 3.7.
 
-# Additionally, we'll test on multiple sites.  To accomplish this, we store the
-# secrets for the second site in a set of additional environment variables,
-# and copy those into the standard environment variables when running on the
-# second site.
 matrix:
   include:
 
@@ -32,16 +28,6 @@ matrix:
       env:
         - RUN_FLAKE=false
 
-    # Python 2.6, Server #2
-    - dist: trusty
-      python: "2.6"
-      # flake8 does not support python 2.6, so only run it on 2.7+
-      env:
-        - RUN_FLAKE=false
-        - SG_SERVER_URL=$SG_SERVER_URL_2
-        - SG_API_KEY=$SG_API_KEY_2
-        - SG_HUMAN_PASSWORD=$SG_HUMAN_PASSWORD_2
-
     # Test python 2.7 and 3.7 on Xenial.
 
     # Python 2.7, Server #1
@@ -50,29 +36,11 @@ matrix:
       env:
         - RUN_FLAKE=true
 
-    # Python 2.7, Server #2
-    - dist: xenial
-      python: "2.7"
-      env:
-        - RUN_FLAKE=true
-        - SG_SERVER_URL=$SG_SERVER_URL_2
-        - SG_API_KEY=$SG_API_KEY_2
-        - SG_HUMAN_PASSWORD=$SG_HUMAN_PASSWORD_2
-
-    # Python 3.7, Server #1
-    - dist: xenial
-      python: "3.7"
-      env:
-        - RUN_FLAKE=true
-
     # Python 3.7, Server #1
     - dist: xenial
       python: "3.7"
       env:
         - RUN_FLAKE=true
-        - SG_SERVER_URL=$SG_SERVER_URL_2
-        - SG_API_KEY=$SG_API_KEY_2
-        - SG_HUMAN_PASSWORD=$SG_HUMAN_PASSWORD_2
 
 # command to install dependencies
 install:

shotgun_api3/lib/certifi/__init__.py

@@ -0,0 +1,3 @@
+from .core import contents, where
+
+__version__ = "2020.06.20"

shotgun_api3/lib/certifi/__main__.py

@@ -0,0 +1,12 @@
+import argparse
+
+from certifi import contents, where
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-c", "--contents", action="store_true")
+args = parser.parse_args()
+
+if args.contents:
+    print(contents())
+else:
+    print(where())

shotgun_api3/lib/certifi/cacert.pem

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+
+"""
+certifi.py
+~~~~~~~~~~
+
+This module returns the installation location of cacert.pem or its contents.
+"""
+import os
+
+try:
+    from importlib.resources import path as get_path, read_text
+
+    _CACERT_CTX = None
+    _CACERT_PATH = None
+
+    def where():
+        # This is slightly terrible, but we want to delay extracting the file
+        # in cases where we're inside of a zipimport situation until someone
+        # actually calls where(), but we don't want to re-extract the file
+        # on every call of where(), so we'll do it once then store it in a
+        # global variable.
+        global _CACERT_CTX
+        global _CACERT_PATH
+        if _CACERT_PATH is None:
+            # This is slightly janky, the importlib.resources API wants you to
+            # manage the cleanup of this file, so it doesn't actually return a
+            # path, it returns a context manager that will give you the path
+            # when you enter it and will do any cleanup when you leave it. In
+            # the common case of not needing a temporary file, it will just
+            # return the file system location and the __exit__() is a no-op.
+            #
+            # We also have to hold onto the actual context manager, because
+            # it will do the cleanup whenever it gets garbage collected, so
+            # we will also store that at the global level as well.
+            _CACERT_CTX = get_path("certifi", "cacert.pem")
+            _CACERT_PATH = str(_CACERT_CTX.__enter__())
+
+        return _CACERT_PATH
+
+
+except ImportError:
+    # This fallback will work for Python versions prior to 3.7 that lack the
+    # importlib.resources module but relies on the existing `where` function
+    # so won't address issues with environments like PyOxidizer that don't set
+    # __file__ on modules.
+    def read_text(_module, _path, encoding="ascii"):
+        with open(where(), "r", encoding=encoding) as data:
+            return data.read()
+
+    # If we don't have importlib.resources, then we will just do the old logic
+    # of assuming we're on the filesystem and munge the path directly.
+    def where():
+        f = os.path.dirname(__file__)
+
+        return os.path.join(f, "cacert.pem")
+
+
+def contents():
+    return read_text("certifi", "cacert.pem", encoding="ascii")

shotgun_api3/lib/certifi/core.py

@@ -29,4 +29,5 @@
 # This file is unused. It is left there so Github can warn us is a CVE is
 # released for our dependencies.
 httplib2==0.18.0
-six==1.12.0
+six==1.13.0
+certifi==2020.06.20

shotgun_api3/lib/requirements.txt

@@ -643,38 +643,7 @@ def __init__(self,
 
         self._connection = None
 
-        # The following lines of code allow to tell the API where to look for
-        # certificate authorities certificates (we will be referring to these
-        # as CAC from now on). Here's how the Python API interacts with those.
-        #
-        # Auth and CRUD operations
-        # ========================
-        # These operations are executed with httplib2. httplib2 ships with a
-        # list of CACs instead of asking Python's ssl module for them.
-        #
-        # Upload/Downloads
-        # ================
-        # These operations are executed using urllib2. urllib2 asks a Python
-        # module called `ssl` for CACs. On Windows, ssl searches for CACs in
-        # the Windows Certificate Store. On Linux/macOS, it asks the OpenSSL
-        # library linked with Python for CACs. Depending on how Python was
-        # compiled for a given DCC, Python may be linked against the OpenSSL
-        # from the OS or a copy of OpenSSL distributed with the DCC. This
-        # impacts which versions of the certificates are available to Python,
-        # as an OS level OpenSSL will be aware of system wide certificates that
-        # have been added, while an OpenSSL that comes with a DCC is likely
-        # bundling a list of certificates that get update with each release and
-        # no not contain system wide certificates.
-        #
-        # Using custom CACs
-        # =================
-        # When a user requires a non-standard CAC, the SHOTGUN_API_CACERTS
-        # environment variable allows to provide an alternate location for
-        # the CACs.
-        if ca_certs is not None:
-            self.__ca_certs = ca_certs
-        else:
-            self.__ca_certs = os.environ.get("SHOTGUN_API_CACERTS")
+        self.__ca_certs = self._get_certs_file(ca_certs)
 
         self.base_url = (base_url or "").lower()
         self.config.set_server_params(self.base_url)
@@ -3249,6 +3218,66 @@ def _build_opener(self, handler):
             handlers.append(handler)
         return urllib.request.build_opener(*handlers)
 
+    @classmethod
+    def _get_certs_file(cls, ca_certs):
+        """
+        The following method tells the API where to look for
+        certificate authorities certificates (we will be referring to these
+        as CAC from now on). Here's how the Python API interacts with those.
+
+        Auth and CRUD operations
+        ========================
+        These operations are executed with httplib2. httplib2 ships with a
+        list of CACs instead of asking Python's ssl module for them.
+
+        Upload/Downloads
+        ================
+        These operations are executed using urllib2. urllib2 asks a Python
+        module called `ssl` for CACs. We have bundled certifi with the API
+        so that we can be sure the certs are correct at the time of the API
+        release. This does however mean when the certs change we must update
+        the API to contain the latest certifi.
+        This approach is preferable to not using certifi since, on Windows,
+        ssl searches for CACs in the Windows Certificate Store, on
+        Linux/macOS, it asks the OpenSSL library linked with Python for CACs.
+        Depending on how Python was compiled for a given DCC, Python may be
+        linked against the OpenSSL from the OS or a copy of OpenSSL distributed
+        with the DCC. This impacts which versions of the certificates are
+        available to Python, as an OS level OpenSSL will be aware of system
+        wide certificates that have been added, while an OpenSSL that comes
+        with a DCC is likely bundling a list of certificates that get update
+        with each release and may not contain system wide certificates.
+
+        Using custom CACs
+        =================
+        When a user requires a non-standard CAC, the SHOTGUN_API_CACERTS
+        environment variable allows to provide an alternate location for
+        the CACs.
+
+        :param ca_certs: A default cert can be provided
+        :return: The cert file path to use.
+        """
+        if ca_certs is not None:
+            # certs were provided up front so use these
+            return ca_certs
+        elif "SHOTGUN_API_CACERTS" in os.environ:
+            return os.environ.get("SHOTGUN_API_CACERTS")
+        else:
+            # No certs have been specifically provided fallback to using the
+            # certs shipped with this API.
+            # We bundle certifi with this API so that we have a higher chance
+            # of using an uptodate certificate, rather than relying
+            # on the certs that are bundled with Python or the OS in some cases.
+            # However we can't use certifi.where() since that searches for the
+            # cacert.pem file using the sys.path and this means that if another
+            # copy of certifi can be found first, then it won't use ours.
+            # So we manually generate the path to the cert, but still use certifi
+            # to make it easier for updating the bundled cert with the API.
+            cur_dir = os.path.dirname(os.path.abspath(__file__))
+            # Now add the rest of the path to the cert file.
+            cert_file = os.path.join(cur_dir, "lib", "certifi", "cacert.pem")
+            return cert_file
+
     def _turn_off_ssl_validation(self):
         """
         Turn off SSL certificate validation.

shotgun_api3/shotgun.py

@@ -10,11 +10,13 @@
 # agreement to the Shotgun Pipeline Toolkit Source Code License. All rights
 # not expressly granted therein are reserved by Shotgun Software Inc.
 
+import os
 import unittest
 from .mock import patch
 import shotgun_api3 as api
 from shotgun_api3.shotgun import _is_mimetypes_broken
-from shotgun_api3.lib.six.moves import range
+from shotgun_api3.lib.six.moves import range, urllib
+from shotgun_api3.lib.httplib2 import Http, ssl_error_classes
 
 
 class TestShotgunInit(unittest.TestCase):
@@ -404,6 +406,88 @@ def test_invalid(self):
         self.assertRaises(api.ShotgunError, api.shotgun._translate_filters, filters, "all")
 
 
+class TestCerts(unittest.TestCase):
+    # A dummy bad url provided by Amazon
+    bad_url = "https://untrusted-root.badssl.com/"
+    # A list of Amazon cert URLS, taken from here:
+    # https://aws.amazon.com/blogs/security/how-to-prepare-for-aws-move-to-its-own-certificate-authority/
+    test_urls = [
+        "https://good.sca1a.amazontrust.com",
+        "https://good.sca2a.amazontrust.com",
+        "https://good.sca3a.amazontrust.com",
+        "https://good.sca4a.amazontrust.com",
+        "https://good.sca0a.amazontrust.com",
+    ]
+
+    def setUp(self):
+        self.sg = api.Shotgun('http://server_path',
+                              'script_name',
+                              'api_key',
+                              connect=False)
+
+        # Get the location of the certs file
+        self.certs = self.sg._get_certs_file(None)
+
+    def _check_url_with_sg_api_httplib2(self, url, certs):
+        """
+        Given a url and the certs file, it will do a simple
+        request and return the result.
+        """
+        http = Http(ca_certs=certs)
+        return http.request(url)
+
+    def _check_url_with_urllib(self, url):
+        """
+        Given a url it will perform a simple request and return a result.
+        """
+        # create a request using the opener generated by the SG API.
+        # The `_build_opener` method internally should use the correct certs.
+        opener = self.sg._build_opener(urllib.request.HTTPHandler)
+        request = urllib.request.Request(url)
+        return opener.open(request)
+
+    def test_found_correct_cert(self):
+        """
+        Checks that the cert file the API is finding,
+        (when a cert path isn't passed and the SHOTGUN_API_CACERTS
+        isn't set), is the one bundled with this API
+        """
+        # Get the path to the cert file we expect the Shotgun API to find
+        cur_path = os.path.dirname(os.path.abspath(__file__))
+        cert_path = os.path.normpath(
+            os.path.join(cur_path, "..", "shotgun_api3", "lib", "certifi", "cacert.pem")
+        )
+        # Now ensure that the path the SG API has found is correct.
+        self.assertEquals(cert_path, self.certs)
+        self.assertTrue(os.path.isfile(self.certs))
+
+    def test_httplib(self):
+        """
+        Checks that we can access the amazon urls using our bundled
+        certificate with httplib.
+        """
+        # First check that we get an error when trying to connect to a known dummy bad URL
+        self.assertRaises(ssl_error_classes, self._check_url_with_sg_api_httplib2, self.bad_url, self.certs)
+
+        # Now check that the good urls connect properly using the certs
+        for url in self.test_urls:
+            response, message = self._check_url_with_sg_api_httplib2(url, self.certs)
+            self.assertEquals(response["status"], "200")
+
+    def test_urlib(self):
+        """
+        Checks that we can access the amazon urls using our bundled
+        certificate with urllib.
+        """
+        # First check that we get an error when trying to connect to a known dummy bad URL
+        self.assertRaises(urllib.error.URLError, self._check_url_with_urllib, self.bad_url)
+
+        # Now check that the good urls connect properly using the certs
+        for url in self.test_urls:
+            response = self._check_url_with_urllib(url)
+            assert (response is not None)
+
+
 class TestMimetypesFix(unittest.TestCase):
     """
     Makes sure that the mimetypes fix will be imported.
@@ -430,6 +514,5 @@ def test_correct_mimetypes_imported(self):
         self._test_mimetypes_import("win32", 3, 0, 0, False)
         self._test_mimetypes_import("darwin", 2, 7, 0, False)
 
-
 if __name__ == '__main__':
     unittest.main()