Use core django for caching signed urls.

.pytest_cache/README.md

@@ -0,0 +1,8 @@
+# pytest cache directory #
+
+This directory contains data from the pytest's cache plugin,
+which provides the `--lf` and `--ff` options, as well as the `cache` fixture.
+
+**Do not** commit this to version control.
+
+See [the docs](https://docs.pytest.org/en/latest/cache.html) for more information.

.pytest_cache/v/cache/lastfailed

@@ -0,0 +1,22 @@
+{
+  "tests/test_class.py::TestGCloudStorageClass::()::test_changed_files_should_be_reuploaded": true,
+  "tests/test_class.py::TestGCloudStorageClass::()::test_exists_method": true,
+  "tests/test_class.py::TestGCloudStorageClass::()::test_open_should_be_able_to_create_new_file": true,
+  "tests/test_class.py::TestGCloudStorageClass::()::test_should_be_able_to_delete_files": true,
+  "tests/test_class.py::TestGCloudStorageClass::()::test_should_be_able_to_list_dirs_and_files": true,
+  "tests/test_class.py::TestGCloudStorageClass::()::test_should_be_able_to_save_and_open_files": true,
+  "tests/test_class.py::TestGCloudStorageClass::()::test_should_create_a_valid_bucket_object": true,
+  "tests/test_class.py::TestGCloudStorageClass::()::test_should_create_a_valid_client_object": true,
+  "tests/test_class.py::TestGCloudStorageClass::()::test_should_create_blob_at_correct_path": true,
+  "tests/test_class.py::TestGCloudStorageClass::()::test_should_not_overwrite_files_on_save": true,
+  "tests/test_class.py::TestGCloudStorageClass::()::test_should_return_correct_file_size": true,
+  "tests/test_class.py::TestGCloudStorageClass::()::test_should_return_created_time": true,
+  "tests/test_class.py::TestGCloudStorageClass::()::test_should_return_modified_time": true,
+  "tests/test_class.py::TestGCloudStorageClass::()::test_should_return_publicly_downloadable_url": true,
+  "tests/test_class.py::TestGCloudStorageClass::()::test_should_work_with_utf8": true,
+  "tests/test_class.py::TestGCloudStorageClass::()::test_use_unsigned_urls_option": true,
+  "tests/test_django.py": true,
+  "tests/test_django.py::TestApp::()::test_file_access": true,
+  "tests/test_django.py::TestApp::()::test_http_upload": true,
+  "tests/test_django.py::TestApp::()::test_manual_upload": true
+}

.pytest_cache/v/cache/nodeids

@@ -0,0 +1,34 @@
+[
+  "tests/test_class.py::TestSafeJoin::()::test_should_join_urls",
+  "tests/test_class.py::TestSafeJoin::()::test_should_not_break_on_slash_on_base",
+  "tests/test_class.py::TestSafeJoin::()::test_should_enforce_no_starting_slash_on_base",
+  "tests/test_class.py::TestSafeJoin::()::test_should_resolve_dots_to_absolute_path",
+  "tests/test_class.py::TestSafeJoin::()::test_should_resolve_multiple_slashes",
+  "tests/test_class.py::TestSafeJoin::()::test_should_not_allow_escaping_base_path",
+  "tests/test_class.py::TestSafeJoin::()::test_should_work_with_bytes",
+  "tests/test_class.py::TestSafeJoin::()::test_should_work_with_unicode_characters",
+  "tests/test_class.py::TestSafeJoin::()::test_should_normalize_system_dependant_slashes",
+  "tests/test_class.py::test_remove_prefix_function",
+  "tests/test_class.py::TestGCloudFile::()::test_should_be_able_to_read_and_write",
+  "tests/test_class.py::TestGCloudFile::()::test_small_temporary_files_should_not_be_rolled_over_to_disk",
+  "tests/test_class.py::TestGCloudFile::()::test_large_temporary_files_should_be_rolled_over_to_disk",
+  "tests/test_class.py::TestGCloudFile::()::test_modified_files_should_be_marked_as_dirty",
+  "tests/test_class.py::TestGCloudStorageClass::()::test_should_create_blob_at_correct_path",
+  "tests/test_class.py::TestGCloudStorageClass::()::test_should_create_a_valid_client_object",
+  "tests/test_class.py::TestGCloudStorageClass::()::test_should_create_a_valid_bucket_object",
+  "tests/test_class.py::TestGCloudStorageClass::()::test_should_be_able_to_save_and_open_files",
+  "tests/test_class.py::TestGCloudStorageClass::()::test_should_return_created_time",
+  "tests/test_class.py::TestGCloudStorageClass::()::test_should_return_modified_time",
+  "tests/test_class.py::TestGCloudStorageClass::()::test_should_be_able_to_delete_files",
+  "tests/test_class.py::TestGCloudStorageClass::()::test_exists_method",
+  "tests/test_class.py::TestGCloudStorageClass::()::test_should_return_correct_file_size",
+  "tests/test_class.py::TestGCloudStorageClass::()::test_should_return_publicly_downloadable_url",
+  "tests/test_class.py::TestGCloudStorageClass::()::test_should_work_with_utf8",
+  "tests/test_class.py::TestGCloudStorageClass::()::test_should_be_able_to_list_dirs_and_files",
+  "tests/test_class.py::TestGCloudStorageClass::()::test_should_not_overwrite_files_on_save",
+  "tests/test_class.py::TestGCloudStorageClass::()::test_changed_files_should_be_reuploaded",
+  "tests/test_class.py::TestGCloudStorageClass::()::test_open_should_be_able_to_create_new_file",
+  "tests/test_class.py::TestGCloudStorageClass::()::test_use_unsigned_urls_option",
+  "tests/test_class.py::TestGCloudStorageClass::()::test_caching_disabled",
+  "tests/test_class.py::TestGCloudStorageClass::()::test_caching_enabled"
+]

django_gcloud_storage/__init__.py

@@ -11,6 +11,7 @@
 from django.core.exceptions import SuspiciousFileOperation
 from django.core.files.base import File
 from django.core.files.storage import Storage
+from django.core.cache import cache
 from django.utils.deconstruct import deconstructible
 from django.utils.encoding import force_text, smart_str
 from google.cloud import _helpers as gcloud_helpers
@@ -29,6 +30,23 @@
     # Fall back to Python 2's urllib2
     import urlparse
 
+DEFAULTS = {
+    # Prefix for cache entries to ensure uniqueness
+    "GCS_CACHE_PREFIX": "DJANGO_GCLOUD_STORAGE_URL_",
+    # How long a signed URL is valid for. Influences
+    # api call to google signing api (timestamp)
+    "GCS_TOKEN_VALIDITY_SECONDS": 3600,
+    # The cache should expire before the timestamp
+    # of the signed url. This value can be adjusted.
+    "GCS_CACHE_GRACE_TIME_SECONDS": 20,
+    # Add ability to disable and set disabled by default
+    "GCS_ENABLE_URL_CACHING": False,
+}
+
+
+def get_config_value(key):
+    return getattr(settings, key, DEFAULTS.get(key))
+
 
 def safe_join(base, path):
     base = force_text(base).replace("\\", "/").lstrip("/").rstrip("/") + "/"
@@ -133,6 +151,11 @@ def __init__(self, project=None, bucket=None, credentials_file_path=None, use_un
         else:
             self.use_unsigned_urls = getattr(settings, "GCS_USE_UNSIGNED_URLS", False)
 
+        self.gcs_cache_prefix = get_config_value("GCS_CACHE_PREFIX")
+        self.gcs_token_validity_seconds = get_config_value("GCS_TOKEN_VALIDITY_SECONDS")
+        self.gcs_cache_grace_time_seconds = get_config_value("GCS_CACHE_GRACE_TIME_SECONDS")
+        self.gcs_enable_cached_urls = get_config_value("GCS_ENABLE_URL_CACHING")
+
         self.bucket_subdir = ''  # TODO should be a parameter
 
     @property
@@ -256,6 +279,31 @@ def url(self, name):
         name = prepare_name(name)
 
         if self.use_unsigned_urls:
-          return "https://storage.googleapis.com/{}/{}".format(self.bucket.name, name)
+            return "https://storage.googleapis.com/{}/{}".format(self.bucket.name, name)
+
+        if self.gcs_enable_cached_urls:
+            # First check the cache for a valid entry
+            cache_key = "%s_%s" % (self.gcs_cache_prefix, name)
+            cached_url = cache.get(cache_key)
+            if cached_url:
+                return cached_url
+
+        # Need to keep "now" around for calculating cache expiry as we want it
+        # to be less than the signed_url cache time
+        now = datetime.datetime.now()
+        # Calculation used to determine when the signed url token expires
+        signature_expires = now + datetime.timedelta(seconds=self.gcs_token_validity_seconds)
+        # Use grace period from settings to work out when to expire cache entry.
+        # This will be before the signature expires
+        cache_expires = signature_expires - datetime.timedelta(seconds=self.gcs_cache_grace_time_seconds) - now
+        # Need the time differences in seconds for the django caching engine
+        cache_expires_seconds = int(cache_expires.total_seconds())
+        # Request the signed URL and store it in the cache.
+        blob = self.bucket.get_blob(name)
+        if not blob:
+            return None
+        signed_url = blob.generate_signed_url(signature_expires)
+        if signed_url and cache_expires_seconds >= 0 and self.gcs_enable_cached_urls:
+            cache.set(cache_key, signed_url, cache_expires_seconds)
 
-        return self.bucket.get_blob(name).generate_signed_url(expiration=datetime.datetime.now() + datetime.timedelta(hours=1))
+        return signed_url

setup.py

@@ -28,7 +28,7 @@ def get_version(*file_paths):
     history = f.read().replace('.. :changelog:', '')
 
 setup(
-    name='django-gcloud-storage',
+    name='django-gcloud-storage-mjc',
     version=version,
     description="""Django storage module implementation for Google Cloud Storage""",
     long_description=readme + '\n\n' + history,

tests/test_class.py

@@ -219,3 +219,17 @@ def test_use_unsigned_urls_option(self, storage, test_file):
         storage.use_unsigned_urls = False
         for i in ["Signature=", "GoogleAccessId=", "Expires="]:
             assert i not in url
+
+    def test_caching_disabled(self, storage, test_file):
+        storage.gcs_enable_cached_urls = False
+        url_short_expiry = storage.url(test_file)
+        storage.gcs_token_validity_seconds = storage.gcs_token_validity_seconds * 2
+        url_long_expiry = storage.url(test_file)
+        assert url_short_expiry != url_long_expiry
+
+    def test_caching_enabled(self, storage, test_file):
+        storage.gcs_enable_cached_urls = True
+        url_short_expiry = storage.url(test_file)
+        storage.gcs_token_validity_seconds = storage.gcs_token_validity_seconds * 2
+        url_long_expiry = storage.url(test_file)
+        assert url_short_expiry == url_long_expiry