diff --git a/.pytest_cache/README.md b/.pytest_cache/README.md new file mode 100644 index 0000000..bb78ba0 --- /dev/null +++ b/.pytest_cache/README.md @@ -0,0 +1,8 @@ +# pytest cache directory # + +This directory contains data from the pytest's cache plugin, +which provides the `--lf` and `--ff` options, as well as the `cache` fixture. + +**Do not** commit this to version control. + +See [the docs](https://docs.pytest.org/en/latest/cache.html) for more information. diff --git a/.pytest_cache/v/cache/lastfailed b/.pytest_cache/v/cache/lastfailed new file mode 100644 index 0000000..0b10245 --- /dev/null +++ b/.pytest_cache/v/cache/lastfailed @@ -0,0 +1,22 @@ +{ + "tests/test_class.py::TestGCloudStorageClass::()::test_changed_files_should_be_reuploaded": true, + "tests/test_class.py::TestGCloudStorageClass::()::test_exists_method": true, + "tests/test_class.py::TestGCloudStorageClass::()::test_open_should_be_able_to_create_new_file": true, + "tests/test_class.py::TestGCloudStorageClass::()::test_should_be_able_to_delete_files": true, + "tests/test_class.py::TestGCloudStorageClass::()::test_should_be_able_to_list_dirs_and_files": true, + "tests/test_class.py::TestGCloudStorageClass::()::test_should_be_able_to_save_and_open_files": true, + "tests/test_class.py::TestGCloudStorageClass::()::test_should_create_a_valid_bucket_object": true, + "tests/test_class.py::TestGCloudStorageClass::()::test_should_create_a_valid_client_object": true, + "tests/test_class.py::TestGCloudStorageClass::()::test_should_create_blob_at_correct_path": true, + "tests/test_class.py::TestGCloudStorageClass::()::test_should_not_overwrite_files_on_save": true, + "tests/test_class.py::TestGCloudStorageClass::()::test_should_return_correct_file_size": true, + "tests/test_class.py::TestGCloudStorageClass::()::test_should_return_created_time": true, + "tests/test_class.py::TestGCloudStorageClass::()::test_should_return_modified_time": true, + "tests/test_class.py::TestGCloudStorageClass::()::test_should_return_publicly_downloadable_url": true, + "tests/test_class.py::TestGCloudStorageClass::()::test_should_work_with_utf8": true, + "tests/test_class.py::TestGCloudStorageClass::()::test_use_unsigned_urls_option": true, + "tests/test_django.py": true, + "tests/test_django.py::TestApp::()::test_file_access": true, + "tests/test_django.py::TestApp::()::test_http_upload": true, + "tests/test_django.py::TestApp::()::test_manual_upload": true +} \ No newline at end of file diff --git a/.pytest_cache/v/cache/nodeids b/.pytest_cache/v/cache/nodeids new file mode 100644 index 0000000..b46b719 --- /dev/null +++ b/.pytest_cache/v/cache/nodeids @@ -0,0 +1,34 @@ +[ + "tests/test_class.py::TestSafeJoin::()::test_should_join_urls", + "tests/test_class.py::TestSafeJoin::()::test_should_not_break_on_slash_on_base", + "tests/test_class.py::TestSafeJoin::()::test_should_enforce_no_starting_slash_on_base", + "tests/test_class.py::TestSafeJoin::()::test_should_resolve_dots_to_absolute_path", + "tests/test_class.py::TestSafeJoin::()::test_should_resolve_multiple_slashes", + "tests/test_class.py::TestSafeJoin::()::test_should_not_allow_escaping_base_path", + "tests/test_class.py::TestSafeJoin::()::test_should_work_with_bytes", + "tests/test_class.py::TestSafeJoin::()::test_should_work_with_unicode_characters", + "tests/test_class.py::TestSafeJoin::()::test_should_normalize_system_dependant_slashes", + "tests/test_class.py::test_remove_prefix_function", + "tests/test_class.py::TestGCloudFile::()::test_should_be_able_to_read_and_write", + "tests/test_class.py::TestGCloudFile::()::test_small_temporary_files_should_not_be_rolled_over_to_disk", + "tests/test_class.py::TestGCloudFile::()::test_large_temporary_files_should_be_rolled_over_to_disk", + "tests/test_class.py::TestGCloudFile::()::test_modified_files_should_be_marked_as_dirty", + "tests/test_class.py::TestGCloudStorageClass::()::test_should_create_blob_at_correct_path", + "tests/test_class.py::TestGCloudStorageClass::()::test_should_create_a_valid_client_object", + "tests/test_class.py::TestGCloudStorageClass::()::test_should_create_a_valid_bucket_object", + "tests/test_class.py::TestGCloudStorageClass::()::test_should_be_able_to_save_and_open_files", + "tests/test_class.py::TestGCloudStorageClass::()::test_should_return_created_time", + "tests/test_class.py::TestGCloudStorageClass::()::test_should_return_modified_time", + "tests/test_class.py::TestGCloudStorageClass::()::test_should_be_able_to_delete_files", + "tests/test_class.py::TestGCloudStorageClass::()::test_exists_method", + "tests/test_class.py::TestGCloudStorageClass::()::test_should_return_correct_file_size", + "tests/test_class.py::TestGCloudStorageClass::()::test_should_return_publicly_downloadable_url", + "tests/test_class.py::TestGCloudStorageClass::()::test_should_work_with_utf8", + "tests/test_class.py::TestGCloudStorageClass::()::test_should_be_able_to_list_dirs_and_files", + "tests/test_class.py::TestGCloudStorageClass::()::test_should_not_overwrite_files_on_save", + "tests/test_class.py::TestGCloudStorageClass::()::test_changed_files_should_be_reuploaded", + "tests/test_class.py::TestGCloudStorageClass::()::test_open_should_be_able_to_create_new_file", + "tests/test_class.py::TestGCloudStorageClass::()::test_use_unsigned_urls_option", + "tests/test_class.py::TestGCloudStorageClass::()::test_caching_disabled", + "tests/test_class.py::TestGCloudStorageClass::()::test_caching_enabled" +] \ No newline at end of file diff --git a/django_gcloud_storage/__init__.py b/django_gcloud_storage/__init__.py index 6f7a1da..c09901b 100644 --- a/django_gcloud_storage/__init__.py +++ b/django_gcloud_storage/__init__.py @@ -11,6 +11,7 @@ from django.core.exceptions import SuspiciousFileOperation from django.core.files.base import File from django.core.files.storage import Storage +from django.core.cache import cache from django.utils.deconstruct import deconstructible from django.utils.encoding import force_text, smart_str from google.cloud import _helpers as gcloud_helpers @@ -29,6 +30,23 @@ # Fall back to Python 2's urllib2 import urlparse +DEFAULTS = { + # Prefix for cache entries to ensure uniqueness + "GCS_CACHE_PREFIX": "DJANGO_GCLOUD_STORAGE_URL_", + # How long a signed URL is valid for. Influences + # api call to google signing api (timestamp) + "GCS_TOKEN_VALIDITY_SECONDS": 3600, + # The cache should expire before the timestamp + # of the signed url. This value can be adjusted. + "GCS_CACHE_GRACE_TIME_SECONDS": 20, + # Add ability to disable and set disabled by default + "GCS_ENABLE_URL_CACHING": False, +} + + +def get_config_value(key): + return getattr(settings, key, DEFAULTS.get(key)) + def safe_join(base, path): base = force_text(base).replace("\\", "/").lstrip("/").rstrip("/") + "/" @@ -133,6 +151,11 @@ def __init__(self, project=None, bucket=None, credentials_file_path=None, use_un else: self.use_unsigned_urls = getattr(settings, "GCS_USE_UNSIGNED_URLS", False) + self.gcs_cache_prefix = get_config_value("GCS_CACHE_PREFIX") + self.gcs_token_validity_seconds = get_config_value("GCS_TOKEN_VALIDITY_SECONDS") + self.gcs_cache_grace_time_seconds = get_config_value("GCS_CACHE_GRACE_TIME_SECONDS") + self.gcs_enable_cached_urls = get_config_value("GCS_ENABLE_URL_CACHING") + self.bucket_subdir = '' # TODO should be a parameter @property @@ -256,6 +279,31 @@ def url(self, name): name = prepare_name(name) if self.use_unsigned_urls: - return "https://storage.googleapis.com/{}/{}".format(self.bucket.name, name) + return "https://storage.googleapis.com/{}/{}".format(self.bucket.name, name) + + if self.gcs_enable_cached_urls: + # First check the cache for a valid entry + cache_key = "%s_%s" % (self.gcs_cache_prefix, name) + cached_url = cache.get(cache_key) + if cached_url: + return cached_url + + # Need to keep "now" around for calculating cache expiry as we want it + # to be less than the signed_url cache time + now = datetime.datetime.now() + # Calculation used to determine when the signed url token expires + signature_expires = now + datetime.timedelta(seconds=self.gcs_token_validity_seconds) + # Use grace period from settings to work out when to expire cache entry. + # This will be before the signature expires + cache_expires = signature_expires - datetime.timedelta(seconds=self.gcs_cache_grace_time_seconds) - now + # Need the time differences in seconds for the django caching engine + cache_expires_seconds = int(cache_expires.total_seconds()) + # Request the signed URL and store it in the cache. + blob = self.bucket.get_blob(name) + if not blob: + return None + signed_url = blob.generate_signed_url(signature_expires) + if signed_url and cache_expires_seconds >= 0 and self.gcs_enable_cached_urls: + cache.set(cache_key, signed_url, cache_expires_seconds) - return self.bucket.get_blob(name).generate_signed_url(expiration=datetime.datetime.now() + datetime.timedelta(hours=1)) + return signed_url diff --git a/setup.py b/setup.py index e930fee..cfaeabb 100755 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ def get_version(*file_paths): history = f.read().replace('.. :changelog:', '') setup( - name='django-gcloud-storage', + name='django-gcloud-storage-mjc', version=version, description="""Django storage module implementation for Google Cloud Storage""", long_description=readme + '\n\n' + history, diff --git a/tests/test_class.py b/tests/test_class.py index b37dcbe..7b229f4 100644 --- a/tests/test_class.py +++ b/tests/test_class.py @@ -219,3 +219,17 @@ def test_use_unsigned_urls_option(self, storage, test_file): storage.use_unsigned_urls = False for i in ["Signature=", "GoogleAccessId=", "Expires="]: assert i not in url + + def test_caching_disabled(self, storage, test_file): + storage.gcs_enable_cached_urls = False + url_short_expiry = storage.url(test_file) + storage.gcs_token_validity_seconds = storage.gcs_token_validity_seconds * 2 + url_long_expiry = storage.url(test_file) + assert url_short_expiry != url_long_expiry + + def test_caching_enabled(self, storage, test_file): + storage.gcs_enable_cached_urls = True + url_short_expiry = storage.url(test_file) + storage.gcs_token_validity_seconds = storage.gcs_token_validity_seconds * 2 + url_long_expiry = storage.url(test_file) + assert url_short_expiry == url_long_expiry