[ENG-11015] [ENG-11692] IDS Phase 1 & 2 Project PR - WB Part

tests/core/test_utils.py

@@ -4,6 +4,10 @@
 import pytest
 
 from waterbutler.core import utils
+from waterbutler.core.path import WaterButlerPath
+from waterbutler.core.streams import EmptyStream
+
+from tests.utils import MockCoroutine, MockFileMetadata, MockProvider
 
 
 class TestAsyncRetry:
@@ -159,3 +163,43 @@ def test_content_disposition(self, filename, expected):
     def test_disposition_encoding(self, filename, expected):
         encoded = utils.encode_for_disposition(filename)
         assert encoded == expected
+
+
+class TestZipStreamGenerator:
+
+    @pytest.mark.asyncio
+    async def test_passes_child_metadata_to_download(self):
+        provider = MockProvider()
+        root_path = WaterButlerPath('/folder/', folder=True)
+        file_meta = MockFileMetadata()
+        provider.metadata = MockCoroutine(return_value=[file_meta])
+        provider.download = MockCoroutine(return_value=EmptyStream())
+
+        generator = utils.ZipStreamGenerator(provider, root_path, minimal=True)
+        zip_name, _ = await generator.__anext__()
+
+        assert zip_name == 'Foo.name'
+        provider.download.assert_called_once()
+        assert provider.download.call_args.kwargs['metadata'] is file_meta
+
+
+class TestZipStreamGeneratorPaginated:
+
+    @pytest.mark.asyncio
+    async def test_passes_child_metadata_to_download(self):
+        provider = MockProvider()
+        root_path = WaterButlerPath('/folder/', folder=True)
+        file_meta = MockFileMetadata()
+        provider.download = MockCoroutine(return_value=EmptyStream())
+
+        async def iter_children_pages(path, **kwargs):
+            yield [file_meta]
+
+        provider.iter_children_pages = iter_children_pages
+
+        generator = utils.ZipStreamGeneratorPaginated(provider, root_path, minimal=True)
+        zip_name, _ = await generator.__anext__()
+
+        assert zip_name == 'Foo.name'
+        provider.download.assert_called_once()
+        assert provider.download.call_args.kwargs['metadata'] is file_meta

tests/providers/osfstorage/fixtures.py

@@ -95,6 +95,11 @@ def folder_children_metadata():
     with open(os.path.join(os.path.dirname(__file__), 'fixtures/root_provider.json')) as fp:
         return json.load(fp)['folder_children_metadata']
 
+@pytest.fixture
+def folder_children_metadata_minimal():
+    with open(os.path.join(os.path.dirname(__file__), 'fixtures/root_provider.json')) as fp:
+        return json.load(fp)['folder_children_metadata_minimal']
+
 
 @pytest.fixture
 def download_response():

tests/providers/osfstorage/fixtures/root_provider.json

@@ -66,6 +66,32 @@
       "latestVersionSeen": null
     }
   ],
+  "folder_children_metadata_minimal": [
+    {
+      "id": "59c0054cb7d1c90114c456af",
+      "kind": "folder",
+      "name": "New Folder",
+      "path": "/59c0054cb7d1c90114c456af/"
+    },
+    {
+      "id": "59a9b637b7d1c903ab5a8f58",
+      "kind": "file",
+      "name": "one",
+      "path": "/59a9b637b7d1c903ab5a8f58"
+    },
+    {
+      "id": "59a9b628b7d1c903ab5a8f52",
+      "kind": "file",
+      "name": "doc.rst",
+      "path": "/59a9b628b7d1c903ab5a8f52"
+    },
+    {
+      "id": "59a5adb9b7d1c901cd40f0e9",
+      "kind": "file",
+      "name": "video.mp4",
+      "path": "/59a5adb9b7d1c901cd40f0e9"
+    }
+  ],
   "file_metadata": {
     "checkout": null,
     "contentType": null,

tests/providers/osfstorage/test_provider.py

@@ -24,7 +24,7 @@
                                                  file_lineage, file_metadata,
                                                  file_metadata_object, file_path,
                                                  folder_lineage, folder_metadata,
-                                                 folder_children_metadata, folder_path,
+                                                 folder_children_metadata, folder_children_metadata_minimal, folder_path,
                                                  revisions_metadata, revision_metadata_object,
                                                  download_response, download_path,
                                                  upload_response, upload_path, root_path,
@@ -70,13 +70,13 @@ class TestDownload:
 
     @pytest.mark.asyncio
     @pytest.mark.aiohttpretty
-    async def test_download_with_auth(self, provider_and_mock_one, download_response,
-                                      download_path, mock_time):
+    async def test_download(self, provider_and_mock_one, download_response,
+                            download_path, mock_time):
 
         provider, inner_provider = provider_and_mock_one
 
-        uri, params = build_signed_url_with_auth(provider, 'GET', download_path.identifier,
-                                                 'download', version=None, mode=None)
+        uri, params = build_signed_url_without_auth(provider, 'GET', download_path.identifier,
+                                                    'download', version=None, mode=None)
 
         aiohttpretty.register_json_uri('GET', uri,  body=download_response, params=params)
 
@@ -92,29 +92,6 @@ async def test_download_with_auth(self, provider_and_mock_one, download_response
         inner_provider.download.assert_called_once_with(path=expected_path,
                                                         display_name=expected_display_name)
 
-    @pytest.mark.asyncio
-    @pytest.mark.aiohttpretty
-    async def test_download_without_auth(self, provider_and_mock_one, download_response,
-                                         download_path, mock_time):
-        provider, inner_provider = provider_and_mock_one
-
-        provider.auth = {}
-        url, params = build_signed_url_without_auth(provider, 'GET', download_path.identifier,
-                                                    'download', version=None, mode=None)
-        aiohttpretty.register_json_uri('GET', url, params=params, body=download_response)
-
-        await provider.download(download_path)
-
-        assert provider.make_provider.called
-        assert inner_provider.download.called
-        assert aiohttpretty.has_call(method='GET', uri=url, params=params)
-        provider.make_provider.assert_called_once_with(download_response['settings'])
-
-        expected_path = WaterButlerPath('/' + download_response['data']['path'])
-        expected_display_name = download_response['data']['name']
-        inner_provider.download.assert_called_once_with(path=expected_path,
-                                                        display_name=expected_display_name)
-
     @pytest.mark.asyncio
     @pytest.mark.aiohttpretty
     async def test_download_without_id(self, provider_one, download_response, file_path,
@@ -141,7 +118,7 @@ async def test_download_with_display_name(self, provider_and_mock_one, download_
 
         provider, inner_provider = provider_and_mock_one
 
-        uri, params = build_signed_url_with_auth(provider, 'GET', download_path.identifier,
+        uri, params = build_signed_url_without_auth(provider, 'GET', download_path.identifier,
                                                  'download', version=None, mode=None)
 
         aiohttpretty.register_json_uri('GET', uri,  body=download_response, params=params)
@@ -157,6 +134,78 @@ async def test_download_with_display_name(self, provider_and_mock_one, download_
         inner_provider.download.assert_called_once_with(path=expected_path,
                                                         display_name=expected_name)
 
+    @pytest.mark.asyncio
+    @pytest.mark.aiohttpretty
+    async def test_download_with_embedded_storage_skips_osf(
+            self, provider_and_mock_one, download_response, download_path, mock_time):
+
+        provider, inner_provider = provider_and_mock_one
+        uri, params = build_signed_url_without_auth(
+            provider, 'GET', download_path.identifier, 'download', version=None, mode=None,
+        )
+
+        metadata = OsfStorageFileMetadata({
+            'kind': 'file',
+            'name': download_response['data']['name'],
+            'path': f'/{download_path.identifier}',
+            'storage': download_response,
+        }, str(download_path))
+
+        await provider.download(download_path, metadata=metadata)
+
+        assert not aiohttpretty.has_call(method='GET', uri=uri, params=params)
+        provider.make_provider.assert_called_once_with(download_response['settings'])
+        expected_path = WaterButlerPath('/' + download_response['data']['path'])
+        inner_provider.download.assert_called_once_with(
+            path=expected_path,
+            display_name=download_response['data']['name'],
+        )
+
+    @pytest.mark.asyncio
+    @pytest.mark.aiohttpretty
+    async def test_download_without_storage_falls_back_to_osf(
+            self, provider_and_mock_one, download_response, download_path, mock_time):
+
+        provider, inner_provider = provider_and_mock_one
+        uri, params = build_signed_url_without_auth(
+            provider, 'GET', download_path.identifier, 'download', version=None, mode=None,
+        )
+        aiohttpretty.register_json_uri('GET', uri, body=download_response, params=params)
+
+        metadata = OsfStorageFileMetadata({
+            'kind': 'file',
+            'name': download_response['data']['name'],
+            'path': f'/{download_path.identifier}',
+        }, str(download_path))
+
+        await provider.download(download_path, metadata=metadata)
+
+        assert aiohttpretty.has_call(method='GET', uri=uri, params=params)
+        inner_provider.download.assert_called_once()
+
+    @pytest.mark.asyncio
+    @pytest.mark.aiohttpretty
+    async def test_download_with_version_ignores_embedded_storage(
+            self, provider_and_mock_one, download_response, download_path, mock_time):
+
+        provider, inner_provider = provider_and_mock_one
+        uri, params = build_signed_url_without_auth(
+            provider, 'GET', download_path.identifier, 'download', version=1, mode=None,
+        )
+        aiohttpretty.register_json_uri('GET', uri, body=download_response, params=params)
+
+        metadata = OsfStorageFileMetadata({
+            'kind': 'file',
+            'name': download_response['data']['name'],
+            'path': f'/{download_path.identifier}',
+            'storage': download_response,
+        }, str(download_path))
+
+        await provider.download(download_path, metadata=metadata, version=1)
+
+        assert aiohttpretty.has_call(method='GET', uri=uri, params=params)
+        inner_provider.download.assert_called_once()
+
 
 class TestDelete:
 
@@ -210,7 +259,7 @@ async def test_delete_folder_contents(self, provider_one, file_path, folder_path
 
         children_url, params = build_signed_url_without_auth(provider_one, 'GET',
                                                              folder_path.identifier, 'children',
-                                                             user_id=provider_one.auth['id'])
+                                                             user_id=provider_one.auth['id'], minimal=False)
         aiohttpretty.register_json_uri('GET', children_url, params=params, status=200,
                                        body=folder_children_metadata)
 
@@ -227,7 +276,7 @@ class TestMetadata:
     async def test_provider_metadata_empty(self, provider_one, folder_path, mock_time):
 
         url, params = build_signed_url_without_auth(provider_one, 'GET', folder_path.identifier,
-                                                    'children', user_id=provider_one.auth['id'])
+                                                    'children', user_id=provider_one.auth['id'], minimal=False)
         aiohttpretty.register_json_uri('GET', url, params=params, status_code=200, body=[])
 
         res = await provider_one.metadata(folder_path)
@@ -241,7 +290,7 @@ async def test_provider_metadata_folder(self, provider_one, folder_path,
                                             folder_children_metadata, mock_time):
 
         url, params = build_signed_url_without_auth(provider_one, 'GET', folder_path.identifier,
-                                                    'children', user_id=provider_one.auth['id'])
+                                                    'children', user_id=provider_one.auth['id'], minimal=False)
         aiohttpretty.register_json_uri('GET', url, params=params, status=200,
                                        body=folder_children_metadata)
 
@@ -276,7 +325,7 @@ async def test_provider_metadata_without_id(self, provider_one, folder_path,
                                                 folder_children_metadata, mock_time):
 
         url, params = build_signed_url_without_auth(provider_one, 'GET', folder_path.identifier,
-                                                    'children')
+                                                    'children', minimal=False)
         aiohttpretty.register_json_uri('GET', url, params=params, status=200,
                                        body=folder_children_metadata)
         folder_path._parts[-1]._id = None
@@ -732,7 +781,7 @@ async def test_validate_path_404s(self, provider_one, file_lineage, mock_time):
     async def test_revalidate_path_new(self, provider_one, folder_path, folder_children_metadata,
                                        mock_time):
         url, params = build_signed_url_without_auth(provider_one, 'GET', folder_path.identifier,
-                                                    'children', user_id=provider_one.auth['id'])
+                                                    'children', user_id=provider_one.auth['id'], minimal=False)
         aiohttpretty.register_json_uri('GET', url, params=params, status=200,
                                        body=folder_children_metadata)
 
@@ -745,7 +794,7 @@ async def test_revalidate_path_new(self, provider_one, folder_path, folder_child
     async def test_revalidate_path_existing(self, provider_one, folder_path,
                                             folder_children_metadata, mock_time):
         url, params = build_signed_url_without_auth(provider_one, 'GET', folder_path.identifier,
-                                                    'children', user_id=provider_one.auth['id'])
+                                                    'children', user_id=provider_one.auth['id'], minimal=False)
         aiohttpretty.register_json_uri('GET', url, params=params, status=200,
                                        body=folder_children_metadata)
 
@@ -1253,3 +1302,38 @@ async def test_intra_copy_reject_by_quota(self, provider_one, auth, credentials,
         src_provider.can_intra_copy.assert_called_once_with(dst_provider, src_path)
         src_provider.intra_copy.assert_not_called()
         src_provider.download.assert_not_called()
+
+class TestMinimalMetadataDAZ:
+
+    @pytest.mark.asyncio
+    async def test_zip_requests_minimal_metadata(self, provider_one, folder_path, monkeypatch):
+        captured = {}
+
+        async def capture_zip(self, path, **kwargs):
+            captured.update(kwargs)
+            return object()
+
+        monkeypatch.setattr(
+            'waterbutler.core.provider.BaseProvider.zip',
+            capture_zip,
+        )
+
+        await provider_one.zip(folder_path)
+
+        assert captured.get('minimal') is True
+
+    @pytest.mark.asyncio
+    @pytest.mark.aiohttpretty
+    async def test_minimal_metadata(self, provider_one, folder_path, folder_children_metadata_minimal):
+
+        url, params = build_signed_url_without_auth(provider_one, 'GET', folder_path.identifier,
+                                                    'children', user_id=provider_one.auth['id'], minimal=True)
+        aiohttpretty.register_json_uri('GET', url, params=params, status=200,
+                                       body=folder_children_metadata_minimal)
+
+        metadata = await provider_one.metadata(folder_path, minimal=True)
+
+        assert [type(metadata_item) for metadata_item in metadata] == [OsfStorageFolderMetadata, OsfStorageFileMetadata, OsfStorageFileMetadata, OsfStorageFileMetadata]
+        assert [metadata_item.name for metadata_item in metadata] == ['New Folder', 'one', 'doc.rst', 'video.mp4']
+        assert [metadata_item.path for metadata_item in metadata] == ['/59c0054cb7d1c90114c456af/', '/59a9b637b7d1c903ab5a8f58', '/59a9b628b7d1c903ab5a8f52', '/59a5adb9b7d1c901cd40f0e9']
+        assert [metadata_item.provider for metadata_item in metadata] == ['osfstorage', 'osfstorage', 'osfstorage', 'osfstorage']

tests/server/api/v1/test_metadata_mixin.py

@@ -233,7 +233,7 @@ async def test_download_folder_as_zip(self, http_request, mock_stream):
         await handler.download_folder_as_zip()
 
         assert handler._headers['Content-Type'] == 'application/zip'
-        expected = 'attachment; filename="test_file.zip"; filename*=UTF-8\'\'test_file.zip'
+        expected = 'attachment; filename="guid1-test_file.zip"; filename*=UTF-8\'\'guid1-test_file.zip'
         assert handler._headers['Content-Disposition'] == expected
 
         handler.write_stream.assert_called_once_with(mock_stream)
@@ -249,7 +249,7 @@ async def test_download_folder_as_zip_root(self, http_request, mock_stream):
         await handler.download_folder_as_zip()
 
         assert handler._headers['Content-Type'] == 'application/zip'
-        expected = 'attachment; filename="MockProvider-archive.zip"; filename*=UTF-8\'\'MockProvider-archive.zip'
+        expected = 'attachment; filename="guid1-MockProvider-archive.zip"; filename*=UTF-8\'\'guid1-MockProvider-archive.zip'
         assert handler._headers['Content-Disposition'] == expected
 
         handler.write_stream.assert_called_once_with(mock_stream)

waterbutler/core/cache.py

@@ -0,0 +1,43 @@
+import asyncio
+
+
+class CacheableMetadataProviderProxy:
+    """Wraps a provider so that folder ``metadata`` requests are cached and the
+    metadata of any child folders is prefetched concurrently.
+
+    When a folder is listed, a metadata task is scheduled for each of its child
+    folders without awaiting them. Those tasks run on the event loop while files
+    are being downloaded, so the generator can issue all of its metadata requests
+    up front instead of fetching folder listings strictly one-at-a-time.
+
+    Any attribute that is not overridden here (``download``, ``path_from_metadata``,
+    etc.) is delegated to the wrapped provider.
+    """
+
+    def __init__(self, provider):
+        self.provider = provider
+        self.__cache = dict[str, asyncio.Task]()
+
+    def metadata_task(self, path, **kwargs):
+        key = path.identifier or str(path)
+        if key not in self.__cache:
+            self.__cache[key] = asyncio.create_task(
+                self.provider.metadata(path, **kwargs)
+            )
+        return self.__cache[key]
+
+    async def metadata(self, path, **kwargs):
+        if path.is_file:
+            return await self.provider.metadata(path, **kwargs)
+
+        items = await self.metadata_task(path, **kwargs)
+
+        for item in items:
+            if item.is_folder:
+                child = self.provider.path_from_metadata(path, item)
+                self.metadata_task(child, **kwargs)
+
+        return items
+
+    def __getattr__(self, name):
+        return getattr(self.provider, name)