Complete adding the optimize flag to the file plugin

Couldn't be done earlier because it was an API-breaking change and
passed an argument into the task which wasn't prepared to handle it.

Author: dralley

CHANGES/pulp_file/+sync-optimization.feature

@@ -0,0 +1 @@
+Added sync optimization that skips re-syncing when the remote manifest has not changed. An `optimize` flag on the sync endpoint (default `True`) allows forcing a full sync when needed.

pulp_file/app/serializers.py

@@ -16,6 +16,7 @@
     PublicationSerializer,
     RemoteSerializer,
     RepositorySerializer,
+    RepositorySyncURLSerializer,
     SingleArtifactContentUploadSerializer,
 )
 from pulpcore.plugin.util import get_domain_pk
@@ -139,11 +140,28 @@ class FileRepositorySerializer(RepositorySerializer):
         allow_null=True,
     )
 
+    last_sync_details = serializers.JSONField(
+        help_text=_("Details about the last sync of this repository."),
+        read_only=True,
+    )
+
     class Meta:
-        fields = RepositorySerializer.Meta.fields + ("autopublish", "manifest")
+        fields = RepositorySerializer.Meta.fields + ("autopublish", "manifest", "last_sync_details")
         model = FileRepository
 
 
+class FileRepositorySyncURLSerializer(RepositorySyncURLSerializer):
+    """
+    Serializer for File Repository Sync URL.
+    """
+
+    optimize = serializers.BooleanField(
+        help_text=_("Whether or not to optimize sync."),
+        required=False,
+        default=True,
+    )
+
+
 class FileRemoteSerializer(RemoteSerializer):
     """
     Serializer for File Remotes.

pulp_file/app/viewsets.py

@@ -18,7 +18,6 @@
 )
 from pulpcore.plugin.serializers import (
     AsyncOperationResponseSerializer,
-    RepositorySyncURLSerializer,
     TaskGroupOperationResponseSerializer,
 )
 from pulpcore.plugin.tasking import dispatch
@@ -55,6 +54,7 @@
     FilePublicationSerializer,
     FileRemoteSerializer,
     FileRepositorySerializer,
+    FileRepositorySyncURLSerializer,
 )
 
 
@@ -257,14 +257,14 @@ class FileRepositoryViewSet(RepositoryViewSet, ModifyRepositoryActionMixin, Role
         summary="Sync from a remote",
         responses={202: AsyncOperationResponseSerializer},
     )
-    @action(detail=True, methods=["post"], serializer_class=RepositorySyncURLSerializer)
+    @action(detail=True, methods=["post"], serializer_class=FileRepositorySyncURLSerializer)
     def sync(self, request, pk):
         """
         Synchronizes a repository.
 
         The ``repository`` field has to be provided.
         """
-        serializer = RepositorySyncURLSerializer(
+        serializer = FileRepositorySyncURLSerializer(
             data=request.data, context={"request": request, "repository_pk": pk}
         )
         serializer.is_valid(raise_exception=True)
@@ -273,6 +273,7 @@ def sync(self, request, pk):
         remote = serializer.validated_data.get("remote", repository.remote)
 
         mirror = serializer.validated_data.get("mirror", False)
+        optimize = serializer.validated_data.get("optimize", True)  # noqa
         if mirror and repository.autopublish:
             raise ValidationError("Cannot use mirror mode with autopublished repository.")
         result = dispatch(
@@ -283,6 +284,7 @@ def sync(self, request, pk):
                 "remote_pk": str(remote.pk),
                 "repository_pk": str(repository.pk),
                 "mirror": mirror,
+                "optimize": optimize,
             },
         )
         return OperationPostponedResponse(result, request)

pulp_file/tests/functional/api/test_acs.py

@@ -4,7 +4,7 @@
 
 import pytest
 
-from pulpcore.client.pulp_file import RepositorySyncURL
+from pulpcore.client.pulp_file import FileRepositorySyncURL
 from pulpcore.client.pulp_file.exceptions import ApiException
 from pulpcore.tests.functional.utils import (
     download_file,
@@ -108,7 +108,7 @@ def test_acs_sync(
     assert acs_server.requests_record[0].path == basic_manifest_path
 
     # Sync the repository
-    repository_sync_data = RepositorySyncURL(remote=main_remote.pulp_href)
+    repository_sync_data = FileRepositorySyncURL(remote=main_remote.pulp_href)
     monitor_task(
         file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, repository_sync_data).task
     )
@@ -167,7 +167,7 @@ def test_acs_sync_with_paths(
     assert expected_request_paths == actual_requested_paths
 
     # Sync the repository
-    repository_sync_data = RepositorySyncURL(remote=main_remote.pulp_href)
+    repository_sync_data = FileRepositorySyncURL(remote=main_remote.pulp_href)
     monitor_task(
         file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, repository_sync_data).task
     )
@@ -231,7 +231,7 @@ def test_serving_acs_content(
     )
 
     # Sync the repository
-    repository_sync_data = RepositorySyncURL(remote=main_remote.pulp_href)
+    repository_sync_data = FileRepositorySyncURL(remote=main_remote.pulp_href)
     monitor_task(
         file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, repository_sync_data).task
     )

pulp_file/tests/functional/api/test_auto_publish.py

@@ -3,7 +3,7 @@
 import pytest
 
 from pulpcore.client.pulp_file import (
-    RepositorySyncURL,
+    FileRepositorySyncURL,
 )
 from pulpcore.tests.functional.utils import get_files_in_manifest
 
@@ -48,7 +48,7 @@ def test_auto_publish_and_distribution(
     expected_files = get_files_in_manifest(remote.url)
 
     # Sync from the remote
-    body = RepositorySyncURL(remote=remote.pulp_href)
+    body = FileRepositorySyncURL(remote=remote.pulp_href)
     monitor_task(file_bindings.RepositoriesFileApi.sync(repo.pulp_href, body).task)
     repo = file_bindings.RepositoriesFileApi.read(repo.pulp_href)
 
@@ -73,7 +73,7 @@ def test_auto_publish_and_distribution(
     assert files_in_first_publication == expected_files
 
     # Assert that mirror=True is not allowed when autopublish=True
-    body = RepositorySyncURL(remote=remote.pulp_href, mirror=True)
+    body = FileRepositorySyncURL(remote=remote.pulp_href, mirror=True)
     with pytest.raises(file_bindings.ApiException) as exc:
         file_bindings.RepositoriesFileApi.sync(repo.pulp_href, body)
     assert exc.value.status == 400

pulp_file/tests/functional/api/test_bad_sync.py

@@ -5,7 +5,7 @@
 import pytest
 from aiohttp import web
 
-from pulpcore.client.pulp_file import RepositorySyncURL
+from pulpcore.client.pulp_file import FileRepositorySyncURL
 
 
 @pytest.fixture
@@ -23,7 +23,7 @@ def _perform_sync(url, policy="immediate"):
         }
         remote = gen_object_with_cleanup(file_bindings.RemotesFileApi, remote_data)
 
-        body = RepositorySyncURL(remote=remote.pulp_href)
+        body = FileRepositorySyncURL(remote=remote.pulp_href)
         monitor_task(file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, body).task)
         return file_repo
 

pulp_file/tests/functional/api/test_distributed_publication.py

@@ -8,7 +8,11 @@
 import requests
 from django.conf import settings
 
-from pulpcore.client.pulp_file import FileFileDistribution, FileFilePublication, RepositorySyncURL
+from pulpcore.client.pulp_file import (
+    FileFileDistribution,
+    FileFilePublication,
+    FileRepositorySyncURL,
+)
 
 
 @dataclass
@@ -82,7 +86,7 @@ def ctx(
         # Sync to get files into the repo
         monitor_task(
             file_bindings.RepositoriesFileApi.sync(
-                repo.pulp_href, RepositorySyncURL(remote=remote.pulp_href)
+                repo.pulp_href, FileRepositorySyncURL(remote=remote.pulp_href)
             ).task
         )
         repo = file_bindings.RepositoriesFileApi.read(repo.pulp_href)

pulp_file/tests/functional/api/test_download_policies.py

@@ -10,7 +10,7 @@
 from aiohttp.client_exceptions import ClientResponseError
 from bs4 import BeautifulSoup
 
-from pulpcore.client.pulp_file import FileFilePublication, RepositorySyncURL
+from pulpcore.client.pulp_file import FileFilePublication, FileRepositorySyncURL
 from pulpcore.tests.functional.utils import download_file, get_files_in_manifest
 
 OBJECT_STORAGES = (
@@ -67,7 +67,7 @@ def test_download_policy(
     expected_files = get_files_in_manifest(remote.url)
 
     # Sync from the remote and assert that a new repository version is created
-    body = RepositorySyncURL(remote=remote.pulp_href)
+    body = FileRepositorySyncURL(remote=remote.pulp_href)
     monitor_task(file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, body).task)
     file_repo = file_bindings.RepositoriesFileApi.read(file_repo.pulp_href)
     assert file_repo.latest_version_href.endswith("/versions/1/")

pulp_file/tests/functional/api/test_git_sync.py

@@ -2,7 +2,7 @@
 
 import pytest
 
-from pulpcore.client.pulp_file import RepositorySyncURL
+from pulpcore.client.pulp_file import FileRepositorySyncURL
 from pulpcore.tests.functional.utils import PulpTaskError
 
 GIT_REMOTE_URL = "https://github.com/pulp/pulp-smash.git"
@@ -65,7 +65,7 @@ def test_git_sync(file_bindings, file_repo, file_git_remote_factory, monitor_tas
     """Test syncing from a public Git repository."""
     remote = file_git_remote_factory(url=GIT_REMOTE_URL, git_ref=git_ref)
 
-    body = RepositorySyncURL(remote=remote.pulp_href)
+    body = FileRepositorySyncURL(remote=remote.pulp_href)
     monitor_task(file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, body).task)
 
     file_repo = file_bindings.RepositoriesFileApi.read(file_repo.pulp_href)
@@ -81,7 +81,7 @@ def test_git_sync_idempotent(file_bindings, file_repo, file_git_remote_factory,
     """Syncing the same Git ref twice should not create a new repository version."""
     remote = file_git_remote_factory(url=GIT_REMOTE_URL, git_ref="main")
 
-    body = RepositorySyncURL(remote=remote.pulp_href)
+    body = FileRepositorySyncURL(remote=remote.pulp_href)
     monitor_task(file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, body).task)
 
     file_repo = file_bindings.RepositoriesFileApi.read(file_repo.pulp_href)
@@ -104,7 +104,7 @@ def test_git_sync_invalid_url(file_bindings, file_repo, file_git_remote_factory,
     """Syncing with an invalid Git URL should raise a task error."""
     remote = file_git_remote_factory(url="https://invalid.example.com/no-such-repo.git")
 
-    body = RepositorySyncURL(remote=remote.pulp_href)
+    body = FileRepositorySyncURL(remote=remote.pulp_href)
     with pytest.raises(PulpTaskError) as exc:
         monitor_task(file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, body).task)
     assert "Failed to clone git repository" in exc.value.task.error["description"]
@@ -115,7 +115,7 @@ def test_git_sync_invalid_ref(file_bindings, file_repo, file_git_remote_factory,
     """Syncing with a non-existent git ref should raise a task error."""
     remote = file_git_remote_factory(url=GIT_REMOTE_URL, git_ref="this-ref-does-not-exist-abc123")
 
-    body = RepositorySyncURL(remote=remote.pulp_href)
+    body = FileRepositorySyncURL(remote=remote.pulp_href)
     with pytest.raises(PulpTaskError) as exc:
         monitor_task(file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, body).task)
     error_desc = exc.value.task.error["description"]
@@ -135,7 +135,7 @@ def test_git_sync_lfs(
     remote = file_git_remote_factory(url=LFS_REMOTE_URL)
     file_repo = file_repo_with_auto_publish
 
-    body = RepositorySyncURL(remote=remote.pulp_href)
+    body = FileRepositorySyncURL(remote=remote.pulp_href)
     monitor_task(file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, body).task)
 
     file_repo = file_bindings.RepositoriesFileApi.read(file_repo.pulp_href)

pulp_file/tests/functional/api/test_publish.py

@@ -8,7 +8,7 @@
 
 from pulpcore.client.pulp_file import (
     FileFilePublication,
-    RepositorySyncURL,
+    FileRepositorySyncURL,
 )
 from pulpcore.client.pulp_file.exceptions import ApiException
 from pulpcore.tests.functional.utils import download_file
@@ -29,7 +29,7 @@ def test_crd_publications(
 
     # Sync from the remote
     initial_repo_version = file_repo.latest_version_href
-    body = RepositorySyncURL(remote=remote.pulp_href)
+    body = FileRepositorySyncURL(remote=remote.pulp_href)
     monitor_task(file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, body).task)
     first_repo_version_href = file_bindings.RepositoriesFileApi.read(
         file_repo.pulp_href