Add ability to sync by digests

fixes: #1909
Assisted by: claude-sonnet-4.6
https://redhat.atlassian.net/browse/PULP-1740

Author: gerrod3

CHANGES/+include_exclude_tags.removal

@@ -0,0 +1 @@
+Renamed `include_tags` and `exclude_tags` to `includes` and `excludes` on the remote.

CHANGES/1909.feature

@@ -0,0 +1 @@
+Added support for syncing manifests by digest through new `includes` field on the remote.

pulp_container/app/migrations/0048_containerremote_includes_excludes.py

@@ -0,0 +1,64 @@
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+def migrate_to_includes_excludes(apps, schema_editor):
+    """Copy include_tags -> includes, exclude_tags -> excludes."""
+    ContainerRemote = apps.get_model("container", "ContainerRemote")
+    remotes = []
+    for remote in ContainerRemote.objects.only("include_tags", "exclude_tags").iterator():
+        remote.includes = remote.include_tags or None
+        remote.excludes = remote.exclude_tags or None
+        remotes.append(remote)
+        if len(remotes) > 1000:
+            ContainerRemote.objects.bulk_update(remotes, fields=["includes", "excludes"])
+            remotes = []
+    if remotes:
+        ContainerRemote.objects.bulk_update(remotes, fields=["includes", "excludes"])
+
+def down_migrate_to_include_exclude_tags(apps, schema_editor):
+    """Copy includes + excludes -> include_tags + exclude_tags."""
+    ContainerRemote = apps.get_model("container", "ContainerRemote")
+    remotes = []
+    for remote in ContainerRemote.objects.only("includes", "excludes").iterator():
+        remote.include_tags = remote.includes or None
+        remote.exclude_tags = remote.excludes or None
+        remotes.append(remote)
+        if len(remotes) > 1000:
+            ContainerRemote.objects.bulk_update(remotes, fields=["include_tags", "exclude_tags"])
+            remotes = []
+    if remotes:
+        ContainerRemote.objects.bulk_update(remotes, fields=["include_tags", "exclude_tags"])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("container", "0047_containernamespace_pulp_labels"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="containerremote",
+            name="includes",
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=models.TextField(null=True),
+                null=True,
+                size=None,
+            ),
+        ),
+        migrations.AddField(
+            model_name="containerremote",
+            name="excludes",
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=models.TextField(null=True),
+                null=True,
+                size=None,
+            ),
+        ),
+        # 2. Copy existing data.
+        migrations.RunPython(
+            migrate_to_includes_excludes,
+            down_migrate_to_include_exclude_tags,
+        ),
+    ]

pulp_container/app/models.py

@@ -483,17 +483,22 @@ class ContainerRemote(Remote, AutoAddObjPermsMixin):
         upstream_name (models.TextField): The name of the image at the remote.
         include_foreign_layers (models.BooleanField): Foreign layers in the remote
             are included. They are not included by default.
-        include_tags (fields.ArrayField): List of tags to include during sync.
-        exclude_tags (fields.ArrayField): List of tags to exclude during sync.
+        includes (fields.ArrayField): List of tags (with optional wildcards) and/or
+            digests (sha256:<hex>) to sync.
+        excludes (fields.ArrayField): List of tag patterns to exclude from sync.
         sigstore (models.TextField): The URL to a sigstore where signatures of container images
             should be synced from.
     """
 
     upstream_name = models.TextField(db_index=True)
     include_foreign_layers = models.BooleanField(default=False)
+    includes = fields.ArrayField(models.TextField(null=True), null=True)
+    excludes = fields.ArrayField(models.TextField(null=True), null=True)
+    sigstore = models.TextField(null=True)
+
+    # Deprecated: kept for ZDT upgrades
     include_tags = fields.ArrayField(models.TextField(null=True), null=True)
     exclude_tags = fields.ArrayField(models.TextField(null=True), null=True)
-    sigstore = models.TextField(null=True)
 
     TYPE = "container"
 

pulp_container/app/serializers.py

@@ -296,27 +296,58 @@ class ContainerRemoteSerializer(RemoteSerializer):
     upstream_name = serializers.CharField(
         required=True, allow_blank=False, help_text=_("Name of the upstream repository")
     )
+    includes = serializers.ListField(
+        child=serializers.CharField(max_length=255),
+        allow_null=True,
+        required=False,
+        help_text=_(
+            "A list of tags (wildcards *, ? are recognized) and/or digests "
+            "(format: 'sha256:<hex>') to include during sync. "
+            "'includes' is evaluated before 'excludes'."
+        ),
+    )
+    excludes = serializers.ListField(
+        child=serializers.CharField(max_length=255),
+        allow_null=True,
+        required=False,
+        help_text=_(
+            "A list of tag patterns to exclude during sync. "
+            "Wildcards *, ? are recognized. "
+            "'excludes' is evaluated after 'includes'."
+        ),
+    )
+
+    # ---------------------------------------------------------------------------
+    # Deprecated write-only fields kept for backwards compatibility.
+    # They are merged into `includes` / `excludes` during validation.
+    # ---------------------------------------------------------------------------
     include_tags = serializers.ListField(
         child=serializers.CharField(max_length=255),
         allow_null=True,
         required=False,
-        help_text=_("""
-            A list of tags to include during sync.
-            Wildcards *, ? are recognized.
-            'include_tags' is evaluated before 'exclude_tags'.
-            """),
+        write_only=True,
+        help_text=_("Deprecated. Use 'includes' instead."),
     )
     exclude_tags = serializers.ListField(
         child=serializers.CharField(max_length=255),
         allow_null=True,
         required=False,
-        help_text=_("""
-            A list of tags to exclude during sync.
-            Wildcards *, ? are recognized.
-            'exclude_tags' is evaluated after 'include_tags'.
-            """),
+        write_only=True,
+        help_text=_("Deprecated. Use 'excludes' instead."),
     )
 
+    def validate(self, data):
+        include_tags = data.pop("include_tags", None)
+        exclude_tags = data.pop("exclude_tags", None)
+
+        if include_tags:
+            data["includes"] = list(data.get("includes") or []) + list(include_tags)
+
+        if exclude_tags:
+            data["excludes"] = list(data.get("excludes") or []) + list(exclude_tags)
+
+        return data
+
     policy = serializers.ChoiceField(
         help_text="""
         immediate - All manifests and blobs are downloaded and saved during a sync.
@@ -337,6 +368,8 @@ class ContainerRemoteSerializer(RemoteSerializer):
     class Meta:
         fields = RemoteSerializer.Meta.fields + (
             "upstream_name",
+            "includes",
+            "excludes",
             "include_tags",
             "exclude_tags",
             "sigstore",