Pull-through caching first cut

pulp_rust/app/migrations/0002_alter_rustcontent_cksum_alter_rustcontent_vers.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.2.5 on 2026-01-28 04:23
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('rust', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='rustcontent',
+            name='cksum',
+            field=models.CharField(db_index=True, max_length=64),
+        ),
+        migrations.AlterField(
+            model_name='rustcontent',
+            name='vers',
+            field=models.CharField(db_index=True, max_length=64),
+        ),
+    ]

pulp_rust/app/models.py

@@ -40,10 +40,10 @@ class RustContent(Content):
     name = models.CharField(max_length=255, blank=False, null=False, db_index=True)
 
     # Semantic version string following SemVer 2.0.0 specification
-    vers = models.CharField(max_length=64, blank=False, null=False)
+    vers = models.CharField(max_length=64, blank=False, null=False, db_index=True)
 
     # SHA256 checksum (hex-encoded) of the .crate tarball file for verification
-    cksum = models.CharField(max_length=64, blank=False, null=False)
+    cksum = models.CharField(max_length=64, blank=False, null=False, db_index=True)
 
     # Indicates if this version has been yanked (deprecated/removed from use)
     # Yanked versions can still be used by existing Cargo.lock files but won't be selected

pulp_rust/app/serializers.py

@@ -1,12 +1,31 @@
-from gettext import gettext as _
+import logging
 
+from gettext import gettext as _
 from rest_framework import serializers
 
 from pulpcore.plugin import models as core_models
 from pulpcore.plugin import serializers as core_serializers
 
 from . import models
 
+log = logging.getLogger(__name__)
+
+
+class IndexRootSerializer(serializers.Serializer):
+    """
+    A Serializer for summary information of an index.
+    """
+
+    dl = serializers.CharField(help_text=_("URL of the index root"), read_only=True)
+    api = serializers.CharField(help_text=_("URL of the API root"), read_only=True)
+    auth_required = serializers.BooleanField(
+        help_text=_(
+            "Indicates whether this is a private registry that requires all operations to "
+            "be authenticated"
+        ),
+        read_only=True,
+    )
+
 
 class RustDependencySerializer(serializers.ModelSerializer):
     """
@@ -181,28 +200,14 @@ class Meta:
 class RustRemoteSerializer(core_serializers.RemoteSerializer):
     """
     A Serializer for RustRemote.
-
-    Add any new fields if defined on RustRemote.
-    Similar to the example above, in RustContentSerializer.
-    Additional validators can be added to the parent validators list
-
-    For example::
-
-    class Meta:
-        validators = core_serializers.RemoteSerializer.Meta.validators
-            + [myValidator1, myValidator2]
-
-    By default the 'policy' field in core_serializers.RemoteSerializer only validates the choice
-    'immediate'. To add on-demand support for more 'policy' options, e.g. 'streamed' or
-    'on_demand', re-define the 'policy' option as follows::
+    """
 
     policy = serializers.ChoiceField(
         help_text="The policy to use when downloading content. The possible values include: "
-                  "'immediate', 'on_demand', and 'streamed'. 'immediate' is the default.",
+        "'immediate', 'on_demand', and 'streamed'. 'streamed' is the default.",
         choices=models.Remote.POLICY_CHOICES,
-        default=models.Remote.IMMEDIATE
+        default=models.Remote.STREAMED,
     )
-    """
 
     class Meta:
         fields = core_serializers.RemoteSerializer.Meta.fields
@@ -258,3 +263,31 @@ class Meta:
     class Meta:
         fields = core_serializers.DistributionSerializer.Meta.fields + ("allow_uploads", "remote")
         model = models.RustDistribution
+
+
+class RepositoryAddCachedContentSerializer(
+    core_serializers.ValidateFieldsMixin, serializers.Serializer
+):
+    remote = core_serializers.DetailRelatedField(
+        required=False,
+        view_name_pattern=r"remotes(-.*/.*)-detail",
+        queryset=models.Remote.objects.all(),
+        help_text=_(
+            "A remote to use to identify content that was cached. This will override a "
+            "remote set on repository."
+        ),
+    )
+
+    def validate(self, data):
+        data = super().validate(data)
+        repository = None
+        if "repository_pk" in self.context:
+            repository = models.Repository.objects.get(pk=self.context["repository_pk"])
+        remote = data.get("remote", None) or getattr(repository, "remote", None)
+
+        if not remote:
+            raise serializers.ValidationError(
+                {"remote": _("This field is required since a remote is not set on the repository.")}
+            )
+        self.check_cross_domains({"repository": repository, "remote": remote})
+        return data

pulp_rust/app/settings.py

@@ -1,6 +1,3 @@
-"""
-Check `Plugin Writer's Guide`_ for more details.
+import socket
 
-.. _Plugin Writer's Guide:
-    https://pulpproject.org/pulpcore/docs/dev/
-"""
+CRATES_IO_API_HOSTNAME = "https://" + socket.getfqdn()

pulp_rust/app/tasks/__init__.py

@@ -1 +1,2 @@
 from .synchronizing import synchronize  # noqa
+from .streaming import add_cached_content_to_repository  # noqa

pulp_rust/app/tasks/streaming.py

@@ -0,0 +1,40 @@
+import datetime
+
+from asgiref.sync import sync_to_async
+
+from pulpcore.plugin.models import Content, ContentArtifact, RemoteArtifact
+from pulpcore.plugin.tasking import add_and_remove
+
+from pulp_rust.app.models import RustRemote, RustRepository
+
+
+async def aadd_and_remove(*args, **kwargs):
+    return await sync_to_async(add_and_remove)(*args, **kwargs)
+
+
+def add_cached_content_to_repository(repository_pk=None, remote_pk=None):
+    """
+    Create a new repository version by adding content that was cached by pulpcore-content when
+    streaming it from a remote.
+
+    Args:
+        repository_pk (uuid): The primary key for a Repository for which a new Repository Version
+            should be created.
+        remote_pk (uuid): The primary key for a Remote which will be used to identify Content
+            created by pulpcore-content when it streamed it to clients.
+    """
+    repository = RustRepository.objects.get(pk=repository_pk)
+    remote = RustRemote.objects.get(pk=remote_pk)
+
+    latest_version = repository.latest_version()
+
+    if latest_version.number == 0:
+        date_min = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)
+    else:
+        date_min = latest_version.pulp_created
+    with repository.new_version(base_version=None) as new_version:
+        ca_id_list = RemoteArtifact.objects.filter(
+            remote=remote, pulp_created__gte=date_min
+        ).values_list("content_artifact")
+        content_list = ContentArtifact.objects.filter(pk__in=ca_id_list).values_list("content")
+        new_version.add_content(Content.objects.filter(pk__in=content_list))

pulp_rust/app/tasks/synchronizing.py

@@ -73,6 +73,8 @@ async def run(self):
         downloader = self.remote.get_downloader(url=self.remote.url)
         result = await downloader.run()
         # Use ProgressReport to report progress
+        raise NotImplementedError("Not implemented")
+
         for entry in self.read_my_metadata_file_somehow(result.path):
             unit = RustContent(entry)  # make the content unit in memory-only
             artifact = Artifact(entry)  # make Artifact in memory-only

pulp_rust/app/urls.py

@@ -0,0 +1,24 @@
+from django.conf import settings
+from django.urls import path
+
+from pulp_rust.app.views import IndexRoot, CargoIndexApiViewSet, CargoDownloadApiView
+
+if settings.DOMAIN_ENABLED:
+    CRATES_IO_URL = "pulp/cargo/<slug:pulp_domain>/<slug:repo>/"
+else:
+    CRATES_IO_URL = "pulp/cargo/<slug:repo>/"
+
+
+urlpatterns = [
+    path(
+        CRATES_IO_URL + "api/v1/crates/<str:package>/<str:version>/<path:rest>",
+        CargoDownloadApiView.as_view(),
+        name="cargo-download-api",
+    ),
+    path(CRATES_IO_URL + "config.json", IndexRoot.as_view({"get": "retrieve"}), name="index-root"),
+    path(
+        CRATES_IO_URL + "<path:path>",
+        CargoIndexApiViewSet.as_view({"get": "retrieve"}),
+        name="cargo-index-api",
+    ),
+]