Rework Rust plugin + domain support + testing

Assisted-By: claude-opus-4.6

Author: dralley

pulp_rust/app/models.py

@@ -1,3 +1,5 @@
+import json
+import urllib.request
 from logging import getLogger
 
 from django.db import models
@@ -13,6 +15,31 @@
 logger = getLogger(__name__)
 
 
+def _strip_sparse_prefix(url):
+    """Strip the sparse+ prefix from a Cargo registry URL."""
+    if url.startswith("sparse+"):
+        return url[len("sparse+") :]
+    return url
+
+
+def _parse_crate_relative_path(relative_path):
+    """
+    Parse crate name and version from a relative path.
+
+    Expected format: {name}/{name}-{version}.crate
+    Returns: (crate_name, version)
+    """
+    # "serde/serde-1.0.0.crate" -> "serde-1.0.0.crate"
+    filename = relative_path.rsplit("/", 1)[-1]
+    # "serde-1.0.0.crate" -> "serde-1.0.0"
+    stem = filename[: -len(".crate")]
+    # "serde/serde-1.0.0.crate" -> "serde"
+    crate_name = relative_path.split("/", 1)[0]
+    # "serde-1.0.0" -> "1.0.0"
+    version = stem[len(crate_name) + 1 :]
+    return crate_name, version
+
+
 class RustContent(Content):
     """
     The "rust" content type representing a Cargo package version.
@@ -73,6 +100,22 @@ class RustContent(Content):
 
     _pulp_domain = models.ForeignKey("core.Domain", default=get_domain_pk, on_delete=models.PROTECT)
 
+    @staticmethod
+    def init_from_artifact_and_relative_path(artifact, relative_path):
+        """
+        Create an unsaved RustContent from a downloaded .crate artifact.
+
+        Called by pulpcore's content handler during pull-through caching.
+        Only populates name, version, and checksum -- dependency and feature
+        metadata is served from the upstream sparse index via the proxy.
+        """
+        crate_name, version = _parse_crate_relative_path(relative_path)
+        return RustContent(
+            name=crate_name,
+            vers=version,
+            cksum=artifact.sha256,
+        )
+
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
         unique_together = (("name", "vers", "_pulp_domain"),)
@@ -159,25 +202,58 @@ class RustRemote(Remote):
     """
     A Remote for RustContent.
 
-    Define any additional fields for your new remote if needed.
+    The `url` field should point to the sparse index root, optionally prefixed
+    with `sparse+` (e.g. `sparse+https://index.crates.io/`).
     """
 
     TYPE = "rust"
 
+    def get_remote_artifact_url(self, relative_path=None, request=None):
+        """
+        Construct the upstream download URL for a .crate file.
+
+        Fetches config.json from the index root to obtain the `dl` template,
+        then substitutes {crate} and {version} markers per the Cargo spec.
+        """
+        if relative_path is None or not relative_path.endswith(".crate"):
+            return None
+
+        crate_name, version = _parse_crate_relative_path(relative_path)
+        index_url = _strip_sparse_prefix(self.url).rstrip("/")
+
+        # TODO: Cache the config.json response to avoid fetching it on every request.
+        config_url = f"{index_url}/config.json"
+        response = urllib.request.urlopen(config_url)
+        config = json.loads(response.read())
+        dl_template = config["dl"]
+
+        if "{crate}" in dl_template or "{version}" in dl_template:
+            return dl_template.replace("{crate}", crate_name).replace("{version}", version)
+        else:
+            # No markers: per Cargo spec, append /{crate}/{version}/download
+            return f"{dl_template.rstrip('/')}/{crate_name}/{version}/download"
+
+    @staticmethod
+    def get_remote_artifact_content_type(relative_path=None):
+        """Return the content type for the given relative path."""
+        if relative_path and relative_path.endswith(".crate"):
+            return RustContent
+        return None
+
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
 
 
 class RustRepository(Repository):
     """
     A Repository for RustContent.
-
-    Define any additional fields for your new repository if needed.
     """
 
     TYPE = "rust"
 
     CONTENT_TYPES = [RustContent]
+    REMOTE_TYPES = [RustRemote]
+    PULL_THROUGH_SUPPORTED = True
 
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"

pulp_rust/app/serializers.py

@@ -204,9 +204,9 @@ class RustRemoteSerializer(core_serializers.RemoteSerializer):
 
     policy = serializers.ChoiceField(
         help_text="The policy to use when downloading content. The possible values include: "
-        "'immediate', 'on_demand', and 'streamed'. 'streamed' is the default.",
+        "'immediate', 'on_demand', and 'streamed'. 'on_demand' is the default.",
         choices=models.Remote.POLICY_CHOICES,
-        default=models.Remote.STREAMED,
+        default=models.Remote.ON_DEMAND,
     )
 
     class Meta:

pulp_rust/app/tasks/streaming.py

@@ -12,6 +12,7 @@ async def aadd_and_remove(*args, **kwargs):
     return await sync_to_async(add_and_remove)(*args, **kwargs)
 
 
+# TODO: look at the version in models/repository.py
 def add_cached_content_to_repository(repository_pk=None, remote_pk=None):
     """
     Create a new repository version by adding content that was cached by pulpcore-content when

pulp_rust/app/tasks/synchronizing.py

@@ -11,7 +11,6 @@
 
 from pulp_rust.app.models import RustContent, RustRemote
 
-
 log = logging.getLogger(__name__)
 
 

pulp_rust/app/urls.py

@@ -11,7 +11,7 @@
 
 urlpatterns = [
     path(
-        CRATES_IO_URL + "api/v1/crates/<str:package>/<str:version>/<path:rest>",
+        CRATES_IO_URL + "api/v1/crates/<str:name>/<str:version>/<path:rest>",
         CargoDownloadApiView.as_view(),
         name="cargo-download-api",
     ),

pulp_rust/app/views.py

@@ -1,5 +1,7 @@
 import json
 import logging
+import urllib.request
+import urllib.error
 
 from rest_framework.views import APIView
 from rest_framework.viewsets import ViewSet
@@ -15,12 +17,11 @@
 )
 from drf_spectacular.utils import extend_schema
 from dynaconf import settings
-from pathlib import PurePath
 from urllib.parse import urljoin
 
 from pulpcore.plugin.util import get_domain
 
-from pulp_rust.app.models import RustDistribution, RustRepository, RustContent
+from pulp_rust.app.models import RustDistribution, RustContent, _strip_sparse_prefix
 from pulp_rust.app.serializers import (
     IndexRootSerializer,
     RustContentSerializer,
@@ -85,7 +86,6 @@ def initial(self, request, *args, **kwargs):
         """Perform common initialization tasks for API endpoints."""
         super().initial(request, *args, **kwargs)
         domain_name = get_domain().name
-        log.warning(self.kwargs)
         repo = self.kwargs["repo"]
         if settings.DOMAIN_ENABLED:
             self.base_content_url = urljoin(BASE_CONTENT_URL, f"pulp/cargo/{domain_name}/{repo}/")
@@ -121,7 +121,7 @@ class CargoIndexApiViewSet(ApiMixin, ViewSet):
         responses={200: RustContentSerializer},
         summary="Get package metadata",
     )
-    def retrieve(self, request, path):
+    def retrieve(self, request, path, **kwargs):
         """
         Retrieve crate metadata for the sparse protocol.
 
@@ -132,42 +132,57 @@ def retrieve(self, request, path):
         - 4+ chars: {first-two}/{second-two}/{crate}
 
         Returns newline-delimited JSON, one version per line.
+
+        If the crate is not found locally and the distribution has a remote,
+        the metadata is proxied from the upstream sparse index.
         """
         repo_ver, content = self.get_rvc()
 
-        if content is None:
-            return HttpResponseNotFound("No content available")
-
-        # Extract crate name from the path
-        meta_path = PurePath(path)
-        crate_name = meta_path.name.lower()
+        # Extract crate name from the path (last component)
+        crate_name = path.rsplit("/", 1)[-1].lower()
 
-        # Query for all versions of this crate
-        crate_versions = content.filter(name=crate_name).order_by("vers")
+        # Try to serve from local content first
+        if content is not None:
+            crate_versions = content.filter(name=crate_name).order_by("vers")
+            if crate_versions.exists():
+                return self._build_index_response(crate_versions)
 
-        if not crate_versions.exists():
-            return HttpResponseNotFound(f"Crate '{crate_name}' not found")
+        # Fall back to proxying from the upstream remote
+        if self.distribution.remote:
+            remote = self.distribution.remote.cast()
+            index_url = _strip_sparse_prefix(remote.url).rstrip("/")
+            upstream_url = f"{index_url}/{path}"
+            try:
+                response = urllib.request.urlopen(upstream_url)
+                return HttpResponse(response.read(), content_type="text/plain")
+            except urllib.error.HTTPError as e:
+                if e.code == 404:
+                    return HttpResponseNotFound(f"Crate '{crate_name}' not found")
+                raise
+
+        return HttpResponseNotFound(f"Crate '{crate_name}' not found")
 
-        # Build newline-delimited JSON response
+    @staticmethod
+    def _build_index_response(crate_versions):
+        """Build a newline-delimited JSON response from local crate versions."""
         lines = []
         for crate_version in crate_versions:
-            # Fetch dependencies for this version
             deps = []
             for dep in crate_version.dependencies.all():
-                dep_obj = {
-                    "name": dep.name,
-                    "req": dep.req,
-                    "features": dep.features,
-                    "optional": dep.optional,
-                    "default_features": dep.default_features,
-                    "target": dep.target,
-                    "kind": dep.kind,
-                    "registry": dep.registry,
-                    "package": dep.package,
-                }
-                deps.append(dep_obj)
-
-            # Build the version object according to sparse protocol
+                deps.append(
+                    {
+                        "name": dep.name,
+                        "req": dep.req,
+                        "features": dep.features,
+                        "optional": dep.optional,
+                        "default_features": dep.default_features,
+                        "target": dep.target,
+                        "kind": dep.kind,
+                        "registry": dep.registry,
+                        "package": dep.package,
+                    }
+                )
+
             version_obj = {
                 "name": crate_version.name,
                 "vers": crate_version.vers,
@@ -179,18 +194,14 @@ def retrieve(self, request, path):
                 "v": crate_version.v,
             }
 
-            # Add optional fields only if present
             if crate_version.features2:
                 version_obj["features2"] = crate_version.features2
             if crate_version.rust_version:
                 version_obj["rust_version"] = crate_version.rust_version
 
-            # Serialize to JSON and add to lines
             lines.append(json.dumps(version_obj))
 
-        # Join with newlines and return as plain text
-        response_text = "\n".join(lines)
-        return HttpResponse(response_text, content_type="text/plain")
+        return HttpResponse("\n".join(lines), content_type="text/plain")
 
 
 class IndexRoot(ApiMixin, ViewSet):
@@ -221,14 +232,9 @@ def retrieve(self, request, repo):
 
 class CargoDownloadApiView(APIView):
     """
-    ViewSet for interacting with Cargo's API API
+    View for Cargo's crate download, readme, yank, and unyank endpoints.
     """
 
-    model = RustRepository
-    queryset = RustRepository.objects.all()
-
-    lookup_field = "name"
-
     # Authentication disabled for now
     authentication_classes = []
     permission_classes = []
@@ -248,23 +254,50 @@ def redirect_to_content_app(self, distribution, relative_path, request):
             f"{self.get_full_path(distribution.base_path)}/{relative_path}"
         )
 
-    def get_repository_and_distributions(self, name):
-        repository = get_object_or_404(RustRepository, name=name, pulp_domain=get_domain())
-        distribution = get_object_or_404(
-            RustDistribution, repository=repository, pulp_domain=get_domain()
+    def get_distribution(self):
+        return get_object_or_404(
+            RustDistribution, base_path=self.kwargs["repo"], pulp_domain=get_domain()
         )
-        return repository, distribution
 
-    def get(self, request, name, version):
+    def get(self, request, name, version, rest, **kwargs):
         """
-        Responds to GET requests about packages by reference
+        Responds to GET requests for crate downloads and readmes.
+
+        Handles:
+        - api/v1/crates/{name}/{version}/download - redirect to .crate file
+        - api/v1/crates/{name}/{version}/readme - not yet implemented
         """
-        repo, distro = self.get_repository_and_distributions(name)
-        content = get_object_or_404(
-            RustContent, name=name, vers=version, pk__in=repo.latest_version().content
-        )
-        relative_path = content.contentartifact_set.get().relative_path
-        return self.redirect_to_content_app(distro, relative_path, request)
+        distro = self.get_distribution()
+
+        if rest == "download":
+            relative_path = f"{name}/{name}-{version}.crate"
+            return self.redirect_to_content_app(distro, relative_path, request)
+        elif rest == "readme":
+            raise NotImplementedError("Readme endpoint is not yet implemented")
+        else:
+            raise Http404(f"Unknown action: {rest}")
+
+    def delete(self, request, name, version, rest, **kwargs):
+        """
+        Responds to DELETE requests for yanking crate versions.
+
+        Handles:
+        - api/v1/crates/{name}/{version}/yank
+        """
+        if rest != "yank":
+            raise Http404(f"Unknown action: {rest}")
+        raise NotImplementedError("Yank endpoint is not yet implemented")
+
+    def put(self, request, name, version, rest, **kwargs):
+        """
+        Responds to PUT requests for unyanking crate versions.
+
+        Handles:
+        - api/v1/crates/{name}/{version}/unyank
+        """
+        if rest != "unyank":
+            raise Http404(f"Unknown action: {rest}")
+        raise NotImplementedError("Unyank endpoint is not yet implemented")
 
 
 def has_task_completed(task):