Force content_ids requirement

fixes: #7466
https://redhat.atlassian.net/browse/PULP-577

Author: gerrod3

CHANGES/plugin_api/7466.removal

@@ -0,0 +1 @@
+Made `content_ids` cache required with forceful migration and removed the `/pulp/api/v3/datarepair/7465/` endpoint. This **will** run a migration that could take a while if you haven't already run the datarepair endpoint.

docs/admin/guides/data-repair.md

@@ -109,3 +109,6 @@ and recomputing the content counts. The repair operates within the current domai
 ## Repair #7465: Populate missing Repository Version Caches
 
 [Issue #7465](https://github.com/pulp/pulpcore/issues/7465) The repository version `content_ids` caches were added in [pulpcore 3.83](https://github.com/pulp/pulpcore/issues/5783) as a way to speed up the content set calculation performed anytime content is accessed from a repository. To ensure a smooth upgrade and maintain backwards compatibility, the new cache was only populated for newly created repository versions after 3.83. Starting in pulpcore 3.115 the cache will become required for all repository versions. This endpoint will spawn a task to populate any missing `content_ids` on all the repository versions in the task's domain.
+
+!!! note
+    This endpoint was removed in 3.115 with the upgrade migration forcefully populating any remaining empty cache.

pulpcore/app/checks.py

@@ -123,22 +123,3 @@ def check_artifact_checksums(app_configs, **kwargs):
         )
 
     return messages
-
-
-@register(Tags.database)
-def check_repository_version_content_ids(app_configs, **kwargs):
-    from pulpcore.app.models import RepositoryVersion
-
-    try:
-        if RepositoryVersion.objects.filter(content_ids=None).exists():
-            return [
-                CheckWarning(
-                    "There are repository versions with no content_ids cache. Run "
-                    "'/pulp/api/v3/datarepair/7465/' to add missing content_ids cache to all "
-                    "repository versions. This will become an error in 3.115",
-                    id="pulpcore.W006",
-                )
-            ]
-    except Exception:
-        pass
-    return []

pulpcore/app/migrations/0152_alter_repositoryversion_content_ids.py

@@ -0,0 +1,35 @@
+# Generated by Django 5.2.13 on 2026-06-08 21:54
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+def populate_content_ids(apps, schema_editor):
+    RepositoryVersion = apps.get_model('core', 'RepositoryVersion')
+    RepositoryContent = apps.get_model('core', 'RepositoryContent')
+    repo_versions = []
+    for rv in RepositoryVersion.objects.filter(content_ids=None).iterator(chunk_size=2000):
+        rv.content_ids = list(RepositoryContent.objects.filter(
+            repository_id=rv.repository_id, version_added__number__lte=rv.number
+        ).exclude(version_removed__number__lte=rv.number).values_list("content_id", flat=True))
+        repo_versions.append(rv)
+        if len(repo_versions) >= 2000:
+            RepositoryVersion.objects.bulk_update(repo_versions, ['content_ids'])
+            repo_versions = []
+    if repo_versions:
+        RepositoryVersion.objects.bulk_update(repo_versions, ['content_ids'])
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0151_upstreampulp_connect_timeout_and_more'),
+    ]
+
+    operations = [
+        migrations.RunPython(populate_content_ids, reverse_code=migrations.RunPython.noop, elidable=True),
+        migrations.AlterField(
+            model_name='repositoryversion',
+            name='content_ids',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.UUIDField(), default=list, size=None),
+        ),
+    ]

pulpcore/app/models/repository.py

@@ -942,7 +942,7 @@ class RepositoryVersion(BaseModel):
     complete = models.BooleanField(db_index=True, default=False)
     base_version = models.ForeignKey("RepositoryVersion", null=True, on_delete=models.SET_NULL)
     info = models.JSONField(default=dict)
-    content_ids = ArrayField(models.UUIDField(), default=None, null=True)
+    content_ids = ArrayField(models.UUIDField(), default=list)
 
     class Meta:
         default_related_name = "versions"
@@ -961,14 +961,6 @@ def _content_relationships(self):
             repository_id=self.repository_id, version_added__number__lte=self.number
         ).exclude(version_removed__number__lte=self.number)
 
-    def _get_content_ids(self):
-        """
-        Returns the content ids for a repository version
-        """
-        if self.content_ids is not None:
-            return self.content_ids
-        return self._content_relationships().values_list("content_id", flat=True)
-
     @hook(BEFORE_CREATE)
     def set_content_ids(self):
         """
@@ -979,12 +971,7 @@ def set_content_ids(self):
         except self.DoesNotExist:
             pass
         else:
-            if previous.content_ids is not None:
-                self.content_ids = previous.content_ids
-        if self.content_ids is None:
-            self.content_ids = list(
-                self._content_relationships().values_list("content_id", flat=True)
-            )
+            self.content_ids = previous.content_ids
 
     def get_content(self, content_qs=None):
         """
@@ -1009,8 +996,8 @@ def get_content(self, content_qs=None):
         if content_qs is None:
             content_qs = Content.objects
 
-        content_ids = self._get_content_ids()
-        if isinstance(content_ids, list) and len(content_ids) >= 65535:
+        content_ids = self.content_ids
+        if len(content_ids) >= 65535:
             # Workaround for PostgreSQL's limit on the number of parameters in a query
             content_ids = (
                 RepositoryVersion.objects.filter(pk=self.pk)
@@ -1131,8 +1118,8 @@ def added(self, base_version=None):
         if not base_version:
             return Content.objects.filter(version_memberships__version_added=self)
 
-        return Content.objects.filter(pk__in=self._get_content_ids()).exclude(
-            pk__in=base_version._get_content_ids()
+        return Content.objects.filter(pk__in=self.content_ids).exclude(
+            pk__in=base_version.content_ids
         )
 
     def removed(self, base_version=None):
@@ -1146,8 +1133,8 @@ def removed(self, base_version=None):
         if not base_version:
             return Content.objects.filter(version_memberships__version_removed=self)
 
-        return Content.objects.filter(pk__in=base_version._get_content_ids()).exclude(
-            pk__in=self._get_content_ids()
+        return Content.objects.filter(pk__in=base_version.content_ids).exclude(
+            pk__in=self.content_ids
         )
 
     def contains(self, content):
@@ -1157,9 +1144,7 @@ def contains(self, content):
         Returns:
             bool: True if the repository version contains the content, False otherwise
         """
-        if self.content_ids is not None:
-            return content.pk in self.content_ids
-        return self.content.filter(pk=content.pk).exists()
+        return content.pk in self.content_ids
 
     def add_content(self, content):
         """
@@ -1184,7 +1169,7 @@ def add_content(self, content):
         )
 
         repo_content = []
-        to_add = set(content.values_list("pk", flat=True)) - set(self._get_content_ids())
+        to_add = set(content.values_list("pk", flat=True)) - set(self.content_ids)
         with transaction.atomic():
             if to_add:
                 self.content_ids += list(to_add)
@@ -1234,7 +1219,7 @@ def remove_content(self, content):
             .exclude(pulp_domain_id=get_domain_pk())
             .exists()
         )
-        content_ids = set(self._get_content_ids())
+        content_ids = set(self.content_ids)
         to_remove = set(content.values_list("pk", flat=True))
         with transaction.atomic():
             if to_remove:

pulpcore/app/serializers/__init__.py

@@ -98,7 +98,7 @@
     RepositoryAddRemoveContentSerializer,
     RepositoryVersionSerializer,
 )
-from .repair import RepairSerializer, DataRepair7272Serializer, DataRepair7465Serializer
+from .repair import RepairSerializer, DataRepair7272Serializer
 from .reclaim import ReclaimSpaceSerializer
 from .task import (
     MinimalTaskSerializer,

pulpcore/app/serializers/repair.py

@@ -26,14 +26,3 @@ class DataRepair7272Serializer(serializers.Serializer, ValidateFieldsMixin):
             "repair the detected issues."
         ),
     )
-
-
-class DataRepair7465Serializer(serializers.Serializer, ValidateFieldsMixin):
-    dry_run = fields.BooleanField(
-        required=False,
-        default=False,
-        help_text=_(
-            "If true, only report issues without fixing them. If false (default), "
-            "repair the detected issues."
-        ),
-    )

pulpcore/app/tasks/datarepair.py

@@ -112,61 +112,3 @@ def repair_7272(dry_run=False):
                 f"Data repair operation for issue #7272 finished. ({number_broken} "
                 f'repository versions fixed in domain "{domain.name}")'
             )
-
-
-def repair_7465(dry_run=False):
-    """
-    Populates the content_ids cache for all repository versions.
-    """
-    number_missing = 0
-    domain = get_domain()
-
-    log.info(f'Performing datarepair for issue #7465 for domain "{domain.name}"')
-
-    repos = models.Repository.objects.filter(pulp_domain=domain)
-    total_versions = models.RepositoryVersion.objects.filter(repository__in=repos).count()
-
-    with (
-        ProgressReport(
-            message="Repositories checked",
-            code="repair.7465.repos_checked",
-            total=repos.count(),
-        ) as repos_progress,
-        ProgressReport(
-            message="Repository versions checked",
-            code="repair.7465.versions_checked",
-            total=total_versions,
-        ) as versions_progress,
-        ProgressReport(
-            message="Repository versions fixed",
-            code="repair.7465.versions_fixed",
-        ) as fixed_progress,
-    ):
-        for repo in repos:
-            for rv in models.RepositoryVersion.objects.filter(repository=repo):
-                versions_progress.increment()
-                if rv.content_ids is None:
-                    number_missing += 1
-                    if not dry_run:
-                        rv.content_ids = list(
-                            rv._content_relationships().values_list("content_id", flat=True)
-                        )
-                        rv.save()
-                        fixed_progress.increment()
-            repos_progress.increment()
-
-        fixed_progress.total = number_missing
-
-    if not number_missing:
-        log.info(f'Data repair operation for issue #7465 for domain "{domain.name}" finished. (OK)')
-    else:
-        if dry_run:
-            log.info(
-                f"Data repair operation for issue #7465 dry run finished. ({number_missing} "
-                f'repository versions need fixing in domain "{domain.name}")'
-            )
-        else:
-            log.info(
-                f"Data repair operation for issue #7465 finished. ({number_missing} "
-                f'repository versions fixed in domain "{domain.name}")'
-            )

pulpcore/app/urls.py

@@ -16,7 +16,6 @@
 from pulpcore.app.apps import pulp_plugin_configs
 from pulpcore.app.views import (
     DataRepair7272View,
-    DataRepair7465View,
     LivezView,
     OrphansView,
     PulpImporterImportCheckView,
@@ -156,7 +155,6 @@ class PulpDefaultRouter(routers.DefaultRouter):
     path("login/", LoginViewSet.as_view()),
     path("repair/", RepairView.as_view()),
     path("datarepair/7272/", DataRepair7272View.as_view()),
-    path("datarepair/7465/", DataRepair7465View.as_view()),
     path(
         "orphans/cleanup/",
         OrphansCleanupViewset.as_view(actions={"post": "cleanup"}),

pulpcore/app/views/__init__.py

@@ -1,6 +1,6 @@
 # ruff: noqa: F401
 # isort: skip_file
-from .datarepair import DataRepair7272View, DataRepair7465View
+from .datarepair import DataRepair7272View
 from .orphans import OrphansView
 from .status import LivezView, StatusView
 from .repair import RepairView