add further sql optimizations

furlongm · furlongm · commit 0376909f7053 · 2026-02-12T23:41:25.000-05:00
diff --git a/errata/managers.py b/errata/managers.py
@@ -19,4 +19,4 @@
 
 class ErratumManager(models.Manager):
     def get_queryset(self):
-        return super().get_queryset().select_related()
+        return super().get_queryset()
diff --git a/errata/views.py b/errata/views.py
@@ -29,7 +29,7 @@
 
 @login_required
 def erratum_list(request):
-    errata = Erratum.objects.select_related()
+    errata = Erratum.objects.all()
 
     if 'e_type' in request.GET:
         errata = errata.filter(e_type=request.GET['e_type']).distinct()
diff --git a/hosts/managers.py b/hosts/managers.py
@@ -20,4 +20,4 @@
 
 class HostManager(models.Manager):
     def get_queryset(self):
-        return super().get_queryset().select_related()
+        return super().get_queryset().select_related('osvariant', 'arch', 'domain')
diff --git a/hosts/models.py b/hosts/models.py
@@ -135,9 +135,9 @@ def clean_reports(self):
         """ Remove all but the last 3 reports for a host
         """
         from reports.models import Report
-        reports = Report.objects.filter(host=self).order_by('-created')[3:]
-        rlen = reports.count()
-        for report in Report.objects.filter(host=self).order_by('-created')[3:]:
+        reports = list(Report.objects.filter(host=self).order_by('-created')[3:])
+        rlen = len(reports)
+        for report in reports:
             report.delete()
         if rlen > 0:
             info_message(text=f'{self.hostname}: removed {rlen} old reports')
@@ -157,14 +157,14 @@ def get_host_repo_packages(self):
                 Q(mirror__repo__in=self.repos.all(),
                   mirror__enabled=True,
                   mirror__repo__enabled=True)
-        return Package.objects.select_related().filter(hostrepos_q).distinct()
+        return Package.objects.select_related('name', 'arch').filter(hostrepos_q).distinct()
 
     def process_update(self, package, highest_package):
         if self.host_repos_only:
             host_repos = Q(repo__host=self)
         else:
             host_repos = Q(repo__osrelease__osvariant__host=self, repo__arch=self.arch) | Q(repo__host=self)
-        mirrors = highest_package.mirror_set.filter(host_repos)
+        mirrors = highest_package.mirror_set.filter(host_repos).select_related('repo')
         security = False
         # if any of the containing repos are security, mark the update as a security update
         for mirror in mirrors:
@@ -223,7 +223,7 @@ def find_host_repo_updates(self, host_packages, repo_packages):
                         repo__mirror__refresh=True,
                         repo__mirror__repo__enabled=True,
                         host=self)
-        hostrepos = HostRepo.objects.select_related().filter(hostrepos_q)
+        hostrepos = HostRepo.objects.select_related('host', 'repo').filter(hostrepos_q)
 
         for package in host_packages:
             highest_package = package
diff --git a/hosts/views.py b/hosts/views.py
@@ -42,7 +42,7 @@ def _get_filtered_hosts(filter_params):
     from urllib.parse import parse_qs
     params = parse_qs(filter_params)
 
-    hosts = Host.objects.select_related()
+    hosts = Host.objects.select_related('osvariant', 'arch', 'domain')
 
     if 'domain_id' in params:
         hosts = hosts.filter(domain=params['domain_id'][0])
@@ -77,7 +77,7 @@ def _get_filtered_hosts(filter_params):
 @login_required
 def host_list(request):
     # Use cached count fields instead of expensive annotations
-    hosts = Host.objects.select_related()
+    hosts = Host.objects.select_related('osvariant', 'arch', 'domain')
 
     if 'domain_id' in request.GET:
         hosts = hosts.filter(domain=request.GET['domain_id'])
@@ -156,7 +156,7 @@ def host_detail(request, hostname):
     hostrepos = HostRepo.objects.filter(host=host)
 
     # Build packages list with update info
-    updates_by_package = {u.oldpackage_id: u for u in host.updates.select_related()}
+    updates_by_package = {u.oldpackage_id: u for u in host.updates.select_related('oldpackage', 'newpackage')}
     packages_with_updates = []
     for package in host.packages.select_related('name', 'arch').order_by('name__name'):
         package.update = updates_by_package.get(package.id)
@@ -294,7 +294,7 @@ class HostViewSet(viewsets.ModelViewSet):
     """
     API endpoint that allows hosts to be viewed or edited.
     """
-    queryset = Host.objects.all()
+    queryset = Host.objects.select_related('osvariant', 'arch', 'domain').all()
     serializer_class = HostSerializer
     filterset_class = HostFilter
 
@@ -303,5 +303,5 @@ class HostRepoViewSet(viewsets.ModelViewSet):
     """
     API endpoint that allows host repos to be viewed or edited.
     """
-    queryset = HostRepo.objects.all()
+    queryset = HostRepo.objects.select_related('host', 'repo').all()
     serializer_class = HostRepoSerializer
diff --git a/modules/managers.py b/modules/managers.py
@@ -19,4 +19,4 @@
 
 class ModuleManager(models.Manager):
     def get_queryset(self):
-        return super().get_queryset().select_related()
+        return super().get_queryset().select_related('arch', 'repo')
diff --git a/modules/views.py b/modules/views.py
@@ -28,7 +28,7 @@
 @login_required
 def module_list(request):
 
-    modules = Module.objects.select_related()
+    modules = Module.objects.select_related('arch', 'repo')
 
     if 'search' in request.GET:
         terms = request.GET['search'].lower()
@@ -62,6 +62,6 @@ class ModuleViewSet(viewsets.ModelViewSet):
     """
     API endpoint that allows modules to be viewed or edited.
     """
-    queryset = Module.objects.all()
+    queryset = Module.objects.select_related('arch', 'repo').all()
     serializer_class = ModuleSerializer
     permission_classes = (permissions.IsAuthenticatedOrReadOnly,)
diff --git a/operatingsystems/forms.py b/operatingsystems/forms.py
@@ -47,7 +47,7 @@ class Meta:
 class AddReposToOSReleaseForm(ModelForm):
 
     repos = ModelMultipleChoiceField(
-        queryset=Repository.objects.select_related(),
+        queryset=Repository.objects.select_related('arch'),
         required=False,
         label=None,
         widget=FilteredSelectMultiple('Repos', False, attrs={'size': '30'}))
diff --git a/operatingsystems/templates/operatingsystems/osvariant_detail.html b/operatingsystems/templates/operatingsystems/osvariant_detail.html
@@ -24,7 +24,7 @@
         <tr><th>Name</th><td> {{ osvariant.name }} </td></tr>
         <tr><th>Architecture</th><td> {{ osvariant.arch }} </td></tr>
         <tr><th>Codename</th><td> {{ osvariant.codename }} </td></tr>
-        <tr><th>Hosts</th><td><a href="{% url 'hosts:host_list' %}?osvariant_id={{ osvariant.id }}">{{ osvariant.host_set.count }}</a></td></tr>
+        <tr><th>Hosts</th><td><a href="{% url 'hosts:host_list' %}?osvariant_id={{ osvariant.id }}">{{ osvariant.hosts_count }}</a></td></tr>
         <tr><th>OS Release</th><td>{% if osvariant.osrelease != None %}<a href="{{ osvariant.osrelease.get_absolute_url }}"> {{ osvariant.osrelease }} </a>{% else %}No OS Release{% endif %}</td></tr>
       </table>
       {% if user.is_authenticated and perms.is_admin %}
diff --git a/operatingsystems/views.py b/operatingsystems/views.py
@@ -40,7 +40,7 @@ def _get_filtered_osvariants(filter_params):
     from urllib.parse import parse_qs
     params = parse_qs(filter_params)
 
-    osvariants = OSVariant.objects.select_related()
+    osvariants = OSVariant.objects.select_related('osrelease', 'arch')
 
     if 'osrelease_id' in params:
         osvariants = osvariants.filter(osrelease=params['osrelease_id'][0])
@@ -60,7 +60,7 @@ def _get_filtered_osreleases(filter_params):
     from urllib.parse import parse_qs
     params = parse_qs(filter_params)
 
-    osreleases = OSRelease.objects.select_related()
+    osreleases = OSRelease.objects.all()
 
     if 'erratum_id' in params:
         osreleases = osreleases.filter(erratum=params['erratum_id'][0])
@@ -78,7 +78,7 @@ def _get_filtered_osreleases(filter_params):
 @login_required
 def osvariant_list(request):
     # Use cached hosts_count instead of expensive annotation
-    osvariants = OSVariant.objects.select_related().annotate(
+    osvariants = OSVariant.objects.select_related('osrelease', 'arch').annotate(
         repos_count=Count('osrelease__repos'),
     )
 
@@ -182,7 +182,7 @@ def delete_nohost_osvariants(request):
 
 @login_required
 def osrelease_list(request):
-    osreleases = OSRelease.objects.select_related()
+    osreleases = OSRelease.objects.all()
 
     if 'erratum_id' in request.GET:
         osreleases = osreleases.filter(erratum=request.GET['erratum_id'])
@@ -347,7 +347,7 @@ class OSVariantViewSet(viewsets.ModelViewSet):
     """
     API endpoint that allows operating system variants to be viewed or edited.
     """
-    queryset = OSVariant.objects.all()
+    queryset = OSVariant.objects.select_related('osrelease', 'arch').all()
     serializer_class = OSVariantSerializer
     filterset_fields = ['name']
 
diff --git a/packages/managers.py b/packages/managers.py
@@ -20,4 +20,4 @@
 
 class PackageManager(models.Manager):
     def get_queryset(self):
-        return super().get_queryset().select_related()
+        return super().get_queryset().select_related('name', 'arch')
diff --git a/packages/utils.py b/packages/utils.py
@@ -281,7 +281,7 @@ def clean_packageupdates():
     """
     package_updates = list(PackageUpdate.objects.all())
     for update in package_updates:
-        if update.host_set.count() == 0:
+        if not update.host_set.exists():
             text = f'Removing unused PackageUpdate {update}'
             info_message(text=text)
             update.delete()
@@ -325,7 +325,8 @@ def clean_packages(remove_duplicates=False):
                 packagetype=package.packagetype,
                 category=package.category,
             )
-            if potential_duplicates.count() > 1:
+            potential_duplicates = list(potential_duplicates)
+            if len(potential_duplicates) > 1:
                 for dupe in potential_duplicates:
                     if dupe.id != package.id:
                         info_message(text=f'Removing duplicate Package {dupe}')
diff --git a/packages/views.py b/packages/views.py
@@ -32,7 +32,7 @@
 
 @login_required
 def package_list(request):
-    packages = Package.objects.select_related()
+    packages = Package.objects.select_related('name', 'arch')
 
     if 'arch_id' in request.GET:
         packages = packages.filter(arch=request.GET['arch_id']).distinct()
@@ -121,7 +121,7 @@ def package_list(request):
 
 @login_required
 def package_name_list(request):
-    packages = PackageName.objects.select_related()
+    packages = PackageName.objects.all()
 
     if 'arch_id' in request.GET:
         packages = packages.filter(package__arch=request.GET['arch_id']).distinct()
@@ -165,7 +165,7 @@ def package_detail(request, package_id):
 @login_required
 def package_name_detail(request, packagename):
     package = get_object_or_404(PackageName, name=packagename)
-    allversions = Package.objects.select_related().filter(name=package.id)
+    allversions = Package.objects.select_related('name', 'arch').filter(name=package.id)
     return render(request,
                   'packages/package_name_detail.html',
                   {'package': package,
@@ -185,7 +185,7 @@ class PackageViewSet(viewsets.ModelViewSet):
     """
     API endpoint that allows packages to be viewed or edited.
     """
-    queryset = Package.objects.all()
+    queryset = Package.objects.select_related('name', 'arch').all()
     serializer_class = PackageSerializer
     filterset_fields = [
         'name',
@@ -201,6 +201,6 @@ class PackageUpdateViewSet(viewsets.ModelViewSet):
     """
     API endpoint that allows packages updates to be viewed or edited.
     """
-    queryset = PackageUpdate.objects.all()
+    queryset = PackageUpdate.objects.select_related('oldpackage', 'newpackage').all()
     serializer_class = PackageUpdateSerializer
     filterset_fields = ['oldpackage', 'newpackage', 'security']
diff --git a/reports/views.py b/reports/views.py
@@ -45,7 +45,7 @@ def _get_filtered_reports(filter_params):
     from urllib.parse import parse_qs
     params = parse_qs(filter_params)
 
-    reports = Report.objects.select_related()
+    reports = Report.objects.all()
 
     if 'host_id' in params:
         reports = reports.filter(hostname=params['host_id'][0])
@@ -108,7 +108,7 @@ def upload(request):
 @login_required
 def report_list(request):
 
-    reports = Report.objects.select_related()
+    reports = Report.objects.all()
 
     if 'host_id' in request.GET:
         reports = reports.filter(hostname=request.GET['host_id'])
diff --git a/repos/forms.py b/repos/forms.py
@@ -28,7 +28,7 @@ class MirrorSelect2Widget(ModelSelect2MultipleWidget):
     model = Mirror
     search_fields = ['url__icontains', 'repo__name__icontains']
     max_results = 50
-    queryset = Mirror.objects.select_related().order_by('repo__name', 'url')
+    queryset = Mirror.objects.select_related('repo').order_by('repo__name', 'url')
 
     def __init__(self, *args, **kwargs):
         kwargs.setdefault('attrs', {})
@@ -41,7 +41,7 @@ def label_from_instance(self, obj):
 
 class EditRepoForm(ModelForm):
     mirrors = ModelMultipleChoiceField(
-        queryset=Mirror.objects.select_related().order_by('repo__name', 'url'),
+        queryset=Mirror.objects.select_related('repo').order_by('repo__name', 'url'),
         required=False,
         widget=MirrorSelect2Widget(attrs={'style': 'width: 100%'}),
     )
diff --git a/repos/managers.py b/repos/managers.py
@@ -20,4 +20,4 @@
 
 class RepositoryManager(models.Manager):
     def get_queryset(self):
-        return super().get_queryset().select_related()
+        return super().get_queryset().select_related('arch')
diff --git a/repos/models.py b/repos/models.py
@@ -82,11 +82,11 @@ def refresh(self, force=False):
             force can be set to force a reset of all the mirrors metadata
         """
         if force:
-            for mirror in self.mirror_set.all():
-                mirror.packages_checksum = None
-                mirror.modules_checksum = None
-                mirror.errata_checksum = None
-                mirror.save()
+            self.mirror_set.all().update(
+                packages_checksum=None,
+                modules_checksum=None,
+                errata_checksum=None
+            )
 
         if not self.auth_required:
             if self.repotype == Repository.DEB:
@@ -108,9 +108,7 @@ def refresh_errata(self, force=False):
         """ Refresh errata metadata for all of a repos mirrors
         """
         if force:
-            for mirror in self.mirror_set.all():
-                mirror.errata_checksum = None
-                mirror.save()
+            self.mirror_set.all().update(errata_checksum=None)
         if self.repotype == Repository.RPM:
             refresh_repo_errata(self)
 
@@ -120,21 +118,15 @@ def disable(self):
             each mirror so that it doesn't try to update its package metadata.
         """
         self.enabled = False
-        for mirror in self.mirror_set.all():
-            mirror.enabled = False
-            mirror.refresh = False
-            mirror.save()
+        self.mirror_set.all().update(enabled=False, refresh=False)
 
     def enable(self):
         """ Enable a repo. This involves enabling each mirror, which allows it
             to be considered for package updates, and enabling refresh for each
             mirror so that it updates its package metadata.
         """
         self.enabled = True
-        for mirror in self.mirror_set.all():
-            mirror.enabled = True
-            mirror.refresh = True
-            mirror.save()
+        self.mirror_set.all().update(enabled=True, refresh=True)
 
 
 class Mirror(models.Model):
diff --git a/repos/utils.py b/repos/utils.py
@@ -273,11 +273,12 @@ def find_best_repo(package, hostrepos):
         repo. Returns the best repo.
     """
     best_repo = None
-    package_repos = hostrepos.filter(repo__mirror__packages=package).distinct()
+    package_repos = hostrepos.filter(repo__mirror__packages=package).select_related('repo').distinct()
 
+    package_repos = list(package_repos)
     if package_repos:
         best_repo = package_repos[0]
-    if package_repos.count() > 1:
+    if len(package_repos) > 1:
         for hostrepo in package_repos:
             if hostrepo.repo.security:
                 best_repo = hostrepo
diff --git a/repos/views.py b/repos/views.py
diff --git a/security/managers.py b/security/managers.py
diff --git a/security/views.py b/security/views.py
diff --git a/util/management/commands/revoke_api_key.py b/util/management/commands/revoke_api_key.py
diff --git a/util/templatetags/common.py b/util/templatetags/common.py
