add further sql optimizations

Author: furlongm

hosts/models.py

@@ -135,9 +135,9 @@ def clean_reports(self):
         """ Remove all but the last 3 reports for a host
         """
         from reports.models import Report
-        reports = Report.objects.filter(host=self).order_by('-created')[3:]
-        rlen = reports.count()
-        for report in Report.objects.filter(host=self).order_by('-created')[3:]:
+        reports = list(Report.objects.filter(host=self).order_by('-created')[3:])
+        rlen = len(reports)
+        for report in reports:
             report.delete()
         if rlen > 0:
             info_message(text=f'{self.hostname}: removed {rlen} old reports')
@@ -164,7 +164,7 @@ def process_update(self, package, highest_package):
             host_repos = Q(repo__host=self)
         else:
             host_repos = Q(repo__osrelease__osvariant__host=self, repo__arch=self.arch) | Q(repo__host=self)
-        mirrors = highest_package.mirror_set.filter(host_repos)
+        mirrors = highest_package.mirror_set.filter(host_repos).select_related('repo')
         security = False
         # if any of the containing repos are security, mark the update as a security update
         for mirror in mirrors:

hosts/views.py

@@ -42,7 +42,7 @@ def _get_filtered_hosts(filter_params):
     from urllib.parse import parse_qs
     params = parse_qs(filter_params)
 
-    hosts = Host.objects.select_related()
+    hosts = Host.objects.select_related('osvariant', 'arch', 'domain')
 
     if 'domain_id' in params:
         hosts = hosts.filter(domain=params['domain_id'][0])
@@ -77,7 +77,7 @@ def _get_filtered_hosts(filter_params):
 @login_required
 def host_list(request):
     # Use cached count fields instead of expensive annotations
-    hosts = Host.objects.select_related()
+    hosts = Host.objects.select_related('osvariant', 'arch', 'domain')
 
     if 'domain_id' in request.GET:
         hosts = hosts.filter(domain=request.GET['domain_id'])
@@ -156,7 +156,7 @@ def host_detail(request, hostname):
     hostrepos = HostRepo.objects.filter(host=host)
 
     # Build packages list with update info
-    updates_by_package = {u.oldpackage_id: u for u in host.updates.select_related()}
+    updates_by_package = {u.oldpackage_id: u for u in host.updates.select_related('oldpackage', 'newpackage')}
     packages_with_updates = []
     for package in host.packages.select_related('name', 'arch').order_by('name__name'):
         package.update = updates_by_package.get(package.id)
@@ -294,7 +294,7 @@ class HostViewSet(viewsets.ModelViewSet):
     """
     API endpoint that allows hosts to be viewed or edited.
     """
-    queryset = Host.objects.all()
+    queryset = Host.objects.select_related('osvariant', 'arch', 'domain').all()
     serializer_class = HostSerializer
     filterset_class = HostFilter
 
@@ -303,5 +303,5 @@ class HostRepoViewSet(viewsets.ModelViewSet):
     """
     API endpoint that allows host repos to be viewed or edited.
     """
-    queryset = HostRepo.objects.all()
+    queryset = HostRepo.objects.select_related('host', 'repo').all()
     serializer_class = HostRepoSerializer

modules/views.py

@@ -62,6 +62,6 @@ class ModuleViewSet(viewsets.ModelViewSet):
     """
     API endpoint that allows modules to be viewed or edited.
     """
-    queryset = Module.objects.all()
+    queryset = Module.objects.select_related('arch', 'repo').all()
     serializer_class = ModuleSerializer
     permission_classes = (permissions.IsAuthenticatedOrReadOnly,)

operatingsystems/templates/operatingsystems/osvariant_detail.html

@@ -24,7 +24,7 @@
         <tr><th>Name</th><td> {{ osvariant.name }} </td></tr>
         <tr><th>Architecture</th><td> {{ osvariant.arch }} </td></tr>
         <tr><th>Codename</th><td> {{ osvariant.codename }} </td></tr>
-        <tr><th>Hosts</th><td><a href="{% url 'hosts:host_list' %}?osvariant_id={{ osvariant.id }}">{{ osvariant.host_set.count }}</a></td></tr>
+        <tr><th>Hosts</th><td><a href="{% url 'hosts:host_list' %}?osvariant_id={{ osvariant.id }}">{{ osvariant.hosts_count }}</a></td></tr>
         <tr><th>OS Release</th><td>{% if osvariant.osrelease != None %}<a href="{{ osvariant.osrelease.get_absolute_url }}"> {{ osvariant.osrelease }} </a>{% else %}No OS Release{% endif %}</td></tr>
       </table>
       {% if user.is_authenticated and perms.is_admin %}

operatingsystems/utils.py

@@ -65,17 +65,17 @@ def get_or_create_osrelease(name, cpe_name=None, codename=None):
         try:
             osrelease, created = OSRelease.objects.get_or_create(name=name, cpe_name=cpe_name)
         except IntegrityError:
-            osreleases = OSRelease.objects.filter(cpe_name=cpe_name)
-            if osreleases.count() == 1:
-                osrelease = osreleases.first()
+            osreleases = list(OSRelease.objects.filter(cpe_name=cpe_name))
+            if len(osreleases) == 1:
+                osrelease = osreleases[0]
                 osrelease.name = name
     if not osrelease and codename:
         try:
             osrelease, created = OSRelease.objects.get_or_create(name=name, codename=codename)
         except IntegrityError:
-            osreleases = OSRelease.objects.filter(codename=codename)
-            if osreleases.count() == 1:
-                osrelease = osreleases.first()
+            osreleases = list(OSRelease.objects.filter(codename=codename))
+            if len(osreleases) == 1:
+                osrelease = osreleases[0]
                 osrelease.name = name
                 osrelease.save()
     if not osrelease:

operatingsystems/views.py

@@ -347,7 +347,7 @@ class OSVariantViewSet(viewsets.ModelViewSet):
     """
     API endpoint that allows operating system variants to be viewed or edited.
     """
-    queryset = OSVariant.objects.all()
+    queryset = OSVariant.objects.select_related('osrelease', 'arch').all()
     serializer_class = OSVariantSerializer
     filterset_fields = ['name']
 

packages/views.py

@@ -185,7 +185,7 @@ class PackageViewSet(viewsets.ModelViewSet):
     """
     API endpoint that allows packages to be viewed or edited.
     """
-    queryset = Package.objects.all()
+    queryset = Package.objects.select_related('name', 'arch').all()
     serializer_class = PackageSerializer
     filterset_fields = [
         'name',
@@ -201,6 +201,6 @@ class PackageUpdateViewSet(viewsets.ModelViewSet):
     """
     API endpoint that allows packages updates to be viewed or edited.
     """
-    queryset = PackageUpdate.objects.all()
+    queryset = PackageUpdate.objects.select_related('oldpackage', 'newpackage').all()
     serializer_class = PackageUpdateSerializer
     filterset_fields = ['oldpackage', 'newpackage', 'security']

repos/models.py

@@ -82,11 +82,11 @@ def refresh(self, force=False):
             force can be set to force a reset of all the mirrors metadata
         """
         if force:
-            for mirror in self.mirror_set.all():
-                mirror.packages_checksum = None
-                mirror.modules_checksum = None
-                mirror.errata_checksum = None
-                mirror.save()
+            self.mirror_set.all().update(
+                packages_checksum=None,
+                modules_checksum=None,
+                errata_checksum=None
+            )
 
         if not self.auth_required:
             if self.repotype == Repository.DEB:
@@ -108,9 +108,7 @@ def refresh_errata(self, force=False):
         """ Refresh errata metadata for all of a repos mirrors
         """
         if force:
-            for mirror in self.mirror_set.all():
-                mirror.errata_checksum = None
-                mirror.save()
+            self.mirror_set.all().update(errata_checksum=None)
         if self.repotype == Repository.RPM:
             refresh_repo_errata(self)
 
@@ -120,21 +118,15 @@ def disable(self):
             each mirror so that it doesn't try to update its package metadata.
         """
         self.enabled = False
-        for mirror in self.mirror_set.all():
-            mirror.enabled = False
-            mirror.refresh = False
-            mirror.save()
+        self.mirror_set.all().update(enabled=False, refresh=False)
 
     def enable(self):
         """ Enable a repo. This involves enabling each mirror, which allows it
             to be considered for package updates, and enabling refresh for each
             mirror so that it updates its package metadata.
         """
         self.enabled = True
-        for mirror in self.mirror_set.all():
-            mirror.enabled = True
-            mirror.refresh = True
-            mirror.save()
+        self.mirror_set.all().update(enabled=True, refresh=True)
 
 
 class Mirror(models.Model):

repos/utils.py

@@ -177,7 +177,8 @@ def add_mirrors_from_urls(repo, mirror_urls):
         mirror_url = mirror_url.replace('$basearch', repo.arch.name)
         mirror_url = mirror_url.rstrip('/')
         q = Q(mirrorlist=False, refresh=True, enabled=True)
-        existing = repo.mirror_set.filter(q).count()
+        existing_mirrors = repo.mirror_set.filter(q)[:max_mirrors]
+        existing = len(existing_mirrors)
         if existing >= max_mirrors:
             text = f'{existing} Mirrors already exist (max={max_mirrors}), not adding more'
             warning_message(text=text)
@@ -273,11 +274,12 @@ def find_best_repo(package, hostrepos):
         repo. Returns the best repo.
     """
     best_repo = None
-    package_repos = hostrepos.filter(repo__mirror__packages=package).distinct()
+    package_repos = hostrepos.filter(repo__mirror__packages=package).select_related('repo').distinct()
 
+    package_repos = list(package_repos)
     if package_repos:
         best_repo = package_repos[0]
-    if package_repos.count() > 1:
+    if len(package_repos) > 1:
         for hostrepo in package_repos:
             if hostrepo.repo.security:
                 best_repo = hostrepo

repos/views.py

@@ -43,7 +43,7 @@
 @login_required
 def repo_list(request):
 
-    repos = Repository.objects.select_related().order_by('name')
+    repos = Repository.objects.select_related('arch').order_by('name')
 
     if 'repotype' in request.GET:
         repos = repos.filter(repotype=request.GET['repotype'])
@@ -146,11 +146,11 @@ def move_mirrors(repo):
                     hostrepo.delete()
             mirror.repo = repo
             mirror.save()
-            if oldrepo.mirror_set.count() == 0:
+            if not oldrepo.mirror_set.exists():
                 oldrepo.delete()
 
     # Use cached packages_count instead of expensive annotation
-    mirrors = Mirror.objects.select_related().order_by('packages_checksum')
+    mirrors = Mirror.objects.select_related('repo').order_by('packages_checksum')
 
     checksum = None
     if 'checksum' in request.GET:
@@ -318,9 +318,7 @@ def repo_edit(request, repo_id):
                 repo = edit_form.save()
                 repo.save()
                 mirrors = edit_form.cleaned_data['mirrors']
-                for mirror in mirrors:
-                    mirror.repo = repo
-                    mirror.save()
+                mirrors.update(repo=repo)
                 if repo.enabled:
                     repo.enable()
                 else:
@@ -418,7 +416,7 @@ def _get_filtered_repos(filter_params):
     from urllib.parse import parse_qs
     params = parse_qs(filter_params)
 
-    repos = Repository.objects.select_related().order_by('name')
+    repos = Repository.objects.select_related('arch').order_by('name')
 
     if 'repotype' in params:
         repos = repos.filter(repotype=params['repotype'][0])
@@ -509,7 +507,7 @@ def _get_filtered_mirrors(filter_params):
     from urllib.parse import parse_qs
     params = parse_qs(filter_params)
 
-    mirrors = Mirror.objects.select_related().order_by('packages_checksum')
+    mirrors = Mirror.objects.select_related('repo').order_by('packages_checksum')
 
     if 'checksum' in params:
         mirrors = mirrors.filter(packages_checksum=params['checksum'][0])
@@ -591,21 +589,21 @@ class RepositoryViewSet(viewsets.ModelViewSet):
     """
     API endpoint that allows repositories to be viewed or edited.
     """
-    queryset = Repository.objects.all()
+    queryset = Repository.objects.select_related('arch').all()
     serializer_class = RepositorySerializer
 
 
 class MirrorViewSet(viewsets.ModelViewSet):
     """
     API endpoint that allows mirrors to be viewed or edited.
     """
-    queryset = Mirror.objects.all()
+    queryset = Mirror.objects.select_related('repo').all()
     serializer_class = MirrorSerializer
 
 
 class MirrorPackageViewSet(viewsets.ModelViewSet):
     """
     API endpoint that allows mirror packages to be viewed or edited.
     """
-    queryset = MirrorPackage.objects.all()
+    queryset = MirrorPackage.objects.select_related('mirror', 'package').all()
     serializer_class = MirrorPackageSerializer