Feature/rebase upstream

hosts/managers.py

@@ -16,8 +16,39 @@
 # along with Patchman. If not, see <http://www.gnu.org/licenses/>
 
 from django.db import models
+from django.db.models import Count, Q
+from django.db.models.functions import Coalesce
 
 
 class HostManager(models.Manager):
     def get_queryset(self):
         return super().get_queryset().select_related()
+
+    def with_counts(self, *properties):
+        """
+        Overwrite count methods with annotated values. Used for
+        template rendering which detects if something is a
+        callable automatically.
+        """
+        available_properties = {
+            'get_num_security_updates': Coalesce(
+                Count(
+                    'updates',
+                    filter=Q(updates__security=True),
+                    distinct=True,
+                ),
+                0,
+            ),
+            'get_num_bugfix_updates': Coalesce(
+                Count(
+                    'updates',
+                    filter=Q(updates__security=False),
+                    distinct=True,
+                ),
+                0,
+            ),
+        }
+
+        return self.get_queryset() \
+            .annotate(**{prop: available_properties[prop] for prop in properties}) \
+            .order_by(*self.model._meta.ordering)  # not sure why, but it's not applied

hosts/models.py

@@ -37,9 +37,11 @@
 from patchman.signals import info_message
 from repos.models import Repository
 from repos.utils import find_best_repo
+from hosts.managers import HostManager
 
 
 class Host(models.Model):
+    objects = HostManager()
 
     hostname = models.CharField(max_length=255, unique=True)
     ipaddress = models.GenericIPAddressField()
@@ -60,9 +62,6 @@ class Host(models.Model):
     updated_at = models.DateTimeField(default=timezone.now)
     errata = models.ManyToManyField(Erratum, blank=True)
 
-    from hosts.managers import HostManager
-    objects = HostManager()
-
     class Meta:
         verbose_name = 'Host'
         verbose_name_plural = 'Hosts'

hosts/views.py

@@ -37,7 +37,9 @@
 
 @login_required
 def host_list(request):
-    hosts = Host.objects.select_related()
+    hosts = Host.objects.with_counts('get_num_security_updates',
+                                     'get_num_bugfix_updates') \
+            .select_related()
 
     if 'domain_id' in request.GET:
         hosts = hosts.filter(domain=request.GET['domain_id'])
@@ -111,7 +113,7 @@ def host_list(request):
 def host_detail(request, hostname):
     host = get_object_or_404(Host, hostname=hostname)
     reports = Report.objects.filter(host=hostname).order_by('-created')[:3]
-    hostrepos = HostRepo.objects.filter(host=host)
+    hostrepos = HostRepo.objects.filter(host=host).select_related('repo')
     return render(request,
                   'hosts/host_detail.html',
                   {'host': host,

operatingsystems/views.py

@@ -18,7 +18,7 @@
 from django.shortcuts import get_object_or_404, render, redirect
 from django.contrib.auth.decorators import login_required
 from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
-from django.db.models import Q
+from django.db.models import Prefetch, Q
 from django.contrib import messages
 from django.urls import reverse
 

repos/views.py

@@ -20,14 +20,14 @@
 from django.contrib.auth.decorators import login_required
 from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
 from django.urls import reverse
-from django.db.models import Q
+from django.db.models import Prefetch, Q
 from django.contrib import messages
 from django.db import IntegrityError
 
 from rest_framework import viewsets
 
 from util.filterspecs import Filter, FilterBar
-from hosts.models import HostRepo
+from hosts.models import Host, HostRepo
 from repos.models import Repository, Mirror, MirrorPackage
 from operatingsystems.models import OSRelease
 from arch.models import MachineArchitecture
@@ -38,7 +38,8 @@
 @login_required
 def repo_list(request):
 
-    repos = Repository.objects.select_related().order_by('name')
+    repos = Repository.objects.select_related() \
+        .prefetch_related('mirror_set').order_by('name')
 
     if 'repotype' in request.GET:
         repos = repos.filter(repotype=request.GET['repotype'])
@@ -273,6 +274,12 @@ def repo_detail(request, repo_id):
 
     repo = get_object_or_404(Repository, id=repo_id)
 
+    hosts = Host.objects.with_counts('get_num_security_updates', 'get_num_bugfix_updates')
+    hosts_prefetch = Prefetch('host_set', queryset=hosts)
+    repo = Repository.objects.select_related() \
+           .prefetch_related(hosts_prefetch) \
+           .get(id=repo_id)
+
     return render(request,
                   'repos/repo_detail.html',
                   {'repo': repo})

util/templates/dashboard.html

@@ -9,7 +9,7 @@
 {% block content %}
 
 {% with count=noosrelease_osvariants.count %}
-  {% if noosrelease_osvariants.count > 0 %}
+  {% if count > 0 %}
     <div class="well well-sm">
       <button class="btn btn-sm" data-toggle="collapse" data-target="#noosrelease_osvariants"><span class="badge">{{ count }}</span> OS Variants are not linked to any OS Release</button>
       <div class="collapse" id="noosrelease_osvariants">
@@ -20,7 +20,7 @@
 {% endwith %}
 
 {% with count=nohost_osvariants.count %}
-  {% if nohost_osvariants.count > 0 %}
+  {% if count > 0 %}
     <div class="well well-sm">
       <button class="btn btn-sm" data-toggle="collapse" data-target="#nohost_osvariants"><span class="badge">{{ count }}</span> OS Variants are not installed on any Host</button>
         <div class="collapse" id="nohost_osvariants">

util/views.py

@@ -19,13 +19,15 @@
 
 from django.shortcuts import render
 from django.contrib.auth.decorators import login_required
+
 from django.contrib.sites.models import Site
-from django.db.models import F
+from django.db.models import Count, Exists, F, OuterRef
+from django.db.models.functions import Coalesce
 
 from hosts.models import Host
 from operatingsystems.models import OSVariant, OSRelease
 from repos.models import Repository, Mirror
-from packages.models import Package
+from packages.models import Package, PackageUpdate
 from reports.models import Report
 from util import get_setting_of_type
 
@@ -38,10 +40,12 @@ def dashboard(request):
     except Site.DoesNotExist:
         site = {'name': '', 'domainname': ''}
 
-    hosts = Host.objects.all()
+    hosts = Host.objects.with_counts('get_num_security_updates',
+                                     'get_num_bugfix_updates') \
+            .select_related()
     osvariants = OSVariant.objects.all()
     osreleases = OSRelease.objects.all()
-    repos = Repository.objects.all()
+    repos = Repository.objects.all().prefetch_related('mirror_set')
     packages = Package.objects.all()
 
     # host issues
@@ -79,23 +83,34 @@ def dashboard(request):
     nohost_repos = repos.filter(host__isnull=True)
 
     # package issues
-    norepo_packages = packages.filter(mirror__isnull=True, oldpackage__isnull=True, host__isnull=False).distinct()  # noqa
-    orphaned_packages = packages.filter(mirror__isnull=True, host__isnull=True).distinct()  # noqa
+    nohost_packages = Host.packages.through.objects \
+        .filter(package=OuterRef('pk'), host__isnull=False)
+    nomirror_packages = Mirror.packages.through.objects \
+        .filter(package=OuterRef('pk'), mirror__isnull=False)
+    nooldpackage_packages = PackageUpdate.objects \
+        .filter(oldpackage=OuterRef('pk'), oldpackage__isnull=False)
+    norepo_packages = packages.filter(Exists(nohost_packages),
+                                      ~Exists(nomirror_packages),
+                                      ~Exists(nooldpackage_packages)) \
+                      .distinct()
+    orphaned_packages = packages.filter(~Exists(nohost_packages),
+                                        ~Exists(nomirror_packages)) \
+                        .distinct()
 
     # report issues
     unprocessed_reports = Report.objects.filter(processed=False)
 
     checksums = {}
     possible_mirrors = {}
 
-    for csvalue in Mirror.objects.all().values('packages_checksum').distinct():
-        checksum = csvalue['packages_checksum']
-        if checksum is not None and checksum != 'yast':
-            for mirror in Mirror.objects.filter(packages_checksum=checksum):
-                if mirror.packages.count() > 0:
-                    if checksum not in checksums:
-                        checksums[checksum] = []
-                    checksums[checksum].append(mirror)
+    mirrors = Mirror.objects.all() \
+              .annotate(packages_count=Coalesce(Count('packages', distinct=True), 0)) \
+              .select_related()
+    for mirror in mirrors:
+        if mirror.packages_checksum != 'yast' and mirror.packages_count > 0:
+            if mirror.packages_checksum not in checksums:
+                checksums[mirror.packages_checksum] = []
+            checksums[mirror.packages_checksum].append(mirror)
 
     for checksum in checksums:
         first_mirror = checksums[checksum][0]