From 7e7bc304bf5b51bdc0242186212b557057006e8b Mon Sep 17 00:00:00 2001 From: cpoppema Date: Mon, 18 Mar 2024 18:38:01 +0100 Subject: [PATCH 1/3] fix: show orphaned packages and unprocessed reports on dashboard, and fix verbose_name for Repository and Mirror --- repos/migrations/0001_initial.py | 2 ++ repos/models.py | 4 ++-- util/templates/dashboard.html | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/repos/migrations/0001_initial.py b/repos/migrations/0001_initial.py index a99f6878..717f4670 100644 --- a/repos/migrations/0001_initial.py +++ b/repos/migrations/0001_initial.py @@ -28,6 +28,7 @@ class Migration(migrations.Migration): ('fail_count', models.IntegerField(default=0)), ], options={ + 'verbose_name': 'Mirror', 'verbose_name_plural': 'Mirrors', }, ), @@ -44,6 +45,7 @@ class Migration(migrations.Migration): ('arch', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='arch.machinearchitecture')), ], options={ + 'verbose_name': 'Repository', 'verbose_name_plural': 'Repositories', }, ), diff --git a/repos/models.py b/repos/models.py index 181a103d..f8e8bcfb 100644 --- a/repos/models.py +++ b/repos/models.py @@ -55,7 +55,7 @@ class Repository(models.Model): objects = RepositoryManager() class Meta: - verbose_name_plural = 'Repository' + verbose_name = 'Repository' verbose_name_plural = 'Repositories' def __str__(self): @@ -153,7 +153,7 @@ class Mirror(models.Model): fail_count = models.IntegerField(default=0) class Meta: - verbose_name_plural = 'Mirror' + verbose_name = 'Mirror' verbose_name_plural = 'Mirrors' def __str__(self): diff --git a/util/templates/dashboard.html b/util/templates/dashboard.html index 631fea36..c43eb576 100644 --- a/util/templates/dashboard.html +++ b/util/templates/dashboard.html @@ -217,7 +217,7 @@ {% endwith %} {% with count=orphaned_packages.count %} - {% if count < 0 %} + {% if count > 0 %}
@@ -232,7 +232,7 @@ {% endwith %} {% with count=unprocessed_reports.count %} - {% if count < 0 %} + {% if count > 0 %}
From bc725d7fa018e7405d1bebaf179cf8931c15fd90 Mon Sep 17 00:00:00 2001 From: cpoppema Date: Mon, 18 Mar 2024 19:41:38 +0100 Subject: [PATCH 2/3] perf: improve page timings by reducing query count --- hosts/managers.py | 31 +++++++++++++++++++++++++ hosts/models.py | 5 ++-- hosts/views.py | 6 +++-- operatingsystems/views.py | 2 +- repos/views.py | 13 ++++++++--- util/templates/dashboard.html | 4 ++-- util/views.py | 43 +++++++++++++++++++++++------------ 7 files changed, 79 insertions(+), 25 deletions(-) diff --git a/hosts/managers.py b/hosts/managers.py index 85a489d4..82d70213 100644 --- a/hosts/managers.py +++ b/hosts/managers.py @@ -16,8 +16,39 @@ # along with Patchman. If not, see from django.db import models +from django.db.models import Count, Q +from django.db.models.functions import Coalesce class HostManager(models.Manager): def get_queryset(self): return super().get_queryset().select_related() + + def with_counts(self, *properties): + """ + Overwrite count methods with annotated values. Used for + template rendering which detects if something is a + callable automatically. + """ + available_properties = { + 'get_num_security_updates': Coalesce( + Count( + 'updates', + filter=Q(updates__security=True), + distinct=True, + ), + 0, + ), + 'get_num_bugfix_updates': Coalesce( + Count( + 'updates', + filter=Q(updates__security=False), + distinct=True, + ), + 0, + ), + } + + return self.get_queryset() \ + .annotate(**{prop: available_properties[prop] for prop in properties}) \ + .order_by(*self.model._meta.ordering) # not sure why, but it's not applied diff --git a/hosts/models.py b/hosts/models.py index a6c451b5..5a1629d8 100644 --- a/hosts/models.py +++ b/hosts/models.py @@ -37,9 +37,11 @@ from patchman.signals import info_message from repos.models import Repository from repos.utils import find_best_repo +from hosts.managers import HostManager class Host(models.Model): + objects = HostManager() hostname = models.CharField(max_length=255, unique=True) ipaddress = models.GenericIPAddressField() @@ -60,9 +62,6 @@ class Host(models.Model): updated_at = models.DateTimeField(default=timezone.now) errata = models.ManyToManyField(Erratum, blank=True) - from hosts.managers import HostManager - objects = HostManager() - class Meta: verbose_name = 'Host' verbose_name_plural = 'Hosts' diff --git a/hosts/views.py b/hosts/views.py index 0fc83ffa..faad410b 100644 --- a/hosts/views.py +++ b/hosts/views.py @@ -37,7 +37,9 @@ @login_required def host_list(request): - hosts = Host.objects.select_related() + hosts = Host.objects.with_counts('get_num_security_updates', + 'get_num_bugfix_updates') \ + .select_related() if 'domain_id' in request.GET: hosts = hosts.filter(domain=request.GET['domain_id']) @@ -111,7 +113,7 @@ def host_list(request): def host_detail(request, hostname): host = get_object_or_404(Host, hostname=hostname) reports = Report.objects.filter(host=hostname).order_by('-created')[:3] - hostrepos = HostRepo.objects.filter(host=host) + hostrepos = HostRepo.objects.filter(host=host).select_related('repo') return render(request, 'hosts/host_detail.html', {'host': host, diff --git a/operatingsystems/views.py b/operatingsystems/views.py index 2b696f92..e244a93c 100644 --- a/operatingsystems/views.py +++ b/operatingsystems/views.py @@ -18,7 +18,7 @@ from django.shortcuts import get_object_or_404, render, redirect from django.contrib.auth.decorators import login_required from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger -from django.db.models import Q +from django.db.models import Prefetch, Q from django.contrib import messages from django.urls import reverse diff --git a/repos/views.py b/repos/views.py index 199c834e..afe8193f 100644 --- a/repos/views.py +++ b/repos/views.py @@ -20,14 +20,14 @@ from django.contrib.auth.decorators import login_required from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger from django.urls import reverse -from django.db.models import Q +from django.db.models import Prefetch, Q from django.contrib import messages from django.db import IntegrityError from rest_framework import viewsets from util.filterspecs import Filter, FilterBar -from hosts.models import HostRepo +from hosts.models import Host, HostRepo from repos.models import Repository, Mirror, MirrorPackage from operatingsystems.models import OSRelease from arch.models import MachineArchitecture @@ -38,7 +38,8 @@ @login_required def repo_list(request): - repos = Repository.objects.select_related().order_by('name') + repos = Repository.objects.select_related() \ + .prefetch_related('mirror_set').order_by('name') if 'repotype' in request.GET: repos = repos.filter(repotype=request.GET['repotype']) @@ -273,6 +274,12 @@ def repo_detail(request, repo_id): repo = get_object_or_404(Repository, id=repo_id) + hosts = Host.objects.with_counts('get_num_security_updates', 'get_num_bugfix_updates') + hosts_prefetch = Prefetch('host_set', queryset=hosts) + repo = Repository.objects.select_related() \ + .prefetch_related(hosts_prefetch) \ + .get(id=repo_id) + return render(request, 'repos/repo_detail.html', {'repo': repo}) diff --git a/util/templates/dashboard.html b/util/templates/dashboard.html index c43eb576..1d16851c 100644 --- a/util/templates/dashboard.html +++ b/util/templates/dashboard.html @@ -9,7 +9,7 @@ {% block content %} {% with count=noosrelease_osvariants.count %} - {% if noosrelease_osvariants.count > 0 %} + {% if count > 0 %}
@@ -20,7 +20,7 @@ {% endwith %} {% with count=nohost_osvariants.count %} - {% if nohost_osvariants.count > 0 %} + {% if count > 0 %}
diff --git a/util/views.py b/util/views.py index b66db6b0..df3b033e 100644 --- a/util/views.py +++ b/util/views.py @@ -19,13 +19,15 @@ from django.shortcuts import render from django.contrib.auth.decorators import login_required + from django.contrib.sites.models import Site -from django.db.models import F +from django.db.models import Count, Exists, F, OuterRef +from django.db.models.functions import Coalesce from hosts.models import Host from operatingsystems.models import OSVariant, OSRelease from repos.models import Repository, Mirror -from packages.models import Package +from packages.models import Package, PackageUpdate from reports.models import Report from util import get_setting_of_type @@ -38,10 +40,12 @@ def dashboard(request): except Site.DoesNotExist: site = {'name': '', 'domainname': ''} - hosts = Host.objects.all() + hosts = Host.objects.with_counts('get_num_security_updates', + 'get_num_bugfix_updates') \ + .select_related() osvariants = OSVariant.objects.all() osreleases = OSRelease.objects.all() - repos = Repository.objects.all() + repos = Repository.objects.all().prefetch_related('mirror_set') packages = Package.objects.all() # host issues @@ -79,8 +83,19 @@ def dashboard(request): nohost_repos = repos.filter(host__isnull=True) # package issues - norepo_packages = packages.filter(mirror__isnull=True, oldpackage__isnull=True, host__isnull=False).distinct() # noqa - orphaned_packages = packages.filter(mirror__isnull=True, host__isnull=True).distinct() # noqa + nohost_packages = Host.packages.through.objects \ + .filter(package=OuterRef('pk'), host__isnull=False) + nomirror_packages = Mirror.packages.through.objects \ + .filter(package=OuterRef('pk'), mirror__isnull=False) + nooldpackage_packages = PackageUpdate.objects \ + .filter(oldpackage=OuterRef('pk'), oldpackage__isnull=False) + norepo_packages = packages.filter(Exists(nohost_packages), + ~Exists(nomirror_packages), + ~Exists(nooldpackage_packages)) \ + .distinct() + orphaned_packages = packages.filter(~Exists(nohost_packages), + ~Exists(nomirror_packages)) \ + .distinct() # report issues unprocessed_reports = Report.objects.filter(processed=False) @@ -88,14 +103,14 @@ def dashboard(request): checksums = {} possible_mirrors = {} - for csvalue in Mirror.objects.all().values('packages_checksum').distinct(): - checksum = csvalue['packages_checksum'] - if checksum is not None and checksum != 'yast': - for mirror in Mirror.objects.filter(packages_checksum=checksum): - if mirror.packages.count() > 0: - if checksum not in checksums: - checksums[checksum] = [] - checksums[checksum].append(mirror) + mirrors = Mirror.objects.all() \ + .annotate(packages_count=Coalesce(Count('packages', distinct=True), 0)) \ + .select_related() + for mirror in mirrors: + if mirror.packages_checksum != 'yast' and mirror.packages_count > 0: + if mirror.packages_checksum not in checksums: + checksums[mirror.packages_checksum] = [] + checksums[mirror.packages_checksum].append(mirror) for checksum in checksums: first_mirror = checksums[checksum][0] From 79723dd758820e123018f9b7efea3e97347e669e Mon Sep 17 00:00:00 2001 From: Jasper Hafkenscheid Date: Fri, 24 May 2024 15:47:55 +0200 Subject: [PATCH 3/3] Revert "fix: show orphaned packages and unprocessed reports on dashboard, and fix verbose_name for Repository and Mirror" This reverts commit 9bb6199c3a19e27f390ed6f3c40671dcf957ba62. --- repos/migrations/0001_initial.py | 2 -- repos/models.py | 4 ++-- util/templates/dashboard.html | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/repos/migrations/0001_initial.py b/repos/migrations/0001_initial.py index 717f4670..a99f6878 100644 --- a/repos/migrations/0001_initial.py +++ b/repos/migrations/0001_initial.py @@ -28,7 +28,6 @@ class Migration(migrations.Migration): ('fail_count', models.IntegerField(default=0)), ], options={ - 'verbose_name': 'Mirror', 'verbose_name_plural': 'Mirrors', }, ), @@ -45,7 +44,6 @@ class Migration(migrations.Migration): ('arch', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='arch.machinearchitecture')), ], options={ - 'verbose_name': 'Repository', 'verbose_name_plural': 'Repositories', }, ), diff --git a/repos/models.py b/repos/models.py index f8e8bcfb..181a103d 100644 --- a/repos/models.py +++ b/repos/models.py @@ -55,7 +55,7 @@ class Repository(models.Model): objects = RepositoryManager() class Meta: - verbose_name = 'Repository' + verbose_name_plural = 'Repository' verbose_name_plural = 'Repositories' def __str__(self): @@ -153,7 +153,7 @@ class Mirror(models.Model): fail_count = models.IntegerField(default=0) class Meta: - verbose_name = 'Mirror' + verbose_name_plural = 'Mirror' verbose_name_plural = 'Mirrors' def __str__(self): diff --git a/util/templates/dashboard.html b/util/templates/dashboard.html index 1d16851c..de9bfc72 100644 --- a/util/templates/dashboard.html +++ b/util/templates/dashboard.html @@ -217,7 +217,7 @@ {% endwith %} {% with count=orphaned_packages.count %} - {% if count > 0 %} + {% if count < 0 %}
@@ -232,7 +232,7 @@ {% endwith %} {% with count=unprocessed_reports.count %} - {% if count > 0 %} + {% if count < 0 %}