add cached count fields to avoid expensive COUNT queries

furlongm · furlongm · commit 9750cfb2fad3 · 2026-02-11T00:28:57.000-05:00
- Add sec_updates_count, bug_updates_count, packages_count, errata_count to Host
- Add packages_count to Mirror
- Add hosts_count to OSVariant
- Add Django signals to auto-update counts on M2M/FK changes
- Update views to use cached fields instead of annotations
diff --git a/hosts/apps.py b/hosts/apps.py
@@ -19,3 +19,6 @@
 
 class HostsConfig(AppConfig):
     name = 'hosts'
+
+    def ready(self):
+        import hosts.signals  # noqa
diff --git a/hosts/migrations/0011_host_bug_updates_count_host_errata_count_and_more.py b/hosts/migrations/0011_host_bug_updates_count_host_errata_count_and_more.py
@@ -0,0 +1,33 @@
+# Generated by Django 4.2.28 on 2026-02-11 04:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('hosts', '0010_alter_hostrepo_options'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='host',
+            name='bug_updates_count',
+            field=models.PositiveIntegerField(db_index=True, default=0),
+        ),
+        migrations.AddField(
+            model_name='host',
+            name='errata_count',
+            field=models.PositiveIntegerField(db_index=True, default=0),
+        ),
+        migrations.AddField(
+            model_name='host',
+            name='packages_count',
+            field=models.PositiveIntegerField(db_index=True, default=0),
+        ),
+        migrations.AddField(
+            model_name='host',
+            name='sec_updates_count',
+            field=models.PositiveIntegerField(db_index=True, default=0),
+        ),
+    ]
diff --git a/hosts/migrations/0012_backfill_cached_counts.py b/hosts/migrations/0012_backfill_cached_counts.py
@@ -0,0 +1,33 @@
+# Generated by Django 4.2.28 on 2026-02-11
+
+from django.db import migrations
+
+
+def backfill_host_counts(apps, schema_editor):
+    """Backfill cached count fields for existing hosts."""
+    Host = apps.get_model('hosts', 'Host')
+    for host in Host.objects.all():
+        host.sec_updates_count = host.updates.filter(security=True).count()
+        host.bug_updates_count = host.updates.filter(security=False).count()
+        host.packages_count = host.packages.count()
+        host.errata_count = host.errata.count()
+        host.save(update_fields=[
+            'sec_updates_count', 'bug_updates_count',
+            'packages_count', 'errata_count'
+        ])
+
+
+def reverse_backfill(apps, schema_editor):
+    """No-op reverse - counts will be recalculated on next report."""
+    pass
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('hosts', '0011_host_bug_updates_count_host_errata_count_and_more'),
+    ]
+
+    operations = [
+        migrations.RunPython(backfill_host_counts, reverse_backfill),
+    ]
diff --git a/hosts/models.py b/hosts/models.py
@@ -60,6 +60,11 @@ class Host(models.Model):
     tags = TaggableManager(blank=True)
     updated_at = models.DateTimeField(default=timezone.now)
     errata = models.ManyToManyField(Erratum, blank=True)
+    # Cached count fields for query optimization
+    sec_updates_count = models.PositiveIntegerField(default=0, db_index=True)
+    bug_updates_count = models.PositiveIntegerField(default=0, db_index=True)
+    packages_count = models.PositiveIntegerField(default=0, db_index=True)
+    errata_count = models.PositiveIntegerField(default=0, db_index=True)
 
     from hosts.managers import HostManager
     objects = HostManager()
@@ -97,20 +102,23 @@ def get_absolute_url(self):
         return reverse('hosts:host_detail', args=[self.hostname])
 
     def get_num_security_updates(self):
-        return self.updates.filter(security=True).count()
+        return self.sec_updates_count
 
     def get_num_bugfix_updates(self):
-        return self.updates.filter(security=False).count()
+        return self.bug_updates_count
 
     def get_num_updates(self):
-        return self.updates.count()
+        return self.sec_updates_count + self.bug_updates_count
 
     def get_num_packages(self):
-        return self.packages.count()
+        return self.packages_count
 
     def get_num_repos(self):
         return self.repos.count()
 
+    def get_num_errata(self):
+        return self.errata_count
+
     def check_rdns(self):
         if self.check_dns:
             update_rdns(self)
diff --git a/hosts/serializers.py b/hosts/serializers.py
@@ -33,10 +33,10 @@ class Meta:
                   'updated_at', 'bugfix_update_count', 'security_update_count')
 
     def get_bugfix_update_count(self, obj):
-        return obj.updates.filter(security=False).count()
+        return obj.bug_updates_count
 
     def get_security_update_count(self, obj):
-        return obj.updates.filter(security=True).count()
+        return obj.sec_updates_count
 
 
 class HostRepoSerializer(serializers.HyperlinkedModelSerializer):
diff --git a/hosts/signals.py b/hosts/signals.py
@@ -0,0 +1,45 @@
+# Copyright 2026 Marcus Furlong <furlongm@gmail.com>
+#
+# This file is part of Patchman.
+#
+# Patchman is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 only.
+#
+# Patchman is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Patchman. If not, see <http://www.gnu.org/licenses/>
+
+from django.db.models.signals import m2m_changed
+from django.dispatch import receiver
+
+from hosts.models import Host
+
+
+@receiver(m2m_changed, sender=Host.packages.through)
+def update_host_packages_count(sender, instance, action, **kwargs):
+    """Update packages_count when Host.packages M2M changes."""
+    if action in ('post_add', 'post_remove', 'post_clear'):
+        instance.packages_count = instance.packages.count()
+        instance.save(update_fields=['packages_count'])
+
+
+@receiver(m2m_changed, sender=Host.updates.through)
+def update_host_updates_count(sender, instance, action, **kwargs):
+    """Update sec_updates_count and bug_updates_count when Host.updates M2M changes."""
+    if action in ('post_add', 'post_remove', 'post_clear'):
+        instance.sec_updates_count = instance.updates.filter(security=True).count()
+        instance.bug_updates_count = instance.updates.filter(security=False).count()
+        instance.save(update_fields=['sec_updates_count', 'bug_updates_count'])
+
+
+@receiver(m2m_changed, sender=Host.errata.through)
+def update_host_errata_count(sender, instance, action, **kwargs):
+    """Update errata_count when Host.errata M2M changes."""
+    if action in ('post_add', 'post_remove', 'post_clear'):
+        instance.errata_count = instance.errata.count()
+        instance.save(update_fields=['errata_count'])
diff --git a/hosts/tables.py b/hosts/tables.py
@@ -31,7 +31,7 @@
     '{% endwith %}'
 )
 AFFECTED_ERRATA_TEMPLATE = (
-    '{% with count=record.errata.count %}'
+    '{% with count=record.errata_count %}'
     '{% if count != 0 %}'
     '<a href="{% url \'errata:erratum_list\' %}?host={{ record.hostname }}">{{ count }}</a>'
     '{% else %}{% endif %}{% endwith %}'
diff --git a/hosts/tasks.py b/hosts/tasks.py
@@ -15,7 +15,6 @@
 # along with Patchman. If not, see <http://www.gnu.org/licenses/>
 
 from celery import shared_task
-from django.db.models import Count
 
 from hosts.models import Host
 from util import get_datetime_now
@@ -51,10 +50,9 @@ def find_all_host_updates_homogenous():
             host.save()
 
             # only include hosts with the exact same number of packages
-            filtered_hosts = Host.objects.annotate(
-                packages_count=Count('packages')).filter(
-                    packages_count=host.packages.count()
-                )
+            filtered_hosts = Host.objects.filter(
+                packages_count=host.packages_count
+            )
             # and exclude hosts with the current timestamp
             filtered_hosts = filtered_hosts.exclude(updated_at=ts)
 
diff --git a/hosts/views.py b/hosts/views.py
@@ -17,7 +17,7 @@
 
 from django.contrib import messages
 from django.contrib.auth.decorators import login_required
-from django.db.models import Count, Q
+from django.db.models import Q
 from django.shortcuts import get_object_or_404, redirect, render
 from django.urls import reverse
 from django_filters import rest_framework as filters
@@ -76,12 +76,8 @@ def _get_filtered_hosts(filter_params):
 
 @login_required
 def host_list(request):
-    hosts = Host.objects.select_related().annotate(
-        sec_updates_count=Count('updates', filter=Q(updates__security=True), distinct=True),
-        bug_updates_count=Count('updates', filter=Q(updates__security=False), distinct=True),
-        errata_count=Count('errata', distinct=True),
-        packages_count=Count('packages', distinct=True),
-    )
+    # Use cached count fields instead of expensive annotations
+    hosts = Host.objects.select_related()
 
     if 'domain_id' in request.GET:
         hosts = hosts.filter(domain=request.GET['domain_id'])
diff --git a/operatingsystems/apps.py b/operatingsystems/apps.py
@@ -19,3 +19,6 @@
 
 class OperatingsystemsConfig(AppConfig):
     name = 'operatingsystems'
+
+    def ready(self):
+        import operatingsystems.signals  # noqa
diff --git a/operatingsystems/migrations/0009_osvariant_hosts_count.py b/operatingsystems/migrations/0009_osvariant_hosts_count.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.2.28 on 2026-02-11 04:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('operatingsystems', '0008_alter_osrelease_options_alter_osvariant_options'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='osvariant',
+            name='hosts_count',
+            field=models.PositiveIntegerField(db_index=True, default=0),
+        ),
+    ]
diff --git a/operatingsystems/migrations/0010_backfill_osvariant_counts.py b/operatingsystems/migrations/0010_backfill_osvariant_counts.py
@@ -0,0 +1,29 @@
+# Generated by Django 4.2.28 on 2026-02-11
+
+from django.db import migrations
+
+
+def backfill_osvariant_counts(apps, schema_editor):
+    """Backfill hosts_count for existing osvariants."""
+    OSVariant = apps.get_model('operatingsystems', 'OSVariant')
+    Host = apps.get_model('hosts', 'Host')
+    for osvariant in OSVariant.objects.all():
+        osvariant.hosts_count = Host.objects.filter(osvariant=osvariant).count()
+        osvariant.save(update_fields=['hosts_count'])
+
+
+def reverse_backfill(apps, schema_editor):
+    """No-op reverse."""
+    pass
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('operatingsystems', '0009_osvariant_hosts_count'),
+        ('hosts', '0012_backfill_cached_counts'),
+    ]
+
+    operations = [
+        migrations.RunPython(backfill_osvariant_counts, reverse_backfill),
+    ]
diff --git a/operatingsystems/models.py b/operatingsystems/models.py
@@ -57,6 +57,8 @@ class OSVariant(models.Model):
     arch = models.ForeignKey(MachineArchitecture, blank=True, null=True, on_delete=models.CASCADE)
     osrelease = models.ForeignKey(OSRelease, blank=True, null=True, on_delete=models.SET_NULL)
     codename = models.CharField(max_length=255, blank=True)
+    # Cached count field for query optimization
+    hosts_count = models.PositiveIntegerField(default=0, db_index=True)
 
     class Meta:
         verbose_name = 'Operating System Variant'
diff --git a/operatingsystems/signals.py b/operatingsystems/signals.py
@@ -0,0 +1,56 @@
+# Copyright 2026 Marcus Furlong <furlongm@gmail.com>
+#
+# This file is part of Patchman.
+#
+# Patchman is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 only.
+#
+# Patchman is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Patchman. If not, see <http://www.gnu.org/licenses/>
+
+from django.db.models.signals import post_delete, post_save, pre_save
+from django.dispatch import receiver
+
+from hosts.models import Host
+
+
+@receiver(pre_save, sender=Host)
+def track_osvariant_change(sender, instance, **kwargs):
+    """Track old osvariant before save to update its count."""
+    if instance.pk:
+        try:
+            old_instance = Host.objects.get(pk=instance.pk)
+            instance._old_osvariant = old_instance.osvariant
+        except Host.DoesNotExist:
+            instance._old_osvariant = None
+    else:
+        instance._old_osvariant = None
+
+
+@receiver(post_save, sender=Host)
+def update_osvariant_count_on_save(sender, instance, created, **kwargs):
+    """Update OSVariant.hosts_count when Host is created or osvariant changes."""
+    # Update new osvariant count
+    if instance.osvariant:
+        instance.osvariant.hosts_count = Host.objects.filter(osvariant=instance.osvariant).count()
+        instance.osvariant.save(update_fields=['hosts_count'])
+
+    # Update old osvariant count if it changed
+    old_osvariant = getattr(instance, '_old_osvariant', None)
+    if old_osvariant and old_osvariant != instance.osvariant:
+        old_osvariant.hosts_count = Host.objects.filter(osvariant=old_osvariant).count()
+        old_osvariant.save(update_fields=['hosts_count'])
+
+
+@receiver(post_delete, sender=Host)
+def update_osvariant_count_on_delete(sender, instance, **kwargs):
+    """Update OSVariant.hosts_count when Host is deleted."""
+    if instance.osvariant:
+        instance.osvariant.hosts_count = Host.objects.filter(osvariant=instance.osvariant).count()
+        instance.osvariant.save(update_fields=['hosts_count'])
diff --git a/operatingsystems/views.py b/operatingsystems/views.py
@@ -77,8 +77,8 @@ def _get_filtered_osreleases(filter_params):
 
 @login_required
 def osvariant_list(request):
+    # Use cached hosts_count instead of expensive annotation
     osvariants = OSVariant.objects.select_related().annotate(
-        hosts_count=Count('host'),
         repos_count=Count('osrelease__repos'),
     )
 
diff --git a/repos/apps.py b/repos/apps.py
@@ -19,3 +19,6 @@
 
 class ReposConfig(AppConfig):
     name = 'repos'
+
+    def ready(self):
+        import repos.signals  # noqa
diff --git a/repos/migrations/0008_mirror_packages_count.py b/repos/migrations/0008_mirror_packages_count.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.2.28 on 2026-02-11 04:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('repos', '0007_alter_mirror_options_alter_mirrorpackage_options_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='mirror',
+            name='packages_count',
+            field=models.PositiveIntegerField(db_index=True, default=0),
+        ),
+    ]
diff --git a/repos/migrations/0009_backfill_mirror_counts.py b/repos/migrations/0009_backfill_mirror_counts.py
diff --git a/repos/models.py b/repos/models.py
diff --git a/repos/signals.py b/repos/signals.py
diff --git a/repos/tables.py b/repos/tables.py
diff --git a/repos/views.py b/repos/views.py
diff --git a/sbin/patchman b/sbin/patchman
diff --git a/util/views.py b/util/views.py

Original file line number	Diff line number	Diff line change
`@@ -31,7 +31,7 @@`
`31`	`31`	`'{% endwith %}'`
`32`	`32`	`)`
`33`	`33`	`AFFECTED_ERRATA_TEMPLATE = (`
`34`		`- '{% with count=record.errata.count %}'`
	`34`	`+ '{% with count=record.errata_count %}'`
`35`	`35`	`'{% if count != 0 %}'`
`36`	`36`	`'<a href="{% url \'errata:erratum_list\' %}?host={{ record.hostname }}">{{ count }}</a>'`
`37`	`37`	`'{% else %}{% endif %}{% endwith %}'`