consolidate and improve homogenous host update logic

Author: furlongm

hosts/tasks.py

@@ -17,8 +17,7 @@
 from celery import shared_task
 
 from hosts.models import Host
-from util import get_datetime_now
-from util.logging import info_message
+from hosts.utils import find_host_updates_homogenous
 
 
 @shared_task(priority=0)
@@ -41,35 +40,4 @@ def find_all_host_updates():
 def find_all_host_updates_homogenous():
     """ Task to find updates for all hosts where hosts are expected to be homogenous
     """
-    updated_host_ids = set()
-    ts = get_datetime_now()
-    for host in Host.objects.all().iterator():
-        if host.id not in updated_host_ids:
-            host.find_updates()
-            host.updated_at = ts
-            host.save()
-
-            # only include hosts with the exact same number of packages
-            filtered_hosts = Host.objects.filter(
-                packages_count=host.packages_count
-            )
-            # and exclude hosts with the current timestamp
-            filtered_hosts = filtered_hosts.exclude(updated_at=ts)
-
-            package_ids = frozenset(host.packages.values_list('id', flat=True))
-            repo_ids = frozenset(host.repos.values_list('id', flat=True))
-            updates = list(host.updates.all())
-
-            for fhost in filtered_hosts.iterator():
-                frepo_ids = frozenset(fhost.repos.values_list('id', flat=True))
-                if repo_ids != frepo_ids:
-                    continue
-                fpackage_ids = frozenset(fhost.packages.values_list('id', flat=True))
-                if package_ids != fpackage_ids:
-                    continue
-
-                fhost.updates.set(updates)
-                fhost.updated_at = ts
-                fhost.save()
-                updated_host_ids.add(fhost.id)
-                info_message(text=f'Added the same updates to {fhost}')
+    find_host_updates_homogenous(Host.objects.all())

hosts/utils.py

@@ -20,6 +20,7 @@
 from django.db import IntegrityError, transaction
 from taggit.models import Tag
 
+from util import get_datetime_now
 from util.logging import error_message, info_message
 
 
@@ -76,6 +77,51 @@ def get_or_create_host(report, arch, osvariant, domain):
         return host
 
 
+def find_host_updates_homogenous(hosts, verbose=False):
+    """ Find updates for hosts, copying updates to homogenous hosts.
+        If a host has the same packages and repos as a previously
+        processed host, it is given the same updates.
+    """
+    from hosts.models import Host
+
+    updated_host_ids = set()
+    ts = get_datetime_now()
+    host_iter = hosts.iterator() if hasattr(hosts, 'iterator') else iter(hosts)
+    for host in host_iter:
+        if verbose:
+            info_message(text=str(host))
+        if host.id not in updated_host_ids:
+            host.find_updates()
+            if verbose:
+                info_message(text='')
+            host.updated_at = ts
+            host.save()
+
+            filtered_hosts = Host.objects.filter(
+                packages_count=host.packages_count)
+            filtered_hosts = filtered_hosts.exclude(updated_at=ts)
+
+            package_ids = frozenset(host.packages.values_list('id', flat=True))
+            repo_ids = frozenset(host.repos.values_list('id', flat=True))
+            updates = list(host.updates.all())
+
+            for fhost in filtered_hosts.iterator():
+                frepo_ids = frozenset(fhost.repos.values_list('id', flat=True))
+                if repo_ids != frepo_ids:
+                    continue
+                fpackage_ids = frozenset(fhost.packages.values_list('id', flat=True))
+                if package_ids != fpackage_ids:
+                    continue
+
+                fhost.updates.set(updates)
+                fhost.updated_at = ts
+                fhost.save()
+                updated_host_ids.add(fhost.id)
+                info_message(text=f'Added the same updates to {fhost}')
+        elif verbose:
+            info_message(text='Updates already added in this run')
+
+
 def clean_tags():
     """ Delete Tags that have no Host
     """

sbin/patchman

@@ -36,7 +36,7 @@ from errata.utils import (
     scan_package_updates_for_affected_packages,
 )
 from hosts.models import Host
-from hosts.utils import clean_tags
+from hosts.utils import clean_tags, find_host_updates_homogenous
 from modules.utils import clean_modules
 from packages.utils import (
     clean_packagenames, clean_packages, clean_packageupdates,
@@ -46,7 +46,6 @@ from reports.tasks import remove_reports_with_no_hosts
 from repos.models import Repository
 from repos.utils import clean_repos
 from security.utils import update_cves, update_cwes
-from util import get_datetime_now
 from util.logging import info_message, set_quiet_mode
 
 
@@ -161,52 +160,8 @@ def clean_reports(hoststr=None):
 def host_updates_alt(host=None):
     """ Find updates for all hosts, specify host for a single host
     """
-    updated_hosts = []
     hosts = get_hosts(host, 'Finding updates')
-    ts = get_datetime_now()
-    for host in hosts.iterator():
-        info_message(text=str(host))
-        if host not in updated_hosts:
-            host.find_updates()
-            info_message(text='')
-            host.updated_at = ts
-            host.save()
-
-            # only include hosts with the same number of packages
-            filtered_hosts = Host.objects.filter(
-                packages_count=host.packages_count)
-            # exclude hosts with the current timestamp
-            filtered_hosts = filtered_hosts.exclude(updated_at=ts)
-
-            packages = set(host.packages.all())
-            repos = set(host.repos.all())
-            updates = host.updates.all()
-
-            phosts = []
-            for fhost in filtered_hosts.iterator():
-
-                frepos = set(fhost.repos.all())
-                rdiff = repos.difference(frepos)
-                if len(rdiff) != 0:
-                    continue
-
-                fpackages = set(fhost.packages.all())
-                pdiff = packages.difference(fpackages)
-                if len(pdiff) != 0:
-                    continue
-
-                phosts.append(fhost)
-
-            for phost in phosts:
-                phost.updates.set(updates)
-                phost.updated_at = ts
-                phost.save()
-                updated_hosts.append(phost)
-                text = f'Added the same updates to {phost}'
-                info_message(text=text)
-        else:
-            text = 'Updates already added in this run'
-            info_message(text=text)
+    find_host_updates_homogenous(hosts, verbose=True)
 
 
 def host_updates(host=None):