add .iterator() to large queryset loops

Author: furlongm

hosts/tasks.py

@@ -33,7 +33,7 @@ def find_host_updates(host_id):
 def find_all_host_updates():
     """ Task to find updates for all hosts
     """
-    for host in Host.objects.all():
+    for host in Host.objects.all().iterator():
         find_host_updates.delay(host.id)
 
 
@@ -43,7 +43,7 @@ def find_all_host_updates_homogenous():
     """
     updated_hosts = []
     ts = get_datetime_now()
-    for host in Host.objects.all():
+    for host in Host.objects.all().iterator():
         if host not in updated_hosts:
             host.find_updates()
             host.updated_at = ts
@@ -61,7 +61,7 @@ def find_all_host_updates_homogenous():
             updates = host.updates.all()
 
             phosts = []
-            for fhost in filtered_hosts:
+            for fhost in filtered_hosts.iterator():
                 frepos = set(fhost.repos.all())
                 if repos != frepos:
                     continue

sbin/patchman

@@ -164,7 +164,7 @@ def host_updates_alt(host=None):
     updated_hosts = []
     hosts = get_hosts(host, 'Finding updates')
     ts = get_datetime_now()
-    for host in hosts:
+    for host in hosts.iterator():
         info_message(text=str(host))
         if host not in updated_hosts:
             host.find_updates()
@@ -183,7 +183,7 @@ def host_updates_alt(host=None):
             updates = host.updates.all()
 
             phosts = []
-            for fhost in filtered_hosts:
+            for fhost in filtered_hosts.iterator():
 
                 frepos = set(fhost.repos.all())
                 rdiff = repos.difference(frepos)
@@ -352,7 +352,7 @@ def process_reports(host=None, force=False):
 
     info_message(text=text)
 
-    for report in reports:
+    for report in reports.iterator():
         report.process(find_updates=False)
 
 

security/tasks.py

@@ -50,7 +50,7 @@ def update_cves():
 
     if cache.add(lock_key, 'true', lock_expire):
         try:
-            for cve in CVE.objects.all():
+            for cve in CVE.objects.all().iterator():
                 update_cve.delay(cve.id)
         finally:
             cache.delete(lock_key)
@@ -87,7 +87,7 @@ def update_cwes():
 
     if cache.add(lock_key, 'true', lock_expire):
         try:
-            for cwe in CWE.objects.all():
+            for cwe in CWE.objects.all().iterator():
                 update_cwe.delay(cwe.id)
         finally:
             cache.delete(lock_key)

security/utils.py

@@ -42,7 +42,7 @@ def update_cves(cve_id=None, fetch_nist_data=False):
         cve = CVE.objects.get(cve_id=cve_id)
         cve.fetch_cve_data(fetch_nist_data, sleep_secs=0)
     else:
-        for cve in CVE.objects.all():
+        for cve in CVE.objects.all().iterator():
             cve.fetch_cve_data(fetch_nist_data)
 
 
@@ -56,7 +56,7 @@ def update_cwes(cve_id=None):
         cwes = cve.cwes.all()
     else:
         cwes = CWE.objects.all()
-    for cwe in cwes:
+    for cwe in cwes.iterator():
         cwe.fetch_cwe_data()