add .iterator() to large queryset loops

Author: furlongm

hosts/tasks.py

@@ -33,7 +33,7 @@ def find_host_updates(host_id):
 def find_all_host_updates():
     """ Task to find updates for all hosts
     """
-    for host in Host.objects.all():
+    for host in Host.objects.all().iterator():
         find_host_updates.delay(host.id)
 
 

sbin/patchman

@@ -352,7 +352,7 @@ def process_reports(host=None, force=False):
 
     info_message(text=text)
 
-    for report in reports:
+    for report in reports.iterator():
         report.process(find_updates=False)
 
 

security/tasks.py

@@ -50,7 +50,7 @@ def update_cves():
 
     if cache.add(lock_key, 'true', lock_expire):
         try:
-            for cve in CVE.objects.all():
+            for cve in CVE.objects.all().iterator():
                 update_cve.delay(cve.id)
         finally:
             cache.delete(lock_key)
@@ -87,7 +87,7 @@ def update_cwes():
 
     if cache.add(lock_key, 'true', lock_expire):
         try:
-            for cwe in CWE.objects.all():
+            for cwe in CWE.objects.all().iterator():
                 update_cwe.delay(cwe.id)
         finally:
             cache.delete(lock_key)

security/utils.py

@@ -42,7 +42,7 @@ def update_cves(cve_id=None, fetch_nist_data=False):
         cve = CVE.objects.get(cve_id=cve_id)
         cve.fetch_cve_data(fetch_nist_data, sleep_secs=0)
     else:
-        for cve in CVE.objects.all():
+        for cve in CVE.objects.all().iterator():
             cve.fetch_cve_data(fetch_nist_data)
 
 
@@ -56,7 +56,7 @@ def update_cwes(cve_id=None):
         cwes = cve.cwes.all()
     else:
         cwes = CWE.objects.all()
-    for cwe in cwes:
+    for cwe in cwes.iterator():
         cwe.fetch_cwe_data()