Merge branch 'main' into issue/1082

Author: jklare

Tests/iaas/openstack_test.py

@@ -227,7 +227,7 @@ def harness(name, *check_fns):
     print(f"{name}: {result}")
 
 
-def run_sanity_checks(container):
+def run_preflight_checks(container):
     # make sure that we can connect to the cloud and that the user doesn't have elevated privileges
     # the former would lead to each testcase aborting with a marginally useful message;
     # the latter would lead to scs_0116_permissions aborting, which we don't want to single out
@@ -274,7 +274,13 @@ def main(argv):
         sys.exit(1)
 
     c = make_container(cloud)
-    run_sanity_checks(c)
+    try:
+        run_preflight_checks(c)
+    except Exception:
+        logger.critical("Pre-flight checks failed. Reporting all testcases as ABORT.")
+        for testcase in testcases:
+            print(f"{testcase}: ABORT")
+        raise
     for testcase in testcases:
         harness(testcase, lambda: getattr(c, testcase.replace('-', '_')))
     return 0

Tests/iaas/scs_0101_entropy/entropy_check.py

@@ -74,28 +74,28 @@ def compute_scs_0101_entropy_avail(collected_vm_output, image_name):
 def compute_scs_0101_fips_test(collected_vm_output, image_name):
     """This test ensures that the 'fips test' via `rngtest` is passed on a test VM."""
     lines = collected_vm_output['fips-test']
-    try:
-        fips_data = '\n'.join(lines)
-        failure_re = re.search(r'failures:\s\d+', fips_data, flags=re.MULTILINE)
-        if failure_re:
-            fips_failures = failure_re.string[failure_re.regs[0][0]:failure_re.regs[0][1]].split(" ")[1]
-            if int(fips_failures) <= 3:
-                return True  # strict test passed
-            logger.info(
-                f"VM '{image_name}' didn't pass the strict FIPS 140-2 testing. "
-                f"Expected a maximum of 3 failures, got {fips_failures}."
-            )
-            if int(fips_failures) <= 5:
-                return True  # lenient test passed
-            logger.error(
-                f"VM '{image_name}' didn't pass the FIPS 140-2 testing. "
-                f"Expected a maximum of 5 failures, got {fips_failures}."
-            )
-        else:
-            logger.error(f"VM '{image_name}': failed to determine fips failures")
-            logger.debug(f"stderr following:\n{fips_data}")
-    except BaseException:
-        logger.critical(f"Couldn't check VM '{image_name}' requirements", exc_info=True)
+    fips_data = '\n'.join(lines)
+    failure_re = re.search(r'failures:\s\d+', fips_data, flags=re.MULTILINE)
+    if not failure_re:
+        # It seems possible that 'failures: 0' is just omitted, and we could check for
+        # 'successes: 1000' to verify that, but I have observed this only once in many
+        # many runs over multiple years. I'm inclined to label this 'inconclusive'
+        # instead of making the code more complex and risking false certainty.
+        logger.debug(f"failed to determine fips failures; stderr following:\n{fips_data}")
+        raise RuntimeError(f"VM '{image_name}': failed to determine fips failures")
+    fips_failures = failure_re.string[failure_re.regs[0][0]:failure_re.regs[0][1]].split(" ")[1]
+    if int(fips_failures) <= 3:
+        return True  # strict test passed
+    logger.info(
+        f"VM '{image_name}' didn't pass the strict FIPS 140-2 testing. "
+        f"Expected a maximum of 3 failures, got {fips_failures}."
+    )
+    if int(fips_failures) <= 5:
+        return True  # lenient test passed
+    logger.error(
+        f"VM '{image_name}' didn't pass the FIPS 140-2 testing. "
+        f"Expected a maximum of 5 failures, got {fips_failures}."
+    )
     return False  # any unsuccessful path should end up here