Improve robustness of SCS test suite (#1227)

* Improve robustness of scs-test-runner.py by saving report before uploading

If upload fails, the report won't be lost.

* Improve robustness of openstack_test.py by reporting tests as ABORT

... when pre-flight checks fail.

---------

Signed-off-by: Matthias Büchse <matthias.buechse@alasca.cloud>

Author: mbuechse

Tests/iaas/openstack_test.py

@@ -231,7 +231,7 @@ def harness(name, *check_fns):
     print(f"{name}: {result}")
 
 
-def run_sanity_checks(container):
+def run_preflight_checks(container):
     # make sure that we can connect to the cloud and that the user doesn't have elevated privileges
     # the former would lead to each testcase aborting with a marginally useful message;
     # the latter would lead to scs_0116_permissions aborting, which we don't want to single out
@@ -278,7 +278,13 @@ def main(argv):
         sys.exit(1)
 
     c = make_container(cloud)
-    run_sanity_checks(c)
+    try:
+        run_preflight_checks(c)
+    except Exception:
+        logger.critical("Pre-flight checks failed. Reporting all testcases as ABORT.")
+        for testcase in testcases:
+            print(f"{testcase}: ABORT")
+        raise
     for testcase in testcases:
         harness(testcase, lambda: getattr(c, testcase.replace('-', '_')))
     return 0

Tests/scs-test-runner.py

@@ -219,10 +219,12 @@ def run(cfg, scopes, subjects, sections, preset, num_workers, monitor_url, repor
         commands = [cfg.build_check_command(job[0], job[1], sections, output) for job, output in zip(jobs, outputs)]
         _run_commands(commands, num_workers=num_workers)
         _concat_files(outputs, report_yaml_tmp)
-        subprocess.run(**cfg.build_sign_command(report_yaml_tmp))
-        subprocess.run(**cfg.build_upload_command(report_yaml_tmp, monitor_url))
-        if report_yaml is not None:
+        if report_yaml is None:
+            report_yaml = report_yaml_tmp
+        else:
             _move_file(report_yaml_tmp, report_yaml)
+        subprocess.run(**cfg.build_sign_command(report_yaml))
+        subprocess.run(**cfg.build_upload_command(report_yaml, monitor_url))
     return 0