Allow retries in flaky k8s e2e tests (#1194)

Signed-off-by: Matthias Büchse <matthias.buechse@alasca.cloud>

Author: mbuechse

Tests/kaas/sonobuoy_handler/run_sonobuoy.py

@@ -21,13 +21,16 @@ def cli():
 @click.option("-r", "--result_dir_name", "result_dir_name", type=str, default="sonobuoy_results", help="directory name to store results at",)
 @click.option("-c", "--check", "check_name", type=str, default="sonobuoy_executor", help="this MUST be the same name as the id in 'scs-compatible-kaas.yaml'",)
 @click.option("--scs-sonobuoy-config", "scs_sonobuoy_config_yaml", required=True, type=click.Path(exists=True), help="Configuration for Sonobuoy (YAML format)")
-@click.option('--execution-mode', 'mode', type=click.Choice(('serial', 'parallel')), default='serial')
+@click.option('--execution-mode', 'mode', type=click.Choice(('serial', 'parallel', 'dry')), default='serial')
 @click.option("-a", "--arg", "args", multiple=True)
 def sonobuoy_run(kubeconfig, result_dir_name, check_name, scs_sonobuoy_config_yaml, mode, args):
     if mode == 'parallel':
         # This is merely a shortcut to simplify commandline in scs-compatible-kaas.yaml
         # For more on parallel execution, see https://github.com/vmware-tanzu/sonobuoy/issues/1435
         args += ('--e2e-parallel=true', )
+    elif mode == 'dry':
+        # mutually exclusive with parallel, it turns out!
+        args += ('--plugin-env e2e.E2E_DRYRUN=true', )
     sonobuoy_handler = SonobuoyHandler(scs_sonobuoy_config_yaml, check_name, kubeconfig, result_dir_name, args)
     sys.exit(sonobuoy_handler.run())
 

Tests/scs-compatible-kaas.yaml

@@ -9,11 +9,17 @@ variables:
     # working directory for the subject under test
     # (note that we consider each kubernetes branch a test subject of its own)
   - kubeconfig
-  - execution_mode: parallel  # alternatively: serial
+  - execution_mode: parallel  # alternatively: serial, dry
 scripts:
 - executable: ./kaas/sonobuoy_handler/run_sonobuoy.py
-  args: run -k {kubeconfig} --scs-sonobuoy-config kaas/scs-sonobuoy-config-v1.yaml -r {subject_root}/sono-results-e2e -c 'cncf-k8s-conformance'  -a '--mode=certified-conformance' --execution-mode {execution_mode}
-  #~ args: -k {kubeconfig} -r {subject_root}/sono-results -c 'cncf-k8s-conformance'  -a '--plugin-env e2e.E2E_DRYRUN=true'
+  args: >-
+    run -k {kubeconfig}
+    --scs-sonobuoy-config kaas/scs-sonobuoy-config-v1.yaml
+    --execution-mode {execution_mode}
+    -r {subject_root}/sono-results-e2e
+    -c 'cncf-k8s-conformance'
+    -a '--mode=certified-conformance'
+    -a '--plugin-env e2e.E2E_EXTRA_ARGS=--ginkgo.flake-attempts=2'
   testcases:
   - id: cncf-k8s-conformance
     lifetime: year
@@ -33,7 +39,14 @@ scripts:
       Manual check: Must fulfill all requirements of scs-0214-v2.
     url: https://docs.scs.community/standards/scs-0214-v2-k8s-node-distribution#decision
 - executable: ./kaas/sonobuoy_handler/run_sonobuoy.py
-  args: run -k {kubeconfig} --scs-sonobuoy-config kaas/scs-sonobuoy-config-v1.yaml -r {subject_root}/sono-results-0219 -c 'kaas-networking-check' -a '--e2e-focus "NetworkPolicy"' --execution-mode {execution_mode}
+  args: >-
+    run -k {kubeconfig}
+    --scs-sonobuoy-config kaas/scs-sonobuoy-config-v1.yaml
+    --execution-mode {execution_mode}
+    -r {subject_root}/sono-results-0219
+    -c 'kaas-networking-check'
+    -a '--e2e-focus "NetworkPolicy"'
+    -a '--plugin-env e2e.E2E_EXTRA_ARGS=--ginkgo.flake-attempts=2'
   testcases:
   - id: kaas-networking-check
     lifetime: month