diff --git a/Tests/kaas/sonobuoy_handler/run_sonobuoy.py b/Tests/kaas/sonobuoy_handler/run_sonobuoy.py index d3f52c47d..a103952d5 100755 --- a/Tests/kaas/sonobuoy_handler/run_sonobuoy.py +++ b/Tests/kaas/sonobuoy_handler/run_sonobuoy.py @@ -21,13 +21,16 @@ def cli(): @click.option("-r", "--result_dir_name", "result_dir_name", type=str, default="sonobuoy_results", help="directory name to store results at",) @click.option("-c", "--check", "check_name", type=str, default="sonobuoy_executor", help="this MUST be the same name as the id in 'scs-compatible-kaas.yaml'",) @click.option("--scs-sonobuoy-config", "scs_sonobuoy_config_yaml", required=True, type=click.Path(exists=True), help="Configuration for Sonobuoy (YAML format)") -@click.option('--execution-mode', 'mode', type=click.Choice(('serial', 'parallel')), default='serial') +@click.option('--execution-mode', 'mode', type=click.Choice(('serial', 'parallel', 'dry')), default='serial') @click.option("-a", "--arg", "args", multiple=True) def sonobuoy_run(kubeconfig, result_dir_name, check_name, scs_sonobuoy_config_yaml, mode, args): if mode == 'parallel': # This is merely a shortcut to simplify commandline in scs-compatible-kaas.yaml # For more on parallel execution, see https://github.com/vmware-tanzu/sonobuoy/issues/1435 args += ('--e2e-parallel=true', ) + elif mode == 'dry': + # mutually exclusive with parallel, it turns out! + args += ('--plugin-env e2e.E2E_DRYRUN=true', ) sonobuoy_handler = SonobuoyHandler(scs_sonobuoy_config_yaml, check_name, kubeconfig, result_dir_name, args) sys.exit(sonobuoy_handler.run()) diff --git a/Tests/scs-compatible-kaas.yaml b/Tests/scs-compatible-kaas.yaml index ec6eba86d..3de98acdc 100644 --- a/Tests/scs-compatible-kaas.yaml +++ b/Tests/scs-compatible-kaas.yaml @@ -9,11 +9,17 @@ variables: # working directory for the subject under test # (note that we consider each kubernetes branch a test subject of its own) - kubeconfig - - execution_mode: parallel # alternatively: serial + - execution_mode: parallel # alternatively: serial, dry scripts: - executable: ./kaas/sonobuoy_handler/run_sonobuoy.py - args: run -k {kubeconfig} --scs-sonobuoy-config kaas/scs-sonobuoy-config-v1.yaml -r {subject_root}/sono-results-e2e -c 'cncf-k8s-conformance' -a '--mode=certified-conformance' --execution-mode {execution_mode} - #~ args: -k {kubeconfig} -r {subject_root}/sono-results -c 'cncf-k8s-conformance' -a '--plugin-env e2e.E2E_DRYRUN=true' + args: >- + run -k {kubeconfig} + --scs-sonobuoy-config kaas/scs-sonobuoy-config-v1.yaml + --execution-mode {execution_mode} + -r {subject_root}/sono-results-e2e + -c 'cncf-k8s-conformance' + -a '--mode=certified-conformance' + -a '--plugin-env e2e.E2E_EXTRA_ARGS=--ginkgo.flake-attempts=2' testcases: - id: cncf-k8s-conformance lifetime: year @@ -33,7 +39,14 @@ scripts: Manual check: Must fulfill all requirements of scs-0214-v2. url: https://docs.scs.community/standards/scs-0214-v2-k8s-node-distribution#decision - executable: ./kaas/sonobuoy_handler/run_sonobuoy.py - args: run -k {kubeconfig} --scs-sonobuoy-config kaas/scs-sonobuoy-config-v1.yaml -r {subject_root}/sono-results-0219 -c 'kaas-networking-check' -a '--e2e-focus "NetworkPolicy"' --execution-mode {execution_mode} + args: >- + run -k {kubeconfig} + --scs-sonobuoy-config kaas/scs-sonobuoy-config-v1.yaml + --execution-mode {execution_mode} + -r {subject_root}/sono-results-0219 + -c 'kaas-networking-check' + -a '--e2e-focus "NetworkPolicy"' + -a '--plugin-env e2e.E2E_EXTRA_ARGS=--ginkgo.flake-attempts=2' testcases: - id: kaas-networking-check lifetime: month