no-wait option in tests.yaml

To run multiple bundles in different models, zaza
need to wait for first bundle applications to acheive
status as specified in target_deploy_status before
deploying second bundle using configure. (Deploying
second bundle using configure is actually a hacky
way). This increases the time of deployment as both
bundles are deployed sequentially. sunbeam-charms
project has total timeout of 2 hours for its
functional test.

Add option no-wait in tests.yaml so that zaza did not
wait for target_deploy_status for the applications and
proceed with configure.

tests_options:
  no_wait_deploy:
    smoke

The option --no-wait already exists for functest-deploy
cli command.

Author: hemanthnakkina

doc/source/addingcharmtests.rst

@@ -173,6 +173,38 @@ In the above case, focal-ussuri will be deployed using the --force parameter.
 i.e. the `tests_options.force_deploy['focal-ussuri']` option applies to the
 `focal-ussuri` bundle whether it appears in any of the bundle sections.
 
+Skipping the post-deploy wait
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+By default, after deploying a bundle zaza waits for all applications to reach
+the states specified in ``target_deploy_status`` (or active/idle if not
+configured). This wait can be skipped on a per-bundle basis using the
+``tests_options.no_wait_deploy`` option.
+
+This is useful when a charm requires post-deploy configuration before it can
+reach its target state, or when you want to proceed immediately to the
+configure step without waiting.
+
+In the ``tests.yaml`` the option is added as a list item::
+
+    charm_name: vault
+    gate_bundles:
+    - focal-ussuri
+
+    target_deploy_status:
+      vault:
+        workload-status: blocked
+        workload-status-message: Vault needs to be initialized
+
+    tests_options:
+      no_wait_deploy:
+        - focal-ussuri
+
+In the above case, zaza will deploy the ``focal-ussuri`` bundle and immediately
+proceed to the configure step without waiting for vault to enter the blocked
+state or for the model to settle (the additional post-deploy
+``block_until_all_units_idle`` wait is also skipped).
+
 Augmenting behaviour of configure steps
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

unit_tests/test_zaza_charm_lifecycle_func_test_runner.py

@@ -85,11 +85,11 @@ def test_func_test_runner(self):
         deploy_calls = [
             mock.call(cwd + '/tests/bundles/bundle1.yaml', 'newmodel',
                       model_ctxt={'default_alias': 'newmodel'},
-                      force=True, test_directory=None, trust=False,
+                      wait=True, force=True, test_directory=None, trust=False,
                       ignore_hard_deploy_errors=False),
             mock.call(cwd + '/tests/bundles/bundle2.yaml', 'newmodel',
                       model_ctxt={'default_alias': 'newmodel'},
-                      force=True, test_directory=None, trust=False,
+                      wait=True, force=True, test_directory=None, trust=False,
                       ignore_hard_deploy_errors=False)]
         configure_calls = [
             mock.call('newmodel', [
@@ -164,24 +164,24 @@ def test_func_test_runner_cmr(self):
         cwd = os.getcwd()
         deploy_calls = [
             mock.call(cwd + '/tests/bundles/bundle1.yaml', 'm1',
-                      model_ctxt={'default_alias': 'm1'}, force=False,
-                      test_directory=None, trust=False,
+                      model_ctxt={'default_alias': 'm1'}, wait=True,
+                      force=False, test_directory=None, trust=False,
                       ignore_hard_deploy_errors=False),
             mock.call(cwd + '/tests/bundles/bundle2.yaml', 'm2',
-                      model_ctxt={'default_alias': 'm2'}, force=False,
-                      test_directory=None, trust=False,
+                      model_ctxt={'default_alias': 'm2'}, wait=True,
+                      force=False, test_directory=None, trust=False,
                       ignore_hard_deploy_errors=False),
             mock.call(
                 cwd + '/tests/bundles/bundle5.yaml',
                 'm3',
                 model_ctxt={'model_alias_5': 'm3', 'model_alias_6': 'm4'},
-                force=False, test_directory=None, trust=False,
+                wait=True, force=False, test_directory=None, trust=False,
                 ignore_hard_deploy_errors=False),
             mock.call(
                 cwd + '/tests/bundles/bundle6.yaml',
                 'm4',
                 model_ctxt={'model_alias_5': 'm3', 'model_alias_6': 'm4'},
-                force=False, test_directory=None, trust=False,
+                wait=True, force=False, test_directory=None, trust=False,
                 ignore_hard_deploy_errors=False)]
         configure_calls = [
             mock.call('m1', [
@@ -256,12 +256,12 @@ def test_func_test_runner_with_before_script(self):
             mock.call('newmodel', 'default_alias', test_directory=None)]
         deploy_calls = [
             mock.call(cwd + '/tests/bundles/bundle1.yaml', 'newmodel',
-                      model_ctxt={'default_alias': 'newmodel'}, force=False,
-                      test_directory=None, trust=False,
+                      model_ctxt={'default_alias': 'newmodel'}, wait=True,
+                      force=False, test_directory=None, trust=False,
                       ignore_hard_deploy_errors=False),
             mock.call(cwd + '/tests/bundles/bundle2.yaml', 'newmodel',
-                      model_ctxt={'default_alias': 'newmodel'}, force=False,
-                      test_directory=None, trust=False,
+                      model_ctxt={'default_alias': 'newmodel'}, wait=True,
+                      force=False, test_directory=None, trust=False,
                       ignore_hard_deploy_errors=False)]
         before_deploy_calls = [
             mock.call('newmodel', [
@@ -319,7 +319,7 @@ def test_func_test_runner_smoke(self):
         deploy_calls = [
             mock.call(cwd + '/tests/bundles/bundle2.yaml', 'newmodel',
                       model_ctxt={'default_alias': 'newmodel'},
-                      force=False,
+                      wait=True, force=False,
                       test_directory=None, trust=False,
                       ignore_hard_deploy_errors=False)]
         self.deploy.assert_has_calls(deploy_calls)
@@ -352,12 +352,12 @@ def test_func_test_runner_dev(self):
         cwd = os.getcwd()
         deploy_calls = [
             mock.call(cwd + '/tests/bundles/bundle3.yaml', 'newmodel',
-                      model_ctxt={'default_alias': 'newmodel'}, force=False,
-                      test_directory=None, trust=False,
+                      model_ctxt={'default_alias': 'newmodel'}, wait=True,
+                      force=False, test_directory=None, trust=False,
                       ignore_hard_deploy_errors=False),
             mock.call(cwd + '/tests/bundles/bundle4.yaml', 'newmodel',
-                      model_ctxt={'default_alias': 'newmodel'}, force=False,
-                      test_directory=None, trust=False,
+                      model_ctxt={'default_alias': 'newmodel'}, wait=True,
+                      force=False, test_directory=None, trust=False,
                       ignore_hard_deploy_errors=False)]
         self.deploy.assert_has_calls(deploy_calls)
 
@@ -392,7 +392,7 @@ def test_func_test_runner_specify_bundle(self):
                 cwd + '/tests/bundles/maveric-filebeat.yaml',
                 'newmodel',
                 model_ctxt={'default_alias': 'newmodel'},
-                force=False,
+                wait=True, force=False,
                 test_directory=None, trust=False,
                 ignore_hard_deploy_errors=False)]
         self.deploy.assert_has_calls(deploy_calls)
@@ -429,7 +429,7 @@ def test_func_test_runner_specify_bundle_with_alias(self):
                 cwd + '/tests/bundles/maveric-filebeat.yaml',
                 'newmodel',
                 model_ctxt={'alias': 'newmodel'},
-                force=False,
+                wait=True, force=False,
                 test_directory=None, trust=False,
                 ignore_hard_deploy_errors=False)]
         self.deploy.assert_has_calls(deploy_calls)
@@ -459,7 +459,7 @@ def test_func_test_runner_specify_bundle_with_implicit_alias(self):
                 cwd + '/tests/bundles/maveric-filebeat.yaml',
                 'newmodel',
                 model_ctxt={'alias': 'newmodel'},
-                force=False,
+                wait=True, force=False,
                 test_directory=None, trust=False,
                 ignore_hard_deploy_errors=False)]
         self.deploy.assert_has_calls(deploy_calls)
@@ -498,15 +498,15 @@ def test_func_test_runner_cmr_specify_bundle_with_alias(self):
                 'newmodel1',
                 model_ctxt={'alias': 'newmodel1',
                             'another_alias': 'newmodel2'},
-                force=False,
+                wait=True, force=False,
                 test_directory=None, trust=False,
                 ignore_hard_deploy_errors=False),
             mock.call(
                 cwd + '/tests/bundles/maverick-things.yaml',
                 'newmodel2',
                 model_ctxt={'alias': 'newmodel1',
                             'another_alias': 'newmodel2'},
-                force=False,
+                wait=True, force=False,
                 test_directory=None, trust=False,
                 ignore_hard_deploy_errors=False)]
         self.deploy.assert_has_calls(deploy_calls)
@@ -604,3 +604,41 @@ def test_main_bundle_keep_model_ambiguous_case2(self):
 
     def test_main_bundle_keep_model_ambiguous_case3(self):
         self.__keep_model_ambiguous(False, True, True)
+
+    def test_func_test_runner_no_wait_deploy(self):
+        self.patch_object(lc_func_test_runner.utils, 'get_charm_config')
+        self.patch_object(lc_func_test_runner.utils, 'generate_model_name')
+        self.patch_object(lc_func_test_runner.prepare, 'prepare')
+        self.patch_object(lc_func_test_runner.before_deploy, 'before_deploy')
+        self.patch_object(lc_func_test_runner.deploy, 'deploy')
+        self.patch_object(lc_func_test_runner.configure, 'configure')
+        self.patch_object(lc_func_test_runner.test, 'test')
+        self.patch_object(lc_func_test_runner.destroy, 'destroy')
+        self.patch_object(
+            lc_func_test_runner.zaza.model,
+            'block_until_all_units_idle')
+        self.generate_model_name.return_value = 'newmodel'
+        self.get_charm_config.return_value = {
+            'charm_name': 'mycharm',
+            'gate_bundles': ['bundle1', 'bundle2'],
+            'tests': [
+                'zaza.charm_tests.mycharm.tests.SmokeTest'],
+            'tests_options': {
+                'no_wait_deploy': ['bundle1']
+            }}
+        lc_func_test_runner.func_test_runner()
+        cwd = os.getcwd()
+        deploy_calls = [
+            mock.call(cwd + '/tests/bundles/bundle1.yaml', 'newmodel',
+                      model_ctxt={'default_alias': 'newmodel'},
+                      wait=False, force=False, test_directory=None,
+                      trust=False, ignore_hard_deploy_errors=False),
+            mock.call(cwd + '/tests/bundles/bundle2.yaml', 'newmodel',
+                      model_ctxt={'default_alias': 'newmodel'},
+                      wait=True, force=False, test_directory=None,
+                      trust=False, ignore_hard_deploy_errors=False)]
+        self.deploy.assert_has_calls(deploy_calls)
+        # block_until_all_units_idle should only be called for bundle2
+        # (bundle1 has no_wait_deploy set)
+        self.block_until_all_units_idle.assert_called_once_with(
+            ignore_hard_errors=False, model_name='newmodel')

unit_tests/test_zaza_charm_lifecycle_utils.py

@@ -304,6 +304,40 @@ def test_is_config_deploy_trusted_for_bundle(self):
         }
         self.assertTrue(lc_utils.is_config_deploy_trusted_for_bundle('x'))
 
+    def test_no_wait_deploy(self):
+        self.patch_object(lc_utils, 'get_charm_config')
+        # test that no options at all returns False
+        self.get_charm_config.return_value = {}
+        self.assertFalse(lc_utils.no_wait_deploy('x'))
+        # test that if options exist but no bundle
+        self.get_charm_config.return_value = {
+            'tests_options': {}
+        }
+        self.assertFalse(lc_utils.no_wait_deploy('x'))
+        self.get_charm_config.return_value = {
+            'tests_options': {
+                'no_wait_deploy': []
+            }
+        }
+        self.assertFalse(lc_utils.no_wait_deploy('x'))
+        # verify that it returns True if the bundle is mentioned
+        self.get_charm_config.return_value = {
+            'tests_options': {
+                'no_wait_deploy': ['x']
+            }
+        }
+        self.assertTrue(lc_utils.no_wait_deploy('x'))
+        # verify that a prefixed bundle 'security:x' does not match 'x'
+        self.assertFalse(lc_utils.no_wait_deploy('security:x'))
+        # verify that listing 'security:x' does not match plain 'x'
+        self.get_charm_config.return_value = {
+            'tests_options': {
+                'no_wait_deploy': ['security:x']
+            }
+        }
+        self.assertFalse(lc_utils.no_wait_deploy('x'))
+        self.assertTrue(lc_utils.no_wait_deploy('security:x'))
+
     def test_get_class(self):
         self.assertEqual(
             type(lc_utils.get_class('unit_tests.'

zaza/charm_lifecycle/func_test_runner.py

@@ -142,21 +142,30 @@ def run_env_deployment(env_deployment, keep_model=DESTROY_MODEL, force=False,
                 deployment.bundle)
             errors_ = (ignore_hard_deploy_errors or
                        utils.ignore_hard_deploy_errors(deployment.bundle))
+            no_wait_ = utils.no_wait_deploy(deployment.bundle)
             deploy.deploy(
                 os.path.join(
                     utils.get_bundle_dir(),
                     '{}.yaml'.format(deployment.bundle)),
                 deployment.model_name,
                 model_ctxt=model_aliases,
+                wait=not no_wait_,
                 force=force_,
                 trust=trust_,
                 test_directory=test_directory,
                 ignore_hard_deploy_errors=errors_)
 
         # When deploying bundles with cross model relations, hooks may be
-        # triggered in already deployedi models so wait for all models to
-        # settle.
+        # triggered in already deployed models so wait for all models to
+        # settle. Bundles with no_wait_deploy set intentionally skip this
+        # settle wait to allow immediate progression to the configure step.
         for deployment in env_deployment.model_deploys:
+            if utils.no_wait_deploy(deployment.bundle):
+                logging.info(
+                    "Skipping post-deploy wait for {} "
+                    "(no_wait_deploy is set)".format(
+                        deployment.model_name))
+                continue
             logging.info("Waiting for {} to settle".format(
                 deployment.model_name))
             errors_ = (ignore_hard_deploy_errors or

zaza/charm_lifecycle/utils.py

@@ -526,6 +526,40 @@ def ignore_hard_deploy_errors(
     return False
 
 
+def no_wait_deploy(
+        bundle_name, yaml_file=None, fatal=True):
+    """Ask if the deploy wait/settle steps should be skipped for a bundle.
+
+    Setting no_wait_deploy for a bundle means that zaza will not:
+      * wait for the applications to reach the states defined in
+        target_deploy_status (or active/idle) after deploying the bundle; or
+      * perform the post-deploy block_until_all_units_idle "settle" wait.
+
+    The tests_options section needs to look like:
+
+        tests_options:
+          no_wait_deploy:
+            - focal-ussuri
+
+    :param bundle_name: bundle to check in the no_wait_deploy list
+    :type bundle_name: str
+    :param yaml_file: the YAML file that contains the tests specification
+    :type yaml_file: Optional[str]
+    :param fatal: whether any errors cause an exception or are just logged.
+    :type fatal: bool
+    :returns: True if the config option is set for the bundle
+    :rtype: bool
+    :raises: OSError if the YAML file doesn't exist and fatal=True
+    """
+    config = get_charm_config(yaml_file, fatal)
+    try:
+        return bundle_name in config['tests_options']['no_wait_deploy']
+    # Type error is if no_wait_deploy is present, but with no list
+    except (KeyError, TypeError):
+        pass
+    return False
+
+
 def get_class(class_str):
     """Get the class represented by the given string.