Merge pull request #1369 from freyes/retry-pacemaker-remote

Retry pacemaker remote

Author: MylesJP

zaza/openstack/charm_tests/pacemaker_remote/tests.py

@@ -18,16 +18,41 @@
 
 import unittest
 
+import tenacity
+import zaza.model
 import zaza.openstack.configure.hacluster
+import zaza.openstack.utilities.generic as generic_utils
 
 
 class PacemakerRemoteTest(unittest.TestCase):
     """Encapsulate pacemaker-remote tests."""
 
     def test_check_nodes_online(self):
         """Test that all nodes are online."""
+        for attempt in tenacity.Retrying(
+            reraise=True,
+            stop=tenacity.stop_after_attempt(5),
+            wait=tenacity.wait_fixed(5),
+        ):
+            with attempt:
+                zaza.openstack.configure.hacluster.check_all_nodes_online(
+                    'api'
+                )
+        units = zaza.model.get_units('pacemaker-remote')
+        last_unit = units[-1]
+        node_names = generic_utils.get_unit_hostnames(units)
+        node_name = node_names[last_unit.entity_id]
         zaza.openstack.configure.hacluster.remove_node(
             'api',
-            'node1')
-        self.assertTrue(
-            zaza.openstack.configure.hacluster.check_all_nodes_online('api'))
+            node_name)
+        for attempt in tenacity.Retrying(
+            reraise=True,
+            stop=tenacity.stop_after_attempt(5),
+            wait=tenacity.wait_fixed(5),
+        ):
+            with attempt:
+                self.assertTrue(
+                    zaza.openstack.configure.hacluster.check_all_nodes_online(
+                        'api'
+                    )
+                )