Skip to content

Commit aa23eaa

Browse files
mbuechsembuechse
committed
Limit waiting times when provisioning/unprovisioning clusters
otherwise this can wait indefinitely which can then stall the whole CI/CD job Signed-off-by: Matthias Büchse <matthias.buechse@alasca.cloud>
1 parent d2fe9a2 commit aa23eaa

2 files changed

Lines changed: 61 additions & 29 deletions

File tree

‎Tests/kaas/plugin/plugin_clusterstacks.py‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616

1717
TEMPLATE_KEYS = ('cluster', 'clusterstack', 'kubeconfig')
18+
TIMEOUTS = [30] * 30 + [0] # wait at most 900 seconds (15 minutes); sentinel value at the end
1819

1920

2021
class _ClusterOps:
@@ -44,6 +45,7 @@ def _get_phase(self, co_api: _csh.CustomObjectsApi):
4445

4546
def create(self, co_api: _csh.CustomObjectsApi, cluster_dict):
4647
# repeat this because it's possible that a cluster object exists that is in Deleting phase
48+
timeouts = iter(TIMEOUTS)
4749
while True:
4850
logger.debug(f'creating cluster object for {self.name}')
4951
try:
@@ -60,8 +62,11 @@ def create(self, co_api: _csh.CustomObjectsApi, cluster_dict):
6062
logger.debug(f'cluster object for {self.name} already present in phase {phase}')
6163
if phase.lower() in ('provisioned', 'provisioning'):
6264
break
63-
logger.debug(f'waiting 30 s for cluster {self.name} to vanish or become provisioned')
64-
time.sleep(30)
65+
timeout = next(timeouts)
66+
if not timeout:
67+
raise RuntimeError(f"Timeout error while waiting on Cluster {self.name}")
68+
logger.debug(f'waiting {timeout} s for cluster {self.name} to vanish or become provisioned')
69+
time.sleep(timeout)
6570
else:
6671
break
6772

@@ -78,23 +83,34 @@ def delete(self, co_api: _csh.CustomObjectsApi):
7883
# it will take at least 5 s (and way longer if the cluster is already running)
7984
logger.debug(f'cluster {self.name} deletion requested; waiting 8 s for it to vanish')
8085
time.sleep(8)
86+
timeouts = iter(TIMEOUTS)
8187
while True:
8288
phase = self._get_phase(co_api)
8389
if phase is None:
90+
logger.debug(f"Cluster {self.name} deleted.")
8491
break
8592
if phase.lower() != 'deleting':
8693
raise RuntimeError(f'cluster {self.name} in phase {phase}; expected: Deleting')
87-
logger.debug(f'cluster {self.name} still deleting; waiting 30 s for it to vanish')
88-
time.sleep(30)
94+
timeout = next(timeouts)
95+
if not timeout:
96+
raise RuntimeError(f"Cluster {self.name} has not gone away in time")
97+
logger.debug(f'cluster {self.name} still deleting; waiting {timeout} s for it to vanish')
98+
time.sleep(timeout)
8999

90100
def get_kubeconfig(self, core_api: _csh.CoreV1Api):
91101
return _csh.get_secret_data(core_api, self.namespace, self.secret_name)
92102

93103
def wait_for_cluster_ready(self, co_api: _csh.CustomObjectsApi):
94-
while not self._get_condition_ready(co_api):
95-
logger.debug(f'waiting 30 s for cluster {self.name} to become ready')
96-
time.sleep(30)
97-
logger.debug(f'cluster {self.name} appears to be ready')
104+
timeouts = iter(TIMEOUTS)
105+
while True:
106+
if self._get_condition_ready(co_api):
107+
logger.debug(f'cluster {self.name} appears to be ready')
108+
break
109+
timeout = next(timeouts)
110+
if not timeout:
111+
raise RuntimeError(f"Cluster {self.name} has not become ready in time")
112+
logger.debug(f'waiting {timeout} s for cluster {self.name} to become ready')
113+
time.sleep(timeout)
98114

99115

100116
def load_templates(env, basepath, fn_map, keys=TEMPLATE_KEYS):

‎Tests/kaas/plugin/plugin_gardener.py‎

Lines changed: 37 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616

1717
TEMPLATE_KEYS = ('shoot', 'kubeconfig')
18+
TIMEOUTS = [30] * 30 + [0] # wait at most 900 seconds (15 minutes); sentinel value at the end
1819

1920

2021
class _ShootOps:
@@ -35,6 +36,7 @@ def create(self, co_api: _gh.CustomObjectsApi, shoot_dict):
3536
# Gardener shoot reconciliation is idempotent, so we can just try to create.
3637
# If it exists, we check its state.
3738
# repeat this because it's possible that a cluster object exists that is in state Delete
39+
timeouts = iter(TIMEOUTS)
3840
while True:
3941
logger.debug(f"creating shoot object for {self.name}")
4042
try:
@@ -55,8 +57,11 @@ def create(self, co_api: _gh.CustomObjectsApi, shoot_dict):
5557
if state == 'Failed':
5658
raise RuntimeError(f"Shoot {self.name} is in Failed state. Please check the Gardener dashboard.")
5759

58-
logger.debug(f'waiting 30 s for shoot {self.name} to proceed')
59-
time.sleep(30)
60+
timeout = next(timeouts)
61+
if not timeout:
62+
raise RuntimeError(f"Timeout error while waiting on shoot {self.name}")
63+
logger.debug(f'waiting {timeout} s for shoot {self.name} to proceed')
64+
time.sleep(timeout)
6065
else:
6166
break
6267

@@ -74,41 +79,52 @@ def delete(self, co_api: _gh.CustomObjectsApi):
7479

7580
logger.debug(f"Shoot {self.name} deletion requested; waiting 8 s for it to start deleting")
7681
time.sleep(8)
82+
timeouts = iter(TIMEOUTS)
7783
while True:
7884
last_op = self._get_last_operation(co_api)
7985
if last_op is None:
8086
logger.info(f"Shoot {self.name} has been deleted.")
8187
break
8288

83-
if last_op.get('type') != 'Delete':
84-
raise RuntimeError(f"Shoot {self.name} in unexpected state during deletion: {last_op!r}")
85-
state = last_op.get('state', 'Unknown')
86-
if state not in ('Processing', 'Succeeded'):
87-
raise RuntimeError(f"Shoot {self.name} in unexpected state during deletion: state={state}")
89+
timeout = next(timeouts)
90+
if not timeout:
91+
raise RuntimeError(f"Shoot {self.name} has not gone away in time")
8892

89-
if state == 'Processing':
93+
state = last_op.get('state', 'Unknown')
94+
if last_op.get('type') != 'Delete':
95+
logger.debug(f"Shoot {self.name} in unexpected state during deletion: {last_op!r}; waiting {timeout} s")
96+
elif state == 'Processing':
9097
progress = last_op.get('progress', 0)
91-
logger.debug(f"Shoot {self.name} is being deleted (progress: {progress} %); waiting 30 s")
98+
logger.debug(f"Shoot {self.name} is being deleted (progress: {progress} %); waiting {timeout} s")
9299
elif state == 'Succeeded':
93-
logger.info(f"Shoot {self.name} deletion succeeded, but object still exists. Waiting 30 s for it to vanish.")
94-
time.sleep(30)
100+
logger.info(f"Shoot {self.name} deletion succeeded, but object still exists. Waiting {timeout} s for it to vanish.")
101+
else:
102+
raise RuntimeError(f"Shoot {self.name} in unexpected state during deletion: state={state}")
103+
time.sleep(timeout)
95104

96105
def get_kubeconfig(self, api_client: ApiClient):
97106
return _gh.request_kubeconfig(api_client, self.namespace, self.name)
98107

99108
def wait_for_shoot_ready(self, co_api: _gh.CustomObjectsApi):
100-
last_op = self._get_last_operation(co_api)
101-
while last_op is not None and last_op.get('state') not in ('Succeeded', 'Failed'):
109+
timeouts = iter(TIMEOUTS)
110+
while True:
111+
last_op = self._get_last_operation(co_api)
112+
if last_op is None:
113+
raise RuntimeError(f"Shoot {self.name} object disappeared")
114+
102115
state = last_op.get('state', 'Unknown')
116+
if state == 'Succeeded':
117+
logger.debug(f'shoot {self.name} appears to be ready')
118+
return
119+
if state not in ('Unknown', 'Processing'):
120+
raise RuntimeError(f"Shoot {self.name} object in unexpected state {last_op.get('state')}")
121+
122+
timeout = next(timeouts)
123+
if not timeout:
124+
raise RuntimeError(f"Shoot {self.name} has not become ready in time")
103125
progress = last_op.get('progress', 0)
104-
logger.debug(f'waiting 30 s for shoot {self.name} to become ready (current state: {state}, progress: {progress}%)')
105-
time.sleep(30)
106-
last_op = self._get_last_operation(co_api)
107-
if last_op is None:
108-
raise RuntimeError(f"Shoot {self.name} object disappeared or in state Failed")
109-
if last_op.get('state') != 'Succeeded':
110-
raise RuntimeError(f"Shoot {self.name} object in unexpected state {last_op.get('state')}")
111-
logger.debug(f'shoot {self.name} appears to be ready')
126+
logger.debug(f'waiting {timeout} s for shoot {self.name} to become ready (current state: {state}, progress: {progress}%)')
127+
time.sleep(timeout)
112128

113129

114130
def load_templates(env, basepath, fn_map, keys=TEMPLATE_KEYS):

0 commit comments

Comments
 (0)