extend timeout for istio proxy sidecars

Author: jcchr

interactive_ai/workflows/common/jobs_common/k8s_helpers/trainer_pod_definition.py

@@ -334,7 +334,7 @@ def create_flyte_container_task(  # noqa: PLR0913
             primary_container_name=CONTAINER_NAME,
             annotations={
                 "proxy.istio.io/config": '{ "holdApplicationUntilProxyStarts": true, '
-                                         '"terminationDrainDuration": "10s" }',
+                                         '"terminationDrainDuration": "30s" }',
                 "karpenter.sh/do-not-disrupt": "true",
             },
         ),

interactive_ai/workflows/common/jobs_common/tasks/primary_container_task.py

@@ -45,7 +45,7 @@
 AWS_CONFIG_VOLUME_NAME = "aws-client-config"
 AWS_CONFIG_MOUNT_PATH = "/home/non-root/.aws"
 FLYTE_POD_ANNOTATIONS = {"proxy.istio.io/config": '{ "holdApplicationUntilProxyStarts": true, '
-                                                  '"terminationDrainDuration": "10s" }'}
+                                                  '"terminationDrainDuration": "30s" }'}
 SPICEDB_TLS_SECRETS_VOLUME_NAME = "tls-secrets"
 SPICEDB_TLS_SECRETS_MOUNT_PATH = "/etc/tls-secrets"
 TERM_ISTIO_PROXY_URL = "http://127.0.0.1:15020/quitquitquit"
@@ -386,21 +386,21 @@ def get_flyte_pod_spec(
 
 def _terminate_istio_proxy(n_trials: int = 10, timeout: float = 1.0, sleep_sec: float = 1.0):
     is_terminated = False
-    sleep(5)
+    logger.info("starting terminating the istio-proxy container - new")
     for step in range(1, n_trials + 1):
-        try:
-            response = requests.post(TERM_ISTIO_PROXY_URL, timeout=timeout)
+       try:
+           response = requests.post(TERM_ISTIO_PROXY_URL, timeout=timeout)
 
-            if response.status_code == 200:
-                is_terminated = True
-                break
-        except Exception as e:
-            logger.warning("[%d/%d] Cannot terminate istio proxy: %s", step, n_trials, e)
+           if response.status_code == 200:
+               is_terminated = True
+               break
+       except Exception as e:
+           logger.warning("[%d/%d] Cannot terminate istio proxy: %s", step, n_trials, e)
 
-        sleep(sleep_sec)
+       sleep(sleep_sec)
 
     if not is_terminated:
-        logger.error("Cannot terminate istio proxy")
+       logger.error("Cannot terminate istio proxy")
 
 
 DEFAULT_TASK_CONFIG = Pod(