Fix terminating TensorDock instances (#2480)

- Do not handle exceptions in
  `terminate_instance`, so that they can be
  detected and termination can be retried
- Ignore termination errors if instance is already
  terminated
- Increase request timeout, termination and other
  requests often take more than 12 seconds

Author: jvstme

src/dstack/_internal/core/backends/tensordock/api_client.py

@@ -8,7 +8,7 @@
 from dstack._internal.utils.logging import get_logger
 
 logger = get_logger(__name__)
-REQUEST_TIMEOUT = 12
+REQUEST_TIMEOUT = 20
 
 
 class TensorDockAPIClient:
@@ -80,7 +80,7 @@ def deploy_single(self, instance_name: str, instance: InstanceType, cloudinit: d
         data["password"] = form["password"]
         return data
 
-    def delete_single(self, instance_id: str):
+    def delete_single_if_exists(self, instance_id: str):
         logger.debug("Deleting instance %s", instance_id)
         resp = self.s.post(
             self._url("/client/delete/single"),
@@ -91,10 +91,11 @@ def delete_single(self, instance_id: str):
             },
             timeout=REQUEST_TIMEOUT,
         )
-        resp.raise_for_status()
         try:
             data = resp.json()
-            if not data["success"]:
+            if "already terminated" in data.get("error", ""):
+                return
+            if not data.get("success"):
                 raise BackendError(data)
         except ValueError:  # json parsing error
             raise BackendError(resp.text)

src/dstack/_internal/core/backends/tensordock/compute.py

@@ -12,7 +12,7 @@
 from dstack._internal.core.backends.base.offers import get_catalog_offers
 from dstack._internal.core.backends.tensordock.api_client import TensorDockAPIClient
 from dstack._internal.core.backends.tensordock.models import TensorDockConfig
-from dstack._internal.core.errors import BackendError, NoCapacityError
+from dstack._internal.core.errors import NoCapacityError
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.instances import (
     InstanceAvailability,
@@ -117,17 +117,4 @@ def create_instance(
     def terminate_instance(
         self, instance_id: str, region: str, backend_data: Optional[str] = None
     ):
-        try:
-            self.api_client.delete_single(instance_id)
-        except requests.HTTPError as e:
-            logger.error(
-                "An HTTP error occurred when trying to terminate TensorDock instance %s: %s",
-                instance_id,
-                e,
-            )
-        except BackendError as e:
-            logger.error(
-                "TensorDock returned an error when trying to terminate instance %s: %s",
-                instance_id,
-                e,
-            )
+        self.api_client.delete_single_if_exists(instance_id)