Kubernetes: add instance volumes support (#3758)

Implemented as inline `hostPath` volume in `PodSpec`.

A new compute feature was added -- previously, instance volume support
was implied by "create instance" feature.

Part-of: #3757

Author: un-def

contributing/BACKENDS.md

@@ -181,8 +181,9 @@ The agent controls the VM and starts Docker containers for users' jobs.
 Since `dstack` controls the entire VM, VM-based backends can support more features,
 such as blocks, instance volumes, privileged containers, and reusable instances.
 
-Note, all VM-based backend `Compute`s should sublass the `ComputeWithPrivilegedSupport` mixin,
-as the `dstack-shim` agent provides this functionality OOTB.
+Note, all VM-based backend `Compute`s should sublass the `ComputeWithPrivilegedSupport`
+and `ComputeWithInstanceVolumesSupport` mixins, as the `dstack-shim` agent
+provides these functionalities OOTB.
 
 To support a VM-based backend, `dstack` expects the following:
 

docs/docs/concepts/volumes.md

@@ -262,7 +262,7 @@ Since persistence isn't guaranteed (instances may be interrupted or runs may occ
 volumes only for caching or with directories manually mounted to network storage.
 
 !!! info "Backends"
-    Instance volumes are currently supported for all backends except `runpod`, `vastai` and `kubernetes`, and can also be used with [SSH fleets](fleets.md#ssh-fleets).
+    Instance volumes are currently supported for all backends except `runpod` and `vastai`, and can also be used with [SSH fleets](fleets.md#ssh-fleets).
 
 ??? info "Optional volumes"
     If the volume is not critical for your workload, you can mark it as `optional`.

src/dstack/_internal/core/backends/aws/compute.py

@@ -25,6 +25,7 @@
     ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
     ComputeWithGatewaySupport,
+    ComputeWithInstanceVolumesSupport,
     ComputeWithMultinodeSupport,
     ComputeWithPlacementGroupSupport,
     ComputeWithPrivateGatewaySupport,
@@ -108,6 +109,7 @@ class AWSCompute(
     ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
     ComputeWithPrivilegedSupport,
+    ComputeWithInstanceVolumesSupport,
     ComputeWithMultinodeSupport,
     ComputeWithReservationSupport,
     ComputeWithPlacementGroupSupport,

src/dstack/_internal/core/backends/azure/compute.py

@@ -43,6 +43,7 @@
     ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
     ComputeWithGatewaySupport,
+    ComputeWithInstanceVolumesSupport,
     ComputeWithMultinodeSupport,
     ComputeWithPrivilegedSupport,
     generate_unique_gateway_instance_name,
@@ -86,6 +87,7 @@ class AzureCompute(
     ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
     ComputeWithPrivilegedSupport,
+    ComputeWithInstanceVolumesSupport,
     ComputeWithMultinodeSupport,
     ComputeWithGatewaySupport,
     Compute,

src/dstack/_internal/core/backends/base/compute.py

@@ -375,6 +375,15 @@ class ComputeWithPrivilegedSupport:
     pass
 
 
+class ComputeWithInstanceVolumesSupport:
+    """
+    Must be subclassed to support runs with `/host/path:/container/path` volumes.
+    All VM-based Computes (that is, Computes that use the shim) should subclass this mixin.
+    """
+
+    pass
+
+
 class ComputeWithMultinodeSupport:
     """
     Must be subclassed to support multinode tasks and cluster fleets.
@@ -668,31 +677,39 @@ def generate_unique_backend_name(
         # project_name is not guaranteed to be valid in all backends,
         # so we add it only if it passes the validation
         prefix = f"dstack-{project_name}-{resource_name}"
-    return _generate_unique_backend_name_with_prefix(
+    return generate_unique_name(
         prefix=prefix,
         max_length=max_length,
     )
 
 
 def generate_unique_short_backend_name() -> str:
     """
-    Generates a unique 15-char resource name of the form "dstack-12345678".
+    Generates a unique 15-char resource name of the form "dstack-12345xyz".
     Can be used for resources that have a very small length limit like AWS LBs.
     """
-    return _generate_unique_backend_name_with_prefix("dstack")
+    return generate_unique_name(prefix="dstack")
 
 
-def _generate_unique_backend_name_with_prefix(
-    prefix: str,
+def generate_unique_name(
+    *,
+    prefix: Optional[str] = None,
+    suffix_length: Optional[int] = None,
     max_length: Optional[int] = None,
 ) -> str:
+    if suffix_length is None:
+        suffix_length = _CLOUD_RESOURCE_SUFFIX_LEN
     if max_length is not None:
-        prefix_len = max_length - _CLOUD_RESOURCE_SUFFIX_LEN - 1
-        prefix = prefix[:prefix_len]
+        assert max_length >= suffix_length
+        if prefix is not None:
+            prefix_len = max_length - suffix_length - 1
+            assert prefix_len > 0
+            prefix = prefix[:prefix_len]
     suffix = "".join(
-        random.choice(string.ascii_lowercase + string.digits)
-        for _ in range(_CLOUD_RESOURCE_SUFFIX_LEN)
+        random.choice(string.ascii_lowercase + string.digits) for _ in range(suffix_length)
     )
+    if prefix is None:
+        return suffix
     return f"{prefix}-{suffix}"
 
 

src/dstack/_internal/core/backends/cloudrift/compute.py

@@ -4,6 +4,7 @@
     Compute,
     ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
+    ComputeWithInstanceVolumesSupport,
     ComputeWithPrivilegedSupport,
     get_shim_commands,
 )
@@ -29,6 +30,7 @@ class CloudRiftCompute(
     ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
     ComputeWithPrivilegedSupport,
+    ComputeWithInstanceVolumesSupport,
     Compute,
 ):
     def __init__(self, config: CloudRiftConfig):

src/dstack/_internal/core/backends/crusoe/compute.py

@@ -8,6 +8,7 @@
 from dstack._internal.core.backends.base.compute import (
     ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
+    ComputeWithInstanceVolumesSupport,
     ComputeWithMultinodeSupport,
     ComputeWithPlacementGroupSupport,
     ComputeWithPrivilegedSupport,
@@ -124,6 +125,7 @@ class CrusoeCompute(
     ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
     ComputeWithPrivilegedSupport,
+    ComputeWithInstanceVolumesSupport,
     ComputeWithMultinodeSupport,
     ComputeWithPlacementGroupSupport,
     Compute,

src/dstack/_internal/core/backends/cudo/compute.py

@@ -6,6 +6,7 @@
 from dstack._internal.core.backends.base.compute import (
     ComputeWithCreateInstanceSupport,
     ComputeWithFilteredOffersCached,
+    ComputeWithInstanceVolumesSupport,
     ComputeWithPrivilegedSupport,
     generate_unique_instance_name,
     get_shim_commands,
@@ -34,6 +35,7 @@ class CudoCompute(
     ComputeWithFilteredOffersCached,
     ComputeWithCreateInstanceSupport,
     ComputeWithPrivilegedSupport,
+    ComputeWithInstanceVolumesSupport,
     Compute,
 ):
     def __init__(self, config: CudoConfig):

src/dstack/_internal/core/backends/digitalocean_base/compute.py

@@ -7,6 +7,7 @@
 from dstack._internal.core.backends.base.compute import (
     ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
+    ComputeWithInstanceVolumesSupport,
     ComputeWithPrivilegedSupport,
     generate_unique_instance_name,
     get_user_data,
@@ -42,6 +43,7 @@ class BaseDigitalOceanCompute(
     ComputeWithAllOffersCached,
     ComputeWithCreateInstanceSupport,
     ComputeWithPrivilegedSupport,
+    ComputeWithInstanceVolumesSupport,
     Compute,
 ):
     def __init__(self, config: BaseDigitalOceanConfig, api_url: str, type: BackendType):

src/dstack/_internal/core/backends/features.py

@@ -2,6 +2,7 @@
     ComputeWithCreateInstanceSupport,
     ComputeWithGatewaySupport,
     ComputeWithGroupProvisioningSupport,
+    ComputeWithInstanceVolumesSupport,
     ComputeWithMultinodeSupport,
     ComputeWithPlacementGroupSupport,
     ComputeWithPrivateGatewaySupport,
@@ -48,6 +49,10 @@ def _get_backends_with_compute_feature(
     configurator_classes=_configurator_classes,
     compute_feature_class=ComputeWithPrivilegedSupport,
 )
+BACKENDS_WITH_INSTANCE_VOLUMES_SUPPORT = _get_backends_with_compute_feature(
+    configurator_classes=_configurator_classes,
+    compute_feature_class=ComputeWithInstanceVolumesSupport,
+)
 BACKENDS_WITH_MULTINODE_SUPPORT = [BackendType.REMOTE] + _get_backends_with_compute_feature(
     configurator_classes=_configurator_classes,
     compute_feature_class=ComputeWithMultinodeSupport,