Add gateway replica statuses and pipeline (#3990)

- Introduce gateway replica statuses.
- Provision and terminate gateway replicas
  independently from each other, in a separate
  pipeline.

In this version, the pipelines have the following
responsibilities.

Gateway pipeline:
- `SUBMITTED` — create replica DB records,
  transition to `PROVISIONING`; in a future
  version — create the load balancer (e.g., AWS
  ALB).
- `PROVISIONING` — once all replicas reach
  `RUNNING`, transition to `RUNNING`; if any
  replica enters `TERMINATING` or `TERMINATED`,
  transition to `FAILED`.
- `RUNNING`, `FAILED` — delete the gateway if
  deletion requested and all replicas are
  `TERMINATED`.

Gateway replica pipeline:
- `SUBMITTED` — call backend to create the cloud
  instance, transition to `PROVISIONING` on
  success, or `TERMINATED` on failure.
- `PROVISIONING` — SSH-connect to the instance and
  configure the gateway, transition to `RUNNING`
  on success, or `TERMINATING` on failure.
- `RUNNING` — nothing to do.
- `TERMINATING` — call backend to destroy the
  cloud instance, transition to `TERMINATED`.
- `TERMINATED` — nothing to do.
- also `SUBMITTED`, `PROVISIONING`, or `RUNNING` —
  transition to `TERMINATING` or `TERMINATED` if
  the gateway is `FAILED` or gateway deletion is
  requested.

Author: jvstme

mkdocs/docs/concepts/gateways.md

@@ -43,8 +43,8 @@ The example-gateway doesn't exist. Create it? [y/n]: y
 Provisioning...
 ---> 100%
 
- BACKEND  REGION     NAME             HOSTNAME  DOMAIN       DEFAULT  STATUS
- aws      eu-west-1  example-gateway            example.com  ✓        submitted
+ NAME             BACKEND          HOSTNAME       DOMAIN       DEFAULT  STATUS
+ example-gateway  aws (eu-west-1)  34.244.128.46  example.com  ✓        running
 ```
 
 </div>
@@ -211,8 +211,8 @@ To balance requests between gateway replicas, add DNS records for each replica o
 $ dstack gateway list
  NAME             BACKEND          HOSTNAME        DOMAIN       DEFAULT  STATUS
  example-gateway                                   example.com  ✓        running
-    replica=0     aws (eu-west-1)  34.244.128.46
-    replica=1     aws (eu-west-1)  18.201.201.174
+    replica=0     aws (eu-west-1)  34.244.128.46                         running
+    replica=1     aws (eu-west-1)  18.201.201.174                        running
 ```
 
 </div>

src/dstack/_internal/cli/utils/gateway.py

@@ -118,6 +118,10 @@ def get_gateways_table(
                 gateway.replicas[0].backend, gateway.replicas[0].region
             )
             gateway_row["HOSTNAME"] = gateway_row.get("HOSTNAME", gateway.replicas[0].hostname)
+            gateway_row["STATUS"] = gateway.replicas[0].status or gateway.status
+            gateway_row["ERROR"] = ". ".join(
+                m for m in [gateway.status_message, gateway.replicas[0].status_message] if m
+            )
         add_row_from_dict(table, gateway_row)
 
         if len(gateway.replicas) > 1:
@@ -126,7 +130,9 @@ def get_gateways_table(
                     "NAME": f"   replica={replica.replica_num}",
                     "BACKEND": format_backend(replica.backend, replica.region),
                     "HOSTNAME": replica.hostname,
+                    "STATUS": replica.status,
                     "CREATED": format_date(replica.created_at),
+                    "ERROR": replica.status_message,
                 }
                 add_row_from_dict(table, replica_row, style="secondary")
 

src/dstack/_internal/core/models/gateways.py

@@ -21,6 +21,14 @@ class GatewayStatus(str, Enum):
     FAILED = "failed"
 
 
+class GatewayReplicaStatus(str, Enum):
+    SUBMITTED = "submitted"
+    PROVISIONING = "provisioning"
+    RUNNING = "running"
+    TERMINATING = "terminating"
+    TERMINATED = "terminated"
+
+
 class LetsEncryptGatewayCertificate(CoreModel):
     type: Annotated[
         Literal["lets-encrypt"], Field(description="Automatic certificates by Let's Encrypt")
@@ -119,11 +127,14 @@ class GatewaySpec(CoreModel):
 
 
 class GatewayReplica(CoreModel):
-    hostname: str
+    hostname: Optional[str] = None
     replica_num: int
-    backend: BackendType
-    region: str
+    backend: Optional[BackendType] = None
+    region: Optional[str] = None
     created_at: datetime.datetime
+    status: Optional[GatewayReplicaStatus] = None
+    """`status` is only optional on the client side for compatibility with 0.20.25 and 0.20.26 servers"""
+    status_message: Optional[str] = None
 
 
 class Gateway(CoreModel):

src/dstack/_internal/server/background/pipeline_tasks/__init__.py

@@ -3,6 +3,9 @@
 from dstack._internal.server.background.pipeline_tasks.base import Pipeline
 from dstack._internal.server.background.pipeline_tasks.compute_groups import ComputeGroupPipeline
 from dstack._internal.server.background.pipeline_tasks.fleets import FleetPipeline
+from dstack._internal.server.background.pipeline_tasks.gateway_replicas import (
+    GatewayReplicaPipeline,
+)
 from dstack._internal.server.background.pipeline_tasks.gateways import GatewayPipeline
 from dstack._internal.server.background.pipeline_tasks.instances import InstancePipeline
 from dstack._internal.server.background.pipeline_tasks.jobs_running import JobRunningPipeline
@@ -33,6 +36,7 @@ def __init__(self) -> None:
             ComputeGroupPipeline(pipeline_hinter=self._hinter),
             FleetPipeline(pipeline_hinter=self._hinter),
             GatewayPipeline(pipeline_hinter=self._hinter),
+            GatewayReplicaPipeline(pipeline_hinter=self._hinter),
             JobSubmittedPipeline(pipeline_hinter=self._hinter),
             JobRunningPipeline(pipeline_hinter=self._hinter),
             JobTerminatingPipeline(pipeline_hinter=self._hinter),