[CLI] Rename stats command to metrics (#2462)

un-def · web-flow · commit ec8a12af1137 · 2025-04-02T07:01:05.000Z
diff --git a/docs/blog/posts/dstack-stats.md b/docs/blog/posts/dstack-stats.md
@@ -15,7 +15,7 @@ categories:
 ## How it works { style="display:none"}
 
 While it's possible to use third-party monitoring tools with `dstack`, it is often more convenient to debug your run and
-track metrics out of the box. That's why, with the latest release, `dstack` introduced [`dstack stats`](../../docs/reference/cli/dstack/stats.md), a new CLI (and API)
+track metrics out of the box. That's why, with the latest release, `dstack` introduced [`dstack stats`](../../docs/reference/cli/dstack/metrics.md), a new CLI (and API)
 for monitoring container metrics, including GPU usage for `NVIDIA`, `AMD`, and other accelerators.
 
 <img src="https://github.com/dstackai/static-assets/blob/main/static-assets/images/dstack-stats-v2.png?raw=true" width="725"/>
diff --git a/docs/blog/posts/prometheus.md b/docs/blog/posts/prometheus.md
@@ -17,7 +17,7 @@ Effective AI infrastructure management requires full visibility into compute per
 detailed insights into container- and GPU-level performance, while managers rely on cost metrics to track resource usage
 across projects.
 
-While `dstack` provides key metrics through its UI and [`dstack stats`](dstack-stats.md) CLI, teams often need more granular data and prefer
+While `dstack` provides key metrics through its UI and [`dstack metrics`](dstack-stats.md) CLI, teams often need more granular data and prefer
 using their own monitoring tools. To support this, we’ve introduced a new endpoint that allows real-time exporting all collected
 metrics—covering fleets and runs—directly to Prometheus.
 
@@ -49,19 +49,19 @@ Overall, `dstack` collects three groups of metrics:
 For a full list of available metrics and labels, check out the [Monitoring](../../docs/guides/monitoring.md) guide.
 
 ??? info "NVIDIA"
-    NVIDIA DCGM metrics are automatically collected for `aws`, `azure`, `gcp`, and `oci` backends, 
+    NVIDIA DCGM metrics are automatically collected for `aws`, `azure`, `gcp`, and `oci` backends,
     as well as for [SSH fleets](../../docs/concepts/fleets.md#ssh).
-    
+
     To ensure NVIDIA DCGM metrics are collected from SSH fleets, ensure the `datacenter-gpu-manager-4-core`,
     `datacenter-gpu-manager-4-proprietary`, and `datacenter-gpu-manager-exporter` packages are installed on the hosts.
 
 ??? info "AMD"
     AMD device metrics are not yet collected for any backends. This support will be available soon. For now, AMD metrics are
-    only accessible through the UI and the [`dstack stats`](dstack-stats.md) CLI.
+    only accessible through the UI and the [`dstack metrics`](dstack-stats.md) CLI.
 
 !!! info "What's next?"
     1. See the [Monitoring](../../docs/guides/monitoring.md) guide
-    1. Check [dev environments](../../docs/concepts/dev-environments.md), 
+    1. Check [dev environments](../../docs/concepts/dev-environments.md),
        [tasks](../../docs/concepts/tasks.md), [services](../../docs/concepts/services.md),
        and [fleets](../../docs/concepts/fleets.md)
     2. Join [Discord :material-arrow-top-right-thin:{ .external }](https://discord.gg/u8SmfwPpMd){:target="_blank"}
diff --git a/docs/docs/guides/protips.md b/docs/docs/guides/protips.md
@@ -312,7 +312,7 @@ The GPU vendor is indicated by one of the following case-insensitive values:
 
 While `dstack` allows the use of any third-party monitoring tools (e.g., Weights and Biases), you can also
 monitor container metrics such as CPU, memory, and GPU usage using the [built-in
-`dstack stats` CLI command](../../blog/posts/dstack-stats.md) or the corresponding API.
+`dstack metrics` CLI command](../../blog/posts/dstack-stats.md) or the corresponding API.
 
 ## Service quotas
 
diff --git a/docs/docs/reference/cli/dstack/metrics.md b/docs/docs/reference/cli/dstack/metrics.md
@@ -1,4 +1,4 @@
-# dstack stats
+# dstack metrics
 
 This command shows run hardware metrics such as CPU, memory, and GPU utilization.
 
@@ -7,7 +7,7 @@ This command shows run hardware metrics such as CPU, memory, and GPU utilization
 <div class="termy">
 
 ```shell
-$ dstack stats --help
+$ dstack metrics --help
 #GENERATE#
 ```
 
diff --git a/mkdocs.yml b/mkdocs.yml
@@ -253,7 +253,7 @@ nav:
               - dstack stop: docs/reference/cli/dstack/stop.md
               - dstack attach: docs/reference/cli/dstack/attach.md
               - dstack logs: docs/reference/cli/dstack/logs.md
-              - dstack stats: docs/reference/cli/dstack/stats.md
+              - dstack metrics: docs/reference/cli/dstack/metrics.md
               - dstack config: docs/reference/cli/dstack/config.md
               - dstack fleet: docs/reference/cli/dstack/fleet.md
               - dstack volume: docs/reference/cli/dstack/volume.md
diff --git a/src/dstack/_internal/cli/commands/metrics.py b/src/dstack/_internal/cli/commands/metrics.py
@@ -0,0 +1,128 @@
+import argparse
+import time
+from typing import Any, Dict, List, Optional, Union
+
+from rich.live import Live
+from rich.table import Table
+
+from dstack._internal.cli.commands import APIBaseCommand
+from dstack._internal.cli.services.completion import RunNameCompleter
+from dstack._internal.cli.utils.common import (
+    LIVE_TABLE_PROVISION_INTERVAL_SECS,
+    LIVE_TABLE_REFRESH_RATE_PER_SEC,
+    add_row_from_dict,
+    console,
+)
+from dstack._internal.core.errors import CLIError
+from dstack._internal.core.models.metrics import JobMetrics
+from dstack.api._public import Client
+from dstack.api._public.runs import Run
+
+
+class MetricsCommand(APIBaseCommand):
+    NAME = "metrics"
+    DESCRIPTION = "Show run metrics"
+
+    def _register(self):
+        super()._register()
+        self._parser.add_argument("run_name").completer = RunNameCompleter()
+        self._parser.add_argument(
+            "-w",
+            "--watch",
+            help="Watch run metrics in realtime",
+            action="store_true",
+        )
+
+    def _command(self, args: argparse.Namespace):
+        super()._command(args)
+        run = self.api.runs.get(run_name=args.run_name)
+        if run is None:
+            raise CLIError(f"Run {args.run_name} not found")
+        if run.status.is_finished():
+            raise CLIError(f"Run {args.run_name} is finished")
+        metrics = _get_run_jobs_metrics(api=self.api, run=run)
+
+        if not args.watch:
+            console.print(_get_metrics_table(run, metrics))
+            return
+
+        try:
+            with Live(console=console, refresh_per_second=LIVE_TABLE_REFRESH_RATE_PER_SEC) as live:
+                while True:
+                    live.update(_get_metrics_table(run, metrics))
+                    time.sleep(LIVE_TABLE_PROVISION_INTERVAL_SECS)
+                    run = self.api.runs.get(run_name=args.run_name)
+                    if run is None:
+                        raise CLIError(f"Run {args.run_name} not found")
+                    if run.status.is_finished():
+                        raise CLIError(f"Run {args.run_name} is finished")
+                    metrics = _get_run_jobs_metrics(api=self.api, run=run)
+        except KeyboardInterrupt:
+            pass
+
+
+def _get_run_jobs_metrics(api: Client, run: Run) -> List[JobMetrics]:
+    metrics = []
+    for job in run._run.jobs:
+        job_metrics = api.client.metrics.get_job_metrics(
+            project_name=api.project,
+            run_name=run.name,
+            replica_num=job.job_spec.replica_num,
+            job_num=job.job_spec.job_num,
+        )
+        metrics.append(job_metrics)
+    return metrics
+
+
+def _get_metrics_table(run: Run, metrics: List[JobMetrics]) -> Table:
+    table = Table(box=None)
+    table.add_column("NAME", style="bold", no_wrap=True)
+    table.add_column("CPU")
+    table.add_column("MEMORY")
+    table.add_column("GPU")
+
+    run_row: Dict[Union[str, int], Any] = {"NAME": run.name}
+    if len(run._run.jobs) != 1:
+        add_row_from_dict(table, run_row)
+
+    for job, job_metrics in zip(run._run.jobs, metrics):
+        cpu_usage = _get_metric_value(job_metrics, "cpu_usage_percent")
+        if cpu_usage is not None:
+            cpu_usage = f"{cpu_usage}%"
+        memory_usage = _get_metric_value(job_metrics, "memory_working_set_bytes")
+        if memory_usage is not None:
+            memory_usage = f"{round(memory_usage / 1024 / 1024)}MB"
+            if job.job_submissions[-1].job_provisioning_data is not None:
+                memory_usage += f"/{job.job_submissions[-1].job_provisioning_data.instance_type.resources.memory_mib}MB"
+        gpu_metrics = ""
+        gpus_detected_num = _get_metric_value(job_metrics, "gpus_detected_num")
+        if gpus_detected_num is not None:
+            for i in range(gpus_detected_num):
+                gpu_memory_usage = _get_metric_value(job_metrics, f"gpu_memory_usage_bytes_gpu{i}")
+                gpu_util_percent = _get_metric_value(job_metrics, f"gpu_util_percent_gpu{i}")
+                if gpu_memory_usage is not None:
+                    if i != 0:
+                        gpu_metrics += "\n"
+                    gpu_metrics += f"#{i} {round(gpu_memory_usage / 1024 / 1024)}MB"
+                    if job.job_submissions[-1].job_provisioning_data is not None:
+                        gpu_metrics += f"/{job.job_submissions[-1].job_provisioning_data.instance_type.resources.gpus[i].memory_mib}MB"
+                    gpu_metrics += f" {gpu_util_percent}% Util"
+
+        job_row: Dict[Union[str, int], Any] = {
+            "NAME": f"  replica={job.job_spec.replica_num} job={job.job_spec.job_num}",
+            "CPU": cpu_usage or "-",
+            "MEMORY": memory_usage or "-",
+            "GPU": gpu_metrics or "-",
+        }
+        if len(run._run.jobs) == 1:
+            job_row.update(run_row)
+        add_row_from_dict(table, job_row)
+
+    return table
+
+
+def _get_metric_value(job_metrics: JobMetrics, name: str) -> Optional[Any]:
+    for metric in job_metrics.metrics:
+        if metric.name == name:
+            return metric.values[-1]
+    return None
diff --git a/src/dstack/_internal/cli/commands/stats.py b/src/dstack/_internal/cli/commands/stats.py
@@ -1,128 +1,14 @@
 import argparse
-import time
-from typing import Any, Dict, List, Optional, Union
 
-from rich.live import Live
-from rich.table import Table
+from dstack._internal.cli.commands.metrics import MetricsCommand
+from dstack._internal.utils.logging import get_logger
 
-from dstack._internal.cli.commands import APIBaseCommand
-from dstack._internal.cli.services.completion import RunNameCompleter
-from dstack._internal.cli.utils.common import (
-    LIVE_TABLE_PROVISION_INTERVAL_SECS,
-    LIVE_TABLE_REFRESH_RATE_PER_SEC,
-    add_row_from_dict,
-    console,
-)
-from dstack._internal.core.errors import CLIError
-from dstack._internal.core.models.metrics import JobMetrics
-from dstack.api._public import Client
-from dstack.api._public.runs import Run
+logger = get_logger(__name__)
 
 
-class StatsCommand(APIBaseCommand):
+class StatsCommand(MetricsCommand):
     NAME = "stats"
-    DESCRIPTION = "Show run stats"
-
-    def _register(self):
-        super()._register()
-        self._parser.add_argument("run_name").completer = RunNameCompleter()
-        self._parser.add_argument(
-            "-w",
-            "--watch",
-            help="Watch run stats in realtime",
-            action="store_true",
-        )
 
     def _command(self, args: argparse.Namespace):
+        logger.warning("`dstack stats` is deprecated in favor of `dstack metrics`")
         super()._command(args)
-        run = self.api.runs.get(run_name=args.run_name)
-        if run is None:
-            raise CLIError(f"Run {args.run_name} not found")
-        if run.status.is_finished():
-            raise CLIError(f"Run {args.run_name} is finished")
-        metrics = _get_run_jobs_metrics(api=self.api, run=run)
-
-        if not args.watch:
-            console.print(_get_stats_table(run, metrics))
-            return
-
-        try:
-            with Live(console=console, refresh_per_second=LIVE_TABLE_REFRESH_RATE_PER_SEC) as live:
-                while True:
-                    live.update(_get_stats_table(run, metrics))
-                    time.sleep(LIVE_TABLE_PROVISION_INTERVAL_SECS)
-                    run = self.api.runs.get(run_name=args.run_name)
-                    if run is None:
-                        raise CLIError(f"Run {args.run_name} not found")
-                    if run.status.is_finished():
-                        raise CLIError(f"Run {args.run_name} is finished")
-                    metrics = _get_run_jobs_metrics(api=self.api, run=run)
-        except KeyboardInterrupt:
-            pass
-
-
-def _get_run_jobs_metrics(api: Client, run: Run) -> List[JobMetrics]:
-    metrics = []
-    for job in run._run.jobs:
-        job_metrics = api.client.metrics.get_job_metrics(
-            project_name=api.project,
-            run_name=run.name,
-            replica_num=job.job_spec.replica_num,
-            job_num=job.job_spec.job_num,
-        )
-        metrics.append(job_metrics)
-    return metrics
-
-
-def _get_stats_table(run: Run, metrics: List[JobMetrics]) -> Table:
-    table = Table(box=None)
-    table.add_column("NAME", style="bold", no_wrap=True)
-    table.add_column("CPU")
-    table.add_column("MEMORY")
-    table.add_column("GPU")
-
-    run_row: Dict[Union[str, int], Any] = {"NAME": run.name}
-    if len(run._run.jobs) != 1:
-        add_row_from_dict(table, run_row)
-
-    for job, job_metrics in zip(run._run.jobs, metrics):
-        cpu_usage = _get_metric_value(job_metrics, "cpu_usage_percent")
-        if cpu_usage is not None:
-            cpu_usage = f"{cpu_usage}%"
-        memory_usage = _get_metric_value(job_metrics, "memory_working_set_bytes")
-        if memory_usage is not None:
-            memory_usage = f"{round(memory_usage / 1024 / 1024)}MB"
-            if job.job_submissions[-1].job_provisioning_data is not None:
-                memory_usage += f"/{job.job_submissions[-1].job_provisioning_data.instance_type.resources.memory_mib}MB"
-        gpu_stats = ""
-        gpus_detected_num = _get_metric_value(job_metrics, "gpus_detected_num")
-        if gpus_detected_num is not None:
-            for i in range(gpus_detected_num):
-                gpu_memory_usage = _get_metric_value(job_metrics, f"gpu_memory_usage_bytes_gpu{i}")
-                gpu_util_percent = _get_metric_value(job_metrics, f"gpu_util_percent_gpu{i}")
-                if gpu_memory_usage is not None:
-                    if i != 0:
-                        gpu_stats += "\n"
-                    gpu_stats += f"#{i} {round(gpu_memory_usage / 1024 / 1024)}MB"
-                    if job.job_submissions[-1].job_provisioning_data is not None:
-                        gpu_stats += f"/{job.job_submissions[-1].job_provisioning_data.instance_type.resources.gpus[i].memory_mib}MB"
-                    gpu_stats += f" {gpu_util_percent}% Util"
-
-        job_row: Dict[Union[str, int], Any] = {
-            "NAME": f"  replica={job.job_spec.replica_num} job={job.job_spec.job_num}",
-            "CPU": cpu_usage or "-",
-            "MEMORY": memory_usage or "-",
-            "GPU": gpu_stats or "-",
-        }
-        if len(run._run.jobs) == 1:
-            job_row.update(run_row)
-        add_row_from_dict(table, job_row)
-
-    return table
-
-
-def _get_metric_value(job_metrics: JobMetrics, name: str) -> Optional[Any]:
-    for metric in job_metrics.metrics:
-        if metric.name == name:
-            return metric.values[-1]
-    return None
diff --git a/src/dstack/_internal/cli/main.py b/src/dstack/_internal/cli/main.py
@@ -13,6 +13,7 @@
 from dstack._internal.cli.commands.gateway import GatewayCommand
 from dstack._internal.cli.commands.init import InitCommand
 from dstack._internal.cli.commands.logs import LogsCommand
+from dstack._internal.cli.commands.metrics import MetricsCommand
 from dstack._internal.cli.commands.ps import PsCommand
 from dstack._internal.cli.commands.server import ServerCommand
 from dstack._internal.cli.commands.stats import StatsCommand
@@ -65,6 +66,7 @@ def main():
     GatewayCommand.register(subparsers)
     InitCommand.register(subparsers)
     LogsCommand.register(subparsers)
+    MetricsCommand.register(subparsers)
     PsCommand.register(subparsers)
     ServerCommand.register(subparsers)
     StatsCommand.register(subparsers)
