Add basic http metrics

Author: Nadine-H

pyproject.toml

@@ -35,6 +35,7 @@ dependencies = [
     "gpuhunt==0.1.6",
     "argcomplete>=3.5.0",
     "ignore-python>=0.2.0",
+    "prometheus-fastapi-instrumentator>=7.1.0",
 ]
 
 [project.urls]

src/dstack/_internal/server/app.py

@@ -11,6 +11,7 @@
 from fastapi.datastructures import URL
 from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
+from prometheus_fastapi_instrumentator import Instrumentator
 
 from dstack._internal.cli.utils.common import console
 from dstack._internal.core.errors import ForbiddenError, ServerClientError
@@ -77,6 +78,7 @@ def create_app() -> FastAPI:
 
     app = FastAPI(docs_url="/api/docs", lifespan=lifespan)
     app.state.proxy_dependency_injector = ServerProxyDependencyInjector()
+    Instrumentator().instrument(app, metric_namespace="dstack", metric_subsystem="server")
     return app
 
 

src/dstack/_internal/server/routers/prometheus.py

@@ -2,7 +2,8 @@
 from typing import Annotated
 
 from fastapi import APIRouter, Depends
-from fastapi.responses import PlainTextResponse
+from fastapi.responses import PlainTextResponse, Response
+from prometheus_client import CONTENT_TYPE_LATEST, generate_latest
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from dstack._internal.server import settings
@@ -23,7 +24,12 @@
 @router.get("/metrics")
 async def get_prometheus_metrics(
     session: Annotated[AsyncSession, Depends(get_session)],
-) -> str:
+):
     if not settings.ENABLE_PROMETHEUS_METRICS:
         raise error_not_found()
-    return await prometheus.get_metrics(session=session)
+    custom_metrics = await prometheus.get_metrics(session=session)
+    instrumentator_metrics = generate_latest().decode()
+    return Response(
+        custom_metrics + instrumentator_metrics,
+        media_type=CONTENT_TYPE_LATEST,
+    )

src/tests/_internal/server/routers/test_prometheus.py

@@ -1,6 +1,7 @@
 from datetime import datetime, timedelta, timezone
 from textwrap import dedent
 from typing import Optional
+from unittest.mock import patch
 
 import pytest
 from freezegun import freeze_time
@@ -35,6 +36,87 @@
     get_run_spec,
 )
 
+BASE_HTTP_METRICS = b"""
+# HELP python_gc_objects_collected_total Objects collected during gc
+# TYPE python_gc_objects_collected_total counter
+python_gc_objects_collected_total{generation="0"} 16262.0
+python_gc_objects_collected_total{generation="1"} 3588.0
+python_gc_objects_collected_total{generation="2"} 325.0
+# HELP python_gc_objects_uncollectable_total Uncollectable objects found during GC
+# TYPE python_gc_objects_uncollectable_total counter
+python_gc_objects_uncollectable_total{generation="0"} 0.0
+python_gc_objects_uncollectable_total{generation="1"} 0.0
+python_gc_objects_uncollectable_total{generation="2"} 0.0
+# HELP python_gc_collections_total Number of times this generation was collected
+# TYPE python_gc_collections_total counter
+python_gc_collections_total{generation="0"} 1687.0
+python_gc_collections_total{generation="1"} 153.0
+python_gc_collections_total{generation="2"} 10.0
+# HELP python_info Python platform information
+# TYPE python_info gauge
+python_info{implementation="CPython",major="3",minor="12",patchlevel="2",version="3.12.2"} 1.0
+# HELP dstack_server_http_requests_total Total number of requests by method, status and handler.
+# TYPE dstack_server_http_requests_total counter
+dstack_server_http_requests_total{handler="/metrics",method="GET",status="2xx"} 1.0
+# HELP dstack_server_http_requests_created Total number of requests by method, status and handler.
+# TYPE dstack_server_http_requests_created gauge
+dstack_server_http_requests_created{handler="/metrics",method="GET",status="2xx"} 1.67262864e+09
+# HELP dstack_server_http_request_size_bytes Content length of incoming requests by handler. Only value of header is respected. Otherwise ignored. No percentile calculated.
+# TYPE dstack_server_http_request_size_bytes summary
+dstack_server_http_request_size_bytes_count{handler="/metrics"} 1.0
+dstack_server_http_request_size_bytes_sum{handler="/metrics"} 0.0
+# HELP dstack_server_http_request_size_bytes_created Content length of incoming requests by handler. Only value of header is respected. Otherwise ignored. No percentile calculated.
+# TYPE dstack_server_http_request_size_bytes_created gauge
+dstack_server_http_request_size_bytes_created{handler="/metrics"} 1.67262864e+09
+# HELP dstack_server_http_response_size_bytes Content length of outgoing responses by handler. Only value of header is respected. Otherwise ignored. No percentile calculated.
+# TYPE dstack_server_http_response_size_bytes summary
+dstack_server_http_response_size_bytes_count{handler="/metrics"} 1.0
+dstack_server_http_response_size_bytes_sum{handler="/metrics"} 17846.0
+# HELP dstack_server_http_response_size_bytes_created Content length of outgoing responses by handler. Only value of header is respected. Otherwise ignored. No percentile calculated.
+# TYPE dstack_server_http_response_size_bytes_created gauge
+dstack_server_http_response_size_bytes_created{handler="/metrics"} 1.67262864e+09
+# HELP dstack_server_http_request_duration_highr_seconds Latency with many buckets but no API specific labels. Made for more accurate percentile calculations.
+# TYPE dstack_server_http_request_duration_highr_seconds histogram
+dstack_server_http_request_duration_highr_seconds_bucket{le="0.01"} 1.0
+dstack_server_http_request_duration_highr_seconds_bucket{le="0.025"} 1.0
+dstack_server_http_request_duration_highr_seconds_bucket{le="0.05"} 1.0
+dstack_server_http_request_duration_highr_seconds_bucket{le="0.075"} 1.0
+dstack_server_http_request_duration_highr_seconds_bucket{le="0.1"} 1.0
+dstack_server_http_request_duration_highr_seconds_bucket{le="0.25"} 1.0
+dstack_server_http_request_duration_highr_seconds_bucket{le="0.5"} 1.0
+dstack_server_http_request_duration_highr_seconds_bucket{le="0.75"} 1.0
+dstack_server_http_request_duration_highr_seconds_bucket{le="1.0"} 1.0
+dstack_server_http_request_duration_highr_seconds_bucket{le="1.5"} 1.0
+dstack_server_http_request_duration_highr_seconds_bucket{le="2.0"} 1.0
+dstack_server_http_request_duration_highr_seconds_bucket{le="2.5"} 1.0
+dstack_server_http_request_duration_highr_seconds_bucket{le="3.0"} 1.0
+dstack_server_http_request_duration_highr_seconds_bucket{le="3.5"} 1.0
+dstack_server_http_request_duration_highr_seconds_bucket{le="4.0"} 1.0
+dstack_server_http_request_duration_highr_seconds_bucket{le="4.5"} 1.0
+dstack_server_http_request_duration_highr_seconds_bucket{le="5.0"} 1.0
+dstack_server_http_request_duration_highr_seconds_bucket{le="7.5"} 1.0
+dstack_server_http_request_duration_highr_seconds_bucket{le="10.0"} 1.0
+dstack_server_http_request_duration_highr_seconds_bucket{le="30.0"} 1.0
+dstack_server_http_request_duration_highr_seconds_bucket{le="60.0"} 1.0
+dstack_server_http_request_duration_highr_seconds_bucket{le="+Inf"} 1.0
+dstack_server_http_request_duration_highr_seconds_count 1.0
+dstack_server_http_request_duration_highr_seconds_sum 0.0
+# HELP dstack_server_http_request_duration_highr_seconds_created Latency with many buckets but no API specific labels. Made for more accurate percentile calculations.
+# TYPE dstack_server_http_request_duration_highr_seconds_created gauge
+dstack_server_http_request_duration_highr_seconds_created 1.67262864e+09
+# HELP dstack_server_http_request_duration_seconds Latency with only few buckets by handler. Made to be only used if aggregation by handler is important.
+# TYPE dstack_server_http_request_duration_seconds histogram
+dstack_server_http_request_duration_seconds_bucket{handler="/metrics",le="0.1",method="GET"} 1.0
+dstack_server_http_request_duration_seconds_bucket{handler="/metrics",le="0.5",method="GET"} 1.0
+dstack_server_http_request_duration_seconds_bucket{handler="/metrics",le="1.0",method="GET"} 1.0
+dstack_server_http_request_duration_seconds_bucket{handler="/metrics",le="+Inf",method="GET"} 1.0
+dstack_server_http_request_duration_seconds_count{handler="/metrics",method="GET"} 1.0
+dstack_server_http_request_duration_seconds_sum{handler="/metrics",method="GET"} 0.0
+# HELP dstack_server_http_request_duration_seconds_created Latency with only few buckets by handler. Made to be only used if aggregation by handler is important.
+# TYPE dstack_server_http_request_duration_seconds_created gauge
+dstack_server_http_request_duration_seconds_created{handler="/metrics",method="GET"} 1.67262864e+09
+"""
+
 
 @pytest.fixture
 def enable_metrics(monkeypatch: pytest.MonkeyPatch):
@@ -50,6 +132,7 @@ def enable_metrics(monkeypatch: pytest.MonkeyPatch):
 @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
 @pytest.mark.usefixtures("image_config_mock", "test_db", "enable_metrics")
 class TestGetPrometheusMetrics:
+    @patch("dstack._internal.server.routers.prometheus.generate_latest", lambda: BASE_HTTP_METRICS)
     async def test_returns_metrics(self, session: AsyncSession, client: AsyncClient):
         user = await create_user(session=session, name="test-user", global_role=GlobalRole.USER)
         offer = get_instance_offer_with_availability(
@@ -200,7 +283,8 @@ async def test_returns_metrics(self, session: AsyncSession, client: AsyncClient)
         response = await client.get("/metrics")
 
         assert response.status_code == 200
-        assert response.text == dedent(f"""\
+        actual = (
+            dedent(f"""\
             # HELP dstack_instance_duration_seconds_total Total seconds the instance is running
             # TYPE dstack_instance_duration_seconds_total counter
             dstack_instance_duration_seconds_total{{dstack_project_name="project-1",dstack_fleet_name="test-fleet",dstack_fleet_id="{fleet.id}",dstack_instance_name="test-instance",dstack_instance_id="{instance.id}",dstack_instance_type="test-type",dstack_backend="aws",dstack_gpu="V4"}} 3600.0
@@ -278,11 +362,16 @@ async def test_returns_metrics(self, session: AsyncSession, client: AsyncClient)
             FIELD_2{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 337325.0 1395066363000
             FIELD_2{{gpu="1",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 987169.0 1395066363010
         """)
+            + "\n"
+            + BASE_HTTP_METRICS.decode().strip()
+        )
+        assert response.text.strip() == actual
 
+    @patch("dstack._internal.server.routers.prometheus.generate_latest", lambda: BASE_HTTP_METRICS)
     async def test_returns_empty_response_if_no_runs(self, client: AsyncClient):
         response = await client.get("/metrics")
         assert response.status_code == 200
-        assert response.text == "\n"
+        assert response.text.strip() == BASE_HTTP_METRICS.decode().strip()
 
     async def test_returns_404_if_not_enabled(
         self, monkeypatch: pytest.MonkeyPatch, client: AsyncClient