Skip to content

Commit 0d53c15

Browse files
Nadine-HNadine-H
committed
Implement custom http metrics
1 parent a5e1429 commit 0d53c15

3 files changed

Lines changed: 86 additions & 78 deletions

File tree

‎src/dstack/_internal/server/app.py‎

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
from fastapi.datastructures import URL
1212
from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
1313
from fastapi.staticfiles import StaticFiles
14-
from prometheus_fastapi_instrumentator import Instrumentator
14+
from prometheus_client import Counter, Histogram
1515

1616
from dstack._internal.cli.utils.common import console
1717
from dstack._internal.core.errors import ForbiddenError, ServerClientError
@@ -64,6 +64,18 @@
6464

6565
logger = get_logger(__name__)
6666

67+
# Server HTTP metrics
68+
REQUESTS_TOTAL = Counter(
69+
"dstack_server_requests_total",
70+
"Total number of HTTP requests",
71+
["method", "endpoint", "http_status", "project_name"],
72+
)
73+
REQUEST_DURATION = Histogram(
74+
"dstack_server_request_duration_seconds",
75+
"HTTP request duration in seconds",
76+
["method", "endpoint", "http_status", "project_name"],
77+
)
78+
6779

6880
def create_app() -> FastAPI:
6981
if settings.SENTRY_DSN is not None:
@@ -78,7 +90,6 @@ def create_app() -> FastAPI:
7890

7991
app = FastAPI(docs_url="/api/docs", lifespan=lifespan)
8092
app.state.proxy_dependency_injector = ServerProxyDependencyInjector()
81-
Instrumentator().instrument(app, metric_namespace="dstack", metric_subsystem="server")
8293
return app
8394

8495

@@ -218,6 +229,8 @@ async def log_request(request: Request, call_next):
218229
start_time = time.time()
219230
response: Response = await call_next(request)
220231
process_time = time.time() - start_time
232+
# log process_time to be used in the log_http_metrics middleware
233+
request.state.process_time = process_time
221234
logger.debug(
222235
"Processed request %s %s in %s. Status: %s",
223236
request.method,
@@ -227,6 +240,36 @@ async def log_request(request: Request, call_next):
227240
)
228241
return response
229242

243+
# this middleware must be defined after the log_request middleware
244+
@app.middleware("http")
245+
async def log_http_metrics(request: Request, call_next):
246+
def _extract_project_name(request: Request):
247+
project_name = None
248+
prefix = "/api/project/"
249+
if request.url.path.startswith(prefix):
250+
rest = request.url.path[len(prefix) :]
251+
project_name = rest.split("/", 1)[0] if rest else None
252+
253+
return project_name
254+
255+
project_name = _extract_project_name(request)
256+
response: Response = await call_next(request)
257+
258+
REQUEST_DURATION.labels(
259+
method=request.method,
260+
endpoint=request.url.path,
261+
http_status=response.status_code,
262+
project_name=project_name,
263+
).observe(request.state.process_time)
264+
265+
REQUESTS_TOTAL.labels(
266+
method=request.method,
267+
endpoint=request.url.path,
268+
http_status=response.status_code,
269+
project_name=project_name,
270+
).inc()
271+
return response
272+
230273
@app.middleware("http")
231274
async def check_client_version(request: Request, call_next):
232275
if (

‎src/dstack/_internal/server/routers/prometheus.py‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22
from typing import Annotated
33

44
from fastapi import APIRouter, Depends
5-
from fastapi.responses import PlainTextResponse, Response
6-
from prometheus_client import CONTENT_TYPE_LATEST, generate_latest
5+
from fastapi.responses import PlainTextResponse
6+
from prometheus_client import generate_latest
77
from sqlalchemy.ext.asyncio import AsyncSession
88

99
from dstack._internal.server import settings
@@ -24,12 +24,9 @@
2424
@router.get("/metrics")
2525
async def get_prometheus_metrics(
2626
session: Annotated[AsyncSession, Depends(get_session)],
27-
):
27+
) -> str:
2828
if not settings.ENABLE_PROMETHEUS_METRICS:
2929
raise error_not_found()
3030
custom_metrics = await prometheus.get_metrics(session=session)
31-
instrumentator_metrics = generate_latest().decode()
32-
return Response(
33-
custom_metrics + instrumentator_metrics,
34-
media_type=CONTENT_TYPE_LATEST,
35-
)
31+
prometheus_metrics = generate_latest()
32+
return custom_metrics + prometheus_metrics.decode()

‎src/tests/_internal/server/routers/test_prometheus.py‎

Lines changed: 36 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -39,82 +39,50 @@
3939
BASE_HTTP_METRICS = b"""
4040
# HELP python_gc_objects_collected_total Objects collected during gc
4141
# TYPE python_gc_objects_collected_total counter
42-
python_gc_objects_collected_total{generation="0"} 16262.0
43-
python_gc_objects_collected_total{generation="1"} 3588.0
44-
python_gc_objects_collected_total{generation="2"} 325.0
42+
python_gc_objects_collected_total{generation="0"} 13159.0
43+
python_gc_objects_collected_total{generation="1"} 1583.0
44+
python_gc_objects_collected_total{generation="2"} 81.0
4545
# HELP python_gc_objects_uncollectable_total Uncollectable objects found during GC
4646
# TYPE python_gc_objects_uncollectable_total counter
4747
python_gc_objects_uncollectable_total{generation="0"} 0.0
4848
python_gc_objects_uncollectable_total{generation="1"} 0.0
4949
python_gc_objects_uncollectable_total{generation="2"} 0.0
5050
# HELP python_gc_collections_total Number of times this generation was collected
5151
# TYPE python_gc_collections_total counter
52-
python_gc_collections_total{generation="0"} 1687.0
53-
python_gc_collections_total{generation="1"} 153.0
54-
python_gc_collections_total{generation="2"} 10.0
52+
python_gc_collections_total{generation="0"} 1609.0
53+
python_gc_collections_total{generation="1"} 146.0
54+
python_gc_collections_total{generation="2"} 9.0
5555
# HELP python_info Python platform information
5656
# TYPE python_info gauge
5757
python_info{implementation="CPython",major="3",minor="12",patchlevel="2",version="3.12.2"} 1.0
58-
# HELP dstack_server_http_requests_total Total number of requests by method, status and handler.
59-
# TYPE dstack_server_http_requests_total counter
60-
dstack_server_http_requests_total{handler="/metrics",method="GET",status="2xx"} 1.0
61-
# HELP dstack_server_http_requests_created Total number of requests by method, status and handler.
62-
# TYPE dstack_server_http_requests_created gauge
63-
dstack_server_http_requests_created{handler="/metrics",method="GET",status="2xx"} 1.67262864e+09
64-
# HELP dstack_server_http_request_size_bytes Content length of incoming requests by handler. Only value of header is respected. Otherwise ignored. No percentile calculated.
65-
# TYPE dstack_server_http_request_size_bytes summary
66-
dstack_server_http_request_size_bytes_count{handler="/metrics"} 1.0
67-
dstack_server_http_request_size_bytes_sum{handler="/metrics"} 0.0
68-
# HELP dstack_server_http_request_size_bytes_created Content length of incoming requests by handler. Only value of header is respected. Otherwise ignored. No percentile calculated.
69-
# TYPE dstack_server_http_request_size_bytes_created gauge
70-
dstack_server_http_request_size_bytes_created{handler="/metrics"} 1.67262864e+09
71-
# HELP dstack_server_http_response_size_bytes Content length of outgoing responses by handler. Only value of header is respected. Otherwise ignored. No percentile calculated.
72-
# TYPE dstack_server_http_response_size_bytes summary
73-
dstack_server_http_response_size_bytes_count{handler="/metrics"} 1.0
74-
dstack_server_http_response_size_bytes_sum{handler="/metrics"} 17846.0
75-
# HELP dstack_server_http_response_size_bytes_created Content length of outgoing responses by handler. Only value of header is respected. Otherwise ignored. No percentile calculated.
76-
# TYPE dstack_server_http_response_size_bytes_created gauge
77-
dstack_server_http_response_size_bytes_created{handler="/metrics"} 1.67262864e+09
78-
# HELP dstack_server_http_request_duration_highr_seconds Latency with many buckets but no API specific labels. Made for more accurate percentile calculations.
79-
# TYPE dstack_server_http_request_duration_highr_seconds histogram
80-
dstack_server_http_request_duration_highr_seconds_bucket{le="0.01"} 1.0
81-
dstack_server_http_request_duration_highr_seconds_bucket{le="0.025"} 1.0
82-
dstack_server_http_request_duration_highr_seconds_bucket{le="0.05"} 1.0
83-
dstack_server_http_request_duration_highr_seconds_bucket{le="0.075"} 1.0
84-
dstack_server_http_request_duration_highr_seconds_bucket{le="0.1"} 1.0
85-
dstack_server_http_request_duration_highr_seconds_bucket{le="0.25"} 1.0
86-
dstack_server_http_request_duration_highr_seconds_bucket{le="0.5"} 1.0
87-
dstack_server_http_request_duration_highr_seconds_bucket{le="0.75"} 1.0
88-
dstack_server_http_request_duration_highr_seconds_bucket{le="1.0"} 1.0
89-
dstack_server_http_request_duration_highr_seconds_bucket{le="1.5"} 1.0
90-
dstack_server_http_request_duration_highr_seconds_bucket{le="2.0"} 1.0
91-
dstack_server_http_request_duration_highr_seconds_bucket{le="2.5"} 1.0
92-
dstack_server_http_request_duration_highr_seconds_bucket{le="3.0"} 1.0
93-
dstack_server_http_request_duration_highr_seconds_bucket{le="3.5"} 1.0
94-
dstack_server_http_request_duration_highr_seconds_bucket{le="4.0"} 1.0
95-
dstack_server_http_request_duration_highr_seconds_bucket{le="4.5"} 1.0
96-
dstack_server_http_request_duration_highr_seconds_bucket{le="5.0"} 1.0
97-
dstack_server_http_request_duration_highr_seconds_bucket{le="7.5"} 1.0
98-
dstack_server_http_request_duration_highr_seconds_bucket{le="10.0"} 1.0
99-
dstack_server_http_request_duration_highr_seconds_bucket{le="30.0"} 1.0
100-
dstack_server_http_request_duration_highr_seconds_bucket{le="60.0"} 1.0
101-
dstack_server_http_request_duration_highr_seconds_bucket{le="+Inf"} 1.0
102-
dstack_server_http_request_duration_highr_seconds_count 1.0
103-
dstack_server_http_request_duration_highr_seconds_sum 0.0
104-
# HELP dstack_server_http_request_duration_highr_seconds_created Latency with many buckets but no API specific labels. Made for more accurate percentile calculations.
105-
# TYPE dstack_server_http_request_duration_highr_seconds_created gauge
106-
dstack_server_http_request_duration_highr_seconds_created 1.67262864e+09
107-
# HELP dstack_server_http_request_duration_seconds Latency with only few buckets by handler. Made to be only used if aggregation by handler is important.
108-
# TYPE dstack_server_http_request_duration_seconds histogram
109-
dstack_server_http_request_duration_seconds_bucket{handler="/metrics",le="0.1",method="GET"} 1.0
110-
dstack_server_http_request_duration_seconds_bucket{handler="/metrics",le="0.5",method="GET"} 1.0
111-
dstack_server_http_request_duration_seconds_bucket{handler="/metrics",le="1.0",method="GET"} 1.0
112-
dstack_server_http_request_duration_seconds_bucket{handler="/metrics",le="+Inf",method="GET"} 1.0
113-
dstack_server_http_request_duration_seconds_count{handler="/metrics",method="GET"} 1.0
114-
dstack_server_http_request_duration_seconds_sum{handler="/metrics",method="GET"} 0.0
115-
# HELP dstack_server_http_request_duration_seconds_created Latency with only few buckets by handler. Made to be only used if aggregation by handler is important.
116-
# TYPE dstack_server_http_request_duration_seconds_created gauge
117-
dstack_server_http_request_duration_seconds_created{handler="/metrics",method="GET"} 1.67262864e+09
58+
# HELP dstack_server_requests_total Total number of HTTP requests
59+
# TYPE dstack_server_requests_total counter
60+
dstack_server_requests_total{endpoint="/metrics",http_status="200",method="GET",project_name="None"} 1.0
61+
# HELP dstack_server_requests_created Total number of HTTP requests
62+
# TYPE dstack_server_requests_created gauge
63+
dstack_server_requests_created{endpoint="/metrics",http_status="200",method="GET",project_name="None"} 1.67262864e+09
64+
# HELP dstack_server_request_duration_seconds HTTP request duration in seconds
65+
# TYPE dstack_server_request_duration_seconds histogram
66+
dstack_server_request_duration_seconds_bucket{endpoint="/metrics",http_status="200",le="0.005",method="GET",project_name="None"} 1.0
67+
dstack_server_request_duration_seconds_bucket{endpoint="/metrics",http_status="200",le="0.01",method="GET",project_name="None"} 1.0
68+
dstack_server_request_duration_seconds_bucket{endpoint="/metrics",http_status="200",le="0.025",method="GET",project_name="None"} 1.0
69+
dstack_server_request_duration_seconds_bucket{endpoint="/metrics",http_status="200",le="0.05",method="GET",project_name="None"} 1.0
70+
dstack_server_request_duration_seconds_bucket{endpoint="/metrics",http_status="200",le="0.075",method="GET",project_name="None"} 1.0
71+
dstack_server_request_duration_seconds_bucket{endpoint="/metrics",http_status="200",le="0.1",method="GET",project_name="None"} 1.0
72+
dstack_server_request_duration_seconds_bucket{endpoint="/metrics",http_status="200",le="0.25",method="GET",project_name="None"} 1.0
73+
dstack_server_request_duration_seconds_bucket{endpoint="/metrics",http_status="200",le="0.5",method="GET",project_name="None"} 1.0
74+
dstack_server_request_duration_seconds_bucket{endpoint="/metrics",http_status="200",le="0.75",method="GET",project_name="None"} 1.0
75+
dstack_server_request_duration_seconds_bucket{endpoint="/metrics",http_status="200",le="1.0",method="GET",project_name="None"} 1.0
76+
dstack_server_request_duration_seconds_bucket{endpoint="/metrics",http_status="200",le="2.5",method="GET",project_name="None"} 1.0
77+
dstack_server_request_duration_seconds_bucket{endpoint="/metrics",http_status="200",le="5.0",method="GET",project_name="None"} 1.0
78+
dstack_server_request_duration_seconds_bucket{endpoint="/metrics",http_status="200",le="7.5",method="GET",project_name="None"} 1.0
79+
dstack_server_request_duration_seconds_bucket{endpoint="/metrics",http_status="200",le="10.0",method="GET",project_name="None"} 1.0
80+
dstack_server_request_duration_seconds_bucket{endpoint="/metrics",http_status="200",le="+Inf",method="GET",project_name="None"} 1.0
81+
dstack_server_request_duration_seconds_count{endpoint="/metrics",http_status="200",method="GET",project_name="None"} 1.0
82+
dstack_server_request_duration_seconds_sum{endpoint="/metrics",http_status="200",method="GET",project_name="None"} 0.0
83+
# HELP dstack_server_request_duration_seconds_created HTTP request duration in seconds
84+
# TYPE dstack_server_request_duration_seconds_created gauge
85+
dstack_server_request_duration_seconds_created{endpoint="/metrics",http_status="200",method="GET",project_name="None"} 1.67262864e+09
11886
"""
11987

12088

@@ -283,7 +251,7 @@ async def test_returns_metrics(self, session: AsyncSession, client: AsyncClient)
283251
response = await client.get("/metrics")
284252

285253
assert response.status_code == 200
286-
actual = (
254+
expected = (
287255
dedent(f"""\
288256
# HELP dstack_instance_duration_seconds_total Total seconds the instance is running
289257
# TYPE dstack_instance_duration_seconds_total counter
@@ -365,7 +333,7 @@ async def test_returns_metrics(self, session: AsyncSession, client: AsyncClient)
365333
+ "\n"
366334
+ BASE_HTTP_METRICS.decode().strip()
367335
)
368-
assert response.text.strip() == actual
336+
assert response.text.strip() == expected
369337

370338
@patch("dstack._internal.server.routers.prometheus.generate_latest", lambda: BASE_HTTP_METRICS)
371339
async def test_returns_empty_response_if_no_runs(self, client: AsyncClient):

0 commit comments

Comments
 (0)