feat(metrics)!: add "server" label to per-worker series

Workers declared in distinct php_server blocks can share a name now that
the per-scope routing landed; without a second label dimension, their
metric series collide on the existing "worker" label. This adds a sibling
"server" label (resolved via ScopeLabel(w.backgroundScope) at every call
site) so each (server, worker) pair stays on its own series.

BREAKING CHANGE: every Metrics interface method that took (name string)
now takes (server, name string). Embedders implementing frankenphp.Metrics
need to widen their signatures; PrometheusMetrics and the null impl are
updated in-tree. Mirrors the shape of #1376 (which introduced the
"worker" label the same way).

Author: nicolas-grekas

docs/metrics.md

@@ -13,13 +13,13 @@ When [Caddy metrics](https://caddyserver.com/docs/metrics) are enabled, FrankenP
 - `frankenphp_total_threads`: The total number of PHP threads.
 - `frankenphp_busy_threads`: The number of PHP threads currently processing a request (running workers always consume a thread).
 - `frankenphp_queue_depth`: The number of regular queued requests
-- `frankenphp_total_workers{worker="[worker_name]"}`: The total number of workers.
-- `frankenphp_busy_workers{worker="[worker_name]"}`: The number of workers currently processing a request.
-- `frankenphp_worker_request_time{worker="[worker_name]"}`: The time spent processing requests by all workers.
-- `frankenphp_worker_request_count{worker="[worker_name]"}`: The number of requests processed by all workers.
-- `frankenphp_ready_workers{worker="[worker_name]"}`: The number of workers that have called `frankenphp_handle_request` at least once.
-- `frankenphp_worker_crashes{worker="[worker_name]"}`: The number of times a worker has unexpectedly terminated.
-- `frankenphp_worker_restarts{worker="[worker_name]"}`: The number of times a worker has been deliberately restarted.
-- `frankenphp_worker_queue_depth{worker="[worker_name]"}`: The number of queued requests.
+- `frankenphp_total_workers{server="[server]",worker="[worker_name]"}`: The total number of workers.
+- `frankenphp_busy_workers{server="[server]",worker="[worker_name]"}`: The number of workers currently processing a request.
+- `frankenphp_worker_request_time{server="[server]",worker="[worker_name]"}`: The time spent processing requests by all workers.
+- `frankenphp_worker_request_count{server="[server]",worker="[worker_name]"}`: The number of requests processed by all workers.
+- `frankenphp_ready_workers{server="[server]",worker="[worker_name]"}`: The number of workers that have called `frankenphp_handle_request` at least once.
+- `frankenphp_worker_crashes{server="[server]",worker="[worker_name]"}`: The number of times a worker has unexpectedly terminated.
+- `frankenphp_worker_restarts{server="[server]",worker="[worker_name]"}`: The number of times a worker has been deliberately restarted.
+- `frankenphp_worker_queue_depth{server="[server]",worker="[worker_name]"}`: The number of queued requests.
 
-For worker metrics, the `[worker_name]` placeholder is replaced by the worker name in the Caddyfile, otherwise the absolute path of the worker file will be used.
+For worker metrics, the `[worker_name]` placeholder is replaced by the worker name in the Caddyfile, otherwise the absolute path of the worker file will be used. The `[server]` label identifies the `php_server` block that declared the worker; the Caddy module resolves it to the first host of the route's host matcher (e.g. `api.example.com`), falling back to the user-set Caddy server name and finally to the first listener address. Same-named workers in distinct `php_server` blocks therefore stay on distinct series.

frankenphp.go

@@ -182,15 +182,15 @@ func calculateMaxThreads(opt *opt) (numWorkers int, _ error) {
 			// Register the expected worker count for metrics too: without
 			// this, a bg-worker-only deployment never initialises
 			// totalWorkers, and StartWorker calls become silent no-ops.
-			metrics.TotalWorkers(w.name, extra)
+			metrics.TotalWorkers(ScopeLabel(w.backgroundScope), w.name, extra)
 			continue
 		}
 
 		if w.num <= 0 {
 			// https://github.com/php/frankenphp/issues/126
 			opt.workers[i].num = maxProcs
 		}
-		metrics.TotalWorkers(w.name, w.num)
+		metrics.TotalWorkers(ScopeLabel(w.backgroundScope), w.name, w.num)
 
 		numWorkers += opt.workers[i].num
 

metrics.go

@@ -16,44 +16,49 @@ const (
 
 type StopReason int
 
+// Metrics is the worker-level instrumentation surface. Every method that
+// identifies a specific worker takes a (server, name) pair: server is the
+// per-php_server label resolved via ScopeLabel, name is the worker name.
+// The pair is what disambiguates same-named workers declared in distinct
+// php_server blocks.
 type Metrics interface {
 	// StartWorker collects started workers
-	StartWorker(name string)
+	StartWorker(server, name string)
 	// ReadyWorker collects ready workers
-	ReadyWorker(name string)
+	ReadyWorker(server, name string)
 	// StopWorker collects stopped workers
-	StopWorker(name string, reason StopReason)
+	StopWorker(server, name string, reason StopReason)
 	// TotalWorkers collects expected workers
-	TotalWorkers(name string, num int)
+	TotalWorkers(server, name string, num int)
 	// TotalThreads collects total threads
 	TotalThreads(num int)
 	// StartRequest collects started requests
 	StartRequest()
 	// StopRequest collects stopped requests
 	StopRequest()
 	// StopWorkerRequest collects stopped worker requests
-	StopWorkerRequest(name string, duration time.Duration)
+	StopWorkerRequest(server, name string, duration time.Duration)
 	// StartWorkerRequest collects started worker requests
-	StartWorkerRequest(name string)
+	StartWorkerRequest(server, name string)
 	Shutdown()
-	QueuedWorkerRequest(name string)
-	DequeuedWorkerRequest(name string)
+	QueuedWorkerRequest(server, name string)
+	DequeuedWorkerRequest(server, name string)
 	QueuedRequest()
 	DequeuedRequest()
 }
 
 type nullMetrics struct{}
 
-func (n nullMetrics) StartWorker(string) {
+func (n nullMetrics) StartWorker(string, string) {
 }
 
-func (n nullMetrics) ReadyWorker(string) {
+func (n nullMetrics) ReadyWorker(string, string) {
 }
 
-func (n nullMetrics) StopWorker(string, StopReason) {
+func (n nullMetrics) StopWorker(string, string, StopReason) {
 }
 
-func (n nullMetrics) TotalWorkers(string, int) {
+func (n nullMetrics) TotalWorkers(string, string, int) {
 }
 
 func (n nullMetrics) TotalThreads(int) {
@@ -65,18 +70,18 @@ func (n nullMetrics) StartRequest() {
 func (n nullMetrics) StopRequest() {
 }
 
-func (n nullMetrics) StopWorkerRequest(string, time.Duration) {
+func (n nullMetrics) StopWorkerRequest(string, string, time.Duration) {
 }
 
-func (n nullMetrics) StartWorkerRequest(string) {
+func (n nullMetrics) StartWorkerRequest(string, string) {
 }
 
 func (n nullMetrics) Shutdown() {
 }
 
-func (n nullMetrics) QueuedWorkerRequest(string) {}
+func (n nullMetrics) QueuedWorkerRequest(string, string) {}
 
-func (n nullMetrics) DequeuedWorkerRequest(string) {}
+func (n nullMetrics) DequeuedWorkerRequest(string, string) {}
 
 func (n nullMetrics) QueuedRequest()   {}
 func (n nullMetrics) DequeuedRequest() {}
@@ -97,54 +102,54 @@ type PrometheusMetrics struct {
 	mu                 sync.Mutex
 }
 
-func (m *PrometheusMetrics) StartWorker(name string) {
+func (m *PrometheusMetrics) StartWorker(server, name string) {
 	m.busyThreads.Inc()
 
 	// tests do not register workers before starting them
 	if m.totalWorkers == nil {
 		return
 	}
 
-	m.totalWorkers.WithLabelValues(name).Inc()
+	m.totalWorkers.WithLabelValues(server, name).Inc()
 }
 
-func (m *PrometheusMetrics) ReadyWorker(name string) {
+func (m *PrometheusMetrics) ReadyWorker(server, name string) {
 	if m.totalWorkers == nil {
 		return
 	}
 
-	m.readyWorkers.WithLabelValues(name).Inc()
+	m.readyWorkers.WithLabelValues(server, name).Inc()
 }
 
-func (m *PrometheusMetrics) StopWorker(name string, reason StopReason) {
+func (m *PrometheusMetrics) StopWorker(server, name string, reason StopReason) {
 	m.busyThreads.Dec()
 
 	// tests do not register workers before starting them
 	if m.totalWorkers == nil {
 		return
 	}
 
-	m.totalWorkers.WithLabelValues(name).Dec()
+	m.totalWorkers.WithLabelValues(server, name).Dec()
 
 	// only decrement readyWorkers if the worker actually reached frankenphp_handle_request
 	if reason != StopReasonBootFailure {
-		m.readyWorkers.WithLabelValues(name).Dec()
+		m.readyWorkers.WithLabelValues(server, name).Dec()
 	}
 
 	switch reason {
 	case StopReasonCrash, StopReasonBootFailure:
-		m.workerCrashes.WithLabelValues(name).Inc()
+		m.workerCrashes.WithLabelValues(server, name).Inc()
 	case StopReasonRestart:
-		m.workerRestarts.WithLabelValues(name).Inc()
+		m.workerRestarts.WithLabelValues(server, name).Inc()
 	}
 }
 
-func (m *PrometheusMetrics) TotalWorkers(string, int) {
+func (m *PrometheusMetrics) TotalWorkers(string, string, int) {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 
 	const ns, sub = "frankenphp", "worker"
-	basicLabels := []string{"worker"}
+	basicLabels := []string{"server", "worker"}
 
 	if m.totalWorkers == nil {
 		m.totalWorkers = prometheus.NewGaugeVec(prometheus.GaugeOpts{
@@ -257,35 +262,35 @@ func (m *PrometheusMetrics) StopRequest() {
 	m.busyThreads.Dec()
 }
 
-func (m *PrometheusMetrics) StopWorkerRequest(name string, duration time.Duration) {
+func (m *PrometheusMetrics) StopWorkerRequest(server, name string, duration time.Duration) {
 	if m.workerRequestTime == nil {
 		return
 	}
 
-	m.workerRequestCount.WithLabelValues(name).Inc()
-	m.busyWorkers.WithLabelValues(name).Dec()
-	m.workerRequestTime.WithLabelValues(name).Add(duration.Seconds())
+	m.workerRequestCount.WithLabelValues(server, name).Inc()
+	m.busyWorkers.WithLabelValues(server, name).Dec()
+	m.workerRequestTime.WithLabelValues(server, name).Add(duration.Seconds())
 }
 
-func (m *PrometheusMetrics) StartWorkerRequest(name string) {
+func (m *PrometheusMetrics) StartWorkerRequest(server, name string) {
 	if m.busyWorkers == nil {
 		return
 	}
-	m.busyWorkers.WithLabelValues(name).Inc()
+	m.busyWorkers.WithLabelValues(server, name).Inc()
 }
 
-func (m *PrometheusMetrics) QueuedWorkerRequest(name string) {
+func (m *PrometheusMetrics) QueuedWorkerRequest(server, name string) {
 	if m.workerQueueDepth == nil {
 		return
 	}
-	m.workerQueueDepth.WithLabelValues(name).Inc()
+	m.workerQueueDepth.WithLabelValues(server, name).Inc()
 }
 
-func (m *PrometheusMetrics) DequeuedWorkerRequest(name string) {
+func (m *PrometheusMetrics) DequeuedWorkerRequest(server, name string) {
 	if m.workerQueueDepth == nil {
 		return
 	}
-	m.workerQueueDepth.WithLabelValues(name).Dec()
+	m.workerQueueDepth.WithLabelValues(server, name).Dec()
 }
 
 func (m *PrometheusMetrics) QueuedRequest() {

metrics_test.go

@@ -32,7 +32,7 @@ func TestPrometheusMetrics_TotalWorkers(t *testing.T) {
 	require.Nil(t, m.workerRequestTime)
 	require.Nil(t, m.workerRequestCount)
 
-	m.TotalWorkers("test_worker", 2)
+	m.TotalWorkers("test_server", "test_worker", 2)
 
 	require.NotNil(t, m.totalWorkers)
 	require.NotNil(t, m.busyWorkers)
@@ -45,8 +45,8 @@ func TestPrometheusMetrics_TotalWorkers(t *testing.T) {
 
 func TestPrometheusMetrics_StopWorkerRequest(t *testing.T) {
 	m := createPrometheusMetrics()
-	m.TotalWorkers("test_worker", 2)
-	m.StopWorkerRequest("test_worker", 2*time.Second)
+	m.TotalWorkers("test_server", "test_worker", 2)
+	m.StopWorkerRequest("test_server", "test_worker", 2*time.Second)
 
 	inputs := []struct {
 		name     string
@@ -62,7 +62,7 @@ func TestPrometheusMetrics_StopWorkerRequest(t *testing.T) {
 				# TYPE frankenphp_worker_request_count counter
 			`,
 			expect: `
-				frankenphp_worker_request_count{worker="test_worker"} 1
+				frankenphp_worker_request_count{server="test_server",worker="test_worker"} 1
 			`,
 		},
 		{
@@ -73,7 +73,7 @@ func TestPrometheusMetrics_StopWorkerRequest(t *testing.T) {
 				# TYPE frankenphp_busy_workers gauge
 			`,
 			expect: `
-				frankenphp_busy_workers{worker="test_worker"} -1
+				frankenphp_busy_workers{server="test_server",worker="test_worker"} -1
 			`,
 		},
 		{
@@ -84,7 +84,7 @@ func TestPrometheusMetrics_StopWorkerRequest(t *testing.T) {
 				# TYPE frankenphp_worker_request_time counter
 			`,
 			expect: `
-				frankenphp_worker_request_time{worker="test_worker"} 2
+				frankenphp_worker_request_time{server="test_server",worker="test_worker"} 2
 			`,
 		},
 	}
@@ -99,8 +99,8 @@ func TestPrometheusMetrics_StopWorkerRequest(t *testing.T) {
 
 func TestPrometheusMetrics_StartWorkerRequest(t *testing.T) {
 	m := createPrometheusMetrics()
-	m.TotalWorkers("test_worker", 2)
-	m.StartWorkerRequest("test_worker")
+	m.TotalWorkers("test_server", "test_worker", 2)
+	m.StartWorkerRequest("test_server", "test_worker")
 
 	inputs := []struct {
 		name     string
@@ -116,7 +116,7 @@ func TestPrometheusMetrics_StartWorkerRequest(t *testing.T) {
 				# TYPE frankenphp_busy_workers gauge
 			`,
 			expect: `
-				frankenphp_busy_workers{worker="test_worker"} 1
+				frankenphp_busy_workers{server="test_server",worker="test_worker"} 1
 			`,
 		},
 	}
@@ -131,8 +131,8 @@ func TestPrometheusMetrics_StartWorkerRequest(t *testing.T) {
 
 func TestPrometheusMetrics_TestStopReasonCrash(t *testing.T) {
 	m := createPrometheusMetrics()
-	m.TotalWorkers("test_worker", 2)
-	m.StopWorker("test_worker", StopReasonCrash)
+	m.TotalWorkers("test_server", "test_worker", 2)
+	m.StopWorker("test_server", "test_worker", StopReasonCrash)
 
 	inputs := []struct {
 		name     string
@@ -159,7 +159,7 @@ func TestPrometheusMetrics_TestStopReasonCrash(t *testing.T) {
 				# TYPE frankenphp_total_workers gauge
 			`,
 			expect: `
-				frankenphp_total_workers{worker="test_worker"} -1
+				frankenphp_total_workers{server="test_server",worker="test_worker"} -1
 			`,
 		},
 		{
@@ -170,7 +170,7 @@ func TestPrometheusMetrics_TestStopReasonCrash(t *testing.T) {
 				# TYPE frankenphp_ready_workers gauge
 			`,
 			expect: `
-				frankenphp_ready_workers{worker="test_worker"} -1
+				frankenphp_ready_workers{server="test_server",worker="test_worker"} -1
 			`,
 		},
 		{
@@ -181,7 +181,7 @@ func TestPrometheusMetrics_TestStopReasonCrash(t *testing.T) {
 				# TYPE frankenphp_worker_crashes counter
 			`,
 			expect: `
-				frankenphp_worker_crashes{worker="test_worker"} 1
+				frankenphp_worker_crashes{server="test_server",worker="test_worker"} 1
 			`,
 		},
 	}

threadbackgroundworker.go

@@ -136,7 +136,7 @@ func (handler *backgroundWorkerThread) setupScript() {
 	if handler.runtimeName == "" {
 		handler.runtimeName = handler.worker.name
 	}
-	metrics.StartWorker(handler.runtimeName)
+	metrics.StartWorker(ScopeLabel(handler.worker.backgroundScope), handler.runtimeName)
 
 	opts := append([]RequestOption(nil), handler.worker.requestOptions...)
 	C.frankenphp_set_background_worker(C._Bool(true))
@@ -185,9 +185,11 @@ func (handler *backgroundWorkerThread) afterScriptExecution(exitStatus int) {
 		return
 	}
 
+	server := ScopeLabel(worker.backgroundScope)
+
 	// Cooperative exit: re-run, reset backoff.
 	if exitStatus == 0 {
-		metrics.StopWorker(runtimeName, StopReasonRestart)
+		metrics.StopWorker(server, runtimeName, StopReasonRestart)
 
 		if globalLogger.Enabled(globalCtx, slog.LevelDebug) {
 			globalLogger.LogAttrs(globalCtx, slog.LevelDebug, "restarting background worker", slog.String("worker", runtimeName), slog.Int("thread", handler.thread.threadIndex), slog.Int("exit_status", exitStatus))
@@ -197,7 +199,7 @@ func (handler *backgroundWorkerThread) afterScriptExecution(exitStatus int) {
 		return
 	}
 
-	metrics.StopWorker(runtimeName, StopReasonCrash)
+	metrics.StopWorker(server, runtimeName, StopReasonCrash)
 
 	// Pre-readiness crash: stash metadata for a timing-out ensure(). Post-
 	// readiness crashes don't update this (the worker already signalled OK).