feat(metrics)!: add "server" label to per-worker series

Workers declared in distinct php_server blocks can share a name now that
the per-scope routing landed; without a second label dimension, their
metric series collide on the existing "worker" label. This adds a sibling
"server" label (resolved via ScopeLabel(w.backgroundScope) at every call
site) so each (server, worker) pair stays on its own series.

BREAKING CHANGE: every Metrics interface method that took (name string)
now takes (server, name string). Embedders implementing frankenphp.Metrics
need to widen their signatures; PrometheusMetrics and the null impl are
updated in-tree. Mirrors the shape of php#1376 (which introduced the
"worker" label the same way).

Author: nicolas-grekas

caddy/caddy_test.go

@@ -828,19 +828,19 @@ func TestWorkerMetrics(t *testing.T) {
 
 	# HELP frankenphp_busy_workers Number of busy PHP workers for this worker
 	# TYPE frankenphp_busy_workers gauge
-	frankenphp_busy_workers{worker="` + workerName + `"} 0
+	frankenphp_busy_workers{server="0",worker="` + workerName + `"} 0
 
 	# HELP frankenphp_total_workers Total number of PHP workers for this worker
 	# TYPE frankenphp_total_workers gauge
-	frankenphp_total_workers{worker="` + workerName + `"} 2
+	frankenphp_total_workers{server="0",worker="` + workerName + `"} 2
 
 	# HELP frankenphp_worker_request_count
 	# TYPE frankenphp_worker_request_count counter
-	frankenphp_worker_request_count{worker="` + workerName + `"} 10
+	frankenphp_worker_request_count{server="0",worker="` + workerName + `"} 10
 
 	# HELP frankenphp_ready_workers Running workers that have successfully called frankenphp_handle_request at least once
 	# TYPE frankenphp_ready_workers gauge
-	frankenphp_ready_workers{worker="` + workerName + `"} 2
+	frankenphp_ready_workers{server="0",worker="` + workerName + `"} 2
 	`
 
 	ctx := caddy.ActiveContext()
@@ -922,19 +922,19 @@ func TestNamedWorkerMetrics(t *testing.T) {
 
 	# HELP frankenphp_busy_workers Number of busy PHP workers for this worker
         # TYPE frankenphp_busy_workers gauge
-        frankenphp_busy_workers{worker="my_app"} 0
+        frankenphp_busy_workers{server="0",worker="my_app"} 0
 
 	# HELP frankenphp_total_workers Total number of PHP workers for this worker
 	# TYPE frankenphp_total_workers gauge
-	frankenphp_total_workers{worker="my_app"} 2
+	frankenphp_total_workers{server="0",worker="my_app"} 2
 
 	# HELP frankenphp_worker_request_count
 	# TYPE frankenphp_worker_request_count counter
-	frankenphp_worker_request_count{worker="my_app"} 10
+	frankenphp_worker_request_count{server="0",worker="my_app"} 10
 
 	# HELP frankenphp_ready_workers Running workers that have successfully called frankenphp_handle_request at least once
 	# TYPE frankenphp_ready_workers gauge
-	frankenphp_ready_workers{worker="my_app"} 2
+	frankenphp_ready_workers{server="0",worker="my_app"} 2
 	`
 
 	ctx := caddy.ActiveContext()
@@ -1018,19 +1018,19 @@ func TestAutoWorkerConfig(t *testing.T) {
 
 	# HELP frankenphp_busy_workers Number of busy PHP workers for this worker
 	# TYPE frankenphp_busy_workers gauge
-	frankenphp_busy_workers{worker="` + workerName + `"} 0
+	frankenphp_busy_workers{server="0",worker="` + workerName + `"} 0
 
 	# HELP frankenphp_total_workers Total number of PHP workers for this worker
 	# TYPE frankenphp_total_workers gauge
-	frankenphp_total_workers{worker="` + workerName + `"} ` + workers + `
+	frankenphp_total_workers{server="0",worker="` + workerName + `"} ` + workers + `
 
 	# HELP frankenphp_worker_request_count
 	# TYPE frankenphp_worker_request_count counter
-	frankenphp_worker_request_count{worker="` + workerName + `"} 10
+	frankenphp_worker_request_count{server="0",worker="` + workerName + `"} 10
 
 	# HELP frankenphp_ready_workers Running workers that have successfully called frankenphp_handle_request at least once
 	# TYPE frankenphp_ready_workers gauge
-	frankenphp_ready_workers{worker="` + workerName + `"} ` + workers + `
+	frankenphp_ready_workers{server="0",worker="` + workerName + `"} ` + workers + `
 	`
 
 	ctx := caddy.ActiveContext()
@@ -1284,7 +1284,7 @@ func TestMaxWaitTimeWorker(t *testing.T) {
 
 	expectedMetrics := `
 	# TYPE frankenphp_worker_queue_depth gauge
-	frankenphp_worker_queue_depth{worker="service"} 0
+	frankenphp_worker_queue_depth{server="0",worker="service"} 0
 	`
 
 	ctx := caddy.ActiveContext()
@@ -1385,21 +1385,21 @@ func TestMultiWorkersMetrics(t *testing.T) {
 
 	# HELP frankenphp_busy_workers Number of busy PHP workers for this worker
 	# TYPE frankenphp_busy_workers gauge
-	frankenphp_busy_workers{worker="service1"} 0
+	frankenphp_busy_workers{server="0",worker="service1"} 0
 
 	# HELP frankenphp_total_workers Total number of PHP workers for this worker
 	# TYPE frankenphp_total_workers gauge
-	frankenphp_total_workers{worker="service1"} 2
-	frankenphp_total_workers{worker="service2"} 3
+	frankenphp_total_workers{server="0",worker="service1"} 2
+	frankenphp_total_workers{server="0",worker="service2"} 3
 
 	# HELP frankenphp_worker_request_count
 	# TYPE frankenphp_worker_request_count counter
-	frankenphp_worker_request_count{worker="service1"} 10
+	frankenphp_worker_request_count{server="0",worker="service1"} 10
 
 	# HELP frankenphp_ready_workers Running workers that have successfully called frankenphp_handle_request at least once
 	# TYPE frankenphp_ready_workers gauge
-	frankenphp_ready_workers{worker="service1"} 2
-	frankenphp_ready_workers{worker="service2"} 3
+	frankenphp_ready_workers{server="0",worker="service1"} 2
+	frankenphp_ready_workers{server="0",worker="service2"} 3
 	`
 
 	ctx := caddy.ActiveContext()
@@ -1552,10 +1552,10 @@ func TestWorkerRestart(t *testing.T) {
 	expectedMetrics := `
 	# HELP frankenphp_ready_workers Running workers that have successfully called frankenphp_handle_request at least once
 	# TYPE frankenphp_ready_workers gauge
-	frankenphp_ready_workers{worker="service"} 1
+	frankenphp_ready_workers{server="0",worker="service"} 1
 	# HELP frankenphp_total_workers Total number of PHP workers for this worker
 	# TYPE frankenphp_total_workers gauge
-	frankenphp_total_workers{worker="service"} 1
+	frankenphp_total_workers{server="0",worker="service"} 1
 	`
 
 	require.NoError(t,
@@ -1580,13 +1580,13 @@ func TestWorkerRestart(t *testing.T) {
 	expectedMetrics = `
 	# HELP frankenphp_ready_workers Running workers that have successfully called frankenphp_handle_request at least once
 	# TYPE frankenphp_ready_workers gauge
-	frankenphp_ready_workers{worker="service"} 1
+	frankenphp_ready_workers{server="0",worker="service"} 1
 	# HELP frankenphp_total_workers Total number of PHP workers for this worker
 	# TYPE frankenphp_total_workers gauge
-	frankenphp_total_workers{worker="service"} 1
+	frankenphp_total_workers{server="0",worker="service"} 1
 	# HELP frankenphp_worker_restarts Number of PHP worker restarts for this worker
 	# TYPE frankenphp_worker_restarts counter
-	frankenphp_worker_restarts{worker="service"} 3
+	frankenphp_worker_restarts{server="0",worker="service"} 3
 	`
 
 	require.NoError(t,

docs/metrics.md

@@ -13,13 +13,13 @@ When [Caddy metrics](https://caddyserver.com/docs/metrics) are enabled, FrankenP
 - `frankenphp_total_threads`: The total number of PHP threads.
 - `frankenphp_busy_threads`: The number of PHP threads currently processing a request (running workers always consume a thread).
 - `frankenphp_queue_depth`: The number of regular queued requests
-- `frankenphp_total_workers{worker="[worker_name]"}`: The total number of workers.
-- `frankenphp_busy_workers{worker="[worker_name]"}`: The number of workers currently processing a request.
-- `frankenphp_worker_request_time{worker="[worker_name]"}`: The time spent processing requests by all workers.
-- `frankenphp_worker_request_count{worker="[worker_name]"}`: The number of requests processed by all workers.
-- `frankenphp_ready_workers{worker="[worker_name]"}`: The number of workers that have called `frankenphp_handle_request` at least once.
-- `frankenphp_worker_crashes{worker="[worker_name]"}`: The number of times a worker has unexpectedly terminated.
-- `frankenphp_worker_restarts{worker="[worker_name]"}`: The number of times a worker has been deliberately restarted.
-- `frankenphp_worker_queue_depth{worker="[worker_name]"}`: The number of queued requests.
+- `frankenphp_total_workers{server="[server]",worker="[worker_name]"}`: The total number of workers.
+- `frankenphp_busy_workers{server="[server]",worker="[worker_name]"}`: The number of workers currently processing a request.
+- `frankenphp_worker_request_time{server="[server]",worker="[worker_name]"}`: The time spent processing requests by all workers.
+- `frankenphp_worker_request_count{server="[server]",worker="[worker_name]"}`: The number of requests processed by all workers.
+- `frankenphp_ready_workers{server="[server]",worker="[worker_name]"}`: The number of workers that have called `frankenphp_handle_request` at least once.
+- `frankenphp_worker_crashes{server="[server]",worker="[worker_name]"}`: The number of times a worker has unexpectedly terminated.
+- `frankenphp_worker_restarts{server="[server]",worker="[worker_name]"}`: The number of times a worker has been deliberately restarted.
+- `frankenphp_worker_queue_depth{server="[server]",worker="[worker_name]"}`: The number of queued requests.
 
-For worker metrics, the `[worker_name]` placeholder is replaced by the worker name in the Caddyfile, otherwise the absolute path of the worker file will be used.
+For worker metrics, the `[worker_name]` placeholder is replaced by the worker name in the Caddyfile, otherwise the absolute path of the worker file will be used. The `[server]` label identifies the `php_server` block that declared the worker; the Caddy module resolves it to the first host of the route's host matcher (e.g. `api.example.com`), falling back to the user-set Caddy server name and finally to the first listener address. Same-named workers in distinct `php_server` blocks therefore stay on distinct series.

frankenphp.go

@@ -182,15 +182,15 @@ func calculateMaxThreads(opt *opt) (numWorkers int, _ error) {
 			// Register the expected worker count for metrics too: without
 			// this, a bg-worker-only deployment never initialises
 			// totalWorkers, and StartWorker calls become silent no-ops.
-			metrics.TotalWorkers(w.name, extra)
+			metrics.TotalWorkers(ScopeLabel(w.backgroundScope), w.name, extra)
 			continue
 		}
 
 		if w.num <= 0 {
 			// https://github.com/php/frankenphp/issues/126
 			opt.workers[i].num = maxProcs
 		}
-		metrics.TotalWorkers(w.name, w.num)
+		metrics.TotalWorkers(ScopeLabel(w.backgroundScope), w.name, w.num)
 
 		numWorkers += opt.workers[i].num
 

metrics.go

@@ -16,44 +16,49 @@ const (
 
 type StopReason int
 
+// Metrics is the worker-level instrumentation surface. Every method that
+// identifies a specific worker takes a (server, name) pair: server is the
+// per-php_server label resolved via ScopeLabel, name is the worker name.
+// The pair is what disambiguates same-named workers declared in distinct
+// php_server blocks.
 type Metrics interface {
 	// StartWorker collects started workers
-	StartWorker(name string)
+	StartWorker(server, name string)
 	// ReadyWorker collects ready workers
-	ReadyWorker(name string)
+	ReadyWorker(server, name string)
 	// StopWorker collects stopped workers
-	StopWorker(name string, reason StopReason)
+	StopWorker(server, name string, reason StopReason)
 	// TotalWorkers collects expected workers
-	TotalWorkers(name string, num int)
+	TotalWorkers(server, name string, num int)
 	// TotalThreads collects total threads
 	TotalThreads(num int)
 	// StartRequest collects started requests
 	StartRequest()
 	// StopRequest collects stopped requests
 	StopRequest()
 	// StopWorkerRequest collects stopped worker requests
-	StopWorkerRequest(name string, duration time.Duration)
+	StopWorkerRequest(server, name string, duration time.Duration)
 	// StartWorkerRequest collects started worker requests
-	StartWorkerRequest(name string)
+	StartWorkerRequest(server, name string)
 	Shutdown()
-	QueuedWorkerRequest(name string)
-	DequeuedWorkerRequest(name string)
+	QueuedWorkerRequest(server, name string)
+	DequeuedWorkerRequest(server, name string)
 	QueuedRequest()
 	DequeuedRequest()
 }
 
 type nullMetrics struct{}
 
-func (n nullMetrics) StartWorker(string) {
+func (n nullMetrics) StartWorker(string, string) {
 }
 
-func (n nullMetrics) ReadyWorker(string) {
+func (n nullMetrics) ReadyWorker(string, string) {
 }
 
-func (n nullMetrics) StopWorker(string, StopReason) {
+func (n nullMetrics) StopWorker(string, string, StopReason) {
 }
 
-func (n nullMetrics) TotalWorkers(string, int) {
+func (n nullMetrics) TotalWorkers(string, string, int) {
 }
 
 func (n nullMetrics) TotalThreads(int) {
@@ -65,18 +70,18 @@ func (n nullMetrics) StartRequest() {
 func (n nullMetrics) StopRequest() {
 }
 
-func (n nullMetrics) StopWorkerRequest(string, time.Duration) {
+func (n nullMetrics) StopWorkerRequest(string, string, time.Duration) {
 }
 
-func (n nullMetrics) StartWorkerRequest(string) {
+func (n nullMetrics) StartWorkerRequest(string, string) {
 }
 
 func (n nullMetrics) Shutdown() {
 }
 
-func (n nullMetrics) QueuedWorkerRequest(string) {}
+func (n nullMetrics) QueuedWorkerRequest(string, string) {}
 
-func (n nullMetrics) DequeuedWorkerRequest(string) {}
+func (n nullMetrics) DequeuedWorkerRequest(string, string) {}
 
 func (n nullMetrics) QueuedRequest()   {}
 func (n nullMetrics) DequeuedRequest() {}
@@ -97,54 +102,54 @@ type PrometheusMetrics struct {
 	mu                 sync.Mutex
 }
 
-func (m *PrometheusMetrics) StartWorker(name string) {
+func (m *PrometheusMetrics) StartWorker(server, name string) {
 	m.busyThreads.Inc()
 
 	// tests do not register workers before starting them
 	if m.totalWorkers == nil {
 		return
 	}
 
-	m.totalWorkers.WithLabelValues(name).Inc()
+	m.totalWorkers.WithLabelValues(server, name).Inc()
 }
 
-func (m *PrometheusMetrics) ReadyWorker(name string) {
+func (m *PrometheusMetrics) ReadyWorker(server, name string) {
 	if m.totalWorkers == nil {
 		return
 	}
 
-	m.readyWorkers.WithLabelValues(name).Inc()
+	m.readyWorkers.WithLabelValues(server, name).Inc()
 }
 
-func (m *PrometheusMetrics) StopWorker(name string, reason StopReason) {
+func (m *PrometheusMetrics) StopWorker(server, name string, reason StopReason) {
 	m.busyThreads.Dec()
 
 	// tests do not register workers before starting them
 	if m.totalWorkers == nil {
 		return
 	}
 
-	m.totalWorkers.WithLabelValues(name).Dec()
+	m.totalWorkers.WithLabelValues(server, name).Dec()
 
 	// only decrement readyWorkers if the worker actually reached frankenphp_handle_request
 	if reason != StopReasonBootFailure {
-		m.readyWorkers.WithLabelValues(name).Dec()
+		m.readyWorkers.WithLabelValues(server, name).Dec()
 	}
 
 	switch reason {
 	case StopReasonCrash, StopReasonBootFailure:
-		m.workerCrashes.WithLabelValues(name).Inc()
+		m.workerCrashes.WithLabelValues(server, name).Inc()
 	case StopReasonRestart:
-		m.workerRestarts.WithLabelValues(name).Inc()
+		m.workerRestarts.WithLabelValues(server, name).Inc()
 	}
 }
 
-func (m *PrometheusMetrics) TotalWorkers(string, int) {
+func (m *PrometheusMetrics) TotalWorkers(string, string, int) {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 
 	const ns, sub = "frankenphp", "worker"
-	basicLabels := []string{"worker"}
+	basicLabels := []string{"server", "worker"}
 
 	if m.totalWorkers == nil {
 		m.totalWorkers = prometheus.NewGaugeVec(prometheus.GaugeOpts{
@@ -257,35 +262,35 @@ func (m *PrometheusMetrics) StopRequest() {
 	m.busyThreads.Dec()
 }
 
-func (m *PrometheusMetrics) StopWorkerRequest(name string, duration time.Duration) {
+func (m *PrometheusMetrics) StopWorkerRequest(server, name string, duration time.Duration) {
 	if m.workerRequestTime == nil {
 		return
 	}
 
-	m.workerRequestCount.WithLabelValues(name).Inc()
-	m.busyWorkers.WithLabelValues(name).Dec()
-	m.workerRequestTime.WithLabelValues(name).Add(duration.Seconds())
+	m.workerRequestCount.WithLabelValues(server, name).Inc()
+	m.busyWorkers.WithLabelValues(server, name).Dec()
+	m.workerRequestTime.WithLabelValues(server, name).Add(duration.Seconds())
 }
 
-func (m *PrometheusMetrics) StartWorkerRequest(name string) {
+func (m *PrometheusMetrics) StartWorkerRequest(server, name string) {
 	if m.busyWorkers == nil {
 		return
 	}
-	m.busyWorkers.WithLabelValues(name).Inc()
+	m.busyWorkers.WithLabelValues(server, name).Inc()
 }
 
-func (m *PrometheusMetrics) QueuedWorkerRequest(name string) {
+func (m *PrometheusMetrics) QueuedWorkerRequest(server, name string) {
 	if m.workerQueueDepth == nil {
 		return
 	}
-	m.workerQueueDepth.WithLabelValues(name).Inc()
+	m.workerQueueDepth.WithLabelValues(server, name).Inc()
 }
 
-func (m *PrometheusMetrics) DequeuedWorkerRequest(name string) {
+func (m *PrometheusMetrics) DequeuedWorkerRequest(server, name string) {
 	if m.workerQueueDepth == nil {
 		return
 	}
-	m.workerQueueDepth.WithLabelValues(name).Dec()
+	m.workerQueueDepth.WithLabelValues(server, name).Dec()
 }
 
 func (m *PrometheusMetrics) QueuedRequest() {