Expose more puma metrics in Prometheus (#4423)

Author: Samze

lib/cloud_controller/metrics/periodic_updater.rb

@@ -165,7 +165,10 @@ def update_webserver_stats
           index: worker_status[:index],
           pid: worker_status[:pid],
           thread_count: worker_status[:last_status][:running],
-          backlog: worker_status[:last_status][:backlog]
+          backlog: worker_status[:last_status][:backlog],
+          pool_capacity: worker_status[:last_status][:pool_capacity],
+          busy_threads: worker_status[:last_status][:busy_threads],
+          requests_count: worker_status[:last_status][:requests_count]
         }
       end
       @prometheus_updater.update_webserver_stats_puma(worker_count, worker_stats)

lib/cloud_controller/metrics/prometheus_updater.rb

@@ -54,7 +54,10 @@ def self.allow_pid_label
       { type: :gauge, name: :cc_puma_worker_count, docstring: 'Puma worker count', aggregation: :most_recent },
       { type: :gauge, name: :cc_puma_worker_started_at, docstring: 'Puma worker: started_at', labels: %i[index pid], aggregation: :most_recent },
       { type: :gauge, name: :cc_puma_worker_thread_count, docstring: 'Puma worker: thread count', labels: %i[index pid], aggregation: :most_recent },
-      { type: :gauge, name: :cc_puma_worker_backlog, docstring: 'Puma worker: backlog', labels: %i[index pid], aggregation: :most_recent }
+      { type: :gauge, name: :cc_puma_worker_backlog, docstring: 'Puma worker: backlog', labels: %i[index pid], aggregation: :most_recent },
+      { type: :gauge, name: :cc_puma_worker_pool_capacity, docstring: 'Puma worker: pool capacity', labels: %i[index pid], aggregation: :most_recent },
+      { type: :gauge, name: :cc_puma_worker_requests_count, docstring: 'Puma worker: requests count', labels: %i[index pid], aggregation: :most_recent },
+      { type: :gauge, name: :cc_puma_worker_busy_threads, docstring: 'Puma worker: busy threads', labels: %i[index pid], aggregation: :most_recent }
     ].freeze
 
     DB_CONNECTION_POOL_METRICS = [

spec/unit/lib/cloud_controller/metrics/periodic_updater_spec.rb

@@ -705,8 +705,8 @@ module VCAP::CloudController::Metrics
           stats_hash = {
             booted_workers: 2,
             worker_status: [
-              { started_at: '2023-11-29T13:15:05Z', index: 0, pid: 123, last_status: { running: 1, backlog: 0 } },
-              { started_at: '2023-11-29T13:15:10Z', index: 1, pid: 234, last_status: { running: 2, backlog: 1 } }
+              { started_at: '2023-11-29T13:15:05Z', index: 0, pid: 123, last_status: { running: 1, backlog: 0, busy_threads: 0, pool_capacity: 1, requests_count: 9 } },
+              { started_at: '2023-11-29T13:15:10Z', index: 1, pid: 234, last_status: { running: 2, backlog: 1, busy_threads: 1, pool_capacity: 2, requests_count: 10 } }
             ]
           }
           allow(Puma).to receive(:stats_hash).and_return(stats_hash)
@@ -715,8 +715,8 @@ module VCAP::CloudController::Metrics
 
           expected_worker_count = 2
           expected_worker_stats = [
-            { started_at: 1_701_263_705, index: 0, pid: 123, thread_count: 1, backlog: 0 },
-            { started_at: 1_701_263_710, index: 1, pid: 234, thread_count: 2, backlog: 1 }
+            { started_at: 1_701_263_705, index: 0, pid: 123, thread_count: 1, backlog: 0, busy_threads: 0, pool_capacity: 1, requests_count: 9 },
+            { started_at: 1_701_263_710, index: 1, pid: 234, thread_count: 2, backlog: 1, busy_threads: 1, pool_capacity: 2, requests_count: 10 }
           ]
           expect(prometheus_updater).to have_received(:update_webserver_stats_puma).with(expected_worker_count, expected_worker_stats)
         end

spec/unit/lib/cloud_controller/metrics/prometheus_updater_spec.rb

@@ -206,8 +206,8 @@ module VCAP::CloudController::Metrics
       it 'contains Puma stats' do
         worker_count = 2
         worker_stats = [
-          { started_at: 1_701_263_705, index: 0, pid: 123, thread_count: 1, backlog: 0 },
-          { started_at: 1_701_263_710, index: 1, pid: 234, thread_count: 2, backlog: 1 }
+          { started_at: 1_701_263_705, index: 0, pid: 123, thread_count: 1, backlog: 0, busy_threads: 0, pool_capacity: 1, requests_count: 9 },
+          { started_at: 1_701_263_710, index: 1, pid: 234, thread_count: 2, backlog: 1, busy_threads: 1, pool_capacity: 2, requests_count: 10 }
         ]
 
         updater.update_webserver_stats_puma(worker_count, worker_stats)
@@ -226,6 +226,18 @@ module VCAP::CloudController::Metrics
         metric = prom_client.metrics.find { |m| m.name == :cc_puma_worker_backlog }
         expect(metric.get(labels: { index: 0, pid: 123 })).to eq(0)
         expect(metric.get(labels: { index: 1, pid: 234 })).to eq(1)
+
+        metric = prom_client.metrics.find { |m| m.name == :cc_puma_worker_pool_capacity }
+        expect(metric.get(labels: { index: 0, pid: 123 })).to eq(1)
+        expect(metric.get(labels: { index: 1, pid: 234 })).to eq(2)
+
+        metric = prom_client.metrics.find { |m| m.name == :cc_puma_worker_requests_count }
+        expect(metric.get(labels: { index: 0, pid: 123 })).to eq(9)
+        expect(metric.get(labels: { index: 1, pid: 234 })).to eq(10)
+
+        metric = prom_client.metrics.find { |m| m.name == :cc_puma_worker_busy_threads }
+        expect(metric.get(labels: { index: 0, pid: 123 })).to eq(0)
+        expect(metric.get(labels: { index: 1, pid: 234 })).to eq(1)
       end
     end