|
| 1 | +require 'prometheus/client' |
| 2 | + |
| 3 | +module VCAP::CloudController::Metrics |
| 4 | + class PrometheusUpdater |
| 5 | + def initialize(registry=Prometheus::Client.registry) |
| 6 | + @registry = registry |
| 7 | + end |
| 8 | + |
| 9 | + def update_gauge_metric(metric, value, message) |
| 10 | + unless @registry.exist?(metric) |
| 11 | + @registry.gauge(metric, docstring: message) |
| 12 | + end |
| 13 | + @registry.get(metric).set(value) |
| 14 | + end |
| 15 | + |
| 16 | + def increment_gauge_metric(metric, message) |
| 17 | + unless @registry.exist?(metric) |
| 18 | + @registry.gauge(metric, docstring: message) |
| 19 | + end |
| 20 | + @registry.get(metric).increment |
| 21 | + end |
| 22 | + |
| 23 | + def decrement_gauge_metric(metric, message) |
| 24 | + unless @registry.exist?(metric) |
| 25 | + @registry.gauge(metric, docstring: message) |
| 26 | + end |
| 27 | + @registry.get(metric).decrement |
| 28 | + end |
| 29 | + |
| 30 | + def increment_counter_metric(metric, message) |
| 31 | + unless @registry.exist?(metric) |
| 32 | + @registry.counter(metric, docstring: message) |
| 33 | + end |
| 34 | + @registry.get(metric).increment |
| 35 | + end |
| 36 | + |
| 37 | + def update_histogram_metric(metric, value, message, buckets) |
| 38 | + unless @registry.exist?(metric) |
| 39 | + @registry.histogram(metric, buckets: buckets, docstring: message) |
| 40 | + end |
| 41 | + @registry.get(metric).observe(value) |
| 42 | + end |
| 43 | + |
| 44 | + def update_summary_metric(metric, value, message) |
| 45 | + unless @registry.exist?(metric) |
| 46 | + @registry.summary(metric, docstring: message) |
| 47 | + end |
| 48 | + @registry.get(metric).observe(value) |
| 49 | + end |
| 50 | + |
| 51 | + def update_deploying_count(deploying_count) |
| 52 | + update_gauge_metric(:cc_deployments_deploying, deploying_count, 'Number of in progress deployments') |
| 53 | + end |
| 54 | + |
| 55 | + def update_user_count(user_count) |
| 56 | + update_gauge_metric(:cc_total_users, user_count, 'Number of users') |
| 57 | + end |
| 58 | + |
| 59 | + def update_job_queue_length(pending_job_count_by_queue, total) |
| 60 | + pending_job_count_by_queue.each do |key, value| |
| 61 | + metric_key = :"cc_job_queue_length_#{key.to_s.underscore}" |
| 62 | + update_gauge_metric(metric_key, value, docstring: "Job queue length for worker #{key}") |
| 63 | + end |
| 64 | + |
| 65 | + update_gauge_metric(:cc_job_queue_length_total, total, 'Total job queue length') |
| 66 | + end |
| 67 | + |
| 68 | + def update_thread_info(thread_info) |
| 69 | + update_gauge_metric(:cc_thread_info_thread_count, thread_info[:thread_count], 'Thread count') |
| 70 | + update_gauge_metric(:cc_thread_info_event_machine_connection_count, thread_info[:event_machine][:connection_count], 'Event Machine connection count') |
| 71 | + update_gauge_metric(:cc_thread_info_event_machine_threadqueue_size, thread_info[:event_machine][:threadqueue][:size], 'EventMachine thread queue size') |
| 72 | + update_gauge_metric(:cc_thread_info_event_machine_threadqueue_num_waiting, thread_info[:event_machine][:threadqueue][:num_waiting], 'EventMachine num waiting in thread') |
| 73 | + update_gauge_metric(:cc_thread_info_event_machine_resultqueue_size, thread_info[:event_machine][:resultqueue][:size], 'EventMachine queue size') |
| 74 | + update_gauge_metric(:cc_thread_info_event_machine_resultqueue_num_waiting, thread_info[:event_machine][:resultqueue][:num_waiting], 'EventMachine requests waiting in queue') |
| 75 | + end |
| 76 | + |
| 77 | + def update_failed_job_count(failed_jobs_by_queue, total) |
| 78 | + failed_jobs_by_queue.each do |key, value| |
| 79 | + metric_key = :"cc_failed_job_count_#{key.to_s.underscore}" |
| 80 | + update_gauge_metric(metric_key, value, "Failed jobs for worker #{key}") |
| 81 | + end |
| 82 | + |
| 83 | + update_gauge_metric(:cc_failed_job_count_total, total, 'Total failed jobs') |
| 84 | + end |
| 85 | + |
| 86 | + def update_vitals(vitals) |
| 87 | + vitals.each do |key, value| |
| 88 | + metric_key = :"cc_vitals_#{key.to_s.underscore}" |
| 89 | + update_gauge_metric(metric_key, value, "CloudController Vitals: #{key}") |
| 90 | + end |
| 91 | + end |
| 92 | + |
| 93 | + def update_log_counts(counts) |
| 94 | + counts.each do |key, value| |
| 95 | + metric_key = :"cc_log_count_#{key.to_s.underscore}" |
| 96 | + update_gauge_metric(metric_key, value, "Log count for log level '#{key}'") |
| 97 | + end |
| 98 | + end |
| 99 | + |
| 100 | + def update_task_stats(total_running_tasks, total_memory_in_mb) |
| 101 | + update_gauge_metric(:cc_tasks_running_count, total_running_tasks, 'Total running tasks') |
| 102 | + update_gauge_metric(:cc_tasks_running_memory_in_mb, total_memory_in_mb, 'Total memory consumed by running tasks') |
| 103 | + end |
| 104 | + |
| 105 | + def update_synced_invalid_lrps(lrp_count) |
| 106 | + update_gauge_metric(:cc_diego_sync_invalid_desired_lrps, lrp_count, 'Invalid Desired LRPs') |
| 107 | + end |
| 108 | + |
| 109 | + def start_staging_request_received |
| 110 | + increment_counter_metric(:cc_staging_requested, 'Number of staging requests') |
| 111 | + end |
| 112 | + |
| 113 | + def report_staging_success_metrics(duration_ns) |
| 114 | + increment_counter_metric(:cc_staging_succeeded, 'Number of successful staging events') |
| 115 | + update_histogram_metric(:cc_staging_succeeded_duration, nanoseconds_to_milliseconds(duration_ns), 'Durations of successful staging events', duration_buckets) |
| 116 | + end |
| 117 | + |
| 118 | + def report_staging_failure_metrics(duration_ns) |
| 119 | + increment_counter_metric(:cc_staging_failed, 'Number of failed staging events') |
| 120 | + update_histogram_metric(:cc_staging_failed_duration, nanoseconds_to_milliseconds(duration_ns), 'Durations of failed staging events', duration_buckets) |
| 121 | + end |
| 122 | + |
| 123 | + def report_diego_cell_sync_duration(duration_ms) |
| 124 | + update_summary_metric(:cc_diego_sync_duration, duration_ms, 'Diego cell sync duration') |
| 125 | + update_gauge_metric(:cc_diego_sync_duration_gauge, duration_ms, 'Diego cell sync duration (gauge metric)') |
| 126 | + end |
| 127 | + |
| 128 | + def report_deployment_duration(duration_ms) |
| 129 | + update_summary_metric(:cc_deployments_update_duration, duration_ms, 'Deployment duration') |
| 130 | + update_gauge_metric(:cc_deployments_update_duration_gauge, duration_ms, 'Deployment duration (gauge metric)') |
| 131 | + end |
| 132 | + |
| 133 | + private |
| 134 | + |
| 135 | + def duration_buckets |
| 136 | + Prometheus::Client::Histogram.linear_buckets(start: 10000, width: 5000, count: 5) |
| 137 | + end |
| 138 | + |
| 139 | + def nanoseconds_to_milliseconds(time_ns) |
| 140 | + (time_ns / 1e6).to_i |
| 141 | + end |
| 142 | + end |
| 143 | +end |
0 commit comments