Add metrics about the solver-service behaviour

Author: moyodiallo

service/service.ml

@@ -6,6 +6,45 @@ module Selection = Worker.Selection
 module Store = Git_unix.Store
 module Worker_process = Internal_worker.Worker_process
 
+module Metrics = struct
+  open Prometheus
+
+  let namespace = "ocluster"
+  let subsystem = "worker"
+
+  let in_use_solver_process =
+    let help = "Number of solver-process currently used for solving" in
+    Gauge.v ~help ~namespace ~subsystem "in_use_solver_process"
+
+  let wait_queue_solver_process_pool =
+    let help = "Number of request in the queue of the solver-process pool" in
+    Gauge.v ~help ~namespace ~subsystem "wait_queue_requests"
+
+  let started_requests =
+    let help = "Number of requests sent on solver-process pool" in
+    Counter.v ~help ~namespace ~subsystem "started_requests_total"
+
+  let success_requests =
+    let help = "Number of solver_process requests success" in
+    Counter.v ~help ~namespace ~subsystem "success_requests_total"
+
+  let failed_requests =
+    let help = "Number of solver_process requests failed" in
+    Counter.v ~help ~namespace ~subsystem "failed_requests_total"
+
+  let canceled_before_started =
+    let help = "Number of solver_process requests canceled before started" in
+    Counter.v ~help ~namespace ~subsystem "canceled_before_started_total"
+
+  let canceled_after_started =
+    let help = "Number of solver_process requests canceled after started" in
+    Counter.v ~help ~namespace ~subsystem "canceled_after_started_total"
+end
+
+let update_wait_queue t =
+  Prometheus.Gauge.set Metrics.wait_queue_solver_process_pool
+    (float_of_int (Lwt_pool.wait_queue_length t))
+
 let oldest_commit = Lwt_pool.create 180 @@ fun _ -> Lwt.return_unit
 (* we are using at most 360 pipes at the same time and that's enough to keep the current
  * performance and prevent some jobs to fail because of file descriptors exceed the limit.*)
@@ -91,7 +130,10 @@ module Make (Opam_repo : Opam_repository_intf.S) = struct
 
     (* Send [request] to [worker] and read the reply. *)
     let process ~switch ~log ~id request worker =
-      if not (Lwt_switch.is_on switch) then Lwt.fail Lwt.Canceled
+      Prometheus.Gauge.inc_one Metrics.in_use_solver_process;
+      if not (Lwt_switch.is_on switch) then (
+        Prometheus.Counter.inc_one Metrics.canceled_before_started;
+        Lwt.fail Lwt.Canceled)
       else
         let request_str =
           Worker.Solve_request.to_yojson request |> Yojson.Safe.to_string
@@ -131,6 +173,7 @@ module Make (Opam_repo : Opam_repository_intf.S) = struct
            * workers's pool choosing the worker for another processing.*)
           if Lwt.state process = Lwt.Sleep then (
             Worker_process.release worker;
+            Prometheus.Counter.inc_one Metrics.canceled_after_started;
             Lwt.cancel process;
             dispose worker)
           else Lwt.return_unit )
@@ -193,9 +236,25 @@ module Make (Opam_repo : Opam_repository_intf.S) = struct
                else compatible_root_pkgs
              in
              let slice = { request with platforms = [ p ]; root_pkgs } in
-             Lwt_pool.use t (process ~switch ~log ~id slice) >>= function
-             | Error _ as e -> Lwt.return (id, e)
+             Lwt.catch
+               (fun () ->
+                 Prometheus.Counter.inc_one Metrics.started_requests;
+                 update_wait_queue t;
+                 Lwt_pool.use t (process ~switch ~log ~id slice))
+               (function
+                 | Lwt.Canceled ->
+                     Prometheus.Gauge.dec_one Metrics.in_use_solver_process;
+                     update_wait_queue t;
+                     Lwt.fail Lwt.Canceled
+                 | ex -> raise ex)
+             >>= fun result ->
+             Prometheus.Gauge.dec_one Metrics.in_use_solver_process;
+             match result with
+             | Error _ as e ->
+                 Prometheus.Counter.inc_one Metrics.failed_requests;
+                 Lwt.return (id, e)
              | Ok packages ->
+                 let _ = Prometheus.Counter.inc_one Metrics.success_requests in
                  let repo_packages =
                    packages
                    |> List.filter_map (fun pkg ->

worker/worker.ml

@@ -24,25 +24,9 @@ module Metrics = struct
     Summary.v_label ~label_name:"result" ~help ~namespace ~subsystem
       "job_time_seconds"
 
-  let docker_push_time =
-    let help = "Time uploading to Docker Hub" in
-    Summary.v ~help ~namespace ~subsystem "docker_push_time_seconds"
-
-  let docker_prune_time =
-    let help = "Time spent pruning Docker cache" in
-    Summary.v ~help ~namespace ~subsystem "docker_prune_time_seconds"
-
   let running_jobs =
     let help = "Number of jobs currently running" in
     Gauge.v ~help ~namespace ~subsystem "running_jobs"
-
-  let healthcheck_time =
-    let help = "Time to perform last healthcheck" in
-    Gauge.v ~help ~namespace ~subsystem "healthcheck_time_seconds"
-
-  let unhealthy =
-    let help = "Number of unhealthy workers" in
-    Gauge.v ~help ~namespace ~subsystem "unhealthy"
 end
 
 type build =