From a8077827a2078dc14480bbc5f817c54c4be00987 Mon Sep 17 00:00:00 2001 From: Harinn Date: Sun, 28 Jun 2026 01:40:27 +0700 Subject: [PATCH] queue-runner: introduce finished/unfinished per machine type metrics --- .../hydra-queue-runner/src/state/metrics.rs | 53 ++++++++++++++++++- .../hydra-queue-runner/src/state/mod.rs | 13 +++-- .../hydra-queue-runner/src/state/step.rs | 17 +++++- 3 files changed, 77 insertions(+), 6 deletions(-) diff --git a/subprojects/hydra-queue-runner/src/state/metrics.rs b/subprojects/hydra-queue-runner/src/state/metrics.rs index b593356e6..9781465d4 100644 --- a/subprojects/hydra-queue-runner/src/state/metrics.rs +++ b/subprojects/hydra-queue-runner/src/state/metrics.rs @@ -62,6 +62,8 @@ pub struct PromMetrics { pub disabled_per_machine_type: prometheus::IntGaugeVec, // hydraqueuerunner_machine_type_disabled pub avg_runnable_time_per_machine_type: prometheus::IntGaugeVec, // hydraqueuerunner_machine_type_avg_runnable_time pub wait_time_per_machine_type: prometheus::IntGaugeVec, // hydraqueuerunner_machine_type_wait_time + pub finished_per_machine_type: prometheus::IntCounterVec, // hydraqueuerunner_machine_type_finished + pub unfinished_per_machine_type: prometheus::IntGaugeVec, // hydraqueuerunner_machine_type_unfinished // Per-machine metrics pub machine_current_jobs: prometheus::IntGaugeVec, // hydraqueuerunner_machine_current_jobs @@ -347,6 +349,20 @@ impl PromMetrics { ), &["machine_type"], )?; + let finished_per_machine_type = prometheus::IntCounterVec::new( + prometheus::Opts::new( + "hydraqueuerunner_machine_type_finished", + "Number of completed build steps per machine type", + ), + &["machine_type"], + )?; + let unfinished_per_machine_type = prometheus::IntGaugeVec::new( + prometheus::Opts::new( + "hydraqueuerunner_machine_type_unfinished", + "Number of unfinished build steps per machine type", + ), + &["machine_type"], + )?; // Per-machine metrics let machine_current_jobs = prometheus::IntGaugeVec::new( @@ -683,6 +699,8 @@ impl PromMetrics { r.register(Box::new(disabled_per_machine_type.clone()))?; r.register(Box::new(avg_runnable_time_per_machine_type.clone()))?; r.register(Box::new(wait_time_per_machine_type.clone()))?; + r.register(Box::new(finished_per_machine_type.clone()))?; + r.register(Box::new(unfinished_per_machine_type.clone()))?; r.register(Box::new(machine_current_jobs.clone()))?; r.register(Box::new(machine_steps_done.clone()))?; r.register(Box::new(machine_total_step_time_ms.clone()))?; @@ -794,6 +812,8 @@ impl PromMetrics { disabled_per_machine_type, avg_runnable_time_per_machine_type, wait_time_per_machine_type, + finished_per_machine_type, + unfinished_per_machine_type, machine_current_jobs, machine_steps_done, machine_total_step_time_ms, @@ -912,6 +932,7 @@ impl PromMetrics { self.disabled_per_machine_type.reset(); self.avg_runnable_time_per_machine_type.reset(); self.wait_time_per_machine_type.reset(); + self.unfinished_per_machine_type.reset(); for (t, s) in state.queues.get_stats_per_queue().await { if let Ok(v) = i64::try_from(s.total_runnable) { self.runnable_per_machine_type @@ -944,6 +965,14 @@ impl PromMetrics { .set(v); } } + + for (t, count) in state.steps.get_unfinished_per_system() { + if let Ok(v) = i64::try_from(count) { + self.unfinished_per_machine_type + .with_label_values(&[&t]) + .set(v); + } + } } fn refresh_per_machine_metrics(&self, state: &Arc) { @@ -1163,20 +1192,40 @@ impl PromMetrics { .observe(wait_seconds); } - pub fn track_build_success(&self, timings: super::build::BuildTimings, total_step_time: u64) { + pub fn track_build_success( + &self, + timings: super::build::BuildTimings, + total_step_time: u64, + machine_type: Option<&str>, + ) { self.nr_builds_succeeded.inc(); self.nr_steps_done.inc(); self.nr_steps_building.sub(1); + if let Some(machine_type) = machine_type { + self.finished_per_machine_type + .with_label_values(&[machine_type]) + .inc(); + } self.add_to_total_step_import_time_ms(timings.import_elapsed.as_millis()); self.add_to_total_step_build_time_ms(timings.build_elapsed.as_millis()); self.add_to_total_step_upload_time_ms(timings.upload_elapsed.as_millis()); self.add_to_total_step_time_ms(total_step_time); } - pub fn track_build_failure(&self, timings: super::build::BuildTimings, total_step_time: u64) { + pub fn track_build_failure( + &self, + timings: super::build::BuildTimings, + total_step_time: u64, + machine_type: Option<&str>, + ) { self.nr_steps_done.inc(); self.nr_steps_building.sub(1); self.nr_builds_failed.inc(); + if let Some(machine_type) = machine_type { + self.finished_per_machine_type + .with_label_values(&[machine_type]) + .inc(); + } self.add_to_total_step_import_time_ms(timings.import_elapsed.as_millis()); self.add_to_total_step_build_time_ms(timings.build_elapsed.as_millis()); self.add_to_total_step_upload_time_ms(timings.upload_elapsed.as_millis()); diff --git a/subprojects/hydra-queue-runner/src/state/mod.rs b/subprojects/hydra-queue-runner/src/state/mod.rs index ad8819725..8cfb5d17d 100644 --- a/subprojects/hydra-queue-runner/src/state/mod.rs +++ b/subprojects/hydra-queue-runner/src/state/mod.rs @@ -1454,8 +1454,11 @@ impl State { item.machine .stats .track_build_success(output.timings, total_step_time); - self.metrics - .track_build_success(output.timings, total_step_time); + self.metrics.track_build_success( + output.timings, + total_step_time, + item.step_info.step.get_system().as_deref(), + ); finish_build_step( &self.db, @@ -1741,7 +1744,11 @@ impl State { item.machine .stats .track_build_failure(timings, total_step_time); - self.metrics.track_build_failure(timings, total_step_time); + self.metrics.track_build_failure( + timings, + total_step_time, + item.step_info.step.get_system().as_deref(), + ); let (max_retries, retry_interval, retry_backoff) = self.config.get_retry(); diff --git a/subprojects/hydra-queue-runner/src/state/step.rs b/subprojects/hydra-queue-runner/src/state/step.rs index 182d49406..cb8d4441a 100644 --- a/subprojects/hydra-queue-runner/src/state/step.rs +++ b/subprojects/hydra-queue-runner/src/state/step.rs @@ -5,7 +5,7 @@ use std::sync::{Arc, Weak}; use hashbrown::{HashMap, HashSet}; -use super::{Build, Jobset}; +use super::{Build, Jobset, System}; use db::models::BuildID; use harmonia_store_derivation::derivation::Derivation; use harmonia_store_derivation::derived_path::OutputName; @@ -520,6 +520,21 @@ impl Steps { .count() } + #[must_use] + pub fn get_unfinished_per_system(&self) -> HashMap { + let mut steps = self.inner.write(); + steps.retain(|_, s| s.upgrade().is_some()); + let mut counts: HashMap = HashMap::new(); + for system in steps + .values() + .filter_map(Weak::upgrade) + .filter_map(|s| s.get_system()) + { + *counts.entry(system).or_default() += 1; + } + counts + } + #[must_use] pub fn clone_as_io(&self) -> Vec { let steps = self.inner.read();