Skip to content

queue-runner: introduce finished/unfinished per machine type metrics #1817

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MiniHarinn wants to merge 1 commit into
base: master
Choose a base branch
from
+77 −6
Open

queue-runner: introduce finished/unfinished per machine type metrics #1817

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 51 additions & 2 deletions subprojects/hydra-queue-runner/src/state/metrics.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ pub struct PromMetrics {
pub disabled_per_machine_type: prometheus::IntGaugeVec, // hydraqueuerunner_machine_type_disabled
pub avg_runnable_time_per_machine_type: prometheus::IntGaugeVec, // hydraqueuerunner_machine_type_avg_runnable_time
pub wait_time_per_machine_type: prometheus::IntGaugeVec, // hydraqueuerunner_machine_type_wait_time
pub finished_per_machine_type: prometheus::IntCounterVec, // hydraqueuerunner_machine_type_finished
pub unfinished_per_machine_type: prometheus::IntGaugeVec, // hydraqueuerunner_machine_type_unfinished

// Per-machine metrics
pub machine_current_jobs: prometheus::IntGaugeVec, // hydraqueuerunner_machine_current_jobs
Expand Down Expand Up @@ -347,6 +349,20 @@ impl PromMetrics {
),
&["machine_type"],
)?;
let finished_per_machine_type = prometheus::IntCounterVec::new(
prometheus::Opts::new(
"hydraqueuerunner_machine_type_finished",
"Number of completed build steps per machine type",
),
&["machine_type"],
)?;
let unfinished_per_machine_type = prometheus::IntGaugeVec::new(
prometheus::Opts::new(
"hydraqueuerunner_machine_type_unfinished",
"Number of unfinished build steps per machine type",
),
&["machine_type"],
)?;

// Per-machine metrics
let machine_current_jobs = prometheus::IntGaugeVec::new(
Expand Down Expand Up @@ -683,6 +699,8 @@ impl PromMetrics {
r.register(Box::new(disabled_per_machine_type.clone()))?;
r.register(Box::new(avg_runnable_time_per_machine_type.clone()))?;
r.register(Box::new(wait_time_per_machine_type.clone()))?;
r.register(Box::new(finished_per_machine_type.clone()))?;
r.register(Box::new(unfinished_per_machine_type.clone()))?;
r.register(Box::new(machine_current_jobs.clone()))?;
r.register(Box::new(machine_steps_done.clone()))?;
r.register(Box::new(machine_total_step_time_ms.clone()))?;
Expand Down Expand Up @@ -794,6 +812,8 @@ impl PromMetrics {
disabled_per_machine_type,
avg_runnable_time_per_machine_type,
wait_time_per_machine_type,
finished_per_machine_type,
unfinished_per_machine_type,
machine_current_jobs,
machine_steps_done,
machine_total_step_time_ms,
Expand Down Expand Up @@ -912,6 +932,7 @@ impl PromMetrics {
self.disabled_per_machine_type.reset();
self.avg_runnable_time_per_machine_type.reset();
self.wait_time_per_machine_type.reset();
self.unfinished_per_machine_type.reset();
for (t, s) in state.queues.get_stats_per_queue().await {
if let Ok(v) = i64::try_from(s.total_runnable) {
self.runnable_per_machine_type
Expand Down Expand Up @@ -944,6 +965,14 @@ impl PromMetrics {
.set(v);
}
}

for (t, count) in state.steps.get_unfinished_per_system() {
if let Ok(v) = i64::try_from(count) {
self.unfinished_per_machine_type
.with_label_values(&[&t])
.set(v);
}
}
}

fn refresh_per_machine_metrics(&self, state: &Arc<super::State>) {
Expand Down Expand Up @@ -1163,20 +1192,40 @@ impl PromMetrics {
.observe(wait_seconds);
}

pub fn track_build_success(&self, timings: super::build::BuildTimings, total_step_time: u64) {
pub fn track_build_success(
&self,
timings: super::build::BuildTimings,
total_step_time: u64,
machine_type: Option<&str>,
) {
self.nr_builds_succeeded.inc();
self.nr_steps_done.inc();
self.nr_steps_building.sub(1);
if let Some(machine_type) = machine_type {
self.finished_per_machine_type
.with_label_values(&[machine_type])
.inc();
}
self.add_to_total_step_import_time_ms(timings.import_elapsed.as_millis());
self.add_to_total_step_build_time_ms(timings.build_elapsed.as_millis());
self.add_to_total_step_upload_time_ms(timings.upload_elapsed.as_millis());
self.add_to_total_step_time_ms(total_step_time);
}

pub fn track_build_failure(&self, timings: super::build::BuildTimings, total_step_time: u64) {
pub fn track_build_failure(
&self,
timings: super::build::BuildTimings,
total_step_time: u64,
machine_type: Option<&str>,
) {
self.nr_steps_done.inc();
self.nr_steps_building.sub(1);
self.nr_builds_failed.inc();
if let Some(machine_type) = machine_type {
self.finished_per_machine_type
.with_label_values(&[machine_type])
.inc();
}
self.add_to_total_step_import_time_ms(timings.import_elapsed.as_millis());
self.add_to_total_step_build_time_ms(timings.build_elapsed.as_millis());
self.add_to_total_step_upload_time_ms(timings.upload_elapsed.as_millis());
Expand Down
13 changes: 10 additions & 3 deletions subprojects/hydra-queue-runner/src/state/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1454,8 +1454,11 @@ impl State {
item.machine
.stats
.track_build_success(output.timings, total_step_time);
self.metrics
.track_build_success(output.timings, total_step_time);
self.metrics.track_build_success(
output.timings,
total_step_time,
item.step_info.step.get_system().as_deref(),
);

finish_build_step(
&self.db,
Expand Down Expand Up @@ -1741,7 +1744,11 @@ impl State {
item.machine
.stats
.track_build_failure(timings, total_step_time);
self.metrics.track_build_failure(timings, total_step_time);
self.metrics.track_build_failure(
timings,
total_step_time,
item.step_info.step.get_system().as_deref(),
);

let (max_retries, retry_interval, retry_backoff) = self.config.get_retry();

Expand Down
17 changes: 16 additions & 1 deletion subprojects/hydra-queue-runner/src/state/step.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use std::sync::{Arc, Weak};

use hashbrown::{HashMap, HashSet};

use super::{Build, Jobset};
use super::{Build, Jobset, System};
use db::models::BuildID;
use harmonia_store_derivation::derivation::Derivation;
use harmonia_store_derivation::derived_path::OutputName;
Expand Down Expand Up @@ -520,6 +520,21 @@ impl Steps {
.count()
}

#[must_use]
pub fn get_unfinished_per_system(&self) -> HashMap<System, u64> {
let mut steps = self.inner.write();
steps.retain(|_, s| s.upgrade().is_some());
let mut counts: HashMap<System, u64> = HashMap::new();
for system in steps
.values()
.filter_map(Weak::upgrade)
.filter_map(|s| s.get_system())
{
*counts.entry(system).or_default() += 1;
}
counts
}

#[must_use]
pub fn clone_as_io(&self) -> Vec<crate::io::Step> {
let steps = self.inner.read();
Expand Down