diff --git a/Cargo.lock b/Cargo.lock index 18cfc8f5..4baeaed1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -330,28 +330,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossbeam" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" -dependencies = [ - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-epoch", - "crossbeam-queue", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -371,15 +349,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "crossbeam-queue" -version = "0.3.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -417,41 +386,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "darling" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" -dependencies = [ - "darling_core", - "darling_macro", -] - -[[package]] -name = "darling_core" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn", -] - -[[package]] -name = "darling_macro" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" -dependencies = [ - "darling_core", - "quote", - "syn", -] - [[package]] name = "deunicode" version = "1.6.2" @@ -552,12 +486,6 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce81f49ae8a0482e4c55ea62ebbd7e5a686af544c00b9d090bba3ff9be97b3d" -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - [[package]] name = "font-kit" version = "0.14.3" @@ -850,12 +778,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - [[package]] name = "idna" version = "1.1.0" @@ -1061,15 +983,6 @@ dependencies = [ "simd-adler32", ] -[[package]] -name = "ntapi" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" -dependencies = [ - "winapi", -] - [[package]] name = "num" version = "0.2.1" @@ -1146,48 +1059,6 @@ dependencies = [ "autocfg 1.5.0", ] -[[package]] -name = "nvml-wrapper" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d5c6c0ef9702176a570f06ad94f3198bc29c524c8b498f1b9346e1b1bdcbb3a" -dependencies = [ - "bitflags 2.9.1", - "libloading", - "nvml-wrapper-sys", - "static_assertions", - "thiserror 1.0.69", - "wrapcenum-derive", -] - -[[package]] -name = "nvml-wrapper-sys" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd23dbe2eb8d8335d2bce0299e0a07d6a63c089243d626ca75b770a962ff49e6" -dependencies = [ - "libloading", -] - -[[package]] -name = "objc2-core-foundation" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c10c2894a6fed806ade6027bcd50662746363a9589d3ec9d9bef30a4e4bc166" -dependencies = [ - "bitflags 2.9.1", -] - -[[package]] -name = "objc2-io-kit" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71c1c64d6120e51cd86033f67176b1cb66780c2efe34dec55176f77befd93c0a" -dependencies = [ - "libc", - "objc2-core-foundation", -] - [[package]] name = "once_cell" version = "1.21.3" @@ -1265,7 +1136,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323" dependencies = [ "memchr", - "thiserror 2.0.12", + "thiserror", "ucd-trie", ] @@ -1600,7 +1471,7 @@ checksum = "dd6f9d3d47bdd2ad6945c5015a226ec6155d0bcdfd8f7cd29f86b71f8de99d2b" dependencies = [ "getrandom 0.2.16", "libredox", - "thiserror 2.0.12", + "thiserror", ] [[package]] @@ -1770,12 +1641,6 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - [[package]] name = "statistical" version = "1.0.0" @@ -1836,20 +1701,6 @@ dependencies = [ "syn", ] -[[package]] -name = "sysinfo" -version = "0.37.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07cec4dc2d2e357ca1e610cfb07de2fa7a10fc3e9fe89f72545f3d244ea87753" -dependencies = [ - "libc", - "memchr", - "ntapi", - "objc2-core-foundation", - "objc2-io-kit", - "windows", -] - [[package]] name = "tera" version = "1.20.0" @@ -1872,33 +1723,13 @@ dependencies = [ "unic-segment", ] -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl 1.0.69", -] - [[package]] name = "thiserror" version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" dependencies = [ - "thiserror-impl 2.0.12", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn", + "thiserror-impl", ] [[package]] @@ -1975,18 +1806,15 @@ version = "0.1.0" dependencies = [ "chrono", "clap", - "crossbeam", "csv", "fs_extra", "git2", - "nvml-wrapper", "plotters", "regex", "serde", "serde_json", "statistical", "strum", - "sysinfo", "tera", "toml", "xshell", @@ -2226,28 +2054,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows" -version = "0.61.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" -dependencies = [ - "windows-collections", - "windows-core", - "windows-future", - "windows-link", - "windows-numerics", -] - -[[package]] -name = "windows-collections" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" -dependencies = [ - "windows-core", -] - [[package]] name = "windows-core" version = "0.61.2" @@ -2261,17 +2067,6 @@ dependencies = [ "windows-strings", ] -[[package]] -name = "windows-future" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" -dependencies = [ - "windows-core", - "windows-link", - "windows-threading", -] - [[package]] name = "windows-implement" version = "0.60.0" @@ -2300,16 +2095,6 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" -[[package]] -name = "windows-numerics" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" -dependencies = [ - "windows-core", - "windows-link", -] - [[package]] name = "windows-result" version = "0.3.4" @@ -2353,15 +2138,6 @@ dependencies = [ "windows_x86_64_msvc", ] -[[package]] -name = "windows-threading" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" -dependencies = [ - "windows-link", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -2434,18 +2210,6 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" -[[package]] -name = "wrapcenum-derive" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a76ff259533532054cfbaefb115c613203c73707017459206380f03b3b3f266e" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "writeable" version = "0.6.2" diff --git a/Cargo.toml b/Cargo.toml index 5dfc0646..5e62eebc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,9 +14,6 @@ plotters = "0.3.7" tera = "1" chrono = "0.4.41" statistical = "1.0.0" -sysinfo = "0.37.0" -crossbeam = "0.8.4" -nvml-wrapper = "0.11.0" regex = "1.11" git2 = "0.20" fs_extra = "1.3.0" diff --git a/src/bin/collect.rs b/src/bin/collect.rs index e885cd1b..51b246f7 100644 --- a/src/bin/collect.rs +++ b/src/bin/collect.rs @@ -104,7 +104,7 @@ impl Commands { ("benchmark".to_string(), None), ("mode".to_string(), Some("sprite".to_string())), ], - 10000, + 20000, )), Box::new(stress_tests::StressTest::on( "bevymark".to_string(), @@ -114,7 +114,7 @@ impl Commands { ("benchmark".to_string(), None), ("mode".to_string(), Some("mesh2d".to_string())), ], - 5000, + 10000, )), Box::new(stress_tests::StressTest::on( "bevymark".to_string(), @@ -125,7 +125,7 @@ impl Commands { ("mode".to_string(), Some("sprite_mesh".to_string())), ("alpha-mode".to_string(), Some("alpha_mask".to_string())), ], - 5000, + 10000, )), Box::new(stress_tests::StressTest::on( "bevymark".to_string(), @@ -136,67 +136,67 @@ impl Commands { ("mode".to_string(), Some("sprite_mesh".to_string())), ("alpha-mode".to_string(), Some("blend".to_string())), ], - 5000, + 10000, )), Box::new(stress_tests::StressTest::on( "many_animated_sprites".to_string(), vec![], - 30000, + 50000, )), Box::new(stress_tests::StressTest::on( "many_buttons".to_string(), vec![], - 5000, + 10000, )), Box::new(stress_tests::StressTest::on( "many_cubes".to_string(), vec![("benchmark".to_string(), None)], - 15000, + 20000, )), Box::new(stress_tests::StressTest::on( "many_foxes".to_string(), vec![], - 15000, + 30000, )), Box::new(stress_tests::StressTest::on( "many_morph_targets".to_string(), vec![], - 15000, + 30000, )), Box::new(stress_tests::StressTest::on( "many_gizmos".to_string(), vec![], - 5000, + 10000, )), Box::new(stress_tests::StressTest::on( "many_glyphs".to_string(), vec![], - 10000, + 15000, )), Box::new(stress_tests::StressTest::on( "many_gradients".to_string(), vec![], - 20000, + 40000, )), Box::new(stress_tests::StressTest::on( "many_lights".to_string(), vec![], - 5000, + 10000, )), Box::new(stress_tests::StressTest::on( "many_materials".to_string(), vec![], - 20000, + 40000, )), Box::new(stress_tests::StressTest::on( "many_sprites".to_string(), vec![], - 30000, + 60000, )), Box::new(stress_tests::StressTest::on( "many_text2d".to_string(), vec![], - 20000, + 30000, )), Box::new(stress_tests::StressTest::on( "bevymark_3d".to_string(), @@ -206,7 +206,7 @@ impl Commands { ("per-wave".to_string(), Some("200".to_string())), ("alpha-mode".to_string(), Some("blend".to_string())), ], - 10000, + 20000, )), Box::new(stress_tests::StressTest::on( "bevymark_3d".to_string(), @@ -216,7 +216,7 @@ impl Commands { ("per-wave".to_string(), Some("500".to_string())), ("alpha-mode".to_string(), Some("opaque".to_string())), ], - 10000, + 20000, )), Box::new(stress_tests::StressTest::on( "bevymark_3d".to_string(), @@ -226,7 +226,7 @@ impl Commands { ("per-wave".to_string(), Some("500".to_string())), ("alpha-mode".to_string(), Some("alpha_mask".to_string())), ], - 10000, + 20000, )), Box::new( stress_tests::StressTest::on( @@ -277,24 +277,24 @@ impl Commands { large_scenes::LargeScene::on( "bistro".to_string(), vec![("compress".to_string(), None)], - 25000, + 30000, ) .with_features(vec!["mipmap_generator/compress"]), ), Box::new(large_scenes::LargeScene::on( "bistro".to_string(), vec![("no-mip-generation".to_string(), None)], - 25000, + 30000, )), Box::new(large_scenes::LargeScene::on( "caldera_hotel".to_string(), vec![], - 25000, + 30000, )), Box::new(large_scenes::LargeScene::on( "bevy_city".to_string(), vec![("no-cpu-culling".into(), None)], - 25000, + 30000, )), ] } else { diff --git a/src/metrics/large_scenes.rs b/src/metrics/large_scenes.rs index 95e7167b..934d6ee1 100644 --- a/src/metrics/large_scenes.rs +++ b/src/metrics/large_scenes.rs @@ -6,7 +6,6 @@ use std::{ time::{Duration, Instant}, }; -use crossbeam::channel::Receiver; use xshell::{Shell, cmd}; use crate::Metrics; @@ -89,9 +88,6 @@ impl Metrics for LargeScene { } fn collect(&self) -> HashMap { - let cpu = cpu_usage(); - let gpu = gpu_usage(); - let key = format!( "large-scene-fps.{}.{}", self.scene, @@ -136,123 +132,36 @@ impl Metrics for LargeScene { .into_iter() .flat_map(|f| ["--features".to_string(), f]); - let _guard = sh.push_env( + let _ = cmd!(sh, "sudo systemctl start lightdm").run(); + thread::sleep(Duration::from_secs(10)); + + let _mangohud_guard = sh.push_env( "MANGOHUD_CONFIG", format!( "output_folder={},autostart_log=10", std::env::current_dir().unwrap().display() ), ); + let _display_guard = sh.push_env("DISPLAY", ":0"); let cmd = cmd!( sh, - "xvfb-run mangohud cargo run --release {features...} --package {scene} -- {parameters...}" + "mangohud cargo run --release {features...} --package {scene} -- {parameters...}" ); let mut results = HashMap::new(); - // Wait for the monitoring threads to start - let _ = cpu.recv(); - let _ = gpu.recv(); - // Clear channels - while cpu.try_recv().is_ok() {} - while gpu.try_recv().is_ok() {} - let start = Instant::now(); - if cmd.run().is_err() { + let cmd_result = cmd.run(); + + let _ = cmd!(sh, "sudo systemctl stop lightdm").run(); + thread::sleep(Duration::from_secs(5)); + + if cmd_result.is_err() { // ignore failure due to a missing scene return results; }; let elapsed = start.elapsed(); - let cpu_usage = cpu.try_iter().skip(2).collect::>(); - while cpu.try_recv().is_ok() {} - std::mem::drop(cpu); - - let gpu_usage = gpu - .try_iter() - .filter(|v| v.sm != 0) - .skip(2) - .collect::>(); - while gpu.try_recv().is_ok() {} - std::mem::drop(gpu); - - let gpu_memory = gpu_usage.iter().map(|v| v.mem as f32).collect::>(); - let gpu_usage = gpu_usage.iter().map(|v| v.sm as f32).collect::>(); - - if cpu_usage.len() > 3 { - results.insert( - format!("{key}.cpu_usage.mean"), - (statistical::mean(&cpu_usage) * 1000.0) as u64, - ); - results.insert( - format!("{key}.cpu_usage.median"), - (statistical::median(&cpu_usage) * 1000.0) as u64, - ); - results.insert( - format!("{key}.cpu_usage.min"), - cpu_usage.iter().map(|d| (d * 1000.0) as u64).min().unwrap(), - ); - results.insert( - format!("{key}.cpu_usage.max"), - cpu_usage.iter().map(|d| (d * 1000.0) as u64).max().unwrap(), - ); - results.insert( - format!("{key}.cpu_usage.std_dev"), - (statistical::standard_deviation(&cpu_usage, None) * 1000.0) as u64, - ); - } - if gpu_usage.len() > 3 { - results.insert( - format!("{key}.gpu_usage.mean"), - (statistical::mean(&gpu_usage) * 1000.0) as u64, - ); - results.insert( - format!("{key}.gpu_usage.median"), - (statistical::median(&gpu_usage) * 1000.0) as u64, - ); - results.insert( - format!("{key}.gpu_usage.min"), - gpu_usage.iter().map(|d| (d * 1000.0) as u64).min().unwrap(), - ); - results.insert( - format!("{key}.gpu_usage.max"), - gpu_usage.iter().map(|d| (d * 1000.0) as u64).max().unwrap(), - ); - results.insert( - format!("{key}.gpu_usage.std_dev"), - (statistical::standard_deviation(&gpu_usage, None) * 1000.0) as u64, - ); - } - if gpu_memory.len() > 3 { - results.insert( - format!("{key}.gpu_memory.mean"), - (statistical::mean(&gpu_memory) * 1000.0) as u64, - ); - results.insert( - format!("{key}.gpu_memory.median"), - (statistical::median(&gpu_memory) * 1000.0) as u64, - ); - results.insert( - format!("{key}.gpu_memory.min"), - gpu_memory - .iter() - .map(|d| (d * 1000.0) as u64) - .min() - .unwrap(), - ); - results.insert( - format!("{key}.gpu_memory.max"), - gpu_memory - .iter() - .map(|d| (d * 1000.0) as u64) - .max() - .unwrap(), - ); - results.insert( - format!("{key}.gpu_memory.std_dev"), - (statistical::standard_deviation(&gpu_memory, None) * 1000.0) as u64, - ); - } results.insert(format!("{key}.duration"), elapsed.as_millis() as u64); results.insert(format!("{key}.frames"), self.nb_frames as u64); @@ -272,121 +181,56 @@ impl Metrics for LargeScene { let _ = reader.read_line(&mut tmp); let _ = reader.read_line(&mut tmp); let mut rdr = csv::ReaderBuilder::new().from_reader(reader); - let frame_times = rdr + let samples: Vec = rdr .records() .flatten() - .flat_map(|record| record.get(1).unwrap().parse::()) - .collect::>(); - - if !frame_times.len() > 3 { - results.insert( - format!("{key}.frame_time.mean"), - (statistical::mean(&frame_times) * 1000.0) as u64, - ); - results.insert( - format!("{key}.frame_time.median"), - (statistical::median(&frame_times) * 1000.0) as u64, - ); - results.insert( - format!("{key}.frame_time.min"), - frame_times - .iter() - .map(|d| (d * 1000.0) as u64) - .min() - .unwrap(), - ); - results.insert( - format!("{key}.frame_time.max"), - frame_times - .iter() - .map(|d| (d * 1000.0) as u64) - .max() - .unwrap(), - ); - results.insert( - format!("{key}.frame_time.std_dev"), - (statistical::standard_deviation(&frame_times, None) * 1000.0) as u64, - ); + .map(|record| super::MangohudSample { + frame_time: record.get(1).unwrap().parse::().unwrap_or_default(), + cpu: record.get(2).unwrap().parse::().unwrap_or_default(), + gpu: record.get(3).unwrap().parse::().unwrap_or_default(), + vram: record.get(8).unwrap().parse::().unwrap_or_default(), + ram: record.get(10).unwrap().parse::().unwrap_or_default(), + }) + .collect(); + + let frame_times: Vec = samples.iter().map(|s| s.frame_time).collect(); + let cpu: Vec = samples.iter().map(|s| s.cpu).collect(); + let gpu: Vec = samples.iter().map(|s| s.gpu).collect(); + let ram: Vec = samples.iter().map(|s| s.ram).collect(); + let vram: Vec = samples.iter().map(|s| s.vram).collect(); + + for (values, name) in [ + (frame_times, "frame_time"), + (cpu, "cpu_load"), + (gpu, "gpu_load"), + (ram, "ram_used"), + (vram, "vram_used"), + ] { + if values.len() > 3 { + results.insert( + format!("{key}.{name}.mean"), + (statistical::mean(&values) * 1000.0) as u64, + ); + results.insert( + format!("{key}.{name}.median"), + (statistical::median(&values) * 1000.0) as u64, + ); + results.insert( + format!("{key}.{name}.min"), + values.iter().map(|d| (d * 1000.0) as u64).min().unwrap(), + ); + results.insert( + format!("{key}.{name}.max"), + values.iter().map(|d| (d * 1000.0) as u64).max().unwrap(), + ); + results.insert( + format!("{key}.{name}.std_dev"), + (statistical::standard_deviation(&values, None) * 1000.0) as u64, + ); + } } } results } } - -fn cpu_usage() -> Receiver { - let (tx, rx) = crossbeam::channel::unbounded(); - - thread::spawn(move || { - let mut sys = sysinfo::System::new(); - let delay = sysinfo::MINIMUM_CPU_UPDATE_INTERVAL.max(Duration::from_secs(1)); - - loop { - sys.refresh_cpu_usage(); - if tx.send(sys.global_cpu_usage()).is_err() { - break; - } - std::thread::sleep(delay); - } - }); - - rx -} - -#[derive(Debug)] -struct GpuUsage { - sm: u32, - mem: u32, -} - -fn gpu_usage() -> Receiver { - let (tx, rx) = crossbeam::channel::unbounded(); - use nvml_wrapper::{Nvml, error::NvmlError}; - - thread::spawn(move || { - let Ok(nvml) = Nvml::init() else { - println!("Couldn't load nvidia driver"); - return; - }; - let device = nvml.device_by_index(0).unwrap(); - let delay = Duration::from_secs(1); - - let mut timestamp = None; - - let _ = tx.try_send(GpuUsage { sm: 0, mem: 0 }); - - loop { - let processes = match device.process_utilization_stats(timestamp) { - Ok(processes) => processes, - Err(NvmlError::NotFound) => { - // No process using the GPU found - if tx.send(GpuUsage { sm: 0, mem: 0 }).is_err() { - break; - } - continue; - } - Err(_) => { - println!("Couldn't get process utilization stats"); - break; - } - }; - - let process = &processes[0]; - timestamp = Some(process.timestamp); - - if tx - .send(GpuUsage { - sm: process.sm_util, - mem: process.mem_util, - }) - .is_err() - { - break; - } - std::thread::sleep(delay); - } - let _ = tx.try_send(GpuUsage { sm: 0, mem: 0 }); - }); - - rx -} diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index c6a53d92..8c34c6e2 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -1,3 +1,11 @@ +pub(crate) struct MangohudSample { + pub frame_time: f32, + pub cpu: f32, + pub gpu: f32, + pub vram: f32, + pub ram: f32, +} + pub mod benchmarks; pub mod binary_size; pub mod compile_time; diff --git a/src/metrics/stress_tests.rs b/src/metrics/stress_tests.rs index 43473ad4..a3ec14c8 100644 --- a/src/metrics/stress_tests.rs +++ b/src/metrics/stress_tests.rs @@ -6,7 +6,6 @@ use std::{ time::{Duration, Instant}, }; -use crossbeam::channel::Receiver; use xshell::{Shell, cmd}; use crate::Metrics; @@ -77,9 +76,6 @@ impl Metrics for StressTest { } fn collect(&self) -> HashMap { - let cpu = cpu_usage(); - let gpu = gpu_usage(); - let key = format!( "stress-test-fps.{}.{}", self.stress_test, @@ -118,159 +114,36 @@ impl Metrics for StressTest { .into_iter() .flat_map(|f| ["--features".to_string(), f]); - let _guard = sh.push_env( + let _ = cmd!(sh, "sudo systemctl start lightdm").run(); + thread::sleep(Duration::from_secs(10)); + + let _mangohud_guard = sh.push_env( "MANGOHUD_CONFIG", format!( "output_folder={},autostart_log=1", std::env::current_dir().unwrap().display() ), ); + let _display_guard = sh.push_env("DISPLAY", ":0"); let cmd = cmd!( sh, - "xvfb-run mangohud cargo run --release {features...} --example {stress_tests} -- {parameters...}" + "mangohud cargo run --release {features...} --example {stress_tests} -- {parameters...}" ); let mut results = HashMap::new(); - // Wait for the monitoring threads to start - let _ = cpu.recv(); - let _ = gpu.recv(); - // Clear channels - while cpu.try_recv().is_ok() {} - while gpu.try_recv().is_ok() {} - let start = Instant::now(); - let Ok(output) = cmd.output() else { - // ignore failure due to a missing stress test + let cmd_result = cmd.run(); + + let _ = cmd!(sh, "sudo systemctl stop lightdm").run(); + thread::sleep(Duration::from_secs(5)); + + if cmd_result.is_err() { + // ignore failure due to a missing scene return results; }; let elapsed = start.elapsed(); - let cpu_usage = cpu.try_iter().skip(2).collect::>(); - while cpu.try_recv().is_ok() {} - std::mem::drop(cpu); - - let gpu_usage = gpu - .try_iter() - .filter(|v| v.sm != 0) - .skip(2) - .collect::>(); - while gpu.try_recv().is_ok() {} - std::mem::drop(gpu); - - let gpu_memory = gpu_usage.iter().map(|v| v.mem as f32).collect::>(); - let gpu_usage = gpu_usage.iter().map(|v| v.sm as f32).collect::>(); - - let fpss = output - .stdout - .lines() - .chain(output.stderr.lines()) - .map_while(|line| line.ok()) - .filter(|line| line.contains("fps")) - .filter(|line| line.contains("avg")) - .map(|line| line.split("fps").nth(1).unwrap().to_string()) - .map(|line| line.split("(").nth(0).unwrap().to_string()) - .map(|line| line.split(":").nth(1).unwrap().to_string()) - .map(|line| line.trim().to_string()) - .map(|line| line.parse::().unwrap()) - .collect::>(); - - if !fpss.is_empty() { - results.insert( - format!("{key}.mean"), - (statistical::mean(&fpss) * 1000.0) as u64, - ); - results.insert( - format!("{key}.median"), - (statistical::median(&fpss) * 1000.0) as u64, - ); - results.insert( - format!("{key}.min"), - fpss.iter().map(|d| (d * 1000.0) as u64).min().unwrap(), - ); - results.insert( - format!("{key}.max"), - fpss.iter().map(|d| (d * 1000.0) as u64).max().unwrap(), - ); - results.insert( - format!("{key}.std_dev"), - (statistical::standard_deviation(&fpss, None) * 1000.0) as u64, - ); - } - if cpu_usage.len() > 3 { - results.insert( - format!("{key}.cpu_usage.mean"), - (statistical::mean(&cpu_usage) * 1000.0) as u64, - ); - results.insert( - format!("{key}.cpu_usage.median"), - (statistical::median(&cpu_usage) * 1000.0) as u64, - ); - results.insert( - format!("{key}.cpu_usage.min"), - cpu_usage.iter().map(|d| (d * 1000.0) as u64).min().unwrap(), - ); - results.insert( - format!("{key}.cpu_usage.max"), - cpu_usage.iter().map(|d| (d * 1000.0) as u64).max().unwrap(), - ); - results.insert( - format!("{key}.cpu_usage.std_dev"), - (statistical::standard_deviation(&cpu_usage, None) * 1000.0) as u64, - ); - } - if gpu_usage.len() > 3 { - results.insert( - format!("{key}.gpu_usage.mean"), - (statistical::mean(&gpu_usage) * 1000.0) as u64, - ); - results.insert( - format!("{key}.gpu_usage.median"), - (statistical::median(&gpu_usage) * 1000.0) as u64, - ); - results.insert( - format!("{key}.gpu_usage.min"), - gpu_usage.iter().map(|d| (d * 1000.0) as u64).min().unwrap(), - ); - results.insert( - format!("{key}.gpu_usage.max"), - gpu_usage.iter().map(|d| (d * 1000.0) as u64).max().unwrap(), - ); - results.insert( - format!("{key}.gpu_usage.std_dev"), - (statistical::standard_deviation(&gpu_usage, None) * 1000.0) as u64, - ); - } - if gpu_memory.len() > 3 { - results.insert( - format!("{key}.gpu_memory.mean"), - (statistical::mean(&gpu_memory) * 1000.0) as u64, - ); - results.insert( - format!("{key}.gpu_memory.median"), - (statistical::median(&gpu_memory) * 1000.0) as u64, - ); - results.insert( - format!("{key}.gpu_memory.min"), - gpu_memory - .iter() - .map(|d| (d * 1000.0) as u64) - .min() - .unwrap(), - ); - results.insert( - format!("{key}.gpu_memory.max"), - gpu_memory - .iter() - .map(|d| (d * 1000.0) as u64) - .max() - .unwrap(), - ); - results.insert( - format!("{key}.gpu_memory.std_dev"), - (statistical::standard_deviation(&gpu_memory, None) * 1000.0) as u64, - ); - } results.insert(format!("{key}.duration"), elapsed.as_millis() as u64); results.insert(format!("{key}.frames"), self.nb_frames as u64); @@ -290,121 +163,56 @@ impl Metrics for StressTest { let _ = reader.read_line(&mut tmp); let _ = reader.read_line(&mut tmp); let mut rdr = csv::ReaderBuilder::new().from_reader(reader); - let frame_times = rdr + let samples: Vec = rdr .records() .flatten() - .flat_map(|record| record.get(1).unwrap().parse::()) - .collect::>(); - - if !frame_times.len() > 3 { - results.insert( - format!("{key}.frame_time.mean"), - (statistical::mean(&frame_times) * 1000.0) as u64, - ); - results.insert( - format!("{key}.frame_time.median"), - (statistical::median(&frame_times) * 1000.0) as u64, - ); - results.insert( - format!("{key}.frame_time.min"), - frame_times - .iter() - .map(|d| (d * 1000.0) as u64) - .min() - .unwrap(), - ); - results.insert( - format!("{key}.frame_time.max"), - frame_times - .iter() - .map(|d| (d * 1000.0) as u64) - .max() - .unwrap(), - ); - results.insert( - format!("{key}.frame_time.std_dev"), - (statistical::standard_deviation(&frame_times, None) * 1000.0) as u64, - ); + .map(|record| super::MangohudSample { + frame_time: record.get(1).unwrap().parse::().unwrap_or_default(), + cpu: record.get(2).unwrap().parse::().unwrap_or_default(), + gpu: record.get(3).unwrap().parse::().unwrap_or_default(), + vram: record.get(8).unwrap().parse::().unwrap_or_default(), + ram: record.get(10).unwrap().parse::().unwrap_or_default(), + }) + .collect(); + + let frame_times: Vec = samples.iter().map(|s| s.frame_time).collect(); + let cpu: Vec = samples.iter().map(|s| s.cpu).collect(); + let gpu: Vec = samples.iter().map(|s| s.gpu).collect(); + let ram: Vec = samples.iter().map(|s| s.ram).collect(); + let vram: Vec = samples.iter().map(|s| s.vram).collect(); + + for (values, name) in [ + (frame_times, "frame_time"), + (cpu, "cpu_load"), + (gpu, "gpu_load"), + (ram, "ram_used"), + (vram, "vram_used"), + ] { + if values.len() > 3 { + results.insert( + format!("{key}.{name}.mean"), + (statistical::mean(&values) * 1000.0) as u64, + ); + results.insert( + format!("{key}.{name}.median"), + (statistical::median(&values) * 1000.0) as u64, + ); + results.insert( + format!("{key}.{name}.min"), + values.iter().map(|d| (d * 1000.0) as u64).min().unwrap(), + ); + results.insert( + format!("{key}.{name}.max"), + values.iter().map(|d| (d * 1000.0) as u64).max().unwrap(), + ); + results.insert( + format!("{key}.{name}.std_dev"), + (statistical::standard_deviation(&values, None) * 1000.0) as u64, + ); + } } } results } } - -fn cpu_usage() -> Receiver { - let (tx, rx) = crossbeam::channel::unbounded(); - - thread::spawn(move || { - let mut sys = sysinfo::System::new(); - let delay = sysinfo::MINIMUM_CPU_UPDATE_INTERVAL.max(Duration::from_secs(1)); - - loop { - sys.refresh_cpu_usage(); - if tx.send(sys.global_cpu_usage()).is_err() { - break; - } - std::thread::sleep(delay); - } - }); - - rx -} - -#[derive(Debug)] -struct GpuUsage { - sm: u32, - mem: u32, -} - -fn gpu_usage() -> Receiver { - let (tx, rx) = crossbeam::channel::unbounded(); - use nvml_wrapper::{Nvml, error::NvmlError}; - - thread::spawn(move || { - let Ok(nvml) = Nvml::init() else { - println!("Couldn't load nvidia driver"); - return; - }; - let device = nvml.device_by_index(0).unwrap(); - let delay = Duration::from_secs(1); - - let mut timestamp = None; - - let _ = tx.try_send(GpuUsage { sm: 0, mem: 0 }); - - loop { - let processes = match device.process_utilization_stats(timestamp) { - Ok(processes) => processes, - Err(NvmlError::NotFound) => { - // No process using the GPU found - if tx.send(GpuUsage { sm: 0, mem: 0 }).is_err() { - break; - } - continue; - } - Err(_) => { - println!("Couldn't get process utilization stats"); - break; - } - }; - - let process = &processes[0]; - timestamp = Some(process.timestamp); - - if tx - .send(GpuUsage { - sm: process.sm_util, - mem: process.mem_util, - }) - .is_err() - { - break; - } - std::thread::sleep(delay); - } - let _ = tx.try_send(GpuUsage { sm: 0, mem: 0 }); - }); - - rx -}