Add --import-json to compare against previously saved benchmark runs

CHANGELOG.md

@@ -1,3 +1,9 @@
+# unreleased
+
+## Features
+
+- Add `--import-json <file>` option that loads benchmarks from a JSON file written by `--export-json` and includes them in the comparison output without re-running them, see #607
+
 # v1.20.0
 
 ## Features

README.md

@@ -153,6 +153,20 @@ multiple benchmarks:
 | ![](doc/histogram.png) | ![](doc/whisker.png) |
 |---:|---:|
 
+### Comparing against a previous run
+
+Use `--import-json <file>` to read a JSON file produced by a previous `--export-json`
+run. The imported benchmarks are not re-executed: they appear in the relative speed
+summary and in every export format alongside any commands you give on the command line.
+This is handy when the baseline takes a long time to run, or lives on another machine,
+or simply already exists from an earlier session:
+
+```sh
+hyperfine 'old-binary args' --export-json baseline.json
+hyperfine --import-json baseline.json 'new-binary args'
+```
+
+`--import-json` may be specified more than once to combine several saved files.
 
 ### Detailed benchmark flowchart
 

src/benchmark/benchmark_result.rs

@@ -1,20 +1,20 @@
 use std::collections::BTreeMap;
 
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
 
 use crate::util::units::Second;
 
 /// Set of values that will be exported.
 // NOTE: `serde` is used for JSON serialization, but not for CSV serialization due to the
 // `parameters` map. Update `src/hyperfine/export/csv.rs` with new fields, as appropriate.
-#[derive(Debug, Default, Clone, Serialize, PartialEq)]
+#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq)]
 pub struct BenchmarkResult {
     /// The full command line of the program that is being benchmarked
     pub command: String,
 
     /// The full command line of the program that is being benchmarked, possibly including a list of
     /// parameters that were not used in the command line template.
-    #[serde(skip_serializing)]
+    #[serde(skip_serializing, default)]
     pub command_with_unused_parameters: String,
 
     /// The average run time
@@ -39,17 +39,17 @@ pub struct BenchmarkResult {
     pub max: Second,
 
     /// All run time measurements
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(skip_serializing_if = "Option::is_none", default)]
     pub times: Option<Vec<Second>>,
 
     /// Maximum memory usage of the process, in bytes
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(skip_serializing_if = "Option::is_none", default)]
     pub memory_usage_byte: Option<Vec<u64>>,
 
     /// Exit codes of all command invocations
     pub exit_codes: Vec<Option<i32>>,
 
     /// Parameter values for this benchmark
-    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
+    #[serde(skip_serializing_if = "BTreeMap::is_empty", default)]
     pub parameters: BTreeMap<String, String>,
 }

src/benchmark/mod.rs

@@ -33,6 +33,7 @@ pub const MIN_EXECUTION_TIME: Second = 5e-3;
 
 pub struct Benchmark<'a> {
     number: usize,
+    display_number: usize,
     command: &'a Command<'a>,
     options: &'a Options,
     executor: &'a dyn Executor,
@@ -47,12 +48,21 @@ impl<'a> Benchmark<'a> {
     ) -> Self {
         Benchmark {
             number,
+            display_number: number,
             command,
             options,
             executor,
         }
     }
 
+    /// Override the integer used for the `Benchmark N:` header. This is used
+    /// when imported benchmarks are listed before live ones, so the live ones
+    /// continue counting from the imported tail rather than restarting at 1.
+    pub fn with_display_number(mut self, display_number: usize) -> Self {
+        self.display_number = display_number;
+        self
+    }
+
     /// Run setup, cleanup, or preparation commands
     fn run_intermediate_command(
         &self,
@@ -143,7 +153,7 @@ impl<'a> Benchmark<'a> {
             println!(
                 "{}{}: {}",
                 "Benchmark ".bold(),
-                (self.number + 1).to_string().bold(),
+                (self.display_number + 1).to_string().bold(),
                 self.command.get_name_with_unused_parameters(),
             );
         }

src/benchmark/scheduler.rs

@@ -17,6 +17,23 @@ pub struct Scheduler<'a> {
     results: Vec<BenchmarkResult>,
 }
 
+/// Print a header for an imported benchmark. Mirrors the format used by
+/// `Benchmark::run` so the imported entries blend in with the live ones.
+fn print_imported_header(number: usize, result: &BenchmarkResult) {
+    let label = if result.command_with_unused_parameters.is_empty() {
+        result.command.as_str()
+    } else {
+        result.command_with_unused_parameters.as_str()
+    };
+    println!(
+        "{}{}: {} {}",
+        "Benchmark ".bold(),
+        (number + 1).to_string().bold(),
+        label,
+        "(imported)".dimmed(),
+    );
+}
+
 impl<'a> Scheduler<'a> {
     pub fn new(
         commands: &'a Commands,
@@ -31,7 +48,29 @@ impl<'a> Scheduler<'a> {
         }
     }
 
+    /// Add benchmark results that were loaded from a previously-saved JSON file.
+    /// These are not re-run, but participate in the relative speed comparison
+    /// and in all configured exports as if they had just finished.
+    pub fn add_imported_results(&mut self, imported: Vec<BenchmarkResult>) {
+        if self.options.output_style != OutputStyleOption::Disabled {
+            for (offset, result) in imported.iter().enumerate() {
+                print_imported_header(self.results.len() + offset, result);
+            }
+        }
+        self.results.extend(imported);
+    }
+
     pub fn run_benchmarks(&mut self) -> Result<()> {
+        // Preserve any pre-populated results (e.g. loaded via --import-json) so
+        // they share the same export pipeline as live benchmarks.
+        if !self.results.is_empty() {
+            self.export_manager.write_results(&self.results, true)?;
+        }
+
+        if self.commands.iter().next().is_none() && self.options.reference_command.is_none() {
+            return Ok(());
+        }
+
         let mut executor: Box<dyn Executor> = match self.options.executor_kind {
             ExecutorKind::Raw => Box::new(RawExecutor::new(self.options)),
             ExecutorKind::Mock(ref shell) => Box::new(MockExecutor::new(shell.clone())),
@@ -46,9 +85,13 @@ impl<'a> Scheduler<'a> {
 
         executor.calibrate()?;
 
+        let display_offset = self.results.len();
         for (number, cmd) in reference.iter().chain(self.commands.iter()).enumerate() {
-            self.results
-                .push(Benchmark::new(number, cmd, self.options, &*executor).run()?);
+            self.results.push(
+                Benchmark::new(number, cmd, self.options, &*executor)
+                    .with_display_number(number + display_offset)
+                    .run()?,
+            );
 
             // We export results after each individual benchmark, because
             // we would risk losing them if a later benchmark fails.

src/cli.rs

@@ -30,7 +30,7 @@ fn build_command() -> Command {
                        The latter is only available if the shell is not explicitly disabled via \
                        '--shell=none'. If multiple commands are given, hyperfine will show a \
                        comparison of the respective runtimes.")
-                .required(true)
+                .required_unless_present("import-json")
                 .action(ArgAction::Append)
                 .value_hint(ValueHint::CommandString)
                 .value_parser(NonEmptyStringValueParser::new()),
@@ -325,6 +325,18 @@ fn build_command() -> Command {
                 .help("Export the timing summary statistics as an Emacs org-mode table to the given FILE. \
                        The output time unit can be changed using the --time-unit option."),
         )
+        .arg(
+            Arg::new("import-json")
+                .long("import-json")
+                .action(ArgAction::Append)
+                .num_args(1)
+                .value_name("FILE")
+                .value_hint(ValueHint::FilePath)
+                .help("Load benchmark results from a JSON file previously written by --export-json. \
+                       The imported benchmarks are not re-run, but appear alongside any commands given \
+                       on the command line in the relative speed comparison and in all export formats. \
+                       This option can be specified multiple times to import several files."),
+        )
         .arg(
             Arg::new("show-output")
                 .long("show-output")

src/import/mod.rs

@@ -0,0 +1,146 @@
+use std::fs::File;
+use std::io::BufReader;
+use std::path::Path;
+
+use anyhow::{Context, Result};
+use serde::Deserialize;
+
+use crate::benchmark::benchmark_result::BenchmarkResult;
+
+/// Mirror of the structure written by `JsonExporter` so we can round-trip
+/// previously-saved benchmark runs back into hyperfine.
+#[derive(Deserialize)]
+struct ImportedSummary {
+    results: Vec<BenchmarkResult>,
+}
+
+/// Read a previously-saved JSON file and return the benchmark results inside it.
+///
+/// The file is expected to follow the same schema written by `--export-json`,
+/// i.e. a top-level object with a `"results"` array of benchmark objects.
+pub fn load_results_from_json<P: AsRef<Path>>(path: P) -> Result<Vec<BenchmarkResult>> {
+    let path = path.as_ref();
+    let file = File::open(path)
+        .with_context(|| format!("Could not open import file '{}'", path.display()))?;
+    let reader = BufReader::new(file);
+    let summary: ImportedSummary = serde_json::from_reader(reader)
+        .with_context(|| format!("Failed to parse JSON from import file '{}'", path.display()))?;
+
+    let mut results = summary.results;
+    for result in &mut results {
+        // The JSON schema does not carry `command_with_unused_parameters`, since
+        // `--export-json` skips it. Fall back to the command line itself so that
+        // the imported entry still has something sensible to display.
+        if result.command_with_unused_parameters.is_empty() {
+            result.command_with_unused_parameters = result.command.clone();
+        }
+    }
+    Ok(results)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::BTreeMap;
+
+    use super::load_results_from_json;
+    use crate::benchmark::benchmark_result::BenchmarkResult;
+
+    fn write_temp_json(contents: &str) -> std::path::PathBuf {
+        use std::io::Write;
+        let dir = tempfile::tempdir().unwrap();
+        let path = dir.path().join("results.json");
+        let mut file = std::fs::File::create(&path).unwrap();
+        file.write_all(contents.as_bytes()).unwrap();
+        // Leak the tempdir so the file survives for the duration of the test.
+        let _ = dir.keep();
+        path
+    }
+
+    #[test]
+    fn round_trip_export_then_import() {
+        use serde_json::json;
+
+        let original = vec![
+            BenchmarkResult {
+                command: "sleep 0.1".into(),
+                command_with_unused_parameters: "sleep 0.1".into(),
+                mean: 0.1,
+                stddev: Some(0.01),
+                median: 0.1,
+                user: 0.05,
+                system: 0.02,
+                min: 0.09,
+                max: 0.11,
+                times: Some(vec![0.1, 0.1]),
+                memory_usage_byte: None,
+                exit_codes: vec![Some(0), Some(0)],
+                parameters: BTreeMap::new(),
+            },
+            BenchmarkResult {
+                command: "sleep 0.2".into(),
+                command_with_unused_parameters: "sleep 0.2".into(),
+                mean: 0.2,
+                stddev: Some(0.02),
+                median: 0.2,
+                user: 0.1,
+                system: 0.05,
+                min: 0.18,
+                max: 0.22,
+                times: Some(vec![0.2, 0.2]),
+                memory_usage_byte: None,
+                exit_codes: vec![Some(0), Some(0)],
+                parameters: BTreeMap::new(),
+            },
+        ];
+
+        // Re-emit via serde to confirm the schema written by `--export-json`
+        // round-trips back to a `BenchmarkResult` without information loss.
+        let summary = json!({ "results": &original });
+        let path = write_temp_json(&summary.to_string());
+
+        let imported = load_results_from_json(&path).unwrap();
+        assert_eq!(imported.len(), original.len());
+        for (a, b) in imported.iter().zip(original.iter()) {
+            assert_eq!(a.command, b.command);
+            assert_eq!(a.mean, b.mean);
+            assert_eq!(a.stddev, b.stddev);
+            assert_eq!(a.median, b.median);
+            assert_eq!(a.times, b.times);
+            assert_eq!(a.exit_codes, b.exit_codes);
+        }
+    }
+
+    #[test]
+    fn missing_optional_fields_are_tolerated() {
+        let path = write_temp_json(
+            r#"{
+                "results": [
+                    {
+                        "command": "echo a",
+                        "mean": 0.5,
+                        "stddev": null,
+                        "median": 0.5,
+                        "user": 0.0,
+                        "system": 0.0,
+                        "min": 0.5,
+                        "max": 0.5,
+                        "exit_codes": [0]
+                    }
+                ]
+            }"#,
+        );
+        let results = load_results_from_json(&path).unwrap();
+        assert_eq!(results.len(), 1);
+        assert_eq!(results[0].command, "echo a");
+        assert_eq!(results[0].command_with_unused_parameters, "echo a");
+        assert_eq!(results[0].mean, 0.5);
+        assert!(results[0].times.is_none());
+    }
+
+    #[test]
+    fn missing_file_produces_a_helpful_error() {
+        let err = load_results_from_json("/no/such/file/hopefully.json").unwrap_err();
+        let msg = format!("{err:#}");
+        assert!(msg.contains("Could not open import file"));
+    }
+}

src/main.rs

@@ -19,6 +19,7 @@ pub mod cli;
 pub mod command;
 pub mod error;
 pub mod export;
+pub mod import;
 pub mod options;
 pub mod outlier_detection;
 pub mod output;
@@ -40,9 +41,22 @@ fn run() -> Result<()> {
         options.sort_order_exports,
     )?;
 
+    let imported_results = if let Some(paths) = cli_arguments.get_many::<String>("import-json") {
+        let mut all = Vec::new();
+        for path in paths {
+            all.extend(import::load_results_from_json(path)?);
+        }
+        all
+    } else {
+        Vec::new()
+    };
+
     options.validate_against_command_list(&commands)?;
 
     let mut scheduler = Scheduler::new(&commands, &options, &export_manager);
+    if !imported_results.is_empty() {
+        scheduler.add_imported_results(imported_results);
+    }
     scheduler.run_benchmarks()?;
     scheduler.print_relative_speed_comparison();
     scheduler.final_export()?;