feat(cli): add --ignore-failure to exec command

Adds -i / --ignore-failure on the exec command, inspired by hyperfine.
When set, a non-zero exit from the benchmarked process is logged as a
warning and execution continues, instead of aborting.

Reworked per review: the flag now lives in the exec-harness crate
(where the user command is directly executed), not at the executor
level. Scoped to codspeed exec only, not codspeed run.

Closes #242

Author: yaowubarbara

crates/exec-harness/src/analysis/mod.rs

@@ -27,7 +27,14 @@ pub fn perform(commands: Vec<BenchmarkCommand>) -> Result<()> {
         let status = status.context("Failed to execute command")?;
 
         if !status.success() {
-            bail!("Command exited with non-zero status: {status}");
+            if benchmark_cmd.ignore_failure {
+                warn!(
+                    "Command exited with non-zero status: {status}; \
+                     continuing because --ignore-failure is set"
+                );
+            } else {
+                bail!("Command exited with non-zero status: {status}");
+            }
         }
 
         hooks.set_executed_benchmark(&name_and_uri.uri).unwrap();
@@ -68,7 +75,14 @@ pub fn perform_with_valgrind(commands: Vec<BenchmarkCommand>) -> Result<()> {
         bail_if_command_spawned_subprocesses_under_valgrind(child.id())?;
 
         if !status.success() {
-            bail!("Command exited with non-zero status: {status}");
+            if benchmark_cmd.ignore_failure {
+                warn!(
+                    "Command exited with non-zero status: {status}; \
+                     continuing because --ignore-failure is set"
+                );
+            } else {
+                bail!("Command exited with non-zero status: {status}");
+            }
         }
     }
 

crates/exec-harness/src/lib.rs

@@ -36,6 +36,11 @@ pub struct BenchmarkCommand {
     /// Walltime execution options (flattened into the JSON object)
     #[serde(default)]
     pub walltime_args: walltime::WalltimeExecutionArgs,
+
+    /// When true, a non-zero exit from the command is logged as a warning
+    /// instead of aborting execution.
+    #[serde(default)]
+    pub ignore_failure: bool,
 }
 
 /// Read and parse benchmark commands from stdin as JSON

crates/exec-harness/src/main.rs

@@ -20,6 +20,13 @@ struct Args {
     #[arg(short, long, global = true, env = "CODSPEED_RUNNER_MODE", hide = true)]
     measurement_mode: Option<MeasurementMode>,
 
+    /// Allow the benchmarked command to exit with a non-zero status code.
+    ///
+    /// When set, a non-zero exit from the benchmarked process is logged as a
+    /// warning and measurement continues, instead of aborting.
+    #[arg(short = 'i', long, default_value = "false")]
+    ignore_failure: bool,
+
     #[command(flatten)]
     walltime_args: WalltimeExecutionArgs,
 
@@ -51,6 +58,7 @@ fn main() -> Result<()> {
             command: args.command,
             name: args.name,
             walltime_args: args.walltime_args,
+            ignore_failure: args.ignore_failure,
         }],
     };
 

crates/exec-harness/src/walltime/benchmark_loop.rs

@@ -10,6 +10,7 @@ pub fn run_rounds(
     bench_uri: String,
     command: Vec<String>,
     config: &ExecutionOptions,
+    ignore_failure: bool,
 ) -> Result<Vec<u128>> {
     let warmup_time_ns = config.warmup_time_ns;
     let hooks = InstrumentHooks::instance(INTEGRATION_NAME, INTEGRATION_VERSION);
@@ -27,7 +28,14 @@ pub fn run_rounds(
         let bench_round_end_ts_ns = InstrumentHooks::current_timestamp();
 
         if !status.success() {
-            bail!("Command exited with non-zero status: {status}");
+            if ignore_failure {
+                warn!(
+                    "Command exited with non-zero status: {status}; \
+                     continuing because --ignore-failure is set"
+                );
+            } else {
+                bail!("Command exited with non-zero status: {status}");
+            }
         }
 
         Ok((bench_round_start_ts_ns, bench_round_end_ts_ns))

crates/exec-harness/src/walltime/mod.rs

@@ -28,8 +28,12 @@ pub fn perform(commands: Vec<BenchmarkCommand>) -> Result<()> {
             ..
         } = name_and_uri;
 
-        let times_per_round_ns =
-            benchmark_loop::run_rounds(bench_uri.clone(), cmd.command, &execution_options)?;
+        let times_per_round_ns = benchmark_loop::run_rounds(
+            bench_uri.clone(),
+            cmd.command,
+            &execution_options,
+            cmd.ignore_failure,
+        )?;
 
         // Collect walltime results
         let max_time_ns = times_per_round_ns.iter().copied().max();

src/cli/exec/mod.rs

@@ -30,6 +30,14 @@ pub struct ExecArgs {
     #[arg(long)]
     pub name: Option<String>,
 
+    /// Allow the benchmarked command to exit with a non-zero status code.
+    ///
+    /// When set, a non-zero exit from the benchmarked process is logged as a
+    /// warning and measurement continues, instead of aborting. Mirrors
+    /// hyperfine's `-i / --ignore-failure`.
+    #[arg(short = 'i', long, default_value = "false")]
+    pub ignore_failure: bool,
+
     /// The command to execute with the exec harness
     pub command: Vec<String>,
 }
@@ -106,6 +114,7 @@ pub async fn run(
         command: merged_args.command.clone(),
         name: merged_args.name.clone(),
         walltime_args: merged_args.walltime_args.clone(),
+        ignore_failure: merged_args.ignore_failure,
     };
     let config = build_orchestrator_config(
         merged_args,

src/cli/exec/multi_targets.rs

@@ -59,6 +59,7 @@ pub fn build_benchmark_targets(
                     command,
                     name: target.name.clone(),
                     walltime_args,
+                    ignore_failure: false,
                 })
             }
             TargetCommand::Entrypoint { entrypoint } => Ok(BenchmarkTarget::Entrypoint {
@@ -80,10 +81,12 @@ pub fn build_exec_targets_pipe_command(
                 command,
                 name,
                 walltime_args,
+                ignore_failure,
             } => Ok(BenchmarkCommand {
                 command: command.clone(),
                 name: name.clone(),
                 walltime_args: walltime_args.clone(),
+                ignore_failure: *ignore_failure,
             }),
             crate::executor::config::BenchmarkTarget::Entrypoint { .. } => {
                 bail!("Entrypoint targets cannot be used with exec-harness pipe command")

src/executor/config.rs

@@ -22,6 +22,8 @@ pub enum BenchmarkTarget {
         command: Vec<String>,
         name: Option<String>,
         walltime_args: exec_harness::walltime::WalltimeExecutionArgs,
+        /// When true, a non-zero exit from the command is tolerated.
+        ignore_failure: bool,
     },
     /// A command with built-in harness (e.g. `pytest --codspeed src`)
     Entrypoint {
@@ -286,11 +288,13 @@ mod tests {
                     command: vec!["exec1".into()],
                     name: None,
                     walltime_args: Default::default(),
+                    ignore_failure: false,
                 },
                 BenchmarkTarget::Exec {
                     command: vec!["exec2".into()],
                     name: None,
                     walltime_args: Default::default(),
+                    ignore_failure: false,
                 },
             ],
             modes: vec![RunnerMode::Simulation],
@@ -305,6 +309,7 @@ mod tests {
                     command: vec!["exec1".into()],
                     name: None,
                     walltime_args: Default::default(),
+                    ignore_failure: false,
                 },
                 BenchmarkTarget::Entrypoint {
                     command: "cmd".into(),
@@ -336,6 +341,7 @@ mod tests {
                     command: vec!["exec1".into()],
                     name: None,
                     walltime_args: Default::default(),
+                    ignore_failure: false,
                 },
                 BenchmarkTarget::Entrypoint {
                     command: "cmd".into(),