feat(cli): add --ignore-failure to allow non-zero exit codes

Adds -i / --ignore-failure on exec and run commands, inspired by
hyperfine flag of the same name. When set, a non-zero exit from
the benchmarked process is logged as a warning and execution
continues, instead of aborting.

The gate lives on ExecutorConfig, mirroring allow_empty, and covers
both the valgrind and walltime paths. A failure from valgrind itself
still aborts - only the wrapped command status is relaxed.

Closes #242

Author: yaowubarbara

src/cli/exec/mod.rs

@@ -85,6 +85,7 @@ fn build_orchestrator_config(
         skip_run: args.shared.skip_run,
         skip_setup: args.shared.skip_setup,
         allow_empty: args.shared.allow_empty,
+        ignore_failure: args.shared.ignore_failure,
         go_runner_version: args.shared.go_runner_version,
         show_full_output: args.shared.show_full_output,
         poll_results_options,

src/cli/run/mod.rs

@@ -66,6 +66,7 @@ impl RunArgs {
                 skip_run: false,
                 skip_setup: false,
                 allow_empty: false,
+                ignore_failure: false,
                 go_runner_version: None,
                 show_full_output: false,
                 base: None,
@@ -119,6 +120,7 @@ fn build_orchestrator_config(
         skip_run: args.shared.skip_run,
         skip_setup: args.shared.skip_setup,
         allow_empty: args.shared.allow_empty,
+        ignore_failure: args.shared.ignore_failure,
         go_runner_version: args.shared.go_runner_version,
         show_full_output: args.shared.show_full_output,
         poll_results_options,

src/cli/shared.rs

@@ -102,6 +102,14 @@ pub struct ExecAndRunSharedArgs {
     #[arg(long, default_value = "false", hide = true)]
     pub allow_empty: bool,
 
+    /// Allow the benchmarked command to exit with a non-zero status code.
+    ///
+    /// When set, a non-zero exit from the benchmarked process is logged as a
+    /// warning and measurement continues, instead of aborting. Mirrors
+    /// hyperfine's `-i / --ignore-failure`.
+    #[arg(short = 'i', long, default_value = "false")]
+    pub ignore_failure: bool,
+
     /// The version of the go-runner to use (e.g., 1.2.3, 1.0.0-beta.1)
     /// If not specified, the latest version will be installed
     #[arg(long, env = "CODSPEED_GO_RUNNER_VERSION", value_parser = parse_version)]

src/executor/config.rs

@@ -71,6 +71,9 @@ pub struct OrchestratorConfig {
     pub skip_setup: bool,
     /// If true, allow execution even when no benchmarks are found
     pub allow_empty: bool,
+    /// If true, the benchmarked command is allowed to exit with a non-zero
+    /// status code; a warning is logged and execution continues.
+    pub ignore_failure: bool,
     /// The version of go-runner to install (if None, installs latest)
     pub go_runner_version: Option<Version>,
     /// If true, show full executor output instead of a rolling buffer window
@@ -106,6 +109,9 @@ pub struct ExecutorConfig {
     pub skip_setup: bool,
     /// If true, allow execution even when no benchmarks are found
     pub allow_empty: bool,
+    /// If true, the benchmarked command is allowed to exit with a non-zero
+    /// status code; a warning is logged and execution continues.
+    pub ignore_failure: bool,
     /// The version of go-runner to install (if None, installs latest)
     pub go_runner_version: Option<Version>,
     /// Additional environment variables forwarded to executor subprocesses.
@@ -187,6 +193,7 @@ impl OrchestratorConfig {
             skip_run: self.skip_run,
             skip_setup: self.skip_setup,
             allow_empty: self.allow_empty,
+            ignore_failure: self.ignore_failure,
             go_runner_version: self.go_runner_version.clone(),
             extra_env: self.extra_env.clone(),
             enable_introspection,
@@ -224,6 +231,7 @@ impl OrchestratorConfig {
             skip_run: false,
             skip_setup: false,
             allow_empty: false,
+            ignore_failure: false,
             go_runner_version: None,
             show_full_output: false,
             poll_results_options: PollResultsOptions::new(false, None),

src/executor/valgrind/measure.rs

@@ -189,7 +189,14 @@ pub async fn measure(
     };
     debug!("Program exit code = {cmd_status}");
     if cmd_status != 0 {
-        bail!("failed to execute the benchmark process, exit code: {cmd_status}");
+        if config.ignore_failure {
+            warn!(
+                "benchmark process exited with non-zero code {cmd_status}; \
+                 continuing because --ignore-failure was set"
+            );
+        } else {
+            bail!("failed to execute the benchmark process, exit code: {cmd_status}");
+        }
     }
 
     Ok(())

src/executor/wall_time/executor.rs

@@ -180,7 +180,14 @@ impl Executor for WallTimeExecutor {
         debug!("cmd exit status: {status:?}");
 
         if !status.success() {
-            bail!("failed to execute the benchmark process: {status}");
+            if execution_context.config.ignore_failure {
+                warn!(
+                    "benchmark process exited with {status}; \
+                     continuing because --ignore-failure was set"
+                );
+            } else {
+                bail!("failed to execute the benchmark process: {status}");
+            }
         }
 
         Ok(())