feat: accept a mix of entrypoint and exec targets in project config

Author: GuillaumeLagrange

schemas/codspeed.schema.json

@@ -80,18 +80,32 @@
       }
     },
     "Target": {
-      "description": "A benchmark target to execute",
+      "description": "A benchmark target to execute.\n\nEither `exec` or `entrypoint` must be specified (mutually exclusive).",
       "type": "object",
-      "required": [
-        "exec"
-      ],
       "properties": {
+        "entrypoint": {
+          "description": "Command with built-in benchmark harness (mutually exclusive with `exec`)",
+          "type": [
+            "string",
+            "null"
+          ]
+        },
         "exec": {
-          "description": "Command to execute",
-          "type": "string"
+          "description": "Command measured by exec-harness (mutually exclusive with `entrypoint`)",
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "id": {
+          "description": "Optional id to run a subset of targets (e.g. `codspeed run --bench my_id`)",
+          "type": [
+            "string",
+            "null"
+          ]
         },
         "name": {
-          "description": "Optional name for this target",
+          "description": "Optional name for this target (display purposes only)",
           "type": [
             "string",
             "null"

src/cli/exec/mod.rs

@@ -21,6 +21,18 @@ pub const DEFAULT_REPOSITORY_NAME: &str = "local-runs";
 pub const EXEC_HARNESS_COMMAND: &str = "exec-harness";
 pub const EXEC_HARNESS_VERSION: &str = "1.2.0";
 
+#[cfg(test)]
+pub fn wrap_with_exec_harness(
+    walltime_args: &exec_harness::walltime::WalltimeExecutionArgs,
+    command: &[String],
+) -> String {
+    shell_words::join(
+        std::iter::once(EXEC_HARNESS_COMMAND)
+            .chain(walltime_args.to_cli_args().iter().map(|s| s.as_str()))
+            .chain(command.iter().map(|s| s.as_str())),
+    )
+}
+
 #[derive(Args, Debug)]
 pub struct ExecArgs {
     #[command(flatten)]
@@ -80,7 +92,7 @@ fn build_orchestrator_config(
             .map(|repo| RepositoryOverride::from_arg(repo, args.shared.provider))
             .transpose()?,
         working_directory: args.shared.working_directory,
-        target,
+        targets: vec![target],
         modes,
         instruments: Instruments { mongodb: None }, // exec doesn't support MongoDB
         perf_unwinding_mode: args.shared.perf_run_args.perf_unwinding_mode,

src/cli/exec/multi_targets.rs

@@ -1,36 +1,10 @@
 use super::EXEC_HARNESS_COMMAND;
+use crate::executor::config::BenchmarkTarget;
 use crate::prelude::*;
 use crate::project_config::Target;
 use crate::project_config::WalltimeOptions;
 use exec_harness::BenchmarkCommand;
 
-/// Convert targets from project config to exec-harness JSON input format
-pub fn targets_to_exec_harness_json(
-    targets: &[Target],
-    default_walltime: Option<&WalltimeOptions>,
-) -> Result<String> {
-    let inputs: Vec<BenchmarkCommand> = targets
-        .iter()
-        .map(|target| {
-            // Parse the exec string into command parts
-            let command = shell_words::split(&target.exec)
-                .with_context(|| format!("Failed to parse command: {}", target.exec))?;
-
-            // Merge target-specific walltime options with defaults
-            let target_walltime = target.options.as_ref().and_then(|o| o.walltime.as_ref());
-            let walltime_args = merge_walltime_options(default_walltime, target_walltime);
-
-            Ok(BenchmarkCommand {
-                command,
-                name: target.name.clone(),
-                walltime_args,
-            })
-        })
-        .collect::<Result<Vec<_>>>()?;
-
-    serde_json::to_string(&inputs).context("Failed to serialize targets to JSON")
-}
-
 /// Merge default walltime options with target-specific overrides
 fn merge_walltime_options(
     default: Option<&WalltimeOptions>,
@@ -66,13 +40,35 @@ fn walltime_options_to_args(
     }
 }
 
-/// Build a shell command string that pipes targets JSON to exec-harness via stdin
-pub fn build_pipe_command(
+/// Convert project config targets into [`BenchmarkTarget`] instances.
+///
+/// Exec targets are each converted to a `BenchmarkTarget::Exec`.
+/// Entrypoint targets are each converted to a `BenchmarkTarget::Entrypoint`.
+pub fn build_benchmark_targets(
     targets: &[Target],
     default_walltime: Option<&WalltimeOptions>,
-) -> Result<String> {
-    let json = targets_to_exec_harness_json(targets, default_walltime)?;
-    Ok(build_pipe_command_from_json(&json))
+) -> Result<Vec<BenchmarkTarget>> {
+    targets
+        .iter()
+        .map(|target| match (&target.exec, &target.entrypoint) {
+            (Some(exec), None) => {
+                let command = shell_words::split(exec)
+                    .with_context(|| format!("Failed to parse command: {exec}"))?;
+                let target_walltime = target.options.as_ref().and_then(|o| o.walltime.as_ref());
+                let walltime_args = merge_walltime_options(default_walltime, target_walltime);
+                Ok(BenchmarkTarget::Exec {
+                    command,
+                    name: target.name.clone(),
+                    walltime_args,
+                })
+            }
+            (None, Some(entrypoint)) => Ok(BenchmarkTarget::Entrypoint {
+                command: entrypoint.clone(),
+                name: target.name.clone(),
+            }),
+            _ => bail!("Benchmark target must have exactly one of 'exec' or 'entrypoint' set"),
+        })
+        .collect()
 }
 
 /// Build a shell command string that pipes BenchmarkTarget::Exec variants to exec-harness via stdin

src/cli/run/mod.rs

@@ -95,7 +95,7 @@ impl RunArgs {
 
 fn build_orchestrator_config(
     args: RunArgs,
-    target: executor::BenchmarkTarget,
+    targets: Vec<executor::BenchmarkTarget>,
 ) -> Result<OrchestratorConfig> {
     let instruments = Instruments::try_from(&args)?;
     let modes = args.shared.resolve_modes()?;
@@ -115,7 +115,7 @@ fn build_orchestrator_config(
             .map(|repo| RepositoryOverride::from_arg(repo, args.shared.provider))
             .transpose()?,
         working_directory: args.shared.working_directory,
-        target,
+        targets,
         modes,
         instruments,
         perf_unwinding_mode: args.shared.perf_run_args.perf_unwinding_mode,
@@ -178,11 +178,14 @@ pub async fn run(
 
     match run_target {
         RunTarget::SingleCommand(args) => {
-            let target = executor::BenchmarkTarget::Entrypoint {
-                command: args.command.join(" "),
-                name: None,
-            };
-            let config = build_orchestrator_config(args, target)?;
+            let command = args.command.join(" ");
+            let config = build_orchestrator_config(
+                args,
+                vec![executor::BenchmarkTarget::Entrypoint {
+                    command,
+                    name: None,
+                }],
+            )?;
             let orchestrator =
                 executor::Orchestrator::new(config, codspeed_config, api_client).await?;
 
@@ -205,28 +208,9 @@ pub async fn run(
             targets,
             default_walltime,
         } => {
-            let pipe_command =
-                super::exec::multi_targets::build_pipe_command(targets, default_walltime)?;
-            // Wrap as Entrypoint since the pipe command string already includes exec-harness
-            let target = executor::BenchmarkTarget::Entrypoint {
-                command: pipe_command,
-                name: None,
-            };
-            let config = build_orchestrator_config(args, target)?;
-
-            // Ensure exec-harness is installed since config targets use it
-            crate::binary_installer::ensure_binary_installed(
-                super::exec::EXEC_HARNESS_COMMAND,
-                super::exec::EXEC_HARNESS_VERSION,
-                || {
-                    format!(
-                        "https://github.com/CodSpeedHQ/codspeed/releases/download/exec-harness-v{}/exec-harness-installer.sh",
-                        super::exec::EXEC_HARNESS_VERSION
-                    )
-                },
-            )
-            .await?;
-
+            let benchmark_targets =
+                super::exec::multi_targets::build_benchmark_targets(targets, default_walltime)?;
+            let config = build_orchestrator_config(args, benchmark_targets)?;
             super::exec::execute_config(config, api_client, codspeed_config, setup_cache_dir)
                 .await?;
         }

src/executor/config.rs

@@ -43,7 +43,7 @@ pub enum SimulationTool {
 /// Run-level configuration owned by the orchestrator.
 ///
 /// Holds all parameters that are constant across benchmark targets within a run,
-/// plus the target(s) to execute.
+/// plus the list of targets to execute.
 /// Constructed from CLI arguments and passed to [`Orchestrator::new`].
 /// Use [`OrchestratorConfig::executor_config_for_command`] to produce a per-execution [`ExecutorConfig`].
 #[derive(Debug, Clone)]
@@ -53,7 +53,7 @@ pub struct OrchestratorConfig {
     pub repository_override: Option<RepositoryOverride>,
     pub working_directory: Option<String>,
 
-    pub target: BenchmarkTarget,
+    pub targets: Vec<BenchmarkTarget>,
 
     pub modes: Vec<RunnerMode>,
     pub instruments: Instruments,
@@ -132,6 +132,25 @@ impl OrchestratorConfig {
         self.token = token;
     }
 
+    /// Compute the total number of executor runs that will be performed.
+    ///
+    /// All `Exec` targets are combined into a single invocation, while each
+    /// `Entrypoint` target runs independently. Both are multiplied by the
+    /// number of configured modes.
+    pub fn expected_run_parts_count(&self) -> u32 {
+        let has_exec = self
+            .targets
+            .iter()
+            .any(|t| matches!(t, BenchmarkTarget::Exec { .. }));
+        let entrypoint_count = self
+            .targets
+            .iter()
+            .filter(|t| matches!(t, BenchmarkTarget::Entrypoint { .. }))
+            .count();
+        let invocation_count = (if has_exec { 1 } else { 0 }) + entrypoint_count;
+        (invocation_count * self.modes.len()) as u32
+    }
+
     /// Produce a per-execution [`ExecutorConfig`] for the given command and mode.
     pub fn executor_config_for_command(&self, command: String) -> ExecutorConfig {
         ExecutorConfig {
@@ -166,10 +185,10 @@ impl OrchestratorConfig {
             token: None,
             repository_override: None,
             working_directory: None,
-            target: BenchmarkTarget::Entrypoint {
+            targets: vec![BenchmarkTarget::Entrypoint {
                 command: String::new(),
                 name: None,
-            },
+            }],
             modes: vec![RunnerMode::Simulation],
             instruments: Instruments::test(),
             perf_unwinding_mode: None,
@@ -197,6 +216,109 @@ impl ExecutorConfig {
 mod tests {
     use super::*;
 
+    #[test]
+    fn test_expected_run_parts_count() {
+        use crate::runner_mode::RunnerMode;
+
+        let base = OrchestratorConfig::test();
+
+        // Single entrypoint, single mode → 1
+        let config = OrchestratorConfig {
+            targets: vec![BenchmarkTarget::Entrypoint {
+                command: "cmd".into(),
+                name: None,
+            }],
+            modes: vec![RunnerMode::Simulation],
+            ..base.clone()
+        };
+        assert_eq!(config.expected_run_parts_count(), 1);
+
+        // Two entrypoints, single mode → 2
+        let config = OrchestratorConfig {
+            targets: vec![
+                BenchmarkTarget::Entrypoint {
+                    command: "cmd1".into(),
+                    name: None,
+                },
+                BenchmarkTarget::Entrypoint {
+                    command: "cmd2".into(),
+                    name: None,
+                },
+            ],
+            modes: vec![RunnerMode::Simulation],
+            ..base.clone()
+        };
+        assert_eq!(config.expected_run_parts_count(), 2);
+
+        // Multiple exec targets count as one invocation, single mode → 1
+        let config = OrchestratorConfig {
+            targets: vec![
+                BenchmarkTarget::Exec {
+                    command: vec!["exec1".into()],
+                    name: None,
+                    walltime_args: Default::default(),
+                },
+                BenchmarkTarget::Exec {
+                    command: vec!["exec2".into()],
+                    name: None,
+                    walltime_args: Default::default(),
+                },
+            ],
+            modes: vec![RunnerMode::Simulation],
+            ..base.clone()
+        };
+        assert_eq!(config.expected_run_parts_count(), 1);
+
+        // Mix of exec and entrypoint, single mode → 2
+        let config = OrchestratorConfig {
+            targets: vec![
+                BenchmarkTarget::Exec {
+                    command: vec!["exec1".into()],
+                    name: None,
+                    walltime_args: Default::default(),
+                },
+                BenchmarkTarget::Entrypoint {
+                    command: "cmd".into(),
+                    name: None,
+                },
+            ],
+            modes: vec![RunnerMode::Simulation],
+            ..base.clone()
+        };
+        assert_eq!(config.expected_run_parts_count(), 2);
+
+        // Single entrypoint, two modes → 2
+        #[allow(deprecated)]
+        let config = OrchestratorConfig {
+            targets: vec![BenchmarkTarget::Entrypoint {
+                command: "cmd".into(),
+                name: None,
+            }],
+            modes: vec![RunnerMode::Simulation, RunnerMode::Walltime],
+            ..base.clone()
+        };
+        assert_eq!(config.expected_run_parts_count(), 2);
+
+        // Mix of exec and entrypoint, two modes → 4
+        #[allow(deprecated)]
+        let config = OrchestratorConfig {
+            targets: vec![
+                BenchmarkTarget::Exec {
+                    command: vec!["exec1".into()],
+                    name: None,
+                    walltime_args: Default::default(),
+                },
+                BenchmarkTarget::Entrypoint {
+                    command: "cmd".into(),
+                    name: None,
+                },
+            ],
+            modes: vec![RunnerMode::Simulation, RunnerMode::Walltime],
+            ..base.clone()
+        };
+        assert_eq!(config.expected_run_parts_count(), 4);
+    }
+
     #[test]
     fn test_repository_override_from_arg() {
         let override_result =