-
Notifications
You must be signed in to change notification settings - Fork 20
Expand file tree
/
Copy pathconfig.rs
More file actions
375 lines (343 loc) · 12.8 KB
/
config.rs
File metadata and controls
375 lines (343 loc) · 12.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
use crate::cli::UnwindingMode;
use crate::instruments::Instruments;
use crate::prelude::*;
use crate::run_environment::RepositoryProvider;
use crate::runner_mode::RunnerMode;
use crate::upload::poll_results::PollResultsOptions;
use clap::ValueEnum;
use semver::Version;
use std::collections::HashMap;
use std::path::PathBuf;
use url::Url;
/// A benchmark target from project configuration.
///
/// Defines how a benchmark is executed:
/// - `Exec`: a plain command measured by exec-harness (all exec targets share one invocation)
/// - `Entrypoint`: a command that already contains benchmark harnessing (run independently)
#[derive(Debug, Clone)]
pub enum BenchmarkTarget {
/// A command measured by exec-harness (e.g. `ls -al /nix/store`)
Exec {
command: Vec<String>,
name: Option<String>,
walltime_args: exec_harness::walltime::WalltimeExecutionArgs,
},
/// A command with built-in harness (e.g. `pytest --codspeed src`)
Entrypoint {
command: String,
// We do not use it yet, temporarily allow
#[allow(dead_code)]
name: Option<String>,
},
}
/// The Valgrind tool to use for simulation mode.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, ValueEnum)]
pub enum SimulationTool {
/// Use Callgrind for aggregated text-based cost profiles (.out files)
#[default]
Callgrind,
/// Use Tracegrind for streaming binary event traces (.tgtrace files)
Tracegrind,
}
/// Run-level configuration owned by the orchestrator.
///
/// Holds all parameters that are constant across benchmark targets within a run,
/// plus the list of targets to execute.
/// Constructed from CLI arguments and passed to [`Orchestrator::new`].
/// Use [`OrchestratorConfig::executor_config_for_command`] to produce a per-execution [`ExecutorConfig`].
#[derive(Debug, Clone)]
pub struct OrchestratorConfig {
pub upload_url: Url,
pub token: Option<String>,
pub repository_override: Option<RepositoryOverride>,
pub working_directory: Option<String>,
pub targets: Vec<BenchmarkTarget>,
pub modes: Vec<RunnerMode>,
pub instruments: Instruments,
pub enable_perf: bool,
/// Stack unwinding mode for perf (if enabled)
pub perf_unwinding_mode: Option<UnwindingMode>,
pub simulation_tool: SimulationTool,
pub profile_folder: Option<PathBuf>,
pub skip_upload: bool,
pub skip_run: bool,
pub skip_setup: bool,
/// If true, allow execution even when no benchmarks are found
pub allow_empty: bool,
/// The version of go-runner to install (if None, installs latest)
pub go_runner_version: Option<Version>,
/// If true, show full executor output instead of a rolling buffer window
pub show_full_output: bool,
/// Options controlling post-upload result polling and display
pub poll_results_options: PollResultsOptions,
/// Additional environment variables forwarded to executor subprocesses.
pub extra_env: HashMap<String, String>,
/// Enable valgrind's --fair-sched option.
pub fair_sched: bool,
}
/// Per-execution configuration passed to executors.
///
/// Produced by [`OrchestratorConfig::executor_config_for_command`]; holds the `command` string
/// that the executor will run, plus executor-specific fields.
/// Fields that are only needed at the orchestrator level (e.g. `upload_url`,
/// `skip_upload`, `repository_override`) live on [`OrchestratorConfig`].
#[derive(Debug, Clone)]
pub struct ExecutorConfig {
pub token: Option<String>,
pub working_directory: Option<String>,
pub command: String,
pub instruments: Instruments,
pub enable_perf: bool,
/// Stack unwinding mode for perf (if enabled)
pub perf_unwinding_mode: Option<UnwindingMode>,
pub simulation_tool: SimulationTool,
pub skip_run: bool,
pub skip_setup: bool,
/// If true, allow execution even when no benchmarks are found
pub allow_empty: bool,
/// The version of go-runner to install (if None, installs latest)
pub go_runner_version: Option<Version>,
/// Additional environment variables forwarded to executor subprocesses.
pub extra_env: HashMap<String, String>,
/// Whether to enable language-level introspection (Node.js, Go wrappers in PATH).
/// Disabled for exec-harness targets since they don't need it.
pub enable_introspection: bool,
/// Enable valgrind's --fair-sched option.
pub fair_sched: bool,
}
#[derive(Debug, Clone, PartialEq)]
pub struct RepositoryOverride {
pub owner: String,
pub repository: String,
pub repository_provider: RepositoryProvider,
}
impl RepositoryOverride {
/// Creates a RepositoryOverride from an "owner/repository" string
pub fn from_arg(
repository_and_owner: String,
provider: Option<RepositoryProvider>,
) -> Result<Self> {
let (owner, repository) = repository_and_owner
.split_once('/')
.context("Invalid owner/repository format")?;
Ok(Self {
owner: owner.to_string(),
repository: repository.to_string(),
repository_provider: provider.unwrap_or_default(),
})
}
}
pub const DEFAULT_UPLOAD_URL: &str = "https://api.codspeed.io/upload";
impl OrchestratorConfig {
pub fn set_token(&mut self, token: Option<String>) {
self.token = token;
}
/// Compute the total number of executor runs that will be performed.
///
/// All `Exec` targets are combined into a single invocation, while each
/// `Entrypoint` target runs independently. Both are multiplied by the
/// number of configured modes.
pub fn expected_run_parts_count(&self) -> u32 {
let has_exec = self
.targets
.iter()
.any(|t| matches!(t, BenchmarkTarget::Exec { .. }));
let entrypoint_count = self
.targets
.iter()
.filter(|t| matches!(t, BenchmarkTarget::Entrypoint { .. }))
.count();
let invocation_count = (if has_exec { 1 } else { 0 }) + entrypoint_count;
(invocation_count * self.modes.len()) as u32
}
/// Produce a per-execution [`ExecutorConfig`] for the given command and mode.
///
/// `enable_introspection` controls whether language-level wrappers (Node.js, Go)
/// are injected into `PATH`. This should be `false` for exec-harness targets.
pub fn executor_config_for_command(
&self,
command: String,
enable_introspection: bool,
) -> ExecutorConfig {
ExecutorConfig {
token: self.token.clone(),
working_directory: self.working_directory.clone(),
command,
instruments: self.instruments.clone(),
enable_perf: self.enable_perf,
perf_unwinding_mode: self.perf_unwinding_mode,
simulation_tool: self.simulation_tool,
skip_run: self.skip_run,
skip_setup: self.skip_setup,
allow_empty: self.allow_empty,
go_runner_version: self.go_runner_version.clone(),
extra_env: self.extra_env.clone(),
enable_introspection,
fair_sched: self.fair_sched,
}
}
}
impl ExecutorConfig {
pub fn set_token(&mut self, token: Option<String>) {
self.token = token;
}
}
#[cfg(test)]
impl OrchestratorConfig {
/// Constructs a new `OrchestratorConfig` with default values for testing purposes
pub fn test() -> Self {
Self {
upload_url: Url::parse(DEFAULT_UPLOAD_URL).unwrap(),
token: None,
repository_override: None,
working_directory: None,
targets: vec![BenchmarkTarget::Entrypoint {
command: String::new(),
name: None,
}],
modes: vec![RunnerMode::Simulation],
instruments: Instruments::test(),
perf_unwinding_mode: None,
enable_perf: false,
simulation_tool: SimulationTool::default(),
profile_folder: None,
skip_upload: false,
skip_run: false,
skip_setup: false,
allow_empty: false,
go_runner_version: None,
show_full_output: false,
poll_results_options: PollResultsOptions::new(false, None),
extra_env: HashMap::new(),
fair_sched: false,
}
}
}
#[cfg(test)]
impl ExecutorConfig {
/// Constructs a new `ExecutorConfig` with default values for testing purposes
pub fn test() -> Self {
OrchestratorConfig::test().executor_config_for_command("".into(), true)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_expected_run_parts_count() {
use crate::runner_mode::RunnerMode;
let base = OrchestratorConfig::test();
// Single entrypoint, single mode → 1
let config = OrchestratorConfig {
targets: vec![BenchmarkTarget::Entrypoint {
command: "cmd".into(),
name: None,
}],
modes: vec![RunnerMode::Simulation],
..base.clone()
};
assert_eq!(config.expected_run_parts_count(), 1);
// Two entrypoints, single mode → 2
let config = OrchestratorConfig {
targets: vec![
BenchmarkTarget::Entrypoint {
command: "cmd1".into(),
name: None,
},
BenchmarkTarget::Entrypoint {
command: "cmd2".into(),
name: None,
},
],
modes: vec![RunnerMode::Simulation],
..base.clone()
};
assert_eq!(config.expected_run_parts_count(), 2);
// Multiple exec targets count as one invocation, single mode → 1
let config = OrchestratorConfig {
targets: vec![
BenchmarkTarget::Exec {
command: vec!["exec1".into()],
name: None,
walltime_args: Default::default(),
},
BenchmarkTarget::Exec {
command: vec!["exec2".into()],
name: None,
walltime_args: Default::default(),
},
],
modes: vec![RunnerMode::Simulation],
..base.clone()
};
assert_eq!(config.expected_run_parts_count(), 1);
// Mix of exec and entrypoint, single mode → 2
let config = OrchestratorConfig {
targets: vec![
BenchmarkTarget::Exec {
command: vec!["exec1".into()],
name: None,
walltime_args: Default::default(),
},
BenchmarkTarget::Entrypoint {
command: "cmd".into(),
name: None,
},
],
modes: vec![RunnerMode::Simulation],
..base.clone()
};
assert_eq!(config.expected_run_parts_count(), 2);
// Single entrypoint, two modes → 2
#[allow(deprecated)]
let config = OrchestratorConfig {
targets: vec![BenchmarkTarget::Entrypoint {
command: "cmd".into(),
name: None,
}],
modes: vec![RunnerMode::Simulation, RunnerMode::Walltime],
..base.clone()
};
assert_eq!(config.expected_run_parts_count(), 2);
// Mix of exec and entrypoint, two modes → 4
#[allow(deprecated)]
let config = OrchestratorConfig {
targets: vec![
BenchmarkTarget::Exec {
command: vec!["exec1".into()],
name: None,
walltime_args: Default::default(),
},
BenchmarkTarget::Entrypoint {
command: "cmd".into(),
name: None,
},
],
modes: vec![RunnerMode::Simulation, RunnerMode::Walltime],
..base.clone()
};
assert_eq!(config.expected_run_parts_count(), 4);
}
#[test]
fn test_repository_override_from_arg() {
let override_result =
RepositoryOverride::from_arg("CodSpeedHQ/codspeed".to_string(), None).unwrap();
assert_eq!(override_result.owner, "CodSpeedHQ");
assert_eq!(override_result.repository, "codspeed");
assert_eq!(
override_result.repository_provider,
RepositoryProvider::GitHub
);
let override_with_provider = RepositoryOverride::from_arg(
"CodSpeedHQ/codspeed".to_string(),
Some(RepositoryProvider::GitLab),
)
.unwrap();
assert_eq!(
override_with_provider.repository_provider,
RepositoryProvider::GitLab
);
let result = RepositoryOverride::from_arg("CodSpeedHQ_runner".to_string(), None);
assert!(result.is_err());
}
}