feat: deduplicate symbol maps, unwind_data and debug info

- Store pid-agnostic data in a file or json map under a mapped
`path_key` for each elf
- For each pid, store pid specific data, mostly the computed
load_bias from where each module was loaded into memory at
runtime, alongside a key to retrieve the pid-agnostic data

This way, we only write to disk relevant parts of the information.

Author: GuillaumeLagrange

.gitattributes

@@ -1,4 +1,4 @@
 testdata/perf_map/* filter=lfs diff=lfs merge=lfs -text
 *.gif binary filter=lfs diff=lfs merge=lfs -text
 src/executor/wall_time/perf/snapshots/codspeed_runner__executor__wall_time__perf__debug_info__tests__ruff_debug_info.snap filter=lfs diff=lfs merge=lfs -text
-src/executor/wall_time/perf/snapshots/codspeed_runner__executor__wall_time__perf__perf_map__tests__ruff_symbols.snap filter=lfs diff=lfs merge=lfs -text
+src/executor/wall_time/perf/snapshots/codspeed_runner__executor__wall_time__perf__module_symbols__tests__ruff_symbols.snap filter=lfs diff=lfs merge=lfs -text

.pre-commit-config.yaml

@@ -11,6 +11,7 @@ repos:
       - id: check-yaml
       - id: check-toml
       - id: check-added-large-files
+        args: ["--maxkb=1000"]
   - repo: https://github.com/doublify/pre-commit-rust
     rev: v1.0
     hooks:

Cargo.toml

@@ -10,6 +10,10 @@ default-run = "codspeed"
 name = "codspeed"
 path = "src/main.rs"
 
+[[bin]]
+name = "compare_walltime_output"
+path = "src/bin/compare_walltime_output.rs"
+
 
 [dependencies]
 anyhow = { workspace = true }

crates/runner-shared/src/debug_info.rs

@@ -21,6 +21,13 @@ impl std::fmt::Debug for DebugInfo {
     }
 }
 
+/// Per-pid mounting info referencing a deduplicated debug info entry.
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct MappedProcessDebugInfo {
+    pub debug_info_key: String,
+    pub load_bias: u64,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct ModuleDebugInfo {
     /// The path to the object file on disk (e.g. `/usr/lib/libc.so.6`)

crates/runner-shared/src/lib.rs

@@ -3,5 +3,6 @@ pub mod debug_info;
 pub mod fifo;
 pub mod metadata;
 pub mod perf_event;
+pub mod perf_map;
 pub mod unwind_data;
 pub mod walltime_results;

crates/runner-shared/src/metadata.rs

@@ -1,10 +1,15 @@
 use anyhow::Context;
+use libc::pid_t;
 use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
 use std::io::BufWriter;
 use std::path::Path;
+use std::path::PathBuf;
 
-use crate::debug_info::ModuleDebugInfo;
+use crate::debug_info::{MappedProcessDebugInfo, ModuleDebugInfo};
 use crate::fifo::MarkerType;
+use crate::perf_map::MappedProcessModuleSymbols;
+use crate::unwind_data::MappedProcessUnwindData;
 
 #[derive(Serialize, Deserialize)]
 pub struct PerfMetadata {
@@ -25,9 +30,31 @@ pub struct PerfMetadata {
     #[deprecated(note = "Use ExecutionTimestamps in the 'artifacts' module instead")]
     pub markers: Vec<MarkerType>,
 
-    /// Debug info for all modules across all processes, mapping PID to module debug info
-    #[serde(default, skip_serializing_if = "std::collections::HashMap::is_empty")]
-    pub debug_info_by_pid: std::collections::HashMap<i32, Vec<ModuleDebugInfo>>,
+    /// Kept for backward compatibility, was used before deduplication of debug info entries.
+    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
+    #[deprecated(note = "Use 'debug_info' + 'mapped_process_debug_info_by_pid' instead")]
+    pub debug_info_by_pid: HashMap<pid_t, Vec<ModuleDebugInfo>>,
+
+    /// Deduplicated debug info entries, keyed by semantic key
+    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
+    pub debug_info: HashMap<String, ModuleDebugInfo>,
+
+    /// Per-pid debug info references, mapping PID to list of debug info index + load bias
+    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
+    pub mapped_process_debug_info_by_pid: HashMap<pid_t, Vec<MappedProcessDebugInfo>>,
+
+    /// Per-pid unwind data references, mapping PID to list of unwind data index + mounting info
+    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
+    pub mapped_process_unwind_data_by_pid: HashMap<pid_t, Vec<MappedProcessUnwindData>>,
+
+    /// Per-pid symbol references, mapping PID to list of perf map index + load bias
+    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
+    pub mapped_process_module_symbols: HashMap<pid_t, Vec<MappedProcessModuleSymbols>>,
+
+    /// Mapping from semantic `path_key` to original binary path on host disk
+    /// Kept for traceability, and if we ever need to reconstruct the original paths from the keys
+    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
+    pub path_key_to_path: HashMap<String, PathBuf>,
 }
 
 impl PerfMetadata {

crates/runner-shared/src/perf_map.rs

@@ -0,0 +1,11 @@
+use serde::{Deserialize, Serialize};
+
+/// File suffix used when registering module symbols in a PID agnostic way.
+pub const SYMBOLS_MAP_SUFFIX: &str = "symbols.map";
+
+/// Per-pid mounting info referencing a deduplicated perf map entry.
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct MappedProcessModuleSymbols {
+    pub perf_map_key: String,
+    pub load_bias: u64,
+}