test: stabilize callgrind topology snapshots

Author: not-matthias

callgrind-utils/src/lib.rs

@@ -2,4 +2,5 @@ pub mod error;
 pub mod model;
 mod normalize;
 pub mod parser;
+mod redact;
 pub mod serialize;

callgrind-utils/src/redact.rs

@@ -0,0 +1,132 @@
+use super::model::{CallGraph, Node};
+
+const UNKNOWN: &str = "???";
+
+impl CallGraph {
+    /// Redact host-specific node identity and rebuild the canonical graph.
+    pub fn redact(self) -> CallGraph {
+        let CallGraph { nodes, edges } = self;
+        let mut nodes = nodes;
+        let mut edges = edges;
+
+        for node in &mut nodes {
+            redact_node(node);
+        }
+
+        for edge in &mut edges {
+            redact_node(&mut edge.caller);
+            redact_node(&mut edge.callee);
+        }
+
+        CallGraph::from_parts(nodes, edges)
+    }
+}
+
+fn redact_node(node: &mut Node) {
+    node.object = redact_object(&node.object);
+
+    if is_runtime_object(&node.object) {
+        node.function = UNKNOWN.to_string();
+        node.file = UNKNOWN.to_string();
+        return;
+    }
+
+    node.function = redact_function(&node.function);
+}
+
+fn redact_function(function: &str) -> String {
+    let function = strip_symbol_version(function);
+    if is_hex_address(function) {
+        return "<unsymbolicated>".to_string();
+    }
+    function.to_string()
+}
+
+fn strip_symbol_version(function: &str) -> &str {
+    for marker in ["@@", "@"] {
+        let Some(index) = function.find(marker) else {
+            continue;
+        };
+        let version = &function[index + marker.len()..];
+        if is_symbol_version(version) {
+            return &function[..index];
+        }
+    }
+    function
+}
+
+fn is_symbol_version(version: &str) -> bool {
+    let Some(first) = version.chars().next() else {
+        return false;
+    };
+    (first.is_ascii_alphanumeric() || first == '_')
+        && version
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '.')
+}
+
+fn is_hex_address(function: &str) -> bool {
+    let Some(hex) = function.strip_prefix("0x") else {
+        return false;
+    };
+    !hex.is_empty() && hex.chars().all(|c| c.is_ascii_hexdigit())
+}
+
+fn redact_object(object: &str) -> String {
+    if is_loader_soname(object) {
+        return "ld-linux".to_string();
+    }
+    if let Some(module) = cpython_extension_module(object) {
+        return format!("{module}.cpython.so");
+    }
+    if is_libffi_soname(object) {
+        return "libffi.so".to_string();
+    }
+    object.to_string()
+}
+
+fn is_runtime_object(object: &str) -> bool {
+    object == "ld-linux" || is_libc_soname(object)
+}
+
+fn is_libc_soname(object: &str) -> bool {
+    let Some(version) = object.strip_prefix("libc.so.") else {
+        return false;
+    };
+    !version.is_empty() && version.chars().all(|c| c.is_ascii_digit())
+}
+
+fn cpython_extension_module(object: &str) -> Option<&str> {
+    let (module, suffix) = object.split_once(".cpython-")?;
+    let abi = suffix.strip_suffix(".so")?;
+    if module.is_empty() || abi.is_empty() {
+        return None;
+    }
+    Some(module)
+}
+
+fn is_libffi_soname(object: &str) -> bool {
+    let Some(version) = object.strip_prefix("libffi.so.") else {
+        return false;
+    };
+    !version.is_empty()
+        && version.chars().all(|c| c.is_ascii_digit() || c == '.')
+        && version.chars().any(|c| c.is_ascii_digit())
+}
+fn is_loader_soname(object: &str) -> bool {
+    let Some(rest) = object.strip_prefix("ld-") else {
+        return false;
+    };
+    let Some(index) = rest.find(".so.") else {
+        return false;
+    };
+
+    let loader_name = &rest[..index];
+    let soname_version = &rest[index + ".so.".len()..];
+    !loader_name.is_empty()
+        && loader_name
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_')
+        && !soname_version.is_empty()
+        && soname_version.chars().all(|c| c.is_ascii_digit())
+}

callgrind-utils/tests/python_callgraph.rs

@@ -1,22 +1,26 @@
-//! Structural test over the Python fixture (`testdata/recursion.py`).
+//! Topology-only snapshot of the Python fixture's call graph.
 //!
-//! Unlike the C snapshot tests, a live Python run can't be a golden snapshot:
-//! Callgrind sees the CPython interpreter's C frames (version- and
-//! platform-specific), never the Python functions. The fixture drives Callgrind
-//! the way pytest-codspeed does: at runtime it adds the Python runtime objects
-//! (libpython + the python executable) to the obj-skip list via the
-//! `CALLGRIND_ADD_OBJ_SKIP` client request, so the interpreter's own frames are
-//! folded into their callers and don't obfuscate the graph. This test profiles
-//! the fixture live and asserts structural properties: instrumentation started
-//! at the shim, and the runtime obj-skip removed the interpreter.
+//! Mirrors `snapshot.rs` for the C fixtures: profile `testdata/recursion.py`
+//! live under the in-repo Callgrind, parse, and snapshot a topology-only
+//! view (nodes + caller/callee indices, no `call_count`; see below).
+//!
+//! Callgrind records the CPython interpreter's C frames, not the Python
+//! functions: the interpreter loop is obj-skipped at runtime via the `clgctl`
+//! shim's `CALLGRIND_ADD_OBJ_SKIP`, so what remains is the ctypes/libffi/libc
+//! C-residual around the `clg_start`/`clg_stop` shim. The graph shape
+//! (nodes + caller/callee indices) is stable after `CallGraph::redact()`: libc/ld
+//! debug-derived fields collapse to `???`, and CPython extension / libffi object
+//! suffixes are normalized. `call_count` on the seed-reconstructed residual edges
+//! drifts run-to-run (loader/PLT timing), so it is stripped and the snapshot is
+//! topology-only.
 //!
 //! Requires a built `./vg-in-place` at the repo root and `cc`. Silently skips
 //! when `python3` is not on PATH (mirrors the `.vgtest` `prereq` guards).
 use std::io::Cursor;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 
-use callgrind_utils::model::{CallGraph, ParseOptions};
+use callgrind_utils::model::{CallGraph, Node, ParseOptions};
 
 /// Repo root: this crate lives at `<repo>/callgrind-utils`.
 fn repo_root() -> PathBuf {
@@ -44,34 +48,6 @@ fn have_python3() -> bool {
         .unwrap_or(false)
 }
 
-/// The basenames of the objects `recursion.py` adds to the obj-skip list:
-/// libpython and the python executable, resolved exactly as the fixture does
-/// (realpath, then basename, matching Callgrind's normalized object names).
-fn skipped_runtime_objects() -> Vec<String> {
-    let script = "\
-import os, sys, sysconfig
-ld = sysconfig.get_config_var('LDLIBRARY')
-libdir = sysconfig.get_config_var('LIBDIR')
-cands = [os.path.join(libdir, ld) if ld and libdir else None,
-         os.path.join(sys.prefix, 'lib', ld) if ld else None]
-lp = next((p for p in cands if p and os.path.exists(p)), None)
-for p in (lp, sys.executable):
-    if p:
-        print(os.path.basename(os.path.realpath(p)))
-";
-    let out = Command::new("python3")
-        .arg("-c")
-        .arg(script)
-        .output()
-        .expect("run python3 to resolve runtime objects");
-    assert!(out.status.success(), "python3 obj resolution failed");
-    String::from_utf8(out.stdout)
-        .expect("utf8")
-        .lines()
-        .map(str::to_owned)
-        .collect()
-}
-
 /// Compile the Callgrind client-request shim the Python fixture loads via
 /// `ctypes`, as a shared library against the in-repo `callgrind.h`.
 fn compile_clgctl() -> PathBuf {
@@ -116,59 +92,54 @@ fn run_python(clgctl: &Path) -> String {
         .status()
         .unwrap_or_else(|e| panic!("failed to spawn vg-in-place: {e}"));
     assert!(status.success(), "vg-in-place exited with {status}");
-    std::fs::read_to_string(&out_file).unwrap_or_else(|e| panic!("read {}: {e}", out_file.display()))
+    std::fs::read_to_string(&out_file)
+        .unwrap_or_else(|e| panic!("read {}: {e}", out_file.display()))
+}
+
+/// Topology-only JSON view: `nodes` then `edges` by index, no `call_count`.
+#[derive(serde::Serialize)]
+struct TopologyEdge {
+    caller: usize,
+    callee: usize,
+}
+
+#[derive(serde::Serialize)]
+struct TopologyGraph<'a> {
+    nodes: &'a [Node],
+    edges: Vec<TopologyEdge>,
 }
 
 #[test]
-fn python_runtime_is_obj_skipped() {
+fn python_topology_json() {
     if !have_python3() {
-        eprintln!("skipping python_runtime_is_obj_skipped: python3 not on PATH");
+        eprintln!("skipping python_topology_json: python3 not on PATH");
         return;
     }
 
-    let skipped = skipped_runtime_objects();
-    assert!(
-        !skipped.is_empty(),
-        "expected at least the python executable to be skipped"
-    );
-
     let clgctl = compile_clgctl();
     let raw = run_python(&clgctl);
     let graph = CallGraph::parse(Cursor::new(raw.as_str()), &ParseOptions::default())
-        .expect("parse python callgrind output");
-
-    assert!(!graph.nodes().is_empty(), "expected a non-empty node set");
-    assert!(!graph.edges().is_empty(), "expected a non-empty edge set");
-
-    // The shim that fired START is captured, so instrumentation began exactly
-    // where the fixture asked, and it is wired into the graph (not dropped as
-    // an orphan root): the seeder reconstructed the native stack at the OFF->ON
-    // transition.
-    assert!(
-        graph.nodes().iter().any(|n| n.function == "clg_start"),
-        "clg_start (instrumentation shim) missing from graph"
-    );
-    assert!(
-        graph
+        .unwrap_or_else(|e| panic!("parse python callgrind output: {e:?}"))
+        .redact();
+    let nodes = graph.nodes();
+    let topology = TopologyGraph {
+        nodes,
+        edges: graph
             .edges()
             .iter()
-            .any(|e| e.caller.function == "clg_start" || e.callee.function == "clg_start"),
-        "clg_start has no edges - START frame was not captured"
-    );
+            .map(|e| TopologyEdge {
+                caller: nodes
+                    .iter()
+                    .position(|x| x == &e.caller)
+                    .expect("caller node present"),
+                callee: nodes
+                    .iter()
+                    .position(|x| x == &e.callee)
+                    .expect("callee node present"),
+            })
+            .collect(),
+    };
+    let json = serde_json::to_string_pretty(&topology).expect("serialize");
 
-    // The runtime obj-skip folded the Python runtime out: no node belongs to
-    // libpython or the python executable, and the interpreter loop is gone.
-    for obj in &skipped {
-        assert!(
-            graph.nodes().iter().all(|n| &n.object != obj),
-            "obj-skip failed: {obj} still present as a node object"
-        );
-    }
-    assert!(
-        !graph
-            .nodes()
-            .iter()
-            .any(|n| n.function.starts_with("_PyEval_EvalFrameDefault")),
-        "interpreter loop _PyEval_EvalFrameDefault should have been obj-skipped"
-    );
+    insta::assert_snapshot!("recursion_py__topology_json", json);
 }

callgrind-utils/tests/snapshot.rs

@@ -89,7 +89,8 @@ fn fixture_canonical_json(#[case] stem: &str) {
     let bin = compile_fixture(stem);
     let raw = run_callgrind(&bin);
     let graph = CallGraph::parse(Cursor::new(raw.as_str()), &ParseOptions::default())
-        .unwrap_or_else(|e| panic!("parse {stem} callgrind output: {e:?}"));
+        .unwrap_or_else(|e| panic!("parse {stem} callgrind output: {e:?}"))
+        .redact();
     let json = graph.to_json().expect("to_json");
 
     insta::assert_snapshot!(format!("{stem}__json"), json);
@@ -129,7 +130,8 @@ fn fixture_full_trace(#[case] stem: &str) {
     let bin = compile_fixture(stem);
     let raw = run_callgrind_full(&bin);
     let graph = CallGraph::parse(Cursor::new(raw.as_str()), &ParseOptions::default())
-        .unwrap_or_else(|e| panic!("parse {stem} full callgrind output: {e:?}"));
+        .unwrap_or_else(|e| panic!("parse {stem} full callgrind output: {e:?}"))
+        .redact();
     let json = graph.to_json().expect("to_json");
 
     insta::assert_snapshot!(format!("{stem}_full__json"), json);