Skip to content

Commit ad995d3

Browse files
nikicatclaude
andcommitted
gix-ref: profiling helper for wide-refs lookup loops
Adds an `#[ignore]`'d test `loose_stat_overhead_profile` that simulates a fetch's `update_refs` loop against the existing 150k-packed-ref fixture, reporting per-strategy timings: - `store.try_find(name)` — current path, stats the loose-ref file before falling through to packed. - A pre-built `HashSet` of loose names + `packed.try_find(name)` direct, which is the lookup shape a hypothetical `file::Store`-level loose-name cache would produce. On warm cache with this PR's packed buffer index applied, ~80% of the remaining lookup wall time is the per-call loose `stat` (most of which returns ENOENT against the dentry cache). The absolute reclaimable time is workload-dependent — typically ~0.3 s warm-cache, more on cold-cache or network filesystems — so the helper lets future work decide whether a loose-refs follow-up is worth its complexity, given local numbers. Run with: cargo test -p gix-ref --release --features sha1 --test refs \\ -- --ignored --nocapture loose_stat_overhead_profile Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
1 parent e9d20d5 commit ad995d3

1 file changed

Lines changed: 106 additions & 0 deletions

File tree

gix-ref/tests/refs/packed/find.rs

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,112 @@ fn index_path_surfaces_parse_failures_on_miss() -> crate::Result {
272272
Ok(())
273273
}
274274

275+
/// Profiling helper: simulate a wide-refs fetch's `update_refs` loop and
276+
/// break down where the time goes. Run with:
277+
///
278+
/// ```text
279+
/// cargo test -p gix-ref --release --features sha1 --test refs \
280+
/// -- --ignored --nocapture loose_stat_overhead_profile
281+
/// ```
282+
///
283+
/// Reports total time for two strategies against a ~150k-packed-ref repo:
284+
/// A. `store.try_find(name)` — the current `try_find_reference` path,
285+
/// which `stat`s the loose-ref file before falling through to packed.
286+
/// B. Pre-built `HashSet` of loose names → bypass the stat when the name
287+
/// isn't in it, then `packed.try_find(name)` directly.
288+
///
289+
/// The delta (A − B) is the loose-stat overhead that a hypothetical
290+
/// `file::Store`-level loose-name cache (or a caller-side enumeration like
291+
/// the original #2605) would reclaim on top of this PR's packed-buffer
292+
/// index. Useful for deciding whether such a follow-up is worth its
293+
/// complexity, since the answer is workload-dependent (warm vs cold cache,
294+
/// loose-ref count, filesystem characteristics).
295+
#[test]
296+
#[ignore = "profiling only; expensive and times vary across machines"]
297+
fn loose_stat_overhead_profile() -> crate::Result {
298+
use std::collections::HashSet;
299+
use std::time::Instant;
300+
301+
let store = store_at("make_repository_with_lots_of_packed_refs.sh")?;
302+
let packed = store.open_packed_buffer()?.expect("packed-refs present");
303+
304+
// Collect all 150k packed ref names — these are what fetch's update_refs
305+
// would iterate over.
306+
let collect_start = Instant::now();
307+
let packed_names: Vec<_> = packed
308+
.iter()?
309+
.filter_map(Result::ok)
310+
.map(|r| r.name.to_owned())
311+
.collect();
312+
eprintln!(
313+
"Collected {} packed ref names in {:?}",
314+
packed_names.len(),
315+
collect_start.elapsed()
316+
);
317+
318+
// Strategy A — current behavior, with loose-stat per call.
319+
let warmup = Instant::now();
320+
for name in packed_names.iter().take(1000) {
321+
let _ = store.try_find(name.as_ref())?;
322+
}
323+
eprintln!("Warmup (1000 lookups): {:?}", warmup.elapsed());
324+
325+
let a_start = Instant::now();
326+
let mut a_found = 0usize;
327+
for name in &packed_names {
328+
if store.try_find(name.as_ref())?.is_some() {
329+
a_found += 1;
330+
}
331+
}
332+
let a_elapsed = a_start.elapsed();
333+
eprintln!(
334+
"Strategy A (store.try_find — per-call loose stat): {:?}, found {} of {} ({:.1} µs/lookup)",
335+
a_elapsed,
336+
a_found,
337+
packed_names.len(),
338+
a_elapsed.as_secs_f64() * 1_000_000.0 / packed_names.len() as f64,
339+
);
340+
341+
// Strategy B — simulated loose-index: pre-enumerate loose names, skip stat
342+
// when name isn't present.
343+
let loose_set_start = Instant::now();
344+
let loose_set: HashSet<_> = store.loose_iter()?.filter_map(Result::ok).map(|r| r.name).collect();
345+
eprintln!(
346+
"Built loose-name set ({} entries) in {:?}",
347+
loose_set.len(),
348+
loose_set_start.elapsed()
349+
);
350+
351+
let b_start = Instant::now();
352+
let mut b_found = 0usize;
353+
for name in &packed_names {
354+
if loose_set.contains(name) {
355+
// Take slow path for shadowed entries.
356+
if store.try_find(name.as_ref())?.is_some() {
357+
b_found += 1;
358+
}
359+
} else if packed.try_find(name.as_ref())?.is_some() {
360+
b_found += 1;
361+
}
362+
}
363+
let b_elapsed = b_start.elapsed();
364+
eprintln!(
365+
"Strategy B (loose-set short-circuit + packed direct): {:?}, found {} of {} ({:.1} µs/lookup)",
366+
b_elapsed,
367+
b_found,
368+
packed_names.len(),
369+
b_elapsed.as_secs_f64() * 1_000_000.0 / packed_names.len() as f64,
370+
);
371+
372+
eprintln!(
373+
"Loose-stat overhead reclaimable by an in-Store loose-name cache: {:?} ({:.1}% of A)",
374+
a_elapsed.saturating_sub(b_elapsed),
375+
(a_elapsed.saturating_sub(b_elapsed).as_secs_f64() / a_elapsed.as_secs_f64()) * 100.0,
376+
);
377+
assert_eq!(a_found, b_found, "both strategies find the same number of refs");
378+
Ok(())
379+
}
380+
275381
#[test]
276382
fn find_speed() -> crate::Result {
277383
let store = store_at("make_repository_with_lots_of_packed_refs.sh")?;

0 commit comments

Comments
 (0)