Skip to content

Commit 3b2f5fa

Browse files
perf(snapshots): Parallelize image hashing with rayon
Use rayon's par_iter to hash all images concurrently instead of sequentially. Also increase the hash read buffer from 8KB to 64KB to reduce syscall overhead. Reduces hashing time from 5.3s to 0.8s (6.6x speedup) on a 753-image / 99MB dataset. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 938a884 commit 3b2f5fa

File tree

1 file changed

+10
-4
lines changed

1 file changed

+10
-4
lines changed

src/commands/build/snapshots.rs

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ use console::style;
1111
use itertools::Itertools as _;
1212
use log::{debug, warn};
1313
use objectstore_client::{ClientBuilder, ExpirationPolicy, Usecase};
14+
use rayon::prelude::*;
1415
use secrecy::ExposeSecret as _;
1516
use serde_json::Value;
1617
use sha2::{Digest as _, Sha256};
@@ -231,7 +232,7 @@ fn compute_sha256_hash(path: &Path) -> Result<String> {
231232
let mut file = std::fs::File::open(path)
232233
.with_context(|| format!("Failed to open image for hashing: {}", path.display()))?;
233234
let mut hasher = Sha256::new();
234-
let mut buffer = [0u8; 8192];
235+
let mut buffer = [0u8; 65536];
235236
loop {
236237
let bytes_read = file
237238
.read(&mut buffer)
@@ -338,9 +339,15 @@ fn upload_images(
338339
let mut kept_paths = HashMap::new();
339340
let mut uploads = Vec::with_capacity(images.len());
340341

341-
for image in images {
342-
debug!("Processing image: {}", image.path.display());
342+
let hashed_images: Vec<_> = images
343+
.into_par_iter()
344+
.map(|image| {
345+
let hash = compute_sha256_hash(&image.path)?;
346+
Ok((image, hash))
347+
})
348+
.collect::<Result<Vec<_>>>()?;
343349

350+
for (image, hash) in hashed_images {
344351
let image_file_name = image
345352
.relative_path
346353
.file_name()
@@ -358,7 +365,6 @@ fn upload_images(
358365
continue;
359366
}
360367

361-
let hash = compute_sha256_hash(&image.path)?;
362368
let key = format!("{org_id}/{project_id}/{hash}");
363369

364370
let mut extra = read_sidecar_metadata(&image.path).unwrap_or_else(|err| {

0 commit comments

Comments
 (0)