Skip to content

Commit 703db7c

Browse files
aneubeckaneubeck
committed
fix linter
1 parent 2b57af8 commit 703db7c

5 files changed

Lines changed: 42 additions & 23 deletions

File tree

‎crates/sparse-ngrams/benchmarks/performance.rs‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
1-
use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main};
2-
use sparse_ngrams::{NGram, collect_sparse_grams_deque, collect_sparse_grams_scan, max_sparse_grams};
1+
use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
2+
use sparse_ngrams::{
3+
collect_sparse_grams_deque, collect_sparse_grams_scan, max_sparse_grams, NGram,
4+
};
35

46
fn bench_collect(c: &mut Criterion) {
57
let inputs: Vec<(&str, Vec<u8>)> = vec![
@@ -10,7 +12,10 @@ fn bench_collect(c: &mut Criterion) {
1012
.repeat(20)
1113
.into_bytes(),
1214
),
13-
("large_15KB", include_str!("../src/lib.rs").as_bytes().to_vec()),
15+
(
16+
"large_15KB",
17+
include_str!("../src/lib.rs").as_bytes().to_vec(),
18+
),
1419
];
1520

1621
let mut group = c.benchmark_group("collect");

‎crates/sparse-ngrams/src/extract.rs‎

Lines changed: 26 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,7 @@ pub fn collect_sparse_grams_deque(content: &[u8], out: &mut [NGram]) -> usize {
5454
out[w] = NGram::from_rolling_hash(bigram_hash, 2);
5555
w += 1;
5656

57-
let v1 =
58-
table[content[idx as usize - 1] as usize * 256 + content[idx as usize] as usize];
57+
let v1 = table[content[idx as usize - 1] as usize * 256 + content[idx as usize] as usize];
5958

6059
if let Some(begin) = queue.front() {
6160
if idx - begin.index + 1 >= MAX_SPARSE_GRAM_SIZE as u32 {
@@ -65,7 +64,8 @@ pub fn collect_sparse_grams_deque(content: &[u8], out: &mut [NGram]) -> usize {
6564
while let Some(begin) = queue.back() {
6665
let start = begin.index as usize - 1;
6766
let len = (idx - begin.index + 2) as usize;
68-
let hash = end_hash.wrapping_sub(prefix_hashes[start & mask].wrapping_mul(POLY_POWERS[len]));
67+
let hash =
68+
end_hash.wrapping_sub(prefix_hashes[start & mask].wrapping_mul(POLY_POWERS[len]));
6969
out[w] = NGram::from_rolling_hash(hash, len);
7070
w += 1;
7171
if begin.value == v1 {
@@ -115,8 +115,7 @@ pub fn collect_sparse_grams_scan(content: &[u8], out: &mut [NGram]) -> usize {
115115
.wrapping_sub(prefix_hashes[(idx as usize - 1) & MASK].wrapping_mul(POLY_POWERS[2]));
116116
out[w] = NGram::from_rolling_hash(bigram_hash, 2);
117117
w += 1;
118-
let v1 =
119-
table[content[idx as usize - 1] as usize * 256 + content[idx as usize] as usize];
118+
let v1 = table[content[idx as usize - 1] as usize * 256 + content[idx as usize] as usize];
120119
priorities[idx as usize & MASK] = v1;
121120
let mut running_min = u16::MAX;
122121
for d in 1..=(MAX_SPARSE_GRAM_SIZE as u32 - 2) {
@@ -129,7 +128,8 @@ pub fn collect_sparse_grams_scan(content: &[u8], out: &mut [NGram]) -> usize {
129128
running_min = v_p;
130129
let start = p.wrapping_sub(1) & MASK;
131130
let len = d as usize + 2;
132-
let hash = end_hash.wrapping_sub(prefix_hashes[start].wrapping_mul(POLY_POWERS[len]));
131+
let hash =
132+
end_hash.wrapping_sub(prefix_hashes[start].wrapping_mul(POLY_POWERS[len]));
133133
out[w] = NGram::from_rolling_hash(hash, len);
134134
w += 1;
135135
if v_p <= v1 {
@@ -178,12 +178,13 @@ mod tests {
178178
break;
179179
}
180180
let left = table[content[start] as usize * 256 + content[start + 1] as usize];
181-
let right =
182-
table[content[start + len - 2] as usize * 256 + content[start + len - 1] as usize];
181+
let right = table
182+
[content[start + len - 2] as usize * 256 + content[start + len - 1] as usize];
183183
let boundary = left.max(right);
184184
// Inner bigrams: bytes [start+1,start+2], ..., [start+len-3,start+len-2]
185185
for k in 1..len - 2 {
186-
let p = table[content[start + k] as usize * 256 + content[start + k + 1] as usize];
186+
let p =
187+
table[content[start + k] as usize * 256 + content[start + k + 1] as usize];
187188
if p <= boundary {
188189
continue 'outer;
189190
}
@@ -225,7 +226,10 @@ mod tests {
225226
let grams = collect_sparse_grams(input);
226227
for gram in &grams {
227228
assert!(gram.len() >= 2, "gram too short: {gram:?}");
228-
assert!(gram.len() <= MAX_SPARSE_GRAM_SIZE, "gram too long: {gram:?}");
229+
assert!(
230+
gram.len() <= MAX_SPARSE_GRAM_SIZE,
231+
"gram too long: {gram:?}"
232+
);
229233
}
230234
}
231235

@@ -309,14 +313,24 @@ mod tests {
309313
if !only_actual.is_empty() || !only_expected.is_empty() {
310314
panic!(
311315
"mismatch on input len={}\n only in algorithm: {:?}\n only in brute force: {:?}",
312-
input.len(), only_actual, only_expected
316+
input.len(),
317+
only_actual,
318+
only_expected
313319
);
314320
}
315321
}
316322

317323
#[test]
318324
fn test_brute_force_small() {
319-
for input in [b"" as &[u8], b"x", b"ab", b"abc", b"abcd", b"abcdefgh", b"abcdefghi"] {
325+
for input in [
326+
b"" as &[u8],
327+
b"x",
328+
b"ab",
329+
b"abc",
330+
b"abcd",
331+
b"abcdefgh",
332+
b"abcdefghi",
333+
] {
320334
assert_matches_brute_force(input);
321335
}
322336
}

‎crates/sparse-ngrams/src/lib.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,5 +43,6 @@ pub const NUM_FREQUENT_BIGRAMS: usize = 65534;
4343
/// Maximum length (in bytes) of a sparse n-gram.
4444
pub const MAX_SPARSE_GRAM_SIZE: usize = 8;
4545

46-
pub use extract::{collect_sparse_grams, collect_sparse_grams_deque, collect_sparse_grams_scan, max_sparse_grams};
47-
46+
pub use extract::{
47+
collect_sparse_grams, collect_sparse_grams_deque, collect_sparse_grams_scan, max_sparse_grams,
48+
};

‎crates/sparse-ngrams/src/ngram.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ pub(crate) const POLY_POWERS: [u32; MAX_SPARSE_GRAM_SIZE + 1] = {
3131
/// would explode the number of ngram keys in a search dictionary. For that reason,
3232
/// we compress ngrams into an u32 which puts a more reasonable upper bound on
3333
/// the number of dictionary keys (~100 million).
34-
///
34+
///
3535
/// Note: By storing the length explicitly in the lower 8 bits, we ensure that
3636
/// only ngrams of the same length collide. This is important because there
3737
/// are exponentially more long ngrams than short ngrams. At the same time,
@@ -128,9 +128,9 @@ mod tests {
128128
.wrapping_mul(POLY_HASH_PRIME)
129129
.wrapping_add(content[idx] as u32);
130130
// Check the bigram content[idx-1..idx+1]
131-
let rolling_hash = end_hash
132-
.wrapping_sub(prefix_hashes[(idx - 1) & (MAX_SPARSE_GRAM_SIZE - 1)]
133-
.wrapping_mul(POLY_POWERS[2]));
131+
let rolling_hash = end_hash.wrapping_sub(
132+
prefix_hashes[(idx - 1) & (MAX_SPARSE_GRAM_SIZE - 1)].wrapping_mul(POLY_POWERS[2]),
133+
);
134134
let rolling = NGram::from_rolling_hash(rolling_hash, 2);
135135
let direct = NGram::from_bytes(&content[idx - 1..idx + 1]);
136136
assert_eq!(rolling, direct, "mismatch at idx={idx}");

‎crates/sparse-ngrams/src/table.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,7 @@ pub(crate) fn get_bigram_table() -> &'static [u16; 256 * 256] {
3434
assert_eq!(table[a as usize * 256 + b as usize], 0);
3535
// Higher-frequency bigrams get HIGHER values so they are more often
3636
// encompassed by longer grams.
37-
table[a as usize * 256 + b as usize] =
38-
(NUM_FREQUENT_BIGRAMS - idx) as u16;
37+
table[a as usize * 256 + b as usize] = (NUM_FREQUENT_BIGRAMS - idx) as u16;
3938
}
4039
table
4140
})

0 commit comments

Comments
 (0)