Skip to content

Commit 59af6a1

Browse files
baflorbaflor
committed
optimize cmplog metadata size
This cuts down memory usage of CmpLog metadata by 5x-20x
1 parent 3ae6545 commit 59af6a1

9 files changed

Lines changed: 173 additions & 111 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ wasmparser = "0.243"
1212
rustc-demangle = "0.1"
1313
clap = { version = "4.4", features = ["derive", "env"] }
1414
md5 = "0.8"
15+
memchr = "2.7.6"
1516
rand = "0.9"
1617
bitvec = "1.0.1"
1718
roaring = "0.11"
@@ -45,7 +46,11 @@ ouroboros = "0.18.3"
4546
libafl = { version = "=0.15.4", default-features = false, features = [
4647
"rand_trait",
4748
] }
48-
libafl_bolts = { version = "0.15", default-features = false, features = ["std", "derive", "serdeany_autoreg"] }
49+
libafl_bolts = { version = "0.15", default-features = false, features = [
50+
"std",
51+
"derive",
52+
"serdeany_autoreg",
53+
] }
4954
cranelift = { version = "0.127", features = [
5055
# cranelift = { path = "../wasmtime/cranelift/umbrella", package = "cranelift", features = [
5156
"jit",
@@ -57,7 +62,7 @@ libc = "0.2.158"
5762

5863
# feat:concolic
5964
# Note: waiting for 0.3.1 upstream
60-
smtlib = { git = "https://github.com/oeb25/smtlib-rs.git", rev = "4f0fad92289ba02931276cbe2bbe45f87f7a6f3e", optional = true}
65+
smtlib = { git = "https://github.com/oeb25/smtlib-rs.git", rev = "4f0fad92289ba02931276cbe2bbe45f87f7a6f3e", optional = true }
6166
z3 = { version = "0.19", optional = true }
6267
bitwuzla = { git = "https://github.com/Mrmaxmeier/bitwuzla-rs.git", optional = true }
6368

‎src/fuzzer/i2s_patches.rs‎

Lines changed: 130 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,99 @@ use libafl::{
55
corpus::Corpus,
66
inputs::{BytesInput, HasMutatorBytes},
77
mutators::{MutationResult, Mutator},
8-
observers::{CmpValues, CmpValuesMetadata},
98
state::{HasCorpus, HasMaxSize, HasRand},
109
};
11-
use libafl_bolts::{AsSlice, HasLen, Named, rands::Rand};
10+
use libafl_bolts::{HasLen, Named, rands::Rand};
11+
12+
use speedy::{Readable, Writable};
13+
use std::io::{Cursor, Seek, SeekFrom};
14+
#[derive(Debug, Clone, Hash, PartialEq, Eq, speedy::Readable, speedy::Writable)]
15+
#[repr(u8)]
16+
pub(crate) enum CmpLog {
17+
U16(u16, u16),
18+
U32(u32, u32),
19+
U64(u64, u64),
20+
Memcmp(Vec<u8>, Vec<u8>),
21+
}
22+
23+
impl CmpLog {
24+
pub fn test_input(&self, input: &[u8]) -> bool {
25+
enum V<'a> {
26+
U16(u16),
27+
U32(u32),
28+
U64(u64),
29+
Bytes(&'a [u8]),
30+
}
31+
impl V<'_> {
32+
fn to_needles(&self) -> Vec<Vec<u8>> {
33+
match self {
34+
V::U16(v) => vec![v.to_le_bytes().to_vec(), v.to_be_bytes().to_vec()],
35+
V::U32(v) => vec![v.to_le_bytes().to_vec(), v.to_be_bytes().to_vec()],
36+
V::U64(v) => vec![v.to_le_bytes().to_vec(), v.to_be_bytes().to_vec()],
37+
V::Bytes(v) => vec![v.to_vec()],
38+
}
39+
}
40+
}
41+
let mut needles = Vec::new();
42+
match self {
43+
CmpLog::U16(a, b) => {
44+
needles.push(V::U16(*a));
45+
needles.push(V::U16(*b));
46+
}
47+
CmpLog::U32(a, b) => {
48+
needles.push(V::U32(*a));
49+
needles.push(V::U32(*b));
50+
}
51+
CmpLog::U64(a, b) => {
52+
needles.push(V::U64(*a));
53+
needles.push(V::U64(*b));
54+
}
55+
CmpLog::Memcmp(a, b) => {
56+
needles.push(V::Bytes(a));
57+
needles.push(V::Bytes(b));
58+
}
59+
}
60+
let needles = needles
61+
.into_iter()
62+
.flat_map(|x| x.to_needles())
63+
.collect::<Vec<_>>();
64+
needles
65+
.iter()
66+
.any(|needle| memchr::memmem::find(input, needle).is_some())
67+
}
68+
}
69+
70+
/// A state metadata holding a list of values logged from comparisons
71+
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
72+
pub struct CmplogStore {
73+
offsets: Vec<u32>,
74+
data: Vec<u8>,
75+
}
76+
libafl_bolts::impl_serdeany!(CmplogStore);
77+
78+
impl CmplogStore {
79+
pub fn new(values: impl Iterator<Item = CmpLog>) -> Self {
80+
let mut data = Vec::new();
81+
let mut offsets = Vec::new();
82+
for value in values {
83+
offsets.push(data.len().try_into().unwrap());
84+
let mut cursor = Cursor::new(&mut data);
85+
cursor.seek(SeekFrom::End(0)).unwrap();
86+
value.write_to_stream(cursor).unwrap();
87+
}
88+
Self { offsets, data }
89+
}
90+
pub fn get(&self, idx: usize) -> CmpLog {
91+
let pos = self.offsets[idx] as usize;
92+
CmpLog::read_from_buffer(&self.data[pos..]).unwrap()
93+
}
94+
pub fn is_empty(&self) -> bool {
95+
self.offsets.is_empty()
96+
}
97+
pub fn len(&self) -> usize {
98+
self.offsets.len()
99+
}
100+
}
12101

13102
/// A `I2SRandReplace` [`Mutator`] replaces a random matching input-2-state comparison operand with the other.
14103
/// It needs a valid [`CmpValuesMetadata`] in the state.
@@ -33,15 +122,15 @@ where
33122
};
34123
let inp_testcase = state.corpus().get(id).unwrap();
35124
let _meta = inp_testcase.borrow();
36-
let meta: &CmpValuesMetadata = match _meta.metadata::<CmpValuesMetadata>().ok() {
125+
let meta: &CmplogStore = match _meta.metadata::<CmplogStore>().ok() {
37126
Some(meta) => meta,
38127
None => return Ok(MutationResult::Skipped),
39128
};
40129

41-
if meta.list.is_empty() {
130+
if meta.is_empty() {
42131
return Ok(MutationResult::Skipped);
43132
}
44-
meta.list.len().try_into().unwrap()
133+
meta.len().try_into().unwrap()
45134
};
46135
let idx = state.rand_mut().below(cmps_len);
47136

@@ -54,136 +143,117 @@ where
54143
};
55144
let inp_testcase = state.corpus().get(id).unwrap();
56145
let _meta = inp_testcase.borrow();
57-
let meta = match _meta.metadata::<CmpValuesMetadata>().ok() {
146+
let meta = match _meta.metadata::<CmplogStore>().ok() {
58147
Some(meta) => meta,
59148
None => return Ok(MutationResult::Skipped),
60149
};
61150

62-
if meta.list.is_empty() {
63-
return Ok(MutationResult::Skipped);
64-
}
65-
66-
let cmp_values = &meta.list[idx];
151+
let cmp_values = &meta.get(idx);
67152

68153
let mut result = MutationResult::Skipped;
69154
match cmp_values {
70-
CmpValues::U8(v) => {
71-
for byte in bytes.iter_mut().take(len).skip(off) {
72-
if *byte == v.0 {
73-
*byte = v.1;
74-
result = MutationResult::Mutated;
75-
break;
76-
} else if *byte == v.1 {
77-
*byte = v.0;
78-
result = MutationResult::Mutated;
79-
break;
80-
}
81-
}
82-
}
83-
CmpValues::U16(v) => {
155+
&CmpLog::U16(a, b) => {
84156
if len >= size_of::<u16>() {
85157
for i in off..len - (size_of::<u16>() - 1) {
86158
let val =
87159
u16::from_ne_bytes(bytes[i..i + size_of::<u16>()].try_into().unwrap());
88-
if val == v.0 {
89-
let new_bytes = v.1.to_ne_bytes();
160+
if val == a {
161+
let new_bytes = b.to_ne_bytes();
90162
bytes[i..i + size_of::<u16>()].copy_from_slice(&new_bytes);
91163
result = MutationResult::Mutated;
92164
break;
93-
} else if val.swap_bytes() == v.0 {
94-
let new_bytes = v.1.swap_bytes().to_ne_bytes();
165+
} else if val.swap_bytes() == a {
166+
let new_bytes = b.swap_bytes().to_ne_bytes();
95167
bytes[i..i + size_of::<u16>()].copy_from_slice(&new_bytes);
96168
result = MutationResult::Mutated;
97169
break;
98-
} else if val == v.1 {
99-
let new_bytes = v.0.to_ne_bytes();
170+
} else if val == b {
171+
let new_bytes = a.to_ne_bytes();
100172
bytes[i..i + size_of::<u16>()].copy_from_slice(&new_bytes);
101173
result = MutationResult::Mutated;
102174
break;
103-
} else if val.swap_bytes() == v.1 {
104-
let new_bytes = v.0.swap_bytes().to_ne_bytes();
175+
} else if val.swap_bytes() == b {
176+
let new_bytes = a.swap_bytes().to_ne_bytes();
105177
bytes[i..i + size_of::<u16>()].copy_from_slice(&new_bytes);
106178
result = MutationResult::Mutated;
107179
break;
108180
}
109181
}
110182
}
111183
}
112-
CmpValues::U32(v) => {
184+
&CmpLog::U32(a, b) => {
113185
if len >= size_of::<u32>() {
114186
for i in off..len - (size_of::<u32>() - 1) {
115187
let val =
116188
u32::from_ne_bytes(bytes[i..i + size_of::<u32>()].try_into().unwrap());
117-
if val == v.0 {
118-
let new_bytes = v.1.to_ne_bytes();
189+
if val == a {
190+
let new_bytes = b.to_ne_bytes();
119191
bytes[i..i + size_of::<u32>()].copy_from_slice(&new_bytes);
120192
result = MutationResult::Mutated;
121193
break;
122-
} else if val.swap_bytes() == v.0 {
123-
let new_bytes = v.1.swap_bytes().to_ne_bytes();
194+
} else if val.swap_bytes() == a {
195+
let new_bytes = b.swap_bytes().to_ne_bytes();
124196
bytes[i..i + size_of::<u32>()].copy_from_slice(&new_bytes);
125197
result = MutationResult::Mutated;
126198
break;
127-
} else if val == v.1 {
128-
let new_bytes = v.0.to_ne_bytes();
199+
} else if val == b {
200+
let new_bytes = a.to_ne_bytes();
129201
bytes[i..i + size_of::<u32>()].copy_from_slice(&new_bytes);
130202
result = MutationResult::Mutated;
131203
break;
132-
} else if val.swap_bytes() == v.1 {
133-
let new_bytes = v.0.swap_bytes().to_ne_bytes();
204+
} else if val.swap_bytes() == b {
205+
let new_bytes = a.swap_bytes().to_ne_bytes();
134206
bytes[i..i + size_of::<u32>()].copy_from_slice(&new_bytes);
135207
result = MutationResult::Mutated;
136208
break;
137209
}
138210
}
139211
}
140212
}
141-
CmpValues::U64(v) => {
213+
&CmpLog::U64(a, b) => {
142214
if len >= size_of::<u64>() {
143215
for i in off..len - (size_of::<u64>() - 1) {
144216
let val =
145217
u64::from_ne_bytes(bytes[i..i + size_of::<u64>()].try_into().unwrap());
146-
if val == v.0 {
147-
let new_bytes = v.1.to_ne_bytes();
218+
if val == a {
219+
let new_bytes = b.to_ne_bytes();
148220
bytes[i..i + size_of::<u64>()].copy_from_slice(&new_bytes);
149221
result = MutationResult::Mutated;
150222
break;
151-
} else if val.swap_bytes() == v.0 {
152-
let new_bytes = v.1.swap_bytes().to_ne_bytes();
223+
} else if val.swap_bytes() == a {
224+
let new_bytes = b.swap_bytes().to_ne_bytes();
153225
bytes[i..i + size_of::<u64>()].copy_from_slice(&new_bytes);
154226
result = MutationResult::Mutated;
155227
break;
156-
} else if val == v.1 {
157-
let new_bytes = v.0.to_ne_bytes();
228+
} else if val == b {
229+
let new_bytes = a.to_ne_bytes();
158230
bytes[i..i + size_of::<u64>()].copy_from_slice(&new_bytes);
159231
result = MutationResult::Mutated;
160232
break;
161-
} else if val.swap_bytes() == v.1 {
162-
let new_bytes = v.0.swap_bytes().to_ne_bytes();
233+
} else if val.swap_bytes() == b {
234+
let new_bytes = a.swap_bytes().to_ne_bytes();
163235
bytes[i..i + size_of::<u64>()].copy_from_slice(&new_bytes);
164236
result = MutationResult::Mutated;
165237
break;
166238
}
167239
}
168240
}
169241
}
170-
CmpValues::Bytes(v) => {
242+
CmpLog::Memcmp(a, b) => {
171243
'outer: for i in off..len {
172-
let mut size = core::cmp::min(v.0.len(), len - i);
244+
let mut size = core::cmp::min(a.len(), len - i);
173245
while size != 0 {
174-
if v.0.as_slice()[0..size] == input.mutator_bytes()[i..i + size] {
175-
input.mutator_bytes_mut()[i..i + size]
176-
.copy_from_slice(&v.1.as_slice()[0..size]);
246+
if &a[0..size] == &input.mutator_bytes()[i..i + size] {
247+
input.mutator_bytes_mut()[i..i + size].copy_from_slice(&b[0..size]);
177248
result = MutationResult::Mutated;
178249
break 'outer;
179250
}
180251
size -= 1;
181252
}
182-
size = core::cmp::min(v.1.len(), len - i);
253+
size = core::cmp::min(b.len(), len - i);
183254
while size != 0 {
184-
if v.1.as_slice()[0..size] == input.mutator_bytes()[i..i + size] {
185-
input.mutator_bytes_mut()[i..i + size]
186-
.copy_from_slice(&v.1.as_slice()[0..size]);
255+
if &b[0..size] == &input.mutator_bytes()[i..i + size] {
256+
input.mutator_bytes_mut()[i..i + size].copy_from_slice(&b[0..size]);
187257
result = MutationResult::Mutated;
188258
break 'outer;
189259
}

‎src/fuzzer/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ use std::sync::Arc;
33
use clap::Parser;
44

55
pub(crate) mod exhaustive;
6-
mod i2s_patches;
6+
pub(crate) mod i2s_patches;
77
pub mod opts;
88
pub(crate) mod orc;
99
mod worker;

‎src/fuzzer/orc.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ use crate::{
2525
pub(crate) struct CliOpts {
2626
#[clap(flatten)]
2727
pub g: super::opts::GeneralOpts,
28-
#[clap(long, default_value = "2m")]
28+
#[clap(long, default_value = "3m")]
2929
pub config_interval: Duration,
3030
#[clap(long)]
3131
pub timeout: Option<Duration>,
@@ -404,7 +404,6 @@ impl Orchestrator {
404404
let mut timeout = *self.opts.config_interval;
405405
if let Some(tm) = self.opts.timeout.as_deref() {
406406
let tm_remaining = (*tm).saturating_sub(self.start.elapsed());
407-
let tm_remaining = tm_remaining.max(std::time::Duration::from_secs(1));
408407
timeout = timeout.min(tm_remaining);
409408
}
410409

0 commit comments

Comments
 (0)