Skip to content

Commit bbcf609

Browse files
oetroetr
committed
test(bench): add LibAFL smoke and anomaly checks
Benchmark both engines against the same target and keep a few anomaly checks close by so backend changes can be compared empirically.
1 parent fc2f7e6 commit bbcf609

9 files changed

Lines changed: 500 additions & 0 deletions

File tree

‎benchmarks/engine_smoke/.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
node_modules/
2+
package-lock.json
3+
work/

‎benchmarks/engine_smoke/anomaly.js‎

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
/*
2+
* Copyright 2026 Code Intelligence GmbH
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
const { spawnSync } = require("child_process");
18+
const fs = require("fs");
19+
const path = require("path");
20+
21+
const benchmarkDirectory = __dirname;
22+
const workDirectory = path.join(benchmarkDirectory, "work", "anomalies");
23+
const engineTarget = path.join(
24+
benchmarkDirectory,
25+
"..",
26+
"..",
27+
"tests",
28+
"engine",
29+
"fuzz.js",
30+
);
31+
const asyncTarget = path.join(benchmarkDirectory, "anomaly_fuzz.js");
32+
33+
function removeIfExists(targetPath) {
34+
fs.rmSync(targetPath, { force: true, recursive: true });
35+
}
36+
37+
function ensureDirectory(targetPath) {
38+
fs.mkdirSync(targetPath, { recursive: true });
39+
}
40+
41+
function runCommand(label, args, cwd, outputDirectory, expectedStatus = 0) {
42+
console.log(`\n[anomaly] ${label}`);
43+
console.log(`[anomaly] command: npx ${args.join(" ")}`);
44+
ensureDirectory(outputDirectory);
45+
const sanitizedLabel = label.replace(/[^a-z0-9]+/gi, "-").toLowerCase();
46+
const stdoutPath = path.join(outputDirectory, `${sanitizedLabel}.stdout.log`);
47+
const stderrPath = path.join(outputDirectory, `${sanitizedLabel}.stderr.log`);
48+
const stdoutFd = fs.openSync(stdoutPath, "w");
49+
const stderrFd = fs.openSync(stderrPath, "w");
50+
const startedAt = Date.now();
51+
const proc = spawnSync("npx", args, {
52+
cwd,
53+
env: { ...process.env },
54+
shell: true,
55+
stdio: ["ignore", stdoutFd, stderrFd],
56+
windowsHide: true,
57+
});
58+
const elapsedMs = Date.now() - startedAt;
59+
fs.closeSync(stdoutFd);
60+
fs.closeSync(stderrFd);
61+
62+
if (proc.status !== expectedStatus) {
63+
throw new Error(
64+
`${label} failed with exit code ${proc.status}\nSTDOUT (${stdoutPath}):\n${fs.readFileSync(stdoutPath, "utf8")}\nSTDERR (${stderrPath}):\n${fs.readFileSync(stderrPath, "utf8")}`,
65+
);
66+
}
67+
68+
return {
69+
elapsedMs,
70+
stderrPath,
71+
stdoutPath,
72+
};
73+
}
74+
75+
function parseExecsPerSecond(stderrPath) {
76+
const stderr = fs.readFileSync(stderrPath, "utf8");
77+
const match = stderr.match(/speed:\s+([0-9.]+) exec\/s/);
78+
if (!match) {
79+
throw new Error(`No LibAFL done line found in ${stderrPath}`);
80+
}
81+
return Number.parseFloat(match[1]);
82+
}
83+
84+
function runGuidedNumericSmoke() {
85+
const outputDirectory = path.join(workDirectory, "guided-numeric");
86+
const corpusDirectory = path.join(outputDirectory, "corpus");
87+
removeIfExists(outputDirectory);
88+
ensureDirectory(corpusDirectory);
89+
fs.writeFileSync(path.join(corpusDirectory, "seed"), Buffer.alloc(4));
90+
91+
const result = runCommand(
92+
"guided numeric solve",
93+
[
94+
"jazzer",
95+
engineTarget,
96+
"-f",
97+
"guided_numeric",
98+
"--engine=afl",
99+
"--sync",
100+
"--disable_bug_detectors=.*",
101+
"--",
102+
corpusDirectory,
103+
"-runs=4000",
104+
"-seed=1337",
105+
"-max_len=16",
106+
`-artifact_prefix=${outputDirectory}${path.sep}`,
107+
],
108+
benchmarkDirectory,
109+
outputDirectory,
110+
77,
111+
);
112+
113+
const output =
114+
fs.readFileSync(result.stdoutPath, "utf8") +
115+
fs.readFileSync(result.stderrPath, "utf8");
116+
if (!output.includes("AFL numeric guidance finding")) {
117+
throw new Error("Guided numeric smoke did not report the expected finding");
118+
}
119+
120+
return {
121+
name: "guided-numeric",
122+
elapsedMs: result.elapsedMs,
123+
};
124+
}
125+
126+
function runAsyncSmoke() {
127+
const outputDirectory = path.join(workDirectory, "async-smoke");
128+
const corpusDirectory = path.join(outputDirectory, "corpus");
129+
removeIfExists(outputDirectory);
130+
ensureDirectory(corpusDirectory);
131+
fs.writeFileSync(path.join(corpusDirectory, "seed"), "async-seed");
132+
133+
const result = runCommand(
134+
"async throughput smoke",
135+
[
136+
"jazzer",
137+
asyncTarget,
138+
"-f",
139+
"async_smoke",
140+
"--engine=afl",
141+
"--disable_bug_detectors=.*",
142+
"--",
143+
corpusDirectory,
144+
"-runs=2000",
145+
"-seed=9001",
146+
"-max_len=128",
147+
`-artifact_prefix=${outputDirectory}${path.sep}`,
148+
],
149+
benchmarkDirectory,
150+
outputDirectory,
151+
);
152+
153+
const execsPerSecond = parseExecsPerSecond(result.stderrPath);
154+
if (execsPerSecond <= 0) {
155+
throw new Error("Async smoke reported a non-positive exec/sec rate");
156+
}
157+
if (result.elapsedMs > 30000) {
158+
throw new Error(
159+
`Async smoke took unexpectedly long: ${result.elapsedMs} ms`,
160+
);
161+
}
162+
163+
return {
164+
name: "async-smoke",
165+
elapsedMs: result.elapsedMs,
166+
execsPerSecond,
167+
};
168+
}
169+
170+
function main() {
171+
ensureDirectory(workDirectory);
172+
const results = [runGuidedNumericSmoke(), runAsyncSmoke()];
173+
for (const result of results) {
174+
const stats = [`elapsed_ms=${result.elapsedMs}`];
175+
if (result.execsPerSecond !== undefined) {
176+
stats.push(`execs_per_second=${result.execsPerSecond}`);
177+
}
178+
console.log(`[anomaly] ${result.name}: ${stats.join(" ")}`);
179+
}
180+
fs.writeFileSync(
181+
path.join(workDirectory, "results.json"),
182+
JSON.stringify(results, null, 2),
183+
);
184+
console.log(
185+
`\n[anomaly] wrote machine-readable results to ${path.join(workDirectory, "results.json")}`,
186+
);
187+
}
188+
189+
main();

‎benchmarks/engine_smoke/anomaly_fuzz.js‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* Copyright 2026 Code Intelligence GmbH
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
module.exports.async_smoke = function (data) {
18+
let checksum = 0;
19+
for (const byte of data) {
20+
checksum = ((checksum * 33) ^ byte) & 0xffff;
21+
}
22+
23+
return new Promise((resolve) => {
24+
setImmediate(() => {
25+
if (checksum === 0x1337) {
26+
// Exercise an extra branch without turning this into a finding target.
27+
checksum ^= data.length;
28+
}
29+
resolve(checksum);
30+
});
31+
});
32+
};

‎benchmarks/engine_smoke/fuzz.js‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
/*
2+
* Copyright 2026 Code Intelligence GmbH
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
const qs = require("qs");
18+
19+
const { FuzzedDataProvider } = require("@jazzer.js/core");
20+
21+
module.exports.fuzz = function (data) {
22+
const provider = new FuzzedDataProvider(data);
23+
const input = provider.consumeRemainingAsString();
24+
25+
const parseOptions = {
26+
allowDots: provider.consumeBoolean(),
27+
allowEmptyArrays: provider.consumeBoolean(),
28+
allowPrototypes: provider.consumeBoolean(),
29+
arrayLimit: provider.consumeIntegralInRange(0, 32),
30+
charset: provider.pickValue(["utf-8", "iso-8859-1"]),
31+
charsetSentinel: provider.consumeBoolean(),
32+
comma: provider.consumeBoolean(),
33+
decodeDotInKeys: provider.consumeBoolean(),
34+
depth: provider.consumeIntegralInRange(0, 16),
35+
duplicates: provider.pickValue(["combine", "first", "last"]),
36+
ignoreQueryPrefix: provider.consumeBoolean(),
37+
interpretNumericEntities: provider.consumeBoolean(),
38+
parameterLimit: provider.consumeIntegralInRange(1, 256),
39+
parseArrays: provider.consumeBoolean(),
40+
plainObjects: provider.consumeBoolean(),
41+
strictDepth: provider.consumeBoolean(),
42+
strictNullHandling: provider.consumeBoolean(),
43+
};
44+
45+
let parsed;
46+
try {
47+
parsed = qs.parse(input, parseOptions);
48+
} catch {
49+
return;
50+
}
51+
52+
try {
53+
qs.stringify(parsed, {
54+
addQueryPrefix: provider.consumeBoolean(),
55+
allowDots: provider.consumeBoolean(),
56+
allowEmptyArrays: provider.consumeBoolean(),
57+
arrayFormat: provider.pickValue([
58+
"indices",
59+
"brackets",
60+
"repeat",
61+
"comma",
62+
]),
63+
charset: provider.pickValue(["utf-8", "iso-8859-1"]),
64+
charsetSentinel: provider.consumeBoolean(),
65+
commaRoundTrip: provider.consumeBoolean(),
66+
delimiter: provider.pickValue(["&", ";"]),
67+
encode: provider.consumeBoolean(),
68+
encodeDotInKeys: provider.consumeBoolean(),
69+
indices: provider.consumeBoolean(),
70+
skipNulls: provider.consumeBoolean(),
71+
strictNullHandling: provider.consumeBoolean(),
72+
});
73+
} catch {
74+
// Smoke target: ignore library-level parse/stringify failures.
75+
}
76+
};

‎benchmarks/engine_smoke/package.json‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"name": "jazzerjs-engine-smoke",
3+
"version": "1.0.0",
4+
"private": true,
5+
"description": "Manual 30-second smoke benchmark for libFuzzer vs LibAFL.",
6+
"scripts": {
7+
"smoke": "node run.js",
8+
"smoke:anomalies": "node anomaly.js"
9+
},
10+
"devDependencies": {
11+
"@jazzer.js/core": "file:../../packages/core",
12+
"istanbul-lib-coverage": "^3.2.2",
13+
"qs": "^6.14.0"
14+
}
15+
}

0 commit comments

Comments
 (0)