Merge pull request #4565 from joostjager/fuzz-crate-split

Split fuzz runners by hash mode

Author: TheBlueMatt

.github/workflows/build.yml

@@ -93,7 +93,8 @@ jobs:
           # Could you use this to fake the coverage report for your PR? Sure.
           # Will anyone be impressed by your amazing coverage? No
           # Maybe if codecov wasn't broken we wouldn't need to do this...
-          ./codecov --verbose upload-process --disable-search --fail-on-error -f fuzz-codecov.json -t "f421b687-4dc2-4387-ac3d-dc3b2528af57" -F 'fuzzing'
+          ./codecov --verbose upload-process --disable-search --fail-on-error -f fuzz-fake-hashes-codecov.json -t "f421b687-4dc2-4387-ac3d-dc3b2528af57" -F 'fuzzing-fake-hashes'
+          ./codecov --verbose upload-process --disable-search --fail-on-error -f fuzz-real-hashes-codecov.json -t "f421b687-4dc2-4387-ac3d-dc3b2528af57" -F 'fuzzing-real-hashes'
 
   benchmark:
     runs-on: ubuntu-latest
@@ -218,7 +219,9 @@ jobs:
       - name: Sanity check fuzz targets on Rust ${{ env.TOOLCHAIN }}
         run: |
           cd fuzz
-          cargo test --quiet --color always --lib --bins -j8
+          RUSTFLAGS="--cfg=fuzzing --cfg=secp256k1_fuzz --cfg=hashes_fuzz" cargo test --quiet --color always --lib -j8
+          RUSTFLAGS="--cfg=fuzzing --cfg=secp256k1_fuzz --cfg=hashes_fuzz" cargo test --manifest-path fuzz-fake-hashes/Cargo.toml --quiet --color always --bins -j8
+          RUSTFLAGS="--cfg=fuzzing --cfg=secp256k1_fuzz" cargo test --manifest-path fuzz-real-hashes/Cargo.toml --quiet --color always --bins -j8
 
   fuzz:
     runs-on: self-hosted

ci/check-compiles.sh

@@ -5,6 +5,10 @@ echo "Testing $(git log -1 --oneline)"
 cargo check
 cargo doc
 cargo doc --document-private-items
-cd fuzz && RUSTFLAGS="--cfg=fuzzing --cfg=secp256k1_fuzz --cfg=hashes_fuzz" cargo check --features=stdin_fuzz
+cd fuzz
+RUSTFLAGS="--cfg=fuzzing --cfg=secp256k1_fuzz --cfg=hashes_fuzz" \
+	cargo check --manifest-path fuzz-fake-hashes/Cargo.toml --features=stdin_fuzz
+RUSTFLAGS="--cfg=fuzzing --cfg=secp256k1_fuzz" \
+	cargo check --manifest-path fuzz-real-hashes/Cargo.toml --features=stdin_fuzz
 cd ../lightning && cargo check --no-default-features
 cd .. && RUSTC_BOOTSTRAP=1 RUSTFLAGS="--cfg=c_bindings" cargo check -Z avoid-dev-deps

contrib/generate_fuzz_coverage.sh

@@ -55,18 +55,37 @@ fi
 # Create output directory if it doesn't exist
 mkdir -p "$OUTPUT_DIR"
 
+generate_coverage_report() {
+    local manifest_path="$1"
+    local output_path="$2"
+    local rustflags="$3"
+
+    cargo llvm-cov clean --workspace
+    RUSTFLAGS="$rustflags" cargo llvm-cov -j8 --manifest-path "$manifest_path" --codecov \
+        --dep-coverage lightning,lightning-invoice,lightning-liquidity,lightning-rapid-gossip-sync,lightning-persister \
+        --no-default-ignore-filename-regex \
+        --ignore-filename-regex "(\.cargo/registry|\.rustup/toolchains|/fuzz/)" \
+        --output-path "$output_path" --tests
+}
+
 # dont run this command when running in CI
 if [ "$OUTPUT_CODECOV_JSON" = "0" ]; then
-    cargo llvm-cov --html \
+    cargo llvm-cov clean --workspace
+    RUSTFLAGS="--cfg=fuzzing --cfg=secp256k1_fuzz --cfg=hashes_fuzz" \
+        cargo llvm-cov --manifest-path fuzz-fake-hashes/Cargo.toml --html \
         --dep-coverage lightning,lightning-invoice,lightning-liquidity,lightning-rapid-gossip-sync,lightning-persister \
         --no-default-ignore-filename-regex \
         --ignore-filename-regex "(\.cargo/registry|\.rustup/toolchains|/fuzz/)" \
-        --output-dir "$OUTPUT_DIR"
-    echo "Coverage report generated in $OUTPUT_DIR/html/index.html"
-else
-    # Clean previous coverage artifacts to ensure a fresh run.
+        --output-dir "$OUTPUT_DIR/fake-hashes" --tests
     cargo llvm-cov clean --workspace
-
+    RUSTFLAGS="--cfg=fuzzing --cfg=secp256k1_fuzz" \
+        cargo llvm-cov --manifest-path fuzz-real-hashes/Cargo.toml --html \
+        --dep-coverage lightning,lightning-invoice,lightning-liquidity,lightning-rapid-gossip-sync,lightning-persister \
+        --no-default-ignore-filename-regex \
+        --ignore-filename-regex "(\.cargo/registry|\.rustup/toolchains|/fuzz/)" \
+        --output-dir "$OUTPUT_DIR/real-hashes" --tests
+    echo "Coverage reports generated in $OUTPUT_DIR/fake-hashes and $OUTPUT_DIR/real-hashes"
+else
     # Import honggfuzz corpus if the artifact was downloaded.
     if [ -d "hfuzz_workspace" ]; then
         echo "Importing corpus from hfuzz_workspace..."
@@ -82,11 +101,14 @@ else
     fi
 
     echo "Replaying imported corpus (if found) via tests to generate coverage..."
-    cargo llvm-cov -j8 --codecov \
-        --dep-coverage lightning,lightning-invoice,lightning-liquidity,lightning-rapid-gossip-sync,lightning-persister \
-        --no-default-ignore-filename-regex \
-        --ignore-filename-regex "(\.cargo/registry|\.rustup/toolchains|/fuzz/)" \
-        --output-path "$OUTPUT_DIR/fuzz-codecov.json" --tests
+    generate_coverage_report \
+        "fuzz-fake-hashes/Cargo.toml" \
+        "$OUTPUT_DIR/fuzz-fake-hashes-codecov.json" \
+        "--cfg=fuzzing --cfg=secp256k1_fuzz --cfg=hashes_fuzz"
+    generate_coverage_report \
+        "fuzz-real-hashes/Cargo.toml" \
+        "$OUTPUT_DIR/fuzz-real-hashes-codecov.json" \
+        "--cfg=fuzzing --cfg=secp256k1_fuzz"
 
-    echo "Fuzz codecov report available at $OUTPUT_DIR/fuzz-codecov.json"
+    echo "Fuzz codecov reports available at $OUTPUT_DIR/fuzz-fake-hashes-codecov.json and $OUTPUT_DIR/fuzz-real-hashes-codecov.json"
 fi

fuzz/.cargo/config.toml

@@ -4,18 +4,6 @@ version = "0.0.1"
 authors = ["Automatically generated"]
 publish = false
 edition = "2021"
-# Because the function is unused it gets dropped before we link lightning, so
-# we have to duplicate build.rs here. Note that this is only required for
-# fuzzing mode.
-
-[package.metadata]
-cargo-fuzz = true
-
-[features]
-afl_fuzz = ["afl"]
-honggfuzz_fuzz = ["honggfuzz"]
-libfuzzer_fuzz = ["libfuzzer-sys"]
-stdin_fuzz = []
 
 [dependencies]
 lightning = { path = "../lightning", features = ["regex", "_test_utils"] }
@@ -27,16 +15,9 @@ bech32 = "0.11.0"
 bitcoin = { version = "0.32.4", features = ["secp-lowmemory"] }
 tokio = { version = "~1.35", default-features = false, features = ["rt-multi-thread"] }
 
-afl = { version = "0.12", optional = true }
-honggfuzz = { version = "0.5", optional = true, default-features = false }
-libfuzzer-sys = { version = "0.4", optional = true }
-
-[build-dependencies]
-cc = "1.0"
-
 # Prevent this from interfering with workspaces
 [workspace]
-members = ["."]
+members = [".", "fuzz-fake-hashes", "fuzz-real-hashes", "write-seeds"]
 
 [profile.release]
 panic = "abort"

fuzz/Cargo.toml

@@ -10,6 +10,11 @@ configured for. Fuzzing is further only effective with a lot of CPU time, indica
 scenarios are discovered on CI with its low runtime constraints, the crash is caused relatively
 easily.
 
+The `fuzz/` directory now contains three crates:
+- `fuzz/`, the shared fuzz target logic and corpus directories
+- `fuzz/fuzz-fake-hashes`, the fuzz targets that require `--cfg=hashes_fuzz`
+- `fuzz/fuzz-real-hashes`, the real-hashes fuzz targets, currently `chanmon_consistency_target`
+
 ## How do I run fuzz tests locally?
 
 We support multiple fuzzing engines such as `honggfuzz`, `libFuzzer` and `AFL`. You typically won't
@@ -47,34 +52,45 @@ cargo install --force cargo-fuzz
 To run fuzzing using `honggfuzz`, do
 
 ```shell
+cd fuzz
 export CPU_COUNT=1 # replace as needed
 export HFUZZ_BUILD_ARGS="--features honggfuzz_fuzz"
 export HFUZZ_RUN_ARGS="-n $CPU_COUNT --exit_upon_crash"
 
 export TARGET="msg_ping_target" # replace with the target to be fuzzed
-cargo hfuzz run $TARGET
+export RUSTFLAGS="--cfg=fuzzing --cfg=secp256k1_fuzz --cfg=hashes_fuzz"
+cargo hfuzz run --manifest-path fuzz-fake-hashes/Cargo.toml $TARGET
 ```
 
-(Or, for a prettier output, replace the last line with `cargo --color always hfuzz run $TARGET`.)
+(For `fuzz-real-hashes`, use
+`RUSTFLAGS="--cfg=fuzzing --cfg=secp256k1_fuzz" cargo hfuzz run --manifest-path fuzz-real-hashes/Cargo.toml chanmon_consistency_target`.)
+For a prettier output, replace the last line with
+`cargo --color always hfuzz run --manifest-path fuzz-fake-hashes/Cargo.toml $TARGET`.
 
 #### cargo-fuzz / libFuzzer
 To run fuzzing using `cargo-fuzz / libFuzzer`, run
 
 ```shell
 rustup install nightly # Note: libFuzzer requires a nightly version of rust.
+cd fuzz
 export RUSTFLAGS="--cfg=fuzzing --cfg=secp256k1_fuzz --cfg=hashes_fuzz"
-cargo +nightly fuzz run --features "libfuzzer_fuzz" msg_ping_target
+cargo +nightly fuzz run --fuzz-dir fuzz-fake-hashes --features "libfuzzer_fuzz" msg_ping_target
 ```
 Note: If you encounter a `SIGKILL` during run/build check for OOM in kernel logs and consider
 increasing RAM size for VM.
 
+For `fuzz-real-hashes`, use
+`RUSTFLAGS="--cfg=fuzzing --cfg=secp256k1_fuzz" cargo +nightly fuzz run --fuzz-dir fuzz-real-hashes --features "libfuzzer_fuzz" chanmon_consistency_target`.
+
 ##### Fast builds for development
 
 The default build uses LTO and single codegen unit, which is slow. For faster iteration during
 development, use the `-D` (dev) flag:
 
 ```shell
-cargo +nightly fuzz run --features "libfuzzer_fuzz" -D msg_ping_target
+cd fuzz
+RUSTFLAGS="--cfg=fuzzing --cfg=secp256k1_fuzz --cfg=hashes_fuzz" \
+	cargo +nightly fuzz run --fuzz-dir fuzz-fake-hashes --features "libfuzzer_fuzz" -D msg_ping_target
 ```
 
 The `-D` flag builds in development mode with faster compilation (still has optimizations via
@@ -83,7 +99,9 @@ sanitizer instrumentation, but subsequent builds will be fast.
 
 If you wish to just generate fuzzing binary executables for `libFuzzer` and not run them:
 ```shell
-cargo +nightly fuzz build --features "libfuzzer_fuzz" msg_ping_target
+cd fuzz
+RUSTFLAGS="--cfg=fuzzing --cfg=secp256k1_fuzz --cfg=hashes_fuzz" \
+	cargo +nightly fuzz build --fuzz-dir fuzz-fake-hashes --features "libfuzzer_fuzz" msg_ping_target
 # Generates binary artifact in path ./target/aarch64-unknown-linux-gnu/release/msg_ping_target
 # Exact path depends on your system architecture.
 ```
@@ -93,7 +111,8 @@ You can upload the build artifact generated above to `ClusterFuzz` for distribut
 To see a list of available fuzzing targets, run:
 
 ```shell
-ls ./src/bin/
+ls ./fuzz-fake-hashes/src/bin/
+ls ./fuzz-real-hashes/src/bin/
 ```
 
 ## A fuzz test failed, what do I do?
@@ -134,7 +153,8 @@ mkdir -p ./test_cases/$TARGET
 echo $HEX | xxd -r -p > ./test_cases/$TARGET/any_filename_works
 
 export RUST_BACKTRACE=1
-cargo test
+RUSTFLAGS="--cfg=fuzzing --cfg=secp256k1_fuzz --cfg=hashes_fuzz" \
+	cargo test --manifest-path fuzz-fake-hashes/Cargo.toml --bin "${TARGET}_target"
 ```
 
 Note that if the fuzz test failed locally, moving the offending run's trace
@@ -151,7 +171,10 @@ Alternatively, you can use the `stdin_fuzz` feature to pipe the crash input dire
 creating test case files on disk:
 
 ```shell
-echo -ne '\x2d\x31\x36\x38\x37\x34\x09\x01...' | cargo run --features stdin_fuzz --bin full_stack_target
+cd fuzz
+echo -ne '\x2d\x31\x36\x38\x37\x34\x09\x01...' | \
+	RUSTFLAGS="--cfg=fuzzing --cfg=secp256k1_fuzz --cfg=hashes_fuzz" \
+	cargo run --manifest-path fuzz-fake-hashes/Cargo.toml --features stdin_fuzz --bin full_stack_target
 ```
 
 Panics will abort the process directly (the crate uses `panic = "abort"`), resulting in a
@@ -171,10 +194,13 @@ file are `do_test`, `my_fuzzy_experiment_test`, and `my_fuzzy_experiment_run`.
 
 3. Adjust the body (not the signature!) of `do_test` as necessary for the new fuzz test.
 
-4. In `fuzz/src/bin/gen_target.sh`, add a line reading `GEN_TEST my_fuzzy_experiment` to the
-first group of `GEN_TEST` lines (starting in line 9).
+4. In `fuzz/src/bin/gen_target.sh`, add a line reading `GEN_FAKE_HASHES_TEST my_fuzzy_experiment`
+to the appropriate target list. Use `GEN_REAL_HASHES_TEST` only for targets that must run without
+`hashes_fuzz`.
 
 5. If your test relies on a new local crate, add that crate as a dependency to `fuzz/Cargo.toml`.
+If the dependency is only needed by a specific runner crate or fuzz engine setup, add it to the
+matching target crate under `fuzz/fuzz-fake-hashes/Cargo.toml` or `fuzz/fuzz-real-hashes/Cargo.toml` instead.
 
 6. In `fuzz/src/lib.rs`, add the line `pub mod my_fuzzy_experiment`. Additionally, if
 you added a new crate dependency, add the `extern crate […]` import line.

fuzz/README.md

@@ -8,16 +8,16 @@ rm msg_*.rs
 [ "$(git diff)" != "" ] && exit 1
 popd
 pushd src/bin
-rm *_target.rs
+rm -f ../../fuzz-fake-hashes/src/bin/*_target.rs ../../fuzz-real-hashes/src/bin/*_target.rs
 ./gen_target.sh
 [ "$(git diff)" != "" ] && exit 1
 popd
 
-export RUSTFLAGS="--cfg=secp256k1_fuzz --cfg=hashes_fuzz"
+export RUSTFLAGS="--cfg=fuzzing --cfg=secp256k1_fuzz --cfg=hashes_fuzz"
 
 mkdir -p hfuzz_workspace/full_stack_target/input
 pushd write-seeds
-RUSTFLAGS="$RUSTFLAGS --cfg=fuzzing" cargo run ../hfuzz_workspace/full_stack_target/input
+cargo run ../hfuzz_workspace/full_stack_target/input
 cargo clean
 popd
 
@@ -27,57 +27,70 @@ cargo install --color always --force honggfuzz --no-default-features
 # compiler optimizations aren't necessary, so we turn off LTO
 sed -i 's/lto = true//' Cargo.toml
 
-export HFUZZ_BUILD_ARGS="--features honggfuzz_fuzz"
-
-cargo --color always hfuzz build -j8
-
 SUMMARY=""
 
 check_crash() {
-	local FILE=$1
-	if [ -f "hfuzz_workspace/$FILE/HONGGFUZZ.REPORT.TXT" ]; then
-		cat "hfuzz_workspace/$FILE/HONGGFUZZ.REPORT.TXT"
-		for CASE in "hfuzz_workspace/$FILE"/SIG*; do
+	local WORKSPACE_DIR=$1
+	local FILE=$2
+	if [ -f "$WORKSPACE_DIR/$FILE/HONGGFUZZ.REPORT.TXT" ]; then
+		cat "$WORKSPACE_DIR/$FILE/HONGGFUZZ.REPORT.TXT"
+		for CASE in "$WORKSPACE_DIR/$FILE"/SIG*; do
 			cat "$CASE" | xxd -p
 		done
 		exit 1
 	fi
 }
 
-for TARGET in src/bin/*.rs; do
-	FILENAME=$(basename $TARGET)
-	FILE="${FILENAME%.*}"
-	CORPUS_DIR="hfuzz_workspace/$FILE/input"
-	CORPUS_COUNT=$(find "$CORPUS_DIR" -type f 2>/dev/null | wc -l)
-	# Run 8x the corpus size plus a baseline, ensuring full corpus replay
-	# with room for new mutations. The 10-minute hard cap (--run_time 600)
-	# prevents slow-per-iteration targets from running too long.
-	ITERATIONS=$((CORPUS_COUNT * 8 + 1000))
-	HFUZZ_RUN_ARGS="--exit_upon_crash -q -n8 -t 3 -N $ITERATIONS --run_time 600"
-	if [ "$FILE" = "chanmon_consistency_target" -o "$FILE" = "fs_store_target" ]; then
-		HFUZZ_RUN_ARGS="$HFUZZ_RUN_ARGS -F 64"
-	fi
-	export HFUZZ_RUN_ARGS
-	FUZZ_START=$(date +%s)
-	cargo --color always hfuzz run $FILE
-	FUZZ_END=$(date +%s)
-	FUZZ_TIME=$((FUZZ_END - FUZZ_START))
-	FUZZ_CORPUS_COUNT=$(find "$CORPUS_DIR" -type f 2>/dev/null | wc -l)
-	check_crash "$FILE"
-	if [ "$GITHUB_REF" = "refs/heads/main" ] || [ "$FUZZ_MINIMIZE" = "true" ]; then
-		HFUZZ_RUN_ARGS="-M -q -n8 -t 3"
+run_targets() {
+	local CRATE_DIR=$1
+	local TARGET_RUSTFLAGS=$2
+
+	pushd "$CRATE_DIR"
+	export HFUZZ_WORKSPACE="../hfuzz_workspace"
+	export HFUZZ_BUILD_ARGS="--features honggfuzz_fuzz"
+	export RUSTFLAGS="$TARGET_RUSTFLAGS"
+	cargo --color always hfuzz build -j8
+
+	for TARGET in src/bin/*.rs; do
+		FILENAME=$(basename "$TARGET")
+		FILE="${FILENAME%.*}"
+		CORPUS_DIR="$HFUZZ_WORKSPACE/$FILE/input"
+		CORPUS_COUNT=$(find "$CORPUS_DIR" -type f 2>/dev/null | wc -l)
+		# Run 8x the corpus size plus a baseline, ensuring full corpus replay
+		# with room for new mutations. The 10-minute hard cap (--run_time 600)
+		# prevents slow-per-iteration targets from running too long.
+		ITERATIONS=$((CORPUS_COUNT * 8 + 1000))
+		HFUZZ_RUN_ARGS="--exit_upon_crash -q -n8 -t 3 -N $ITERATIONS --run_time 600"
+		if [ "$FILE" = "chanmon_consistency_target" -o "$FILE" = "fs_store_target" ]; then
+			HFUZZ_RUN_ARGS="$HFUZZ_RUN_ARGS -F 64"
+		fi
 		export HFUZZ_RUN_ARGS
-		MIN_START=$(date +%s)
-		cargo --color always hfuzz run $FILE
-		MIN_END=$(date +%s)
-		MIN_TIME=$((MIN_END - MIN_START))
-		MIN_CORPUS_COUNT=$(find "$CORPUS_DIR" -type f 2>/dev/null | wc -l)
-		check_crash "$FILE"
-		SUMMARY="${SUMMARY}${FILE}|${ITERATIONS}|${CORPUS_COUNT}|${FUZZ_CORPUS_COUNT}|${FUZZ_TIME}|${MIN_CORPUS_COUNT}|${MIN_TIME}\n"
-	else
-		SUMMARY="${SUMMARY}${FILE}|${ITERATIONS}|${CORPUS_COUNT}|${FUZZ_CORPUS_COUNT}|${FUZZ_TIME}|-|-\n"
-	fi
-done
+		FUZZ_START=$(date +%s)
+		cargo --color always hfuzz run "$FILE"
+		FUZZ_END=$(date +%s)
+		FUZZ_TIME=$((FUZZ_END - FUZZ_START))
+		FUZZ_CORPUS_COUNT=$(find "$CORPUS_DIR" -type f 2>/dev/null | wc -l)
+		check_crash "$HFUZZ_WORKSPACE" "$FILE"
+		if [ "$GITHUB_REF" = "refs/heads/main" ] || [ "$FUZZ_MINIMIZE" = "true" ]; then
+			HFUZZ_RUN_ARGS="-M -q -n8 -t 3"
+			export HFUZZ_RUN_ARGS
+			MIN_START=$(date +%s)
+			cargo --color always hfuzz run "$FILE"
+			MIN_END=$(date +%s)
+			MIN_TIME=$((MIN_END - MIN_START))
+			MIN_CORPUS_COUNT=$(find "$CORPUS_DIR" -type f 2>/dev/null | wc -l)
+			check_crash "$HFUZZ_WORKSPACE" "$FILE"
+			SUMMARY="${SUMMARY}${FILE}|${ITERATIONS}|${CORPUS_COUNT}|${FUZZ_CORPUS_COUNT}|${FUZZ_TIME}|${MIN_CORPUS_COUNT}|${MIN_TIME}\n"
+		else
+			SUMMARY="${SUMMARY}${FILE}|${ITERATIONS}|${CORPUS_COUNT}|${FUZZ_CORPUS_COUNT}|${FUZZ_TIME}|-|-\n"
+		fi
+	done
+
+	popd
+}
+
+run_targets fuzz-fake-hashes "--cfg=fuzzing --cfg=secp256k1_fuzz --cfg=hashes_fuzz"
+run_targets fuzz-real-hashes "--cfg=fuzzing --cfg=secp256k1_fuzz"
 
 fmt_time() {
 	local secs=$1