Merge rust-bitcoin#467: Upstream minurl::Url type for url parsing

7d0945e Add percent decoding to `query_pairs()` for parity with `url::Url` (Elias Rohrer)
1bdf361 Apply trimming and filtering to URL accessor methods (Elias Rohrer)
8664a2a Trim whitespace from URL input in `Url::parse` (Elias Rohrer)
d47c77a Add a simple fuzz test for url parsing (Matt Corallo)
4fa9e80 Integrate percent encoding into `Url` type (Elias Rohrer)
cb3ce89 Replace `HttpUrl` with new `Url` type (Elias Rohrer)
921e06c Add `Url` property and parity tests to bitreq (Elias Rohrer)
66337c0 Add `Url` type to bitreq (Elias Rohrer)

Pull request description:

  Fixes rust-bitcoin#468.

  The `url` crate has a notoriously large dependency tree which is why we want to avoid it as far as possible. However, we found us then re-implementing several aspects of URL parsing in different places. To this end, I recently (mostly vibe-)coded the 0-dependency [`minurl`](https://crates.io/crates/minurl) crate (https://github.com/tnull/minurl) which is meant as a drop-in replacement for the popular `url` crate.

  To this end, we kept the `Url` API completely compatible, and even added parity tests ensuring both APIs return exactly the same output given the same input.

  While I'm generally fine maintaining this as a separate crate, it makes a lot of sense to have this live as part of `bitreq` and hence have it available everywhere in the ecosystem. Here I propose to upstream the `minurl::Url` type (and ofc the corresponding test code). This also allows us to replace the ~half-done `http_url::HttpUrl` type.

ACKs for top commit:
  TheBlueMatt:
    ACK 7d0945e needs some of the followups before we can release but this looks good as-is so far.

Tree-SHA512: 9d48c556c620b385fe9f24d27154b0faf4528416a645ed4857bd494a00ae622a793f991bab9f68d8e85d11a48cd1766105c382acd943bcfd6f66ad319331fb8d

Author: tcharding

Cargo-minimal.lock

@@ -42,6 +42,8 @@ log = { version = "0.4.0", default-features = false, optional = true }
 [dev-dependencies]
 tiny_http = "0.12"
 tokio = { version = "1.0", default-features = false, features = ["macros", "rt-multi-thread", "time"] }
+proptest = { version = "1", default-features = false, features = ["std"] }
+url = { version = "2.4" }
 
 [package.metadata.docs.rs]
 all-features = true

Cargo-recent.lock

@@ -0,0 +1,48 @@
+[package]
+name = "bitreq-fuzz"
+version = "0.0.1"
+authors = ["Automatically generated"]
+publish = false
+edition = "2021"
+
+[package.metadata]
+cargo-fuzz = true
+
+[features]
+afl_fuzz = ["afl"]
+honggfuzz_fuzz = ["honggfuzz"]
+libfuzzer_fuzz = ["libfuzzer-sys"]
+stdin_fuzz = []
+
+[dependencies]
+bitreq = { path = ".." }
+url = "2.5"
+
+afl = { version = "0.12", optional = true }
+honggfuzz = { version = "0.5", optional = true, default-features = false }
+libfuzzer-sys = { version = "0.4", optional = true }
+
+# Prevent this from interfering with workspaces
+[workspace]
+members = ["."]
+
+[profile.release]
+lto = true
+codegen-units = 1
+debug-assertions = true
+overflow-checks = true
+
+# When testing a large fuzz corpus, -O1 offers a nice speedup
+[profile.dev]
+opt-level = 1
+
+[lib]
+name = "bitreq_fuzz"
+path = "src/lib.rs"
+crate-type = ["rlib", "dylib", "staticlib"]
+
+[lints.rust.unexpected_cfgs]
+level = "forbid"
+check-cfg = [
+    "cfg(fuzzing)",
+]

bitreq/Cargo.toml

@@ -0,0 +1,73 @@
+# Fuzzing bitreq
+
+This directory contains fuzzing infrastructure for the `bitreq` crate, specifically targeting the `Url` parser.
+
+## Structure
+
+- `src/url_parse.rs` - Fuzz target for the URL parser
+- `src/bin/target_template.txt` - Template for generating fuzz target binaries
+- `src/bin/gen_target.sh` - Script to generate target binaries from template
+- `ci-fuzz.sh` - CI script for running fuzzing tests
+
+## Running Fuzz Tests
+
+### With stdin (for quick testing)
+
+```bash
+cd fuzz
+RUSTFLAGS="--cfg=fuzzing" cargo build --features stdin_fuzz --bin url_parse_target
+echo "http://example.com" | ./target/debug/url_parse_target
+```
+
+### With honggfuzz (for comprehensive fuzzing)
+
+```bash
+cd fuzz
+cargo install honggfuzz
+export RUSTFLAGS="--cfg=fuzzing"
+export HFUZZ_BUILD_ARGS="--features honggfuzz_fuzz"
+cargo hfuzz build
+HFUZZ_RUN_ARGS="--exit_upon_crash -v -n8 --run_time 30" cargo hfuzz run url_parse_target
+```
+
+### With AFL (alternative fuzzer)
+
+```bash
+cd fuzz
+cargo install afl
+export RUSTFLAGS="--cfg=fuzzing"
+cargo afl build --features afl_fuzz --bin url_parse_target
+cargo afl fuzz -i seeds -o findings target/debug/url_parse_target
+```
+
+### Running CI Fuzzing
+
+The `ci-fuzz.sh` script automates the fuzzing process with honggfuzz:
+
+```bash
+cd fuzz
+./ci-fuzz.sh
+```
+
+This will:
+1. Regenerate fuzz targets
+2. Install honggfuzz
+3. Build fuzz targets
+4. Run each target for 30 seconds
+5. Report any crashes found
+
+## Running Tests
+
+The fuzz targets include unit tests that can be run with:
+
+```bash
+cd fuzz
+RUSTFLAGS="--cfg=fuzzing" cargo test
+```
+
+## Adding New Fuzz Targets
+
+1. Create a new module in `src/` (e.g., `src/my_target.rs`)
+2. Export it from `src/lib.rs`
+3. Add a `GEN_TEST my_target` line to `src/bin/gen_target.sh`
+4. Run `cd src/bin && ./gen_target.sh` to generate the target binary

bitreq/fuzz/Cargo.toml

@@ -0,0 +1,36 @@
+#!/bin/bash
+set -e
+set -x
+
+# Regenerate targets to ensure they're up to date
+pushd src/bin
+rm -f *_target.rs
+./gen_target.sh
+[ "$(git diff)" != "" ] && exit 1
+popd
+
+export RUSTFLAGS="--cfg=fuzzing"
+
+cargo install --color always --force honggfuzz --no-default-features
+
+# Because we're fuzzing relatively few iterations, the maximum possible
+# compiler optimizations aren't necessary, so we turn off LTO
+sed -i 's/lto = true//' Cargo.toml
+
+export HFUZZ_BUILD_ARGS="--features honggfuzz_fuzz"
+
+cargo --color always hfuzz build -j8
+for TARGET in src/bin/*_target.rs; do
+	FILENAME=$(basename $TARGET)
+	FILE="${FILENAME%.*}"
+	HFUZZ_RUN_ARGS="--exit_upon_crash -v -n8 --run_time 30"
+	export HFUZZ_RUN_ARGS
+	cargo --color always hfuzz run $FILE
+	if [ -f hfuzz_workspace/$FILE/HONGGFUZZ.REPORT.TXT ]; then
+		cat hfuzz_workspace/$FILE/HONGGFUZZ.REPORT.TXT
+		for CASE in hfuzz_workspace/$FILE/SIG*; do
+			cat $CASE | xxd -p
+		done
+		exit 1
+	fi
+done

bitreq/fuzz/README.md

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+GEN_TEST() {
+	cat target_template.txt | sed s/TARGET_NAME/$1/g > $1_target.rs
+}
+
+GEN_TEST url_parse

bitreq/fuzz/ci-fuzz.sh

@@ -0,0 +1,66 @@
+// This file is auto-generated by gen_target.sh based on target_template.txt
+// To modify it, modify target_template.txt and run gen_target.sh instead.
+
+#![cfg_attr(feature = "libfuzzer_fuzz", no_main)]
+#![cfg_attr(rustfmt, rustfmt_skip)]
+
+#[cfg(not(fuzzing))]
+compile_error!("Fuzz targets need cfg=fuzzing");
+
+extern crate bitreq_fuzz;
+use bitreq_fuzz::TARGET_NAME::*;
+
+#[cfg(feature = "afl")]
+#[macro_use] extern crate afl;
+#[cfg(feature = "afl")]
+fn main() {
+	fuzz!(|data| {
+		TARGET_NAME_run(data.as_ptr(), data.len());
+	});
+}
+
+#[cfg(feature = "honggfuzz")]
+#[macro_use] extern crate honggfuzz;
+#[cfg(feature = "honggfuzz")]
+fn main() {
+	loop {
+		fuzz!(|data| {
+			TARGET_NAME_run(data.as_ptr(), data.len());
+		});
+	}
+}
+
+#[cfg(feature = "libfuzzer_fuzz")]
+#[macro_use] extern crate libfuzzer_sys;
+#[cfg(feature = "libfuzzer_fuzz")]
+fuzz_target!(|data: &[u8]| {
+	TARGET_NAME_run(data.as_ptr(), data.len());
+});
+
+#[cfg(feature = "stdin_fuzz")]
+fn main() {
+	use std::io::Read;
+
+	let mut data = Vec::with_capacity(8192);
+	std::io::stdin().read_to_end(&mut data).unwrap();
+	TARGET_NAME_run(data.as_ptr(), data.len());
+}
+
+#[test]
+fn run_test_cases() {
+	use std::fs;
+	use std::io::Read;
+
+	{
+		let data: Vec<u8> = vec![0];
+		TARGET_NAME_run(data.as_ptr(), data.len());
+	}
+	if let Ok(tests) = fs::read_dir("test_cases/TARGET_NAME") {
+		for test in tests {
+			let mut data: Vec<u8> = Vec::new();
+			let path = test.unwrap().path();
+			fs::File::open(&path).unwrap().read_to_end(&mut data).unwrap();
+			TARGET_NAME_run(data.as_ptr(), data.len());
+		}
+	}
+}

bitreq/fuzz/src/bin/gen_target.sh

@@ -0,0 +1,66 @@
+// This file is auto-generated by gen_target.sh based on target_template.txt
+// To modify it, modify target_template.txt and run gen_target.sh instead.
+
+#![cfg_attr(feature = "libfuzzer_fuzz", no_main)]
+#![cfg_attr(rustfmt, rustfmt_skip)]
+
+#[cfg(not(fuzzing))]
+compile_error!("Fuzz targets need cfg=fuzzing");
+
+extern crate bitreq_fuzz;
+use bitreq_fuzz::url_parse::*;
+
+#[cfg(feature = "afl")]
+#[macro_use] extern crate afl;
+#[cfg(feature = "afl")]
+fn main() {
+	fuzz!(|data| {
+		url_parse_run(data.as_ptr(), data.len());
+	});
+}
+
+#[cfg(feature = "honggfuzz")]
+#[macro_use] extern crate honggfuzz;
+#[cfg(feature = "honggfuzz")]
+fn main() {
+	loop {
+		fuzz!(|data| {
+			url_parse_run(data.as_ptr(), data.len());
+		});
+	}
+}
+
+#[cfg(feature = "libfuzzer_fuzz")]
+#[macro_use] extern crate libfuzzer_sys;
+#[cfg(feature = "libfuzzer_fuzz")]
+fuzz_target!(|data: &[u8]| {
+	url_parse_run(data.as_ptr(), data.len());
+});
+
+#[cfg(feature = "stdin_fuzz")]
+fn main() {
+	use std::io::Read;
+
+	let mut data = Vec::with_capacity(8192);
+	std::io::stdin().read_to_end(&mut data).unwrap();
+	url_parse_run(data.as_ptr(), data.len());
+}
+
+#[test]
+fn run_test_cases() {
+	use std::fs;
+	use std::io::Read;
+
+	{
+		let data: Vec<u8> = vec![0];
+		url_parse_run(data.as_ptr(), data.len());
+	}
+	if let Ok(tests) = fs::read_dir("test_cases/url_parse") {
+		for test in tests {
+			let mut data: Vec<u8> = Vec::new();
+			let path = test.unwrap().path();
+			fs::File::open(&path).unwrap().read_to_end(&mut data).unwrap();
+			url_parse_run(data.as_ptr(), data.len());
+		}
+	}
+}

bitreq/fuzz/src/bin/target_template.txt

@@ -0,0 +1,9 @@
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+#![cfg_attr(rustfmt, rustfmt_skip)]
+
+pub mod url_parse;