fix: resolve flaky tests and add library unit test coverage

- fix(hpx/tests): fix with_concurrency_limit_layer_success by increasing
  concurrency limit from 1 to 2 and inner timeout from 200ms to 300ms,
  preventing third concurrent request from exceeding the timeout bound
- fix(hpx/tests): replace external httpbin.org dependency in
  http1_send_case_sensitive_headers with a local low_level_with_response
  server that inspects raw HTTP/1.1 bytes for case-sensitive headers
- test(hpx): add unit tests for cookie::Jar (set/retrieve, domain scoping)
- test(hpx): add unit tests for redirect (sensitive header stripping,
  same-origin preservation, referer policy for https-to-http)
- test(hpx): add unit tests for retry::Policy (default, never, scoped,
  classify_fn, budget, max_retries, first_retry_delay_clamping)
- test(hpx): add unit tests for proxy (no_proxy exclusion, basic auth,
  custom auth, http-only interception, socks default ports)
- test(hpx): add NSS keylog unit tests for TLS debugging
- test(hpx): add proxy pool failure classifier and proxy auth tests
- feat(hpx-browser): extend SSRF protection with private/loopback/link-local
  address checks, IPv6 mapped IPv4 detection, and additional block ranges
- feat(hpx-dl): add checksum verification variants (Sha256Truncated,
  LowerHexCaseInsensitive) and engine concurrency controls
- chore: add specs context and README for SSRF documentation

Author: Akagi201

AGENTS.md

@@ -30,24 +30,24 @@
 
 When introducing new dependencies, prefer these versions unless compatibility requires an upgrade:
 
-- `clap = "4.5.60"`
+- `clap = "4.6.1"`
 - `config = "0.15.19"`
 - `eyre = "0.6.12"`
 - `serde = "1.0.228"`
 - `thiserror = "2.0.18"`
-- `tokio = "1.49.0"`
+- `tokio = "1.52.1"`
 - `tracing = "0.1.44"`
-- `tracing-subscriber = "0.3.22"`
+- `tracing-subscriber = "0.3.23"`
 - `tracing-opentelemetry = "0.32.1"`
 - `opentelemetry = "0.31.0"`
 - `opentelemetry-otlp = "0.31.0"`
-- `sqlx = "=0.9.0-alpha.1"`
+- `sqlx = "0.9.0"`
 - `utoipa = "5.4.0"`
 - `utoipa-swagger-ui = "9.0.2"`
-- `arc-swap = "1.8.2"`
-- `hpx = "2.3.1"`
-- `scc = "3.6.5"`
-- `winnow = "0.7.14"`
+- `arc-swap = "1.9.2"`
+- `hpx = "2.5.4"`
+- `scc = "3.8.4"`
+- `winnow = "1.0.3"`
 - `shadow-rs = "1.7.0"`
 - `ecdysis = "1.0.1"`
 

Cargo.toml

@@ -3,7 +3,7 @@ members = ["bin/*", "crates/*"]
 resolver = "3"
 
 [workspace.package]
-version = "2.5.3"
+version = "2.5.4"
 edition = "2024"
 authors = ["Akagi201 <akagi201@gmail.com>"]
 license = "Apache-2.0"
@@ -184,11 +184,11 @@ all = "warn"
 
 [workspace.dependencies]
 # local crates
-hpx = { path = "crates/hpx", version = "2.5.3" }
-hpx-browser = { path = "crates/hpx-browser", version = "2.5.3" }
-hpx-cli = { path = "crates/hpx-cli", version = "2.5.3" }
-hpx-dl = { path = "crates/hpx-dl", version = "2.5.3" }
-hpx-yawc = { path = "crates/yawc", version = "2.5.3" }
+hpx = { path = "crates/hpx", version = "2.5.4" }
+hpx-browser = { path = "crates/hpx-browser", version = "2.5.4" }
+hpx-cli = { path = "crates/hpx-cli", version = "2.5.4" }
+hpx-dl = { path = "crates/hpx-dl", version = "2.5.4" }
+hpx-yawc = { path = "crates/yawc", version = "2.5.4" }
 
 # external crates
 ahash = "0.8.12"
@@ -199,9 +199,9 @@ async-trait = "0.1.80"
 axum = "0.8.9"
 axum-core = "0.5.6"
 base64 = "0.22.1"
-blitz-dom = { git = "https://github.com/DioxusLabs/blitz.git", rev = "77435aadb92c6bf47e50d74810c52120f3aa3f37", version = "=0.3.0-alpha.6", features = ["floats"] }
-blitz-html = { git = "https://github.com/DioxusLabs/blitz.git", rev = "77435aadb92c6bf47e50d74810c52120f3aa3f37", version = "=0.3.0-alpha.6" }
-blitz-traits = { git = "https://github.com/DioxusLabs/blitz.git", rev = "77435aadb92c6bf47e50d74810c52120f3aa3f37", version = "=0.3.0-alpha.6" }
+blitz-dom = { git = "https://github.com/DioxusLabs/blitz.git", rev = "406a571491523d69d52c396f240b14edd1a1974f", version = "=0.3.0-alpha.6", features = ["floats"] }
+blitz-html = { git = "https://github.com/DioxusLabs/blitz.git", rev = "406a571491523d69d52c396f240b14edd1a1974f", version = "=0.3.0-alpha.6" }
+blitz-traits = { git = "https://github.com/DioxusLabs/blitz.git", rev = "406a571491523d69d52c396f240b14edd1a1974f", version = "=0.3.0-alpha.6" }
 boring = "5.1.0"
 brotli = "8.0.4"
 bytes = "1.12.0"
@@ -213,7 +213,7 @@ criterion = "0.8.2"
 crossbeam-channel = "0.5.16"
 csv = "1.4.0"
 cucumber = "0.23.0"
-deno_core = "0.406.0"
+deno_core = "0.407.0"
 der = "0.8.0"
 digest = "0.11.3"
 encoding_rs = "0.8.35"

Justfile

@@ -10,6 +10,7 @@ lint:
     cargo +nightly fmt --all -- --check
     cargo +nightly clippy --all -- -D warnings
     cargo shear
+    just check-agents-md
 test:
     #!/usr/bin/env bash
     set -euo pipefail
@@ -51,6 +52,30 @@ test-coverage:
     else
         cargo tarpaulin --all-features --workspace --timeout 300
     fi
+# Check that AGENTS.md dependency versions match Cargo.toml
+check-agents-md:
+    #!/usr/bin/env bash
+    errors=0
+    while IFS= read -r line; do
+        crate=$(echo "$line" | sed -n 's/.*`\([^ ]*\) = ".*/\1/p')
+        agents_ver=$(echo "$line" | sed -n 's/.*"\([^"]*\)".*/\1/p')
+        if [ -z "$crate" ] || [ -z "$agents_ver" ]; then
+            continue
+        fi
+        cargo_ver=$(grep -E "^${crate} = " Cargo.toml 2>/dev/null | sed -n 's/.*"\([^"]*\)".*/\1/p' | head -1)
+        if [ -z "$cargo_ver" ]; then
+            continue
+        fi
+        if [ "$agents_ver" != "$cargo_ver" ]; then
+            echo "MISMATCH: ${crate}: AGENTS.md=${agents_ver} Cargo.toml=${cargo_ver}"
+            errors=$((errors + 1))
+        fi
+    done < <(sed -n '/^## Preferred Dependencies/,/^##/p' AGENTS.md | grep '^\-.*\`.*=.*"')
+    if [ $errors -gt 0 ]; then
+        echo "FAIL: ${errors} version mismatch(es) between AGENTS.md and Cargo.toml"
+        exit 1
+    fi
+    echo "OK: AGENTS.md versions match Cargo.toml"
 check-feature:
     #!/usr/bin/env bash
     set -euo pipefail

README.md

@@ -291,6 +291,7 @@ Default: `boring-tls`, `http1`, `http2`, `stream`, `tracing`.
 | `rustls-tls` | No | Rustls TLS backend (pure Rust) |
 | `openssl-tls` | No | OpenSSL TLS backend |
 | `openssl-vendored` | No | OpenSSL with vendored static linking |
+| `keylog` | No | TLS key logging to file (for debugging) — NOT for production |
 | `webpki-roots` | No | WebPKI root certificates |
 | **HTTP** | | |
 | `http1` | **Yes** | HTTP/1.1 support |

bin/hpx-cli/Cargo.toml

@@ -45,6 +45,7 @@ tokio = { workspace = true, features = [
     "process",
 ] }
 tracing-subscriber = { workspace = true }
+url = { workspace = true }
 uuid = { workspace = true }
 
 [dev-dependencies]

bin/hpx-cli/src/browser.rs

@@ -23,7 +23,7 @@ pub(crate) async fn handle_fetch(
     output: Option<PathBuf>,
     quiet: bool,
     _obey_robots: bool,
-    _allow_private_network: bool,
+    allow_private_network: bool,
     _v8_flags: Option<String>,
     _storage_dir: Option<PathBuf>,
 ) -> eyre::Result<()> {
@@ -42,6 +42,18 @@ pub(crate) async fn handle_fetch(
 
     // Original format: short-circuit browser, use raw HTTP client.
     if dump == DumpFormat::Original {
+        // SSRF check — reject private/special-use IPs unless explicitly allowed.
+        // Uses DNS resolution to prevent DNS rebinding attacks.
+        let parsed = url::Url::parse(&url)?;
+        if !allow_private_network
+            && let Some(host) = parsed.host_str()
+            && hpx_browser::net::ssrf::is_forbidden_resolved(host).await
+        {
+            eyre::bail!(
+                "SSRF protection: request to {host} is forbidden (use --allow-private-network to bypass)"
+            );
+        }
+
         let client = hpx::Client::new();
         let resp = client.get(&url).send().await.wrap_err("fetch failed")?;
         let bytes = resp.bytes().await.wrap_err("reading response body")?;
@@ -260,10 +272,6 @@ pub(crate) async fn handle_scrape(
         }
     }
 
-    let _ = (obey_robots, allow_private_network, v8_flags, storage_dir);
-    // ponytail: obey_robots / allow_private_network / v8_flags / storage_dir
-    // not yet wired to in-process worker. Ignored for now.
-
     let sem = std::sync::Arc::new(Semaphore::new(concurrency));
     let eval = std::sync::Arc::new(eval);
 
@@ -396,10 +404,11 @@ pub(crate) async fn handle_serve(
     _storage_dir: Option<PathBuf>,
     quiet: bool,
     _obey_robots: bool,
-    _allow_private_network: bool,
+    allow_private_network: bool,
     _v8_flags: Option<String>,
 ) -> eyre::Result<()> {
-    // ponytail: allow_file_access, storage_dir, obey_robots, allow_private_network, v8_flags not wired yet.
+    // ponytail: allow_file_access, storage_dir, obey_robots, v8_flags not wired yet.
+    let _ = allow_private_network; // CDP server uses Page which has SSRF via HttpClient
 
     let html = "<html><body></body></html>";
 

bin/hpx-cli/src/cli.rs

@@ -357,7 +357,7 @@ pub(crate) struct Cli {
     #[arg(long, global = true, env = "HPX_OBEY_ROBOTS")]
     pub obey_robots: bool,
 
-    /// Allow fetching from private/local network IPs.
+    /// Allow requests to private/internal IP addresses (disabled by default for security).
     #[arg(long, global = true, env = "HPX_ALLOW_PRIVATE_NETWORK")]
     pub allow_private_network: bool,
 

crates/hpx-browser/Cargo.toml

@@ -66,7 +66,9 @@ which = { workspace = true, optional = true }
 winnow = { workspace = true }
 
 [dev-dependencies]
+axum = { workspace = true }
 criterion = { workspace = true, features = ["html_reports"] }
+hpx = { workspace = true }
 proptest = { workspace = true }
 sha1 = { workspace = true }
 

crates/hpx-browser/src/net/mod.rs

@@ -59,6 +59,9 @@ pub enum NetError {
 
     #[error("hpx client error: {0}")]
     Client(#[from] hpx::Error),
+
+    #[error("SSRF protection: request to {0} is forbidden (use --allow-private-network to bypass)")]
+    Ssrf(String),
 }
 
 // ---------------------------------------------------------------------------
@@ -145,6 +148,7 @@ pub struct HttpClient {
     accept_ch_origins: scc::HashSet<String>,
     h1_only_hosts: scc::HashSet<String>,
     browser_profile: hpx::BrowserProfile,
+    allow_private_network: bool,
 }
 
 impl HttpClient {
@@ -169,6 +173,7 @@ impl HttpClient {
             accept_ch_origins: session.accept_ch.clone(),
             h1_only_hosts: session.h1_only_hosts.clone(),
             browser_profile,
+            allow_private_network: false,
         })
     }
 
@@ -180,6 +185,12 @@ impl HttpClient {
         &self.browser_profile
     }
 
+    /// Allow requests to private/internal IP addresses (disabled by default).
+    pub fn allow_private_network(mut self, allow: bool) -> Self {
+        self.allow_private_network = allow;
+        self
+    }
+
     /// Whether `host` has previously sent `Accept-CH`.
     pub fn has_accept_ch(&self, host: &str) -> bool {
         self.accept_ch_origins.contains_sync(host)
@@ -476,6 +487,17 @@ impl HttpClient {
         extra_headers: &[(String, String)],
     ) -> Result<hpx::Response, NetError> {
         let parsed = Url::parse(url)?;
+
+        // SSRF check — reject private/special-use IPs unless explicitly allowed.
+        // Uses DNS resolution to prevent DNS rebinding attacks.
+        if !self.allow_private_network {
+            if let Some(host) = parsed.host_str() {
+                if crate::net::ssrf::is_forbidden_resolved(host).await {
+                    return Err(NetError::Ssrf(host.to_string()));
+                }
+            }
+        }
+
         let builder = match method {
             "GET" | "HEAD" => self.inner.get(url),
             "POST" => self.inner.post(url),
@@ -823,4 +845,65 @@ mod tests {
         let q = p;
         assert_eq!(p, q); // Copy semantics — both still usable
     }
+
+    // --- SSRF integration tests ---
+
+    #[tokio::test]
+    async fn ssrf_blocks_loopback_by_default() {
+        let client = HttpClient::new(hpx::BrowserProfile::Chrome).unwrap();
+        let result = client
+            .execute_single_request("GET", "http://127.0.0.1", None, &[])
+            .await;
+        assert!(result.is_err());
+        let err = result.unwrap_err().to_string();
+        assert!(
+            err.contains("127.0.0.1"),
+            "error should mention the blocked host, got: {err}"
+        );
+        assert!(
+            err.contains("--allow-private-network"),
+            "error should mention --allow-private-network bypass, got: {err}"
+        );
+    }
+
+    #[tokio::test]
+    async fn ssrf_allows_private_network_when_enabled() {
+        let client = HttpClient::new(hpx::BrowserProfile::Chrome)
+            .unwrap()
+            .allow_private_network(true);
+        // This will fail with a connection error (no server listening), NOT an SSRF error
+        let result = client
+            .execute_single_request("GET", "http://127.0.0.1", None, &[])
+            .await;
+        match result {
+            Ok(_) => {}
+            Err(e) => {
+                let msg = e.to_string();
+                assert!(
+                    !msg.contains("SSRF"),
+                    "should not be SSRF rejection, got: {msg}"
+                );
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn ssrf_allows_public_ip() {
+        let client = HttpClient::new(hpx::BrowserProfile::Chrome).unwrap();
+        // 93.184.216.34 is example.com — real public IP; may fail with
+        // connection error in CI but must NOT be SSRF rejection.
+        let result = client
+            .execute_single_request("GET", "http://93.184.216.34", None, &[])
+            .await;
+        match result {
+            Ok(_) => {}
+            Err(e) => {
+                let msg = e.to_string();
+                assert!(
+                    !msg.contains("SSRF"),
+                    "public IP should not trigger SSRF, got: {msg}"
+                );
+            }
+        }
+    }
 }