fix(detect): skip leading ASCII whitespace in is_binary_file + 0.2.4

j-mendez · claude · j-mendez · commit 8b4755d8b206 · 2026-04-29T15:16:18.000-04:00
Server-padded HTML payloads (Aestiva HTML/OS engines emit
`\n\n\n\n\n\n\n\n\n&lt;!doctype html&gt;...` — observed in the wild on
seeleylake.com and similar legacy WordPress-adjacent stacks)
previously dodged magic-byte sniffing because the first byte was
whitespace, not a magic prefix.  Worse, the small fixed sniff
windows that downstream consumers (e.g. spider's 16-byte spool head
sample) feed in could leave no headroom for an actual signature
match if the padding ate into the window.

The fix adds a 256-byte compile-time `INTERESTING_FIRST_BYTE`
classifier (magic-table first bytes ∪ ASCII whitespace) that gates
the magic search.  Any byte not in either set short-circuits to
false in a single L1-resident lookup.  Whitespace bytes hand off to
a `#[cold]` helper that trims and re-runs the search.  Symmetrical
treatment for the legacy `is_binary_file_phf` sibling.

Bench delta (criterion, sub-bench `is_binary_file`):
  not_binary  3.86 ns -&gt; 0.78 ns   (-80%, 5x faster)
  png_match   7.33 ns -&gt; 9.10 ns   (+25%)
  jpeg_match 10.40 ns -&gt; 11.81 ns  (+16%)

The dominant real-world case is "not binary" (HTML responses), so
this is a net speedup; the absolute regression on binary headers is
~1-2 ns, well under the cost of any actual byte-level work the
caller is doing on the body.

Tests cover Aestiva-padded HTML, padded PNG, padded PDF, empty,
whitespace-only, and the bare-magic / non-binary regressions.

Co-Authored-By: Claude Opus 4.7 (1M context) &lt;noreply@anthropic.com&gt;
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "auto_encoder"
-version = "0.2.3"
+version = "0.2.4"
 edition = "2021"
 description = "Auto encoding library"
 repository = "https://github.com/spider-rs/auto-encoder"
diff --git a/src/detect.rs b/src/detect.rs
@@ -23,40 +23,149 @@ static MAGIC_TABLE: &[(u8, &[&[u8]])] = &[
     (0xFF, &[&[0xFF, 0xD8, 0xFF], &[0xFF, 0xFB]]),
 ];
 
+/// 256-entry classifier built at compile time.  A byte is "interesting"
+/// iff it's the first byte of a magic signature in [`MAGIC_TABLE`] or
+/// ASCII whitespace (` `, `\t`, `\n`, `\r`, `\x0c` — matches
+/// `u8::is_ascii_whitespace`).  All other bytes short-circuit to
+/// `false` without touching the magic search or the whitespace retry.
+///
+/// One byte per entry, so the table is a single 256-byte cache line —
+/// the lookup is one L1-resident load plus a branch.  This is faster
+/// than the original `binary_search_by_key` for the dominant
+/// "definitely-not-binary" HTML case (`<` etc.) by a factor of ~5x in
+/// micro-benches, more than amortizing the small overhead added to the
+/// magic-hit path for actual binary content.
+static INTERESTING_FIRST_BYTE: [bool; 256] = {
+    let mut t = [false; 256];
+    // Magic-table first bytes — keep in sync with `MAGIC_TABLE`.
+    let magic = [
+        0x00u8, 0x1A, 0x1F, 0x25, 0x42, 0x46, 0x47, 0x49, 0x4C, 0x4D, 0x4F, 0x50, 0x52, 0x66,
+        0x7F, 0x89, 0xCA, 0xFF,
+    ];
+    let mut i = 0;
+    while i < magic.len() {
+        t[magic[i] as usize] = true;
+        i += 1;
+    }
+    // ASCII whitespace bytes.
+    t[b' ' as usize] = true;
+    t[b'\t' as usize] = true;
+    t[b'\n' as usize] = true;
+    t[b'\r' as usize] = true;
+    t[0x0C] = true;
+    t
+};
+
 /// Checks if the file is a known binary format using its initial bytes.
+///
+/// Performance shape:
+/// * **Not binary, no whitespace** (the dominant HTML case): one
+///   bounds-check, one `INTERESTING_FIRST_BYTE` lookup, return false.
+/// * **Magic hit** (PNG/JPEG/PDF/...): same lookup then a binary search
+///   plus per-signature compare, identical to the original logic
+///   except for the leading single-cycle table load.
+/// * **Whitespace-padded** (Aestiva HTML/OS:
+///   `\n\n\n\n\n\n\n\n\n<!doctype html>`): hands off to a `#[cold]`
+///   helper that trims and re-runs the search.  Catches binaries with
+///   stray leading whitespace and stops whitespace-padded text bodies
+///   from being mis-classified as binary.
 #[inline]
 pub fn is_binary_file(content: &[u8]) -> bool {
-    if content.is_empty() {
+    let first = match content.first() {
+        Some(&b) => b,
+        None => return false,
+    };
+    if !INTERESTING_FIRST_BYTE[first as usize] {
         return false;
     }
-    let first = content[0];
     if let Ok(idx) = MAGIC_TABLE.binary_search_by_key(&first, |&(b, _)| b) {
         let (_, signatures) = MAGIC_TABLE[idx];
         for sig in signatures.iter() {
             if content.len() >= sig.len() && &content[..sig.len()] == *sig {
                 return true;
             }
         }
+        return false;
     }
-    false
+    // Reached only when the byte is in `INTERESTING_FIRST_BYTE` but not
+    // in `MAGIC_TABLE` — i.e. ASCII whitespace.  Cold path.
+    is_binary_file_ws_cold(content)
 }
 
 /// Checks if the file is a known binary format using its initial bytes.
 /// Uses the original PHF map implementation for backwards compatibility.
+///
+/// Same fast-path / cold-path shape as [`is_binary_file`].
 #[inline]
 pub fn is_binary_file_phf(content: &[u8]) -> bool {
-    if content.is_empty() {
+    let first = match content.first() {
+        Some(&b) => b,
+        None => return false,
+    };
+    if !INTERESTING_FIRST_BYTE[first as usize] {
         return false;
     }
-
-    if let Some(&keys) = FIRST_BYTE_MAP.get(&content[0]) {
+    if let Some(&keys) = FIRST_BYTE_MAP.get(&first) {
         for &key in keys {
             if let Some(&k) = ASSET_NUMBERS.get(key) {
                 if content.len() >= k.len() && &content[..k.len()] == k {
                     return true;
                 }
             }
         }
+        return false;
+    }
+    is_binary_file_phf_ws_cold(content)
+}
+
+/// Cold whitespace-retry helper for [`is_binary_file`].  Reached only
+/// when the first byte is ASCII whitespace.
+#[cold]
+#[inline(never)]
+fn is_binary_file_ws_cold(content: &[u8]) -> bool {
+    let mut i = 1;
+    while i < content.len() && content[i].is_ascii_whitespace() {
+        i += 1;
+    }
+    if i >= content.len() {
+        return false;
+    }
+    // SAFETY: i ∈ [1, content.len()).
+    let trimmed = unsafe { content.get_unchecked(i..) };
+    let first = trimmed[0];
+    if let Ok(idx) = MAGIC_TABLE.binary_search_by_key(&first, |&(b, _)| b) {
+        let (_, signatures) = MAGIC_TABLE[idx];
+        for sig in signatures.iter() {
+            if trimmed.len() >= sig.len() && &trimmed[..sig.len()] == *sig {
+                return true;
+            }
+        }
+    }
+    false
+}
+
+/// Cold whitespace-retry helper for [`is_binary_file_phf`].
+#[cold]
+#[inline(never)]
+fn is_binary_file_phf_ws_cold(content: &[u8]) -> bool {
+    let mut i = 1;
+    while i < content.len() && content[i].is_ascii_whitespace() {
+        i += 1;
+    }
+    if i >= content.len() {
+        return false;
+    }
+    // SAFETY: i ∈ [1, content.len()).
+    let trimmed = unsafe { content.get_unchecked(i..) };
+    let first = trimmed[0];
+    if let Some(&keys) = FIRST_BYTE_MAP.get(&first) {
+        for &key in keys {
+            if let Some(&k) = ASSET_NUMBERS.get(key) {
+                if trimmed.len() >= k.len() && &trimmed[..k.len()] == k {
+                    return true;
+                }
+            }
+        }
     }
     false
 }
diff --git a/src/lib.rs b/src/lib.rs
@@ -235,6 +235,36 @@ mod tests {
         assert!(!is_binary_file(&[0x01, 0x02, 0x03]));
     }
 
+    #[test]
+    fn test_is_binary_file_skips_leading_whitespace() {
+        use crate::detect::is_binary_file_phf;
+
+        // Aestiva HTML/OS-style padded HTML (the shape served by
+        // seeleylake.com): nine leading newlines then `<!doctype html>`.
+        // Must not be classified as binary.
+        let padded = b"\n\n\n\n\n\n\n\n\n<!doctype html>\n<html></html>";
+        assert!(!is_binary_file(padded));
+        assert!(!is_binary_file_phf(padded));
+
+        // Whitespace prefix in front of a real magic header still detects
+        // the binary correctly.
+        let mut padded_png = Vec::from(&b"  \t\r\n"[..]);
+        padded_png.extend_from_slice(&[0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]);
+        assert!(is_binary_file(&padded_png));
+        assert!(is_binary_file_phf(&padded_png));
+
+        let mut padded_pdf = Vec::from(&b"\n\n"[..]);
+        padded_pdf.extend_from_slice(b"%PDF-1.4\n");
+        assert!(is_binary_file(&padded_pdf));
+        assert!(is_binary_file_phf(&padded_pdf));
+
+        // Whitespace-only / empty bodies are not binary.
+        assert!(!is_binary_file(b""));
+        assert!(!is_binary_file(b"   \t\n\r"));
+        assert!(!is_binary_file_phf(b""));
+        assert!(!is_binary_file_phf(b"   \t\n\r"));
+    }
+
     #[test]
     fn test_encode_bytes() {
         let html_content = b"hello";
