Fix 10 code review findings from Rust/PyO3 core review

Correctness fixes:
- token.rs: implement distinct truncate_smart strategy (2:1 head/tail
  weighting) — was identical to middle strategy
- chunk.rs: always flush on max_tokens overflow; remove min_tokens guard
  that silently allowed chunks to exceed the hard cap
- chunk.rs: fix find_table_end CRLF handling — advance by actual
  terminator byte count (2 for CRLF, 1 for LF) instead of always +1
- encoding.rs: detect UTF-8 BOM before chardetng so decode uses
  utf-8-sig (strips BOM) rather than utf-8 (preserves it)
- normalize.rs: seed strip_headers_footers candidates from all pages,
  not just page 0, so cover-page documents strip running headers

Fallback path fixes (_fallback.py):
- detect_encoding: try cp1252 before latin-1 so Windows smart-quote
  bytes (0x80-0x9F) decode to printable chars, not C1 control chars
- chunk(): oversized paragraphs now set metadata={oversized: True}
  to match the Rust interface contract
- chunk(): use re.split capturing group to track actual separator
  lengths; fixes char_start/char_end drift after 3+-newline gaps
- count/truncate/chunk._count: pass allowed_special=all to tiktoken
  encode() to match Rust encode_with_special_tokens behaviour

core.py:
- _decode_bytes chain updated to (utf-8, cp1252, latin-1) so text/CSV
  files correctly handle Windows-encoded content

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: fsecada01

TextSpitter/_fallback.py

@@ -15,7 +15,7 @@ def detect_encoding(data: bytes) -> str:
     """Detect encoding by trying common codecs in priority order."""
     if not data:
         return "utf-8"
-    for enc in ("utf-8", "utf-8-sig", "latin-1", "cp1252"):
+    for enc in ("utf-8", "utf-8-sig", "cp1252", "latin-1"):
         try:
             data.decode(enc)
             return enc
@@ -150,24 +150,74 @@ def _count(self, text: str) -> int:
             import tiktoken
 
             enc = tiktoken.get_encoding(self.tokenizer)
-            return len(enc.encode(text))
+            # Mirror Rust encode_with_special_tokens: allow all special tokens.
+            return len(enc.encode(text, allowed_special="all"))
         except Exception:
-            # Rough character-based fallback: ~4 chars per token
             return len(text) // 4
 
     def chunk(self, text: str) -> list[Chunk]:
         import re
 
-        paragraphs = [p.strip() for p in re.split(r"\n\n+", text) if p.strip()]
+        # Split with a capturing group so we can measure the actual separator
+        # length (2+ newlines). Without this, char_cursor drifts when gaps use
+        # 3+ newlines because the old code always added a fixed +2.
+        pieces = re.split(r"(\n\n+)", text)
         chunks: list[Chunk] = []
         current_parts: list[str] = []
         current_tokens = 0
         char_cursor = 0
         current_start = 0
         section_title: str | None = None
 
-        for para in paragraphs:
+        for idx, piece in enumerate(pieces):
+            if idx % 2 == 1:
+                # Odd indices are separator strings ("\n\n", "\n\n\n", …)
+                char_cursor += len(piece)
+                continue
+
+            para = piece.strip()
+            if not para:
+                char_cursor += len(piece)
+                continue
+
             para_tokens = self._count(para)
+
+            # Single paragraph exceeds max_tokens — emit as oversized.
+            if para_tokens > self.max_tokens:
+                if current_parts:
+                    chunk_text = "\n\n".join(current_parts)
+                    chunks.append(
+                        Chunk(
+                            text=chunk_text,
+                            token_count=current_tokens,
+                            char_start=current_start,
+                            char_end=char_cursor,
+                            section_title=section_title,
+                            chunk_index=0,
+                            total_chunks=None,
+                            metadata={},
+                        )
+                    )
+                    current_parts = []
+                    current_tokens = 0
+                    current_start = char_cursor
+                end = char_cursor + len(piece)
+                chunks.append(
+                    Chunk(
+                        text=para,
+                        token_count=para_tokens,
+                        char_start=char_cursor,
+                        char_end=end,
+                        section_title=section_title,
+                        chunk_index=0,
+                        total_chunks=None,
+                        metadata={"oversized": True},
+                    )
+                )
+                char_cursor = end
+                current_start = char_cursor
+                continue
+
             if current_tokens + para_tokens > self.max_tokens and current_parts:
                 chunk_text = "\n\n".join(current_parts)
                 chunks.append(
@@ -188,7 +238,7 @@ def chunk(self, text: str) -> list[Chunk]:
 
             current_parts.append(para)
             current_tokens += para_tokens
-            char_cursor += len(para) + 2  # +2 for \n\n
+            char_cursor += len(piece)
 
         if current_parts:
             chunk_text = "\n\n".join(current_parts)
@@ -235,7 +285,13 @@ def _bpe(self):
 
     def count(self, text: str) -> int:
         bpe = self._bpe()
-        return len(bpe.encode(text)) if bpe else len(text) // 4
+        # allowed_special="all" mirrors Rust encode_with_special_tokens and
+        # prevents ValueError when text contains tokens like <|endoftext|>.
+        return (
+            len(bpe.encode(text, allowed_special="all"))
+            if bpe
+            else len(text) // 4
+        )
 
     def count_batch(self, texts: list[str]) -> list[int]:
         return [self.count(t) for t in texts]
@@ -255,7 +311,7 @@ def truncate(
             else:
                 kept = words[:max_tokens]
             return " ".join(kept)
-        tokens = bpe.encode(text)
+        tokens = bpe.encode(text, allowed_special="all")
         if len(tokens) <= max_tokens:
             return text
         if strategy == "middle":

TextSpitter/core.py

@@ -400,8 +400,12 @@ def docx_file_read(self) -> str:  # Added return type hint
 
     def _decode_bytes(self, data: bytes, label: str) -> str:
         """
-        Decode bytes to str, trying UTF-8 then latin-1 then UTF-8 with
-        replacement characters.
+        Decode bytes to str, trying UTF-8, cp1252, then latin-1, then UTF-8
+        with replacement characters.
+
+        cp1252 is tried before latin-1 so Windows smart-quote bytes (0x80-0x9F)
+        decode to printable characters instead of C1 control characters.
+        latin-1 always succeeds and acts as the final deterministic fallback.
 
         Args:
             data: Raw bytes to decode.
@@ -410,16 +414,13 @@ def _decode_bytes(self, data: bytes, label: str) -> str:
         Returns:
             str
         """
-        try:
-            return data.decode("utf-8")
-        except UnicodeDecodeError:
-            pass
-        try:
-            return data.decode("latin-1")
-        except UnicodeDecodeError:
-            pass
+        for enc in ("utf-8", "cp1252", "latin-1"):
+            try:
+                return data.decode(enc)
+            except (UnicodeDecodeError, LookupError):
+                continue
         logger.warning(
-            f"Could not decode {label} with utf-8 or latin-1, "
+            f"Could not decode {label} with utf-8, cp1252, or latin-1, "
             f"using utf-8 with replacement characters."
         )
         return data.decode("utf-8", errors="replace")

src/chunk.rs

@@ -189,21 +189,19 @@ impl TextChunker {
 
             let pending_tokens = bpe.encode_with_special_tokens(&current_text).len();
 
+            // Always flush on overflow — max_tokens is a hard cap; min_tokens
+            // is a soft target that must not allow chunks to exceed max_tokens.
             if pending_tokens + unit_tokens > self.max_tokens && !current_text.is_empty() {
-                // Emit pending chunk if it meets min_tokens.
-                let pending_count = bpe.encode_with_special_tokens(&current_text).len();
-                if pending_count >= self.min_tokens || chunks.is_empty() {
-                    chunks.push(self.make_chunk(
-                        &current_text,
-                        &bpe,
-                        current_start,
-                        char_cursor,
-                        current_section.clone(),
-                        false,
-                    ));
-                    current_text.clear();
-                    current_start = char_cursor;
-                }
+                chunks.push(self.make_chunk(
+                    &current_text,
+                    &bpe,
+                    current_start,
+                    char_cursor,
+                    current_section.clone(),
+                    false,
+                ));
+                current_text.clear();
+                current_start = char_cursor;
             }
 
             if let Some(title) = &unit.section_title {
@@ -338,15 +336,20 @@ fn push_text_units(
 
 fn find_table_end(text: &str, start: usize) -> usize {
     let from = &text[start..];
-    let mut end = start;
+    let mut offset = 0usize;
     for line in from.lines() {
-        if line.trim_start().starts_with('|') {
-            end += line.len() + 1; // +1 for newline
-        } else if line.trim().is_empty() {
-            end += line.len() + 1;
+        if line.trim_start().starts_with('|') || line.trim().is_empty() {
+            offset += line.len();
+            // lines() strips line terminators; advance past the actual bytes
+            // so CRLF (2 bytes) is handled correctly, not just LF (1 byte).
+            if from[offset..].starts_with("\r\n") {
+                offset += 2;
+            } else if offset < from.len() {
+                offset += 1;
+            }
         } else {
             break;
         }
     }
-    end.min(text.len())
+    (start + offset).min(text.len())
 }

src/encoding.rs

@@ -25,6 +25,13 @@ pub fn detect_encoding(data: &[u8]) -> String {
         return "utf-8".into();
     }
 
+    // Explicit BOM check before chardetng: chardetng returns "UTF-8" for
+    // BOM-prefixed files, but Python's "utf-8" codec preserves the BOM at
+    // position 0. "utf-8-sig" strips it during decode.
+    if data.starts_with(b"\xef\xbb\xbf") {
+        return "utf-8-sig".into();
+    }
+
     // Feed the entire buffer; last=true signals end-of-stream.
     let mut detector = EncodingDetector::new();
     detector.feed(data, true);

src/normalize.rs

@@ -82,20 +82,25 @@ fn strip_headers_footers(text: &str) -> String {
         .map(|p| p.lines().collect())
         .collect();
 
-    let candidate_lines: std::collections::HashSet<&str> = all_lines[0]
-        .iter()
+    // Collect every unique non-empty line from all pages, then keep only those
+    // present on more than half the pages. Seeding from page 0 alone misses
+    // running headers when page 0 is a cover page with no shared lines.
+    let all_unique: std::collections::HashSet<&str> = all_lines.iter()
+        .flat_map(|pg| pg.iter().copied())
+        .filter(|l| !l.trim().is_empty())
+        .collect();
+
+    let candidate_lines: std::collections::HashSet<&str> = all_unique
+        .into_iter()
         .filter(|line| {
             let trimmed = line.trim();
-            if trimmed.is_empty() { return false; }
             let count = all_lines.iter()
                 .filter(|page_lines| {
                     page_lines.iter().any(|l| l.trim() == trimmed)
                 })
                 .count();
-            // Present on more than half the pages = header/footer
             count * 2 > pages.len()
         })
-        .copied()
         .collect();
 
     if candidate_lines.is_empty() {

src/token.rs

@@ -94,9 +94,18 @@ impl TokenCounter {
 }
 
 fn truncate_smart(tokens: &[usize], max_tokens: usize) -> Vec<usize> {
-    let keep_start = max_tokens / 2;
+    // Weight the head 2:1 over the tail — beginning of document carries more
+    // context; middle is dropped first, then tail is trimmed before head.
+    let n = tokens.len();
+    let keep_start = (max_tokens * 2).div_ceil(3);
     let keep_end = max_tokens - keep_start;
-    let mut result = tokens[..keep_start].to_vec();
-    result.extend_from_slice(&tokens[tokens.len() - keep_end..]);
-    result
+    let tail_start = n.saturating_sub(keep_end);
+
+    if keep_end == 0 || tail_start <= keep_start {
+        tokens[..max_tokens.min(n)].to_vec()
+    } else {
+        let mut result = tokens[..keep_start].to_vec();
+        result.extend_from_slice(&tokens[tail_start..]);
+        result
+    }
 }

tests/test_file_extractor.py

@@ -456,7 +456,7 @@ def mock_decode_side_effect(encoding, errors=None):
     result = extractor.text_file_read()
 
     assert (
-        "Could not decode text file badtext.txt with utf-8 or latin-1"
+        "Could not decode text file badtext.txt with utf-8, cp1252, or latin-1"
         in "\n".join(log_capture)
     )
     mock_bytes_instance.decode.assert_any_call("utf-8", errors="replace")
@@ -506,7 +506,7 @@ def mock_decode_side_effect(encoding, errors=None):
     result = extractor.csv_file_read()
 
     assert (
-        "Could not decode CSV file bad.csv with utf-8 or latin-1"
+        "Could not decode CSV file bad.csv with utf-8, cp1252, or latin-1"
         in "\n".join(log_capture)
     )
     mock_bytes_instance.decode.assert_any_call("utf-8", errors="replace")