fix: CSVToDocument row mode no longer crashes the batch on a row with extra fields (#11944)

Author: otiscuilei

haystack/components/converters/csv.py

@@ -163,7 +163,12 @@ def run(
 
             # Create DictReader; if this fails, raise (no fallback)
             try:
-                reader = csv.DictReader(io.StringIO(data), delimiter=self.delimiter, quotechar=self.quotechar)
+                # ``restkey`` ensures surplus fields on ragged rows (rows with more values than the
+                # header, e.g. an unquoted comma inside a value) land under an explicit string key
+                # instead of the default ``None`` key, which would break ``Document`` id generation.
+                reader = csv.DictReader(
+                    io.StringIO(data), delimiter=self.delimiter, quotechar=self.quotechar, restkey="extra_columns"
+                )
             except Exception as e:
                 raise RuntimeError(f"CSVToDocument(row): could not parse CSV rows for {source}: {e}") from e
 

releasenotes/notes/csv-to-document-row-mode-extra-fields-8783f95b2b564383.yaml

@@ -0,0 +1,7 @@
+---
+fixes:
+  - |
+    Fixed ``CSVToDocument`` in ``conversion_mode="row"`` crashing the entire batch when a CSV row
+    has more fields than the header (ragged rows, e.g. an unquoted comma inside a value). Surplus
+    values are now stored under an explicit ``extra_columns`` metadata key instead of the ``None``
+    key, which previously broke Document id generation and discarded results from all sources.

test/components/converters/test_csv_to_document.py

@@ -219,3 +219,26 @@ def broken_reader(*_args, **_kwargs):  # noqa: D401
         monkeypatch.setattr(csv_mod.csv, "DictReader", broken_reader, raising=True)
         with pytest.raises(RuntimeError):
             _ = conv.run(sources=[f], content_column="a")
+
+    def test_row_mode_ragged_row_does_not_crash(self):
+        # A data row with more fields than the header (e.g. an unquoted comma inside a value).
+        # Previously the surplus value landed under the None key, which broke Document id
+        # generation (TypeError sorting None against str keys) and aborted the whole batch.
+        valid = ByteStream(data=b"text,author\r\nfine,Ada\r\n", meta={"file_path": "valid.csv"})
+        ragged = ByteStream(data=b"text,note\r\nhello,city,state\r\n", meta={"file_path": "ragged.csv"})
+
+        conv = CSVToDocument(conversion_mode="row")
+        out = conv.run(sources=[valid, ragged], content_column="text")
+        docs = out["documents"]
+
+        # Both sources yielded a Document; the earlier valid source is not lost.
+        assert len(docs) == 2
+        assert docs[0].content == "fine"
+        assert docs[0].meta["author"] == "Ada"
+
+        ragged_doc = docs[1]
+        assert ragged_doc.content == "hello"
+        assert ragged_doc.meta["note"] == "city"
+        # Surplus value is preserved under an explicit (non-None) string meta key.
+        assert None not in ragged_doc.meta
+        assert "state" in ragged_doc.meta["extra_columns"]