Fix DuckDB bulk_insert failing when quoted CSV cells fall outside sniffer sample (#64)

* Fix DuckDB bulk_insert failing when quoted CSV cells fall outside sniffer sample (#62)

DuckDB's read_csv sniffer only examines the first ~20 480 rows; if none are
quoted, it sets quote=(empty) and then errors on any later quoted cell with a
column-count mismatch.  Passing quote='"' explicitly bypasses auto-detection.
Adds a regression test that inserts 25 000 plain rows followed by one row
whose value contains a comma (triggering csv.writer quoting).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* Bump version to 0.13.3

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* Add explicit escape='\"' to read_csv alongside quote='\"'

Makes the RFC 4180 doubling behaviour explicit rather than relying on
DuckDB defaulting escape to the same char as quote.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: martinv13

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "xml2db"
-version = "0.13.2"
+version = "0.13.3"
 authors = [
   { name="Commission de régulation de l'énergie", email="opensource@cre.fr" },
 ]

src/xml2db/dialect/duckdb.py

@@ -158,7 +158,7 @@ def bulk_insert(self, conn: Any, table: Any, records: list) -> None:
             sql = text(
                 f"INSERT INTO {full_name} ({insert_cols}) "
                 f"SELECT {select_exprs} "
-                f"FROM read_csv('{safe_path}', header=true, nullstr='', all_varchar=true)"
+                f"FROM read_csv('{safe_path}', header=true, nullstr='', all_varchar=true, quote='\"', escape='\"')"
             )
             conn.execute(sql)
         finally:

tests/test_bulk_insert.py

@@ -168,6 +168,27 @@ def test_duckdb_bulk_insert_scalar_column_default(duckdb_engine):
     assert rows[1]["flag"] is False
 
 
+def test_duckdb_bulk_insert_quoted_csv_field_after_large_unquoted_sample(duckdb_engine):
+    """Regression: DuckDB's CSV sniffer uses only the first ~20k rows as a sample.
+
+    If all sampled rows are unquoted, the sniffer sets quote=(empty), causing a
+    column-count error when it later hits a row whose cell value contains a comma
+    (making csv.writer emit a quoted field).  Explicitly passing quote='"' to
+    read_csv bypasses auto-detection and must always be present.
+    """
+    table = _make_table(duckdb_engine, "quoted_field_test")
+    # 'vals' value that contains a comma — document.py's 'join' transform can produce
+    # strings like '"val,ue",other' which csv.writer then wraps in outer quotes,
+    # yielding a quoted CSV cell.
+    problematic_value = '"val,ue",other_value'
+    records = [
+        {"id": i, "label": "simple"} for i in range(25_000)  # exceeds sniffer sample
+    ] + [{"id": 25_000, "label": problematic_value}]
+    rows = _roundtrip(duckdb_engine, table, records)
+    assert len(rows) == 25_001
+    assert rows[-1]["label"] == problematic_value
+
+
 def test_duckdb_bulk_insert_empty(duckdb_engine):
     table = _make_table(duckdb_engine, "empty_test")
     dialect = DuckDBDialect()