Add CodSpeed perf-regression benchmark suite

Nine modules over the duckdb-python binding hot paths: fetch (OUT-row),
arrow, pandas, produce_numpy (df/fetchnumpy columnar), ingest_native
(values/executemany), ingest_numpy (numpy scan + analyzer bind), udf (native +
vectorized arrow), types_roundtrip (type x direction matrix), cardinality
(LIMIT sweep). Full-consume discipline, warmup, real-null gotchas, tracemalloc
memory guard. See benchmarks/PLAN.md. Standalone (not yet wired into CI).

Author: evertlammerts

benchmarks/PLAN.md

@@ -0,0 +1,119 @@
+"""Standalone CodSpeed benchmark module for the Arrow read/write binding paths — NOT integrated
+(not in pyproject, not in CI, not committed). Run under each build's interpreter and compare:
+
+  M=/Users/evert/projects/duckdb-python/main/.venv-release/bin/python
+  C=/Users/evert/projects/duckdb-python/wt-cutover/.venv-release/bin/python
+  cd /Users/evert/projects/duckdb-python/wt-cutover
+  $M -m pytest benchmarks/test_arrow_perf.py --codspeed --codspeed-mode=walltime -o addopts=
+  $C -m pytest benchmarks/test_arrow_perf.py --codspeed --codspeed-mode=walltime -o addopts=
+
+DESIGN — the data must be FULLY MOVED, not lazily wrapped, or the benchmark measures nothing:
+  * READ (arrow -> duckdb): the duckdb ENGINE must scan every value. We aggregate over the actual
+    columns (sum/length), NOT count(*) -- count(*) is answered from arrow metadata without touching data.
+  * WRITE (duckdb -> arrow): the CONSUMER must materialize everything.
+      - to_arrow_table() / pl() are EAGER (the full table / polars DataFrame is built).
+      - to_arrow_reader() is LAZY -- duckdb only produces a batch when it is pulled -- so we iterate the
+        whole stream to actually exercise and consume the write path.
+
+pyarrow/polars are pinned to the SAME version in both .venv-release, so the A/B delta is purely the binding.
+"""
+
+import duckdb
+import pyarrow as pa
+import pytest
+
+N = 500_000
+WRITE_Q_NUM = "SELECT i::BIGINT AS a, (i * 1.5)::DOUBLE AS b FROM range(500000) t(i)"
+WRITE_Q_STR = "SELECT ('str_value_' || i) AS s FROM range(500000) t(i)"
+
+
+@pytest.fixture
+def con():
+    c = duckdb.connect()
+    yield c
+    c.close()
+
+
+@pytest.fixture(scope="module")
+def arrow_numeric():
+    return pa.table(
+        {
+            "a": pa.array(range(N), type=pa.int64()),
+            "b": pa.array([i * 1.5 for i in range(N)], type=pa.float64()),
+        }
+    )
+
+
+@pytest.fixture(scope="module")
+def arrow_string():
+    return pa.table({"s": pa.array([f"str_value_{i}" for i in range(N)], type=pa.string())})
+
+
+@pytest.fixture(scope="module")
+def arrow_numeric_batches(arrow_numeric):
+    # RecordBatches are immutable/re-readable, so a fresh reader can be built from them every round
+    return arrow_numeric.schema, arrow_numeric.to_batches(max_chunksize=50_000)
+
+
+# --------------------------------------------------------------------------- #
+# READ: arrow -> duckdb. The engine must scan every value (sum/length force it).
+# --------------------------------------------------------------------------- #
+
+
+def test_read_arrow_numeric(benchmark, con, arrow_numeric):
+    con.register("t_num", arrow_numeric)
+    benchmark(lambda: con.execute("SELECT sum(a), sum(b) FROM t_num").fetchall())
+
+
+def test_read_arrow_string(benchmark, con, arrow_string):
+    con.register("t_str", arrow_string)
+    benchmark(lambda: con.execute("SELECT count(s), sum(length(s)) FROM t_str").fetchall())
+
+
+# ADDED: RecordBatchReader ingest -- the SAME PythonTableArrowArrayStreamFactory but STREAMING (distinct from
+# the materialized Table read above). A fresh reader is built per round (the engine drains it); sum() forces a
+# full scan of every value.
+
+
+def test_read_arrow_reader_numeric(benchmark, con, arrow_numeric_batches):
+    schema, batches = arrow_numeric_batches
+
+    def run():
+        reader = pa.RecordBatchReader.from_batches(schema, iter(batches))
+        con.register("t_rdr", reader)
+        return con.execute("SELECT sum(a), sum(b) FROM t_rdr").fetchall()
+
+    run()  # warm
+    benchmark(run)
+
+
+# --------------------------------------------------------------------------- #
+# WRITE: duckdb -> arrow, consumer fully materializes / fully drains the stream.
+# --------------------------------------------------------------------------- #
+
+
+def test_write_arrow_table_numeric(benchmark, con):
+    benchmark(lambda: con.sql(WRITE_Q_NUM).to_arrow_table())
+
+
+def test_write_arrow_table_string(benchmark, con):
+    benchmark(lambda: con.sql(WRITE_Q_STR).to_arrow_table())
+
+
+def test_write_arrow_reader_consumed(benchmark, con):
+    def run():
+        reader = con.sql(WRITE_Q_NUM).to_arrow_reader(100_000)
+        rows = 0
+        for batch in reader:  # drain the lazy stream so duckdb actually produces every batch
+            rows += batch.num_rows
+        return rows
+
+    benchmark(run)
+
+
+def test_write_polars_numeric(benchmark, con):
+    benchmark(lambda: con.sql(WRITE_Q_NUM).pl())
+
+
+def test_write_polars_string(benchmark, con):
+    benchmark(lambda: con.sql(WRITE_Q_STR).pl())

benchmarks/test_arrow_perf.py

@@ -0,0 +1,63 @@
+"""Standalone CodSpeed benchmark module: the RESULT-CARDINALITY (top-N) sweep — NOT integrated (not in
+pyproject, not in CI, not committed). Run under each build's interpreter and compare:
+
+  M=/Users/evert/projects/duckdb-python/main/.venv-release/bin/python
+  C=/Users/evert/projects/duckdb-python/wt-codspeed/.venv-release/bin/python
+  cd /Users/evert/projects/duckdb-python/wt-codspeed
+  $M -m pytest benchmarks/test_cardinality_perf.py --codspeed --codspeed-mode=walltime -o addopts= -p no:cacheprovider
+  $C -m pytest benchmarks/test_cardinality_perf.py --codspeed --codspeed-mode=walltime -o addopts= -p no:cacheprovider
+
+WHY THIS MODULE (adopted from iqmo-org/bareduckdb): hold the SOURCE fixed and sweep only the number of rows
+materialized to Python via ORDER BY ... LIMIT n for n in {100, 1k, 10k, 100k}, through fetchall / df /
+to_arrow_table. The engine cost (scan the fixed SRC + top-N heap) stays ~constant, so the walltime delta
+across n is dominated by the per-row binding conversion -> a clean per-row slope. The n=100 end is the
+noise-free overhead regime (the natural instruction-count-gate point); the n=100k end is throughput.
+
+A clean monotone slope (and ~parity slope between the two builds) is the signal we report; a build whose slope
+is steeper has a per-row conversion regression. Source held constant rules out scan-cost as the confound (a
+cleaner axis than varying range(), which also changes scan cost).
+
+numpy/pandas/pyarrow are pinned to the SAME versions in both .venv-release, so the A/B delta is purely the binding.
+"""
+
+import duckdb
+import pytest
+
+SRC = 200_000  # fixed source size -> constant engine scan + top-N across all n
+LIMITS = [100, 1_000, 10_000, 100_000]
+
+# 3 columns (BIGINT, DOUBLE, VARCHAR) so the per-row conversion is non-trivial; source is a fixed inline
+# subquery (no table state) and ORDER BY forces a full scan + top-N of the same SRC rows every time.
+_SRC_SUBQ = f"(SELECT i::BIGINT AS a, (i * 1.5)::DOUBLE AS b, ('s_' || i) AS s FROM range({SRC}) t(i))"
+
+
+def _query(n):
+    return f"SELECT a, b, s FROM {_SRC_SUBQ} ORDER BY a DESC LIMIT {n}"
+
+
+@pytest.fixture
+def con():
+    c = duckdb.connect()
+    yield c
+    c.close()
+
+
+@pytest.mark.parametrize("n", LIMITS)
+def test_limit_fetchall(benchmark, con, n):
+    q = _query(n)
+    con.execute(q).fetchall()  # warm
+    benchmark(lambda: con.execute(q).fetchall())
+
+
+@pytest.mark.parametrize("n", LIMITS)
+def test_limit_df(benchmark, con, n):
+    q = _query(n)
+    con.sql(q).df()  # warm
+    benchmark(lambda: con.sql(q).df())
+
+
+@pytest.mark.parametrize("n", LIMITS)
+def test_limit_to_arrow(benchmark, con, n):
+    q = _query(n)
+    con.sql(q).to_arrow_table()  # warm
+    benchmark(lambda: con.sql(q).to_arrow_table())

benchmarks/test_cardinality_perf.py

@@ -0,0 +1,141 @@
+"""Standalone CodSpeed benchmark module — NOT integrated (not in pyproject, not in CI, not committed).
+
+Purpose: A/B the binding-layer perf between the two builds (pybind11 `main` vs nanobind cutover), in particular
+the narrow-column `fetchall` regression. Run the SAME file under each build's interpreter and compare:
+
+  M=/Users/evert/projects/duckdb-python/main/.venv-release/bin/python
+  C=/Users/evert/projects/duckdb-python/wt-cutover/.venv-release/bin/python
+  cd /Users/evert/projects/duckdb-python/wt-cutover
+  $M -m pytest benchmarks/test_fetch_perf.py --codspeed --codspeed-mode=walltime -o addopts=
+  $C -m pytest benchmarks/test_fetch_perf.py --codspeed --codspeed-mode=walltime -o addopts=
+
+NOTE: macOS arm64 has no Valgrind, so only `--codspeed-mode=walltime` works locally (wall-clock stats). The
+deterministic instruction-count mode (`--codspeed-mode=simulation`) needs Linux + the CodSpeed instrument
+(CI, or `codspeed run` in a Linux container). In CI/cloud, CodSpeed compares each run against a git baseline;
+locally we get the same benchmark workflow but A/B by running the file under the two interpreters by hand.
+"""
+
+import duckdb
+import pytest
+
+
+@pytest.fixture
+def con():
+    c = duckdb.connect()
+    yield c
+    c.close()
+
+
+def _bench_fetchall(benchmark, con, query):
+    con.execute(query).fetchall()  # warm the engine before measuring
+    benchmark(lambda: con.execute(query).fetchall())
+
+
+def test_fetchall_int(benchmark, con):
+    _bench_fetchall(benchmark, con, "SELECT i::BIGINT AS a FROM range(200000) t(i)")
+
+
+def test_fetchall_smallint(benchmark, con):
+    _bench_fetchall(benchmark, con, "SELECT (i % 100)::INTEGER AS a FROM range(200000) t(i)")
+
+
+def test_fetchall_double(benchmark, con):
+    _bench_fetchall(benchmark, con, "SELECT (i * 1.5)::DOUBLE AS a FROM range(200000) t(i)")
+
+
+def test_fetchall_2int(benchmark, con):
+    _bench_fetchall(benchmark, con, "SELECT i::BIGINT AS a, (i + 1)::BIGINT AS b FROM range(200000) t(i)")
+
+
+def test_fetchall_str(benchmark, con):
+    _bench_fetchall(benchmark, con, "SELECT ('str_value_' || i) AS s FROM range(100000) t(i)")
+
+
+def test_fetchall_mixed(benchmark, con):
+    query = (
+        "SELECT i::BIGINT AS bi, ('str_' || i) AS s, [i, i + 1, i + 2] AS lst, "
+        "{'a': i, 'b': i + 1} AS st FROM range(50000) t(i)"
+    )
+    _bench_fetchall(benchmark, con, query)
+
+
+def test_fetchone_iter(benchmark, con):
+    query = "SELECT i::BIGINT AS a, (i * 1.5)::DOUBLE AS b FROM range(100000) t(i)"
+
+    def run():
+        rel = con.execute(query)
+        while rel.fetchone() is not None:
+            pass
+
+    benchmark(run)
+
+
+# --------------------------------------------------------------------------- #
+# ADDED: small-N instruction-count-gate variants (the narrow-numeric fixed-cost path, noise-free at range(2048)
+# under simulation mode in CI), expensive scalar OUT-row types (timestamptz pytz-per-row, blob, null-heavy), a
+# heterogeneous per-cell-dispatch row (hugeint+uuid+decimal128+varchar, distinct from homogeneous columns), and
+# the batched fetchmany loop.
+# --------------------------------------------------------------------------- #
+
+
+def test_fetchall_int_gate(benchmark, con):
+    _bench_fetchall(benchmark, con, "SELECT i::BIGINT AS a FROM range(2048) t(i)")
+
+
+def test_fetchall_2int_gate(benchmark, con):
+    _bench_fetchall(benchmark, con, "SELECT i::BIGINT AS a, (i + 1)::BIGINT AS b FROM range(2048) t(i)")
+
+
+def test_fetchall_null_heavy(benchmark, con):
+    _bench_fetchall(
+        benchmark, con, "SELECT CASE WHEN i % 2 = 0 THEN NULL ELSE i::BIGINT END FROM range(200000) t(i)"
+    )
+
+
+def test_fetchall_timestamptz(benchmark, con):
+    _bench_fetchall(
+        benchmark, con, "SELECT (TIMESTAMPTZ '2020-01-01' + (i * INTERVAL 1 SECOND)) FROM range(100000) t(i)"
+    )
+
+
+def test_fetchall_decimal128(benchmark, con):
+    _bench_fetchall(benchmark, con, "SELECT ((i * 1.5)::DECIMAL(28, 6)) FROM range(200000) t(i)")
+
+
+def test_fetchall_blob(benchmark, con):
+    _bench_fetchall(benchmark, con, "SELECT ('blob_value_' || i)::BLOB FROM range(100000) t(i)")
+
+
+def test_fetchall_mixed_wide(benchmark, con):
+    # heterogeneous row -> per-cell type dispatch in the Fetchone column loop (distinct branch/cache profile
+    # from the homogeneous single-type columns above)
+    query = (
+        "SELECT (i::HUGEINT * 1000000000000) AS h, gen_random_uuid() AS u, "
+        "((i * 1.5)::DECIMAL(28, 6)) AS d, ('string_' || i) AS s FROM range(100000) t(i)"
+    )
+    _bench_fetchall(benchmark, con, query)
+
+
+def test_fetchmany_batched(benchmark, con):
+    query = "SELECT i::BIGINT AS a, (i * 1.5)::DOUBLE AS b FROM range(100000) t(i)"
+
+    def run():
+        rel = con.execute(query)
+        while True:
+            rows = rel.fetchmany(10_000)
+            if not rows:
+                break
+
+    benchmark(run)
+
+
+def test_expr_many(benchmark):
+    def run():
+        out = []
+        for i in range(2000):
+            col = duckdb.ColumnExpression(f"col_{i}")
+            const = duckdb.ConstantExpression(i)
+            out.append(((col + const) * duckdb.ConstantExpression(2)).alias(f"a{i}"))
+        return len(out)
+
+    benchmark(run)