Add TPC-DS Phase 2: store_sales and inventory generators

Adds a complete tpcds_benchmark executable (built with -DTPCDS_ENABLE=ON)
that generates TPC-DS store_sales and inventory tables into Parquet, CSV,
ORC, Lance, Paimon, or Iceberg via the existing tpch::WriterInterface.

Build infrastructure (third_party/dsdgen/):
- CMakeLists.txt: mkheader → tables.h/streams.h/columns.h → distcomp →
  tpcds.idx → gen_dsts.py → dsts_generated.c → dsdgen_objs object library
- dsdgen_stubs.c: thin file-I/O stubs (not needed in embedded mode)
- tpcds_dsdgen.h: C++-safe forward declarations that bypass the fragile
  config.h/porting.h LINUX/HUGE_TYPE dependency chain; defines ds_key_t,
  decimal_t, ds_pricing_t, W_STORE_SALES_TBL, W_INVENTORY_TBL, and the
  EMBEDDED_DSDGEN callback globals
- cmake/gen_dsts.py: embeds tpcds.idx as a C byte array (dsts_generated.c)

C++ wrappers (include/tpch/, src/dsdgen/):
- dsdgen_wrapper.hpp/.cpp: DSDGenWrapper class — initialises dsdgen from
  the embedded tpcds.idx (via mkstemp temp file), exposes Arrow schemas
  for STORE_SALES and INVENTORY, and drives generation via the
  EMBEDDED_DSDGEN callback trampoline (store_sales is master-detail:
  8-16 line items per ticket; callback fires once per line item)
- dsdgen_converter.hpp/.cpp: append_*_to_builders() helpers mapping the
  dsdgen C structs to Arrow array builders; dec_to_double() correctly
  converts decimal_t scaled integers (avoids buggy upstream dectoflt)

Executable (src/tpcds_main.cpp):
- CLI mirrors tpch_benchmark: --format, --table, --scale-factor,
  --output-dir, --max-rows, --verbose
- Batched Arrow generation loop (10 000 rows/batch) → writer->write_batch

third_party/tpcds submodule updated to branch tpcds_cpp_embedded which
adds EMBEDDED_DSDGEN guards to w_store_sales.c (callback + suppressed
file I/O for both store_sales and store_returns output files).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: tsafin

.gitmodules

@@ -16,3 +16,6 @@
 [submodule "third_party/lance"]
 	path = third_party/lance
 	url = https://github.com/tsafin/lance.git
+[submodule "third_party/tpcds"]
+	path = third_party/tpcds
+	url = https://github.com/tsafin/tpchds-tools.git

CMakeLists.txt

@@ -28,6 +28,7 @@ option(TPCH_ENABLE_ICEBERG "Enable Apache Iceberg table format support" OFF)
 option(TPCH_ENABLE_LANCE "Enable Lance columnar format support (requires Rust)" OFF)
 option(TPCH_ENABLE_PERF_COUNTERS "Enable performance counters instrumentation" OFF)
 option(TPCH_ENABLE_MOLD "Enable mold linker if available (incompatible with GTest in this project)" ON)
+option(TPCDS_ENABLE "Enable TPC-DS data generation (tpcds_benchmark executable)" OFF)
 
 # Compiler configuration
 include(cmake/CompilerWarnings.cmake)
@@ -258,6 +259,11 @@ set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS
 add_subdirectory(third_party/dbgen EXCLUDE_FROM_ALL)
 include_directories(${DBGEN_INCLUDE_DIRS})
 
+# TPC-DS dsdgen objects (built only when TPCDS_ENABLE=ON)
+if(TPCDS_ENABLE)
+    add_subdirectory(third_party/dsdgen EXCLUDE_FROM_ALL)
+endif()
+
 # Copy TPC-H distribution file to build directory
 # Required by dbgen for loading nations, regions, and other lookup tables
 configure_file(
@@ -559,6 +565,46 @@ if(TPCH_ENABLE_PERF_COUNTERS)
     target_compile_definitions(tpch_benchmark PRIVATE TPCH_ENABLE_PERF_COUNTERS)
 endif()
 
+# TPC-DS benchmark executable
+if(TPCDS_ENABLE)
+    add_executable(tpcds_benchmark
+        src/tpcds_main.cpp
+        src/dsdgen/dsdgen_wrapper.cpp
+        src/dsdgen/dsdgen_converter.cpp
+        ${DSDGEN_OBJECTS}
+    )
+    target_link_libraries(tpcds_benchmark PRIVATE tpch_core)
+    target_include_directories(tpcds_benchmark PRIVATE ${DSDGEN_INCLUDE_DIRS})
+    # dsdgen upstream has several globals defined in multiple source files
+    # (pCurrentFile in driver.c and grammar_support.c, etc.). Allow duplicates
+    # at link time — the old GCC linker accepted these by default via -fcommon.
+    target_link_options(tpcds_benchmark PRIVATE -Wl,--allow-multiple-definition)
+    # dsdgen headers require LINUX=1 and TPCDS=1 to define ds_key_t and enable
+    # 64-bit support (config.h/#ifdef LINUX). Also needed by dsdgen_wrapper.cpp
+    # and dsdgen_converter.cpp which include dsdgen C headers.
+    target_compile_definitions(tpcds_benchmark PRIVATE TPCDS_ENABLE LINUX=1 TPCDS=1 EMBEDDED_DSDGEN=1)
+    if(TPCH_ENABLE_ORC)
+        target_compile_definitions(tpcds_benchmark PRIVATE TPCH_ENABLE_ORC)
+    endif()
+    if(TPCH_ENABLE_PAIMON)
+        target_compile_definitions(tpcds_benchmark PRIVATE TPCH_ENABLE_PAIMON)
+    endif()
+    if(TPCH_ENABLE_ICEBERG)
+        target_compile_definitions(tpcds_benchmark PRIVATE TPCH_ENABLE_ICEBERG)
+    endif()
+    if(TPCH_ENABLE_LANCE)
+        target_compile_definitions(tpcds_benchmark PRIVATE TPCH_ENABLE_LANCE)
+    endif()
+    if(TPCH_ENABLE_ASYNC_IO AND Uring_FOUND)
+        target_compile_definitions(tpcds_benchmark PRIVATE TPCH_ENABLE_ASYNC_IO)
+    endif()
+    if(TPCH_ENABLE_PERF_COUNTERS)
+        target_compile_definitions(tpcds_benchmark PRIVATE TPCH_ENABLE_PERF_COUNTERS)
+    endif()
+    message(STATUS "TPC-DS support enabled: tpcds_benchmark target added")
+    install(TARGETS tpcds_benchmark RUNTIME DESTINATION bin)
+endif()
+
 # Examples
 if(TPCH_BUILD_EXAMPLES)
     add_subdirectory(examples)

cmake/gen_dsts.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+"""
+gen_dsts.py - Embed the TPC-DS binary distribution index (tpcds.idx) into a
+              C source file as a static byte array.
+
+Usage: gen_dsts.py <tpcds.idx> <output.c>
+
+Background
+----------
+TPC-DS's dsdgen reads distribution data from a compiled binary file (tpcds.idx)
+produced by the 'distcomp' tool.  At runtime, dist.c opens this file via:
+
+    fopen(get_str("DISTRIBUTIONS"), "rb")
+
+To avoid shipping tpcds.idx as a separate runtime file, we embed its bytes here
+as a C uint8_t array.  DSDGenWrapper writes the embedded bytes to a tmpfile on
+first use and points the DISTRIBUTIONS param at that tmpfile.
+
+This mirrors the approach used by cmake/gen_dists.py for TPC-H's dists.dss.
+"""
+
+import sys
+import os
+
+
+def embed_binary(input_path: str, output_path: str) -> None:
+    with open(input_path, "rb") as f:
+        data = f.read()
+
+    size = len(data)
+    filename = os.path.basename(input_path)
+
+    lines = []
+    lines.append(
+        "/* Auto-generated from {} by cmake/gen_dsts.py -- do not edit */".format(filename)
+    )
+    lines.append("")
+    lines.append("#include <stddef.h>")
+    lines.append("#include <stdint.h>")
+    lines.append("")
+    lines.append("/* Embedded binary content of {} ({} bytes) */".format(filename, size))
+    lines.append("const uint8_t tpcds_idx_data[] = {")
+
+    # 16 bytes per row for readability
+    for i in range(0, size, 16):
+        chunk = data[i : i + 16]
+        hex_vals = ", ".join("0x{:02x}".format(b) for b in chunk)
+        comma = "," if i + 16 < size else ""
+        lines.append("    {}{}".format(hex_vals, comma))
+
+    lines.append("};")
+    lines.append("")
+    lines.append(
+        "const size_t tpcds_idx_size = {};".format(size)
+    )
+    lines.append("")
+
+    out_dir = os.path.dirname(output_path)
+    if out_dir:
+        os.makedirs(out_dir, exist_ok=True)
+
+    with open(output_path, "w") as f:
+        f.write("\n".join(lines) + "\n")
+
+    print(
+        "Embedded {} ({} bytes) -> {}".format(filename, size, os.path.basename(output_path))
+    )
+
+
+def main() -> None:
+    if len(sys.argv) != 3:
+        print("Usage: gen_dsts.py <tpcds.idx> <output.c>", file=sys.stderr)
+        sys.exit(1)
+
+    input_path = sys.argv[1]
+    output_path = sys.argv[2]
+
+    if not os.path.exists(input_path):
+        print("Error: input file not found: {}".format(input_path), file=sys.stderr)
+        sys.exit(1)
+
+    embed_binary(input_path, output_path)
+
+
+if __name__ == "__main__":
+    main()

include/tpch/dsdgen_converter.hpp

@@ -0,0 +1,41 @@
+#pragma once
+
+#include <memory>
+#include <map>
+#include <string>
+#include <arrow/builder.h>
+
+namespace tpcds {
+
+/**
+ * Convert dsdgen C struct rows to Arrow array builders.
+ *
+ * Each function casts void* to the appropriate dsdgen struct, extracts
+ * fields, and appends to the matching Arrow builders.
+ */
+
+/**
+ * Append a store_sales row (W_STORE_SALES_TBL*) to Arrow builders.
+ * Schema matches DSDGenWrapper::get_schema(TableType::STORE_SALES).
+ */
+void append_store_sales_to_builders(
+    const void* row,
+    std::map<std::string, std::shared_ptr<arrow::ArrayBuilder>>& builders);
+
+/**
+ * Append an inventory row (W_INVENTORY_TBL*) to Arrow builders.
+ * Schema matches DSDGenWrapper::get_schema(TableType::INVENTORY).
+ */
+void append_inventory_to_builders(
+    const void* row,
+    std::map<std::string, std::shared_ptr<arrow::ArrayBuilder>>& builders);
+
+/**
+ * Generic dispatcher by table name.
+ */
+void append_dsdgen_row_to_builders(
+    const std::string& table_name,
+    const void* row,
+    std::map<std::string, std::shared_ptr<arrow::ArrayBuilder>>& builders);
+
+}  // namespace tpcds

include/tpch/dsdgen_wrapper.hpp

@@ -0,0 +1,116 @@
+#pragma once
+
+#include <memory>
+#include <functional>
+#include <string>
+#include <cstdint>
+
+#include <arrow/api.h>
+
+namespace tpcds {
+
+/**
+ * TPC-DS table identifiers for the 24 standard W_ (warehouse) tables.
+ * Numeric values match the generated tables.h constants (STORE_SALES=17, etc.).
+ */
+enum class TableType {
+    CALL_CENTER          = 0,
+    CATALOG_PAGE         = 1,
+    CATALOG_RETURNS      = 2,
+    CATALOG_SALES        = 3,
+    CUSTOMER             = 4,
+    CUSTOMER_ADDRESS     = 5,
+    CUSTOMER_DEMOGRAPHICS = 6,
+    DATE_DIM             = 7,
+    HOUSEHOLD_DEMOGRAPHICS = 8,
+    INCOME_BAND          = 9,
+    INVENTORY            = 10,
+    ITEM                 = 11,
+    PROMOTION            = 12,
+    REASON               = 13,
+    SHIP_MODE            = 14,
+    STORE                = 15,
+    STORE_RETURNS        = 16,
+    STORE_SALES          = 17,
+    TIME_DIM             = 18,
+    WAREHOUSE            = 19,
+    WEB_PAGE             = 20,
+    WEB_RETURNS          = 21,
+    WEB_SALES            = 22,
+    WEB_SITE             = 23,
+    COUNT_
+};
+
+/**
+ * C++ wrapper around the TPC-DS dsdgen reference implementation.
+ *
+ * Initializes dsdgen global state (embedded distribution data, scale factor,
+ * RNG seeds) and provides per-table generation methods with callback API.
+ *
+ * THREAD-SAFETY: NOT thread-safe. dsdgen uses global mutable state.
+ * Use one DSDGenWrapper per process, generate tables sequentially.
+ */
+class DSDGenWrapper {
+public:
+    /**
+     * Construct wrapper for the given scale factor.
+     * @param scale_factor TPC-DS scale factor (1 = ~1GB baseline)
+     * @param verbose      Print verbose diagnostic messages
+     */
+    explicit DSDGenWrapper(long scale_factor, bool verbose = false);
+    ~DSDGenWrapper();
+
+    DSDGenWrapper(const DSDGenWrapper&) = delete;
+    DSDGenWrapper& operator=(const DSDGenWrapper&) = delete;
+
+    /**
+     * Generate store_sales rows.
+     * Calls callback once per row with a const W_STORE_SALES_TBL*.
+     * @param callback  Invoked for each generated row.
+     * @param max_rows  Limit; -1 or 0 means generate all rows.
+     */
+    void generate_store_sales(
+        std::function<void(const void* row)> callback,
+        long max_rows = -1);
+
+    /**
+     * Generate inventory rows.
+     * Calls callback once per row with a const W_INVENTORY_TBL*.
+     */
+    void generate_inventory(
+        std::function<void(const void* row)> callback,
+        long max_rows = -1);
+
+    long scale_factor() const { return scale_factor_; }
+
+    /**
+     * Return the Arrow schema for a table type.
+     */
+    static std::shared_ptr<arrow::Schema> get_schema(TableType table);
+
+    /**
+     * Return expected row count for a table at the given scale factor.
+     * Uses dsdgen's get_rowcount() after initialization.
+     */
+    long get_row_count(TableType table) const;
+
+    /**
+     * Return the dsdgen integer table ID for a TableType.
+     */
+    static int table_id(TableType table);
+
+    /**
+     * Return the canonical lower-case table name string.
+     */
+    static std::string table_name(TableType table);
+
+private:
+    long scale_factor_;
+    bool verbose_;
+    bool initialized_;
+    std::string tmp_dist_path_;  // path to temporary tpcds.idx file
+
+    void init_dsdgen();
+};
+
+}  // namespace tpcds