feat: add YAML input support (-I yaml) with bundled libyaml 0.2.5 (#191)

Author: vmvarela

.github/workflows/ci.yml

@@ -54,6 +54,22 @@ jobs:
           print('SQLite amalgamation ready')
           "
 
+      - name: Download libyaml amalgamation
+        shell: bash
+        run: |
+          mkdir -p lib/yaml
+          curl -fsSL "https://github.com/yaml/libyaml/archive/refs/tags/0.2.5.tar.gz" -o /tmp/yaml-0.2.5.tar.gz
+          tar -xzf /tmp/yaml-0.2.5.tar.gz -C /tmp
+          cp /tmp/libyaml-0.2.5/include/yaml.h lib/yaml/yaml.h
+          cp /tmp/libyaml-0.2.5/src/yaml_private.h lib/yaml/yaml_private.h
+          cp /tmp/libyaml-0.2.5/src/api.c lib/yaml/api.c
+          cp /tmp/libyaml-0.2.5/src/parser.c lib/yaml/parser.c
+          cp /tmp/libyaml-0.2.5/src/scanner.c lib/yaml/scanner.c
+          cp /tmp/libyaml-0.2.5/src/reader.c lib/yaml/reader.c
+          cp /tmp/libyaml-0.2.5/src/writer.c lib/yaml/writer.c
+          cp /tmp/libyaml-0.2.5/src/loader.c lib/yaml/loader.c
+          echo "libyaml 0.2.5 ready"
+
       - name: Build
         run: zig build -Dbundle-sqlite=true -Doptimize=ReleaseSafe
 

.github/workflows/release.yml

@@ -86,6 +86,21 @@ jobs:
           unzip -j sqlite.zip "*/sqlite3.c" "*/sqlite3.h" -d lib/
           echo "SQLite $(grep -m1 SQLITE_VERSION lib/sqlite3.h | awk '{print $3}')"
 
+      - name: Download libyaml amalgamation
+        run: |
+          mkdir -p lib/yaml
+          curl -fsSL "https://github.com/yaml/libyaml/archive/refs/tags/0.2.5.tar.gz" -o /tmp/yaml-0.2.5.tar.gz
+          tar -xzf /tmp/yaml-0.2.5.tar.gz -C /tmp
+          cp /tmp/libyaml-0.2.5/include/yaml.h lib/yaml/yaml.h
+          cp /tmp/libyaml-0.2.5/src/yaml_private.h lib/yaml/yaml_private.h
+          cp /tmp/libyaml-0.2.5/src/api.c lib/yaml/api.c
+          cp /tmp/libyaml-0.2.5/src/parser.c lib/yaml/parser.c
+          cp /tmp/libyaml-0.2.5/src/scanner.c lib/yaml/scanner.c
+          cp /tmp/libyaml-0.2.5/src/reader.c lib/yaml/reader.c
+          cp /tmp/libyaml-0.2.5/src/writer.c lib/yaml/writer.c
+          cp /tmp/libyaml-0.2.5/src/loader.c lib/yaml/loader.c
+          echo "libyaml 0.2.5 ready"
+
       - name: Build
         run: |
           VERSION="${GITHUB_REF_NAME#v}"

README.md

@@ -4,7 +4,7 @@
 [![Release](https://img.shields.io/github/v/release/vmvarela/sql-pipe)](https://github.com/vmvarela/sql-pipe/releases/latest)
 [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
 
-`sql-pipe` reads CSV, JSON, or NDJSON from stdin or file arguments, loads it into an in-memory SQLite database, runs a SQL query, and prints the results. No server, no schema files, no setup.
+`sql-pipe` reads CSV, JSON, NDJSON, YAML, or XML from stdin or file arguments, loads it into an in-memory SQLite database, runs a SQL query, and prints the results. No server, no schema files, no setup.
 
 It exists because `awk` is cryptic, spinning up a Python interpreter for a one-liner feels wrong, and `sqlite3 :memory:` takes four commands before you can query anything. If you know SQL and work with CSV in the terminal, this is the tool you've been reaching for.
 
@@ -196,6 +196,24 @@ $ printf '[{"name":"Alice","score":95},{"name":"Bob","score":72}]' \
 Alice,95
 ```
 
+For YAML input, pass `-I yaml` (reads a top-level sequence of mappings). Both `.yaml` and `.yml` extensions are auto-detected. All values are loaded as `TEXT`:
+
+```sh
+$ cat users.yaml
+- name: Alice
+  country: US
+- name: Bob
+  country: UK
+- name: Carol
+  country: DE
+
+$ cat users.yaml | sql-pipe -I yaml 'SELECT name FROM t WHERE country != "US" ORDER BY name'
+Bob
+Carol
+```
+
+YAML is auto-detected from the `.yaml` and `.yml` extensions, or use `-I yaml` explicitly. Comments (`#`), flow-style mappings (`[{name: Alice}]`), and standard scalar types (strings, numbers, booleans) work transparently. Multi-document streams and anchors/aliases are rejected with a clear error.
+
 Columns are auto-detected as `INTEGER`, `REAL`, `DATE`, `DATETIME`, or `TEXT` based on the first 100 rows. Date and datetime values are normalized to ISO 8601 on insert, so SQLite date functions (`date()`, `strftime()`, `julianday()`) work immediately. Use `--no-type-inference` to force all columns to `TEXT`:
 
 ```sh
@@ -258,7 +276,7 @@ $ cat events.xml | sql-pipe -I xml --xml-root events --xml-row event \
 
 ### File arguments
 
-Pass files as positional arguments instead of piping through stdin. Each file becomes a table named after its basename (without extension). The input format is auto-detected from the file extension (`.csv`, `.tsv`, `.json`, `.ndjson`, `.xml`):
+Pass files as positional arguments instead of piping through stdin. Each file becomes a table named after its basename (without extension). The input format is auto-detected from the file extension (`.csv`, `.tsv`, `.json`, `.ndjson`, `.yaml`, `.yml`, `.xml`):
 
 ```sh
 # Single file — no more cat
@@ -267,6 +285,9 @@ $ sql-pipe orders.csv 'SELECT * FROM orders WHERE amount > 100'
 # JSON file — extension tells sql-pipe the format, no -I needed
 $ sql-pipe data.json 'SELECT * FROM data WHERE score > 80'
 
+# YAML file — .yaml and .yml both auto-detect
+$ sql-pipe users.yaml 'SELECT name FROM users WHERE country = "US"'
+
 # Multi-file join — the #1 reason people reach for DuckDB
 $ sql-pipe orders.csv customers.csv \
     'SELECT c.name, SUM(o.amount) FROM orders o
@@ -321,15 +342,15 @@ When `-f` is used, all positional arguments are treated as data files (no positi
 |------|-------------|
 | `-d`, `--delimiter <char>` | Input field delimiter (single character, default `,`) |
 | `--tsv` | Alias for `--delimiter '\t'` |
-| `-I`, `--input-format <fmt>` | Input format: `csv` (default), `tsv`, `json`, `ndjson`, `xml`. Overrides file extension auto-detection. |
+| `-I`, `--input-format <fmt>` | Input format: `csv` (default), `tsv`, `json`, `ndjson`, `yaml`, `xml`. Overrides file extension auto-detection. Both `.yaml` and `.yml` file extensions auto-detect to `yaml`. |
 | `-O`, `--output-format <fmt>` | Output format: `csv` (default), `tsv`, `json`, `ndjson`, `xml`, `markdown` (alias: `md`), `html`, `sql` |
 | `--sql-table <name>` | Target table name for `-O sql` INSERT output (default: `t`) |
 | `--no-type-inference` | Treat all columns as TEXT (skip auto-detection) |
 | `-H`, `--header` | Print column names as the first output row (CSV/TSV/HTML) |
 | `--json` | Alias for `--output-format json` (mutually exclusive with `-H`) |
 | `--max-rows <n>` | Stop if more than `n` data rows are read (exit 1) |
-| `--validate` | Parse the entire input and print a summary (`OK: <n> rows, <m> columns (col TYPE, ...)`) to stdout. Exit 0 on success, exit 2 on parse error. No query required. Compatible with `--delimiter`, `--tsv`, `--no-type-inference`, `-I`/`--input-format` (csv, tsv, json, ndjson, xml). JSON/NDJSON/XML columns are reported as TEXT. |
-| `--columns` | Read the input header, print each column name on its own line, and exit 0. Supports CSV, TSV, JSON, NDJSON, and XML input. With `-v`/`--verbose`, also shows the inferred type per column (`name INTEGER`). Respects `--delimiter` and `--tsv`. Mutually exclusive with a query argument. |
+| `--validate` | Parse the entire input and print a summary (`OK: <n> rows, <m> columns (col TYPE, ...)`) to stdout. Exit 0 on success, exit 2 on parse error. No query required. Compatible with `--delimiter`, `--tsv`, `--no-type-inference`, `-I`/`--input-format` (csv, tsv, json, ndjson, yaml, xml). JSON/NDJSON/YAML/XML columns are reported as TEXT. |
+| `--columns` | Read the input header, print each column name on its own line, and exit 0. Supports CSV, TSV, JSON, NDJSON, YAML, and XML input. With `-v`/`--verbose`, also shows the inferred type per column (`name INTEGER`). Respects `--delimiter` and `--tsv`. Mutually exclusive with a query argument. |
 | `--sample [<n>]` | Print a schema comment block to stderr and the first `<n>` data rows to stdout as CSV (default: `n=10`). The schema block lists each column name and its inferred type, prefixed with `#`. Implies `--header`. Compatible with `--delimiter` and `--tsv`. Mutually exclusive with `--json` and a query argument. No query required. |
 | `--stats` | Load input and print per-column statistics (column name, type, non-null count, min, max, mean) as a formatted table. Mean is blank for non-numeric columns. Compatible with `--delimiter`, `--tsv`, `--no-type-inference`, `-I`/`--input-format`. Mutually exclusive with a query, `--columns`, `--validate`, `--sample`, and `--output`. |
 | `--profile` | Alias for `--stats` |
@@ -643,10 +664,10 @@ The database never touches disk and vanishes when the process exits. No state, n
 
 ## Limitations
 
-- **File format auto-detection** is based on file extension. Files without a recognized extension (`.csv`, `.tsv`, `.json`, `.ndjson`, `.xml`) default to CSV. Use `-I` to override.
+- **File format auto-detection** is based on file extension. Files without a recognized extension (`.csv`, `.tsv`, `.json`, `.ndjson`, `.yaml`, `.yml`, `.xml`) default to CSV. Use `-I` to override.
 
 ## Related
 
 - **[q](https://harelba.github.io/q/)** — Python-based SQL on tabular data. Similar concept, but requires Python runtime. Better if you're already in a Python environment or need Python-specific integrations.
-- **[trdsql](https://github.com/noborus/trdsql)** — Go alternative with broader format support (LTSV, TBLN) and more output options. Better if you need formats beyond CSV/JSON/NDJSON/XML or want more output formatting choices.
+- **[trdsql](https://github.com/noborus/trdsql)** — Go alternative with broader format support (LTSV, TBLN) and more output options. Better if you need formats beyond CSV/JSON/NDJSON/YAML/XML or want more output formatting choices.
 - **[sqlite-utils](https://sqlite-utils.datasette.io/)** — better if you need persistent databases, schema management, or Python scripting. sql-pipe is designed for one-shot queries on ephemeral in-memory data.

build.zig

@@ -44,6 +44,16 @@ pub fn build(b: *std.Build) void {
     });
     exe.root_module.addImport("c", translate_c.createModule());
 
+    // Translate yaml.h to Zig declarations, exposed as @import("yaml").
+    // libyaml is always bundled (no --bundle-yaml option) — it's tiny and
+    // not always available as a system library.
+    const translate_yaml = b.addTranslateC(.{
+        .root_source_file = b.path("lib/yaml/yaml_translate.h"),
+        .target = target,
+        .optimize = optimize,
+    });
+    exe.root_module.addImport("yaml", translate_yaml.createModule());
+
     if (bundle_sqlite) {
         exe.root_module.addIncludePath(b.path("lib"));
         exe.root_module.addCSourceFile(.{
@@ -54,6 +64,22 @@ pub fn build(b: *std.Build) void {
         exe.root_module.linkSystemLibrary("sqlite3", .{});
     }
 
+    // Bundle libyaml — only used for loading, so compile api.c, parser.c,
+    // scanner.c, reader.c, writer.c, loader.c. Skip dumper.c, emitter.c
+    // (we don't emit YAML).
+    exe.root_module.addIncludePath(b.path("lib/yaml"));
+    exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/api.c"),    .flags = &.{
+        "-DYAML_VERSION_MAJOR=0",
+        "-DYAML_VERSION_MINOR=2",
+        "-DYAML_VERSION_PATCH=5",
+        "-DYAML_VERSION_STRING=\"0.2.5\"",
+    } });
+    exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/parser.c"),  .flags = &.{} });
+    exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/scanner.c"), .flags = &.{} });
+    exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/reader.c"),  .flags = &.{} });
+    exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/writer.c"),  .flags = &.{} });
+    exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/loader.c"),  .flags = &.{} });
+
     b.installArtifact(exe);
 
     // Generate man page from scdoc source if scdoc (and gzip) are available (optional dependencies)
@@ -2643,6 +2669,77 @@ pub fn build(b: *std.Build) void {
     });
     fixture_test_step.dependOn(&fixture_sample_file.step);
 
+    // Fixture test 168a: YAML file argument — auto-detected from .yaml extension
+    // Note: YAML preserves original number formatting (e.g., 25.00 stays 25.00)
+    const fixture_yaml_file = b.addSystemCommand(&.{
+        "bash", "-c",
+        \\result=$(./zig-out/bin/sql-pipe tests/fixtures/users.yaml 'SELECT name, price FROM users ORDER BY CAST(price AS REAL) DESC')
+        \\expected=$(printf 'Thingamajig,60.00\nDoohickey,50.00\nGadget,40.25\nWidget,25.00')
+        \\[ "$result" = "$expected" ]
+    });
+    fixture_test_step.dependOn(&fixture_yaml_file.step);
+
+    // Fixture test 168b: YAML file argument — .yml extension alias
+    const fixture_yaml_yml_alias = b.addSystemCommand(&.{
+        "bash", "-c",
+        \\result=$(./zig-out/bin/sql-pipe tests/fixtures/users.yml 'SELECT name FROM users WHERE category = "hardware" ORDER BY name')
+        \\expected=$(printf 'Doohickey\nWidget')
+        \\[ "$result" = "$expected" ]
+    });
+    fixture_test_step.dependOn(&fixture_yaml_yml_alias.step);
+
+    // Fixture test 168c: YAML file via stdin with -I yaml
+    const fixture_yaml_stdin = b.addSystemCommand(&.{
+        "bash", "-c",
+        \\result=$(cat tests/fixtures/users.yaml | ./zig-out/bin/sql-pipe -I yaml 'SELECT name FROM t WHERE CAST(stock AS INTEGER) > 0 ORDER BY name')
+        \\expected=$(printf 'Doohickey\nGadget\nWidget')
+        \\[ "$result" = "$expected" ]
+    });
+    fixture_test_step.dependOn(&fixture_yaml_stdin.step);
+
+    // Fixture test 168d: YAML file — filter and aggregate
+    const fixture_yaml_aggregate = b.addSystemCommand(&.{
+        "bash", "-c",
+        \\result=$(./zig-out/bin/sql-pipe tests/fixtures/users.yaml 'SELECT category, COUNT(*) FROM users GROUP BY category ORDER BY category')
+        \\expected=$(printf 'electronics,2\nhardware,2')
+        \\[ "$result" = "$expected" ]
+    });
+    fixture_test_step.dependOn(&fixture_yaml_aggregate.step);
+
+    // Fixture test 168e: YAML comments are transparent
+    const fixture_yaml_comments = b.addSystemCommand(&.{
+        "bash", "-c",
+        \\result=$(printf -- '# This is a comment\n# Another comment\n- name: Alice\n  age: 30\n# Mid comment\n- name: Bob\n  age: 25\n' \
+        \\    | ./zig-out/bin/sql-pipe -I yaml 'SELECT name FROM t ORDER BY name')
+        \\expected=$(printf 'Alice\nBob')
+        \\[ "$result" = "$expected" ]
+    });
+    fixture_test_step.dependOn(&fixture_yaml_comments.step);
+
+    // Fixture test 168f: Malformed YAML — error exit 2 with line number
+    const fixture_yaml_malformed = b.addSystemCommand(&.{
+        "bash", "-c",
+        \\msg=$(printf -- '- name: Alice\n\tbad: indent\n' | ./zig-out/bin/sql-pipe -I yaml 'SELECT 1' 2>&1 >/dev/null; echo "EXIT:$?")
+        \\echo "$msg" | grep -q 'YAML parse error at line' && echo "$msg" | grep -q 'EXIT:2'
+    });
+    fixture_test_step.dependOn(&fixture_yaml_malformed.step);
+
+    // Fixture test 168g: Empty YAML input — graceful (returns 0, no crash)
+    const fixture_yaml_empty = b.addSystemCommand(&.{
+        "bash", "-c",
+        \\result=$(./zig-out/bin/sql-pipe -I yaml 'SELECT 1 AS one' < /dev/null 2>/dev/null)
+        \\[ "$result" = "1" ]
+    });
+    fixture_test_step.dependOn(&fixture_yaml_empty.step);
+
+    // Fixture test 168h: Multi-document YAML stream — error exit 2
+    const fixture_yaml_multidoc = b.addSystemCommand(&.{
+        "bash", "-c",
+        \\msg=$(printf -- '- name: Alice\n---\n- name: Bob\n' | ./zig-out/bin/sql-pipe -I yaml 'SELECT 1' 2>&1 >/dev/null; echo "EXIT:$?")
+        \\echo "$msg" | grep -q 'multiple documents not supported' && echo "$msg" | grep -q 'EXIT:2'
+    });
+    fixture_test_step.dependOn(&fixture_yaml_multidoc.step);
+
     const unit_tests = b.addTest(.{
         .root_module = b.createModule(.{
             .root_source_file = b.path("src/csv.zig"),
@@ -2664,6 +2761,19 @@ pub fn build(b: *std.Build) void {
         }),
     });
     xml_unit_tests.root_module.addImport("c", translate_c.createModule());
+    xml_unit_tests.root_module.addImport("yaml", translate_yaml.createModule());
+    xml_unit_tests.root_module.addIncludePath(b.path("lib/yaml"));
+    xml_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/api.c"),    .flags = &.{
+        "-DYAML_VERSION_MAJOR=0",
+        "-DYAML_VERSION_MINOR=2",
+        "-DYAML_VERSION_PATCH=5",
+        "-DYAML_VERSION_STRING=\"0.2.5\"",
+    } });
+    xml_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/parser.c"),  .flags = &.{} });
+    xml_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/scanner.c"), .flags = &.{} });
+    xml_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/reader.c"),  .flags = &.{} });
+    xml_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/writer.c"),  .flags = &.{} });
+    xml_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/loader.c"),  .flags = &.{} });
     if (bundle_sqlite) {
         xml_unit_tests.root_module.addIncludePath(b.path("lib"));
         xml_unit_tests.root_module.addCSourceFile(.{
@@ -2687,6 +2797,19 @@ pub fn build(b: *std.Build) void {
         }),
     });
     loader_unit_tests.root_module.addImport("c", translate_c.createModule());
+    loader_unit_tests.root_module.addImport("yaml", translate_yaml.createModule());
+    loader_unit_tests.root_module.addIncludePath(b.path("lib/yaml"));
+    loader_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/api.c"),    .flags = &.{
+        "-DYAML_VERSION_MAJOR=0",
+        "-DYAML_VERSION_MINOR=2",
+        "-DYAML_VERSION_PATCH=5",
+        "-DYAML_VERSION_STRING=\"0.2.5\"",
+    } });
+    loader_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/parser.c"),  .flags = &.{} });
+    loader_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/scanner.c"), .flags = &.{} });
+    loader_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/reader.c"),  .flags = &.{} });
+    loader_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/writer.c"),  .flags = &.{} });
+    loader_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/loader.c"),  .flags = &.{} });
     if (bundle_sqlite) {
         loader_unit_tests.root_module.addIncludePath(b.path("lib"));
         loader_unit_tests.root_module.addCSourceFile(.{