Use jetro 0.5.9

Author: mitghi

Cargo.lock

@@ -8,7 +8,7 @@ homepage = "https://github.com/mitghi/jetrocli"
 repository = "https://github.com/mitghi/jetrocli"
 
 [dependencies]
-jetro-core    = "0.5.8"
+jetro-core    = "0.5.9"
 serde_json    = "1"
 indexmap      = "2"
 ratatui       = "0.29"

Cargo.toml

@@ -1,12 +1,17 @@
 # jetrocli
 
+New to jetro? [**jetro-book**](https://mitghi.github.io/jetro-book/) is the best place to start — guide, tour, and documentation for learning the expression language.
+
 <p align="center">
   <img src="media/jetrocli.png" alt="jetrocli">
 </p>
 
-Interactive split-pane TUI for [jetro](https://github.com/mitghi/jetro) — paste JSON, write expressions, see results live. Built on `ratatui` + `crossterm`.
 
-New to jetro? [**jetro-book**](https://mitghi.github.io/jetro-book/) is the best place to start — guide, tour, and documentation for learning the expression language.
+`jetrocli` is a terminal companion for [jetro](https://github.com/mitghi/jetro), a JSON expression language. It gives you two ways to work with JSON: an interactive TUI for exploring data and a command-line program for processing JSON in scripts, pipes, files, and NDJSON datasets.
+
+Run it directly and it opens a split-pane JSON workbench: paste or load JSON, write a jetro expression, and see the result update live as you type. Pipe or redirect input and it skips the TUI, evaluates once, and prints the result like a regular Unix command.
+
+For large files, `jetrocli` can memory map regular-file input. In `--ndjson` mode it scans one JSON document per line, supports reverse reads from the end of log-style files, and can stop early with `--limit` for bounded queries over very large datasets.
 
 ## Features
 
@@ -15,6 +20,11 @@ New to jetro? [**jetro-book**](https://mitghi.github.io/jetro-book/) is the best
 - **Structural folding** in JSON editor. Fold any `{…}` / `[…]` block, with gutter triangles (`▾` / `▸`) and inline `⋯ N lines` markers.
 - **Schema-aware completion** — suggests fields at the current path, auto-unwraps element fields inside array chains, filters builtins by receiver type.
 - **Inline docs pane** next to completions — every jetro builtin ships with signature, summary, and example.
+- **Pipe / batch mode without TUI** — when stdin is piped or redirected, jetrocli evaluates once and prints the result directly for shell workflows.
+- **File-backed large JSON reads** — regular-file stdin is memory mapped for zero-copy loading instead of forcing the interactive path.
+- **Fast NDJSON scans** — `--ndjson` evaluates one JSON document per line from `-i <FILE>` and emits one compact result per row.
+- **Reverse NDJSON reads** — `--ndjson -r` scans from tail to head, useful for log-style files where the newest rows matter first.
+- **Bounded NDJSON filters** — combine `--ndjson` with `--limit <N>` to stop after the first `N` emitted rows, including reverse scans for "latest matching rows" queries.
 - **Emacs-style bindings** throughout (`C-a/C-e`, `C-f/C-b`, `M-f/M-b`, `C-n/C-p`, `C-g`, `C-c` prefix chord).
 - **Expression formatter** — breaks long jetro chains onto indented lines (`C-c C-f`).
 
@@ -92,6 +102,41 @@ jetrocli --ndjson -i app.log -r --limit 50 '$.msg'         # last 50 matches
 | `--max-line-bytes <BYTES>` | Per-line byte cap. Default 64 MiB. |
 | `--reverse-chunk <BYTES>` | Reverse reader chunk size. Tune for very wide rows. |
 
+## Performance
+
+NDJSON mode is built for file-backed batch scans. It memory maps the input file, evaluates the expression independently for each line, and emits compact JSON results without starting the interactive TUI.
+
+The repository includes a reproducible benchmark in `benchmark/`:
+
+```sh
+rustc -O benchmark/gen_ndjson.rs -o /tmp/gen_ndjson
+/tmp/gen_ndjson /tmp/big.ndjson 1000000000
+benchmark/bench.sh
+```
+
+`benchmark/bench.sh` compares only the `jetrocli` and [`jaq`](https://github.com/01mf02/jaq) program. The generator writes roughly 1 GB of NDJSON shaped like:
+
+```json
+{"id":1,"name":"user_1","attributes":[{"key":"k1","value":"v_1_1"}]}
+```
+
+One run on an Apple M1 laptop over 4,764,404 rows produced:
+
+| Query | jetro expression | jaq expression | jetrocli | jaq | Speedup |
+| --- | --- | --- | ---: | ---: | ---: |
+| Project id | `$.id` | `.id` | 0.72s | 27.89s | **38.7x** |
+| Project name | `$.name` | `.name` | 0.30s | 28.99s | **96.6x** |
+| Attributes count | `$.attributes.len()` | `.attributes \| length` | 1.52s | 28.19s | **18.5x** |
+| Attribute keys list | `$.attributes.map(@.key)` | `.attributes \| map(.key)` | 2.06s | 39.98s | **19.4x** |
+| First attr value | `$.attributes.first().value` | `.attributes[0].value` | 0.80s | 29.22s | **36.5x** |
+| Last attr value | `$.attributes.last().value` | `.attributes[-1].value` | 1.61s | 29.25s | **18.2x** |
+| Uppercase name | `$.name.upper()` | `.name \| ascii_upcase` | 0.41s | 28.42s | **69.3x** |
+| `[key,value]` pairs | `$.attributes.map([@.key, @.value])` | `.attributes \| map([.key, .value])` | 2.99s | 54.26s | **18.1x** |
+| Count attrs matching `_3` | `$.attributes.filter(@.value.contains("_3")).len()` | `[.attributes[] \| select(.value \| contains("_3"))] \| length` | 1.52s | 48.15s | **31.7x** |
+| Object keys | `$.keys()` | `keys` | 1.05s | 28.48s | **27.1x** |
+
+In practice, expect NDJSON mode to be especially strong for field projection, string transforms, row-local indexing, and shallow array operations over large files. Queries that allocate larger derived arrays or inspect more nested values naturally move more bytes and take longer.
+
 ## License
 
 MIT

README.md

@@ -0,0 +1,41 @@
+#!/bin/bash
+# jetrocli vs jaq, 10 per-row queries over 1 GB NDJSON.
+# Output: per-tool wall time + emitted row count, written to /tmp/bench_results.txt.
+
+set -u
+FILE=/tmp/big.ndjson
+OUT=/tmp/bench_results.txt
+> "$OUT"
+
+run() {
+    local idx=$1 jq=$2 jeq=$3 desc=$4
+    echo "=== Q$idx: $desc ===" | tee -a "$OUT"
+    echo "  jetro: $jeq" | tee -a "$OUT"
+    echo "  jaq  : $jq"  | tee -a "$OUT"
+
+    local out_j=/tmp/bench_q${idx}_jetro.out
+    local out_a=/tmp/bench_q${idx}_jaq.out
+
+    /usr/bin/time -p jetrocli --ndjson -i "$FILE" "$jeq" > "$out_j" 2>/tmp/jc_t
+    local jc_real=$(awk '/real/ {print $2}' /tmp/jc_t)
+    local jc_rows=$(wc -l < "$out_j")
+
+    /usr/bin/time -p jaq -c "$jq" < "$FILE" > "$out_a" 2>/tmp/ja_t
+    local ja_real=$(awk '/real/ {print $2}' /tmp/ja_t)
+    local ja_rows=$(wc -l < "$out_a")
+
+    printf "  jetrocli %ss  rows=%s\n" "$jc_real" "$jc_rows" | tee -a "$OUT"
+    printf "  jaq      %ss  rows=%s\n" "$ja_real" "$ja_rows" | tee -a "$OUT"
+    echo | tee -a "$OUT"
+}
+
+run 1  '.id'                                                    '$.id'                                                    "project id"
+run 2  '.name'                                                  '$.name'                                                  "project name"
+run 3  '.attributes | length'                                   '$.attributes.len()'                                      "attributes count"
+run 4  '.attributes | map(.key)'                                '$.attributes.map(@.key)'                                 "attribute keys list"
+run 5  '.attributes[0].value'                                   '$.attributes.first().value'                              "first attr value"
+run 6  '.attributes[-1].value'                                  '$.attributes.last().value'                               "last attr value"
+run 7  '.name | ascii_upcase'                                   '$.name.upper()'                                          "uppercase name"
+run 8  '.attributes | map([.key, .value])'                      '$.attributes.map([@.key, @.value])'                      "[key,value] pairs"
+run 9  '[.attributes[] | select(.value | contains("_3"))] | length' '$.attributes.filter(@.value.contains("_3")).len()'   "count attrs matching _3"
+run 10 'keys'                                                   '$.keys()'                                                "object keys"

benchmark/bench.sh

@@ -0,0 +1,67 @@
+// Standalone NDJSON generator. Writes ~target_bytes worth of rows.
+// Row shape: {"id":N,"name":"user_N","attributes":[{"key":"kI","value":"v_N_I"}, ...]}
+//
+// Build: rustc -O /tmp/gen_ndjson.rs -o /tmp/gen_ndjson
+// Run:   /tmp/gen_ndjson <out_path> <target_bytes>
+
+use std::env;
+use std::fs::File;
+use std::io::{BufWriter, Write};
+
+const ATTRS_PER_ROW: usize = 5;
+
+fn main() {
+    let mut args = env::args().skip(1);
+    let path = args.next().expect("usage: gen_ndjson <path> <bytes>");
+    let target: u64 = args
+        .next()
+        .expect("usage: gen_ndjson <path> <bytes>")
+        .parse()
+        .expect("bytes must be integer");
+
+    let file = File::create(&path).expect("create");
+    let mut w = BufWriter::with_capacity(1 << 20, file);
+    let mut written: u64 = 0;
+    let mut id: u64 = 0;
+    let mut buf: Vec<u8> = Vec::with_capacity(512);
+
+    while written < target {
+        id += 1;
+        buf.clear();
+        buf.extend_from_slice(b"{\"id\":");
+        write_u64(&mut buf, id);
+        buf.extend_from_slice(b",\"name\":\"user_");
+        write_u64(&mut buf, id);
+        buf.extend_from_slice(b"\",\"attributes\":[");
+        for i in 1..=ATTRS_PER_ROW {
+            if i > 1 {
+                buf.push(b',');
+            }
+            buf.extend_from_slice(b"{\"key\":\"k");
+            write_u64(&mut buf, i as u64);
+            buf.extend_from_slice(b"\",\"value\":\"v_");
+            write_u64(&mut buf, id);
+            buf.push(b'_');
+            write_u64(&mut buf, i as u64);
+            buf.extend_from_slice(b"\"}");
+        }
+        buf.extend_from_slice(b"]}\n");
+        w.write_all(&buf).expect("write");
+        written += buf.len() as u64;
+    }
+    w.flush().expect("flush");
+    eprintln!("wrote {} bytes, {} rows", written, id);
+}
+
+fn write_u64(buf: &mut Vec<u8>, mut n: u64) {
+    if n == 0 {
+        buf.push(b'0');
+        return;
+    }
+    let start = buf.len();
+    while n > 0 {
+        buf.push(b'0' + (n % 10) as u8);
+        n /= 10;
+    }
+    buf[start..].reverse();
+}