Skip to content

Commit dfc5d65

Browse files
trielofffactorydroidnickxiao-zoomfactory-droid[bot]
authored
feat(jq): add raw input mode (-R/--raw-input) (#260)
* feat(jq): add raw input mode (-R/--raw-input) Builds on #189 by nicoster, rebased onto the current monorepo layout (packages/just-bash/...) and fixing a file-boundary correctness bug: real jq concatenates all inputs into a single stream and splits on newlines, so a raw line spans a file boundary when a file lacks a trailing newline. Inputs are now concatenated before splitting instead of being processed per-file. Adds unit and comparison coverage for stdin, files, blank lines, the '-' stdin marker, CRLF preservation, and the cross-file-boundary case. Co-authored-by: Nick Xiao <nick.xiao@zoom.us> Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Signed-off-by: Lars Trieloff <lars@trieloff.net> * fix(jq): align raw-input help with jq, scan raw input incrementally Address PR review: - Help: usage FILTER [FILE...], -c long flag is --compact-output (matches both the implementation and real jq), and -R wording mirrors jq ("read each line as string instead of JSON"). - Raw input now scans inputs incrementally, carrying only the unterminated trailing fragment across file boundaries, instead of allocating the full concatenated string and a complete array of lines. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Signed-off-by: Lars Trieloff <lars@trieloff.net> --------- Signed-off-by: Lars Trieloff <lars@trieloff.net> Co-authored-by: Droid <droid@factory.ai> Co-authored-by: Nick Xiao <nick.xiao@zoom.us> Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
1 parent d64009a commit dfc5d65

5 files changed

Lines changed: 311 additions & 4 deletions

File tree

‎packages/just-bash/src/commands/jq/jq.raw-input.test.ts‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
import { describe, expect, it } from "vitest";
2+
import { Bash } from "../../Bash.js";
3+
4+
describe("jq raw input (-R)", () => {
5+
it("should read stdin lines as strings with -R", async () => {
6+
const env = new Bash();
7+
const result = await env.exec("printf 'a\\nb\\n' | jq -R '.'");
8+
expect(result.stdout).toBe('"a"\n"b"\n');
9+
expect(result.stderr).toBe("");
10+
expect(result.exitCode).toBe(0);
11+
});
12+
13+
it("should preserve blank lines with -R", async () => {
14+
const env = new Bash();
15+
const result = await env.exec("printf 'a\\n\\nb' | jq -R '.'");
16+
expect(result.stdout).toBe('"a"\n""\n"b"\n');
17+
expect(result.stderr).toBe("");
18+
expect(result.exitCode).toBe(0);
19+
});
20+
21+
it("should slurp raw stdin into a single string with -Rs", async () => {
22+
const env = new Bash();
23+
const result = await env.exec("printf 'a\\nb\\n' | jq -Rs '.'");
24+
expect(result.stdout).toBe('"a\\nb\\n"\n');
25+
expect(result.stderr).toBe("");
26+
expect(result.exitCode).toBe(0);
27+
});
28+
29+
it("should read raw input from files line by line", async () => {
30+
const env = new Bash({
31+
files: {
32+
"/a.txt": "first\n",
33+
"/b.txt": "second",
34+
},
35+
});
36+
const result = await env.exec("jq -R '.' /a.txt /b.txt");
37+
expect(result.stdout).toBe('"first"\n"second"\n');
38+
expect(result.stderr).toBe("");
39+
expect(result.exitCode).toBe(0);
40+
});
41+
42+
it("should join a line across a file boundary when no trailing newline", async () => {
43+
const env = new Bash({
44+
files: {
45+
"/a.txt": "first",
46+
"/b.txt": "second\n",
47+
},
48+
});
49+
const result = await env.exec("jq -R '.' /a.txt /b.txt");
50+
expect(result.stdout).toBe('"firstsecond"\n');
51+
expect(result.stderr).toBe("");
52+
expect(result.exitCode).toBe(0);
53+
});
54+
55+
it("should slurp raw input across files without inserting separators", async () => {
56+
const env = new Bash({
57+
files: {
58+
"/a.txt": "first\n",
59+
"/b.txt": "second",
60+
},
61+
});
62+
const result = await env.exec("jq -Rs '.' /a.txt /b.txt");
63+
expect(result.stdout).toBe('"first\\nsecond"\n');
64+
expect(result.stderr).toBe("");
65+
expect(result.exitCode).toBe(0);
66+
});
67+
68+
it("should handle stdin marker together with files in raw mode", async () => {
69+
const env = new Bash({
70+
files: {
71+
"/file.txt": "file\n",
72+
},
73+
});
74+
const result = await env.exec("printf 'stdin\\n' | jq -R '.' - /file.txt");
75+
expect(result.stdout).toBe('"stdin"\n"file"\n');
76+
expect(result.stderr).toBe("");
77+
expect(result.exitCode).toBe(0);
78+
});
79+
80+
it("should allow raw input with raw output", async () => {
81+
const env = new Bash();
82+
const result = await env.exec("printf 'hello\\n' | jq -Rr '.'");
83+
expect(result.stdout).toBe("hello\n");
84+
expect(result.stderr).toBe("");
85+
expect(result.exitCode).toBe(0);
86+
});
87+
88+
it("should keep carriage returns inside raw lines", async () => {
89+
const env = new Bash();
90+
const result = await env.exec("printf 'a\\r\\nb\\r\\n' | jq -R '.'");
91+
expect(result.stdout).toBe('"a\\r"\n"b\\r"\n');
92+
expect(result.stderr).toBe("");
93+
expect(result.exitCode).toBe(0);
94+
});
95+
96+
it("should produce no output for empty raw input", async () => {
97+
const env = new Bash();
98+
const result = await env.exec("printf '' | jq -R '.'");
99+
expect(result.stdout).toBe("");
100+
expect(result.stderr).toBe("");
101+
expect(result.exitCode).toBe(0);
102+
});
103+
104+
it("should slurp empty raw input into an empty string", async () => {
105+
const env = new Bash();
106+
const result = await env.exec("printf '' | jq -Rs '.'");
107+
expect(result.stdout).toBe('""\n');
108+
expect(result.stderr).toBe("");
109+
expect(result.exitCode).toBe(0);
110+
});
111+
112+
it("should build an object from raw slurped input", async () => {
113+
const env = new Bash();
114+
const result = await env.exec(
115+
"printf 'console.log(1)\\n' | jq -Rs '{action:\"load-code\", code: .}'",
116+
);
117+
expect(result.stdout).toBe(
118+
'{\n "action": "load-code",\n "code": "console.log(1)\\n"\n}\n',
119+
);
120+
expect(result.stderr).toBe("");
121+
expect(result.exitCode).toBe(0);
122+
});
123+
});

‎packages/just-bash/src/commands/jq/jq.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -335,6 +335,7 @@ describe("jq", () => {
335335
const env = new Bash();
336336
const result = await env.exec("jq --help");
337337
expect(result.stdout).toMatch(/jq.*JSON/);
338+
expect(result.stdout).toContain("-R, --raw-input");
338339
expect(result.exitCode).toBe(0);
339340
});
340341
});

‎packages/just-bash/src/commands/jq/jq.ts‎

Lines changed: 36 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -178,10 +178,11 @@ function parseJsonStream(input: string): unknown[] {
178178
const jqHelp = {
179179
name: "jq",
180180
summary: "command-line JSON processor",
181-
usage: "jq [OPTIONS] FILTER [FILE]",
181+
usage: "jq [OPTIONS] FILTER [FILE...]",
182182
options: [
183+
"-R, --raw-input read each line as string instead of JSON",
183184
"-r, --raw-output output strings without quotes",
184-
"-c, --compact compact output (no pretty printing)",
185+
"-c, --compact-output compact instead of pretty-printed output",
185186
"-e, --exit-status set exit status based on output",
186187
"-s, --slurp read entire input into array",
187188
"-n, --null-input don't read any input",
@@ -267,6 +268,7 @@ export const jqCommand: Command = {
267268
if (hasHelpFlag(args)) return showHelp(jqHelp);
268269

269270
let raw = false;
271+
let rawInput = false;
270272
let compact = false;
271273
let exitStatus = false;
272274
let slurp = false;
@@ -280,7 +282,8 @@ export const jqCommand: Command = {
280282

281283
for (let i = 0; i < args.length; i++) {
282284
const a = args[i];
283-
if (a === "-r" || a === "--raw-output") raw = true;
285+
if (a === "-R" || a === "--raw-input") rawInput = true;
286+
else if (a === "-r" || a === "--raw-output") raw = true;
284287
else if (a === "-c" || a === "--compact-output") compact = true;
285288
else if (a === "-e" || a === "--exit-status") exitStatus = true;
286289
else if (a === "-s" || a === "--slurp") slurp = true;
@@ -298,7 +301,8 @@ export const jqCommand: Command = {
298301
else if (a.startsWith("--")) return unknownOption("jq", a);
299302
else if (a.startsWith("-")) {
300303
for (const c of a.slice(1)) {
301-
if (c === "r") raw = true;
304+
if (c === "R") rawInput = true;
305+
else if (c === "r") raw = true;
302306
else if (c === "c") compact = true;
303307
else if (c === "e") exitStatus = true;
304308
else if (c === "s") slurp = true;
@@ -365,6 +369,33 @@ export const jqCommand: Command = {
365369

366370
if (nullInput) {
367371
values = evaluate(null, ast, evalOptions);
372+
} else if (rawInput && slurp) {
373+
// Raw slurp: the entire concatenated input becomes one JSON string.
374+
const rawText = inputs.map(({ content }) => content).join("");
375+
values = evaluate(rawText, ast, evalOptions);
376+
} else if (rawInput) {
377+
// Raw input: real jq concatenates all inputs into a single stream and
378+
// splits on newlines, so a line can span a file boundary when a file
379+
// lacks a trailing newline. Scan incrementally, carrying only the
380+
// unterminated trailing fragment across inputs, instead of building the
381+
// full concatenated string and a complete array of lines. A trailing
382+
// newline does not yield a final empty string, but interior blank
383+
// lines are preserved.
384+
let remainder = "";
385+
for (const { content } of inputs) {
386+
const text = remainder + content;
387+
let start = 0;
388+
let nl = text.indexOf("\n", start);
389+
while (nl !== -1) {
390+
values.push(...evaluate(text.slice(start, nl), ast, evalOptions));
391+
start = nl + 1;
392+
nl = text.indexOf("\n", start);
393+
}
394+
remainder = text.slice(start);
395+
}
396+
if (remainder !== "") {
397+
values.push(...evaluate(remainder, ast, evalOptions));
398+
}
368399
} else if (slurp) {
369400
// Slurp mode: combine all inputs into single array
370401
// Use JSON stream parser to handle concatenated JSON (not just NDJSON)
@@ -457,6 +488,7 @@ import type { CommandFuzzInfo } from "../fuzz-flags-types.js";
457488
export const flagsForFuzzing: CommandFuzzInfo = {
458489
name: "jq",
459490
flags: [
491+
{ flag: "-R", type: "boolean" },
460492
{ flag: "-r", type: "boolean" },
461493
{ flag: "-c", type: "boolean" },
462494
{ flag: "-e", type: "boolean" },

‎packages/just-bash/src/comparison-tests/fixtures/jq.raw-input.comparison.fixtures.json‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
{
2+
"03c428e42ab1f036": {
3+
"command": "printf 'console.log(1)\\n' | jq -Rs '{action:\"load-code\", code: .}'",
4+
"files": {},
5+
"stdout": "{\n \"action\": \"load-code\",\n \"code\": \"console.log(1)\\n\"\n}\n",
6+
"stderr": "",
7+
"exitCode": 0
8+
},
9+
"2774b60ac30dbf90": {
10+
"command": "jq -R '.' a.txt b.txt",
11+
"files": {
12+
"a.txt": "first",
13+
"b.txt": "second\n"
14+
},
15+
"stdout": "\"firstsecond\"\n",
16+
"stderr": "",
17+
"exitCode": 0
18+
},
19+
"352162774da404df": {
20+
"command": "printf 'a\\n\\nb' | jq -R '.'",
21+
"files": {},
22+
"stdout": "\"a\"\n\"\"\n\"b\"\n",
23+
"stderr": "",
24+
"exitCode": 0
25+
},
26+
"3eccd43ed92a3c69": {
27+
"command": "jq -Rs '.' a.txt b.txt",
28+
"files": {
29+
"a.txt": "first\n",
30+
"b.txt": "second"
31+
},
32+
"stdout": "\"first\\nsecond\"\n",
33+
"stderr": "",
34+
"exitCode": 0
35+
},
36+
"4527747d8e92f72c": {
37+
"command": "printf 'stdin\\n' | jq -R '.' - file.txt",
38+
"files": {
39+
"file.txt": "file\n"
40+
},
41+
"stdout": "\"stdin\"\n\"file\"\n",
42+
"stderr": "",
43+
"exitCode": 0
44+
},
45+
"4ef4020b99945f34": {
46+
"command": "printf 'a\\nb\\n' | jq -Rs '.'",
47+
"files": {},
48+
"stdout": "\"a\\nb\\n\"\n",
49+
"stderr": "",
50+
"exitCode": 0
51+
},
52+
"ed6d0745a5d92cef": {
53+
"command": "printf 'a\\nb\\n' | jq -R '.'",
54+
"files": {},
55+
"stdout": "\"a\"\n\"b\"\n",
56+
"stderr": "",
57+
"exitCode": 0
58+
},
59+
"ff334c372880449c": {
60+
"command": "jq -R '.' a.txt b.txt",
61+
"files": {
62+
"a.txt": "first\n",
63+
"b.txt": "second"
64+
},
65+
"stdout": "\"first\"\n\"second\"\n",
66+
"stderr": "",
67+
"exitCode": 0
68+
}
69+
}

‎packages/just-bash/src/comparison-tests/jq.raw-input.comparison.test.ts‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
import { afterEach, beforeEach, describe, it } from "vitest";
2+
import {
3+
cleanupTestDir,
4+
compareOutputs,
5+
createTestDir,
6+
setupFiles,
7+
} from "./fixture-runner.js";
8+
9+
describe("jq raw input - Real Bash Comparison", () => {
10+
let testDir: string;
11+
12+
beforeEach(async () => {
13+
testDir = await createTestDir();
14+
});
15+
16+
afterEach(async () => {
17+
await cleanupTestDir(testDir);
18+
});
19+
20+
describe("stdin", () => {
21+
it("should read stdin lines as strings with -R", async () => {
22+
const env = await setupFiles(testDir, {});
23+
await compareOutputs(env, testDir, "printf 'a\\nb\\n' | jq -R '.'");
24+
});
25+
26+
it("should preserve blank lines with -R", async () => {
27+
const env = await setupFiles(testDir, {});
28+
await compareOutputs(env, testDir, "printf 'a\\n\\nb' | jq -R '.'");
29+
});
30+
31+
it("should slurp raw stdin with -Rs", async () => {
32+
const env = await setupFiles(testDir, {});
33+
await compareOutputs(env, testDir, "printf 'a\\nb\\n' | jq -Rs '.'");
34+
});
35+
36+
it("should build an object from raw slurped stdin", async () => {
37+
const env = await setupFiles(testDir, {});
38+
await compareOutputs(
39+
env,
40+
testDir,
41+
"printf 'console.log(1)\\n' | jq -Rs '{action:\"load-code\", code: .}'",
42+
);
43+
});
44+
});
45+
46+
describe("files", () => {
47+
it("should read files line by line with -R", async () => {
48+
const env = await setupFiles(testDir, {
49+
"a.txt": "first\n",
50+
"b.txt": "second",
51+
});
52+
await compareOutputs(env, testDir, "jq -R '.' a.txt b.txt");
53+
});
54+
55+
it("should join a line across a file boundary without trailing newline", async () => {
56+
const env = await setupFiles(testDir, {
57+
"a.txt": "first",
58+
"b.txt": "second\n",
59+
});
60+
await compareOutputs(env, testDir, "jq -R '.' a.txt b.txt");
61+
});
62+
63+
it("should slurp files without separators with -Rs", async () => {
64+
const env = await setupFiles(testDir, {
65+
"a.txt": "first\n",
66+
"b.txt": "second",
67+
});
68+
await compareOutputs(env, testDir, "jq -Rs '.' a.txt b.txt");
69+
});
70+
71+
it("should support stdin marker with files in raw mode", async () => {
72+
const env = await setupFiles(testDir, {
73+
"file.txt": "file\n",
74+
});
75+
await compareOutputs(
76+
env,
77+
testDir,
78+
"printf 'stdin\\n' | jq -R '.' - file.txt",
79+
);
80+
});
81+
});
82+
});

0 commit comments

Comments
 (0)