Skip to content

Commit 81f61be

Browse files
committed
fix(yq): support file operands and add e2e
1 parent 32def6a commit 81f61be

4 files changed

Lines changed: 263 additions & 24 deletions

File tree

docs-internal/registry-parity-worklist.md

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -609,6 +609,19 @@ real e2e tests that prove Linux-parity behavior — not smoke tests.
609609
Biome is not applicable for this package test path; it reported the file is
610610
ignored by config in
611611
`2026-07-08T13-25-12-0700-item12-gzip-biome-check.log`. Rev: `tlstlwvy`.
612+
- **yq — DONE.** Added package-local VM e2e coverage for the staged `yq`
613+
command and fixed the wrapper to accept file operands instead of only
614+
stdin. The suite proves YAML filtering, YAML-to-JSON query output,
615+
explicit JSON/TOML/XML input formats, and invalid YAML parse errors through
616+
the packaged WASM command. Proof:
617+
`2026-07-08T13-27-36-0700-item12-yq-toolchain-cmd-build.log`;
618+
`2026-07-08T13-28-59-0700-item12-yq-package-build-after-file-operands.log`;
619+
`2026-07-08T13-29-10-0700-item12-yq-package-e2e-final.log`;
620+
`2026-07-08T13-29-10-0700-item12-yq-check-types-final.log`;
621+
`2026-07-08T13-29-11-0700-item12-yq-cargo-fmt-check-final.log`.
622+
Biome is not applicable for this package test path; it reported the file is
623+
ignored by config in
624+
`2026-07-08T13-29-23-0700-item12-yq-biome-check.log`. Rev: `znlmtymu`.
612625
- **jq — DONE.** Added package-local VM e2e coverage for the staged `jq`
613626
command and fixed the jaq-backed CLI wrapper to accept Linux-style file
614627
operands instead of only stdin. The suite now proves version output,

software/yq/bin/yq

8.21 KB
Binary file not shown.

software/yq/native/crates/yq/src/lib.rs

Lines changed: 69 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
66
use std::ffi::OsString;
77
use std::fmt;
8+
use std::fs::File as FsFile;
89
use std::io::{self, Read, Write};
910

1011
use jaq_core::load::{Arena, File, Loader};
@@ -35,6 +36,7 @@ struct YqOptions {
3536
compact: bool,
3637
null_input: bool,
3738
slurp: bool,
39+
input_paths: Vec<String>,
3840
}
3941

4042
/// Entry point for yq command.
@@ -76,6 +78,7 @@ fn parse_args(args: &[String]) -> Result<YqOptions, String> {
7678
compact: false,
7779
null_input: false,
7880
slurp: false,
81+
input_paths: Vec::new(),
7982
};
8083

8184
let mut filter_set = false;
@@ -85,6 +88,16 @@ fn parse_args(args: &[String]) -> Result<YqOptions, String> {
8588
let arg = &args[i];
8689

8790
if arg == "--" {
91+
let remaining = &args[i + 1..];
92+
if !filter_set {
93+
if let Some(filter) = remaining.first() {
94+
opts.filter = filter.clone();
95+
filter_set = true;
96+
opts.input_paths.extend(remaining.iter().skip(1).cloned());
97+
}
98+
} else {
99+
opts.input_paths.extend(remaining.iter().cloned());
100+
}
88101
break;
89102
}
90103

@@ -151,7 +164,7 @@ fn parse_args(args: &[String]) -> Result<YqOptions, String> {
151164
opts.filter = arg.clone();
152165
filter_set = true;
153166
} else {
154-
return Err(format!("unexpected argument: {}", arg));
167+
opts.input_paths.push(arg.clone());
155168
}
156169

157170
i += 1;
@@ -404,17 +417,47 @@ fn record_output_value(output_count: &mut usize) -> Result<(), String> {
404417
}
405418

406419
fn read_limited_string<R: Read>(reader: R) -> Result<String, String> {
420+
read_limited_source(reader, "stdin", MAX_INPUT_BYTES)
421+
}
422+
423+
fn read_limited_source<R: Read>(reader: R, label: &str, limit: usize) -> Result<String, String> {
407424
let mut input = String::new();
408425
reader
409-
.take((MAX_INPUT_BYTES + 1) as u64)
426+
.take((limit + 1) as u64)
410427
.read_to_string(&mut input)
411-
.map_err(|e| format!("failed to read stdin: {}", e))?;
412-
if input.len() > MAX_INPUT_BYTES {
413-
return Err("stdin exceeds size limit".to_string());
428+
.map_err(|e| format!("failed to read {}: {}", label, e))?;
429+
if input.len() > limit {
430+
return Err(format!("{} exceeds size limit", label));
414431
}
415432
Ok(input)
416433
}
417434

435+
fn read_sources(opts: &YqOptions) -> Result<Vec<String>, String> {
436+
if opts.input_paths.is_empty() {
437+
return Ok(vec![read_limited_string(io::stdin())?]);
438+
}
439+
440+
let mut total = 0usize;
441+
let mut sources = Vec::new();
442+
for path in &opts.input_paths {
443+
let remaining = MAX_INPUT_BYTES.saturating_sub(total);
444+
let data = if path == "-" {
445+
read_limited_source(io::stdin(), "stdin", remaining)?
446+
} else {
447+
let file = FsFile::open(path).map_err(|e| format!("failed to open {}: {}", path, e))?;
448+
read_limited_source(file, path, remaining)?
449+
};
450+
total = total
451+
.checked_add(data.len())
452+
.ok_or_else(|| "input exceeds size limit".to_string())?;
453+
if total > MAX_INPUT_BYTES {
454+
return Err("input exceeds size limit".to_string());
455+
}
456+
sources.push(data);
457+
}
458+
Ok(sources)
459+
}
460+
418461
fn insert_or_array(
419462
map: &mut serde_json::Map<String, serde_json::Value>,
420463
key: String,
@@ -810,37 +853,39 @@ fn write_toml_string(out: &mut LimitedString, s: &str) -> Result<(), String> {
810853
fn run_yq(args: &[String]) -> Result<i32, String> {
811854
let opts = parse_args(args)?;
812855

813-
// Read input
814-
let stdin_data = if opts.null_input {
815-
String::new()
856+
let sources = if opts.null_input {
857+
Vec::new()
816858
} else {
817-
read_limited_string(io::stdin())?
859+
read_sources(&opts)?
818860
};
819861

820-
// Determine input format
821-
let in_format = opts.input_format.unwrap_or_else(|| {
822-
if opts.null_input {
823-
Format::Yaml
824-
} else {
825-
detect_format(&stdin_data)
826-
}
827-
});
862+
let input_formats: Vec<Format> = if opts.null_input {
863+
vec![opts.input_format.unwrap_or(Format::Yaml)]
864+
} else {
865+
sources
866+
.iter()
867+
.map(|source| opts.input_format.unwrap_or_else(|| detect_format(source)))
868+
.collect()
869+
};
870+
let default_input_format = input_formats.first().copied().unwrap_or(Format::Yaml);
828871

829872
// Default output format: YAML for YAML input, otherwise matches input
830-
let out_format = opts.output_format.unwrap_or(in_format);
873+
let out_format = opts.output_format.unwrap_or(default_input_format);
831874

832875
// Parse input to JSON, then convert to jaq Val
833876
let inputs = if opts.null_input {
834877
vec![Val::from(serde_json::Value::Null)]
835878
} else {
836-
let json_val = parse_input(&stdin_data, in_format)?;
879+
let json_values: Result<Vec<_>, _> = sources
880+
.iter()
881+
.zip(input_formats.iter().copied())
882+
.map(|(source, format)| parse_input(source, format))
883+
.collect();
884+
let mut json_values = json_values?;
837885
if opts.slurp {
838-
match json_val {
839-
serde_json::Value::Array(_) => vec![Val::from(json_val)],
840-
_ => vec![Val::from(serde_json::Value::Array(vec![json_val]))],
841-
}
886+
vec![Val::from(serde_json::Value::Array(json_values))]
842887
} else {
843-
vec![Val::from(json_val)]
888+
json_values.drain(..).map(Val::from).collect()
844889
}
845890
};
846891

software/yq/test/yq.test.ts

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
import { existsSync } from "node:fs";
2+
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
3+
import { tmpdir } from "node:os";
4+
import { dirname, join } from "node:path";
5+
import { fileURLToPath } from "node:url";
6+
import {
7+
NodeFileSystem,
8+
createKernel,
9+
createWasmVmRuntime,
10+
describeIf,
11+
} from "@agentos/test-harness";
12+
import type { Kernel } from "@agentos/test-harness";
13+
import { afterEach, expect, it } from "vitest";
14+
15+
const YQ_COMMAND_DIR = fileURLToPath(new URL("../bin", import.meta.url));
16+
const hasYqPackageBinary = existsSync(join(YQ_COMMAND_DIR, "yq"));
17+
18+
let tempRoot: string | undefined;
19+
20+
async function writeFixture(path: string, contents: string): Promise<void> {
21+
if (!tempRoot) throw new Error("fixture root not initialized");
22+
const hostPath = join(tempRoot, path.replace(/^\/+/, ""));
23+
await mkdir(dirname(hostPath), { recursive: true });
24+
await writeFile(hostPath, contents);
25+
}
26+
27+
async function createTestVFS(): Promise<NodeFileSystem> {
28+
tempRoot = await mkdtemp(join(tmpdir(), "agentos-yq-"));
29+
await writeFixture(
30+
"/project/services.yaml",
31+
[
32+
"services:",
33+
" - name: api",
34+
" enabled: true",
35+
" port: 8080",
36+
" - name: worker",
37+
" enabled: false",
38+
" port: 9090",
39+
].join("\n") + "\n",
40+
);
41+
await writeFixture(
42+
"/project/services.json",
43+
JSON.stringify({ services: [{ name: "api" }, { name: "worker" }] }) + "\n",
44+
);
45+
await writeFixture(
46+
"/project/config.toml",
47+
["[server]", 'name = "agentos"', "port = 7331", "enabled = true"].join("\n") +
48+
"\n",
49+
);
50+
await writeFixture(
51+
"/project/inventory.xml",
52+
'<inventory><item id="a">hammer</item><item id="b">nail</item></inventory>\n',
53+
);
54+
await writeFixture("/project/broken.yaml", "services:\n - name: ok\n bad");
55+
return new NodeFileSystem({ root: tempRoot });
56+
}
57+
58+
function lines(stdout: string): string[] {
59+
return stdout.split("\n").filter((line) => line.length > 0);
60+
}
61+
62+
describeIf(hasYqPackageBinary, "yq command", { timeout: 10_000 }, () => {
63+
let kernel: Kernel | undefined;
64+
65+
afterEach(async () => {
66+
await kernel?.dispose();
67+
kernel = undefined;
68+
if (tempRoot) {
69+
await rm(tempRoot, { recursive: true, force: true });
70+
tempRoot = undefined;
71+
}
72+
});
73+
74+
async function mountFixture(): Promise<void> {
75+
const vfs = await createTestVFS();
76+
kernel = createKernel({ filesystem: vfs });
77+
await kernel.mount(createWasmVmRuntime({ commandDirs: [YQ_COMMAND_DIR] }));
78+
}
79+
80+
async function runYq(args: string[]) {
81+
if (!kernel) throw new Error("kernel not mounted");
82+
let stdout = "";
83+
let stderr = "";
84+
const proc = kernel.spawn("yq", args, {
85+
onStdout: (chunk) => {
86+
stdout += Buffer.from(chunk).toString("utf8");
87+
},
88+
onStderr: (chunk) => {
89+
stderr += Buffer.from(chunk).toString("utf8");
90+
},
91+
});
92+
const exitCode = await proc.wait();
93+
await new Promise<void>((resolve) => setTimeout(resolve, 0));
94+
return { stdout, stderr, exitCode };
95+
}
96+
97+
it("filters YAML files and emits raw strings", async () => {
98+
await mountFixture();
99+
100+
const result = await runYq([
101+
"-r",
102+
".services[] | select(.enabled) | .name",
103+
"/project/services.yaml",
104+
]);
105+
expect(result.exitCode, result.stderr || result.stdout).toBe(0);
106+
expect(lines(result.stdout)).toEqual(["api"]);
107+
});
108+
109+
it("converts YAML query results to compact JSON", async () => {
110+
await mountFixture();
111+
112+
const result = await runYq([
113+
"-o",
114+
"json",
115+
"-c",
116+
"{names: [.services[].name], ports: [.services[].port]}",
117+
"/project/services.yaml",
118+
]);
119+
expect(result.exitCode, result.stderr || result.stdout).toBe(0);
120+
expect(JSON.parse(result.stdout.trim())).toEqual({
121+
names: ["api", "worker"],
122+
ports: [8080, 9090],
123+
});
124+
});
125+
126+
it("reads JSON files explicitly", async () => {
127+
await mountFixture();
128+
129+
const result = await runYq([
130+
"-p",
131+
"json",
132+
"-r",
133+
".services[].name",
134+
"/project/services.json",
135+
]);
136+
expect(result.exitCode, result.stderr || result.stdout).toBe(0);
137+
expect(lines(result.stdout)).toEqual(["api", "worker"]);
138+
});
139+
140+
it("reads TOML files explicitly", async () => {
141+
await mountFixture();
142+
143+
const result = await runYq([
144+
"-p",
145+
"toml",
146+
"-o",
147+
"json",
148+
"-c",
149+
".server",
150+
"/project/config.toml",
151+
]);
152+
expect(result.exitCode, result.stderr || result.stdout).toBe(0);
153+
expect(JSON.parse(result.stdout.trim())).toEqual({
154+
name: "agentos",
155+
port: 7331,
156+
enabled: true,
157+
});
158+
});
159+
160+
it("reads XML files explicitly", async () => {
161+
await mountFixture();
162+
163+
const result = await runYq([
164+
"-p",
165+
"xml",
166+
"-r",
167+
'.inventory.item[]["#text"]',
168+
"/project/inventory.xml",
169+
]);
170+
expect(result.exitCode, result.stderr || result.stdout).toBe(0);
171+
expect(lines(result.stdout)).toEqual(["hammer", "nail"]);
172+
});
173+
174+
it("fails with a parse error for invalid YAML", async () => {
175+
await mountFixture();
176+
177+
const result = await runYq([".", "/project/broken.yaml"]);
178+
expect(result.exitCode).not.toBe(0);
179+
expect(result.stderr).toContain("invalid YAML");
180+
});
181+
});

0 commit comments

Comments
 (0)