Skip to content

Commit 48871f0

Browse files
author
catlog22
committed
feat(tests): add CLI API response format tests and output format detection
1 parent a54246a commit 48871f0

3 files changed

Lines changed: 263 additions & 2 deletions

File tree

‎ccw/frontend/src/lib/api.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1571,11 +1571,17 @@ export interface ConversationTurn {
15711571
stdout: string;
15721572
stderr?: string;
15731573
truncated?: boolean;
1574+
cached?: boolean;
1575+
stdout_full?: string;
1576+
stderr_full?: string;
1577+
parsed_output?: string;
1578+
final_output?: string;
15741579
structured?: unknown[];
15751580
};
15761581
timestamp: string;
15771582
duration_ms: number;
15781583
status?: 'success' | 'error' | 'timeout';
1584+
exit_code?: number;
15791585
}
15801586

15811587
// ========== CLI Tools Config API ==========

‎ccw/src/tools/cli-executor-utils.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -386,8 +386,14 @@ export function buildCommand(params: {
386386
fullCommand: `${command} ${args.join(' ')}${useStdin ? ' (stdin)' : ''}`,
387387
});
388388

389-
// Auto-detect output format: Codex uses --json flag for JSONL output
390-
const outputFormat = tool.toLowerCase() === 'codex' ? 'json-lines' : 'text';
389+
// Auto-detect output format: All CLI tools use JSON lines output
390+
// - Codex: --json
391+
// - Gemini: -o stream-json
392+
// - Qwen: -o stream-json
393+
// - Claude: --output-format stream-json
394+
// - OpenCode: --format json
395+
const jsonLineTools = ['codex', 'gemini', 'qwen', 'claude', 'opencode'];
396+
const outputFormat = jsonLineTools.includes(tool.toLowerCase()) ? 'json-lines' : 'text';
391397

392398
return { command, args, useStdin, outputFormat };
393399
}

‎ccw/tests/api-response-test.ts‎

Lines changed: 249 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,249 @@
1+
/**
2+
* Test script to verify CLI API response format
3+
* Tests that the API returns properly parsed JSON without double-serialization
4+
*/
5+
6+
import { join } from 'path';
7+
8+
async function testApiResponse() {
9+
console.log('=== API Response Format Test ===\n');
10+
11+
// Use parent directory as project root (D:\Claude_dms3 instead of D:\Claude_dms3\ccw)
12+
const projectPath = join(process.cwd(), '..');
13+
14+
// Test 1: Get a sample execution (you'll need to replace with an actual ID)
15+
console.log('Test 1: Get conversation detail');
16+
console.log('Project path:', projectPath);
17+
18+
try {
19+
// Get the most recent execution for testing
20+
const { getHistoryStore } = await import('../src/tools/cli-history-store.js');
21+
const store = getHistoryStore(projectPath);
22+
const history = store.getHistory({ limit: 1 });
23+
24+
if (history.total === 0 || history.executions.length === 0) {
25+
console.log('❌ No execution history found. Please run a CLI command first.');
26+
console.log('Example: ccw cli -p "test" --tool gemini --mode analysis\n');
27+
return;
28+
}
29+
30+
const executionId = history.executions[0].id;
31+
console.log('Testing with execution ID:', executionId, '\n');
32+
33+
// Get conversation detail - use getConversationWithNativeInfo from store directly
34+
const conversation = store.getConversationWithNativeInfo(executionId);
35+
36+
if (!conversation) {
37+
console.log('❌ Conversation not found');
38+
return;
39+
}
40+
41+
console.log('✅ Conversation retrieved');
42+
console.log(' - ID:', conversation.id);
43+
console.log(' - Tool:', conversation.tool);
44+
console.log(' - Mode:', conversation.mode);
45+
console.log(' - Turns:', conversation.turns.length);
46+
console.log();
47+
48+
// Test 2: Check turn output structure
49+
console.log('Test 2: Verify turn output structure');
50+
51+
if (conversation.turns.length > 0) {
52+
const firstTurn = conversation.turns[0];
53+
console.log('First turn output keys:', Object.keys(firstTurn.output));
54+
console.log();
55+
56+
// Test 3: Check for double-serialization
57+
console.log('Test 3: Check for JSON double-serialization');
58+
59+
const outputFields = [
60+
'stdout',
61+
'stderr',
62+
'parsed_output',
63+
'final_output'
64+
];
65+
66+
let hasDoubleSerializtion = false;
67+
68+
for (const field of outputFields) {
69+
const value = firstTurn.output[field as keyof typeof firstTurn.output];
70+
if (value && typeof value === 'string') {
71+
// Check if the string starts with a JSON structure indicator
72+
const trimmed = value.trim();
73+
if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
74+
try {
75+
const parsed = JSON.parse(trimmed);
76+
console.log(`⚠️ ${field}: Contains JSON string (length: ${trimmed.length})`);
77+
console.log(` First 100 chars: ${trimmed.substring(0, 100)}...`);
78+
console.log(` Parsed type: ${typeof parsed}, keys: ${Object.keys(parsed).slice(0, 5).join(', ')}`);
79+
hasDoubleSerializtion = true;
80+
} catch {
81+
// Not JSON, this is fine
82+
console.log(`✅ ${field}: Plain text (length: ${trimmed.length})`);
83+
}
84+
} else {
85+
console.log(`✅ ${field}: Plain text (length: ${trimmed.length})`);
86+
}
87+
} else if (value) {
88+
console.log(`ℹ️ ${field}: Type ${typeof value}`);
89+
}
90+
}
91+
92+
console.log();
93+
94+
if (hasDoubleSerializtion) {
95+
console.log('❌ ISSUE FOUND: Some fields contain JSON strings instead of plain text');
96+
console.log(' This suggests double-serialization or incorrect parsing.');
97+
} else {
98+
console.log('✅ No double-serialization detected');
99+
}
100+
}
101+
102+
// Test 4: Simulate API JSON.stringify
103+
console.log('\nTest 4: Simulate API response serialization');
104+
const apiResponse = JSON.stringify(conversation);
105+
console.log('API response length:', apiResponse.length);
106+
107+
// Parse it back (like frontend would)
108+
const parsed = JSON.parse(apiResponse);
109+
console.log('✅ Can be parsed back');
110+
console.log('Parsed turn count:', parsed.turns.length);
111+
112+
if (parsed.turns.length > 0) {
113+
const parsedTurn = parsed.turns[0];
114+
console.log('Parsed turn output keys:', Object.keys(parsedTurn.output));
115+
116+
// Check if parsed_output is accessible
117+
if (parsedTurn.output.parsed_output) {
118+
console.log('✅ parsed_output field is accessible');
119+
console.log(' Length:', parsedTurn.output.parsed_output.length);
120+
} else {
121+
console.log('❌ parsed_output field is missing or undefined');
122+
}
123+
}
124+
125+
// Test 5: Check stdout content - is it JSON lines or plain text?
126+
console.log('\nTest 5: Check stdout content format');
127+
if (conversation.turns.length > 0) {
128+
const stdout = conversation.turns[0].output.stdout;
129+
const firstLines = stdout.split('\n').slice(0, 5);
130+
console.log('First 5 lines of stdout:');
131+
for (const line of firstLines) {
132+
const trimmed = line.trim();
133+
if (!trimmed) continue;
134+
let isJson = false;
135+
try {
136+
JSON.parse(trimmed);
137+
isJson = true;
138+
} catch {}
139+
console.log(` ${isJson ? '⚠️ JSON' : '✅ TEXT'}: ${trimmed.substring(0, 120)}${trimmed.length > 120 ? '...' : ''}`);
140+
}
141+
142+
// Compare stdout vs parsed_output
143+
const parsedOutput = conversation.turns[0].output.parsed_output;
144+
console.log('\nTest 6: Compare stdout vs parsed_output');
145+
console.log(` stdout length: ${stdout.length}`);
146+
console.log(` parsed_output length: ${parsedOutput?.length || 0}`);
147+
if (parsedOutput) {
148+
const parsedFirstLines = parsedOutput.split('\n').slice(0, 3);
149+
console.log(' First 3 lines of parsed_output:');
150+
for (const line of parsedFirstLines) {
151+
console.log(` ${line.substring(0, 120)}${line.length > 120 ? '...' : ''}`);
152+
}
153+
}
154+
}
155+
156+
} catch (error) {
157+
console.error('❌ Test failed:', error);
158+
}
159+
}
160+
161+
/**
162+
* Test that buildCommand returns correct outputFormat for all tools
163+
*/
164+
async function testOutputFormatDetection() {
165+
console.log('\n=== Output Format Detection Test ===\n');
166+
167+
const { buildCommand } = await import('../src/tools/cli-executor-utils.js');
168+
169+
const tools = ['gemini', 'qwen', 'codex', 'claude', 'opencode'];
170+
171+
for (const tool of tools) {
172+
try {
173+
const result = buildCommand({
174+
tool,
175+
prompt: 'test prompt',
176+
mode: 'analysis',
177+
});
178+
const expected = 'json-lines';
179+
const status = result.outputFormat === expected ? '✅' : '❌';
180+
console.log(` ${status} ${tool}: outputFormat = "${result.outputFormat}" (expected: "${expected}")`);
181+
} catch (err) {
182+
console.log(` ⚠️ ${tool}: buildCommand error (${(err as Error).message})`);
183+
}
184+
}
185+
}
186+
187+
/**
188+
* Test that JsonLinesParser correctly extracts text from Gemini JSON lines
189+
*/
190+
async function testJsonLinesParsing() {
191+
console.log('\n=== JSON Lines Parser Test ===\n');
192+
193+
const { createOutputParser, flattenOutputUnits } = await import('../src/tools/cli-output-converter.js');
194+
195+
const parser = createOutputParser('json-lines');
196+
197+
// Simulate Gemini stream-json output
198+
const geminiLines = [
199+
'{"type":"init","timestamp":"2026-01-01T00:00:00.000Z","session_id":"test-session","model":"gemini-2.5-pro"}',
200+
'{"type":"message","timestamp":"2026-01-01T00:00:01.000Z","role":"user","content":"test prompt"}',
201+
'{"type":"message","timestamp":"2026-01-01T00:00:02.000Z","role":"assistant","content":"Hello, this is the response text.","delta":true}',
202+
'{"type":"message","timestamp":"2026-01-01T00:00:03.000Z","role":"assistant","content":" More response text here.","delta":true}',
203+
'{"type":"result","timestamp":"2026-01-01T00:00:04.000Z","status":"success","stats":{"input_tokens":100,"output_tokens":50}}',
204+
];
205+
206+
const input = Buffer.from(geminiLines.join('\n') + '\n');
207+
const units = parser.parse(input, 'stdout');
208+
const remaining = parser.flush();
209+
const allUnits = [...units, ...remaining];
210+
211+
console.log(` Total IR units created: ${allUnits.length}`);
212+
for (const unit of allUnits) {
213+
const contentPreview = typeof unit.content === 'string'
214+
? unit.content.substring(0, 80)
215+
: JSON.stringify(unit.content).substring(0, 80);
216+
console.log(` Type: ${unit.type.padEnd(20)} Content: ${contentPreview}`);
217+
}
218+
219+
// Test flattenOutputUnits with same filters as cli-executor-core.ts
220+
const parsedOutput = flattenOutputUnits(allUnits, {
221+
excludeTypes: ['stderr', 'progress', 'metadata', 'system', 'tool_call', 'thought', 'code', 'file_diff', 'streaming_content'],
222+
stripCommandJsonBlocks: true
223+
});
224+
225+
console.log();
226+
console.log(` parsed_output result:`);
227+
console.log(` "${parsedOutput}"`);
228+
229+
// Verify it's NOT JSON lines
230+
const firstLine = parsedOutput.split('\n')[0]?.trim();
231+
let isJson = false;
232+
try {
233+
JSON.parse(firstLine);
234+
isJson = true;
235+
} catch {}
236+
237+
if (isJson) {
238+
console.log(` ❌ parsed_output still contains JSON lines!`);
239+
} else {
240+
console.log(` ✅ parsed_output contains plain text (not JSON lines)`);
241+
}
242+
}
243+
244+
// Run all tests
245+
(async () => {
246+
await testApiResponse();
247+
await testOutputFormatDetection();
248+
await testJsonLinesParsing();
249+
})().catch(console.error);

0 commit comments

Comments
 (0)