Merge pull request #82 from eviltester/69-add-streaming-output-for-all-export-formats

more streaming output

Author: eviltester

README.md

@@ -359,6 +359,13 @@ Notes:
 - for JSON requests, newline characters inside string literals must be escaped as `\\n`
 - use `/v1/generate/fromschema` when you want to paste raw multiline text directly
 
+Streaming notes for CLI/core generation:
+
+- stream mode supports: `csv`, `jsonl`, `dsv`, `json`, `xml`
+- `json` stream mode emits a valid JSON array payload
+- incompatible format options in stream mode are warned and ignored where needed
+- REST API generation endpoints currently run in buffered mode
+
 OpenAPI spec:
 
 `GET http://localhost:3000/v1/openapi.json`

apps/api/src/openapi.js

@@ -32,6 +32,8 @@ const openApiDocument = {
     '/v1/generate': {
       post: {
         summary: 'Generate data from text spec with selectable response shape',
+        description:
+          'Generation requests are currently processed in buffered mode for all formats. Stream-mode behavior is available in the core helper and CLI, not this REST endpoint.',
         requestBody: {
           required: true,
           content: {

apps/cli/README.md

@@ -83,8 +83,15 @@ Streaming is currently supported for:
 
 - `csv`
 - `jsonl`
+- `dsv`
+- `json`
+- `xml`
 
-Other formats use buffered generation.
+Notes:
+
+- `json` stream mode emits a valid JSON array payload.
+- For stream-mode option mismatches, generation continues and warnings are reported when applicable.
+- Other formats use buffered generation.
 
 For `amend`, streaming flags are accepted for compatibility but ignored (always buffered).
 
@@ -93,6 +100,9 @@ Examples:
 ```bash
 anywaydata generate -i ./apps/cli/examples/company-literal.txt -n 100000 -f jsonl -o output.jsonl --stream
 anywaydata generate -i ./apps/cli/examples/company-literal.txt -n 100000 -f csv -o output.csv --stream-threshold 1000
+anywaydata generate -i ./apps/cli/examples/company-literal.txt -n 100000 -f dsv -o output.dsv --stream
+anywaydata generate -i ./apps/cli/examples/company-literal.txt -n 100000 -f json -o output.json --stream
+anywaydata generate -i ./apps/cli/examples/company-literal.txt -n 100000 -f xml -o output.xml --stream
 ```
 
 ## Spec File Format

apps/cli/src/cli-options.js

@@ -37,7 +37,7 @@ export function parseCliOptions(argvInput = process.argv) {
     .option('stream', {
       type: 'boolean',
       default: false,
-      describe: 'Use streaming generation path when available (exporting CSV or JSONL)',
+      describe: 'Use streaming generation path when available (exporting CSV, JSONL, DSV, JSON or XML)',
     })
     .option('stream-threshold', {
       type: 'number',

apps/cli/src/run-cli.js

@@ -106,7 +106,7 @@ export async function runCliCommand({ options, platform }) {
     }
   }
 
-  if (useStreamMode && (outputFormat === 'csv' || outputFormat === 'jsonl')) {
+  if (useStreamMode && ['csv', 'jsonl', 'dsv', 'json', 'xml'].includes(outputFormat)) {
     const streamedLines = [];
     const writer = options.outputFile ? platform.createLineWriter(options.outputFile) : null;
     let writerClosed = false;
@@ -133,6 +133,11 @@ export async function runCliCommand({ options, platform }) {
       }
 
       progress(streamResult.diagnostics.report);
+      if (Array.isArray(streamResult.diagnostics?.warnings)) {
+        for (const warning of streamResult.diagnostics.warnings) {
+          progress(`WARNING: ${warning}`);
+        }
+      }
       if (options.testMode && streamResult.diagnostics.firstRow) {
         progress('e.g.');
         progress(JSON.stringify(streamResult.diagnostics.firstRow));

apps/cli/src/tests/integration.cli-params.test.js

@@ -102,28 +102,30 @@ test('param --show-progress false suppresses progress logs', () => {
   expect(result.stdout).toContain('"Company"');
 });
 
-test('params --stream and --show-progress true use streaming path for csv/jsonl', async () => {
+test('params --stream and --show-progress true use streaming path for csv/jsonl/dsv/json/xml', async () => {
   const inputPath = path.join(repoRoot, 'apps', 'cli', 'examples', 'company-literal.txt');
-  const outputPath = tempFile('stream');
-  const result = runCli([
-    'generate',
-    '-i',
-    inputPath,
-    '-n',
-    '2',
-    '-f',
-    'csv',
-    '-o',
-    outputPath,
-    '--stream',
-    '--show-progress',
-    'true',
-  ]);
-  expect(result.status).toBe(0);
-  expect(result.stdout).toContain('using stream mode');
-
-  const written = await fs.readFile(outputPath, 'utf8');
-  expect(written).toContain('"Company"');
+  const formats = ['csv', 'jsonl', 'dsv', 'json', 'xml'];
+  for (const format of formats) {
+    const outputPath = tempFile(`stream-${format}`);
+    const result = runCli([
+      'generate',
+      '-i',
+      inputPath,
+      '-n',
+      '2',
+      '-f',
+      format,
+      '-o',
+      outputPath,
+      '--stream',
+      '--show-progress',
+      'true',
+    ]);
+    expect(result.status).toBe(0);
+    expect(result.stdout).toContain('using stream mode');
+    const written = await fs.readFile(outputPath, 'utf8');
+    expect(written.length).toBeGreaterThan(0);
+  }
 });
 
 test('param --stream-threshold auto-enables stream mode when threshold reached', () => {

docs-src/docs/070-interfaces-and-deployment/030-rest-api.md

@@ -148,6 +148,11 @@ Both endpoints generate data from the same schema language and output formats. T
 - `stream` is accepted for compatibility and ignored
 - `inputFormat` is normalized (trimmed and lower-cased), so values like `" csv "` are accepted
 
+Generation mode behavior:
+
+- REST generation endpoints currently run in buffered mode.
+- Stream-mode generation behavior is available via the core helper/CLI paths, not via `/v1/generate`.
+
 ## Schema Formatting
 
 Schema text supports:

docs-src/docs/070-interfaces-and-deployment/050-cli-node-and-bun.md

@@ -44,7 +44,7 @@ Parameter guide for the examples:
 - `-o, --outputfile`: optional output file path. If omitted, output is written to stdout.
 - `-t, --testMode`: generate one row and print diagnostics for troubleshooting.
 - `--show-progress`: explicitly control progress logs (for example `--show-progress true` or `--show-progress false`).
-- `--stream`: enable streaming generation when supported (currently `csv` and `jsonl`).
+- `--stream`: enable streaming generation when supported (`csv`, `jsonl`, `dsv`, `json`, `xml`).
 - `--stream-threshold`: auto-enable streaming when `rowCount >= threshold` and `--outputfile` is set (default `5000`).
 - `--unsafe-faker-expressions`: opt-in to expression-style faker arguments (unsafe for untrusted input).
 - `--help`: show CLI usage and options.
@@ -65,7 +65,9 @@ Schema text supports:
   - on for stdout mode (no `--outputfile`)
   - on for `--testMode`
   - off for file output unless `--show-progress true` is provided
-- Streaming is currently implemented for `csv` and `jsonl` exports. Other formats use buffered generation.
+- Streaming is currently implemented for `csv`, `jsonl`, `dsv`, `json`, and `xml` exports. Other formats use buffered generation.
+- JSON stream mode emits a valid JSON array payload.
+- If a stream mode cannot honor some format options, generation continues and warnings are reported.
 - `amend` always uses buffered generation; stream flags are ignored for this command.
 - Auto-streaming (`--stream-threshold`) applies only when writing to a file. For stdout workflows, use `--stream` explicitly.
 - For `amend`, if `-n/--numberOfLines` is omitted, all imported rows are amended.