streaming

Author: eviltester

README.md

@@ -359,11 +359,12 @@ Notes:
 - for JSON requests, newline characters inside string literals must be escaped as `\\n`
 - use `/v1/generate/fromschema` when you want to paste raw multiline text directly
 
-Streaming notes for generation:
+Streaming notes for CLI/core generation:
 
 - stream mode supports: `csv`, `jsonl`, `dsv`, `json`, `xml`
 - `json` stream mode emits a valid JSON array payload
 - incompatible format options in stream mode are warned and ignored where needed
+- REST API generation endpoints currently run in buffered mode
 
 OpenAPI spec:
 

apps/api/src/openapi.js

@@ -33,7 +33,7 @@ const openApiDocument = {
       post: {
         summary: 'Generate data from text spec with selectable response shape',
         description:
-          'Streaming generation is available for csv, jsonl, dsv, json and xml. In stream mode, incompatible format options are reported as warnings and ignored where needed.',
+          'Generation requests are currently processed in buffered mode for all formats. Stream-mode behavior is available in the core helper and CLI, not this REST endpoint.',
         requestBody: {
           required: true,
           content: {

docs-src/docs/070-interfaces-and-deployment/030-rest-api.md

@@ -148,11 +148,10 @@ Both endpoints generate data from the same schema language and output formats. T
 - `stream` is accepted for compatibility and ignored
 - `inputFormat` is normalized (trimmed and lower-cased), so values like `" csv "` are accepted
 
-Generation streaming behavior:
+Generation mode behavior:
 
-- stream mode supports: `csv`, `jsonl`, `dsv`, `json`, `xml`
-- `json` stream mode emits a valid JSON array payload
-- if stream mode cannot honor some format options, generation continues and warnings are reported
+- REST generation endpoints currently run in buffered mode.
+- Stream-mode generation behavior is available via the core helper/CLI paths, not via `/v1/generate`.
 
 ## Schema Formatting
 

package-lock.json

@@ -553,15 +553,16 @@ function quoteCsvValue(value, { quoteChar, escapeChar }) {
 }
 
 function rowToCsv(headers, row, csvSettings) {
-  return headers.map((header) => quoteCsvValue(row[header], csvSettings)).join(',');
+  return headers.map((_, index) => quoteCsvValue(row[index], csvSettings)).join(',');
 }
 
 function rowToDsv(headers, row, dsvSettings) {
-  return headers.map((header) => quoteCsvValue(row[header], dsvSettings)).join(dsvSettings.delimiter);
+  return headers.map((_, index) => quoteCsvValue(row[index], dsvSettings)).join(dsvSettings.delimiter);
 }
 
-function rowToJsonLine(row) {
-  return JSON.stringify(row);
+function rowToJsonLine(headers, rowArray, options = {}) {
+  const rowObject = rowArrayToObject(headers, rowArray, options);
+  return JSON.stringify(rowObject);
 }
 
 function rowArrayToObject(headers, rowArray, options = {}) {
@@ -571,10 +572,13 @@ function rowArrayToObject(headers, rowArray, options = {}) {
     const header = headers[i];
     const value = rowArray[i] === undefined || rowArray[i] === null ? '' : rowArray[i];
     if (makeNumbersNumeric) {
-      if (value * 1 == 0) {
-        rowObject[header] = 0;
+      const text = String(value);
+      if (text.trim().length === 0) {
+        rowObject[header] = value;
+      } else if (Number.isFinite(Number(text))) {
+        rowObject[header] = Number(text);
       } else {
-        rowObject[header] = value == value * 1 ? value * 1 : value;
+        rowObject[header] = value;
       }
     } else {
       rowObject[header] = value;
@@ -789,7 +793,7 @@ export async function streamFromTextSpec({
     } else if (outputFormat === 'dsv') {
       await onChunk(rowToDsv(headers, rowArray, dsvSettings));
     } else if (outputFormat === 'jsonl') {
-      await onChunk(rowToJsonLine(rowArray));
+      await onChunk(rowToJsonLine(headers, rowArray, options));
     } else if (outputFormat === 'json') {
       const rowObject = rowArrayToObject(headers, rowArray, options);
       await onChunk(index === 0 ? JSON.stringify(rowObject) : `,${JSON.stringify(rowObject)}`);

packages/core/src/index.js

@@ -14,6 +14,8 @@ test('streamFromTextSpec streams csv rows with header', async () => {
   expect(result.ok).toBe(true);
   expect(chunks.length).toBe(3);
   expect(chunks[0]).toContain('"Name"');
+  expect(chunks[1]).toContain('"Bob"');
+  expect(chunks[2]).toContain('"Bob"');
 });
 
 test('streamFromTextSpec streams jsonl rows', async () => {
@@ -30,7 +32,10 @@ test('streamFromTextSpec streams jsonl rows', async () => {
   expect(result.ok).toBe(true);
   expect(chunks.length).toBe(2);
   for (const chunk of chunks) {
-    expect(() => JSON.parse(chunk)).not.toThrow();
+    const parsed = JSON.parse(chunk);
+    expect(Array.isArray(parsed)).toBe(false);
+    expect(parsed).toHaveProperty('Name');
+    expect(parsed.Name).toBe('Bob');
   }
 });
 
@@ -73,6 +78,8 @@ test('streamFromTextSpec streams dsv rows with delimiter and header', async () =
   expect(result.ok).toBe(true);
   expect(chunks.length).toBe(3);
   expect(chunks[0]).toContain('"Name"');
+  expect(chunks[1]).toContain('"Bob"');
+  expect(chunks[2]).toContain('"Bob"');
 });
 
 test('streamFromTextSpec streams json as valid array payload', async () => {
@@ -91,6 +98,8 @@ test('streamFromTextSpec streams json as valid array payload', async () => {
   expect(Array.isArray(parsed)).toBe(true);
   expect(parsed.length).toBe(2);
   expect(parsed[0]).toHaveProperty('Name');
+  expect(parsed[0].Name).toBe('Bob');
+  expect(parsed[1].Name).toBe('Bob');
 });
 
 test('streamFromTextSpec streams xml as a well-formed document', async () => {
@@ -109,6 +118,7 @@ test('streamFromTextSpec streams xml as a well-formed document', async () => {
   expect(payload).toContain('<root>');
   expect(payload).toContain('</root>');
   expect(payload).toContain('<item>');
+  expect(payload).toContain('<Name>Bob</Name>');
 });
 
 test('streamFromTextSpec warns for json options that are unsupported in stream mode', async () => {
@@ -126,3 +136,20 @@ test('streamFromTextSpec warns for json options that are unsupported in stream m
   expect(Array.isArray(result.diagnostics.warnings)).toBe(true);
   expect(result.diagnostics.warnings.join(' ')).toContain('ignores option asObject');
 });
+
+test('streamFromTextSpec json makeNumbersNumeric preserves empty strings', async () => {
+  const chunks = [];
+  const result = await streamFromTextSpec({
+    textSpec: 'Quantity\n()',
+    rowCount: 1,
+    outputFormat: 'json',
+    options: { makeNumbersNumeric: true },
+    onChunk: async (chunk) => {
+      chunks.push(chunk);
+    },
+  });
+
+  expect(result.ok).toBe(true);
+  const parsed = JSON.parse(chunks.join(''));
+  expect(parsed[0].Quantity).toBe('');
+});