Merge pull request #166 from eviltester/142-export-settings-line-endings-bom

Add shared export encoding settings

Author: eviltester

apps/api/jest.config.cjs

@@ -1,5 +1,5 @@
 module.exports = {
   testEnvironment: "node",
-  testMatch: ["<rootDir>/src/*.test.js"],
+  testRegex: "src[/\\\\].*\\.test\\.js$",
   transform: {},
 };

apps/cli/README.md

@@ -61,6 +61,12 @@ Write to an output file:
 anywaydata generate -i exampleTestDataSpec.txt -n 3 -o output.csv
 ```
 
+Write CRLF line endings and a UTF-8 BOM to an output file:
+
+```bash
+anywaydata generate -i exampleTestDataSpec.txt -n 3 -o output.csv --line-endings crlf --bom
+```
+
 Redirect stdout:
 
 ```bash
@@ -82,6 +88,8 @@ anywaydata amend --schema-file schema.txt --data-file input.csv --input-format c
 - `-n, --numberOfLines` row count (required, overridden to `1` by `--testMode`)
 - `-f, --format` output format (e.g. `csv`, `json`, `jsonl`, `xml`, `sql`)
 - `-o, --outputfile` optional output file path
+- `--line-endings` file-output line endings: `lf` or `crlf` (defaults to the current OS)
+- `--bom` write a UTF-8 BOM when output is written to a file
 - `-t, --testMode` generate one row and print diagnostics/example output
 - `--show-progress` force progress logs on/off
 - `--stream` enable stream mode when supported

apps/cli/jest.config.cjs

@@ -4,5 +4,5 @@ module.exports = {
   transform: {
     '^.+\\.js$': 'babel-jest',
   },
-  testMatch: ['<rootDir>/src/tests/**/*.test.js'],
+  testRegex: 'src[/\\\\]tests[/\\\\].*\\.test\\.js$',
 };

apps/cli/src/cli-options.js

@@ -49,6 +49,16 @@ export function parseCliOptions(argvInput = process.argv) {
       default: false,
       describe: 'Allow expression-style faker arguments (unsafe for untrusted input)',
     })
+    .option('line-endings', {
+      type: 'string',
+      choices: ['lf', 'crlf'],
+      describe: 'Line endings for file output: lf or crlf (defaults to the current OS)',
+    })
+    .option('bom', {
+      type: 'boolean',
+      default: false,
+      describe: 'Write a UTF-8 BOM when output is written to a file',
+    })
     .option('pairwise', {
       type: 'boolean',
       default: false,
@@ -79,6 +89,8 @@ export function parseCliOptions(argvInput = process.argv) {
     testMode: parsed.testMode === true,
     showProgress,
     shouldStream,
+    lineEndings: parsed['line-endings'] ? String(parsed['line-endings']).toLowerCase() : undefined,
+    bom: parsed.bom === true,
     unsafeFakerExpressions: parsed['unsafe-faker-expressions'] === true,
     pairwise: parsed.pairwise === true,
   };

apps/cli/src/platform/bun-platform.js

@@ -1,18 +1,31 @@
+import { applyExportTextEncoding, resolveExportTextEncodingSettings } from '@anywaydata/core';
+
 export function createBunPlatform() {
   const bunRef = globalThis.Bun;
   return {
     name: 'bun',
+    getRuntimePlatform() {
+      return process.platform;
+    },
     async readText(path) {
       const file = bunRef.file(path);
       return file.text();
     },
-    async writeText(path, content) {
-      await bunRef.write(path, content);
+    async writeText(path, content, exportEncodingSettings = {}) {
+      await bunRef.write(
+        path,
+        applyExportTextEncoding(content, exportEncodingSettings, { platform: process.platform })
+      );
     },
-    createLineWriter(path) {
+    createLineWriter(path, exportEncodingSettings = {}) {
+      const resolvedEncodingSettings = resolveExportTextEncodingSettings(exportEncodingSettings, {
+        platform: process.platform,
+      });
+      const lineSuffix = resolvedEncodingSettings.lineEnding === 'crlf' ? '\r\n' : '\n';
       const writer = bunRef.file(path).writer();
       let writerError = null;
       let closed = false;
+      let wroteBom = false;
       return {
         async writeLine(line) {
           if (closed) {
@@ -22,7 +35,9 @@ export function createBunPlatform() {
             throw writerError;
           }
           try {
-            writer.write(`${line}\n`);
+            const prefix = !wroteBom && resolvedEncodingSettings.includeBom ? '\uFEFF' : '';
+            writer.write(`${prefix}${String(line ?? '')}${lineSuffix}`);
+            wroteBom = true;
           } catch (error) {
             writerError = error;
             throw error;

apps/cli/src/platform/node-platform.js

@@ -1,18 +1,63 @@
 import fs from 'node:fs/promises';
 import { createWriteStream } from 'node:fs';
+import { applyExportTextEncoding, resolveExportTextEncodingSettings } from '@anywaydata/core';
+
+function waitForStreamEvent(stream, eventName, startOperation, { shouldResolveImmediately = () => false } = {}) {
+  return new Promise((resolve, reject) => {
+    let settled = false;
+
+    const cleanup = () => {
+      stream.off(eventName, handleEvent);
+      stream.off('error', handleError);
+    };
+
+    const settle = (callback, value) => {
+      if (settled) {
+        return;
+      }
+      settled = true;
+      cleanup();
+      callback(value);
+    };
+
+    const handleEvent = () => settle(resolve);
+    const handleError = (error) => settle(reject, error);
+
+    stream.once(eventName, handleEvent);
+    stream.once('error', handleError);
+
+    try {
+      const result = startOperation();
+      if (shouldResolveImmediately(result)) {
+        settle(resolve);
+      }
+    } catch (error) {
+      settle(reject, error);
+    }
+  });
+}
 
 export function createNodePlatform() {
   return {
     name: 'node',
+    getRuntimePlatform() {
+      return process.platform;
+    },
     async readText(path) {
       return fs.readFile(path, 'utf8');
     },
-    async writeText(path, content) {
-      await fs.writeFile(path, content, 'utf8');
+    async writeText(path, content, exportEncodingSettings = {}) {
+      const encodedContent = applyExportTextEncoding(content, exportEncodingSettings, { platform: process.platform });
+      await fs.writeFile(path, encodedContent, 'utf8');
     },
-    createLineWriter(path) {
+    createLineWriter(path, exportEncodingSettings = {}) {
+      const resolvedEncodingSettings = resolveExportTextEncodingSettings(exportEncodingSettings, {
+        platform: process.platform,
+      });
+      const lineSuffix = resolvedEncodingSettings.lineEnding === 'crlf' ? '\r\n' : '\n';
       const stream = createWriteStream(path, { encoding: 'utf8' });
       let streamError = null;
+      let wroteBom = false;
       stream.on('error', (error) => {
         streamError = error;
       });
@@ -21,31 +66,27 @@ export function createNodePlatform() {
           if (streamError) {
             throw streamError;
           }
-          await new Promise((resolve, reject) => {
-            const ok = stream.write(`${line}\n`);
-            if (ok) {
-              resolve();
-              return;
+          await waitForStreamEvent(
+            stream,
+            'drain',
+            () => {
+              const prefix = !wroteBom && resolvedEncodingSettings.includeBom ? '\uFEFF' : '';
+              const ok = stream.write(`${prefix}${String(line ?? '')}${lineSuffix}`);
+              wroteBom = true;
+              return ok;
+            },
+            {
+              shouldResolveImmediately(result) {
+                return result !== false;
+              },
             }
-            stream.once('drain', () => {
-              if (streamError) {
-                reject(streamError);
-                return;
-              }
-              resolve();
-            });
-          });
+          );
         },
         async close() {
           if (streamError) {
             throw streamError;
           }
-          await new Promise((resolve, reject) => {
-            stream.end(() => resolve());
-            if (streamError) {
-              reject(streamError);
-            }
-          });
+          await waitForStreamEvent(stream, 'finish', () => stream.end());
         },
       };
     },

apps/cli/src/run-cli.js

@@ -1,6 +1,7 @@
 import {
   amendFromTextSpecAndData,
   generateFromTextSpec,
+  resolveExportTextEncodingSettings,
   streamFromTextSpec,
   SUPPORTED_FORMATS,
 } from '@anywaydata/core';
@@ -25,6 +26,13 @@ function formatCanonicalErrors(errors = []) {
 }
 
 export async function runCliCommand({ options, platform }) {
+  const outputEncodingSettings = resolveExportTextEncodingSettings(
+    {
+      lineEnding: options.lineEndings,
+      includeBom: options.bom === true,
+    },
+    { platform: platform.getRuntimePlatform?.() }
+  );
   const progress = (message) => {
     if (options.showProgress) {
       writeLine(platform.stdout, message);
@@ -102,7 +110,7 @@ export async function runCliCommand({ options, platform }) {
       }
     }
     if (options.outputFile) {
-      await platform.writeText(options.outputFile, result.rendered);
+      await platform.writeText(options.outputFile, result.rendered, outputEncodingSettings);
       progress(`> Writing output to ${options.outputFile}`);
     } else {
       platform.stdout(`${result.rendered}\n`);
@@ -122,7 +130,7 @@ export async function runCliCommand({ options, platform }) {
 
   if (useStreamMode && ['csv', 'jsonl', 'dsv', 'json', 'xml'].includes(outputFormat)) {
     const streamedLines = [];
-    const writer = options.outputFile ? platform.createLineWriter(options.outputFile) : null;
+    const writer = options.outputFile ? platform.createLineWriter(options.outputFile, outputEncodingSettings) : null;
     let writerClosed = false;
     try {
       const streamResult = await streamFromTextSpec({
@@ -206,7 +214,7 @@ export async function runCliCommand({ options, platform }) {
   }
 
   if (options.outputFile) {
-    await platform.writeText(options.outputFile, result.rendered);
+    await platform.writeText(options.outputFile, result.rendered, outputEncodingSettings);
     progress(`> Writing output to ${options.outputFile}`);
   } else {
     platform.stdout(`${result.rendered}\n`);

apps/cli/src/tests/cli-options.test.js

@@ -64,3 +64,21 @@ test('amend command options are parsed', () => {
   expect(opts.rowCount).toBe(3);
   expect(opts.shouldStream).toBe(false);
 });
+
+test('export encoding flags are parsed', () => {
+  const opts = parseCliOptions([
+    'node',
+    'cli',
+    'generate',
+    '-i',
+    'spec.txt',
+    '-n',
+    '3',
+    '--line-endings',
+    'crlf',
+    '--bom',
+  ]);
+
+  expect(opts.lineEndings).toBe('crlf');
+  expect(opts.bom).toBe(true);
+});

apps/cli/src/tests/integration.cli-output.test.js

@@ -8,6 +8,7 @@ const thisFilePath = fileURLToPath(import.meta.url);
 const thisDir = path.dirname(thisFilePath);
 const repoRoot = path.resolve(thisDir, '../../../../');
 const cliEntry = path.join(repoRoot, 'apps', 'cli', 'src', 'node-entry.js');
+const utf8BomBytes = [0xef, 0xbb, 0xbf];
 
 function runCli(args) {
   return spawnSync('node', [cliEntry, 'generate', ...args], {
@@ -16,6 +17,23 @@ function runCli(args) {
   });
 }
 
+async function withTempOutputPath(name, run) {
+  const outputPath = path.join(os.tmpdir(), `anywaydata-cli-${name}-${Date.now()}.csv`);
+  try {
+    await run(outputPath);
+  } finally {
+    try {
+      await fs.unlink(outputPath);
+    } catch {
+      // Ignore missing file cleanup races.
+    }
+  }
+}
+
+function hasUtf8Bom(buffer) {
+  return utf8BomBytes.every((byte, index) => buffer[index] === byte);
+}
+
 test('integration: writes generated content to stdout when no output file is provided', () => {
   const inputPath = path.join(repoRoot, 'apps', 'cli', 'examples', 'company-literal.txt');
   const result = runCli(['-i', inputPath, '-n', '2', '-f', 'csv']);
@@ -50,3 +68,49 @@ test('integration: writes generated content to file and keeps stdout progress-on
     }
   }
 });
+
+test('integration: writes a UTF-8 BOM when --bom is true', async () => {
+  const inputPath = path.join(repoRoot, 'apps', 'cli', 'examples', 'company-literal.txt');
+  await withTempOutputPath('bom-only', async (outputPath) => {
+    const result = runCli(['-i', inputPath, '-n', '2', '-f', 'csv', '-o', outputPath, '--bom']);
+
+    expect(result.status).toBe(0);
+    const writtenBuffer = await fs.readFile(outputPath);
+    expect(hasUtf8Bom(writtenBuffer)).toBe(true);
+  });
+});
+
+test('integration: does not write a UTF-8 BOM by default', async () => {
+  const inputPath = path.join(repoRoot, 'apps', 'cli', 'examples', 'company-literal.txt');
+  await withTempOutputPath('default-no-bom', async (outputPath) => {
+    const result = runCli(['-i', inputPath, '-n', '2', '-f', 'csv', '-o', outputPath]);
+
+    expect(result.status).toBe(0);
+    const writtenBuffer = await fs.readFile(outputPath);
+    expect(hasUtf8Bom(writtenBuffer)).toBe(false);
+  });
+});
+
+test('integration: writes CRLF line endings only when requested', async () => {
+  const inputPath = path.join(repoRoot, 'apps', 'cli', 'examples', 'company-literal.txt');
+  await withTempOutputPath('crlf-only', async (outputPath) => {
+    const result = runCli(['-i', inputPath, '-n', '2', '-f', 'csv', '-o', outputPath, '--line-endings', 'crlf']);
+
+    expect(result.status).toBe(0);
+    const written = await fs.readFile(outputPath, 'utf8');
+    expect(written).toContain('\r\n');
+    expect(written.replace(/\r\n/g, '')).not.toContain('\n');
+  });
+});
+
+test('integration: writes LF line endings only when requested', async () => {
+  const inputPath = path.join(repoRoot, 'apps', 'cli', 'examples', 'company-literal.txt');
+  await withTempOutputPath('lf-only', async (outputPath) => {
+    const result = runCli(['-i', inputPath, '-n', '2', '-f', 'csv', '-o', outputPath, '--line-endings', 'lf']);
+
+    expect(result.status).toBe(0);
+    const written = await fs.readFile(outputPath, 'utf8');
+    expect(written).toContain('\n');
+    expect(written).not.toContain('\r\n');
+  });
+});