Merge pull request #180 from eviltester/codex/import-trim-options

[codex] add import trim options

Author: eviltester

README.md

@@ -298,11 +298,17 @@ Supported options:
 - `-o, --outputfile` write output to file instead of stdout
 - `-t, --testMode` enable diagnostics mode and generate one row
 - `--unsafe-faker-expressions` allow expression-style faker args (disabled by default)
+- `--trim-input` trim whitespace from every imported field value before amend processing
+- `--trim-input-fields` comma-separated imported field names to trim before amend processing
 
 Amend existing data with a schema:
 
 `anywaydata amend --schema-file schema.txt --data-file input.csv --input-format csv -f json -o amended.json`
 
+Trim imported input values during amend:
+
+`anywaydata amend --schema-file schema.txt --data-file input.csv --input-format csv --trim-input-fields Name,Email -f json`
+
 Pairwise note:
 
 - when `--pairwise` is enabled, `--numberOfLines` is accepted but ignored (the pairwise engine determines row count)
@@ -370,6 +376,7 @@ Amend imported data endpoint:
 `POST http://localhost:3000/v1/generate/amend`
 
 - JSON body fields: `textSpec` (required), `inputData` (required raw text), `inputFormat` (required), `rowCount` (optional, defaults to input row count), `outputFormat` (optional), `responseFormat` (optional), `stream` (optional and ignored)
+- amend trim controls: `trimInput` trims all imported field values; `trimInputFieldsCsv` trims only the listed imported field names
 - `rowCount` must be `<=` imported row count
 - response returns the full resulting dataset after amendment
 
@@ -499,6 +506,8 @@ Inputs:
 - `outputFormat` (required string e.g. `csv`, `json`, `jsonl`, `xml`, `sql`)
 - `options` (optional object)
 - `seed` (optional number)
+- `trimInput` (optional boolean for `amend_data_from_spec`)
+- `trimInputFieldsCsv` (optional string for `amend_data_from_spec`)
 
 Discoverability support:
 

apps/api/src/api-service.js

@@ -211,6 +211,8 @@ export function createApiService({
       outputFormat = 'csv',
       options,
       seed,
+      trimInput,
+      trimInputFieldsCsv,
       unsafeFakerExpressions,
       stream,
     } = payload;
@@ -242,6 +244,8 @@ export function createApiService({
         outputFormat: concreteOutputFormat,
         options: effectiveOptions,
         seed: parsedSeed.seed,
+        trimInput: parseBooleanFlag(trimInput),
+        trimInputFieldsCsv,
         unsafeFakerExpressions: unsafeFakerExpressions || false,
         stream,
       });

apps/api/src/openapi.js

@@ -418,6 +418,16 @@ const openApiDocument = {
                   outputFormat: { type: 'string', default: 'csv' },
                   options: { type: 'object' },
                   seed: { type: 'number' },
+                  trimInput: {
+                    type: 'boolean',
+                    default: false,
+                    description: 'Trim whitespace from every imported input field value before amend processing.',
+                  },
+                  trimInputFieldsCsv: {
+                    type: 'string',
+                    description:
+                      'Comma-separated imported field names whose values should be trimmed before amend processing.',
+                  },
                   unsafeFakerExpressions: { type: 'boolean', default: false },
                   responseFormat: { type: 'string', enum: ['rows', 'rendered', 'all', 'raw'], default: 'rows' },
                   stream: { type: 'boolean', description: 'Accepted for compatibility and ignored for amend mode.' },

apps/api/src/tests/generate/v1-generate-amend.spec.js

@@ -41,4 +41,21 @@ test.describe('POST /v1/generate/amend', () => {
     expect(Array.isArray(body.diagnostics)).toBe(false);
     expect(typeof body.diagnostics).toBe('object');
   });
+
+  test('amend trimInputFieldsCsv trims only listed imported columns', async ({ request }) => {
+    const response = await request.post(apiUrl('/v1/generate/amend'), {
+      data: {
+        textSpec: 'Status\nActive',
+        inputData: '"Name","Role"\n"  Alice  ","  Engineer  "',
+        inputFormat: 'csv',
+        rowCount: 0,
+        outputFormat: 'json',
+        trimInputFieldsCsv: 'Name',
+      },
+    });
+
+    expect(response.status()).toBe(200);
+    const body = await response.json();
+    expect(body.rows).toEqual([['Alice', '  Engineer  ', '']]);
+  });
 });

apps/api/src/tests/health/health-endpoints.spec.js

@@ -24,6 +24,12 @@ test.describe('Health and docs endpoints', () => {
     expect(body.openapi).toBe('3.0.3');
     expect(body.paths['/v1/generate']).toBeTruthy();
     expect(body.paths['/v1/generate/amend']).toBeTruthy();
+    expect(
+      body.paths['/v1/generate/amend'].post.requestBody.content['application/json'].schema.properties.trimInput
+    ).toBeTruthy();
+    expect(
+      body.paths['/v1/generate/amend'].post.requestBody.content['application/json'].schema.properties.trimInputFieldsCsv
+    ).toBeTruthy();
   });
 
   test('GET /v1/docs serves Swagger UI html', async ({ request }) => {

apps/cli/src/cli-options.js

@@ -11,6 +11,15 @@ export function parseCliOptions(argvInput = process.argv) {
     .option('schema-file', { type: 'string', describe: 'Schema file path for amend command' })
     .option('data-file', { type: 'string', describe: 'Input data file path for amend command' })
     .option('input-format', { type: 'string', describe: 'Input data format for amend command (e.g. csv, json)' })
+    .option('trim-input', {
+      type: 'boolean',
+      default: false,
+      describe: 'Trim whitespace from every imported input field value before amend processing',
+    })
+    .option('trim-input-fields', {
+      type: 'string',
+      describe: 'Comma-separated imported field names whose values should be trimmed before amend processing',
+    })
     .option('n', {
       alias: 'numberOfLines',
       type: 'number',
@@ -93,5 +102,7 @@ export function parseCliOptions(argvInput = process.argv) {
     bom: parsed.bom === true,
     unsafeFakerExpressions: parsed['unsafe-faker-expressions'] === true,
     pairwise: parsed.pairwise === true,
+    trimInput: command === 'amend' && parsed['trim-input'] === true,
+    trimInputFieldsCsv: command === 'amend' ? parsed['trim-input-fields'] || '' : '',
   };
 }

apps/cli/src/run-cli.js

@@ -95,6 +95,8 @@ export async function runCliCommand({ options, platform }) {
       rowCount: options.rowCount,
       outputFormat,
       options: formatterOptions,
+      trimInput: options.trimInput === true,
+      trimInputFieldsCsv: options.trimInputFieldsCsv || '',
       unsafeFakerExpressions: options.unsafeFakerExpressions,
       stream: options.shouldStream,
     });

apps/cli/src/tests/cli-options.test.js

@@ -52,6 +52,9 @@ test('amend command options are parsed', () => {
     'data.csv',
     '--input-format',
     'csv',
+    '--trim-input',
+    '--trim-input-fields',
+    'Name,Role',
     '-n',
     '3',
     '-f',
@@ -63,6 +66,8 @@ test('amend command options are parsed', () => {
   expect(opts.inputFormat).toBe('csv');
   expect(opts.rowCount).toBe(3);
   expect(opts.shouldStream).toBe(false);
+  expect(opts.trimInput).toBe(true);
+  expect(opts.trimInputFieldsCsv).toBe('Name,Role');
 });
 
 test('export encoding flags are parsed', () => {

apps/cli/src/tests/integration.cli-params.test.js

@@ -205,6 +205,33 @@ test('amend command applies schema to existing data', async () => {
   expect(JSON.parse(result.stdout.trim())).toEqual([{ Name: 'Bob' }, { Name: 'Eve' }]);
 });
 
+test('amend command trim flags affect imported input values before amend processing', async () => {
+  const schemaPath = tempFile('trim-schema');
+  const dataPath = tempFile('trim-data');
+  await fs.writeFile(schemaPath, 'Status\nActive', 'utf8');
+  await fs.writeFile(dataPath, '"Name","Role"\n"  Alice  ","  Engineer  "', 'utf8');
+
+  const result = runCli([
+    'amend',
+    '--schema-file',
+    schemaPath,
+    '--data-file',
+    dataPath,
+    '--input-format',
+    'csv',
+    '--trim-input-fields',
+    'Name',
+    '-n',
+    '0',
+    '-f',
+    'json',
+    '--show-progress',
+    'false',
+  ]);
+  expect(result.status).toBe(0);
+  expect(JSON.parse(result.stdout.trim())).toEqual([{ Name: 'Alice', Role: '  Engineer  ', Status: '' }]);
+});
+
 test('amend command fixture flow: CSV input to DSV output on stdout', async () => {
   const schemaPath = path.join(amendFixturesDir, 'schema.txt');
   const dataPath = path.join(amendFixturesDir, 'input.csv');

apps/mcp/src/index.js

@@ -328,6 +328,15 @@ function handleRequest(request) {
                 outputFormat: { type: 'string', enum: SUPPORTED_FORMATS },
                 options: GENERATE_OPTIONS_SCHEMA,
                 seed: { type: 'number' },
+                trimInput: {
+                  type: 'boolean',
+                  description: 'Trim whitespace from every imported input field value before amend processing.',
+                },
+                trimInputFieldsCsv: {
+                  type: 'string',
+                  description:
+                    'Comma-separated imported field names whose values should be trimmed before amend processing.',
+                },
                 stream: {
                   type: 'boolean',
                   description: 'Accepted for compatibility and ignored for amend operation (always buffered).',