Merge pull request #75 from eviltester/48-allow-comments-in-schema

comments and spaces in schema

Author: eviltester

README.md

@@ -64,9 +64,19 @@ You can then create a new table, or amend the existing table or selected rows.
 
 The spec is a paragraph of text where each line is either a 'name' or a 'rule':
 
+### Schema Formatting
+
+- **Comments**: lines starting with `#` (optionally prefixed by whitespace) are treated as comments.
+- **Blank lines**: blank lines are allowed and ignored, so you can separate column groups for readability.
+- **Column definitions**: each column is defined as `name` followed by `rule` on the next logical content line.
+
 ```
+# optional comment
+
 name
 rule
+
+# another comment
 name
 rule
 ```
@@ -102,12 +112,19 @@ The `fake` method is also supported, which takes a mustache template style strin
 So a sample test data spec might look like:
 
 ```
+# person details
 name
 helpers.fake("{{name.lastName}}, {{name.firstName}}")
+
+# profile text
 desc
 faker.lorem.paragraph
+
+# preference data
 collects
 hacker.noun
+
+# regex example
 prefers
 (Connie|Bob)
 ```
@@ -119,10 +136,15 @@ When you have 2 or more enum fields (comma-separated values), you can generate p
 For enum data, use comma-separated values in your spec:
 
 ```
+# pairwise parameters
 browser
 chrome,firefox,safari,edge
+
+# viewport class
 device
 desktop,tablet,mobile
+
+# style variant
 theme
 light,dark
 ```
@@ -337,7 +359,7 @@ Example request body:
 
 ```json
 {
-  "textSpec": "Name\nBob",
+  "textSpec": "# literal example\\n\\nName\\nBob",
   "rowCount": 3,
   "outputFormat": "json"
 }
@@ -368,7 +390,7 @@ Add `unsafeFakerExpressions: true` to individual requests:
 
 ```json
 {
-  "textSpec": "Name\nperson.firstName\nScore\nnumber.int({\"min\": 18, \"max\": 65})",
+  "textSpec": "# faker + numeric range\\n\\nName\\nperson.firstName\\n\\nScore\\nnumber.int({\"min\": 18, \"max\": 65})",
   "rowCount": 5,
   "outputFormat": "json",
   "unsafeFakerExpressions": true

apps/api/src/fromschema.route.test.js

@@ -176,3 +176,16 @@ test('/v1/generate/fromschema supports pairwise query flag', async () => {
     ['Safari', 'Dark'],
   ]);
 });
+
+test('/v1/generate/fromschema accepts comments and blank lines in schema text', async () => {
+  const response = await fetch(url('/v1/generate/fromschema?rowCount=2&outputFormat=json'), {
+    method: 'POST',
+    headers: { 'content-type': 'text/plain' },
+    body: '# skip me\n\nPriority\nenum(high,medium,low)\n\n# and me\nStatus\nenum(active,inactive,pending)',
+  });
+
+  expect(response.status).toBe(200);
+  const body = await response.json();
+  expect(body.headers).toEqual(['Priority', 'Status']);
+  expect(body.rows).toHaveLength(2);
+});

apps/cli/src/tests/run-cli.test.js

@@ -177,3 +177,27 @@ test('generates deterministic pairwise output in buffered mode', async () => {
     { Browser: 'Safari', Theme: 'Dark' },
   ]);
 });
+
+test('supports comments and blank lines in input schema', async () => {
+  const platform = makePlatform({
+    textSpec: '# comment\n\nPriority\nenum(high,medium,low)\n\nStatus\nenum(active,inactive,pending)',
+  });
+  const code = await runCliCommand({
+    platform,
+    options: {
+      inputFile: 'spec.txt',
+      outputFile: null,
+      format: 'json',
+      rowCount: 2,
+      testMode: false,
+      showProgress: false,
+      shouldStream: false,
+      unsafeFakerExpressions: false,
+      pairwise: false,
+    },
+  });
+  expect(code).toBe(0);
+  const parsed = JSON.parse(platform.out.join('').trim());
+  expect(parsed).toHaveLength(2);
+  expect(Object.keys(parsed[0])).toEqual(['Priority', 'Status']);
+});

apps/mcp/src/mcp.test.js

@@ -49,6 +49,26 @@ test('MCP server handles generate_data_from_spec tool call', () => {
   expect(response?.result?.structuredContent?.ok).toBe(true);
 });
 
+test('MCP server accepts comments and blank lines in textSpec', () => {
+  const response = requestServer({
+    jsonrpc: '2.0',
+    id: 201,
+    method: 'tools/call',
+    params: {
+      name: 'generate_data_from_spec',
+      arguments: {
+        textSpec: '# comment\n\nPriority\nenum(high,medium,low)\nStatus\nenum(active,inactive,pending)',
+        rowCount: 1,
+        outputFormat: 'json',
+      },
+    },
+  });
+  const payload = JSON.parse(response?.result?.content?.[0]?.text || '{}');
+  expect(payload.ok).toBe(true);
+  expect(payload.headers).toEqual(['Priority', 'Status']);
+  expect(payload.rows).toHaveLength(1);
+});
+
 test('MCP server supports pairwise generation', () => {
   const response = requestServer({
     jsonrpc: '2.0',

apps/web/src/tests/browser/planned-functional/test-data/text-schema.spec.js

@@ -9,18 +9,18 @@ test.describe('7. Test Data Generation', () => {
     await appPage.testDataPanel.expectExpanded();
     const beforeSchema = await appPage.testDataPanel.getSchemaRowCount();
 
-    await appPage.testDataPanel.addSchemaRow();
-    await expect.poll(async () => appPage.testDataPanel.getSchemaRowCount()).toBe(beforeSchema + 1);
-
-    const schemaRowIndex = beforeSchema;
-    await appPage.testDataPanel.setSchemaCell(schemaRowIndex, 'columnName', 'First Name');
-    await appPage.testDataPanel.setSchemaTypeValue(schemaRowIndex, 'faker');
-    await appPage.testDataPanel.setSchemaCell(schemaRowIndex, 'value', 'faker.person.firstName');
+    await appPage.testDataPanel.setSchemaText(
+      '# this comment should be ignored\n\nFirst Name\nperson.firstName\n\n# second comment\nStatus\nenum(active,inactive,pending)'
+    );
+    await expect.poll(async () => appPage.testDataPanel.getSchemaRowCount()).toBe(beforeSchema + 2);
     await appPage.testDataPanel.setGenerateCount(5);
 
     await appPage.testDataPanel.clickGenerate();
     await expect.poll(async () => appPage.gridEditor.renderer.countRows()).toBe(5);
     await expect.poll(async () => appPage.gridEditor.header.getColumnNames()).toContain('First Name');
+    await expect.poll(async () => appPage.gridEditor.header.getColumnNames()).toContain('Status');
+    // comments are not rendered as schema rows
+    expect(await appPage.testDataPanel.getSchemaRowCount()).toBe(beforeSchema + 2);
 
     const values = await appPage.gridEditor.renderer.getColumnTextsByName('First Name');
     expect(values.filter(Boolean).length).toBe(5);

docs-src/docs/040-test-data/040-literal-test-data.md

@@ -31,6 +31,7 @@ Schema text format uses:
 For a literal column:
 
 ```text
+# deployment context
 Environment
 UAT
 ```
@@ -42,10 +43,15 @@ If you generate 5 rows, every row in `Environment` will be `UAT`.
 You can define several static columns at once:
 
 ```text
+# locale defaults
 Country
 UK
+
+# currency defaults
 Currency
 GBP
+
+# workflow state
 Status
 ACTIVE
 ```
@@ -57,12 +63,19 @@ This is useful for creating baseline datasets quickly.
 Literal values are often combined with Faker and Regex columns.
 
 ```text
+# constant label
 Environment
 UAT
+
+# generated identity
 Customer Name
 person.fullName
+
+# generated order id
 Order Ref
 [A-Z]{3}-[0-9]{6}
+
+# constant flag
 Is Premium
 true
 ```

docs-src/docs/040-test-data/050-pairwise-testing.md

@@ -31,15 +31,28 @@ Pairwise testing is most effective when:
 
 ## How to Generate Pairwise Data
 
+## Schema Formatting
+
+Schema text supports:
+
+- **Comments**: lines starting with `#` (after optional leading whitespace) are treated as comments.
+- **Blank lines**: allowed and ignored, useful for readability between column groups.
+- **Column definitions**: each column name must be followed by its generation rule on the next logical content line.
+
 ### In the Main App (app.html)
 
 1. **Open the Test Data section** by clicking the "Test Data" header
 2. **Define your enum parameters** in the Test Data Text Schema:
    ```
+   # categorical parameters
    color
    red,blue,green,yellow
+
+   # sizing options
    size  
    small,medium,large
+
+   # material choices
    material
    wood,metal
    ```
@@ -66,8 +79,11 @@ AnyWayData supports multiple formats for specifying enum values in pairwise test
 The basic format uses comma-separated values:
 
 ```
+# pairwise field 1
 Priority
 high,medium,low
+
+# pairwise field 2
 Status  
 active,inactive,pending
 ```
@@ -78,24 +94,33 @@ For more complex scenarios, you can use function-based formats:
 
 #### Basic enum() Function
 ```
+# function style enum
 Priority
 enum(high,medium,low)
+
+# another function style enum
 Status
 enum(active,inactive,pending)
 ```
 
 #### Datatype enum() Function  
 ```
+# datatype enum form
 Priority
 datatype.enum(high,medium,low)
+
+# second datatype enum
 Status
 datatype.enum(active,inactive,pending)
 ```
 
 #### Full AWD enum() Function
 ```
+# fully-qualified enum form
 Priority
 awd.datatype.enum(high,medium,low)
+
+# fully-qualified enum form
 Status
 awd.datatype.enum(active,inactive,pending)
 ```
@@ -105,10 +130,15 @@ awd.datatype.enum(active,inactive,pending)
 When your enum values contain commas or special characters, use quoted values:
 
 ```
+# values containing commas
 Product Category
 enum("Hardware, Electronics","Software, Applications","Books, Media")
+
+# geo values containing commas
 Location
 enum("New York, NY","Los Angeles, CA","Chicago, IL")
+
+# status text with commas
 Status Message
 enum("Ready, waiting for input","Processing, please wait","Error, check configuration")
 ```
@@ -118,8 +148,11 @@ enum("Ready, waiting for input","Processing, please wait","Error, check configur
 You can mix quoted and unquoted values in the same enum:
 
 ```
+# mixed quoting example
 Environment
 enum("Production, Live",staging,development,"Test, QA")
+
+# mixed quoting example
 User Type
 enum("Admin, Full Access",editor,viewer,"Guest, Limited")
 ```
@@ -136,10 +169,15 @@ Consider testing an API with both categorical parameters (that need pairwise cov
 - Then random values User ID, etc.
 
 ```
+# pairwise enums
 HTTP Method
 enum(GET,POST,PUT,DELETE)
+
+# pairwise enums
 Content Type  
 enum("application/json","application/xml","text/plain")
+
+# randomized fields
 User ID
 faker.number.int
 Email Address
@@ -200,8 +238,11 @@ AnyWayData uses a greedy set cover approximation algorithm that:
 Example Input Schema:
 
 ```csv
+# simple pairwise schema
 color
 red,blue,green,yellow
+
+# simple pairwise schema
 size  
 small,medium,large
 ```

docs-src/docs/070-interfaces-and-deployment/030-rest-api.md

@@ -137,6 +137,14 @@ Use endpoints at `http://localhost:8082`.
 
 Both endpoints generate data from the same schema language and output formats. The key difference is request shape and content type, not generation capability.
 
+## Schema Formatting
+
+Schema text supports:
+
+- **Comments**: lines starting with `#` (after optional leading whitespace) are treated as comments.
+- **Blank lines**: allowed and ignored, useful for readability between column groups.
+- **Column definitions**: each column name must be followed by its generation rule on the next logical content line.
+
 ## API Examples
 
 Health check:
@@ -151,7 +159,7 @@ Generate JSON output with a JSON payload:
 curl -X POST http://localhost:3000/v1/generate \
   -H "Content-Type: application/json" \
   -d '{
-    "textSpec": "Name\nBob\n\nCity\nlondon",
+    "textSpec": "# literal values\n\nName\nBob\n\n# lower-case city\nCity\nlondon",
     "rowCount": 3,
     "outputFormat": "json"
   }'
@@ -163,7 +171,7 @@ Generate CSV output with CSV-specific options:
 curl -X POST http://localhost:3000/v1/generate \
   -H "Content-Type: application/json" \
   -d '{
-    "textSpec": "Name\nBob",
+    "textSpec": "# basic csv schema\n\nName\nBob",
     "rowCount": 2,
     "outputFormat": "csv",
     "options": {
@@ -180,7 +188,7 @@ Generate using raw schema text (`fromschema`):
 ```bash
 curl -X POST "http://localhost:3000/v1/generate/fromschema?outputFormat=markdown&rowCount=2" \
   -H "Content-Type: text/plain" \
-  --data-binary $'Name\nBob\n\nId\n1'
+  --data-binary $'# markdown sample\n\nName\nBob\n\n# numeric id\nId\n1'
 ```
 
 Get current options for a format: