Add shared schema acceptance tests (#224)

* Add shared schema acceptance tests

* Derive schema acceptance headers from output

* Clarify CLI schema acceptance parse failures

* Move API acceptance describe config first

* Handle MCP schema acceptance RPC errors

Author: eviltester

apps/api/src/tests/fromschema/schema-acceptance.spec.js

@@ -0,0 +1,38 @@
+import { test, expect } from '@playwright/test';
+import { createRequire } from 'node:module';
+import { setupApiServer, teardownApiServer, apiUrl } from '../api-test-setup.js';
+
+const require = createRequire(import.meta.url);
+const {
+  SCHEMA_ACCEPTANCE_SCENARIOS,
+} = require('../../../../../tests/integration/support/schema-acceptance-fixtures.cjs');
+const { normalizeApiBody } = require('../../../../../tests/integration/support/schema-acceptance-assertions.cjs');
+
+test.describe('POST /v1/generate/fromschema schema acceptance', () => {
+  test.describe.configure({ mode: 'serial' });
+
+  test.beforeAll(async () => {
+    await setupApiServer();
+  });
+
+  test.afterAll(async () => {
+    await teardownApiServer();
+  });
+
+  for (const scenario of SCHEMA_ACCEPTANCE_SCENARIOS) {
+    test(`${scenario.id} matches shared acceptance criteria`, async ({ request }) => {
+      const response = await request.post(
+        apiUrl(`/v1/generate/fromschema?rowCount=${scenario.rowCount}&outputFormat=${scenario.outputFormat}`),
+        {
+          data: scenario.schemaText,
+          headers: { 'content-type': 'text/plain' },
+        }
+      );
+
+      expect([200, 400]).toContain(response.status());
+      const body = await response.json();
+      const normalized = normalizeApiBody(body);
+      scenario.assertAcceptance(expect, normalized);
+    });
+  }
+});

apps/web/src/tests/browser/app/functional/test-data/schema-acceptance.spec.js

@@ -0,0 +1,53 @@
+const { test } = require('@playwright/test');
+const { openApp, expectNoPageErrors, expect } = require('../../abstractions/helpers/scenario-helpers');
+const {
+  SCHEMA_ACCEPTANCE_SCENARIOS,
+} = require('../../../../../../../../tests/integration/support/schema-acceptance-fixtures.cjs');
+const {
+  normalizeUiGridResult,
+  normalizeUiErrorText,
+} = require('../../../../../../../../tests/integration/support/schema-acceptance-assertions.cjs');
+
+async function readGridRows(appPage, headers) {
+  const rowCount = await appPage.gridEditor.renderer.countRows();
+  const columns = await Promise.all(headers.map((header) => appPage.gridEditor.renderer.getColumnTextsByName(header)));
+  const rows = [];
+  for (let rowIndex = 0; rowIndex < rowCount; rowIndex += 1) {
+    rows.push(columns.map((column) => column[rowIndex]));
+  }
+  return rows;
+}
+
+test.describe('App test-data schema acceptance', () => {
+  for (const scenario of SCHEMA_ACCEPTANCE_SCENARIOS) {
+    test(`${scenario.id} matches shared acceptance criteria`, async ({ page }) => {
+      const { appPage, pageErrors } = await openApp(page);
+
+      await appPage.testDataPanel.expand();
+      await appPage.testDataPanel.expectExpanded();
+      await appPage.testDataPanel.setSchemaText(scenario.schemaText);
+
+      if (scenario.kind === 'good') {
+        await appPage.testDataPanel.setGenerateCount(scenario.rowCount);
+        await appPage.testDataPanel.clickGenerate();
+
+        await expect.poll(async () => appPage.gridEditor.renderer.countRows()).toBe(scenario.rowCount);
+        await expect.poll(async () => appPage.gridEditor.header.getColumnNames()).toEqual(scenario.expectedHeaders);
+
+        const rows = await readGridRows(appPage, scenario.expectedHeaders);
+        const normalized = normalizeUiGridResult({
+          headers: scenario.expectedHeaders,
+          rows,
+        });
+        scenario.assertAcceptance(expect, normalized);
+      } else {
+        await appPage.testDataPanel.clickGenerate();
+        await expect.poll(async () => await appPage.testDataPanel.getSchemaErrorText()).not.toBe('');
+        const normalized = normalizeUiErrorText(await appPage.testDataPanel.getSchemaErrorText());
+        scenario.assertAcceptance(expect, normalized);
+      }
+
+      expectNoPageErrors(pageErrors);
+    });
+  }
+});

apps/web/src/tests/browser/generator/functional/schema-acceptance.spec.js

@@ -0,0 +1,44 @@
+const fs = require('node:fs');
+const { test } = require('@playwright/test');
+const { openGenerator, expectNoPageErrors, expect } = require('../abstractions/helpers/scenario-helpers');
+const {
+  SCHEMA_ACCEPTANCE_SCENARIOS,
+} = require('../../../../../../../tests/integration/support/schema-acceptance-fixtures.cjs');
+const {
+  normalizeUiDownloadedJson,
+  normalizeUiErrorText,
+} = require('../../../../../../../tests/integration/support/schema-acceptance-assertions.cjs');
+
+async function downloadGeneratedDataText(generatorPage) {
+  const download = await generatorPage.downloadGeneratedData();
+  const filePath = await download.path();
+  return fs.readFileSync(filePath, 'utf8');
+}
+
+test.describe('Generator schema acceptance', () => {
+  for (const scenario of SCHEMA_ACCEPTANCE_SCENARIOS) {
+    test(`${scenario.id} matches shared acceptance criteria`, async ({ page }) => {
+      const { generatorPage, pageErrors } = await openGenerator(page);
+
+      await generatorPage.schema.setSchemaText(scenario.schemaText);
+
+      if (scenario.kind === 'good') {
+        await generatorPage.generateOptions.setRowsCount(scenario.rowCount);
+        await generatorPage.generateOptions.setOutputFormat(scenario.outputFormat);
+
+        const outputText = await downloadGeneratedDataText(generatorPage);
+        const normalized = normalizeUiDownloadedJson(outputText, scenario.expectedHeaders);
+        scenario.assertAcceptance(expect, normalized);
+      } else {
+        await generatorPage.preview.clickPreview();
+        await expect
+          .poll(async () => (await generatorPage.schema.errorStatus.textContent())?.trim() || '')
+          .not.toBe('');
+        const normalized = normalizeUiErrorText((await generatorPage.schema.errorStatus.textContent())?.trim() || '');
+        scenario.assertAcceptance(expect, normalized);
+      }
+
+      expectNoPageErrors(pageErrors);
+    });
+  }
+});

test-fixtures/schema-acceptance/bad/invalid-literal-constraint.schema.txt

@@ -0,0 +1,6 @@
+# Bad schema: the constraint contradicts the literal value.
+# All surfaces should reject this during validation/generation.
+Status
+literal(closed)
+
+IF [Status] = "closed" THEN [Status] = "open" ENDIF

test-fixtures/schema-acceptance/bad/missing-definition.schema.txt

@@ -0,0 +1,4 @@
+# Bad schema: the field name is present but the rule definition is missing.
+# All surfaces should reject this and surface a "requires a data definition"
+# style validation error.
+OnlyName

test-fixtures/schema-acceptance/good/autoincrement-sequence.schema.txt

@@ -0,0 +1,4 @@
+# Good schema: autoIncrement.sequence should generate a deterministic
+# sequence when start, step, prefix, suffix, and zero padding are fixed.
+Ticket
+autoIncrement.sequence(start=1, step=5, prefix="T-", suffix=".txt", zeropadding=3)

test-fixtures/schema-acceptance/good/autoincrement-timestamp.schema.txt

@@ -0,0 +1,4 @@
+# Good schema: autoIncrement.timestamp should generate deterministic
+# timestamps when the start time, step, type, and output format are fixed.
+Created At
+autoIncrement.timestamp(start="2026-06-12T12:39:23Z", step=2, type="days", outputFormat="yyyy-MM-dd")

test-fixtures/schema-acceptance/good/comments-and-enum.schema.txt

@@ -0,0 +1,9 @@
+# Good schema: comments and blank lines should be ignored consistently.
+# Priority is intentionally enum-based so acceptance checks validate
+# allowed values instead of one exact row ordering.
+Priority
+enum(high,medium,low)
+
+# This literal column gives one stable value across all rows.
+Status
+literal(active)

test-fixtures/schema-acceptance/good/deterministic-literals.schema.txt

@@ -0,0 +1,7 @@
+# Good schema: deterministic literals so every surface should generate
+# the same headers, row count, and values.
+First Name
+literal(Alice)
+
+Status
+literal(active)

tests/integration/cross-surface-schema-acceptance.test.js

@@ -0,0 +1,136 @@
+import { execFileSync } from 'node:child_process';
+import { createRequire } from 'node:module';
+import path from 'node:path';
+
+const CROSS_SURFACE_PROCESS_TIMEOUT_MS = 60000;
+
+const require = createRequire(import.meta.url);
+const {
+  repoRoot,
+  SCHEMA_ACCEPTANCE_SCENARIOS,
+} = require('./support/schema-acceptance-fixtures.cjs');
+const {
+  normalizeCliSuccess,
+  normalizeCliFailure,
+  normalizeMcpPayload,
+} = require('./support/schema-acceptance-assertions.cjs');
+
+function jsonRpcMessages(output) {
+  return output
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter((line) => line.startsWith('{'))
+    .map((line) => {
+      try {
+        return JSON.parse(line);
+      } catch {
+        return null;
+      }
+    })
+    .filter(Boolean);
+}
+
+function requestMcpServer(payload) {
+  const scriptPath = path.join(repoRoot, 'apps', 'mcp', 'src', 'index.js');
+  const output = execFileSync(process.execPath, [scriptPath], {
+    input: `${JSON.stringify(payload)}\n`,
+    encoding: 'utf8',
+    cwd: repoRoot,
+    // Full-suite runs can be CPU-bound; keep the cross-surface contract stable under load.
+    timeout: CROSS_SURFACE_PROCESS_TIMEOUT_MS,
+  });
+  const messages = jsonRpcMessages(output);
+  const response = messages.find((message) => message?.id === payload.id);
+  expect(response).toBeTruthy();
+  return response;
+}
+
+function parseMcpScenarioPayload(response, scenarioId) {
+  if (response?.error) {
+    const code = response.error.code ?? 'unknown';
+    const message = response.error.message || JSON.stringify(response.error);
+    throw new Error(
+      `MCP JSON-RPC error for schema acceptance scenario "${scenarioId}" [${code}]: ${message}`
+    );
+  }
+
+  const text = response?.result?.content?.[0]?.text;
+  if (typeof text !== 'string' || text.trim() === '') {
+    throw new Error(
+      `MCP returned no result payload for schema acceptance scenario "${scenarioId}".`
+    );
+  }
+
+  try {
+    return JSON.parse(text);
+  } catch (error) {
+    throw new Error(
+      `MCP returned non-JSON payload for schema acceptance scenario "${scenarioId}": ${error.message}`
+    );
+  }
+}
+
+function runCliScenario(scenario) {
+  const cliEntry = path.join(repoRoot, 'apps', 'cli', 'src', 'node-entry.js');
+
+  let stdout;
+  try {
+    stdout = execFileSync(
+      process.execPath,
+      [
+        cliEntry,
+        'generate',
+        '-i',
+        scenario.schemaPath,
+        '-n',
+        String(scenario.rowCount),
+        '-f',
+        scenario.outputFormat,
+        '--show-progress',
+        'false',
+      ],
+      {
+        cwd: repoRoot,
+        encoding: 'utf8',
+        // Full-suite runs can be CPU-bound; keep the cross-surface contract stable under load.
+        timeout: CROSS_SURFACE_PROCESS_TIMEOUT_MS,
+      }
+    );
+  } catch (error) {
+    return normalizeCliFailure(error.stderr || error.stdout || error.message);
+  }
+
+  try {
+    return normalizeCliSuccess(stdout, scenario.expectedHeaders);
+  } catch (error) {
+    throw new Error(
+      `CLI returned non-JSON output for schema acceptance scenario "${scenario.id}": ${error.message}`
+    );
+  }
+}
+
+describe('cross-surface schema acceptance (CLI + MCP)', () => {
+  test.each(SCHEMA_ACCEPTANCE_SCENARIOS)('$id CLI matches shared acceptance criteria', (scenario) => {
+    const normalized = runCliScenario(scenario);
+    scenario.assertAcceptance(expect, normalized);
+  });
+
+  test.each(SCHEMA_ACCEPTANCE_SCENARIOS)('$id MCP matches shared acceptance criteria', (scenario) => {
+    const response = requestMcpServer({
+      jsonrpc: '2.0',
+      id: `schema-acceptance-${scenario.id}`,
+      method: 'tools/call',
+      params: {
+        name: 'generate_data_from_spec',
+        arguments: {
+          textSpec: scenario.schemaText,
+          rowCount: scenario.rowCount,
+          outputFormat: scenario.outputFormat,
+        },
+      },
+    });
+    const payload = parseMcpScenarioPayload(response, scenario.id);
+    const normalized = normalizeMcpPayload(payload);
+    scenario.assertAcceptance(expect, normalized);
+  });
+});