Add special case for to-app-state to convert a project to a v1 spec stucture

* Add validation to mock provisioner

* experiment with using Project to generate a spec file from state

* format

* restore tests

* little style tweak

* remove state.json

* handle credentials properly in spec

* update tests

* mock: handle deleted edges

* fix tests

* correct project credential name

* one more test for luck

* one more test fix and log removal

Author: josephjclark

.changeset/forty-areas-remain.md

@@ -0,0 +1,5 @@
+---
+'@openfn/lightning-mock': patch
+---
+
+Add validation to the provisioner endpoint

integration-tests/cli/test/deploy.test.ts

@@ -1,6 +1,6 @@
 import test from 'ava';
 import path from 'node:path';
-import fs from 'node:fs/promises';
+import fs, { rm } from 'node:fs/promises';
 import run from '../src/run';
 import createLightningServer from '@openfn/lightning-mock';
 import { extractLogs, assertLog } from '../src/util';
@@ -12,6 +12,50 @@ const port = 8967;
 const endpoint = `http://localhost:${port}`;
 let tmpDir = path.resolve('tmp/deploy');
 
+const testProjectV2 = `
+id: my-project
+name: My Project
+schema_version: '4.0'
+workflows:
+  - id: my-workflow
+    name: My Workflow
+    start: webhook
+    steps:
+      - id: webhook
+        type: webhook
+        enabled: true
+        next:
+          transform-data: {}
+      - id: transform-data
+        name: Transform data
+        expression: 'fn(s => s)'
+        adaptor: '@openfn/language-common@latest'
+`.trim();
+
+const testProjectV2WithCredential = `
+id: my-project
+name: My Project
+schema_version: '4.0'
+credentials:
+  - name: http1
+    owner: super@openfn.org
+workflows:
+  - id: my-workflow
+    name: My Workflow
+    start: webhook
+    steps:
+      - id: webhook
+        type: webhook
+        enabled: true
+        next:
+          transform-data: {}
+      - id: transform-data
+        name: Transform data
+        expression: 'fn(s => s)'
+        adaptor: '@openfn/language-common@latest'
+        configuration: super@openfn.org|http1
+`.trim();
+
 const testProject = `
 name: test-project
 workflows:
@@ -97,7 +141,6 @@ test.serial('deploy a local project', async (t) => {
       --log-json \
       -l debug`
   );
-
   t.falsy(stderr);
 
   const logs = extractLogs(stdout);
@@ -301,7 +344,6 @@ workflows:
       --log-json \
       -l debug`
   );
-
   t.falsy(stderr);
 
   const logs = extractLogs(stdout);
@@ -364,7 +406,6 @@ test.serial('deploy then pull, changes one workflow, deploy', async (t) => {
 
   // And deploy those changes
   const { stdout, stderr } = await run(deployCmd);
-
   t.falsy(stderr);
 
   const logs = extractLogs(stdout);
@@ -378,3 +419,63 @@ test.serial('deploy then pull, changes one workflow, deploy', async (t) => {
   t.is(Object.keys(server.state.projects).length, 1);
   t.truthy(server.state.projects[projectId]);
 });
+
+test.serial('deploy a v2 project.yaml', async (t) => {
+  await fs.writeFile(path.join(tmpDir, 'project.yaml'), testProjectV2);
+
+  const { stdout, stderr } = await run(
+    `openfn deploy \
+      --project-path ${tmpDir}/project.yaml \
+      --state-path ${tmpDir}/.state.json \
+      --no-confirm \
+      --log-json \
+      -l debug`
+  );
+
+  t.falsy(stderr);
+
+  const logs = extractLogs(stdout);
+  assertLog(t, logs, /Deployed/);
+
+  t.is(Object.keys(server.state.projects).length, 1);
+  const [project] = Object.values(server.state.projects) as any[];
+  t.is(project.name, 'My Project');
+});
+
+test.serial('deploy a new v2 project.yaml with credentials', async (t) => {
+  await fs.writeFile(
+    path.join(tmpDir, 'project.yaml'),
+    testProjectV2WithCredential
+  );
+
+  try {
+    await rm(`${tmpDir}/.state.json`);
+  } catch (e) {
+    // ignore
+  }
+
+  const { stdout, stderr } = await run(
+    `openfn deploy \
+      --project-path ${tmpDir}/project.yaml \
+      --no-confirm \
+      --log-json \
+      -l debug`
+  );
+
+  t.falsy(stderr);
+
+  const logs = extractLogs(stdout);
+  assertLog(t, logs, /Deployed/);
+
+  t.is(Object.keys(server.state.projects).length, 1);
+  const [project] = Object.values(server.state.projects) as any[];
+  t.is(project.name, 'My Project');
+
+  t.is(project.project_credentials[0].name, 'http1');
+  t.is(project.project_credentials[0].owner, 'super@openfn.org');
+
+  const uuid = project.project_credentials[0].id;
+
+  const workflow: any = Object.values(project.workflows).pop();
+  t.is(workflow.jobs[0].project_credential_id, uuid);
+});

packages/cli/src/deploy/handler.ts

@@ -150,7 +150,10 @@ export const maybeConvertV2spec = async (yaml: string): Promise<string> => {
   const json = yamlToJson(yaml) as any;
   if (detectVersion(json) > 1) {
     const project = await Project.from('project', json);
-    return project.serialize('state', { format: 'yaml' }) as string;
+    return project.serialize('state', {
+      format: 'yaml',
+      asSpec: true,
+    }) as string;
   }
   return yaml;
 };

packages/cli/test/deploy/deploy.test.ts

@@ -188,8 +188,6 @@ test.serial('catches DeployErrors', async (t) => {
   process.exitCode = origExitCode;
 });
 
-// maybeConvertV2spec
-
 const v1Yaml = `id: '1234'
 name: My Project
 workflows:
@@ -221,6 +219,9 @@ project_credentials: []
 const v2Yaml = `id: my-project
 name: My Project
 schema_version: '4.0'
+credentials:
+  - name: http1
+    owner: super@openfn.org
 workflows:
   - id: my-workflow
     name: My Workflow
@@ -235,14 +236,15 @@ workflows:
         name: Transform data
         expression: 'fn(s => s)'
         adaptor: '@openfn/language-common@latest'
+        configuration: super@openfn.org|http1
 `;
 
 test('maybeConvertV2spec: returns v1 yaml unchanged', async (t) => {
   const result = await maybeConvertV2spec(v1Yaml);
   t.is(result, v1Yaml);
 });
 
-test('maybeConvertV2spec: converts v2 (schema_version) to v1', async (t) => {
+test('maybeConvertV2spec: converts v2 to v1', async (t) => {
   const result = await maybeConvertV2spec(v2Yaml);
   const json = yamlToJson(result) as any;
 
@@ -256,10 +258,48 @@ test('maybeConvertV2spec: converts v2 (schema_version) to v1', async (t) => {
   t.falsy(workflow.steps);
   t.truthy(workflow.triggers);
 
+  // no uuids
+  const edge = workflow.edges['webhook->transform-data'];
+  t.is(edge.target_job, 'transform-data');
+  t.is(edge.source_trigger, 'webhook');
+  t.falsy(workflow.jobs['transform-data'].id);
+
   // no v2 marker
   t.falsy(json.schema_version);
 });
 
+test('maybeConvertV2spec: converts with credentials', async (t) => {
+  const result = await maybeConvertV2spec(v2Yaml);
+  const json = yamlToJson(result) as any;
+
+  t.deepEqual(json.credentials, {
+    'super@openfn.org|http1': { name: 'http1', owner: 'super@openfn.org' },
+  });
+
+  t.is(
+    json.workflows['my-workflow'].jobs['transform-data'].credential,
+    'super@openfn.org|http1'
+  );
+});
+
+test('maybeConvertV2spec: converted edges use key references, not UUIDs', async (t) => {
+  const result = await maybeConvertV2spec(v2Yaml);
+  const json = yamlToJson(result) as any;
+
+  const workflow = Object.values(json.workflows)[0] as any;
+  const edge = Object.values(workflow.edges)[0] as any;
+
+  // edge must use spec format (key references) so mergeSpecIntoState can resolve them
+  t.truthy(edge.source_trigger);
+  t.truthy(edge.target_job);
+  t.falsy(edge.source_trigger_id);
+  t.falsy(edge.target_job_id);
+
+  // source_trigger must match a trigger key; target_job must match a job key
+  t.truthy(workflow.triggers[edge.source_trigger]);
+  t.truthy(workflow.jobs[edge.target_job]);
+});
+
 test('maybeConvertV2spec: converts legacy v2 (cli.version: 2) to v1', async (t) => {
   const legacyV2Yaml = `id: my-project
 name: My Project

packages/cli/test/projects/deploy.test.ts

@@ -19,7 +19,6 @@ import {
   TWO_WORKFLOWS_UUID,
 } from './fixtures';
 import { checkout } from '../../src/projects';
-import { readFileSync } from 'node:fs';
 
 let server: any;
 const logger = createMockLogger(undefined, { level: 'debug' });

packages/deploy/src/index.ts

@@ -118,7 +118,6 @@ export async function deploy(config: DeployConfig, logger: Logger) {
     throw new DeployError(`${config.specPath} has errors`, 'VALIDATION_ERROR');
   }
   const nextState = mergeSpecIntoState(state, spec.doc);
-
   validateProjectState(nextState);
 
   // Convert the state to a payload for the API.

packages/lightning-mock/src/api-rest.ts

@@ -83,6 +83,45 @@ workflows:
         enabled: true
 `;
 
+// Validates a provisioner payload, returning an error body if invalid or null if valid.
+// Mirrors Lightning's error format so deploy code sees realistic rejection responses.
+export function validateProvisionPayload(
+  incoming: any
+): Record<string, any> | null {
+  const workflowErrors: Record<string, any> = {};
+
+  const wfList: any[] = Array.isArray(incoming.workflows)
+    ? incoming.workflows
+    : Object.values(incoming.workflows ?? {});
+
+  for (const wf of wfList) {
+    const edgeErrors: Record<string, any> = {};
+    const edgeList: any[] = Array.isArray(wf.edges)
+      ? wf.edges
+      : Object.values(wf.edges ?? {});
+
+    for (const edge of edgeList) {
+      if (!edge.delete && !edge.source_trigger_id && !edge.source_job_id) {
+        const key = edge.id ?? '->';
+        edgeErrors[key] = {
+          source_job_id: ['source_job_id or source_trigger_id must be present'],
+        };
+      }
+    }
+
+    if (Object.keys(edgeErrors).length > 0) {
+      const wfKey = wf.name ?? wf.id ?? 'unknown';
+      workflowErrors[wfKey] = { edges: edgeErrors };
+    }
+  }
+
+  if (Object.keys(workflowErrors).length > 0) {
+    return { errors: { workflows: workflowErrors } };
+  }
+
+  return null;
+}
+
 export default (
   app: DevServer,
   state: ServerState,
@@ -121,6 +160,14 @@ export default (
 
   router.post('/api/provision', (ctx) => {
     const incoming: any = ctx.request.body;
+
+    const validationErrors = validateProvisionPayload(incoming);
+    if (validationErrors) {
+      ctx.response.status = 422;
+      ctx.response.body = validationErrors;
+      return;
+    }
+
     const now = new Date().toISOString();
 
     if (!state.projects[incoming.id]) {