CLI: Deploy through v1 API with v2 spec (#1465)

* convert v2 spec down to v1

* refactor to make a version-sniffing util

* integrate the new feature

* test

* Add special case for  to-app-state to convert a project to a v1 spec stucture

* Add validation to mock provisioner

* experiment with using Project to generate a spec file from state

* format

* restore tests

* little style tweak

* remove state.json

* handle credentials properly in spec

* update tests

* mock: handle deleted edges

* fix tests

* correct project credential name

* one more test for luck

* one more test fix and log removal

Author: josephjclark

.changeset/forty-areas-remain.md

@@ -0,0 +1,5 @@
+---
+'@openfn/lightning-mock': patch
+---
+
+Add validation to the provisioner endpoint

.claude/yaml-formats.md

@@ -0,0 +1,96 @@
+# OpenFn Project YAML Formats
+
+Two YAML formats are used across the monorepo. The key distinction: **v1** uses objects keyed by ID; **v2** uses arrays.
+
+## v1 (Lightning app state)
+
+Used by `packages/deploy` and sent to/from the Lightning API (`Provisioner.Project` type from `@openfn/lexicon/lightning`).
+
+- `workflows` is a keyed object (`{ [slug]: Workflow }`)
+- Each workflow has `jobs`, `triggers`, and `edges` as keyed objects
+- Steps are called `jobs`; code is stored in `body`
+- Credentials referenced by UUID (`project_credential_id`)
+- No version marker — absence of `schema_version`/`cli.version` means v1
+
+```yaml
+id: abc-123
+name: My Project
+project_credentials:
+  - id: cred-uuid
+    name: My Credential
+    owner: admin@openfn.org
+workflows:
+  my-workflow:
+    id: wf-uuid
+    name: My Workflow
+    jobs:
+      transform-data:
+        id: job-uuid
+        name: Transform data
+        body: 'fn(s => s)'
+        adaptor: '@openfn/language-common@latest'
+        project_credential_id: cred-uuid
+        keychain_credential_id: null
+    triggers:
+      webhook:
+        id: trig-uuid
+        type: webhook
+        enabled: true
+    edges:
+      trigger->transform-data:
+        id: edge-uuid
+        enabled: true
+        source_trigger_id: trig-uuid
+        target_job_id: job-uuid
+```
+
+## v2 (local project state)
+
+Used by `packages/project` and the CLI project subcommands (`ProjectState` type from `@openfn/lexicon`).
+
+- Identified by `schema_version` field (current: `'4.0'`) or legacy `cli.version: 2`
+- `workflows` is an array
+- Each workflow has a `steps` array; triggers are steps with a `type` field
+- Code stored in `expression`; edges expressed inline via `next` map on each step
+- Credentials referenced by name string (`configuration`)
+
+```yaml
+id: my-project
+name: My Project
+schema_version: '4.0'
+credentials:
+  - uuid: cred-uuid
+    name: My Credential
+    owner: admin@openfn.org
+workflows:
+  - id: my-workflow
+    name: My Workflow
+    start: webhook
+    steps:
+      - id: webhook
+        type: webhook
+        enabled: true
+        next:
+          transform-data:
+            condition: always
+      - id: transform-data
+        name: Transform data
+        expression: 'fn(s => s)'
+        adaptor: '@openfn/language-common@latest'
+        configuration: 'admin@openfn.org|My Credential'
+```
+
+## Detection logic
+
+Use `detectVersion(data)` from `@openfn/project` — returns `1` or `2`. Accepts YAML/JSON string or pre-parsed object.
+
+```typescript
+import { detectVersion } from '@openfn/project';
+if (detectVersion(json) === 2) { /* v2 */ }
+```
+
+## Conversion
+
+- **v2 → v1**: `Project.from('project', json).then(p => p.serialize('state', { format: 'yaml' }))` — see `maybeConvertV2spec` in `packages/cli/src/deploy/handler.ts`
+- **v1 → v2**: `Project.from('state', json)` — see `packages/project/src/parse/from-app-state.ts`
+- Full conversion logic: `packages/project/src/serialize/to-app-state.ts` (v2→v1) and `packages/project/src/parse/from-app-state.ts` (v1→v2)

CLAUDE.md

@@ -131,6 +131,7 @@ cd packages/cli && pnpm test:watch  # Watch mode
 The [.claude](.claude) folder contains detailed guides:
 
 - **[event-processor.md](.claude/event-processor.md)** - Worker event processing deep-dive (ordering, batching) — companion to `packages/ws-worker/CLAUDE.md`
+- **[yaml-formats.md](.claude/yaml-formats.md)** - v1 vs v2 project YAML formats: structure, detection logic, and conversion paths
 
 Key packages also carry their own `CLAUDE.md` (runtime, engine-multi, ws-worker), auto-loaded when you work in them.
 

integration-tests/cli/test/deploy.test.ts

@@ -1,6 +1,6 @@
 import test from 'ava';
 import path from 'node:path';
-import fs from 'node:fs/promises';
+import fs, { rm } from 'node:fs/promises';
 import run from '../src/run';
 import createLightningServer from '@openfn/lightning-mock';
 import { extractLogs, assertLog } from '../src/util';
@@ -12,6 +12,50 @@ const port = 8967;
 const endpoint = `http://localhost:${port}`;
 let tmpDir = path.resolve('tmp/deploy');
 
+const testProjectV2 = `
+id: my-project
+name: My Project
+schema_version: '4.0'
+workflows:
+  - id: my-workflow
+    name: My Workflow
+    start: webhook
+    steps:
+      - id: webhook
+        type: webhook
+        enabled: true
+        next:
+          transform-data: {}
+      - id: transform-data
+        name: Transform data
+        expression: 'fn(s => s)'
+        adaptor: '@openfn/language-common@latest'
+`.trim();
+
+const testProjectV2WithCredential = `
+id: my-project
+name: My Project
+schema_version: '4.0'
+credentials:
+  - name: http1
+    owner: super@openfn.org
+workflows:
+  - id: my-workflow
+    name: My Workflow
+    start: webhook
+    steps:
+      - id: webhook
+        type: webhook
+        enabled: true
+        next:
+          transform-data: {}
+      - id: transform-data
+        name: Transform data
+        expression: 'fn(s => s)'
+        adaptor: '@openfn/language-common@latest'
+        configuration: super@openfn.org|http1
+`.trim();
+
 const testProject = `
 name: test-project
 workflows:
@@ -97,7 +141,6 @@ test.serial('deploy a local project', async (t) => {
       --log-json \
       -l debug`
   );
-
   t.falsy(stderr);
 
   const logs = extractLogs(stdout);
@@ -269,6 +312,51 @@ test.serial('redirect to v2 protocol if openfn.yaml is present', async (t) => {
   );
 });
 
+test.serial('deploy a v2 spec file', async (t) => {
+  const testProjectV2 = `
+name: test-project
+schema_version: '4.0'
+workflows:
+  - id: my-workflow
+    name: My Workflow
+    start: webhook
+    steps:
+      - id: webhook
+        type: webhook
+        enabled: true
+        next:
+          my-job: {}
+      - id: my-job
+        name: My Job
+        expression: 'fn(s => s)'
+        adaptor: '@openfn/language-common@latest'
+`.trim();
+
+  await fs.writeFile(path.join(tmpDir, 'project.yaml'), testProjectV2);
+
+  t.is(Object.keys(server.state.projects).length, 0);
+
+  const { stdout, stderr } = await run(
+    `openfn deploy \
+      --project-path ${tmpDir}/project.yaml \
+      --state-path ${tmpDir}/.state.json \
+      --no-confirm \
+      --log-json \
+      -l debug`
+  );
+  t.falsy(stderr);
+
+  const logs = extractLogs(stdout);
+  assertLog(t, logs, /v2 spec/i);
+  assertLog(t, logs, /Deployed/);
+
+  t.is(Object.keys(server.state.projects).length, 1);
+  const [project] = Object.values(server.state.projects) as any[];
+  t.is(project.name, 'test-project');
+  const [workflow] = Object.values(project.workflows) as any[];
+  t.is(workflow.name, 'My Workflow');
+});
+
 test.serial('deploy then pull, changes one workflow, deploy', async (t) => {
   t.is(Object.keys(server.state.projects).length, 0);
 
@@ -318,7 +406,6 @@ test.serial('deploy then pull, changes one workflow, deploy', async (t) => {
 
   // And deploy those changes
   const { stdout, stderr } = await run(deployCmd);
-
   t.falsy(stderr);
 
   const logs = extractLogs(stdout);
@@ -332,3 +419,63 @@ test.serial('deploy then pull, changes one workflow, deploy', async (t) => {
   t.is(Object.keys(server.state.projects).length, 1);
   t.truthy(server.state.projects[projectId]);
 });
+
+test.serial('deploy a v2 project.yaml', async (t) => {
+  await fs.writeFile(path.join(tmpDir, 'project.yaml'), testProjectV2);
+
+  const { stdout, stderr } = await run(
+    `openfn deploy \
+      --project-path ${tmpDir}/project.yaml \
+      --state-path ${tmpDir}/.state.json \
+      --no-confirm \
+      --log-json \
+      -l debug`
+  );
+
+  t.falsy(stderr);
+
+  const logs = extractLogs(stdout);
+  assertLog(t, logs, /Deployed/);
+
+  t.is(Object.keys(server.state.projects).length, 1);
+  const [project] = Object.values(server.state.projects) as any[];
+  t.is(project.name, 'My Project');
+});
+
+test.serial('deploy a new v2 project.yaml with credentials', async (t) => {
+  await fs.writeFile(
+    path.join(tmpDir, 'project.yaml'),
+    testProjectV2WithCredential
+  );
+
+  try {
+    await rm(`${tmpDir}/.state.json`);
+  } catch (e) {
+    // ignore
+  }
+
+  const { stdout, stderr } = await run(
+    `openfn deploy \
+      --project-path ${tmpDir}/project.yaml \
+      --no-confirm \
+      --log-json \
+      -l debug`
+  );
+
+  t.falsy(stderr);
+
+  const logs = extractLogs(stdout);
+  assertLog(t, logs, /Deployed/);
+
+  t.is(Object.keys(server.state.projects).length, 1);
+  const [project] = Object.values(server.state.projects) as any[];
+  t.is(project.name, 'My Project');
+
+  t.is(project.project_credentials[0].name, 'http1');
+  t.is(project.project_credentials[0].owner, 'super@openfn.org');
+
+  const uuid = project.project_credentials[0].id;
+
+  const workflow: any = Object.values(project.workflows).pop();
+  t.is(workflow.jobs[0].project_credential_id, uuid);
+});

packages/cli/src/deploy/handler.ts

@@ -10,7 +10,7 @@ import { DeployOptions } from './command';
 import * as beta from '../projects/deploy';
 import path from 'node:path';
 import { fileExists } from '../util/file-exists';
-import { yamlToJson } from '@openfn/project';
+import Project, { detectVersion, yamlToJson } from '@openfn/project';
 import fs from 'node:fs/promises';
 
 export type DeployFn = typeof deploy;
@@ -62,6 +62,15 @@ async function deployHandler(
       config.endpoint = process.env['OPENFN_ENDPOINT'];
     }
 
+    const rawSpec = await fs.readFile(config.specPath, 'utf-8');
+    const convertedSpec = await maybeConvertV2spec(rawSpec);
+    if (convertedSpec !== rawSpec) {
+      logger.info(
+        'Detected v2 spec file - converting to legacy format; validation will be skipped.'
+      );
+      config.spec = convertedSpec;
+    }
+
     logger.debug('Deploying with config', config);
     logger.info(`Deploying`);
 
@@ -137,4 +146,16 @@ const redirectTov2 = async (
   );
 };
 
+export const maybeConvertV2spec = async (yaml: string): Promise<string> => {
+  const json = yamlToJson(yaml) as any;
+  if (detectVersion(json) > 1) {
+    const project = await Project.from('project', json);
+    return project.serialize('state', {
+      format: 'yaml',
+      asSpec: true,
+    }) as string;
+  }
+  return yaml;
+};
+
 export default deployHandler;