Merge pull request #1406 from OpenFn/schema-version-4

josephjclark · web-flow · commit 66c7f79df277 · 2026-05-12T17:36:26.000+01:00
CLI 1.35.3 Schema version 4 &amp; supporting fies
diff --git a/.tool-versions b/.tool-versions
@@ -1 +1 @@
-nodejs 24.14.0
+nodejs 24.15.0
diff --git a/integration-tests/cli/CHANGELOG.md b/integration-tests/cli/CHANGELOG.md
@@ -1,5 +1,13 @@
 # @openfn/integration-tests-cli
 
+## 1.0.22
+
+### Patch Changes
+
+- Updated dependencies
+  - @openfn/project@0.15.2
+  - @openfn/lightning-mock@2.4.17
+
 ## 1.0.21
 
 ### Patch Changes
diff --git a/integration-tests/cli/package.json b/integration-tests/cli/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@openfn/integration-tests-cli",
   "private": true,
-  "version": "1.0.21",
+  "version": "1.0.22",
   "description": "CLI integration tests",
   "author": "Open Function Group <admin@openfn.org>",
   "license": "ISC",
diff --git a/integration-tests/cli/test/project-v1.test.ts b/integration-tests/cli/test/project-v1.test.ts
@@ -142,7 +142,6 @@ test.serial('Checkout a project', async (t) => {
     `id: my-workflow
 name: my workflow
 start: webhook
-options: {}
 steps:
   - id: webhook
     type: webhook
@@ -153,7 +152,7 @@ steps:
         condition: always
   - id: transform-data
     name: Transform data
-    adaptor: "@openfn/language-common@latest"
+    adaptor: '@openfn/language-common@latest'
     expression: ./transform-data.js
 `
   );
diff --git a/integration-tests/cli/test/project-v2.test.ts b/integration-tests/cli/test/project-v2.test.ts
@@ -151,7 +151,6 @@ test.serial('Checkout a project', async (t) => {
     `id: hello-workflow
 name: Hello Workflow
 start: trigger
-options: {}
 steps:
   - id: trigger
     type: webhook
@@ -161,7 +160,7 @@ steps:
         condition: true
   - id: transform-data
     name: Transform data
-    adaptor: "@openfn/language-dhis2@8.0.4"
+    adaptor: '@openfn/language-dhis2@8.0.4'
     expression: ./transform-data.js
 `
   );
diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md
@@ -1,5 +1,15 @@
 # @openfn/cli
 
+## 1.35.3
+
+### Patch Changes
+
+- Include version number in serialized project files
+- Don't validate workflows on execution unless `--validate` is explicitly passed
+- Allow setting `PREFER_LEGACY_SYNC=1` to strictly disable v2 sync when calling the v1 API
+- Updated dependencies
+  - @openfn/project@0.15.2
+
 ## 1.35.2
 
 ### Patch Changes
diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "1.35.2",
+  "version": "1.35.3",
   "description": "CLI devtools for the OpenFn toolchain",
   "engines": {
     "node": ">=18",
diff --git a/packages/cli/src/deploy/handler.ts b/packages/cli/src/deploy/handler.ts
@@ -40,15 +40,15 @@ async function deployHandler(
       options.workspace || process.cwd(),
       'openfn.yaml'
     );
-    if (await fileExists(v2ConfigPath)) {
+    if (!process.env.PREFER_LEGACY_SYNC && (await fileExists(v2ConfigPath))) {
       // override endpoint with one from openfn.yaml
       const config = yamlToJson(await fs.readFile(v2ConfigPath, 'utf-8'));
       if (config?.project?.endpoint) {
         config.endpoint = config.project.endpoint;
       }
 
       logger.always(
-        'Detected openfn.yaml file - switching to v2 deploy (openfn project deploy)'
+        'Detected openfn.yaml file - switching to v2 deploy (openfn project deploy). Set PREFER_LEGACY_SYNC to disable this.'
       );
       return beta.handler(
         {
diff --git a/packages/cli/src/execute/command.ts b/packages/cli/src/execute/command.ts
@@ -8,39 +8,40 @@ import type { Opts } from '../options';
 export type ExecuteOptions = { workspace?: string } & Required<
   Pick<
     Opts,
-    | 'apiKey'
     | 'adaptors'
+    | 'apiKey'
     | 'autoinstall'
     | 'baseDir'
     | 'cacheSteps'
+    | 'collectionsEndpoint'
+    | 'collectionsVersion'
     | 'command'
     | 'compile'
     | 'credentials'
-    | 'collectionsEndpoint'
-    | 'collectionsVersion'
-    | 'expandAdaptors'
     | 'end'
-    | 'immutable'
-    | 'ignoreImports'
+    | 'expandAdaptors'
     | 'expressionPath'
+    | 'globals'
+    | 'ignoreImports'
+    | 'immutable'
     | 'log'
     | 'logJson'
+    | 'only'
     | 'outputPath'
     | 'outputStdout'
-    | 'only'
     | 'path'
     | 'repoDir'
+    | 'sanitize'
     | 'skipAdaptorValidation'
     | 'start'
     | 'statePath'
     | 'stateStdin'
-    | 'sanitize'
     | 'timeout'
     | 'trace'
     | 'useAdaptorsMonorepo'
-    | 'workflowPath'
+    | 'validate'
     | 'workflowName'
-    | 'globals'
+    | 'workflowPath'
   >
 > &
   Pick<Opts, 'monorepoPath' | 'repoDir'>;
@@ -77,6 +78,7 @@ const options = [
   o.timeout,
   o.trace,
   o.useAdaptorsMonorepo,
+  o.validate,
 
   po.workspace,
 ];
diff --git a/packages/cli/src/execute/handler.ts b/packages/cli/src/execute/handler.ts
@@ -87,7 +87,9 @@ const executeHandler = async (options: ExecuteOptions, logger: Logger) => {
   await validateAdaptors(options, logger);
 
   let plan = await loadPlan(options, logger);
-  validatePlan(plan, logger);
+  if (options.validate) {
+    validatePlan(plan, logger);
+  }
   await loadAndApplyCredentialMap(plan, options, logger);
   if (options.cacheSteps) {
     await clearCache(plan, options, logger);
diff --git a/packages/cli/src/options.ts b/packages/cli/src/options.ts
@@ -70,6 +70,7 @@ export type Opts = {
   useAdaptorsMonorepo?: boolean;
   workflow: string;
   workflowName?: string;
+  validate?: boolean;
 
   // deprecated
   workflowPath?: string;
@@ -613,6 +614,15 @@ export const sanitize: CLIOption = {
   },
 };
 
+export const validate: CLIOption = {
+  name: 'validate',
+  yargs: {
+    boolean: true,
+    default: false,
+    description: 'Validate workflows before executing',
+  },
+};
+
 export const workflow: CLIOption = {
   name: 'workflow',
   yargs: {
diff --git a/packages/cli/src/pull/handler.ts b/packages/cli/src/pull/handler.ts
@@ -26,15 +26,15 @@ async function pullHandler(options: PullOptions, logger: Logger) {
       options.workspace || process.cwd(),
       'openfn.yaml'
     );
-    if (await fileExists(v2ConfigPath)) {
+    if (!process.env.PREFER_LEGACY_SYNC && (await fileExists(v2ConfigPath))) {
       // override endpoint with one from openfn.yaml
       const config = yamlToJson(await fs.readFile(v2ConfigPath, 'utf-8'));
       if (config?.project?.endpoint) {
         config.endpoint = config.project.endpoint;
       }
 
       logger.always(
-        'Detected openfn.yaml file - switching to v2 pull (openfn project pull)'
+        'Detected openfn.yaml file - switching to v2 pull (openfn project pull). Set PREFER_LEGACY_SYNC to disable this.'
       );
       return beta(
         {
diff --git a/packages/cli/test/projects/checkout.test.ts b/packages/cli/test/projects/checkout.test.ts
@@ -476,7 +476,6 @@ workspace:
   t.deepEqual(JSON.parse(wf), {
     id: 'simple-workflow',
     name: 'Simple Workflow',
-    options: {},
     start: 'webhook',
     steps: [
       {
diff --git a/packages/cli/test/projects/fetch.test.ts b/packages/cli/test/projects/fetch.test.ts
@@ -402,9 +402,7 @@ test.serial(
     const json = {
       id: 'my-project',
       name: 'My Project',
-      cli: {
-        version: 2,
-      },
+      schema_version: '4.0',
       description: 'my lovely project',
       collections: [],
       credentials: [],
diff --git a/packages/cli/test/projects/fixtures.ts b/packages/cli/test/projects/fixtures.ts
@@ -61,8 +61,7 @@ export const myProject_v1: Provisioner.Project = {
 
 export const myProject_yaml = `id: my-project
 name: My Project
-cli:
-  version: 2
+schema_version: '4.0'
 description: my lovely project
 collections: []
 credentials: []
@@ -81,7 +80,7 @@ workflows:
       - id: transform-data
         name: Transform data
         expression: fn()
-        adaptor: "@openfn/language-common@latest"
+        adaptor: '@openfn/language-common@latest'
         openfn:
           uuid: 66add020-e6eb-4eec-836b-20008afca816
       - id: webhook
diff --git a/packages/lexicon/portability.d.ts b/packages/lexicon/portability.d.ts
@@ -1,15 +1,13 @@
-/**
- * Typings which define the Portability Spec
- * See https://docs.openfn.org/documentation/deploy/portability
- */
-
 /**
  * Schema for portable representation of a Project
  * AKA Project Spec
  * Can serialize to JSON or YAML or whatever you like
  *
+ * This interface describes the 4.0 Portability Spec: https://docs.openfn.org/documentation/deploy/portability
+ *
  * If you serialize a v2 project file without state, this is what you get
  */
+
 export interface ProjectSpec {
   /** Single-word identifier */
   id: string;
@@ -19,6 +17,8 @@ export interface ProjectSpec {
 
   description?: string;
 
+  schema_version?: string;
+
   workflows: WorkflowSpec[];
 
   credentials?: Credential[];
@@ -33,6 +33,8 @@ export interface WorkflowSpec {
   /** Human readable label. Can be used to generate an internal id */
   name?: string;
 
+  schema_version?: string;
+
   steps: Array<Job | Trigger>;
 
   /** Global functions (path to a js file, unsupported in app) */
diff --git a/packages/lightning-mock/CHANGELOG.md b/packages/lightning-mock/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @openfn/lightning-mock
 
+## 2.4.17
+
+### Patch Changes
+
+- Updated dependencies
+  - @openfn/project@0.15.2
+
 ## 2.4.16
 
 ### Patch Changes
diff --git a/packages/lightning-mock/package.json b/packages/lightning-mock/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/lightning-mock",
-  "version": "2.4.16",
+  "version": "2.4.17",
   "private": true,
   "description": "A mock Lightning server",
   "main": "dist/index.js",
diff --git a/packages/project/CHANGELOG.md b/packages/project/CHANGELOG.md
@@ -1,5 +1,11 @@
 # @openfn/project
 
+## 0.15.2
+
+### Patch Changes
+
+- Include version number in serialized project files
+
 ## 0.15.1
 
 ### Patch Changes
diff --git a/packages/project/package.json b/packages/project/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/project",
-  "version": "0.15.1",
+  "version": "0.15.2",
   "description": "Read, serialize, replicate and sync OpenFn projects",
   "scripts": {
     "test": "pnpm ava",
diff --git a/packages/project/src/parse/from-project.ts b/packages/project/src/parse/from-project.ts
@@ -4,26 +4,15 @@ import Project from '../Project';
 import ensureJson from '../util/ensure-json';
 import { Provisioner } from '@openfn/lexicon/lightning';
 import fromAppState, { fromAppStateConfig } from './from-app-state';
-import { WithMeta } from '../Workflow';
 
 // Load a project from any JSON or yaml representation
 // This is backwards-compatible with v1 state.json files
 // But is really designed for v2 project.yaml files
 
 // TODO move these types to a common types.ts, or maybe Project.ts
-export type SerializedProject = Omit<Partial<l.ProjectState>, 'workflows'> & {
-  version: number;
-  workflows: SerializedWorkflow[];
-};
-
-export type SerializedWorkflow = {
-  id: string;
-  name: string;
+export type SerializedProject = l.ProjectState;
 
-  steps: WithMeta<Array<l.Job | l.Trigger>>;
-
-  openfn?: l.ProjectMeta;
-};
+export type SerializedWorkflow = l.WorkflowState;
 
 export default (
   data: l.ProjectState | SerializedProject | string,
@@ -32,8 +21,11 @@ export default (
   // first ensure the data is in JSON format
   let rawJson = ensureJson<any>(data);
 
-  if (rawJson.cli?.version ?? rawJson.version /*deprecated*/) {
-    // If there's any version key at all, its at least v2
+  if (
+    rawJson.schema_version ||
+    rawJson.cli?.version === 2 ||
+    rawJson.version /*deprecated*/
+  ) {
     return new Project(from_v2(rawJson as SerializedProject), config);
   }
 
diff --git a/packages/project/src/serialize/to-fs.ts b/packages/project/src/serialize/to-fs.ts
diff --git a/packages/project/src/serialize/to-project.ts b/packages/project/src/serialize/to-project.ts
diff --git a/packages/project/src/util/yaml.ts b/packages/project/src/util/yaml.ts
diff --git a/packages/project/test/fixtures/sample-v2-project.ts b/packages/project/test/fixtures/sample-v2-project.ts
diff --git a/packages/project/test/parse/from-project.test.ts b/packages/project/test/parse/from-project.test.ts
diff --git a/packages/project/test/serialize/to-fs.test.ts b/packages/project/test/serialize/to-fs.test.ts
diff --git a/packages/project/test/util/yaml.test.ts b/packages/project/test/util/yaml.test.ts

Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,7 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@openfn/integration-tests-cli",`
`3`	`3`	`"private": true,`
`4`		`- "version": "1.0.21",`
	`4`	`+ "version": "1.0.22",`
`5`	`5`	`"description": "CLI integration tests",`
`6`	`6`	`"author": "Open Function Group <admin@openfn.org>",`
`7`	`7`	`"license": "ISC",`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@openfn/cli",`
`3`		`- "version": "1.35.2",`
	`3`	`+ "version": "1.35.3",`
`4`	`4`	`"description": "CLI devtools for the OpenFn toolchain",`
`5`	`5`	`"engines": {`
`6`	`6`	`"node": ">=18",`