Skip to content

Commit e8dafa7

Browse files
josephjclarkjosephjclark
authored
Merge pull request #1142 from OpenFn/project-v2
Support a project v2 file
2 parents 4df8a35 + a66abf9 commit e8dafa7

29 files changed

Lines changed: 886 additions & 194 deletions

‎.changeset/icy-shrimps-serve.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@openfn/cli': minor
3+
---
4+
5+
In pull --beta, use an improved and updated structure for project.yaml files

‎.changeset/neat-parents-type.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
'@openfn/lexicon': patch
3+
'@openfn/project': patch
4+
---
5+
6+
Add a new project.yaml structure, which is the default "v2 state" used by each project. For now, this mirrors the internal structure of the runtime, rather than Lightning's structure

integration-tests/cli/test/project.test.ts renamed to integration-tests/cli/test/project-v1.test.ts

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ import { rm, mkdir, writeFile, readFile } from 'node:fs/promises';
33
import path from 'node:path';
44
import run from '../src/run';
55

6+
// These tests use the legacy v1 yaml structure
7+
68
const mainYaml = `
79
id: 8dbc4349-52b4-4bf2-be10-fdf06da52c46
810
name: hello-world
@@ -107,6 +109,15 @@ test.before(async () => {
107109
);
108110
});
109111

112+
test.serial('list available projects', async (t) => {
113+
const { stdout } = await run(`openfn projects -p ${projectsPath}`);
114+
115+
t.regex(stdout, /hello-world/);
116+
t.regex(stdout, /8dbc4349-52b4-4bf2-be10-fdf06da52c46/);
117+
t.regex(stdout, /hello-world-staging/);
118+
t.regex(stdout, /5deddbfa-c63f-4dbc-98b5-a49d3395a488/);
119+
});
120+
110121
// checkout a project from a yaml file
111122
test.serial('Checkout a project', async (t) => {
112123
await run(`openfn checkout hello-world -p ${projectsPath}`);
@@ -128,8 +139,6 @@ steps:
128139
transform-data:
129140
disabled: false
130141
condition: true
131-
openfn:
132-
uuid: 33dce70f-047f-4508-82fd-950eb508519b
133142
- id: transform-data
134143
name: Transform data
135144
adaptor: "@openfn/language-common@latest"

‎integration-tests/cli/test/project-v2.test.ts‎

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
import test from 'ava';
2+
import { rm, mkdir, writeFile, readFile } from 'node:fs/promises';
3+
import path from 'node:path';
4+
import run from '../src/run';
5+
6+
const mainYaml = `
7+
id: sandboxing-simple
8+
name: Sandboxing Simple
9+
version: 2
10+
collections: []
11+
credentials:
12+
- id: 10a50683-78b0-4ddf-9c14-23a1fb21074a
13+
name: name
14+
owner: editor@openfn.org
15+
openfn:
16+
uuid: a272a529-716a-4de7-a01c-a082916c6d23
17+
endpoint: http://localhost:4000
18+
env: project
19+
fetched_at: 2025-11-26T17:55:09.716Z
20+
inserted_at: 2025-10-17T10:30:44Z
21+
updated_at: 2025-10-24T14:52:13Z
22+
options:
23+
env: main
24+
allow_support_access: false
25+
requires_mfa: false
26+
retention_policy: retain_all
27+
version_history: []
28+
workflows:
29+
- name: Hello Workflow
30+
steps:
31+
- id: trigger
32+
type: webhook
33+
openfn:
34+
enabled: true
35+
uuid: 9c0a4e8a-b82f-4fa5-8d12-419da143cd04
36+
next:
37+
transform-data:
38+
disabled: false
39+
condition: true
40+
openfn:
41+
uuid: add150e9-8616-48ca-844e-8aaa489c7a10
42+
- id: transform-data
43+
name: Transform data
44+
expression: |-
45+
// TODO
46+
adaptor: "@openfn/language-dhis2@8.0.4"
47+
openfn:
48+
uuid: a9f64216-7974-469d-8415-d6d9baf2f92e
49+
project_credential_id: null
50+
openfn:
51+
uuid: af697653-98d2-46e4-912f-e1cb6bf8f4b4
52+
concurrency: 5
53+
inserted_at: 2025-10-17T10:30:51Z
54+
updated_at: 2025-10-24T14:52:13Z
55+
deleted_at: null
56+
lock_version: 16
57+
id: hello-workflow
58+
history: []
59+
`;
60+
61+
const stagingYaml = `id: staging
62+
name: staging
63+
version: 2
64+
collections: []
65+
credentials:
66+
- id: 07c0baa7-c1d7-44b2-abb6-3446defbe3e3
67+
name: name
68+
owner: editor@openfn.org
69+
openfn:
70+
uuid: bc6629fb-7dc8-4b28-93af-901e2bd58dc4
71+
endpoint: http://localhost:4000
72+
env: staging
73+
fetched_at: 2025-11-27T11:40:37.670Z
74+
inserted_at: 2025-11-27T11:39:33Z
75+
updated_at: 2025-11-27T11:39:33Z
76+
options:
77+
env: main
78+
color: "#F39B33"
79+
parent_id: a272a529-716a-4de7-a01c-a082916c6d23
80+
allow_support_access: false
81+
requires_mfa: false
82+
retention_policy: retain_all
83+
version_history:
84+
- 7b0f5af558f5
85+
workflows:
86+
- name: Hello Workflow
87+
steps:
88+
- id: trigger
89+
type: webhook
90+
openfn:
91+
enabled: false
92+
uuid: 9ef55dca-16a3-480c-a807-2d37744e6e53
93+
next:
94+
transform-data:
95+
disabled: false
96+
condition: true
97+
openfn:
98+
uuid: f34146b5-de43-4b05-ac00-3b4f327e62ec
99+
- id: transform-data
100+
name: Transform data
101+
expression: |-
102+
fn()
103+
adaptor: "@openfn/language-dhis2@8.0.4"
104+
openfn:
105+
uuid: 5b4c74f9-76ac-4715-bd45-04b130ca549c
106+
project_credential_id: null
107+
108+
openfn:
109+
uuid: 10ce2914-16aa-4e00-b746-47678b1c60d4
110+
concurrency: 5
111+
inserted_at: 2025-11-27T11:39:33Z
112+
updated_at: 2025-11-27T11:39:47Z
113+
deleted_at: null
114+
lock_version: 1
115+
id: hello-workflow
116+
history: []
117+
`;
118+
const projectsPath = path.resolve('tmp/project');
119+
120+
test.before(async () => {
121+
await rm('tmp/project', { recursive: true });
122+
await mkdir('tmp/project/.projects', { recursive: true });
123+
124+
await writeFile('tmp/project/openfn.yaml', '');
125+
await writeFile('tmp/project/.projects/main@app.openfn.org.yaml', mainYaml);
126+
await writeFile(
127+
'tmp/project/.projects/staging@app.openfn.org.yaml',
128+
stagingYaml
129+
);
130+
});
131+
132+
test.serial('list available projects', async (t) => {
133+
const { stdout } = await run(`openfn projects -p ${projectsPath}`);
134+
t.regex(stdout, /sandboxing-simple/);
135+
t.regex(stdout, /a272a529-716a-4de7-a01c-a082916c6d23/);
136+
t.regex(stdout, /staging/);
137+
t.regex(stdout, /bc6629fb-7dc8-4b28-93af-901e2bd58dc4/);
138+
});
139+
140+
test.serial('Checkout a project', async (t) => {
141+
await run(`openfn checkout staging -p ${projectsPath}`);
142+
143+
// check workflow.yaml
144+
const workflowYaml = await readFile(
145+
path.resolve(projectsPath, 'workflows/hello-workflow/hello-workflow.yaml'),
146+
'utf8'
147+
);
148+
t.is(
149+
workflowYaml,
150+
`id: hello-workflow
151+
name: Hello Workflow
152+
options:
153+
history: []
154+
steps:
155+
- id: trigger
156+
type: webhook
157+
next:
158+
transform-data:
159+
disabled: false
160+
condition: true
161+
- id: transform-data
162+
name: Transform data
163+
adaptor: "@openfn/language-dhis2@8.0.4"
164+
expression: ./transform-data.js
165+
`
166+
);
167+
168+
const expr = await readFile(
169+
path.resolve(projectsPath, 'workflows/hello-workflow/transform-data.js'),
170+
'utf8'
171+
);
172+
t.is(expr.trim(), 'fn()');
173+
});
174+
175+
// requires the prior test to run
176+
test.serial('merge a project', async (t) => {
177+
const readStep = () =>
178+
readFile(
179+
path.resolve(projectsPath, 'workflows/hello-workflow/transform-data.js'),
180+
'utf8'
181+
).then((str) => str.trim());
182+
183+
await run(`openfn checkout sandboxing-simple -p ${projectsPath}`);
184+
185+
// assert the initial step code
186+
const initial = await readStep();
187+
t.is(initial, '// TODO');
188+
189+
// Run the merge
190+
const { stdout } = await run(
191+
`openfn merge staging -p ${projectsPath} --force`
192+
);
193+
194+
// Check the step is updated
195+
const merged = await readStep();
196+
t.is(merged, 'fn()');
197+
});

‎packages/cli/src/checkout/handler.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ const checkoutHandler = async (options: CheckoutOptions, logger: Logger) => {
4040
await rimraf(path.join(commandPath, config.workflowRoot ?? 'workflows'));
4141

4242
// expand project into directory
43-
const files = switchProject.serialize('fs');
43+
const files: any = switchProject.serialize('fs');
4444
for (const f in files) {
4545
if (files[f]) {
4646
fs.mkdirSync(path.join(commandPath, path.dirname(f)), {

‎packages/cli/src/merge/handler.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ const mergeHandler = async (options: MergeOptions, logger: Logger) => {
7777
if (outputFormat === 'json') {
7878
finalState = JSON.stringify(finalState, null, 2);
7979
}
80-
await fs.writeFile(finalPath, finalState);
80+
await fs.writeFile(finalPath, finalState as string);
8181

8282
logger.info(`Updated statefile at `, finalPath);
8383

‎packages/cli/src/pull/beta.ts‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,16 @@
11
// beta v2 version of CLI pull
2-
import { confirm } from '@inquirer/prompts';
3-
import path from 'path';
2+
import path from 'node:path';
43
import fs from 'node:fs/promises';
4+
import { rimraf } from 'rimraf';
5+
import { confirm } from '@inquirer/prompts';
6+
57
import { DeployConfig, getProject } from '@openfn/deploy';
68
import Project, { Workspace } from '@openfn/project';
7-
import type { Logger } from '../util/logger';
8-
import { rimraf } from 'rimraf';
9-
import { Opts } from '../options';
109
import { Provisioner } from '@openfn/lexicon/lightning';
1110

11+
import type { Logger } from '../util/logger';
12+
import type { Opts } from '../options';
13+
1214
// new config
1315
type Config = {
1416
endpoint: string;
@@ -83,7 +85,7 @@ export async function handler(options: PullOptionsBeta, logger: Logger) {
8385
const projectFileName = project.getIdentifier();
8486

8587
await fs.mkdir(`${outputRoot}/.projects`, { recursive: true });
86-
let stateOutputPath = `${outputRoot}/.projects/${projectFileName}`;
88+
const stateOutputPath = `${outputRoot}/.projects/${projectFileName}`;
8789

8890
const workflowsRoot = path.resolve(
8991
outputRoot,
@@ -95,7 +97,7 @@ export async function handler(options: PullOptionsBeta, logger: Logger) {
9597
!(await confirm({
9698
message: `This will remove all files in ${path.resolve(
9799
workflowsRoot
98-
)} and rebuild the workflow. Are you sure you wish to proceed?
100+
)}. Are you sure you wish to proceed?
99101
`,
100102
default: true,
101103
}))
@@ -105,19 +107,19 @@ export async function handler(options: PullOptionsBeta, logger: Logger) {
105107
}
106108
await rimraf(workflowsRoot);
107109

108-
const state = project?.serialize('state');
110+
const projFile = project?.serialize('project');
109111

110-
if (project.config.formats.project === 'yaml') {
111-
await fs.writeFile(`${stateOutputPath}.yaml`, state);
112+
if (typeof projFile === 'string') {
113+
await fs.writeFile(`${stateOutputPath}.yaml`, projFile);
112114
} else {
113115
await fs.writeFile(
114116
`${stateOutputPath}.json`,
115-
JSON.stringify(state, null, 2)
117+
JSON.stringify(projFile, null, 2)
116118
);
117119
}
118120
logger.success(`Saved project file to ${stateOutputPath}`);
119121

120-
const files = project?.serialize('fs');
122+
const files = project?.serialize('fs') as any;
121123
for (const f in files) {
122124
if (files[f]) {
123125
await fs.mkdir(path.join(outputRoot, path.dirname(f)), {

‎packages/cli/test/checkout/handler.test.ts‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -399,9 +399,6 @@ workspace:
399399
'transform-data-to-fhir-standard': {
400400
disabled: false,
401401
condition: true,
402-
openfn: {
403-
uuid: 'edge-id',
404-
},
405402
},
406403
},
407404
},

0 commit comments

Comments
 (0)