Skip to content

Commit 4d9e6fd

Browse files
josephjclarkjosephjclark
authored
Fix v2 sync (#1425)
* fix gh sync * fix endpoint argument tracing and a little refactor to the v2 pull proxy * refactor deploy function * integration tests for deploy * changeset * remove log * version
1 parent 76ad967 commit 4d9e6fd

10 files changed

Lines changed: 281 additions & 136 deletions

File tree

‎integration-tests/cli/test/deploy.test.ts‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import run from '../src/run';
55
import createLightningServer from '@openfn/lightning-mock';
66
import { extractLogs, assertLog } from '../src/util';
77
import { rimraf } from 'rimraf';
8+
import { makeProject } from './fixtures/projects';
89

910
let server: any;
1011
const port = 8967;
@@ -196,6 +197,78 @@ test.serial('pull a project', async (t) => {
196197
t.is(workflow.version_history.length, 1);
197198
});
198199

200+
test.serial('redirect to v2 protocol if openfn.yaml is present', async (t) => {
201+
const projectId = 'redirect-test-1';
202+
server.addProject(makeProject(projectId) as any);
203+
204+
// create an empty openfn.yaml to trigger the v1 -> v2 redirect
205+
await fs.writeFile(path.join(tmpDir, 'openfn.yaml'), '');
206+
207+
const bootstrap = await run(
208+
`openfn pull ${projectId} --workspace ${tmpDir} --log-json -l debug`
209+
);
210+
t.falsy(bootstrap.stderr);
211+
assertLog(t, extractLogs(bootstrap.stdout), /Detected openfn.yaml file/i);
212+
213+
const yaml = await fs.readFile(path.join(tmpDir, 'openfn.yaml'), 'utf8');
214+
t.regex(yaml, new RegExp(`uuid\\: ${projectId}`));
215+
216+
const workflowYaml = await fs.readFile(
217+
path.join(tmpDir, 'workflows/my-workflow/my-workflow.yaml'),
218+
'utf8'
219+
);
220+
t.regex(workflowYaml, /id: my-workflow/);
221+
t.regex(workflowYaml, /name: My Workflow/);
222+
t.regex(workflowYaml, /expression: \.\/my-job\.js/);
223+
224+
const stepJs = await fs.readFile(
225+
path.join(tmpDir, 'workflows/my-workflow/my-job.js'),
226+
'utf8'
227+
);
228+
t.is(stepJs, 'fn(s => s)');
229+
230+
// simulate a remote change
231+
const remoteProject = server.state.projects[projectId];
232+
const wf = Object.values(remoteProject.workflows as any).find(
233+
(w: any) => w.id === 'my-workflow-1'
234+
) as any;
235+
server.updateWorkflow(projectId, {
236+
...wf,
237+
jobs: Object.values(wf.jobs ?? {}).map((j: any) =>
238+
j.id === 'my-job-1'
239+
? { ...j, body: 'fn(s => ({ ...s, remote: true }))' }
240+
: j
241+
),
242+
});
243+
244+
// v1 pull -> should redirect to v2 because openfn.yaml exists
245+
const pullResult = await run(
246+
`openfn pull ${projectId} --workspace ${tmpDir} --log-json -l debug`
247+
);
248+
t.falsy(pullResult.stderr);
249+
assertLog(t, extractLogs(pullResult.stdout), /Detected openfn.yaml file/i);
250+
251+
const exprPath = path.join(tmpDir, 'workflows/my-workflow/my-job.js');
252+
t.regex(await fs.readFile(exprPath, 'utf8'), /remote: true/);
253+
254+
// make a local change
255+
await fs.writeFile(exprPath, 'fn(s => ({ ...s, local: true }))');
256+
257+
// v1 deploy -> should redirect to v2
258+
const { stdout, stderr } = await run(
259+
`openfn deploy --workspace ${tmpDir} --no-confirm --log-json -l debug`
260+
);
261+
t.falsy(stderr);
262+
assertLog(t, extractLogs(stdout), /Detected openfn.yaml file/i);
263+
264+
// confirm the local change made it to the server
265+
const serverProj = server.state.projects[projectId];
266+
t.regex(
267+
serverProj.workflows['my-workflow-1'].jobs['my-job'].body,
268+
/local: true/
269+
);
270+
});
271+
199272
test.serial('deploy then pull, changes one workflow, deploy', async (t) => {
200273
t.is(Object.keys(server.state.projects).length, 0);
201274

‎integration-tests/cli/test/deploy.v2.test.ts‎

Lines changed: 1 addition & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import createLightningServer, {
88
import Project from '@openfn/project';
99
import { extractLogs, assertLog } from '../src/util';
1010
import { rimraf } from 'rimraf';
11+
import { makeProject, makeMultiProject } from './fixtures/projects';
1112

1213
let server: ReturnType<typeof createLightningServer>;
1314

@@ -16,100 +17,6 @@ const endpoint = `http://localhost:${port}`;
1617

1718
const tmpDir = path.resolve('tmp/deploy-v2');
1819

19-
const makeProject = (id: string) => ({
20-
id,
21-
name: 'test-project',
22-
workflows: [
23-
{
24-
id: 'my-workflow-1',
25-
name: 'My Workflow',
26-
jobs: [
27-
{
28-
id: 'my-job-1',
29-
name: 'My Job',
30-
body: 'fn(s => s)',
31-
adaptor: '@openfn/language-common@latest',
32-
project_credential_id: null,
33-
},
34-
],
35-
triggers: [{ id: 'my-trigger-1', type: 'webhook', enabled: true }],
36-
edges: [
37-
{
38-
id: 'my-edge-1',
39-
condition_type: 'always',
40-
source_trigger_id: 'my-trigger-1',
41-
target_job_id: 'my-job-1',
42-
enabled: true,
43-
},
44-
],
45-
lock_version: 1,
46-
deleted_at: null,
47-
},
48-
],
49-
project_credentials: [],
50-
collections: [],
51-
});
52-
53-
// A two-workflow project for isolation/divergence tests
54-
const makeMultiProject = (id: string): any => ({
55-
id,
56-
name: 'test-project',
57-
workflows: [
58-
{
59-
id: 'my-workflow-1',
60-
name: 'My Workflow',
61-
jobs: [
62-
{
63-
id: 'my-job-1',
64-
name: 'My Job',
65-
body: 'fn(s => s)',
66-
adaptor: '@openfn/language-common@latest',
67-
project_credential_id: null,
68-
},
69-
],
70-
triggers: [{ id: 'my-trigger-1', type: 'webhook', enabled: true }],
71-
edges: [
72-
{
73-
id: 'my-edge-1',
74-
condition_type: 'always',
75-
source_trigger_id: 'my-trigger-1',
76-
target_job_id: 'my-job-1',
77-
enabled: true,
78-
},
79-
],
80-
lock_version: 1,
81-
deleted_at: null,
82-
},
83-
{
84-
id: 'another-workflow-1',
85-
name: 'Another Workflow',
86-
jobs: [
87-
{
88-
id: 'another-job-1',
89-
name: 'Another Job',
90-
body: "get('http://example.com')",
91-
adaptor: '@openfn/language-http@latest',
92-
project_credential_id: null,
93-
},
94-
],
95-
triggers: [{ id: 'another-trigger-1', type: 'webhook', enabled: true }],
96-
edges: [
97-
{
98-
id: 'another-edge-1',
99-
condition_type: 'always',
100-
source_trigger_id: 'another-trigger-1',
101-
target_job_id: 'another-job-1',
102-
enabled: true,
103-
},
104-
],
105-
lock_version: 1,
106-
deleted_at: null,
107-
},
108-
],
109-
project_credentials: [],
110-
collections: [],
111-
});
112-
11320
test.before(async () => {
11421
server = await createLightningServer({ port });
11522

‎integration-tests/cli/test/fixtures/projects.ts‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
export const makeProject = (id: string) => ({
2+
id,
3+
name: 'test-project',
4+
workflows: [
5+
{
6+
id: 'my-workflow-1',
7+
name: 'My Workflow',
8+
jobs: [
9+
{
10+
id: 'my-job-1',
11+
name: 'My Job',
12+
body: 'fn(s => s)',
13+
adaptor: '@openfn/language-common@latest',
14+
project_credential_id: null,
15+
},
16+
],
17+
triggers: [{ id: 'my-trigger-1', type: 'webhook', enabled: true }],
18+
edges: [
19+
{
20+
id: 'my-edge-1',
21+
condition_type: 'always',
22+
source_trigger_id: 'my-trigger-1',
23+
target_job_id: 'my-job-1',
24+
enabled: true,
25+
},
26+
],
27+
lock_version: 1,
28+
deleted_at: null,
29+
},
30+
],
31+
project_credentials: [],
32+
collections: [],
33+
});
34+
35+
export const makeMultiProject = (id: string): any => ({
36+
id,
37+
name: 'test-project',
38+
workflows: [
39+
{
40+
id: 'my-workflow-1',
41+
name: 'My Workflow',
42+
jobs: [
43+
{
44+
id: 'my-job-1',
45+
name: 'My Job',
46+
body: 'fn(s => s)',
47+
adaptor: '@openfn/language-common@latest',
48+
project_credential_id: null,
49+
},
50+
],
51+
triggers: [{ id: 'my-trigger-1', type: 'webhook', enabled: true }],
52+
edges: [
53+
{
54+
id: 'my-edge-1',
55+
condition_type: 'always',
56+
source_trigger_id: 'my-trigger-1',
57+
target_job_id: 'my-job-1',
58+
enabled: true,
59+
},
60+
],
61+
lock_version: 1,
62+
deleted_at: null,
63+
},
64+
{
65+
id: 'another-workflow-1',
66+
name: 'Another Workflow',
67+
jobs: [
68+
{
69+
id: 'another-job-1',
70+
name: 'Another Job',
71+
body: "get('http://example.com')",
72+
adaptor: '@openfn/language-http@latest',
73+
project_credential_id: null,
74+
},
75+
],
76+
triggers: [{ id: 'another-trigger-1', type: 'webhook', enabled: true }],
77+
edges: [
78+
{
79+
id: 'another-edge-1',
80+
condition_type: 'always',
81+
source_trigger_id: 'another-trigger-1',
82+
target_job_id: 'another-job-1',
83+
enabled: true,
84+
},
85+
],
86+
lock_version: 1,
87+
deleted_at: null,
88+
},
89+
],
90+
project_credentials: [],
91+
collections: [],
92+
});

‎packages/cli/CHANGELOG.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
# @openfn/cli
22

3+
## 1.36.2
4+
5+
### Patch Changes
6+
7+
- 8ebd5c9: Fix an issue where pull and deploy do not track the endpoint argument properly when redirecting to v2
8+
39
## 1.36.1
410

511
### Patch Changes

‎packages/cli/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@openfn/cli",
3-
"version": "1.36.1",
3+
"version": "1.36.2",
44
"description": "CLI devtools for the OpenFn toolchain",
55
"engines": {
66
"node": ">=18",

‎packages/cli/src/deploy/handler.ts‎

Lines changed: 28 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -41,24 +41,7 @@ async function deployHandler(
4141
'openfn.yaml'
4242
);
4343
if (!process.env.PREFER_LEGACY_SYNC && (await fileExists(v2ConfigPath))) {
44-
// default endpoint to one from openfn.yaml
45-
const v2config = yamlToJson(await fs.readFile(v2ConfigPath, 'utf-8'));
46-
if (!config.endpoint && v2config?.project?.endpoint) {
47-
config.endpoint = v2config.project.endpoint;
48-
}
49-
50-
logger.always(
51-
'Detected openfn.yaml file - switching to v2 deploy (openfn project deploy). Set PREFER_LEGACY_SYNC to disable this.'
52-
);
53-
return beta.handler(
54-
{
55-
...options,
56-
force: true,
57-
endpoint: config.endpoint,
58-
apiKey: config.apiKey ?? undefined,
59-
},
60-
logger
61-
);
44+
return redirectTov2(v2ConfigPath, options, config, logger);
6245
}
6346

6447
if (options.confirm === false) {
@@ -123,4 +106,31 @@ function pickFirst<T>(...args: (T | null | undefined)[]): T {
123106
return args.find((arg) => arg !== undefined && arg !== null) as T;
124107
}
125108

109+
const redirectTov2 = async (
110+
v2ConfigPath: string,
111+
options: DeployOptions,
112+
config: DeployConfig,
113+
logger: Logger
114+
) => {
115+
logger.always(
116+
'Detected openfn.yaml file - switching to v2 deploy (openfn project deploy). Set PREFER_LEGACY_SYNC to disable this.'
117+
);
118+
119+
// default endpoint to one from openfn.yaml
120+
const v2config = yamlToJson(await fs.readFile(v2ConfigPath, 'utf-8'));
121+
if (!config.endpoint && v2config?.project?.endpoint) {
122+
config.endpoint = v2config.project.endpoint;
123+
}
124+
125+
return beta.handler(
126+
{
127+
...options,
128+
force: true,
129+
endpoint: config.endpoint,
130+
apiKey: config.apiKey ?? undefined,
131+
},
132+
logger
133+
);
134+
};
135+
126136
export default deployHandler;

‎packages/cli/src/projects/fetch.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ const fetchV1 = async (options: FetchOptions, logger: Logger) => {
9393
const config = loadAppAuthConfig(options, logger);
9494

9595
const { data } = await fetchProject(
96-
options.endpoint ?? localProject?.openfn?.endpoint!,
96+
config.endpoint ?? localProject?.openfn?.endpoint!,
9797
config.apiKey,
9898
localProject?.uuid ?? options.project!,
9999
logger
@@ -265,7 +265,10 @@ export async function fetchRemoteProject(
265265
);
266266
}
267267

268-
const projectEndpoint = localProject?.openfn?.endpoint ?? config.endpoint;
268+
// TODO this resolution is pretty awkward. The problem is we don't
269+
// know if config.endpoint comes from an env var or explicit option
270+
const projectEndpoint =
271+
options.endpoint ?? localProject?.openfn?.endpoint ?? config.endpoint;
269272

270273
const { data } = await fetchProject(
271274
projectEndpoint,

‎packages/cli/src/pull/command.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,10 @@ import * as po from '../projects/options';
88
export type PullOptions = Required<
99
Pick<
1010
Opts & POpts,
11+
| 'apiKey'
1112
| 'beta'
1213
| 'command'
14+
| 'endpoint'
1315
| 'log'
1416
| 'logJson'
1517
| 'statePath'

0 commit comments

Comments
 (0)