Next Release

integration-tests/cli/test/project.test.ts

@@ -152,12 +152,14 @@ test.serial('merge a project', async (t) => {
       'utf8'
     ).then((str) => str.trim());
 
-  // assert the intial step code
+  // assert the initial step code
   const initial = await readStep();
   t.is(initial, '// TODO');
 
   // Run the merge
-  await run(`openfn merge hello-world-staging -p ${projectsPath}`);
+  const { stdout } = await run(
+    `openfn merge hello-world-staging -p ${projectsPath} --force`
+  );
 
   // Check the step is updated
   const merged = await readStep();

packages/cli/CHANGELOG.md

@@ -1,5 +1,29 @@
 # @openfn/cli
 
+## 1.18.0
+
+### Minor Changes
+
+- 16da2ef: Warn when merging two projects might result in lost work
+
+### Patch Changes
+
+- Updated dependencies [16da2ef]
+- Updated dependencies [16da2ef]
+  - @openfn/project@0.7.0
+
+## 1.17.2
+
+### Patch Changes
+
+- edfc759: Update Project dependency
+- 6a68759: Respect openfn.yaml options in pull --beta
+- f4209dd: Warn when merging projects which may have diverged
+- Updated dependencies [f955548]
+- Updated dependencies [edfc759]
+- Updated dependencies [329d29d]
+  - @openfn/project@0.6.1
+
 ## 1.17.1
 
 ### Patch Changes

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "1.17.1",
+  "version": "1.18.0",
   "description": "CLI devtools for the OpenFn toolchain",
   "engines": {
     "node": ">=18",

packages/cli/src/checkout/handler.ts

@@ -15,7 +15,7 @@ const checkoutHandler = async (options: CheckoutOptions, logger: Logger) => {
 
   // get the config
   // TODO: try to retain the endpoint for the projects
-  const { project: _, ...config } = workspace.getConfig() ?? {};
+  const { project: _, ...config } = workspace.getConfig() as any;
 
   // get the project
   let switchProject;

packages/cli/src/merge/command.ts

@@ -1,6 +1,6 @@
 import yargs from 'yargs';
 import { Opts } from '../options';
-import { ensure, build } from '../util/command-builders';
+import { ensure, build, override } from '../util/command-builders';
 import * as o from '../options';
 
 export type MergeOptions = Required<
@@ -12,6 +12,7 @@ export type MergeOptions = Required<
     | 'removeUnmapped'
     | 'workflowMappings'
     | 'log'
+    | 'force'
   >
 >;
 
@@ -21,6 +22,9 @@ const options = [
   o.removeUnmapped,
   o.workflowMappings,
   o.log,
+  override(o.force, {
+    description: 'Force a merge even when workflows are incompatible',
+  }),
 ];
 
 const mergeCommand: yargs.CommandModule = {

packages/cli/src/merge/handler.ts

@@ -51,10 +51,10 @@ const mergeHandler = async (options: MergeOptions, logger: Logger) => {
     return;
   }
 
-  // TODO pick options from the terminal
   const final = Project.merge(sourceProject, targetProject, {
     removeUnmapped: options.removeUnmapped,
     workflowMappings: options.workflowMappings,
+    force: options.force,
   });
   const yaml = final.serialize('state', { format: 'yaml' });
   await fs.writeFile(finalPath, yaml);
@@ -65,6 +65,7 @@ const mergeHandler = async (options: MergeOptions, logger: Logger) => {
       command: 'checkout',
       projectPath: commandPath,
       projectName: final.name || '',
+      log: options.log,
     },
     logger
   );

packages/cli/src/pull/beta.ts

@@ -3,7 +3,7 @@ import { confirm } from '@inquirer/prompts';
 import path from 'path';
 import fs from 'node:fs/promises';
 import { DeployConfig, getProject } from '@openfn/deploy';
-import Project from '@openfn/project';
+import Project, { Workspace } from '@openfn/project';
 import type { Logger } from '../util/logger';
 import { rimraf } from 'rimraf';
 import { Opts } from '../options';
@@ -37,47 +37,52 @@ export type PullOptionsBeta = Required<
 export async function handler(options: PullOptionsBeta, logger: Logger) {
   const { OPENFN_API_KEY, OPENFN_ENDPOINT } = process.env;
 
-  const config: Partial<Config> = {
+  const cfg: Partial<Config> = {
     apiKey: options.apiKey,
     endpoint: options.endpoint,
   };
 
   if (!options.apiKey && OPENFN_API_KEY) {
     logger.info('Using OPENFN_API_KEY environment variable');
-    config.apiKey = OPENFN_API_KEY;
+    cfg.apiKey = OPENFN_API_KEY;
   }
 
   if (!options.endpoint && OPENFN_ENDPOINT) {
     logger.info('Using OPENFN_ENDPOINT environment variable');
-    config.endpoint = OPENFN_ENDPOINT;
+    cfg.endpoint = OPENFN_ENDPOINT;
   }
 
+  // TODO `path` or `output` ?
+  // I don't think I want to model this as output. deploy is really
+  // designed to run from the working folder
+  // could be projectPath or repoPath too
+  const outputRoot = path.resolve(options.path || '.');
+
+  // TODO is outputRoot the right dir for this?
+  const workspace = new Workspace(outputRoot);
+  const config = workspace.getConfig();
+
   // download the state.json from lightning
-  const { data } = await getProject(config as DeployConfig, options.projectId);
+  const { data } = await getProject(cfg as DeployConfig, options.projectId);
 
   // TODO if the user doesn't specify an env name, prompt for one
   const name = options.env || 'project';
 
   const project = Project.from('state', data, {
-    endpoint: config.endpoint,
+    config,
+    endpoint: cfg.endpoint,
     env: name,
     fetched_at: new Date().toISOString(),
   });
 
-  // TODO `path` or `output` ?
-  // I don't think I want to model this as output. deploy is really
-  // designed to run from the working folder
-  // could be projectPath or repoPath too
-  const outputRoot = path.resolve(options.path || '.');
-
   const projectFileName = project.getIdentifier();
 
   await fs.mkdir(`${outputRoot}/.projects`, { recursive: true });
   let stateOutputPath = `${outputRoot}/.projects/${projectFileName}`;
 
   const workflowsRoot = path.resolve(
     outputRoot,
-    project.repo?.workflowRoot ?? 'workflows'
+    project.config.dirs.workflows ?? 'workflows'
   );
   // Prompt before deleting
   // TODO this is actually the wrong path
@@ -96,7 +101,8 @@ export async function handler(options: PullOptionsBeta, logger: Logger) {
   await rimraf(workflowsRoot);
 
   const state = project?.serialize('state');
-  if (project.repo?.formats.project === 'yaml') {
+
+  if (project.config.formats.project === 'yaml') {
     await fs.writeFile(`${stateOutputPath}.yaml`, state);
   } else {
     await fs.writeFile(

packages/cli/test/checkout/handler.test.ts

@@ -164,7 +164,7 @@ test.serial('checkout: invalid project id', (t) => {
 test.serial('checkout: to a different valid project', async (t) => {
   // before checkout. some-project-name is active and expanded
   const bcheckout = new Workspace('/ws');
-  t.is(bcheckout.getConfig()?.name, 'some-project-name');
+  t.is(bcheckout.projectMeta.name, 'some-project-name');
   t.is(bcheckout.getActiveProject()?.name, 'some-project-name');
 
   await checkoutHandler(
@@ -176,7 +176,7 @@ test.serial('checkout: to a different valid project', async (t) => {
 
   // after checkout. main-project-id is active and expanded
   const acheckout = new Workspace('/ws');
-  t.is(acheckout.getConfig()?.name, 'main-project-id');
+  t.is(acheckout.projectMeta.name, 'main-project-id');
   t.is(acheckout.getActiveProject()?.name, 'main-project-id');
 
   // check if files where well expanded
@@ -189,19 +189,23 @@ test.serial('checkout: to a different valid project', async (t) => {
 test.serial('checkout: same id as active', async (t) => {
   // before checkout. some-project-name is active and expanded
   const bcheckout = new Workspace('/ws');
-  t.is(bcheckout.getConfig()?.name, 'some-project-name');
+  t.is(bcheckout.projectMeta.name, 'some-project-name');
   t.is(bcheckout.getActiveProject()?.name, 'some-project-name');
 
   await checkoutHandler(
-    { command: 'checkout', projectName: 'some-project-name', projectPath: '/ws' },
+    {
+      command: 'checkout',
+      projectName: 'some-project-name',
+      projectPath: '/ws',
+    },
     logger
   );
   const { message } = logger._parse(logger._last);
   t.is(message, 'Expanded project to /ws');
 
   // after checkout. main-project-id is active and expanded
   const acheckout = new Workspace('/ws');
-  t.is(acheckout.getConfig()?.name, 'some-project-name');
+  t.is(acheckout.projectMeta.name, 'some-project-name');
   t.is(acheckout.getActiveProject()?.name, 'some-project-name');
 
   // check if files where well expanded
@@ -214,7 +218,7 @@ test.serial('checkout: same id as active', async (t) => {
 test.serial('checkout: switching to and back between projects', async (t) => {
   // before checkout. some-project-name is active and expanded
   const bcheckout = new Workspace('/ws');
-  t.is(bcheckout.getConfig()?.name, 'some-project-name');
+  t.is(bcheckout.projectMeta.name, 'some-project-name');
   t.is(bcheckout.getActiveProject()?.name, 'some-project-name');
 
   // 1. switch from some-project-name to main-project-id
@@ -227,7 +231,7 @@ test.serial('checkout: switching to and back between projects', async (t) => {
 
   // after checkout. main-project-id is active and expanded
   const acheckout = new Workspace('/ws');
-  t.is(acheckout.getConfig()?.name, 'main-project-id');
+  t.is(acheckout.projectMeta.name, 'main-project-id');
   t.is(acheckout.getActiveProject()?.name, 'main-project-id');
 
   // check if files where well expanded
@@ -238,15 +242,19 @@ test.serial('checkout: switching to and back between projects', async (t) => {
 
   // 2. switch back from main-project-id to some-project-name
   await checkoutHandler(
-    { command: 'checkout', projectName: 'some-project-name', projectPath: '/ws' },
+    {
+      command: 'checkout',
+      projectName: 'some-project-name',
+      projectPath: '/ws',
+    },
     logger
   );
   const { message: lastMsg } = logger._parse(logger._last);
   t.is(lastMsg, 'Expanded project to /ws');
 
   // after checkout. main-project-id is active and expanded
   const fcheckout = new Workspace('/ws');
-  t.is(fcheckout.getConfig()?.name, 'some-project-name');
+  t.is(fcheckout.projectMeta.name, 'some-project-name');
   t.is(fcheckout.getActiveProject()?.name, 'some-project-name');
 
   // check if files where well expanded

packages/cli/test/merge/handler.test.ts

@@ -113,7 +113,7 @@ test('merging into the same project', async (t) => {
 test('merging a different project into checked-out', async (t) => {
   // state of main projects workflow before sandbox is merged in
   const bworkspace = new Workspace('/ws');
-  t.is(bworkspace.getConfig()?.name, 'main-project-id');
+  t.is(bworkspace.projectMeta.name, 'main-project-id');
   t.is(bworkspace.getActiveProject()?.name, 'main-project-id');
   const bprojects = bworkspace.list();
   t.is(bprojects[0].workflows[0].steps.length, 2);
@@ -133,13 +133,13 @@ test('merging a different project into checked-out', async (t) => {
 
   // state of main projects workflow before sandbox is merged in
   const workspace = new Workspace('/ws');
-  t.is(workspace.getConfig()?.name, 'main-project-id');
+  t.is(workspace.projectMeta.name, 'main-project-id');
   t.is(workspace.getActiveProject()?.name, 'main-project-id');
   const projects = workspace.list();
   t.is(projects[0].workflows[0].steps.length, 3);
   t.is(projects[0].workflows[0].steps[1].name, 'Job X');
   t.is(projects[0].workflows[0].steps[1].openfn?.uuid, 'job-a'); // id got retained
-  t.is(projects[0].workflows[0].steps[2].name, 'Job Y'); 
+  t.is(projects[0].workflows[0].steps[2].name, 'Job Y');
   t.is(projects[0].workflows[0].steps[2].openfn?.uuid, 'job-y'); // id not retained - new nod
 
   const { message, level } = logger._parse(logger._last);

packages/engine-multi/CHANGELOG.md

@@ -1,5 +1,11 @@
 # engine-multi
 
+## 1.7.1
+
+### Patch Changes
+
+- b61bf9b: Fix an issue where memory may not be released after runs
+
 ## 1.7.0
 
 ### Minor Changes

packages/engine-multi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/engine-multi",
-  "version": "1.7.0",
+  "version": "1.7.1",
   "description": "Multi-process runtime engine",
   "main": "dist/index.js",
   "type": "module",

packages/engine-multi/src/api/lifecycle.ts

@@ -52,23 +52,9 @@ export const workflowComplete = (
   const { workflowId, state: result, threadId } = event;
 
   logger.success('complete workflow ', workflowId);
-  //logger.info(event.state);
-
-  // TODO I don't know how we'd get here in this architecture
-  // if (!allWorkflows.has(workflowId)) {
-  //   throw new Error(`Workflow with id ${workflowId} is not defined`);
-  // }
-
   state.status = 'done';
   state.duration = Date.now() - state.startTime!;
 
-  // Important! We do NOT write the result back to this state object
-  // It has a tendency to not get garbage collected and causing memory problems
-
-  // TODO do we have to remove this from the active workflows array?
-  // const idx = activeWorkflows.findIndex((id) => id === workflowId);
-  // activeWorkflows.splice(idx, 1);
-
   // forward the event on to any external listeners
   context.emit(externalEvents.WORKFLOW_COMPLETE, {
     threadId,