feat: openfn merge command (#1035)

* feat: init merge command

* tests: update

* feat: add removeUnmapped option

* feat: accept workflow-mappings as cli arguments

* tests: cli object parsing

* tests: merge handler test

* Merge: fix pathing and add a basic integration test (#1050)

* basic integration test

* remove log line

* remove more logging

* version: cli@1.16.0

---------

Co-authored-by: Joe Clark <jclark@openfn.org>

Author: doc-han

integration-tests/cli/test/project.test.ts

@@ -0,0 +1,165 @@
+import test from 'ava';
+import { rm, mkdir, writeFile, readFile } from 'node:fs/promises';
+import path from 'node:path';
+import run from '../src/run';
+
+const mainYaml = `
+id: 8dbc4349-52b4-4bf2-be10-fdf06da52c46
+name: hello-world
+description: Simple test project to test sandboxing and merging
+project_credentials: []
+collections: []
+inserted_at: 2025-10-07T09:47:41Z
+updated_at: 2025-10-07T09:47:41Z
+env: null
+color: null
+concurrency: null
+scheduled_deletion: null
+parent_id: null
+history_retention_period: null
+allow_support_access: false
+dataclip_retention_period: null
+requires_mfa: false
+retention_policy: retain_all
+version_history: []
+workflows:
+  - name: my workflow
+    id: 0afbefab-5824-4911-aaae-a19f20106dec
+    concurrency: null
+    inserted_at: 2025-10-07T10:00:23Z
+    updated_at: 2025-10-07T10:00:29Z
+    deleted_at: null
+    lock_version: 2
+    jobs:
+      - name: Transform data
+        body: |
+         // TODO
+        adaptor: "@openfn/language-common@latest"
+        id: b8b780f3-98dd-4244-880b-e534d8f24547
+        project_credential_id: null
+    triggers:
+      - type: webhook
+        enabled: true
+        id: 3b4a47c0-7242-4f0c-8886-838e34762654
+    edges:
+      - id: 33dce70f-047f-4508-82fd-950eb508519b
+        target_job_id: b8b780f3-98dd-4244-880b-e534d8f24547
+        enabled: true
+        source_trigger_id: 3b4a47c0-7242-4f0c-8886-838e34762654
+        condition_type: always
+`;
+
+const stagingYaml = `
+id: 5deddbfa-c63f-4dbc-98b5-a49d3395a488
+name: hello-world-staging
+description: "simulate a staging project "
+project_credentials: []
+collections: []
+inserted_at: 2025-10-07T10:00:13Z
+updated_at: 2025-10-07T10:00:13Z
+env: null
+color: null
+concurrency: null
+scheduled_deletion: null
+parent_id: null
+history_retention_period: null
+allow_support_access: false
+dataclip_retention_period: null
+requires_mfa: false
+retention_policy: retain_all
+version_history: []
+workflows:
+  - name: my workflow
+    id: 9e2cc86a-8896-4a5a-9467-9c4128207fd3
+    concurrency: null
+    inserted_at: 2025-10-07T10:00:36Z
+    updated_at: 2025-10-07T10:00:53Z
+    deleted_at: null
+    lock_version: 3
+    jobs:
+      - name: Transform data
+        body: log('hello world')
+        adaptor: "@openfn/language-common@latest"
+        id: 8d627978-ebb9-4fb2-8cda-9b31c10c963e
+        project_credential_id: null
+    triggers:
+      - type: webhook
+        enabled: true
+        id: 7bb476cc-0292-4573-89d0-b13417bc648e
+    edges:
+      - id: 4c68d22a-4ba7-4d8f-8103-6f4f15c4e7d2
+        target_job_id: 8d627978-ebb9-4fb2-8cda-9b31c10c963e
+        enabled: true
+        source_trigger_id: 7bb476cc-0292-4573-89d0-b13417bc648e
+        condition_type: always
+`;
+const projectsPath = path.resolve('tmp/project');
+
+test.before(async () => {
+  // await rm('tmp/project', { recursive: true });
+  await mkdir('tmp/project/.projects', { recursive: true });
+
+  await writeFile('tmp/project/openfn.yaml', '');
+  await writeFile('tmp/project/.projects/main@app.openfn.org.yaml', mainYaml);
+  await writeFile(
+    'tmp/project/.projects/staging@app.openfn.org.yaml',
+    stagingYaml
+  );
+});
+
+// checkout a project from a yaml file
+test.serial('Checkout a project', async (t) => {
+  await run(`openfn checkout hello-world -p ${projectsPath}`);
+
+  // check workflow.yaml
+  const workflowYaml = await readFile(
+    path.resolve(projectsPath, 'workflows/my-workflow/my-workflow.yaml'),
+    'utf8'
+  );
+  t.is(
+    workflowYaml,
+    `id: my-workflow
+name: my workflow
+options: {}
+steps:
+  - id: trigger
+    type: webhook
+    next:
+      transform-data:
+        disabled: false
+        condition: true
+        openfn:
+          uuid: 33dce70f-047f-4508-82fd-950eb508519b
+  - id: transform-data
+    name: Transform data
+    adaptor: "@openfn/language-common@latest"
+    expression: ./transform-data.js
+`
+  );
+
+  const expr = await readFile(
+    path.resolve(projectsPath, 'workflows/my-workflow/transform-data.js'),
+    'utf8'
+  );
+  t.is(expr.trim(), '// TODO');
+});
+
+// requires the prior test to run
+test.serial('merge a project', async (t) => {
+  const readStep = () =>
+    readFile(
+      path.resolve(projectsPath, 'workflows/my-workflow/transform-data.js'),
+      'utf8'
+    ).then((str) => str.trim());
+
+  // assert the intial step code
+  const initial = await readStep();
+  t.is(initial, '// TODO');
+
+  // Run the merge
+  await run(`openfn merge hello-world-staging -p ${projectsPath}`);
+
+  // Check the step is updated
+  const merged = await readStep();
+  t.is(merged, "log('hello world')");
+});

packages/cli/CHANGELOG.md

@@ -1,11 +1,15 @@
 # @openfn/cli
 
-## 1.15.2
+## 1.16.0
+
+### Minor Changes
+
+- Support merge command, to merge two projects while preserving UUIDs
 
 ### Patch Changes
 
-- Updated dependencies [84bebf4]
-  - @openfn/runtime@1.7.3
+- Updated dependencies
+  - @openfn/project@0.4.1
 
 ## 1.15.1
 

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "1.15.2",
+  "version": "1.16.0",
   "description": "CLI devtools for the OpenFn toolchain",
   "engines": {
     "node": ">=18",

packages/cli/src/checkout/handler.ts

@@ -6,7 +6,7 @@ import fs from 'fs';
 import { rimraf } from 'rimraf';
 
 const checkoutHandler = async (options: CheckoutOptions, logger: Logger) => {
-  const commandPath = path.resolve(process.cwd(), options.projectPath ?? '.');
+  const commandPath = path.resolve(options.projectPath ?? '.');
   const workspace = new Workspace(commandPath);
   if (!workspace.valid) {
     logger.error('Command was run in an invalid openfn workspace');
@@ -17,7 +17,7 @@ const checkoutHandler = async (options: CheckoutOptions, logger: Logger) => {
   const switchProject = workspace.get(options.projectName);
   if (!switchProject) {
     logger.error(
-      `Project with id ${options.projectName} not found in the workspace`
+      `Project with id/name ${options.projectName} not found in the workspace`
     );
     return;
   }

packages/cli/src/cli.ts

@@ -15,6 +15,7 @@ import { install as installCommand, repo as repoCommand } from './repo/command';
 import testCommand from './test/command';
 import projectsCommand from './projects/command';
 import checkoutCommand from './checkout/command';
+import mergeCommand from './merge/command';
 
 const y = yargs(hideBin(process.argv));
 
@@ -34,6 +35,7 @@ export const cmd = y
   .command(pullCommand as any)
   .command(projectsCommand)
   .command(checkoutCommand)
+  .command(mergeCommand)
   .command({
     command: 'version',
     describe:

packages/cli/src/commands.ts

@@ -11,6 +11,7 @@ import metadata from './metadata/handler';
 import pull from './pull/handler';
 import projects from './projects/handler';
 import checkout from './checkout/handler';
+import merge from './merge/handler';
 import { clean, install, pwd, list } from './repo/handler';
 
 import createLogger, { CLI, Logger } from './util/logger';
@@ -34,6 +35,7 @@ export type CommandList =
   | 'pull'
   | 'projects'
   | 'checkout'
+  | 'merge'
   | 'repo-clean'
   | 'repo-install'
   | 'repo-list'
@@ -53,6 +55,7 @@ const handlers = {
   pull,
   projects,
   checkout,
+  merge,
   ['collections-get']: collections.get,
   ['collections-set']: collections.set,
   ['collections-remove']: collections.remove,

packages/cli/src/merge/command.ts

@@ -0,0 +1,31 @@
+import yargs from 'yargs';
+import { Opts } from '../options';
+import { ensure, build } from '../util/command-builders';
+import * as o from '../options';
+
+export type MergeOptions = Required<
+  Pick<
+    Opts,
+    | 'command'
+    | 'projectName'
+    | 'projectPath'
+    | 'removeUnmapped'
+    | 'workflowMappings'
+  >
+>;
+
+const options = [
+  o.projectName,
+  o.projectPath,
+  o.removeUnmapped,
+  o.workflowMappings,
+];
+
+const mergeCommand: yargs.CommandModule = {
+  command: 'merge [project-name]',
+  describe: 'Merges the specified project into the checked out project',
+  handler: ensure('merge', options),
+  builder: (yargs) => build(options, yargs),
+};
+
+export default mergeCommand;

packages/cli/src/merge/handler.ts

@@ -0,0 +1,68 @@
+import Project, { Workspace } from '@openfn/project';
+import path from 'path';
+import type { Logger } from '../util/logger';
+import type { MergeOptions } from './command';
+import { promises as fs } from 'fs';
+import checkoutHandler from '../checkout/handler';
+
+const mergeHandler = async (options: MergeOptions, logger: Logger) => {
+  const commandPath = path.resolve(options.projectPath ?? '.');
+  const workspace = new Workspace(commandPath);
+  if (!workspace.valid) {
+    logger.error('Command was run in an invalid openfn workspace');
+    return;
+  }
+
+  const checkedProject = workspace.getActiveProject();
+  if (!checkedProject) {
+    logger.error(`No project currently checked out`);
+    return;
+  }
+
+  const mProject = workspace.get(options.projectName);
+  if (!mProject) {
+    logger.error(
+      `Project with id/name ${options.projectName} not found in the workspace`
+    );
+    return;
+  }
+
+  if (checkedProject.name === mProject.name) {
+    logger.error('Merging into the same project not allowed');
+    return;
+  }
+
+  if (!checkedProject.name) {
+    logger.error('The checked out project has no name/id');
+    return;
+  }
+
+  const finalPath = workspace.getProjectPath(checkedProject.name);
+  if (!finalPath) {
+    logger.error('Path to checked out project not found.');
+    return;
+  }
+
+  // TODO pick options from the terminal
+  const final = Project.merge(mProject, checkedProject, {
+    removeUnmapped: options.removeUnmapped,
+    workflowMappings: options.workflowMappings,
+  });
+  const yaml = final.serialize('state', { format: 'yaml' });
+  await fs.writeFile(finalPath, yaml);
+
+  // Checkout after merge. to unwrap updated files into filesystem
+  await checkoutHandler(
+    {
+      command: 'checkout',
+      projectPath: commandPath,
+      projectName: final.name || '',
+    },
+    logger
+  );
+  logger.success(
+    `Project ${mProject.name} has been merged into Project ${checkedProject.name} successfully`
+  );
+};
+
+export default mergeHandler;

packages/cli/src/options.ts

@@ -8,6 +8,7 @@ import {
   ensureLogOpts,
   LogLevel,
 } from './util';
+import getCLIOptionObject from './util/get-cli-option-object';
 
 // Central type definition for the main options
 // This represents the types coming out of yargs,
@@ -64,6 +65,9 @@ export type Opts = {
   trace?: boolean;
   useAdaptorsMonorepo?: boolean;
   workflow: string;
+  // merge options
+  removeUnmapped?: boolean | undefined;
+  workflowMappings?: Record<string, string> | undefined;
 
   // deprecated
   workflowPath?: string;
@@ -581,3 +585,23 @@ export const workflow: CLIOption = {
     description: 'Name of the workflow to execute',
   },
 };
+
+// merge options
+export const removeUnmapped: CLIOption = {
+  name: 'remove-unmapped',
+  yargs: {
+    boolean: true,
+    description:
+      "Removes all workflows that didn't get mapped from the final project after merge",
+  },
+};
+
+export const workflowMappings: CLIOption = {
+  name: 'workflow-mappings',
+  yargs: {
+    type: 'string',
+    coerce: getCLIOptionObject,
+    description:
+      'A manual object mapping of which workflows in source and target should be matched for a merge.',
+  },
+};