Release/next (#1454)

* create output directory if it doesn't exist (#1452)

* fix(cli): create output directory if it doesn't exist (#1445)

The compile and collections commands threw ENOENT when --output-path
pointed to a directory that hadn't been created yet. Add
mkdir({ recursive: true }) before writeFile in both handlers, matching
the pattern already used in the execute command's serialize-output.ts.

Fixes #859

* changeset

---------

Co-authored-by: Lamine Gueye <51838776+lamine2000@users.noreply.github.com>

* Fix gitignore cache (#1453)

* fix(cli): fix cache .gitignore written to fs root and undefined workflow name (#1444)

* fix(cli): fix cache .gitignore written to fs root and undefined workflow name

- Replace ensureGitIgnore path-walking logic with a direct getCacheRoot
  helper. The old loop used endsWith('.cli-cache') to find the cache
  root, but if the path never contained that segment (custom cachePath)
  it would walk all the way to '/' and write /.gitignore
- Fall back to 'workflow' when plan.workflow.name is undefined, avoiding
  .cli-cache/undefined directories
- Add cachePath to saveToCache/clearCache options types so custom cache
  paths set by workspace projects are correctly forwarded to getCachePath
- Remove spurious await on the now-synchronous getCachePath call
- Add regression tests covering expressionPath + cacheSteps (the exact
  scenario reported in #669, which had no test coverage)

Fixes #669

* fix test: use correct cache subdir name derived from filename

* refactor(cli): address review feedback on cache fix

- Eliminate getCacheRoot helper: getCachePath(options) with no
  workflowName/stepId already returns the CACHE_DIR root naturally
- Use .filter(Boolean) spread into path.resolve so undefined workflowName
  does not cause ERR_INVALID_ARG_TYPE (path.resolve throws on undefined)
- Keep cachePath in saveToCache/clearCache Pick types so custom paths work
- Rename gitignore test name to be more accurate

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* style(cli): fix prettier formatting

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* Revert "style(cli): fix prettier formatting"

This reverts commit 1e47fad.

* style(cli): fix prettier v2 formatting

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* refactor(cli): simplify workflowName fallback per review

Use workflowName ?? '' instead of .filter(Boolean) spread.
path.resolve treats '' as a no-op so undefined workflowName
still resolves to the bare CACHE_DIR root.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>

* clean up test

* changeset

* tidying up

* typing

---------

Co-authored-by: Lamine Gueye <51838776+lamine2000@users.noreply.github.com>
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>

* Merge & Sync improvements (#1450)

* CLI: add --workflows flag to merge and deploy (#1437)

* add --workflows flag

* rename --workflows to --workflow and alias

* changelog

* update tests

* format

* mock: fix an issue where project obejts can get scribbed on

* fix tests

* update integration tests

* improve error messages for invalid workspace

* better diverence message

* Fix incorrect divergence warnings on checkout (#1446)

* improve error messages for invalid workspace

* better diverence message

* add failing test and notes

* project: set the alias on a checked out project

* fix divergence on checkout

* fixes for tests

* fix tests and enable project alias to be null

* update tests

* update more tests

* relax alias lookup to fix tests

* when merging, force-checkout the result

* remove comment

* types

* fix one more test

* update test

* hide merge command

* chore: ignore sentry errors by regexp (#1440)

* feat: ignore sentry errors by regexp

* chore: remove debug

* feat: add severity override to ignored errors

* changeset

---------

Co-authored-by: Joe Clark <jclark@openfn.org>

* versions

---------

Co-authored-by: Lamine Gueye <51838776+lamine2000@users.noreply.github.com>
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
Co-authored-by: Farhan Y. <yahyafarhan48@gmail.com>

Author: 4 people

integration-tests/cli/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @openfn/integration-tests-cli
 
+## 1.0.24
+
+### Patch Changes
+
+- Updated dependencies [25e01c9]
+- Updated dependencies [25e01c9]
+  - @openfn/lightning-mock@2.4.19
+  - @openfn/project@0.17.1
+
 ## 1.0.23
 
 ### Patch Changes

integration-tests/cli/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openfn/integration-tests-cli",
   "private": true,
-  "version": "1.0.23",
+  "version": "1.0.24",
   "description": "CLI integration tests",
   "author": "Open Function Group <admin@openfn.org>",
   "license": "ISC",

integration-tests/cli/test/errors.test.ts

@@ -22,7 +22,6 @@ test.serial('expression not found', async (t) => {
   const stdlogs = extractLogs(stdout);
   assertLog(t, stdlogs, /expression not found/i);
   assertLog(t, stdlogs, /failed to load the expression from blah.js/i);
-  assertLog(t, stdlogs, /critical error: aborting command/i);
 });
 
 test.serial('workflow not found', async (t) => {
@@ -33,7 +32,6 @@ test.serial('workflow not found', async (t) => {
 
   assertLog(t, stdlogs, /workflow not found/i);
   assertLog(t, stdlogs, /failed to load a workflow from blah.json/i);
-  assertLog(t, stdlogs, /critical error: aborting command/i);
 });
 
 test.serial('job contains invalid js', async (t) => {
@@ -45,7 +43,6 @@ test.serial('job contains invalid js', async (t) => {
   assertLog(t, stdlogs, /failed to compile job/i);
   assertLog(t, stdlogs, /unexpected token \(2:10\)/i);
   assertLog(t, stdlogs, /check the syntax of the job expression/i);
-  assertLog(t, stdlogs, /critical error: aborting command/i);
 });
 
 // TODO this should really mention which job threw the error
@@ -60,7 +57,6 @@ test.serial('workflow references a job with invalid js', async (t) => {
   assertLog(t, stdlogs, /failed to compile job/i);
   assertLog(t, stdlogs, /unexpected token \(2:10\)/i);
   assertLog(t, stdlogs, /check the syntax of the job expression/i);
-  assertLog(t, stdlogs, /critical error: aborting command/i);
 });
 
 test.serial("can't find an expression referenced in a workflow", async (t) => {
@@ -77,7 +73,6 @@ test.serial("can't find an expression referenced in a workflow", async (t) => {
     stdlogs,
     /This workflow references a file which cannot be found at does-not-exist.js/i
   );
-  assertLog(t, stdlogs, /critical error: aborting command/i);
 });
 
 test.serial("can't find config referenced in a workflow", async (t) => {
@@ -98,7 +93,6 @@ test.serial("can't find config referenced in a workflow", async (t) => {
     stdlogs,
     /This workflow references a file which cannot be found at does-not-exist.js/i
   );
-  assertLog(t, stdlogs, /critical error: aborting command/i);
 });
 
 test.serial('circular workflow', async (t) => {
@@ -141,7 +135,6 @@ test.serial('invalid end (ambiguous)', async (t) => {
   const stdlogs = extractLogs(stdout);
 
   assertLog(t, stdlogs, /Error: end pattern matched multiple steps/i);
-  assertLog(t, stdlogs, /aborting/i);
 });
 
 // These test error outputs within valid workflows

integration-tests/cli/test/project-v1.test.ts

@@ -108,7 +108,9 @@ workflows:
 const projectsPath = path.resolve(TMP_DIR);
 
 test.before(async () => {
-  // await rm(TMP_DIR, { recursive: true });
+  try {
+    await rm(TMP_DIR, { recursive: true });
+  } catch (e) {}
   await mkdir(`${TMP_DIR}/.projects`, { recursive: true });
 
   await writeFile(`${TMP_DIR}/openfn.yaml`, '');
@@ -120,7 +122,7 @@ test.before(async () => {
 });
 
 test.serial('list available projects', async (t) => {
-  const { stdout } = await run(`openfn projects -w ${projectsPath}`);
+  const { stdout } = await run(`openfn projects --workspace ${projectsPath}`);
 
   t.regex(stdout, /hello-world/);
   t.regex(stdout, /8dbc4349-52b4-4bf2-be10-fdf06da52c46/);
@@ -130,7 +132,7 @@ test.serial('list available projects', async (t) => {
 
 // checkout a project from a yaml file
 test.serial('Checkout a project', async (t) => {
-  await run(`openfn checkout hello-world -w ${projectsPath}`);
+  await run(`openfn checkout hello-world --workspace ${projectsPath}`);
 
   // check workflow.yaml
   const workflowYaml = await readFile(
@@ -167,7 +169,7 @@ steps:
 // note: order of tests is important here
 test.serial('execute a workflow from the checked out project', async (t) => {
   // cheeky bonus test of checkout by alias
-  await run(`openfn checkout main -w ${projectsPath}`);
+  await run(`openfn checkout main --workspace ${projectsPath}`);
 
   // execute a workflow
   const { stdout } = await run(
@@ -191,8 +193,9 @@ test.serial('merge a project', async (t) => {
   t.is(initial, 'fn(() => ({ x: 1}))');
 
   // Run the merge
-  await run(`openfn merge hello-world-staging -w ${projectsPath} --force`);
-
+  const { stdout } = await run(
+    `openfn merge hello-world-staging --workspace ${projectsPath} --force --log debug`
+  );
   // Check the step is updated
   const merged = await readStep();
   t.is(merged, "log('hello world')");

integration-tests/cli/test/project-v2.test.ts

@@ -131,15 +131,15 @@ test.before(async () => {
 });
 
 test.serial('list available projects', async (t) => {
-  const { stdout } = await run(`openfn projects -w ${TMP_DIR}`);
+  const { stdout } = await run(`openfn projects --workspace ${TMP_DIR}`);
   t.regex(stdout, /sandboxing-simple/);
   t.regex(stdout, /a272a529-716a-4de7-a01c-a082916c6d23/);
   t.regex(stdout, /staging/);
   t.regex(stdout, /bc6629fb-7dc8-4b28-93af-901e2bd58dc4/);
 });
 
 test.serial('Checkout a project', async (t) => {
-  await run(`openfn checkout staging -w ${TMP_DIR}`);
+  await run(`openfn checkout staging --workspace ${TMP_DIR}`);
 
   // check workflow.yaml
   const workflowYaml = await readFile(
@@ -174,12 +174,13 @@ steps:
 
 test.serial('execute a workflow from the checked out project', async (t) => {
   // cheeky bonus test of checkout by alias
-  await run(`openfn checkout main -w ${TMP_DIR}`);
+  await run(`openfn checkout main --workspace ${TMP_DIR} --force`);
 
   // execute a workflow
-  await run(
+  const { stdout } = await run(
     `openfn hello-workflow  -o ${TMP_DIR}/output.json  --workspace ${TMP_DIR}`
   );
+  console.log(stdout);
 
   const output = await readFile(`${TMP_DIR}/output.json`, 'utf8');
   const finalState = JSON.parse(output);
@@ -189,7 +190,7 @@ test.serial('execute a workflow from the checked out project', async (t) => {
 test.serial(
   'execute a workflow from the checked out project with a credential map',
   async (t) => {
-    await run(`openfn checkout main --log debug -w ${TMP_DIR}`);
+    await run(`openfn checkout main --log debug --workspace ${TMP_DIR}`);
 
     // Modify the checked out workflow code
     await writeFile(
@@ -248,7 +249,7 @@ test.serial(
     // Important: the collection value MUST be as string
     server.collections.upsert('stuff', 'x', JSON.stringify({ id: 'x' }));
 
-    await run(`openfn checkout main --log debug -w ${TMP_DIR}`);
+    await run(`openfn checkout main --log debug --workspace ${TMP_DIR}`);
 
     // Modify the checked out workflow code
     await writeFile(
@@ -313,7 +314,7 @@ workspace:
 );
 
 test.serial('merge a project', async (t) => {
-  await run(`openfn checkout main -w ${TMP_DIR}`);
+  await run(`openfn checkout main --workspace ${TMP_DIR}`);
 
   const readStep = () =>
     readFile(
@@ -326,7 +327,9 @@ test.serial('merge a project', async (t) => {
   t.is(initial, 'fn(() => ({ x: 1}))');
 
   // Run the merge
-  const { stdout } = await run(`openfn merge staging -w ${TMP_DIR} --force`);
+  const { stdout } = await run(
+    `openfn merge staging --workspace ${TMP_DIR} --force`
+  );
 
   // Check the step is updated
   const merged = await readStep();

packages/cli/CHANGELOG.md

@@ -1,5 +1,23 @@
 # @openfn/cli
 
+## 1.38.0
+
+### Minor Changes
+
+- 25e01c9: Allow users to specifiy which workflows to deploy or merge by passing `-w`.
+
+  NOTE: the `-w` alias has been repurposed from `--workspace` to `--workflow`. This may affect your local development environment. If so, just expand `-w` to `--workspace`.
+
+### Patch Changes
+
+- 4bf5394: Write the .gitignore file for cli cache to the cache root. This means the command will generate more ignore files, but prevents them getting generated at the system root
+- 8180b70: Ensure output path exists on compile and collections commands
+- 25e01c9: Fix an issue on checkout where incorrect divergence warnings can be shown
+- Updated dependencies [25e01c9]
+- Updated dependencies [25e01c9]
+  - @openfn/lexicon@2.2.1
+  - @openfn/project@0.17.1
+
 ## 1.37.0
 
 ### Minor Changes

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "1.37.0",
+  "version": "1.38.0",
   "description": "CLI devtools for the OpenFn toolchain",
   "engines": {
     "node": ">=18",

packages/cli/src/collections/handler.ts

@@ -1,5 +1,5 @@
 import path from 'node:path';
-import { readFile, writeFile } from 'node:fs/promises';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
 
 import { Logger } from '../util/logger';
 import request from './request';
@@ -94,6 +94,7 @@ export const get = async (options: GetOptions, logger: Logger) => {
       null,
       options.pretty ? 2 : undefined
     );
+    await mkdir(path.dirname(options.outputPath!), { recursive: true });
     await writeFile(options.outputPath!, content);
     logger.always(`Wrote items to ${options.outputPath}`);
   } else {

packages/cli/src/compile/handler.ts

@@ -1,4 +1,5 @@
-import { writeFile } from 'node:fs/promises';
+import { mkdir, writeFile } from 'node:fs/promises';
+import { dirname } from 'node:path';
 import type { CompileOptions } from './command';
 import type { Logger } from '../util/logger';
 
@@ -22,6 +23,7 @@ const compileHandler = async (options: CompileOptions, logger: Logger) => {
   if (options.outputStdout) {
     logger.success('Result:\n\n' + result);
   } else {
+    await mkdir(dirname(options.outputPath!), { recursive: true });
     await writeFile(options.outputPath!, result as string);
     logger.success(`Compiled to ${options.outputPath}`);
   }

packages/cli/src/execute/handler.ts

@@ -15,7 +15,7 @@ import loadState from '../util/load-state';
 import validateAdaptors from '../util/validate-adaptors';
 import loadPlan from '../util/load-plan';
 import assertPath from '../util/assert-path';
-import { clearCache, getCachePath } from '../util/cache';
+import { clearCache } from '../util/cache';
 import fuzzyMatchStep from '../util/fuzzy-match-step';
 import abort from '../util/abort';
 import validatePlan from '../util/validate-plan';
@@ -175,15 +175,6 @@ const executeHandler = async (options: ExecuteOptions, logger: Logger) => {
   try {
     const result = await execute(finalPlan, state, options, logger);
 
-    if (options.cacheSteps) {
-      logger.success(
-        `Cached output written to ${getCachePath(
-          options,
-          plan.workflow.name
-        )} (see info logs for details)`
-      );
-    }
-
     await serializeOutput(options, result, logger);
     const duration = printDuration(new Date().getTime() - start);
     if (result?.errors) {