Merge pull request #1144 from OpenFn/dotenv

Support .env files

Author: josephjclark

.changeset/sixty-books-tell.md

@@ -0,0 +1,5 @@
+---
+'@openfn/logger': minor
+---
+
+Add colour to output

.changeset/twelve-boats-matter.md

@@ -0,0 +1,5 @@
+---
+'@openfn/cli': minor
+---
+
+Support .env files in all CLI commands

.gitignore

@@ -1,3 +1,5 @@
+.env
+
 # dev
 *.code-workspace
 examples/**/dist
@@ -35,3 +37,5 @@ tmp
 
 *.drawio.bkp
 .fuse
+
+

packages/cli/package.json

@@ -57,6 +57,7 @@
     "@openfn/project": "workspace:^",
     "@openfn/runtime": "workspace:*",
     "chalk": "^5.6.2",
+    "dotenv": "^17.2.3",
     "figures": "^5.0.0",
     "rimraf": "^6.0.1",
     "treeify": "^1.1.0",

packages/cli/src/cli.ts

@@ -1,6 +1,7 @@
 import yargs, { Arguments } from 'yargs';
 import { hideBin } from 'yargs/helpers';
 
+import loadDotEnv from './env';
 import apolloCommand from './apollo/command';
 import collectionsCommand from './collections/command';
 import compileCommand from './compile/command';
@@ -18,6 +19,13 @@ import checkoutCommand from './checkout/command';
 import mergeCommand from './merge/command';
 import workflowVersionCommand from './version/command';
 
+const env = loadDotEnv();
+if (env) {
+  // Write the env so that the inner process can use it
+  // This is just a convenience for logging back to the user
+  process.env.$DOT_ENV_OVERRIDES = Object.keys(env).join(',');
+}
+
 const y = yargs(hideBin(process.argv));
 
 export const cmd = y

packages/cli/src/commands.ts

@@ -21,6 +21,7 @@ import mapAdaptorsToMonorepo, {
 } from './util/map-adaptors-to-monorepo';
 import printVersions from './util/print-versions';
 import abort from './util/abort';
+import { report } from './env';
 
 export type CommandList =
   | 'apollo'
@@ -80,6 +81,9 @@ const parse = async (options: Opts, log?: Logger) => {
     await printVersions(logger, options);
   }
 
+  // Tell the user whether we're using env vars
+  report(logger);
+
   const { monorepoPath } = options;
   if (monorepoPath) {
     // TODO how does this occur?

packages/cli/src/env.ts

@@ -0,0 +1,30 @@
+import { Logger } from '@openfn/logger';
+import dotenv from 'dotenv';
+
+let env: any;
+
+export default (path?: string) => {
+  env = dotenv.config({ path, override: true, debug: false, quiet: true });
+  if (env.error) {
+    return null;
+  }
+  return env.parsed;
+};
+
+export const report = (logger?: Logger) => {
+  // workaround for the CLI's inner process
+
+  let envs = [];
+  if (process.env.$DOT_ENV_OVERRIDES) {
+    envs = process.env.$DOT_ENV_OVERRIDES.split(',').map((s) => s.trim());
+  } else {
+    envs = Object.keys(env?.parsed ?? {});
+  }
+
+  if (envs.length) {
+    logger?.always(`Imported ${envs.length} env vars from .env file`);
+    logger?.debug('Envs set from .env: ', envs.join(', '));
+  } else if (env && env.error) {
+    logger?.debug('.env not found');
+  }
+};

packages/cli/test/commands.test.ts

@@ -761,7 +761,7 @@ test.serial('docs should print documentation with full names', async (t) => {
   opts.repoDir = '/repo';
 
   await commandParser(opts, logger);
-  const docs = logger._parse(logger._history[2]).message as string;
+  const docs = logger._find('print', /API Reference/).message;
 
   // match the signature
   t.regex(docs, /\#\# fn\(\)/);
@@ -796,7 +796,7 @@ test.serial('docs adaptor should list operations', async (t) => {
 
   await commandParser(opts, logger);
 
-  const docs = logger._parse(logger._history[3]).message as string;
+  const docs = logger._find('print', /language-common/).message;
   t.notRegex(docs, /\[object Object\]/);
   t.notRegex(docs, /\#\#\# Usage Examples/);
   t.regex(docs, /fn.+\(a, b\)/);
@@ -823,7 +823,8 @@ test.serial(
     opts.repoDir = '/repo';
 
     await commandParser(opts, logger);
-    const docs = logger._parse(logger._history[2]).message as string;
+
+    const docs = logger._find('print', /API Reference/).message;
     // match the signature
     t.regex(docs, /\#\# fn\(\)/);
     // Match usage examples

packages/cli/test/env.test.ts

@@ -0,0 +1,83 @@
+import test from 'ava';
+import mock from 'mock-fs';
+import { createMockLogger } from '@openfn/logger';
+
+import load, { report } from '../src/env';
+
+const logger = createMockLogger(undefined, { level: 'debug' });
+
+test.afterEach(() => {
+  logger._reset();
+  mock.restore();
+});
+
+test('should load and return values from .env', (t) => {
+  mock({
+    '/.env': `
+      FOO: BAR
+    `,
+  });
+
+  const env = load('/.env');
+  t.deepEqual(env, {
+    FOO: 'BAR',
+  });
+});
+
+test('should report values loaded from .env', (t) => {
+  mock({
+    '/.env': `
+      FOO: BAR
+    `,
+  });
+
+  load('/.env');
+
+  report(logger);
+
+  const a = logger._find('always', /imported 1 env vars/i);
+  t.truthy(a);
+
+  const b = logger._find('debug', /set from .env:/i);
+  t.truthy(b);
+
+  const c = logger._find('debug', /FOO/);
+  t.truthy(c);
+});
+
+test('should return null if no env is found', (t) => {
+  mock({});
+
+  const env = load('/.env');
+  t.is(env, null);
+});
+
+test('should log if no env is found', (t) => {
+  mock({});
+
+  load('/.env');
+
+  report(logger);
+
+  const a = logger._find('debug', /.env not found/i);
+  t.truthy(a);
+});
+
+test('should use $DOT_ENV_OVERRIDES found', (t) => {
+  mock({});
+
+  // don't load .env
+  // but set this magic env var instead
+  process.env.$DOT_ENV_OVERRIDES = 'FOO,BAR';
+
+  report(logger);
+
+  const a = logger._find('always', /imported 1 env vars/i);
+  t.truthy(a);
+
+  const b = logger._find('debug', /set from .env:/i);
+  t.truthy(b);
+
+  const c = logger._find('debug', /FOO, BAR/);
+  t.truthy(c);
+});

packages/logger/src/logger.ts

@@ -53,6 +53,21 @@ export type JSONLog = {
 
 export type StringLog = [LogFns | 'confirm' | 'print', ...any];
 
+export const colourize = (level: string, str: string) => {
+  if (typeof str === 'string') {
+    if (level === DEBUG) {
+      return c.grey(str);
+    }
+    if (level === ERROR) {
+      return c.red(str);
+    }
+    if (level === WARN) {
+      return c.yellow(str);
+    }
+  }
+  return str;
+};
+
 // Design for a logger
 // some inputs:
 // This will be passed into a job, so must look like the console object
@@ -224,7 +239,9 @@ export default function (name?: string, options: LogOptions = {}): Logger {
           output.push(styleLevel(level));
         }
 
-        emitter[level](...output.concat(cleanedArgs));
+        emitter[level](
+          ...output.concat(cleanedArgs).map((s) => colourize(level, s))
+        );
       }
     }
   };