Merge pull request #1066 from OpenFn/release/next

Release/next

Author: josephjclark

packages/cli/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @openfn/cli
 
+## 1.16.1
+
+### Patch Changes
+
+- Updated dependencies [cb97744]
+- Updated dependencies [49caf75]
+  - @openfn/compiler@1.1.5
+  - @openfn/project@0.5.0
+
 ## 1.16.0
 
 ### Minor Changes

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "1.16.0",
+  "version": "1.16.1",
   "description": "CLI devtools for the OpenFn toolchain",
   "engines": {
     "node": ">=18",

packages/compiler/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @openfn/compiler
 
+## 1.1.5
+
+### Patch Changes
+
+- cb97744: Improve compiler performance for large files
+
 ## 1.1.4
 
 ### Patch Changes

packages/compiler/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/compiler",
-  "version": "1.1.4",
+  "version": "1.1.5",
   "description": "Compiler and language tooling for openfn jobs.",
   "author": "Open Function Group <admin@openfn.org>",
   "license": "ISC",

packages/compiler/src/transform-ignore.ts

@@ -0,0 +1,20 @@
+import { namedTypes as n } from 'ast-types';
+import { NodePath } from 'ast-types/lib/node-path';
+
+const IgnoreRules = (path: NodePath<n.Expression>) => {
+  return {
+    check: n.ObjectExpression.check(path.node),
+    shouldSkip: () => {
+      if (
+        n.VariableDeclarator.check(path.parentPath.node) &&
+        n.VariableDeclaration.check(path.parentPath.parentPath.node) &&
+        n.Program.check(path.parentPath.parentPath.parentPath.parentPath.node)
+      ) {
+        return true;
+      }
+      return false;
+    },
+  };
+};
+
+export default IgnoreRules;

packages/compiler/src/transform.ts

@@ -68,7 +68,6 @@ export default function transform(
     ] as Transformer[];
   }
   const logger = options.logger || defaultLogger;
-
   transformers
     // Ignore transformers which are explicitly disabled
     .filter(({ id }) => options[id] ?? true)
@@ -82,11 +81,13 @@ export default function transform(
     })
     // Run each transformer
     .forEach(({ id, types, visitor }) => {
+      const t = `transformer ${id}`;
+      console.time?.(t);
       const astTypes: Visitor = {};
       for (const type of types) {
         const name = `visit${type}` as keyof Visitor;
         astTypes[name] = function (path: NodePath) {
-          printHeap(`visit: ${path.name}`);
+          // printHeap(`visit: ${path.name}`);
           const opts = options[id] || {};
           const abort = visitor!(path, logger, opts);
           if (abort) {
@@ -98,8 +99,15 @@ export default function transform(
 
       // @ts-ignore
       visit(ast, astTypes);
+      if (options.trace) {
+        console.log();
+        printHeap(`${t}`);
+        console.timeEnd?.(t);
+      }
     });
-
+  if (options.trace) {
+    console.log();
+  }
   printHeap(`finished`);
   const duration = (Date.now() - start) / 1000;
   logger.debug(`Finished in ${duration}s`);

packages/compiler/src/transforms/add-imports.ts

@@ -107,6 +107,17 @@ export type IdentifierList = Record<string, true>;
 export function findAllDanglingIdentifiers(ast: ASTNode) {
   const result: IdentifierList = {};
   visit(ast, {
+    visitObjectExpression(path) {
+      // excaping top-level object variable declarations.
+      if (
+        n.VariableDeclarator.check(path.parentPath.node) &&
+        n.VariableDeclaration.check(path.parentPath.parentPath.node) &&
+        n.Program.check(path.parentPath.parentPath.parentPath.parentPath.node)
+      ) {
+        return false;
+      }
+      this.traverse(path);
+    },
     visitIdentifier: function (path) {
       // If this is part of an import statement, do nothing
       if (n.ImportSpecifier.check(path.parent.node)) {

packages/compiler/src/transforms/lazy-state.ts

@@ -9,6 +9,7 @@
 import { builders as b, namedTypes as n } from 'ast-types';
 import type { NodePath } from 'ast-types/lib/node-path';
 import type { Transformer } from '../transform';
+import IgnoreRules from '../transform-ignore';
 
 // Walk up the AST and work out where the parent arrow function should go
 const ensureParentArrow = (path: NodePath<n.MemberExpression>) => {
@@ -79,6 +80,11 @@ const isOpenFunction = (path: NodePath) => {
 };
 
 function visitor(path: NodePath<n.MemberExpression>) {
+  // if it was called for an ObjectExpression
+  const ignoreRule = IgnoreRules(path);
+  if (ignoreRule.check) {
+    return ignoreRule.shouldSkip();
+  }
   let first = path.node.object;
   while (first.hasOwnProperty('object')) {
     first = (first as n.MemberExpression).object;
@@ -106,7 +112,7 @@ function visitor(path: NodePath<n.MemberExpression>) {
 
 export default {
   id: 'lazy-state',
-  types: ['MemberExpression'],
+  types: ['MemberExpression', 'ObjectExpression'],
   visitor,
   // It's important that $ symbols are escaped before any other transformations can run
   order: 0,

packages/compiler/src/transforms/promises.ts

@@ -1,6 +1,7 @@
 import { namedTypes as n, builders as b } from 'ast-types';
 
 import type { NodePath } from 'ast-types/lib/node-path';
+import IgnoreRules from '../transform-ignore';
 
 const NO_DEFER_DECLARATION_ERROR = 'No defer declaration found';
 
@@ -146,6 +147,10 @@ const isTopScope = (path: NodePath<any>) => {
 };
 
 const visitor = (path: NodePath<n.CallExpression>) => {
+  const ignoreRule = IgnoreRules(path);
+  if (ignoreRule.check) {
+    return ignoreRule.shouldSkip();
+  }
   let root: NodePath<any> = path;
   while (!n.Program.check(root.node)) {
     root = root.parent;
@@ -166,7 +171,7 @@ const visitor = (path: NodePath<n.CallExpression>) => {
 
 export default {
   id: 'promises',
-  types: ['CallExpression'],
+  types: ['CallExpression', 'ObjectExpression'],
   visitor,
   // this should run before top-level operations are moved into the exports array
   order: 0,

packages/compiler/src/transforms/top-level-operations.ts

@@ -8,71 +8,53 @@ import type { Transformer } from '../transform';
 // Note that the validator should complain if it see anything other than export default []
 // What is the relationship between the validator and the compiler?
 
-export type ExtendedProgram = NodePath<namedTypes.Program> & {
-  operations: Array<{ line: number; name: string; order: number }>;
-};
+export type ExtendedProgram = NodePath<
+  namedTypes.Program & {
+    operations: Array<{ line: number; name: string; order: number }>;
+  }
+>;
 
 export type TopLevelOpsOptions = {
   // Wrap operations in a `(state) => op` wrapper
   wrap: boolean; // TODO
 };
 
-function visitor(path: NodePath<namedTypes.CallExpression>) {
-  const root = path.parent.parent.node;
-  // Check if the node is a top level Operation
+function visitor(programPath: ExtendedProgram) {
+  const operations: Array<{ line: number; name: string; order: number }> = [];
+  const children = programPath.node.body;
+  const rem = [];
+
+  const target = programPath.node.body.at(-1);
   if (
-    // Check this is a top level call expression
-    // ie, the parent must be an ExpressionStatement, and the statement's parent must be a Program
-    n.Program.check(root) &&
-    n.Statement.check(path.parent.node) &&
-    // An Operation could be an Identifier (fn()) or Member Expression (http.get())
-    (n.Identifier.check(path.node.callee) ||
-      n.MemberExpression.check(path.node.callee))
+    n.ExportDefaultDeclaration.check(target) &&
+    n.ArrayExpression.check(target.declaration)
   ) {
-    appendOperationMetadata(path, root as unknown as ExtendedProgram);
-
-    // Now Find the top level exports array
-    const target = root.body.at(-1);
-    if (
-      n.ExportDefaultDeclaration.check(target) &&
-      n.ArrayExpression.check(target.declaration)
-    ) {
-      const statement = path.parent;
-      target.declaration.elements.push(path.node);
-      // orphan the original expression statement parent
-      statement.prune();
-    } else {
-      // error! there isn't an appropriate export statement
-      // What do we do?
+    for (const child of children) {
+      if (
+        n.ExpressionStatement.check(child) &&
+        n.CallExpression.check(child.expression)
+      ) {
+        const order = operations.length + 1;
+        // @ts-ignore
+        const name = child.expression.callee.name;
+        const line = child.expression.loc?.start.line ?? -1;
+        operations.push({ name, line, order });
+        target.declaration.elements.push(child.expression as any);
+      } else rem.push(child);
     }
+    programPath.node.body = rem;
+  } else {
+    // error! there isn't an appropriate export statement
+    // What do we do?
   }
+  programPath.node.operations = operations;
 
   // if not (for now) we should cancel traversal
   return true;
 }
 
-// Add metadata to each operation for:
-// - the order (do we know? We can guess from the state of the exports array
-// - the name of the operation
-// - the original line number (do we know?)
-function appendOperationMetadata(
-  path: NodePath<namedTypes.CallExpression>,
-  root: ExtendedProgram
-) {
-  if (!root.operations) {
-    root.operations = [];
-  }
-
-  const { operations } = root;
-  const order = operations.length + 1;
-  const name = path.value.callee.name;
-  const line = path.value.loc?.start.line ?? -1;
-
-  operations.push({ line, order, name });
-}
-
 export default {
   id: 'top-level-operations',
-  types: ['CallExpression'],
+  types: ['Program'],
   visitor,
-} as Transformer;
+} as unknown as Transformer;