New Workflow Generator (#1034)

* use ohmjs to build a workflow parser

* update tests

* support multiple pairs

* more tests

* unstable commit: testing props

* get a single prop working

* support multiple props

* tidy up

* add project generation util

* return proper  Workflow instance

* support attributes on workflows

* support comments

* docs

* update tests, allow prop values to be written into strings

* update tests

* changeset

Author: josephjclark

.changeset/ten-hoops-search.md

@@ -0,0 +1,5 @@
+---
+'@openfn/project': minor
+---
+
+Add project and workflow generator utility

packages/project/README.md

@@ -0,0 +1,81 @@
+## openfn/project
+
+A package to track, parse and serialize OpenFn project definitions.
+
+A PROJECT is defined as a set of connected workflows with a single billing account, like a project in the app.
+
+A single Project can be Checked Out to disk at a time, meaning its source workflows and expressions will be expanded nicely onto the file system.
+
+A Workspace is a set of related Projects , including a Project and its associated Sandboxes, or a Project deployed to apps in multiple web domains
+
+### Serializing and Parsing
+
+The main idea of Projects is that a Project represents a set of OpenFn workflows defined in any format and present a standard JS-friendly interface to manipulate and reason about them.
+
+The from/to serializers are designed to support the following formats:
+
+- Projects expanded to the file system (through CLI or hand-written)
+- v1 JSON state files generated by `openfn pull`
+- v2 Project files (basically v1 state with some extra props)
+
+Serializers and parsers also support JSON and YAML formats interchangeably.
+
+### Project & Workflow Generation
+
+`project` exports utility functions to generate Projects and Workflows from a simple syntax. This is useful for testing.
+
+Really it's a Workflow generator, but you can have it wrapped in a Project if you like.
+
+Use it like this:
+
+```
+import { generateProject, generateWorkflow } from '@openfn/project'
+import type { Project, Workflow } from '@openfn/project'
+
+const proj: Project = generateProject('my-project', ['a-b b-c'])
+const wfL Workflow = generateWorkflow('a-b b-c')
+```
+
+Project generation uses a simple string language to represent a workflow structure:
+
+Define nodes in pairs seperated by a dash (no whitespace)
+
+```
+a-b # parent-child
+```
+
+For multiple children, define multiple pairs:
+
+```
+a-b
+a-c
+```
+
+You can set properties on the workflow itself - probably the name, with `@attributes`
+
+```
+@name my-cool-workflow
+```
+
+You can also set properties on a node by putting comma seperated key-value pairs in brackets
+
+```
+a(adaptor=http)
+```
+
+You can use quotes to include spaces and brackets in a property value - great for expressions:
+
+```
+a(expression="fn(s => s)")
+```
+
+You can comment inside the string with `#`, which is a basic single-line comment
+
+Reference:
+
+```
+# Comments behind hashes
+@attribute-name attribute-value
+parent(propName=propValue,x=y)-child
+a-b # can comment here to
+```

packages/project/package.json

@@ -38,7 +38,9 @@
     "@openfn/lexicon": "workspace:^",
     "@openfn/logger": "workspace:*",
     "glob": "^11.0.2",
+    "lodash": "^4.17.21",
     "lodash-es": "^4.17.21",
+    "ohm-js": "^17.2.1",
     "yaml": "^2.2.2"
   },
   "files": [

packages/project/src/Workflow.ts

@@ -1,4 +1,5 @@
 import * as l from '@openfn/lexicon';
+import slugify from './util/slugify';
 
 const clone = (obj) => JSON.parse(JSON.stringify(obj));
 
@@ -29,8 +30,8 @@ class Workflow {
     this.workflow = clone(workflow);
 
     const { id, name, openfn, steps, ...options } = workflow;
-    this.id = id;
-    this.name = name;
+    this.id = id ?? slugify(name);
+    this.name = name ?? id;
     this.openfn = openfn;
     this.options = options;
 

packages/project/src/gen/generator.ts

@@ -0,0 +1,196 @@
+import { randomUUID } from 'node:crypto';
+import path from 'node:path';
+import { readFileSync } from 'node:fs';
+import { grammar } from 'ohm-js';
+import Project from '../Project';
+import Workflow from '../Workflow';
+import slugify from '../util/slugify';
+
+type GenerateWorkflowOptions = {
+  name: string;
+  uuidSeed: number;
+  openfnUuid: boolean;
+  printErrors: boolean; // true by default
+};
+
+let parser;
+
+const initOperations = (options = {}) => {
+  let nodes = {};
+
+  // Sets values available to this inside semantic actions
+  const uuid = () => {
+    return options.uuidSeed ? options.uuidSeed++ : randomUUID();
+  };
+
+  const buildNode = (name: string) => {
+    if (!nodes[name]) {
+      nodes[name] = {
+        name: name,
+        id: slugify(name),
+        openfn: {
+          uuid: uuid(),
+        },
+      };
+    }
+    return nodes[name];
+  };
+
+  // These are functions which run on matched parse trees
+  const operations = {
+    Workflow(attrs, pair) {
+      pair.children.forEach((child) => child.buildWorkflow());
+
+      const steps = Object.values(nodes);
+
+      const attributes = attrs.children
+        .map((c) => c.buildWorkflow())
+        .reduce((obj, next) => {
+          const [key, value] = next;
+          obj[key] = value;
+          return obj;
+        }, {});
+
+      return { ...attributes, steps: steps };
+    },
+    comment(_a, _b) {
+      return null;
+    },
+    attribute(_, name, _space, value) {
+      return [name.sourceString, value.sourceString];
+    },
+    Pair(parent, edge, child) {
+      const n1 = parent.buildWorkflow();
+      const n2 = child.buildWorkflow();
+      const e = edge.buildWorkflow();
+
+      n1.next ??= {};
+
+      n1.next[n2.name] = e;
+
+      return [n1, n2];
+    },
+    // node could just be a node name, or a node with props
+    // different results have different requirements
+    // Not sure the best way to handle this, but this seems to work
+    node(node) {
+      if (node._node.ruleName === 'node_name') {
+        return buildNode(node.sourceString);
+      }
+      return node.buildWorkflow();
+    },
+    nodeWithProps(nameNode, props) {
+      const name = nameNode.sourceString;
+      const node = buildNode(name);
+      props.buildWorkflow().forEach(([key, value]) => {
+        nodes[name][key] = value;
+      });
+      return node;
+    },
+    node_name(n) {
+      return n.sourceString;
+    },
+    props(_lbr, props, _rbr) {
+      return props.asIteration().children.map((c) => c.buildWorkflow());
+    },
+    prop(key, _op, value) {
+      if (value._iter) {
+        console.log('>>>> ITER');
+      }
+      return [key.sourceString, value.buildWorkflow()];
+    },
+    // Bit flaky - we need this to handle quoted props
+    _iter(...items) {
+      return items.map((i) => i.buildWorkflow()).join('');
+    },
+    alnum(a) {
+      return a.sourceString;
+    },
+    quotedProp(_left, value, _right) {
+      return value.sourceString;
+    },
+    edge(_) {
+      return {
+        openfn: {
+          uuid: uuid(),
+        },
+      };
+    },
+  };
+
+  return operations;
+};
+
+export const createParser = () => {
+  // Load the grammar
+  // TODO: is there any way I can compile/serialize the grammar into JS?
+  const contents = readFileSync(path.resolve('src/gen/workflow.ohm'), 'utf-8');
+  const parser = grammar(contents);
+
+  return {
+    parse(str, options) {
+      const { printErrors = true } = options;
+
+      // Setup semantic actions (which run against an AST and build stuff)
+      // Do this on each parse so we can maintain state
+      const semantics = parser.createSemantics();
+
+      semantics.addOperation('buildWorkflow', initOperations(options));
+
+      // First we parse the source
+      const result = parser.match(str);
+      if (!result.succeeded()) {
+        if (printErrors) {
+          console.error(result.shortMessage);
+          console.error(result.message);
+        }
+        // TODO can we be more helpful here?
+        throw new Error('Parsing failed!' + result.shortMessage);
+      }
+
+      // Then we pass the AST into an operation factory
+      const adaptor = semantics(result);
+
+      // Finally we trigger a semantic action to build a workflow
+      return adaptor.buildWorkflow();
+    },
+  };
+};
+
+/**
+ * Generate a Workflow from a simple text based representation
+ * eg, `a-b b-c a-c`
+ */
+function generateWorkflow(
+  def: string,
+  options: Partial<GenerateWorkflowOptions> = {}
+) {
+  if (!parser) {
+    parser = createParser();
+  }
+
+  const wf = new Workflow(parser.parse(def, options));
+  if (options.openfnUuid) {
+    wf.openfn = {
+      uuid: randomUUID(),
+    };
+  }
+  return wf;
+}
+
+function generateProject(
+  name: string,
+  workflowDefs: string[],
+  options: Partial<GenerateWorkflowOptions>
+) {
+  const workflows = workflowDefs.map((w) => generateWorkflow(w, options));
+
+  return new Project({
+    name,
+    workflows,
+  });
+}
+
+export default generateWorkflow;
+
+export { generateWorkflow, generateProject };