Merge pull request #1057 from OpenFn/projects-root

Support new Project & CLI features for integration tests

Author: josephjclark

packages/cli/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @openfn/cli
 
+## 1.16.2
+
+### Patch Changes
+
+- 8a50703: Allow a project to be checked out from a direct path to a project file
+- 7d16875: When merging, allow the source project to specified as a path to a project file
+- Updated dependencies [81b97c3]
+- Updated dependencies [43be979]
+- Updated dependencies [1f8d65e]
+- Updated dependencies [25c7a2b]
+- Updated dependencies [04a89e2]
+- Updated dependencies [7d16875]
+  - @openfn/project@0.5.1
+  - @openfn/lexicon@1.2.4
+
 ## 1.16.1
 
 ### Patch Changes

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "1.16.1",
+  "version": "1.16.2",
   "description": "CLI devtools for the OpenFn toolchain",
   "engines": {
     "node": ">=18",

packages/cli/src/checkout/handler.ts

@@ -1,4 +1,4 @@
-import { Workspace } from '@openfn/project';
+import Project, { Workspace } from '@openfn/project';
 import path from 'path';
 import type { Logger } from '../util/logger';
 import type { CheckoutOptions } from './command';
@@ -13,20 +13,32 @@ const checkoutHandler = async (options: CheckoutOptions, logger: Logger) => {
     return;
   }
 
+  // get the config
+  // TODO: try to retain the endpoint for the projects
+  const { project: _, ...config } = workspace.getConfig() ?? {};
+
   // get the project
-  const switchProject = workspace.get(options.projectName);
+  let switchProject;
+  if (/\.(yaml|json)$/.test(options.projectName)) {
+    // TODO: should we allow checkout into an arbitrary folder?
+    const filePath = path.join(commandPath, options.projectName);
+    logger.debug('Loading project from path ', filePath);
+    switchProject = await Project.from('path', filePath, {
+      config,
+    });
+  } else {
+    switchProject = workspace.get(options.projectName);
+  }
+
   if (!switchProject) {
     logger.error(
       `Project with id/name ${options.projectName} not found in the workspace`
     );
     return;
   }
-  // get the config
-  // TODO: try to retain the endpoint for the projects
-  const config = workspace.getConfig();
 
   // delete workflow dir before expanding project
-  await rimraf(path.join(commandPath, config?.workflowRoot || 'workflows'));
+  await rimraf(path.join(commandPath, config.workflowRoot ?? 'workflows'));
 
   // expand project into directory
   const files = switchProject.serialize('fs');

packages/cli/src/merge/handler.ts

@@ -13,38 +13,46 @@ const mergeHandler = async (options: MergeOptions, logger: Logger) => {
     return;
   }
 
-  const checkedProject = workspace.getActiveProject();
-  if (!checkedProject) {
+  // The target project - the think we apply changes to - is
+  // whatever is checked out
+  const targetProject = workspace.getActiveProject();
+  if (!targetProject) {
     logger.error(`No project currently checked out`);
     return;
   }
 
-  const mProject = workspace.get(options.projectName);
-  if (!mProject) {
-    logger.error(
-      `Project with id/name ${options.projectName} not found in the workspace`
-    );
+  // Lookup the source project - the thing we are getting changes from
+  let sourceProject;
+  if (/\.(yaml|json)$/.test(options.projectName)) {
+    const filePath = path.join(commandPath, options.projectName);
+    logger.debug('Loading source project from path ', filePath);
+    sourceProject = await Project.from('path', filePath);
+  } else {
+    sourceProject = workspace.get(options.projectName);
+  }
+  if (!sourceProject) {
+    logger.error(`Project "${options.projectName}" not found in the workspace`);
     return;
   }
 
-  if (checkedProject.name === mProject.name) {
+  if (targetProject.name === sourceProject.name) {
     logger.error('Merging into the same project not allowed');
     return;
   }
 
-  if (!checkedProject.name) {
+  if (!targetProject.name) {
     logger.error('The checked out project has no name/id');
     return;
   }
 
-  const finalPath = workspace.getProjectPath(checkedProject.name);
+  const finalPath = workspace.getProjectPath(targetProject.name);
   if (!finalPath) {
     logger.error('Path to checked out project not found.');
     return;
   }
 
   // TODO pick options from the terminal
-  const final = Project.merge(mProject, checkedProject, {
+  const final = Project.merge(sourceProject, targetProject, {
     removeUnmapped: options.removeUnmapped,
     workflowMappings: options.workflowMappings,
   });
@@ -61,7 +69,7 @@ const mergeHandler = async (options: MergeOptions, logger: Logger) => {
     logger
   );
   logger.success(
-    `Project ${mProject.name} has been merged into Project ${checkedProject.name} successfully`
+    `Project ${sourceProject.name} has been merged into Project ${targetProject.name} successfully`
   );
 };
 

packages/lexicon/CHANGELOG.md

@@ -1,5 +1,11 @@
 # lexicon
 
+## 1.2.4
+
+### Patch Changes
+
+- 1f8d65e: Clarify naming conventions for id/name
+
 ## 1.2.3
 
 ### Patch Changes

packages/lexicon/core.d.ts

@@ -1,6 +1,9 @@
 import { SanitizePolicies } from '@openfn/logger';
 import type { RawSourceMap } from 'source-map';
 
+/** UUID v4 */
+export type UUID = string;
+
 export type SourceMap = RawSourceMap;
 
 export type SourceMapWithOperations = RawSourceMap & {
@@ -10,10 +13,17 @@ export type SourceMapWithOperations = RawSourceMap & {
 // The serialised shape of of a project, as JSON
 // this is what is saved to project.yaml
 export type Project = {
-  name: string; // human readable name
+  /** Single-word identifier */
+  id: string;
+
+  /** human readable name */
+  name?: string;
+
+  /** Shorthand identifier used by CLI commands */
+  alias?: string;
+
   description?: string;
 
-  // object or array?
   workflows: Workflow[];
 
   options: {};
@@ -24,8 +34,11 @@ export type Project = {
   openfn?: ProjectConfig;
 };
 
-export type ProjectConfig = {
-  uuid: string;
+export type OpenFnMetadata = {
+  uuid?: UUID;
+};
+
+export type ProjectConfig = OpenFnMetadata & {
   endpoint: string;
   name: string;
   env: string;
@@ -42,7 +55,7 @@ export type ProjectConfig = {
  * (ie, the CLI or Worker)
  */
 export type ExecutionPlan = {
-  id?: UUID; // this would be the run (nee attempt) id
+  id?: UUID; // TODO make required
   workflow: Workflow;
   options?: WorkflowOptions;
 };
@@ -51,11 +64,15 @@ export type ExecutionPlan = {
  * A workflow is just a series of steps, executed start to finish
  */
 export type Workflow = {
-  id?: UUID; // unique id used to track this workflow. Could be autogenerated
+  /** The ID is the primary internal identifier for a Workflow */
+  id?: string;
 
-  // TODO: make required (worker and cli may have to generate a name)
+  /** Human readable name, like display. Can be used to generate an internal id */
   name?: string;
 
+  /** Local shorthand name for use in CLI commands. Not used by Lightning */
+  alias?: string;
+
   steps: Array<Job | Trigger>;
 
   // global credentials
@@ -64,6 +81,8 @@ export type Workflow = {
 
   // a path to a file where functions are defined
   globals?: string;
+
+  openfn?: OpenFnMetadata;
 };
 
 export type StepId = string;
@@ -73,6 +92,7 @@ export type StepId = string;
  * (usually a job)
  */
 export interface Step {
+  // TODO a Step must ALWAYS have an id (util functions can default it)
   id?: StepId;
   name?: string; // user-friendly name used in logging
 
@@ -88,7 +108,7 @@ export type StepEdge = boolean | string | ConditionalStepEdge;
 
 export type ConditionalStepEdge = {
   condition?: string; // Javascript expression (function body, not function)
-  label?: string;
+  label?: string; // TODO this is probably the name
   disabled?: boolean;
 };
 
@@ -208,6 +228,4 @@ export type ErrorReport = {
 
 // TODO standard shape of error object in our stack
 
-type UUID = string;
-
 export type Lazy<T> = T | string;

packages/lexicon/lightning.d.ts

@@ -17,15 +17,19 @@ type TimeInMicroSeconds = string;
  */
 
 /**
- * An execution plan representing a Lightning 'Run'.
- * This represents the execution of a workflow.
+ * The Lightning Plan is the data structure sent by lightning in response
+ * to the fetch:plan event. It represents a single Run, or an execution
+ * of a workflow given some input dataclip.
  *
- * The data structure that Lightning sends is converted by the Worker into
- * a runtime ExecutionPlan (as found in Core)
+ * This structure is converted by the Worker into a runtime ExecutionPlan (as found in Core)
  */
 export type LightningPlan = {
+  /** The Lightning UUID for this Run (not the same as the workflow id/UUID) */
   id: string;
+
+  /** The human readable name of the workflow. Not currently sent by Lightning. */
   name?: string;
+
   dataclip_id: string;
   starting_node_id: string;
 

packages/lexicon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/lexicon",
-  "version": "1.2.3",
+  "version": "1.2.4",
   "description": "Central repo of names and type definitions",
   "author": "Open Function Group <admin@openfn.org>",
   "license": "ISC",

packages/project/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @openfn/project
 
+## 0.5.1
+
+### Patch Changes
+
+- 81b97c3: Allow projects to be loaded directly from a file
+- 43be979: Fix an issue loading the ohm grammer
+- 25c7a2b: Allow a UUID map to be passed to the project generator
+- 04a89e2: Add support for dirs in config
+- 7d16875: When merging, allow the source project to specified as a path to a project file
+- Updated dependencies [1f8d65e]
+  - @openfn/lexicon@1.2.4
+
 ## 0.5.0
 
 ### Minor Changes

packages/project/README.md

@@ -51,10 +51,10 @@ a-b
 a-c
 ```
 
-You can set properties on the workflow itself - probably the name, with `@attributes`
+You can set properties on the workflow itself - probably the id, with `@attributes`
 
 ```
-@name my-cool-workflow
+@id my-cool-workflow
 ```
 
 You can also set properties on a node by putting comma seperated key-value pairs in brackets