Support channels in provisioner (#1412)

* feat(deploy): support channels in project state

Add ChannelSpec/ChannelState types, merge channels in mergeSpecIntoState
and mergeProjectPayloadIntoState, include them in toProjectPayload (drop
when empty), read them in getStateFromProjectPayload, and accept null
channels in the YAML validator.

* feat(project,lexicon): propagate channels through parse, serialize, merge

Add Provisioner.Channel type and channels field on Provisioner.Project_v1
in lexicon. Carry channels through Project, from-app-state, to-app-state,
to-project, and merge-project (REPLACE mode prefers source, falls back to
target; baseMerge treats channels as opaque like collections).

* add collections to portability projectspec

* style(deploy): format stateTransform with prettier

* move channel typedef into portability layer

* add extra tests

* tweak typings

* changesets

* types

---------

Co-authored-by: Joe Clark <jclark@openfn.org>

Author: midigofrank

.changeset/dull-meals-stare.md

@@ -0,0 +1,8 @@
+---
+'@openfn/lexicon': minor
+'@openfn/project': minor
+'@openfn/deploy': minor
+'@openfn/cli': minor
+---
+
+Add support for channels

packages/deploy/src/stateTransform.ts

@@ -374,6 +374,58 @@ export function mergeSpecIntoState(
       }
     )
   );
+  const nextChannels = Object.fromEntries(
+    splitZip(oldState.channels || {}, spec.channels || {}).map(
+      ([channelKey, stateChannel, specChannel]) => {
+        if (specChannel && !stateChannel) {
+          return [
+            channelKey,
+            {
+              id: crypto.randomUUID(),
+              name: specChannel.name,
+              destination_url: specChannel.destination_url,
+              enabled: specChannel.enabled,
+              destination_credential_id:
+                specChannel.destination_credential &&
+                getStateJobCredential(
+                  specChannel.destination_credential,
+                  nextCredentials
+                ),
+            },
+          ];
+        }
+
+        if (specChannel && stateChannel) {
+          return [
+            channelKey,
+            {
+              id: stateChannel.id,
+              name: specChannel.name,
+              destination_url: specChannel.destination_url,
+              enabled: specChannel.enabled,
+              destination_credential_id:
+                specChannel.destination_credential &&
+                getStateJobCredential(
+                  specChannel.destination_credential,
+                  nextCredentials
+                ),
+            },
+          ];
+        }
+
+        if (!specChannel && stateChannel) {
+          return [channelKey, { id: stateChannel.id, delete: true }];
+        }
+
+        throw new DeployError(
+          `Invalid channel spec or corrupted state for channel: ${
+            stateChannel?.name || specChannel?.name
+          }`,
+          'VALIDATION_ERROR'
+        );
+      }
+    )
+  );
 
   const nextWorkflows = Object.fromEntries(
     splitZip(oldState.workflows, spec.workflows).map(
@@ -442,6 +494,7 @@ export function mergeSpecIntoState(
     workflows: nextWorkflows,
     project_credentials: nextCredentials,
     collections: nextCollections,
+    channels: nextChannels,
   };
 
   if (spec.description) projectState.description = spec.description;
@@ -490,9 +543,12 @@ export function getStateFromProjectPayload(
 
   const collections = reduceByKey('name', project.collections || []);
 
+  const channels = reduceByKey('name', project.channels || []);
+
   return {
     ...project,
     collections,
+    channels,
     project_credentials,
     workflows,
   };
@@ -561,9 +617,18 @@ export function mergeProjectPayloadIntoState(
     )
   );
 
+  const nextChannels = Object.fromEntries(
+    idKeyPairs(project.channels || [], state.channels || {}).map(
+      ([key, nextChannel, _state]) => {
+        return [key, nextChannel];
+      }
+    )
+  );
+
   return {
     ...project,
     collections: nextCollections,
+    channels: nextChannels,
     project_credentials: nextCredentials,
     workflows: nextWorkflows,
   };
@@ -609,12 +674,17 @@ export function toProjectPayload(state: ProjectState): ProjectPayload {
     state.collections
   );
 
-  const { collections: _, ...stateWithoutCollections } = state;
+  const channels: ProjectPayload['channels'] = Object.values(
+    state.channels || {}
+  );
+
+  const { collections: _, channels: __, ...stateWithoutOptionals } = state;
 
   return {
-    ...stateWithoutCollections,
+    ...stateWithoutOptionals,
     project_credentials,
     workflows,
     ...(collections.length > 0 && { collections }),
+    ...(channels.length > 0 && { channels }),
   };
 }

packages/deploy/src/types.ts

@@ -117,12 +117,29 @@ export type CollectionState = {
   delete?: boolean;
 };
 
+export type ChannelSpec = {
+  name: string;
+  destination_url: string;
+  enabled: boolean;
+  destination_credential: string | null;
+};
+
+export type ChannelState = {
+  id: string;
+  name: string;
+  destination_url: string;
+  enabled: boolean;
+  destination_credential_id: string | null;
+  delete?: boolean;
+};
+
 export interface ProjectSpec {
   name: string;
   description: string;
   workflows: Record<string | symbol, WorkflowSpec>;
   credentials: Record<string | symbol, CredentialSpec>;
   collections: Record<string | symbol, CollectionSpec>;
+  channels: Record<string | symbol, ChannelSpec>;
 }
 
 export interface WorkflowState {
@@ -146,13 +163,15 @@ export interface ProjectState {
   workflows: Record<string | symbol, WorkflowState>;
   project_credentials: Record<string | symbol, CredentialState>;
   collections: Record<string | symbol, CollectionState>;
+  channels: Record<string | symbol, ChannelState>;
 }
 
 export interface ProjectPayload {
   id: string;
   name: string;
   description: string;
   collections?: Concrete<CollectionState>[];
+  channels?: Concrete<ChannelState>[];
   project_credentials: Concrete<CredentialState>[];
   workflows: {
     id: string;

packages/deploy/src/validator.ts

@@ -129,6 +129,12 @@ export async function parseAndValidate(
         }
       }
 
+      if (pair.key && pair.key.value === 'channels') {
+        if (pair.value.value === null) {
+          return doc.createPair('channels', {});
+        }
+      }
+
       if (pair.key && pair.key.value === 'jobs') {
         if (pair.value.value === null) {
           errors.push({

packages/deploy/test/fixtures.ts

@@ -5,6 +5,7 @@ export function fullExampleSpec() {
     name: 'my project',
     description: 'some helpful description',
     collections: {},
+    channels: {},
     credentials: {},
     workflows: {
       'workflow-one': {
@@ -58,6 +59,7 @@ export function fullExampleState() {
     name: 'my project',
     description: 'some helpful description',
     collections: {},
+    channels: {},
     project_credentials: {},
     workflows: {
       'workflow-one': {
@@ -254,6 +256,7 @@ export const lightningProjectState = {
       name: 'collection-one',
     },
   },
+  channels: {},
   project_credentials: {
     'email@test.com-Basic-Auth': {
       id: '25f48989-d349-4eb8-99c3-923ebba5b116',