Next CLI Release (#1413)

* deploy: support webhook_response in triggers

* update portability spec

* add canonical v2 test

* project: support extra state keys

* duplicate types

* add support for new keys in v2

* update version hash

* revert job code

* webhook_response -> webhook_response_config

* fix tests

* fix step complete

* update deploy test

* changesets

* enable type checking on project tests and fix a bunch of types

* fix more typings

* Support channels in provisioner (#1412)

* feat(deploy): support channels in project state

Add ChannelSpec/ChannelState types, merge channels in mergeSpecIntoState
and mergeProjectPayloadIntoState, include them in toProjectPayload (drop
when empty), read them in getStateFromProjectPayload, and accept null
channels in the YAML validator.

* feat(project,lexicon): propagate channels through parse, serialize, merge

Add Provisioner.Channel type and channels field on Provisioner.Project_v1
in lexicon. Carry channels through Project, from-app-state, to-app-state,
to-project, and merge-project (REPLACE mode prefers source, falls back to
target; baseMerge treats channels as opaque like collections).

* add collections to portability projectspec

* style(deploy): format stateTransform with prettier

* move channel typedef into portability layer

* add extra tests

* tweak typings

* changesets

* types

---------

Co-authored-by: Joe Clark <jclark@openfn.org>

* versions

---------

Co-authored-by: Frank Midigo <midigofrank@gmail.com>
Co-authored-by: Midigo Frank <39288959+midigofrank@users.noreply.github.com>

Author: josephjclark

.changeset/dull-meals-stare.md

@@ -0,0 +1,8 @@
+---
+'@openfn/lexicon': minor
+'@openfn/project': minor
+'@openfn/deploy': minor
+'@openfn/cli': minor
+---
+
+Add support for channels

.changeset/frank-canyons-report.md

@@ -0,0 +1,5 @@
+---
+'@openfn/deploy': minor
+---
+
+Add support for webhook_response_config

.changeset/salty-areas-juggle.md

@@ -0,0 +1,5 @@
+---
+'@openfn/lexicon': minor
+---
+
+Support `cron_cursor_job_id`, `webhook_reply` and `webhook_response_config` in Provisioner types

.changeset/shy-needles-attack.md

@@ -0,0 +1,5 @@
+---
+'@openfn/cli': minor
+---
+
+Support webhook responses in sync & deploy

.changeset/wide-seals-tease.md

@@ -0,0 +1,5 @@
+---
+'@openfn/project': patch
+---
+
+Support more trigger keys

packages/deploy/src/stateTransform.ts

@@ -164,6 +164,13 @@ function mergeTriggers(
           if (specTrigger.type === 'webhook' && specTrigger.webhook_reply) {
             trigger.webhook_reply = specTrigger.webhook_reply;
           }
+          if (
+            specTrigger.type === 'webhook' &&
+            specTrigger.webhook_response_config
+          ) {
+            trigger.webhook_response_config =
+              specTrigger.webhook_response_config;
+          }
 
           if (specTrigger.type === 'cron') {
             trigger.cron_expression = specTrigger.cron_expression;
@@ -202,6 +209,13 @@ function mergeTriggers(
         if (specTrigger!.type === 'webhook' && specTrigger!.webhook_reply) {
           trigger.webhook_reply = specTrigger!.webhook_reply;
         }
+        if (
+          specTrigger!.type === 'webhook' &&
+          specTrigger!.webhook_response_config
+        ) {
+          trigger.webhook_response_config =
+            specTrigger!.webhook_response_config;
+        }
 
         if (specTrigger!.type === 'cron') {
           trigger.cron_expression = specTrigger!.cron_expression;
@@ -360,6 +374,58 @@ export function mergeSpecIntoState(
       }
     )
   );
+  const nextChannels = Object.fromEntries(
+    splitZip(oldState.channels || {}, spec.channels || {}).map(
+      ([channelKey, stateChannel, specChannel]) => {
+        if (specChannel && !stateChannel) {
+          return [
+            channelKey,
+            {
+              id: crypto.randomUUID(),
+              name: specChannel.name,
+              destination_url: specChannel.destination_url,
+              enabled: specChannel.enabled,
+              destination_credential_id:
+                specChannel.destination_credential &&
+                getStateJobCredential(
+                  specChannel.destination_credential,
+                  nextCredentials
+                ),
+            },
+          ];
+        }
+
+        if (specChannel && stateChannel) {
+          return [
+            channelKey,
+            {
+              id: stateChannel.id,
+              name: specChannel.name,
+              destination_url: specChannel.destination_url,
+              enabled: specChannel.enabled,
+              destination_credential_id:
+                specChannel.destination_credential &&
+                getStateJobCredential(
+                  specChannel.destination_credential,
+                  nextCredentials
+                ),
+            },
+          ];
+        }
+
+        if (!specChannel && stateChannel) {
+          return [channelKey, { id: stateChannel.id, delete: true }];
+        }
+
+        throw new DeployError(
+          `Invalid channel spec or corrupted state for channel: ${
+            stateChannel?.name || specChannel?.name
+          }`,
+          'VALIDATION_ERROR'
+        );
+      }
+    )
+  );
 
   const nextWorkflows = Object.fromEntries(
     splitZip(oldState.workflows, spec.workflows).map(
@@ -428,6 +494,7 @@ export function mergeSpecIntoState(
     workflows: nextWorkflows,
     project_credentials: nextCredentials,
     collections: nextCollections,
+    channels: nextChannels,
   };
 
   if (spec.description) projectState.description = spec.description;
@@ -476,9 +543,12 @@ export function getStateFromProjectPayload(
 
   const collections = reduceByKey('name', project.collections || []);
 
+  const channels = reduceByKey('name', project.channels || []);
+
   return {
     ...project,
     collections,
+    channels,
     project_credentials,
     workflows,
   };
@@ -547,9 +617,18 @@ export function mergeProjectPayloadIntoState(
     )
   );
 
+  const nextChannels = Object.fromEntries(
+    idKeyPairs(project.channels || [], state.channels || {}).map(
+      ([key, nextChannel, _state]) => {
+        return [key, nextChannel];
+      }
+    )
+  );
+
   return {
     ...project,
     collections: nextCollections,
+    channels: nextChannels,
     project_credentials: nextCredentials,
     workflows: nextWorkflows,
   };
@@ -595,12 +674,17 @@ export function toProjectPayload(state: ProjectState): ProjectPayload {
     state.collections
   );
 
-  const { collections: _, ...stateWithoutCollections } = state;
+  const channels: ProjectPayload['channels'] = Object.values(
+    state.channels || {}
+  );
+
+  const { collections: _, channels: __, ...stateWithoutOptionals } = state;
 
   return {
-    ...stateWithoutCollections,
+    ...stateWithoutOptionals,
     project_credentials,
     workflows,
     ...(collections.length > 0 && { collections }),
+    ...(channels.length > 0 && { channels }),
   };
 }

packages/deploy/src/types.ts

@@ -38,13 +38,19 @@ export type SpecKafkaConfiguration = {
   connect_timeout: number;
 };
 
+export type WebhookResponseConfig = {
+  error_code: number | null;
+  success_code: number | null;
+};
+
 export type WebhookReply = 'before_start' | 'after_completion';
 
 export type SpecTrigger = {
   type: string;
   cron_expression?: string;
   cron_cursor_job?: string;
   webhook_reply?: WebhookReply;
+  webhook_response_config?: WebhookResponseConfig | null;
   enabled?: boolean;
   kafka_configuration?: SpecKafkaConfiguration;
 };
@@ -55,6 +61,7 @@ export type StateTrigger = {
   cron_expression?: string;
   cron_cursor_job_id?: string | null;
   webhook_reply?: WebhookReply;
+  webhook_response_config?: WebhookResponseConfig | null;
   delete?: boolean;
   enabled?: boolean;
   kafka_configuration?: StateKafkaConfiguration;
@@ -110,12 +117,29 @@ export type CollectionState = {
   delete?: boolean;
 };
 
+export type ChannelSpec = {
+  name: string;
+  destination_url: string;
+  enabled: boolean;
+  destination_credential: string | null;
+};
+
+export type ChannelState = {
+  id: string;
+  name: string;
+  destination_url: string;
+  enabled: boolean;
+  destination_credential_id: string | null;
+  delete?: boolean;
+};
+
 export interface ProjectSpec {
   name: string;
   description: string;
   workflows: Record<string | symbol, WorkflowSpec>;
   credentials: Record<string | symbol, CredentialSpec>;
   collections: Record<string | symbol, CollectionSpec>;
+  channels: Record<string | symbol, ChannelSpec>;
 }
 
 export interface WorkflowState {
@@ -139,13 +163,15 @@ export interface ProjectState {
   workflows: Record<string | symbol, WorkflowState>;
   project_credentials: Record<string | symbol, CredentialState>;
   collections: Record<string | symbol, CollectionState>;
+  channels: Record<string | symbol, ChannelState>;
 }
 
 export interface ProjectPayload {
   id: string;
   name: string;
   description: string;
   collections?: Concrete<CollectionState>[];
+  channels?: Concrete<ChannelState>[];
   project_credentials: Concrete<CredentialState>[];
   workflows: {
     id: string;

packages/deploy/src/validator.ts

@@ -129,6 +129,12 @@ export async function parseAndValidate(
         }
       }
 
+      if (pair.key && pair.key.value === 'channels') {
+        if (pair.value.value === null) {
+          return doc.createPair('channels', {});
+        }
+      }
+
       if (pair.key && pair.key.value === 'jobs') {
         if (pair.value.value === null) {
           errors.push({

packages/deploy/test/fixtures.ts

@@ -5,6 +5,7 @@ export function fullExampleSpec() {
     name: 'my project',
     description: 'some helpful description',
     collections: {},
+    channels: {},
     credentials: {},
     workflows: {
       'workflow-one': {
@@ -58,6 +59,7 @@ export function fullExampleState() {
     name: 'my project',
     description: 'some helpful description',
     collections: {},
+    channels: {},
     project_credentials: {},
     workflows: {
       'workflow-one': {
@@ -254,6 +256,7 @@ export const lightningProjectState = {
       name: 'collection-one',
     },
   },
+  channels: {},
   project_credentials: {
     'email@test.com-Basic-Auth': {
       id: '25f48989-d349-4eb8-99c3-923ebba5b116',