add option to send to another sns topic on a per-form basis

Author: alexluckett

docs/sns-per-form-topic-routing.md

@@ -0,0 +1,35 @@
+# SNS per-form topic routing
+
+By default, all form submissions are published to a single global SNS topic configured via `SNS_ADAPTER_TOPIC_ARN`, however we can additionally route specific forms to their own SNS topics.
+
+When a form is submitted, the runner always publishes to the global topic. If the submitted form's ID is present in the per-form topic map, the exact same payload is also published to the configured topic for that form.
+
+## Configuration
+
+Set the `SNS_FORM_TOPIC_ARN_MAP` environment variable to a JSON object that maps form IDs to SNS topic ARNs. Environment variables are managed in the [cdp-app-config](https://github.com/defra/cdp-app-config) repository.
+
+```
+SNS_FORM_TOPIC_ARN_MAP='{"<formId>":"<topicArn>"}'
+```
+
+### Example
+
+```
+SNS_FORM_TOPIC_ARN_MAP='{"abc123":"arn:aws:sns:eu-west-2:123456789012:my-form-topic","def456":"arn:aws:sns:eu-west-2:123456789012:another-form-topic"}'
+```
+
+In this example:
+
+- Form `abc123` publishes to the global topic and to `my-form-topic`
+- Form `def456` publishes to the global topic and to `another-form-topic`
+- All other forms publish to the global topic only
+
+## Future direction
+
+The longer-term intention is to move this configuration into Designer, allowing form authors to add additional outputs directly against a form without requiring an environment variable change. This approach was not implemented due to time constraints and a deadline. The environment variable solution described here is the interim approach until that work is prioritised.
+
+## Notes
+
+- The variable must be valid JSON. An invalid value will cause the application to fail on startup.
+- The payload published to the form-specific topic is identical to the one sent to the global topic.
+- If `SNS_FORM_TOPIC_ARN_MAP` is not set, all forms publish to the global topic only — there is no change in behaviour.

jest.setup.cjs

@@ -19,3 +19,5 @@ process.env.SNS_SAVE_TOPIC_ARN =
 process.env.SNS_ADAPTER_TOPIC_ARN =
   'arn:aws:sns:eu-west-2:123456789012:test-adapter-topic'
 process.env.SNS_ENDPOINT = 'http://localhost:4566'
+process.env.SNS_FORM_TOPIC_ARN_MAP =
+  '{"507f1f77bcf86cd799439099":"arn:aws:sns:eu-west-2:123456789012:form-specific-topic"}'

src/config/index.ts

@@ -184,6 +184,12 @@ export const config = convict({
     default: null,
     env: 'SNS_ENDPOINT'
   } as SchemaObj<string>,
+  snsFormTopicArnMap: {
+    doc: 'JSON object mapping formId to SNS topic ARN for per-form additional topic routing',
+    format: String,
+    default: '',
+    env: 'SNS_FORM_TOPIC_ARN_MAP'
+  } as SchemaObj<string>,
 
   /**
    * API integrations

src/server/messaging/formAdapterEventPublisher.test.js

@@ -24,6 +24,32 @@ async function expectValidationError(invalidPayload) {
 jest.mock('@aws-sdk/client-sns')
 jest.mock('~/src/server/messaging/sns.ts')
 
+/** @type {FormAdapterSubmissionMessagePayload} */
+const basePayload = {
+  meta: {
+    schemaVersion: 1,
+    timestamp: new Date(),
+    formId: '507f1f77bcf86cd799439011',
+    formSlug: 'test-form',
+    formName: 'Test Form',
+    referenceNumber: 'REF-123456',
+    status: 'live',
+    isPreview: false,
+    notificationEmail: 'test@example.com'
+  },
+  data: {
+    main: { field1: 'value1' },
+    repeaters: {},
+    files: {}
+  },
+  result: {
+    files: {
+      main: 'main-file-path',
+      repeaters: {}
+    }
+  }
+}
+
 describe('formAdapterEventPublisher', () => {
   /** @type {FormAdapterSubmissionMessagePayload} */
   let mockPayload
@@ -35,28 +61,8 @@ describe('formAdapterEventPublisher', () => {
     jest.clearAllMocks()
 
     mockPayload = /** @type {FormAdapterSubmissionMessagePayload} */ ({
-      meta: {
-        schemaVersion: 1,
-        timestamp: new Date(),
-        formId: '507f1f77bcf86cd799439011',
-        formSlug: 'test-form',
-        formName: 'Test Form',
-        referenceNumber: 'REF-123456',
-        status: 'live',
-        isPreview: false,
-        notificationEmail: 'test@example.com'
-      },
-      data: {
-        main: { field1: 'value1' },
-        repeaters: {},
-        files: {}
-      },
-      result: {
-        files: {
-          main: 'main-file-path',
-          repeaters: {}
-        }
-      }
+      ...basePayload,
+      meta: { ...basePayload.meta }
     })
 
     mockSnsClient = {
@@ -105,7 +111,7 @@ describe('formAdapterEventPublisher', () => {
       expect(mockSnsClient.send).toHaveBeenCalled()
     })
 
-    it('serializes entire payload as JSON message', async () => {
+    it('serialises entire payload as JSON message', async () => {
       mockSnsClient.send.mockResolvedValue({ MessageId: 'msg-789' })
 
       const complexPayload =
@@ -245,6 +251,76 @@ describe('formAdapterEventPublisher', () => {
   })
 })
 
+describe('per-form topic routing (SNS_FORM_TOPIC_ARN_MAP)', () => {
+  // formId must match the entry in jest.setup.cjs SNS_FORM_TOPIC_ARN_MAP
+  const formSpecificArn =
+    'arn:aws:sns:eu-west-2:123456789012:form-specific-topic'
+  const mappedFormId = '507f1f77bcf86cd799439099'
+
+  /** @type {FormAdapterSubmissionMessagePayload} */
+  const mappedPayload = {
+    ...basePayload,
+    meta: { ...basePayload.meta, formId: mappedFormId }
+  }
+
+  /** @type {any} */
+  let mockSnsClient
+
+  beforeEach(() => {
+    mockSnsClient = { send: jest.fn() }
+    jest.mocked(getSNSClient).mockReturnValue(mockSnsClient)
+  })
+
+  it('publishes to form-specific topic in addition to global topic when formId is mapped', async () => {
+    mockSnsClient.send
+      .mockResolvedValueOnce({ MessageId: 'global-msg-id' })
+      .mockResolvedValueOnce({ MessageId: 'form-specific-msg-id' })
+
+    const result = await publishFormAdapterEvent(mappedPayload)
+
+    expect(result).toBe('global-msg-id')
+    expect(mockSnsClient.send).toHaveBeenCalledTimes(2)
+    expect(PublishCommand).toHaveBeenNthCalledWith(1, {
+      TopicArn: 'arn:aws:sns:eu-west-2:123456789012:test-adapter-topic',
+      Message: JSON.stringify(mappedPayload)
+    })
+    expect(PublishCommand).toHaveBeenNthCalledWith(2, {
+      TopicArn: formSpecificArn,
+      Message: JSON.stringify(mappedPayload)
+    })
+  })
+
+  it('does not publish to form-specific topic when formId is not in the map', async () => {
+    mockSnsClient.send.mockResolvedValueOnce({ MessageId: 'global-msg-id' })
+
+    // basePayload uses formId 507f1f77bcf86cd799439011 which is not in the map
+    await publishFormAdapterEvent(basePayload)
+
+    expect(mockSnsClient.send).toHaveBeenCalledTimes(1)
+  })
+
+  it('throws when form-specific publish returns no MessageId', async () => {
+    mockSnsClient.send
+      .mockResolvedValueOnce({ MessageId: 'global-msg-id' })
+      .mockResolvedValueOnce({})
+
+    await expect(publishFormAdapterEvent(mappedPayload)).rejects.toThrow(
+      'Failed to publish form adapter event to form-specific topic - no message ID returned'
+    )
+  })
+
+  it('sends the exact same message string to both topics', async () => {
+    mockSnsClient.send
+      .mockResolvedValueOnce({ MessageId: 'global-msg-id' })
+      .mockResolvedValueOnce({ MessageId: 'form-specific-msg-id' })
+
+    await publishFormAdapterEvent(mappedPayload)
+
+    const calls = /** @type {any[]} */ (PublishCommand).mock.calls
+    expect(calls[0][0].Message).toBe(calls[1][0].Message)
+  })
+})
+
 /**
  * @import { FormAdapterSubmissionMessagePayload } from '@defra/forms-engine-plugin/engine/types.js'
  */

src/server/messaging/formAdapterEventPublisher.ts

@@ -8,6 +8,10 @@ import { getSNSClient } from '~/src/server/messaging/sns.js'
 
 const logger = createLogger()
 const snsAdapterTopicArn = config.get('snsAdapterTopicArn')
+const snsFormTopicArnMapRaw = config.get('snsFormTopicArnMap')
+const snsFormTopicArnMap: Record<string, string> = snsFormTopicArnMapRaw
+  ? (JSON.parse(snsFormTopicArnMapRaw) as Record<string, string>)
+  : {}
 
 /**
  * Validate form adapter submission payload against schema
@@ -55,12 +59,13 @@ export async function publishFormAdapterEvent(
   }
 
   const validatedPayload = validateFormAdapterPayload(submissionPayload)
+  const message = JSON.stringify(validatedPayload)
 
   const snsClient = getSNSClient()
   const result = await snsClient.send(
     new PublishCommand({
       TopicArn: snsAdapterTopicArn,
-      Message: JSON.stringify(validatedPayload)
+      Message: message
     })
   )
 
@@ -74,5 +79,25 @@ export async function publishFormAdapterEvent(
     `Published form adapter event for submission ${validatedPayload.meta.referenceNumber}. MessageId: ${result.MessageId}`
   )
 
+  const formSpecificTopicArn = snsFormTopicArnMap[validatedPayload.meta.formId]
+  if (formSpecificTopicArn) {
+    const formSpecificResult = await snsClient.send(
+      new PublishCommand({
+        TopicArn: formSpecificTopicArn,
+        Message: message
+      })
+    )
+
+    if (!formSpecificResult.MessageId) {
+      throw new Error(
+        'Failed to publish form adapter event to form-specific topic - no message ID returned'
+      )
+    }
+
+    logger.info(
+      `Published form adapter event to form-specific topic for submission ${validatedPayload.meta.referenceNumber}. MessageId: ${formSpecificResult.MessageId}`
+    )
+  }
+
   return result.MessageId
 }