Merge pull request #502 from DEFRA/feat/df-975-metrics

feat/df-975: Added report endpoint

Author: jbarnsley10

package-lock.json

@@ -44,7 +44,7 @@
     "@aws-sdk/client-sqs": "^3.982.0",
     "@aws-sdk/s3-request-presigner": "^3.679.0",
     "@defra/forms-engine-plugin": "^4.5.5",
-    "@defra/forms-model": "^3.0.642",
+    "@defra/forms-model": "^3.0.644",
     "@defra/hapi-tracing": "^1.12.0",
     "@elastic/ecs-pino-format": "^1.5.0",
     "@hapi/boom": "^10.0.1",

package.json

@@ -76,6 +76,10 @@
  * @property {number} numberOfRejectedFiles - Total number of files that have been rejected by the uploader
  */
 
+/**
+ * @typedef {{ Query: { date: Date }}} GetReportTimelineRequest
+ */
+
 /**
  * @import { SaveAndExitRecord } from '@defra/forms-model'
  * @import { FormAdapterSubmissionMessagePayload } from '@defra/forms-engine-plugin/engine/types.js'

src/api/types.js

@@ -69,6 +69,17 @@ export const getSubmissionByReferenceResponseSchema = Joi.object()
   .concat(formAdapterSubmissionMessagePayloadSchema)
   .label('getSubmissionByReferenceResponseSchema')
 
+export const generateReportTimelineResponseSchema = Joi.object({
+  timeline: Joi.array().items({
+    type: Joi.string().required(),
+    formId: Joi.string().required(),
+    formStatus: Joi.string().required(),
+    metricName: Joi.string().required(),
+    metricValue: Joi.number().required(),
+    createdAt: Joi.date().required()
+  })
+}).label('generateReportTimelineResponse')
+
 /**
  * @import { FormSubmissionDocument } from '~/src/api/types.js'
  */

src/models/form.js

@@ -7,6 +7,28 @@ export const STUB_FORM_ID = '688131eeff67f889d52c66cc'
 export const STUB_SUBMISSION_REF = '365-DFR-C67'
 export const STUB_SUBMISSION_RECORD_ID = '68d284ef5fa1a0fb2ede066a'
 
+/**
+ * @param {string} formId
+ * @param {FormStatus} formStatus
+ * @param {Date} timestamp
+ * @returns {WithId<FormSubmissionDocument>}
+ */
+export function buildCustomisedSubmissionDocument(
+  formId,
+  formStatus,
+  timestamp
+) {
+  const doc = /** @type {WithId<FormSubmissionDocument>} */ ({
+    ...buildFormAdapterSubmissionMessagePayloadStub(),
+    recordCreatedAt: new Date(),
+    expireAt: addDays(new Date(), 28)
+  })
+  doc.meta.formId = formId
+  doc.meta.timestamp = timestamp
+  doc.meta.status = formStatus
+  return doc
+}
+
 /**
  * @returns {WithId<FormSubmissionDocument>}
  */

src/repositories/__stubs__/submission.js

@@ -36,6 +36,43 @@ export function getSubmissionRecords(formId, filter) {
   }
 }
 
+/**
+ * Gets all submission records for a single day
+ * @param {Date} date - the specified day
+ * @returns { FindCursor<WithId<FormSubmissionDocument>> }
+ */
+export function getSubmissionRecordsForDate(date) {
+  logger.info(`Reading submission records for date ${date.toISOString()}`)
+
+  const coll = /** @type {Collection<FormSubmissionDocument>} */ (
+    db.collection(SUBMISSIONS_COLLECTION_NAME)
+  )
+
+  const withoutTime = date.toISOString().substring(0, 10)
+  const startOfDay = `${withoutTime}T00:00:00.000Z`
+  const endOfDay = `${withoutTime}T23:59:59.999Z`
+  try {
+    const result = coll
+      .find({
+        'meta.timestamp': {
+          $gte: new Date(startOfDay),
+          $lte: new Date(endOfDay)
+        }
+      })
+      .sort('meta.timestamp', 'desc')
+
+    logger.info(`Read submission records for date ${date.toISOString()}`)
+
+    return result
+  } catch (err) {
+    logger.error(
+      err,
+      `Failed to read submission records for date ${date.toISOString()} - ${getErrorMessage(err)}`
+    )
+    throw err
+  }
+}
+
 /**
  * Creates a form submission record
  * @param {FormSubmissionDocument} document

src/repositories/submission-repository.js

@@ -8,7 +8,8 @@ import {
 import {
   createSubmissionRecord,
   getSubmissionRecordByReference,
-  getSubmissionRecords
+  getSubmissionRecords,
+  getSubmissionRecordsForDate
 } from '~/src/repositories/submission-repository.js'
 
 const mockCollection = buildMockCollection()
@@ -125,4 +126,31 @@ describe('submission repository', () => {
       ).rejects.toThrow(new Error('an error'))
     })
   })
+
+  describe('getSubmissionRecordsForDate', () => {
+    it('should get submission records cursor', () => {
+      mockCollection.find.mockReturnValueOnce({
+        sort: jest.fn(() => {
+          return { next: () => submissionDocument }
+        })
+      })
+      const date = new Date('2026-02-15')
+      const submissionRecord = getSubmissionRecordsForDate(date)
+      expect(submissionRecord.next()).toEqual(submissionDocument)
+      expect(mockCollection.find).toHaveBeenCalledWith({
+        'meta.timestamp': {
+          $gte: new Date('2026-02-15T00:00:00.000Z'),
+          $lte: new Date('2026-02-15T23:59:59.999Z')
+        }
+      })
+    })
+
+    it('should log and throw when error', () => {
+      mockCollection.find.mockImplementationOnce(() => {
+        throw new Error('db error')
+      })
+      const date = new Date()
+      expect(() => getSubmissionRecordsForDate(date)).toThrow('db error')
+    })
+  })
 })

src/repositories/submission-repository.test.js

@@ -3,12 +3,14 @@ import Joi from 'joi'
 
 import {
   formSubmitResponseSchema,
+  generateReportTimelineResponseSchema,
   getSavedLinkGoneSchema,
   getSavedLinkResponseSchema,
   magicLinkSchema,
   validateSavedLinkResponseSchema
 } from '~/src/models/form.js'
 import { submit } from '~/src/services/file-service.js'
+import { generateReportTimeline } from '~/src/services/report.js'
 import {
   getSavedLinkDetails,
   validateSavedLinkCredentials
@@ -107,11 +109,40 @@ export default [
         }
       }
     }
+  }),
+
+  /**
+   * @type {ServerRoute<GetReportTimelineRequest>}
+   */
+  ({
+    method: 'GET',
+    path: '/report/timeline',
+    handler(request) {
+      const { date } = request.query
+
+      return generateReportTimeline(date)
+    },
+    options: {
+      tags: ['api'],
+      auth: false,
+      validate: {
+        query: Joi.object()
+          .keys({
+            date: Joi.date().required()
+          })
+          .label('getReportTimelineQuery')
+      },
+      response: {
+        status: {
+          200: generateReportTimelineResponseSchema
+        }
+      }
+    }
   })
 ]
 
 /**
  * @import { ServerRoute } from '@hapi/hapi'
  * @import { SubmitPayload } from '@defra/forms-model'
- * @import { GetSavedLinkParams, ValidateSaveAndExit } from '~/src/api/types.js'
+ * @import { GetSavedLinkParams, GetReportTimelineRequest, ValidateSaveAndExit } from '~/src/api/types.js'
  */

src/routes/form.js

@@ -4,6 +4,7 @@ import { StatusCodes } from 'http-status-codes'
 
 import { createServer } from '~/src/api/server.js'
 import { submit } from '~/src/services/file-service.js'
+import { generateReportTimeline } from '~/src/services/report.js'
 import {
   getSavedLinkDetails,
   validateSavedLinkCredentials
@@ -15,6 +16,7 @@ jest.mock('~/src/services/save-and-exit-service.js')
 jest.mock('~/src/tasks/receive-save-and-exit-messages.js')
 jest.mock('~/src/tasks/receive-submission-messages.js')
 jest.mock('~/src/services/submission-service.js')
+jest.mock('~/src/services/report.js')
 jest.mock('~/src/helpers/logging/logger.js', () => ({
   createLogger: () => ({
     error: jest.fn(),
@@ -253,6 +255,23 @@ describe('Forms route', () => {
       })
     })
   })
+
+  describe('report', () => {
+    test('Testing GET /report/timeline route returns data', async () => {
+      jest.mocked(generateReportTimeline).mockResolvedValueOnce({
+        timeline: []
+      })
+      const response = await server.inject({
+        method: 'GET',
+        url: '/report/timeline?date=2025-05-04'
+      })
+
+      expect(response.statusCode).toEqual(StatusCodes.OK)
+      expect(response.result).toMatchObject({
+        timeline: []
+      })
+    })
+  })
 })
 
 /**

src/routes/form.test.js

@@ -0,0 +1,106 @@
+import { FormMetricType, FormStatus, getErrorMessage } from '@defra/forms-model'
+
+import { createLogger } from '~/src/helpers/logging/logger.js'
+import { getSubmissionRecordsForDate } from '~/src/repositories/submission-repository.js'
+
+const logger = createLogger()
+
+/**
+ * @param {Map<string, number>} map
+ * @param {string} formId
+ */
+function incrementFormCount(map, formId) {
+  const current = map.get(formId) ?? 0
+  map.set(formId, current + 1)
+}
+
+/**
+ * @param {any[]} timelineMetrics
+ * @param {string} formId
+ * @param {FormStatus} formStatus
+ * @param {number} count
+ * @param {Date} date
+ */
+function pushTimelineMetric(timelineMetrics, formId, formStatus, count, date) {
+  timelineMetrics.push(
+    /** @type {FormTimelineMetric} */ ({
+      type: FormMetricType.TimelineMetric,
+      formId,
+      formStatus,
+      metricName: 'Submissions',
+      metricValue: count,
+      createdAt: date
+    })
+  )
+}
+
+/**
+ * Generates a set of timeline metrics
+ * @param {Date} date - date on which to gather the metrics for
+ */
+export async function generateReportTimeline(date) {
+  logger.info(
+    `[report] Generating timeline report for date ${date.toUTCString()}`
+  )
+
+  try {
+    const submissionsCursor = getSubmissionRecordsForDate(date)
+
+    const timelineMapDraft = new Map()
+    const timelineMapLive = new Map()
+
+    for await (const submission of submissionsCursor) {
+      const status = submission.meta.status
+      if (status === FormStatus.Draft) {
+        incrementFormCount(timelineMapDraft, submission.meta.formId)
+      } else {
+        incrementFormCount(timelineMapLive, submission.meta.formId)
+      }
+    }
+
+    const timelineMetrics = /** @type {FormTimelineMetric[]} */ ([])
+
+    if (timelineMapDraft.size) {
+      for (const [formId, count] of timelineMapDraft) {
+        pushTimelineMetric(
+          timelineMetrics,
+          formId,
+          FormStatus.Draft,
+          count,
+          date
+        )
+      }
+    }
+
+    if (timelineMapLive.size) {
+      for (const [formId, count] of timelineMapLive) {
+        pushTimelineMetric(
+          timelineMetrics,
+          formId,
+          FormStatus.Live,
+          count,
+          date
+        )
+      }
+    }
+
+    logger.info(
+      `[report] Generated timeline report for date ${date.toString()}`
+    )
+
+    return {
+      timeline: timelineMetrics
+    }
+  } catch (err) {
+    logger.error(
+      err,
+      `[report] Failed to generate timeline report for date ${date.toString()} - ${getErrorMessage(err)}`
+    )
+
+    throw err
+  }
+}
+
+/**
+ * @import { FormTimelineMetric } from '@defra/forms-model'
+ */