feat: replace file attachment content by download link when retrieving submission (#202)

* feat: replace file attachment content by download link when retrieving submission

* feat: updade Node.js example to work with new submission attachment download links

Author: craigzour

examples/nodejs/index.ts

@@ -1,6 +1,6 @@
 import readline from "node:readline";
-import filesystem from "node:fs/promises";
 import type {
+  Attachment,
   FormSubmission,
   FormSubmissionProblem,
   PrivateApiKey,
@@ -10,6 +10,10 @@ import { GCFormsApiClient } from "./gcFormsApiClient.js";
 import { decryptFormSubmission } from "./formSubmissionDecrypter.js";
 import { verifyIntegrity } from "./formSubmissionIntegrityVerifier.js";
 import path from "node:path";
+import { createWriteStream } from "node:fs";
+import { mkdir, readdir, readFile, writeFile } from "node:fs/promises";
+import axios from "axios";
+import type { Readable } from "node:stream";
 
 const IDENTITY_PROVIDER_URL = "https://auth.forms-formulaires.alpha.canada.ca";
 const PROJECT_IDENTIFIER = "284778202772022819";
@@ -106,7 +110,9 @@ async function runRetrieveDecryptAndConfirmFormSubmissions(
       const endGetFormSubmissionTimer = performance.now();
 
       console.info(
-        `\nForm submission retrieved in ${Math.round(endGetFormSubmissionTimer - startGetFormSubmissionTimer)} ms`,
+        `\nForm submission retrieved in ${Math.round(
+          endGetFormSubmissionTimer - startGetFormSubmissionTimer,
+        )} ms`,
       );
 
       console.info("\nEncrypted submission:");
@@ -134,7 +140,9 @@ async function runRetrieveDecryptAndConfirmFormSubmissions(
       );
 
       console.info(
-        `\nIntegrity verification result: ${integrityVerificationResult ? "OK" : "INVALID"}`,
+        `\nIntegrity verification result: ${
+          integrityVerificationResult ? "OK" : "INVALID"
+        }`,
       );
 
       await saveSubmissionLocally(newFormSubmission.name, formSubmission);
@@ -165,36 +173,61 @@ async function saveSubmissionLocally(
 
   const submissionFolderPath = path.join("./", submissionName);
 
-  await filesystem.mkdir(submissionFolderPath, {
+  await mkdir(submissionFolderPath, {
     recursive: true,
   });
 
-  await filesystem.writeFile(
-    path.join(`./${submissionName}`, "answers.json"),
+  await writeFile(
+    path.join(submissionFolderPath, "answers.json"),
     submission.answers,
   );
 
   if (submission.attachments) {
     console.info("\nSaving submission attachments...\n");
 
-    for (const attachment of submission.attachments) {
-      const attachmentBuffer = Buffer.from(
-        attachment.base64EncodedContent,
-        "base64",
-      );
+    await Promise.all(
+      submission.attachments.map((attachment) =>
+        downloadAndSaveAttachment(attachment, submissionFolderPath),
+      ),
+    );
+  }
+
+  console.info(`\nSubmission saved in folder named '${submissionFolderPath}'`);
+}
 
-      await filesystem.writeFile(
-        path.join(`./${submissionName}`, attachment.name),
-        attachmentBuffer,
+async function downloadAndSaveAttachment(
+  attachment: Attachment,
+  submissionFolderPath: string,
+): Promise<void> {
+  return axios
+    .get<Readable>(attachment.downloadLink, { responseType: "stream" })
+    .then((response) => {
+      const writableStream = createWriteStream(
+        path.join(submissionFolderPath, attachment.name),
       );
 
+      return new Promise<void>((resolve, reject) => {
+        writableStream.on("finish", resolve);
+        writableStream.on("error", reject);
+
+        response.data.pipe(writableStream);
+      });
+    })
+    .then(() => {
       console.info(
-        `Submission attachment '${attachment.name}' has been saved ${attachment.isPotentiallyMalicious ? "(flagged as potentially malicious)" : ""}`,
+        `Submission attachment '${attachment.name}' has been saved ${
+          attachment.isPotentiallyMalicious
+            ? "(flagged as potentially malicious)"
+            : ""
+        }`,
       );
-    }
-  }
-
-  console.info(`\nSubmission saved in folder named '${submissionFolderPath}'`);
+    })
+    .catch((error) => {
+      throw new Error(
+        `Failed to download and save submission attachment ${attachment.name}`,
+        { cause: error },
+      );
+    });
 }
 
 async function runReportProblemWithFormSubmission(
@@ -240,8 +273,7 @@ async function runReportProblemWithFormSubmission(
 }
 
 function loadPrivateApiKey(): Promise<PrivateApiKey> {
-  return filesystem
-    .readdir(".")
+  return readdir(".")
     .then((files) => {
       const validFiles = files.filter((fileName) =>
         fileName.endsWith("_private_api_key.json"),
@@ -256,7 +288,7 @@ function loadPrivateApiKey(): Promise<PrivateApiKey> {
       return validFiles[0];
     })
     .then((privateApiKeyFileName) => {
-      return filesystem.readFile(`./${privateApiKeyFileName}`, {
+      return readFile(`./${privateApiKeyFileName}`, {
         encoding: "utf8",
       });
     })

examples/nodejs/types.d.ts

@@ -26,7 +26,7 @@ export enum FormSubmissionStatus {
 
 export type Attachment = {
   name: string;
-  base64EncodedContent: string;
+  downloadLink: string;
   isPotentiallyMalicious: boolean;
 };
 

package.json

@@ -24,6 +24,7 @@
     "@aws-sdk/client-secrets-manager": "^3.823.0",
     "@aws-sdk/client-sqs": "^3.823.0",
     "@aws-sdk/lib-dynamodb": "^3.823.0",
+    "@aws-sdk/s3-request-presigner": "^3.859.0",
     "cors": "^2.8.5",
     "dotenv": "^16.4.5",
     "express": "^4.21.0",
@@ -38,7 +39,6 @@
   "devDependencies": {
     "@aws-sdk/types": "^3.662.0",
     "@biomejs/biome": "^1.9.0",
-    "@smithy/util-stream": "^4.2.2",
     "@types/cors": "^2.8.18",
     "@types/express": "^4.17.21",
     "@types/node": "^22.7.4",

pnpm-lock.yaml

@@ -0,0 +1,25 @@
+import { AwsServicesConnector } from "@lib/integration/awsServicesConnector.js";
+import { GetObjectCommand } from "@aws-sdk/client-s3";
+import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
+import { VAULT_FILE_STORAGE_BUCKET_NAME } from "@config";
+import { logMessage } from "@lib/logging/logger.js";
+
+export function getFormSubmissionAttachmentDownloadLink(
+  path: string,
+): Promise<string> {
+  return getSignedUrl(
+    AwsServicesConnector.getInstance().s3Client,
+    new GetObjectCommand({
+      Bucket: VAULT_FILE_STORAGE_BUCKET_NAME,
+      Key: path,
+    }),
+    { expiresIn: 10 }, // 10 seconds
+  ).catch((error) => {
+    logMessage.info(
+      error,
+      `[s3] Failed to generate form submission attachment download link. Path: ${path}.`,
+    );
+
+    throw error;
+  });
+}

src/lib/vault/getFormSubmissionAttachmentContent.ts

@@ -4,16 +4,16 @@ import { encryptResponse } from "@lib/encryption/encryptResponse.js";
 import { auditLog } from "@lib/logging/auditLogs.js";
 import type { ApiOperation } from "@operations/types/operation.js";
 import { FormSubmissionNotFoundException } from "@lib/vault/types/exceptions.types.js";
-import { getFormSubmissionAttachmentContent } from "@lib/vault/getFormSubmissionAttachmentContent.js";
+import { getFormSubmissionAttachmentDownloadLink } from "@lib/vault/getFormSubmissionAttachmentDownloadLink.js";
 import { getPublicKey } from "@lib/formsClient/getPublicKey.js";
 import {
   AttachmentScanStatus,
   type FormSubmission,
   type PartialAttachment,
 } from "@lib/vault/types/formSubmission.types.js";
 
-type CompleteFormSubmissionAttachment = PartialAttachment & {
-  base64EncodedContent: string;
+type CompleteAttachment = PartialAttachment & {
+  downloadLink: string;
 };
 
 async function v1(
@@ -70,21 +70,21 @@ async function v1(
 
 function getCompleteFormSubmissionAttachments(
   partialAttachments: PartialAttachment[],
-): Promise<CompleteFormSubmissionAttachment[]> {
+): Promise<CompleteAttachment[]> {
   return Promise.all(
     partialAttachments.map((partialAttachment) => {
-      return getFormSubmissionAttachmentContent(partialAttachment.path).then(
-        ({ base64EncodedContent }) => {
-          return { ...partialAttachment, base64EncodedContent };
-        },
-      );
+      return getFormSubmissionAttachmentDownloadLink(
+        partialAttachment.path,
+      ).then((downloadLink) => {
+        return { ...partialAttachment, downloadLink };
+      });
     }),
   );
 }
 
 function buildJsonResponse(
   formSubmission: FormSubmission,
-  attachments: CompleteFormSubmissionAttachment[] | undefined,
+  attachments: CompleteAttachment[] | undefined,
 ): string {
   return JSON.stringify({
     createdAt: formSubmission.createdAt,
@@ -95,7 +95,7 @@ function buildJsonResponse(
     ...(attachments && {
       attachments: attachments.map((attachment) => ({
         name: attachment.name,
-        base64EncodedContent: attachment.base64EncodedContent,
+        downloadLink: attachment.downloadLink,
         isPotentiallyMalicious: isAttachmentPotentiallyMalicious(
           attachment.scanStatus,
         ),