Allow blank file attachments in bulk upload

This commit allows for a bulk upload to proceed when a cell is blank
for a file. This was a requested feature for the service by @grant-minor-sntialtech
!

Issue #2429 Allow null or blank file attachments for some rows in bulk upload

Author: hminsky2002

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Fixed
+
+- Bulk upload processing no longer fails when a row leaves a file-typed field blank. The empty cell is stored as a non-file value (`value: ""`, `isValid: false`) and no attachment lookup is attempted for that field.
+
 ### Changed
 
 - Creating an entity now automatically grants the creator a `manage` permission with `any` scope on the new entity. This applies to opportunities, changemakers, proposals, sources, bulk upload tasks, application forms (and their fields), proposal versions (and their field values), and changemaker field values created via the HTTP API, as well as proposals, proposal versions, proposal field values, and newly inserted changemakers created during bulk upload processing.

src/tasks/__tests__/fixtures/processBulkUploadTask/validCsvTemplateWithBlankFile.csv

@@ -0,0 +1,5 @@
+proposal_submitter_email,organization_name,favorite_file
+foo@example.com,Foo LLC.,one.txt
+foo@example.com,Bar Inc.,
+foo@example.com,Baz Co.,"   "
+foo@example.com,Qux Ltd.,"  one.txt  "

src/tasks/__tests__/processBulkUploadTask.int.test.ts

@@ -748,6 +748,117 @@ describe('processBulkUploadTask', () => {
 		expect(updatedBulkUploadTask.status).toEqual(TaskStatus.COMPLETED);
 	});
 
+	it('should process rows with blank or whitespace-only file attachment cells without failing the bulk upload', async () => {
+		const db = getDatabase();
+		await createTestBaseFields(db);
+		const testAuthContext = await getTestAuthContext(db);
+		const systemUser = await loadSystemUser(db, null);
+		const systemUserAuthContext = getAuthContext(systemUser);
+		const proposalsDataFile = await createTestFile(db, systemUserAuthContext);
+		const attachmentsArchiveFile = await createTestFile(
+			db,
+			systemUserAuthContext,
+		);
+		const { applicationFormId } = await createTestApplicationForm(
+			db,
+			systemUserAuthContext,
+			['proposal_submitter_email', 'organization_name', 'favorite_file'],
+		);
+		const bulkUploadTask = await createTestBulkUploadTask(
+			db,
+			systemUserAuthContext,
+			{
+				proposalsDataFileId: proposalsDataFile.id,
+				applicationFormId,
+				overrideValues: {
+					attachmentsArchiveFileId: attachmentsArchiveFile.id,
+				},
+			},
+		);
+
+		s3Mock
+			.on(GetObjectCommand, { Key: proposalsDataFile.storageKey })
+			.resolves({
+				Body: sdkStreamMixin(
+					fs.createReadStream(
+						path.join(
+							__dirname,
+							'fixtures',
+							'processBulkUploadTask',
+							'validCsvTemplateWithBlankFile.csv',
+						),
+					),
+				),
+			});
+
+		s3Mock
+			.on(GetObjectCommand, { Key: attachmentsArchiveFile.storageKey })
+			.resolves({
+				Body: sdkStreamMixin(
+					fs.createReadStream(
+						path.join(
+							__dirname,
+							'fixtures',
+							'processBulkUploadTask',
+							'attachments.zip',
+						),
+					),
+				),
+			});
+
+		await processBulkUploadTask(
+			{ bulkUploadId: bulkUploadTask.id },
+			getMockJobHelpers(),
+		);
+
+		const updatedBulkUploadTask = await loadBulkUploadTask(
+			db,
+			testAuthContext,
+			bulkUploadTask.id,
+		);
+		expect(updatedBulkUploadTask.status).toEqual(TaskStatus.COMPLETED);
+
+		const proposalBundle = await loadProposalBundle(
+			db,
+			testAuthContext,
+			undefined,
+			undefined,
+			undefined,
+			undefined,
+			NO_LIMIT,
+			NO_OFFSET,
+		);
+		const favoriteFileValues = proposalBundle.entries
+			.flatMap((proposal) => proposal.versions)
+			.flatMap((version) => version.fieldValues)
+			.filter(
+				(fv) => fv.applicationFormField.baseFieldShortCode === 'favorite_file',
+			)
+			.sort((a, b) => a.proposalVersionId - b.proposalVersionId);
+		expect(favoriteFileValues).toEqual([
+			expectObjectContaining({
+				value: expectString(),
+				isValid: true,
+				file: expectObjectContaining({ name: 'one.txt' }),
+			}),
+			expectObjectContaining({
+				value: '',
+				isValid: false,
+				file: null,
+			}),
+			expectObjectContaining({
+				value: '   ',
+				isValid: false,
+				file: null,
+			}),
+			expectObjectContaining({
+				value: '  one.txt  ',
+				isValid: true,
+				file: expectObjectContaining({ name: 'one.txt' }),
+			}),
+		]);
+	});
+
 	it('should create changemakers and changemaker-proposal relationships', async () => {
 		const db = getDatabase();
 		await createTestBaseFields(db);

src/tasks/processBulkUploadTask.ts

@@ -632,14 +632,17 @@ export const processBulkUploadTask = async (
 
 						// File attachments are the one unique case where we need to convert the bulk upload value
 						// into a PDC File ID.  The value in the CSV is the relative path of the file within the
-						// attachments archive.
+						// attachments archive.  A blank or whitespace-only cell means no attachment for that row.
 						if (
 							applicationFormField.baseField.dataType === BaseFieldDataType.FILE
 						) {
-							const attachmentFile =
-								await attachmentsManager.getAttachmentFile(fieldValue);
-							processedFieldValue = attachmentFile.id.toString();
-							isValid = true;
+							const trimmedPath = fieldValue.trim();
+							if (trimmedPath !== '') {
+								const attachmentFile =
+									await attachmentsManager.getAttachmentFile(trimmedPath);
+								processedFieldValue = attachmentFile.id.toString();
+								isValid = true;
+							}
 						}
 
 						const proposalFieldValue = await createProposalFieldValue(