feat(website): XLSX submission template with Data + Guidance sheets and dropdown enforcement

integration-tests/tests/pages/revision.page.ts

@@ -72,7 +72,7 @@ export class RevisionPage {
      */
     async downloadTsvTemplate() {
         const downloadPromise = this.page.waitForEvent('download');
-        await this.page.getByText('TSV', { exact: true }).click();
+        await this.page.locator('a[href*="fileType=tsv"]').click();
         return downloadPromise;
     }
 
@@ -81,7 +81,7 @@ export class RevisionPage {
      */
     async downloadXlsxTemplate() {
         const downloadPromise = this.page.waitForEvent('download');
-        await this.page.getByText('XLSX', { exact: true }).click();
+        await this.page.locator('a[href*="fileType=xlsx"]').click();
         return downloadPromise;
     }
 

website/package.json

@@ -36,6 +36,7 @@
         "chart.js": "^4.5.1",
         "chartjs-adapter-date-fns": "^3.0.0",
         "cookie": "^1.1.1",
+        "exceljs": "^4.4.0",
         "fflate": "^0.8.3",
         "flowbite-react": "^0.12.17",
         "fzstd": "^0.1.1",

website/src/components/Submission/FileUpload/SequenceEntryUploadComponent.tsx

@@ -82,12 +82,12 @@ export const SequenceEntryUpload: FC<SequenceEntryUploadProps> = ({
                         metadata format
                     </a>{' '}
                     including a list of all supported metadata. You can download a{' '}
+                    <a href={routes.metadataTemplate(organism, action, 'xlsx')} className='text-primary-700 opacity-90'>
+                        XLSX template
+                    </a>
+                    {' (recommended) or a minimal '}
                     <a href={routes.metadataTemplate(organism, action, 'tsv')} className='text-primary-700 opacity-90'>
                         TSV
-                    </a>
-                    {' or '}
-                    <a href={routes.metadataTemplate(organism, action, 'xlsx')} className='text-primary-700 opacity-90'>
-                        XLSX
                     </a>{' '}
                     template with column headings for the metadata file.
                 </p>

website/src/components/Submission/FileUpload/fileProcessing.spec.ts

@@ -1,10 +1,23 @@
 import { fail } from 'assert';
 import { promises as fs } from 'fs';
 
+import ExcelJS from 'exceljs';
 import { describe, expect, test } from 'vitest';
 
 import { METADATA_FILE_KIND, PLAIN_SEGMENT_KIND } from './fileProcessing';
 
+async function buildWorkbookFile(extraSheetNames: string[]): Promise<File> {
+    const workbook = new ExcelJS.Workbook();
+    const dataSheet = workbook.addWorksheet('Data');
+    dataSheet.addRow(['submissionId', 'country']);
+    dataSheet.addRow(['sample1', 'Germany']);
+    for (const sheetName of extraSheetNames) {
+        workbook.addWorksheet(sheetName).addRow(['ignored']);
+    }
+    const buffer = await workbook.xlsx.writeBuffer();
+    return new File([buffer], 'template.xlsx');
+}
+
 async function loadTestFile(fileName: string): Promise<File> {
     const path = `${import.meta.dirname}/test_files/${fileName}`;
     const contents = await fs.readFile(path);
@@ -82,4 +95,40 @@ describe('fileProcessing', () => {
         expect(processedText).toBe('ACTGACTGACTG');
         expect(processedFile.fastaHeader()).toBe('fooid description');
     });
+
+    test('template reference sheets (Guidance, _lists) do not trigger a multi-sheet warning', async () => {
+        const file = await buildWorkbookFile(['Guidance', '_lists']);
+        const processingResult = await METADATA_FILE_KIND.processRawFile(file);
+
+        expect(processingResult.isOk()).toBe(true);
+        expect(processingResult._unsafeUnwrap().warnings()).toHaveLength(0);
+    });
+
+    test('an unexpected extra sheet still triggers a multi-sheet warning', async () => {
+        const file = await buildWorkbookFile(['Guidance', 'My other data']);
+        const processingResult = await METADATA_FILE_KIND.processRawFile(file);
+
+        expect(processingResult.isOk()).toBe(true);
+        expect(processingResult._unsafeUnwrap().warnings()).toHaveLength(1);
+    });
+
+    test('parses the Data sheet by name even when it is not the first sheet', async () => {
+        // Reference sheet dragged in front of Data — the Data sheet must still be the one parsed.
+        const workbook = new ExcelJS.Workbook();
+        workbook.addWorksheet('Guidance').addRow(['Field name', 'Display name']);
+        const data = workbook.addWorksheet('Data');
+        data.addRow(['submissionId', 'country']);
+        data.addRow(['sample1', 'Germany']);
+        const buffer = await workbook.xlsx.writeBuffer();
+        const file = new File([buffer], 'template.xlsx');
+
+        const processingResult = await METADATA_FILE_KIND.processRawFile(file);
+        expect(processingResult.isOk()).toBe(true);
+        const processedFile = processingResult._unsafeUnwrap();
+
+        expect(processedFile.warnings()).toHaveLength(0);
+        const text = await processedFile.text();
+        expect(text.split('\n')[0]).toContain('submissionId'); // parsed Data...
+        expect(text).not.toContain('Field name'); // ...not the Guidance reference sheet
+    });
 });

website/src/components/Submission/FileUpload/fileProcessing.ts

@@ -5,6 +5,7 @@ import JSZip from 'jszip';
 import { Result, ok, err } from 'neverthrow';
 import { type SVGProps, type ForwardRefExoticComponent } from 'react';
 
+import { DATA_SHEET_NAME, TEMPLATE_REFERENCE_SHEET_NAMES } from '../../../utils/metadataTemplateSheets';
 import MaterialSymbolsLightDataTableOutline from '~icons/material-symbols-light/data-table-outline';
 import PhDnaLight from '~icons/ph/dna-light';
 
@@ -241,8 +242,11 @@ export class ExcelFile implements ProcessedFile {
             cellDates: true,
         });
 
-        const firstSheetName = workbook.SheetNames[0];
-        let sheet = workbook.Sheets[firstSheetName];
+        // Parse the `Data` sheet by name when present, so reordering the template's sheets (e.g.
+        // dragging `Guidance` to the front) does not cause the reference sheet to be read as
+        // metadata. Fall back to the first sheet for arbitrary, non-template workbooks.
+        const dataSheetName = workbook.SheetNames.includes(DATA_SHEET_NAME) ? DATA_SHEET_NAME : workbook.SheetNames[0];
+        let sheet = workbook.Sheets[dataSheetName];
 
         // convert to JSON and back due to date formatting not working well otherwise
         const json = XLSX.utils.sheet_to_json(sheet);
@@ -258,16 +262,21 @@ export class ExcelFile implements ProcessedFile {
         });
         const rowCount = tsvContent.split('\n').length - 1;
         if (rowCount <= 0) {
-            throw new Error(`Sheet ${firstSheetName} is empty.`);
+            throw new Error(`Sheet ${dataSheetName} is empty.`);
         }
 
         const tsvBlob = new Blob([tsvContent], { type: 'text/tab-separated-values' });
         // filename needs to end in 'tsv' for the uploaded file
         const tsvFile = new File([tsvBlob], 'converted.tsv', { type: 'text/tab-separated-values' });
         this.tsvFile = tsvFile;
-        if (workbook.SheetNames.length > 1) {
+        // Any sheet other than the parsed one and the template's reference/lookup sheets is
+        // unexpected and should trigger the "you have unprocessed sheets" warning.
+        const unexpectedSheets = workbook.SheetNames.filter(
+            (sheetName) => sheetName !== dataSheetName && !TEMPLATE_REFERENCE_SHEET_NAMES.has(sheetName),
+        );
+        if (unexpectedSheets.length > 0) {
             this.processingWarnings.push(
-                `The file contains ${workbook.SheetNames.length} sheets, only the first sheet (${firstSheetName}; ${rowCount} rows) was processed.`,
+                `The file contains ${workbook.SheetNames.length} sheets, only the '${dataSheetName}' sheet (${rowCount} rows) was processed.`,
             );
         }
     }

website/src/config.ts

@@ -7,6 +7,7 @@ import { ACCESSION_FIELD, FASTA_IDS_FIELD, SUBMISSION_ID_INPUT_FIELD } from './s
 import {
     type InputField,
     type InstanceConfig,
+    type MetadataType,
     type Schema,
     type SequenceFlaggingConfig,
     type WebsiteConfig,
@@ -166,6 +167,52 @@ export function getMetadataTemplateFields(
     return fieldsToDisplaynames;
 }
 
+/**
+ * An {@link InputField} as it should appear in the downloadable submission template, tagged with
+ * whether it is one of the fields enabled by default (a "template field") and with the field's
+ * metadata `type` (used e.g. to format date columns). Default-enabled fields are ordered before the
+ * remaining, opt-in fields.
+ */
+export type TemplateInputField = InputField & { isTemplateField: boolean; metadataType?: MetadataType };
+
+/**
+ * Returns every submittable input field for the template download, in column order:
+ * submission-detail fields first, then the default-enabled "template" fields
+ * (`schema.metadataTemplate`, or all input fields if unset), then the remaining opt-in fields.
+ * Fields enabled by default are tagged with `isTemplateField: true`.
+ */
+export function getOrderedTemplateInputFields(organism: string, action: 'submit' | 'revise'): TemplateInputField[] {
+    const schema = getConfig(organism).schema;
+
+    const submissionIdInputFields = getSubmissionIdInputFields(schema);
+    const accessionFields = action === 'revise' ? [getAccessionInputField()] : [];
+    const detailFields = [...accessionFields, ...submissionIdInputFields];
+
+    const detailFieldNames = new Set(detailFields.map((field) => field.name));
+    const nonDetailFields = schema.inputFields.filter((field) => !detailFieldNames.has(field.name));
+    const fieldsByName = new Map(nonDetailFields.map((field) => [field.name, field]));
+
+    const templateFieldNames = schema.metadataTemplate ?? nonDetailFields.map((field) => field.name);
+    const templateFieldNameSet = new Set(templateFieldNames);
+    const templateFields = templateFieldNames
+        .map((name) => fieldsByName.get(name))
+        .filter((field): field is InputField => field !== undefined);
+    const restFields = nonDetailFields.filter((field) => !templateFieldNameSet.has(field.name));
+
+    const metadataTypeByName = new Map(schema.metadata.map((entry) => [entry.name, entry.type] as const));
+    const decorate = (field: InputField, isTemplateField: boolean): TemplateInputField => ({
+        ...field,
+        isTemplateField,
+        metadataType: metadataTypeByName.get(field.name),
+    });
+
+    return [
+        ...detailFields.map((field) => decorate(field, true)),
+        ...templateFields.map((field) => decorate(field, true)),
+        ...restFields.map((field) => decorate(field, false)),
+    ];
+}
+
 function getAccessionInputField(): InputField {
     const accessionPrefix = getWebsiteConfig().accessionPrefix;
     const instanceName = getWebsiteConfig().name;