From 777972d115605ceb27252f874261573a94eeb5f6 Mon Sep 17 00:00:00 2001 From: theosanderson-agent Date: Tue, 9 Jun 2026 15:23:11 +0100 Subject: [PATCH 01/16] feat(website): XLSX template with Data + Config sheets and dropdown enforcement Rework the downloadable XLSX submission template into a richer workbook: - `Data` sheet (first): every input field as a column, ordered template (default-enabled) fields first, then the remaining opt-in fields. Headers remain machine field names so the file round-trips through upload. - `Config` sheet (read-only): human-readable reference listing every field with its display name, whether it is enabled by default, whether it is required, and its allowed values. - `_lists` sheet (hidden): one column per field with a controlled vocabulary, the lookup source for dropdowns. Dropdown enforcement is a single, header-driven data validation over the whole Data grid: each cell reads its own column header and looks it up in `_lists`, so the dropdown follows a column even if the user reorders or renames it. Free-text columns are left unconstrained (the lookup errors, which Excel treats permissively). Generation switches to ExcelJS, which (unlike @lokalise/xlsx / SheetJS CE) can write data validations, hidden sheets and sheet protection. Upload parsing is unchanged and the multi-sheet upload warning now ignores the `Config`/`_lists` reference sheets. TSV output is unchanged. Refs #6636 Co-Authored-By: Claude Opus 4.8 (1M context) --- website/package-lock.json | 822 +++++++++++++++++- website/package.json | 1 + .../FileUpload/fileProcessing.spec.ts | 29 + .../Submission/FileUpload/fileProcessing.ts | 12 +- website/src/config.ts | 38 + .../submission/template/index.spec.ts | 172 ++++ .../[organism]/submission/template/index.ts | 150 +++- 7 files changed, 1202 insertions(+), 22 deletions(-) create mode 100644 website/src/pages/[organism]/submission/template/index.spec.ts diff --git a/website/package-lock.json b/website/package-lock.json index 1478a250ad..7e309125e1 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -26,6 +26,7 @@ "chart.js": "^4.5.1", "chartjs-adapter-date-fns": "^3.0.0", "cookie": "^1.1.1", + "exceljs": "^4.4.0", "fflate": "^0.8.3", "flowbite-react": "^0.12.17", "fzstd": "^0.1.1", @@ -1466,6 +1467,47 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@fast-csv/format": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-4.3.5.tgz", + "integrity": "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A==", + "license": "MIT", + "dependencies": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.isboolean": "^3.0.3", + "lodash.isequal": "^4.5.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0" + } + }, + "node_modules/@fast-csv/format/node_modules/@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==", + "license": "MIT" + }, + "node_modules/@fast-csv/parse": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-4.3.6.tgz", + "integrity": "sha512-uRsLYksqpbDmWaSmzvJcuApSEe38+6NQZBUsuAyMZKqHxH0g1wcJgsKUvN3WC8tewaqFjBMMGrkHmC+T7k8LvA==", + "license": "MIT", + "dependencies": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.groupby": "^4.6.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0", + "lodash.isundefined": "^3.0.1", + "lodash.uniq": "^4.5.0" + } + }, + "node_modules/@fast-csv/parse/node_modules/@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==", + "license": "MIT" + }, "node_modules/@floating-ui/core": { "version": "1.7.5", "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.7.5.tgz", @@ -5480,6 +5522,81 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/archiver": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-5.3.2.tgz", + "integrity": "sha512-+25nxyyznAXF7Nef3y0EbBeqmGZgeN/BxHX29Rs39djAfaFalmQ89SE6CWyDCHzGL0yt/ycBtNOmGTW0FyGWNw==", + "license": "MIT", + "dependencies": { + "archiver-utils": "^2.1.0", + "async": "^3.2.4", + "buffer-crc32": "^0.2.1", + "readable-stream": "^3.6.0", + "readdir-glob": "^1.1.2", + "tar-stream": "^2.2.0", + "zip-stream": "^4.1.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/archiver-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-2.1.0.tgz", + "integrity": "sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==", + "license": "MIT", + "dependencies": { + "glob": "^7.1.4", + "graceful-fs": "^4.2.0", + "lazystream": "^1.0.0", + "lodash.defaults": "^4.2.0", + "lodash.difference": "^4.5.0", + "lodash.flatten": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.union": "^4.6.0", + "normalize-path": "^3.0.0", + "readable-stream": "^2.0.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/archiver-utils/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/archiver-utils/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/archiver-utils/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/archiver-utils/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -6148,7 +6265,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true, "license": "MIT" }, "node_modules/base-64": { @@ -6157,6 +6273,26 @@ "integrity": "sha512-kwDPIFCGx0NZHog36dj+tHiwP4QMzsZ3AgMViUBKI0+V5n4U0ufTCUMhnQ04diaRI8EX/QcPfql7zlhZ7j4zgg==", "license": "MIT" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/bezier-easing": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/bezier-easing/-/bezier-easing-2.1.0.tgz", @@ -6164,6 +6300,45 @@ "dev": true, "license": "MIT" }, + "node_modules/big-integer": { + "version": "1.6.52", + "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.52.tgz", + "integrity": "sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==", + "license": "Unlicense", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/binary": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", + "integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==", + "license": "MIT", + "dependencies": { + "buffers": "~0.1.1", + "chainsaw": "~0.1.0" + }, + "engines": { + "node": "*" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/bluebird": { + "version": "3.4.7", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", + "integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==", + "license": "MIT" + }, "node_modules/boolbase": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", @@ -6284,12 +6459,62 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", "license": "BSD-3-Clause" }, + "node_modules/buffer-indexof-polyfill": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.2.tgz", + "integrity": "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A==", + "license": "MIT", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/buffers": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", + "integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==", + "engines": { + "node": ">=0.2.0" + } + }, "node_modules/call-bind": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", @@ -6400,6 +6625,18 @@ "node": ">=18" } }, + "node_modules/chainsaw": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", + "integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==", + "license": "MIT/X11", + "dependencies": { + "traverse": ">=0.3.0 <0.4" + }, + "engines": { + "node": "*" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -6704,11 +6941,25 @@ "integrity": "sha512-L3sHRo1pXXEqX8VU28kfgUY+YGsk09hPqZiZmLacNib6XNTCM8ubYeT7ryXQw8asB1sKgcU5lkB7ONug08aB8w==", "license": "ISC" }, + "node_modules/compress-commons": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-4.1.2.tgz", + "integrity": "sha512-D3uMHtGc/fcO1Gt1/L7i1e33VOvD4A9hfQLP+6ewd+BvG/gQ84Yh4oftEhAdjSMgBgwGL+jsppT7JYNpo6MHHg==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "^0.2.13", + "crc32-stream": "^4.0.2", + "normalize-path": "^3.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true, "license": "MIT" }, "node_modules/confbox": { @@ -6774,6 +7025,31 @@ } } }, + "node_modules/crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "license": "Apache-2.0", + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/crc32-stream": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-4.0.3.tgz", + "integrity": "sha512-NT7w2JVU7DFroFdYkeq8cywxrgjPHWkdX1wjpRQXPX5Asews3tA+Ght6lddQO5Mkumffp3X7GEqku3epj2toIw==", + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "readable-stream": "^3.4.0" + }, + "engines": { + "node": ">= 10" + } + }, "node_modules/cross-env": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-10.1.0.tgz", @@ -7327,6 +7603,51 @@ "node": ">= 0.4" } }, + "node_modules/duplexer2": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", + "integrity": "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==", + "license": "BSD-3-Clause", + "dependencies": { + "readable-stream": "^2.0.2" + } + }, + "node_modules/duplexer2/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/duplexer2/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/duplexer2/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/duplexer2/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/ecdsa-sig-formatter": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", @@ -7387,6 +7708,15 @@ "node": ">= 0.8" } }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/enhanced-resolve": { "version": "5.22.1", "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.22.1.tgz", @@ -8414,6 +8744,36 @@ "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==", "license": "MIT" }, + "node_modules/exceljs": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/exceljs/-/exceljs-4.4.0.tgz", + "integrity": "sha512-XctvKaEMaj1Ii9oDOqbW/6e1gXknSY4g/aLCDicOXqBE4M0nRWkUu0PTp++UPNzoFY12BNHMfs/VadKIS6llvg==", + "license": "MIT", + "dependencies": { + "archiver": "^5.0.0", + "dayjs": "^1.8.34", + "fast-csv": "^4.3.1", + "jszip": "^3.10.1", + "readable-stream": "^3.6.0", + "saxes": "^5.0.1", + "tmp": "^0.2.0", + "unzipper": "^0.10.11", + "uuid": "^8.3.0" + }, + "engines": { + "node": ">=8.3.0" + } + }, + "node_modules/exceljs/node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/expect-type": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", @@ -8436,6 +8796,19 @@ "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", "license": "MIT" }, + "node_modules/fast-csv": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-4.3.6.tgz", + "integrity": "sha512-2RNSpuwwsJGP0frGsOmTb9oUF+VkFSM4SyLTDgwf2ciHWTarN0lQTC+F2f/t5J9QjW+c65VFIAAu85GsvMIusw==", + "license": "MIT", + "dependencies": { + "@fast-csv/format": "4.3.5", + "@fast-csv/parse": "4.3.6" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -8790,6 +9163,18 @@ "node": ">= 0.8" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "license": "ISC" + }, "node_modules/fsevents": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", @@ -8804,6 +9189,22 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/fstream": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz", + "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==", + "deprecated": "This package is no longer supported.", + "license": "ISC", + "dependencies": { + "graceful-fs": "^4.1.2", + "inherits": "~2.0.0", + "mkdirp": ">=0.5 0", + "rimraf": "2" + }, + "engines": { + "node": ">=0.6" + } + }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", @@ -8942,6 +9343,27 @@ "integrity": "sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw==", "license": "ISC" }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -8955,6 +9377,28 @@ "node": ">=10.13.0" } }, + "node_modules/glob/node_modules/brace-expansion": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.15.tgz", + "integrity": "sha512-EwOCDEex4quD37XhqM3omwtMoJjr//isUZz1JopUNWms+4Z2ViyM/k1YIRePpoVNnQhENnxtFjLaxNHrT7xIUg==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/globals": { "version": "14.0.0", "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", @@ -9001,7 +9445,6 @@ "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true, "license": "ISC" }, "node_modules/graphql": { @@ -9544,6 +9987,26 @@ "node": ">= 6" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -9623,6 +10086,17 @@ "node": ">=8" } }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "license": "ISC", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -10541,6 +11015,54 @@ "dayjs": "^1.11.7" } }, + "node_modules/lazystream": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", + "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", + "license": "MIT", + "dependencies": { + "readable-stream": "^2.0.5" + }, + "engines": { + "node": ">= 0.6.3" + } + }, + "node_modules/lazystream/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/lazystream/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/lazystream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/lazystream/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -10835,6 +11357,12 @@ "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", "license": "MIT" }, + "node_modules/listenercount": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz", + "integrity": "sha512-3mk/Zag0+IJxeDrxSgaDPy4zZ3w05PRZeJNnlWhzFz5OkX49J4krc+A8X2d2M69vGMBEX0uyl8M+W+8gH+kBqQ==", + "license": "ISC" + }, "node_modules/local-pkg": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/local-pkg/-/local-pkg-1.2.1.tgz", @@ -10880,6 +11408,36 @@ "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==", "license": "MIT" }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "license": "MIT" + }, + "node_modules/lodash.difference": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", + "integrity": "sha512-dS2j+W26TQ7taQBGN8Lbbq04ssV3emRw4NY58WErlTO29pIqS0HmoT5aJ9+TUQ1N3G+JOZSji4eugsWwGp9yPA==", + "license": "MIT" + }, + "node_modules/lodash.escaperegexp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", + "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==", + "license": "MIT" + }, + "node_modules/lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==", + "license": "MIT" + }, + "node_modules/lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==", + "license": "MIT" + }, "node_modules/lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", @@ -10892,12 +11450,31 @@ "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==", "license": "MIT" }, + "node_modules/lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==", + "deprecated": "This package is deprecated. Use require('node:util').isDeepStrictEqual instead.", + "license": "MIT" + }, + "node_modules/lodash.isfunction": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", + "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==", + "license": "MIT" + }, "node_modules/lodash.isinteger": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==", "license": "MIT" }, + "node_modules/lodash.isnil": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz", + "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==", + "license": "MIT" + }, "node_modules/lodash.isnumber": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", @@ -10916,6 +11493,12 @@ "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==", "license": "MIT" }, + "node_modules/lodash.isundefined": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz", + "integrity": "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==", + "license": "MIT" + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -10929,6 +11512,18 @@ "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", "license": "MIT" }, + "node_modules/lodash.union": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz", + "integrity": "sha512-c4pB2CdGrGdjMKYLA+XiRDO7Y0PRQbm/Gzg8qMj+QH+pFVAoTp5sBpO0odL3FjoPCGjK96p6qsP+yQoiLoOBcw==", + "license": "MIT" + }, + "node_modules/lodash.uniq": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", + "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==", + "license": "MIT" + }, "node_modules/logform": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/logform/-/logform-2.7.0.tgz", @@ -12216,7 +12811,6 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -12228,6 +12822,18 @@ "integrity": "sha512-ihFnidEeU8iXzcVHy74dhkxh/dn8Dc08ERl0xwoMMGqp4+LvRSCgicb+zGqWthVokQKvCSxITlh3P08OzdTYCQ==", "license": "MIT" }, + "node_modules/mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "license": "MIT", + "dependencies": { + "minimist": "^1.2.6" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, "node_modules/mlly": { "version": "1.8.2", "resolved": "https://registry.npmjs.org/mlly/-/mlly-1.8.2.tgz", @@ -12636,6 +13242,15 @@ "node": ">= 0.8" } }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, "node_modules/one-time": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", @@ -12989,6 +13604,15 @@ "node": ">=8" } }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -13524,6 +14148,36 @@ "node": ">= 6" } }, + "node_modules/readdir-glob": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.3.tgz", + "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", + "license": "Apache-2.0", + "dependencies": { + "minimatch": "^5.1.0" + } + }, + "node_modules/readdir-glob/node_modules/brace-expansion": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.1.tgz", + "integrity": "sha512-WR1cURNjuvBLMZBMbqM0UoE+WAfdUcEV1ccD8PVBVOI+Z3ND4+SZbN8RsfT2bMuG1qwz5RFvPukSZm5fF2D5eA==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/readdir-glob/node_modules/minimatch": { + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", + "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/readdirp": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.0.2.tgz", @@ -14005,6 +14659,19 @@ "node": ">=0.10.0" } }, + "node_modules/rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "license": "ISC", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, "node_modules/rollup": { "version": "4.59.0", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.59.0.tgz", @@ -14301,6 +14968,18 @@ "node": ">=11.0.0" } }, + "node_modules/saxes": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz", + "integrity": "sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw==", + "license": "ISC", + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/scheduler": { "version": "0.23.2", "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", @@ -15191,6 +15870,22 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/text-hex": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", @@ -15265,6 +15960,15 @@ "dev": true, "license": "MIT" }, + "node_modules/tmp": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.7.tgz", + "integrity": "sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw==", + "license": "MIT", + "engines": { + "node": ">=14.14" + } + }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -15300,6 +16004,15 @@ "node": ">=16" } }, + "node_modules/traverse": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", + "integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==", + "license": "MIT/X11", + "engines": { + "node": "*" + } + }, "node_modules/trim-lines": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz", @@ -15834,6 +16547,60 @@ "url": "https://github.com/sponsors/kettanaito" } }, + "node_modules/unzipper": { + "version": "0.10.14", + "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.10.14.tgz", + "integrity": "sha512-ti4wZj+0bQTiX2KmKWuwj7lhV+2n//uXEotUmGuQqrbVZSEGFMbI68+c6JCQ8aAmUWYvtHEz2A8K6wXvueR/6g==", + "license": "MIT", + "dependencies": { + "big-integer": "^1.6.17", + "binary": "~0.3.0", + "bluebird": "~3.4.1", + "buffer-indexof-polyfill": "~1.0.0", + "duplexer2": "~0.1.4", + "fstream": "^1.0.12", + "graceful-fs": "^4.2.2", + "listenercount": "~1.0.1", + "readable-stream": "~2.3.6", + "setimmediate": "~1.0.4" + } + }, + "node_modules/unzipper/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/unzipper/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/unzipper/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/unzipper/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/update-browserslist-db": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.2.tgz", @@ -16736,6 +17503,12 @@ "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, "node_modules/ws": { "version": "8.19.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz", @@ -16758,6 +17531,12 @@ } } }, + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", + "license": "MIT" + }, "node_modules/xxhash-wasm": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-1.1.0.tgz", @@ -16966,6 +17745,41 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/zip-stream": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-4.1.1.tgz", + "integrity": "sha512-9qv4rlDiopXg4E69k+vMHjNN63YFMe9sZMrdlvKnCjlCRWeCBswPPMPUfx+ipsAWq1LXHe70RcbaHdJJpS6hyQ==", + "license": "MIT", + "dependencies": { + "archiver-utils": "^3.0.4", + "compress-commons": "^4.1.2", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/zip-stream/node_modules/archiver-utils": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-3.0.4.tgz", + "integrity": "sha512-KVgf4XQVrTjhyWmx6cte4RxonPLR9onExufI1jhvw/MQ4BB6IsZD5gT8Lq+u/+pRkWna/6JoHpiQioaqFP5Rzw==", + "license": "MIT", + "dependencies": { + "glob": "^7.2.3", + "graceful-fs": "^4.2.0", + "lazystream": "^1.0.0", + "lodash.defaults": "^4.2.0", + "lodash.difference": "^4.5.0", + "lodash.flatten": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.union": "^4.6.0", + "normalize-path": "^3.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } + }, "node_modules/zod": { "version": "3.25.76", "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", diff --git a/website/package.json b/website/package.json index 7e75931a72..b1f6553ce1 100644 --- a/website/package.json +++ b/website/package.json @@ -36,6 +36,7 @@ "chart.js": "^4.5.1", "chartjs-adapter-date-fns": "^3.0.0", "cookie": "^1.1.1", + "exceljs": "^4.4.0", "fflate": "^0.8.3", "flowbite-react": "^0.12.17", "fzstd": "^0.1.1", diff --git a/website/src/components/Submission/FileUpload/fileProcessing.spec.ts b/website/src/components/Submission/FileUpload/fileProcessing.spec.ts index 5dc4e8784b..39c346f394 100644 --- a/website/src/components/Submission/FileUpload/fileProcessing.spec.ts +++ b/website/src/components/Submission/FileUpload/fileProcessing.spec.ts @@ -1,10 +1,23 @@ import { fail } from 'assert'; import { promises as fs } from 'fs'; +import ExcelJS from 'exceljs'; import { describe, expect, test } from 'vitest'; import { METADATA_FILE_KIND, PLAIN_SEGMENT_KIND } from './fileProcessing'; +async function buildWorkbookFile(extraSheetNames: string[]): Promise { + const workbook = new ExcelJS.Workbook(); + const dataSheet = workbook.addWorksheet('Data'); + dataSheet.addRow(['submissionId', 'country']); + dataSheet.addRow(['sample1', 'Germany']); + for (const sheetName of extraSheetNames) { + workbook.addWorksheet(sheetName).addRow(['ignored']); + } + const buffer = await workbook.xlsx.writeBuffer(); + return new File([buffer], 'template.xlsx'); +} + async function loadTestFile(fileName: string): Promise { const path = `${import.meta.dirname}/test_files/${fileName}`; const contents = await fs.readFile(path); @@ -82,4 +95,20 @@ describe('fileProcessing', () => { expect(processedText).toBe('ACTGACTGACTG'); expect(processedFile.fastaHeader()).toBe('fooid description'); }); + + test('template reference sheets (Config, _lists) do not trigger a multi-sheet warning', async () => { + const file = await buildWorkbookFile(['Config', '_lists']); + const processingResult = await METADATA_FILE_KIND.processRawFile(file); + + expect(processingResult.isOk()).toBe(true); + expect(processingResult._unsafeUnwrap().warnings()).toHaveLength(0); + }); + + test('an unexpected extra sheet still triggers a multi-sheet warning', async () => { + const file = await buildWorkbookFile(['Config', 'My other data']); + const processingResult = await METADATA_FILE_KIND.processRawFile(file); + + expect(processingResult.isOk()).toBe(true); + expect(processingResult._unsafeUnwrap().warnings()).toHaveLength(1); + }); }); diff --git a/website/src/components/Submission/FileUpload/fileProcessing.ts b/website/src/components/Submission/FileUpload/fileProcessing.ts index f3c1000868..48081cdedb 100644 --- a/website/src/components/Submission/FileUpload/fileProcessing.ts +++ b/website/src/components/Submission/FileUpload/fileProcessing.ts @@ -19,6 +19,11 @@ export type FileKind = { const COMPRESSION_EXTENSIONS = ['zst', 'gz', 'zip', 'xz']; +// Sheets that the downloadable XLSX template adds alongside the `Data` sheet (see +// `pages/[organism]/submission/template/index.ts`). Kept in sync manually rather than imported, to +// avoid pulling the server-only template endpoint (and its dependencies) into the client bundle. +const TEMPLATE_REFERENCE_SHEET_NAMES = new Set(['Config', '_lists']); + export const METADATA_FILE_KIND: FileKind = { type: 'metadata', icon: MaterialSymbolsLightDataTableOutline, @@ -265,7 +270,12 @@ export class ExcelFile implements ProcessedFile { // filename needs to end in 'tsv' for the uploaded file const tsvFile = new File([tsvBlob], 'converted.tsv', { type: 'text/tab-separated-values' }); this.tsvFile = tsvFile; - if (workbook.SheetNames.length > 1) { + // Sheets that the downloadable template adds for reference/lookup purposes are expected and + // should not trigger the "you have unprocessed sheets" warning. + const unexpectedSheets = workbook.SheetNames.slice(1).filter( + (sheetName) => !TEMPLATE_REFERENCE_SHEET_NAMES.has(sheetName), + ); + if (unexpectedSheets.length > 0) { this.processingWarnings.push( `The file contains ${workbook.SheetNames.length} sheets, only the first sheet (${firstSheetName}; ${rowCount} rows) was processed.`, ); diff --git a/website/src/config.ts b/website/src/config.ts index 69a635febe..8b9bf6849d 100644 --- a/website/src/config.ts +++ b/website/src/config.ts @@ -166,6 +166,44 @@ export function getMetadataTemplateFields( return fieldsToDisplaynames; } +/** + * An {@link InputField} as it should appear in the downloadable submission template, tagged with + * whether it is one of the fields enabled by default (a "template field"). Default-enabled fields + * are ordered before the remaining, opt-in fields. + */ +export type TemplateInputField = InputField & { isTemplateField: boolean }; + +/** + * Returns every submittable input field for the template download, in column order: + * submission-detail fields first, then the default-enabled "template" fields + * (`schema.metadataTemplate`, or all input fields if unset), then the remaining opt-in fields. + * Fields enabled by default are tagged with `isTemplateField: true`. + */ +export function getOrderedTemplateInputFields(organism: string, action: 'submit' | 'revise'): TemplateInputField[] { + const schema = getConfig(organism).schema; + + const submissionIdInputFields = getSubmissionIdInputFields(schema); + const accessionFields = action === 'revise' ? [getAccessionInputField()] : []; + const detailFields = [...accessionFields, ...submissionIdInputFields]; + + const detailFieldNames = new Set(detailFields.map((field) => field.name)); + const nonDetailFields = schema.inputFields.filter((field) => !detailFieldNames.has(field.name)); + const fieldsByName = new Map(nonDetailFields.map((field) => [field.name, field])); + + const templateFieldNames = schema.metadataTemplate ?? nonDetailFields.map((field) => field.name); + const templateFieldNameSet = new Set(templateFieldNames); + const templateFields = templateFieldNames + .map((name) => fieldsByName.get(name)) + .filter((field): field is InputField => field !== undefined); + const restFields = nonDetailFields.filter((field) => !templateFieldNameSet.has(field.name)); + + return [ + ...detailFields.map((field) => ({ ...field, isTemplateField: true })), + ...templateFields.map((field) => ({ ...field, isTemplateField: true })), + ...restFields.map((field) => ({ ...field, isTemplateField: false })), + ]; +} + function getAccessionInputField(): InputField { const accessionPrefix = getWebsiteConfig().accessionPrefix; const instanceName = getWebsiteConfig().name; diff --git a/website/src/pages/[organism]/submission/template/index.spec.ts b/website/src/pages/[organism]/submission/template/index.spec.ts new file mode 100644 index 0000000000..3d6b72db8a --- /dev/null +++ b/website/src/pages/[organism]/submission/template/index.spec.ts @@ -0,0 +1,172 @@ +import type { APIContext } from 'astro'; +import ExcelJS from 'exceljs'; +import JSZip from 'jszip'; +import { describe, expect, test, vi } from 'vitest'; + +import { CONFIG_SHEET_NAME, DATA_SHEET_NAME, GET, LISTS_SHEET_NAME } from './index'; +import type { TemplateInputField } from '../../../../config'; + +const submissionDetailFields: TemplateInputField[] = [{ name: 'submissionId', required: true, isTemplateField: true }]; + +const templateFields: TemplateInputField[] = [ + { name: 'date', displayName: 'Collection date', required: true, isTemplateField: true }, + { + name: 'country', + displayName: 'Country', + isTemplateField: true, + options: [{ name: 'Germany' }, { name: 'France' }], + }, +]; + +const restFields: TemplateInputField[] = [ + { + name: 'host', + displayName: 'Host', + isTemplateField: false, + options: [{ name: 'Homo sapiens' }, { name: 'Sus scrofa' }], + }, + { name: 'notes', displayName: 'Notes', isTemplateField: false }, +]; + +const orderedFields = [...submissionDetailFields, ...templateFields, ...restFields]; + +vi.mock('../../../../config', () => ({ + getMetadataTemplateFields: () => + new Map([ + ['submissionId', undefined], + ['date', 'Collection date'], + ]), + getOrderedTemplateInputFields: (organism: string) => { + if (organism === 'test-organism') { + return orderedFields; + } + throw new Error(`Unknown organism: ${organism}`); + }, +})); + +vi.mock('../../../../components/Navigation/cleanOrganism', () => ({ + cleanOrganism: (rawOrganism: string) => { + if (rawOrganism === 'test-organism') { + return { organism: { key: 'test-organism', displayName: 'Test Organism' } }; + } + return { organism: undefined }; + }, +})); + +function callGet(organism: string, params: Record = {}): Promise { + const search = new URLSearchParams(params).toString(); + return GET({ + params: { organism }, + request: new Request(`http://localhost/${organism}/submission/template?${search}`), + } as unknown as APIContext) as Promise; +} + +async function loadWorkbook(response: Response): Promise { + const buffer = await response.arrayBuffer(); + const workbook = new ExcelJS.Workbook(); + await workbook.xlsx.load(buffer); + return workbook; +} + +describe('submission template API route', () => { + test('returns 404 for an unknown organism', async () => { + const response = await callGet('invalid-organism', { fileType: 'xlsx' }); + expect(response.status).toBe(404); + }); + + test('TSV template keeps the existing (template-only) field set and headers', async () => { + const response = await callGet('test-organism', { fileType: 'tsv' }); + + expect(response.headers.get('Content-Type')).toBe('text/tab-separated-values'); + expect(response.headers.get('Content-Disposition')).toBe( + 'attachment; filename="Test_Organism_metadata_template.tsv"', + ); + const text = await response.text(); + expect(text).toBe('submissionId\tdate\n'); + }); + + test('XLSX template has Data, Config and hidden _lists sheets, with Data first', async () => { + const response = await callGet('test-organism', { fileType: 'xlsx' }); + expect(response.headers.get('Content-Type')).toBe( + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + ); + + const workbook = await loadWorkbook(response); + const sheetNames = workbook.worksheets.map((sheet) => sheet.name); + expect(sheetNames[0]).toBe(DATA_SHEET_NAME); + expect(sheetNames).toContain(CONFIG_SHEET_NAME); + expect(sheetNames).toContain(LISTS_SHEET_NAME); + + expect(workbook.getWorksheet(LISTS_SHEET_NAME)!.state).toBe('hidden'); + }); + + test('Data sheet headers are machine names, template fields before the rest', async () => { + const response = await callGet('test-organism', { fileType: 'xlsx' }); + const workbook = await loadWorkbook(response); + + const headerRow = workbook.getWorksheet(DATA_SHEET_NAME)!.getRow(1); + const headers = (headerRow.values as unknown[]).slice(1); + expect(headers).toEqual(['submissionId', 'date', 'country', 'host', 'notes']); + }); + + test('_lists holds an options column per field with choices, keyed by field name', async () => { + const response = await callGet('test-organism', { fileType: 'xlsx' }); + const workbook = await loadWorkbook(response); + const listsSheet = workbook.getWorksheet(LISTS_SHEET_NAME)!; + + expect(listsSheet.getCell('A1').value).toBe('country'); + expect(listsSheet.getCell('A2').value).toBe('Germany'); + expect(listsSheet.getCell('A3').value).toBe('France'); + expect(listsSheet.getCell('B1').value).toBe('host'); + expect(listsSheet.getCell('B2').value).toBe('Homo sapiens'); + }); + + test('Data grid carries a single header-driven list validation pointing at _lists', async () => { + // Asserted against the raw written XML: ExcelJS' own re-read expands a range `sqref` into + // per-cell entries, so it cannot tell us the validation was written as one compact rule. + const response = await callGet('test-organism', { fileType: 'xlsx' }); + const zip = await JSZip.loadAsync(await response.arrayBuffer()); + // `Data` is the first sheet, so it is sheet1.xml. + const sheetXml = await zip.file('xl/worksheets/sheet1.xml')!.async('string'); + + const validationBlock = //.exec(sheetXml)?.[0] ?? ''; + expect(validationBlock).toContain('count="1"'); + expect(validationBlock).toContain('type="list"'); + // 5 columns (submissionId, date, country, host, notes) -> A..E. + expect(validationBlock).toContain('sqref="A2:E100000"'); + expect(validationBlock).toContain('MATCH(A$1'); + expect(validationBlock).toContain(LISTS_SHEET_NAME); + }); + + test('Config sheet lists every field with enabled/required flags and allowed values', async () => { + const response = await callGet('test-organism', { fileType: 'xlsx' }); + const workbook = await loadWorkbook(response); + const configSheet = workbook.getWorksheet(CONFIG_SHEET_NAME)!; + + expect((configSheet.getRow(1).values as unknown[]).slice(1)).toEqual([ + 'Field name', + 'Display name', + 'Enabled by default', + 'Required', + 'Allowed values', + ]); + + // country: enabled by default, optional, with allowed values + const countryRow = configSheet.getRow(4).values as unknown[]; + expect(countryRow.slice(1)).toEqual(['country', 'Country', 'Yes', 'No', 'Germany, France']); + + // host: opt-in (not a template field), free text label only when no options + const hostRow = configSheet.getRow(5).values as unknown[]; + expect(hostRow.slice(1, 4)).toEqual(['host', 'Host', 'No']); + + const notesRow = configSheet.getRow(6).values as unknown[]; + expect(notesRow[5]).toBe('free text'); + }); + + test('revision template filename includes "revision"', async () => { + const response = await callGet('test-organism', { fileType: 'xlsx', format: 'revise' }); + expect(response.headers.get('Content-Disposition')).toBe( + 'attachment; filename="Test_Organism_metadata_revision_template.xlsx"', + ); + }); +}); diff --git a/website/src/pages/[organism]/submission/template/index.ts b/website/src/pages/[organism]/submission/template/index.ts index f0f403b8d9..3146745836 100644 --- a/website/src/pages/[organism]/submission/template/index.ts +++ b/website/src/pages/[organism]/submission/template/index.ts @@ -1,9 +1,9 @@ -import * as XLSX from '@lokalise/xlsx'; import type { APIRoute } from 'astro'; +import ExcelJS from 'exceljs'; import { cleanOrganism } from '../../../../components/Navigation/cleanOrganism'; import type { UploadAction } from '../../../../components/Submission/DataUploadForm.tsx'; -import { getMetadataTemplateFields } from '../../../../config'; +import { getMetadataTemplateFields, getOrderedTemplateInputFields, type TemplateInputField } from '../../../../config'; export type TemplateFileType = 'tsv' | 'xlsx'; const VALID_FILE_TYPES = ['tsv', 'xlsx']; @@ -12,8 +12,26 @@ const CONTENT_TYPES = new Map([ ['xlsx', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'], ]); -/** The TSV template file that users can download from the submission page. */ -export const GET: APIRoute = ({ params, request }) => { +/** + * Sheet names of the XLSX template. `Data` MUST be added to the workbook first, because the upload + * parser only reads the first sheet. `Config` and `_lists` are recognised as reference sheets by + * the upload parser, which therefore does not warn about them (see `fileProcessing.ts`). + */ +export const DATA_SHEET_NAME = 'Data'; +export const CONFIG_SHEET_NAME = 'Config'; +export const LISTS_SHEET_NAME = '_lists'; + +/** + * The dropdown validation is applied to data rows 2..MAX_DATA_ROWS. We deliberately do not use + * Excel's full column height (1048576): the on-disk file is the same size either way, but a + * full-column `sqref` forces any tool that re-parses the workbook to materialise ~1M cells per + * column. 100k rows far exceeds a realistic single metadata file; rows beyond it simply lack the + * dropdown (free text), which is graceful degradation, not data loss. + */ +const MAX_DATA_ROWS = 100000; + +/** The TSV/XLSX template file that users can download from the submission page. */ +export const GET: APIRoute = async ({ params, request }) => { const rawOrganism = params.organism!; const { organism } = cleanOrganism(rawOrganism); if (organism === undefined) { @@ -38,27 +56,125 @@ export const GET: APIRoute = ({ params, request }) => { }; /* eslint-enable @typescript-eslint/naming-convention */ - const columnNames = Array.from(getMetadataTemplateFields(organism.key, action).keys()); - - const fileBuffer = createTemplateFile(fileType, columnNames); + const fileBuffer = + fileType === 'tsv' ? createTsvTemplate(organism.key, action) : await createXlsxTemplate(organism.key, action); return new Response(fileBuffer, { headers, }); }; -function createTemplateFile(fileType: TemplateFileType, columnNames: string[]): ArrayBuffer { - if (fileType === 'tsv') { - const content = columnNames.join('\t') + '\n'; - return new TextEncoder().encode(content).buffer; +function createTsvTemplate(organism: string, action: UploadAction): ArrayBuffer { + const columnNames = Array.from(getMetadataTemplateFields(organism, action).keys()); + const content = columnNames.join('\t') + '\n'; + return new TextEncoder().encode(content).buffer; +} + +/** + * Builds a workbook with three sheets: + * - `Data`: the sheet submitters fill in. Columns are the machine field names (so the file + * round-trips through upload), ordered template fields first, then the remaining opt-in fields. + * - `_lists` (hidden): one column per field that has a controlled vocabulary; the column header is + * the field name and the values below are its allowed options. This is the lookup source. + * - `Config` (read-only): a human-readable reference of every available field. + * + * Each `Data` cell carries a single, header-driven dropdown validation: the formula reads the cell's + * own column header and looks it up in `_lists`, so the dropdown follows a column even if the user + * reorders or renames it. Columns whose header is not in `_lists` (free-text fields) are left + * unconstrained because the lookup yields an error, which Excel treats permissively. + */ +async function createXlsxTemplate(organism: string, action: UploadAction): Promise { + const fields = getOrderedTemplateInputFields(organism, action); + const workbook = new ExcelJS.Workbook(); + + // --- Data sheet (must be added first) --- + const dataSheet = workbook.addWorksheet(DATA_SHEET_NAME); + dataSheet.addRow(fields.map((field) => field.name)); + dataSheet.getRow(1).font = { bold: true }; + + const optionFields = fields.filter((field) => (field.options?.length ?? 0) > 0); + + if (optionFields.length > 0) { + addOptionsLookup(workbook, dataSheet, fields, optionFields); } - const worksheetData = [columnNames]; // Add headers as the first row - const worksheet = XLSX.utils.aoa_to_sheet(worksheetData); + addConfigSheet(workbook, fields); - const workbook = XLSX.utils.book_new(); - XLSX.utils.book_append_sheet(workbook, worksheet, 'Template'); + const buffer = await workbook.xlsx.writeBuffer(); + return buffer; +} - const buffer = XLSX.write(workbook, { type: 'array', bookType: fileType }); - return new Uint8Array(buffer as number[]).buffer; +/** + * Adds the hidden `_lists` lookup sheet and the whole-grid, header-driven dropdown validation on the + * Data sheet. + */ +function addOptionsLookup( + workbook: ExcelJS.Workbook, + dataSheet: ExcelJS.Worksheet, + fields: TemplateInputField[], + optionFields: TemplateInputField[], +): void { + const listsSheet = workbook.addWorksheet(LISTS_SHEET_NAME, { state: 'hidden' }); + optionFields.forEach((field, columnIndex) => { + listsSheet.getCell(1, columnIndex + 1).value = field.name; + field.options!.forEach((option, optionIndex) => { + listsSheet.getCell(optionIndex + 2, columnIndex + 1).value = option.name; + }); + }); + void listsSheet.protect('', { selectLockedCells: false, selectUnlockedCells: false }); + + // The relative `A$1` resolves to each column's own header cell, so a single rule covers the whole + // grid: look the header up in the `_lists` header row and return that field's column of options. + const sheet = `'${LISTS_SHEET_NAME}'`; + const matchColumn = `MATCH(A$1,${sheet}!$1:$1,0)-1`; + const formula = + `OFFSET(${sheet}!$A$1,1,${matchColumn},` + + `COUNTA(OFFSET(${sheet}!$A$1,1,${matchColumn},${MAX_DATA_ROWS},1)),1)`; + const range = `A2:${columnLetter(fields.length)}${MAX_DATA_ROWS}`; + // `dataValidations.add` exists at runtime but is missing from ExcelJS' type definitions. Using it + // (rather than per-cell `cell.dataValidation`) writes one compact range rule instead of + // materialising a validation object for every cell in the grid. + const dataValidations = (dataSheet as unknown as { dataValidations: WorksheetDataValidations }).dataValidations; + dataValidations.add(range, { + type: 'list', + allowBlank: true, + formulae: [formula], + showErrorMessage: true, + errorStyle: 'stop', + errorTitle: 'Invalid value', + error: 'Please choose a value from the dropdown list for this field.', + }); +} + +interface WorksheetDataValidations { + add(sqref: string, validation: ExcelJS.DataValidation): void; +} + +/** Adds the read-only `Config` reference sheet listing every available field. */ +function addConfigSheet(workbook: ExcelJS.Workbook, fields: TemplateInputField[]): void { + const configSheet = workbook.addWorksheet(CONFIG_SHEET_NAME); + configSheet.addRow(['Field name', 'Display name', 'Enabled by default', 'Required', 'Allowed values']); + configSheet.getRow(1).font = { bold: true }; + fields.forEach((field) => { + configSheet.addRow([ + field.name, + field.displayName ?? '', + field.isTemplateField ? 'Yes' : 'No', + field.required === true ? 'Yes' : 'No', + (field.options ?? []).map((option) => option.name).join(', ') || 'free text', + ]); + }); + void configSheet.protect('', { selectLockedCells: true, selectUnlockedCells: true }); +} + +/** Converts a 1-based column index to its Excel column letters (1 -> A, 27 -> AA). */ +function columnLetter(columnIndex: number): string { + let letters = ''; + let remaining = columnIndex; + while (remaining > 0) { + const modulo = (remaining - 1) % 26; + letters = String.fromCharCode(65 + modulo) + letters; + remaining = Math.floor((remaining - modulo - 1) / 26); + } + return letters; } From 65c21d088b2413319b89932dd154ab65d5bdd492 Mon Sep 17 00:00:00 2001 From: theosanderson-agent Date: Tue, 9 Jun 2026 15:50:29 +0100 Subject: [PATCH 02/16] fix(website): per-column advisory dropdowns and wider template columns Only fields with a controlled vocabulary now get a dropdown, each pointing directly at that field's column in `_lists`. Free-text columns get no validation, so they accept any value (a single grid-wide `list` rule rejected free text, because list validation cannot express "allow anything"). The dropdowns are advisory rather than strict: a value typed outside the list warns ("Yes" keeps it) instead of being rejected. Also widen the Data and Config columns so headers and option values are readable. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../submission/template/index.spec.ts | 38 ++++++++--- .../[organism]/submission/template/index.ts | 63 +++++++++++-------- 2 files changed, 68 insertions(+), 33 deletions(-) diff --git a/website/src/pages/[organism]/submission/template/index.spec.ts b/website/src/pages/[organism]/submission/template/index.spec.ts index 3d6b72db8a..7396204b7a 100644 --- a/website/src/pages/[organism]/submission/template/index.spec.ts +++ b/website/src/pages/[organism]/submission/template/index.spec.ts @@ -121,21 +121,43 @@ describe('submission template API route', () => { expect(listsSheet.getCell('B2').value).toBe('Homo sapiens'); }); - test('Data grid carries a single header-driven list validation pointing at _lists', async () => { + test('only option columns get an advisory dropdown, each pointing at its _lists range', async () => { // Asserted against the raw written XML: ExcelJS' own re-read expands a range `sqref` into - // per-cell entries, so it cannot tell us the validation was written as one compact rule. + // per-cell entries, which makes the compact per-column rules unreadable. const response = await callGet('test-organism', { fileType: 'xlsx' }); const zip = await JSZip.loadAsync(await response.arrayBuffer()); // `Data` is the first sheet, so it is sheet1.xml. const sheetXml = await zip.file('xl/worksheets/sheet1.xml')!.async('string'); const validationBlock = //.exec(sheetXml)?.[0] ?? ''; - expect(validationBlock).toContain('count="1"'); - expect(validationBlock).toContain('type="list"'); - // 5 columns (submissionId, date, country, host, notes) -> A..E. - expect(validationBlock).toContain('sqref="A2:E100000"'); - expect(validationBlock).toContain('MATCH(A$1'); - expect(validationBlock).toContain(LISTS_SHEET_NAME); + // Only the two option fields (country -> col C, host -> col D) are validated; the free-text + // columns (submissionId/A, date/B, notes/E) get no validation so they accept any value. + expect(validationBlock).toContain('count="2"'); + expect(validationBlock).toContain('sqref="C2:C100000"'); + expect(validationBlock).toContain('sqref="D2:D100000"'); + expect(validationBlock).not.toContain('sqref="A2'); + expect(validationBlock).not.toContain('sqref="E2'); + // Advisory, not strict: a warning the user can dismiss, rather than a hard rejection. + expect(validationBlock).toContain('errorStyle="warning"'); + expect(validationBlock).not.toContain('errorStyle="stop"'); + // Each dropdown points directly at the field's column of options in `_lists` + // (the single quotes around the sheet name are XML-escaped to '). + expect(validationBlock).toContain(`${LISTS_SHEET_NAME}'!$A$2:$A$3`); + expect(validationBlock).toContain(`${LISTS_SHEET_NAME}'!$B$2:$B$3`); + }); + + test('Data columns are widened to fit field names and their options', async () => { + const response = await callGet('test-organism', { fileType: 'xlsx' }); + const workbook = await loadWorkbook(response); + const dataSheet = workbook.getWorksheet(DATA_SHEET_NAME)!; + + // Every column is at least the minimum width... + for (let column = 1; column <= 5; column++) { + expect(dataSheet.getColumn(column).width).toBeGreaterThanOrEqual(16); + } + // ...and the `host` column (col D) widens to fit "Homo sapiens" (12) -> 14 < 16, stays at 16, + // while a longer option would push it wider. Sanity-check it is a finite, bounded number. + expect(dataSheet.getColumn(4).width).toBeLessThanOrEqual(45); }); test('Config sheet lists every field with enabled/required flags and allowed values', async () => { diff --git a/website/src/pages/[organism]/submission/template/index.ts b/website/src/pages/[organism]/submission/template/index.ts index 3146745836..441944e25b 100644 --- a/website/src/pages/[organism]/submission/template/index.ts +++ b/website/src/pages/[organism]/submission/template/index.ts @@ -75,13 +75,14 @@ function createTsvTemplate(organism: string, action: UploadAction): ArrayBuffer * - `Data`: the sheet submitters fill in. Columns are the machine field names (so the file * round-trips through upload), ordered template fields first, then the remaining opt-in fields. * - `_lists` (hidden): one column per field that has a controlled vocabulary; the column header is - * the field name and the values below are its allowed options. This is the lookup source. + * the field name and the values below are its allowed options. This is the dropdown source. * - `Config` (read-only): a human-readable reference of every available field. * - * Each `Data` cell carries a single, header-driven dropdown validation: the formula reads the cell's - * own column header and looks it up in `_lists`, so the dropdown follows a column even if the user - * reorders or renames it. Columns whose header is not in `_lists` (free-text fields) are left - * unconstrained because the lookup yields an error, which Excel treats permissively. + * Only columns whose field has a controlled vocabulary get a dropdown validation, each pointing + * directly at that field's column in `_lists`. Free-text columns deliberately get no validation so + * they accept any value — a `list` validation cannot express "allow anything", so a single grid-wide + * rule would reject free text. Excel relocates a validation with its column when the user inserts or + * moves columns, so the dropdowns stay attached to the right field. */ async function createXlsxTemplate(organism: string, action: UploadAction): Promise { const fields = getOrderedTemplateInputFields(organism, action); @@ -91,6 +92,9 @@ async function createXlsxTemplate(organism: string, action: UploadAction): Promi const dataSheet = workbook.addWorksheet(DATA_SHEET_NAME); dataSheet.addRow(fields.map((field) => field.name)); dataSheet.getRow(1).font = { bold: true }; + fields.forEach((field, index) => { + dataSheet.getColumn(index + 1).width = columnWidthFor(field); + }); const optionFields = fields.filter((field) => (field.options?.length ?? 0) > 0); @@ -105,8 +109,8 @@ async function createXlsxTemplate(organism: string, action: UploadAction): Promi } /** - * Adds the hidden `_lists` lookup sheet and the whole-grid, header-driven dropdown validation on the - * Data sheet. + * Adds the hidden `_lists` lookup sheet and a dropdown validation for each field that has options, + * pointing at that field's column of allowed values in `_lists`. */ function addOptionsLookup( workbook: ExcelJS.Workbook, @@ -123,29 +127,35 @@ function addOptionsLookup( }); void listsSheet.protect('', { selectLockedCells: false, selectUnlockedCells: false }); - // The relative `A$1` resolves to each column's own header cell, so a single rule covers the whole - // grid: look the header up in the `_lists` header row and return that field's column of options. - const sheet = `'${LISTS_SHEET_NAME}'`; - const matchColumn = `MATCH(A$1,${sheet}!$1:$1,0)-1`; - const formula = - `OFFSET(${sheet}!$A$1,1,${matchColumn},` + - `COUNTA(OFFSET(${sheet}!$A$1,1,${matchColumn},${MAX_DATA_ROWS},1)),1)`; - const range = `A2:${columnLetter(fields.length)}${MAX_DATA_ROWS}`; // `dataValidations.add` exists at runtime but is missing from ExcelJS' type definitions. Using it - // (rather than per-cell `cell.dataValidation`) writes one compact range rule instead of - // materialising a validation object for every cell in the grid. + // (rather than per-cell `cell.dataValidation`) writes one compact range rule per option field + // instead of materialising a validation object for every cell. const dataValidations = (dataSheet as unknown as { dataValidations: WorksheetDataValidations }).dataValidations; - dataValidations.add(range, { - type: 'list', - allowBlank: true, - formulae: [formula], - showErrorMessage: true, - errorStyle: 'stop', - errorTitle: 'Invalid value', - error: 'Please choose a value from the dropdown list for this field.', + optionFields.forEach((field, listsIndex) => { + const dataColumn = columnLetter(fields.indexOf(field) + 1); + const listsColumn = columnLetter(listsIndex + 1); + const lastOptionRow = field.options!.length + 1; // row 1 is the header + const source = `'${LISTS_SHEET_NAME}'!$${listsColumn}$2:$${listsColumn}$${lastOptionRow}`; + dataValidations.add(`${dataColumn}2:${dataColumn}${MAX_DATA_ROWS}`, { + type: 'list', + allowBlank: true, + formulae: [source], + // Advisory, not strict: the dropdown offers the controlled vocabulary, but a value typed + // outside the list only warns ("Yes" keeps it) rather than being rejected. + showErrorMessage: true, + errorStyle: 'warning', + errorTitle: 'Value not in the suggested list', + error: `"${field.name}" has a suggested list of values. You can pick one from the dropdown, or keep what you typed.`, + }); }); } +/** A roomy-but-bounded Data column width that fits the field name and its longest dropdown option. */ +function columnWidthFor(field: TemplateInputField): number { + const longestOption = (field.options ?? []).reduce((max, option) => Math.max(max, option.name.length), 0); + return Math.min(45, Math.max(16, field.name.length + 2, longestOption + 2)); +} + interface WorksheetDataValidations { add(sqref: string, validation: ExcelJS.DataValidation): void; } @@ -155,6 +165,9 @@ function addConfigSheet(workbook: ExcelJS.Workbook, fields: TemplateInputField[] const configSheet = workbook.addWorksheet(CONFIG_SHEET_NAME); configSheet.addRow(['Field name', 'Display name', 'Enabled by default', 'Required', 'Allowed values']); configSheet.getRow(1).font = { bold: true }; + [28, 28, 18, 12, 60].forEach((width, index) => { + configSheet.getColumn(index + 1).width = width; + }); fields.forEach((field) => { configSheet.addRow([ field.name, From f1097f43e403e250bdb5f2a267188eaaf8506622 Mon Sep 17 00:00:00 2001 From: theosanderson-agent Date: Tue, 9 Jun 2026 15:55:07 +0100 Subject: [PATCH 03/16] feat(website): make option dropdowns header-driven so they follow renamed columns Each option field's dropdown now resolves its allowed values by looking its own column header up in `_lists` (via MATCH/OFFSET) instead of referencing a fixed `_lists` range. The header reference is column-relative to the validation range, so the dropdown follows the field when the column is renamed or moved. Still per-column and advisory: free-text columns get no validation (so they never warn), and off-list values on option columns warn rather than reject. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../submission/template/index.spec.ts | 11 +++---- .../[organism]/submission/template/index.ts | 30 ++++++++++++------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/website/src/pages/[organism]/submission/template/index.spec.ts b/website/src/pages/[organism]/submission/template/index.spec.ts index 7396204b7a..ec9702b083 100644 --- a/website/src/pages/[organism]/submission/template/index.spec.ts +++ b/website/src/pages/[organism]/submission/template/index.spec.ts @@ -121,7 +121,7 @@ describe('submission template API route', () => { expect(listsSheet.getCell('B2').value).toBe('Homo sapiens'); }); - test('only option columns get an advisory dropdown, each pointing at its _lists range', async () => { + test('only option columns get an advisory, header-driven dropdown looking up _lists', async () => { // Asserted against the raw written XML: ExcelJS' own re-read expands a range `sqref` into // per-cell entries, which makes the compact per-column rules unreadable. const response = await callGet('test-organism', { fileType: 'xlsx' }); @@ -140,10 +140,11 @@ describe('submission template API route', () => { // Advisory, not strict: a warning the user can dismiss, rather than a hard rejection. expect(validationBlock).toContain('errorStyle="warning"'); expect(validationBlock).not.toContain('errorStyle="stop"'); - // Each dropdown points directly at the field's column of options in `_lists` - // (the single quotes around the sheet name are XML-escaped to '). - expect(validationBlock).toContain(`${LISTS_SHEET_NAME}'!$A$2:$A$3`); - expect(validationBlock).toContain(`${LISTS_SHEET_NAME}'!$B$2:$B$3`); + // Header-driven: each validation looks up its own column header (C$1 / D$1) in `_lists`, + // so the dropdown follows the field if the column is renamed or moved. + expect(validationBlock).toContain('MATCH(C$1'); + expect(validationBlock).toContain('MATCH(D$1'); + expect(validationBlock).toContain(LISTS_SHEET_NAME); }); test('Data columns are widened to fit field names and their options', async () => { diff --git a/website/src/pages/[organism]/submission/template/index.ts b/website/src/pages/[organism]/submission/template/index.ts index 441944e25b..b129807bc1 100644 --- a/website/src/pages/[organism]/submission/template/index.ts +++ b/website/src/pages/[organism]/submission/template/index.ts @@ -78,11 +78,12 @@ function createTsvTemplate(organism: string, action: UploadAction): ArrayBuffer * the field name and the values below are its allowed options. This is the dropdown source. * - `Config` (read-only): a human-readable reference of every available field. * - * Only columns whose field has a controlled vocabulary get a dropdown validation, each pointing - * directly at that field's column in `_lists`. Free-text columns deliberately get no validation so - * they accept any value — a `list` validation cannot express "allow anything", so a single grid-wide - * rule would reject free text. Excel relocates a validation with its column when the user inserts or - * moves columns, so the dropdowns stay attached to the right field. + * Only columns whose field has a controlled vocabulary get a dropdown validation. Each such + * validation is header-driven: its source formula reads that column's own header cell and looks it + * up in `_lists`, so the dropdown follows the field even if the user renames or reorders the column. + * Free-text columns deliberately get no validation so they accept any value — a `list` validation + * cannot express "allow anything", so applying a header-driven rule to them would warn on every + * entry (the lookup errors). Limiting it to option columns keeps free-text columns silent. */ async function createXlsxTemplate(organism: string, action: UploadAction): Promise { const fields = getOrderedTemplateInputFields(organism, action); @@ -109,8 +110,9 @@ async function createXlsxTemplate(organism: string, action: UploadAction): Promi } /** - * Adds the hidden `_lists` lookup sheet and a dropdown validation for each field that has options, - * pointing at that field's column of allowed values in `_lists`. + * Adds the hidden `_lists` lookup sheet and a header-driven dropdown validation for each field that + * has options. The validation reads its column's own header and resolves that field's option list in + * `_lists`, so the dropdown follows the field when the column is renamed or moved. */ function addOptionsLookup( workbook: ExcelJS.Workbook, @@ -131,11 +133,17 @@ function addOptionsLookup( // (rather than per-cell `cell.dataValidation`) writes one compact range rule per option field // instead of materialising a validation object for every cell. const dataValidations = (dataSheet as unknown as { dataValidations: WorksheetDataValidations }).dataValidations; - optionFields.forEach((field, listsIndex) => { + const sheet = `'${LISTS_SHEET_NAME}'`; + optionFields.forEach((field) => { const dataColumn = columnLetter(fields.indexOf(field) + 1); - const listsColumn = columnLetter(listsIndex + 1); - const lastOptionRow = field.options!.length + 1; // row 1 is the header - const source = `'${LISTS_SHEET_NAME}'!$${listsColumn}$2:$${listsColumn}$${lastOptionRow}`; + // Reference to this column's own header cell. It is column-relative to the validation range, + // so Excel keeps it pointing at the header even after the column is moved. + const headerCell = `${dataColumn}$1`; + // Find the field's column in `_lists` by matching the header text, then return its options. + const matchOffset = `MATCH(${headerCell},${sheet}!$1:$1,0)-1`; + const source = + `OFFSET(${sheet}!$A$1,1,${matchOffset},` + + `COUNTA(OFFSET(${sheet}!$A$1,1,${matchOffset},${MAX_DATA_ROWS},1)),1)`; dataValidations.add(`${dataColumn}2:${dataColumn}${MAX_DATA_ROWS}`, { type: 'list', allowBlank: true, From 41b767fa182135cdf2a2bfa16bef77ba2a7e7071 Mon Sep 17 00:00:00 2001 From: theosanderson-agent Date: Tue, 9 Jun 2026 16:04:35 +0100 Subject: [PATCH 04/16] =?UTF-8?q?feat(website):=20polish=20XLSX=20template?= =?UTF-8?q?=20=E2=80=94=20header=20notes,=20colour=20tiers,=20freeze,=20da?= =?UTF-8?q?te=20format?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the generated template friendlier to fill in: - Header hover notes carrying each field's definition, guidance and example. - Colour-coded headers by tier (required / included-by-default / opt-in), with a matching colour legend on the Config sheet. - Frozen header row and an auto-filter on the Data sheet. - Date-typed columns formatted as yyyy-mm-dd (the format upload expects), via a new `metadataType` on the ordered template fields. - Config header row styled and frozen. Co-Authored-By: Claude Opus 4.8 (1M context) --- website/src/config.ts | 21 ++++-- .../submission/template/index.spec.ts | 50 ++++++++++++- .../[organism]/submission/template/index.ts | 72 +++++++++++++++++-- 3 files changed, 131 insertions(+), 12 deletions(-) diff --git a/website/src/config.ts b/website/src/config.ts index 8b9bf6849d..ae2d4dda54 100644 --- a/website/src/config.ts +++ b/website/src/config.ts @@ -7,6 +7,7 @@ import { ACCESSION_FIELD, FASTA_IDS_FIELD, SUBMISSION_ID_INPUT_FIELD } from './s import { type InputField, type InstanceConfig, + type MetadataType, type Schema, type SequenceFlaggingConfig, type WebsiteConfig, @@ -168,10 +169,11 @@ export function getMetadataTemplateFields( /** * An {@link InputField} as it should appear in the downloadable submission template, tagged with - * whether it is one of the fields enabled by default (a "template field"). Default-enabled fields - * are ordered before the remaining, opt-in fields. + * whether it is one of the fields enabled by default (a "template field") and with the field's + * metadata `type` (used e.g. to format date columns). Default-enabled fields are ordered before the + * remaining, opt-in fields. */ -export type TemplateInputField = InputField & { isTemplateField: boolean }; +export type TemplateInputField = InputField & { isTemplateField: boolean; metadataType?: MetadataType }; /** * Returns every submittable input field for the template download, in column order: @@ -197,10 +199,17 @@ export function getOrderedTemplateInputFields(organism: string, action: 'submit' .filter((field): field is InputField => field !== undefined); const restFields = nonDetailFields.filter((field) => !templateFieldNameSet.has(field.name)); + const metadataTypeByName = new Map(schema.metadata.map((entry) => [entry.name, entry.type] as const)); + const decorate = (field: InputField, isTemplateField: boolean): TemplateInputField => ({ + ...field, + isTemplateField, + metadataType: metadataTypeByName.get(field.name), + }); + return [ - ...detailFields.map((field) => ({ ...field, isTemplateField: true })), - ...templateFields.map((field) => ({ ...field, isTemplateField: true })), - ...restFields.map((field) => ({ ...field, isTemplateField: false })), + ...detailFields.map((field) => decorate(field, true)), + ...templateFields.map((field) => decorate(field, true)), + ...restFields.map((field) => decorate(field, false)), ]; } diff --git a/website/src/pages/[organism]/submission/template/index.spec.ts b/website/src/pages/[organism]/submission/template/index.spec.ts index ec9702b083..4a4774d2b0 100644 --- a/website/src/pages/[organism]/submission/template/index.spec.ts +++ b/website/src/pages/[organism]/submission/template/index.spec.ts @@ -9,11 +9,13 @@ import type { TemplateInputField } from '../../../../config'; const submissionDetailFields: TemplateInputField[] = [{ name: 'submissionId', required: true, isTemplateField: true }]; const templateFields: TemplateInputField[] = [ - { name: 'date', displayName: 'Collection date', required: true, isTemplateField: true }, + { name: 'date', displayName: 'Collection date', required: true, isTemplateField: true, metadataType: 'date' }, { name: 'country', displayName: 'Country', isTemplateField: true, + definition: 'Country where the sample was collected', + example: 'Germany', options: [{ name: 'Germany' }, { name: 'France' }], }, ]; @@ -186,6 +188,52 @@ describe('submission template API route', () => { expect(notesRow[5]).toBe('free text'); }); + test('Data sheet freezes the header row and adds an auto-filter', async () => { + const response = await callGet('test-organism', { fileType: 'xlsx' }); + const workbook = await loadWorkbook(response); + const dataSheet = workbook.getWorksheet(DATA_SHEET_NAME)!; + + expect(dataSheet.views[0]).toMatchObject({ state: 'frozen', ySplit: 1 }); + expect(dataSheet.autoFilter).toBeTruthy(); + }); + + test('headers are colour-coded by tier and carry hover notes', async () => { + const response = await callGet('test-organism', { fileType: 'xlsx' }); + const workbook = await loadWorkbook(response); + const header = workbook.getWorksheet(DATA_SHEET_NAME)!.getRow(1); + + // date (required) -> required tier; country (default template) -> default tier; + // notes (opt-in) -> optional tier. + const fillArgb = (column: number) => (header.getCell(column).fill as ExcelJS.FillPattern).fgColor?.argb; + expect(fillArgb(2)).toBe('FFFCE4D6'); // date, required + expect(fillArgb(3)).toBe('FFDDEBF7'); // country, default + expect(fillArgb(5)).toBe('FFF2F2F2'); // notes, optional + + // The country header note surfaces its definition and example. + const note = JSON.stringify(header.getCell(3).note); + expect(note).toContain('Country where the sample was collected'); + expect(note).toContain('Example: Germany'); + }); + + test('date columns are formatted as yyyy-mm-dd', async () => { + const response = await callGet('test-organism', { fileType: 'xlsx' }); + const workbook = await loadWorkbook(response); + const dataSheet = workbook.getWorksheet(DATA_SHEET_NAME)!; + expect(dataSheet.getColumn(2).numFmt).toBe('yyyy-mm-dd'); // date is column B + }); + + test('Config sheet includes a colour legend', async () => { + const response = await callGet('test-organism', { fileType: 'xlsx' }); + const workbook = await loadWorkbook(response); + const configSheet = workbook.getWorksheet(CONFIG_SHEET_NAME)!; + + const cellTexts: string[] = []; + configSheet.eachRow((row) => row.eachCell((cell) => cellTexts.push(cell.text))); + expect(cellTexts).toContain('Colour key'); + expect(cellTexts).toContain('Required field'); + expect(cellTexts).toContain('Included in the template by default'); + }); + test('revision template filename includes "revision"', async () => { const response = await callGet('test-organism', { fileType: 'xlsx', format: 'revise' }); expect(response.headers.get('Content-Disposition')).toBe( diff --git a/website/src/pages/[organism]/submission/template/index.ts b/website/src/pages/[organism]/submission/template/index.ts index b129807bc1..1efa85ffbf 100644 --- a/website/src/pages/[organism]/submission/template/index.ts +++ b/website/src/pages/[organism]/submission/template/index.ts @@ -92,9 +92,19 @@ async function createXlsxTemplate(organism: string, action: UploadAction): Promi // --- Data sheet (must be added first) --- const dataSheet = workbook.addWorksheet(DATA_SHEET_NAME); dataSheet.addRow(fields.map((field) => field.name)); - dataSheet.getRow(1).font = { bold: true }; + dataSheet.views = [{ state: 'frozen', ySplit: 1 }]; // keep the header row visible while scrolling + dataSheet.autoFilter = `A1:${columnLetter(fields.length)}1`; + + const headerRow = dataSheet.getRow(1); fields.forEach((field, index) => { dataSheet.getColumn(index + 1).width = columnWidthFor(field); + if (field.metadataType === 'date') { + dataSheet.getColumn(index + 1).numFmt = 'yyyy-mm-dd'; + } + const headerCell = headerRow.getCell(index + 1); + headerCell.font = { bold: true }; + headerCell.fill = tierFill(field); // colour-code required / default / opt-in + headerCell.note = headerNoteFor(field); // hover help: definition, example, guidance }); const optionFields = fields.filter((field) => (field.options?.length ?? 0) > 0); @@ -168,23 +178,75 @@ interface WorksheetDataValidations { add(sqref: string, validation: ExcelJS.DataValidation): void; } -/** Adds the read-only `Config` reference sheet listing every available field. */ +/** Visual tiers used to colour-code field headers (and explained by the legend on `Config`). */ +type FieldTier = 'required' | 'default' | 'optional'; +const FIELD_TIERS: Record = { + required: { argb: 'FFFCE4D6', label: 'Required field' }, + default: { argb: 'FFDDEBF7', label: 'Included in the template by default' }, + optional: { argb: 'FFF2F2F2', label: 'Optional — opt in by filling the column' }, +}; + +function fieldTier(field: TemplateInputField): FieldTier { + if (field.required === true) return 'required'; + return field.isTemplateField ? 'default' : 'optional'; +} + +function solidFill(argb: string): ExcelJS.Fill { + return { type: 'pattern', pattern: 'solid', fgColor: { argb } }; +} + +function tierFill(field: TemplateInputField): ExcelJS.Fill { + return solidFill(FIELD_TIERS[fieldTier(field)].argb); +} + +/** Hover help for a Data header: display name, definition, guidance, an example, and the field tier. */ +function headerNoteFor(field: TemplateInputField): string { + const lines: string[] = []; + if (field.displayName !== undefined && field.displayName !== field.name) lines.push(field.displayName); + if (field.definition !== undefined) lines.push(field.definition); + if (field.guidance !== undefined) lines.push(field.guidance); + if (field.example !== undefined && field.example !== '') lines.push(`Example: ${field.example}`); + lines.push(FIELD_TIERS[fieldTier(field)].label); + return lines.join('\n'); +} + +/** Adds the read-only `Config` reference sheet listing every available field, plus a colour legend. */ function addConfigSheet(workbook: ExcelJS.Workbook, fields: TemplateInputField[]): void { const configSheet = workbook.addWorksheet(CONFIG_SHEET_NAME); - configSheet.addRow(['Field name', 'Display name', 'Enabled by default', 'Required', 'Allowed values']); - configSheet.getRow(1).font = { bold: true }; + configSheet.views = [{ state: 'frozen', ySplit: 1 }]; + const headerRow = configSheet.addRow([ + 'Field name', + 'Display name', + 'Enabled by default', + 'Required', + 'Allowed values', + ]); + headerRow.font = { bold: true }; + headerRow.eachCell((cell) => { + cell.fill = solidFill('FFD9D9D9'); + }); [28, 28, 18, 12, 60].forEach((width, index) => { configSheet.getColumn(index + 1).width = width; }); fields.forEach((field) => { - configSheet.addRow([ + const row = configSheet.addRow([ field.name, field.displayName ?? '', field.isTemplateField ? 'Yes' : 'No', field.required === true ? 'Yes' : 'No', (field.options ?? []).map((option) => option.name).join(', ') || 'free text', ]); + row.getCell(1).fill = tierFill(field); // mirror the Data header colour }); + + // Colour legend. + configSheet.addRow([]); + configSheet.addRow(['Colour key']).getCell(1).font = { bold: true }; + (Object.keys(FIELD_TIERS) as FieldTier[]).forEach((tier) => { + const row = configSheet.addRow(['', FIELD_TIERS[tier].label]); + row.getCell(1).fill = solidFill(FIELD_TIERS[tier].argb); + }); + void configSheet.protect('', { selectLockedCells: true, selectUnlockedCells: true }); } From 1ea6fbfd26274f228444709f348591e8a9cd06f8 Mon Sep 17 00:00:00 2001 From: theosanderson-agent Date: Tue, 9 Jun 2026 16:26:59 +0100 Subject: [PATCH 05/16] feat(website): rename Config sheet to Guidance with descriptions; drop filter - Remove the Data sheet auto-filter (its header arrows clutter the sheet and are easily confused with the validation dropdowns); keep the frozen header. - Rename the `Config` reference sheet to `Guidance` (and update the upload parser's reference-sheet allowlist accordingly). - Drop the "Enabled by default" column; relabel the blue tier to "Priority fields" in the colour key. - Add Definition, Guidance and Example columns to the Guidance sheet so the field descriptions are readable in the workbook, not only in header notes. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../FileUpload/fileProcessing.spec.ts | 6 +-- .../Submission/FileUpload/fileProcessing.ts | 2 +- .../submission/template/index.spec.ts | 53 +++++++++++-------- .../[organism]/submission/template/index.ts | 48 ++++++++++------- 4 files changed, 63 insertions(+), 46 deletions(-) diff --git a/website/src/components/Submission/FileUpload/fileProcessing.spec.ts b/website/src/components/Submission/FileUpload/fileProcessing.spec.ts index 39c346f394..4c6b238dfb 100644 --- a/website/src/components/Submission/FileUpload/fileProcessing.spec.ts +++ b/website/src/components/Submission/FileUpload/fileProcessing.spec.ts @@ -96,8 +96,8 @@ describe('fileProcessing', () => { expect(processedFile.fastaHeader()).toBe('fooid description'); }); - test('template reference sheets (Config, _lists) do not trigger a multi-sheet warning', async () => { - const file = await buildWorkbookFile(['Config', '_lists']); + test('template reference sheets (Guidance, _lists) do not trigger a multi-sheet warning', async () => { + const file = await buildWorkbookFile(['Guidance', '_lists']); const processingResult = await METADATA_FILE_KIND.processRawFile(file); expect(processingResult.isOk()).toBe(true); @@ -105,7 +105,7 @@ describe('fileProcessing', () => { }); test('an unexpected extra sheet still triggers a multi-sheet warning', async () => { - const file = await buildWorkbookFile(['Config', 'My other data']); + const file = await buildWorkbookFile(['Guidance', 'My other data']); const processingResult = await METADATA_FILE_KIND.processRawFile(file); expect(processingResult.isOk()).toBe(true); diff --git a/website/src/components/Submission/FileUpload/fileProcessing.ts b/website/src/components/Submission/FileUpload/fileProcessing.ts index 48081cdedb..3e4d98d39e 100644 --- a/website/src/components/Submission/FileUpload/fileProcessing.ts +++ b/website/src/components/Submission/FileUpload/fileProcessing.ts @@ -22,7 +22,7 @@ const COMPRESSION_EXTENSIONS = ['zst', 'gz', 'zip', 'xz']; // Sheets that the downloadable XLSX template adds alongside the `Data` sheet (see // `pages/[organism]/submission/template/index.ts`). Kept in sync manually rather than imported, to // avoid pulling the server-only template endpoint (and its dependencies) into the client bundle. -const TEMPLATE_REFERENCE_SHEET_NAMES = new Set(['Config', '_lists']); +const TEMPLATE_REFERENCE_SHEET_NAMES = new Set(['Guidance', '_lists']); export const METADATA_FILE_KIND: FileKind = { type: 'metadata', diff --git a/website/src/pages/[organism]/submission/template/index.spec.ts b/website/src/pages/[organism]/submission/template/index.spec.ts index 4a4774d2b0..a1b3b7f7f4 100644 --- a/website/src/pages/[organism]/submission/template/index.spec.ts +++ b/website/src/pages/[organism]/submission/template/index.spec.ts @@ -3,7 +3,7 @@ import ExcelJS from 'exceljs'; import JSZip from 'jszip'; import { describe, expect, test, vi } from 'vitest'; -import { CONFIG_SHEET_NAME, DATA_SHEET_NAME, GET, LISTS_SHEET_NAME } from './index'; +import { GUIDANCE_SHEET_NAME, DATA_SHEET_NAME, GET, LISTS_SHEET_NAME } from './index'; import type { TemplateInputField } from '../../../../config'; const submissionDetailFields: TemplateInputField[] = [{ name: 'submissionId', required: true, isTemplateField: true }]; @@ -87,7 +87,7 @@ describe('submission template API route', () => { expect(text).toBe('submissionId\tdate\n'); }); - test('XLSX template has Data, Config and hidden _lists sheets, with Data first', async () => { + test('XLSX template has Data, Guidance and hidden _lists sheets, with Data first', async () => { const response = await callGet('test-organism', { fileType: 'xlsx' }); expect(response.headers.get('Content-Type')).toBe( 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', @@ -96,7 +96,7 @@ describe('submission template API route', () => { const workbook = await loadWorkbook(response); const sheetNames = workbook.worksheets.map((sheet) => sheet.name); expect(sheetNames[0]).toBe(DATA_SHEET_NAME); - expect(sheetNames).toContain(CONFIG_SHEET_NAME); + expect(sheetNames).toContain(GUIDANCE_SHEET_NAME); expect(sheetNames).toContain(LISTS_SHEET_NAME); expect(workbook.getWorksheet(LISTS_SHEET_NAME)!.state).toBe('hidden'); @@ -163,38 +163,47 @@ describe('submission template API route', () => { expect(dataSheet.getColumn(4).width).toBeLessThanOrEqual(45); }); - test('Config sheet lists every field with enabled/required flags and allowed values', async () => { + test('Guidance sheet lists every field with description columns and allowed values', async () => { const response = await callGet('test-organism', { fileType: 'xlsx' }); const workbook = await loadWorkbook(response); - const configSheet = workbook.getWorksheet(CONFIG_SHEET_NAME)!; + const guidanceSheet = workbook.getWorksheet(GUIDANCE_SHEET_NAME)!; - expect((configSheet.getRow(1).values as unknown[]).slice(1)).toEqual([ + expect((guidanceSheet.getRow(1).values as unknown[]).slice(1)).toEqual([ 'Field name', 'Display name', - 'Enabled by default', 'Required', + 'Definition', + 'Guidance', + 'Example', 'Allowed values', ]); - // country: enabled by default, optional, with allowed values - const countryRow = configSheet.getRow(4).values as unknown[]; - expect(countryRow.slice(1)).toEqual(['country', 'Country', 'Yes', 'No', 'Germany, France']); - - // host: opt-in (not a template field), free text label only when no options - const hostRow = configSheet.getRow(5).values as unknown[]; - expect(hostRow.slice(1, 4)).toEqual(['host', 'Host', 'No']); + // country: optional, carries its definition + example, with allowed values + const countryRow = guidanceSheet.getRow(4).values as unknown[]; + expect(countryRow.slice(1)).toEqual([ + 'country', + 'Country', + 'No', + 'Country where the sample was collected', + '', + 'Germany', + 'Germany, France', + ]); - const notesRow = configSheet.getRow(6).values as unknown[]; - expect(notesRow[5]).toBe('free text'); + // notes: no description, free text label only when no options + const notesRow = guidanceSheet.getRow(6).values as unknown[]; + expect(notesRow.slice(1)).toEqual(['notes', 'Notes', 'No', '', '', '', 'free text']); }); - test('Data sheet freezes the header row and adds an auto-filter', async () => { + test('Data sheet freezes the header row and is not in filter mode', async () => { const response = await callGet('test-organism', { fileType: 'xlsx' }); const workbook = await loadWorkbook(response); const dataSheet = workbook.getWorksheet(DATA_SHEET_NAME)!; expect(dataSheet.views[0]).toMatchObject({ state: 'frozen', ySplit: 1 }); - expect(dataSheet.autoFilter).toBeTruthy(); + // No auto-filter: its dropdown arrows clutter the header and confuse with the validation + // dropdowns. + expect(dataSheet.autoFilter).toBeFalsy(); }); test('headers are colour-coded by tier and carry hover notes', async () => { @@ -222,16 +231,16 @@ describe('submission template API route', () => { expect(dataSheet.getColumn(2).numFmt).toBe('yyyy-mm-dd'); // date is column B }); - test('Config sheet includes a colour legend', async () => { + test('Guidance sheet includes a colour legend', async () => { const response = await callGet('test-organism', { fileType: 'xlsx' }); const workbook = await loadWorkbook(response); - const configSheet = workbook.getWorksheet(CONFIG_SHEET_NAME)!; + const guidanceSheet = workbook.getWorksheet(GUIDANCE_SHEET_NAME)!; const cellTexts: string[] = []; - configSheet.eachRow((row) => row.eachCell((cell) => cellTexts.push(cell.text))); + guidanceSheet.eachRow((row) => row.eachCell((cell) => cellTexts.push(cell.text))); expect(cellTexts).toContain('Colour key'); expect(cellTexts).toContain('Required field'); - expect(cellTexts).toContain('Included in the template by default'); + expect(cellTexts).toContain('Priority fields'); }); test('revision template filename includes "revision"', async () => { diff --git a/website/src/pages/[organism]/submission/template/index.ts b/website/src/pages/[organism]/submission/template/index.ts index 1efa85ffbf..9573934975 100644 --- a/website/src/pages/[organism]/submission/template/index.ts +++ b/website/src/pages/[organism]/submission/template/index.ts @@ -14,11 +14,11 @@ const CONTENT_TYPES = new Map([ /** * Sheet names of the XLSX template. `Data` MUST be added to the workbook first, because the upload - * parser only reads the first sheet. `Config` and `_lists` are recognised as reference sheets by + * parser only reads the first sheet. `Guidance` and `_lists` are recognised as reference sheets by * the upload parser, which therefore does not warn about them (see `fileProcessing.ts`). */ export const DATA_SHEET_NAME = 'Data'; -export const CONFIG_SHEET_NAME = 'Config'; +export const GUIDANCE_SHEET_NAME = 'Guidance'; export const LISTS_SHEET_NAME = '_lists'; /** @@ -76,7 +76,7 @@ function createTsvTemplate(organism: string, action: UploadAction): ArrayBuffer * round-trips through upload), ordered template fields first, then the remaining opt-in fields. * - `_lists` (hidden): one column per field that has a controlled vocabulary; the column header is * the field name and the values below are its allowed options. This is the dropdown source. - * - `Config` (read-only): a human-readable reference of every available field. + * - `Guidance` (read-only): a human-readable reference of every available field. * * Only columns whose field has a controlled vocabulary get a dropdown validation. Each such * validation is header-driven: its source formula reads that column's own header cell and looks it @@ -93,7 +93,6 @@ async function createXlsxTemplate(organism: string, action: UploadAction): Promi const dataSheet = workbook.addWorksheet(DATA_SHEET_NAME); dataSheet.addRow(fields.map((field) => field.name)); dataSheet.views = [{ state: 'frozen', ySplit: 1 }]; // keep the header row visible while scrolling - dataSheet.autoFilter = `A1:${columnLetter(fields.length)}1`; const headerRow = dataSheet.getRow(1); fields.forEach((field, index) => { @@ -113,7 +112,7 @@ async function createXlsxTemplate(organism: string, action: UploadAction): Promi addOptionsLookup(workbook, dataSheet, fields, optionFields); } - addConfigSheet(workbook, fields); + addGuidanceSheet(workbook, fields); const buffer = await workbook.xlsx.writeBuffer(); return buffer; @@ -182,7 +181,7 @@ interface WorksheetDataValidations { type FieldTier = 'required' | 'default' | 'optional'; const FIELD_TIERS: Record = { required: { argb: 'FFFCE4D6', label: 'Required field' }, - default: { argb: 'FFDDEBF7', label: 'Included in the template by default' }, + default: { argb: 'FFDDEBF7', label: 'Priority fields' }, optional: { argb: 'FFF2F2F2', label: 'Optional — opt in by filling the column' }, }; @@ -210,44 +209,53 @@ function headerNoteFor(field: TemplateInputField): string { return lines.join('\n'); } -/** Adds the read-only `Config` reference sheet listing every available field, plus a colour legend. */ -function addConfigSheet(workbook: ExcelJS.Workbook, fields: TemplateInputField[]): void { - const configSheet = workbook.addWorksheet(CONFIG_SHEET_NAME); - configSheet.views = [{ state: 'frozen', ySplit: 1 }]; - const headerRow = configSheet.addRow([ +/** Adds the read-only `Guidance` reference sheet listing every available field, plus a colour legend. */ +function addGuidanceSheet(workbook: ExcelJS.Workbook, fields: TemplateInputField[]): void { + const guidanceSheet = workbook.addWorksheet(GUIDANCE_SHEET_NAME); + guidanceSheet.views = [{ state: 'frozen', ySplit: 1 }]; + const headerRow = guidanceSheet.addRow([ 'Field name', 'Display name', - 'Enabled by default', 'Required', + 'Definition', + 'Guidance', + 'Example', 'Allowed values', ]); headerRow.font = { bold: true }; headerRow.eachCell((cell) => { cell.fill = solidFill('FFD9D9D9'); }); - [28, 28, 18, 12, 60].forEach((width, index) => { - configSheet.getColumn(index + 1).width = width; + [28, 24, 10, 55, 55, 22, 55].forEach((width, index) => { + const column = guidanceSheet.getColumn(index + 1); + column.width = width; + // Wrap the long free-text columns (definition, guidance, allowed values) so they stay readable. + if (index === 3 || index === 4 || index === 6) { + column.alignment = { wrapText: true, vertical: 'top' }; + } }); fields.forEach((field) => { - const row = configSheet.addRow([ + const row = guidanceSheet.addRow([ field.name, field.displayName ?? '', - field.isTemplateField ? 'Yes' : 'No', field.required === true ? 'Yes' : 'No', + field.definition ?? '', + field.guidance ?? '', + field.example !== undefined ? String(field.example) : '', (field.options ?? []).map((option) => option.name).join(', ') || 'free text', ]); row.getCell(1).fill = tierFill(field); // mirror the Data header colour }); // Colour legend. - configSheet.addRow([]); - configSheet.addRow(['Colour key']).getCell(1).font = { bold: true }; + guidanceSheet.addRow([]); + guidanceSheet.addRow(['Colour key']).getCell(1).font = { bold: true }; (Object.keys(FIELD_TIERS) as FieldTier[]).forEach((tier) => { - const row = configSheet.addRow(['', FIELD_TIERS[tier].label]); + const row = guidanceSheet.addRow(['', FIELD_TIERS[tier].label]); row.getCell(1).fill = solidFill(FIELD_TIERS[tier].argb); }); - void configSheet.protect('', { selectLockedCells: true, selectUnlockedCells: true }); + void guidanceSheet.protect('', { selectLockedCells: true, selectUnlockedCells: true }); } /** Converts a 1-based column index to its Excel column letters (1 -> A, 27 -> AA). */ From f2ff3bd8efa83925d445f9eca4461000696a69a2 Mon Sep 17 00:00:00 2001 From: theosanderson-agent Date: Tue, 9 Jun 2026 16:39:54 +0100 Subject: [PATCH 06/16] fix(website): size Data columns to field name, not widest option Long option values (e.g. country names) were stretching columns to the 45-char cap. Width is now based on the field name only. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../[organism]/submission/template/index.spec.ts | 14 ++++++++------ .../pages/[organism]/submission/template/index.ts | 5 ++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/website/src/pages/[organism]/submission/template/index.spec.ts b/website/src/pages/[organism]/submission/template/index.spec.ts index a1b3b7f7f4..01d288e4ef 100644 --- a/website/src/pages/[organism]/submission/template/index.spec.ts +++ b/website/src/pages/[organism]/submission/template/index.spec.ts @@ -149,18 +149,20 @@ describe('submission template API route', () => { expect(validationBlock).toContain(LISTS_SHEET_NAME); }); - test('Data columns are widened to fit field names and their options', async () => { + test('Data columns are sized to the field name, bounded and not widened by options', async () => { const response = await callGet('test-organism', { fileType: 'xlsx' }); const workbook = await loadWorkbook(response); const dataSheet = workbook.getWorksheet(DATA_SHEET_NAME)!; - // Every column is at least the minimum width... + // Every column is between the minimum and maximum width... for (let column = 1; column <= 5; column++) { - expect(dataSheet.getColumn(column).width).toBeGreaterThanOrEqual(16); + const width = dataSheet.getColumn(column).width!; + expect(width).toBeGreaterThanOrEqual(16); + expect(width).toBeLessThanOrEqual(45); } - // ...and the `host` column (col D) widens to fit "Homo sapiens" (12) -> 14 < 16, stays at 16, - // while a longer option would push it wider. Sanity-check it is a finite, bounded number. - expect(dataSheet.getColumn(4).width).toBeLessThanOrEqual(45); + // ...and short-named option fields stay at the minimum rather than widening to fit a long + // option value (e.g. country -> "Germany"/"France" does not stretch column C). + expect(dataSheet.getColumn(3).width).toBe(16); }); test('Guidance sheet lists every field with description columns and allowed values', async () => { diff --git a/website/src/pages/[organism]/submission/template/index.ts b/website/src/pages/[organism]/submission/template/index.ts index 9573934975..3da61940a1 100644 --- a/website/src/pages/[organism]/submission/template/index.ts +++ b/website/src/pages/[organism]/submission/template/index.ts @@ -167,10 +167,9 @@ function addOptionsLookup( }); } -/** A roomy-but-bounded Data column width that fits the field name and its longest dropdown option. */ +/** A roomy-but-bounded Data column width that fits the field name (not its dropdown options). */ function columnWidthFor(field: TemplateInputField): number { - const longestOption = (field.options ?? []).reduce((max, option) => Math.max(max, option.name.length), 0); - return Math.min(45, Math.max(16, field.name.length + 2, longestOption + 2)); + return Math.min(45, Math.max(16, field.name.length + 2)); } interface WorksheetDataValidations { From 96fcf07cc2c6ad9563ea6fe5cc7b6b3ba03cb634 Mon Sep 17 00:00:00 2001 From: theosanderson-agent Date: Tue, 9 Jun 2026 16:43:17 +0100 Subject: [PATCH 07/16] feat(website): enforce controlled-vocabulary dropdowns (reject off-list values) Switch the per-column option dropdowns from advisory to strict (errorStyle stop): a value outside a field's controlled vocabulary is rejected. Only columns with options are validated, so free-text columns remain unrestricted. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../[organism]/submission/template/index.spec.ts | 8 ++++---- .../pages/[organism]/submission/template/index.ts | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/website/src/pages/[organism]/submission/template/index.spec.ts b/website/src/pages/[organism]/submission/template/index.spec.ts index 01d288e4ef..71688321b6 100644 --- a/website/src/pages/[organism]/submission/template/index.spec.ts +++ b/website/src/pages/[organism]/submission/template/index.spec.ts @@ -123,7 +123,7 @@ describe('submission template API route', () => { expect(listsSheet.getCell('B2').value).toBe('Homo sapiens'); }); - test('only option columns get an advisory, header-driven dropdown looking up _lists', async () => { + test('only option columns get a strict, header-driven dropdown looking up _lists', async () => { // Asserted against the raw written XML: ExcelJS' own re-read expands a range `sqref` into // per-cell entries, which makes the compact per-column rules unreadable. const response = await callGet('test-organism', { fileType: 'xlsx' }); @@ -139,9 +139,9 @@ describe('submission template API route', () => { expect(validationBlock).toContain('sqref="D2:D100000"'); expect(validationBlock).not.toContain('sqref="A2'); expect(validationBlock).not.toContain('sqref="E2'); - // Advisory, not strict: a warning the user can dismiss, rather than a hard rejection. - expect(validationBlock).toContain('errorStyle="warning"'); - expect(validationBlock).not.toContain('errorStyle="stop"'); + // Strict: off-list values on a controlled-vocabulary column are rejected. + expect(validationBlock).toContain('errorStyle="stop"'); + expect(validationBlock).not.toContain('errorStyle="warning"'); // Header-driven: each validation looks up its own column header (C$1 / D$1) in `_lists`, // so the dropdown follows the field if the column is renamed or moved. expect(validationBlock).toContain('MATCH(C$1'); diff --git a/website/src/pages/[organism]/submission/template/index.ts b/website/src/pages/[organism]/submission/template/index.ts index 3da61940a1..1e2b24b075 100644 --- a/website/src/pages/[organism]/submission/template/index.ts +++ b/website/src/pages/[organism]/submission/template/index.ts @@ -82,8 +82,8 @@ function createTsvTemplate(organism: string, action: UploadAction): ArrayBuffer * validation is header-driven: its source formula reads that column's own header cell and looks it * up in `_lists`, so the dropdown follows the field even if the user renames or reorders the column. * Free-text columns deliberately get no validation so they accept any value — a `list` validation - * cannot express "allow anything", so applying a header-driven rule to them would warn on every - * entry (the lookup errors). Limiting it to option columns keeps free-text columns silent. + * cannot express "allow anything", so applying a (strict) header-driven rule to them would reject + * every entry (the lookup errors). Limiting it to option columns keeps free-text columns unrestricted. */ async function createXlsxTemplate(organism: string, action: UploadAction): Promise { const fields = getOrderedTemplateInputFields(organism, action); @@ -157,12 +157,12 @@ function addOptionsLookup( type: 'list', allowBlank: true, formulae: [source], - // Advisory, not strict: the dropdown offers the controlled vocabulary, but a value typed - // outside the list only warns ("Yes" keeps it) rather than being rejected. + // Strict: a value outside the controlled vocabulary is rejected. Only columns with a + // controlled vocabulary get a validation, so free-text columns are unaffected. showErrorMessage: true, - errorStyle: 'warning', - errorTitle: 'Value not in the suggested list', - error: `"${field.name}" has a suggested list of values. You can pick one from the dropdown, or keep what you typed.`, + errorStyle: 'stop', + errorTitle: 'Invalid value', + error: `"${field.name}" must be one of the values in the dropdown list.`, }); }); } From d61ed2eee8208305d30547d5a9ce6278c472818a Mon Sep 17 00:00:00 2001 From: theosanderson-agent Date: Tue, 9 Jun 2026 16:50:41 +0100 Subject: [PATCH 08/16] feat(website): recommend XLSX template in the upload help text Lead with the XLSX template (recommended) and call the TSV one a minimal template, on the metadata upload page. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../FileUpload/SequenceEntryUploadComponent.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/website/src/components/Submission/FileUpload/SequenceEntryUploadComponent.tsx b/website/src/components/Submission/FileUpload/SequenceEntryUploadComponent.tsx index b5214282d9..8b8679bf56 100644 --- a/website/src/components/Submission/FileUpload/SequenceEntryUploadComponent.tsx +++ b/website/src/components/Submission/FileUpload/SequenceEntryUploadComponent.tsx @@ -82,12 +82,12 @@ export const SequenceEntryUpload: FC = ({ metadata format {' '} including a list of all supported metadata. You can download a{' '} + + XLSX template (recommended) + + {' or a minimal '} TSV - - {' or '} - - XLSX {' '} template with column headings for the metadata file.

From 9b8d8fae9154f5a8c09052052fdf58317ac14072 Mon Sep 17 00:00:00 2001 From: theosanderson-agent Date: Tue, 9 Jun 2026 17:03:47 +0100 Subject: [PATCH 09/16] fix(website): parse the Data sheet by name on upload, regardless of order The XLSX upload parser read the first sheet by position, so reordering the template's sheets (e.g. dragging Guidance to the front) would parse the reference sheet as metadata. Select the 'Data' sheet by name when present, falling back to the first sheet for non-template workbooks, and compute the multi-sheet warning relative to the parsed sheet. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../FileUpload/fileProcessing.spec.ts | 20 +++++++++++++++ .../Submission/FileUpload/fileProcessing.ts | 25 ++++++++++++------- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/website/src/components/Submission/FileUpload/fileProcessing.spec.ts b/website/src/components/Submission/FileUpload/fileProcessing.spec.ts index 4c6b238dfb..cd62da4392 100644 --- a/website/src/components/Submission/FileUpload/fileProcessing.spec.ts +++ b/website/src/components/Submission/FileUpload/fileProcessing.spec.ts @@ -111,4 +111,24 @@ describe('fileProcessing', () => { expect(processingResult.isOk()).toBe(true); expect(processingResult._unsafeUnwrap().warnings()).toHaveLength(1); }); + + test('parses the Data sheet by name even when it is not the first sheet', async () => { + // Reference sheet dragged in front of Data — the Data sheet must still be the one parsed. + const workbook = new ExcelJS.Workbook(); + workbook.addWorksheet('Guidance').addRow(['Field name', 'Display name']); + const data = workbook.addWorksheet('Data'); + data.addRow(['submissionId', 'country']); + data.addRow(['sample1', 'Germany']); + const buffer = await workbook.xlsx.writeBuffer(); + const file = new File([buffer], 'template.xlsx'); + + const processingResult = await METADATA_FILE_KIND.processRawFile(file); + expect(processingResult.isOk()).toBe(true); + const processedFile = processingResult._unsafeUnwrap(); + + expect(processedFile.warnings()).toHaveLength(0); + const text = await processedFile.text(); + expect(text.split('\n')[0]).toContain('submissionId'); // parsed Data... + expect(text).not.toContain('Field name'); // ...not the Guidance reference sheet + }); }); diff --git a/website/src/components/Submission/FileUpload/fileProcessing.ts b/website/src/components/Submission/FileUpload/fileProcessing.ts index 3e4d98d39e..46f48f51cb 100644 --- a/website/src/components/Submission/FileUpload/fileProcessing.ts +++ b/website/src/components/Submission/FileUpload/fileProcessing.ts @@ -19,9 +19,11 @@ export type FileKind = { const COMPRESSION_EXTENSIONS = ['zst', 'gz', 'zip', 'xz']; -// Sheets that the downloadable XLSX template adds alongside the `Data` sheet (see +// Names of the sheets in the downloadable XLSX template (see // `pages/[organism]/submission/template/index.ts`). Kept in sync manually rather than imported, to // avoid pulling the server-only template endpoint (and its dependencies) into the client bundle. +// `Data` is the sheet to parse; the others are reference/lookup sheets that should be ignored. +const TEMPLATE_DATA_SHEET_NAME = 'Data'; const TEMPLATE_REFERENCE_SHEET_NAMES = new Set(['Guidance', '_lists']); export const METADATA_FILE_KIND: FileKind = { @@ -246,8 +248,13 @@ export class ExcelFile implements ProcessedFile { cellDates: true, }); - const firstSheetName = workbook.SheetNames[0]; - let sheet = workbook.Sheets[firstSheetName]; + // Parse the `Data` sheet by name when present, so reordering the template's sheets (e.g. + // dragging `Guidance` to the front) does not cause the reference sheet to be read as + // metadata. Fall back to the first sheet for arbitrary, non-template workbooks. + const dataSheetName = workbook.SheetNames.includes(TEMPLATE_DATA_SHEET_NAME) + ? TEMPLATE_DATA_SHEET_NAME + : workbook.SheetNames[0]; + let sheet = workbook.Sheets[dataSheetName]; // convert to JSON and back due to date formatting not working well otherwise const json = XLSX.utils.sheet_to_json(sheet); @@ -263,21 +270,21 @@ export class ExcelFile implements ProcessedFile { }); const rowCount = tsvContent.split('\n').length - 1; if (rowCount <= 0) { - throw new Error(`Sheet ${firstSheetName} is empty.`); + throw new Error(`Sheet ${dataSheetName} is empty.`); } const tsvBlob = new Blob([tsvContent], { type: 'text/tab-separated-values' }); // filename needs to end in 'tsv' for the uploaded file const tsvFile = new File([tsvBlob], 'converted.tsv', { type: 'text/tab-separated-values' }); this.tsvFile = tsvFile; - // Sheets that the downloadable template adds for reference/lookup purposes are expected and - // should not trigger the "you have unprocessed sheets" warning. - const unexpectedSheets = workbook.SheetNames.slice(1).filter( - (sheetName) => !TEMPLATE_REFERENCE_SHEET_NAMES.has(sheetName), + // Any sheet other than the parsed one and the template's reference/lookup sheets is + // unexpected and should trigger the "you have unprocessed sheets" warning. + const unexpectedSheets = workbook.SheetNames.filter( + (sheetName) => sheetName !== dataSheetName && !TEMPLATE_REFERENCE_SHEET_NAMES.has(sheetName), ); if (unexpectedSheets.length > 0) { this.processingWarnings.push( - `The file contains ${workbook.SheetNames.length} sheets, only the first sheet (${firstSheetName}; ${rowCount} rows) was processed.`, + `The file contains ${workbook.SheetNames.length} sheets, only the '${dataSheetName}' sheet (${rowCount} rows) was processed.`, ); } } From eb4b9380d995b152392c5efa1752d437bec43f27 Mon Sep 17 00:00:00 2001 From: theosanderson-agent Date: Tue, 9 Jun 2026 17:08:23 +0100 Subject: [PATCH 10/16] docs(website): correct stale 'Data must be first' comments The upload parser now selects the Data sheet by name, so its position is no longer load-bearing; it is still added first so the workbook opens on it. Co-Authored-By: Claude Opus 4.8 (1M context) --- website/src/pages/[organism]/submission/template/index.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/website/src/pages/[organism]/submission/template/index.ts b/website/src/pages/[organism]/submission/template/index.ts index 1e2b24b075..12f7f019d6 100644 --- a/website/src/pages/[organism]/submission/template/index.ts +++ b/website/src/pages/[organism]/submission/template/index.ts @@ -13,9 +13,9 @@ const CONTENT_TYPES = new Map([ ]); /** - * Sheet names of the XLSX template. `Data` MUST be added to the workbook first, because the upload - * parser only reads the first sheet. `Guidance` and `_lists` are recognised as reference sheets by - * the upload parser, which therefore does not warn about them (see `fileProcessing.ts`). + * Sheet names of the XLSX template. `Data` is added to the workbook first so the file opens on it, + * but the upload parser selects it by name (see `fileProcessing.ts`), so the order is not + * load-bearing. `Guidance` and `_lists` are recognised as reference sheets and ignored on upload. */ export const DATA_SHEET_NAME = 'Data'; export const GUIDANCE_SHEET_NAME = 'Guidance'; @@ -89,7 +89,7 @@ async function createXlsxTemplate(organism: string, action: UploadAction): Promi const fields = getOrderedTemplateInputFields(organism, action); const workbook = new ExcelJS.Workbook(); - // --- Data sheet (must be added first) --- + // --- Data sheet (added first so the workbook opens on it; parser finds it by name) --- const dataSheet = workbook.addWorksheet(DATA_SHEET_NAME); dataSheet.addRow(fields.map((field) => field.name)); dataSheet.views = [{ state: 'frozen', ySplit: 1 }]; // keep the header row visible while scrolling From 11e95ae221a575b600d86a278a392cd9ae40ddc9 Mon Sep 17 00:00:00 2001 From: theosanderson-agent Date: Tue, 9 Jun 2026 17:09:15 +0100 Subject: [PATCH 11/16] docs(website): drop firstness mentions from Data sheet comments Co-Authored-By: Claude Opus 4.8 (1M context) --- website/src/pages/[organism]/submission/template/index.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/website/src/pages/[organism]/submission/template/index.ts b/website/src/pages/[organism]/submission/template/index.ts index 12f7f019d6..f9c1c5bf95 100644 --- a/website/src/pages/[organism]/submission/template/index.ts +++ b/website/src/pages/[organism]/submission/template/index.ts @@ -13,9 +13,9 @@ const CONTENT_TYPES = new Map([ ]); /** - * Sheet names of the XLSX template. `Data` is added to the workbook first so the file opens on it, - * but the upload parser selects it by name (see `fileProcessing.ts`), so the order is not - * load-bearing. `Guidance` and `_lists` are recognised as reference sheets and ignored on upload. + * Sheet names of the XLSX template. The upload parser selects the `Data` sheet by name (see + * `fileProcessing.ts`); `Guidance` and `_lists` are recognised as reference sheets and ignored on + * upload. */ export const DATA_SHEET_NAME = 'Data'; export const GUIDANCE_SHEET_NAME = 'Guidance'; @@ -89,7 +89,7 @@ async function createXlsxTemplate(organism: string, action: UploadAction): Promi const fields = getOrderedTemplateInputFields(organism, action); const workbook = new ExcelJS.Workbook(); - // --- Data sheet (added first so the workbook opens on it; parser finds it by name) --- + // --- Data sheet --- const dataSheet = workbook.addWorksheet(DATA_SHEET_NAME); dataSheet.addRow(fields.map((field) => field.name)); dataSheet.views = [{ state: 'frozen', ySplit: 1 }]; // keep the header row visible while scrolling From b0ef53ee69dce3c27aaf7d42189144cbbf8a91cc Mon Sep 17 00:00:00 2001 From: theosanderson-agent Date: Tue, 9 Jun 2026 17:31:26 +0100 Subject: [PATCH 12/16] fix(website): adjust template link copy and integration-test locators - Link text is 'XLSX template' with '(recommended)' outside the link. - Make the revision page object click the template links by href (fileType query param) instead of exact 'TSV'/'XLSX' text, which broke when the copy changed. Co-Authored-By: Claude Opus 4.8 (1M context) --- integration-tests/tests/pages/revision.page.ts | 4 ++-- .../Submission/FileUpload/SequenceEntryUploadComponent.tsx | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/integration-tests/tests/pages/revision.page.ts b/integration-tests/tests/pages/revision.page.ts index 73b0651851..24606f9243 100644 --- a/integration-tests/tests/pages/revision.page.ts +++ b/integration-tests/tests/pages/revision.page.ts @@ -72,7 +72,7 @@ export class RevisionPage { */ async downloadTsvTemplate() { const downloadPromise = this.page.waitForEvent('download'); - await this.page.getByText('TSV', { exact: true }).click(); + await this.page.locator('a[href*="fileType=tsv"]').click(); return downloadPromise; } @@ -81,7 +81,7 @@ export class RevisionPage { */ async downloadXlsxTemplate() { const downloadPromise = this.page.waitForEvent('download'); - await this.page.getByText('XLSX', { exact: true }).click(); + await this.page.locator('a[href*="fileType=xlsx"]').click(); return downloadPromise; } diff --git a/website/src/components/Submission/FileUpload/SequenceEntryUploadComponent.tsx b/website/src/components/Submission/FileUpload/SequenceEntryUploadComponent.tsx index 8b8679bf56..ff55c0edeb 100644 --- a/website/src/components/Submission/FileUpload/SequenceEntryUploadComponent.tsx +++ b/website/src/components/Submission/FileUpload/SequenceEntryUploadComponent.tsx @@ -83,9 +83,9 @@ export const SequenceEntryUpload: FC = ({ {' '} including a list of all supported metadata. You can download a{' '} - XLSX template (recommended) + XLSX template - {' or a minimal '} + {' (recommended) or a minimal '} TSV {' '} From 3ab97b313f6569b67ec595e489b282820349acaf Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Tue, 9 Jun 2026 17:37:28 +0100 Subject: [PATCH 13/16] Update website/src/pages/[organism]/submission/template/index.ts Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com> --- website/src/pages/[organism]/submission/template/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/website/src/pages/[organism]/submission/template/index.ts b/website/src/pages/[organism]/submission/template/index.ts index f9c1c5bf95..0de146d187 100644 --- a/website/src/pages/[organism]/submission/template/index.ts +++ b/website/src/pages/[organism]/submission/template/index.ts @@ -136,6 +136,7 @@ function addOptionsLookup( listsSheet.getCell(optionIndex + 2, columnIndex + 1).value = option.name; }); }); + // Empty-string password: protection is cosmetic (guard against accidental edits), not a security boundary. void listsSheet.protect('', { selectLockedCells: false, selectUnlockedCells: false }); // `dataValidations.add` exists at runtime but is missing from ExcelJS' type definitions. Using it From 24f612747bcf9bf270d93dafb559f6fa4d087e77 Mon Sep 17 00:00:00 2001 From: theosanderson-agent Date: Tue, 9 Jun 2026 17:43:50 +0100 Subject: [PATCH 14/16] refactor(website): share template sheet-name constants via a dedicated module Move DATA_SHEET_NAME / GUIDANCE_SHEET_NAME / LISTS_SHEET_NAME and TEMPLATE_REFERENCE_SHEET_NAMES into utils/metadataTemplateSheets.ts, imported by both the server-side template generator and the client-side upload parser, instead of duplicating them. The module is dependency-free, so importing it does not pull ExcelJS into the client bundle. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Submission/FileUpload/fileProcessing.ts | 12 ++---------- .../[organism]/submission/template/index.spec.ts | 3 ++- .../pages/[organism]/submission/template/index.ts | 10 +--------- website/src/utils/metadataTemplateSheets.ts | 14 ++++++++++++++ 4 files changed, 19 insertions(+), 20 deletions(-) create mode 100644 website/src/utils/metadataTemplateSheets.ts diff --git a/website/src/components/Submission/FileUpload/fileProcessing.ts b/website/src/components/Submission/FileUpload/fileProcessing.ts index 46f48f51cb..9af27a257c 100644 --- a/website/src/components/Submission/FileUpload/fileProcessing.ts +++ b/website/src/components/Submission/FileUpload/fileProcessing.ts @@ -5,6 +5,7 @@ import JSZip from 'jszip'; import { Result, ok, err } from 'neverthrow'; import { type SVGProps, type ForwardRefExoticComponent } from 'react'; +import { DATA_SHEET_NAME, TEMPLATE_REFERENCE_SHEET_NAMES } from '../../../utils/metadataTemplateSheets'; import MaterialSymbolsLightDataTableOutline from '~icons/material-symbols-light/data-table-outline'; import PhDnaLight from '~icons/ph/dna-light'; @@ -19,13 +20,6 @@ export type FileKind = { const COMPRESSION_EXTENSIONS = ['zst', 'gz', 'zip', 'xz']; -// Names of the sheets in the downloadable XLSX template (see -// `pages/[organism]/submission/template/index.ts`). Kept in sync manually rather than imported, to -// avoid pulling the server-only template endpoint (and its dependencies) into the client bundle. -// `Data` is the sheet to parse; the others are reference/lookup sheets that should be ignored. -const TEMPLATE_DATA_SHEET_NAME = 'Data'; -const TEMPLATE_REFERENCE_SHEET_NAMES = new Set(['Guidance', '_lists']); - export const METADATA_FILE_KIND: FileKind = { type: 'metadata', icon: MaterialSymbolsLightDataTableOutline, @@ -251,9 +245,7 @@ export class ExcelFile implements ProcessedFile { // Parse the `Data` sheet by name when present, so reordering the template's sheets (e.g. // dragging `Guidance` to the front) does not cause the reference sheet to be read as // metadata. Fall back to the first sheet for arbitrary, non-template workbooks. - const dataSheetName = workbook.SheetNames.includes(TEMPLATE_DATA_SHEET_NAME) - ? TEMPLATE_DATA_SHEET_NAME - : workbook.SheetNames[0]; + const dataSheetName = workbook.SheetNames.includes(DATA_SHEET_NAME) ? DATA_SHEET_NAME : workbook.SheetNames[0]; let sheet = workbook.Sheets[dataSheetName]; // convert to JSON and back due to date formatting not working well otherwise diff --git a/website/src/pages/[organism]/submission/template/index.spec.ts b/website/src/pages/[organism]/submission/template/index.spec.ts index 71688321b6..b3d04afd3a 100644 --- a/website/src/pages/[organism]/submission/template/index.spec.ts +++ b/website/src/pages/[organism]/submission/template/index.spec.ts @@ -3,8 +3,9 @@ import ExcelJS from 'exceljs'; import JSZip from 'jszip'; import { describe, expect, test, vi } from 'vitest'; -import { GUIDANCE_SHEET_NAME, DATA_SHEET_NAME, GET, LISTS_SHEET_NAME } from './index'; +import { GET } from './index'; import type { TemplateInputField } from '../../../../config'; +import { DATA_SHEET_NAME, GUIDANCE_SHEET_NAME, LISTS_SHEET_NAME } from '../../../../utils/metadataTemplateSheets'; const submissionDetailFields: TemplateInputField[] = [{ name: 'submissionId', required: true, isTemplateField: true }]; diff --git a/website/src/pages/[organism]/submission/template/index.ts b/website/src/pages/[organism]/submission/template/index.ts index 0de146d187..dce5cddaed 100644 --- a/website/src/pages/[organism]/submission/template/index.ts +++ b/website/src/pages/[organism]/submission/template/index.ts @@ -4,6 +4,7 @@ import ExcelJS from 'exceljs'; import { cleanOrganism } from '../../../../components/Navigation/cleanOrganism'; import type { UploadAction } from '../../../../components/Submission/DataUploadForm.tsx'; import { getMetadataTemplateFields, getOrderedTemplateInputFields, type TemplateInputField } from '../../../../config'; +import { DATA_SHEET_NAME, GUIDANCE_SHEET_NAME, LISTS_SHEET_NAME } from '../../../../utils/metadataTemplateSheets'; export type TemplateFileType = 'tsv' | 'xlsx'; const VALID_FILE_TYPES = ['tsv', 'xlsx']; @@ -12,15 +13,6 @@ const CONTENT_TYPES = new Map([ ['xlsx', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'], ]); -/** - * Sheet names of the XLSX template. The upload parser selects the `Data` sheet by name (see - * `fileProcessing.ts`); `Guidance` and `_lists` are recognised as reference sheets and ignored on - * upload. - */ -export const DATA_SHEET_NAME = 'Data'; -export const GUIDANCE_SHEET_NAME = 'Guidance'; -export const LISTS_SHEET_NAME = '_lists'; - /** * The dropdown validation is applied to data rows 2..MAX_DATA_ROWS. We deliberately do not use * Excel's full column height (1048576): the on-disk file is the same size either way, but a diff --git a/website/src/utils/metadataTemplateSheets.ts b/website/src/utils/metadataTemplateSheets.ts new file mode 100644 index 0000000000..a8821e5cfb --- /dev/null +++ b/website/src/utils/metadataTemplateSheets.ts @@ -0,0 +1,14 @@ +/** + * Sheet names used by the downloadable XLSX metadata template + * (`pages/[organism]/submission/template/index.ts`). + * + * Kept in this dependency-free module so that both the server-side generator and the client-side + * upload parser (`components/Submission/FileUpload/fileProcessing.ts`) can import them, without the + * client bundle pulling in the generator's server-only dependencies (ExcelJS). + */ +export const DATA_SHEET_NAME = 'Data'; +export const GUIDANCE_SHEET_NAME = 'Guidance'; +export const LISTS_SHEET_NAME = '_lists'; + +/** Reference/lookup sheets that the upload parser should ignore (everything except `Data`). */ +export const TEMPLATE_REFERENCE_SHEET_NAMES = new Set([GUIDANCE_SHEET_NAME, LISTS_SHEET_NAME]); From 92f9c1b6f9e9f514460c092b1739149a363c0599 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Thu, 11 Jun 2026 16:03:32 +0100 Subject: [PATCH 15/16] refactor(website): address review comments on template field ordering and docs - Split getOrderedTemplateInputFields into explicit template set/unset branches and rename detailFields -> requiredInternalFields for clarity - Reference sheet-name constants via {@link} in template doc comments - Fix legend typo: Config -> Guidance Co-Authored-By: Claude Opus 4.8 (1M context) --- website/src/config.ts | 32 ++++++++++++------- .../[organism]/submission/template/index.ts | 17 ++++++---- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/website/src/config.ts b/website/src/config.ts index ae2d4dda54..d7af27da56 100644 --- a/website/src/config.ts +++ b/website/src/config.ts @@ -186,18 +186,10 @@ export function getOrderedTemplateInputFields(organism: string, action: 'submit' const submissionIdInputFields = getSubmissionIdInputFields(schema); const accessionFields = action === 'revise' ? [getAccessionInputField()] : []; - const detailFields = [...accessionFields, ...submissionIdInputFields]; + const requiredInternalFields = [...accessionFields, ...submissionIdInputFields]; - const detailFieldNames = new Set(detailFields.map((field) => field.name)); - const nonDetailFields = schema.inputFields.filter((field) => !detailFieldNames.has(field.name)); - const fieldsByName = new Map(nonDetailFields.map((field) => [field.name, field])); - - const templateFieldNames = schema.metadataTemplate ?? nonDetailFields.map((field) => field.name); - const templateFieldNameSet = new Set(templateFieldNames); - const templateFields = templateFieldNames - .map((name) => fieldsByName.get(name)) - .filter((field): field is InputField => field !== undefined); - const restFields = nonDetailFields.filter((field) => !templateFieldNameSet.has(field.name)); + const requiredInternalFieldNames = new Set(requiredInternalFields.map((field) => field.name)); + const nonInternalInputFields = schema.inputFields.filter((field) => !requiredInternalFieldNames.has(field.name)); const metadataTypeByName = new Map(schema.metadata.map((entry) => [entry.name, entry.type] as const)); const decorate = (field: InputField, isTemplateField: boolean): TemplateInputField => ({ @@ -206,8 +198,24 @@ export function getOrderedTemplateInputFields(organism: string, action: 'submit' metadataType: metadataTypeByName.get(field.name), }); + // Required internal fields (submissionId/fastaIds, plus accession on revise) always lead the columns. + const orderedRequiredFields = requiredInternalFields.map((field) => decorate(field, true)); + + // Without an explicit template, every input field is enabled by default. + if (schema.metadataTemplate === undefined || schema.metadataTemplate === null) { + return [...orderedRequiredFields, ...nonInternalInputFields.map((field) => decorate(field, true))]; + } + + // With a template, the listed fields are enabled (in template order) and the rest become opt-in. + const fieldsByName = new Map(nonInternalInputFields.map((field) => [field.name, field])); + const templateFieldNameSet = new Set(schema.metadataTemplate); + const templateFields = schema.metadataTemplate + .map((name) => fieldsByName.get(name)) + .filter((field): field is InputField => field !== undefined); + const restFields = nonInternalInputFields.filter((field) => !templateFieldNameSet.has(field.name)); + return [ - ...detailFields.map((field) => decorate(field, true)), + ...orderedRequiredFields, ...templateFields.map((field) => decorate(field, true)), ...restFields.map((field) => decorate(field, false)), ]; diff --git a/website/src/pages/[organism]/submission/template/index.ts b/website/src/pages/[organism]/submission/template/index.ts index dce5cddaed..b42bfa0f45 100644 --- a/website/src/pages/[organism]/submission/template/index.ts +++ b/website/src/pages/[organism]/submission/template/index.ts @@ -64,15 +64,18 @@ function createTsvTemplate(organism: string, action: UploadAction): ArrayBuffer /** * Builds a workbook with three sheets: - * - `Data`: the sheet submitters fill in. Columns are the machine field names (so the file - * round-trips through upload), ordered template fields first, then the remaining opt-in fields. - * - `_lists` (hidden): one column per field that has a controlled vocabulary; the column header is - * the field name and the values below are its allowed options. This is the dropdown source. - * - `Guidance` (read-only): a human-readable reference of every available field. + * - {@link DATA_SHEET_NAME}: the sheet submitters fill in. Columns are the machine field names (so + * the file round-trips through upload), ordered template fields first, then the remaining opt-in + * fields. + * - {@link LISTS_SHEET_NAME} (hidden): one column per field that has a controlled vocabulary; the + * column header is the field name and the values below are its allowed options. This is the + * dropdown source. + * - {@link GUIDANCE_SHEET_NAME} (read-only): a human-readable reference of every available field. * * Only columns whose field has a controlled vocabulary get a dropdown validation. Each such * validation is header-driven: its source formula reads that column's own header cell and looks it - * up in `_lists`, so the dropdown follows the field even if the user renames or reorders the column. + * up in {@link LISTS_SHEET_NAME}, so the dropdown follows the field even if the user renames or + * reorders the column. * Free-text columns deliberately get no validation so they accept any value — a `list` validation * cannot express "allow anything", so applying a (strict) header-driven rule to them would reject * every entry (the lookup errors). Limiting it to option columns keeps free-text columns unrestricted. @@ -169,7 +172,7 @@ interface WorksheetDataValidations { add(sqref: string, validation: ExcelJS.DataValidation): void; } -/** Visual tiers used to colour-code field headers (and explained by the legend on `Config`). */ +/** Visual tiers used to colour-code field headers (and explained by the legend on {@link GUIDANCE_SHEET_NAME}). */ type FieldTier = 'required' | 'default' | 'optional'; const FIELD_TIERS: Record = { required: { argb: 'FFFCE4D6', label: 'Required field' }, From 230b08e110112855a18467b7eff66fa51ecbb9cf Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Thu, 11 Jun 2026 16:06:08 +0100 Subject: [PATCH 16/16] fix(website): drop impossible null check on metadataTemplate metadataTemplate is string[] | undefined, so the === null branch tripped @typescript-eslint/no-unnecessary-condition. Co-Authored-By: Claude Opus 4.8 (1M context) --- website/src/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/src/config.ts b/website/src/config.ts index d7af27da56..6ced779bd1 100644 --- a/website/src/config.ts +++ b/website/src/config.ts @@ -202,7 +202,7 @@ export function getOrderedTemplateInputFields(organism: string, action: 'submit' const orderedRequiredFields = requiredInternalFields.map((field) => decorate(field, true)); // Without an explicit template, every input field is enabled by default. - if (schema.metadataTemplate === undefined || schema.metadataTemplate === null) { + if (schema.metadataTemplate === undefined) { return [...orderedRequiredFields, ...nonInternalInputFields.map((field) => decorate(field, true))]; }