fix(plugin-grid): sniff CSV encoding in import wizard (GB18030 fallback) (#2557)

The import wizard read delimited files with file.text(), which always
decodes UTF-8. CSVs saved by zh-CN Excel ("CSV (comma delimited)") are
GBK-encoded, so every non-ASCII header turned into mojibake, required
columns could never be mapped, and the import could never succeed.

Decode bytes instead: honor an explicit BOM (UTF-8 / UTF-16 LE / BE),
then try strict UTF-8, then fall back to GB18030 via the native
TextDecoder (no new dependency).

Reported in steedos-labs/os-project-titanwind-ehr#185.

Author: baozhoutao

packages/plugin-grid/src/importParsers.test.ts

@@ -109,6 +109,38 @@ describe('parseSpreadsheetFile', () => {
     const grid = await parseSpreadsheetFile(makeFile('data.csv', 'a,b\n1,2'));
     expect(grid).toEqual([['a', 'b'], ['1', '2']]);
   });
+
+  // Encoding sniffing (#185): zh-CN Excel's "save as CSV" writes GBK, which a
+  // plain UTF-8 read turns into unmappable mojibake headers.
+  it('decodes a GBK/GB18030 .csv (zh-CN Excel default) via fallback', async () => {
+    // '名称 *,编号 *\n测试岛2,TEST-001' encoded as GBK
+    const gbk = new Uint8Array([
+      0xc3, 0xfb, 0xb3, 0xc6, 0x20, 0x2a, 0x2c, 0xb1, 0xe0, 0xba, 0xc5, 0x20, 0x2a, 0x0a,
+      0xb2, 0xe2, 0xca, 0xd4, 0xb5, 0xba, 0x32, 0x2c, 0x54, 0x45, 0x53, 0x54, 0x2d, 0x30, 0x30, 0x31,
+    ]);
+    const grid = await parseSpreadsheetFile(makeFile('data.csv', gbk));
+    expect(grid).toEqual([['名称 *', '编号 *'], ['测试岛2', 'TEST-001']]);
+  });
+
+  it('keeps plain UTF-8 Chinese without a BOM intact', async () => {
+    const grid = await parseSpreadsheetFile(makeFile('data.csv', '名称,城市\n张三,北京'));
+    expect(grid).toEqual([['名称', '城市'], ['张三', '北京']]);
+  });
+
+  it('strips a UTF-8 BOM (our own downloaded template round-trips)', async () => {
+    const grid = await parseSpreadsheetFile(makeFile('data.csv', 'a,b\n1,2'));
+    expect(grid).toEqual([['a', 'b'], ['1', '2']]);
+  });
+
+  it('decodes UTF-16 LE with a BOM', async () => {
+    // 'name,城市\n1,北京' encoded as UTF-16 LE with BOM
+    const utf16 = new Uint8Array([
+      0xff, 0xfe, 0x6e, 0x00, 0x61, 0x00, 0x6d, 0x00, 0x65, 0x00, 0x2c, 0x00,
+      0xce, 0x57, 0x02, 0x5e, 0x0a, 0x00, 0x31, 0x00, 0x2c, 0x00, 0x17, 0x53, 0xac, 0x4e,
+    ]);
+    const grid = await parseSpreadsheetFile(makeFile('data.csv', utf16));
+    expect(grid).toEqual([['name', '城市'], ['1', '北京']]);
+  });
 });
 
 describe('suggestColumnMappings', () => {

packages/plugin-grid/src/importParsers.ts

@@ -104,14 +104,43 @@ export async function parseExcelArrayBuffer(buffer: ArrayBuffer): Promise<string
 }
 
 /**
- * Parse any supported import file into `string[][]`. Delimited text is read as
- * UTF-8; .xlsx is parsed via {@link parseExcelArrayBuffer}. Throws an
+ * Decode a delimited-text import file's bytes into a string. Real-world
+ * spreadsheet CSVs are not reliably UTF-8 — zh-CN Excel's "另存为 CSV" writes
+ * GBK, which `file.text()` (always UTF-8) turns into mojibake headers that can
+ * never be mapped — so the encoding is sniffed instead of assumed:
+ *   1. An explicit BOM (UTF-8 / UTF-16 LE / UTF-16 BE) wins.
+ *   2. Otherwise try strict UTF-8; its validation rejects GBK multi-byte runs.
+ *   3. Fall back to GB18030 (a superset of GBK/GB2312), the dominant
+ *      non-UTF-8 CSV encoding in practice.
+ */
+export function decodeSpreadsheetText(buffer: ArrayBuffer): string {
+  const bytes = new Uint8Array(buffer);
+  if (bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) return new TextDecoder('utf-8').decode(bytes);
+  if (bytes[0] === 0xff && bytes[1] === 0xfe) return new TextDecoder('utf-16le').decode(bytes);
+  if (bytes[0] === 0xfe && bytes[1] === 0xff) return new TextDecoder('utf-16be').decode(bytes);
+  try {
+    return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
+  } catch {
+    try {
+      return new TextDecoder('gb18030').decode(bytes);
+    } catch {
+      // Runtimes without the gb18030 decoder (e.g. small-ICU Node) — degrade
+      // to lossy UTF-8 rather than failing the whole import.
+      return new TextDecoder('utf-8').decode(bytes);
+    }
+  }
+}
+
+/**
+ * Parse any supported import file into `string[][]`. Delimited text is
+ * decoded via {@link decodeSpreadsheetText} (UTF-8 with GB18030 fallback);
+ * .xlsx is parsed via {@link parseExcelArrayBuffer}. Throws an
  * {@link ImportParseError} code for unsupported / legacy formats.
  */
 export async function parseSpreadsheetFile(file: File): Promise<string[][]> {
   const name = file.name.toLowerCase();
   if (name.endsWith('.csv') || name.endsWith('.tsv') || name.endsWith('.txt')) {
-    const text = await file.text();
+    const text = decodeSpreadsheetText(await file.arrayBuffer());
     return parseDelimited(text, name.endsWith('.tsv') ? '\t' : ',');
   }
   if (name.endsWith('.xlsx') || name.endsWith('.xlsm')) {