feat: support fatal utf-8 on Node.js without ICU

Author: ChALkeR

tests/utf8.test.js

@@ -145,6 +145,16 @@ describe('random data', () => {
   const restored = []
   const ignoreBOM = true
 
+  let nativeFatal = nativeDecoder
+  if (nativeFatal) {
+    try {
+      // Non-fixed Node.js without ICU doesn't have 'fatal' option support
+      new TextDecoder('utf8', { fatal: true }) // eslint-disable-line no-new
+    } catch {
+      nativeFatal = false
+    }
+  }
+
   test('utf8toStringLoose', (t) => {
     const textDecoder = nativeDecoder ? new TextDecoder('utf8', { ignoreBOM }) : null // polyfilled might be wrong
     const NativeBuffer = globalThis.Buffer && !globalThis.Buffer.TYPED_ARRAY_SUPPORT ? Buffer : null
@@ -158,7 +168,7 @@ describe('random data', () => {
   })
 
   test('utf8toString (ascii)', (t) => {
-    const textDecoder = nativeDecoder ? new TextDecoder('utf8', { fatal: true, ignoreBOM }) : null
+    const textDecoder = nativeFatal ? new TextDecoder('utf8', { fatal: true, ignoreBOM }) : null
     for (const u8 of poolAscii) {
       const str = utf8toString(u8)
       t.assert.strictEqual(str, utf8toStringLoose(u8))
@@ -171,7 +181,7 @@ describe('random data', () => {
   })
 
   test('utf8toString', (t) => {
-    const textDecoder = nativeDecoder ? new TextDecoder('utf8', { fatal: true, ignoreBOM }) : null
+    const textDecoder = nativeFatal ? new TextDecoder('utf8', { fatal: true, ignoreBOM }) : null
     t.assert.strictEqual(strings.length, pool.length)
     for (let i = 0; i < pool.length; i++) {
       const u8 = pool[i]
@@ -227,7 +237,7 @@ describe('random data', () => {
   })
 
   test('utf8toString / utf8toStringLoose', (t) => {
-    const textDecoder = nativeDecoder ? new TextDecoder('utf8', { fatal: true, ignoreBOM }) : null
+    const textDecoder = nativeFatal ? new TextDecoder('utf8', { fatal: true, ignoreBOM }) : null
     t.assert.strictEqual(strings.length, pool.length)
     for (let i = 0; i < pool.length; i++) {
       const str = strings[i]

utf8.node.js

@@ -1,15 +1,21 @@
 import { assertUint8 } from './assert.js'
 import { typedView } from './array.js'
-import { E_STRICT_UNICODE } from './fallback/utf8.js'
+import { E_STRICT, E_STRICT_UNICODE } from './fallback/utf8.js'
 import { isAscii } from 'node:buffer'
 
 if (Buffer.TYPED_ARRAY_SUPPORT) throw new Error('Unexpected Buffer polyfill')
 
-const decoderFatal = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true })
+let decoderFatal
 const decoderLoose = new TextDecoder('utf-8', { ignoreBOM: true })
 const { isWellFormed } = String.prototype
 const isDeno = Boolean(globalThis.Deno)
 
+try {
+  decoderFatal = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true })
+} catch {
+  // Without ICU, Node.js doesn't support fatal option for utf-8
+}
+
 function encode(str, loose = false) {
   if (typeof str !== 'string') throw new TypeError('Input is not a string')
   const strLength = str.length
@@ -45,7 +51,14 @@ function decode(arr, loose = false) {
     return buf.latin1Slice(0, arr.byteLength) // .latin1Slice is faster than .asciiSlice
   }
 
-  return loose ? decoderLoose.decode(arr) : decoderFatal.decode(arr)
+  if (loose) return decoderLoose.decode(arr)
+  if (decoderFatal) return decoderFatal.decode(arr)
+
+  // We are in an env without native fatal decoder support (non-fixed Node.js without ICU)
+  // Well, just recheck against encode if it contains replacement then, this is still faster than js impl
+  const str = decoderLoose.decode(arr)
+  if (str.includes('\uFFFD') && !Buffer.from(str).equals(arr)) throw new TypeError(E_STRICT)
+  return str
 }
 
 export const utf8fromString = (str, format = 'uint8') => typedView(encode(str, false), format)