Skip to content

Commit b8a1e7c

Browse files
ChALkeRChALkeR
committed
test: test various utf16 codepaths on Node.js
1 parent 8d8d08b commit b8a1e7c

2 files changed

Lines changed: 60 additions & 7 deletions

File tree

‎tests/utf16.noenc.test.cjs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
delete globalThis.TextEncoder
2+
delete globalThis.TextDecoder
3+
require('./utf16.test.js')

‎tests/utf16.test.js‎

Lines changed: 57 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import * as utf16 from '@exodus/bytes/utf16.js'
2+
import * as lib from '../utf16.js'
23
import { nativeDecoder } from '../fallback/_utils.js'
34
import { randomValues } from '@exodus/crypto/randomBytes'
45
import * as js from '../fallback/utf16.js'
@@ -44,7 +45,12 @@ const { Buffer, TextDecoder } = globalThis
4445

4546
describe('utf16toString', () => {
4647
describe('invalid input', () => {
47-
for (const method of [utf16.utf16toString, utf16.utf16toStringLoose]) {
48+
for (const method of [
49+
utf16.utf16toString,
50+
utf16.utf16toStringLoose,
51+
lib.utf16toString,
52+
lib.utf16toStringLoose,
53+
]) {
4854
test(method.name, (t) => {
4955
for (const x of [null, undefined, [], [1, 2], 'string']) t.assert.throws(() => method(x))
5056
t.assert.throws(() => method(new Uint8Array(0)))
@@ -60,7 +66,7 @@ describe('utf16toString', () => {
6066
})
6167

6268
test('orphans throw in utf16toString', (t) => {
63-
for (const utf16toString of [utf16.utf16toString, js.decode]) {
69+
for (const utf16toString of [utf16.utf16toString, lib.utf16toString, js.decode]) {
6470
for (const { invalid } of orphans) {
6571
t.assert.throws(() => utf16toString(Uint16Array.of(...invalid)))
6672

@@ -79,7 +85,11 @@ describe('utf16toString', () => {
7985

8086
test('non-utf16 bytes get replaced in utf16toStringLoose', (t) => {
8187
for (const { invalid, replaced } of orphans) {
82-
for (const utf16toStringLoose of [utf16.utf16toStringLoose, (u16) => js.decode(u16, true)]) {
88+
for (const utf16toStringLoose of [
89+
utf16.utf16toStringLoose,
90+
lib.utf16toStringLoose,
91+
(u16) => js.decode(u16, true),
92+
]) {
8393
const res = utf16toStringLoose(Uint16Array.of(...invalid))
8494
t.assert.strictEqual(res.length, invalid.length)
8595
t.assert.strictEqual(res, String.fromCharCode(...replaced))
@@ -106,7 +116,12 @@ describe('utf16toString', () => {
106116

107117
describe('utf16fromString', () => {
108118
describe('invalid input', () => {
109-
for (const method of [utf16.utf16fromString, utf16.utf16fromStringLoose]) {
119+
for (const method of [
120+
utf16.utf16fromString,
121+
utf16.utf16fromStringLoose,
122+
lib.utf16fromString,
123+
lib.utf16fromStringLoose,
124+
]) {
110125
test(method.name, (t) => {
111126
for (const input of [...[null, undefined, [], [1, 2], ['00'], new Uint8Array()]]) {
112127
t.assert.throws(() => method(input))
@@ -120,15 +135,19 @@ describe('utf16fromString', () => {
120135

121136
test('orphans throw in utf16fromString', (t) => {
122137
for (const { invalid } of orphans) {
123-
for (const utf16fromString of [utf16.utf16fromString, js.encode]) {
138+
for (const utf16fromString of [utf16.utf16fromString, lib.utf16fromString, js.encode]) {
124139
t.assert.throws(() => utf16fromString(String.fromCharCode(...invalid)))
125140
}
126141
}
127142
})
128143

129144
test('orphans get replaced in utf16fromStringLoose', (t) => {
130145
for (const { invalid, replaced } of orphans) {
131-
for (const utf16fromStringLoose of [utf16.utf16fromStringLoose, (s) => js.encode(s, true)]) {
146+
for (const utf16fromStringLoose of [
147+
utf16.utf16fromStringLoose,
148+
lib.utf16fromStringLoose,
149+
(s) => js.encode(s, true),
150+
]) {
132151
const input = String.fromCharCode(...invalid)
133152
t.assert.deepStrictEqual(utf16fromStringLoose(input), Uint16Array.from(replaced))
134153
}
@@ -145,17 +164,21 @@ describe('random data', () => {
145164
let nativeIsOk = nativeDecoder
146165
// Non-native might be wrong
147166
// Also, Node.js without ICU has native not performing replacement, so we can't compare against that
148-
nativeIsOk &= new TextDecoder('utf-16le').decode(Uint8Array.of(0x00, 0xd8)) === '\uFFFD'
167+
nativeIsOk &=
168+
TextDecoder && new TextDecoder('utf-16le').decode(Uint8Array.of(0x00, 0xd8)) === '\uFFFD'
149169

150170
test('utf16toStringLoose', (t) => {
151171
const decoderLE = nativeIsOk ? new TextDecoder('utf-16le', { ignoreBOM }) : null // polyfilled might be wrong
152172
const decoderBE = nativeIsOk ? new TextDecoder('utf-16be', { ignoreBOM }) : null // polyfilled might be wrong
153173
for (const u16 of pool) {
154174
const str = utf16.utf16toStringLoose(u16)
175+
t.assert.strictEqual(str, lib.utf16toStringLoose(u16))
155176
t.assert.strictEqual(str, js.decode(u16, true))
156177
const u8 = new Uint8Array(u16.buffer, u16.byteOffset, u16.byteLength)
157178
const strLE = utf16.utf16toStringLoose(u8, 'uint8-le')
179+
t.assert.strictEqual(strLE, lib.utf16toStringLoose(u8, 'uint8-le'))
158180
const strBE = utf16.utf16toStringLoose(u8, 'uint8-be')
181+
t.assert.strictEqual(strBE, lib.utf16toStringLoose(u8, 'uint8-be'))
159182
t.assert.strictEqual(str, isLE ? strLE : strBE)
160183
if (decoderLE) t.assert.strictEqual(strLE, decoderLE.decode(u8))
161184
if (decoderBE) t.assert.strictEqual(strBE, decoderBE.decode(u8))
@@ -170,13 +193,19 @@ describe('random data', () => {
170193
for (const u16 of poolAscii) {
171194
const str = utf16.utf16toString(u16)
172195
t.assert.strictEqual(str, utf16.utf16toStringLoose(u16))
196+
t.assert.strictEqual(str, lib.utf16toString(u16))
197+
t.assert.strictEqual(str, lib.utf16toStringLoose(u16))
173198
t.assert.strictEqual(str, js.decode(u16, false))
174199
t.assert.strictEqual(str, js.decode(u16, true))
175200
const u8 = new Uint8Array(u16.buffer, u16.byteOffset, u16.byteLength)
176201
const strLE = utf16.utf16toString(u8, 'uint8-le')
177202
const strBE = utf16.utf16toString(u8, 'uint8-be')
178203
t.assert.strictEqual(strLE, utf16.utf16toStringLoose(u8, 'uint8-le'))
204+
t.assert.strictEqual(strLE, lib.utf16toString(u8, 'uint8-le'))
205+
t.assert.strictEqual(strLE, lib.utf16toStringLoose(u8, 'uint8-le'))
179206
t.assert.strictEqual(strBE, utf16.utf16toStringLoose(u8, 'uint8-be'))
207+
t.assert.strictEqual(strBE, lib.utf16toString(u8, 'uint8-be'))
208+
t.assert.strictEqual(strBE, lib.utf16toStringLoose(u8, 'uint8-be'))
180209
t.assert.strictEqual(str, isLE ? strLE : strBE)
181210
if (decoderLE) t.assert.strictEqual(strLE, decoderLE.decode(u8))
182211
if (decoderBE) t.assert.strictEqual(strBE, decoderBE.decode(u8))
@@ -200,16 +229,22 @@ describe('random data', () => {
200229
}
201230

202231
if (str === undefined) {
232+
t.assert.throws(() => lib.utf16toString(u16))
203233
t.assert.throws(() => js.decode(u16, false))
204234
t.assert.throws(() => utf16.utf16toString(u8, isLE ? 'uint8-le' : 'uint8-be'))
235+
t.assert.throws(() => lib.utf16toString(u8, isLE ? 'uint8-le' : 'uint8-be'))
205236
if (nativeIsOk) t.assert.throws(() => (isLE ? decoderLE : decoderBE).decode(u8))
206237
} else {
207238
t.assert.strictEqual(str, strings[i])
208239
t.assert.strictEqual(str, utf16.utf16toStringLoose(u16))
240+
t.assert.strictEqual(str, lib.utf16toString(u16))
241+
t.assert.strictEqual(str, lib.utf16toStringLoose(u16))
209242
t.assert.strictEqual(str, js.decode(u16, false))
210243
t.assert.strictEqual(str, js.decode(u16, true))
211244
t.assert.strictEqual(str, utf16.utf16toString(u8, isLE ? 'uint8-le' : 'uint8-be'))
212245
t.assert.strictEqual(str, utf16.utf16toStringLoose(u8, isLE ? 'uint8-le' : 'uint8-be'))
246+
t.assert.strictEqual(str, lib.utf16toString(u8, isLE ? 'uint8-le' : 'uint8-be'))
247+
t.assert.strictEqual(str, lib.utf16toStringLoose(u8, isLE ? 'uint8-le' : 'uint8-be'))
213248
if (nativeIsOk) t.assert.strictEqual(str, (isLE ? decoderLE : decoderBE).decode(u8))
214249
if (Buffer && isLE) t.assert.strictEqual(str, Buffer.from(u8).toString('utf-16le'))
215250
}
@@ -224,10 +259,14 @@ describe('random data', () => {
224259
const str = stringsAscii[i]
225260
t.assert.deepStrictEqual(u16, utf16.utf16fromString(str))
226261
t.assert.deepStrictEqual(u16, utf16.utf16fromStringLoose(str))
262+
t.assert.deepStrictEqual(u16, lib.utf16fromString(str))
263+
t.assert.deepStrictEqual(u16, lib.utf16fromStringLoose(str))
227264
t.assert.deepStrictEqual(u16, js.encode(str, false))
228265
t.assert.deepStrictEqual(u16, js.encode(str, true))
229266
t.assert.deepStrictEqual(u8, utf16.utf16fromString(str, isLE ? 'uint8-le' : 'uint8-be'))
230267
t.assert.deepStrictEqual(u8, utf16.utf16fromStringLoose(str, isLE ? 'uint8-le' : 'uint8-be'))
268+
t.assert.deepStrictEqual(u8, lib.utf16fromString(str, isLE ? 'uint8-le' : 'uint8-be'))
269+
t.assert.deepStrictEqual(u8, lib.utf16fromStringLoose(str, isLE ? 'uint8-le' : 'uint8-be'))
231270
if (Buffer && isLE) t.assert.deepEqual(u8, Buffer.from(str, 'utf-16le'))
232271
}
233272
})
@@ -239,10 +278,14 @@ describe('random data', () => {
239278
const u16 = utf16.utf16fromString(str)
240279
const u8 = new Uint8Array(u16.buffer, u16.byteOffset, u16.byteLength)
241280
t.assert.deepStrictEqual(u16, utf16.utf16fromStringLoose(str))
281+
t.assert.deepStrictEqual(u16, lib.utf16fromString(str))
282+
t.assert.deepStrictEqual(u16, lib.utf16fromStringLoose(str))
242283
t.assert.deepStrictEqual(u16, js.encode(str, false))
243284
t.assert.deepStrictEqual(u16, js.encode(str, true))
244285
t.assert.deepStrictEqual(u8, utf16.utf16fromString(str, isLE ? 'uint8-le' : 'uint8-be'))
245286
t.assert.deepStrictEqual(u8, utf16.utf16fromStringLoose(str, isLE ? 'uint8-le' : 'uint8-be'))
287+
t.assert.deepStrictEqual(u8, lib.utf16fromString(str, isLE ? 'uint8-le' : 'uint8-be'))
288+
t.assert.deepStrictEqual(u8, lib.utf16fromStringLoose(str, isLE ? 'uint8-le' : 'uint8-be'))
246289
if (Buffer && isLE) t.assert.deepEqual(u8, Buffer.from(str, 'utf-16le'))
247290
restored.push(u16)
248291
}
@@ -258,10 +301,14 @@ describe('random data', () => {
258301
const u8 = new Uint8Array(u16.buffer, u16.byteOffset, u16.byteLength)
259302
t.assert.strictEqual(str, utf16.utf16toString(u16))
260303
t.assert.strictEqual(str, utf16.utf16toStringLoose(u16))
304+
t.assert.strictEqual(str, lib.utf16toString(u16))
305+
t.assert.strictEqual(str, lib.utf16toStringLoose(u16))
261306
t.assert.strictEqual(str, js.decode(u16, false))
262307
t.assert.strictEqual(str, js.decode(u16, true))
263308
t.assert.strictEqual(str, utf16.utf16toString(u8, isLE ? 'uint8-le' : 'uint8-be'))
264309
t.assert.strictEqual(str, utf16.utf16toStringLoose(u8, isLE ? 'uint8-le' : 'uint8-be'))
310+
t.assert.strictEqual(str, lib.utf16toString(u8, isLE ? 'uint8-le' : 'uint8-be'))
311+
t.assert.strictEqual(str, lib.utf16toStringLoose(u8, isLE ? 'uint8-le' : 'uint8-be'))
265312
if (nativeIsOk) t.assert.strictEqual(str, (isLE ? decoderLE : decoderBE).decode(u8))
266313
if (Buffer && isLE) t.assert.strictEqual(str, Buffer.from(u8).toString('utf-16le'))
267314
}
@@ -276,4 +323,7 @@ test('large strings', { skip: skipLarge }, (t) => {
276323
const s = 'abcde01234'.repeat(12e6) // 120e6 total
277324
// e.g. npmjs.com/buffer fails on this
278325
t.assert.strictEqual(s, utf16.utf16toString(utf16.utf16fromString(s)))
326+
if (lib.utf16toString !== utf16.utf16toString || lib.utf16fromString !== utf16.utf16fromString) {
327+
t.assert.strictEqual(s, lib.utf16toString(lib.utf16fromString(s)))
328+
}
279329
})

0 commit comments

Comments
 (0)