feat: bundle fixtures dir when file ops are used, add readdirSync

Author: ChALkeR

bundler/bundle.js

@@ -1,5 +1,5 @@
 import assert from 'node:assert/strict'
-import { readFile } from 'node:fs/promises'
+import { readFile, readdir } from 'node:fs/promises'
 import { existsSync } from 'node:fs'
 import { fileURLToPath, pathToFileURL } from 'node:url'
 import { basename, dirname, extname, resolve, join, relative } from 'node:path'
@@ -175,11 +175,15 @@ export const build = async (...files) => {
 
   const fsfiles = await getPackageFiles(filename ? dirname(resolve(filename)) : process.cwd())
   const fsFilesContents = new Map()
+  const fsFilesDirs = new Map()
   const cwd = process.cwd()
-  const fsFilesAdd = async (fileRelative) => {
-    if (!fileRelative || !/^[a-z0-9@_./-]+$/iu.test(fileRelative)) return
-    const file = resolve(fileRelative)
-    if (!file.startsWith(`${cwd}/`)) return
+  const fixturesRegex = /fixtures/u
+  const aggressiveExtensions = /\.(json|txt|hex)$/u // These are bundled when just used in path.join and by wildcard from fixtures/
+  const fileAllowed = (f) =>
+    f && f.startsWith(`${cwd}/`) && resolve(f) === f && /^[a-z0-9@_./-]+$/iu.test(relative(cwd, f))
+
+  const fsFilesAdd = async (file) => {
+    if (!fileAllowed(file)) return
     try {
       const data = await readFile(file, 'base64')
       if (fsFilesContents.has(file)) {
@@ -192,19 +196,46 @@ export const build = async (...files) => {
     }
   }
 
+  const fixturesSeen = { fs: false, fixtures: false, bundled: false }
+  const fsFilesBundleFixtures = async (reason) => {
+    if (fixturesSeen.bundled || !filename) return
+    if (reason === 'fs' || reason === 'fixtures') fixturesSeen[reason] = true
+    if (!fixturesSeen.fs || !fixturesSeen.fixtures) return
+    fixturesSeen.bundled = true
+    const dir = dirname(resolve(filename))
+    for (const name of await readdir(dir, { recursive: true })) {
+      const parent = dirname(name)
+      if (!fixturesRegex.test(parent)) continue // relative dir path should look like a fixtures dir
+      // Save as directiry entry into parent dir
+      const subdir = resolve(dir, parent)
+      if (fileAllowed(subdir)) {
+        if (!fsFilesDirs.has(subdir)) fsFilesDirs.set(subdir, [])
+        fsFilesDirs.get(subdir).push(basename(name))
+      }
+
+      // Save to files
+      const file = resolve(dir, name)
+      if (aggressiveExtensions.test(file)) await fsFilesAdd(file)
+    }
+  }
+
   specificLoadPipeline.push(async (source, filepath) => {
     for (const m of source.matchAll(/readFileSync\(\s*(?:"([^"\\]+)"|'([^'\\]+)')[),]/gu)) {
-      await fsFilesAdd(m[1] || m[2])
+      await fsFilesAdd(resolve(m[1] || m[2])) // resolves from cwd
     }
 
     // E.g. path.join(import.meta.dirname, './fixtures/data.json'), dirname is inlined by loadPipeline already
     const dir = dirname(filepath)
     for (const m of source.matchAll(/join\(\s*("[^"\\]+"),\s*(?:"([^"\\]+)"|'([^'\\]+)')\s*\)/gu)) {
       if (m[1] !== JSON.stringify(dir)) continue // only allow files relative to dirname, from loadPipeline
-      const file = relative(cwd, join(dir, m[2] || m[3]))
-      if (/\.(json|txt|hex)$/u.test(file)) await fsFilesAdd(file) // only allow specific extensions used as test fixtures
+      const file = resolve(dir, m[2] || m[3])
+      if (aggressiveExtensions.test(file)) await fsFilesAdd(file) // only bundle path.join for specific extensions used as test fixtures
     }
 
+    // Both conditions should happen for deep fixtures inclusion
+    if (/(readdir|readFile|exists)Sync/u.test(source)) await fsFilesBundleFixtures('fs')
+    if (fixturesRegex.test(source)) await fsFilesBundleFixtures('fixtures')
+
     return source
   })
 
@@ -284,6 +315,7 @@ export const build = async (...files) => {
       EXODUS_TEST_RECORDINGS: stringify(EXODUS_TEST_RECORDINGS),
       EXODUS_TEST_FSFILES: stringify(fsfiles), // TODO: can we safely use relative paths?
       EXODUS_TEST_FSFILES_CONTENTS: stringify([...fsFilesContents.entries()]),
+      EXODUS_TEST_FSDIRS: stringify([...fsFilesDirs.entries()]),
     },
     alias: {
       // Jest, tape and node:test
@@ -352,9 +384,10 @@ export const build = async (...files) => {
   let res = await buildWrap(config)
   assert.equal(res instanceof Error, res.errors.length > 0)
 
-  if (fsFilesContents.size > 0) {
+  if (fsFilesContents.size > 0 || fsFilesDirs.size > 0) {
     // re-run as we detected that tests depend on fsReadFileSync contents
     config.define.EXODUS_TEST_FSFILES_CONTENTS = stringify([...fsFilesContents.entries()])
+    config.define.EXODUS_TEST_FSDIRS = stringify([...fsFilesDirs.entries()])
     res = await buildWrap(config)
     assert.equal(res instanceof Error, res.errors.length > 0)
   }

bundler/modules/fs.cjs

@@ -75,38 +75,49 @@ const stubs = Object.fromEntries(mainKeys.map((key) => [key, () => err(key)]))
 const stubsPromises = Object.fromEntries(promisesKeys.map((key) => [key, async () => err(key)]))
 const promises = { ...stubsPromises, constants }
 
-// eslint-disable-next-line no-undef
-const fsFiles = typeof EXODUS_TEST_FSFILES === 'undefined' ? null : new Set(EXODUS_TEST_FSFILES)
-const existsSync = (file) => {
-  if (fsFiles?.has(file) || fsFilesContents?.has(file)) return true
-  err('existsSync', file)
+const decode = (source, sourceEncoding, encoding) => {
+  if (encoding && sourceEncoding === encoding) return source
+  const data = Buffer.from(source, sourceEncoding)
+  return encoding === undefined ? data : data.toString(encoding)
+}
+
+const getOptions = (arg, options) => {
+  if (typeof arg !== 'string') throw new Error('first argument should be string')
+  const file = resolve(process.cwd(), arg)
+  if (typeof options === 'string') return { file, encoding: options, rest: {} }
+  if (options === undefined) return { file, rest: {} }
+  if (typeof options !== 'object') throw new Error('Unexpected options')
+  const { encoding: enc, ...rest } = options
+  if (enc !== undefined && typeof enc !== 'string') throw new Error('encoding should be a string')
+  return { file, encoding: enc, rest }
 }
 
 const fsFilesContents =
   // eslint-disable-next-line no-undef
   typeof EXODUS_TEST_FSFILES_CONTENTS === 'undefined' ? null : new Map(EXODUS_TEST_FSFILES_CONTENTS)
-const readFileSync = (file, options) => {
-  let encoding
-  if (typeof options === 'string') {
-    encoding = options
-  } else if (options !== undefined) {
-    if (typeof options !== 'object') throw new Error('Unexpected readFileSync options')
-    const { encoding: enc, ...rest } = options
-    if (enc !== undefined && typeof enc !== 'string') throw new Error('encoding should be a string')
-    encoding = enc
-    if (Object.keys(rest).length > 0) throw new Error('Unsupported readFileSync options')
-  }
+const readFileSync = (arg, options) => {
+  const { file, encoding, rest } = getOptions(arg, options)
+  if (Object.keys(rest).length > 0) throw new Error('Unsupported readFileSync options')
+  if (fsFilesContents?.has(file)) return decode(fsFilesContents.get(file), 'base64', encoding)
+  err('readFileSync', file)
+}
 
-  if (typeof file !== 'string') throw new Error('file argument should be string')
-  file = resolve(process.cwd(), file)
-  if (fsFilesContents?.has(file)) {
-    const data = Buffer.from(fsFilesContents.get(file), 'base64')
-    if (encoding?.toLowerCase().replace('-', '') === 'utf8') return data.toString('utf8')
-    if (encoding === undefined) return data
-    throw new Error('Unsupported encoding')
-  }
+// eslint-disable-next-line no-undef
+const fsDir = typeof EXODUS_TEST_FSDIRS === 'undefined' ? null : new Map(EXODUS_TEST_FSDIRS)
+const readdirSync = (arg, options) => {
+  const { file: dir, encoding, rest } = getOptions(arg, options)
+  if (Object.keys(rest).length > 0) throw new Error('Unsupported readdirSync options')
+  const enc = encoding === 'buffer' ? undefined : encoding || 'utf8'
+  if (fsDir?.has(dir)) return fsDir.get(dir).map((name) => decode(name, 'utf8', enc))
+  err('readdirSync', dir)
+}
 
-  err('readFileSync', file)
+// eslint-disable-next-line no-undef
+const fsFiles = typeof EXODUS_TEST_FSFILES === 'undefined' ? null : new Set(EXODUS_TEST_FSFILES)
+const existsSync = (file) => {
+  if (fsFiles?.has(file) || fsFilesContents?.has(file) || fsDir?.has(file)) return true
+  err('existsSync', file)
 }
 
-module.exports = { ...stubs, existsSync, readFileSync, promises, constants, F_OK, R_OK, W_OK, X_OK }
+const implemented = { existsSync, readFileSync, readdirSync }
+module.exports = { ...stubs, ...implemented, promises, constants, F_OK, R_OK, W_OK, X_OK }