feat(babel): add parallel processing via worker threads

Add a `parallel` option that processes files concurrently using Node.js worker threads. This reduces build times for large projects where Babel transformation is a bottleneck. This is similar to the existing parallel behavior of `@rollup/plugin-terser`, but uses the `workerpool` package instead of a custom implementation.

This required some fairly significant refactoring, because we can only pass serializable objects between the main thread and the worker threads. It also required changes to the plugin's own build config, so that we can generate a dedicated worker entrypoint.

Validations are added to ensure that unserializable config (e.g. inline babel plugins) cannot be used alongside the new parallel mode. For people using dedicated babel config files, this isn't a problem, because they are loaded directly by babel in the worker thread itself.

The worker threads do have a setup cost, so this only makes sense for large projects. In Discourse, enabling this parallel mode cuts our overall vite (rolldown) build time by about 45% (from ~11s to ~6s) on my machine.

Author: davidtaylorhq

packages/babel/README.md

@@ -135,6 +135,17 @@ Default: `false`
 
 Before transpiling your input files this plugin also transpile a short piece of code **for each** input file. This is used to validate some misconfiguration errors, but for sufficiently big projects it can slow your build times so if you are confident about your configuration then you might disable those checks with this option.
 
+### `parallel`
+
+Type: `Boolean | number`<br>
+Default: `false`
+
+Enable parallel processing of files in worker threads. This has a setup cost, so is best suited for larger projects. Pass an integer to set the number of workers. Set `true` for the default number of workers (based on CPU cores, capped at 4).
+
+This option is available for both the input plugin (`babel()`) and the output plugin (`getBabelOutputPlugin()`).
+
+This option cannot be used alongside custom overrides or non-serializable Babel options.
+
 ### External dependencies
 
 Ideally, you should only be transforming your source code, rather than running all of your external dependencies through Babel (to ignore external dependencies from being handled by this plugin you might use `exclude: 'node_modules/**'` option). If you have a dependency that exposes untranspiled ES6 source code that doesn't run in your target environment, then you may need to break this rule, but it often causes problems with unusual `.babelrc` files or mismatched versions of Babel.

packages/babel/package.json

@@ -67,7 +67,8 @@
   },
   "dependencies": {
     "@babel/helper-module-imports": "^7.18.6",
-    "@rollup/pluginutils": "^5.0.1"
+    "@rollup/pluginutils": "^5.0.1",
+    "workerpool": "^9.0.0"
   },
   "devDependencies": {
     "@babel/core": "^7.19.1",

packages/babel/rollup.config.mjs

@@ -1,14 +1,37 @@
 import { readFileSync } from 'fs';
 
-import { createConfig } from '../../shared/rollup.config.mjs';
+import { createConfig, emitModulePackageFile } from '../../shared/rollup.config.mjs';
 
 import { babel } from './src/index.js';
 
 const pkg = JSON.parse(readFileSync(new URL('./package.json', import.meta.url), 'utf8'));
 
 export default {
   ...createConfig({ pkg }),
-  input: './src/index.js',
+  input: {
+    index: './src/index.js',
+    worker: './src/worker.js'
+  },
+  output: [
+    {
+      format: 'cjs',
+      dir: 'dist/cjs',
+      exports: 'named',
+      footer(chunkInfo) {
+        if (chunkInfo.name === 'index') {
+          return 'module.exports = Object.assign(exports.default, exports);';
+        }
+        return null;
+      },
+      sourcemap: true
+    },
+    {
+      format: 'es',
+      dir: 'dist/es',
+      plugins: [emitModulePackageFile()],
+      sourcemap: true
+    }
+  ],
   plugins: [
     babel({
       presets: [['@babel/preset-env', { targets: { node: 14 } }]],

packages/babel/src/index.js

@@ -1,11 +1,13 @@
+import { cpus } from 'os';
+import { fileURLToPath } from 'url';
+
 import * as babel from '@babel/core';
 import { createFilter } from '@rollup/pluginutils';
+import workerpool from 'workerpool';
 
 import { BUNDLED, HELPERS } from './constants.js';
-import bundledHelpersPlugin from './bundledHelpersPlugin.js';
-import preflightCheck from './preflightCheck.js';
 import transformCode from './transformCode.js';
-import { addBabelPlugin, escapeRegExpCharacters, warnOnce } from './utils.js';
+import { escapeRegExpCharacters, warnOnce } from './utils.js';
 
 const unpackOptions = ({
   extensions = babel.DEFAULT_EXTENSIONS,
@@ -100,6 +102,68 @@ const returnObject = () => {
   return {};
 };
 
+function findNonSerializableOption(obj) {
+  const isSerializable = (value) => {
+    if (value === null) return true;
+    if (Array.isArray(value)) return value.every(isSerializable);
+    switch (typeof value) {
+      case 'undefined':
+      case 'string':
+      case 'number':
+      case 'boolean':
+        return true;
+      case 'object':
+        return Object.values(value).every(isSerializable);
+      default:
+        return false;
+    }
+  };
+
+  for (const key of Object.keys(obj)) {
+    if (!isSerializable(obj[key])) return key;
+  }
+  return null;
+}
+
+const WORKER_PATH = fileURLToPath(new URL('./worker.js', import.meta.url));
+
+function createParallelWorkerPool(parallel, overrides) {
+  if (typeof parallel === 'number' && (!Number.isInteger(parallel) || parallel < 1)) {
+    throw new Error(
+      'The "parallel" option must be true or a positive integer specifying the number of workers.'
+    );
+  }
+
+  if (!parallel) return null;
+
+  if (overrides?.config) {
+    throw new Error('Cannot use "parallel" mode with a custom "config" override.');
+  }
+  if (overrides?.result) {
+    throw new Error('Cannot use "parallel" mode with a custom "result" override.');
+  }
+
+  // Default limits to 4 workers. Benefits diminish after this point, because of the setup cost.
+  const workerCount = typeof parallel === 'number' ? parallel : Math.min(cpus().length, 4);
+  return workerpool.pool(WORKER_PATH, {
+    maxWorkers: workerCount,
+    workerType: 'thread'
+  });
+}
+
+function transformWithWorkerPool(workerPool, context, transformOpts, babelOptions) {
+  const nonSerializableKey = findNonSerializableOption(babelOptions);
+  if (nonSerializableKey) {
+    return Promise.reject(
+      new Error(
+        `Cannot use "parallel" mode because the "${nonSerializableKey}" option is not serializable.`
+      )
+    );
+  }
+
+  return workerPool.exec('transform', [transformOpts]).catch((err) => context.error(err.message));
+}
+
 function createBabelInputPluginFactory(customCallback = returnObject) {
   const overrides = customCallback(babel);
 
@@ -116,9 +180,12 @@ function createBabelInputPluginFactory(customCallback = returnObject) {
       include,
       filter: customFilter,
       skipPreflightCheck,
+      parallel,
       ...babelOptions
     } = unpackInputPluginOptions(pluginOptionsWithOverrides);
 
+    const workerPool = createParallelWorkerPool(parallel, overrides);
+
     const extensionRegExp = new RegExp(
       `(${extensions.map(escapeRegExpCharacters).join('|')})(\\?.*)?(#.*)?$`
     );
@@ -162,23 +229,45 @@ function createBabelInputPluginFactory(customCallback = returnObject) {
           if (!(await filter(filename, code))) return null;
           if (filename === HELPERS) return null;
 
-          return transformCode(
-            code,
-            { ...babelOptions, filename },
-            overrides,
+          const resolvedBabelOptions = { ...babelOptions, filename };
+
+          if (workerPool) {
+            return transformWithWorkerPool(
+              workerPool,
+              this,
+              {
+                inputCode: code,
+                babelOptions: resolvedBabelOptions,
+                skipPreflightCheck,
+                babelHelpers
+              },
+              resolvedBabelOptions
+            );
+          }
+
+          return transformCode({
+            inputCode: code,
+            babelOptions: resolvedBabelOptions,
+            overrides: {
+              config: overrides.config?.bind(this),
+              result: overrides.result?.bind(this)
+            },
             customOptions,
-            this,
-            async (transformOptions) => {
-              if (!skipPreflightCheck) {
-                await preflightCheck(this, babelHelpers, transformOptions);
-              }
-
-              return babelHelpers === BUNDLED
-                ? addBabelPlugin(transformOptions, bundledHelpersPlugin)
-                : transformOptions;
-            }
-          );
+            error: this.error.bind(this),
+            skipPreflightCheck,
+            babelHelpers
+          });
+        }
+      },
+
+      async closeBundle() {
+        if (!this.meta.watchMode) {
+          await workerPool?.terminate();
         }
+      },
+
+      async closeWatcher() {
+        await workerPool?.terminate();
       }
     };
   };
@@ -207,6 +296,8 @@ function createBabelOutputPluginFactory(customCallback = returnObject) {
       overrides
     );
 
+    const workerPool = createParallelWorkerPool(pluginOptionsWithOverrides.parallel, overrides);
+
     // cache for chunk name filter (includeChunks/excludeChunks)
     let chunkNameFilter;
 
@@ -242,6 +333,7 @@ function createBabelOutputPluginFactory(customCallback = returnObject) {
           externalHelpers,
           externalHelpersWhitelist,
           include,
+          parallel,
           runtimeHelpers,
           ...babelOptions
         } = unpackOutputPluginOptions(pluginOptionsWithOverrides, outputOptions);
@@ -257,7 +349,36 @@ function createBabelOutputPluginFactory(customCallback = returnObject) {
           }
         }
 
-        return transformCode(code, babelOptions, overrides, customOptions, this);
+        if (workerPool) {
+          return transformWithWorkerPool(
+            workerPool,
+            this,
+            {
+              inputCode: code,
+              babelOptions,
+              skipPreflightCheck: true
+            },
+            babelOptions
+          );
+        }
+
+        return transformCode({
+          inputCode: code,
+          babelOptions,
+          overrides: {
+            config: overrides.config?.bind(this),
+            result: overrides.result?.bind(this)
+          },
+          customOptions,
+          error: this.error.bind(this),
+          skipPreflightCheck: true
+        });
+      },
+
+      async generateBundle() {
+        if (!this.meta.watchMode) {
+          await workerPool?.terminate();
+        }
       }
     };
   };

packages/babel/src/preflightCheck.js

@@ -37,27 +37,27 @@ const mismatchError = (actual, expected, filename) =>
 // Revert to /\/helpers\/(esm\/)?inherits/ when Babel 8 gets released, this was fixed in https://github.com/babel/babel/issues/14185
 const inheritsHelperRe = /[\\/]+helpers[\\/]+(esm[\\/]+)?inherits/;
 
-export default async function preflightCheck(ctx, babelHelpers, transformOptions) {
+export default async function preflightCheck(error, babelHelpers, transformOptions) {
   const finalOptions = addBabelPlugin(transformOptions, helpersTestTransform);
   const check = (await babel.transformAsync(PREFLIGHT_INPUT, finalOptions)).code;
 
   // Babel sometimes splits ExportDefaultDeclaration into 2 statements, so we also check for ExportNamedDeclaration
   if (!/export (d|{)/.test(check)) {
-    ctx.error(MODULE_ERROR);
+    error(MODULE_ERROR);
   }
 
   if (inheritsHelperRe.test(check)) {
     if (babelHelpers === RUNTIME) {
       return;
     }
-    ctx.error(mismatchError(RUNTIME, babelHelpers, transformOptions.filename));
+    error(mismatchError(RUNTIME, babelHelpers, transformOptions.filename));
   }
 
   if (check.includes('babelHelpers.inherits')) {
     if (babelHelpers === EXTERNAL) {
       return;
     }
-    ctx.error(mismatchError(EXTERNAL, babelHelpers, transformOptions.filename));
+    error(mismatchError(EXTERNAL, babelHelpers, transformOptions.filename));
   }
 
   // test unminifiable string content
@@ -66,12 +66,12 @@ export default async function preflightCheck(ctx, babelHelpers, transformOptions
       return;
     }
     if (babelHelpers === RUNTIME && !transformOptions.plugins.length) {
-      ctx.error(
+      error(
         `You must use the \`@babel/plugin-transform-runtime\` plugin when \`babelHelpers\` is "${RUNTIME}".\n`
       );
     }
-    ctx.error(mismatchError(INLINE, babelHelpers, transformOptions.filename));
+    error(mismatchError(INLINE, babelHelpers, transformOptions.filename));
   }
 
-  ctx.error(UNEXPECTED_ERROR);
+  error(UNEXPECTED_ERROR);
 }

packages/babel/src/transformCode.js

@@ -1,13 +1,19 @@
 import * as babel from '@babel/core';
 
-export default async function transformCode(
+import bundledHelpersPlugin from './bundledHelpersPlugin.js';
+import preflightCheck from './preflightCheck.js';
+import { BUNDLED } from './constants.js';
+import { addBabelPlugin } from './utils.js';
+
+export default async function transformCode({
   inputCode,
   babelOptions,
   overrides,
   customOptions,
-  ctx,
-  finalizeOptions
-) {
+  error,
+  skipPreflightCheck,
+  babelHelpers
+}) {
   // loadPartialConfigAsync has become available in @babel/core@7.8.0
   const config = await (babel.loadPartialConfigAsync || babel.loadPartialConfig)(babelOptions);
 
@@ -16,18 +22,23 @@ export default async function transformCode(
     return null;
   }
 
-  let transformOptions = !overrides.config
+  let transformOptions = !overrides?.config
     ? config.options
-    : await overrides.config.call(ctx, config, {
+    : await overrides.config(config, {
         code: inputCode,
         customOptions
       });
 
-  if (finalizeOptions) {
-    transformOptions = await finalizeOptions(transformOptions);
+  if (!skipPreflightCheck) {
+    await preflightCheck(error, babelHelpers, transformOptions);
   }
 
-  if (!overrides.result) {
+  transformOptions =
+    babelHelpers === BUNDLED
+      ? addBabelPlugin(transformOptions, bundledHelpersPlugin)
+      : transformOptions;
+
+  if (!overrides?.result) {
     const { code, map } = await babel.transformAsync(inputCode, transformOptions);
     return {
       code,
@@ -36,7 +47,7 @@ export default async function transformCode(
   }
 
   const result = await babel.transformAsync(inputCode, transformOptions);
-  const { code, map } = await overrides.result.call(ctx, result, {
+  const { code, map } = await overrides.result(result, {
     code: inputCode,
     customOptions,
     config,