module: synchronously load most ES modules

Author: GeoffreyBooth

benchmark/esm/startup-esm-graph.js

@@ -0,0 +1,59 @@
+'use strict';
+
+const common = require('../common');
+const fs = require('fs');
+const path = require('path');
+const { spawnSync } = require('child_process');
+const tmpdir = require('../../test/common/tmpdir');
+
+const bench = common.createBenchmark(main, {
+  modules: [250, 500, 1000, 2000],
+  n: [30],
+});
+
+function prepare(count) {
+  tmpdir.refresh();
+  const dir = tmpdir.resolve('esm-graph');
+  fs.mkdirSync(dir, { recursive: true });
+
+  // Create a flat ESM graph: entry imports all modules directly.
+  // Each module is independent, maximizing the number of resolve/load/link
+  // operations in the loader pipeline.
+  const imports = [];
+  for (let i = 0; i < count; i++) {
+    fs.writeFileSync(
+      path.join(dir, `mod${i}.mjs`),
+      `export const value${i} = ${i};\n`,
+    );
+    imports.push(`import './mod${i}.mjs';`);
+  }
+
+  const entry = path.join(dir, 'entry.mjs');
+  fs.writeFileSync(entry, imports.join('\n') + '\n');
+  return entry;
+}
+
+function main({ n, modules }) {
+  const entry = prepare(modules);
+  const cmd = process.execPath || process.argv[0];
+  const warmup = 3;
+  const state = { finished: -warmup };
+
+  while (state.finished < n) {
+    const child = spawnSync(cmd, [entry]);
+    if (child.status !== 0) {
+      console.log('---- STDOUT ----');
+      console.log(child.stdout.toString());
+      console.log('---- STDERR ----');
+      console.log(child.stderr.toString());
+      throw new Error(`Child process stopped with exit code ${child.status}`);
+    }
+    state.finished++;
+    if (state.finished === 0) {
+      bench.start();
+    }
+    if (state.finished === n) {
+      bench.end(n);
+    }
+  }
+}

lib/internal/modules/esm/loader.js

@@ -366,6 +366,32 @@ class ModuleLoader {
     return { wrap: job.module, namespace: job.runSync(parent).namespace };
   }
 
+  /**
+   * Synchronously load and evaluate the entry point module.
+   * This avoids creating any promises when no TLA is present
+   * and no async customization hooks are registered.
+   * @param {string} url The URL of the entry point module.
+   * @returns {{ module: ModuleWrap, completed: boolean }} The entry module and whether
+   *   evaluation completed synchronously. When false, the caller should fall back to
+   *   async evaluation (TLA detected).
+   */
+  importSyncForEntryPoint(url) {
+    return onImport.traceSync(() => {
+      const request = { specifier: url, phase: kEvaluationPhase, attributes: kEmptyObject, __proto__: null };
+      const job = this.getOrCreateModuleJob(undefined, request, kImportInImportedESM);
+      job.module.instantiate();
+      if (job.module.hasAsyncGraph) {
+        return { __proto__: null, module: job.module, completed: false };
+      }
+      job.runSync();
+      return { __proto__: null, module: job.module, completed: true };
+    }, {
+      __proto__: null,
+      parentURL: undefined,
+      url,
+    });
+  }
+
   /**
    * Check invariants on a cached module job when require()'d from ESM.
    * @param {string} specifier The first parameter of require().
@@ -561,10 +587,15 @@ class ModuleLoader {
     }
 
     const { ModuleJob, ModuleJobSync } = require('internal/modules/esm/module_job');
-    // TODO(joyeecheung): use ModuleJobSync for kRequireInImportedCJS too.
-    const ModuleJobCtor = (requestType === kImportInRequiredESM ? ModuleJobSync : ModuleJob);
     const isMain = (parentURL === undefined);
     const inspectBrk = (isMain && getOptionValue('--inspect-brk'));
+    // Use ModuleJobSync whenever we're on the main thread (not the async loader hook worker),
+    // except for kRequireInImportedCJS (TODO: consolidate that case too) and --inspect-brk
+    // (which needs the async ModuleJob to pause on the first line).
+    // TODO(joyeecheung): use ModuleJobSync for kRequireInImportedCJS too.
+    const ModuleJobCtor = (!this.isForAsyncLoaderHookWorker && !inspectBrk &&
+                           requestType !== kRequireInImportedCJS) ?
+      ModuleJobSync : ModuleJob;
     job = new ModuleJobCtor(
       this,
       url,
@@ -591,8 +622,9 @@ class ModuleLoader {
    */
   getOrCreateModuleJob(parentURL, request, requestType) {
     let maybePromise;
-    if (requestType === kRequireInImportedCJS || requestType === kImportInRequiredESM) {
-      // In these two cases, resolution must be synchronous.
+    if (!this.isForAsyncLoaderHookWorker) {
+      // On the main thread, always resolve synchronously;
+      // `resolveSync` coordinates with the async loader hook worker if needed.
       maybePromise = this.resolveSync(parentURL, request);
       assert(!isPromise(maybePromise));
     } else {

lib/internal/modules/esm/module_job.js

@@ -46,7 +46,7 @@ const {
   getSourceMapsSupport,
 } = require('internal/source_map/source_map_cache');
 const assert = require('internal/assert');
-const resolvedPromise = PromiseResolve();
+let resolvedPromise;
 const {
   setHasStartedUserESMExecution,
   urlToFilename,
@@ -374,7 +374,7 @@ class ModuleJob extends ModuleJobBase {
     for (const dependencyJob of jobsInGraph) {
       // Calling `this.module.instantiate()` instantiates not only the
       // ModuleWrap in this module, but all modules in the graph.
-      dependencyJob.instantiated = resolvedPromise;
+      dependencyJob.instantiated = resolvedPromise ??= PromiseResolve();
     }
   }
 
@@ -445,12 +445,14 @@ class ModuleJob extends ModuleJobBase {
 
 /**
  * This is a fully synchronous job and does not spawn additional threads in any way.
- * All the steps are ensured to be synchronous and it throws on instantiating
- * an asynchronous graph. It also disallows CJS <-> ESM cycles.
+ * Loading and linking are always synchronous. Evaluation via runSync() throws on an
+ * asynchronous graph; evaluation via run() falls back to async for top-level await.
+ * It also disallows CJS <-> ESM cycles.
  *
- * This is used for ES modules loaded via require(esm). Modules loaded by require() in
- * imported CJS are handled by ModuleJob with the isForRequireInImportedCJS set to true instead.
- * The two currently have different caching behaviors.
+ * Used for all ES module imports on the main thread, regardless of how the import was
+ * triggered (entry point, import(), require(esm), --import, etc.).
+ * Modules loaded by require() in imported CJS are handled by ModuleJob with the
+ * isForRequireInImportedCJS set to true instead. The two currently have different caching behaviors.
  * TODO(joyeecheung): consolidate this with the isForRequireInImportedCJS variant of ModuleJob.
  */
 class ModuleJobSync extends ModuleJobBase {
@@ -491,32 +493,49 @@ class ModuleJobSync extends ModuleJobBase {
     return PromiseResolve(this.module);
   }
 
-  async run() {
+  async run(isEntryPoint = false) {
     assert(this.phase === kEvaluationPhase);
-    // This path is hit by a require'd module that is imported again.
     const status = this.module.getStatus();
     debug('ModuleJobSync.run()', status, this.module);
     // If the module was previously required and errored, reject from import() again.
     if (status === kErrored) {
       throw this.module.getError();
-    } else if (status > kInstantiated) {
+    }
+    if (status > kInstantiated) {
+      // Already evaluated (e.g. previously require()'d and now import()'d again).
       if (this.evaluationPromise) {
         await this.evaluationPromise;
       }
       return { __proto__: null, module: this.module };
-    } else if (status === kInstantiated) {
-      // The evaluation may have been canceled because instantiate() detected TLA first.
-      // But when it is imported again, it's fine to re-evaluate it asynchronously.
+    }
+    if (status < kInstantiated) {
+      // Fresh module: instantiate it now (links were already resolved synchronously in constructor)
+      this.module.instantiate();
+    }
+    // `status === kInstantiated`: either just instantiated above, or previously instantiated
+    // but evaluation was deferred (e.g. TLA detected by a prior `runSync()` call)
+    if (isEntryPoint) {
+      globalThis[entry_point_module_private_symbol] = this.module;
+    }
+    setHasStartedUserESMExecution();
+    if (this.module.hasAsyncGraph) {
+      // Has top-level `await`: fall back to async evaluation
       const timeout = -1;
       const breakOnSigint = false;
       this.evaluationPromise = this.module.evaluate(timeout, breakOnSigint);
       await this.evaluationPromise;
       this.evaluationPromise = undefined;
       return { __proto__: null, module: this.module };
     }
-
-    assert.fail('Unexpected status of a module that is imported again after being required. ' +
-                `Status = ${status}`);
+    // No top-level `await`: evaluate synchronously
+    const filename = urlToFilename(this.url);
+    try {
+      this.module.evaluateSync(filename, undefined);
+    } catch (evaluateError) {
+      explainCommonJSGlobalLikeNotDefinedError(evaluateError, this.module.url, this.module.hasTopLevelAwait);
+      throw evaluateError;
+    }
+    return { __proto__: null, module: this.module };
   }
 
   runSync(parent) {

lib/internal/modules/run_main.js

@@ -19,6 +19,7 @@ const {
 const {
   privateSymbols: {
     entry_point_promise_private_symbol,
+    entry_point_module_private_symbol,
   },
 } = internalBinding('util');
 /**
@@ -156,6 +157,20 @@ function executeUserEntryPoint(main = process.argv[1]) {
     const mainPath = resolvedMain || main;
     const mainURL = getOptionValue('--entry-url') ? new URL(mainPath, getCWDURL()) : pathToFileURL(mainPath);
 
+    // When no async hooks or --inspect-brk are needed, try the fully synchronous path first.
+    // This avoids creating any promises during startup.
+    if (!getOptionValue('--inspect-brk') &&
+        getOptionValue('--experimental-loader').length === 0 &&
+        getOptionValue('--import').length === 0) {
+      const cascadedLoader = require('internal/modules/esm/loader').getOrInitializeCascadedLoader();
+      const { module: entryModule, completed } = cascadedLoader.importSyncForEntryPoint(mainURL.href);
+      globalThis[entry_point_module_private_symbol] = entryModule;
+      if (completed) {
+        return;
+      }
+      // TLA detected — fall through to async path.
+    }
+
     runEntryPointWithESMLoader((cascadedLoader) => {
       // Note that if the graph contains unsettled TLA, this may never resolve
       // even after the event loop stops running.

test/es-module/test-esm-sync-import.mjs

@@ -0,0 +1,54 @@
+// Flags: --no-warnings
+import { spawnPromisified } from '../common/index.mjs';
+import * as fixtures from '../common/fixtures.mjs';
+import { describe, it } from 'node:test';
+import assert from 'node:assert';
+
+
+describe('synchronous ESM loading', () => {
+  it('should create minimal promises for ESM importing ESM', async () => {
+    // import-esm.mjs imports imported-esm.mjs — a pure ESM graph.
+    const count = await getPromiseCount(fixtures.path('es-modules', 'import-esm.mjs'));
+    // V8's Module::Evaluate returns one promise for the entire graph.
+    assert.strictEqual(count, 1);
+  });
+
+  it('should create minimal promises for ESM importing CJS', async () => {
+    // builtin-imports-case.mjs imports node:assert (builtin) + dep1.js and dep2.js (CJS).
+    const count = await getPromiseCount(fixtures.path('es-modules', 'builtin-imports-case.mjs'));
+    // V8 creates one promise for the ESM entry evaluation, plus one per CJS module
+    // in the graph (each CJS namespace is wrapped in a promise).
+    // entry (ESM, 1) + node:assert (CJS, 1) + dep1.js (CJS, 1) + dep2.js (CJS, 1) = 4.
+    assert.strictEqual(count, 4);
+  });
+
+  it('should fall back to async evaluation for top-level await', async () => {
+    // tla/resolved.mjs uses top-level await, so the sync path detects TLA
+    // and falls back to async evaluation.
+    const count = await getPromiseCount(fixtures.path('es-modules', 'tla', 'resolved.mjs'));
+    // The async fallback creates more promises — just verify the module
+    // still runs successfully. The promise count will be higher than the
+    // sync path but should remain bounded.
+    assert(count > 1, `Expected TLA fallback to create multiple promises, got ${count}`);
+  });
+
+  it('should create minimal promises when entry point is CJS importing ESM', async () => {
+    // When a CJS entry point uses require(esm), the ESM module is loaded via
+    // ModuleJobSync, so the same promise minimization applies.
+    const count = await getPromiseCount(fixtures.path('es-modules', 'require-esm-entry.cjs'));
+    // V8's Module::Evaluate returns one promise for the ESM module.
+    assert.strictEqual(count, 1);
+  });
+});
+
+
+async function getPromiseCount(entry) {
+  const { stdout, stderr, code } = await spawnPromisified(process.execPath, [
+    '--require', fixtures.path('es-modules', 'promise-counter.cjs'),
+    entry,
+  ]);
+  assert.strictEqual(code, 0, `child failed:\nstdout: ${stdout}\nstderr: ${stderr}`);
+  const match = stdout.match(/PROMISE_COUNT=(\d+)/);
+  assert(match, `Expected PROMISE_COUNT in output, got: ${stdout}`);
+  return Number(match[1]);
+}

test/fixtures/es-modules/promise-counter.cjs

@@ -0,0 +1,11 @@
+// Counts PROMISE async resources created during the process lifetime.
+// Used by test-esm-sync-entry-point.mjs to verify the sync ESM loader
+// path does not create unnecessary promises.
+'use strict';
+let count = 0;
+require('async_hooks').createHook({
+  init(id, type) { if (type === 'PROMISE') count++; },
+}).enable();
+process.on('exit', () => {
+  process.stdout.write(`PROMISE_COUNT=${count}\n`);
+});

test/fixtures/es-modules/require-esm-entry.cjs

@@ -0,0 +1 @@
+require('./imported-esm.mjs');