feat: introduce lazy generators (#609)

* feat: introduce lazy generators

* chore: code review

Co-authored-by: Aviv Keller <me@aviv.sh>

* fix: tests

* chore: use lazy

---------

Co-authored-by: Aviv Keller <me@aviv.sh>

Author: ovflowd

package.json

@@ -1,5 +1,6 @@
 {
   "name": "@nodejs/doc-kit",
+  "type": "module",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/nodejs/api-docs-tooling.git"

src/generators.mjs

@@ -48,16 +48,17 @@ const createGenerator = () => {
    * @param {string} generatorName - Generator to schedule
    * @param {import('./utils/configuration/types').Configuration} configuration - Runtime options
    */
-  const scheduleGenerator = (generatorName, configuration) => {
+  const scheduleGenerator = async (generatorName, configuration) => {
     if (generatorName in cachedGenerators) {
       return;
     }
 
-    const { dependsOn, generate, processChunk } = allGenerators[generatorName];
+    const { dependsOn, generate, processChunk } =
+      await allGenerators[generatorName]();
 
     // Schedule dependency first
     if (dependsOn && !(dependsOn in cachedGenerators)) {
-      scheduleGenerator(dependsOn, configuration);
+      await scheduleGenerator(dependsOn, configuration);
     }
 
     generatorsLogger.debug(`Scheduling "${generatorName}"`, {
@@ -74,9 +75,9 @@ const createGenerator = () => {
       // Create parallel worker for streaming generators
       const worker = processChunk
         ? createParallelWorker(generatorName, pool, configuration)
-        : null;
+        : Promise.resolve(null);
 
-      const result = await generate(dependencyInput, worker);
+      const result = await generate(dependencyInput, await worker);
 
       // For streaming generators, "Completed" is logged when collection finishes
       // (in streamingCache.getOrCollect), not here when the generator returns
@@ -107,7 +108,7 @@ const createGenerator = () => {
 
     // Schedule all generators
     for (const name of generators) {
-      scheduleGenerator(name, configuration);
+      await scheduleGenerator(name, configuration);
     }
 
     // Start all collections in parallel (don't await sequentially)

src/generators/__tests__/index.test.mjs

@@ -6,11 +6,17 @@ import semver from 'semver';
 import { allGenerators } from '../index.mjs';
 
 const validDependencies = Object.keys(allGenerators);
-const generatorEntries = Object.entries(allGenerators);
+
+const allGeneratorsReaolved = await Promise.all(
+  Object.entries(allGenerators).map(async ([key, loader]) => [
+    key,
+    await loader(),
+  ])
+);
 
 describe('All Generators', () => {
-  it('should have keys matching their name property', () => {
-    generatorEntries.forEach(([key, generator]) => {
+  it('should have keys matching their name property', async () => {
+    allGeneratorsReaolved.forEach(([key, generator]) => {
       assert.equal(
         key,
         generator.name,
@@ -19,8 +25,8 @@ describe('All Generators', () => {
     });
   });
 
-  it('should have valid semver versions', () => {
-    generatorEntries.forEach(([key, generator]) => {
+  it('should have valid semver versions', async () => {
+    allGeneratorsReaolved.forEach(([key, generator]) => {
       const isValid = semver.valid(generator.version);
       assert.ok(
         isValid,
@@ -29,8 +35,8 @@ describe('All Generators', () => {
     });
   });
 
-  it('should have valid dependsOn references', () => {
-    generatorEntries.forEach(([key, generator]) => {
+  it('should have valid dependsOn references', async () => {
+    allGeneratorsReaolved.forEach(([key, generator]) => {
       if (generator.dependsOn) {
         assert.ok(
           validDependencies.includes(generator.dependsOn),
@@ -40,10 +46,11 @@ describe('All Generators', () => {
     });
   });
 
-  it('should have ast generator as a top-level generator with no dependencies', () => {
-    assert.ok(allGenerators.ast, 'ast generator should exist');
+  it('should have ast generator as a top-level generator with no dependencies', async () => {
+    const ast = await allGenerators.ast();
+    assert.ok(ast, 'ast generator should exist');
     assert.equal(
-      allGenerators.ast.dependsOn,
+      ast.dependsOn,
       undefined,
       'ast generator should have no dependencies'
     );

src/generators/api-links/__tests__/fixtures.test.mjs

@@ -35,7 +35,7 @@ describe('api links', () => {
           join(relativePath, 'fixtures', sourceFile).replaceAll(sep, '/'),
         ];
 
-        const worker = createParallelWorker('ast-js', pool, config);
+        const worker = await createParallelWorker('ast-js', pool, config);
 
         // Collect results from the async generator
         const astJsResults = [];

src/generators/index.mjs

@@ -1,44 +1,38 @@
 'use strict';
 
-import addonVerify from './addon-verify/index.mjs';
-import apiLinks from './api-links/index.mjs';
-import ast from './ast/index.mjs';
-import astJs from './ast-js/index.mjs';
-import jsonSimple from './json-simple/index.mjs';
-import jsxAst from './jsx-ast/index.mjs';
-import legacyHtml from './legacy-html/index.mjs';
-import legacyHtmlAll from './legacy-html-all/index.mjs';
-import legacyJson from './legacy-json/index.mjs';
-import legacyJsonAll from './legacy-json-all/index.mjs';
-import llmsTxt from './llms-txt/index.mjs';
-import manPage from './man-page/index.mjs';
-import metadata from './metadata/index.mjs';
-import oramaDb from './orama-db/index.mjs';
-import sitemap from './sitemap/index.mjs';
-import web from './web/index.mjs';
+import { lazy } from '../utils/misc.mjs';
+
+/**
+ * Wraps a dynamic import into a lazy loader that resolves to the default export.
+ *
+ * @template T
+ * @param {() => Promise<{default: T}>} loader
+ * @returns {() => Promise<T>}
+ */
+const lazyDefault = loader => lazy(() => loader().then(m => m.default));
 
 export const publicGenerators = {
-  'json-simple': jsonSimple,
-  'legacy-html': legacyHtml,
-  'legacy-html-all': legacyHtmlAll,
-  'man-page': manPage,
-  'legacy-json': legacyJson,
-  'legacy-json-all': legacyJsonAll,
-  'addon-verify': addonVerify,
-  'api-links': apiLinks,
-  'orama-db': oramaDb,
-  'llms-txt': llmsTxt,
-  sitemap,
-  web,
+  'json-simple': lazyDefault(() => import('./json-simple/index.mjs')),
+  'legacy-html': lazyDefault(() => import('./legacy-html/index.mjs')),
+  'legacy-html-all': lazyDefault(() => import('./legacy-html-all/index.mjs')),
+  'man-page': lazyDefault(() => import('./man-page/index.mjs')),
+  'legacy-json': lazyDefault(() => import('./legacy-json/index.mjs')),
+  'legacy-json-all': lazyDefault(() => import('./legacy-json-all/index.mjs')),
+  'addon-verify': lazyDefault(() => import('./addon-verify/index.mjs')),
+  'api-links': lazyDefault(() => import('./api-links/index.mjs')),
+  'orama-db': lazyDefault(() => import('./orama-db/index.mjs')),
+  'llms-txt': lazyDefault(() => import('./llms-txt/index.mjs')),
+  sitemap: lazyDefault(() => import('./sitemap/index.mjs')),
+  web: lazyDefault(() => import('./web/index.mjs')),
 };
 
 // These ones are special since they don't produce standard output,
 // and hence, we don't expose them to the CLI.
 const internalGenerators = {
-  ast,
-  metadata,
-  'jsx-ast': jsxAst,
-  'ast-js': astJs,
+  ast: lazyDefault(() => import('./ast/index.mjs')),
+  metadata: lazyDefault(() => import('./metadata/index.mjs')),
+  'jsx-ast': lazyDefault(() => import('./jsx-ast/index.mjs')),
+  'ast-js': lazyDefault(() => import('./ast-js/index.mjs')),
 };
 
 export const allGenerators = {

src/generators/jsx-ast/utils/__tests__/buildBarProps.test.mjs

@@ -18,7 +18,7 @@ mock.module('../../../../utils/generators.mjs', {
       { version: '19.0.0', isLts: false, isCurrent: true },
     ],
     leftHandAssign: Object.assign,
-    getVersionFromSemVer: version => version.split('.')[0],
+    getVersionFromSemVer: version => `${version.major}.x`,
     getVersionURL: (version, api) => `/api/${version}/${api}`,
   },
 });
@@ -94,7 +94,7 @@ describe('buildMetaBarProps', () => {
     const result = buildMetaBarProps(head, entries);
 
     assert.equal(result.addedIn, 'v1.0.0');
-    assert.equal(result.readingTime, '1 min read');
+    assert.equal(result.readingTime, '5 min read');
     assert.deepEqual(result.viewAs, [
       ['JSON', 'fs.json'],
       ['MD', 'fs.md'],
@@ -134,15 +134,15 @@ describe('formatVersionOptions', () => {
     assert.deepStrictEqual(result, [
       {
         label: 'v16.x (LTS)',
-        value: 'https://nodejs.org/docs/latest-v16.x/api/http.html',
+        value: '/api/16.x/http',
       },
       {
         label: 'v17.x (Current)',
-        value: 'https://nodejs.org/docs/latest-v17.x/api/http.html',
+        value: '/api/17.x/http',
       },
       {
         label: 'v18.x',
-        value: 'https://nodejs.org/docs/latest-v18.x/api/http.html',
+        value: '/api/18.x/http',
       },
     ]);
   });

src/generators/types.d.ts

@@ -1,12 +1,23 @@
 import type { publicGenerators, allGenerators } from './index.mjs';
 
 declare global {
+  /**
+   * A lazy generator loader that returns a promise resolving to the generator metadata.
+   */
+  export type LazyGenerator<T = GeneratorMetadata<any, any, any>> =
+    () => Promise<T>;
+
   // Public generators exposed to the CLI
   export type AvailableGenerators = typeof publicGenerators;
 
   // All generators including internal ones (metadata, jsx-ast, ast-js)
   export type AllGenerators = typeof allGenerators;
 
+  // The resolved type of a loaded generator
+  export type ResolvedGenerator<K extends keyof AllGenerators> = Awaited<
+    ReturnType<AllGenerators[K]>
+  >;
+
   /**
    * ParallelWorker interface for distributing work across Node.js worker threads.
    * Streams results as chunks complete, enabling pipeline parallelism.

src/threading/__tests__/parallel.test.mjs

@@ -41,7 +41,7 @@ async function collectChunks(generator) {
 describe('createParallelWorker', () => {
   it('should create a ParallelWorker with stream method', async () => {
     const pool = createWorkerPool(2);
-    const worker = createParallelWorker('metadata', pool, { threads: 2 });
+    const worker = await createParallelWorker('metadata', pool, { threads: 2 });
 
     ok(worker);
     strictEqual(typeof worker.stream, 'function');
@@ -51,7 +51,7 @@ describe('createParallelWorker', () => {
 
   it('should handle empty items array', async () => {
     const pool = createWorkerPool(2);
-    const worker = createParallelWorker('ast-js', pool, {
+    const worker = await createParallelWorker('ast-js', pool, {
       threads: 2,
       chunkSize: 10,
     });
@@ -65,7 +65,7 @@ describe('createParallelWorker', () => {
 
   it('should distribute items to multiple worker threads', async () => {
     const pool = createWorkerPool(4);
-    const worker = createParallelWorker('metadata', pool, {
+    const worker = await createParallelWorker('metadata', pool, {
       threads: 4,
       chunkSize: 1,
     });
@@ -104,7 +104,7 @@ describe('createParallelWorker', () => {
 
   it('should yield results as chunks complete', async () => {
     const pool = createWorkerPool(2);
-    const worker = createParallelWorker('metadata', pool, {
+    const worker = await createParallelWorker('metadata', pool, {
       threads: 2,
       chunkSize: 1,
     });
@@ -131,7 +131,7 @@ describe('createParallelWorker', () => {
 
   it('should work with single thread and items', async () => {
     const pool = createWorkerPool(2);
-    const worker = createParallelWorker('metadata', pool, {
+    const worker = await createParallelWorker('metadata', pool, {
       threads: 2,
       chunkSize: 5,
     });
@@ -155,7 +155,7 @@ describe('createParallelWorker', () => {
 
   it('should use sliceInput for metadata generator', async () => {
     const pool = createWorkerPool(2);
-    const worker = createParallelWorker('metadata', pool, {
+    const worker = await createParallelWorker('metadata', pool, {
       threads: 2,
       chunkSize: 1,
     });

src/threading/chunk-worker.mjs

@@ -17,7 +17,7 @@ export default async ({
 }) => {
   await setConfig(configuration);
 
-  const generator = allGenerators[generatorName];
+  const generator = await allGenerators[generatorName]();
 
   return generator.processChunk(input, itemIndices, extra);
 };

src/threading/parallel.mjs

@@ -63,14 +63,14 @@ const createTask = (
  * @param {import('../utils/configuration/types').Configuration} configuration - Generator options
  * @returns {ParallelWorker}
  */
-export default function createParallelWorker(
+export default async function createParallelWorker(
   generatorName,
   pool,
   configuration
 ) {
   const { threads, chunkSize } = configuration;
 
-  const generator = allGenerators[generatorName];
+  const generator = await allGenerators[generatorName]();
 
   return {
     /**