feat: chunked processing (streaming)

bin/cli.mjs

@@ -5,13 +5,27 @@ import process from 'node:process';
 import { Command, Option } from 'commander';
 
 import commands from './commands/index.mjs';
-import interactive from './commands/interactive.mjs';
 import { errorWrap } from './utils.mjs';
+import logger from '../src/logger/index.mjs';
 
 const program = new Command()
   .name('@nodejs/doc-kit')
   .description('CLI tool to generate the Node.js API documentation');
 
+// Add global log level option
+program.addOption(
+  new Option('--log-level <level>', 'Log level')
+    .choices(['debug', 'info', 'warn', 'error', 'fatal'])
+    .default('info')
+);
+
+// Set log level before any command runs
+program.hook('preAction', thisCommand => {
+  const { logLevel } = thisCommand.opts();
+
+  logger.setLogLevel(logLevel);
+});
+
 // Registering commands
 commands.forEach(({ name, description, options, action }) => {
   const cmd = program.command(name).description(description);
@@ -37,11 +51,5 @@ commands.forEach(({ name, description, options, action }) => {
   cmd.action(errorWrap(action));
 });
 
-// Register the interactive command
-program
-  .command('interactive')
-  .description('Launch guided CLI wizard')
-  .action(errorWrap(interactive));
-
 // Parse and execute command-line arguments
 program.parse(process.argv);

bin/commands/generate.mjs

@@ -3,34 +3,28 @@ import { resolve } from 'node:path';
 
 import { coerce } from 'semver';
 
-import {
-  DOC_NODE_CHANGELOG_URL,
-  DOC_NODE_VERSION,
-} from '../../src/constants.mjs';
+import { NODE_CHANGELOG_URL, NODE_VERSION } from '../../src/constants.mjs';
 import { publicGenerators } from '../../src/generators/index.mjs';
 import createGenerator from '../../src/generators.mjs';
+import logger from '../../src/logger/index.mjs';
 import { parseChangelog, parseIndex } from '../../src/parsers/markdown.mjs';
 import { DEFAULT_TYPE_MAP } from '../../src/utils/parser/constants.mjs';
 import { loadFromURL } from '../../src/utils/parser.mjs';
-import { loadAndParse } from '../utils.mjs';
 
 const availableGenerators = Object.keys(publicGenerators);
 
-// Half of available logical CPUs guarantees in general all physical CPUs are being used
-// which in most scenarios is the best way to maximize performance
-const optimalThreads = Math.floor(cpus().length / 2) + 1;
-
 /**
- * @typedef {Object} Options
- * @property {Array<string>|string} input - Specifies the glob/path for input files.
- * @property {Array<string>|string} [ignore] - Specifies the glob/path for ignoring files.
- * @property {Array<keyof publicGenerators>} target - Specifies the generator target mode.
- * @property {string} version - Specifies the target Node.js version.
- * @property {string} changelog - Specifies the path to the Node.js CHANGELOG.md file.
- * @property {string} typeMap - Specifies the path to the Node.js Type Map.
- * @property {string} [gitRef] - Git ref/commit URL.
- * @property {number} [threads] - Number of threads to allow.
- * @property {number} [chunkSize] - Number of items to process per worker thread.
+ * @typedef {{
+ * input: Array<string> | string;
+ * ignore?: Array<string> | string;
+ * target: Array<keyof publicGenerators>;
+ * version: string;
+ * changelog: string;
+ * typeMap: string;
+ * gitRef?: string;
+ * threads?: number;
+ * chunkSize?: number;
+ * }} Options
  */
 
 /**
@@ -66,11 +60,11 @@ export default {
     },
     threads: {
       flags: ['-p', '--threads <number>'],
-      desc: 'Number of worker threads to use',
+      desc: 'Number of worker threads to use (minimum: 2)',
       prompt: {
         type: 'text',
         message: 'How many threads to allow',
-        initialValue: String(Math.max(optimalThreads, 1)),
+        initialValue: String(cpus().length),
       },
     },
     chunkSize: {
@@ -88,7 +82,7 @@ export default {
       prompt: {
         type: 'text',
         message: 'Enter Node.js version',
-        initialValue: DOC_NODE_VERSION,
+        initialValue: NODE_VERSION,
       },
     },
     changelog: {
@@ -97,7 +91,7 @@ export default {
       prompt: {
         type: 'text',
         message: 'Enter changelog URL',
-        initialValue: DOC_NODE_CHANGELOG_URL,
+        initialValue: NODE_CHANGELOG_URL,
       },
     },
     gitRef: {
@@ -146,15 +140,17 @@ export default {
    * @returns {Promise<void>}
    */
   async action(opts) {
-    const docs = await loadAndParse(opts.input, opts.ignore);
+    logger.debug('Starting doc-kit', opts);
+
+    const { runGenerators } = createGenerator();
+
     const releases = await parseChangelog(opts.changelog);
 
-    const rawTypeMap = await loadFromURL(opts.typeMap);
-    const typeMap = JSON.parse(rawTypeMap);
+    const typeMap = JSON.parse(await loadFromURL(opts.typeMap));
 
     const index = opts.index && (await parseIndex(opts.index));
 
-    const { runGenerators } = createGenerator(docs);
+    logger.debug('Starting generation', { targets: opts.target });
 
     await runGenerators({
       generators: opts.target,
@@ -163,7 +159,7 @@ export default {
       version: coerce(opts.version),
       releases,
       gitRef: opts.gitRef,
-      threads: parseInt(opts.threads, 10),
+      threads: Math.max(parseInt(opts.threads, 10), 2),
       chunkSize: parseInt(opts.chunkSize, 10),
       index,
       typeMap,

bin/commands/index.mjs

@@ -1,3 +1,4 @@
 import generate from './generate.mjs';
+import interactive from './interactive.mjs';
 
-export default [generate];
+export default [generate, interactive];