chore(memory): improve (#842)

Author: avivkeller

src/generators/jsx-ast/generate.mjs

@@ -1,3 +1,5 @@
+import { jsx, toJs } from 'estree-util-to-js';
+
 import buildContent from './utils/buildContent.mjs';
 import { getSortedHeadNodes } from './utils/getSortedHeadNodes.mjs';
 import { buildNotFoundPage } from './utils/synthetic/404.mjs';
@@ -7,30 +9,32 @@ import getConfig from '../../utils/configuration/index.mjs';
 import { groupNodesByModule } from '../../utils/generators.mjs';
 
 /**
- * Builds JSX content for all configured synthetic pages.
+ * Builds the `{ head, entries }` page descriptors for all configured synthetic
+ * pages. The descriptors are cheap to build; the expensive `buildContent` step
+ * runs later in a worker (via `processChunk`), so the very large synthetic
+ * `all` page is never built on the main thread.
  *
  * @param {Array<import('../metadata/types').MetadataEntry>} input
  */
-const buildSyntheticEntries = async input => {
+const buildSyntheticDescriptors = input => {
   const config = getConfig('jsx-ast');
 
-  const descriptors = [
+  return [
     config.generateAllPage && buildAllPage(input),
     config.generateIndexPage && buildIndexPage(input),
     config.generateNotFoundPage && buildNotFoundPage(),
   ].filter(Boolean);
-
-  return Promise.all(
-    descriptors.map(({ head, entries }) => buildContent(entries, head))
-  );
 };
 
 /**
  * Process a chunk of items in a worker thread.
- * Transforms metadata entries into JSX AST nodes.
  *
- * Each item is a SlicedModuleInput containing the head node
- * and all entries for that module - no need to recompute grouping.
+ * Each item is a `{ head, entries }` descriptor (one module, or a synthetic
+ * page). The JSX AST is built AND serialized to a code string here, inside the
+ * worker, so the heavy AST — most notably the giant `all` page, which
+ * concatenates every module — is dropped in the worker and never crosses back
+ * to or accumulates on the main thread. Only the much smaller code string and
+ * the page metadata are returned.
  *
  * @type {import('./types').Generator['processChunk']}
  */
@@ -42,14 +46,16 @@ export async function processChunk(slicedInput, itemIndices) {
 
     const content = await buildContent(entries, head);
 
-    results.push(content);
+    const { value: code } = toJs(content, { handlers: jsx });
+
+    results.push({ data: content.data, code });
   }
 
   return results;
 }
 
 /**
- * Generates a JSX AST
+ * Generates per-page JSX code from API metadata.
  *
  * @type {import('./types').Generator['generate']}
  */
@@ -60,18 +66,16 @@ export async function* generate(input, worker) {
   // Create sliced input: each item contains head + its module's entries
   // This avoids sending all 4700+ entries to every worker
   const groupedModules = groupNodesByModule(input);
-  const entries = getSortedHeadNodes(input).map(head => ({
+  const descriptors = getSortedHeadNodes(input).map(head => ({
     head,
     entries: groupedModules.get(head.api),
   }));
 
-  for await (const chunkResult of worker.stream(entries)) {
-    yield chunkResult;
-  }
-
-  const syntheticEntries = await buildSyntheticEntries(moduleInput);
+  // Process the synthetic pages through the worker pool as well, so their
+  // (potentially enormous) content is built and converted off the main thread.
+  descriptors.push(...buildSyntheticDescriptors(moduleInput));
 
-  if (syntheticEntries.length > 0) {
-    yield syntheticEntries;
+  for await (const chunkResult of worker.stream(descriptors)) {
+    yield chunkResult;
   }
 }

src/generators/web/__tests__/generate.test.mjs

@@ -1,11 +1,22 @@
 import assert from 'node:assert/strict';
 import { describe, it } from 'node:test';
 
+import { jsx, toJs } from 'estree-util-to-js';
+
 import { setConfig } from '../../../utils/configuration/index.mjs';
 import buildContent from '../../jsx-ast/utils/buildContent.mjs';
 import { buildNotFoundPage } from '../../jsx-ast/utils/synthetic/404.mjs';
 import { generate } from '../generate.mjs';
 
+/**
+ * Converts a JSX AST entry into the `{ data, code }` shape `web` now consumes,
+ * mirroring the conversion the jsx-ast worker performs before streaming.
+ */
+const toCodeItem = content => ({
+  data: content.data,
+  code: toJs(content, { handlers: jsx }).value,
+});
+
 const createEntry = (api, name) => {
   const heading = {
     type: 'heading',
@@ -39,10 +50,11 @@ describe('web generate', () => {
 
     const fs = createEntry('fs', 'File system');
     const notFoundPage = buildNotFoundPage();
-    const input = await Promise.all([
+    const contents = await Promise.all([
       buildContent([fs], fs),
       buildContent(notFoundPage.entries, notFoundPage.head),
     ]);
+    const input = contents.map(toCodeItem);
 
     const [fsPage, notFoundResult] = await generate(input);
 
@@ -69,7 +81,7 @@ describe('web generate', () => {
     };
 
     const fs = createEntry('fs', 'File system');
-    const [fsPage] = await generate([await buildContent([fs], fs)]);
+    const [fsPage] = await generate([toCodeItem(await buildContent([fs], fs))]);
 
     assert.match(fsPage.html, /Custom project docs/);
     assert.match(fsPage.html, /https:\/\/example\.com\/og\.png/);

src/generators/web/generate.mjs

@@ -4,15 +4,17 @@ import { readFile } from 'node:fs/promises';
 import { join } from 'node:path';
 
 import { copyStaticAssets } from './utils/copying.mjs';
-import { processJSXEntries } from './utils/processing.mjs';
+import { createCodeConverter, processBundles } from './utils/processing.mjs';
 import getConfig from '../../utils/configuration/index.mjs';
 import { writeFile } from '../../utils/file.mjs';
 
 /**
- * Main generation function that processes JSX AST entries into web bundles.
+ * Main generation function that bundles per-page JSX code into web output.
  *
- * Bundles all JSX AST entries in a single pass so shared component chunks and
- * CSS are produced once.
+ * Receives `jsx-ast`'s output as `{ data, code }` items — the JSX AST was
+ * already serialized to `code` in the jsx-ast worker, so no AST is held here.
+ * Bundling and rendering then run once over the accumulated code, since shared
+ * component chunks, CSS, and the sidebar need every entry together.
  *
  * @type {import('./types').Generator['generate']}
  */
@@ -21,14 +23,30 @@ export async function generate(input) {
 
   const template = await readFile(config.templatePath, 'utf-8');
 
+  const converter = createCodeConverter();
+
+  // Per-page metadata, in render order. Each item is already just
+  // `{ data, code }` — the heavy JSX AST was converted to `code` and discarded
+  // in the jsx-ast worker, so nothing large is held here.
+  const datas = [];
+
+  for (const item of input) {
+    converter.add(item);
+    datas.push(item.data);
+  }
+
   // Sidebar lists only the real module pages.
-  const sidebarEntries = input.filter(entry => entry.data.synthetic !== true);
+  const sidebarEntries = datas
+    .filter(data => data.synthetic !== true)
+    .map(data => ({ data }));
 
-  const { results, css, chunks } = await processJSXEntries(
-    input,
+  const { results, css, chunks } = await processBundles({
+    serverCodeMap: converter.serverCodeMap,
+    clientCodeMap: converter.clientCodeMap,
+    datas,
+    sidebarEntries,
     template,
-    sidebarEntries
-  );
+  });
 
   if (config.output) {
     for (const { html, path } of results) {

src/generators/web/index.mjs

@@ -13,8 +13,11 @@ import { createLazyGenerator } from '../../utils/generators.mjs';
  * - Client-side JavaScript with code splitting
  * - Bundled CSS styles
  *
- * Note: This generator does NOT support streaming/chunked processing because
- * processJSXEntries needs all entries together to generate code-split bundles.
+ * `jsx-ast` serializes each page's JSX AST to a `code` string inside its worker,
+ * so this generator only ever handles small `{ data, code }` items — the heavy
+ * ASTs (notably the giant `all` page) never reach the main thread. Bundling and
+ * rendering run once over the accumulated code, since code-splitting and the
+ * sidebar need every entry together.
  *
  * @type {import('./types').Generator}
  */

src/generators/web/utils/processing.mjs

@@ -1,7 +1,6 @@
 import { randomUUID } from 'node:crypto';
 import { createRequire } from 'node:module';
 
-import { jsx, toJs } from 'estree-util-to-js';
 import { transform } from 'lightningcss-wasm';
 
 import bundleCode from './bundle.mjs';
@@ -81,31 +80,37 @@ export const buildHead = ({ meta = [], links = [], html = [] }) =>
   ].join('\n  ');
 
 /**
- * Converts JSX AST entries to server and client JavaScript code.
+ * Creates an accumulator that wraps per-page JSX code into server and client
+ * programs one at a time. The JSX AST has already been serialized to a code
+ * string upstream (in the `jsx-ast` worker), so the heavy AST never reaches
+ * the main thread — only the code string and page metadata stream in here.
  *
- * @param {Array<import('../../jsx-ast/utils/buildContent.mjs').JSXContent>} entries - JSX AST entries
- * @param {function} buildServerProgram - Wraps code for server execution
- * @param {function} buildClientProgram - Wraps code for client hydration
- * @returns {{serverCodeMap: Map<string, string>, clientCodeMap: Map<string, string>}}
+ * @returns {{ add: (item: { data: import('../../metadata/types').MetadataEntry, code: string }) => void, serverCodeMap: Map<string, string>, clientCodeMap: Map<string, string> }}
  */
-function convertJSXToCode(entries, { buildServerProgram, buildClientProgram }) {
+export function createCodeConverter() {
+  const { buildServerProgram, buildClientProgram } = createASTBuilder();
+
   const serverCodeMap = new Map();
   const clientCodeMap = new Map();
 
-  for (const entry of entries) {
-    const fileName = `${entry.data.api}.jsx`;
-
-    // Convert AST to JavaScript string with JSX syntax
-    const { value: code } = toJs(entry, { handlers: jsx });
-
-    // Prepare code for server-side execution (wrapped for SSR)
-    serverCodeMap.set(fileName, buildServerProgram(code));
-
-    // Prepare code for client-side execution (wrapped for hydration)
-    clientCodeMap.set(fileName, buildClientProgram(code));
-  }
-
-  return { serverCodeMap, clientCodeMap };
+  return {
+    /**
+     * Records the server/client programs for a single page's JSX code.
+     *
+     * @param {{ data: import('../../metadata/types').MetadataEntry, code: string }} item
+     */
+    add: ({ data, code }) => {
+      const fileName = `${data.api}.jsx`;
+
+      // Prepare code for server-side execution (wrapped for SSR)
+      serverCodeMap.set(fileName, buildServerProgram(code));
+
+      // Prepare code for client-side execution (wrapped for hydration)
+      clientCodeMap.set(fileName, buildClientProgram(code));
+    },
+    serverCodeMap,
+    clientCodeMap,
+  };
 }
 
 /**
@@ -141,32 +146,34 @@ async function executeServerCode(serverCodeMap, requireFn, virtualImports) {
 }
 
 /**
- * Processes JSX AST entries into complete HTML pages, client JS bundles, and CSS.
+ * Bundles pre-converted JSX code into complete HTML pages, client JS bundles,
+ * and CSS. Conversion (JSX AST → code) happens upstream via
+ * {@link createCodeConverter} so the heavy ASTs are already discarded; this
+ * step needs every entry together for code-splitting and the shared sidebar.
  *
- * @param {Array<import('../../jsx-ast/utils/buildContent.mjs').JSXContent>} entries - The JSX AST entries to process.
- * @param {string} template - The HTML template string for the output pages.
- * @param {Array<{ data: import('../../metadata/types').MetadataEntry }>} [sidebarEntries] - Entries used to build the sidebar page list. Defaults to `entries`. Pass the full set when rendering a subset (e.g. the `all` page) so the sidebar still links to every module.
+ * @param {object} params
+ * @param {Map<string, string>} params.serverCodeMap - Server-side code per page.
+ * @param {Map<string, string>} params.clientCodeMap - Client-side code per page.
+ * @param {Array<import('../../metadata/types').MetadataEntry>} params.datas - Per-page metadata, in render order.
+ * @param {Array<{ data: import('../../metadata/types').MetadataEntry }>} params.sidebarEntries - Entries used to build the sidebar page list (real module pages only).
+ * @param {string} params.template - The HTML template string for the output pages.
  */
-export async function processJSXEntries(
-  entries,
+export async function processBundles({
+  serverCodeMap,
+  clientCodeMap,
+  datas,
+  sidebarEntries,
   template,
-  sidebarEntries = entries
-) {
+}) {
   const config = getConfig('web');
-  const astBuilders = createASTBuilder();
   const requireFn = createRequire(import.meta.url);
   const virtualImports = {
     '#theme/config': createConfigSource(sidebarEntries),
     ...config.virtualImports,
   };
-  // Step 1: Convert JSX AST to JavaScript
-  const { serverCodeMap, clientCodeMap } = convertJSXToCode(
-    entries,
-    astBuilders
-  );
 
-  // Step 2: Bundle server and client code in parallel
-  // Both need all entries for code-splitting, but are independent of each other
+  // Bundle server and client code in parallel. Both need all entries for
+  // code-splitting, but are independent of each other.
   const [serverBundle, clientBundle] = await Promise.all([
     executeServerCode(serverCodeMap, requireFn, virtualImports),
     bundleCode(clientCodeMap, virtualImports),
@@ -182,9 +189,9 @@ export async function processJSXEntries(
   // template authors avoid nested template-literal escaping.
   const head = buildHead(config.head);
 
-  // Step 3: Render final HTML pages
+  // Render final HTML pages
   const results = await Promise.all(
-    entries.map(async ({ data }) => {
+    datas.map(async data => {
       const root = resolvePageRoot(data);
 
       // Replace template placeholders with actual content