feat(web): add all, index, and 404 generation (#780)

* feat(web): add all.html generation

Author: avivkeller

package-lock.json

@@ -6,7 +6,10 @@ The `jsx-ast` generator converts MDAST (Markdown Abstract Syntax Tree) to JSX AS
 
 The `jsx-ast` generator accepts the following configuration options:
 
-| Name    | Type     | Default  | Description                                                              |
-| ------- | -------- | -------- | ------------------------------------------------------------------------ |
-| `ref`   | `string` | `'main'` | Git reference/branch for linking to source files                         |
-| `index` | `array`  | -        | Array of `{ section, api }` objects defining the documentation structure |
+| Name                   | Type      | Default  | Description                                                              |
+| ---------------------- | --------- | -------- | ------------------------------------------------------------------------ |
+| `ref`                  | `string`  | `'main'` | Git reference/branch for linking to source files                         |
+| `index`                | `array`   | -        | Array of `{ section, api }` objects defining the documentation structure |
+| `generateAllPage`      | `boolean` | `true`   | When `true`, creates a synthetic JSX AST entry for `all.html`            |
+| `generateIndexPage`    | `boolean` | `true`   | When `true`, creates a synthetic JSX AST entry for `index.html`          |
+| `generateNotFoundPage` | `boolean` | `true`   | When `true`, creates a synthetic JSX AST entry for `404.html`            |

src/generators/jsx-ast/README.md

@@ -0,0 +1,95 @@
+import assert from 'node:assert/strict';
+import { describe, it } from 'node:test';
+
+import getConfig, { setConfig } from '../../../utils/configuration/index.mjs';
+import { generate, processChunk } from '../generate.mjs';
+
+const createEntry = (api, name, { stabilityIndex = '2' } = {}) => {
+  const heading = {
+    type: 'heading',
+    depth: 1,
+    children: [{ type: 'text', value: name }],
+    data: { name, text: name, slug: api },
+  };
+
+  return {
+    api,
+    path: `/${api}`,
+    basename: api,
+    heading,
+    stability:
+      stabilityIndex == null
+        ? null
+        : {
+            data: {
+              index: stabilityIndex,
+              description: `${name} stable. Longer description.`,
+            },
+          },
+    content: {
+      type: 'root',
+      children: [
+        heading,
+        {
+          type: 'paragraph',
+          children: [{ type: 'text', value: `${name} body` }],
+        },
+      ],
+    },
+  };
+};
+
+const collect = async generator => {
+  const results = [];
+
+  for await (const chunk of generator) {
+    results.push(...chunk);
+  }
+
+  return results;
+};
+
+const createWorker = seenItems => ({
+  async *stream(items) {
+    seenItems.push(...items);
+    yield items.map(({ head }) => ({ type: 'JSXElement', data: head }));
+  },
+});
+
+describe('jsx-ast generate', () => {
+  it('does not attach raw section entries to regular JSX content', async () => {
+    await setConfig({});
+
+    const fs = createEntry('fs', 'File system');
+    const [content] = await processChunk([{ head: fs, entries: [fs] }], [0]);
+
+    assert.equal(content.data.api, 'fs');
+    assert.equal('sectionEntries' in content, false);
+  });
+
+  it('respects jsx-ast synthetic page flags', async () => {
+    await setConfig({});
+
+    const jsxAstConfig = getConfig('jsx-ast');
+    jsxAstConfig.generateAllPage = false;
+    jsxAstConfig.generateIndexPage = false;
+    jsxAstConfig.generateNotFoundPage = false;
+
+    const seenItems = [];
+    const results = await collect(
+      generate(
+        [createEntry('index', 'Index'), createEntry('fs', 'File system')],
+        createWorker(seenItems)
+      )
+    );
+
+    assert.deepEqual(
+      seenItems.map(({ head }) => head.api),
+      ['index', 'fs']
+    );
+    assert.deepEqual(
+      results.map(({ data }) => data.api),
+      ['index', 'fs']
+    );
+  });
+});

src/generators/jsx-ast/__tests__/generate.test.mjs

@@ -1,7 +1,30 @@
 import buildContent from './utils/buildContent.mjs';
 import { getSortedHeadNodes } from './utils/getSortedHeadNodes.mjs';
+import { buildNotFoundPage } from './utils/synthetic/404.mjs';
+import { buildAllPage } from './utils/synthetic/all.mjs';
+import { buildIndexPage } from './utils/synthetic/index.mjs';
+import getConfig from '../../utils/configuration/index.mjs';
 import { groupNodesByModule } from '../../utils/generators.mjs';
 
+/**
+ * Builds JSX content for all configured synthetic pages.
+ *
+ * @param {Array<import('../metadata/types').MetadataEntry>} input
+ */
+const buildSyntheticEntries = async input => {
+  const config = getConfig('jsx-ast');
+
+  const descriptors = [
+    config.generateAllPage && buildAllPage(input),
+    config.generateIndexPage && buildIndexPage(input),
+    config.generateNotFoundPage && buildNotFoundPage(),
+  ].filter(Boolean);
+
+  return Promise.all(
+    descriptors.map(({ head, entries }) => buildContent(entries, head))
+  );
+};
+
 /**
  * Process a chunk of items in a worker thread.
  * Transforms metadata entries into JSX AST nodes.
@@ -31,18 +54,24 @@ export async function processChunk(slicedInput, itemIndices) {
  * @type {import('./types').Generator['generate']}
  */
 export async function* generate(input, worker) {
-  const groupedModules = groupNodesByModule(input);
-
-  const headNodes = getSortedHeadNodes(input);
+  // The synthetic `index` page replaces the Core `index` document.
+  const moduleInput = input.filter(entry => entry.api !== 'index');
 
   // Create sliced input: each item contains head + its module's entries
   // This avoids sending all 4700+ entries to every worker
-  const entries = headNodes.map(head => ({
+  const groupedModules = groupNodesByModule(input);
+  const entries = getSortedHeadNodes(input).map(head => ({
     head,
     entries: groupedModules.get(head.api),
   }));
 
   for await (const chunkResult of worker.stream(entries)) {
     yield chunkResult;
   }
+
+  const syntheticEntries = await buildSyntheticEntries(moduleInput);
+
+  if (syntheticEntries.length > 0) {
+    yield syntheticEntries;
+  }
 }

src/generators/jsx-ast/generate.mjs

@@ -18,6 +18,9 @@ export default createLazyGenerator({
 
   defaultConfiguration: {
     ref: 'main',
+    generateAllPage: true,
+    generateIndexPage: true,
+    generateNotFoundPage: true,
   },
 
   hasParallelProcessor: true,

src/generators/jsx-ast/index.mjs

@@ -3,8 +3,10 @@ import type { JSXContent } from './utils/buildContent.mjs';
 
 export type Generator = GeneratorMetadata<
   {
-    pageURL: string;
-    editURL: string;
+    ref: string;
+    generateAllPage: boolean;
+    generateIndexPage: boolean;
+    generateNotFoundPage: boolean;
   },
   Generate<Array<MetadataEntry>, AsyncGenerator<JSXContent>>,
   ProcessChunk<

src/generators/jsx-ast/types.d.ts

@@ -0,0 +1,37 @@
+'use strict';
+
+import { createSyntheticHead, wrapAsEntry } from './synthetic.mjs';
+
+/**
+ * Builds the page descriptor for `404.html`
+ */
+export const buildNotFoundPage = () => {
+  const head = createSyntheticHead('404', 'Page Not Found');
+
+  return {
+    head,
+    entries: [
+      wrapAsEntry(head, [
+        {
+          type: 'paragraph',
+          children: [
+            {
+              type: 'text',
+              value:
+                'The page you requested could not be found. Use the navigation to find the documentation you are looking for, or return to the ',
+            },
+            {
+              type: 'link',
+              url: 'index.html',
+              children: [{ type: 'text', value: 'API index' }],
+            },
+            {
+              type: 'text',
+              value: '.',
+            },
+          ],
+        },
+      ]),
+    ],
+  };
+};