feat(fs): file system improvements (#672)

* feat(fs): file system improvements

* fixup!

Author: avivkeller

package-lock.json

@@ -1,7 +1,7 @@
 {
   "name": "@node-core/doc-kit",
   "type": "module",
-  "version": "1.0.2",
+  "version": "1.1.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/nodejs/doc-kit.git"
@@ -57,6 +57,7 @@
     "estree-util-to-js": "^2.0.0",
     "estree-util-visit": "^2.0.0",
     "github-slugger": "^2.0.0",
+    "glob-parent": "^6.0.2",
     "globals": "^17.3.0",
     "hast-util-to-string": "^3.0.1",
     "hastscript": "^9.0.1",
@@ -85,7 +86,6 @@
     "unist-util-remove": "^4.0.0",
     "unist-util-select": "^5.1.0",
     "unist-util-visit": "^5.1.0",
-    "vfile": "^6.0.3",
     "yaml": "^2.8.2"
   }
 }

package.json

@@ -1,6 +1,6 @@
 'use strict';
 
-import { mkdir, writeFile } from 'node:fs/promises';
+import { mkdir } from 'node:fs/promises';
 import { join } from 'node:path';
 
 import { visit } from 'unist-util-visit';
@@ -13,6 +13,7 @@ import {
   normalizeSectionName,
 } from './utils/section.mjs';
 import getConfig from '../../utils/configuration/index.mjs';
+import { writeFile } from '../../utils/file.mjs';
 
 /**
  * Generates a file list from code blocks.

src/generators/addon-verify/generate.mjs

@@ -1,6 +1,5 @@
 'use strict';
 
-import { writeFile } from 'node:fs/promises';
 import { basename, join } from 'node:path';
 
 import { checkIndirectReferences } from './utils/checkIndirectReferences.mjs';
@@ -11,6 +10,7 @@ import {
   GITHUB_BLOB_URL,
   populate,
 } from '../../utils/configuration/templates.mjs';
+import { writeFile } from '../../utils/file.mjs';
 
 /**
  * Generates the `apilinks.json` file.

src/generators/api-links/generate.mjs

@@ -1,13 +1,14 @@
 'use strict';
 
 import { readFile } from 'node:fs/promises';
-import { extname } from 'node:path';
+import { relative, sep } from 'node:path/posix';
 
+import globParent from 'glob-parent';
 import { globSync } from 'tinyglobby';
-import { VFile } from 'vfile';
 
 import { STABILITY_INDEX_URL } from './constants.mjs';
 import getConfig from '../../utils/configuration/index.mjs';
+import { withExt } from '../../utils/file.mjs';
 import { QUERIES } from '../../utils/queries/index.mjs';
 import { getRemark } from '../../utils/remark.mjs';
 
@@ -24,19 +25,19 @@ export async function processChunk(inputSlice, itemIndices) {
 
   const results = [];
 
-  for (const path of filePaths) {
+  for (const [path, parent] of filePaths) {
     const content = await readFile(path, 'utf-8');
-    const vfile = new VFile({
-      path,
-      value: content.replace(
-        QUERIES.stabilityIndexPrefix,
-        match => `[${match}](${STABILITY_INDEX_URL})`
-      ),
-    });
+    const value = content.replace(
+      QUERIES.stabilityIndexPrefix,
+      match => `[${match}](${STABILITY_INDEX_URL})`
+    );
+
+    const relativePath = sep + withExt(relative(parent, path));
 
     results.push({
-      tree: remarkProcessor.parse(vfile),
-      file: { stem: vfile.stem, basename: vfile.basename },
+      tree: remarkProcessor.parse(value),
+      // The path is the relative path minus the extension
+      path: relativePath,
     });
   }
 
@@ -51,9 +52,14 @@ export async function processChunk(inputSlice, itemIndices) {
 export async function* generate(_, worker) {
   const { ast: config } = getConfig();
 
-  const files = globSync(config.input, { ignore: config.ignore }).filter(
-    p => extname(p) === '.md'
-  );
+  const files = config.input.flatMap(input => {
+    const parent = globParent(input);
+
+    return globSync(input, { ignore: config.ignore }).map(child => [
+      child,
+      parent,
+    ]);
+  });
 
   // Parse markdown files in parallel using worker threads
   for await (const chunkResult of worker.stream(files)) {

src/generators/ast/generate.mjs

@@ -1,11 +1,11 @@
 'use strict';
 
-import { writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
 
 import { remove } from 'unist-util-remove';
 
 import getConfig from '../../utils/configuration/index.mjs';
+import { writeFile } from '../../utils/file.mjs';
 import { UNIST } from '../../utils/queries/index.mjs';
 
 /**

src/generators/json-simple/generate.mjs

@@ -1,9 +1,9 @@
 import { buildSideBarProps } from './utils/buildBarProps.mjs';
 import buildContent from './utils/buildContent.mjs';
 import { getSortedHeadNodes } from './utils/getSortedHeadNodes.mjs';
-import getConfig from '../../utils/configuration/index.mjs';
 import { groupNodesByModule } from '../../utils/generators.mjs';
 import { getRemarkRecma } from '../../utils/remark.mjs';
+import { relative } from '../../utils/url.mjs';
 
 const remarkRecma = getRemarkRecma();
 
@@ -22,7 +22,15 @@ export async function processChunk(slicedInput, itemIndices, docPages) {
   for (const idx of itemIndices) {
     const { head, entries } = slicedInput[idx];
 
-    const sideBarProps = buildSideBarProps(head, docPages);
+    const sideBarProps = buildSideBarProps(
+      head,
+      docPages.map(([heading, path]) => [
+        heading,
+        head.path === path
+          ? `${head.basename}.html`
+          : `${relative(path, head.path)}.html`,
+      ])
+    );
 
     const content = await buildContent(
       entries,
@@ -43,17 +51,11 @@ export async function processChunk(slicedInput, itemIndices, docPages) {
  * @type {import('./types').Generator['generate']}
  */
 export async function* generate(input, worker) {
-  const config = getConfig('jsx-ast');
-
   const groupedModules = groupNodesByModule(input);
 
   const headNodes = getSortedHeadNodes(input);
 
-  // Pre-compute docPages once in main thread
-  // TODO(@avivkeller): Load the index file here instead of during configuration
-  const docPages = config.index
-    ? config.index.map(({ section, api }) => [section, `${api}.html`])
-    : headNodes.map(node => [node.heading.data.name, `${node.api}.html`]);
+  const docPages = headNodes.map(node => [node.heading.data.name, node.path]);
 
   // Create sliced input: each item contains head + its module's entries
   // This avoids sending all 4700+ entries to every worker

src/generators/jsx-ast/generate.mjs

@@ -69,7 +69,8 @@ describe('extractTextContent', () => {
 describe('buildMetaBarProps', () => {
   it('creates meta bar properties from entries', () => {
     const head = {
-      api: 'fs',
+      basename: 'fs',
+      path: '/fs',
       added: 'v1.0.0',
     };
 
@@ -151,7 +152,8 @@ describe('formatVersionOptions', () => {
 describe('buildSideBarProps', () => {
   it('creates sidebar properties with versions and navigation', () => {
     const entry = {
-      api: 'http',
+      path: 'http',
+      basename: 'http',
       introduced_in: 'v0.10.0',
     };
 

src/generators/jsx-ast/utils/__tests__/buildBarProps.test.mjs

@@ -90,25 +90,25 @@ export const buildMetaBarProps = (head, entries) => {
     addedIn: head.added || head.introduced_in || '',
     readingTime: readingTime(extractTextContent(entries)).text,
     viewAs: [
-      ['JSON', `${head.api}.json`],
-      ['MD', `${head.api}.md`],
+      ['JSON', `${head.basename}.json`],
+      ['MD', `${head.basename}.md`],
     ],
-    editThisPage: `${populate(GITHUB_EDIT_URL, config)}${head.api}.md`,
+    editThisPage: `${populate(GITHUB_EDIT_URL, config)}${head.path}.md`,
   };
 };
 
 /**
  * Converts a compatible version entry into a version label and link.
  *
  * @param {Array<import('../../../parsers/types').ReleaseEntry>} compatibleVersions - Compatible versions
- * @param {string} api - API identifier (used in link)
+ * @param {string} path - path for the version URL
  */
-export const formatVersionOptions = (compatibleVersions, api) => {
+export const formatVersionOptions = (compatibleVersions, path) => {
   const config = getConfig('jsx-ast');
 
   return compatibleVersions.map(({ version, isLts, isCurrent }) => {
     const parsed = getVersionFromSemVer(version);
-    const value = getVersionURL(parsed, api, config.baseURL);
+    const value = getVersionURL(parsed, path, config.baseURL);
 
     let label = `v${parsed}`;
 
@@ -143,9 +143,9 @@ export const buildSideBarProps = (entry, docPages) => {
   );
 
   return {
-    versions: formatVersionOptions(compatibleVersions, entry.api),
+    versions: formatVersionOptions(compatibleVersions, entry.path),
     currentVersion: `v${config.version.version}`,
-    pathname: `${entry.api}.html`,
+    pathname: `${entry.basename}.html`,
     docPages,
   };
 };

src/generators/jsx-ast/utils/buildBarProps.mjs

@@ -49,6 +49,7 @@ export async function generate(input) {
 
   const templateValues = {
     api: 'all',
+    path: 'all',
     added: '',
     section: 'All',
     version: `v${config.version.version}`,