code review

Author: avivkeller

.github/workflows/sync-orama.yml

@@ -17,6 +17,7 @@ on:
       - main
     types:
       - labeled
+  pull_request:
 
 permissions:
   contents: read

apps/site/package.json

@@ -88,6 +88,7 @@
     "@testing-library/user-event": "~14.6.1",
     "@types/mdx": "^2.0.13",
     "@types/semver": "~7.7.0",
+    "dedent": "^1.6.0",
     "eslint-config-next": "15.5.0",
     "eslint-import-resolver-typescript": "~4.4.4",
     "eslint-plugin-mdx": "~3.6.2",
@@ -96,6 +97,9 @@
     "global-jsdom": "^26.0.0",
     "handlebars": "4.7.8",
     "jsdom": "^26.0.0",
+    "mdast-util-from-markdown": "^2.0.2",
+    "mdast-util-to-string": "^4.0.0",
+    "nock": "^14.0.10",
     "remark-frontmatter": "^5.0.0",
     "stylelint": "16.23.0",
     "stylelint-config-standard": "39.0.0",

apps/site/scripts/orama-search/__tests__/get-documents.test.mjs

@@ -0,0 +1,38 @@
+import assert from 'node:assert/strict';
+import { test, mock } from 'node:test';
+
+import nock from 'nock';
+
+mock.module('node:fs/promises', {
+  namedExports: {
+    glob: () => ['filename'],
+    readFile: name => name.endsWith('filename') && 'content',
+  },
+});
+
+const { getAPIDocs, getArticles } = await import('../get-documents.mjs');
+
+test('getAPIDocs', async () => {
+  nock('https://api.github.com')
+    .get('/repos/nodejs/node/contents/doc/api')
+    .query(true)
+    .reply(200, [
+      {
+        name: 'fs.md',
+        download_url: 'data:text/plain,fs',
+      },
+    ]);
+
+  const result = await getAPIDocs();
+
+  assert.equal(result.length, 1);
+  assert.equal(result[0].content, 'fs');
+  assert.match(result[0].pathname, /^docs\/v[^/]+\/api\/fs\.html$/);
+});
+
+test('getArticles', async () => {
+  const result = await getArticles();
+  assert.deepStrictEqual(result, [
+    { content: 'content', pathname: 'filename' },
+  ]);
+});

apps/site/scripts/orama-search/__tests__/process-documents.test.mjs

@@ -0,0 +1,104 @@
+import assert from 'node:assert';
+import test from 'node:test';
+
+import dedent from 'dedent';
+
+import { processDocument } from '../process-documents.mjs';
+
+const testCases = [
+  {
+    name: 'Uses front matter title if available',
+    input: {
+      pathname: 'blog/my-post.html',
+      content: dedent`
+        ---
+        title: Custom Title
+        ---
+        # Intro
+        Hello world
+      `,
+    },
+    expected: [
+      {
+        path: 'blog/my-post.html#intro',
+        siteSection: 'Blog',
+        pageTitle: 'Custom Title',
+        pageSectionTitle: 'Intro',
+        pageSectionContent: 'Hello world',
+      },
+    ],
+  },
+  {
+    name: 'Falls back to filename for title',
+    input: {
+      pathname: 'docs/another-post.html',
+      content: dedent`
+        # Start
+        Content here
+      `,
+    },
+    expected: [
+      {
+        path: 'docs/another-post.html#start',
+        siteSection: 'Docs',
+        pageTitle: 'another post',
+        pageSectionTitle: 'Start',
+        pageSectionContent: 'Content here',
+      },
+    ],
+  },
+  {
+    name: 'Handles multiple sections',
+    input: {
+      pathname: 'guides/test.html',
+      content: dedent`
+        # First
+        Paragraph A
+
+        # Second
+        Paragraph B
+      `,
+    },
+    expected: [
+      {
+        path: 'guides/test.html#first',
+        siteSection: 'Guides',
+        pageTitle: 'test',
+        pageSectionTitle: 'First',
+        pageSectionContent: 'Paragraph A',
+      },
+      {
+        path: 'guides/test.html#second',
+        siteSection: 'Guides',
+        pageTitle: 'test',
+        pageSectionTitle: 'Second',
+        pageSectionContent: 'Paragraph B',
+      },
+    ],
+  },
+  {
+    name: 'Section with no heading',
+    input: {
+      pathname: 'misc/untitled.html',
+      content: dedent`
+        Just some text without a heading
+      `,
+    },
+    expected: [
+      {
+        path: 'misc/untitled.html#',
+        siteSection: 'Misc',
+        pageTitle: 'untitled',
+        pageSectionTitle: '',
+        pageSectionContent: 'Just some text without a heading',
+      },
+    ],
+  },
+];
+
+for (const { name, input, expected } of testCases) {
+  test(name, () => {
+    const result = processDocument(input);
+    assert.deepStrictEqual(result, expected);
+  });
+}

apps/site/scripts/orama-search/get-documents.mjs

@@ -1,5 +1,5 @@
 import { readFile, glob } from 'node:fs/promises';
-import { join, basename } from 'node:path';
+import { join, basename, posix, win32 } from 'node:path';
 
 import generateReleaseData from '#site/next-data/generators/releaseData.mjs';
 import { getRelativePath } from '#site/next.helpers.mjs';
@@ -15,16 +15,16 @@ const fetchOptions = process.env.GITHUB_TOKEN
  * Fetch Node.js API documentation directly from GitHub
  * for the current Active LTS version.
  */
-const getAPIDocs = async () => {
+export const getAPIDocs = async () => {
   // Find the current Active LTS version
   const releaseData = await generateReleaseData();
-  const version = releaseData.find(
+  const { versionWithPrefix } = releaseData.find(
     r => r.status === 'Active LTS'
-  ).versionWithPrefix;
+  );
 
   // Get list of API docs from the Node.js repo
   const fetchResponse = await fetch(
-    `https://api.github.com/repos/nodejs/node/contents/doc/api?ref=${version}`,
+    `https://api.github.com/repos/nodejs/node/contents/doc/api?ref=${versionWithPrefix}`,
     fetchOptions
   );
   const documents = await fetchResponse.json();
@@ -34,10 +34,10 @@ const getAPIDocs = async () => {
     documents.map(async ({ name, download_url }) => {
       const res = await fetch(download_url, fetchOptions);
 
-      return processDocument({
+      return {
         content: await res.text(),
-        pathname: `docs/${version}/api/${basename(name, '.md')}.html`,
-      });
+        pathname: `docs/${versionWithPrefix}/api/${basename(name, '.md')}.html`,
+      };
     })
   );
 };
@@ -46,7 +46,7 @@ const getAPIDocs = async () => {
  * Collect all local markdown/mdx articles under /pages/en,
  * excluding blog content.
  */
-const getArticles = async () => {
+export const getArticles = async () => {
   const relativePath = getRelativePath(import.meta.url);
   const root = join(relativePath, '..', '..', 'pages', 'en');
 
@@ -57,15 +57,14 @@ const getArticles = async () => {
   return Promise.all(
     files
       .filter(path => !path.startsWith('blog'))
-      .map(async path =>
-        processDocument(
-          {
-            content: await readFile(join(root, path), 'utf8'),
-            pathname: path.replace(/\.mdx?$/, ''),
-          },
-          true
-        )
-      )
+      .map(async path => ({
+        content: await readFile(join(root, path), 'utf8'),
+        pathname: path
+          // Strip the extension
+          .replace(/\.mdx?$/, '')
+          // Normalize to a POSIX path
+          .replaceAll(win32.sep, posix.sep),
+      }))
   );
 };
 
@@ -74,5 +73,7 @@ const getArticles = async () => {
  */
 export const getDocuments = async () => {
   const documentPromises = await Promise.all([getAPIDocs(), getArticles()]);
-  return documentPromises.flatMap(documents => documents.flat());
+  return documentPromises.flatMap(documents =>
+    documents.flatMap(processDocument)
+  );
 };