Improve MDX parsing, local source, and repo config

Enhanced MDX parsing to clean TOC titles and better handle type annotations and style attributes. Improved local source handling for meta JSON files and added support for writing meta files. Updated repository config to include LICENSE.md and docsPath, and adjusted scripts to set NEXT_DOC_SOURCE for static builds. Also improved relative link resolution for local sources.

Author: cklei-carly

.gitignore

@@ -12,6 +12,7 @@
 # test & build
 /coverage
 /fi-plugins/
+/fi-plugins*/
 /.next/
 /out/
 /build

components/mdx.tsx

@@ -14,6 +14,7 @@ import { CodeBlock, Pre } from "fumadocs-ui/components/codeblock";
 import type { MDXComponents } from "mdx/types";
 import { siteConfig } from "@/lib/site-config";
 import { RepositoryConfig, VersionConfig } from "@/lib/repo-config";
+import { source, isLocal } from "@/lib/source";
 
 const githubCalloutRegex =
   /^\[!(NOTE|TIP|IMPORTANT|WARNING|CAUTION)\]\s*(.*)$/s;
@@ -247,5 +248,26 @@ export function createRelativeLink(
   let repositoryUrl = `${repository.repository_url}/blob/${version.github_branch || version.version}/${href}`;
   // console.debug("### Creating relative link for:", href, repositoryUrl);
 
+  if (isLocal) {
+    // 1. limited_files take precedence
+    let localPage = source.getPage([repository.repo, version.version, href.replace(/\.mdx?$/, "").toLowerCase()])
+    if (!localPage &&repository.docsPath) {
+      // 2. search in the docsPath
+      // (replace leading 'repository.docsPath' if any)
+      localPage = source.getPage([repository.repo, version.version, href.replace(new RegExp(`^${repository.docsPath}/?`), "").replace(/\.mdx?$/, "").toLowerCase()])
+    }
+    
+    if (localPage) {
+      console.debug("### Found local page for link:", href, "->", localPage.url);
+      // Convert to internal docs link
+      repositoryUrl = `${siteConfig.baseUrl}/${localPage.url.replace(/^\//, "")}`;
+    }
+  }
+
+  // Converting relative links to absolute links for Vercel's preview image generation
+  if (repositoryUrl.startsWith("/")) {
+    repositoryUrl = siteConfig.baseUrl + repositoryUrl;
+  }
+
   return repositoryUrl;
 }

lib/compile-md.ts

@@ -20,6 +20,56 @@ export interface CompiledPage {
   body: FC<{ components?: MDXComponents }>;
 }
 
+// Function to clean TOC items and convert them to plain text
+function cleanTocItems(toc: TableOfContents): TableOfContents {
+  return toc.map(item => ({
+    ...item,
+    title: cleanTocTitle(item.title)
+  }));
+}
+
+// Function to convert TOC title to plain text
+function cleanTocTitle(title: any): string {
+  if (typeof title === 'string') {
+    // Remove HTML tags and decode entities
+    return title
+      .replace(/<[^>]*>/g, '') // Remove HTML tags
+      .replace(/&lt;/g, '<')
+      .replace(/&gt;/g, '>')
+      .replace(/&amp;/g, '&')
+      .replace(/&quot;/g, '"')
+      .replace(/&#39;/g, "'")
+      .trim();
+  }
+  
+  if (title && typeof title === 'object') {
+    // Handle React nodes - extract text content
+    if (title.props && title.props.children) {
+      return cleanTocTitle(title.props.children);
+    }
+    
+    if (Array.isArray(title)) {
+      return title.map(cleanTocTitle).filter(Boolean).join('');
+    }
+    
+    // If it's a React element with type and props
+    if (title.type === 'a' && title.props && title.props.children) {
+      return cleanTocTitle(title.props.children);
+    }
+    
+    // Handle other React node types
+    if (title.toString && typeof title.toString === 'function') {
+      return cleanTocTitle(title.toString());
+    }
+  }
+  
+  if (Array.isArray(title)) {
+    return title.map(cleanTocTitle).filter(Boolean).join('');
+  }
+  
+  return String(title || '').trim();
+}
+
 // Create a cached compile function using React cache
 const cachedCompile = reactCache(async (filePath: string, source: string): Promise<CompiledPage> => {
   // console.time(`compile md: ${filePath}`);
@@ -31,7 +81,7 @@ const cachedCompile = reactCache(async (filePath: string, source: string): Promi
     })
     .then((compiled) => ({
       body: compiled.body,
-      toc: compiled.toc,
+      toc: cleanTocItems(compiled.toc), // Clean TOC items to ensure plain text titles
       ...compiled.frontmatter,
     }))
     .catch((error) => {
@@ -130,11 +180,51 @@ function parseSourceBeforeCompile(filePath: string, source: string): string {
     '<div class="note">$1</div>',
   );
 
-  // Replace "style="xx" as mdx format
+  // Fix type annotations that get interpreted as HTML tags
+  // Replace array<type> with array&lt;type&gt; to prevent HTML parsing
   parsedSource = parsedSource.replace(
-    /style="([^"]+)"/g,
-    (match, p1) => {
-      return `style={{ ${p1} }}`;
+    /array<([^>]+)>/g,
+    'array&lt;$1&gt;',
+  );
+
+  // Fix standalone type annotations at start of lines or after colons/spaces
+  // This handles cases like "- `id`: int" -> "- `id`: `int`"
+  parsedSource = parsedSource.replace(
+    /(\s+- `[^`]+`):\s+(int|string|bool|float|double|object|array)(\s|$)/g,
+    '$1: `$2`$3',
+  );
+
+  // Fix type annotations in parameter lists
+  // This handles cases like "- **tags**: array<int> (optional)"
+  parsedSource = parsedSource.replace(
+    /(\*\*[^*]+\*\*):\s+(array&lt;[^&]+&gt;|int|string|bool|float|double|object|array)(\s)/g,
+    '$1: `$2`$3',
+  );
+
+  // Replace "style="xx" as mdx format  
+  // Convert CSS style attributes to JSX format
+  parsedSource = parsedSource.replace(
+    /style\s*=\s*"([^"]+)"/g,
+    (match: string, styleValue: string): string => {
+      // Parse CSS properties and convert to JSX object
+      const styles = styleValue
+        .split(';')
+        .filter((prop: string) => prop.trim())
+        .map((prop: string) => {
+          const [property, ...valueParts] = prop.split(':');
+          const value = valueParts.join(':').trim(); // Handle values with colons
+          const trimmedProperty = property.trim();
+          
+          if (!trimmedProperty || !value) return '';
+          
+          // Convert kebab-case to camelCase
+          const camelProperty = trimmedProperty.replace(/-([a-z])/g, (_, letter: string) => letter.toUpperCase());
+          return `${camelProperty}: "${value}"`;
+        })
+        .filter((prop: string) => prop)
+        .join(', ');
+      
+      return `style={{ ${styles} }}`;
     }
   );
 
@@ -144,9 +234,10 @@ function parseSourceBeforeCompile(filePath: string, source: string): string {
     '\\{\\{$1\\}\\}',
   );
 
+
   return parsedSource;
 }
 
 export async function compile(filePath: string, source: string) {
   return cachedCompile(filePath, source);
-}
+}

lib/meta.ts

@@ -87,6 +87,7 @@ export const fumadocMeta: VirtualFile[] = [
             root: false,
             pages: [
               `./index`, // e.g. /docs/repo-slug/3.x/ 
+              `./overview`, // e.g. /docs/repo-slug/3.x/overview 
               "...", // all other files in this version
             ],
           },

lib/repo-config.ts

@@ -40,13 +40,15 @@ const rawRepositories = [
       "https://github.com/solutionforest/filament-cms-website-plugin",
     latest_version: "3.x",
     is_private: true,
+    docsPath: "docs", // include docs
     versions: [
       {
         version: "1.x",
         github_branch: "1.x",
         limited_files: [
           { name: "README.md", title: "Overview", slug: "overview" },
           { name: "Documentation.md", title: "Documentation", slug: "docs" },
+          { name: "LICENSE.md", title: "License", slug: "license" },
         ],
       },
       {
@@ -55,14 +57,15 @@ const rawRepositories = [
         limited_files: [
           { name: "README.md", title: "Overview", slug: "overview" },
           { name: "Documentation.md", title: "Documentation", slug: "docs" },
+          { name: "LICENSE.md", title: "License", slug: "license" },
         ],
       },
       {
         version: "3.x",
         github_branch: "3.x",
         limited_files: [
           { name: "README.md", title: "Overview", slug: "overview" },
-          { name: "Documentation.md", title: "Documentation", slug: "docs" },
+          { name: "LICENSE.md", title: "License", slug: "license" },
         ],
       },
     ],
@@ -71,7 +74,7 @@ const rawRepositories = [
   //   description: 'Effortlessly manage your newsletters with our Filament Newsletter package, designed for seamless integration with the Filament Admin Panel. Enjoy flexible and user-friendly email marketing directly within your admin panel.',
   //   repository_url: 'https://github.com/solutionforest/filaletter',
   //   latest_version: '3.x',
-  //   is_private: true,
+  //   is_private: false,
   //   versions: [
   //     {
   //       version: '2.x',

lib/source.ts

@@ -8,8 +8,6 @@ const FileNameRegex = /^\d\d-(.+)$/;
 export const isLocal =
   process.env.NEXT_DOC_SOURCE === 'local'; // || process.env.NEXT_PHASE === "phase-production-build";
 
-console.debug('#### Is Local: ', isLocal);
-
 export const source = loader({
   baseUrl: "/docs",
   // source: await createLocalSource(),
@@ -19,16 +17,12 @@ export const source = loader({
     try {
       const segments = info.path
         .split("/")
-        .filter((seg) => !(seg.startsWith("(") && seg.endsWith(")")))
-        .map((seg) => {
-          const res = FileNameRegex.exec(seg);
-
-          return res ? res[1] : seg;
-        });
+        .filter((seg) => !(seg.startsWith("(") && seg.endsWith(")")));
 
       if (segments.at(-1) === "index") {
         segments.pop();
       }
+      // console.debug("Generating slugs for:", info.path, "->", segments);
 
       return segments;
     } catch (error) {

lib/sources/github.ts

@@ -11,7 +11,7 @@ import {
   type VersionConfig,
 } from "../repo-config";
 import { cache as reactCache } from 'react';
-import { buildMarkdownFileToLocal } from "./local";
+import { buildMarkdownFileToLocal, buildMetaJson } from "./local";
 
 const token = process.env.GITHUB_TOKEN;
 if (!token) throw new Error(`environment variable GITHUB_TOKEN is needed.`);
@@ -283,6 +283,7 @@ async function fetchRepositoryFiles(
   const docsSha = config.docsPath
     ? await getDirectorySha(config, version, config.docsPath)
     : null;
+  console.debug(`Docs SHA for ${config.owner}/${config.repo} (${version.version}) [${config.docsPath}]:`, docsSha);
   if (docsSha) {
     try {
       // Use cached GitHub tree function for better performance
@@ -317,8 +318,11 @@ async function fetchRepositoryFiles(
         // Parsing file path
         const parsedFilePath = path.parse(file.path);
         const pageTitle = getTitleFromFile(parsedFilePath.name);
+        const cleanFileRelativePath = parsedFilePath.dir.length > 0
+          ? `${parsedFilePath.dir}/${parsedFilePath.name}`
+          : parsedFilePath.name;
 
-        const pagePath = `${repoSlug}/${versionSlug}/${parsedFilePath.dir}/${parsedFilePath.name}`;
+        const pagePath = `${repoSlug}/${versionSlug}/${cleanFileRelativePath}`;
 
         if (!docFolderPages.has(parsedFilePath.dir)) {
           docFolderPages.set(parsedFilePath.dir, []);
@@ -327,7 +331,7 @@ async function fetchRepositoryFiles(
 
         // console.debug(`Docs file found: ${file.path} (Parsed: ${formattedFilePath})`);
 
-        // console.debug(`2| Adding doc file: ${pagePath} (title: ${pageTitle})`);
+        console.debug(`2| Adding doc file: ${pagePath} (title: ${pageTitle})`);
 
         const content = await fetchBlob(file.url as string);
 
@@ -389,6 +393,8 @@ async function fetchRepositoryFiles(
 
   await buildMarkdownFileToLocal(rawMarkdownFiles);
 
+  buildMetaJson(allFiles);
+
   return allFiles;
 }
 
@@ -436,4 +442,4 @@ export async function createGitHubSource(): Promise<
   return {
     files: [...allPages, ...fumadocMeta],
   };
-}
+}