Merge pull request #178 from ut-code/docs-meta-script

ドキュメントのメタデータ生成をシンプルに & バージョン管理を追加

Author: na-trium-144

.gitignore

@@ -3,11 +3,9 @@
 /.open-next
 /cloudflare-env.d.ts
 
-# generated docs section file lists (regenerated by npm run generateSections)
-/public/docs/**/sections.yml
-
-# generated languages list (regenerated by npm run generateLanguages)
-/public/docs/languages.yml
+# generated docs section file lists (regenerated by npm run generateDocsMeta)
+/public/docs/**/sections.json
+/public/docs/languages.json
 
 # dependencies
 /node_modules

app/lib/docs.ts

@@ -4,12 +4,15 @@ import { join } from "node:path";
 import yaml from "js-yaml";
 import { isCloudflare } from "./detectCloudflare";
 import { notFound } from "next/navigation";
+import crypto from "node:crypto";
 
 export interface MarkdownSection {
+  file: string; // ファイル名
   id: string;
   level: number;
   title: string;
   rawContent: string; // 見出しも含めたもとのmarkdownの内容
+  md5: string; // mdファイル全体のmd5
 }
 
 export interface PageEntry {
@@ -36,6 +39,16 @@ interface IndexYml {
   }[];
 }
 
+export interface RevisionYmlEntry {
+  page: string;
+  rev: SectionRevision[];
+}
+export interface SectionRevision {
+  md5: string; // mdファイル全体のmd5
+  git: string; // git上のコミットハッシュ
+  path: string;
+}
+
 async function readPublicFile(path: string): Promise<string> {
   try {
     if (isCloudflare()) {
@@ -59,8 +72,8 @@ async function readPublicFile(path: string): Promise<string> {
 
 async function getLanguageIds(): Promise<string[]> {
   if (isCloudflare()) {
-    const raw = await readPublicFile("docs/languages.yml");
-    return yaml.load(raw) as string[];
+    const raw = await readPublicFile("docs/languages.json");
+    return JSON.parse(raw) as string[];
   } else {
     const docsDir = join(process.cwd(), "public", "docs");
     const entries = await readdir(docsDir, { withFileTypes: true });
@@ -90,28 +103,66 @@ export async function getPagesList(): Promise<LanguageEntry[]> {
   );
 }
 
+export async function getSectionsList(
+  lang: string,
+  pageId: string
+): Promise<string[]> {
+  if (isCloudflare()) {
+    const sectionsJson = await readPublicFile(
+      `docs/${lang}/${pageId}/sections.json`
+    );
+    return JSON.parse(sectionsJson) as string[];
+  } else {
+    function naturalSortMdFiles(a: string, b: string): number {
+      // -intro.md always comes first
+      if (a === "-intro.md") return -1;
+      if (b === "-intro.md") return 1;
+      // Sort numerically by leading N1-N2 prefix
+      const aMatch = a.match(/^(\d+)-(\d+)/);
+      const bMatch = b.match(/^(\d+)-(\d+)/);
+      if (aMatch && bMatch) {
+        const n1Diff = parseInt(aMatch[1]) - parseInt(bMatch[1]);
+        if (n1Diff !== 0) return n1Diff;
+        return parseInt(aMatch[2]) - parseInt(bMatch[2]);
+      }
+      return a.localeCompare(b);
+    }
+    return (await readdir(join(process.cwd(), "public", "docs", lang, pageId)))
+      .filter((f) => f.endsWith(".md"))
+      .sort(naturalSortMdFiles);
+  }
+}
+
+export async function getRevisions(
+  sectionId: string
+): Promise<RevisionYmlEntry | undefined> {
+  const revisionsYml = await readPublicFile(`docs/revisions.yml`);
+  return (yaml.load(revisionsYml) as Record<string, RevisionYmlEntry>)[
+    sectionId
+  ];
+}
+
 /**
  * public/docs/{lang}/{pageId}/ 以下のmdファイルを結合して MarkdownSection[] を返す。
  */
 export async function getMarkdownSections(
   lang: string,
   pageId: string
 ): Promise<MarkdownSection[]> {
-  const sectionsYml = await readPublicFile(
-    `docs/${lang}/${pageId}/sections.yml`
-  );
-  const files = yaml.load(sectionsYml) as string[];
+  const files = await getSectionsList(lang, pageId);
 
   const sections: MarkdownSection[] = [];
   for (const file of files) {
     const raw = await readPublicFile(`docs/${lang}/${pageId}/${file}`);
     if (file === "-intro.md") {
       // イントロセクションはフロントマターなし・見出しなし
       sections.push({
+        file,
         id: `${lang}-${pageId}-intro`,
         level: 1,
         title: "",
         rawContent: raw,
+        md5: crypto.createHash("md5").update(raw).digest("base64"),
       });
     } else {
       sections.push(parseFrontmatter(raw, file));
@@ -143,9 +194,39 @@ function parseFrontmatter(content: string, file: string): MarkdownSection {
     .slice(endIdx + 5)
     .replace(/-repl\s*\n/, `-repl:${fm?.id ?? ""}\n`);
   return {
+    file,
     id: fm?.id ?? "",
     title: fm?.title ?? "",
     level: fm?.level ?? 2,
     rawContent,
+    md5: crypto.createHash("md5").update(rawContent).digest("base64"),
   };
 }
+
+export async function getRevisionOfMarkdownSection(
+  sectionId: string,
+  md5: string
+): Promise<MarkdownSection> {
+  const revisions = await getRevisions(sectionId);
+  const targetRevision = revisions?.rev.find((r) => r.md5 === md5);
+  if (targetRevision) {
+    const rawRes = await fetch(
+      `https://raw.githubusercontent.com/ut-code/my-code/${targetRevision.git}/${targetRevision.path}`
+    );
+    if (rawRes.ok) {
+      const raw = await rawRes.text();
+      return parseFrontmatter(
+        raw,
+        `${targetRevision.git}/${targetRevision.path}`
+      );
+    } else {
+      throw new Error(
+        `Failed to fetch ${targetRevision.git}/${targetRevision.path}. ${rawRes.status}: ${await rawRes.text()}`
+      );
+    }
+  } else {
+    throw new Error(
+      `Revision for sectionId=${sectionId}, md5=${md5} not found`
+    );
+  }
+}

package.json

@@ -7,14 +7,13 @@
     "packages/*"
   ],
   "scripts": {
-    "dev": "npm run cf-typegen && npm run generateLanguages && npm run generateSections && npm run copyAllDTSFiles && npm run removeHinting && next dev",
-    "build": "npm run cf-typegen && npm run generateLanguages && npm run generateSections && npm run copyAllDTSFiles && npm run removeHinting && next build",
+    "dev": "npm run cf-typegen && npm run generateDocsMeta && npm run copyAllDTSFiles && npm run removeHinting && next dev",
+    "build": "npm run cf-typegen && npm run generateDocsMeta && npm run copyAllDTSFiles && npm run removeHinting && next build",
     "start": "next start",
     "lint": "npm run cf-typegen && next lint",
     "tsc": "npm run cf-typegen && tsc",
     "format": "prettier --write app/",
-    "generateLanguages": "tsx ./scripts/generateLanguagesList.ts",
-    "generateSections": "tsx ./scripts/generateSectionsList.ts",
+    "generateDocsMeta": "tsx ./scripts/generateDocsMeta.ts",
     "copyAllDTSFiles": "tsx ./scripts/copyAllDTSFiles.ts",
     "removeHinting": "tsx ./scripts/removeHinting.ts",
     "cf-preview": "opennextjs-cloudflare build && opennextjs-cloudflare preview --port 3000",

public/docs/ruby/6-classes/3-0-accessor.md

@@ -1,6 +1,6 @@
 ---
 id: ruby-classes-accessor
-title: '🔐 アクセサメソッド
+title: '🔐 アクセサメソッド'
 level: 2
 ---
 

scripts/generateDocsMeta.ts

@@ -0,0 +1,31 @@
+// Generates public/docs/{lang}/{pageId}/sections.yml for each page directory.
+// Each sections.yml lists the .md files in that directory in display order.
+
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { getPagesList, getSectionsList } from "@/lib/docs";
+
+const docsDir = join(process.cwd(), "public", "docs");
+
+const langEntries = await getPagesList();
+
+const langIdsJson = JSON.stringify(langEntries.map((lang) => lang.id));
+await writeFile(join(docsDir, "languages.json"), langIdsJson, "utf-8");
+console.log(
+  `Generated languages.json (${langEntries.length} languages: ${langEntries.map((lang) => lang.id).join(", ")})`
+);
+
+for (const lang of langEntries) {
+  for (const page of lang.pages) {
+    const files = await getSectionsList(lang.id, page.slug);
+    const filesJson = JSON.stringify(files);
+    await writeFile(
+      join(docsDir, lang.id, page.slug, "sections.json"),
+      filesJson,
+      "utf-8"
+    );
+    console.log(
+      `Generated ${lang.id}/${page.slug}/sections.json (${files.length} files)`
+    );
+  }
+}

scripts/generateLanguagesList.ts

@@ -0,0 +1,79 @@
+import { readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import {
+  getMarkdownSections,
+  getPagesList,
+  RevisionYmlEntry,
+} from "@/lib/docs";
+import yaml from "js-yaml";
+import { execFileSync } from "node:child_process";
+import { existsSync } from "node:fs";
+
+const docsDir = join(process.cwd(), "public", "docs");
+
+const commit = execFileSync("git", ["rev-parse", "--short", "HEAD"], {
+  encoding: "utf8",
+}).trim();
+
+const langEntries = await getPagesList();
+
+const revisionsPrevYml = existsSync(join(docsDir, "revisions.yml"))
+  ? await readFile(join(docsDir, "revisions.yml"), "utf-8")
+  : "{}";
+const revisions = yaml.load(revisionsPrevYml) as Record<
+  string,
+  RevisionYmlEntry
+>;
+
+for (const lang of langEntries) {
+  for (const page of lang.pages) {
+    const sections = await getMarkdownSections(lang.id, page.slug);
+    for (const section of sections) {
+      if (section.id in revisions) {
+        revisions[section.id].page = `${lang.id}/${page.slug}`;
+        if (!revisions[section.id].rev.some((r) => r.md5 === section.md5)) {
+          // ドキュメントが変更された場合
+          console.log(`${section.id} has new md5: ${section.md5}`);
+          revisions[section.id].rev.push({
+            md5: section.md5,
+            git: commit,
+            path: `public/docs/${lang.id}/${page.slug}/${section.file}`,
+          });
+        }
+      } else {
+        // ドキュメントが新規追加された場合
+        console.log(`${section.id} is new, adding to revisions.yml`);
+        revisions[section.id] = {
+          rev: [
+            {
+              md5: section.md5,
+              git: commit,
+              path: `public/docs/${lang.id}/${page.slug}/${section.file}`,
+            },
+          ],
+          page: `${lang.id}/${page.slug}`,
+        };
+      }
+    }
+  }
+}
+
+for (const id in revisions) {
+  if (!existsSync(join(docsDir, revisions[id].page))) {
+    throw new Error(
+      `The page slug ${revisions[id].page} previously used by section ${id} does not exist. ` +
+        `Please replace 'page: ${revisions[id].page}' in public/docs/revisions.yml with new page path manually.`
+    );
+  }
+}
+
+const revisionsYml = yaml.dump(revisions, {
+  sortKeys: true,
+  noArrayIndent: true,
+});
+await writeFile(
+  join(docsDir, "revisions.yml"),
+  "# This file will be updated by CI. Do not edit manually, unless CI failed.\n" +
+    revisionsYml,
+  "utf-8"
+);