feat: improve indexing and embedded search (#2734)

Author: sy-records

src/plugins/search/search.js

@@ -4,6 +4,11 @@ import {
   removeAtag,
   escapeHtml,
 } from '../../core/render/utils.js';
+import {
+  getPath,
+  getParentPath,
+  isAbsolutePath,
+} from '../../core/router/util.js';
 import { markdownToTxt } from './markdown-to-txt.js';
 import Dexie from 'dexie';
 
@@ -16,10 +21,29 @@ db.version(1).stores({
 });
 
 async function saveData(maxAge, expireKey) {
-  INDEXES = Object.values(INDEXES).flatMap(innerData =>
-    Object.values(innerData),
-  );
-  await /** @type {any} */ (db).search.bulkPut(INDEXES);
+  const records = [];
+
+  Object.values(INDEXES).forEach(entry => {
+    if (!entry || typeof entry !== 'object') {
+      return;
+    }
+
+    // Entry may already be a flat record read from IndexedDB.
+    if ('slug' in entry) {
+      records.push(entry);
+      return;
+    }
+
+    // Entry may be a per-path map of slug -> record produced by genIndex().
+    Object.values(entry).forEach(item => {
+      if (item && typeof item === 'object' && 'slug' in item) {
+        records.push(item);
+      }
+    });
+  });
+
+  INDEXES = records;
+  await /** @type {any} */ (db).search.bulkPut(records);
   await /** @type {any} */ (db).expires.put({
     key: expireKey,
     value: Date.now() + maxAge,
@@ -96,6 +120,111 @@ function getListData(token) {
   return token.text;
 }
 
+function extractFragmentContent(text, fragment, fullLine) {
+  if (!fragment) {
+    return text;
+  }
+
+  let fragmentRegex = `(?:###|\\/\\/\\/)\\s*\\[${fragment}\\]`;
+  if (fullLine) {
+    fragmentRegex = `.*${fragmentRegex}.*\n`;
+  }
+
+  const pattern = new RegExp(
+    `(?:${fragmentRegex})([\\s\\S]*?)(?:${fragmentRegex})`,
+  );
+  const match = text.match(pattern);
+  return ((match || [])[1] || '').trim();
+}
+
+function collectEmbedRequests(raw = '', path, vm) {
+  const tokens = window.marked.lexer(raw);
+  const requests = [];
+
+  const maybePushEmbed = inlineToken => {
+    if (
+      !inlineToken ||
+      (inlineToken.type !== 'link' && inlineToken.type !== 'image')
+    ) {
+      return;
+    }
+
+    const { config } = getAndRemoveConfig(inlineToken.title || '');
+    if (!config.include || !inlineToken.href) {
+      return;
+    }
+
+    const href = isAbsolutePath(inlineToken.href)
+      ? inlineToken.href
+      : getPath(vm.router.getBasePath(), getParentPath(path), inlineToken.href);
+
+    let type = 'code';
+    if (/\.(md|markdown)/.test(href)) {
+      type = 'markdown';
+    } else if (/\.mmd/.test(href)) {
+      type = 'mermaid';
+    }
+
+    requests.push({
+      url: href,
+      type,
+      fragment: config.fragment,
+      omitFragmentLine: config.omitFragmentLine,
+    });
+  };
+
+  tokens.forEach(token => {
+    if (token.type === 'paragraph') {
+      (token.tokens || []).forEach(maybePushEmbed);
+    } else if (token.type === 'table') {
+      (token.header || []).forEach(cell => {
+        (cell.tokens || []).forEach(maybePushEmbed);
+      });
+      (token.rows || []).forEach(row => {
+        row.forEach(cell => {
+          (cell.tokens || []).forEach(maybePushEmbed);
+        });
+      });
+    }
+  });
+
+  return requests;
+}
+
+async function getEmbeddedContent(raw = '', path, vm) {
+  const requests = collectEmbedRequests(raw, path, vm);
+  if (!requests.length) {
+    return '';
+  }
+
+  const results = await Promise.all(
+    requests.map(
+      request =>
+        new Promise(resolve => {
+          Docsify.get(request.url, false, vm.config.requestHeaders).then(
+            text => {
+              let content = text || '';
+              if (request.fragment) {
+                content = extractFragmentContent(
+                  content,
+                  request.fragment,
+                  request.omitFragmentLine,
+                );
+              }
+
+              resolve(
+                request.type === 'markdown' ? content : markdownToTxt(content),
+              );
+            },
+            () => resolve(''),
+          );
+        }),
+    ),
+  );
+
+  return results.filter(Boolean).join('\n');
+}
+
 export function genIndex(path, content = '', router, depth, indexKey) {
   const tokens = window.marked.lexer(content);
   const slugify = window.Docsify.slugify;
@@ -205,29 +334,24 @@ export function search(query) {
           ),
           'gi',
         );
-        let indexTitle = -1;
-        let indexContent = -1;
         handlePostTitle = postTitle
           ? escapeHtml(ignoreDiacriticalMarks(postTitle))
           : postTitle;
         handlePostContent = postContent
           ? escapeHtml(ignoreDiacriticalMarks(postContent))
           : postContent;
 
-        indexTitle = postTitle ? handlePostTitle.search(regEx) : -1;
-        indexContent = postContent ? handlePostContent.search(regEx) : -1;
+        const indexTitle = postTitle ? handlePostTitle.search(regEx) : -1;
+        let indexContent = postContent ? handlePostContent.search(regEx) : -1;
 
         if (indexTitle >= 0 || indexContent >= 0) {
           matchesScore += indexTitle >= 0 ? 3 : indexContent >= 0 ? 2 : 0;
           if (indexContent < 0) {
             indexContent = 0;
           }
 
-          let start = 0;
-          let end = 0;
-
-          start = indexContent < 11 ? 0 : indexContent - 10;
-          end = start === 0 ? 100 : indexContent + keyword.length + 90;
+          const start = indexContent < 11 ? 0 : indexContent - 10;
+          let end = start === 0 ? 100 : indexContent + keyword.length + 90;
 
           if (handlePostContent && end > handlePostContent.length) {
             end = handlePostContent.length;
@@ -306,26 +430,38 @@ export async function init(config, vm) {
   const len = paths.length;
   let count = 0;
 
+  const markComplete = async () => {
+    if (len === ++count) {
+      await saveData(config.maxAge, expireKey);
+    }
+  };
+
   paths.forEach(path => {
     const pathExists = Array.isArray(INDEXES)
       ? INDEXES.some(obj => obj.path === path)
       : false;
     if (pathExists) {
-      return count++;
+      void markComplete();
+      return;
     }
 
     Docsify.get(vm.router.getFile(path), false, vm.config.requestHeaders).then(
       async result => {
+        const embeddedContent = await getEmbeddedContent(result, path, vm);
+        const contentToIndex = embeddedContent
+          ? `${result}\n${embeddedContent}`
+          : result;
         INDEXES[path] = genIndex(
           path,
-          result,
+          contentToIndex,
           vm.router,
           config.depth,
           indexKey,
         );
-        if (len === ++count) {
-          await saveData(config.maxAge, expireKey);
-        }
+        return markComplete();
+      },
+      () => {
+        return markComplete();
       },
     );
   });