Skip to content

Commit 5c51ae3

Browse files
tjbcktjbck
committed
refac
1 parent 7b4c569 commit 5c51ae3

9 files changed

Lines changed: 324 additions & 1 deletion

File tree

‎.github/workflows/ci.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,3 +32,6 @@ jobs:
3232

3333
- name: Test build
3434
run: npm run build
35+
36+
- name: Test docs search
37+
run: npm run test:docs-search

‎package.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@
66
"docusaurus": "docusaurus",
77
"start": "docusaurus start",
88
"build": "docusaurus build",
9-
"postbuild": "node scripts/generate-raw-docs.mjs",
9+
"postbuild": "node scripts/generate-raw-docs.mjs && node scripts/generate-search-corpus.mjs",
1010
"raw-docs": "node scripts/generate-raw-docs.mjs",
11+
"search-corpus": "node scripts/generate-search-corpus.mjs",
12+
"test:docs-search": "node scripts/test-docs-search.mjs",
1113
"swizzle": "docusaurus swizzle",
1214
"deploy": "docusaurus deploy",
1315
"clear": "docusaurus clear",

‎scripts/generate-search-corpus.mjs‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import fs from "node:fs";
2+
import path from "node:path";
3+
4+
const BUILD_DIR = "build";
5+
const SITE_URL = "https://docs.openwebui.com";
6+
const OUTPUT_PATH = path.join(BUILD_DIR, "search-corpus.json");
7+
8+
function walk(dir) {
9+
return fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
10+
const fullPath = path.join(dir, entry.name);
11+
return entry.isDirectory() ? walk(fullPath) : [fullPath];
12+
});
13+
}
14+
15+
function isRawDoc(filePath) {
16+
if (!filePath.endsWith(".md")) {
17+
return false;
18+
}
19+
return !filePath.endsWith("README.md");
20+
}
21+
22+
function routeFromFile(filePath) {
23+
const relative = path.relative(BUILD_DIR, filePath).split(path.sep).join("/");
24+
if (relative === "index.md") {
25+
return "/";
26+
}
27+
return `/${relative.replace(/\.md$/, "")}`;
28+
}
29+
30+
function titleFromMarkdown(markdown) {
31+
return markdown.match(/^#\s+(.+)$/m)?.[1]?.trim() || "Untitled";
32+
}
33+
34+
function markdownUrl(route) {
35+
return route === `${SITE_URL}/` ? `${SITE_URL}/index.md` : `${route}.md`;
36+
}
37+
38+
function textUrl(route) {
39+
return route === `${SITE_URL}/` ? `${SITE_URL}/index.txt` : `${route}.txt`;
40+
}
41+
42+
if (!fs.existsSync(BUILD_DIR)) {
43+
throw new Error("Run `npm run build` before generating the search corpus.");
44+
}
45+
46+
const docs = walk(BUILD_DIR)
47+
.filter(isRawDoc)
48+
.map((filePath) => {
49+
const content = fs.readFileSync(filePath, "utf8");
50+
const route = routeFromFile(filePath);
51+
const url = `${SITE_URL}${route === "/" ? "/" : route}`;
52+
53+
return {
54+
title: titleFromMarkdown(content),
55+
path: route,
56+
url,
57+
markdown_url: markdownUrl(url),
58+
text_url: textUrl(url),
59+
content,
60+
};
61+
})
62+
.sort((a, b) => a.path.localeCompare(b.path));
63+
64+
fs.writeFileSync(
65+
OUTPUT_PATH,
66+
`${JSON.stringify({ generated_at: new Date().toISOString(), docs })}\n`
67+
);
68+
console.log(`Generated search corpus for ${docs.length} docs pages.`);

‎scripts/test-docs-search.mjs‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
import fs from "node:fs";
2+
import { searchDocs } from "../workers/docs-search/src/search.mjs";
3+
4+
const corpus = JSON.parse(fs.readFileSync("build/search-corpus.json", "utf8"));
5+
6+
const cases = [
7+
["notion mcp", "mcp-notion"],
8+
["sso keycloak", "sso"],
9+
["rebrand logo", "brand"],
10+
["mcp timeout", "mcp"],
11+
];
12+
13+
for (const [query, expectedPathPart] of cases) {
14+
const { results } = searchDocs(corpus, query, 8);
15+
if (!results.length) {
16+
throw new Error(`No results for ${query}`);
17+
}
18+
if (!results.some((result) => result.path.includes(expectedPathPart))) {
19+
throw new Error(
20+
`Expected ${query} to match ${expectedPathPart}; got ${results
21+
.map((result) => result.path)
22+
.join(", ")}`
23+
);
24+
}
25+
}
26+
27+
console.log("Docs search smoke tests passed.");

‎workers/docs-search/README.md‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Open WebUI Docs Search Worker
2+
3+
This Worker serves `https://docs.openwebui.com/search?q=...`.
4+
5+
Deploy it from the Cloudflare dashboard using Workers Builds:
6+
7+
1. Create or open the `open-webui-docs-search` Worker.
8+
2. Connect this GitHub repository under **Settings > Builds**.
9+
3. Set the Worker root directory to `workers/docs-search`.
10+
4. Use the production branch `main`.
11+
5. Use `npm install` as the build command.
12+
6. Use `npx wrangler deploy` as the deploy command.
13+
7. Save and deploy.
14+
15+
No GitHub Cloudflare secrets are needed. The Worker fetches the static corpus
16+
from `https://docs.openwebui.com/search-corpus.json`, which is generated by the
17+
normal GitHub Pages docs build.

‎workers/docs-search/package.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"private": true,
3+
"type": "module",
4+
"devDependencies": {
5+
"wrangler": "^4.0.0"
6+
}
7+
}

‎workers/docs-search/src/index.mjs‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
import { searchDocs } from "./search.mjs";
2+
3+
const DEFAULT_CORPUS_URL = "https://docs.openwebui.com/search-corpus.json";
4+
let cachedCorpus;
5+
6+
function json(data, init = {}) {
7+
return new Response(JSON.stringify(data, null, 2), {
8+
...init,
9+
headers: {
10+
"content-type": "application/json; charset=utf-8",
11+
"access-control-allow-origin": "*",
12+
"cache-control": "public, max-age=60",
13+
...(init.headers ?? {}),
14+
},
15+
});
16+
}
17+
18+
async function loadCorpus(env) {
19+
if (cachedCorpus) {
20+
return cachedCorpus;
21+
}
22+
23+
const corpusUrl = env.SEARCH_CORPUS_URL || DEFAULT_CORPUS_URL;
24+
const response = await fetch(corpusUrl, {
25+
headers: { accept: "application/json" },
26+
cf: { cacheTtl: 300, cacheEverything: true },
27+
});
28+
29+
if (!response.ok) {
30+
throw new Error(`Failed to fetch search corpus: HTTP ${response.status}`);
31+
}
32+
33+
cachedCorpus = await response.json();
34+
return cachedCorpus;
35+
}
36+
37+
export default {
38+
async fetch(request, env) {
39+
const url = new URL(request.url);
40+
41+
if (request.method === "OPTIONS") {
42+
return new Response(null, {
43+
headers: {
44+
"access-control-allow-origin": "*",
45+
"access-control-allow-methods": "GET, OPTIONS",
46+
"access-control-allow-headers": "content-type",
47+
},
48+
});
49+
}
50+
51+
if (request.method !== "GET") {
52+
return json({ error: "Method not allowed" }, { status: 405 });
53+
}
54+
55+
const query = (url.searchParams.get("q") || "").trim();
56+
const limit = Math.min(
57+
20,
58+
Math.max(1, Number.parseInt(url.searchParams.get("limit") || "8", 10))
59+
);
60+
61+
if (!query) {
62+
return json(
63+
{
64+
error: "Missing query",
65+
example: "https://docs.openwebui.com/search?q=notion%20mcp",
66+
},
67+
{ status: 400 }
68+
);
69+
}
70+
71+
if (query.length > 200) {
72+
return json({ error: "Query is too long" }, { status: 400 });
73+
}
74+
75+
try {
76+
const corpus = await loadCorpus(env);
77+
const { keywords, results } = searchDocs(corpus, query, limit);
78+
return json({
79+
query,
80+
keywords,
81+
count: results.length,
82+
results,
83+
});
84+
} catch (error) {
85+
return json({ error: error.message }, { status: 502 });
86+
}
87+
},
88+
};

‎workers/docs-search/src/search.mjs‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
export function keywordsFor(query) {
2+
return Array.from(
3+
new Set(
4+
query
5+
.toLowerCase()
6+
.split(/[^a-z0-9._/-]+/)
7+
.map((keyword) => keyword.trim())
8+
.filter(Boolean)
9+
)
10+
);
11+
}
12+
13+
function distinctHits(text, keywords) {
14+
const lower = text.toLowerCase();
15+
return keywords.filter((keyword) => lower.includes(keyword));
16+
}
17+
18+
function rankMatches(matches) {
19+
return matches.sort((a, b) => {
20+
if (b.score !== a.score) {
21+
return b.score - a.score;
22+
}
23+
return a.path.length - b.path.length;
24+
});
25+
}
26+
27+
function makeSnippet(content, keywords) {
28+
const lower = content.toLowerCase();
29+
const index = keywords.reduce((best, keyword) => {
30+
const found = lower.indexOf(keyword);
31+
if (found === -1) {
32+
return best;
33+
}
34+
return best === -1 ? found : Math.min(best, found);
35+
}, -1);
36+
37+
if (index === -1) {
38+
return content.slice(0, 240).replace(/\s+/g, " ").trim();
39+
}
40+
41+
const start = Math.max(0, index - 120);
42+
const end = Math.min(content.length, index + 240);
43+
const prefix = start > 0 ? "..." : "";
44+
const suffix = end < content.length ? "..." : "";
45+
return `${prefix}${content.slice(start, end).replace(/\s+/g, " ").trim()}${suffix}`;
46+
}
47+
48+
function toResult(doc, keywords, hits, match) {
49+
return {
50+
title: doc.title,
51+
path: doc.path,
52+
url: doc.url,
53+
markdown_url: doc.markdown_url,
54+
text_url: doc.text_url,
55+
match,
56+
score: hits.length,
57+
matched_keywords: hits,
58+
snippet: makeSnippet(doc.content, hits.length ? hits : keywords),
59+
};
60+
}
61+
62+
export function searchDocs(corpus, query, limit = 8) {
63+
const keywords = keywordsFor(query);
64+
if (!keywords.length) {
65+
return { keywords, results: [] };
66+
}
67+
68+
const docs = Array.isArray(corpus?.docs) ? corpus.docs : [];
69+
const pathMatches = [];
70+
const contentMatches = [];
71+
72+
for (const doc of docs) {
73+
const pathHits = distinctHits(doc.path, keywords);
74+
if (pathHits.length) {
75+
pathMatches.push(toResult(doc, keywords, pathHits, "path"));
76+
}
77+
78+
const contentHits = distinctHits(doc.content, keywords);
79+
if (contentHits.length) {
80+
contentMatches.push(toResult(doc, keywords, contentHits, "content"));
81+
}
82+
}
83+
84+
const seen = new Set();
85+
const results = [];
86+
for (const result of [
87+
...rankMatches(pathMatches),
88+
...rankMatches(contentMatches),
89+
]) {
90+
if (seen.has(result.path)) {
91+
continue;
92+
}
93+
seen.add(result.path);
94+
results.push(result);
95+
if (results.length >= limit) {
96+
break;
97+
}
98+
}
99+
100+
return { keywords, results };
101+
}

‎workers/docs-search/wrangler.toml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
name = "open-webui-docs-search"
2+
main = "src/index.mjs"
3+
compatibility_date = "2026-07-02"
4+
5+
routes = [
6+
{ pattern = "docs.openwebui.com/search", zone_name = "openwebui.com" }
7+
]
8+
9+
[vars]
10+
SEARCH_CORPUS_URL = "https://docs.openwebui.com/search-corpus.json"

0 commit comments

Comments
 (0)