From 5c51ae32015ddd9ecaa3018f786e705dce752f7a Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Thu, 2 Jul 2026 16:06:24 -0500 Subject: [PATCH] refac --- .github/workflows/ci.yml | 3 + package.json | 4 +- scripts/generate-search-corpus.mjs | 68 +++++++++++++++++++ scripts/test-docs-search.mjs | 27 ++++++++ workers/docs-search/README.md | 17 +++++ workers/docs-search/package.json | 7 ++ workers/docs-search/src/index.mjs | 88 +++++++++++++++++++++++++ workers/docs-search/src/search.mjs | 101 +++++++++++++++++++++++++++++ workers/docs-search/wrangler.toml | 10 +++ 9 files changed, 324 insertions(+), 1 deletion(-) create mode 100644 scripts/generate-search-corpus.mjs create mode 100644 scripts/test-docs-search.mjs create mode 100644 workers/docs-search/README.md create mode 100644 workers/docs-search/package.json create mode 100644 workers/docs-search/src/index.mjs create mode 100644 workers/docs-search/src/search.mjs create mode 100644 workers/docs-search/wrangler.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 069a0764..0cad2d6c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,3 +32,6 @@ jobs: - name: Test build run: npm run build + + - name: Test docs search + run: npm run test:docs-search diff --git a/package.json b/package.json index 9146b53c..4d99fb3c 100644 --- a/package.json +++ b/package.json @@ -6,8 +6,10 @@ "docusaurus": "docusaurus", "start": "docusaurus start", "build": "docusaurus build", - "postbuild": "node scripts/generate-raw-docs.mjs", + "postbuild": "node scripts/generate-raw-docs.mjs && node scripts/generate-search-corpus.mjs", "raw-docs": "node scripts/generate-raw-docs.mjs", + "search-corpus": "node scripts/generate-search-corpus.mjs", + "test:docs-search": "node scripts/test-docs-search.mjs", "swizzle": "docusaurus swizzle", "deploy": "docusaurus deploy", "clear": "docusaurus clear", diff --git a/scripts/generate-search-corpus.mjs b/scripts/generate-search-corpus.mjs new file mode 100644 index 00000000..2c2e51c8 --- /dev/null +++ b/scripts/generate-search-corpus.mjs @@ -0,0 +1,68 @@ +import fs from "node:fs"; +import path from "node:path"; + +const BUILD_DIR = "build"; +const SITE_URL = "https://docs.openwebui.com"; +const OUTPUT_PATH = path.join(BUILD_DIR, "search-corpus.json"); + +function walk(dir) { + return fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + const fullPath = path.join(dir, entry.name); + return entry.isDirectory() ? walk(fullPath) : [fullPath]; + }); +} + +function isRawDoc(filePath) { + if (!filePath.endsWith(".md")) { + return false; + } + return !filePath.endsWith("README.md"); +} + +function routeFromFile(filePath) { + const relative = path.relative(BUILD_DIR, filePath).split(path.sep).join("/"); + if (relative === "index.md") { + return "/"; + } + return `/${relative.replace(/\.md$/, "")}`; +} + +function titleFromMarkdown(markdown) { + return markdown.match(/^#\s+(.+)$/m)?.[1]?.trim() || "Untitled"; +} + +function markdownUrl(route) { + return route === `${SITE_URL}/` ? `${SITE_URL}/index.md` : `${route}.md`; +} + +function textUrl(route) { + return route === `${SITE_URL}/` ? `${SITE_URL}/index.txt` : `${route}.txt`; +} + +if (!fs.existsSync(BUILD_DIR)) { + throw new Error("Run `npm run build` before generating the search corpus."); +} + +const docs = walk(BUILD_DIR) + .filter(isRawDoc) + .map((filePath) => { + const content = fs.readFileSync(filePath, "utf8"); + const route = routeFromFile(filePath); + const url = `${SITE_URL}${route === "/" ? "/" : route}`; + + return { + title: titleFromMarkdown(content), + path: route, + url, + markdown_url: markdownUrl(url), + text_url: textUrl(url), + content, + }; + }) + .sort((a, b) => a.path.localeCompare(b.path)); + +fs.writeFileSync( + OUTPUT_PATH, + `${JSON.stringify({ generated_at: new Date().toISOString(), docs })}\n` +); +console.log(`Generated search corpus for ${docs.length} docs pages.`); diff --git a/scripts/test-docs-search.mjs b/scripts/test-docs-search.mjs new file mode 100644 index 00000000..a840853e --- /dev/null +++ b/scripts/test-docs-search.mjs @@ -0,0 +1,27 @@ +import fs from "node:fs"; +import { searchDocs } from "../workers/docs-search/src/search.mjs"; + +const corpus = JSON.parse(fs.readFileSync("build/search-corpus.json", "utf8")); + +const cases = [ + ["notion mcp", "mcp-notion"], + ["sso keycloak", "sso"], + ["rebrand logo", "brand"], + ["mcp timeout", "mcp"], +]; + +for (const [query, expectedPathPart] of cases) { + const { results } = searchDocs(corpus, query, 8); + if (!results.length) { + throw new Error(`No results for ${query}`); + } + if (!results.some((result) => result.path.includes(expectedPathPart))) { + throw new Error( + `Expected ${query} to match ${expectedPathPart}; got ${results + .map((result) => result.path) + .join(", ")}` + ); + } +} + +console.log("Docs search smoke tests passed."); diff --git a/workers/docs-search/README.md b/workers/docs-search/README.md new file mode 100644 index 00000000..9d7e15fe --- /dev/null +++ b/workers/docs-search/README.md @@ -0,0 +1,17 @@ +# Open WebUI Docs Search Worker + +This Worker serves `https://docs.openwebui.com/search?q=...`. + +Deploy it from the Cloudflare dashboard using Workers Builds: + +1. Create or open the `open-webui-docs-search` Worker. +2. Connect this GitHub repository under **Settings > Builds**. +3. Set the Worker root directory to `workers/docs-search`. +4. Use the production branch `main`. +5. Use `npm install` as the build command. +6. Use `npx wrangler deploy` as the deploy command. +7. Save and deploy. + +No GitHub Cloudflare secrets are needed. The Worker fetches the static corpus +from `https://docs.openwebui.com/search-corpus.json`, which is generated by the +normal GitHub Pages docs build. diff --git a/workers/docs-search/package.json b/workers/docs-search/package.json new file mode 100644 index 00000000..21da920c --- /dev/null +++ b/workers/docs-search/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "type": "module", + "devDependencies": { + "wrangler": "^4.0.0" + } +} diff --git a/workers/docs-search/src/index.mjs b/workers/docs-search/src/index.mjs new file mode 100644 index 00000000..489cf3bb --- /dev/null +++ b/workers/docs-search/src/index.mjs @@ -0,0 +1,88 @@ +import { searchDocs } from "./search.mjs"; + +const DEFAULT_CORPUS_URL = "https://docs.openwebui.com/search-corpus.json"; +let cachedCorpus; + +function json(data, init = {}) { + return new Response(JSON.stringify(data, null, 2), { + ...init, + headers: { + "content-type": "application/json; charset=utf-8", + "access-control-allow-origin": "*", + "cache-control": "public, max-age=60", + ...(init.headers ?? {}), + }, + }); +} + +async function loadCorpus(env) { + if (cachedCorpus) { + return cachedCorpus; + } + + const corpusUrl = env.SEARCH_CORPUS_URL || DEFAULT_CORPUS_URL; + const response = await fetch(corpusUrl, { + headers: { accept: "application/json" }, + cf: { cacheTtl: 300, cacheEverything: true }, + }); + + if (!response.ok) { + throw new Error(`Failed to fetch search corpus: HTTP ${response.status}`); + } + + cachedCorpus = await response.json(); + return cachedCorpus; +} + +export default { + async fetch(request, env) { + const url = new URL(request.url); + + if (request.method === "OPTIONS") { + return new Response(null, { + headers: { + "access-control-allow-origin": "*", + "access-control-allow-methods": "GET, OPTIONS", + "access-control-allow-headers": "content-type", + }, + }); + } + + if (request.method !== "GET") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const query = (url.searchParams.get("q") || "").trim(); + const limit = Math.min( + 20, + Math.max(1, Number.parseInt(url.searchParams.get("limit") || "8", 10)) + ); + + if (!query) { + return json( + { + error: "Missing query", + example: "https://docs.openwebui.com/search?q=notion%20mcp", + }, + { status: 400 } + ); + } + + if (query.length > 200) { + return json({ error: "Query is too long" }, { status: 400 }); + } + + try { + const corpus = await loadCorpus(env); + const { keywords, results } = searchDocs(corpus, query, limit); + return json({ + query, + keywords, + count: results.length, + results, + }); + } catch (error) { + return json({ error: error.message }, { status: 502 }); + } + }, +}; diff --git a/workers/docs-search/src/search.mjs b/workers/docs-search/src/search.mjs new file mode 100644 index 00000000..79aad23c --- /dev/null +++ b/workers/docs-search/src/search.mjs @@ -0,0 +1,101 @@ +export function keywordsFor(query) { + return Array.from( + new Set( + query + .toLowerCase() + .split(/[^a-z0-9._/-]+/) + .map((keyword) => keyword.trim()) + .filter(Boolean) + ) + ); +} + +function distinctHits(text, keywords) { + const lower = text.toLowerCase(); + return keywords.filter((keyword) => lower.includes(keyword)); +} + +function rankMatches(matches) { + return matches.sort((a, b) => { + if (b.score !== a.score) { + return b.score - a.score; + } + return a.path.length - b.path.length; + }); +} + +function makeSnippet(content, keywords) { + const lower = content.toLowerCase(); + const index = keywords.reduce((best, keyword) => { + const found = lower.indexOf(keyword); + if (found === -1) { + return best; + } + return best === -1 ? found : Math.min(best, found); + }, -1); + + if (index === -1) { + return content.slice(0, 240).replace(/\s+/g, " ").trim(); + } + + const start = Math.max(0, index - 120); + const end = Math.min(content.length, index + 240); + const prefix = start > 0 ? "..." : ""; + const suffix = end < content.length ? "..." : ""; + return `${prefix}${content.slice(start, end).replace(/\s+/g, " ").trim()}${suffix}`; +} + +function toResult(doc, keywords, hits, match) { + return { + title: doc.title, + path: doc.path, + url: doc.url, + markdown_url: doc.markdown_url, + text_url: doc.text_url, + match, + score: hits.length, + matched_keywords: hits, + snippet: makeSnippet(doc.content, hits.length ? hits : keywords), + }; +} + +export function searchDocs(corpus, query, limit = 8) { + const keywords = keywordsFor(query); + if (!keywords.length) { + return { keywords, results: [] }; + } + + const docs = Array.isArray(corpus?.docs) ? corpus.docs : []; + const pathMatches = []; + const contentMatches = []; + + for (const doc of docs) { + const pathHits = distinctHits(doc.path, keywords); + if (pathHits.length) { + pathMatches.push(toResult(doc, keywords, pathHits, "path")); + } + + const contentHits = distinctHits(doc.content, keywords); + if (contentHits.length) { + contentMatches.push(toResult(doc, keywords, contentHits, "content")); + } + } + + const seen = new Set(); + const results = []; + for (const result of [ + ...rankMatches(pathMatches), + ...rankMatches(contentMatches), + ]) { + if (seen.has(result.path)) { + continue; + } + seen.add(result.path); + results.push(result); + if (results.length >= limit) { + break; + } + } + + return { keywords, results }; +} diff --git a/workers/docs-search/wrangler.toml b/workers/docs-search/wrangler.toml new file mode 100644 index 00000000..f1b6b42a --- /dev/null +++ b/workers/docs-search/wrangler.toml @@ -0,0 +1,10 @@ +name = "open-webui-docs-search" +main = "src/index.mjs" +compatibility_date = "2026-07-02" + +routes = [ + { pattern = "docs.openwebui.com/search", zone_name = "openwebui.com" } +] + +[vars] +SEARCH_CORPUS_URL = "https://docs.openwebui.com/search-corpus.json"