mirror of
https://github.com/open-webui/docs.git
synced 2026-07-19 22:53:44 -04:00
Replace lunr search with local search and exact-phrase ranking
Swap docusaurus-lunr-search for @easyops-cn/docusaurus-search-local. All query words are now required to match instead of being OR-ed, so a page spamming one common word no longer outranks the page containing the whole query. Underscore-joined names like CONTEXT_COMPACTION_TOKEN_CAP are indexed both whole and as sub-words, matched terms are highlighted on the target page, and a full results page is available at /search. Add an exact-match layer on top (src/client/exactSearch.js, wired via webpack module replacement). Multi-word or quoted queries that occur verbatim on a page always rank first, rendered as a single marked phrase with a section-anchor deep link, with token-based results following. A SearchBar wrapper (src/theme/SearchBar) highlights exactly the searched sentence on the target page and scrolls it into view, replacing the per-word highlight parameters for these results. Also mention the Builtin Tools capability name on the tools page so the in-product term is findable by search.
This commit is contained in:
Classic298
@@ -251,7 +251,7 @@ Legacy Mode is **not** a supported workaround even for DeepSeek; it is unsupport
|
||||
|
||||
### Built-in System Tools (Native/Agentic Mode)
|
||||
|
||||
🛠️ When **Native Mode (Agentic Mode)** is enabled, Open WebUI automatically injects powerful system tools. This unlocks truly agentic behaviors where capable models (like GPT-5, Claude 4.5 Sonnet, Gemini 3 Flash, or MiniMax M2.5) can perform multi-step research, explore knowledge bases, or manage user memory autonomously.
|
||||
🛠️ These are the **Builtin Tools** (the capability name shown in the Model Editor). When **Native Mode (Agentic Mode)** is enabled, Open WebUI automatically injects powerful system tools. This unlocks truly agentic behaviors where capable models (like GPT-5, Claude 4.5 Sonnet, Gemini 3 Flash, or MiniMax M2.5) can perform multi-step research, explore knowledge bases, or manage user memory autonomously.
|
||||
|
||||
| Tool | Purpose |
|
||||
|------|---------|
|
||||
|
||||
+35
-1
@@ -1,3 +1,5 @@
|
||||
import path from "path";
|
||||
import webpack from "webpack";
|
||||
import { Config } from "@docusaurus/types";
|
||||
import type * as Preset from "@docusaurus/preset-classic";
|
||||
|
||||
@@ -50,8 +52,24 @@ const config: Config = {
|
||||
markdown: {
|
||||
mermaid: true,
|
||||
},
|
||||
<<<<<<< Updated upstream
|
||||
clientModules: ["./src/clientModules/ensure-gtag.js"],
|
||||
themes: ["@docusaurus/theme-mermaid"],
|
||||
=======
|
||||
themes: [
|
||||
"@docusaurus/theme-mermaid",
|
||||
[
|
||||
require.resolve("@easyops-cn/docusaurus-search-local"),
|
||||
{
|
||||
hashed: true,
|
||||
indexBlog: false,
|
||||
docsRouteBasePath: "/",
|
||||
highlightSearchTermsOnTargetPage: true,
|
||||
explicitSearchResultPath: true,
|
||||
},
|
||||
],
|
||||
],
|
||||
>>>>>>> Stashed changes
|
||||
|
||||
presets: [
|
||||
[
|
||||
@@ -198,7 +216,23 @@ const config: Config = {
|
||||
},
|
||||
},
|
||||
} satisfies Preset.ThemeConfig,
|
||||
plugins: [require.resolve("docusaurus-lunr-search")],
|
||||
|
||||
plugins: [
|
||||
// Rank verbatim phrase matches above token results (see src/client/exactSearch.js).
|
||||
() => ({
|
||||
name: "docs-exact-search",
|
||||
configureWebpack() {
|
||||
return {
|
||||
plugins: [
|
||||
new webpack.NormalModuleReplacementPlugin(
|
||||
/searchByWorker$/,
|
||||
path.resolve(__dirname, "src/client/exactSearch.js")
|
||||
),
|
||||
],
|
||||
};
|
||||
},
|
||||
}),
|
||||
],
|
||||
};
|
||||
|
||||
export default config;
|
||||
|
||||
Generated
+499
-744
File diff suppressed because it is too large
Load Diff
+1
-1
@@ -30,10 +30,10 @@
|
||||
"@docusaurus/plugin-google-gtag": "^3.7.0",
|
||||
"@docusaurus/preset-classic": "^3.5.2",
|
||||
"@docusaurus/theme-mermaid": "^3.5.2",
|
||||
"@easyops-cn/docusaurus-search-local": "^0.55.2",
|
||||
"@mdx-js/react": "^3.0.1",
|
||||
"@shikijs/rehype": "^4.0.2",
|
||||
"clsx": "^2.1.1",
|
||||
"docusaurus-lunr-search": "^3.6.0",
|
||||
"docusaurus-plugin-sass": "^0.2.5",
|
||||
"marked": "^17.0.1",
|
||||
"react": "^18.3.1",
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
// Drop-in replacement for @easyops-cn/docusaurus-search-local's theme/searchByWorker
|
||||
// (wired via NormalModuleReplacementPlugin in docusaurus.config.ts).
|
||||
// Adds an exact-substring layer over search-corpus.json: when a multi-word or
|
||||
// quoted query occurs verbatim in a page, those hits rank first; token-based
|
||||
// results from the original worker follow.
|
||||
import * as Comlink from "comlink";
|
||||
import Slugger from "github-slugger";
|
||||
|
||||
let remoteWorkerPromise;
|
||||
function getRemoteWorker() {
|
||||
if (process.env.NODE_ENV === "production" && !remoteWorkerPromise) {
|
||||
remoteWorkerPromise = (async () => {
|
||||
const Remote = Comlink.wrap(
|
||||
new Worker(
|
||||
new URL(
|
||||
"../../node_modules/@easyops-cn/docusaurus-search-local/dist/client/client/theme/worker.js",
|
||||
import.meta.url
|
||||
)
|
||||
)
|
||||
);
|
||||
return await new Remote();
|
||||
})();
|
||||
}
|
||||
return remoteWorkerPromise;
|
||||
}
|
||||
|
||||
export async function fetchIndexesByWorker(baseUrl, searchContext) {
|
||||
if (process.env.NODE_ENV === "production") {
|
||||
const remoteWorker = await getRemoteWorker();
|
||||
await remoteWorker.fetchIndexes(baseUrl, searchContext);
|
||||
}
|
||||
}
|
||||
|
||||
let corpusPromise;
|
||||
function fetchCorpus(baseUrl) {
|
||||
if (!corpusPromise) {
|
||||
corpusPromise = fetch(`${baseUrl}search-corpus.json`)
|
||||
.then((res) => (res.ok ? res.json() : { docs: [] }))
|
||||
.then((json) => (json.docs || []).map(prepareDoc))
|
||||
.catch(() => []);
|
||||
}
|
||||
return corpusPromise;
|
||||
}
|
||||
|
||||
export function prepareDoc(doc) {
|
||||
// Strip link targets and emphasis markers (outside code fences) so matching
|
||||
// approximates rendered text; keep fence lines so the heading scan can skip blocks.
|
||||
// Collapse spaces/tabs but keep newlines (text and matchable stay offset-aligned).
|
||||
let inFence = false;
|
||||
const lines = (doc.content || "")
|
||||
.replace(/\r\n?/g, "\n")
|
||||
.split("\n")
|
||||
.map((line) => {
|
||||
if (/^ {0,3}(```|~~~)/.test(line)) {
|
||||
inFence = !inFence;
|
||||
return line;
|
||||
}
|
||||
if (inFence) {
|
||||
return line;
|
||||
}
|
||||
return line.replace(/\[([^\]]*)\]\([^)]*\)/g, "$1").replace(/[*`]/g, "");
|
||||
});
|
||||
const text = lines.join("\n").replace(/[^\S\n]+/g, " ");
|
||||
return {
|
||||
title: doc.title || "Untitled",
|
||||
path: doc.path || "/",
|
||||
text,
|
||||
matchable: text.replace(/\n/g, " ").toLowerCase(),
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeQuery(input) {
|
||||
let query = input.trim();
|
||||
const quoted = query.length > 2 && query.startsWith('"') && query.endsWith('"');
|
||||
if (quoted) {
|
||||
query = query.slice(1, -1);
|
||||
}
|
||||
query = query.replace(/\s+/g, " ").trim();
|
||||
return { query, eligible: query.length >= 3 && (quoted || /\s/.test(query)) };
|
||||
}
|
||||
|
||||
function nearestHeadingAnchor(text, matchIndex) {
|
||||
const slugger = new Slugger();
|
||||
const lineRegex = /^ {0,3}(?:(```|~~~)[^\r\n]*|(#{1,6}) +(.+?) *)$/gm;
|
||||
let inFence = false;
|
||||
let anchor;
|
||||
let heading;
|
||||
let match;
|
||||
while ((match = lineRegex.exec(text)) !== null) {
|
||||
if (match[1]) {
|
||||
inFence = !inFence;
|
||||
continue;
|
||||
}
|
||||
if (inFence) {
|
||||
continue;
|
||||
}
|
||||
if (match.index > matchIndex) {
|
||||
break;
|
||||
}
|
||||
const explicit = match[3].match(/^(.*?) *\{#([^}]+)\} *$/);
|
||||
heading = (explicit ? explicit[1] : match[3]).trim();
|
||||
anchor = explicit ? explicit[2] : slugger.slug(heading);
|
||||
}
|
||||
return anchor ? { anchor: `#${anchor}`, heading } : {};
|
||||
}
|
||||
|
||||
function makeSnippet(text, index, length) {
|
||||
let start = Math.max(0, index - 40);
|
||||
let end = Math.min(text.length, index + length + 90);
|
||||
if (start > 0) {
|
||||
const ws = text.indexOf(" ", start);
|
||||
if (ws !== -1 && ws < index) start = ws + 1;
|
||||
}
|
||||
if (end < text.length) {
|
||||
const ws = text.lastIndexOf(" ", end);
|
||||
if (ws > index + length) end = ws;
|
||||
}
|
||||
const prefix = start > 0 ? "… " : "";
|
||||
const suffix = end < text.length ? " …" : "";
|
||||
const body = text.slice(start, end).replace(/\n/g, " ");
|
||||
return { snippet: prefix + body + suffix, matchStart: prefix.length + (index - start) };
|
||||
}
|
||||
|
||||
export function searchExact(docs, query, limit) {
|
||||
const q = query.toLowerCase();
|
||||
const hits = [];
|
||||
for (const doc of docs) {
|
||||
const firstIndex = doc.matchable.indexOf(q);
|
||||
if (firstIndex === -1) {
|
||||
continue;
|
||||
}
|
||||
let count = 0;
|
||||
for (let i = firstIndex; i !== -1 && count < 50; i = doc.matchable.indexOf(q, i + q.length)) {
|
||||
count += 1;
|
||||
}
|
||||
const titleHit = doc.title.toLowerCase().includes(q);
|
||||
hits.push({ doc, firstIndex, score: count + (titleHit ? 100 : 0) });
|
||||
}
|
||||
hits.sort((a, b) => b.score - a.score || a.doc.path.length - b.doc.path.length);
|
||||
|
||||
return hits.slice(0, limit).map(({ doc, firstIndex }) => {
|
||||
const { anchor, heading } = nearestHeadingAnchor(doc.text, firstIndex);
|
||||
const { snippet, matchStart } = makeSnippet(doc.text, firstIndex, q.length);
|
||||
// Empty tokens: the dropdown then marks the whole matched range as one
|
||||
// <mark>, and the plugin skips its per-word _highlight params. The phrase
|
||||
// travels in our own param, marked on the target page by the SearchBar wrapper.
|
||||
return {
|
||||
document: {
|
||||
i: `exact:${doc.path}`,
|
||||
t: snippet,
|
||||
s: heading || doc.title,
|
||||
u: `${doc.path}?_highlight_phrase=${encodeURIComponent(query)}`,
|
||||
h: anchor,
|
||||
b: [],
|
||||
},
|
||||
type: 4, // SearchDocumentType.Content
|
||||
page: { i: `exact-page:${doc.path}`, t: doc.title, u: doc.path, b: [] },
|
||||
metadata: { [q]: { t: { position: [[matchStart, q.length]] } } },
|
||||
tokens: [],
|
||||
score: 0,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export async function searchByWorker(baseUrl, searchContext, input, limit) {
|
||||
if (process.env.NODE_ENV !== "production") {
|
||||
return [];
|
||||
}
|
||||
const { query, eligible } = normalizeQuery(input);
|
||||
const tokenPromise = getRemoteWorker().then((worker) =>
|
||||
worker.search(baseUrl, searchContext, input, limit)
|
||||
);
|
||||
if (!eligible) {
|
||||
return tokenPromise;
|
||||
}
|
||||
const [exact, tokenResults] = await Promise.all([
|
||||
fetchCorpus(baseUrl).then((docs) => searchExact(docs, query, limit)),
|
||||
tokenPromise,
|
||||
]);
|
||||
if (!exact.length) {
|
||||
return tokenResults;
|
||||
}
|
||||
const routeKey = (r) => r.document.u.split("?")[0] + (r.document.h || "");
|
||||
const seen = new Set(exact.map(routeKey));
|
||||
return exact.concat(tokenResults.filter((r) => !seen.has(routeKey(r)))).slice(0, limit);
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
// Wraps the search plugin's SearchBar. Exact-phrase results (src/client/exactSearch.js)
|
||||
// navigate with ?_highlight_phrase=<phrase>; this marks that phrase on the target page
|
||||
// as one unit and scrolls to it, instead of the plugin's per-word _highlight marking.
|
||||
import React, { useEffect } from "react";
|
||||
import { useLocation } from "@docusaurus/router";
|
||||
import Mark from "mark.js";
|
||||
import OriginalSearchBar from "@easyops-cn/docusaurus-search-local/dist/client/client/theme/SearchBar";
|
||||
|
||||
function escapeRegExp(text) {
|
||||
return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
|
||||
export default function SearchBar(props) {
|
||||
const location = useLocation();
|
||||
useEffect(() => {
|
||||
const raw = new URLSearchParams(location.search).get("_highlight_phrase");
|
||||
// The /search page appends its own query string after ours; keep our part.
|
||||
const phrase = raw ? raw.split("?")[0].trim() : "";
|
||||
if (!phrase) {
|
||||
return undefined;
|
||||
}
|
||||
// After the plugin's own mark effect (setTimeout 0) so its unmark() runs first.
|
||||
const timer = setTimeout(() => {
|
||||
const root = document.querySelector("article");
|
||||
if (!root) {
|
||||
return;
|
||||
}
|
||||
const pattern = new RegExp(
|
||||
phrase.split(/\s+/).map(escapeRegExp).join("[\\s\\u00A0]+"),
|
||||
"i"
|
||||
);
|
||||
new Mark(root).markRegExp(pattern, {
|
||||
acrossElements: true,
|
||||
done: () => {
|
||||
root.querySelector("mark")?.scrollIntoView({ block: "center" });
|
||||
},
|
||||
});
|
||||
}, 150);
|
||||
return () => clearTimeout(timer);
|
||||
}, [location.search, location.pathname]);
|
||||
return <OriginalSearchBar {...props} />;
|
||||
}
|
||||
Reference in New Issue
Block a user