mirror of
https://github.com/open-webui/docs.git
synced 2026-07-21 01:55:22 -04:00
refac
This commit is contained in:
Timothy Jaeryang Baek
@@ -6,6 +6,8 @@
|
||||
"docusaurus": "docusaurus",
|
||||
"start": "docusaurus start",
|
||||
"build": "docusaurus build",
|
||||
"postbuild": "node scripts/generate-raw-docs.mjs",
|
||||
"raw-docs": "node scripts/generate-raw-docs.mjs",
|
||||
"swizzle": "docusaurus swizzle",
|
||||
"deploy": "docusaurus deploy",
|
||||
"clear": "docusaurus clear",
|
||||
|
||||
@@ -0,0 +1,335 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import * as cheerio from "cheerio";
|
||||
|
||||
const DOCS_DIR = "docs";
|
||||
const BUILD_DIR = "build";
|
||||
const SITE_URL = "https://docs.openwebui.com";
|
||||
|
||||
function walk(dir) {
|
||||
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
return entries.flatMap((entry) => {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
return walk(fullPath);
|
||||
}
|
||||
return [fullPath];
|
||||
});
|
||||
}
|
||||
|
||||
function isDocPage(filePath) {
|
||||
if (!/\.(md|mdx)$/.test(filePath)) {
|
||||
return false;
|
||||
}
|
||||
if (filePath.includes(`${path.sep}tab-`)) {
|
||||
return false;
|
||||
}
|
||||
if (path.basename(filePath).startsWith("_template.")) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function parseFrontMatter(source) {
|
||||
if (!source.startsWith("---\n")) {
|
||||
return { frontMatter: {}, body: source };
|
||||
}
|
||||
|
||||
const end = source.indexOf("\n---", 4);
|
||||
if (end === -1) {
|
||||
return { frontMatter: {}, body: source };
|
||||
}
|
||||
|
||||
const raw = source.slice(4, end).trim();
|
||||
const body = source.slice(end + 4).replace(/^\s*\n/, "");
|
||||
const frontMatter = {};
|
||||
|
||||
for (const line of raw.split("\n")) {
|
||||
const match = line.match(/^([A-Za-z0-9_-]+):\s*(.*)$/);
|
||||
if (match) {
|
||||
frontMatter[match[1]] = match[2].replace(/^['"]|['"]$/g, "");
|
||||
}
|
||||
}
|
||||
|
||||
return { frontMatter, body };
|
||||
}
|
||||
|
||||
function routeForDoc(filePath) {
|
||||
const source = fs.readFileSync(filePath, "utf8");
|
||||
const { frontMatter } = parseFrontMatter(source);
|
||||
|
||||
if (frontMatter.slug) {
|
||||
return normalizeRoute(frontMatter.slug);
|
||||
}
|
||||
|
||||
let route = filePath
|
||||
.replace(
|
||||
new RegExp(
|
||||
`^${DOCS_DIR.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}${path.sep}`
|
||||
),
|
||||
""
|
||||
)
|
||||
.replace(/\.(md|mdx)$/, "")
|
||||
.split(path.sep)
|
||||
.join("/");
|
||||
|
||||
if (route.endsWith("/index")) {
|
||||
route = route.slice(0, -"/index".length);
|
||||
}
|
||||
|
||||
return normalizeRoute(`/${route}`);
|
||||
}
|
||||
|
||||
function normalizeRoute(route) {
|
||||
if (!route || route === "/") {
|
||||
return "/";
|
||||
}
|
||||
return `/${route.replace(/^\/+|\/+$/g, "")}`;
|
||||
}
|
||||
|
||||
function routeToHtmlPath(route) {
|
||||
if (route === "/") {
|
||||
return path.join(BUILD_DIR, "index.html");
|
||||
}
|
||||
return path.join(BUILD_DIR, route.slice(1), "index.html");
|
||||
}
|
||||
|
||||
function routeToRawPath(route, extension) {
|
||||
if (route === "/") {
|
||||
return path.join(BUILD_DIR, `index.${extension}`);
|
||||
}
|
||||
return path.join(BUILD_DIR, `${route.slice(1)}.${extension}`);
|
||||
}
|
||||
|
||||
function routeToRawHref(route, extension) {
|
||||
if (route === "/") {
|
||||
return `/index.${extension}`;
|
||||
}
|
||||
return `${route}.${extension}`;
|
||||
}
|
||||
|
||||
function htmlUrl(route) {
|
||||
return `${SITE_URL}${route === "/" ? "/" : route}`;
|
||||
}
|
||||
|
||||
function textOf($, node) {
|
||||
return $(node).text().replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function tableToMarkdown($, table) {
|
||||
const rows = [];
|
||||
$(table)
|
||||
.find("tr")
|
||||
.each((_, tr) => {
|
||||
const cells = $(tr)
|
||||
.children("th,td")
|
||||
.map((__, cell) => textOf($, cell).replace(/\|/g, "\\|"))
|
||||
.get();
|
||||
if (cells.length) {
|
||||
rows.push(cells);
|
||||
}
|
||||
});
|
||||
|
||||
if (!rows.length) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const width = Math.max(...rows.map((row) => row.length));
|
||||
const normalized = rows.map((row) => [
|
||||
...row,
|
||||
...Array.from({ length: width - row.length }, () => ""),
|
||||
]);
|
||||
const header = normalized[0];
|
||||
const separator = Array.from({ length: width }, () => "---");
|
||||
const body = normalized.slice(1);
|
||||
|
||||
return [header, separator, ...body]
|
||||
.map((row) => `| ${row.join(" | ")} |`)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function convertChildren($, node) {
|
||||
return $(node)
|
||||
.contents()
|
||||
.map((_, child) => convertNode($, child))
|
||||
.get()
|
||||
.join("");
|
||||
}
|
||||
|
||||
function convertNode($, node) {
|
||||
if (node.type === "text") {
|
||||
return node.data.replace(/\s+/g, " ");
|
||||
}
|
||||
|
||||
if (node.type !== "tag") {
|
||||
return "";
|
||||
}
|
||||
|
||||
const tag = node.name.toLowerCase();
|
||||
const className = $(node).attr("class") ?? "";
|
||||
const role = $(node).attr("role") ?? "";
|
||||
const inner = convertChildren($, node).trim();
|
||||
|
||||
if (className.includes("hash-link") || role === "tablist") {
|
||||
return "";
|
||||
}
|
||||
if (tag === "ul" && className.split(/\s+/).includes("tabs")) {
|
||||
return "";
|
||||
}
|
||||
if (className.includes("theme-admonition")) {
|
||||
const title = $(node).find(".admonitionHeading_Gvgb").first().text().trim();
|
||||
const content = $(node).find(".admonitionContent_BuS1").first();
|
||||
const body = content.length ? convertChildren($, content).trim() : inner;
|
||||
return `\n\n> **${title}**\n>\n${body
|
||||
.split("\n")
|
||||
.map((line) => `> ${line}`)
|
||||
.join("\n")}\n\n`;
|
||||
}
|
||||
|
||||
if (/^h[1-6]$/.test(tag)) {
|
||||
return `\n\n${"#".repeat(Number(tag[1]))} ${inner}\n\n`;
|
||||
}
|
||||
if (tag === "p") {
|
||||
return `\n\n${inner}\n\n`;
|
||||
}
|
||||
if (tag === "br") {
|
||||
return "\n";
|
||||
}
|
||||
if (tag === "strong" || tag === "b") {
|
||||
return `**${inner}**`;
|
||||
}
|
||||
if (tag === "em" || tag === "i") {
|
||||
return `_${inner}_`;
|
||||
}
|
||||
if (tag === "code") {
|
||||
if ($(node).parents("pre").length) {
|
||||
return $(node).text();
|
||||
}
|
||||
return `\`${$(node).text().replace(/`/g, "\\`")}\``;
|
||||
}
|
||||
if (tag === "pre") {
|
||||
const code = $(node).text().replace(/\n+$/, "");
|
||||
const language =
|
||||
$(node)
|
||||
.find("code")
|
||||
.attr("class")
|
||||
?.match(/language-([^\s]+)/)?.[1] ?? "";
|
||||
return `\n\n\`\`\`${language}\n${code}\n\`\`\`\n\n`;
|
||||
}
|
||||
if (tag === "a") {
|
||||
const href = $(node).attr("href");
|
||||
return href ? `[${inner || href}](${href})` : inner;
|
||||
}
|
||||
if (tag === "img") {
|
||||
const src = $(node).attr("src");
|
||||
const alt = $(node).attr("alt") ?? "";
|
||||
return src ? `` : "";
|
||||
}
|
||||
if (tag === "ul" || tag === "ol") {
|
||||
return `\n${$(node)
|
||||
.children("li")
|
||||
.map((index, li) => {
|
||||
const marker = tag === "ol" ? `${index + 1}.` : "-";
|
||||
return `${marker} ${convertChildren($, li).trim()}`;
|
||||
})
|
||||
.get()
|
||||
.join("\n")}\n\n`;
|
||||
}
|
||||
if (tag === "blockquote") {
|
||||
return `\n\n${inner
|
||||
.split("\n")
|
||||
.map((line) => `> ${line}`)
|
||||
.join("\n")}\n\n`;
|
||||
}
|
||||
if (tag === "table") {
|
||||
return `\n\n${tableToMarkdown($, node)}\n\n`;
|
||||
}
|
||||
if (tag === "hr") {
|
||||
return "\n\n---\n\n";
|
||||
}
|
||||
if (tag === "svg" || tag === "button") {
|
||||
return "";
|
||||
}
|
||||
if (tag === "div") {
|
||||
return inner ? `\n\n${inner}\n\n` : "";
|
||||
}
|
||||
|
||||
return inner;
|
||||
}
|
||||
|
||||
function compactMarkdown(markdown) {
|
||||
return markdown
|
||||
.replace(/[ \t]+\n/g, "\n")
|
||||
.replace(/^[ \t]+(?=```)/gm, "")
|
||||
.replace(/^[ \t]+(?=#{1,6} )/gm, "")
|
||||
.replace(/\n{3,}/g, "\n\n")
|
||||
.replace(/^\s+|\s+$/g, "")
|
||||
.concat("\n");
|
||||
}
|
||||
|
||||
function markdownFromHtml(htmlPath, route) {
|
||||
const $ = cheerio.load(fs.readFileSync(htmlPath, "utf8"));
|
||||
const content = $(".theme-doc-markdown.markdown").first();
|
||||
|
||||
if (!content.length) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const markdown = compactMarkdown(convertChildren($, content));
|
||||
return markdown.replace(/^(# .+\n)/, `$1\nSource: ${htmlUrl(route)}\n`);
|
||||
}
|
||||
|
||||
function injectAlternateLinks(htmlPath, route) {
|
||||
const html = fs.readFileSync(htmlPath, "utf8");
|
||||
const mdHref = routeToRawHref(route, "md");
|
||||
const txtHref = routeToRawHref(route, "txt");
|
||||
const links = [
|
||||
`<link rel="alternate" type="text/markdown" href="${mdHref}">`,
|
||||
`<link rel="alternate" type="text/plain" href="${txtHref}">`,
|
||||
].join("");
|
||||
|
||||
if (html.includes(`href="${mdHref}"`) && html.includes(`href="${txtHref}"`)) {
|
||||
return;
|
||||
}
|
||||
|
||||
fs.writeFileSync(htmlPath, html.replace("</head>", `${links}</head>`));
|
||||
}
|
||||
|
||||
function writeRawFiles(route, markdown) {
|
||||
for (const extension of ["md", "txt"]) {
|
||||
const outputPath = routeToRawPath(route, extension);
|
||||
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
||||
fs.writeFileSync(outputPath, markdown);
|
||||
}
|
||||
}
|
||||
|
||||
if (!fs.existsSync(BUILD_DIR)) {
|
||||
throw new Error("Run `npm run build` before generating raw docs.");
|
||||
}
|
||||
|
||||
let count = 0;
|
||||
const routes = new Set();
|
||||
|
||||
for (const filePath of walk(DOCS_DIR).filter(isDocPage)) {
|
||||
const route = routeForDoc(filePath);
|
||||
if (routes.has(route)) {
|
||||
continue;
|
||||
}
|
||||
routes.add(route);
|
||||
|
||||
const htmlPath = routeToHtmlPath(route);
|
||||
if (!fs.existsSync(htmlPath)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const markdown = markdownFromHtml(htmlPath, route);
|
||||
if (!markdown) {
|
||||
continue;
|
||||
}
|
||||
|
||||
writeRawFiles(route, markdown);
|
||||
injectAlternateLinks(htmlPath, route);
|
||||
count += 1;
|
||||
}
|
||||
|
||||
console.log(`Generated raw Markdown/Text for ${count} docs pages.`);
|
||||
Reference in New Issue
Block a user