feat: build api reference pages for new documentation site (#1551)

.changeset/sharp-ties-jog.md

@@ -0,0 +1,7 @@
+---
+"@llamaindex/doc": patch
+"@llamaindex/core": patch
+"@llamaindex/readers": patch
+---
+
+feat: build api reference pages for new documentation site

apps/next/.gitignore

@@ -29,4 +29,5 @@ next-env.d.ts
 
 # build
 /src/content/docs/cloud/api
+/src/content/docs/api
 ./types

apps/next/package.json

@@ -8,7 +8,7 @@
     "start": "next start",
     "postdev": "fumadocs-mdx",
     "postbuild": "fumadocs-mdx && tsx scripts/post-build.mts",
-    "build:docs": "node ./scripts/generate-docs.mjs"
+    "build:docs": "cross-env NODE_OPTIONS=\"--max-old-space-size=8192\" typedoc && node ./scripts/generate-docs.mjs"
   },
   "dependencies": {
     "@icons-pack/react-simple-icons": "^10.1.0",
@@ -33,13 +33,13 @@
     "clsx": "2.1.1",
     "foxact": "^0.2.41",
     "framer-motion": "^11.11.17",
-    "fumadocs-core": "14.4.2",
+    "fumadocs-core": "^14.6.0",
     "fumadocs-docgen": "^1.3.2",
-    "fumadocs-mdx": "^11.1.1",
-    "fumadocs-openapi": "^5.7.0",
-    "fumadocs-twoslash": "^2.0.1",
+    "fumadocs-mdx": "^11.1.2",
+    "fumadocs-openapi": "^5.8.2",
+    "fumadocs-twoslash": "^2.0.2",
     "fumadocs-typescript": "^3.0.2",
-    "fumadocs-ui": "14.4.2",
+    "fumadocs-ui": "^14.6.0",
     "hast-util-to-jsx-runtime": "^2.3.2",
     "llamaindex": "workspace:*",
     "lucide-react": "^0.460.0",
@@ -72,6 +72,7 @@
     "@types/react": "^18.3.12",
     "@types/react-dom": "^18.3.1",
     "autoprefixer": "^10.4.20",
+    "cross-env": "^7.0.3",
     "fast-glob": "^3.3.2",
     "gray-matter": "^4.0.3",
     "monaco-editor-webpack-plugin": "^7.1.0",
@@ -82,6 +83,9 @@
     "remark-stringify": "^11.0.0",
     "tailwindcss": "^3.4.15",
     "tsx": "^4.19.2",
+    "typedoc": "^0.26.11",
+    "typedoc-plugin-markdown": "^4.3.1",
+    "typedoc-plugin-merge-modules": "^6.1.0",
     "typescript": "^5.6.3"
   }
 }

apps/next/scripts/generate-docs.mjs

@@ -1,8 +1,12 @@
 import * as OpenAPI from "fumadocs-openapi";
+import { generateFiles } from "fumadocs-typescript";
+import fs from "node:fs";
+import * as path from "node:path";
 import { fileURLToPath } from "node:url";
 import { rimrafSync } from "rimraf";
 
 const out = "./src/content/docs/cloud/api";
+const apiRefOut = "./src/content/docs/api";
 
 // clean generated files
 rimrafSync(out, {
@@ -20,3 +24,63 @@ void OpenAPI.generateFiles({
   output: out,
   groupBy: "tag",
 });
+
+void generateFiles({
+  input: ["./src/content/docs/api/**/*.mdx"],
+  output: (file) => path.resolve(path.dirname(file), path.basename(file)),
+  transformOutput,
+});
+
+function transformOutput(filePath, content) {
+  const fileName = path.basename(filePath);
+  let title = fileName.split(".")[0];
+  let pageContent = content;
+  if (title === "index") title = "LlamaIndex API Reference";
+  return `---\ntitle: ${title}\n---\n\n${transformAbsoluteUrl(pageContent, filePath)}`;
+}
+
+/**
+ * Transforms the content by converting relative MDX links to absolute docs API links
+ * Example: [text](../type-aliases/TaskHandler.mdx) -> [text](/docs/api/type-aliases/TaskHandler)
+ * [text](BaseChatEngine.mdx) -> [text](/docs/api/classes/BaseChatEngine)
+ * [text](BaseVectorStore.mdx#constructors) -> [text](/docs/api/classes/BaseVectorStore#constructors)
+ * [text](TaskStep.mdx) -> [text](/docs/api/type-aliases/TaskStep)
+ */
+function transformAbsoluteUrl(content, filePath) {
+  const group = path.dirname(filePath).split(path.sep).pop();
+  return content.replace(
+    /\]\(([^)]+)\.mdx([^)]*)\)/g,
+    (match, slug, anchor) => {
+      const slugParts = slug.split("/");
+      const fileName = slugParts[slugParts.length - 1];
+      const fileGroup = slugParts[slugParts.length - 2] ?? group;
+      const result = ["/docs/api", fileGroup, fileName, anchor]
+        .filter(Boolean)
+        .join("/");
+      return `](${result})`;
+    },
+  );
+}
+
+// append meta.json for API page
+fs.writeFileSync(
+  path.resolve(apiRefOut, "meta.json"),
+  JSON.stringify(
+    {
+      title: "API Reference",
+      description: "LlamaIndex API Reference",
+      root: true,
+      pages: [
+        "index",
+        "classes",
+        "enumerations",
+        "functions",
+        "interfaces",
+        "type-aliases",
+        "variables",
+      ],
+    },
+    null,
+    2,
+  ),
+);

apps/next/src/app/global.css

@@ -94,4 +94,33 @@
   body {
     @apply bg-background text-foreground;
   }
+
+  /*
+  * Override default styles for Markdown
+  */
+  .prose
+    :where(blockquote):not(
+      :where([class~="not-prose"], [class~="not-prose"] *)
+    ) {
+    font-style: normal !important;
+  }
+
+  .prose
+    :where(blockquote p:first-of-type):not(
+      :where([class~="not-prose"], [class~="not-prose"] *)
+    ):before {
+    content: none !important;
+  }
+
+  .prose
+    :where(blockquote p:first-of-type):not(
+      :where([class~="not-prose"], [class~="not-prose"] *)
+    ):after {
+    content: none !important;
+  }
+
+  .prose
+    :where(code):not(:where([class~="not-prose"], [class~="not-prose"] *)) {
+    @apply text-blue-600 !important;
+  }
 }

apps/next/src/content/docs/meta.json

@@ -1,3 +1,3 @@
 {
-  "pages": ["llamaindex", "cloud"]
+  "pages": ["llamaindex", "cloud", "api"]
 }

apps/next/typedoc.json

@@ -0,0 +1,18 @@
+{
+  "plugin": ["typedoc-plugin-markdown", "typedoc-plugin-merge-modules"],
+  "entryPoints": ["../../packages/llamaindex/src/index.ts"],
+  "tsconfig": "../../tsconfig.json",
+  "readme": "none",
+  "sourceLinkTemplate": "https://github.com/run-llama/LlamaIndexTS/blob/{gitRevision}/{path}#L{line}",
+  "out": "./src/content/docs/api",
+  "outputFileStrategy": "members",
+  "categorizeByGroup": true,
+  "categoryOrder": ["Classes", "Enums", "Functions", "Interfaces", "Types"],
+  "sort": ["source-order"],
+  "entryFileName": "index.md",
+  "fileExtension": ".mdx",
+  "hidePageTitle": true,
+  "hidePageHeader": true,
+  "hideGroupHeadings": true,
+  "hideBreadcrumbs": true
+}

packages/core/src/chat-engine/context-chat-engine.ts

@@ -22,7 +22,8 @@ import type { ContextGenerator } from "./type";
 
 /**
  * ContextChatEngine uses the Index to get the appropriate context for each query.
- * The context is stored in the system prompt, and the chat history is chunk: ChatResponseChunk, nodes?: NodeWithScore<import("/Users/marcus/code/llamaindex/LlamaIndexTS/packages/core/src/Node").Metadata>[], nodes?: NodeWithScore<import("/Users/marcus/code/llamaindex/LlamaIndexTS/packages/core/src/Node").Metadata>[]lowing the appropriate context to be surfaced for each query.
+ * The context is stored in the system prompt, and the chat history is chunk,
+ * allowing the appropriate context to be surfaced for each query.
  */
 export class ContextChatEngine extends PromptMixin implements BaseChatEngine {
   chatModel: LLM;

packages/readers/src/html.ts

@@ -13,7 +13,7 @@ export class HTMLReader extends FileReader<Document> {
    * Public method for this reader.
    * Required by BaseReader interface.
    * @param fileContent - The content of the file.
-   * @returns Promise<Document[]> A Promise object, eventually yielding zero or one Document parsed from the HTML content of the specified file.
+   * @returns `Promise<Document[]>` A Promise object, eventually yielding zero or one Document parsed from the HTML content of the specified file.
    */
   async loadDataAsContent(fileContent: Uint8Array): Promise<Document[]> {
     const decoder = new TextDecoder("utf-8");

packages/readers/src/image.ts

@@ -8,7 +8,7 @@ export class ImageReader extends FileReader<ImageDocument> {
    * Public method for this reader.
    * Required by BaseReader interface.
    * @param fileContent - The content of the file.
-   * @returns Promise<Document[]> A Promise object, eventually yielding zero or one ImageDocument of the specified file.
+   * @returns `Promise<Document[]>` A Promise object, eventually yielding zero or one ImageDocument of the specified file.
    */
   async loadDataAsContent(fileContent: Uint8Array): Promise<ImageDocument[]> {
     const blob = new Blob([fileContent]);