Compare commits

..

2 Commits

Author SHA1 Message Date
github-actions[bot] 5bb4531245 Release 0.8.3 (#1427)
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
2024-11-02 19:05:28 -07:00
Alex Yang 2ff0a89891 fix: expose @llamaindex/node-parser (#1426) 2024-11-02 18:57:05 -07:00
53 changed files with 830 additions and 102 deletions
+6
View File
@@ -1,5 +1,11 @@
# docs
## 0.0.107
### Patch Changes
- llamaindex@0.8.3
## 0.0.106
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "docs",
"version": "0.0.106",
"version": "0.0.107",
"private": true,
"scripts": {
"docusaurus": "docusaurus",
+8
View File
@@ -1,5 +1,13 @@
# @llamaindex/doc
## 0.0.5
### Patch Changes
- Updated dependencies [2ff0a89]
- @llamaindex/node-parser@0.0.2
- llamaindex@0.8.3
## 0.0.4
### Patch Changes
+17 -2
View File
@@ -1,16 +1,31 @@
import { createMDX } from "fumadocs-mdx/next";
import MonacoWebpackPlugin from "monaco-editor-webpack-plugin";
const withMDX = createMDX();
/** @type {import('next').NextConfig} */
const config = {
reactStrictMode: true,
webpack: (config) => {
transpilePackages: ["monaco-editor"],
webpack: (config, { isServer }) => {
if (Array.isArray(config.target) && config.target.includes("web")) {
config.target = ["web", "es2020"];
}
config.resolve.alias = {
...config.resolve.alias,
sharp$: false,
"onnxruntime-node$": false,
};
config.resolve.fallback ??= {};
config.resolve.fallback.fs = false;
if (!isServer) {
config.plugins.push(
new MonacoWebpackPlugin({
languages: ["typescript"],
filename: "static/[name].worker.js",
}),
);
}
return config;
},
};
+9 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/doc",
"version": "0.0.4",
"version": "0.0.5",
"private": true,
"scripts": {
"build": "pnpm run build:docs && next build",
@@ -14,6 +14,7 @@
"@icons-pack/react-simple-icons": "^10.1.0",
"@llamaindex/cloud": "workspace:*",
"@llamaindex/core": "workspace:*",
"@llamaindex/node-parser": "workspace:*",
"@llamaindex/openai": "workspace:*",
"@llamaindex/readers": "workspace:*",
"@llamaindex/workflow": "workspace:*",
@@ -21,6 +22,8 @@
"@number-flow/react": "^0.3.0",
"@radix-ui/react-dialog": "^1.1.2",
"@radix-ui/react-icons": "^1.3.1",
"@radix-ui/react-label": "^2.1.0",
"@radix-ui/react-slider": "^1.2.1",
"@radix-ui/react-slot": "^1.1.0",
"@radix-ui/react-tooltip": "^1.1.3",
"@vercel/functions": "^1.5.0",
@@ -44,6 +47,7 @@
"react": "^18.3.1",
"react-dom": "^18.3.1",
"react-icons": "^5.3.0",
"react-monaco-editor": "^0.56.2",
"react-text-transition": "^3.1.0",
"react-use-measure": "^2.1.1",
"rehype-katex": "^7.0.1",
@@ -54,7 +58,10 @@
"swr": "^2.2.5",
"tailwind-merge": "^2.5.2",
"tailwindcss-animate": "^1.0.7",
"tree-sitter": "^0.22.0",
"tree-sitter-typescript": "^0.23.0",
"use-stick-to-bottom": "^1.0.41",
"web-tree-sitter": "^0.24.3",
"zod": "^3.23.8"
},
"devDependencies": {
@@ -66,6 +73,7 @@
"autoprefixer": "^10.4.20",
"fast-glob": "^3.3.2",
"gray-matter": "^4.0.3",
"monaco-editor-webpack-plugin": "^7.1.0",
"postcss": "^8.4.47",
"remark": "^15.0.1",
"remark-gfm": "^4.0.0",
Binary file not shown.
BIN
View File
Binary file not shown.
+1 -2
View File
@@ -11,7 +11,6 @@ import { NpmInstall } from "@/components/npm-install";
import { TextEffect } from "@/components/text-effect";
import { Button } from "@/components/ui/button";
import { Skeleton } from "@/components/ui/skeleton";
import { DOCUMENT_URL } from "@/lib/const";
import { SiStackblitz } from "@icons-pack/react-simple-icons";
import {
CodeBlock as FumaCodeBlock,
@@ -38,7 +37,7 @@ export default function HomePage() {
</div>
<div className="flex flex-wrap justify-center gap-4">
<Link href={DOCUMENT_URL}>
<Link href="/docs/llamaindex">
<Button variant="outline">Get Started</Button>
</Link>
<NpmInstall />
+2 -2
View File
@@ -1,4 +1,4 @@
import { DOCUMENT_URL } from "@/lib/const";
import { LEGACY_DOCUMENT_URL } from "@/lib/const";
import { redirect } from "next/navigation";
export default async function Page(props: {
@@ -7,5 +7,5 @@ export default async function Page(props: {
}>;
}) {
const path = await props.params.then(({ any }) => any.join("/"));
return redirect(new URL(path, DOCUMENT_URL).toString());
return redirect(new URL(path, LEGACY_DOCUMENT_URL).toString());
}
+40 -24
View File
@@ -1,38 +1,54 @@
import { baseOptions } from "@/app/layout.config";
import { AITrigger } from "@/components/ai-chat";
import { buttonVariants } from "@/components/ui/button";
import { LEGACY_DOCUMENT_URL } from "@/lib/const";
import { source } from "@/lib/source";
import { cn } from "@/lib/utils";
import "fumadocs-twoslash/twoslash.css";
import { Banner } from "fumadocs-ui/components/banner";
import { DocsLayout } from "fumadocs-ui/layouts/docs";
import { MessageCircle } from "lucide-react";
import type { ReactNode } from "react";
export default function Layout({ children }: { children: ReactNode }) {
return (
<DocsLayout
tree={source.pageTree}
{...baseOptions}
nav={{
...baseOptions.nav,
children: (
<AITrigger
className={cn(
buttonVariants({
variant: "secondary",
size: "xs",
className:
"md:flex-1 px-2 ms-2 gap-1.5 text-fd-muted-foreground rounded-full",
}),
)}
>
<MessageCircle className="size-3" />
Ask LlamaCloud
</AITrigger>
),
}}
>
{children}
</DocsLayout>
<>
<Banner variant="rainbow" id="welcome">
Welcome to the new LlamaIndex.TS documentation! 🎉 If you are looking
for the old documentation
<a
className="underline text-blue-500 ml-1"
target="_blank"
href={LEGACY_DOCUMENT_URL}
>
check it here
</a>
.
</Banner>
<DocsLayout
tree={source.pageTree}
{...baseOptions}
nav={{
...baseOptions.nav,
children: (
<AITrigger
className={cn(
buttonVariants({
variant: "secondary",
size: "xs",
className:
"md:flex-1 px-2 ms-2 gap-1.5 text-fd-muted-foreground rounded-full",
}),
)}
>
<MessageCircle className="size-3" />
Ask LlamaCloud
</AITrigger>
),
}}
>
{children}
</DocsLayout>
</>
);
}
+13 -3
View File
@@ -1,4 +1,4 @@
import { DOCUMENT_URL } from "@/lib/const";
import { Badge } from "@/components/ui/badge";
import type { BaseLayoutProps } from "fumadocs-ui/layouts/shared";
import Image from "next/image";
@@ -27,8 +27,18 @@ export const baseOptions: BaseLayoutProps = {
githubUrl: "https://github.com/run-llama/LlamaIndexTS",
links: [
{
text: "Documentation",
url: DOCUMENT_URL,
text: (
<div className="relative">
Docs
<Badge
variant="outline"
className="text-blue-500 absolute -top-5 -left-5 bg-fd-background hover:scale-125 transition-transform -rotate-3 hover:-rotate-12"
>
new
</Badge>
</div>
),
url: "/docs/llamaindex",
active: "nested-url",
},
],
+1 -9
View File
@@ -1,6 +1,5 @@
import { AIProvider } from "@/actions";
import { TooltipProvider } from "@/components/ui/tooltip";
import { Banner } from "fumadocs-ui/components/banner";
import { RootProvider } from "fumadocs-ui/provider";
import { Inter } from "next/font/google";
import type { ReactNode } from "react";
@@ -36,14 +35,7 @@ export default function Layout({ children }: { children: ReactNode }) {
<body className="flex flex-col min-h-screen">
<TooltipProvider>
<AIProvider>
<RootProvider>
<Banner variant="rainbow" id="experimental">
Welcome to the experimental LlamaIndex.TS documentation site.
Some content are still in progress, you are welcome to help
contribute to the documentation
</Banner>
{children}
</RootProvider>
<RootProvider>{children}</RootProvider>
</AIProvider>
</TooltipProvider>
</body>
@@ -0,0 +1,182 @@
"use client";
import { createContextState } from "foxact/context-state";
import { useIsClient } from "foxact/use-is-client";
import { useShiki } from "fumadocs-core/utils/use-shiki";
import { CodeBlock, Pre } from "fumadocs-ui/components/codeblock";
import { lazy, Suspense, use, useMemo } from "react";
import { StickToBottom, useStickToBottomContext } from "use-stick-to-bottom";
import Parser from "web-tree-sitter";
import { Label } from "@/components/ui/label";
import { Skeleton } from "@/components/ui/skeleton";
import { Slider } from "@/components/ui/slider";
import { CodeSplitter } from "@llamaindex/node-parser/code";
let promise: Promise<CodeSplitter>;
if (typeof window !== "undefined") {
promise = Parser.init({
locateFile(scriptName: string) {
return "/" + scriptName;
},
}).then(async () => {
const parser = new Parser();
const Lang = await Parser.Language.load("/tree-sitter-typescript.wasm");
parser.setLanguage(Lang);
return new CodeSplitter({
getParser: () => parser,
maxChars: 100,
});
});
}
const [SliderProvider, useSlider, useSetSlider] = createContextState(100);
const [CodeProvider, useCode, useSetCode] =
createContextState(`interface Person {
name: string;
age: number;
}
function greet(person: Person): string {
return \`Hello, \${person.name}! You are \${person.age} years old.\`;
}
const john: Person = {
name: "John Doe",
age: 30
};
console.log(greet(john));`);
const Editor = lazy(() => import("react-monaco-editor"));
export const IDE = () => {
const codeSplitter = use(promise);
const code = useCode();
const setCode = useSetCode();
const maxChars = useSlider();
const useSetMaxChars = useSetSlider();
return (
<div className="flex flex-col p-4 border-r max-h-96 overflow-scroll">
<div>
<Label>Max Chars {maxChars}</Label>
<Slider
className="my-4"
min={10}
max={300}
step={10}
value={[maxChars]}
onValueChange={(value) => {
useSetMaxChars(value[0]);
codeSplitter.maxChars = value[0];
}}
/>
</div>
<Editor
editorWillMount={() => {}}
editorDidMount={() => {
window.MonacoEnvironment!.getWorkerUrl = (
_moduleId: string,
label: string,
) => {
if (label === "json") return "/_next/static/json.worker.js";
if (label === "css") return "/_next/static/css.worker.js";
if (label === "html") return "/_next/static/html.worker.js";
if (label === "typescript" || label === "javascript")
return "/_next/static/ts.worker.js";
return "/_next/static/editor.worker.js";
};
}}
editorWillUnmount={() => {}}
options={{
minimap: {
enabled: false,
},
}}
theme="vs-dark"
height="100%"
width="100%"
language="typescript"
onChange={setCode}
value={code}
/>
</div>
);
};
const Preview = ({ text }: { text: string }) => {
const rendered = useShiki(text, {
lang: "ts",
components: {
pre: (props) => {
return <Pre {...props}>{props.children}</Pre>;
},
},
});
return <CodeBlock className="py-0 m-2">{rendered}</CodeBlock>;
};
function ScrollToBottom() {
const { isAtBottom, scrollToBottom } = useStickToBottomContext();
return (
!isAtBottom && (
<button
className="absolute i-ph-arrow-circle-down-fill text-4xl rounded-lg left-[50%] translate-x-[-50%] bottom-0"
onClick={() => scrollToBottom()}
/>
)
);
}
export const NodePreview = () => {
const parser = use(promise);
const code = useCode();
const maxChars = useSlider();
const textChunks = useMemo(() => parser.splitText(code), [code, maxChars]);
return (
<StickToBottom
className="block relative max-h-96 overflow-scroll"
resize="smooth"
initial="smooth"
>
<StickToBottom.Content>
{textChunks.map((chunk, i) => (
<Preview key={i} text={chunk} />
))}
</StickToBottom.Content>
<ScrollToBottom />
</StickToBottom>
);
};
export const CodeNodeParserDemo = () => {
const isClient = useIsClient();
if (!isClient) {
return (
<div className="my-2 grid grid-cols-1 md:grid-cols-2 gap-2 border rounded-xl w-full max-h-96">
<Skeleton className="h-96" />
<Skeleton className="h-96" />
</div>
);
}
return (
<SliderProvider>
<CodeProvider>
<Suspense
fallback={
<div className="my-2 grid grid-cols-1 md:grid-cols-2 gap-2 border rounded-xl w-full max-h-96">
<Skeleton className="h-96" />
<Skeleton className="h-96" />
</div>
}
>
<div className="my-2 grid grid-cols-1 md:grid-cols-2 gap-2 border rounded-xl w-full max-h-96">
<IDE />
<NodePreview />
</div>
</Suspense>
</CodeProvider>
</SliderProvider>
);
};
@@ -84,7 +84,7 @@ function ScrollToBottom() {
export function WorkflowStreamingDemo() {
const [ui, setUI] = useState<ReactNode[]>([
<div key={0} className="bg-gray-100">
<div key={0} className="bg-gray-100 dark:bg-gray-800">
Waiting for workflow to start
</div>,
]);
@@ -110,28 +110,28 @@ export function WorkflowStreamingDemo() {
if (event instanceof ComputeEvent) {
setUI((ui) => [
...ui,
<div key={i++} className="bg-yellow-100">
<div key={i++} className="bg-yellow-100 dark:bg-yellow-800">
Computing task id: {event.data}
</div>,
]);
} else if (event instanceof ComputeResultEvent) {
setUI((ui) => [
...ui,
<div key={i++} className="bg-green-100">
<div key={i++} className="bg-green-100 dark:bg-green-800">
Computed task id: {event.data}
</div>,
]);
} else if (event instanceof StartEvent) {
setUI((ui) => [
...ui,
<div key={i++} className="bg-blue-100">
<div key={i++} className="bg-blue-100 dark:bg-blue-800">
Started workflow with total {event.data}
</div>,
]);
} else if (event instanceof StopEvent) {
setUI((ui) => [
...ui,
<div key={i++} className="bg-red-100">
<div key={i++} className="bg-red-100 dark:bg-red-800">
Workflow stopped
</div>,
]);
+36
View File
@@ -0,0 +1,36 @@
import { cva, type VariantProps } from "class-variance-authority";
import * as React from "react";
import { cn } from "@/lib/utils";
const badgeVariants = cva(
"inline-flex items-center rounded-md border px-2.5 py-0.5 text-xs font-semibold transition-colors focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2",
{
variants: {
variant: {
default:
"border-transparent bg-primary text-primary-foreground shadow hover:bg-primary/80",
secondary:
"border-transparent bg-secondary text-secondary-foreground hover:bg-secondary/80",
destructive:
"border-transparent bg-destructive text-destructive-foreground shadow hover:bg-destructive/80",
outline: "text-foreground",
},
},
defaultVariants: {
variant: "default",
},
},
);
export interface BadgeProps
extends React.HTMLAttributes<HTMLDivElement>,
VariantProps<typeof badgeVariants> {}
function Badge({ className, variant, ...props }: BadgeProps) {
return (
<div className={cn(badgeVariants({ variant }), className)} {...props} />
);
}
export { Badge, badgeVariants };
+26
View File
@@ -0,0 +1,26 @@
"use client";
import * as LabelPrimitive from "@radix-ui/react-label";
import { cva, type VariantProps } from "class-variance-authority";
import * as React from "react";
import { cn } from "@/lib/utils";
const labelVariants = cva(
"text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70",
);
const Label = React.forwardRef<
React.ElementRef<typeof LabelPrimitive.Root>,
React.ComponentPropsWithoutRef<typeof LabelPrimitive.Root> &
VariantProps<typeof labelVariants>
>(({ className, ...props }, ref) => (
<LabelPrimitive.Root
ref={ref}
className={cn(labelVariants(), className)}
{...props}
/>
));
Label.displayName = LabelPrimitive.Root.displayName;
export { Label };
+28
View File
@@ -0,0 +1,28 @@
"use client";
import * as SliderPrimitive from "@radix-ui/react-slider";
import * as React from "react";
import { cn } from "@/lib/utils";
const Slider = React.forwardRef<
React.ElementRef<typeof SliderPrimitive.Root>,
React.ComponentPropsWithoutRef<typeof SliderPrimitive.Root>
>(({ className, ...props }, ref) => (
<SliderPrimitive.Root
ref={ref}
className={cn(
"relative flex w-full touch-none select-none items-center",
className,
)}
{...props}
>
<SliderPrimitive.Track className="relative h-1.5 w-full grow overflow-hidden rounded-full bg-primary/20">
<SliderPrimitive.Range className="absolute h-full bg-primary" />
</SliderPrimitive.Track>
<SliderPrimitive.Thumb className="block h-4 w-4 rounded-full border border-primary/50 bg-background shadow transition-colors focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50" />
</SliderPrimitive.Root>
));
Slider.displayName = SliderPrimitive.Root.displayName;
export { Slider };
@@ -22,20 +22,10 @@ import { Tab, Tabs } from "fumadocs-ui/components/tabs";
## What's next?
<Cards>
<Card
title="I'm new to RAG"
description="Learn more about RAG (Retrieval-augmented generation) using LlamaIndex.TS."
href="/docs/llamaindex/setup/what-is-rag"
/>
<Card
title="I want to try LlamaIndex.TS"
description="Learn how to use LlamaIndex.TS with different JS runtime and frameworks."
href="/docs/llamaindex/setup"
/>
<Card
title="I want to improve performance when using LlamaIndex.TS"
description="Learn how to improve performance, reduce bundle size, improve accuracy."
href="/docs/llamaindex/performance"
href="/docs/llamaindex/setup/getting-started"
/>
<Card
title="Show me code examples"
@@ -2,6 +2,7 @@
title: Node Parsers / Text Splitters
description: Learn how to use Node Parsers and Text Splitters to extract data from documents.
---
import { CodeNodeParserDemo } from '../../../../components/demo/code-node-parser';
import { Tab, Tabs } from "fumadocs-ui/components/tabs";
Node parsers are a simple abstraction that take a list of documents, and chunk them into `Node` objects, such that each node is a specific chunk of the parent document. When a document is broken into nodes, all of it's attributes are inherited to the children nodes (i.e. `metadata`, text and metadata templates, etc.). You can read more about `Node` and `Document` properties [here](./).
@@ -10,30 +11,17 @@ Node parsers are a simple abstraction that take a list of documents, and chunk t
The `NodeParser` in LlamaIndex is responsible for splitting `Document` objects into more manageable `Node` objects.
<Tabs items={["with reader", "with node:fs"]}>
```ts twoslash tab="with reader"
import { TextFileReader } from '@llamaindex/readers/text'
import {SentenceSplitter} from '@llamaindex/core/node-parser';
By default, we will use `Settings.nodeParser` to split the document into nodes. You can also assign a custom `NodeParser` to the `Settings` object.
const nodeParser = new SentenceSplitter();
const reader = new TextFileReader();
const documents = await reader.loadData('path/to/file.txt');
```ts twoslash
import { TextFileReader } from '@llamaindex/readers/text'
import { SentenceSplitter } from '@llamaindex/core/node-parser';
import { Settings } from 'llamaindex';
const parsedDocuments = nodeParser(documents);
// ^?
```
```ts twoslash tab="with node:fs"
import fs from 'node:fs/promises';
import { SentenceSplitter } from '@llamaindex/core/node-parser';
const nodeParser = new SentenceSplitter();
const texts = nodeParser.splitText(await fs.readFile('path/to/file.txt', 'utf-8'));
// ^?
```
</Tabs>
const nodeParser = new SentenceSplitter();
Settings.nodeParser = nodeParser;
// ^?
```
## TextSplitter
@@ -79,4 +67,92 @@ The `MarkdownNodeParser` is a more advanced `NodeParser` that can handle markdow
// ^?
```
</Tabs>
</Tabs>
## CodeSplitter
The `CodeSplitter` is a more advanced `NodeParser` that can handle code documents.
It will split the code by AST nodes and then parse the nodes into a `Document` object.
<Tabs items={["with reader", "with node:fs"]}>
```ts twoslash tab="with reader"
import { TextFileReader } from '@llamaindex/readers/text'
import { CodeSplitter } from '@llamaindex/node-parser/code'
import Parser from "tree-sitter";
import TS from "tree-sitter-typescript";
const parser = new Parser();
parser.setLanguage(TS.typescript);
const codeSplitter = new CodeSplitter({
getParser: () => parser,
});
const reader = new TextFileReader();
const documents = await reader.loadData('path/to/file.ts');
const parsedDocuments = codeSplitter(documents);
// ^?
```
```ts twoslash tab="with node:fs"
import fs from 'node:fs/promises';
import { CodeSplitter } from '@llamaindex/node-parser/code'
import Parser from "tree-sitter";
import TS from "tree-sitter-typescript";
const parser = new Parser();
parser.setLanguage(TS.typescript);
const codeSplitter = new CodeSplitter({
getParser: () => parser,
});
const parsedDocuments = codeSplitter.splitText(await fs.readFile('path/to/file.ts', 'utf-8'));
// ^?
```
</Tabs>
Try it out ⬇️
<CodeNodeParserDemo/>
import { Accordion, Accordions } from 'fumadocs-ui/components/accordion';
<Accordions>
<Accordion title="Use it in browser">
You might setup WASM files for `web-tree-sitter` and use it in the browser.
```ts
import Parser from 'web-tree-sitter';
Parser.init({
locateFile(scriptName: string) {
return '/' + scriptName
},
}).then(async () => {
const parser = new Parser();
const Lang = await Parser.Language.load('/tree-sitter-typescript.wasm');
parser.setLanguage(Lang);
return new CodeSplitter({
getParser: () => parser,
maxChars: 100
});
});
```
In this example, you should put `tree-sitter-typescript.wasm` to the `public` folder for Next.js.
And also update the `next.config.js` to make `@llamaindex/env` work properly.
```js
const config = {
webpack: (config) => {
if (Array.isArray(config.target) && config.target.includes('web')) {
config.target = ["web", "es2020"];
}
return config;
}
}
export default config;
```
</Accordion>
</Accordions>
+1 -1
View File
@@ -1,2 +1,2 @@
// when we are ready, change to /docs/llamaindex
export const DOCUMENT_URL = 'https://legacy.ts.llamaindex.ai/'
export const LEGACY_DOCUMENT_URL = 'https://legacy.ts.llamaindex.ai/'
+135
View File
@@ -0,0 +1,135 @@
// Type definitions for Next.js cacheLife configs
declare module "next/cache" {
export { cacheTag as unstable_cacheTag } from "next/dist/server/use-cache/cache-tag";
export {
revalidatePath,
revalidateTag,
} from "next/dist/server/web/spec-extension/revalidate";
export { unstable_cache } from "next/dist/server/web/spec-extension/unstable-cache";
export { unstable_noStore } from "next/dist/server/web/spec-extension/unstable-no-store";
/**
* Cache this `"use cache"` for a timespan defined by the `"default"` profile.
* ```
* stale: 300 seconds (5 minutes)
* revalidate: 900 seconds (15 minutes)
* expire: never
* ```
*
* This cache may be stale on clients for 5 minutes before checking with the server.
* If the server receives a new request after 15 minutes, start revalidating new values in the background.
* It lives for the maximum age of the server cache. If this entry has no traffic for a while, it may serve an old value the next request.
*/
export function unstable_cacheLife(profile: "default"): void;
/**
* Cache this `"use cache"` for a timespan defined by the `"seconds"` profile.
* ```
* stale: 0 seconds
* revalidate: 1 seconds
* expire: 1 seconds
* ```
*
* This cache may be stale on clients for 0 seconds before checking with the server.
* This cache will expire after 1 seconds. The next request will recompute it.
*/
export function unstable_cacheLife(profile: "seconds"): void;
/**
* Cache this `"use cache"` for a timespan defined by the `"minutes"` profile.
* ```
* stale: 300 seconds (5 minutes)
* revalidate: 60 seconds (1 minute)
* expire: 3600 seconds (1 hour)
* ```
*
* This cache may be stale on clients for 5 minutes before checking with the server.
* If the server receives a new request after 1 minute, start revalidating new values in the background.
* If this entry has no traffic for 1 hour it will expire. The next request will recompute it.
*/
export function unstable_cacheLife(profile: "minutes"): void;
/**
* Cache this `"use cache"` for a timespan defined by the `"hours"` profile.
* ```
* stale: 300 seconds (5 minutes)
* revalidate: 3600 seconds (1 hour)
* expire: 86400 seconds (1 day)
* ```
*
* This cache may be stale on clients for 5 minutes before checking with the server.
* If the server receives a new request after 1 hour, start revalidating new values in the background.
* If this entry has no traffic for 1 day it will expire. The next request will recompute it.
*/
export function unstable_cacheLife(profile: "hours"): void;
/**
* Cache this `"use cache"` for a timespan defined by the `"days"` profile.
* ```
* stale: 300 seconds (5 minutes)
* revalidate: 86400 seconds (1 day)
* expire: 604800 seconds (1 week)
* ```
*
* This cache may be stale on clients for 5 minutes before checking with the server.
* If the server receives a new request after 1 day, start revalidating new values in the background.
* If this entry has no traffic for 1 week it will expire. The next request will recompute it.
*/
export function unstable_cacheLife(profile: "days"): void;
/**
* Cache this `"use cache"` for a timespan defined by the `"weeks"` profile.
* ```
* stale: 300 seconds (5 minutes)
* revalidate: 604800 seconds (1 week)
* expire: 2592000 seconds (30 days)
* ```
*
* This cache may be stale on clients for 5 minutes before checking with the server.
* If the server receives a new request after 1 week, start revalidating new values in the background.
* If this entry has no traffic for 30 days it will expire. The next request will recompute it.
*/
export function unstable_cacheLife(profile: "weeks"): void;
/**
* Cache this `"use cache"` for a timespan defined by the `"max"` profile.
* ```
* stale: 300 seconds (5 minutes)
* revalidate: 2592000 seconds (30 days)
* expire: never
* ```
*
* This cache may be stale on clients for 5 minutes before checking with the server.
* If the server receives a new request after 30 days, start revalidating new values in the background.
* It lives for the maximum age of the server cache. If this entry has no traffic for a while, it may serve an old value the next request.
*/
export function unstable_cacheLife(profile: "max"): void;
/**
* Cache this `"use cache"` using a custom timespan.
* ```
* stale: ... // seconds
* revalidate: ... // seconds
* expire: ... // seconds
* ```
*
* This is similar to Cache-Control: max-age=`stale`,s-max-age=`revalidate`,stale-while-revalidate=`expire-revalidate`
*
* If a value is left out, the lowest of other cacheLife() calls or the default, is used instead.
*/
export function unstable_cacheLife(profile: {
/**
* This cache may be stale on clients for ... seconds before checking with the server.
*/
stale?: number;
/**
* If the server receives a new request after ... seconds, start revalidating new values in the background.
*/
revalidate?: number;
/**
* If this entry has no traffic for ... seconds it will expire. The next request will recompute it.
*/
expire?: number;
}): void;
}
+3
View File
@@ -0,0 +1,3 @@
{
"type": "module"
}
+6
View File
@@ -1,5 +1,11 @@
# @llamaindex/autotool
## 5.0.3
### Patch Changes
- llamaindex@0.8.3
## 5.0.2
### Patch Changes
@@ -1,5 +1,12 @@
# @llamaindex/autotool-01-node-example
## 0.0.46
### Patch Changes
- llamaindex@0.8.3
- @llamaindex/autotool@5.0.3
## 0.0.45
### Patch Changes
@@ -13,5 +13,5 @@
"scripts": {
"start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
},
"version": "0.0.45"
"version": "0.0.46"
}
@@ -1,5 +1,12 @@
# @llamaindex/autotool-02-next-example
## 0.1.90
### Patch Changes
- llamaindex@0.8.3
- @llamaindex/autotool@5.0.3
## 0.1.89
### Patch Changes
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/autotool-02-next-example",
"private": true,
"version": "0.1.89",
"version": "0.1.90",
"scripts": {
"dev": "next dev",
"build": "next build",
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/autotool",
"type": "module",
"version": "5.0.2",
"version": "5.0.3",
"description": "auto transpile your JS function to LLM Agent compatible",
"files": [
"dist",
+6
View File
@@ -1,5 +1,11 @@
# @llamaindex/experimental
## 0.0.115
### Patch Changes
- llamaindex@0.8.3
## 0.0.114
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/experimental",
"description": "Experimental package for LlamaIndexTS",
"version": "0.0.114",
"version": "0.0.115",
"type": "module",
"types": "dist/type/index.d.ts",
"main": "dist/cjs/index.js",
+7
View File
@@ -1,5 +1,12 @@
# llamaindex
## 0.8.3
### Patch Changes
- Updated dependencies [2ff0a89]
- @llamaindex/node-parser@0.0.2
## 0.8.2
### Patch Changes
@@ -1,5 +1,11 @@
# @llamaindex/cloudflare-worker-agent-test
## 0.0.99
### Patch Changes
- llamaindex@0.8.3
## 0.0.98
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/cloudflare-worker-agent-test",
"version": "0.0.98",
"version": "0.0.99",
"type": "module",
"private": true,
"scripts": {
@@ -1,5 +1,11 @@
# @llamaindex/next-agent-test
## 0.1.99
### Patch Changes
- llamaindex@0.8.3
## 0.1.98
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/next-agent-test",
"version": "0.1.98",
"version": "0.1.99",
"private": true,
"scripts": {
"dev": "next dev",
@@ -1,5 +1,11 @@
# test-edge-runtime
## 0.1.98
### Patch Changes
- llamaindex@0.8.3
## 0.1.97
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/nextjs-edge-runtime-test",
"version": "0.1.97",
"version": "0.1.98",
"private": true,
"scripts": {
"dev": "next dev",
@@ -1,5 +1,11 @@
# @llamaindex/next-node-runtime
## 0.0.80
### Patch Changes
- llamaindex@0.8.3
## 0.0.79
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/next-node-runtime-test",
"version": "0.0.79",
"version": "0.0.80",
"private": true,
"scripts": {
"dev": "next dev",
@@ -1,5 +1,11 @@
# @llamaindex/waku-query-engine-test
## 0.0.99
### Patch Changes
- llamaindex@0.8.3
## 0.0.98
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/waku-query-engine-test",
"version": "0.0.98",
"version": "0.0.99",
"type": "module",
"private": true,
"scripts": {
+2 -1
View File
@@ -1,6 +1,6 @@
{
"name": "llamaindex",
"version": "0.8.2",
"version": "0.8.3",
"license": "MIT",
"type": "module",
"keywords": [
@@ -37,6 +37,7 @@
"@llamaindex/env": "workspace:*",
"@llamaindex/groq": "workspace:*",
"@llamaindex/huggingface": "workspace:*",
"@llamaindex/node-parser": "workspace:*",
"@llamaindex/ollama": "workspace:*",
"@llamaindex/openai": "workspace:*",
"@llamaindex/portkey-ai": "workspace:*",
+1 -1
View File
@@ -68,7 +68,7 @@ export * from "./indices/index.js";
export * from "./ingestion/index.js";
export { imageToDataUrl } from "./internal/utils.js";
export * from "./llm/index.js";
export * from "./nodeParsers/index.js";
export * from "./node-parser.js";
export * from "./objects/index.js";
export * from "./OutputParser.js";
export * from "./postprocessors/index.js";
+3
View File
@@ -0,0 +1,3 @@
export * from "@llamaindex/core/node-parser";
export * from "@llamaindex/node-parser/code";
export * from "@llamaindex/node-parser/html";
@@ -1 +0,0 @@
export * from "@llamaindex/core/node-parser";
@@ -10,7 +10,7 @@ import {
TitleExtractor,
} from "llamaindex/extractors/index";
import { OpenAI } from "llamaindex/llm/openai";
import { SentenceSplitter } from "llamaindex/nodeParsers/index";
import { SentenceSplitter } from "llamaindex/node-parser";
import { afterAll, beforeAll, describe, expect, test, vi } from "vitest";
import {
DEFAULT_LLM_TEXT_OUTPUT,
@@ -5,7 +5,7 @@ import {
IngestionCache,
getTransformationHash,
} from "llamaindex/ingestion/IngestionCache";
import { SentenceSplitter } from "llamaindex/nodeParsers/index";
import { SentenceSplitter } from "llamaindex/node-parser";
import { beforeAll, describe, expect, test } from "vitest";
describe("IngestionCache", () => {
+7
View File
@@ -0,0 +1,7 @@
# @llamaindex/node-parser
## 0.0.2
### Patch Changes
- 2ff0a89: feat: add code splitter and html node parser
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/node-parser",
"version": "0.0.1",
"version": "0.0.2",
"description": "Node parser for LlamaIndex",
"type": "module",
"exports": {
+7 -2
View File
@@ -1,7 +1,12 @@
import { Settings } from "@llamaindex/core/global";
import { TextSplitter } from "@llamaindex/core/node-parser";
import type Parser from "tree-sitter";
import type { SyntaxNode } from "tree-sitter";
import type NodeParser from "tree-sitter";
import type { SyntaxNode as NodeSyntaxNode } from "tree-sitter";
import type WebParser from "web-tree-sitter";
import type { SyntaxNode as WebSyntaxNode } from "web-tree-sitter";
type SyntaxNode = NodeSyntaxNode | WebSyntaxNode;
type Parser = NodeParser | WebParser;
export type CodeSplitterParam = {
getParser: () => Parser;
+112
View File
@@ -136,6 +136,9 @@ importers:
'@llamaindex/core':
specifier: workspace:*
version: link:../../packages/core
'@llamaindex/node-parser':
specifier: workspace:*
version: link:../../packages/node-parser
'@llamaindex/openai':
specifier: workspace:*
version: link:../../packages/providers/openai
@@ -157,6 +160,12 @@ importers:
'@radix-ui/react-icons':
specifier: ^1.3.1
version: 1.3.1(react@18.3.1)
'@radix-ui/react-label':
specifier: ^2.1.0
version: 2.1.0(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
'@radix-ui/react-slider':
specifier: ^1.2.1
version: 1.2.1(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
'@radix-ui/react-slot':
specifier: ^1.1.0
version: 1.1.0(@types/react@18.3.12)(react@18.3.1)
@@ -226,6 +235,9 @@ importers:
react-icons:
specifier: ^5.3.0
version: 5.3.0(react@18.3.1)
react-monaco-editor:
specifier: ^0.56.2
version: 0.56.2(@types/react@18.3.12)(monaco-editor@0.52.0)(react@18.3.1)
react-text-transition:
specifier: ^3.1.0
version: 3.1.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
@@ -256,9 +268,18 @@ importers:
tailwindcss-animate:
specifier: ^1.0.7
version: 1.0.7(tailwindcss@3.4.14)
tree-sitter:
specifier: ^0.22.0
version: 0.22.0
tree-sitter-typescript:
specifier: ^0.23.0
version: 0.23.0(tree-sitter@0.22.0)
use-stick-to-bottom:
specifier: ^1.0.41
version: 1.0.41(react@18.3.1)
web-tree-sitter:
specifier: ^0.24.3
version: 0.24.3
zod:
specifier: ^3.23.8
version: 3.23.8
@@ -287,6 +308,9 @@ importers:
gray-matter:
specifier: ^4.0.3
version: 4.0.3
monaco-editor-webpack-plugin:
specifier: ^7.1.0
version: 7.1.0(monaco-editor@0.52.0)(webpack@5.96.1)
postcss:
specifier: ^8.4.47
version: 8.4.47
@@ -745,6 +769,9 @@ importers:
'@llamaindex/huggingface':
specifier: workspace:*
version: link:../providers/huggingface
'@llamaindex/node-parser':
specifier: workspace:*
version: link:../node-parser
'@llamaindex/ollama':
specifier: workspace:*
version: link:../providers/ollama
@@ -4258,6 +4285,19 @@ packages:
'@types/react':
optional: true
'@radix-ui/react-label@2.1.0':
resolution: {integrity: sha512-peLblDlFw/ngk3UWq0VnYaOLy6agTZZ+MUO/WhVfm14vJGML+xH4FAl2XQGLqdefjNb7ApRg6Yn7U42ZhmYXdw==}
peerDependencies:
'@types/react': '*'
'@types/react-dom': '*'
react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
peerDependenciesMeta:
'@types/react':
optional: true
'@types/react-dom':
optional: true
'@radix-ui/react-navigation-menu@1.2.1':
resolution: {integrity: sha512-egDo0yJD2IK8L17gC82vptkvW1jLeni1VuqCyzY727dSJdk5cDjINomouLoNk8RVF7g2aNIfENKWL4UzeU9c8Q==}
peerDependencies:
@@ -4375,6 +4415,19 @@ packages:
'@types/react-dom':
optional: true
'@radix-ui/react-slider@1.2.1':
resolution: {integrity: sha512-bEzQoDW0XP+h/oGbutF5VMWJPAl/UU8IJjr7h02SOHDIIIxq+cep8nItVNoBV+OMmahCdqdF38FTpmXoqQUGvw==}
peerDependencies:
'@types/react': '*'
'@types/react-dom': '*'
react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
peerDependenciesMeta:
'@types/react':
optional: true
'@types/react-dom':
optional: true
'@radix-ui/react-slot@1.1.0':
resolution: {integrity: sha512-FUCf5XMfmW4dtYl69pdS4DbxKy8nj4M7SafBgPllysxmdachynNflAdp/gCsnYWNDnge6tI9onzMp5ARYc1KNw==}
peerDependencies:
@@ -9736,6 +9789,15 @@ packages:
engines: {node: '>=18'}
hasBin: true
monaco-editor-webpack-plugin@7.1.0:
resolution: {integrity: sha512-ZjnGINHN963JQkFqjjcBtn1XBtUATDZBMgNQhDQwd78w2ukRhFXAPNgWuacaQiDZsUr4h1rWv5Mv6eriKuOSzA==}
peerDependencies:
monaco-editor: '>= 0.31.0'
webpack: ^4.5.0 || 5.x
monaco-editor@0.52.0:
resolution: {integrity: sha512-OeWhNpABLCeTqubfqLMXGsqf6OmPU6pHM85kF3dhy6kq5hnhuVS1p3VrEW/XhWHc71P2tHyS5JFySD8mgs1crw==}
mongodb-connection-string-url@3.0.1:
resolution: {integrity: sha512-XqMGwRX0Lgn05TDB4PyG2h2kKO/FfWJyCzYQbIhXUxz7ETt0I/FqHjUeqj37irJ+Dl1ZtU82uYyj14u2XsZKfg==}
@@ -11129,6 +11191,13 @@ packages:
react: ^16.8.0 || ^17.0.0 || ^18.0.0
react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0
react-monaco-editor@0.56.2:
resolution: {integrity: sha512-Tp5U3QF9h92Cuf0eIhGd8Jyef8tPMlEJC2Dk1GeuR/hj6WoFn8AgjVX/2dv+3l5DvpMUpAECcFarc3eFKTBZ5w==}
peerDependencies:
'@types/react': '>=16 <= 18'
monaco-editor: ^0.52.0
react: '>=16 <= 18'
react-refresh@0.14.2:
resolution: {integrity: sha512-jCvmsr+1IUSMUyzOkRcvnVbX3ZYC6g9TDrDbFuFmRDq7PD4yaGbLKNQL6k2jnArV8hjYxh7hVhAZB6s9HDGpZA==}
engines: {node: '>=0.10.0'}
@@ -17430,6 +17499,15 @@ snapshots:
optionalDependencies:
'@types/react': 18.3.12
'@radix-ui/react-label@2.1.0(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)':
dependencies:
'@radix-ui/react-primitive': 2.0.0(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
react: 18.3.1
react-dom: 18.3.1(react@18.3.1)
optionalDependencies:
'@types/react': 18.3.12
'@types/react-dom': 18.3.1
'@radix-ui/react-navigation-menu@1.2.1(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)':
dependencies:
'@radix-ui/primitive': 1.1.0
@@ -17585,6 +17663,25 @@ snapshots:
'@types/react': 18.3.12
'@types/react-dom': 18.3.1
'@radix-ui/react-slider@1.2.1(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)':
dependencies:
'@radix-ui/number': 1.1.0
'@radix-ui/primitive': 1.1.0
'@radix-ui/react-collection': 1.1.0(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
'@radix-ui/react-compose-refs': 1.1.0(@types/react@18.3.12)(react@18.3.1)
'@radix-ui/react-context': 1.1.1(@types/react@18.3.12)(react@18.3.1)
'@radix-ui/react-direction': 1.1.0(@types/react@18.3.12)(react@18.3.1)
'@radix-ui/react-primitive': 2.0.0(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
'@radix-ui/react-use-controllable-state': 1.1.0(@types/react@18.3.12)(react@18.3.1)
'@radix-ui/react-use-layout-effect': 1.1.0(@types/react@18.3.12)(react@18.3.1)
'@radix-ui/react-use-previous': 1.1.0(@types/react@18.3.12)(react@18.3.1)
'@radix-ui/react-use-size': 1.1.0(@types/react@18.3.12)(react@18.3.1)
react: 18.3.1
react-dom: 18.3.1(react@18.3.1)
optionalDependencies:
'@types/react': 18.3.12
'@types/react-dom': 18.3.1
'@radix-ui/react-slot@1.1.0(@types/react@18.3.12)(react@18.3.1)':
dependencies:
'@radix-ui/react-compose-refs': 1.1.0(@types/react@18.3.12)(react@18.3.1)
@@ -24189,6 +24286,14 @@ snapshots:
requirejs: 2.3.7
requirejs-config-file: 4.0.0
monaco-editor-webpack-plugin@7.1.0(monaco-editor@0.52.0)(webpack@5.96.1):
dependencies:
loader-utils: 2.0.4
monaco-editor: 0.52.0
webpack: 5.96.1
monaco-editor@0.52.0: {}
mongodb-connection-string-url@3.0.1:
dependencies:
'@types/whatwg-url': 11.0.5
@@ -25682,6 +25787,13 @@ snapshots:
react: 18.3.1
react-dom: 18.3.1(react@18.3.1)
react-monaco-editor@0.56.2(@types/react@18.3.12)(monaco-editor@0.52.0)(react@18.3.1):
dependencies:
'@types/react': 18.3.12
monaco-editor: 0.52.0
prop-types: 15.8.1
react: 18.3.1
react-refresh@0.14.2: {}
react-remove-scroll-bar@2.3.6(@types/react@18.3.12)(react@18.3.1):
+8
View File
@@ -1,5 +1,13 @@
# @llamaindex/unit-test
## 0.0.12
### Patch Changes
- Updated dependencies [2ff0a89]
- @llamaindex/node-parser@0.0.2
- llamaindex@0.8.3
## 0.0.11
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/unit-test",
"private": true,
"version": "0.0.11",
"version": "0.0.12",
"type": "module",
"scripts": {
"test": "vitest run"